github: update bug-report issue template

Tweaked wording slightly to ask for information about managed devices; explain why we need a bug report identifier.

Also updated a link to the bug report KB.

Signed-off-by: Andrea Gottardo <andrea@tailscale.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,5 +1,5 @@
 name: Bug report
-description: File a bug report. If you need help, contact support instead
+description: File a bug report. If you need help, contact support at https://tailscale.com/support instead.
 labels: [needs-triage, bug]
 body:
   - type: markdown
@@ -39,6 +39,7 @@ body:
         - macOS
         - Windows
         - iOS
+        - tvOS
         - Android
         - Synology
         - Other
@@ -57,21 +58,21 @@ body:
     attributes:
       label: Tailscale version
       description: What Tailscale version are you using?
-      placeholder: e.g., 1.14.4
+      placeholder: e.g., 1.68.1
     validations:
       required: false
   - type: textarea
     id: other-software
     attributes:
       label: Other software
-      description: What [other software](https://github.com/tailscale/tailscale/wiki/OtherSoftwareInterop) (networking, security, etc) are you running?
+      description: What [other software](https://github.com/tailscale/tailscale/wiki/OtherSoftwareInterop) (networking, security, etc) are you running? Are you using a work device managed by your employer?
     validations:
       required: false
   - type: input
     id: bug-report
     attributes:
       label: Bug report
-      description: Please run [`tailscale bugreport`](https://tailscale.com/kb/1080/cli/?q=Cli#bugreport) and share the bug identifier. The identifier is a random string which allows Tailscale support to locate your account and gives a point to focus on when looking for errors.
+      description: [Generate a bug report identifier](https://tailscale.com/kb/1227/bug-report) and attach it here. The identifier is a random string which allows Tailscale support to locate your account, and gives us a point in time to focus on when looking for errors. If you do not share a bug report identifier, the Tailscale team won't be able to investigate your issue.
       placeholder: e.g., BUG-1b7641a16971a9cd75822c0ed8043fee70ae88cf05c52981dc220eb96a5c49a8-20210427151443Z-fbcd4fd3a4b7ad94
     validations:
       required: false

.github/workflows/ssh-integrationtest.yml

@@ -0,0 +1,23 @@
+# Run the ssh integration tests with `make sshintegrationtest`.
+# These tests can also be running locally.
+name: "ssh-integrationtest"
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - "ssh/**"
+      - "tempfork/gliderlabs/ssh/**"
+      - ".github/workflows/ssh-integrationtest"
+jobs:
+  ssh-integrationtest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Run SSH integration tests
+        run: |
+          make sshintegrationtest

.github/workflows/test.yml

@@ -194,7 +194,7 @@ jobs:
     - name: chown
       run: chown -R $(id -u):$(id -g) $PWD
     - name: privileged tests
-      run: ./tool/go test ./util/linuxfw
+      run: ./tool/go test ./util/linuxfw ./derp/xdp
 
   vm:
     runs-on: ["self-hosted", "linux", "vm"]
@@ -480,7 +480,7 @@ jobs:
       uses: actions/checkout@v4
     - name: check that 'go generate' is clean
       run: |
-        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator')
+        pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp')
         ./tool/go generate $pkgs
         echo
         echo

Makefile

@@ -110,8 +110,8 @@ publishdevnameserver: ## Build and publish k8s-nameserver image to location spec
 
 .PHONY: sshintegrationtest
 sshintegrationtest: ## Run the SSH integration tests in various Docker containers
-	@GOOS=linux GOARCH=amd64 go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \
-	GOOS=linux GOARCH=amd64 go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
+	@GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \
+	GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
 	echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
 	echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \

VERSION.txt

@@ -1 +1 @@
-1.67.0
+1.69.0

appc/appconnector.go

@@ -15,6 +15,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"time"
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
@@ -26,6 +27,46 @@ import (
 	"tailscale.com/util/slicesx"
 )
 
+// rateLogger responds to calls to update by adding a count for the current period and
+// calling the callback if any previous period has finished since update was last called
+type rateLogger struct {
+	interval    time.Duration
+	start       time.Time
+	periodStart time.Time
+	periodCount int64
+	now         func() time.Time
+	callback    func(int64, time.Time, int64)
+}
+
+func (rl *rateLogger) currentIntervalStart(now time.Time) time.Time {
+	millisSince := now.Sub(rl.start).Milliseconds() % rl.interval.Milliseconds()
+	return now.Add(-(time.Duration(millisSince)) * time.Millisecond)
+}
+
+func (rl *rateLogger) update(numRoutes int64) {
+	now := rl.now()
+	periodEnd := rl.periodStart.Add(rl.interval)
+	if periodEnd.Before(now) {
+		if rl.periodCount != 0 {
+			rl.callback(rl.periodCount, rl.periodStart, numRoutes)
+		}
+		rl.periodCount = 0
+		rl.periodStart = rl.currentIntervalStart(now)
+	}
+	rl.periodCount++
+}
+
+func newRateLogger(now func() time.Time, interval time.Duration, callback func(int64, time.Time, int64)) *rateLogger {
+	nowTime := now()
+	return &rateLogger{
+		callback:    callback,
+		now:         now,
+		interval:    interval,
+		start:       nowTime,
+		periodStart: nowTime,
+	}
+}
+
 // RouteAdvertiser is an interface that allows the AppConnector to advertise
 // newly discovered routes that need to be served through the AppConnector.
 type RouteAdvertiser interface {
@@ -81,6 +122,9 @@ type AppConnector struct {
 
 	// queue provides ordering for update operations
 	queue execqueue.ExecQueue
+
+	writeRateMinute *rateLogger
+	writeRateDay    *rateLogger
 }
 
 // NewAppConnector creates a new AppConnector.
@@ -95,6 +139,12 @@ func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, routeInf
 		ac.wildcards = routeInfo.Wildcards
 		ac.controlRoutes = routeInfo.Control
 	}
+	ac.writeRateMinute = newRateLogger(time.Now, time.Minute, func(c int64, s time.Time, l int64) {
+		ac.logf("routeInfo write rate: %d in minute starting at %v (%d routes)", c, s, l)
+	})
+	ac.writeRateDay = newRateLogger(time.Now, 24*time.Hour, func(c int64, s time.Time, l int64) {
+		ac.logf("routeInfo write rate: %d in 24 hours starting at %v (%d routes)", c, s, l)
+	})
 	return ac
 }
 
@@ -109,6 +159,15 @@ func (e *AppConnector) storeRoutesLocked() error {
 	if !e.ShouldStoreRoutes() {
 		return nil
 	}
+
+	// log write rate and write size
+	numRoutes := int64(len(e.controlRoutes))
+	for _, rs := range e.domains {
+		numRoutes += int64(len(rs))
+	}
+	e.writeRateMinute.update(numRoutes)
+	e.writeRateDay.update(numRoutes)
+
 	return e.storeRoutesFunc(&RouteInfo{
 		Control:   e.controlRoutes,
 		Domains:   e.domains,

appc/appconnector_test.go

@@ -9,10 +9,12 @@ import (
 	"reflect"
 	"slices"
 	"testing"
+	"time"
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/tstest"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
@@ -520,3 +522,50 @@ func TestRoutesWithout(t *testing.T) {
 	assert("a has fewer", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32"), prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32")), []netip.Prefix{})
 	assert("a has more", routesWithout(prefixes("1.1.1.1/32", "1.1.1.2/32", "1.1.1.3/32", "1.1.1.4/32"), prefixes("1.1.1.1/32", "1.1.1.3/32")), prefixes("1.1.1.2/32", "1.1.1.4/32"))
 }
+
+func TestRateLogger(t *testing.T) {
+	clock := tstest.Clock{}
+	wasCalled := false
+	rl := newRateLogger(func() time.Time { return clock.Now() }, 1*time.Second, func(count int64, _ time.Time, _ int64) {
+		if count != 3 {
+			t.Fatalf("count for prev period: got %d, want 3", count)
+		}
+		wasCalled = true
+	})
+
+	for i := 0; i < 3; i++ {
+		clock.Advance(1 * time.Millisecond)
+		rl.update(0)
+		if wasCalled {
+			t.Fatalf("wasCalled: got true, want false")
+		}
+	}
+
+	clock.Advance(1 * time.Second)
+	rl.update(0)
+	if !wasCalled {
+		t.Fatalf("wasCalled: got false, want true")
+	}
+
+	wasCalled = false
+	rl = newRateLogger(func() time.Time { return clock.Now() }, 1*time.Hour, func(count int64, _ time.Time, _ int64) {
+		if count != 3 {
+			t.Fatalf("count for prev period: got %d, want 3", count)
+		}
+		wasCalled = true
+	})
+
+	for i := 0; i < 3; i++ {
+		clock.Advance(1 * time.Minute)
+		rl.update(0)
+		if wasCalled {
+			t.Fatalf("wasCalled: got true, want false")
+		}
+	}
+
+	clock.Advance(1 * time.Hour)
+	rl.update(0)
+	if !wasCalled {
+		t.Fatalf("wasCalled: got false, want true")
+	}
+}

client/tailscale/devices.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"net/url"
 
@@ -39,6 +40,7 @@ type Device struct {
 	// It's currently just 1 element, the 100.x.y.z Tailscale IP.
 	Addresses []string `json:"addresses"`
 	DeviceID  string   `json:"id"`
+	NodeID    string   `json:"nodeId"`
 	User      string   `json:"user"`
 	Name      string   `json:"name"`
 	Hostname  string   `json:"hostname"`
@@ -213,6 +215,9 @@ func (c *Client) DeleteDevice(ctx context.Context, deviceID string) (err error)
 	if err != nil {
 		return err
 	}
+
+	log.Printf("RESP: %di, path: %s", resp.StatusCode, path)
+
 	// If status code was not successful, return the error.
 	// TODO: Change the check for the StatusCode to include other 2XX success codes.
 	if resp.StatusCode != http.StatusOK {

client/tailscale/localclient.go

@@ -253,11 +253,16 @@ func (lc *LocalClient) sendWithHeaders(
 	}
 	if res.StatusCode != wantStatus {
 		err = fmt.Errorf("%v: %s", res.Status, bytes.TrimSpace(slurp))
-		return nil, nil, bestError(err, slurp)
+		return nil, nil, httpStatusError{bestError(err, slurp), res.StatusCode}
 	}
 	return slurp, res.Header, nil
 }
 
+type httpStatusError struct {
+	error
+	HTTPStatus int
+}
+
 func (lc *LocalClient) get200(ctx context.Context, path string) ([]byte, error) {
 	return lc.send(ctx, "GET", path, 200, nil)
 }
@@ -278,9 +283,50 @@ func decodeJSON[T any](b []byte) (ret T, err error) {
 }
 
 // WhoIs returns the owner of the remoteAddr, which must be an IP or IP:port.
+//
+// If not found, the error is ErrPeerNotFound.
+//
+// For connections proxied by tailscaled, this looks up the owner of the given
+// address as TCP first, falling back to UDP; if you want to only check a
+// specific address family, use WhoIsProto.
 func (lc *LocalClient) WhoIs(ctx context.Context, remoteAddr string) (*apitype.WhoIsResponse, error) {
 	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(remoteAddr))
 	if err != nil {
+		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
+			return nil, ErrPeerNotFound
+		}
+		return nil, err
+	}
+	return decodeJSON[*apitype.WhoIsResponse](body)
+}
+
+// ErrPeerNotFound is returned by WhoIs and WhoIsNodeKey when a peer is not found.
+var ErrPeerNotFound = errors.New("peer not found")
+
+// WhoIsNodeKey returns the owner of the given wireguard public key.
+//
+// If not found, the error is ErrPeerNotFound.
+func (lc *LocalClient) WhoIsNodeKey(ctx context.Context, key key.NodePublic) (*apitype.WhoIsResponse, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/whois?addr="+url.QueryEscape(key.String()))
+	if err != nil {
+		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
+			return nil, ErrPeerNotFound
+		}
+		return nil, err
+	}
+	return decodeJSON[*apitype.WhoIsResponse](body)
+}
+
+// WhoIsProto returns the owner of the remoteAddr, which must be an IP or
+// IP:port, for the given protocol (tcp or udp).
+//
+// If not found, the error is ErrPeerNotFound.
+func (lc *LocalClient) WhoIsProto(ctx context.Context, proto, remoteAddr string) (*apitype.WhoIsResponse, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/whois?proto="+url.QueryEscape(proto)+"&addr="+url.QueryEscape(remoteAddr))
+	if err != nil {
+		if hs, ok := err.(httpStatusError); ok && hs.HTTPStatus == http.StatusNotFound {
+			return nil, ErrPeerNotFound
+		}
 		return nil, err
 	}
 	return decodeJSON[*apitype.WhoIsResponse](body)
@@ -699,6 +745,27 @@ func (lc *LocalClient) CheckUDPGROForwarding(ctx context.Context) error {
 	return nil
 }
 
+// SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this
+// node. This can be done to improve performance of tailnet nodes acting as exit
+// nodes or subnet routers.
+// See https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+func (lc *LocalClient) SetUDPGROForwarding(ctx context.Context) error {
+	body, err := lc.get200(ctx, "/localapi/v0/set-udp-gro-forwarding")
+	if err != nil {
+		return err
+	}
+	var jres struct {
+		Warning string
+	}
+	if err := json.Unmarshal(body, &jres); err != nil {
+		return fmt.Errorf("invalid JSON from set-udp-gro-forwarding: %w", err)
+	}
+	if jres.Warning != "" {
+		return errors.New(jres.Warning)
+	}
+	return nil
+}
+
 // CheckPrefs validates the provided preferences, without making any changes.
 //
 // The CLI uses this before a Start call to fail fast if the preferences won't

client/tailscale/localclient_test.go

@@ -6,9 +6,14 @@
 package tailscale
 
 import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"tailscale.com/tstest/deptest"
+	"tailscale.com/types/key"
 )
 
 func TestGetServeConfigFromJSON(t *testing.T) {
@@ -30,6 +35,32 @@ func TestGetServeConfigFromJSON(t *testing.T) {
 	}
 }
 
+func TestWhoIsPeerNotFound(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+	}))
+	defer ts.Close()
+
+	lc := &LocalClient{
+		Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			var std net.Dialer
+			return std.DialContext(ctx, network, ts.Listener.Addr().(*net.TCPAddr).String())
+		},
+	}
+	var k key.NodePublic
+	if err := k.UnmarshalText([]byte("nodekey:5c8f86d5fc70d924e55f02446165a5dae8f822994ad26bcf4b08fd841f9bf261")); err != nil {
+		t.Fatal(err)
+	}
+	res, err := lc.WhoIsNodeKey(context.Background(), k)
+	if err != ErrPeerNotFound {
+		t.Errorf("got (%v, %v), want ErrPeerNotFound", res, err)
+	}
+	res, err = lc.WhoIs(context.Background(), "1.2.3.4:5678")
+	if err != ErrPeerNotFound {
+		t.Errorf("got (%v, %v), want ErrPeerNotFound", res, err)
+	}
+}
+
 func TestDeps(t *testing.T) {
 	deptest.DepChecker{
 		BadDeps: map[string]string{

clientupdate/clientupdate.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/google/uuid"
 	"tailscale.com/clientupdate/distsign"
-	"tailscale.com/hostinfo"
 	"tailscale.com/types/logger"
 	"tailscale.com/util/cmpver"
 	"tailscale.com/util/winutil"
@@ -38,11 +37,18 @@ import (
 )
 
 const (
-	CurrentTrack  = ""
 	StableTrack   = "stable"
 	UnstableTrack = "unstable"
 )
 
+var CurrentTrack = func() string {
+	if version.IsUnstableBuild() {
+		return UnstableTrack
+	} else {
+		return StableTrack
+	}
+}()
+
 func versionToTrack(v string) (string, error) {
 	_, rest, ok := strings.Cut(v, ".")
 	if !ok {
@@ -107,7 +113,7 @@ func (args Arguments) validate() error {
 		return fmt.Errorf("only one of Version(%q) or Track(%q) can be set", args.Version, args.Track)
 	}
 	switch args.Track {
-	case StableTrack, UnstableTrack, CurrentTrack:
+	case StableTrack, UnstableTrack, "":
 		// All valid values.
 	default:
 		return fmt.Errorf("unsupported track %q", args.Track)
@@ -120,11 +126,17 @@ type Updater struct {
 	// Update is a platform-specific method that updates the installation. May be
 	// nil (not all platforms support updates from within Tailscale).
 	Update func() error
+
+	// currentVersion is the short form of the current client version as
+	// returned by version.Short(), typically "x.y.z". Used for tests to
+	// override the actual current version.
+	currentVersion string
 }
 
 func NewUpdater(args Arguments) (*Updater, error) {
 	up := Updater{
-		Arguments: args,
+		Arguments:      args,
+		currentVersion: version.Short(),
 	}
 	if up.Stdout == nil {
 		up.Stdout = os.Stdout
@@ -140,18 +152,15 @@ func NewUpdater(args Arguments) (*Updater, error) {
 	if args.ForAutoUpdate && !canAutoUpdate {
 		return nil, errors.ErrUnsupported
 	}
-	if up.Track == CurrentTrack {
-		switch {
-		case up.Version != "":
+	if up.Track == "" {
+		if up.Version != "" {
 			var err error
 			up.Track, err = versionToTrack(args.Version)
 			if err != nil {
 				return nil, err
 			}
-		case version.IsUnstableBuild():
-			up.Track = UnstableTrack
-		default:
-			up.Track = StableTrack
+		} else {
+			up.Track = CurrentTrack
 		}
 	}
 	if up.Arguments.PkgsAddr == "" {
@@ -163,10 +172,9 @@ func NewUpdater(args Arguments) (*Updater, error) {
 type updateFunction func() error
 
 func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
-	canAutoUpdate = !hostinfo.New().Container.EqualBool(true) // EqualBool(false) would return false if the value is not set.
 	switch runtime.GOOS {
 	case "windows":
-		return up.updateWindows, canAutoUpdate
+		return up.updateWindows, true
 	case "linux":
 		switch distro.Get() {
 		case distro.NixOS:
@@ -180,20 +188,20 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 			// auto-update mechanism.
 			return up.updateSynology, false
 		case distro.Debian: // includes Ubuntu
-			return up.updateDebLike, canAutoUpdate
+			return up.updateDebLike, true
 		case distro.Arch:
 			if up.archPackageInstalled() {
 				// Arch update func just prints a message about how to update,
 				// it doesn't support auto-updates.
 				return up.updateArchLike, false
 			}
-			return up.updateLinuxBinary, canAutoUpdate
+			return up.updateLinuxBinary, true
 		case distro.Alpine:
-			return up.updateAlpineLike, canAutoUpdate
+			return up.updateAlpineLike, true
 		case distro.Unraid:
-			return up.updateUnraid, canAutoUpdate
+			return up.updateUnraid, true
 		case distro.QNAP:
-			return up.updateQNAP, canAutoUpdate
+			return up.updateQNAP, true
 		}
 		switch {
 		case haveExecutable("pacman"):
@@ -202,21 +210,21 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 				// it doesn't support auto-updates.
 				return up.updateArchLike, false
 			}
-			return up.updateLinuxBinary, canAutoUpdate
+			return up.updateLinuxBinary, true
 		case haveExecutable("apt-get"): // TODO(awly): add support for "apt"
 			// The distro.Debian switch case above should catch most apt-based
 			// systems, but add this fallback just in case.
-			return up.updateDebLike, canAutoUpdate
+			return up.updateDebLike, true
 		case haveExecutable("dnf"):
-			return up.updateFedoraLike("dnf"), canAutoUpdate
+			return up.updateFedoraLike("dnf"), true
 		case haveExecutable("yum"):
-			return up.updateFedoraLike("yum"), canAutoUpdate
+			return up.updateFedoraLike("yum"), true
 		case haveExecutable("apk"):
-			return up.updateAlpineLike, canAutoUpdate
+			return up.updateAlpineLike, true
 		}
 		// If nothing matched, fall back to tarball updates.
 		if up.Update == nil {
-			return up.updateLinuxBinary, canAutoUpdate
+			return up.updateLinuxBinary, true
 		}
 	case "darwin":
 		switch {
@@ -232,7 +240,7 @@ func (up *Updater) getUpdateFunction() (fn updateFunction, canAutoUpdate bool) {
 			return nil, false
 		}
 	case "freebsd":
-		return up.updateFreeBSD, canAutoUpdate
+		return up.updateFreeBSD, true
 	}
 	return nil, false
 }
@@ -261,13 +269,16 @@ func Update(args Arguments) error {
 }
 
 func (up *Updater) confirm(ver string) bool {
-	switch cmpver.Compare(version.Short(), ver) {
-	case 0:
-		up.Logf("already running %v version %v; no update needed", up.Track, ver)
-		return false
-	case 1:
-		up.Logf("installed %v version %v is newer than the latest available version %v; no update needed", up.Track, version.Short(), ver)
-		return false
+	// Only check version when we're not switching tracks.
+	if up.Track == "" || up.Track == CurrentTrack {
+		switch c := cmpver.Compare(up.currentVersion, ver); {
+		case c == 0:
+			up.Logf("already running %v version %v; no update needed", up.Track, ver)
+			return false
+		case c > 0:
+			up.Logf("installed %v version %v is newer than the latest available version %v; no update needed", up.Track, up.currentVersion, ver)
+			return false
+		}
 	}
 	if up.Confirm != nil {
 		return up.Confirm(ver)
@@ -683,7 +694,7 @@ func parseAlpinePackageVersion(out []byte) (string, error) {
 			return "", fmt.Errorf("malformed info line: %q", line)
 		}
 		ver := parts[1]
-		if cmpver.Compare(ver, maxVer) == 1 {
+		if cmpver.Compare(ver, maxVer) > 0 {
 			maxVer = ver
 		}
 	}
@@ -882,7 +893,7 @@ func (up *Updater) installMSI(msi string) error {
 			break
 		}
 		up.Logf("Install attempt failed: %v", err)
-		uninstallVersion := version.Short()
+		uninstallVersion := up.currentVersion
 		if v := os.Getenv("TS_DEBUG_UNINSTALL_VERSION"); v != "" {
 			uninstallVersion = v
 		}
@@ -1333,12 +1344,8 @@ func requestedTailscaleVersion(ver, track string) (string, error) {
 // LatestTailscaleVersion returns the latest released version for the given
 // track from pkgs.tailscale.com.
 func LatestTailscaleVersion(track string) (string, error) {
-	if track == CurrentTrack {
-		if version.IsUnstableBuild() {
-			track = UnstableTrack
-		} else {
-			track = StableTrack
-		}
+	if track == "" {
+		track = CurrentTrack
 	}
 
 	latest, err := latestPackages(track)

clientupdate/clientupdate_test.go

@@ -846,3 +846,107 @@ func TestParseUnraidPluginVersion(t *testing.T) {
 		})
 	}
 }
+
+func TestConfirm(t *testing.T) {
+	curTrack := CurrentTrack
+	defer func() { CurrentTrack = curTrack }()
+
+	tests := []struct {
+		desc      string
+		fromTrack string
+		toTrack   string
+		fromVer   string
+		toVer     string
+		confirm   func(string) bool
+		want      bool
+	}{
+		{
+			desc:      "on latest stable",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.66.0",
+			want:      false,
+		},
+		{
+			desc:      "stable upgrade",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.68.0",
+			want:      true,
+		},
+		{
+			desc:      "unstable upgrade",
+			fromTrack: UnstableTrack,
+			toTrack:   UnstableTrack,
+			fromVer:   "1.67.1",
+			toVer:     "1.67.2",
+			want:      true,
+		},
+		{
+			desc:      "from stable to unstable",
+			fromTrack: StableTrack,
+			toTrack:   UnstableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.67.1",
+			want:      true,
+		},
+		{
+			desc:      "from unstable to stable",
+			fromTrack: UnstableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.67.1",
+			toVer:     "1.66.0",
+			want:      true,
+		},
+		{
+			desc:      "confirm callback rejects",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.66.1",
+			confirm: func(string) bool {
+				return false
+			},
+			want: false,
+		},
+		{
+			desc:      "confirm callback allows",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.0",
+			toVer:     "1.66.1",
+			confirm: func(string) bool {
+				return true
+			},
+			want: true,
+		},
+		{
+			desc:      "downgrade",
+			fromTrack: StableTrack,
+			toTrack:   StableTrack,
+			fromVer:   "1.66.1",
+			toVer:     "1.66.0",
+			want:      false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			CurrentTrack = tt.fromTrack
+			up := Updater{
+				currentVersion: tt.fromVer,
+				Arguments: Arguments{
+					Track:   tt.toTrack,
+					Confirm: tt.confirm,
+					Logf:    t.Logf,
+				},
+			}
+
+			if got := up.confirm(tt.toVer); got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/containerboot/kube.go

@@ -19,22 +19,20 @@ import (
 	"tailscale.com/tailcfg"
 )
 
-// storeDeviceInfo writes deviceID into the "device_id" data field of the kube
-// secret secretName.
-func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID, fqdn string, addresses []netip.Prefix) error {
-	// First check if the secret exists at all. Even if running on
-	// kubernetes, we do not necessarily store state in a k8s secret.
-	if _, err := kc.GetSecret(ctx, secretName); err != nil {
-		if s, ok := err.(*kube.Status); ok {
-			if s.Code >= 400 && s.Code <= 499 {
-				// Assume the secret doesn't exist, or we don't have
-				// permission to access it.
-				return nil
-			}
-		}
-		return err
+// storeDeviceID writes deviceID to 'device_id' data field of the named
+// Kubernetes Secret.
+func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
+	s := &kube.Secret{
+		Data: map[string][]byte{
+			"device_id": []byte(deviceID),
+		},
 	}
+	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
+}
 
+// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields
+// 'device_ips', 'device_fqdn' of the named Kubernetes Secret.
+func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, addresses []netip.Prefix) error {
 	var ips []string
 	for _, addr := range addresses {
 		ips = append(ips, addr.Addr().String())
@@ -44,14 +42,13 @@ func storeDeviceInfo(ctx context.Context, secretName string, deviceID tailcfg.St
 		return err
 	}
 
-	m := &kube.Secret{
+	s := &kube.Secret{
 		Data: map[string][]byte{
-			"device_id":   []byte(deviceID),
 			"device_fqdn": []byte(fqdn),
 			"device_ips":  deviceIPs,
 		},
 	}
-	return kc.StrategicMergePatchSecret(ctx, secretName, m, "tailscale-container")
+	return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
 }
 
 // deleteAuthKey deletes the 'authkey' field of the given kube

cmd/containerboot/main.go

@@ -61,6 +61,11 @@
 //     and not `tailscale up` or `tailscale set`.
 //     The config file contents are currently read once on container start.
 //     NB: This env var is currently experimental and the logic will likely change!
+//     TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS: set to true to
+//     autoconfigure the default network interface for optimal performance for
+//     Tailscale subnet router/exit node.
+//     https://tailscale.com/kb/1320/performance-best-practices#linux-optimizations-for-subnet-routers-and-exit-nodes
+//     NB: This env var is currently experimental and the logic will likely change!
 //   - EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS: if set to true
 //     and if this containerboot instance is an L7 ingress proxy (created by
 //     the Kubernetes operator), set up rules to allow proxying cluster traffic,
@@ -152,6 +157,7 @@ func main() {
 		TailscaledConfigFilePath:              tailscaledConfigFilePath(),
 		AllowProxyingClusterTrafficViaIngress: defaultBool("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS", false),
 		PodIP:                                 defaultEnv("POD_IP", ""),
+		EnableForwardingOptimizations:         defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -199,6 +205,12 @@ func main() {
 	}
 	defer killTailscaled()
 
+	if cfg.EnableForwardingOptimizations {
+		if err := client.SetUDPGROForwarding(bootCtx); err != nil {
+			log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
+		}
+	}
+
 	w, err := client.WatchIPNBus(bootCtx, ipn.NotifyInitialNetMap|ipn.NotifyInitialPrefs|ipn.NotifyInitialState)
 	if err != nil {
 		log.Fatalf("failed to watch tailscaled for updates: %v", err)
@@ -309,7 +321,7 @@ authLoop:
 		}
 	}
 
-	if cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch && isTwoStepConfigAuthOnce(cfg) {
+	if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) {
 		// We were told to only auth once, so any secret-bound
 		// authkey is no longer needed. We don't strictly need to
 		// wipe it, but it's good hygiene.
@@ -325,11 +337,10 @@ authLoop:
 	}
 
 	var (
-		wantProxy         = cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
-		wantDeviceInfo    = cfg.InKubernetes && cfg.KubeSecret != "" && cfg.KubernetesCanPatch
-		startupTasksDone  = false
-		currentIPs        deephash.Sum // tailscale IPs assigned to device
-		currentDeviceInfo deephash.Sum // device ID and fqdn
+		startupTasksDone       = false
+		currentIPs             deephash.Sum // tailscale IPs assigned to device
+		currentDeviceID        deephash.Sum // device ID
+		currentDeviceEndpoints deephash.Sum // device FQDN and IPs
 
 		currentEgressIPs deephash.Sum
 
@@ -343,7 +354,7 @@ authLoop:
 		go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
 	}
 	var nfr linuxfw.NetfilterRunner
-	if wantProxy {
+	if isL3Proxy(cfg) {
 		nfr, err = newNetfilterRunner(log.Printf)
 		if err != nil {
 			log.Fatalf("error creating new netfilter runner: %v", err)
@@ -428,6 +439,20 @@ runLoop:
 				newCurrentIPs := deephash.Hash(&addrs)
 				ipsHaveChanged := newCurrentIPs != currentIPs
 
+				// Store device ID in a Kubernetes Secret before
+				// setting up any routing rules. This ensures
+				// that, for containerboot instances that are
+				// Kubernetes operator proxies, the operator is
+				// able to retrieve the device ID from the
+				// Kubernetes Secret to clean up tailnet nodes
+				// for proxies whose route setup continuously
+				// fails.
+				deviceID := n.NetMap.SelfNode.StableID()
+				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceID, &deviceID) {
+					if err := storeDeviceID(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID()); err != nil {
+						log.Fatalf("storing device ID in Kubernetes Secret: %v", err)
+					}
+				}
 				if cfg.TailnetTargetFQDN != "" {
 					var (
 						egressAddrs          []netip.Prefix
@@ -521,15 +546,36 @@ runLoop:
 				}
 				currentIPs = newCurrentIPs
 
-				deviceInfo := []any{n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name()}
-				if cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != "" && deephash.Update(&currentDeviceInfo, &deviceInfo) {
-					if err := storeDeviceInfo(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID(), n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
-						log.Fatalf("storing device ID in kube secret: %v", err)
+				// Only store device FQDN and IP addresses to
+				// Kubernetes Secret when any required proxy
+				// route setup has succeeded. IPs and FQDN are
+				// read from the Secret by the Tailscale
+				// Kubernetes operator and, for some proxy
+				// types, such as Tailscale Ingress, advertized
+				// on the Ingress status. Writing them to the
+				// Secret only after the proxy routing has been
+				// set up ensures that the operator does not
+				// advertize endpoints of broken proxies.
+				// TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'.
+				deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()}
+				if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceEndpoints, &deviceEndpoints) {
+					if err := storeDeviceEndpoints(ctx, cfg.KubeSecret, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
+						log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
 					}
 				}
 			}
 			if !startupTasksDone {
-				if (!wantProxy || currentIPs != deephash.Sum{}) && (!wantDeviceInfo || currentDeviceInfo != deephash.Sum{}) {
+				// For containerboot instances that act as TCP
+				// proxies (proxying traffic to an endpoint
+				// passed via one of the env vars that
+				// containerbot reads) and store state in a
+				// Kubernetes Secret, we consider startup tasks
+				// done at the point when device info has been
+				// successfully stored to state Secret.
+				// For all other containerboot instances, if we
+				// just get to this point the startup tasks can
+				// be considered done.
+				if !isL3Proxy(cfg) || !hasKubeStateStore(cfg) || (currentDeviceEndpoints != deephash.Sum{} && currentDeviceID != deephash.Sum{}) {
 					// This log message is used in tests to detect when all
 					// post-auth configuration is done.
 					log.Println("Startup complete, waiting for shutdown signal")
@@ -1080,22 +1126,23 @@ type settings struct {
 	// TailnetTargetFQDN is an MagicDNS name to which all incoming
 	// non-Tailscale traffic should be proxied. This must be a full Tailnet
 	// node FQDN.
-	TailnetTargetFQDN        string
-	ServeConfigPath          string
-	DaemonExtraArgs          string
-	ExtraArgs                string
-	InKubernetes             bool
-	UserspaceMode            bool
-	StateDir                 string
-	AcceptDNS                *bool
-	KubeSecret               string
-	SOCKSProxyAddr           string
-	HTTPProxyAddr            string
-	Socket                   string
-	AuthOnce                 bool
-	Root                     string
-	KubernetesCanPatch       bool
-	TailscaledConfigFilePath string
+	TailnetTargetFQDN             string
+	ServeConfigPath               string
+	DaemonExtraArgs               string
+	ExtraArgs                     string
+	InKubernetes                  bool
+	UserspaceMode                 bool
+	StateDir                      string
+	AcceptDNS                     *bool
+	KubeSecret                    string
+	SOCKSProxyAddr                string
+	HTTPProxyAddr                 string
+	Socket                        string
+	AuthOnce                      bool
+	Root                          string
+	KubernetesCanPatch            bool
+	TailscaledConfigFilePath      string
+	EnableForwardingOptimizations bool
 	// If set to true and, if this containerboot instance is a Kubernetes
 	// ingress proxy, set up rules to forward incoming cluster traffic to be
 	// forwarded to the ingress target in cluster.
@@ -1149,6 +1196,9 @@ func (s *settings) validate() error {
 	if s.AllowProxyingClusterTrafficViaIngress && s.PodIP == "" {
 		return errors.New("EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS is set but POD_IP is not set")
 	}
+	if s.EnableForwardingOptimizations && s.UserspaceMode {
+		return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
+	}
 	return nil
 }
 
@@ -1271,6 +1321,19 @@ func isOneStepConfig(cfg *settings) bool {
 	return cfg.TailscaledConfigFilePath != ""
 }
 
+// isL3Proxy returns true if the Tailscale node needs to be configured to act
+// as an L3 proxy, proxying to an endpoint provided via one of the config env
+// vars.
+func isL3Proxy(cfg *settings) bool {
+	return cfg.ProxyTargetIP != "" || cfg.ProxyTargetDNSName != "" || cfg.TailnetTargetIP != "" || cfg.TailnetTargetFQDN != "" || cfg.AllowProxyingClusterTrafficViaIngress
+}
+
+// hasKubeStateStore returns true if the state must be stored in a Kubernetes
+// Secret.
+func hasKubeStateStore(cfg *settings) bool {
+	return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
+}
+
 // tailscaledConfigFilePath returns the path to the tailscaled config file that
 // should be used for the current capability version. It is determined by the
 // TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR environment variable and looks for a

cmd/derper/README.md

@@ -0,0 +1,74 @@
+# DERP
+
+This is the code for the [Tailscale DERP server](https://tailscale.com/kb/1232/derp-servers).
+
+In general, you should not need to nor want to run this code. The overwhelming majority of Tailscale users (both individuals and companies) do not.
+
+In the happy path, Tailscale establishes direct connections between peers and
+data plane traffic flows directly between them, without using DERP for more than
+acting as a low bandwidth side channel to bootstrap the NAT traversal. If you
+find yourself wanting DERP for more bandwidth, the real problem is usually the
+network configuration of your Tailscale node(s), making sure that Tailscale can
+get direction connections via some mechanism.
+
+But if you've decided or been advised to run your own `derper`, then read on.
+
+## Caveats
+
+* Node sharing and other cross-Tailnet features don't work when using custom
+  DERP servers.
+
+* DERP servers only see encrypted WireGuard packets and thus are not useful for
+  network-level debugging.
+
+* The Tailscale control plane does certain geo-level steering features and
+  optimizations that are not available when using custom DERP servers.
+
+## Guide to running `cmd/derper`
+
+* You must build and update the `cmd/derper` binary yourself. There are no
+  packages. Use `go install tailscale.com/cmd/derper@latest` with the latest
+  version of Go.
+
+* The DERP protocol does a protocol switch inside TLS from HTTP to a custom
+  bidirectional binary protocol. It is thus incompatible with many HTTP proxies.
+  Do not put `derper` behind another HTTP proxy.
+
+* The `tailscaled` client does its own selection of the fastest/nearest DERP
+  server based on latency measurements. Do not put `derper` behind a global load
+  balancer.
+
+* DERP servers should ideally have both a static IPv4 and static IPv6 address.
+Both of those should be listed in the DERP map so the client doesn't need to
+rely on its DNS which might be broken and dependent on DERP to get back up.
+
+* A DERP server should not share an IP address with any other DERP server.
+
+* Avoid having multiple DERP nodes in a region. If you must, they all need to be
+  meshed with each other and monitored. Having two one-node "regions" in the
+  same datacenter is usually easier and more reliable than meshing, at the cost
+  of more required connections from clients in some cases. If your clients
+  aren't mobile (battery constrained), one node regions are definitely
+  preferred. If you really need multiple nodes in a region for HA reasons, two
+  is sufficient.
+
+* Monitor your DERP servers with [`cmd/derpprobe`](../derpprobe/).
+
+* If using `--verify-clients`, a `tailscaled` must be running alongside the
+  `derper`.
+
+* If using `--verify-clients`, a `tailscaled` must also be running alongside
+  your `derpprobe`, and `derpprobe` needs to use `--derp-map=local`.
+
+* The firewall on the `derper` should permit TCP ports 80 and 443 and UDP port
+  3478.
+
+* Only LetsEncrypt certs are rotated automatically. Other cert updates require a
+  restart.
+
+* Don't use a firewall in front of `derper` that suppresses `RST`s upon
+  receiving traffic to a dead or unknown connection.
+
+* Don't rate-limit UDP STUN packets.
+
+* Don't rate-limit outbound TCP traffic (only inbound).

cmd/derper/depaware.txt

@@ -46,7 +46,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
-        go4.org/netipx                                               from tailscale.com/net/tsaddr+
+        go4.org/netipx                                               from tailscale.com/net/tsaddr
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         google.golang.org/protobuf/encoding/protodelim               from github.com/prometheus/common/expfmt
         google.golang.org/protobuf/encoding/prototext                from github.com/prometheus/common/expfmt+
@@ -95,14 +95,12 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
         tailscale.com/metrics                                        from tailscale.com/cmd/derper+
         tailscale.com/net/dnscache                                   from tailscale.com/derp/derphttp
-        tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
         tailscale.com/net/ktimeout                                   from tailscale.com/cmd/derper
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netknob                                    from tailscale.com/net/netns
      💣 tailscale.com/net/netmon                                     from tailscale.com/derp/derphttp+
         tailscale.com/net/netns                                      from tailscale.com/derp/derphttp
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale
-        tailscale.com/net/packet                                     from tailscale.com/wgengine/filter
         tailscale.com/net/sockstats                                  from tailscale.com/derp/derphttp
         tailscale.com/net/stun                                       from tailscale.com/net/stunserver
         tailscale.com/net/stunserver                                 from tailscale.com/cmd/derper
@@ -119,13 +117,13 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
    W    tailscale.com/tsconst                                        from tailscale.com/net/netmon
         tailscale.com/tstime                                         from tailscale.com/derp+
         tailscale.com/tstime/mono                                    from tailscale.com/tstime/rate
-        tailscale.com/tstime/rate                                    from tailscale.com/derp+
+        tailscale.com/tstime/rate                                    from tailscale.com/derp
         tailscale.com/tsweb                                          from tailscale.com/cmd/derper
         tailscale.com/tsweb/promvarz                                 from tailscale.com/tsweb
         tailscale.com/tsweb/varz                                     from tailscale.com/tsweb+
         tailscale.com/types/dnstype                                  from tailscale.com/tailcfg
         tailscale.com/types/empty                                    from tailscale.com/ipn
-        tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
+        tailscale.com/types/ipproto                                  from tailscale.com/tailcfg+
         tailscale.com/types/key                                      from tailscale.com/client/tailscale+
         tailscale.com/types/lazy                                     from tailscale.com/version+
         tailscale.com/types/logger                                   from tailscale.com/cmd/derper+
@@ -157,10 +155,10 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/syspolicy                                 from tailscale.com/ipn
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+
-   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo
+   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/derp+
         tailscale.com/version/distro                                 from tailscale.com/envknob+
-        tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
+        tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap
         golang.org/x/crypto/acme                                     from golang.org/x/crypto/acme/autocert
         golang.org/x/crypto/acme/autocert                            from tailscale.com/cmd/derper
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
@@ -175,6 +173,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         golang.org/x/crypto/nacl/box                                 from tailscale.com/types/key
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
+   W    golang.org/x/exp/constraints                                 from tailscale.com/util/winutil
    L    golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http

cmd/derper/derper_test.go

@@ -104,6 +104,8 @@ func TestDeps(t *testing.T) {
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
+			"tailscale.com/net/packet":           "not needed in derper",
+			"github.com/gaissmai/bart":           "not needed in derper",
 		},
 	}.Check(t)
 }

cmd/derpprobe/derpprobe.go

@@ -20,7 +20,7 @@ import (
 )
 
 var (
-	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://)")
+	derpMapURL   = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
 	versionFlag  = flag.Bool("version", false, "print version and exit")
 	listen       = flag.String("listen", ":8030", "HTTP listen address")
 	probeOnce    = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")

cmd/k8s-nameserver/main.go

@@ -153,11 +153,36 @@ func (n *nameserver) handleFunc() func(w dns.ResponseWriter, r *dns.Msg) {
 				m.Answer = append(m.Answer, rr)
 			}
 		case dns.TypeAAAA:
-			// TODO (irbekrm): implement IPv6 support.
-			// Kubernetes distributions that I am most familiar with
-			// default to IPv4 for Pod CIDR ranges and often many cases don't
-			// support IPv6 at all, so this should not be crucial for now.
-			fallthrough
+			// TODO (irbekrm): add IPv6 support.
+			// The nameserver currently does not support IPv6
+			// (records are not being created for IPv6 Pod addresses).
+			// However, we can expect that some callers will
+			// nevertheless send AAAA queries.
+			// We have to return NOERROR if a query is received for
+			// an AAAA record for a DNS name that we have an A
+			// record for- else the caller might not follow with an
+			// A record query.
+			// https://github.com/tailscale/tailscale/issues/12321
+			// https://datatracker.ietf.org/doc/html/rfc4074
+			q := r.Question[0].Name
+			fqdn, err := dnsname.ToFQDN(q)
+			if err != nil {
+				m = r.SetRcodeFormatError(r)
+				return
+			}
+			// The only supported use of this nameserver is as a
+			// single source of truth for MagicDNS names by
+			// non-tailnet Kubernetes workloads.
+			m.Authoritative = true
+			ips := n.lookupIP4(fqdn)
+			if len(ips) == 0 {
+				// As we are the authoritative nameserver for MagicDNS
+				// names, if we do not have a record for this MagicDNS
+				// name, it does not exist.
+				m = m.SetRcode(r, dns.RcodeNameError)
+				return
+			}
+			m.SetRcode(r, dns.RcodeSuccess)
 		default:
 			log.Printf("[unexpected] nameserver received a query for an unsupported record type: %s", r.Question[0].String())
 			m.SetRcode(r, dns.RcodeNotImplemented)

cmd/k8s-nameserver/main_test.go

@@ -79,7 +79,7 @@ func TestNameserver(t *testing.T) {
 				}},
 		},
 		{
-			name: "AAAA record query",
+			name: "AAAA record query, A record exists",
 			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
 			query: &dns.Msg{
 				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
@@ -88,26 +88,28 @@ func TestNameserver(t *testing.T) {
 			wantResp: &dns.Msg{
 				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
 				MsgHdr: dns.MsgHdr{
-					Id:       1,
-					Rcode:    dns.RcodeNotImplemented,
-					Response: true,
-					Opcode:   dns.OpcodeQuery,
+					Id:            1,
+					Rcode:         dns.RcodeSuccess,
+					Response:      true,
+					Opcode:        dns.OpcodeQuery,
+					Authoritative: true,
 				}},
 		},
 		{
-			name: "AAAA record query",
+			name: "AAAA record query, A record does not exist",
 			ip4:  map[dnsname.FQDN][]net.IP{dnsname.FQDN("foo.bar.com."): {{1, 2, 3, 4}}},
 			query: &dns.Msg{
-				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeAAAA}},
 				MsgHdr:   dns.MsgHdr{Id: 1},
 			},
 			wantResp: &dns.Msg{
-				Question: []dns.Question{{Name: "foo.bar.com", Qtype: dns.TypeAAAA}},
+				Question: []dns.Question{{Name: "baz.bar.com", Qtype: dns.TypeAAAA}},
 				MsgHdr: dns.MsgHdr{
-					Id:       1,
-					Rcode:    dns.RcodeNotImplemented,
-					Response: true,
-					Opcode:   dns.OpcodeQuery,
+					Id:            1,
+					Rcode:         dns.RcodeNameError,
+					Response:      true,
+					Opcode:        dns.OpcodeQuery,
+					Authoritative: true,
 				}},
 		},
 		{

cmd/k8s-operator/connector.go

@@ -33,11 +33,8 @@ import (
 
 const (
 	reasonConnectorCreationFailed = "ConnectorCreationFailed"
-
-	reasonConnectorCreated           = "ConnectorCreated"
-	reasonConnectorCleanupFailed     = "ConnectorCleanupFailed"
-	reasonConnectorCleanupInProgress = "ConnectorCleanupInProgress"
-	reasonConnectorInvalid           = "ConnectorInvalid"
+	reasonConnectorCreated        = "ConnectorCreated"
+	reasonConnectorInvalid        = "ConnectorInvalid"
 
 	messageConnectorCreationFailed = "Failed creating Connector: %v"
 	messageConnectorInvalid        = "Connector is invalid: %v"
@@ -108,7 +105,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 	}
 
 	oldCnStatus := cn.Status.DeepCopy()
-	setStatus := func(cn *tsapi.Connector, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+	setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
 			// An error encountered here should get returned by the Reconcile function.
@@ -184,7 +181,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 		Connector: &connector{
 			isExitNode: cn.Spec.ExitNode,
 		},
-		ProxyClass: proxyClass,
+		ProxyClassName: proxyClass,
 	}
 
 	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
@@ -211,7 +208,27 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
 	gaugeConnectorResources.Set(int64(connectors.Len()))
 
 	_, err := a.ssr.Provision(ctx, logger, sts)
-	return err
+	if err != nil {
+		return err
+	}
+
+	_, tsHost, ips, err := a.ssr.DeviceInfo(ctx, crl)
+	if err != nil {
+		return err
+	}
+
+	if tsHost == "" {
+		logger.Debugf("no Tailscale hostname known yet, waiting for connector pod to finish auth")
+		// No hostname yet. Wait for the connector pod to auth.
+		cn.Status.TailnetIPs = nil
+		cn.Status.Hostname = ""
+		return nil
+	}
+
+	cn.Status.TailnetIPs = ips
+	cn.Status.Hostname = tsHost
+
+	return nil
 }
 
 func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {

cmd/k8s-operator/connector_test.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/tstest"
+	"tailscale.com/util/mak"
 )
 
 func TestConnector(t *testing.T) {
@@ -29,7 +30,7 @@ func TestConnector(t *testing.T) {
 		},
 		TypeMeta: metav1.TypeMeta{
 			Kind:       tsapi.ConnectorKind,
-			APIVersion: "tailscale.io/v1alpha1",
+			APIVersion: "tailscale.com/v1alpha1",
 		},
 		Spec: tsapi.ConnectorSpec{
 			SubnetRouter: &tsapi.SubnetRouter{
@@ -74,9 +75,26 @@ func TestConnector(t *testing.T) {
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
+	// Connector status should get updated with the IP/hostname info when available.
+	const hostname = "foo.tailnetxyz.ts.net"
+	mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
+		mak.Set(&secret.Data, "device_id", []byte("1234"))
+		mak.Set(&secret.Data, "device_fqdn", []byte(hostname))
+		mak.Set(&secret.Data, "device_ips", []byte(`["127.0.0.1", "::1"]`))
+	})
+	expectReconciled(t, cr, "", "test")
+	cn.Finalizers = append(cn.Finalizers, "tailscale.com/finalizer")
+	cn.Status.IsExitNode = cn.Spec.ExitNode
+	cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
+	cn.Status.Hostname = hostname
+	cn.Status.TailnetIPs = []string{"127.0.0.1", "::1"}
+	expectEqual(t, fc, cn, func(o *tsapi.Connector) {
+		o.Status.Conditions = nil
+	})
+
 	// Add another route to be advertised.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
 		conn.Spec.SubnetRouter.AdvertiseRoutes = []tsapi.Route{"10.40.0.0/14", "10.44.0.0/20"}
@@ -152,7 +170,7 @@ func TestConnector(t *testing.T) {
 		subnetRoutes: "10.40.0.0/14",
 		hostname:     "test-connector",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
 	// Add an exit node.
@@ -237,7 +255,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
 		isExitNode:   true,
 		subnetRoutes: "10.40.0.0/14",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
 	// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet
@@ -254,9 +272,9 @@ func TestConnectorWithProxyClass(t *testing.T) {
 	// its resources.
 	mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
 		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []tsapi.ConnectorCondition{{
+			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               tsapi.ProxyClassready,
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -49,7 +49,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           {{- $operatorTag:= printf ":%s" ( .Values.operatorConfig.image.tag | default .Chart.AppVersion )}}
-          image: {{ .Values.operatorConfig.image.repo }}{{- if .Values.operatorConfig.image.digest -}}{{ printf "@%s" .Values.operatorConfig.image.digest}}{{- else -}}{{ printf "%s" $operatorTag }}{{- end }}
+          image: {{ coalesce .Values.operatorConfig.image.repo .Values.operatorConfig.image.repository }}{{- if .Values.operatorConfig.image.digest -}}{{ printf "@%s" .Values.operatorConfig.image.digest}}{{- else -}}{{ printf "%s" $operatorTag }}{{- end }}
           imagePullPolicy: {{ .Values.operatorConfig.image.pullPolicy }}
           env:
             - name: OPERATOR_INITIAL_TAGS
@@ -70,7 +70,7 @@ spec:
               value: /oauth/client_secret
             {{- $proxyTag := printf ":%s" ( .Values.proxyConfig.image.tag | default .Chart.AppVersion )}}
             - name: PROXY_IMAGE
-              value: {{ .Values.proxyConfig.image.repo }}{{- if .Values.proxyConfig.image.digest -}}{{ printf "@%s" .Values.proxyConfig.image.digest}}{{- else -}}{{ printf "%s" $proxyTag }}{{- end }}
+              value: {{ coalesce .Values.proxyConfig.image.repo .Values.proxyConfig.image.repository }}{{- if .Values.proxyConfig.image.digest -}}{{ printf "@%s" .Values.proxyConfig.image.digest}}{{- else -}}{{ printf "%s" $proxyTag }}{{- end }}
             - name: PROXY_TAGS
               value: {{ .Values.proxyConfig.defaultTags }}
             - name: APISERVER_PROXY

cmd/k8s-operator/deploy/chart/values.yaml

@@ -23,7 +23,8 @@ operatorConfig:
     - "tag:k8s-operator"
 
   image:
-    repo: tailscale/k8s-operator
+    # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/k8s-operator.
+    repository: tailscale/k8s-operator
     # Digest will be prioritized over tag. If neither are set appVersion will be
     # used.
     tag: ""
@@ -57,7 +58,8 @@ operatorConfig:
 # https://tailscale.com/kb/1236/kubernetes-operator#cluster-resource-customization-using-proxyclass-custom-resource
 proxyConfig:
   image:
-    repo: tailscale/tailscale
+    # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale.
+    repository: tailscale/tailscale
     # Digest will be prioritized over tag. If neither are set appVersion will be
     # used.
     tag: ""

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.13.0
+    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
   name: connectors.tailscale.com
 spec:
   group: tailscale.com
@@ -31,48 +31,95 @@ spec:
       name: v1alpha1
       schema:
         openAPIV3Schema:
-          description: 'Connector defines a Tailscale node that will be deployed in the cluster. The node can be configured to act as a Tailscale subnet router and/or a Tailscale exit node. Connector is a cluster-scoped resource. More info: https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource'
+          description: |-
+            Connector defines a Tailscale node that will be deployed in the cluster. The
+            node can be configured to act as a Tailscale subnet router and/or a Tailscale
+            exit node.
+            Connector is a cluster-scoped resource.
+            More info:
+            https://tailscale.com/kb/1236/kubernetes-operator#deploying-exit-nodes-and-subnet-routers-on-kubernetes-using-connector-custom-resource
           type: object
           required:
             - spec
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
             spec:
-              description: 'ConnectorSpec describes the desired Tailscale component. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
+              description: |-
+                ConnectorSpec describes the desired Tailscale component.
+                More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               properties:
                 exitNode:
-                  description: ExitNode defines whether the Connector node should act as a Tailscale exit node. Defaults to false. https://tailscale.com/kb/1103/exit-nodes
+                  description: |-
+                    ExitNode defines whether the Connector node should act as a
+                    Tailscale exit node. Defaults to false.
+                    https://tailscale.com/kb/1103/exit-nodes
                   type: boolean
                 hostname:
-                  description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
+                  description: |-
+                    Hostname is the tailnet hostname that should be assigned to the
+                    Connector node. If unset, hostname defaults to <connector
+                    name>-connector. Hostname can contain lower case letters, numbers and
+                    dashes, it must not start or end with a dash and must be between 2
+                    and 63 characters long.
                   type: string
                   pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
                 proxyClass:
-                  description: ProxyClass is the name of the ProxyClass custom resource that contains configuration options that should be applied to the resources created for this Connector. If unset, the operator will create resources with the default configuration.
+                  description: |-
+                    ProxyClass is the name of the ProxyClass custom resource that
+                    contains configuration options that should be applied to the
+                    resources created for this Connector. If unset, the operator will
+                    create resources with the default configuration.
                   type: string
                 subnetRouter:
-                  description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
+                  description: |-
+                    SubnetRouter defines subnet routes that the Connector node should
+                    expose to tailnet. If unset, none are exposed.
+                    https://tailscale.com/kb/1019/subnets/
                   type: object
                   required:
                     - advertiseRoutes
                   properties:
                     advertiseRoutes:
-                      description: AdvertiseRoutes refer to CIDRs that the subnet router should make available. Route values must be strings that represent a valid IPv4 or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. https://tailscale.com/kb/1201/4via6-subnets/
+                      description: |-
+                        AdvertiseRoutes refer to CIDRs that the subnet router should make
+                        available. Route values must be strings that represent a valid IPv4
+                        or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes.
+                        https://tailscale.com/kb/1201/4via6-subnets/
                       type: array
                       minItems: 1
                       items:
                         type: string
                         format: cidr
                 tags:
-                  description: Tags that the Tailscale node will be tagged with. Defaults to [tag:k8s]. To autoapprove the subnet routes or exit node defined by a Connector, you can configure Tailscale ACLs to give these tags the necessary permissions. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes. If you specify custom tags here, you must also make the operator an owner of these tags. See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector node has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                  description: |-
+                    Tags that the Tailscale node will be tagged with.
+                    Defaults to [tag:k8s].
+                    To autoapprove the subnet routes or exit node defined by a Connector,
+                    you can configure Tailscale ACLs to give these tags the necessary
+                    permissions.
+                    See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes.
+                    If you specify custom tags here, you must also make the operator an owner of these tags.
+                    See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+                    Tags cannot be changed once a Connector node has been created.
+                    Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
                   type: array
                   items:
                     type: string
@@ -81,48 +128,93 @@ spec:
                 - rule: has(self.subnetRouter) || self.exitNode == true
                   message: A Connector needs to be either an exit node or a subnet router, or both.
             status:
-              description: ConnectorStatus describes the status of the Connector. This is set and managed by the Tailscale operator.
+              description: |-
+                ConnectorStatus describes the status of the Connector. This is set
+                and managed by the Tailscale operator.
               type: object
               properties:
                 conditions:
-                  description: List of status conditions to indicate the status of the Connector. Known condition types are `ConnectorReady`.
+                  description: |-
+                    List of status conditions to indicate the status of the Connector.
+                    Known condition types are `ConnectorReady`.
                   type: array
                   items:
-                    description: ConnectorCondition contains condition information for a Connector.
+                    description: Condition contains details for one aspect of the current state of this API Resource.
                     type: object
                     required:
+                      - lastTransitionTime
+                      - message
+                      - reason
                       - status
                       - type
                     properties:
                       lastTransitionTime:
-                        description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
                         type: string
                         format: date-time
                       message:
-                        description: Message is a human readable description of the details of the last transition, complementing reason.
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
                         type: string
+                        maxLength: 32768
                       observedGeneration:
-                        description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
                         type: integer
                         format: int64
+                        minimum: 0
                       reason:
-                        description: Reason is a brief machine readable explanation for the condition's last transition.
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
                         type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
                       status:
-                        description: Status of the condition, one of ('True', 'False', 'Unknown').
+                        description: status of the condition, one of True, False, Unknown.
                         type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
                       type:
-                        description: Type of the condition, known values are (`SubnetRouterReady`).
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
                         type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                   x-kubernetes-list-map-keys:
                     - type
                   x-kubernetes-list-type: map
+                hostname:
+                  description: |-
+                    Hostname is the fully qualified domain name of the Connector node.
+                    If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
+                    node.
+                  type: string
                 isExitNode:
                   description: IsExitNode is set to true if the Connector acts as an exit node.
                   type: boolean
                 subnetRoutes:
-                  description: SubnetRoutes are the routes currently exposed to tailnet via this Connector instance.
+                  description: |-
+                    SubnetRoutes are the routes currently exposed to tailnet via this
+                    Connector instance.
                   type: string
+                tailnetIPs:
+                  description: |-
+                    TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
+                    assigned to the Connector node.
+                  type: array
+                  items:
+                    type: string
       served: true
       storage: true
       subresources:

cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.13.0
+    controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
   name: dnsconfigs.tailscale.com
 spec:
   group: tailscale.com
@@ -23,72 +23,157 @@ spec:
       name: v1alpha1
       schema:
         openAPIV3Schema:
+          description: |-
+            DNSConfig can be deployed to cluster to make a subset of Tailscale MagicDNS
+            names resolvable by cluster workloads. Use this if: A) you need to refer to
+            tailnet services, exposed to cluster via Tailscale Kubernetes operator egress
+            proxies by the MagicDNS names of those tailnet services (usually because the
+            services run over HTTPS)
+            B) you have exposed a cluster workload to the tailnet using Tailscale Ingress
+            and you also want to refer to the workload from within the cluster over the
+            Ingress's MagicDNS name (usually because you have some callback component
+            that needs to use the same URL as that used by a non-cluster client on
+            tailnet).
+            When a DNSConfig is applied to a cluster, Tailscale Kubernetes operator will
+            deploy a nameserver for ts.net DNS names and automatically populate it with records
+            for any Tailscale egress or Ingress proxies deployed to that cluster.
+            Currently you must manually update your cluster DNS configuration to add the
+            IP address of the deployed nameserver as a ts.net stub nameserver.
+            Instructions for how to do it:
+            https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configuration-of-stub-domain-and-upstream-nameserver-using-coredns (for CoreDNS),
+            https://cloud.google.com/kubernetes-engine/docs/how-to/kube-dns (for kube-dns).
+            Tailscale Kubernetes operator will write the address of a Service fronting
+            the nameserver to dsnconfig.status.nameserver.ip.
+            DNSConfig is a singleton - you must not create more than one.
+            NB: if you want cluster workloads to be able to refer to Tailscale Ingress
+            using its MagicDNS name, you must also annotate the Ingress resource with
+            tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation to
+            ensure that the proxy created for the Ingress listens on its Pod IP address.
+            NB: Clusters where Pods get assigned IPv6 addresses only are currently not supported.
           type: object
           required:
             - spec
           properties:
             apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
               type: string
             kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
               type: string
             metadata:
               type: object
             spec:
+              description: |-
+                Spec describes the desired DNS configuration.
+                More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
               type: object
               required:
                 - nameserver
               properties:
                 nameserver:
+                  description: |-
+                    Configuration for a nameserver that can resolve ts.net DNS names
+                    associated with in-cluster proxies for Tailscale egress Services and
+                    Tailscale Ingresses. The operator will always deploy this nameserver
+                    when a DNSConfig is applied.
                   type: object
                   properties:
                     image:
+                      description: Nameserver image.
                       type: object
                       properties:
                         repo:
+                          description: Repo defaults to tailscale/k8s-nameserver.
                           type: string
                         tag:
+                          description: Tag defaults to operator's own tag.
                           type: string
             status:
+              description: |-
+                Status describes the status of the DNSConfig. This is set
+                and managed by the Tailscale operator.
               type: object
               properties:
                 conditions:
                   type: array
                   items:
-                    description: ConnectorCondition contains condition information for a Connector.
+                    description: Condition contains details for one aspect of the current state of this API Resource.
                     type: object
                     required:
+                      - lastTransitionTime
+                      - message
+                      - reason
                       - status
                       - type
                     properties:
                       lastTransitionTime:
-                        description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
                         type: string
                         format: date-time
                       message:
-                        description: Message is a human readable description of the details of the last transition, complementing reason.
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
                         type: string
+                        maxLength: 32768
                       observedGeneration:
-                        description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
                         type: integer
                         format: int64
+                        minimum: 0
                       reason:
-                        description: Reason is a brief machine readable explanation for the condition's last transition.
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
                         type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
                       status:
-                        description: Status of the condition, one of ('True', 'False', 'Unknown').
+                        description: status of the condition, one of True, False, Unknown.
                         type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
                       type:
-                        description: Type of the condition, known values are (`SubnetRouterReady`).
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
                         type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                   x-kubernetes-list-map-keys:
                     - type
                   x-kubernetes-list-type: map
                 nameserver:
+                  description: Nameserver describes the status of nameserver cluster resources.
                   type: object
                   properties:
                     ip:
+                      description: |-
+                        IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
+                        Currently you must manually update your cluster DNS config to add
+                        this address as a stub nameserver for ts.net for cluster workloads to be
+                        able to resolve MagicDNS names associated with egress or Ingress
+                        proxies.
+                        The IP address will change if you delete and recreate the DNSConfig.
                       type: string
       served: true
       storage: true

cmd/k8s-operator/deploy/examples/dnsconfig.yaml

@@ -3,7 +3,4 @@ kind: DNSConfig
 metadata:
   name: ts-dns
 spec:
-  nameserver:
-    image:
-      repo: tailscale/k8s-nameserver
-      tag: unstable-v1.65
+  nameserver: {}

cmd/k8s-operator/deploy/examples/proxyclass.yaml

@@ -15,3 +15,9 @@ spec:
         kubernetes.io/os: "linux"
       imagePullSecrets:
       - name: "foo"
+      tailscaleContainer:
+        image: "ghcr.io/tailscale/tailscale:v1.64"
+        imagePullPolicy: IfNotPresent
+      tailscaleInitContainer:
+        image: "ghcr.io/tailscale/tailscale:v1.64"
+        imagePullPolicy: IfNotPresent

cmd/k8s-operator/ingress.go

@@ -264,7 +264,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		ServeConfig:         sc,
 		Tags:                tags,
 		ChildResourceLabels: crl,
-		ProxyClass:          proxyClass,
+		ProxyClassName:      proxyClass,
 	}
 
 	if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

cmd/k8s-operator/ingress_test.go

@@ -100,7 +100,7 @@ func TestTailscaleIngress(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
 
@@ -231,7 +231,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	}
 	opts.serveConfig = serveConfig
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
 	expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
 
@@ -248,9 +248,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
 	// created proxy resources.
 	mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
 		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []tsapi.ConnectorCondition{{
+			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               tsapi.ProxyClassready,
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})

cmd/k8s-operator/nameserver.go

@@ -101,7 +101,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 	}
 
 	oldCnStatus := dnsCfg.Status.DeepCopy()
-	setStatus := func(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConnectorConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
+	setStatus := func(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
 		tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, a.clock, logger)
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, dnsCfg.Status) {
 			// An error encountered here should get returned by the Reconcile function.

cmd/k8s-operator/nameserver_test.go

@@ -81,12 +81,12 @@ func TestNameserverReconciler(t *testing.T) {
 		IP: "1.2.3.4",
 	}
 	dnsCfg.Finalizers = []string{FinalizerName}
-	dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, tsapi.ConnectorCondition{
-		Type:               tsapi.NameserverReady,
+	dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{
+		Type:               string(tsapi.NameserverReady),
 		Status:             metav1.ConditionTrue,
 		Reason:             reasonNameserverCreated,
 		Message:            reasonNameserverCreated,
-		LastTransitionTime: &metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
 	})
 	expectEqual(t, fc, dnsCfg, nil)
 

cmd/k8s-operator/operator.go

@@ -278,6 +278,7 @@ func runReconcilers(opts reconcilerOpts) {
 			isDefaultLoadBalancer: opts.proxyActAsDefaultLoadBalancer,
 			recorder:              eventRecorder,
 			tsNamespace:           opts.tailscaleNamespace,
+			clock:                 tstime.DefaultClock{},
 		})
 	if err != nil {
 		startlog.Fatalf("could not create service reconciler: %v", err)

cmd/k8s-operator/operator_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
@@ -17,10 +18,13 @@ import (
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/net/dns/resolvconffile"
+	"tailscale.com/tstest"
+	"tailscale.com/tstime"
 	"tailscale.com/types/ptr"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
@@ -33,6 +37,7 @@ func TestLoadBalancerClass(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -42,11 +47,13 @@ func TestLoadBalancerClass(t *testing.T) {
 			operatorNamespace: "operator-ns",
 			proxyImage:        "tailscale/tailscale",
 		},
-		logger: zl.Sugar(),
+		logger:   zl.Sugar(),
+		clock:    clock,
+		recorder: record.NewFakeRecorder(100),
 	}
 
-	// Create a service that we should manage, and check that the initial round
-	// of objects looks right.
+	// Create a service that we should manage, but start with a miconfiguration
+	// in the annotations.
 	mustCreate(t, fc, &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
@@ -55,6 +62,9 @@ func TestLoadBalancerClass(t *testing.T) {
 			// doesn't. So, set it explicitly because other code later depends
 			// on it being set.
 			UID: types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: "invalid.example.com",
+			},
 		},
 		Spec: corev1.ServiceSpec{
 			ClusterIP:         "10.20.30.40",
@@ -65,6 +75,46 @@ func TestLoadBalancerClass(t *testing.T) {
 
 	expectReconciled(t, sr, "default", "test")
 
+	// The expected value of .status.conditions[0].LastTransitionTime until the
+	// proxy becomes ready.
+	t0 := conditionTime(clock)
+
+	// Should have an error about invalid config.
+	want := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "default",
+			UID:       types.UID("1234-UID"),
+			Annotations: map[string]string{
+				AnnotationTailnetTargetFQDN: "invalid.example.com",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP:         "10.20.30.40",
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+		},
+		Status: corev1.ServiceStatus{
+			Conditions: []metav1.Condition{{
+				Type:               string(tsapi.ProxyReady),
+				Status:             metav1.ConditionFalse,
+				LastTransitionTime: t0,
+				Reason:             reasonProxyInvalid,
+				Message:            `unable to provision proxy resources: invalid Service: invalid value of annotation tailscale.com/tailnet-fqdn: "invalid.example.com" does not appear to be a valid MagicDNS name`,
+			}},
+		},
+	}
+	expectEqual(t, fc, want, nil)
+
+	// Delete the misconfiguration so the proxy starts getting created on the
+	// next reconcile.
+	mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
+		s.ObjectMeta.Annotations = nil
+	})
+
+	clock.Advance(time.Second)
+	expectReconciled(t, sr, "default", "test")
+
 	fullName, shortName := findGenName(t, fc, "default", "test", "svc")
 	opts := configOpts{
 		stsName:         shortName,
@@ -75,10 +125,23 @@ func TestLoadBalancerClass(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
+	want.Annotations = nil
+	want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
+	want.Status = corev1.ServiceStatus{
+		Conditions: []metav1.Condition{{
+			Type:               string(tsapi.ProxyReady),
+			Status:             metav1.ConditionFalse,
+			LastTransitionTime: t0, // Status is still false, no update to transition time
+			Reason:             reasonProxyPending,
+			Message:            "no Tailscale hostname known yet, waiting for proxy pod to finish auth",
+		}},
+	}
+	expectEqual(t, fc, want, nil)
+
 	// Normally the Tailscale proxy pod would come up here and write its info
 	// into the secret. Simulate that, then verify reconcile again and verify
 	// that we get to the end.
@@ -90,33 +153,16 @@ func TestLoadBalancerClass(t *testing.T) {
 		s.Data["device_fqdn"] = []byte("tailscale.device.name.")
 		s.Data["device_ips"] = []byte(`["100.99.98.97", "2c0a:8083:94d4:2012:3165:34a5:3616:5fdf"]`)
 	})
+	clock.Advance(time.Second)
 	expectReconciled(t, sr, "default", "test")
-	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:       "test",
-			Namespace:  "default",
-			Finalizers: []string{"tailscale.com/finalizer"},
-			UID:        types.UID("1234-UID"),
-		},
-		Spec: corev1.ServiceSpec{
-			ClusterIP:         "10.20.30.40",
-			Type:              corev1.ServiceTypeLoadBalancer,
-			LoadBalancerClass: ptr.To("tailscale"),
-		},
-		Status: corev1.ServiceStatus{
-			LoadBalancer: corev1.LoadBalancerStatus{
-				Ingress: []corev1.LoadBalancerIngress{
-					{
-						Hostname: "tailscale.device.name",
-					},
-					{
-						IP: "100.99.98.97",
-					},
-				},
+	want.Status.Conditions = proxyCreatedCondition(clock)
+	want.Status.LoadBalancer = corev1.LoadBalancerStatus{
+		Ingress: []corev1.LoadBalancerIngress{
+			{
+				Hostname: "tailscale.device.name",
+			},
+			{
+				IP: "100.99.98.97",
 			},
 		},
 	}
@@ -144,11 +190,9 @@ func TestLoadBalancerClass(t *testing.T) {
 	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
+
+	// Note that the Tailscale-specific condition status should be gone now.
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -170,6 +214,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		t.Fatal(err)
 	}
 	tailnetTargetFQDN := "foo.bar.ts.net."
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -180,6 +225,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -216,14 +262,10 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 		hostname:          "default-test",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -238,9 +280,12 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
 			Type:         corev1.ServiceTypeExternalName,
 			Selector:     nil,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -280,6 +325,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		t.Fatal(err)
 	}
 	tailnetTargetIP := "100.66.66.66"
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -290,6 +336,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -326,14 +373,10 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 		hostname:        "default-test",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -348,9 +391,12 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
 			Type:         corev1.ServiceTypeExternalName,
 			Selector:     nil,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -389,6 +435,7 @@ func TestAnnotations(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -399,6 +446,7 @@ func TestAnnotations(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -433,14 +481,10 @@ func TestAnnotations(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -454,6 +498,9 @@ func TestAnnotations(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 
@@ -473,10 +520,6 @@ func TestAnnotations(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -497,6 +540,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -507,6 +551,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -541,7 +586,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -558,10 +603,6 @@ func TestAnnotationIntoLB(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -575,6 +616,9 @@ func TestAnnotationIntoLB(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 
@@ -592,10 +636,6 @@ func TestAnnotationIntoLB(t *testing.T) {
 	// ... but the service should have a LoadBalancer status.
 
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -618,6 +658,7 @@ func TestAnnotationIntoLB(t *testing.T) {
 					},
 				},
 			},
+			Conditions: proxyCreatedCondition(clock),
 		},
 	}
 	expectEqual(t, fc, want, nil)
@@ -630,6 +671,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -640,6 +682,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -672,7 +715,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
@@ -689,10 +732,6 @@ func TestLBIntoAnnotation(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -715,6 +754,7 @@ func TestLBIntoAnnotation(t *testing.T) {
 					},
 				},
 			},
+			Conditions: proxyCreatedCondition(clock),
 		},
 	}
 	expectEqual(t, fc, want, nil)
@@ -740,10 +780,6 @@ func TestLBIntoAnnotation(t *testing.T) {
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -757,6 +793,9 @@ func TestLBIntoAnnotation(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 }
@@ -768,6 +807,7 @@ func TestCustomHostname(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -778,6 +818,7 @@ func TestCustomHostname(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -813,14 +854,10 @@ func TestCustomHostname(t *testing.T) {
 		clusterTargetIP: "10.20.30.40",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, o), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, o), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
 	want := &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       "test",
 			Namespace:  "default",
@@ -835,6 +872,9 @@ func TestCustomHostname(t *testing.T) {
 			ClusterIP: "10.20.30.40",
 			Type:      corev1.ServiceTypeClusterIP,
 		},
+		Status: corev1.ServiceStatus{
+			Conditions: proxyCreatedCondition(clock),
+		},
 	}
 	expectEqual(t, fc, want, nil)
 
@@ -854,10 +894,6 @@ func TestCustomHostname(t *testing.T) {
 	expectMissing[corev1.Service](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 	want = &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test",
 			Namespace: "default",
@@ -881,6 +917,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -892,6 +929,7 @@ func TestCustomPriorityClassName(t *testing.T) {
 			proxyPriorityClassName: "custom-priority-class-name",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// Create a service that we should manage, and check that the initial round
@@ -935,10 +973,14 @@ func TestProxyClassForService(t *testing.T) {
 	// Setup
 	pc := &tsapi.ProxyClass{
 		ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
-		Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
-			Labels:      map[string]string{"foo": "bar"},
-			Annotations: map[string]string{"bar.io/foo": "some-val"},
-			Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
+		Spec: tsapi.ProxyClassSpec{
+			TailscaleConfig: &tsapi.TailscaleConfig{
+				AcceptRoutes: true,
+			},
+			StatefulSet: &tsapi.StatefulSet{
+				Labels:      map[string]string{"foo": "bar"},
+				Annotations: map[string]string{"bar.io/foo": "some-val"},
+				Pod:         &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
 	}
 	fc := fake.NewClientBuilder().
 		WithScheme(tsapi.GlobalScheme).
@@ -950,6 +992,7 @@ func TestProxyClassForService(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -960,6 +1003,7 @@ func TestProxyClassForService(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// 1. A new tailscale LoadBalancer Service is created without any
@@ -989,7 +1033,7 @@ func TestProxyClassForService(t *testing.T) {
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
 	}
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
@@ -1001,21 +1045,23 @@ func TestProxyClassForService(t *testing.T) {
 	})
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 
 	// 3. ProxyClass is set to Ready, the Service gets reconciled by the
 	// services-reconciler and the customization from the ProxyClass is
 	// applied to the proxy resources.
 	mustUpdateStatus(t, fc, "", "custom-metadata", func(pc *tsapi.ProxyClass) {
 		pc.Status = tsapi.ProxyClassStatus{
-			Conditions: []tsapi.ConnectorCondition{{
+			Conditions: []metav1.Condition{{
 				Status:             metav1.ConditionTrue,
-				Type:               tsapi.ProxyClassready,
+				Type:               string(tsapi.ProxyClassready),
 				ObservedGeneration: pc.Generation,
 			}}}
 	})
 	opts.proxyClass = pc.Name
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
 
 	// 4. tailscale.com/proxy-class label is removed from the Service, the
 	// configuration from the ProxyClass is removed from the cluster
@@ -1035,6 +1081,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1045,6 +1092,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger:                zl.Sugar(),
+		clock:                 clock,
 		isDefaultLoadBalancer: true,
 	}
 
@@ -1089,6 +1137,7 @@ func TestProxyFirewallMode(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1100,6 +1149,7 @@ func TestProxyFirewallMode(t *testing.T) {
 			tsFirewallMode:    "nftables",
 		},
 		logger:                zl.Sugar(),
+		clock:                 clock,
 		isDefaultLoadBalancer: true,
 	}
 
@@ -1142,6 +1192,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1152,6 +1203,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger:                zl.Sugar(),
+		clock:                 clock,
 		isDefaultLoadBalancer: true,
 	}
 
@@ -1433,6 +1485,7 @@ func Test_externalNameService(t *testing.T) {
 
 	// 1. A External name Service that should be exposed via Tailscale gets
 	// created.
+	clock := tstest.NewClock(tstest.ClockOpts{})
 	sr := &ServiceReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -1443,6 +1496,7 @@ func Test_externalNameService(t *testing.T) {
 			proxyImage:        "tailscale/tailscale",
 		},
 		logger: zl.Sugar(),
+		clock:  clock,
 	}
 
 	// 1. Create an ExternalName Service that we should manage, and check that the initial round
@@ -1477,7 +1531,7 @@ func Test_externalNameService(t *testing.T) {
 		clusterTargetDNS: "foo.com",
 	}
 
-	expectEqual(t, fc, expectedSecret(t, opts), nil)
+	expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
 	expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
 	expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
 
@@ -1498,3 +1552,18 @@ func toFQDN(t *testing.T, s string) dnsname.FQDN {
 	}
 	return fqdn
 }
+
+func proxyCreatedCondition(clock tstime.Clock) []metav1.Condition {
+	return []metav1.Condition{{
+		Type:               string(tsapi.ProxyReady),
+		Status:             metav1.ConditionTrue,
+		ObservedGeneration: 0,
+		LastTransitionTime: conditionTime(clock),
+		Reason:             reasonProxyCreated,
+		Message:            reasonProxyCreated,
+	}}
+}
+
+func conditionTime(clock tstime.Clock) metav1.Time {
+	return metav1.NewTime(clock.Now().Truncate(time.Second))
+}

cmd/k8s-operator/proxy.go

@@ -20,6 +20,7 @@ import (
 	"k8s.io/client-go/transport"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
+	tskube "tailscale.com/kube"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
 	"tailscale.com/util/clientmetric"
@@ -207,32 +208,24 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 }
 
 const (
-	capabilityName    = "tailscale.com/cap/kubernetes"
-	oldCapabilityName = "https://" + capabilityName
+	// oldCapabilityName is a legacy form of
+	// tailfcg.PeerCapabilityKubernetes capability. The only capability rule
+	// that is respected for this form is group impersonation - for
+	// backwards compatibility reasons.
+	// TODO (irbekrm): determine if anyone uses this and remove if possible.
+	oldCapabilityName = "https://" + tailcfg.PeerCapabilityKubernetes
 )
 
-type capRule struct {
-	// Impersonate is a list of rules that specify how to impersonate the caller
-	// when proxying to the Kubernetes API.
-	Impersonate *impersonateRule `json:"impersonate,omitempty"`
-}
-
-// TODO(maisem): move this to some well-known location so that it can be shared
-// with control.
-type impersonateRule struct {
-	Groups []string `json:"groups,omitempty"`
-}
-
 // addImpersonationHeaders adds the appropriate headers to r to impersonate the
 // caller when proxying to the Kubernetes API. It uses the WhoIsResponse stashed
 // in the context by the apiserverProxy.
 func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	log = log.With("remote", r.RemoteAddr)
 	who := whoIsKey.Value(r.Context())
-	rules, err := tailcfg.UnmarshalCapJSON[capRule](who.CapMap, capabilityName)
+	rules, err := tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, tailcfg.PeerCapabilityKubernetes)
 	if len(rules) == 0 && err == nil {
 		// Try the old capability name for backwards compatibility.
-		rules, err = tailcfg.UnmarshalCapJSON[capRule](who.CapMap, oldCapabilityName)
+		rules, err = tailcfg.UnmarshalCapJSON[tskube.KubernetesCapRule](who.CapMap, oldCapabilityName)
 	}
 	if err != nil {
 		return fmt.Errorf("failed to unmarshal capability: %v", err)

cmd/k8s-operator/proxy_test.go

@@ -49,7 +49,7 @@ func TestImpersonationHeaders(t *testing.T) {
 			name:     "user-with-cap",
 			emailish: "foo@example.com",
 			capMap: tailcfg.PeerCapMap{
-				capabilityName: {
+				tailcfg.PeerCapabilityKubernetes: {
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group1","group2"]}}`),
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group1","group3"]}}`), // One group is duplicated.
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group4"]}}`),
@@ -71,7 +71,7 @@ func TestImpersonationHeaders(t *testing.T) {
 			emailish: "tagged-device",
 			tags:     []string{"tag:foo", "tag:bar"},
 			capMap: tailcfg.PeerCapMap{
-				capabilityName: {
+				tailcfg.PeerCapabilityKubernetes: {
 					tailcfg.RawMessage(`{"impersonate":{"groups":["group1"]}}`),
 				},
 			},
@@ -80,12 +80,26 @@ func TestImpersonationHeaders(t *testing.T) {
 				"Impersonate-User":  {"node.ts.net"},
 			},
 		},
+		{
+			name:     "mix-of-caps",
+			emailish: "tagged-device",
+			tags:     []string{"tag:foo", "tag:bar"},
+			capMap: tailcfg.PeerCapMap{
+				tailcfg.PeerCapabilityKubernetes: {
+					tailcfg.RawMessage(`{"impersonate":{"groups":["group1"]},"recorder":["tag:foo"],"enforceRecorder":true}`),
+				},
+			},
+			wantHeaders: http.Header{
+				"Impersonate-Group": {"group1"},
+				"Impersonate-User":  {"node.ts.net"},
+			},
+		},
 		{
 			name:     "bad-cap",
 			emailish: "tagged-device",
 			tags:     []string{"tag:foo", "tag:bar"},
 			capMap: tailcfg.PeerCapMap{
-				capabilityName: {
+				tailcfg.PeerCapabilityKubernetes: {
 					tailcfg.RawMessage(`[]`),
 				},
 			},

cmd/k8s-operator/proxyclass.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"strings"
 
+	dockerref "github.com/distribution/reference"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
@@ -111,6 +112,20 @@ func (a *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.
 						a.recorder.Event(pc, corev1.EventTypeWarning, reasonCustomTSEnvVar, fmt.Sprintf(messageCustomTSEnvVar, string(e.Name), "tailscale"))
 					}
 				}
+				if tc.Image != "" {
+					// Same validation as used by kubelet https://github.com/kubernetes/kubernetes/blob/release-1.30/pkg/kubelet/images/image_manager.go#L212
+					if _, err := dockerref.ParseNormalizedNamed(tc.Image); err != nil {
+						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleContainer", "image"), tc.Image, err.Error()))
+					}
+				}
+			}
+			if tc := pod.TailscaleInitContainer; tc != nil {
+				if tc.Image != "" {
+					// Same validation as used by kubelet https://github.com/kubernetes/kubernetes/blob/release-1.30/pkg/kubelet/images/image_manager.go#L212
+					if _, err := dockerref.ParseNormalizedNamed(tc.Image); err != nil {
+						violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
+					}
+				}
 			}
 		}
 	}

cmd/k8s-operator/proxyclass_test.go

@@ -36,9 +36,13 @@ func TestProxyClass(t *testing.T) {
 				Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
 				Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
 				Pod: &tsapi.Pod{
-					Labels:             map[string]string{"foo": "bar", "xyz1234": "abc567"},
-					Annotations:        map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
-					TailscaleContainer: &tsapi.Container{Env: []tsapi.Env{{Name: "FOO", Value: "BAR"}}},
+					Labels:      map[string]string{"foo": "bar", "xyz1234": "abc567"},
+					Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
+					TailscaleContainer: &tsapi.Container{
+						Env:             []tsapi.Env{{Name: "FOO", Value: "BAR"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.my-repo/tailscale:v0.01testsomething",
+					},
 				},
 			},
 		},
@@ -63,17 +67,17 @@ func TestProxyClass(t *testing.T) {
 
 	// 1. A valid ProxyClass resource gets its status updated to Ready.
 	expectReconciled(t, pcr, "", "test")
-	pc.Status.Conditions = append(pc.Status.Conditions, tsapi.ConnectorCondition{
-		Type:               tsapi.ProxyClassready,
+	pc.Status.Conditions = append(pc.Status.Conditions, metav1.Condition{
+		Type:               string(tsapi.ProxyClassready),
 		Status:             metav1.ConditionTrue,
 		Reason:             reasonProxyClassValid,
 		Message:            reasonProxyClassValid,
-		LastTransitionTime: &metav1.Time{Time: cl.Now().Truncate(time.Second)},
+		LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
 	})
 
 	expectEqual(t, fc, pc, nil)
 
-	// 2. An invalid ProxyClass resource gets its status updated to Invalid.
+	// 2. A ProxyClass resource with invalid labels gets its status updated to Invalid with an error message.
 	pc.Spec.StatefulSet.Labels["foo"] = "?!someVal"
 	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
 		proxyClass.Spec.StatefulSet.Labels = pc.Spec.StatefulSet.Labels
@@ -85,9 +89,43 @@ func TestProxyClass(t *testing.T) {
 	expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue',  or 'my_value',  or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
 	expectEvents(t, fr, []string{expectedEvent})
 
-	// 2. An valid ProxyClass but with a Tailscale env vars set results in warning events.
+	// 3. A ProxyClass resource with invalid image reference gets it status updated to Invalid with an error message.
+	pc.Spec.StatefulSet.Labels = nil
+	pc.Spec.StatefulSet.Pod.TailscaleContainer.Image = "FOO bar"
 	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
-		proxyClass.Spec.StatefulSet.Labels = nil // unset invalid labels from the previous test
+		proxyClass.Spec.StatefulSet.Labels = nil
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleContainer.Image
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 4. A ProxyClass resource with invalid init container image reference gets it status updated to Invalid with an error message.
+	pc.Spec.StatefulSet.Labels = nil
+	pc.Spec.StatefulSet.Pod.TailscaleContainer.Image = ""
+	pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{
+		Image: "FOO bar",
+	}
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleContainer.Image
+		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{
+			Image: pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image,
+		}
+	})
+	expectReconciled(t, pcr, "", "test")
+	msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassready, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
+	expectEqual(t, fc, pc, nil)
+	expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
+	expectEvents(t, fr, []string{expectedEvent})
+
+	// 5. An valid ProxyClass but with a Tailscale env vars set results in warning events.
+	pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = "" // unset previous test
+	mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
+		proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image
 		proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}}
 	})
 	expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.",

cmd/k8s-operator/sts.go

@@ -29,14 +29,12 @@ import (
 	"sigs.k8s.io/yaml"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
-	kubeutils "tailscale.com/k8s-operator"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/opt"
 	"tailscale.com/types/ptr"
-	"tailscale.com/util/dnsname"
 	"tailscale.com/util/mak"
 )
 
@@ -125,7 +123,9 @@ type tailscaleSTSConfig struct {
 	// what this StatefulSet should be created for.
 	Connector *connector
 
-	ProxyClass string
+	ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy
+
+	ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
 }
 
 type connector struct {
@@ -171,6 +171,18 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
 		return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
 	}
 
+	proxyClass := new(tsapi.ProxyClass)
+	if sts.ProxyClassName != "" {
+		if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClassName}, proxyClass); err != nil {
+			return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
+		}
+		if !tsoperator.ProxyClassIsReady(proxyClass) {
+			logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
+			return nil, nil
+		}
+	}
+	sts.ProxyClass = proxyClass
+
 	secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
@@ -346,7 +358,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 	latest := tailcfg.CapabilityVersion(-1)
 	var latestConfig ipn.ConfigVAlpha
 	for key, val := range configs {
-		fn := kubeutils.TailscaledConfigFileNameForCap(key)
+		fn := tsoperator.TailscaledConfigFileNameForCap(key)
 		b, err := json.Marshal(val)
 		if err != nil {
 			return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
@@ -393,8 +405,10 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
 	return string(sanitizedBytes)
 }
 
-// DeviceInfo returns the device ID and hostname for the Tailscale device
-// associated with the given labels.
+// DeviceInfo returns the device ID, hostname and IPs for the Tailscale device
+// that acts as an operator proxy. It retrieves info from a Kubernetes Secret
+// labeled with the provided labels.
+// Either of device ID, hostname and IPs can be empty string if not found in the Secret.
 func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
 	sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
 	if err != nil {
@@ -411,7 +425,12 @@ func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map
 	// to remove it.
 	hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
 	if hostname == "" {
-		return "", "", nil, nil
+		// Device ID gets stored and retrieved in a different flow than
+		// FQDN and IPs. A device that acts as Kubernetes operator
+		// proxy, but whose route setup has failed might have an device
+		// ID, but no FQDN/IPs. If so, return the ID, to allow the
+		// operator to clean up such devices.
+		return id, "", nil, nil
 	}
 	if rawDeviceIPs, ok := sec.Data["device_ips"]; ok {
 		if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
@@ -465,16 +484,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	}
 	pod := &ss.Spec.Template
 	container := &pod.Spec.Containers[0]
-	proxyClass := new(tsapi.ProxyClass)
-	if sts.ProxyClass != "" {
-		if err := a.Get(ctx, types.NamespacedName{Name: sts.ProxyClass}, proxyClass); err != nil {
-			return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
-		}
-		if !tsoperator.ProxyClassIsReady(proxyClass) {
-			logger.Infof("ProxyClass %s specified for the proxy, but it is not (yet) in a ready state, waiting..")
-			return nil, nil
-		}
-	}
 	container.Image = a.proxyImage
 	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:      headlessSvc.Name,
@@ -589,9 +598,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 		})
 	}
 	logger.Debugf("reconciling statefulset %s/%s", ss.GetNamespace(), ss.GetName())
-	if sts.ProxyClass != "" {
-		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClass)
-		ss = applyProxyClassToStatefulSet(proxyClass, ss, sts, logger)
+	if sts.ProxyClassName != "" {
+		logger.Debugf("configuring proxy resources with ProxyClass %s", sts.ProxyClassName)
+		ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger)
 	}
 	updateSS := func(s *appsv1.StatefulSet) {
 		s.Spec = ss.Spec
@@ -691,6 +700,12 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
 			// in the env var list overrides an earlier one.
 			base.Env = append(base.Env, corev1.EnvVar{Name: string(e.Name), Value: e.Value})
 		}
+		if overlay.Image != "" {
+			base.Image = overlay.Image
+		}
+		if overlay.ImagePullPolicy != "" {
+			base.ImagePullPolicy = overlay.ImagePullPolicy
+		}
 		return base
 	}
 	for i, c := range ss.Spec.Template.Spec.Containers {
@@ -765,6 +780,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 		}
 		conf.AdvertiseRoutes = routes
 	}
+	if shouldAcceptRoutes(stsC.ProxyClass) {
+		conf.AcceptRoutes = "true"
+	}
+
 	if newAuthkey != "" {
 		conf.AuthKey = &newAuthkey
 	} else if oldSecret != nil {
@@ -776,7 +795,7 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 			if len(data) == 0 {
 				continue
 			}
-			v, err := kubeutils.CapVerFromFileName(k)
+			v, err := tsoperator.CapVerFromFileName(k)
 			if err != nil {
 				continue
 			}
@@ -803,6 +822,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 	return capVerConfigs, nil
 }
 
+func shouldAcceptRoutes(pc *tsapi.ProxyClass) bool {
+	return pc != nil && pc.Spec.TailscaleConfig != nil && pc.Spec.TailscaleConfig.AcceptRoutes
+}
+
 // ptrObject is a type constraint for pointer types that implement
 // client.Object.
 type ptrObject[T any] interface {
@@ -923,14 +946,11 @@ func defaultEnv(envName, defVal string) string {
 	return v
 }
 
-func nameForService(svc *corev1.Service) (string, error) {
+func nameForService(svc *corev1.Service) string {
 	if h, ok := svc.Annotations[AnnotationHostname]; ok {
-		if err := dnsname.ValidLabel(h); err != nil {
-			return "", fmt.Errorf("invalid Tailscale hostname %q: %w", h, err)
-		}
-		return h, nil
+		return h
 	}
-	return svc.Namespace + "-" + svc.Name, nil
+	return svc.Namespace + "-" + svc.Name
 }
 
 func isValidFirewallMode(m string) bool {

cmd/k8s-operator/sts_test.go

@@ -81,7 +81,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 							Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
 							Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
 						},
-						Env: []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						Env:             []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.io/my-repo/tailscale:v0.01testsomething",
 					},
 					TailscaleInitContainer: &tsapi.Container{
 						SecurityContext: &corev1.SecurityContext{
@@ -92,7 +94,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 							Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1000m"), corev1.ResourceMemory: resource.MustParse("128Mi")},
 							Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
 						},
-						Env: []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						Env:             []tsapi.Env{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}},
+						ImagePullPolicy: "IfNotPresent",
+						Image:           "ghcr.io/my-repo/tailscale:v0.01testsomething",
 					},
 				},
 			},
@@ -135,10 +139,12 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	env := []corev1.EnvVar{{Name: "TS_HOSTNAME", Value: "nginx"}}
 	userspaceProxySS.Labels = labels
 	userspaceProxySS.Annotations = annots
+	userspaceProxySS.Spec.Template.Spec.Containers[0].Image = "tailscale/tailscale:v0.0.1"
 	userspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
 	nonUserspaceProxySS.ObjectMeta.Labels = labels
 	nonUserspaceProxySS.ObjectMeta.Annotations = annots
 	nonUserspaceProxySS.Spec.Template.Spec.Containers[0].Env = env
+	nonUserspaceProxySS.Spec.Template.Spec.InitContainers[0].Image = "tailscale/tailscale:v0.0.1"
 
 	// 1. Test that a ProxyClass with all fields set gets correctly applied
 	// to a Statefulset built from non-userspace proxy template.
@@ -159,6 +165,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.InitContainers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleInitContainer.Resources
 	wantSS.Spec.Template.Spec.InitContainers[0].Env = append(wantSS.Spec.Template.Spec.InitContainers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent"
+	wantSS.Spec.Template.Spec.InitContainers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
+	wantSS.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = "IfNotPresent"
 
 	gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
@@ -194,9 +204,11 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
 	wantSS.Spec.Template.Spec.Containers[0].SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.SecurityContext
 	wantSS.Spec.Template.Spec.Containers[0].Resources = proxyClassAllOpts.Spec.StatefulSet.Pod.TailscaleContainer.Resources
 	wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "foo", Value: "bar"}, {Name: "TS_USERSPACE", Value: "true"}, {Name: "bar"}}...)
+	wantSS.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent"
+	wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
 	gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
 	if diff := cmp.Diff(gotSS, wantSS); diff != "" {
-		t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
+		t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
 	}
 
 	// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied

cmd/k8s-operator/svc.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/netip"
 	"slices"
@@ -15,7 +16,9 @@ import (
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -23,13 +26,20 @@ import (
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/net/dns/resolvconffile"
+	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/dnsname"
 	"tailscale.com/util/set"
 )
 
 const (
 	resolvConfPath       = "/etc/resolv.conf"
 	defaultClusterDomain = "cluster.local"
+
+	reasonProxyCreated = "ProxyCreated"
+	reasonProxyInvalid = "ProxyInvalid"
+	reasonProxyFailed  = "ProxyFailed"
+	reasonProxyPending = "ProxyPending"
 )
 
 type ServiceReconciler struct {
@@ -50,6 +60,8 @@ type ServiceReconciler struct {
 	recorder record.EventRecorder
 
 	tsNamespace string
+
+	clock tstime.Clock
 }
 
 var (
@@ -76,6 +88,12 @@ func childResourceLabels(name, ns, typ string) map[string]string {
 	}
 }
 
+func (a *ServiceReconciler) isTailscaleService(svc *corev1.Service) bool {
+	targetIP := tailnetTargetAnnotation(svc)
+	targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN]
+	return a.shouldExpose(svc) || targetIP != "" || targetFQDN != ""
+}
+
 func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
 	logger := a.logger.With("service-ns", req.Namespace, "service-name", req.Name)
 	logger.Debugf("starting reconcile")
@@ -90,9 +108,8 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	} else if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to get svc: %w", err)
 	}
-	targetIP := tailnetTargetAnnotation(svc)
-	targetFQDN := svc.Annotations[AnnotationTailnetTargetFQDN]
-	if !svc.DeletionTimestamp.IsZero() || !a.shouldExpose(svc) && targetIP == "" && targetFQDN == "" {
+
+	if !svc.DeletionTimestamp.IsZero() || !a.isTailscaleService(svc) {
 		logger.Debugf("service is being deleted or is (no longer) referring to Tailscale ingress/egress, ensuring any created resources are cleaned up")
 		return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
 	}
@@ -104,7 +121,14 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 //
 // This function is responsible for removing the finalizer from the service,
 // once all associated resources are gone.
-func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
+	oldSvcStatus := svc.Status.DeepCopy()
+	defer func() {
+		if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			err = errors.Join(err, a.Client.Status().Update(ctx, svc))
+		}
+	}()
 	ix := slices.Index(svc.Finalizers, FinalizerName)
 	if ix < 0 {
 		logger.Debugf("no finalizer, nothing to do")
@@ -114,6 +138,10 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 		a.managedEgressProxies.Remove(svc.UID)
 		gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
 		gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+
+		if !a.isTailscaleService(svc) {
+			tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady)
+		}
 		return nil
 	}
 
@@ -133,7 +161,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 	// exactly once at the very end of cleanup, because the final step of
 	// cleanup removes the tailscale finalizer, which will make all future
 	// reconciles exit early.
-	logger.Infof("unexposed service from tailnet")
+	logger.Infof("unexposed Service from tailnet")
 
 	a.mu.Lock()
 	defer a.mu.Unlock()
@@ -141,6 +169,10 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 	a.managedEgressProxies.Remove(svc.UID)
 	gaugeIngressProxies.Set(int64(a.managedIngressProxies.Len()))
 	gaugeEgressProxies.Set(int64(a.managedEgressProxies.Len()))
+
+	if !a.isTailscaleService(svc) {
+		tsoperator.RemoveServiceCondition(svc, tsapi.ProxyReady)
+	}
 	return nil
 }
 
@@ -149,7 +181,15 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
 //
 // This function adds a finalizer to svc, ensuring that we can handle orderly
 // deprovisioning later.
-func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) error {
+func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
+	oldSvcStatus := svc.Status.DeepCopy()
+	defer func() {
+		if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
+			// An error encountered here should get returned by the Reconcile function.
+			err = errors.Join(err, a.Client.Status().Update(ctx, svc))
+		}
+	}()
+
 	// Run for proxy config related validations here as opposed to running
 	// them earlier. This is to prevent cleanup being blocked on a
 	// misconfigured proxy param.
@@ -157,30 +197,31 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid config: %v", err)
 		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDCONFIG", msg)
 		a.logger.Error(msg)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
 		return nil
 	}
 	if violations := validateService(svc); len(violations) > 0 {
 		msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", "))
 		a.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
 		a.logger.Error(msg)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyInvalid, msg, a.clock, logger)
 		return nil
 	}
 
 	proxyClass := proxyClassForObject(svc)
 	if proxyClass != "" {
 		if ready, err := proxyClassIsReady(ctx, proxyClass, a.Client); err != nil {
-			return fmt.Errorf("error verifying ProxyClass for Service: %w", err)
+			errMsg := fmt.Errorf("error verifying ProxyClass for Service: %w", err)
+			tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+			return errMsg
 		} else if !ready {
-			logger.Infof("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass)
+			msg := fmt.Sprintf("ProxyClass %s specified for the Service, but is not (yet) Ready, waiting..", proxyClass)
+			tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
+			logger.Info(msg)
 			return nil
 		}
 	}
 
-	hostname, err := nameForService(svc)
-	if err != nil {
-		return err
-	}
-
 	if !slices.Contains(svc.Finalizers, FinalizerName) {
 		// This log line is printed exactly once during initial provisioning,
 		// because once the finalizer is in place this block gets skipped. So,
@@ -189,7 +230,9 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		logger.Infof("exposing service over tailscale")
 		svc.Finalizers = append(svc.Finalizers, FinalizerName)
 		if err := a.Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to add finalizer: %w", err)
+			errMsg := fmt.Errorf("failed to add finalizer: %w", err)
+			tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+			return errMsg
 		}
 	}
 	crl := childResourceLabels(svc.Name, svc.Namespace, "svc")
@@ -201,10 +244,10 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 	sts := &tailscaleSTSConfig{
 		ParentResourceName:  svc.Name,
 		ParentResourceUID:   string(svc.UID),
-		Hostname:            hostname,
+		Hostname:            nameForService(svc),
 		Tags:                tags,
 		ChildResourceLabels: crl,
-		ProxyClass:          proxyClass,
+		ProxyClassName:      proxyClass,
 	}
 
 	a.mu.Lock()
@@ -233,10 +276,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 
 	var hsvc *corev1.Service
 	if hsvc, err = a.ssr.Provision(ctx, logger, sts); err != nil {
-		return fmt.Errorf("failed to provision: %w", err)
+		errMsg := fmt.Errorf("failed to provision: %w", err)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+		return errMsg
 	}
 
-	if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" {
+	if sts.TailnetTargetIP != "" || sts.TailnetTargetFQDN != "" { // if an egress proxy
 		clusterDomain := retrieveClusterDomain(a.tsNamespace, logger)
 		headlessSvcName := hsvc.Name + "." + hsvc.Namespace + ".svc." + clusterDomain
 		if svc.Spec.ExternalName != headlessSvcName || svc.Spec.Type != corev1.ServiceTypeExternalName {
@@ -244,14 +289,18 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 			svc.Spec.Selector = nil
 			svc.Spec.Type = corev1.ServiceTypeExternalName
 			if err := a.Update(ctx, svc); err != nil {
-				return fmt.Errorf("failed to update service: %w", err)
+				errMsg := fmt.Errorf("failed to update service: %w", err)
+				tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, errMsg.Error(), a.clock, logger)
+				return errMsg
 			}
 		}
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
 		return nil
 	}
 
 	if !isTailscaleLoadBalancerService(svc, a.isDefaultLoadBalancer) {
 		logger.Debugf("service is not a LoadBalancer, so not updating ingress")
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
 		return nil
 	}
 
@@ -260,22 +309,23 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		return fmt.Errorf("failed to get device ID: %w", err)
 	}
 	if tsHost == "" {
-		logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
+		msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
+		logger.Debug(msg)
 		// No hostname yet. Wait for the proxy pod to auth.
 		svc.Status.LoadBalancer.Ingress = nil
-		if err := a.Status().Update(ctx, svc); err != nil {
-			return fmt.Errorf("failed to update service status: %w", err)
-		}
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyPending, msg, a.clock, logger)
 		return nil
 	}
 
-	logger.Debugf("setting ingress to %q, %s", tsHost, strings.Join(tsIPs, ", "))
+	logger.Debugf("setting Service LoadBalancer status to %q, %s", tsHost, strings.Join(tsIPs, ", "))
 	ingress := []corev1.LoadBalancerIngress{
 		{Hostname: tsHost},
 	}
 	clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
 	if err != nil {
-		return fmt.Errorf("failed to parse cluster IP: %w", err)
+		msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
+		tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
+		return fmt.Errorf(msg)
 	}
 	for _, ip := range tsIPs {
 		addr, err := netip.ParseAddr(ip)
@@ -287,22 +337,28 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		}
 	}
 	svc.Status.LoadBalancer.Ingress = ingress
-	if err := a.Status().Update(ctx, svc); err != nil {
-		return fmt.Errorf("failed to update service status: %w", err)
-	}
+	tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
 	return nil
 }
 
 func validateService(svc *corev1.Service) []string {
 	violations := make([]string, 0)
 	if svc.Annotations[AnnotationTailnetTargetFQDN] != "" && svc.Annotations[AnnotationTailnetTargetIP] != "" {
-		violations = append(violations, "only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN)
+		violations = append(violations, fmt.Sprintf("only one of annotations %s and %s can be set", AnnotationTailnetTargetIP, AnnotationTailnetTargetFQDN))
 	}
 	if fqdn := svc.Annotations[AnnotationTailnetTargetFQDN]; fqdn != "" {
 		if !isMagicDNSName(fqdn) {
 			violations = append(violations, fmt.Sprintf("invalid value of annotation %s: %q does not appear to be a valid MagicDNS name", AnnotationTailnetTargetFQDN, fqdn))
 		}
 	}
+	svcName := nameForService(svc)
+	if err := dnsname.ValidLabel(svcName); err != nil {
+		if _, ok := svc.Annotations[AnnotationHostname]; ok {
+			violations = append(violations, fmt.Sprintf("invalid Tailscale hostname specified %q: %s", svcName, err))
+		} else {
+			violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err))
+		}
+	}
 	return violations
 }
 
@@ -334,7 +390,7 @@ func hasExposeAnnotation(svc *corev1.Service) bool {
 	return svc != nil && svc.Annotations[AnnotationExpose] == "true"
 }
 
-// hasTailnetTargetAnnotation returns the value of tailscale.com/tailnet-ip
+// tailnetTargetAnnotation returns the value of tailscale.com/tailnet-ip
 // annotation or of the deprecated tailscale.com/ts-tailnet-target-ip
 // annotation. If neither is set, it returns an empty string. If both are set,
 // it returns the value of the new annotation.

cmd/k8s-operator/testutils_test.go

@@ -304,10 +304,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 
 func expectedHeadlessService(name string, parentType string) *corev1.Service {
 	return &corev1.Service{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:         name,
 			GenerateName: "ts-test-",
@@ -328,13 +324,9 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
 	}
 }
 
-func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
+func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
 	t.Helper()
 	s := &corev1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: "v1",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      opts.secretName,
 			Namespace: "operator-ns",
@@ -355,6 +347,16 @@ func expectedSecret(t *testing.T, opts configOpts) *corev1.Secret {
 		AuthKey:      ptr.To("secret-authkey"),
 		AcceptRoutes: "false",
 	}
+	if opts.proxyClass != "" {
+		t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
+		proxyClass := new(tsapi.ProxyClass)
+		if err := cl.Get(context.Background(), types.NamespacedName{Name: opts.proxyClass}, proxyClass); err != nil {
+			t.Fatalf("error getting ProxyClass: %v", err)
+		}
+		if proxyClass.Spec.TailscaleConfig != nil && proxyClass.Spec.TailscaleConfig.AcceptRoutes {
+			conf.AcceptRoutes = "true"
+		}
+	}
 	var routes []netip.Prefix
 	if opts.subnetRoutes != "" || opts.isExitNode {
 		r := opts.subnetRoutes
@@ -455,10 +457,10 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client,
 
 // expectEqual accepts a Kubernetes object and a Kubernetes client. It tests
 // whether an object with equivalent contents can be retrieved by the passed
-// client. If you want to NOT test some object fields for equality, ensure that
-// they are not present in the passed object and use the modify func to remove
-// them from the cluster object. If no such modifications are needed, you can
-// pass nil in place of the modify function.
+// client. If you want to NOT test some object fields for equality, use the
+// modify func to ensure that they are removed from the cluster object and the
+// object passed as 'want'. If no such modifications are needed, you can pass
+// nil in place of the modify function.
 func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) {
 	t.Helper()
 	got := O(new(T))
@@ -474,6 +476,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
 	got.SetResourceVersion("")
 	want.SetResourceVersion("")
 	if modifier != nil {
+		modifier(want)
 		modifier(got)
 	}
 	if diff := cmp.Diff(got, want); diff != "" {
@@ -608,3 +611,33 @@ func (c *fakeTSClient) Deleted() []string {
 func removeHashAnnotation(sts *appsv1.StatefulSet) {
 	delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
 }
+
+func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
+	return func(secret *corev1.Secret) {
+		t.Helper()
+		if len(secret.StringData["tailscaled"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
+				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
+			}
+			mak.Set(&secret.StringData, "tailscaled", string(b))
+		}
+		if len(secret.StringData["cap-95.hujson"]) != 0 {
+			conf := &ipn.ConfigVAlpha{}
+			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {
+				t.Fatalf("error umarshalling 'cap-95.hujson' contents: %v", err)
+			}
+			conf.AuthKey = nil
+			b, err := json.Marshal(conf)
+			if err != nil {
+				t.Fatalf("error marshalling 'cap-95.huson' contents: %v", err)
+			}
+			mak.Set(&secret.StringData, "cap-95.hujson", string(b))
+		}
+	}
+}

cmd/natc/natc.go

@@ -0,0 +1,567 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The natc command is a work-in-progress implementation of a NAT based
+// connector for Tailscale. It is intended to be used to route traffic to a
+// specific domain through a specific node.
+package main
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand/v2"
+	"net"
+	"net/http"
+	"net/netip"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gaissmai/bart"
+	"github.com/inetaf/tcpproxy"
+	"github.com/peterbourgon/ff/v3"
+	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/client/tailscale"
+	"tailscale.com/envknob"
+	"tailscale.com/hostinfo"
+	"tailscale.com/ipn"
+	"tailscale.com/net/netutil"
+	"tailscale.com/syncs"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tsnet"
+	"tailscale.com/tsweb"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/mak"
+)
+
+func main() {
+	hostinfo.SetApp("natc")
+	if !envknob.UseWIPCode() {
+		log.Fatal("cmd/natc is a work in progress and has not been security reviewed;\nits use requires TAILSCALE_USE_WIP_CODE=1 be set in the environment for now.")
+	}
+
+	// Parse flags
+	fs := flag.NewFlagSet("natc", flag.ExitOnError)
+	var (
+		debugPort       = fs.Int("debug-port", 8893, "Listening port for debug/metrics endpoint")
+		hostname        = fs.String("hostname", "", "Hostname to register the service under")
+		siteID          = fs.Uint("site-id", 1, "an integer site ID to use for the ULA prefix which allows for multiple proxies to act in a HA configuration")
+		v4PfxStr        = fs.String("v4-pfx", "100.64.1.0/24", "comma-separated list of IPv4 prefixes to advertise")
+		verboseTSNet    = fs.Bool("verbose-tsnet", false, "enable verbose logging in tsnet")
+		printULA        = fs.Bool("print-ula", false, "print the ULA prefix and exit")
+		ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore")
+		wgPort          = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic")
+	)
+	ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC"))
+
+	if *printULA {
+		fmt.Println(ula(uint16(*siteID)))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	if *siteID == 0 {
+		log.Fatalf("site-id must be set")
+	} else if *siteID > 0xffff {
+		log.Fatalf("site-id must be in the range [0, 65535]")
+	}
+
+	var ignoreDstTable *bart.Table[bool]
+	for _, s := range strings.Split(*ignoreDstPfxStr, ",") {
+		s := strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		if ignoreDstTable == nil {
+			ignoreDstTable = &bart.Table[bool]{}
+		}
+		pfx, err := netip.ParsePrefix(s)
+		if err != nil {
+			log.Fatalf("unable to parse prefix: %v", err)
+		}
+		if pfx.Masked() != pfx {
+			log.Fatalf("prefix %v is not normalized (bits are set outside the mask)", pfx)
+		}
+		ignoreDstTable.Insert(pfx, true)
+	}
+	var v4Prefixes []netip.Prefix
+	for _, s := range strings.Split(*v4PfxStr, ",") {
+		p := netip.MustParsePrefix(strings.TrimSpace(s))
+		if p.Masked() != p {
+			log.Fatalf("v4 prefix %v is not a masked prefix", p)
+		}
+		v4Prefixes = append(v4Prefixes, p)
+	}
+	if len(v4Prefixes) == 0 {
+		log.Fatalf("no v4 prefixes specified")
+	}
+	dnsAddr := v4Prefixes[0].Addr()
+	ts := &tsnet.Server{
+		Hostname: *hostname,
+	}
+	if *wgPort != 0 {
+		if *wgPort >= 1<<16 {
+			log.Fatalf("wg-port must be in the range [0, 65535]")
+		}
+		ts.Port = uint16(*wgPort)
+	}
+	defer ts.Close()
+	if *verboseTSNet {
+		ts.Logf = log.Printf
+	}
+
+	// Start special-purpose listeners: dns, http promotion, debug server
+	if *debugPort != 0 {
+		mux := http.NewServeMux()
+		tsweb.Debugger(mux)
+		dln, err := ts.Listen("tcp", fmt.Sprintf(":%d", *debugPort))
+		if err != nil {
+			log.Fatalf("failed listening on debug port: %v", err)
+		}
+		defer dln.Close()
+		go func() {
+			log.Fatalf("debug serve: %v", http.Serve(dln, mux))
+		}()
+	}
+	lc, err := ts.LocalClient()
+	if err != nil {
+		log.Fatalf("LocalClient() failed: %v", err)
+	}
+	if _, err := ts.Up(ctx); err != nil {
+		log.Fatalf("ts.Up: %v", err)
+	}
+
+	c := &connector{
+		ts:         ts,
+		lc:         lc,
+		dnsAddr:    dnsAddr,
+		v4Ranges:   v4Prefixes,
+		v6ULA:      ula(uint16(*siteID)),
+		ignoreDsts: ignoreDstTable,
+	}
+	c.run(ctx)
+}
+
+type connector struct {
+	// ts is the tsnet.Server used to host the connector.
+	ts *tsnet.Server
+	// lc is the LocalClient used to interact with the tsnet.Server hosting this
+	// connector.
+	lc *tailscale.LocalClient
+
+	// dnsAddr is the IPv4 address to listen on for DNS requests. It is used to
+	// prevent the app connector from assigning it to a domain.
+	dnsAddr netip.Addr
+
+	// v4Ranges is the list of IPv4 ranges to advertise and assign addresses from.
+	// These are masked prefixes.
+	v4Ranges []netip.Prefix
+	// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
+	v6ULA netip.Prefix
+
+	perPeerMap syncs.Map[tailcfg.NodeID, *perPeerState]
+
+	// ignoreDsts is initialized at start up with the contents of --ignore-destinations (if none it is nil)
+	// It is never mutated, only used for lookups.
+	// Users who want to natc a DNS wildcard but not every address record in that domain can supply the
+	// exceptions in --ignore-destinations. When we receive a dns request we will look up the fqdn
+	// and if any of the ip addresses in response to the lookup match any 'ignore destinations' prefix we will
+	// return a dns response that contains the ip addresses we discovered with the lookup (ie not the
+	// natc behavior, which would return a dummy ip address pointing at natc).
+	ignoreDsts *bart.Table[bool]
+}
+
+// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
+// The 8th and 9th bytes are used to encode the site ID which allows for
+// multiple proxies to act in a HA configuration.
+// mnemonic: a99c = appc
+var v6ULA = netip.MustParsePrefix("fd7a:115c:a1e0:a99c::/64")
+
+func ula(siteID uint16) netip.Prefix {
+	as16 := v6ULA.Addr().As16()
+	as16[8] = byte(siteID >> 8)
+	as16[9] = byte(siteID)
+	return netip.PrefixFrom(netip.AddrFrom16(as16), 64+16)
+}
+
+// run runs the connector.
+//
+// The passed in context is only used for the initial setup. The connector runs
+// forever.
+func (c *connector) run(ctx context.Context) {
+	if _, err := c.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
+		AdvertiseRoutesSet: true,
+		Prefs: ipn.Prefs{
+			AdvertiseRoutes: append(c.v4Ranges, c.v6ULA),
+		},
+	}); err != nil {
+		log.Fatalf("failed to advertise routes: %v", err)
+	}
+	c.ts.RegisterFallbackTCPHandler(c.handleTCPFlow)
+	c.serveDNS()
+}
+
+func (c *connector) serveDNS() {
+	pc, err := c.ts.ListenPacket("udp", net.JoinHostPort(c.dnsAddr.String(), "53"))
+	if err != nil {
+		log.Fatalf("failed listening on port 53: %v", err)
+	}
+	defer pc.Close()
+	log.Printf("Listening for DNS on %s", pc.LocalAddr().String())
+	for {
+		buf := make([]byte, 1500)
+		n, addr, err := pc.ReadFrom(buf)
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			log.Printf("serveDNS.ReadFrom failed: %v", err)
+			continue
+		}
+		go c.handleDNS(pc, buf[:n], addr.(*net.UDPAddr))
+	}
+}
+
+func lookupDestinationIP(domain string) ([]netip.Addr, error) {
+	netIPs, err := net.LookupIP(domain)
+	if err != nil {
+		var dnsError *net.DNSError
+		if errors.As(err, &dnsError) && dnsError.IsNotFound {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+	var addrs []netip.Addr
+	for _, ip := range netIPs {
+		a, ok := netip.AddrFromSlice(ip)
+		if ok {
+			addrs = append(addrs, a)
+		}
+	}
+	return addrs, nil
+}
+
+// handleDNS handles a DNS request to the app connector.
+// It generates a response based on the request and the node that sent it.
+//
+// Each node is assigned a unique pair of IP addresses for each domain it
+// queries. This assignment is done lazily and is not persisted across restarts.
+// A per-peer assignment allows the connector to reuse a limited number of IP
+// addresses across multiple nodes and domains. It also allows for clear
+// failover behavior when an app connector is restarted.
+//
+// This assignment later allows the connector to determine where to forward
+// traffic based on the destination IP address.
+func (c *connector) handleDNS(pc net.PacketConn, buf []byte, remoteAddr *net.UDPAddr) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	who, err := c.lc.WhoIs(ctx, remoteAddr.String())
+	if err != nil {
+		log.Printf("HandleDNS: WhoIs failed: %v\n", err)
+		return
+	}
+
+	var msg dnsmessage.Message
+	err = msg.Unpack(buf)
+	if err != nil {
+		log.Printf("HandleDNS: dnsmessage unpack failed: %v\n ", err)
+		return
+	}
+
+	// If there are destination ips that we don't want to route, we
+	// have to do a dns lookup here to find the destination ip.
+	if c.ignoreDsts != nil {
+		if len(msg.Questions) > 0 {
+			q := msg.Questions[0]
+			switch q.Type {
+			case dnsmessage.TypeAAAA, dnsmessage.TypeA:
+				dstAddrs, err := lookupDestinationIP(q.Name.String())
+				if err != nil {
+					log.Printf("HandleDNS: lookup destination failed: %v\n ", err)
+					return
+				}
+				if c.ignoreDestination(dstAddrs) {
+					bs, err := dnsResponse(&msg, dstAddrs)
+					// TODO (fran): treat as SERVFAIL
+					if err != nil {
+						log.Printf("HandleDNS: generate ignore response failed: %v\n", err)
+						return
+					}
+					_, err = pc.WriteTo(bs, remoteAddr)
+					if err != nil {
+						log.Printf("HandleDNS: write failed: %v\n", err)
+					}
+					return
+				}
+			}
+		}
+	}
+	// None of the destination IP addresses match an ignore destination prefix, do
+	// the natc thing.
+
+	resp, err := c.generateDNSResponse(&msg, who.Node.ID)
+	// TODO (fran): treat as SERVFAIL
+	if err != nil {
+		log.Printf("HandleDNS: connector handling failed: %v\n", err)
+		return
+	}
+	// TODO (fran): treat as NXDOMAIN
+	if len(resp) == 0 {
+		return
+	}
+	// This connector handled the DNS request
+	_, err = pc.WriteTo(resp, remoteAddr)
+	if err != nil {
+		log.Printf("HandleDNS: write failed: %v\n", err)
+	}
+}
+
+// tsMBox is the mailbox used in SOA records.
+// The convention is to replace the @ symbol with a dot.
+// So in this case, the mailbox is support.tailscale.com. with the trailing dot
+// to indicate that it is a fully qualified domain name.
+var tsMBox = dnsmessage.MustNewName("support.tailscale.com.")
+
+// generateDNSResponse generates a DNS response for the given request. The from
+// argument is the NodeID of the node that sent the request.
+func (c *connector) generateDNSResponse(req *dnsmessage.Message, from tailcfg.NodeID) ([]byte, error) {
+	pm, _ := c.perPeerMap.LoadOrStore(from, &perPeerState{c: c})
+	var addrs []netip.Addr
+	if len(req.Questions) > 0 {
+		switch req.Questions[0].Type {
+		case dnsmessage.TypeAAAA, dnsmessage.TypeA:
+			var err error
+			addrs, err = pm.ipForDomain(req.Questions[0].Name.String())
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return dnsResponse(req, addrs)
+}
+
+// dnsResponse makes a DNS response for the natc. If the dnsmessage is requesting TypeAAAA
+// or TypeA the provided addrs of the requested type will be used.
+func dnsResponse(req *dnsmessage.Message, addrs []netip.Addr) ([]byte, error) {
+	b := dnsmessage.NewBuilder(nil,
+		dnsmessage.Header{
+			ID:            req.Header.ID,
+			Response:      true,
+			Authoritative: true,
+		})
+	b.EnableCompression()
+
+	if len(req.Questions) == 0 {
+		return b.Finish()
+	}
+	q := req.Questions[0]
+	if err := b.StartQuestions(); err != nil {
+		return nil, err
+	}
+	if err := b.Question(q); err != nil {
+		return nil, err
+	}
+	if err := b.StartAnswers(); err != nil {
+		return nil, err
+	}
+	switch q.Type {
+	case dnsmessage.TypeAAAA, dnsmessage.TypeA:
+		want6 := q.Type == dnsmessage.TypeAAAA
+		for _, ip := range addrs {
+			if want6 != ip.Is6() {
+				continue
+			}
+			if want6 {
+				if err := b.AAAAResource(
+					dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 5},
+					dnsmessage.AAAAResource{AAAA: ip.As16()},
+				); err != nil {
+					return nil, err
+				}
+			} else {
+				if err := b.AResource(
+					dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 5},
+					dnsmessage.AResource{A: ip.As4()},
+				); err != nil {
+					return nil, err
+				}
+			}
+		}
+	case dnsmessage.TypeSOA:
+		if err := b.SOAResource(
+			dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 120},
+			dnsmessage.SOAResource{NS: q.Name, MBox: tsMBox, Serial: 2023030600,
+				Refresh: 120, Retry: 120, Expire: 120, MinTTL: 60},
+		); err != nil {
+			return nil, err
+		}
+	case dnsmessage.TypeNS:
+		if err := b.NSResource(
+			dnsmessage.ResourceHeader{Name: q.Name, Class: q.Class, TTL: 120},
+			dnsmessage.NSResource{NS: tsMBox},
+		); err != nil {
+			return nil, err
+		}
+	}
+	return b.Finish()
+}
+
+// handleTCPFlow handles a TCP flow from the given source to the given
+// destination. It uses the source address to determine the node that sent the
+// request and the destination address to determine the domain that the request
+// is for based on the IP address assigned to the destination in the DNS
+// response.
+func (c *connector) handleTCPFlow(src, dst netip.AddrPort) (handler func(net.Conn), intercept bool) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	who, err := c.lc.WhoIs(ctx, src.Addr().String())
+	cancel()
+	if err != nil {
+		log.Printf("HandleTCPFlow: WhoIs failed: %v\n", err)
+		return nil, false
+	}
+
+	from := who.Node.ID
+	ps, ok := c.perPeerMap.Load(from)
+	if !ok {
+		log.Printf("handleTCPFlow: no perPeerState for %v", from)
+		return nil, false
+	}
+	domain, ok := ps.domainForIP(dst.Addr())
+	if !ok {
+		log.Printf("handleTCPFlow: no domain for IP %v\n", dst.Addr())
+		return nil, false
+	}
+	return func(conn net.Conn) {
+		proxyTCPConn(conn, domain)
+	}, true
+}
+
+// ignoreDestination reports whether any of the provided dstAddrs match the prefixes configured
+// in --ignore-destinations
+func (c *connector) ignoreDestination(dstAddrs []netip.Addr) bool {
+	for _, a := range dstAddrs {
+		if _, ok := c.ignoreDsts.Get(a); ok {
+			return true
+		}
+	}
+	return false
+}
+
+func proxyTCPConn(c net.Conn, dest string) {
+	addrPortStr := c.LocalAddr().String()
+	_, port, err := net.SplitHostPort(addrPortStr)
+	if err != nil {
+		log.Printf("tcpRoundRobinHandler.Handle: bogus addrPort %q", addrPortStr)
+		c.Close()
+		return
+	}
+
+	p := &tcpproxy.Proxy{
+		ListenFunc: func(net, laddr string) (net.Listener, error) {
+			return netutil.NewOneConnListener(c, nil), nil
+		},
+	}
+	p.AddRoute(addrPortStr, &tcpproxy.DialProxy{
+		Addr: fmt.Sprintf("%s:%s", dest, port),
+	})
+	p.Start()
+}
+
+// perPeerState holds the state for a single peer.
+type perPeerState struct {
+	c *connector
+
+	mu           sync.Mutex
+	domainToAddr map[string][]netip.Addr
+	addrToDomain *bart.Table[string]
+}
+
+// domainForIP returns the domain name assigned to the given IP address and
+// whether it was found.
+func (ps *perPeerState) domainForIP(ip netip.Addr) (_ string, ok bool) {
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+	return ps.addrToDomain.Get(ip)
+}
+
+// ipForDomain assigns a pair of unique IP addresses for the given domain and
+// returns them. The first address is an IPv4 address and the second is an IPv6
+// address. If the domain already has assigned addresses, it returns them.
+func (ps *perPeerState) ipForDomain(domain string) ([]netip.Addr, error) {
+	fqdn, err := dnsname.ToFQDN(domain)
+	if err != nil {
+		return nil, err
+	}
+	domain = fqdn.WithoutTrailingDot()
+
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+	if addrs, ok := ps.domainToAddr[domain]; ok {
+		return addrs, nil
+	}
+	addrs := ps.assignAddrsLocked(domain)
+	return addrs, nil
+}
+
+// isIPUsedLocked reports whether the given IP address is already assigned to a
+// domain.
+// ps.mu must be held.
+func (ps *perPeerState) isIPUsedLocked(ip netip.Addr) bool {
+	_, ok := ps.addrToDomain.Get(ip)
+	return ok
+}
+
+// unusedIPv4Locked returns an unused IPv4 address from the available ranges.
+func (ps *perPeerState) unusedIPv4Locked() netip.Addr {
+	// TODO: skip ranges that have been exhausted
+	for _, r := range ps.c.v4Ranges {
+		ip := randV4(r)
+		for r.Contains(ip) {
+			if !ps.isIPUsedLocked(ip) && ip != ps.c.dnsAddr {
+				return ip
+			}
+			ip = ip.Next()
+		}
+	}
+	return netip.Addr{}
+}
+
+// randV4 returns a random IPv4 address within the given prefix.
+func randV4(maskedPfx netip.Prefix) netip.Addr {
+	bits := 32 - maskedPfx.Bits()
+	randBits := rand.Uint32N(1 << uint(bits))
+
+	ip4 := maskedPfx.Addr().As4()
+	pn := binary.BigEndian.Uint32(ip4[:])
+	binary.BigEndian.PutUint32(ip4[:], randBits|pn)
+	return netip.AddrFrom4(ip4)
+}
+
+// assignAddrsLocked assigns a pair of unique IP addresses for the given domain
+// and returns them. The first address is an IPv4 address and the second is an
+// IPv6 address. It does not check if the domain already has assigned addresses.
+// ps.mu must be held.
+func (ps *perPeerState) assignAddrsLocked(domain string) []netip.Addr {
+	if ps.addrToDomain == nil {
+		ps.addrToDomain = &bart.Table[string]{}
+	}
+	v4 := ps.unusedIPv4Locked()
+	as16 := ps.c.v6ULA.Addr().As16()
+	as4 := v4.As4()
+	copy(as16[12:], as4[:])
+	v6 := netip.AddrFrom16(as16)
+	addrs := []netip.Addr{v4, v6}
+	mak.Set(&ps.domainToAddr, domain, addrs)
+	for _, a := range addrs {
+		ps.addrToDomain.Insert(netip.PrefixFrom(a, a.BitLen()), domain)
+	}
+	return addrs
+}

cmd/proxy-to-grafana/proxy-to-grafana.go

@@ -46,6 +46,7 @@ var (
 	backendAddr  = flag.String("backend-addr", "", "Address of the Grafana server served over HTTP, in host:port format. Typically localhost:nnnn.")
 	tailscaleDir = flag.String("state-dir", "./", "Alternate directory to use for Tailscale state storage. If empty, a default is used.")
 	useHTTPS     = flag.Bool("use-https", false, "Serve over HTTPS via your *.ts.net subdomain if enabled in Tailscale admin.")
+	loginServer  = flag.String("login-server", "", "URL to alternative control server. If empty, the default Tailscale control is used.")
 )
 
 func main() {
@@ -57,8 +58,9 @@ func main() {
 		log.Fatal("missing --backend-addr")
 	}
 	ts := &tsnet.Server{
-		Dir:      *tailscaleDir,
-		Hostname: *hostname,
+		Dir:        *tailscaleDir,
+		Hostname:   *hostname,
+		ControlURL: *loginServer,
 	}
 
 	// TODO(bradfitz,maisem): move this to a method on tsnet.Server probably.

cmd/sniproxy/handlers.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"math/rand"
+	"math/rand/v2"
 	"net"
 	"net/netip"
 	"slices"
@@ -47,7 +47,7 @@ func (h *tcpRoundRobinHandler) Handle(c net.Conn) {
 		return netutil.NewOneConnListener(c, nil), nil
 	}
 
-	dest := h.To[rand.Intn(len(h.To))]
+	dest := h.To[rand.IntN(len(h.To))]
 	dial := &tcpproxy.DialProxy{
 		Addr:        fmt.Sprintf("%s:%s", dest, port),
 		DialContext: h.DialContext,

cmd/stunc/stunc.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"strconv"
 
 	"tailscale.com/net/stun"
 )
@@ -15,12 +16,20 @@ import (
 func main() {
 	log.SetFlags(0)
 
-	if len(os.Args) != 2 {
-		log.Fatalf("usage: %s <hostname>", os.Args[0])
+	if len(os.Args) < 2 || len(os.Args) > 3 {
+		log.Fatalf("usage: %s <hostname> [port]", os.Args[0])
 	}
 	host := os.Args[1]
+	port := "3478"
+	if len(os.Args) == 3 {
+		port = os.Args[2]
+	}
+	_, err := strconv.ParseUint(port, 10, 16)
+	if err != nil {
+		log.Fatalf("invalid port: %v", err)
+	}
 
-	uaddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(host, "3478"))
+	uaddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(host, port))
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/stund/depaware.txt

@@ -153,7 +153,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from math/big+
-        math/rand/v2                                                 from tailscale.com/util/fastuuid
+        math/rand/v2                                                 from tailscale.com/util/fastuuid+
         mime                                                         from github.com/prometheus/common/expfmt+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/stunstamp/api.go

@@ -0,0 +1,142 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	sq "github.com/Masterminds/squirrel"
+)
+
+type api struct {
+	db  *db
+	mux *http.ServeMux
+}
+
+func newAPI(db *db) *api {
+	a := &api{
+		db: db,
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/query", a.query)
+	a.mux = mux
+	return a
+}
+
+type apiResult struct {
+	At         int    `json:"at"` // time.Time.Unix()
+	RegionID   int    `json:"regionID"`
+	Hostname   string `json:"hostname"`
+	Af         int    `json:"af"` // 4 or 6
+	Addr       string `json:"addr"`
+	Source     int    `json:"source"` // timestampSourceUserspace (0) or timestampSourceKernel (1)
+	StableConn bool   `json:"stableConn"`
+	DstPort    int    `json:"dstPort"`
+	RttNS      *int   `json:"rttNS"`
+}
+
+func getTimeBounds(vals url.Values) (from time.Time, to time.Time, err error) {
+	lastForm, ok := vals["last"]
+	if ok && len(lastForm) > 0 {
+		dur, err := time.ParseDuration(lastForm[0])
+		if err != nil {
+			return time.Time{}, time.Time{}, err
+		}
+		now := time.Now()
+		return now.Add(-dur), now, nil
+	}
+
+	fromForm, ok := vals["from"]
+	if ok && len(fromForm) > 0 {
+		fromUnixSec, err := strconv.Atoi(fromForm[0])
+		if err != nil {
+			return time.Time{}, time.Time{}, err
+		}
+		from = time.Unix(int64(fromUnixSec), 0)
+		toForm, ok := vals["to"]
+		if ok && len(toForm) > 0 {
+			toUnixSec, err := strconv.Atoi(toForm[0])
+			if err != nil {
+				return time.Time{}, time.Time{}, err
+			}
+			to = time.Unix(int64(toUnixSec), 0)
+		} else {
+			return time.Time{}, time.Time{}, errors.New("from specified without to")
+		}
+		return from, to, nil
+	}
+
+	// no time bounds specified, default to last 1h
+	now := time.Now()
+	return now.Add(-time.Hour), now, nil
+}
+
+func (a *api) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	a.mux.ServeHTTP(w, r)
+}
+
+func (a *api) query(w http.ResponseWriter, r *http.Request) {
+	err := r.ParseForm()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	from, to, err := getTimeBounds(r.Form)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	sb := sq.Select("at_unix", "region_id", "hostname", "af", "address", "timestamp_source", "stable_conn", "dst_port", "rtt_ns").From("rtt")
+	sb = sb.Where(sq.And{
+		sq.GtOrEq{"at_unix": from.Unix()},
+		sq.LtOrEq{"at_unix": to.Unix()},
+	})
+	query, args, err := sb.ToSql()
+	if err != nil {
+		return
+	}
+
+	rows, err := a.db.Query(query, args...)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	results := make([]apiResult, 0)
+	for rows.Next() {
+		rtt := 0
+		result := apiResult{
+			RttNS: &rtt,
+		}
+		err = rows.Scan(&result.At, &result.RegionID, &result.Hostname, &result.Af, &result.Addr, &result.Source, &result.StableConn, &result.DstPort, &result.RttNS)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		results = append(results, result)
+	}
+	if rows.Err() != nil {
+		http.Error(w, rows.Err().Error(), 500)
+		return
+	}
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		gz := gzip.NewWriter(w)
+		defer gz.Close()
+		w.Header().Set("Content-Encoding", "gzip")
+		err = json.NewEncoder(gz).Encode(&results)
+	} else {
+		err = json.NewEncoder(w).Encode(&results)
+	}
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+}

cmd/stunstamp/stunstamp.go

@@ -0,0 +1,886 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The stunstamp binary measures STUN round-trip latency with DERPs.
+package main
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"math/rand/v2"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"os"
+	"os/signal"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/golang/snappy"
+	"github.com/prometheus/prometheus/prompb"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/net/stun"
+	"tailscale.com/tailcfg"
+)
+
+var (
+	flagDERPMap        = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map")
+	flagOut            = flag.String("out", "", "output sqlite filename")
+	flagInterval       = flag.Duration("interval", time.Minute, "interval to probe at in time.ParseDuration() format")
+	flagAPI            = flag.String("api", "", "listen addr for HTTP API")
+	flagIPv6           = flag.Bool("ipv6", false, "probe IPv6 addresses")
+	flagRetention      = flag.Duration("retention", time.Hour*24*7, "sqlite retention period in time.ParseDuration() format")
+	flagRemoteWriteURL = flag.String("rw-url", "", "prometheus remote write URL")
+	flagInstance       = flag.String("instance", "", "instance label value; defaults to hostname if unspecified")
+	flagDstPorts       = flag.String("dst-ports", "", "comma-separated list of destination ports to monitor")
+)
+
+const (
+	minInterval       = time.Second
+	maxBufferDuration = time.Hour
+)
+
+func getDERPMap(ctx context.Context, url string) (*tailcfg.DERPMap, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	dm := tailcfg.DERPMap{}
+	err = json.NewDecoder(resp.Body).Decode(&dm)
+	if err != nil {
+		return nil, nil
+	}
+	return &dm, nil
+}
+
+type timestampSource int
+
+const (
+	timestampSourceUserspace timestampSource = iota
+	timestampSourceKernel
+)
+
+func (t timestampSource) String() string {
+	switch t {
+	case timestampSourceUserspace:
+		return "userspace"
+	case timestampSourceKernel:
+		return "kernel"
+	default:
+		return "unknown"
+	}
+}
+
+// resultKey contains the stable dimensions and their values for a given
+// timeseries, i.e. not time and not rtt/timeout.
+type resultKey struct {
+	meta            nodeMeta
+	timestampSource timestampSource
+	connStability   connStability
+	dstPort         int
+}
+
+type result struct {
+	key resultKey
+	at  time.Time
+	rtt *time.Duration // nil signifies failure, e.g. timeout
+}
+
+func measureRTT(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+	uconn, ok := conn.(*net.UDPConn)
+	if !ok {
+		return 0, fmt.Errorf("unexpected conn type: %T", conn)
+	}
+	err = uconn.SetReadDeadline(time.Now().Add(time.Second * 2))
+	if err != nil {
+		return 0, fmt.Errorf("error setting read deadline: %w", err)
+	}
+	txID := stun.NewTxID()
+	req := stun.Request(txID)
+	txAt := time.Now()
+	_, err = uconn.WriteToUDP(req, dst)
+	if err != nil {
+		return 0, fmt.Errorf("error writing to udp socket: %w", err)
+	}
+	b := make([]byte, 1460)
+	for {
+		n, err := uconn.Read(b)
+		rxAt := time.Now()
+		if err != nil {
+			return 0, fmt.Errorf("error reading from udp socket: %w", err)
+		}
+		gotTxID, _, err := stun.ParseResponse(b[:n])
+		if err != nil || gotTxID != txID {
+			continue
+		}
+		return rxAt.Sub(txAt), nil
+	}
+
+}
+
+func isTemporaryOrTimeoutErr(err error) bool {
+	if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	if err, ok := err.(interface{ Temporary() bool }); ok {
+		return err.Temporary()
+	}
+	return false
+}
+
+type nodeMeta struct {
+	regionID   int
+	regionCode string
+	hostname   string
+	addr       netip.Addr
+}
+
+type measureFn func(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error)
+
+// probe measures STUN round trip time for the node described by meta over
+// conn against dstPort. It may return a nil duration and nil error if the
+// STUN request timed out. A non-nil error indicates an unrecoverable or
+// non-temporary error.
+func probe(meta nodeMeta, conn io.ReadWriteCloser, fn measureFn, dstPort int) (*time.Duration, error) {
+	ua := &net.UDPAddr{
+		IP:   net.IP(meta.addr.AsSlice()),
+		Port: dstPort,
+	}
+
+	time.Sleep(rand.N(200 * time.Millisecond)) // jitter across tx
+	rtt, err := fn(conn, ua)
+	if err != nil {
+		if isTemporaryOrTimeoutErr(err) {
+			log.Printf("temp error measuring RTT to %s(%s): %v", meta.hostname, ua.String(), err)
+			return nil, nil
+		}
+		return nil, err
+	}
+	return &rtt, nil
+}
+
+// nodeMetaFromDERPMap parses the provided DERP map in order to update nodeMeta
+// in the provided nodeMetaByAddr. It returns a slice of nodeMeta containing
+// the nodes that are no longer seen in the DERP map, but were previously held
+// in nodeMetaByAddr.
+func nodeMetaFromDERPMap(dm *tailcfg.DERPMap, nodeMetaByAddr map[netip.Addr]nodeMeta, ipv6 bool) (stale []nodeMeta, err error) {
+	// Parse the new derp map before making any state changes in nodeMetaByAddr.
+	// If parse fails we just stick with the old state.
+	updated := make(map[netip.Addr]nodeMeta)
+	for regionID, region := range dm.Regions {
+		for _, node := range region.Nodes {
+			v4, err := netip.ParseAddr(node.IPv4)
+			if err != nil || !v4.Is4() {
+				return nil, fmt.Errorf("invalid ipv4 addr for node in derp map: %v", node.Name)
+			}
+			metas := make([]nodeMeta, 0, 2)
+			metas = append(metas, nodeMeta{
+				regionID:   regionID,
+				regionCode: region.RegionCode,
+				hostname:   node.HostName,
+				addr:       v4,
+			})
+			if ipv6 {
+				v6, err := netip.ParseAddr(node.IPv6)
+				if err != nil || !v6.Is6() {
+					return nil, fmt.Errorf("invalid ipv6 addr for node in derp map: %v", node.Name)
+				}
+				metas = append(metas, metas[0])
+				metas[1].addr = v6
+			}
+			for _, meta := range metas {
+				updated[meta.addr] = meta
+			}
+		}
+	}
+
+	// Find nodeMeta that have changed
+	for addr, updatedMeta := range updated {
+		previousMeta, ok := nodeMetaByAddr[addr]
+		if ok {
+			if previousMeta == updatedMeta {
+				continue
+			}
+			stale = append(stale, previousMeta)
+			nodeMetaByAddr[addr] = updatedMeta
+		} else {
+			nodeMetaByAddr[addr] = updatedMeta
+		}
+	}
+
+	// Find nodeMeta that no longer exist
+	for addr, potentialStale := range nodeMetaByAddr {
+		_, ok := updated[addr]
+		if !ok {
+			stale = append(stale, potentialStale)
+		}
+	}
+
+	return stale, nil
+}
+
+func getStableConns(stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, addr netip.Addr, dstPort int) ([2]io.ReadWriteCloser, error) {
+	conns := [2]io.ReadWriteCloser{}
+	byDstPort, ok := stableConns[addr]
+	if ok {
+		conns, ok = byDstPort[dstPort]
+		if ok {
+			return conns, nil
+		}
+	}
+	if supportsKernelTS() {
+		kconn, err := getConnKernelTimestamp()
+		if err != nil {
+			return conns, err
+		}
+		conns[timestampSourceKernel] = kconn
+	}
+	uconn, err := net.ListenUDP("udp", &net.UDPAddr{})
+	if err != nil {
+		if supportsKernelTS() {
+			conns[timestampSourceKernel].Close()
+		}
+		return conns, err
+	}
+	conns[timestampSourceUserspace] = uconn
+	if byDstPort == nil {
+		byDstPort = make(map[int][2]io.ReadWriteCloser)
+	}
+	byDstPort[dstPort] = conns
+	stableConns[addr] = byDstPort
+	return conns, nil
+}
+
+// probeNodes measures the round-trip time for STUN binding requests against the
+// DERP nodes described by nodeMetaByAddr while using/updating stableConns for
+// UDP sockets that should be recycled across runs. It returns the results or
+// an error if one occurs.
+func probeNodes(nodeMetaByAddr map[netip.Addr]nodeMeta, stableConns map[netip.Addr]map[int][2]io.ReadWriteCloser, dstPorts []int) ([]result, error) {
+	wg := sync.WaitGroup{}
+	results := make([]result, 0)
+	resultsCh := make(chan result)
+	errCh := make(chan error)
+	doneCh := make(chan struct{})
+	numProbes := 0
+	at := time.Now()
+	addrsToProbe := make(map[netip.Addr]bool)
+
+	doProbe := func(conn io.ReadWriteCloser, meta nodeMeta, source timestampSource, dstPort int) {
+		defer wg.Done()
+		r := result{
+			key: resultKey{
+				meta:            meta,
+				timestampSource: source,
+				dstPort:         dstPort,
+			},
+			at: at,
+		}
+		if conn == nil {
+			var err error
+			if source == timestampSourceKernel {
+				conn, err = getConnKernelTimestamp()
+			} else {
+				conn, err = net.ListenUDP("udp", &net.UDPAddr{})
+			}
+			if err != nil {
+				select {
+				case <-doneCh:
+					return
+				case errCh <- err:
+					return
+				}
+			}
+			defer conn.Close()
+		} else {
+			r.key.connStability = stableConn
+		}
+		fn := measureRTT
+		if source == timestampSourceKernel {
+			fn = measureRTTKernel
+		}
+		rtt, err := probe(meta, conn, fn, dstPort)
+		if err != nil {
+			select {
+			case <-doneCh:
+				return
+			case errCh <- err:
+				return
+			}
+		}
+		r.rtt = rtt
+		select {
+		case <-doneCh:
+		case resultsCh <- r:
+		}
+	}
+
+	for _, meta := range nodeMetaByAddr {
+		addrsToProbe[meta.addr] = true
+		for _, port := range dstPorts {
+			stable, err := getStableConns(stableConns, meta.addr, port)
+			if err != nil {
+				close(doneCh)
+				wg.Wait()
+				return nil, err
+			}
+
+			wg.Add(2)
+			numProbes += 2
+			go doProbe(stable[timestampSourceUserspace], meta, timestampSourceUserspace, port)
+			go doProbe(nil, meta, timestampSourceUserspace, port)
+			if supportsKernelTS() {
+				wg.Add(2)
+				numProbes += 2
+				go doProbe(stable[timestampSourceKernel], meta, timestampSourceKernel, port)
+				go doProbe(nil, meta, timestampSourceKernel, port)
+			}
+		}
+	}
+
+	// cleanup conns we no longer need
+	for k, byDstPort := range stableConns {
+		if !addrsToProbe[k] {
+			for _, conns := range byDstPort {
+				if conns[timestampSourceKernel] != nil {
+					conns[timestampSourceKernel].Close()
+				}
+				conns[timestampSourceUserspace].Close()
+				delete(stableConns, k)
+			}
+		}
+	}
+
+	for {
+		select {
+		case err := <-errCh:
+			close(doneCh)
+			wg.Wait()
+			return nil, err
+		case result := <-resultsCh:
+			results = append(results, result)
+			if len(results) == numProbes {
+				return results, nil
+			}
+		}
+	}
+}
+
+type connStability bool
+
+const (
+	unstableConn connStability = false
+	stableConn   connStability = true
+)
+
+const (
+	rttMetricName      = "stunstamp_derp_stun_rtt_ns"
+	timeoutsMetricName = "stunstamp_derp_stun_timeouts_total"
+)
+
+func timeSeriesLabels(metricName string, meta nodeMeta, instance string, source timestampSource, stability connStability, dstPort int) []prompb.Label {
+	addressFamily := "ipv4"
+	if meta.addr.Is6() {
+		addressFamily = "ipv6"
+	}
+	labels := make([]prompb.Label, 0)
+	labels = append(labels, prompb.Label{
+		Name:  "job",
+		Value: "stunstamp-rw",
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "instance",
+		Value: instance,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "region_id",
+		Value: fmt.Sprintf("%d", meta.regionID),
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "region_code",
+		Value: meta.regionCode,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "address_family",
+		Value: addressFamily,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "hostname",
+		Value: meta.hostname,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "dst_port",
+		Value: strconv.Itoa(dstPort),
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "__name__",
+		Value: metricName,
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "timestamp_source",
+		Value: source.String(),
+	})
+	labels = append(labels, prompb.Label{
+		Name:  "stable_conn",
+		Value: fmt.Sprintf("%v", stability),
+	})
+	slices.SortFunc(labels, func(a, b prompb.Label) int {
+		// prometheus remote-write spec requires lexicographically sorted label names
+		return cmp.Compare(a.Name, b.Name)
+	})
+	return labels
+}
+
+const (
+	// https://prometheus.io/docs/concepts/remote_write_spec/#stale-markers
+	staleNaN uint64 = 0x7ff0000000000002
+)
+
+func staleMarkersFromNodeMeta(stale []nodeMeta, instance string, dstPorts []int) []prompb.TimeSeries {
+	staleMarkers := make([]prompb.TimeSeries, 0)
+	now := time.Now()
+	for _, s := range stale {
+		for _, dstPort := range dstPorts {
+			samples := []prompb.Sample{
+				{
+					Timestamp: now.UnixMilli(),
+					Value:     math.Float64frombits(staleNaN),
+				},
+			}
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
+				Samples: samples,
+			})
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
+				Samples: samples,
+			})
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, unstableConn, dstPort),
+				Samples: samples,
+			})
+			staleMarkers = append(staleMarkers, prompb.TimeSeries{
+				Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceUserspace, stableConn, dstPort),
+				Samples: samples,
+			})
+			if supportsKernelTS() {
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
+					Samples: samples,
+				})
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(rttMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
+					Samples: samples,
+				})
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, unstableConn, dstPort),
+					Samples: samples,
+				})
+				staleMarkers = append(staleMarkers, prompb.TimeSeries{
+					Labels:  timeSeriesLabels(timeoutsMetricName, s, instance, timestampSourceKernel, stableConn, dstPort),
+					Samples: samples,
+				})
+			}
+		}
+	}
+	return staleMarkers
+}
+
+// resultsToPromTimeSeries returns a slice of prometheus TimeSeries for the
+// provided results and instance. timeouts is updated based on results, i.e.
+// all result.key's are added to timeouts if they do not exist, and removed
+// from timeouts if they are not present in results.
+func resultsToPromTimeSeries(results []result, instance string, timeouts map[resultKey]uint64) []prompb.TimeSeries {
+	all := make([]prompb.TimeSeries, 0, len(results)*2)
+	seenKeys := make(map[resultKey]bool)
+	for _, r := range results {
+		timeoutsCount := timeouts[r.key] // a non-existent key will return a zero val
+		seenKeys[r.key] = true
+		rttLabels := timeSeriesLabels(rttMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		rttSamples := make([]prompb.Sample, 1)
+		rttSamples[0].Timestamp = r.at.UnixMilli()
+		if r.rtt != nil {
+			rttSamples[0].Value = float64(*r.rtt)
+		} else {
+			rttSamples[0].Value = math.NaN()
+			timeoutsCount++
+		}
+		rttTS := prompb.TimeSeries{
+			Labels:  rttLabels,
+			Samples: rttSamples,
+		}
+		all = append(all, rttTS)
+		timeouts[r.key] = timeoutsCount
+		timeoutsLabels := timeSeriesLabels(timeoutsMetricName, r.key.meta, instance, r.key.timestampSource, r.key.connStability, r.key.dstPort)
+		timeoutsSamples := make([]prompb.Sample, 1)
+		timeoutsSamples[0].Timestamp = r.at.UnixMilli()
+		timeoutsSamples[0].Value = float64(timeoutsCount)
+		timeoutsTS := prompb.TimeSeries{
+			Labels:  timeoutsLabels,
+			Samples: timeoutsSamples,
+		}
+		all = append(all, timeoutsTS)
+	}
+	for k := range timeouts {
+		if !seenKeys[k] {
+			delete(timeouts, k)
+		}
+	}
+	return all
+}
+
+type remoteWriteClient struct {
+	c   *http.Client
+	url string
+}
+
+type recoverableErr struct {
+	error
+}
+
+func newRemoteWriteClient(url string) *remoteWriteClient {
+	return &remoteWriteClient{
+		c: &http.Client{
+			Timeout: time.Second * 30,
+		},
+		url: url,
+	}
+}
+
+func (r *remoteWriteClient) write(ctx context.Context, ts []prompb.TimeSeries) error {
+	wr := &prompb.WriteRequest{
+		Timeseries: ts,
+	}
+	b, err := wr.Marshal()
+	if err != nil {
+		return fmt.Errorf("unable to marshal write request: %w", err)
+	}
+	compressed := snappy.Encode(nil, b)
+	req, err := http.NewRequestWithContext(ctx, "POST", r.url, bytes.NewReader(compressed))
+	if err != nil {
+		return fmt.Errorf("unable to create write request: %w", err)
+	}
+	req.Header.Add("Content-Encoding", "snappy")
+	req.Header.Set("Content-Type", "application/x-protobuf")
+	req.Header.Set("User-Agent", "stunstamp")
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	resp, err := r.c.Do(req)
+	if err != nil {
+		return recoverableErr{fmt.Errorf("error performing write request: %w", err)}
+	}
+	if resp.StatusCode/100 != 2 {
+		err = fmt.Errorf("remote server %s returned HTTP status %d", r.url, resp.StatusCode)
+	}
+	if resp.StatusCode/100 == 5 || resp.StatusCode == http.StatusTooManyRequests {
+		return recoverableErr{err}
+	}
+	return err
+}
+
+func remoteWriteTimeSeries(client *remoteWriteClient, tsCh chan []prompb.TimeSeries) {
+	bo := backoff.NewBackoff("remote-write", log.Printf, time.Second*30)
+	// writeErr may contribute to bo's backoff schedule across tsCh read ops,
+	// i.e. if an unrecoverable error occurs for client.write(ctx, A), that
+	// should be accounted against bo prior to attempting to
+	// client.write(ctx, B).
+	var writeErr error
+	for ts := range tsCh {
+		for {
+			bo.BackOff(context.Background(), writeErr)
+			reqCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+			writeErr = client.write(reqCtx, ts)
+			cancel()
+			var re recoverableErr
+			recoverable := errors.As(writeErr, &re)
+			if writeErr != nil {
+				log.Printf("remote write error(recoverable=%v): %v", recoverable, writeErr)
+			}
+			if !recoverable {
+				// a nil err is not recoverable
+				break
+			}
+		}
+	}
+}
+
+func main() {
+	flag.Parse()
+	if len(*flagDstPorts) == 0 {
+		log.Fatal("dst-ports flag is unset")
+	}
+	dstPortsSplit := strings.Split(*flagDstPorts, ",")
+	slices.Sort(dstPortsSplit)
+	dstPortsSplit = slices.Compact(dstPortsSplit)
+	dstPorts := make([]int, 0, len(dstPortsSplit))
+	for _, d := range dstPortsSplit {
+		i, err := strconv.ParseUint(d, 10, 16)
+		if err != nil {
+			log.Fatal("invalid dst-ports")
+		}
+		dstPorts = append(dstPorts, int(i))
+	}
+	if len(*flagDERPMap) < 1 {
+		log.Fatal("derp-map flag is unset")
+	}
+	if len(*flagOut) < 1 {
+		log.Fatal("out flag is unset")
+	}
+	if *flagInterval < minInterval || *flagInterval > maxBufferDuration {
+		log.Fatalf("interval must be >= %s and <= %s", minInterval, maxBufferDuration)
+	}
+	if *flagRetention < *flagInterval {
+		log.Fatal("retention must be >= interval")
+	}
+	if len(*flagRemoteWriteURL) < 1 {
+		log.Fatal("rw-url flag is unset")
+	}
+	_, err := url.Parse(*flagRemoteWriteURL)
+	if err != nil {
+		log.Fatalf("invalid rw-url flag value: %v", err)
+	}
+	if len(*flagInstance) < 1 {
+		hostname, err := os.Hostname()
+		if err != nil {
+			log.Fatalf("failed to get hostname: %v", err)
+		}
+		*flagInstance = hostname
+	}
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	dmCh := make(chan *tailcfg.DERPMap)
+
+	go func() {
+		bo := backoff.NewBackoff("derp-map", log.Printf, time.Second*30)
+		for {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+			dm, err := getDERPMap(ctx, *flagDERPMap)
+			cancel()
+			bo.BackOff(context.Background(), err)
+			if err != nil {
+				continue
+			}
+			dmCh <- dm
+			return
+		}
+	}()
+
+	nodeMetaByAddr := make(map[netip.Addr]nodeMeta)
+	select {
+	case <-sigCh:
+		return
+	case dm := <-dmCh:
+		_, err := nodeMetaFromDERPMap(dm, nodeMetaByAddr, *flagIPv6)
+		if err != nil {
+			log.Fatalf("error parsing derp map on startup: %v", err)
+		}
+	}
+
+	db, err := newDB(*flagOut)
+	if err != nil {
+		log.Fatalf("error opening output file for writing: %v", err)
+	}
+	defer db.Close()
+
+	_, err = db.Exec("PRAGMA journal_mode=WAL")
+	if err != nil {
+		log.Fatalf("error enabling WAL mode: %v", err)
+	}
+
+	// No indices or primary key. Keep it simple for now. Reads will be full
+	// scans. We can AUTOINCREMENT rowid in the future and hold an in-memory
+	// index to at_unix if needed as reads are almost always going to be
+	// time-bound (e.g. WHERE at_unix >= ?). At the time of authorship we have
+	// ~300 data points per-interval w/o ipv6 w/kernel timestamping resulting
+	// in ~2.6m rows in 24h w/a 10s probe interval.
+	_, err = db.Exec(`
+CREATE TABLE IF NOT EXISTS rtt(at_unix INT, region_id INT, hostname TEXT, af INT, address TEXT, timestamp_source INT, stable_conn INT, dst_port INT, rtt_ns INT)
+`)
+	if err != nil {
+		log.Fatalf("error initializing db: %v", err)
+	}
+
+	wg := sync.WaitGroup{}
+	httpErrCh := make(chan error, 1)
+	var httpServer *http.Server
+	if len(*flagAPI) > 0 {
+		api := newAPI(db)
+		httpServer = &http.Server{
+			Addr:         *flagAPI,
+			Handler:      api,
+			ReadTimeout:  time.Second * 60,
+			WriteTimeout: time.Second * 60,
+		}
+		wg.Add(1)
+		go func() {
+			err := httpServer.ListenAndServe()
+			httpErrCh <- err
+			wg.Done()
+		}()
+	}
+
+	tsCh := make(chan []prompb.TimeSeries, maxBufferDuration / *flagInterval)
+	remoteWriteDoneCh := make(chan struct{})
+	rwc := newRemoteWriteClient(*flagRemoteWriteURL)
+	go func() {
+		remoteWriteTimeSeries(rwc, tsCh)
+		close(remoteWriteDoneCh)
+	}()
+
+	shutdown := func() {
+		if httpServer != nil {
+			httpServer.Close()
+		}
+		close(tsCh)
+		select {
+		case <-time.After(time.Second * 10): // give goroutine some time to flush
+		case <-remoteWriteDoneCh:
+		}
+
+		// send stale markers on shutdown
+		staleMeta := make([]nodeMeta, 0, len(nodeMetaByAddr))
+		for _, v := range nodeMetaByAddr {
+			staleMeta = append(staleMeta, v)
+		}
+		staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+		if len(staleMarkers) > 0 {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			rwc.write(ctx, staleMarkers)
+			cancel()
+		}
+
+		wg.Wait()
+		return
+	}
+
+	log.Println("stunstamp started")
+
+	// Re-using sockets means we get the same 5-tuple across runs. This results
+	// in a higher probability of the packets traversing the same underlay path.
+	// Comparison of stable and unstable 5-tuple results can shed light on
+	// differences between paths where hashing (multipathing/load balancing)
+	// comes into play.
+	stableConns := make(map[netip.Addr]map[int][2]io.ReadWriteCloser)
+
+	// timeouts holds counts of timeout events. Values are persisted for the
+	// lifetime of the related node in the DERP map.
+	timeouts := make(map[resultKey]uint64)
+
+	derpMapTicker := time.NewTicker(time.Minute * 5)
+	defer derpMapTicker.Stop()
+	probeTicker := time.NewTicker(*flagInterval)
+	defer probeTicker.Stop()
+	cleanupTicker := time.NewTicker(time.Hour)
+	defer cleanupTicker.Stop()
+
+	for {
+		select {
+		case <-cleanupTicker.C:
+			older := time.Now().Add(-*flagRetention)
+			log.Printf("cleaning up measurements older than %v", older)
+			_, err := db.Exec("DELETE FROM rtt WHERE at_unix < ?", older.Unix())
+			if err != nil {
+				log.Printf("error cleaning up old data: %v", err)
+				shutdown()
+				return
+			}
+		case <-probeTicker.C:
+			results, err := probeNodes(nodeMetaByAddr, stableConns, dstPorts)
+			if err != nil {
+				log.Printf("unrecoverable error while probing: %v", err)
+				shutdown()
+				return
+			}
+			ts := resultsToPromTimeSeries(results, *flagInstance, timeouts)
+			select {
+			case tsCh <- ts:
+			default:
+				select {
+				case <-tsCh:
+					log.Println("prometheus remote-write buffer full, dropped measurements")
+				default:
+					tsCh <- ts
+				}
+			}
+			tx, err := db.Begin()
+			if err != nil {
+				log.Printf("error beginning sqlite tx: %v", err)
+				shutdown()
+				return
+			}
+			for _, result := range results {
+				af := 4
+				if result.key.meta.addr.Is6() {
+					af = 6
+				}
+				_, err = tx.Exec("INSERT INTO rtt(at_unix, region_id, hostname, af, address, timestamp_source, stable_conn, dst_port, rtt_ns) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)",
+					result.at.Unix(), result.key.meta.regionID, result.key.meta.hostname, af, result.key.meta.addr.String(), result.key.timestampSource, result.key.connStability, result.key.dstPort, result.rtt)
+				if err != nil {
+					tx.Rollback()
+					log.Printf("error adding result to tx: %v", err)
+					shutdown()
+					return
+				}
+			}
+			err = tx.Commit()
+			if err != nil {
+				log.Printf("error committing tx: %v", err)
+				shutdown()
+				return
+			}
+		case dm := <-dmCh:
+			staleMeta, err := nodeMetaFromDERPMap(dm, nodeMetaByAddr, *flagIPv6)
+			if err != nil {
+				log.Printf("error parsing DERP map, continuing with stale map: %v", err)
+				continue
+			}
+			staleMarkers := staleMarkersFromNodeMeta(staleMeta, *flagInstance, dstPorts)
+			if len(staleMarkers) < 1 {
+				continue
+			}
+			select {
+			case tsCh <- staleMarkers:
+			default:
+				select {
+				case <-tsCh:
+					log.Println("prometheus remote-write buffer full, dropped measurements")
+				default:
+					tsCh <- staleMarkers
+				}
+			}
+		case <-derpMapTicker.C:
+			go func() {
+				ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+				defer cancel()
+				updatedDM, err := getDERPMap(ctx, *flagDERPMap)
+				if err != nil {
+					dmCh <- updatedDM
+				}
+			}()
+		case err := <-httpErrCh:
+			log.Printf("http server error: %v", err)
+			shutdown()
+			return
+		case <-sigCh:
+			shutdown()
+			return
+		}
+	}
+}

cmd/stunstamp/stunstamp_db_default.go

@@ -0,0 +1,26 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !(windows && 386)
+
+package main
+
+import (
+	"database/sql"
+
+	_ "modernc.org/sqlite"
+)
+
+type db struct {
+	*sql.DB
+}
+
+func newDB(path string) (*db, error) {
+	d, err := sql.Open("sqlite", *flagOut)
+	if err != nil {
+		return nil, err
+	}
+	return &db{
+		DB: d,
+	}, nil
+}

cmd/stunstamp/stunstamp_db_windows_386.go

@@ -0,0 +1,17 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"database/sql"
+	"errors"
+)
+
+type db struct {
+	*sql.DB
+}
+
+func newDB(path string) (*db, error) {
+	return nil, errors.New("unsupported platform")
+}

cmd/stunstamp/stunstamp_default.go

@@ -0,0 +1,25 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package main
+
+import (
+	"errors"
+	"io"
+	"net"
+	"time"
+)
+
+func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unimplemented")
+}
+
+func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+	return 0, errors.New("unimplemented")
+}
+
+func supportsKernelTS() bool {
+	return false
+}

cmd/stunstamp/stunstamp_linux.go

@@ -0,0 +1,143 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"github.com/mdlayher/socket"
+	"golang.org/x/sys/unix"
+	"tailscale.com/net/stun"
+)
+
+const (
+	flags = unix.SOF_TIMESTAMPING_TX_SOFTWARE | // tx timestamp generation in device driver
+		unix.SOF_TIMESTAMPING_RX_SOFTWARE | // rx timestamp generation in the kernel
+		unix.SOF_TIMESTAMPING_SOFTWARE // report software timestamps
+)
+
+func getConnKernelTimestamp() (io.ReadWriteCloser, error) {
+	sconn, err := socket.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP, "udp", nil)
+	if err != nil {
+		return nil, err
+	}
+	sa := unix.SockaddrInet6{}
+	err = sconn.Bind(&sa)
+	if err != nil {
+		return nil, err
+	}
+	err = sconn.SetsockoptInt(unix.SOL_SOCKET, unix.SO_TIMESTAMPING_NEW, flags)
+	if err != nil {
+		return nil, err
+	}
+	return sconn, nil
+}
+
+func parseTimestampFromCmsgs(oob []byte) (time.Time, error) {
+	msgs, err := unix.ParseSocketControlMessage(oob)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("error parsing oob as cmsgs: %w", err)
+	}
+	for _, msg := range msgs {
+		if msg.Header.Level == unix.SOL_SOCKET && msg.Header.Type == unix.SO_TIMESTAMPING_NEW && len(msg.Data) >= 16 {
+			sec := int64(binary.NativeEndian.Uint64(msg.Data[:8]))
+			ns := int64(binary.NativeEndian.Uint64(msg.Data[8:16]))
+			return time.Unix(sec, ns), nil
+		}
+	}
+	return time.Time{}, errors.New("failed to parse timestamp from cmsgs")
+}
+
+func measureRTTKernel(conn io.ReadWriteCloser, dst *net.UDPAddr) (rtt time.Duration, err error) {
+	sconn, ok := conn.(*socket.Conn)
+	if !ok {
+		return 0, fmt.Errorf("conn of unexpected type: %T", conn)
+	}
+
+	var to unix.Sockaddr
+	to4 := dst.IP.To4()
+	if to4 != nil {
+		to = &unix.SockaddrInet4{
+			Port: dst.Port,
+		}
+		copy(to.(*unix.SockaddrInet4).Addr[:], to4)
+	} else {
+		to = &unix.SockaddrInet6{
+			Port: dst.Port,
+		}
+		copy(to.(*unix.SockaddrInet6).Addr[:], dst.IP)
+	}
+
+	txID := stun.NewTxID()
+	req := stun.Request(txID)
+
+	err = sconn.Sendto(context.Background(), req, 0, to)
+	if err != nil {
+		return 0, fmt.Errorf("sendto error: %v", err) // don't wrap
+	}
+
+	txCtx, txCancel := context.WithTimeout(context.Background(), time.Second*2)
+	defer txCancel()
+
+	buf := make([]byte, 1024)
+	oob := make([]byte, 1024)
+	var txAt time.Time
+
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(txCtx, buf, oob, unix.MSG_ERRQUEUE)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg (MSG_ERRQUEUE) error: %v", err) // don't wrap
+		}
+
+		buf = buf[:n]
+		if n < len(req) || !bytes.Equal(req, buf[len(buf)-len(req):]) {
+			// Spin until we find the message we sent. We get the full packet
+			// looped including eth header so match against the tail.
+			continue
+		}
+		txAt, err = parseTimestampFromCmsgs(oob[:oobn])
+		if err != nil {
+			return 0, fmt.Errorf("failed to get tx timestamp: %v", err) // don't wrap
+		}
+		break
+	}
+
+	rxCtx, rxCancel := context.WithTimeout(context.Background(), time.Second*2)
+	defer rxCancel()
+
+	for {
+		n, oobn, _, _, err := sconn.Recvmsg(rxCtx, buf, oob, 0)
+		if err != nil {
+			return 0, fmt.Errorf("recvmsg error: %w", err) // wrap for timeout-related error unwrapping
+		}
+
+		gotTxID, _, err := stun.ParseResponse(buf[:n])
+		if err != nil || gotTxID != txID {
+			// Spin until we find the txID we sent. We may end up reading
+			// extremely late arriving responses from previous intervals. As
+			// such, we can't be certain if we're parsing the "current"
+			// response, so spin for parse errors too.
+			continue
+		}
+
+		rxAt, err := parseTimestampFromCmsgs(oob[:oobn])
+		if err != nil {
+			return 0, fmt.Errorf("failed to get rx timestamp: %v", err) // don't wrap
+		}
+
+		return rxAt.Sub(txAt), nil
+	}
+
+}
+
+func supportsKernelTS() bool {
+	return true
+}

cmd/tailscale/cli/debug.go

@@ -28,10 +28,12 @@ import (
 
 	"github.com/peterbourgon/ff/v3/ffcli"
 	"golang.org/x/net/http/httpproxy"
+	"golang.org/x/net/http2"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/control/controlhttp"
 	"tailscale.com/hostinfo"
+	"tailscale.com/internal/noiseconn"
 	"tailscale.com/ipn"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tshttpproxy"
@@ -801,7 +803,10 @@ func runTS2021(ctx context.Context, args []string) error {
 		log.Printf("Dial(%q, %q) ...", network, address)
 		c, err := dialer.DialContext(ctx, network, address)
 		if err != nil {
-			log.Printf("Dial(%q, %q) = %v", network, address, err)
+			// skip logging context cancellation errors
+			if !errors.Is(err, context.Canceled) {
+				log.Printf("Dial(%q, %q) = %v", network, address, err)
+			}
 		} else {
 			log.Printf("Dial(%q, %q) = %v / %v", network, address, c.LocalAddr(), c.RemoteAddr())
 		}
@@ -834,6 +839,52 @@ func runTS2021(ctx context.Context, args []string) error {
 	}
 
 	log.Printf("final underlying conn: %v / %v", conn.LocalAddr(), conn.RemoteAddr())
+
+	h2Transport, err := http2.ConfigureTransports(&http.Transport{
+		IdleConnTimeout: time.Second,
+	})
+	if err != nil {
+		return fmt.Errorf("http2.ConfigureTransports: %w", err)
+	}
+
+	// Now, create a Noise conn over the existing conn.
+	nc, err := noiseconn.New(conn.Conn, h2Transport, 0, nil)
+	if err != nil {
+		return fmt.Errorf("noiseconn.New: %w", err)
+	}
+	defer nc.Close()
+
+	// Reserve a RoundTrip for the whoami request.
+	ok, _, err := nc.ReserveNewRequest(ctx)
+	if err != nil {
+		return fmt.Errorf("ReserveNewRequest: %w", err)
+	}
+	if !ok {
+		return errors.New("ReserveNewRequest failed")
+	}
+
+	// Make a /whoami request to the server to verify that we can actually
+	// communicate over the newly-established connection.
+	whoamiURL := "http://" + ts2021Args.host + "/machine/whoami"
+	req, err = http.NewRequestWithContext(ctx, "GET", whoamiURL, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := nc.RoundTrip(req)
+	if err != nil {
+		return fmt.Errorf("RoundTrip whoami request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		log.Printf("whoami request returned status %v", resp.Status)
+	} else {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("reading whoami response: %w", err)
+		}
+		log.Printf("whoami response: %q", body)
+	}
 	return nil
 }
 

cmd/tailscale/cli/netcheck.go

@@ -78,9 +78,13 @@ func runNetcheck(ctx context.Context, args []string) error {
 		log.Printf("No DERP map from tailscaled; using default.")
 	}
 	if err != nil || noRegions {
-		hc := &http.Client{Transport: tlsdial.NewTransport()}
+		hc := &http.Client{
+			Transport: tlsdial.NewTransport(),
+			Timeout:   10 * time.Second,
+		}
 		dm, err = prodDERPMap(ctx, hc)
 		if err != nil {
+			log.Println("Failed to fetch a DERP map, so netcheck cannot continue. Check your Internet connection.")
 			return err
 		}
 	}
@@ -209,6 +213,7 @@ func portMapping(r *netcheck.Report) string {
 }
 
 func prodDERPMap(ctx context.Context, httpc *http.Client) (*tailcfg.DERPMap, error) {
+	log.Printf("attempting to fetch a DERPMap from %s", ipn.DefaultControlURL)
 	req, err := http.NewRequestWithContext(ctx, "GET", ipn.DefaultControlURL+"/derpmap/default", nil)
 	if err != nil {
 		return nil, fmt.Errorf("create prodDERPMap request: %w", err)

cmd/tailscale/cli/update.go

@@ -33,11 +33,13 @@ var updateCmd = &ffcli.Command{
 		//  - Alpine (and other apk-based distros)
 		//  - FreeBSD (and other pkg-based distros)
 		//  - Unraid/QNAP/Synology
+		//  - macOS
 		if distro.Get() != distro.Arch &&
 			distro.Get() != distro.Alpine &&
 			distro.Get() != distro.QNAP &&
 			distro.Get() != distro.Synology &&
-			runtime.GOOS != "freebsd" {
+			runtime.GOOS != "freebsd" &&
+			runtime.GOOS != "darwin" {
 			fs.StringVar(&updateArgs.track, "track", "", `which track to check for updates: "stable" or "unstable" (dev); empty means same as current`)
 			fs.StringVar(&updateArgs.version, "version", "", `explicit version to update/downgrade to`)
 		}

cmd/tailscale/cli/whois.go

@@ -26,12 +26,14 @@ var whoisCmd = &ffcli.Command{
 	FlagSet: func() *flag.FlagSet {
 		fs := newFlagSet("whois")
 		fs.BoolVar(&whoIsArgs.json, "json", false, "output in JSON format")
+		fs.StringVar(&whoIsArgs.proto, "proto", "", `protocol; one of "tcp" or "udp"; empty mans both `)
 		return fs
 	}(),
 }
 
 var whoIsArgs struct {
-	json bool // output in JSON format
+	json  bool   // output in JSON format
+	proto string // "tcp" or "udp"
 }
 
 func runWhoIs(ctx context.Context, args []string) error {
@@ -40,7 +42,7 @@ func runWhoIs(ctx context.Context, args []string) error {
 	} else if len(args) == 0 {
 		return errors.New("missing argument, expected one peer")
 	}
-	who, err := localClient.WhoIs(ctx, args[0])
+	who, err := localClient.WhoIsProto(ctx, whoIsArgs.proto, args[0])
 	if err != nil {
 		return err
 	}

cmd/tailscale/depaware.txt

@@ -57,7 +57,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    L    github.com/vishvananda/netns                                 from github.com/tailscale/netlink+
         github.com/x448/float16                                      from github.com/fxamacker/cbor/v2
      💣 go4.org/mem                                                  from tailscale.com/client/tailscale+
-        go4.org/netipx                                               from tailscale.com/net/tsaddr+
+        go4.org/netipx                                               from tailscale.com/net/tsaddr
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/netmon+
         k8s.io/client-go/util/homedir                                from tailscale.com/cmd/tailscale/cli
         nhooyr.io/websocket                                          from tailscale.com/control/controlhttp+
@@ -78,7 +78,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/cmd/tailscale/cli                              from tailscale.com/cmd/tailscale
         tailscale.com/cmd/tailscale/cli/ffcomplete                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/cmd/tailscale/cli/ffcomplete/internal          from tailscale.com/cmd/tailscale/cli/ffcomplete
-        tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp
+        tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlhttp                            from tailscale.com/cmd/tailscale/cli
         tailscale.com/control/controlknobs                           from tailscale.com/net/portmapper
         tailscale.com/derp                                           from tailscale.com/derp/derphttp
@@ -89,6 +89,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/health                                         from tailscale.com/net/tlsdial+
         tailscale.com/health/healthmsg                               from tailscale.com/cmd/tailscale/cli
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
+        tailscale.com/internal/noiseconn                             from tailscale.com/cmd/tailscale/cli
         tailscale.com/ipn                                            from tailscale.com/client/tailscale+
         tailscale.com/ipn/ipnstate                                   from tailscale.com/client/tailscale+
         tailscale.com/licenses                                       from tailscale.com/client/web+
@@ -96,7 +97,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/dns/recursive                              from tailscale.com/net/dnsfallback
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlhttp+
         tailscale.com/net/dnsfallback                                from tailscale.com/control/controlhttp
-        tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
+        tailscale.com/net/flowtrack                                  from tailscale.com/net/packet
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netcheck                                   from tailscale.com/cmd/tailscale/cli
         tailscale.com/net/neterror                                   from tailscale.com/net/netcheck+
@@ -104,7 +105,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
      💣 tailscale.com/net/netmon                                     from tailscale.com/cmd/tailscale/cli+
         tailscale.com/net/netns                                      from tailscale.com/derp/derphttp+
         tailscale.com/net/netutil                                    from tailscale.com/client/tailscale+
-        tailscale.com/net/packet                                     from tailscale.com/wgengine/capture+
+        tailscale.com/net/packet                                     from tailscale.com/wgengine/capture
         tailscale.com/net/ping                                       from tailscale.com/net/netcheck
         tailscale.com/net/portmapper                                 from tailscale.com/cmd/tailscale/cli+
         tailscale.com/net/sockstats                                  from tailscale.com/control/controlhttp+
@@ -164,11 +165,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/util/vizerror                                  from tailscale.com/tailcfg+
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate
-   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo
+   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
         tailscale.com/wgengine/capture                               from tailscale.com/cmd/tailscale/cli
-        tailscale.com/wgengine/filter                                from tailscale.com/types/netmap
+        tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap
         golang.org/x/crypto/argon2                                   from tailscale.com/tka
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/argon2+
         golang.org/x/crypto/blake2s                                  from tailscale.com/clientupdate/distsign+
@@ -182,13 +183,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
         golang.org/x/crypto/pbkdf2                                   from software.sslmate.com/src/go-pkcs12
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
-   W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe
+   W    golang.org/x/exp/constraints                                 from github.com/dblohm7/wingoes/pe+
         golang.org/x/exp/maps                                        from tailscale.com/cmd/tailscale/cli
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
         golang.org/x/net/http/httpguts                               from net/http+
         golang.org/x/net/http/httpproxy                              from net/http+
-        golang.org/x/net/http2/hpack                                 from net/http
+        golang.org/x/net/http2                                       from tailscale.com/cmd/tailscale/cli+
+        golang.org/x/net/http2/hpack                                 from net/http+
         golang.org/x/net/icmp                                        from tailscale.com/net/ping
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
         golang.org/x/net/ipv4                                        from github.com/miekg/dns+
@@ -277,6 +279,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
+        math/rand/v2                                                 from tailscale.com/derp+
         mime                                                         from golang.org/x/oauth2/internal+
         mime/multipart                                               from net/http
         mime/quotedprintable                                         from mime/multipart

cmd/tailscale/tailscale_test.go

@@ -17,6 +17,10 @@ func TestDeps(t *testing.T) {
 			"gvisor.dev/gvisor/pkg/cpuid":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip":        "https://github.com/tailscale/tailscale/issues/9756",
 			"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
+			"tailscale.com/wgengine/filter":      "brings in bart, etc",
+			"github.com/bits-and-blooms/bitset":  "unneeded in CLI",
+			"github.com/gaissmai/bart":           "unneeded in CLI",
+			"tailscale.com/net/ipset":            "unneeded in CLI",
 		},
 	}.Check(t)
 }

cmd/tailscaled/depaware.txt

@@ -245,7 +245,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/clientupdate                                   from tailscale.com/client/web+
         tailscale.com/clientupdate/distsign                          from tailscale.com/clientupdate
         tailscale.com/cmd/tailscaled/childproc                       from tailscale.com/cmd/tailscaled+
-        tailscale.com/control/controlbase                            from tailscale.com/control/controlclient+
+        tailscale.com/control/controlbase                            from tailscale.com/control/controlhttp+
         tailscale.com/control/controlclient                          from tailscale.com/cmd/tailscaled+
         tailscale.com/control/controlhttp                            from tailscale.com/control/controlclient
         tailscale.com/control/controlknobs                           from tailscale.com/control/controlclient+
@@ -265,6 +265,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/health                                         from tailscale.com/control/controlclient+
         tailscale.com/health/healthmsg                               from tailscale.com/ipn/ipnlocal
         tailscale.com/hostinfo                                       from tailscale.com/client/web+
+        tailscale.com/internal/noiseconn                             from tailscale.com/control/controlclient
         tailscale.com/ipn                                            from tailscale.com/client/tailscale+
         tailscale.com/ipn/conffile                                   from tailscale.com/cmd/tailscaled+
      💣 tailscale.com/ipn/ipnauth                                    from tailscale.com/ipn/ipnlocal+
@@ -295,6 +296,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/net/dnscache                                   from tailscale.com/control/controlclient+
         tailscale.com/net/dnsfallback                                from tailscale.com/cmd/tailscaled+
         tailscale.com/net/flowtrack                                  from tailscale.com/net/packet+
+        tailscale.com/net/ipset                                      from tailscale.com/ipn/ipnlocal+
         tailscale.com/net/netaddr                                    from tailscale.com/ipn+
         tailscale.com/net/netcheck                                   from tailscale.com/wgengine/magicsock+
         tailscale.com/net/neterror                                   from tailscale.com/net/dns/resolver+
@@ -400,7 +402,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/util/winutil                                   from tailscale.com/clientupdate+
    W 💣 tailscale.com/util/winutil/authenticode                      from tailscale.com/clientupdate+
    W    tailscale.com/util/winutil/policy                            from tailscale.com/ipn/ipnlocal
-   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo
+   W 💣 tailscale.com/util/winutil/winenv                            from tailscale.com/hostinfo+
         tailscale.com/util/zstdframe                                 from tailscale.com/control/controlclient+
         tailscale.com/version                                        from tailscale.com/client/web+
         tailscale.com/version/distro                                 from tailscale.com/client/web+
@@ -408,6 +410,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine                                       from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/capture                               from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
+        tailscale.com/wgengine/filter/filtertype                     from tailscale.com/types/netmap+
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled
@@ -514,7 +517,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         hash                                                         from compress/zlib+
         hash/adler32                                                 from compress/zlib+
         hash/crc32                                                   from compress/gzip+
-        hash/fnv                                                     from tailscale.com/wgengine/magicsock
         hash/maphash                                                 from go4.org/mem
         html                                                         from html/template+
         html/template                                                from github.com/gorilla/csrf
@@ -529,7 +531,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         math/big                                                     from crypto/dsa+
         math/bits                                                    from compress/flate+
         math/rand                                                    from github.com/mdlayher/netlink+
-        math/rand/v2                                                 from tailscale.com/util/rands
+        math/rand/v2                                                 from tailscale.com/util/rands+
         mime                                                         from github.com/tailscale/xnet/webdav+
         mime/multipart                                               from net/http+
         mime/quotedprintable                                         from mime/multipart

cmd/tailscaled/tailscaled.go

@@ -698,7 +698,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 			// configuration being unavailable (from the noop
 			// manager). More in Issue 4017.
 			// TODO(bradfitz): add a Synology-specific DNS manager.
-			conf.DNS, err = dns.NewOSConfigurator(logf, sys.HealthTracker(), "") // empty interface name
+			conf.DNS, err = dns.NewOSConfigurator(logf, sys.HealthTracker(), sys.ControlKnobs(), "") // empty interface name
 			if err != nil {
 				return false, fmt.Errorf("dns.NewOSConfigurator: %w", err)
 			}
@@ -726,7 +726,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo
 			return false, fmt.Errorf("creating router: %w", err)
 		}
 
-		d, err := dns.NewOSConfigurator(logf, sys.HealthTracker(), devName)
+		d, err := dns.NewOSConfigurator(logf, sys.HealthTracker(), sys.ControlKnobs(), devName)
 		if err != nil {
 			dev.Close()
 			r.Close()

cmd/tailscaled/tailscaled_windows.go

@@ -435,6 +435,9 @@ func babysitProc(ctx context.Context, args []string, logf logger.Logf) {
 		startTime := time.Now()
 		log.Printf("exec: %#v %v", executable, args)
 		cmd := exec.Command(executable, args...)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			CreationFlags: windows.DETACHED_PROCESS,
+		}
 
 		// Create a pipe object to use as the subproc's stdin.
 		// When the writer goes away, the reader gets EOF.

cmd/tsconnect/wasm/wasm_js.go

@@ -16,7 +16,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"math/rand"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/netip"
@@ -604,7 +604,7 @@ func filterSlice[T any](a []T, f func(T) bool) []T {
 func generateHostname() string {
 	tails := words.Tails()
 	scales := words.Scales()
-	if rand.Int()%2 == 0 {
+	if rand.IntN(2) == 0 {
 		// JavaScript
 		tails = filterSlice(tails, func(s string) bool { return strings.HasPrefix(s, "j") })
 		scales = filterSlice(scales, func(s string) bool { return strings.HasPrefix(s, "s") })
@@ -614,8 +614,8 @@ func generateHostname() string {
 		scales = filterSlice(scales, func(s string) bool { return strings.HasPrefix(s, "a") })
 	}
 
-	tail := tails[rand.Intn(len(tails))]
-	scale := scales[rand.Intn(len(scales))]
+	tail := tails[rand.IntN(len(tails))]
+	scale := scales[rand.IntN(len(scales))]
 	return fmt.Sprintf("%s-%s", tail, scale)
 }
 

cmd/viewer/tests/tests_view.go

@@ -188,11 +188,7 @@ func (v *MapView) UnmarshalJSON(b []byte) error {
 
 func (v MapView) Int() views.Map[string, int] { return views.MapOf(v.ж.Int) }
 
-func (v MapView) SliceInt() views.MapFn[string, []int, views.Slice[int]] {
-	return views.MapFnOf(v.ж.SliceInt, func(t []int) views.Slice[int] {
-		return views.SliceOf(t)
-	})
-}
+func (v MapView) SliceInt() views.MapSlice[string, int] { return views.MapSliceOf(v.ж.SliceInt) }
 
 func (v MapView) StructPtrWithPtr() views.MapFn[string, *StructWithPtrs, StructWithPtrsView] {
 	return views.MapFnOf(v.ж.StructPtrWithPtr, func(t *StructWithPtrs) StructWithPtrsView {

cmd/viewer/viewer.go

@@ -92,6 +92,9 @@ func(v {{.ViewName}}) {{.FieldName}}() views.MapFn[{{.MapKeyType}},{{.MapValueTy
 	return {{.MapFn}}
 })}
 {{end}}
+{{define "mapSliceField"}}
+func(v {{.ViewName}}) {{.FieldName}}() views.MapSlice[{{.MapKeyType}},{{.MapValueType}}] { return views.MapSliceOf(v.ж.{{.FieldName}}) }
+{{end}}
 {{define "unsupportedField"}}func(v {{.ViewName}}) {{.FieldName}}() {{.FieldType}} {panic("unsupported")}
 {{end}}
 {{define "stringFunc"}}func(v {{.ViewName}}) String() string { return v.ж.String() }
@@ -241,9 +244,8 @@ func genView(buf *bytes.Buffer, it *codegen.ImportTracker, typ *types.Named, thi
 				case *types.Basic, *types.Named:
 					sElem := it.QualifiedName(sElem)
 					args.MapValueView = fmt.Sprintf("views.Slice[%v]", sElem)
-					args.MapValueType = "[]" + sElem
-					args.MapFn = "views.SliceOf(t)"
-					template = "mapFnField"
+					args.MapValueType = sElem
+					template = "mapSliceField"
 				case *types.Pointer:
 					ptr := x
 					pElem := ptr.Elem()

cmd/xdpderper/xdpderper.go

@@ -0,0 +1,96 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"flag"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"tailscale.com/derp/xdp"
+	"tailscale.com/tsweb"
+)
+
+var (
+	flagDevice  = flag.String("device", "", "target device name")
+	flagPort    = flag.Int("dst-port", 0, "destination UDP port to serve")
+	flagVerbose = flag.Bool("verbose", false, "verbose output including verifier errors")
+	flagMode    = flag.String("mode", "xdp", "XDP mode; valid modes: [xdp, xdpgeneric, xdpdrv, xdpoffload]")
+	flagHTTP    = flag.String("http", ":8230", "HTTP listen address")
+)
+
+func main() {
+	flag.Parse()
+	var attachFlags xdp.XDPAttachFlags
+	switch strings.ToLower(*flagMode) {
+	case "xdp":
+		attachFlags = 0
+	case "xdpgeneric":
+		attachFlags = xdp.XDPGenericMode
+	case "xdpdrv":
+		attachFlags = xdp.XDPDriverMode
+	case "xdpoffload":
+		attachFlags = xdp.XDPOffloadMode
+	default:
+		log.Fatal("invalid mode")
+	}
+	server, err := xdp.NewSTUNServer(&xdp.STUNServerConfig{
+		DeviceName:      *flagDevice,
+		DstPort:         *flagPort,
+		AttachFlags:     attachFlags,
+		FullVerifierErr: *flagVerbose,
+	})
+	if err != nil {
+		log.Fatalf("failed to init XDP STUN server: %v", err)
+	}
+	defer server.Close()
+	err = prometheus.Register(server)
+	if err != nil {
+		log.Fatalf("failed to register XDP STUN server as a prometheus collector: %v", err)
+	}
+	log.Println("XDP STUN server started")
+
+	mux := http.NewServeMux()
+	debug := tsweb.Debugger(mux)
+	debug.KVFunc("Drop STUN", func() any {
+		return server.GetDropSTUN()
+	})
+	debug.Handle("drop-stun-on", "Drop STUN packets", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := server.SetDropSTUN(true)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+		} else {
+			io.WriteString(w, "STUN packets are now being dropped.")
+		}
+	}))
+	debug.Handle("drop-stun-off", "Handle STUN packets", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := server.SetDropSTUN(false)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+		} else {
+			io.WriteString(w, "STUN packets are now being handled.")
+		}
+	}))
+	errCh := make(chan error, 1)
+	go func() {
+		err := http.ListenAndServe(*flagHTTP, mux)
+		errCh <- err
+	}()
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	select {
+	case err := <-errCh:
+		log.Printf("HTTP serve err: %v", err)
+	case sig := <-sigCh:
+		log.Printf("received signal: %s", sig)
+	}
+
+}

control/controlclient/direct.go

@@ -15,6 +15,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
@@ -25,6 +26,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go4.org/mem"
@@ -62,6 +64,7 @@ import (
 // Direct is the client that connects to a tailcontrol server for a node.
 type Direct struct {
 	httpc                      *http.Client // HTTP client used to talk to tailcontrol
+	interceptedDial            *atomic.Bool // if non-nil, pointer to bool whether ScreenTime intercepted our dial
 	dialer                     *tsdial.Dialer
 	dnsCache                   *dnscache.Resolver
 	controlKnobs               *controlknobs.Knobs // always non-nil
@@ -258,23 +261,28 @@ func NewDirect(opts Options) (*Direct, error) {
 		// etc set).
 		httpc = http.DefaultClient
 	}
+	var interceptedDial *atomic.Bool
 	if httpc == nil {
 		tr := http.DefaultTransport.(*http.Transport).Clone()
 		tr.Proxy = tshttpproxy.ProxyFromEnvironment
 		tshttpproxy.SetTransportGetProxyConnectHeader(tr)
 		tr.TLSClientConfig = tlsdial.Config(serverURL.Hostname(), opts.HealthTracker, tr.TLSClientConfig)
-		tr.DialContext = dnscache.Dialer(opts.Dialer.SystemDial, dnsCache)
-		tr.DialTLSContext = dnscache.TLSDialer(opts.Dialer.SystemDial, dnsCache, tr.TLSClientConfig)
+		var dialFunc dialFunc
+		dialFunc, interceptedDial = makeScreenTimeDetectingDialFunc(opts.Dialer.SystemDial)
+		tr.DialContext = dnscache.Dialer(dialFunc, dnsCache)
+		tr.DialTLSContext = dnscache.TLSDialer(dialFunc, dnsCache, tr.TLSClientConfig)
 		tr.ForceAttemptHTTP2 = true
 		// Disable implicit gzip compression; the various
 		// handlers (register, map, set-dns, etc) do their own
 		// zstd compression per naclbox.
 		tr.DisableCompression = true
+
 		httpc = &http.Client{Transport: tr}
 	}
 
 	c := &Direct{
 		httpc:                      httpc,
+		interceptedDial:            interceptedDial,
 		controlKnobs:               opts.ControlKnobs,
 		getMachinePrivKey:          opts.GetMachinePrivateKey,
 		serverURL:                  opts.ServerURL,
@@ -464,6 +472,16 @@ func (c *Direct) hostInfoLocked() *tailcfg.Hostinfo {
 	return hi
 }
 
+var macOSScreenTime = health.Register(&health.Warnable{
+	Code:     "macos-screen-time-controlclient",
+	Severity: health.SeverityHigh,
+	Title:    "Tailscale blocked by Screen Time",
+	Text: func(args health.Args) string {
+		return "macOS Screen Time seems to be blocking Tailscale. Try disabling Screen Time in System Settings > Screen Time > Content & Privacy > Access to Web Content."
+	},
+	ImpactsConnectivity: true,
+})
+
 func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, newURL string, nks tkatype.MarshaledSignature, err error) {
 	if c.panicOnUse {
 		panic("tainted client")
@@ -505,6 +523,11 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	c.logf("doLogin(regen=%v, hasUrl=%v)", regen, opt.URL != "")
 	if serverKey.IsZero() {
 		keys, err := loadServerPubKeys(ctx, c.httpc, c.serverURL)
+		if err != nil && c.interceptedDial != nil && c.interceptedDial.Load() {
+			c.health.SetUnhealthy(macOSScreenTime, nil)
+		} else {
+			c.health.SetHealthy(macOSScreenTime)
+		}
 		if err != nil {
 			return regen, opt.URL, nil, err
 		}
@@ -1581,9 +1604,9 @@ func postPingResult(start time.Time, logf logger.Logf, c *http.Client, pr *tailc
 }
 
 // ReportHealthChange reports to the control plane a change to this node's
-// health.
-func (c *Direct) ReportHealthChange(sys health.Subsystem, sysErr error) {
-	if sys == health.SysOverall {
+// health. w must be non-nil. us can be nil to indicate a healthy state for w.
+func (c *Direct) ReportHealthChange(w *health.Warnable, us *health.UnhealthyState) {
+	if w == health.NetworkStatusWarnable || w == health.IPNStateWarnable || w == health.LoginStateWarnable {
 		// We don't report these. These include things like the network is down
 		// (in which case we can't report anyway) or the user wanted things
 		// stopped, as opposed to the more unexpected failure types in the other
@@ -1602,12 +1625,13 @@ func (c *Direct) ReportHealthChange(sys health.Subsystem, sysErr error) {
 	if c.panicOnUse {
 		panic("tainted client")
 	}
+	// TODO(angott): at some point, update `Subsys` in the request to be `Warnable`
 	req := &tailcfg.HealthChangeRequest{
-		Subsys:  string(sys),
+		Subsys:  string(w.Code),
 		NodeKey: nodeKey,
 	}
-	if sysErr != nil {
-		req.Error = sysErr.Error()
+	if us != nil {
+		req.Error = us.Text
 	}
 
 	// Best effort, no logging:
@@ -1663,6 +1687,38 @@ func addLBHeader(req *http.Request, nodeKey key.NodePublic) {
 	}
 }
 
+type dialFunc = func(ctx context.Context, network, addr string) (net.Conn, error)
+
+// makeScreenTimeDetectingDialFunc returns dialFunc, optionally wrapped (on
+// Apple systems) with a func that sets the returned atomic.Bool for whether
+// Screen Time seemed to intercept the connection.
+//
+// The returned *atomic.Bool is nil on non-Apple systems.
+func makeScreenTimeDetectingDialFunc(dial dialFunc) (dialFunc, *atomic.Bool) {
+	switch runtime.GOOS {
+	case "darwin", "ios":
+		// Continue below.
+	default:
+		return dial, nil
+	}
+	ab := new(atomic.Bool)
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		c, err := dial(ctx, network, addr)
+		if err != nil {
+			return nil, err
+		}
+		ab.Store(isTCPLoopback(c.LocalAddr()) && isTCPLoopback(c.RemoteAddr()))
+		return c, nil
+	}, ab
+}
+
+func isTCPLoopback(a net.Addr) bool {
+	if ta, ok := a.(*net.TCPAddr); ok {
+		return ta.IP.IsLoopback()
+	}
+	return false
+}
+
 var (
 	metricMapRequestsActive = clientmetric.NewGauge("controlclient_map_requests_active")
 

control/controlclient/noise.go

@@ -6,10 +6,8 @@ package controlclient
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"encoding/json"
 	"errors"
-	"io"
 	"math"
 	"net/http"
 	"net/url"
@@ -17,9 +15,9 @@ import (
 	"time"
 
 	"golang.org/x/net/http2"
-	"tailscale.com/control/controlbase"
 	"tailscale.com/control/controlhttp"
 	"tailscale.com/health"
+	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
@@ -32,113 +30,6 @@ import (
 	"tailscale.com/util/singleflight"
 )
 
-// noiseConn is a wrapper around controlbase.Conn.
-// It allows attaching an ID to a connection to allow
-// cleaning up references in the pool when the connection
-// is closed.
-type noiseConn struct {
-	*controlbase.Conn
-	id   int
-	pool *NoiseClient
-	h2cc *http2.ClientConn
-
-	readHeaderOnce    sync.Once     // guards init of reader field
-	reader            io.Reader     // (effectively Conn.Reader after header)
-	earlyPayloadReady chan struct{} // closed after earlyPayload is set (including set to nil)
-	earlyPayload      *tailcfg.EarlyNoise
-	earlyPayloadErr   error
-}
-
-func (c *noiseConn) RoundTrip(r *http.Request) (*http.Response, error) {
-	return c.h2cc.RoundTrip(r)
-}
-
-// getEarlyPayload waits for the early noise payload to arrive.
-// It may return (nil, nil) if the server begins HTTP/2 without one.
-func (c *noiseConn) getEarlyPayload(ctx context.Context) (*tailcfg.EarlyNoise, error) {
-	select {
-	case <-c.earlyPayloadReady:
-		return c.earlyPayload, c.earlyPayloadErr
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-}
-
-// The first 9 bytes from the server to client over Noise are either an HTTP/2
-// settings frame (a normal HTTP/2 setup) or, as we added later, an "early payload"
-// header that's also 9 bytes long: 5 bytes (earlyPayloadMagic) followed by 4 bytes
-// of length. Then that many bytes of JSON-encoded tailcfg.EarlyNoise.
-// The early payload is optional. Some servers may not send it.
-const (
-	hdrLen            = 9 // http2 frame header size; also size of our early payload size header
-	earlyPayloadMagic = "\xff\xff\xffTS"
-)
-
-// returnErrReader is an io.Reader that always returns an error.
-type returnErrReader struct {
-	err error // the error to return
-}
-
-func (r returnErrReader) Read([]byte) (int, error) { return 0, r.err }
-
-// Read is basically the same as controlbase.Conn.Read, but it first reads the
-// "early payload" header from the server which may or may not be present,
-// depending on the server.
-func (c *noiseConn) Read(p []byte) (n int, err error) {
-	c.readHeaderOnce.Do(c.readHeader)
-	return c.reader.Read(p)
-}
-
-// readHeader reads the optional "early payload" from the server that arrives
-// after the Noise handshake but before the HTTP/2 session begins.
-//
-// readHeader is responsible for reading the header (if present), initializing
-// c.earlyPayload, closing c.earlyPayloadReady, and initializing c.reader for
-// future reads.
-func (c *noiseConn) readHeader() {
-	defer close(c.earlyPayloadReady)
-
-	setErr := func(err error) {
-		c.reader = returnErrReader{err}
-		c.earlyPayloadErr = err
-	}
-
-	var hdr [hdrLen]byte
-	if _, err := io.ReadFull(c.Conn, hdr[:]); err != nil {
-		setErr(err)
-		return
-	}
-	if string(hdr[:len(earlyPayloadMagic)]) != earlyPayloadMagic {
-		// No early payload. We have to return the 9 bytes read we already
-		// consumed.
-		c.reader = io.MultiReader(bytes.NewReader(hdr[:]), c.Conn)
-		return
-	}
-	epLen := binary.BigEndian.Uint32(hdr[len(earlyPayloadMagic):])
-	if epLen > 10<<20 {
-		setErr(errors.New("invalid early payload length"))
-		return
-	}
-	payBuf := make([]byte, epLen)
-	if _, err := io.ReadFull(c.Conn, payBuf); err != nil {
-		setErr(err)
-		return
-	}
-	if err := json.Unmarshal(payBuf, &c.earlyPayload); err != nil {
-		setErr(err)
-		return
-	}
-	c.reader = c.Conn
-}
-
-func (c *noiseConn) Close() error {
-	if err := c.Conn.Close(); err != nil {
-		return err
-	}
-	c.pool.connClosed(c.id)
-	return nil
-}
-
 // NoiseClient provides a http.Client to connect to tailcontrol over
 // the ts2021 protocol.
 type NoiseClient struct {
@@ -158,7 +49,7 @@ type NoiseClient struct {
 
 	// sfDial ensures that two concurrent requests for a noise connection only
 	// produce one shared one between the two callers.
-	sfDial singleflight.Group[struct{}, *noiseConn]
+	sfDial singleflight.Group[struct{}, *noiseconn.Conn]
 
 	dialer       *tsdial.Dialer
 	dnsCache     *dnscache.Resolver
@@ -180,9 +71,9 @@ type NoiseClient struct {
 	// mu only protects the following variables.
 	mu       sync.Mutex
 	closed   bool
-	last     *noiseConn // or nil
+	last     *noiseconn.Conn // or nil
 	nextID   int
-	connPool map[int]*noiseConn // active connections not yet closed; see noiseConn.Close
+	connPool map[int]*noiseconn.Conn // active connections not yet closed; see noiseconn.Conn.Close
 }
 
 // NoiseOpts contains options for the NewNoiseClient function. All fields are
@@ -283,11 +174,11 @@ func (nc *NoiseClient) GetSingleUseRoundTripper(ctx context.Context) (http.Round
 		if err != nil {
 			return nil, nil, err
 		}
-		earlyPayloadMaybeNil, err := conn.getEarlyPayload(ctx)
+		ok, earlyPayloadMaybeNil, err := conn.ReserveNewRequest(ctx)
 		if err != nil {
 			return nil, nil, err
 		}
-		if conn.h2cc.ReserveNewRequest() {
+		if ok {
 			return conn, earlyPayloadMaybeNil, nil
 		}
 	}
@@ -308,14 +199,14 @@ func (e contextErr) Unwrap() error {
 	return e.err
 }
 
-// getConn returns a noiseConn that can be used to make requests to the
+// getConn returns a noiseconn.Conn that can be used to make requests to the
 // coordination server. It may return a cached connection or create a new one.
 // Dials are singleflighted, so concurrent calls to getConn may only dial once.
 // As such, context values may not be respected as there are no guarantees that
 // the context passed to getConn is the same as the context passed to dial.
-func (nc *NoiseClient) getConn(ctx context.Context) (*noiseConn, error) {
+func (nc *NoiseClient) getConn(ctx context.Context) (*noiseconn.Conn, error) {
 	nc.mu.Lock()
-	if last := nc.last; last != nil && last.canTakeNewRequest() {
+	if last := nc.last; last != nil && last.CanTakeNewRequest() {
 		nc.mu.Unlock()
 		return last, nil
 	}
@@ -327,7 +218,7 @@ func (nc *NoiseClient) getConn(ctx context.Context) (*noiseConn, error) {
 		// canceled. Instead, we have to additionally check that the context
 		// which was canceled is our context and retry if our context is still
 		// valid.
-		conn, err, _ := nc.sfDial.Do(struct{}{}, func() (*noiseConn, error) {
+		conn, err, _ := nc.sfDial.Do(struct{}{}, func() (*noiseconn.Conn, error) {
 			c, err := nc.dial(ctx)
 			if err != nil {
 				if ctx.Err() != nil {
@@ -395,7 +286,7 @@ func (nc *NoiseClient) Close() error {
 
 // dial opens a new connection to tailcontrol, fetching the server noise key
 // if not cached.
-func (nc *NoiseClient) dial(ctx context.Context) (*noiseConn, error) {
+func (nc *NoiseClient) dial(ctx context.Context) (*noiseconn.Conn, error) {
 	nc.mu.Lock()
 	connID := nc.nextID
 	nc.nextID++
@@ -465,18 +356,10 @@ func (nc *NoiseClient) dial(ctx context.Context) (*noiseConn, error) {
 		return nil, err
 	}
 
-	ncc := &noiseConn{
-		Conn:              clientConn.Conn,
-		id:                connID,
-		pool:              nc,
-		earlyPayloadReady: make(chan struct{}),
-	}
-
-	h2cc, err := nc.h2t.NewClientConn(ncc)
+	ncc, err := noiseconn.New(clientConn.Conn, nc.h2t, connID, nc.connClosed)
 	if err != nil {
 		return nil, err
 	}
-	ncc.h2cc = h2cc
 
 	nc.mu.Lock()
 	if nc.closed {
@@ -485,7 +368,7 @@ func (nc *NoiseClient) dial(ctx context.Context) (*noiseConn, error) {
 		return nil, errors.New("noise client closed")
 	}
 	defer nc.mu.Unlock()
-	mak.Set(&nc.connPool, ncc.id, ncc)
+	mak.Set(&nc.connPool, connID, ncc)
 	nc.last = ncc
 	return ncc, nil
 }
@@ -508,9 +391,5 @@ func (nc *NoiseClient) post(ctx context.Context, path string, nodeKey key.NodePu
 	if err != nil {
 		return nil, err
 	}
-	return conn.h2cc.RoundTrip(req)
-}
-
-func (c *noiseConn) canTakeNewRequest() bool {
-	return c.h2cc.CanTakeNewRequest()
+	return conn.RoundTrip(req)
 }

control/controlclient/noise_test.go

@@ -16,6 +16,7 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlhttp"
+	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/netmon"
 	"tailscale.com/net/tsdial"
 	"tailscale.com/tailcfg"
@@ -93,22 +94,21 @@ func (tt noiseClientTest) run(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	select {
-	case <-c.earlyPayloadReady:
-		gotNonNil := c.earlyPayload != nil
-		if gotNonNil != tt.sendEarlyPayload {
-			t.Errorf("sendEarlyPayload = %v but got earlyPayload = %T", tt.sendEarlyPayload, c.earlyPayload)
-		}
-		if c.earlyPayload != nil {
-			if c.earlyPayload.NodeKeyChallenge != chalPrivate.Public() {
-				t.Errorf("earlyPayload.NodeKeyChallenge = %v; want %v", c.earlyPayload.NodeKeyChallenge, chalPrivate.Public())
-			}
-		}
-
-	case <-ctx.Done():
+	payload, err := c.GetEarlyPayload(ctx)
+	if err != nil {
 		t.Fatal("timed out waiting for didReadHeaderCh")
 	}
 
+	gotNonNil := payload != nil
+	if gotNonNil != tt.sendEarlyPayload {
+		t.Errorf("sendEarlyPayload = %v but got earlyPayload = %T", tt.sendEarlyPayload, payload)
+	}
+	if payload != nil {
+		if payload.NodeKeyChallenge != chalPrivate.Public() {
+			t.Errorf("earlyPayload.NodeKeyChallenge = %v; want %v", payload.NodeKeyChallenge, chalPrivate.Public())
+		}
+	}
+
 	checkRes := func(t *testing.T, res *http.Response) {
 		t.Helper()
 		defer res.Body.Close()
@@ -184,7 +184,7 @@ func (up *Upgrader) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		// https://httpwg.org/specs/rfc7540.html#rfc.section.4.1 (Especially not
 		// an HTTP/2 settings frame, which isn't of type 'T')
 		var notH2Frame [5]byte
-		copy(notH2Frame[:], earlyPayloadMagic)
+		copy(notH2Frame[:], noiseconn.EarlyPayloadMagic)
 		var lenBuf [4]byte
 		binary.BigEndian.PutUint32(lenBuf[:], uint32(len(earlyJSON)))
 		// These writes are all buffered by caller, so fine to do them

control/controlhttp/client.go

@@ -32,12 +32,14 @@ import (
 	"net/http/httptrace"
 	"net/netip"
 	"net/url"
+	"runtime"
 	"sort"
 	"sync/atomic"
 	"time"
 
 	"tailscale.com/control/controlbase"
 	"tailscale.com/envknob"
+	"tailscale.com/health"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/netutil"
@@ -396,12 +398,29 @@ func (a *Dialer) resolver() *dnscache.Resolver {
 	}
 }
 
+func isLoopback(a net.Addr) bool {
+	if ta, ok := a.(*net.TCPAddr); ok {
+		return ta.IP.IsLoopback()
+	}
+	return false
+}
+
+var macOSScreenTime = health.Register(&health.Warnable{
+	Code:     "macos-screen-time",
+	Severity: health.SeverityHigh,
+	Title:    "Tailscale blocked by Screen Time",
+	Text: func(args health.Args) string {
+		return "macOS Screen Time seems to be blocking Tailscale. Try disabling Screen Time in System Settings > Screen Time > Content & Privacy > Access to Web Content."
+	},
+	ImpactsConnectivity: true,
+})
+
 // tryURLUpgrade connects to u, and tries to upgrade it to a net.Conn. If addr
 // is valid, then no DNS is used and the connection will be made to the
 // provided address.
 //
 // Only the provided ctx is used, not a.ctx.
-func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr, init []byte) (net.Conn, error) {
+func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr, init []byte) (_ net.Conn, retErr error) {
 	var dns *dnscache.Resolver
 
 	// If we were provided an address to dial, then create a resolver that just
@@ -423,6 +442,30 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, addr netip.Addr,
 		dialer = stdDialer.DialContext
 	}
 
+	// On macOS, see if Screen Time is blocking things.
+	if runtime.GOOS == "darwin" {
+		var proxydIntercepted atomic.Bool // intercepted by macOS webfilterproxyd
+		origDialer := dialer
+		dialer = func(ctx context.Context, network, address string) (net.Conn, error) {
+			c, err := origDialer(ctx, network, address)
+			if err != nil {
+				return nil, err
+			}
+			if isLoopback(c.LocalAddr()) && isLoopback(c.RemoteAddr()) {
+				proxydIntercepted.Store(true)
+			}
+			return c, nil
+		}
+		defer func() {
+			if retErr != nil && proxydIntercepted.Load() {
+				a.HealthTracker.SetUnhealthy(macOSScreenTime, nil)
+				retErr = fmt.Errorf("macOS Screen Time is blocking network access: %w", retErr)
+			} else {
+				a.HealthTracker.SetHealthy(macOSScreenTime)
+			}
+		}()
+	}
+
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	defer tr.CloseIdleConnections()
 	tr.Proxy = a.getProxyFunc()

control/controlknobs/controlknobs.go

@@ -81,6 +81,24 @@ type Knobs struct {
 	// how to dial the destination address. When true, it also makes the DNS forwarder
 	// use UserDial instead of SystemDial when dialing resolvers.
 	UserDialUseRoutes atomic.Bool
+
+	// DisableSplitDNSWhenNoCustomResolvers indicates that the node's DNS manager
+	// should not adopt a split DNS configuration even though the Config of the
+	// resolver only contains routes that do not specify custom resolver(s), hence
+	// all DNS queries can be safely sent to the upstream DNS resolver and the
+	// node's DNS forwarder doesn't need to handle all DNS traffic.
+	// This is for now (2024-06-06) an iOS-specific battery life optimization,
+	// and this knob allows us to disable the optimization remotely if needed.
+	DisableSplitDNSWhenNoCustomResolvers atomic.Bool
+
+	// DisableLocalDNSOverrideViaNRPT indicates that the node's DNS manager should not
+	// create a default (catch-all) Windows NRPT rule when "Override local DNS" is enabled.
+	// Without this rule, Windows 8.1 and newer devices issue parallel DNS requests to DNS servers
+	// associated with all network adapters, even when "Override local DNS" is enabled and/or
+	// a Mullvad exit node is being used, resulting in DNS leaks.
+	// We began creating this rule on 2024-06-14, and this knob
+	// allows us to disable the new behavior remotely if needed.
+	DisableLocalDNSOverrideViaNRPT atomic.Bool
 }
 
 // UpdateFromNodeAttributes updates k (if non-nil) based on the provided self
@@ -91,22 +109,24 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 	}
 	has := capMap.Contains
 	var (
-		keepFullWG                    = has(tailcfg.NodeAttrDebugDisableWGTrim)
-		disableDRPO                   = has(tailcfg.NodeAttrDebugDisableDRPO)
-		disableUPnP                   = has(tailcfg.NodeAttrDisableUPnP)
-		randomizeClientPort           = has(tailcfg.NodeAttrRandomizeClientPort)
-		disableDeltaUpdates           = has(tailcfg.NodeAttrDisableDeltaUpdates)
-		oneCGNAT                      opt.Bool
-		forceBackgroundSTUN           = has(tailcfg.NodeAttrDebugForceBackgroundSTUN)
-		peerMTUEnable                 = has(tailcfg.NodeAttrPeerMTUEnable)
-		dnsForwarderDisableTCPRetries = has(tailcfg.NodeAttrDNSForwarderDisableTCPRetries)
-		silentDisco                   = has(tailcfg.NodeAttrSilentDisco)
-		forceIPTables                 = has(tailcfg.NodeAttrLinuxMustUseIPTables)
-		forceNfTables                 = has(tailcfg.NodeAttrLinuxMustUseNfTables)
-		seamlessKeyRenewal            = has(tailcfg.NodeAttrSeamlessKeyRenewal)
-		probeUDPLifetime              = has(tailcfg.NodeAttrProbeUDPLifetime)
-		appCStoreRoutes               = has(tailcfg.NodeAttrStoreAppCRoutes)
-		userDialUseRoutes             = has(tailcfg.NodeAttrUserDialUseRoutes)
+		keepFullWG                           = has(tailcfg.NodeAttrDebugDisableWGTrim)
+		disableDRPO                          = has(tailcfg.NodeAttrDebugDisableDRPO)
+		disableUPnP                          = has(tailcfg.NodeAttrDisableUPnP)
+		randomizeClientPort                  = has(tailcfg.NodeAttrRandomizeClientPort)
+		disableDeltaUpdates                  = has(tailcfg.NodeAttrDisableDeltaUpdates)
+		oneCGNAT                             opt.Bool
+		forceBackgroundSTUN                  = has(tailcfg.NodeAttrDebugForceBackgroundSTUN)
+		peerMTUEnable                        = has(tailcfg.NodeAttrPeerMTUEnable)
+		dnsForwarderDisableTCPRetries        = has(tailcfg.NodeAttrDNSForwarderDisableTCPRetries)
+		silentDisco                          = has(tailcfg.NodeAttrSilentDisco)
+		forceIPTables                        = has(tailcfg.NodeAttrLinuxMustUseIPTables)
+		forceNfTables                        = has(tailcfg.NodeAttrLinuxMustUseNfTables)
+		seamlessKeyRenewal                   = has(tailcfg.NodeAttrSeamlessKeyRenewal)
+		probeUDPLifetime                     = has(tailcfg.NodeAttrProbeUDPLifetime)
+		appCStoreRoutes                      = has(tailcfg.NodeAttrStoreAppCRoutes)
+		userDialUseRoutes                    = has(tailcfg.NodeAttrUserDialUseRoutes)
+		disableSplitDNSWhenNoCustomResolvers = has(tailcfg.NodeAttrDisableSplitDNSWhenNoCustomResolvers)
+		disableLocalDNSOverrideViaNRPT       = has(tailcfg.NodeAttrDisableLocalDNSOverrideViaNRPT)
 	)
 
 	if has(tailcfg.NodeAttrOneCGNATEnable) {
@@ -131,6 +151,8 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 	k.ProbeUDPLifetime.Store(probeUDPLifetime)
 	k.AppCStoreRoutes.Store(appCStoreRoutes)
 	k.UserDialUseRoutes.Store(userDialUseRoutes)
+	k.DisableSplitDNSWhenNoCustomResolvers.Store(disableSplitDNSWhenNoCustomResolvers)
+	k.DisableLocalDNSOverrideViaNRPT.Store(disableLocalDNSOverrideViaNRPT)
 }
 
 // AsDebugJSON returns k as something that can be marshalled with json.Marshal
@@ -140,21 +162,23 @@ func (k *Knobs) AsDebugJSON() map[string]any {
 		return nil
 	}
 	return map[string]any{
-		"DisableUPnP":                   k.DisableUPnP.Load(),
-		"DisableDRPO":                   k.DisableDRPO.Load(),
-		"KeepFullWGConfig":              k.KeepFullWGConfig.Load(),
-		"RandomizeClientPort":           k.RandomizeClientPort.Load(),
-		"OneCGNAT":                      k.OneCGNAT.Load(),
-		"ForceBackgroundSTUN":           k.ForceBackgroundSTUN.Load(),
-		"DisableDeltaUpdates":           k.DisableDeltaUpdates.Load(),
-		"PeerMTUEnable":                 k.PeerMTUEnable.Load(),
-		"DisableDNSForwarderTCPRetries": k.DisableDNSForwarderTCPRetries.Load(),
-		"SilentDisco":                   k.SilentDisco.Load(),
-		"LinuxForceIPTables":            k.LinuxForceIPTables.Load(),
-		"LinuxForceNfTables":            k.LinuxForceNfTables.Load(),
-		"SeamlessKeyRenewal":            k.SeamlessKeyRenewal.Load(),
-		"ProbeUDPLifetime":              k.ProbeUDPLifetime.Load(),
-		"AppCStoreRoutes":               k.AppCStoreRoutes.Load(),
-		"UserDialUseRoutes":             k.UserDialUseRoutes.Load(),
+		"DisableUPnP":                          k.DisableUPnP.Load(),
+		"DisableDRPO":                          k.DisableDRPO.Load(),
+		"KeepFullWGConfig":                     k.KeepFullWGConfig.Load(),
+		"RandomizeClientPort":                  k.RandomizeClientPort.Load(),
+		"OneCGNAT":                             k.OneCGNAT.Load(),
+		"ForceBackgroundSTUN":                  k.ForceBackgroundSTUN.Load(),
+		"DisableDeltaUpdates":                  k.DisableDeltaUpdates.Load(),
+		"PeerMTUEnable":                        k.PeerMTUEnable.Load(),
+		"DisableDNSForwarderTCPRetries":        k.DisableDNSForwarderTCPRetries.Load(),
+		"SilentDisco":                          k.SilentDisco.Load(),
+		"LinuxForceIPTables":                   k.LinuxForceIPTables.Load(),
+		"LinuxForceNfTables":                   k.LinuxForceNfTables.Load(),
+		"SeamlessKeyRenewal":                   k.SeamlessKeyRenewal.Load(),
+		"ProbeUDPLifetime":                     k.ProbeUDPLifetime.Load(),
+		"AppCStoreRoutes":                      k.AppCStoreRoutes.Load(),
+		"UserDialUseRoutes":                    k.UserDialUseRoutes.Load(),
+		"DisableSplitDNSWhenNoCustomResolvers": k.DisableSplitDNSWhenNoCustomResolvers.Load(),
+		"DisableLocalDNSOverrideViaNRPT":       k.DisableLocalDNSOverrideViaNRPT.Load(),
 	}
 }

derp/derp_server.go

@@ -22,7 +22,7 @@ import (
 	"log"
 	"math"
 	"math/big"
-	"math/rand"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/netip"
@@ -725,7 +725,7 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 
 	clientAP, _ := netip.ParseAddrPort(remoteAddr)
 	if err := s.verifyClient(ctx, clientKey, clientInfo, clientAP.Addr()); err != nil {
-		return fmt.Errorf("client %x rejected: %v", clientKey, err)
+		return fmt.Errorf("client %v rejected: %v", clientKey, err)
 	}
 
 	// At this point we trust the client so we don't time out.
@@ -751,7 +751,7 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
 		discoSendQueue: make(chan pkt, perClientSendQueueDepth),
 		sendPongCh:     make(chan [8]byte, 1),
 		peerGone:       make(chan peerGoneMsg),
-		canMesh:        clientInfo.MeshKey != "" && clientInfo.MeshKey == s.meshKey,
+		canMesh:        s.isMeshPeer(clientInfo),
 		peerGoneLim:    rate.NewLimiter(rate.Every(time.Second), 3),
 	}
 
@@ -939,7 +939,7 @@ func (c *sclient) handleFrameForwardPacket(ft frameType, fl uint32) error {
 
 	srcKey, dstKey, contents, err := s.recvForwardPacket(c.br, fl)
 	if err != nil {
-		return fmt.Errorf("client %x: recvForwardPacket: %v", c.key, err)
+		return fmt.Errorf("client %v: recvForwardPacket: %v", c.key, err)
 	}
 	s.packetsForwardedIn.Add(1)
 
@@ -994,7 +994,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 
 	dstKey, contents, err := s.recvPacket(c.br, fl)
 	if err != nil {
-		return fmt.Errorf("client %x: recvPacket: %v", c.key, err)
+		return fmt.Errorf("client %v: recvPacket: %v", c.key, err)
 	}
 
 	var fwd PacketForwarder
@@ -1151,20 +1151,32 @@ func (c *sclient) requestMeshUpdate() {
 	}
 }
 
+var localClient tailscale.LocalClient
+
+// isMeshPeer reports whether the client is a trusted mesh peer
+// node in the DERP region.
+func (s *Server) isMeshPeer(info *clientInfo) bool {
+	return info != nil && info.MeshKey != "" && info.MeshKey == s.meshKey
+}
+
 // verifyClient checks whether the client is allowed to connect to the derper,
 // depending on how & whether the server's been configured to verify.
 func (s *Server) verifyClient(ctx context.Context, clientKey key.NodePublic, info *clientInfo, clientIP netip.Addr) error {
+	if s.isMeshPeer(info) {
+		// Trusted mesh peer. No need to verify further. In fact, verifying
+		// further wouldn't work: it's not part of the tailnet so tailscaled and
+		// likely the admission control URL wouldn't know about it.
+		return nil
+	}
+
 	// tailscaled-based verification:
 	if s.verifyClientsLocalTailscaled {
-		status, err := tailscale.Status(ctx)
+		_, err := localClient.WhoIsNodeKey(ctx, clientKey)
+		if err == tailscale.ErrPeerNotFound {
+			return fmt.Errorf("peer %v not authorized (not found in local tailscaled)", clientKey)
+		}
 		if err != nil {
-			return fmt.Errorf("failed to query local tailscaled status: %w", err)
-		}
-		if clientKey == status.Self.PublicKey {
-			return nil
-		}
-		if _, exists := status.Peer[clientKey]; !exists {
-			return fmt.Errorf("client %v not in set of peers", clientKey)
+			return fmt.Errorf("failed to query local tailscaled status for %v: %w", clientKey, err)
 		}
 	}
 
@@ -1514,7 +1526,7 @@ func (c *sclient) sendLoop(ctx context.Context) error {
 		}
 	}()
 
-	jitter := time.Duration(rand.Intn(5000)) * time.Millisecond
+	jitter := rand.N(5 * time.Second)
 	keepAliveTick, keepAliveTickChannel := c.s.clock.NewTicker(keepAlive + jitter)
 	defer keepAliveTick.Stop()
 
@@ -1956,12 +1968,37 @@ func (s *Server) ConsistencyCheck() error {
 			s.curClients.Value(),
 			len(s.clients)))
 	}
+
+	if s.verifyClientsLocalTailscaled {
+		if err := s.checkVerifyClientsLocalTailscaled(); err != nil {
+			errs = append(errs, err.Error())
+		}
+	}
+
 	if len(errs) == 0 {
 		return nil
 	}
 	return errors.New(strings.Join(errs, ", "))
 }
 
+// checkVerifyClientsLocalTailscaled checks that a verifyClients call can be made successfully for the derper hosts own node key.
+func (s *Server) checkVerifyClientsLocalTailscaled() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	status, err := localClient.StatusWithoutPeers(ctx)
+	if err != nil {
+		return fmt.Errorf("localClient.Status: %w", err)
+	}
+	info := &clientInfo{
+		IsProber: true,
+	}
+	clientIP := netip.IPv6Loopback()
+	if err := s.verifyClient(ctx, status.Self.PublicKey, info, clientIP); err != nil {
+		return fmt.Errorf("verifyClient for self nodekey: %w", err)
+	}
+	return nil
+}
+
 const minTimeBetweenLogs = 2 * time.Second
 
 // BytesSentRecv records the number of bytes that have been sent since the last traffic check

derp/xdp/bpf_bpfeb.go

@@ -0,0 +1,170 @@
+// Code generated by bpf2go; DO NOT EDIT.
+//go:build mips || mips64 || ppc64 || s390x
+
+package xdp
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+
+	"github.com/cilium/ebpf"
+)
+
+type bpfConfig struct {
+	DstPort  uint16
+	DropStun uint16
+}
+
+type bpfCounterKeyAf uint32
+
+const (
+	bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN bpfCounterKeyAf = 0
+	bpfCounterKeyAfCOUNTER_KEY_AF_IPV4    bpfCounterKeyAf = 1
+	bpfCounterKeyAfCOUNTER_KEY_AF_IPV6    bpfCounterKeyAf = 2
+	bpfCounterKeyAfCOUNTER_KEY_AF_LEN     bpfCounterKeyAf = 3
+)
+
+type bpfCounterKeyPacketsBytesAction uint32
+
+const (
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL       bpfCounterKeyPacketsBytesAction = 0
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL         bpfCounterKeyPacketsBytesAction = 1
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL    bpfCounterKeyPacketsBytesAction = 2
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL      bpfCounterKeyPacketsBytesAction = 3
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL         bpfCounterKeyPacketsBytesAction = 4
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL           bpfCounterKeyPacketsBytesAction = 5
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL       bpfCounterKeyPacketsBytesAction = 6
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL         bpfCounterKeyPacketsBytesAction = 7
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN bpfCounterKeyPacketsBytesAction = 8
+)
+
+type bpfCounterKeyProgEnd uint32
+
+const (
+	bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED                bpfCounterKeyProgEnd = 0
+	bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR bpfCounterKeyProgEnd = 1
+	bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM           bpfCounterKeyProgEnd = 2
+	bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM            bpfCounterKeyProgEnd = 3
+	bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT              bpfCounterKeyProgEnd = 4
+	bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL        bpfCounterKeyProgEnd = 5
+	bpfCounterKeyProgEndCOUNTER_KEY_END_DROP_STUN                  bpfCounterKeyProgEnd = 6
+	bpfCounterKeyProgEndCOUNTER_KEY_END_LEN                        bpfCounterKeyProgEnd = 7
+)
+
+type bpfCountersKey struct {
+	Unused  uint8
+	Af      uint8
+	Pba     uint8
+	ProgEnd uint8
+}
+
+// loadBpf returns the embedded CollectionSpec for bpf.
+func loadBpf() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_BpfBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load bpf: %w", err)
+	}
+
+	return spec, err
+}
+
+// loadBpfObjects loads bpf and converts it into a struct.
+//
+// The following types are suitable as obj argument:
+//
+//	*bpfObjects
+//	*bpfPrograms
+//	*bpfMaps
+//
+// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
+func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
+	spec, err := loadBpf()
+	if err != nil {
+		return err
+	}
+
+	return spec.LoadAndAssign(obj, opts)
+}
+
+// bpfSpecs contains maps and programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfSpecs struct {
+	bpfProgramSpecs
+	bpfMapSpecs
+}
+
+// bpfSpecs contains programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfProgramSpecs struct {
+	XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"`
+}
+
+// bpfMapSpecs contains maps before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfMapSpecs struct {
+	ConfigMap   *ebpf.MapSpec `ebpf:"config_map"`
+	CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
+}
+
+// bpfObjects contains all objects after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfObjects struct {
+	bpfPrograms
+	bpfMaps
+}
+
+func (o *bpfObjects) Close() error {
+	return _BpfClose(
+		&o.bpfPrograms,
+		&o.bpfMaps,
+	)
+}
+
+// bpfMaps contains all maps after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfMaps struct {
+	ConfigMap   *ebpf.Map `ebpf:"config_map"`
+	CountersMap *ebpf.Map `ebpf:"counters_map"`
+}
+
+func (m *bpfMaps) Close() error {
+	return _BpfClose(
+		m.ConfigMap,
+		m.CountersMap,
+	)
+}
+
+// bpfPrograms contains all programs after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfPrograms struct {
+	XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"`
+}
+
+func (p *bpfPrograms) Close() error {
+	return _BpfClose(
+		p.XdpProgFunc,
+	)
+}
+
+func _BpfClose(closers ...io.Closer) error {
+	for _, closer := range closers {
+		if err := closer.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Do not access this directly.
+//
+//go:embed bpf_bpfeb.o
+var _BpfBytes []byte

derp/xdp/bpf_bpfel.go

@@ -0,0 +1,170 @@
+// Code generated by bpf2go; DO NOT EDIT.
+//go:build 386 || amd64 || arm || arm64 || loong64 || mips64le || mipsle || ppc64le || riscv64
+
+package xdp
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+
+	"github.com/cilium/ebpf"
+)
+
+type bpfConfig struct {
+	DstPort  uint16
+	DropStun uint16
+}
+
+type bpfCounterKeyAf uint32
+
+const (
+	bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN bpfCounterKeyAf = 0
+	bpfCounterKeyAfCOUNTER_KEY_AF_IPV4    bpfCounterKeyAf = 1
+	bpfCounterKeyAfCOUNTER_KEY_AF_IPV6    bpfCounterKeyAf = 2
+	bpfCounterKeyAfCOUNTER_KEY_AF_LEN     bpfCounterKeyAf = 3
+)
+
+type bpfCounterKeyPacketsBytesAction uint32
+
+const (
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL       bpfCounterKeyPacketsBytesAction = 0
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL         bpfCounterKeyPacketsBytesAction = 1
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL    bpfCounterKeyPacketsBytesAction = 2
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL      bpfCounterKeyPacketsBytesAction = 3
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL         bpfCounterKeyPacketsBytesAction = 4
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL           bpfCounterKeyPacketsBytesAction = 5
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL       bpfCounterKeyPacketsBytesAction = 6
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL         bpfCounterKeyPacketsBytesAction = 7
+	bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN bpfCounterKeyPacketsBytesAction = 8
+)
+
+type bpfCounterKeyProgEnd uint32
+
+const (
+	bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED                bpfCounterKeyProgEnd = 0
+	bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR bpfCounterKeyProgEnd = 1
+	bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM           bpfCounterKeyProgEnd = 2
+	bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM            bpfCounterKeyProgEnd = 3
+	bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT              bpfCounterKeyProgEnd = 4
+	bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL        bpfCounterKeyProgEnd = 5
+	bpfCounterKeyProgEndCOUNTER_KEY_END_DROP_STUN                  bpfCounterKeyProgEnd = 6
+	bpfCounterKeyProgEndCOUNTER_KEY_END_LEN                        bpfCounterKeyProgEnd = 7
+)
+
+type bpfCountersKey struct {
+	Unused  uint8
+	Af      uint8
+	Pba     uint8
+	ProgEnd uint8
+}
+
+// loadBpf returns the embedded CollectionSpec for bpf.
+func loadBpf() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_BpfBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load bpf: %w", err)
+	}
+
+	return spec, err
+}
+
+// loadBpfObjects loads bpf and converts it into a struct.
+//
+// The following types are suitable as obj argument:
+//
+//	*bpfObjects
+//	*bpfPrograms
+//	*bpfMaps
+//
+// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
+func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
+	spec, err := loadBpf()
+	if err != nil {
+		return err
+	}
+
+	return spec.LoadAndAssign(obj, opts)
+}
+
+// bpfSpecs contains maps and programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfSpecs struct {
+	bpfProgramSpecs
+	bpfMapSpecs
+}
+
+// bpfSpecs contains programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfProgramSpecs struct {
+	XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"`
+}
+
+// bpfMapSpecs contains maps before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfMapSpecs struct {
+	ConfigMap   *ebpf.MapSpec `ebpf:"config_map"`
+	CountersMap *ebpf.MapSpec `ebpf:"counters_map"`
+}
+
+// bpfObjects contains all objects after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfObjects struct {
+	bpfPrograms
+	bpfMaps
+}
+
+func (o *bpfObjects) Close() error {
+	return _BpfClose(
+		&o.bpfPrograms,
+		&o.bpfMaps,
+	)
+}
+
+// bpfMaps contains all maps after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfMaps struct {
+	ConfigMap   *ebpf.Map `ebpf:"config_map"`
+	CountersMap *ebpf.Map `ebpf:"counters_map"`
+}
+
+func (m *bpfMaps) Close() error {
+	return _BpfClose(
+		m.ConfigMap,
+		m.CountersMap,
+	)
+}
+
+// bpfPrograms contains all programs after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfPrograms struct {
+	XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"`
+}
+
+func (p *bpfPrograms) Close() error {
+	return _BpfClose(
+		p.XdpProgFunc,
+	)
+}
+
+func _BpfClose(closers ...io.Closer) error {
+	for _, closer := range closers {
+		if err := closer.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Do not access this directly.
+//
+//go:embed bpf_bpfel.o
+var _BpfBytes []byte

derp/xdp/headers/LICENSE.BSD-2-Clause

@@ -0,0 +1,32 @@
+Valid-License-Identifier: BSD-2-Clause
+SPDX-URL: https://spdx.org/licenses/BSD-2-Clause.html
+Usage-Guide:
+  To use the BSD 2-clause "Simplified" License put the following SPDX
+  tag/value pair into a comment according to the placement guidelines in
+  the licensing rules documentation:
+    SPDX-License-Identifier: BSD-2-Clause
+License-Text:
+
+Copyright (c) 2015 The Libbpf Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

derp/xdp/headers/bpf_endian.h

@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __BPF_ENDIAN__
+#define __BPF_ENDIAN__
+
+/*
+ * Isolate byte #n and put it into byte #m, for __u##b type.
+ * E.g., moving byte #6 (nnnnnnnn) into byte #1 (mmmmmmmm) for __u64:
+ * 1) xxxxxxxx nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx
+ * 2) nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx 00000000
+ * 3) 00000000 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn
+ * 4) 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn 00000000
+ */
+#define ___bpf_mvb(x, b, n, m) ((__u##b)(x) << (b-(n+1)*8) >> (b-8) << (m*8))
+
+#define ___bpf_swab16(x) ((__u16)(			\
+			  ___bpf_mvb(x, 16, 0, 1) |	\
+			  ___bpf_mvb(x, 16, 1, 0)))
+
+#define ___bpf_swab32(x) ((__u32)(			\
+			  ___bpf_mvb(x, 32, 0, 3) |	\
+			  ___bpf_mvb(x, 32, 1, 2) |	\
+			  ___bpf_mvb(x, 32, 2, 1) |	\
+			  ___bpf_mvb(x, 32, 3, 0)))
+
+#define ___bpf_swab64(x) ((__u64)(			\
+			  ___bpf_mvb(x, 64, 0, 7) |	\
+			  ___bpf_mvb(x, 64, 1, 6) |	\
+			  ___bpf_mvb(x, 64, 2, 5) |	\
+			  ___bpf_mvb(x, 64, 3, 4) |	\
+			  ___bpf_mvb(x, 64, 4, 3) |	\
+			  ___bpf_mvb(x, 64, 5, 2) |	\
+			  ___bpf_mvb(x, 64, 6, 1) |	\
+			  ___bpf_mvb(x, 64, 7, 0)))
+
+/* LLVM's BPF target selects the endianness of the CPU
+ * it compiles on, or the user specifies (bpfel/bpfeb),
+ * respectively. The used __BYTE_ORDER__ is defined by
+ * the compiler, we cannot rely on __BYTE_ORDER from
+ * libc headers, since it doesn't reflect the actual
+ * requested byte order.
+ *
+ * Note, LLVM's BPF target has different __builtin_bswapX()
+ * semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE
+ * in bpfel and bpfeb case, which means below, that we map
+ * to cpu_to_be16(). We could use it unconditionally in BPF
+ * case, but better not rely on it, so that this header here
+ * can be used from application and BPF program side, which
+ * use different targets.
+ */
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+# define __bpf_ntohs(x)			__builtin_bswap16(x)
+# define __bpf_htons(x)			__builtin_bswap16(x)
+# define __bpf_constant_ntohs(x)	___bpf_swab16(x)
+# define __bpf_constant_htons(x)	___bpf_swab16(x)
+# define __bpf_ntohl(x)			__builtin_bswap32(x)
+# define __bpf_htonl(x)			__builtin_bswap32(x)
+# define __bpf_constant_ntohl(x)	___bpf_swab32(x)
+# define __bpf_constant_htonl(x)	___bpf_swab32(x)
+# define __bpf_be64_to_cpu(x)		__builtin_bswap64(x)
+# define __bpf_cpu_to_be64(x)		__builtin_bswap64(x)
+# define __bpf_constant_be64_to_cpu(x)	___bpf_swab64(x)
+# define __bpf_constant_cpu_to_be64(x)	___bpf_swab64(x)
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# define __bpf_ntohs(x)			(x)
+# define __bpf_htons(x)			(x)
+# define __bpf_constant_ntohs(x)	(x)
+# define __bpf_constant_htons(x)	(x)
+# define __bpf_ntohl(x)			(x)
+# define __bpf_htonl(x)			(x)
+# define __bpf_constant_ntohl(x)	(x)
+# define __bpf_constant_htonl(x)	(x)
+# define __bpf_be64_to_cpu(x)		(x)
+# define __bpf_cpu_to_be64(x)		(x)
+# define __bpf_constant_be64_to_cpu(x)  (x)
+# define __bpf_constant_cpu_to_be64(x)  (x)
+#else
+# error "Fix your compiler's __BYTE_ORDER__?!"
+#endif
+
+#define bpf_htons(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_htons(x) : __bpf_htons(x))
+#define bpf_ntohs(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_ntohs(x) : __bpf_ntohs(x))
+#define bpf_htonl(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_htonl(x) : __bpf_htonl(x))
+#define bpf_ntohl(x)				\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_ntohl(x) : __bpf_ntohl(x))
+#define bpf_cpu_to_be64(x)			\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x))
+#define bpf_be64_to_cpu(x)			\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x))
+
+#endif /* __BPF_ENDIAN__ */

derp/xdp/headers/bpf_helpers.h

@@ -0,0 +1,410 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __BPF_HELPERS__
+#define __BPF_HELPERS__
+
+/*
+ * Note that bpf programs need to include either
+ * vmlinux.h (auto-generated from BTF) or linux/types.h
+ * in advance since bpf_helper_defs.h uses such types
+ * as __u64.
+ */
+#include "bpf_helper_defs.h"
+
+#define __uint(name, val) int (*name)[val]
+#define __type(name, val) typeof(val) *name
+#define __array(name, val) typeof(val) *name[]
+#define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name
+
+/*
+ * Helper macro to place programs, maps, license in
+ * different sections in elf_bpf file. Section names
+ * are interpreted by libbpf depending on the context (BPF programs, BPF maps,
+ * extern variables, etc).
+ * To allow use of SEC() with externs (e.g., for extern .maps declarations),
+ * make sure __attribute__((unused)) doesn't trigger compilation warning.
+ */
+#if __GNUC__ && !__clang__
+
+/*
+ * Pragma macros are broken on GCC
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400
+ */
+#define SEC(name) __attribute__((section(name), used))
+
+#else
+
+#define SEC(name) \
+	_Pragma("GCC diagnostic push")					    \
+	_Pragma("GCC diagnostic ignored \"-Wignored-attributes\"")	    \
+	__attribute__((section(name), used))				    \
+	_Pragma("GCC diagnostic pop")					    \
+
+#endif
+
+/* Avoid 'linux/stddef.h' definition of '__always_inline'. */
+#undef __always_inline
+#define __always_inline inline __attribute__((always_inline))
+
+#ifndef __noinline
+#define __noinline __attribute__((noinline))
+#endif
+#ifndef __weak
+#define __weak __attribute__((weak))
+#endif
+
+/*
+ * Use __hidden attribute to mark a non-static BPF subprogram effectively
+ * static for BPF verifier's verification algorithm purposes, allowing more
+ * extensive and permissive BPF verification process, taking into account
+ * subprogram's caller context.
+ */
+#define __hidden __attribute__((visibility("hidden")))
+
+/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include
+ * any system-level headers (such as stddef.h, linux/version.h, etc), and
+ * commonly-used macros like NULL and KERNEL_VERSION aren't available through
+ * vmlinux.h. This just adds unnecessary hurdles and forces users to re-define
+ * them on their own. So as a convenience, provide such definitions here.
+ */
+#ifndef NULL
+#define NULL ((void *)0)
+#endif
+
+#ifndef KERNEL_VERSION
+#define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
+#endif
+
+/*
+ * Helper macros to manipulate data structures
+ */
+
+/* offsetof() definition that uses __builtin_offset() might not preserve field
+ * offset CO-RE relocation properly, so force-redefine offsetof() using
+ * old-school approach which works with CO-RE correctly
+ */
+#undef offsetof
+#define offsetof(type, member)	((unsigned long)&((type *)0)->member)
+
+/* redefined container_of() to ensure we use the above offsetof() macro */
+#undef container_of
+#define container_of(ptr, type, member)				\
+	({							\
+		void *__mptr = (void *)(ptr);			\
+		((type *)(__mptr - offsetof(type, member)));	\
+	})
+
+/*
+ * Compiler (optimization) barrier.
+ */
+#ifndef barrier
+#define barrier() asm volatile("" ::: "memory")
+#endif
+
+/* Variable-specific compiler (optimization) barrier. It's a no-op which makes
+ * compiler believe that there is some black box modification of a given
+ * variable and thus prevents compiler from making extra assumption about its
+ * value and potential simplifications and optimizations on this variable.
+ *
+ * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of
+ * a variable, making some code patterns unverifiable. Putting barrier_var()
+ * in place will ensure that cast is performed before the barrier_var()
+ * invocation, because compiler has to pessimistically assume that embedded
+ * asm section might perform some extra operations on that variable.
+ *
+ * This is a variable-specific variant of more global barrier().
+ */
+#ifndef barrier_var
+#define barrier_var(var) asm volatile("" : "+r"(var))
+#endif
+
+/*
+ * Helper macro to throw a compilation error if __bpf_unreachable() gets
+ * built into the resulting code. This works given BPF back end does not
+ * implement __builtin_trap(). This is useful to assert that certain paths
+ * of the program code are never used and hence eliminated by the compiler.
+ *
+ * For example, consider a switch statement that covers known cases used by
+ * the program. __bpf_unreachable() can then reside in the default case. If
+ * the program gets extended such that a case is not covered in the switch
+ * statement, then it will throw a build error due to the default case not
+ * being compiled out.
+ */
+#ifndef __bpf_unreachable
+# define __bpf_unreachable()	__builtin_trap()
+#endif
+
+/*
+ * Helper function to perform a tail call with a constant/immediate map slot.
+ */
+#if __clang_major__ >= 8 && defined(__bpf__)
+static __always_inline void
+bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
+{
+	if (!__builtin_constant_p(slot))
+		__bpf_unreachable();
+
+	/*
+	 * Provide a hard guarantee that LLVM won't optimize setting r2 (map
+	 * pointer) and r3 (constant map index) from _different paths_ ending
+	 * up at the _same_ call insn as otherwise we won't be able to use the
+	 * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
+	 * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
+	 * tracking for prog array pokes") for details on verifier tracking.
+	 *
+	 * Note on clobber list: we need to stay in-line with BPF calling
+	 * convention, so even if we don't end up using r0, r4, r5, we need
+	 * to mark them as clobber so that LLVM doesn't end up using them
+	 * before / after the call.
+	 */
+	asm volatile("r1 = %[ctx]\n\t"
+		     "r2 = %[map]\n\t"
+		     "r3 = %[slot]\n\t"
+		     "call 12"
+		     :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
+		     : "r0", "r1", "r2", "r3", "r4", "r5");
+}
+#endif
+
+enum libbpf_pin_type {
+	LIBBPF_PIN_NONE,
+	/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
+	LIBBPF_PIN_BY_NAME,
+};
+
+enum libbpf_tristate {
+	TRI_NO = 0,
+	TRI_YES = 1,
+	TRI_MODULE = 2,
+};
+
+#define __kconfig __attribute__((section(".kconfig")))
+#define __ksym __attribute__((section(".ksyms")))
+#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))
+#define __kptr __attribute__((btf_type_tag("kptr")))
+#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))
+
+#define bpf_ksym_exists(sym) ({									\
+	_Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak");	\
+	!!sym;											\
+})
+
+#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx")))
+#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull")))
+#define __arg_nullable __attribute((btf_decl_tag("arg:nullable")))
+#define __arg_trusted __attribute((btf_decl_tag("arg:trusted")))
+#define __arg_arena __attribute((btf_decl_tag("arg:arena")))
+
+#ifndef ___bpf_concat
+#define ___bpf_concat(a, b) a ## b
+#endif
+#ifndef ___bpf_apply
+#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
+#endif
+#ifndef ___bpf_nth
+#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
+#endif
+#ifndef ___bpf_narg
+#define ___bpf_narg(...) \
+	___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+#endif
+
+#define ___bpf_fill0(arr, p, x) do {} while (0)
+#define ___bpf_fill1(arr, p, x) arr[p] = x
+#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args)
+#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args)
+#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args)
+#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args)
+#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args)
+#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args)
+#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args)
+#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args)
+#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args)
+#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args)
+#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args)
+#define ___bpf_fill(arr, args...) \
+	___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args)
+
+/*
+ * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values
+ * in a structure.
+ */
+#define BPF_SEQ_PRINTF(seq, fmt, args...)			\
+({								\
+	static const char ___fmt[] = fmt;			\
+	unsigned long long ___param[___bpf_narg(args)];		\
+								\
+	_Pragma("GCC diagnostic push")				\
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")	\
+	___bpf_fill(___param, args);				\
+	_Pragma("GCC diagnostic pop")				\
+								\
+	bpf_seq_printf(seq, ___fmt, sizeof(___fmt),		\
+		       ___param, sizeof(___param));		\
+})
+
+/*
+ * BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of
+ * an array of u64.
+ */
+#define BPF_SNPRINTF(out, out_size, fmt, args...)		\
+({								\
+	static const char ___fmt[] = fmt;			\
+	unsigned long long ___param[___bpf_narg(args)];		\
+								\
+	_Pragma("GCC diagnostic push")				\
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")	\
+	___bpf_fill(___param, args);				\
+	_Pragma("GCC diagnostic pop")				\
+								\
+	bpf_snprintf(out, out_size, ___fmt,			\
+		     ___param, sizeof(___param));		\
+})
+
+#ifdef BPF_NO_GLOBAL_DATA
+#define BPF_PRINTK_FMT_MOD
+#else
+#define BPF_PRINTK_FMT_MOD static const
+#endif
+
+#define __bpf_printk(fmt, ...)				\
+({							\
+	BPF_PRINTK_FMT_MOD char ____fmt[] = fmt;	\
+	bpf_trace_printk(____fmt, sizeof(____fmt),	\
+			 ##__VA_ARGS__);		\
+})
+
+/*
+ * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments
+ * instead of an array of u64.
+ */
+#define __bpf_vprintk(fmt, args...)				\
+({								\
+	static const char ___fmt[] = fmt;			\
+	unsigned long long ___param[___bpf_narg(args)];		\
+								\
+	_Pragma("GCC diagnostic push")				\
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")	\
+	___bpf_fill(___param, args);				\
+	_Pragma("GCC diagnostic pop")				\
+								\
+	bpf_trace_vprintk(___fmt, sizeof(___fmt),		\
+			  ___param, sizeof(___param));		\
+})
+
+/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args
+ * Otherwise use __bpf_vprintk
+ */
+#define ___bpf_pick_printk(...) \
+	___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk,	\
+		   __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk,		\
+		   __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\
+		   __bpf_printk /*1*/, __bpf_printk /*0*/)
+
+/* Helper macro to print out debug messages */
+#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args)
+
+struct bpf_iter_num;
+
+extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __weak __ksym;
+extern int *bpf_iter_num_next(struct bpf_iter_num *it) __weak __ksym;
+extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym;
+
+#ifndef bpf_for_each
+/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for
+ * using BPF open-coded iterators without having to write mundane explicit
+ * low-level loop logic. Instead, it provides for()-like generic construct
+ * that can be used pretty naturally. E.g., for some hypothetical cgroup
+ * iterator, you'd write:
+ *
+ * struct cgroup *cg, *parent_cg = <...>;
+ *
+ * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) {
+ *     bpf_printk("Child cgroup id = %d", cg->cgroup_id);
+ *     if (cg->cgroup_id == 123)
+ *         break;
+ * }
+ *
+ * I.e., it looks almost like high-level for each loop in other languages,
+ * supports continue/break, and is verifiable by BPF verifier.
+ *
+ * For iterating integers, the difference betwen bpf_for_each(num, i, N, M)
+ * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to
+ * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int
+ * *`, not just `int`. So for integers bpf_for() is more convenient.
+ *
+ * Note: this macro relies on C99 feature of allowing to declare variables
+ * inside for() loop, bound to for() loop lifetime. It also utilizes GCC
+ * extension: __attribute__((cleanup(<func>))), supported by both GCC and
+ * Clang.
+ */
+#define bpf_for_each(type, cur, args...) for (							\
+	/* initialize and define destructor */							\
+	struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */,	\
+						    cleanup(bpf_iter_##type##_destroy))),	\
+	/* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */		\
+			       *___p __attribute__((unused)) = (				\
+					bpf_iter_##type##_new(&___it, ##args),			\
+	/* this is a workaround for Clang bug: it currently doesn't emit BTF */			\
+	/* for bpf_iter_##type##_destroy() when used from cleanup() attribute */		\
+					(void)bpf_iter_##type##_destroy, (void *)0);		\
+	/* iteration and termination check */							\
+	(((cur) = bpf_iter_##type##_next(&___it)));						\
+)
+#endif /* bpf_for_each */
+
+#ifndef bpf_for
+/* bpf_for(i, start, end) implements a for()-like looping construct that sets
+ * provided integer variable *i* to values starting from *start* through,
+ * but not including, *end*. It also proves to BPF verifier that *i* belongs
+ * to range [start, end), so this can be used for accessing arrays without
+ * extra checks.
+ *
+ * Note: *start* and *end* are assumed to be expressions with no side effects
+ * and whose values do not change throughout bpf_for() loop execution. They do
+ * not have to be statically known or constant, though.
+ *
+ * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
+ * loop bound variables and cleanup attribute, supported by GCC and Clang.
+ */
+#define bpf_for(i, start, end) for (								\
+	/* initialize and define destructor */							\
+	struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */	\
+						 cleanup(bpf_iter_num_destroy))),		\
+	/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */		\
+			    *___p __attribute__((unused)) = (					\
+				bpf_iter_num_new(&___it, (start), (end)),			\
+	/* this is a workaround for Clang bug: it currently doesn't emit BTF */			\
+	/* for bpf_iter_num_destroy() when used from cleanup() attribute */			\
+				(void)bpf_iter_num_destroy, (void *)0);				\
+	({											\
+		/* iteration step */								\
+		int *___t = bpf_iter_num_next(&___it);						\
+		/* termination and bounds check */						\
+		(___t && ((i) = *___t, (i) >= (start) && (i) < (end)));				\
+	});											\
+)
+#endif /* bpf_for */
+
+#ifndef bpf_repeat
+/* bpf_repeat(N) performs N iterations without exposing iteration number
+ *
+ * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
+ * loop bound variables and cleanup attribute, supported by GCC and Clang.
+ */
+#define bpf_repeat(N) for (									\
+	/* initialize and define destructor */							\
+	struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */	\
+						 cleanup(bpf_iter_num_destroy))),		\
+	/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */		\
+			    *___p __attribute__((unused)) = (					\
+				bpf_iter_num_new(&___it, 0, (N)),				\
+	/* this is a workaround for Clang bug: it currently doesn't emit BTF */			\
+	/* for bpf_iter_num_destroy() when used from cleanup() attribute */			\
+				(void)bpf_iter_num_destroy, (void *)0);				\
+	bpf_iter_num_next(&___it);								\
+	/* nothing here  */									\
+)
+#endif /* bpf_repeat */
+
+#endif

derp/xdp/headers/bpf_tracing.h

@@ -0,0 +1,922 @@
+/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
+#ifndef __BPF_TRACING_H__
+#define __BPF_TRACING_H__
+
+#include "bpf_helpers.h"
+
+/* Scan the ARCH passed in from ARCH env variable (see Makefile) */
+#if defined(__TARGET_ARCH_x86)
+	#define bpf_target_x86
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_s390)
+	#define bpf_target_s390
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_arm)
+	#define bpf_target_arm
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_arm64)
+	#define bpf_target_arm64
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_mips)
+	#define bpf_target_mips
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_powerpc)
+	#define bpf_target_powerpc
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_sparc)
+	#define bpf_target_sparc
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_riscv)
+	#define bpf_target_riscv
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_arc)
+	#define bpf_target_arc
+	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_loongarch)
+	#define bpf_target_loongarch
+	#define bpf_target_defined
+#else
+
+/* Fall back to what the compiler says */
+#if defined(__x86_64__)
+	#define bpf_target_x86
+	#define bpf_target_defined
+#elif defined(__s390__)
+	#define bpf_target_s390
+	#define bpf_target_defined
+#elif defined(__arm__)
+	#define bpf_target_arm
+	#define bpf_target_defined
+#elif defined(__aarch64__)
+	#define bpf_target_arm64
+	#define bpf_target_defined
+#elif defined(__mips__)
+	#define bpf_target_mips
+	#define bpf_target_defined
+#elif defined(__powerpc__)
+	#define bpf_target_powerpc
+	#define bpf_target_defined
+#elif defined(__sparc__)
+	#define bpf_target_sparc
+	#define bpf_target_defined
+#elif defined(__riscv) && __riscv_xlen == 64
+	#define bpf_target_riscv
+	#define bpf_target_defined
+#elif defined(__arc__)
+	#define bpf_target_arc
+	#define bpf_target_defined
+#elif defined(__loongarch__)
+	#define bpf_target_loongarch
+	#define bpf_target_defined
+#endif /* no compiler target */
+
+#endif
+
+#ifndef __BPF_TARGET_MISSING
+#define __BPF_TARGET_MISSING "GCC error \"Must specify a BPF target arch via __TARGET_ARCH_xxx\""
+#endif
+
+#if defined(bpf_target_x86)
+
+/*
+ * https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI
+ */
+
+#if defined(__KERNEL__) || defined(__VMLINUX_H__)
+
+#define __PT_PARM1_REG di
+#define __PT_PARM2_REG si
+#define __PT_PARM3_REG dx
+#define __PT_PARM4_REG cx
+#define __PT_PARM5_REG r8
+#define __PT_PARM6_REG r9
+/*
+ * Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_64
+ * comments in Linux sources. And refer to syscall(2) manpage.
+ */
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG r10
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+
+#define __PT_RET_REG sp
+#define __PT_FP_REG bp
+#define __PT_RC_REG ax
+#define __PT_SP_REG sp
+#define __PT_IP_REG ip
+
+#else
+
+#ifdef __i386__
+
+/* i386 kernel is built with -mregparm=3 */
+#define __PT_PARM1_REG eax
+#define __PT_PARM2_REG edx
+#define __PT_PARM3_REG ecx
+/* i386 syscall ABI is very different, refer to syscall(2) manpage */
+#define __PT_PARM1_SYSCALL_REG ebx
+#define __PT_PARM2_SYSCALL_REG ecx
+#define __PT_PARM3_SYSCALL_REG edx
+#define __PT_PARM4_SYSCALL_REG esi
+#define __PT_PARM5_SYSCALL_REG edi
+#define __PT_PARM6_SYSCALL_REG ebp
+
+#define __PT_RET_REG esp
+#define __PT_FP_REG ebp
+#define __PT_RC_REG eax
+#define __PT_SP_REG esp
+#define __PT_IP_REG eip
+
+#else /* __i386__ */
+
+#define __PT_PARM1_REG rdi
+#define __PT_PARM2_REG rsi
+#define __PT_PARM3_REG rdx
+#define __PT_PARM4_REG rcx
+#define __PT_PARM5_REG r8
+#define __PT_PARM6_REG r9
+
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG r10
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+
+#define __PT_RET_REG rsp
+#define __PT_FP_REG rbp
+#define __PT_RC_REG rax
+#define __PT_SP_REG rsp
+#define __PT_IP_REG rip
+
+#endif /* __i386__ */
+
+#endif /* __KERNEL__ || __VMLINUX_H__ */
+
+#elif defined(bpf_target_s390)
+
+/*
+ * https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf
+ */
+
+struct pt_regs___s390 {
+	unsigned long orig_gpr2;
+};
+
+/* s390 provides user_pt_regs instead of struct pt_regs to userspace */
+#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x))
+#define __PT_PARM1_REG gprs[2]
+#define __PT_PARM2_REG gprs[3]
+#define __PT_PARM3_REG gprs[4]
+#define __PT_PARM4_REG gprs[5]
+#define __PT_PARM5_REG gprs[6]
+
+#define __PT_PARM1_SYSCALL_REG orig_gpr2
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG gprs[7]
+#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
+#define PT_REGS_PARM1_CORE_SYSCALL(x) \
+	BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG)
+
+#define __PT_RET_REG gprs[14]
+#define __PT_FP_REG gprs[11]	/* Works only with CONFIG_FRAME_POINTER */
+#define __PT_RC_REG gprs[2]
+#define __PT_SP_REG gprs[15]
+#define __PT_IP_REG psw.addr
+
+#elif defined(bpf_target_arm)
+
+/*
+ * https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers
+ */
+
+#define __PT_PARM1_REG uregs[0]
+#define __PT_PARM2_REG uregs[1]
+#define __PT_PARM3_REG uregs[2]
+#define __PT_PARM4_REG uregs[3]
+
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG uregs[4]
+#define __PT_PARM6_SYSCALL_REG uregs[5]
+#define __PT_PARM7_SYSCALL_REG uregs[6]
+
+#define __PT_RET_REG uregs[14]
+#define __PT_FP_REG uregs[11]	/* Works only with CONFIG_FRAME_POINTER */
+#define __PT_RC_REG uregs[0]
+#define __PT_SP_REG uregs[13]
+#define __PT_IP_REG uregs[12]
+
+#elif defined(bpf_target_arm64)
+
+/*
+ * https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers
+ */
+
+struct pt_regs___arm64 {
+	unsigned long orig_x0;
+};
+
+/* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
+#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
+#define __PT_PARM1_REG regs[0]
+#define __PT_PARM2_REG regs[1]
+#define __PT_PARM3_REG regs[2]
+#define __PT_PARM4_REG regs[3]
+#define __PT_PARM5_REG regs[4]
+#define __PT_PARM6_REG regs[5]
+#define __PT_PARM7_REG regs[6]
+#define __PT_PARM8_REG regs[7]
+
+#define __PT_PARM1_SYSCALL_REG orig_x0
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
+#define PT_REGS_PARM1_CORE_SYSCALL(x) \
+	BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG)
+
+#define __PT_RET_REG regs[30]
+#define __PT_FP_REG regs[29]	/* Works only with CONFIG_FRAME_POINTER */
+#define __PT_RC_REG regs[0]
+#define __PT_SP_REG sp
+#define __PT_IP_REG pc
+
+#elif defined(bpf_target_mips)
+
+/*
+ * N64 ABI is assumed right now.
+ * https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions
+ */
+
+#define __PT_PARM1_REG regs[4]
+#define __PT_PARM2_REG regs[5]
+#define __PT_PARM3_REG regs[6]
+#define __PT_PARM4_REG regs[7]
+#define __PT_PARM5_REG regs[8]
+#define __PT_PARM6_REG regs[9]
+#define __PT_PARM7_REG regs[10]
+#define __PT_PARM8_REG regs[11]
+
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */
+
+#define __PT_RET_REG regs[31]
+#define __PT_FP_REG regs[30]	/* Works only with CONFIG_FRAME_POINTER */
+#define __PT_RC_REG regs[2]
+#define __PT_SP_REG regs[29]
+#define __PT_IP_REG cp0_epc
+
+#elif defined(bpf_target_powerpc)
+
+/*
+ * http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14,
+ * section "Function Calling Sequence")
+ */
+
+#define __PT_PARM1_REG gpr[3]
+#define __PT_PARM2_REG gpr[4]
+#define __PT_PARM3_REG gpr[5]
+#define __PT_PARM4_REG gpr[6]
+#define __PT_PARM5_REG gpr[7]
+#define __PT_PARM6_REG gpr[8]
+#define __PT_PARM7_REG gpr[9]
+#define __PT_PARM8_REG gpr[10]
+
+/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */
+#define PT_REGS_SYSCALL_REGS(ctx) ctx
+#define __PT_PARM1_SYSCALL_REG orig_gpr3
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+#if !defined(__arch64__)
+#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */
+#endif
+
+#define __PT_RET_REG regs[31]
+#define __PT_FP_REG __unsupported__
+#define __PT_RC_REG gpr[3]
+#define __PT_SP_REG sp
+#define __PT_IP_REG nip
+
+#elif defined(bpf_target_sparc)
+
+/*
+ * https://en.wikipedia.org/wiki/Calling_convention#SPARC
+ */
+
+#define __PT_PARM1_REG u_regs[UREG_I0]
+#define __PT_PARM2_REG u_regs[UREG_I1]
+#define __PT_PARM3_REG u_regs[UREG_I2]
+#define __PT_PARM4_REG u_regs[UREG_I3]
+#define __PT_PARM5_REG u_regs[UREG_I4]
+#define __PT_PARM6_REG u_regs[UREG_I5]
+
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+
+#define __PT_RET_REG u_regs[UREG_I7]
+#define __PT_FP_REG __unsupported__
+#define __PT_RC_REG u_regs[UREG_I0]
+#define __PT_SP_REG u_regs[UREG_FP]
+/* Should this also be a bpf_target check for the sparc case? */
+#if defined(__arch64__)
+#define __PT_IP_REG tpc
+#else
+#define __PT_IP_REG pc
+#endif
+
+#elif defined(bpf_target_riscv)
+
+/*
+ * https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions
+ */
+
+/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */
+#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
+#define __PT_PARM1_REG a0
+#define __PT_PARM2_REG a1
+#define __PT_PARM3_REG a2
+#define __PT_PARM4_REG a3
+#define __PT_PARM5_REG a4
+#define __PT_PARM6_REG a5
+#define __PT_PARM7_REG a6
+#define __PT_PARM8_REG a7
+
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+
+#define __PT_RET_REG ra
+#define __PT_FP_REG s0
+#define __PT_RC_REG a0
+#define __PT_SP_REG sp
+#define __PT_IP_REG pc
+
+#elif defined(bpf_target_arc)
+
+/*
+ * Section "Function Calling Sequence" (page 24):
+ * https://raw.githubusercontent.com/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf
+ */
+
+/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */
+#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
+#define __PT_PARM1_REG scratch.r0
+#define __PT_PARM2_REG scratch.r1
+#define __PT_PARM3_REG scratch.r2
+#define __PT_PARM4_REG scratch.r3
+#define __PT_PARM5_REG scratch.r4
+#define __PT_PARM6_REG scratch.r5
+#define __PT_PARM7_REG scratch.r6
+#define __PT_PARM8_REG scratch.r7
+
+/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */
+#define PT_REGS_SYSCALL_REGS(ctx) ctx
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+
+#define __PT_RET_REG scratch.blink
+#define __PT_FP_REG scratch.fp
+#define __PT_RC_REG scratch.r0
+#define __PT_SP_REG scratch.sp
+#define __PT_IP_REG scratch.ret
+
+#elif defined(bpf_target_loongarch)
+
+/*
+ * https://docs.kernel.org/loongarch/introduction.html
+ * https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html
+ */
+
+/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */
+#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x))
+#define __PT_PARM1_REG regs[4]
+#define __PT_PARM2_REG regs[5]
+#define __PT_PARM3_REG regs[6]
+#define __PT_PARM4_REG regs[7]
+#define __PT_PARM5_REG regs[8]
+#define __PT_PARM6_REG regs[9]
+#define __PT_PARM7_REG regs[10]
+#define __PT_PARM8_REG regs[11]
+
+/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */
+#define PT_REGS_SYSCALL_REGS(ctx) ctx
+#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG
+#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG
+#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG
+#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG
+#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG
+#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG
+
+#define __PT_RET_REG regs[1]
+#define __PT_FP_REG regs[22]
+#define __PT_RC_REG regs[4]
+#define __PT_SP_REG regs[3]
+#define __PT_IP_REG csr_era
+
+#endif
+
+#if defined(bpf_target_defined)
+
+struct pt_regs;
+
+/* allow some architectures to override `struct pt_regs` */
+#ifndef __PT_REGS_CAST
+#define __PT_REGS_CAST(x) (x)
+#endif
+
+/*
+ * Different architectures support different number of arguments passed
+ * through registers. i386 supports just 3, some arches support up to 8.
+ */
+#ifndef __PT_PARM4_REG
+#define __PT_PARM4_REG __unsupported__
+#endif
+#ifndef __PT_PARM5_REG
+#define __PT_PARM5_REG __unsupported__
+#endif
+#ifndef __PT_PARM6_REG
+#define __PT_PARM6_REG __unsupported__
+#endif
+#ifndef __PT_PARM7_REG
+#define __PT_PARM7_REG __unsupported__
+#endif
+#ifndef __PT_PARM8_REG
+#define __PT_PARM8_REG __unsupported__
+#endif
+/*
+ * Similarly, syscall-specific conventions might differ between function call
+ * conventions within each architecutre. All supported architectures pass
+ * either 6 or 7 syscall arguments in registers.
+ *
+ * See syscall(2) manpage for succinct table with information on each arch.
+ */
+#ifndef __PT_PARM7_SYSCALL_REG
+#define __PT_PARM7_SYSCALL_REG __unsupported__
+#endif
+
+#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG)
+#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG)
+#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG)
+#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG)
+#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG)
+#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG)
+#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG)
+#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG)
+#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG)
+#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG)
+#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG)
+#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG)
+#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG)
+
+#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG)
+#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG)
+#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG)
+#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG)
+#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG)
+#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG)
+#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG)
+#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG)
+#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG)
+#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG)
+#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG)
+#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG)
+#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG)
+
+#if defined(bpf_target_powerpc)
+
+#define BPF_KPROBE_READ_RET_IP(ip, ctx)		({ (ip) = (ctx)->link; })
+#define BPF_KRETPROBE_READ_RET_IP		BPF_KPROBE_READ_RET_IP
+
+#elif defined(bpf_target_sparc)
+
+#define BPF_KPROBE_READ_RET_IP(ip, ctx)		({ (ip) = PT_REGS_RET(ctx); })
+#define BPF_KRETPROBE_READ_RET_IP		BPF_KPROBE_READ_RET_IP
+
+#else
+
+#define BPF_KPROBE_READ_RET_IP(ip, ctx)					    \
+	({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); })
+#define BPF_KRETPROBE_READ_RET_IP(ip, ctx)				    \
+	({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); })
+
+#endif
+
+#ifndef PT_REGS_PARM1_SYSCALL
+#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG)
+#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG)
+#endif
+#ifndef PT_REGS_PARM2_SYSCALL
+#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG)
+#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG)
+#endif
+#ifndef PT_REGS_PARM3_SYSCALL
+#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG)
+#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG)
+#endif
+#ifndef PT_REGS_PARM4_SYSCALL
+#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG)
+#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG)
+#endif
+#ifndef PT_REGS_PARM5_SYSCALL
+#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG)
+#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG)
+#endif
+#ifndef PT_REGS_PARM6_SYSCALL
+#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG)
+#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG)
+#endif
+#ifndef PT_REGS_PARM7_SYSCALL
+#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG)
+#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG)
+#endif
+
+#else /* defined(bpf_target_defined) */
+
+#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+
+#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+
+#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+
+#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+
+#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; })
+
+#endif /* defined(bpf_target_defined) */
+
+/*
+ * When invoked from a syscall handler kprobe, returns a pointer to a
+ * struct pt_regs containing syscall arguments and suitable for passing to
+ * PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL().
+ */
+#ifndef PT_REGS_SYSCALL_REGS
+/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */
+#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx))
+#endif
+
+#ifndef ___bpf_concat
+#define ___bpf_concat(a, b) a ## b
+#endif
+#ifndef ___bpf_apply
+#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
+#endif
+#ifndef ___bpf_nth
+#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
+#endif
+#ifndef ___bpf_narg
+#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+#endif
+
+#define ___bpf_ctx_cast0()            ctx
+#define ___bpf_ctx_cast1(x)           ___bpf_ctx_cast0(), (void *)ctx[0]
+#define ___bpf_ctx_cast2(x, args...)  ___bpf_ctx_cast1(args), (void *)ctx[1]
+#define ___bpf_ctx_cast3(x, args...)  ___bpf_ctx_cast2(args), (void *)ctx[2]
+#define ___bpf_ctx_cast4(x, args...)  ___bpf_ctx_cast3(args), (void *)ctx[3]
+#define ___bpf_ctx_cast5(x, args...)  ___bpf_ctx_cast4(args), (void *)ctx[4]
+#define ___bpf_ctx_cast6(x, args...)  ___bpf_ctx_cast5(args), (void *)ctx[5]
+#define ___bpf_ctx_cast7(x, args...)  ___bpf_ctx_cast6(args), (void *)ctx[6]
+#define ___bpf_ctx_cast8(x, args...)  ___bpf_ctx_cast7(args), (void *)ctx[7]
+#define ___bpf_ctx_cast9(x, args...)  ___bpf_ctx_cast8(args), (void *)ctx[8]
+#define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), (void *)ctx[9]
+#define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), (void *)ctx[10]
+#define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), (void *)ctx[11]
+#define ___bpf_ctx_cast(args...)      ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args)
+
+/*
+ * BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and
+ * similar kinds of BPF programs, that accept input arguments as a single
+ * pointer to untyped u64 array, where each u64 can actually be a typed
+ * pointer or integer of different size. Instead of requring user to write
+ * manual casts and work with array elements by index, BPF_PROG macro
+ * allows user to declare a list of named and typed input arguments in the
+ * same syntax as for normal C function. All the casting is hidden and
+ * performed transparently, while user code can just assume working with
+ * function arguments of specified type and name.
+ *
+ * Original raw context argument is preserved as well as 'ctx' argument.
+ * This is useful when using BPF helpers that expect original context
+ * as one of the parameters (e.g., for bpf_perf_event_output()).
+ */
+#define BPF_PROG(name, args...)						    \
+name(unsigned long long *ctx);						    \
+static __always_inline typeof(name(0))					    \
+____##name(unsigned long long *ctx, ##args);				    \
+typeof(name(0)) name(unsigned long long *ctx)				    \
+{									    \
+	_Pragma("GCC diagnostic push")					    \
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")		    \
+	return ____##name(___bpf_ctx_cast(args));			    \
+	_Pragma("GCC diagnostic pop")					    \
+}									    \
+static __always_inline typeof(name(0))					    \
+____##name(unsigned long long *ctx, ##args)
+
+#ifndef ___bpf_nth2
+#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13,	\
+		    _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N
+#endif
+#ifndef ___bpf_narg2
+#define ___bpf_narg2(...)	\
+	___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7,	\
+		    6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0)
+#endif
+
+#define ___bpf_treg_cnt(t) \
+	__builtin_choose_expr(sizeof(t) == 1, 1,	\
+	__builtin_choose_expr(sizeof(t) == 2, 1,	\
+	__builtin_choose_expr(sizeof(t) == 4, 1,	\
+	__builtin_choose_expr(sizeof(t) == 8, 1,	\
+	__builtin_choose_expr(sizeof(t) == 16, 2,	\
+			      (void)0)))))
+
+#define ___bpf_reg_cnt0()		(0)
+#define ___bpf_reg_cnt1(t, x)		(___bpf_reg_cnt0() + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt2(t, x, args...)	(___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt3(t, x, args...)	(___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt4(t, x, args...)	(___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt5(t, x, args...)	(___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt6(t, x, args...)	(___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt7(t, x, args...)	(___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt8(t, x, args...)	(___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt9(t, x, args...)	(___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt10(t, x, args...)	(___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt11(t, x, args...)	(___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt12(t, x, args...)	(___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt(args...)	 ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args)
+
+#define ___bpf_union_arg(t, x, n) \
+	__builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \
+			      (void)0)))))
+
+#define ___bpf_ctx_arg0(n, args...)
+#define ___bpf_ctx_arg1(n, t, x)		, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x))
+#define ___bpf_ctx_arg2(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args)
+#define ___bpf_ctx_arg3(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args)
+#define ___bpf_ctx_arg4(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args)
+#define ___bpf_ctx_arg5(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args)
+#define ___bpf_ctx_arg6(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args)
+#define ___bpf_ctx_arg7(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args)
+#define ___bpf_ctx_arg8(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args)
+#define ___bpf_ctx_arg9(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args)
+#define ___bpf_ctx_arg10(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args)
+#define ___bpf_ctx_arg11(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args)
+#define ___bpf_ctx_arg12(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args)
+#define ___bpf_ctx_arg(args...)	___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args)
+
+#define ___bpf_ctx_decl0()
+#define ___bpf_ctx_decl1(t, x)			, t x
+#define ___bpf_ctx_decl2(t, x, args...)		, t x ___bpf_ctx_decl1(args)
+#define ___bpf_ctx_decl3(t, x, args...)		, t x ___bpf_ctx_decl2(args)
+#define ___bpf_ctx_decl4(t, x, args...)		, t x ___bpf_ctx_decl3(args)
+#define ___bpf_ctx_decl5(t, x, args...)		, t x ___bpf_ctx_decl4(args)
+#define ___bpf_ctx_decl6(t, x, args...)		, t x ___bpf_ctx_decl5(args)
+#define ___bpf_ctx_decl7(t, x, args...)		, t x ___bpf_ctx_decl6(args)
+#define ___bpf_ctx_decl8(t, x, args...)		, t x ___bpf_ctx_decl7(args)
+#define ___bpf_ctx_decl9(t, x, args...)		, t x ___bpf_ctx_decl8(args)
+#define ___bpf_ctx_decl10(t, x, args...)	, t x ___bpf_ctx_decl9(args)
+#define ___bpf_ctx_decl11(t, x, args...)	, t x ___bpf_ctx_decl10(args)
+#define ___bpf_ctx_decl12(t, x, args...)	, t x ___bpf_ctx_decl11(args)
+#define ___bpf_ctx_decl(args...)	___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args)
+
+/*
+ * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct
+ * arguments. Since each struct argument might take one or two u64 values
+ * in the trampoline stack, argument type size is needed to place proper number
+ * of u64 values for each argument. Therefore, BPF_PROG2 has different
+ * syntax from BPF_PROG. For example, for the following BPF_PROG syntax:
+ *
+ *   int BPF_PROG(test2, int a, int b) { ... }
+ *
+ * the corresponding BPF_PROG2 syntax is:
+ *
+ *   int BPF_PROG2(test2, int, a, int, b) { ... }
+ *
+ * where type and the corresponding argument name are separated by comma.
+ *
+ * Use BPF_PROG2 macro if one of the arguments might be a struct/union larger
+ * than 8 bytes:
+ *
+ *   int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b,
+ *		   int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret)
+ *   {
+ *        // access a, b, c, d, e, and ret directly
+ *        ...
+ *   }
+ */
+#define BPF_PROG2(name, args...)						\
+name(unsigned long long *ctx);							\
+static __always_inline typeof(name(0))						\
+____##name(unsigned long long *ctx ___bpf_ctx_decl(args));			\
+typeof(name(0)) name(unsigned long long *ctx)					\
+{										\
+	return ____##name(ctx ___bpf_ctx_arg(args));				\
+}										\
+static __always_inline typeof(name(0))						\
+____##name(unsigned long long *ctx ___bpf_ctx_decl(args))
+
+struct pt_regs;
+
+#define ___bpf_kprobe_args0()           ctx
+#define ___bpf_kprobe_args1(x)          ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx)
+#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx)
+#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx)
+#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx)
+#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx)
+#define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (void *)PT_REGS_PARM6(ctx)
+#define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (void *)PT_REGS_PARM7(ctx)
+#define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (void *)PT_REGS_PARM8(ctx)
+#define ___bpf_kprobe_args(args...)     ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args)
+
+/*
+ * BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for
+ * tp_btf/fentry/fexit BPF programs. It hides the underlying platform-specific
+ * low-level way of getting kprobe input arguments from struct pt_regs, and
+ * provides a familiar typed and named function arguments syntax and
+ * semantics of accessing kprobe input paremeters.
+ *
+ * Original struct pt_regs* context is preserved as 'ctx' argument. This might
+ * be necessary when using BPF helpers like bpf_perf_event_output().
+ */
+#define BPF_KPROBE(name, args...)					    \
+name(struct pt_regs *ctx);						    \
+static __always_inline typeof(name(0))					    \
+____##name(struct pt_regs *ctx, ##args);				    \
+typeof(name(0)) name(struct pt_regs *ctx)				    \
+{									    \
+	_Pragma("GCC diagnostic push")					    \
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")		    \
+	return ____##name(___bpf_kprobe_args(args));			    \
+	_Pragma("GCC diagnostic pop")					    \
+}									    \
+static __always_inline typeof(name(0))					    \
+____##name(struct pt_regs *ctx, ##args)
+
+#define ___bpf_kretprobe_args0()       ctx
+#define ___bpf_kretprobe_args1(x)      ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx)
+#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args)
+
+/*
+ * BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional
+ * return value (in addition to `struct pt_regs *ctx`), but no input
+ * arguments, because they will be clobbered by the time probed function
+ * returns.
+ */
+#define BPF_KRETPROBE(name, args...)					    \
+name(struct pt_regs *ctx);						    \
+static __always_inline typeof(name(0))					    \
+____##name(struct pt_regs *ctx, ##args);				    \
+typeof(name(0)) name(struct pt_regs *ctx)				    \
+{									    \
+	_Pragma("GCC diagnostic push")					    \
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")		    \
+	return ____##name(___bpf_kretprobe_args(args));			    \
+	_Pragma("GCC diagnostic pop")					    \
+}									    \
+static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args)
+
+/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */
+#define ___bpf_syscall_args0()           ctx
+#define ___bpf_syscall_args1(x)          ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs)
+#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs)
+#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs)
+#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs)
+#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs)
+#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (void *)PT_REGS_PARM6_SYSCALL(regs)
+#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (void *)PT_REGS_PARM7_SYSCALL(regs)
+#define ___bpf_syscall_args(args...)     ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args)
+
+/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */
+#define ___bpf_syswrap_args0()           ctx
+#define ___bpf_syswrap_args1(x)          ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (void *)PT_REGS_PARM6_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (void *)PT_REGS_PARM7_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args(args...)     ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args)
+
+/*
+ * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for
+ * tracing syscall functions, like __x64_sys_close. It hides the underlying
+ * platform-specific low-level way of getting syscall input arguments from
+ * struct pt_regs, and provides a familiar typed and named function arguments
+ * syntax and semantics of accessing syscall input parameters.
+ *
+ * Original struct pt_regs * context is preserved as 'ctx' argument. This might
+ * be necessary when using BPF helpers like bpf_perf_event_output().
+ *
+ * At the moment BPF_KSYSCALL does not transparently handle all the calling
+ * convention quirks for the following syscalls:
+ *
+ * - mmap(): __ARCH_WANT_SYS_OLD_MMAP.
+ * - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and
+ *            CONFIG_CLONE_BACKWARDS3.
+ * - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL.
+ * - compat syscalls.
+ *
+ * This may or may not change in the future. User needs to take extra measures
+ * to handle such quirks explicitly, if necessary.
+ *
+ * This macro relies on BPF CO-RE support and virtual __kconfig externs.
+ */
+#define BPF_KSYSCALL(name, args...)					    \
+name(struct pt_regs *ctx);						    \
+extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig;			    \
+static __always_inline typeof(name(0))					    \
+____##name(struct pt_regs *ctx, ##args);				    \
+typeof(name(0)) name(struct pt_regs *ctx)				    \
+{									    \
+	struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER		    \
+			       ? (struct pt_regs *)PT_REGS_PARM1(ctx)	    \
+			       : ctx;					    \
+	_Pragma("GCC diagnostic push")					    \
+	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")		    \
+	if (LINUX_HAS_SYSCALL_WRAPPER)					    \
+		return ____##name(___bpf_syswrap_args(args));		    \
+	else								    \
+		return ____##name(___bpf_syscall_args(args));		    \
+	_Pragma("GCC diagnostic pop")					    \
+}									    \
+static __always_inline typeof(name(0))					    \
+____##name(struct pt_regs *ctx, ##args)
+
+#define BPF_KPROBE_SYSCALL BPF_KSYSCALL
+
+/* BPF_UPROBE and BPF_URETPROBE are identical to BPF_KPROBE and BPF_KRETPROBE,
+ * but are named way less confusingly for SEC("uprobe") and SEC("uretprobe")
+ * use cases.
+ */
+#define BPF_UPROBE(name, args...)  BPF_KPROBE(name, ##args)
+#define BPF_URETPROBE(name, args...)  BPF_KRETPROBE(name, ##args)
+
+#endif

derp/xdp/headers/update.go

@@ -0,0 +1,99 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	libbpfVersion = "1.4.3"
+	prefix        = "libbpf-" + libbpfVersion
+)
+
+var (
+	filesToExtract = map[string]struct{}{
+		prefix + "/LICENSE.BSD-2-Clause":  {},
+		prefix + "/src/bpf_endian.h":      {},
+		prefix + "/src/bpf_helper_defs.h": {},
+		prefix + "/src/bpf_helpers.h":     {},
+		prefix + "/src/bpf_tracing.h":     {},
+	}
+)
+
+var (
+	flagDest = flag.String("dest", ".", "destination directory")
+)
+
+// TODO(jwhited): go generate strategy for derp/xdp
+func main() {
+	flag.Parse()
+
+	f, err := os.CreateTemp("", "libbpf")
+	if err != nil {
+		log.Panic(err)
+	}
+	defer os.Remove(f.Name())
+
+	resp, err := http.Get(fmt.Sprintf("https://github.com/libbpf/libbpf/archive/refs/tags/v%s.tar.gz", libbpfVersion))
+	if err != nil {
+		log.Panic(err)
+	}
+	defer resp.Body.Close()
+
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	_, err = f.Seek(0, 0)
+	if err != nil {
+		log.Panic(err)
+	}
+	g, err := gzip.NewReader(f)
+	if err != nil {
+		log.Panic(err)
+	}
+	defer g.Close()
+	t := tar.NewReader(g)
+
+	seen := make(map[string]bool, len(filesToExtract))
+	for {
+		h, err := t.Next()
+		if err != nil {
+			log.Panic(err)
+		}
+		if strings.Contains(h.Name, "..") {
+			continue
+		}
+		_, ok := filesToExtract[h.Name]
+		if ok {
+			if seen[h.Name] {
+				log.Panicf("saw %s more than once in archive", h.Name)
+			}
+			seen[h.Name] = true
+			p := filepath.Join(*flagDest, filepath.Base(h.Name))
+			e, err := os.Create(p)
+			if err != nil {
+				log.Panic(err)
+			}
+			_, err = io.Copy(e, t)
+			if err != nil {
+				log.Panic(err)
+			}
+			if len(seen) == len(filesToExtract) {
+				break
+			}
+		}
+	}
+}

derp/xdp/xdp.c

@@ -0,0 +1,595 @@
+//go:build ignore
+
+#include <linux/bpf.h>
+#include <linux/if_ether.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/udp.h>
+#include <bpf_endian.h>
+#include <bpf_helpers.h>
+
+struct config {
+	// TODO(jwhited): if we add more fields consider endianness consistency in
+	// the context of the data. cilium/ebpf uses native endian encoding for map
+	// encoding even if we use big endian types here, e.g. __be16.
+	__u16 dst_port;
+	// If drop_stun is set to a nonzero value all UDP packets destined to
+	// dst_port will be dropped. This is useful for shedding home client load
+	// during maintenance.
+	__u16 drop_stun;
+};
+struct config *unused_config __attribute__((unused)); // required by bpf2go -type
+
+struct {
+      __uint(type, BPF_MAP_TYPE_ARRAY);
+      __uint(key_size, sizeof(__u32));
+      __uint(value_size, sizeof(struct config));
+      __uint(max_entries, 1);
+} config_map SEC(".maps");
+
+struct counters_key {
+	__u8 unused;
+	__u8 af;
+	__u8 pba;
+	__u8 prog_end;
+};
+struct counters_key *unused_counters_key __attribute__((unused)); // required by bpf2go -type
+
+enum counter_key_af {
+	COUNTER_KEY_AF_UNKNOWN,
+	COUNTER_KEY_AF_IPV4,
+	COUNTER_KEY_AF_IPV6,
+	COUNTER_KEY_AF_LEN
+};
+enum counter_key_af *unused_counter_key_af __attribute__((unused)); // required by bpf2go -type
+
+enum counter_key_packets_bytes_action {
+	COUNTER_KEY_PACKETS_PASS_TOTAL,
+	COUNTER_KEY_BYTES_PASS_TOTAL,
+	COUNTER_KEY_PACKETS_ABORTED_TOTAL,
+	COUNTER_KEY_BYTES_ABORTED_TOTAL,
+	COUNTER_KEY_PACKETS_TX_TOTAL,
+	COUNTER_KEY_BYTES_TX_TOTAL,
+	COUNTER_KEY_PACKETS_DROP_TOTAL,
+	COUNTER_KEY_BYTES_DROP_TOTAL,
+	COUNTER_KEY_PACKETS_BYTES_ACTION_LEN
+};
+enum counter_key_packets_bytes_action *unused_counter_key_packets_bytes_action __attribute__((unused)); // required by bpf2go -type
+
+enum counter_key_prog_end {
+	COUNTER_KEY_END_UNSPECIFIED,
+	COUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR,
+	COUNTER_KEY_END_INVALID_UDP_CSUM,
+	COUNTER_KEY_END_INVALID_IP_CSUM,
+	COUNTER_KEY_END_NOT_STUN_PORT,
+	COUNTER_KEY_END_INVALID_SW_ATTR_VAL,
+	COUNTER_KEY_END_DROP_STUN,
+	COUNTER_KEY_END_LEN
+};
+enum counter_key_prog_end *unused_counter_key_prog_end __attribute__((unused)); // required by bpf2go -type
+
+#define COUNTERS_MAP_MAX_ENTRIES ((COUNTER_KEY_AF_LEN - 1) << 16) | \
+                                 ((COUNTER_KEY_PACKETS_BYTES_ACTION_LEN - 1) << 8) | \
+                                 (COUNTER_KEY_END_LEN - 1)
+
+struct {
+      __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
+      __uint(key_size, sizeof(struct counters_key));
+      __uint(value_size, sizeof(__u64));
+      __uint(max_entries, COUNTERS_MAP_MAX_ENTRIES);
+} counters_map SEC(".maps");
+
+struct stunreq {
+	__be16 type;
+	__be16 length;
+	__be32 magic;
+	__be32 txid[3];
+	// attributes follow
+};
+
+struct stunattr {
+	__be16 num;
+	__be16 length;
+};
+
+struct stunxor {
+	__u8 unused;
+	__u8 family;
+	__be16 port;
+	__be32 addr;
+};
+
+struct stunxor6 {
+	__u8 unused;
+	__u8 family;
+	__be16 port;
+	__be32 addr[4];
+};
+
+#define STUN_BINDING_REQUEST 1
+
+#define STUN_MAGIC 0x2112a442
+
+#define STUN_ATTR_SW 0x8022
+
+#define STUN_ATTR_XOR_MAPPED_ADDR 0x0020
+
+#define STUN_BINDING_RESPONSE 0x0101
+
+#define STUN_MAGIC_FOR_PORT_XOR 0x2112
+
+#define MAX_UDP_LEN_IPV4 1480
+
+#define MAX_UDP_LEN_IPV6 1460
+
+#define IP_MF 0x2000
+#define IP_OFFSET 0x1fff
+
+static __always_inline __u16 csum_fold_flip(__u32 csum) {
+	__u32 sum;
+	sum = (csum >> 16) + (csum & 0xffff); // maximum value 0x1fffe
+	sum += (sum >> 16); // maximum value 0xffff
+	return ~sum;
+}
+
+// csum_const_size is an alternative to bpf_csum_diff. It's a verifier
+// workaround for when we are forced to use a constant max_size + bounds
+// checking. The alternative being passing a dynamic length to bpf_csum_diff
+// {from,to}_size arguments, which the verifier can't follow. For further info
+// see: https://github.com/iovisor/bcc/issues/2463#issuecomment-512503958
+static __always_inline __u16 csum_const_size(__u32 seed, void* from, void* data_end, int max_size) {
+	__u16 *buf = from;
+	for (int i = 0; i < max_size; i += 2) {
+		if ((void *)(buf + 1) > data_end) {
+			break;
+		}
+		seed += *buf;
+		buf++;
+	}
+	if ((void *)buf + 1 <= data_end) {
+		seed += *(__u8 *)buf;
+	}
+	return csum_fold_flip(seed);
+}
+
+static __always_inline __u32 pseudo_sum_ipv6(struct ipv6hdr* ip6, __u16 udp_len) {
+	__u32 pseudo = 0; // TODO(jwhited): __u64 for intermediate checksum values to reduce number of ops
+	for (int i = 0; i < 8; i ++) {
+		pseudo += ip6->saddr.in6_u.u6_addr16[i];
+		pseudo += ip6->daddr.in6_u.u6_addr16[i];
+	}
+	pseudo += bpf_htons(ip6->nexthdr);
+	pseudo += udp_len;
+	return pseudo;
+}
+
+static __always_inline __u32 pseudo_sum_ipv4(struct iphdr* ip, __u16 udp_len) {
+	__u32 pseudo = (__u16)ip->saddr;
+	pseudo += (__u16)(ip->saddr >> 16);
+	pseudo += (__u16)ip->daddr;
+	pseudo += (__u16)(ip->daddr >> 16);
+	pseudo += bpf_htons(ip->protocol);
+	pseudo += udp_len;
+	return pseudo;
+}
+
+struct packet_context {
+	enum counter_key_af af;
+	enum counter_key_prog_end prog_end;
+};
+
+static __always_inline int inc_counter(struct counters_key key, __u64 val) {
+	__u64 *counter = bpf_map_lookup_elem(&counters_map, &key);
+	if (!counter) {
+		return bpf_map_update_elem(&counters_map, &key, &val, BPF_ANY);
+	}
+	*counter += val;
+	return bpf_map_update_elem(&counters_map, &key, counter, BPF_ANY);
+}
+
+static __always_inline int handle_counters(struct xdp_md *ctx, int action, struct packet_context *pctx) {
+	void *data_end = (void *)(long)ctx->data_end;
+	void *data     = (void *)(long)ctx->data;
+
+	__u64 bytes = data_end - data;
+	enum counter_key_packets_bytes_action packets_pba = COUNTER_KEY_PACKETS_PASS_TOTAL;
+	enum counter_key_packets_bytes_action bytes_pba = COUNTER_KEY_BYTES_PASS_TOTAL;
+
+	switch (action) {
+		case XDP_ABORTED:
+			packets_pba = COUNTER_KEY_PACKETS_ABORTED_TOTAL;
+			bytes_pba = COUNTER_KEY_BYTES_ABORTED_TOTAL;
+			break;
+		case XDP_PASS:
+			packets_pba = COUNTER_KEY_PACKETS_PASS_TOTAL;
+			bytes_pba = COUNTER_KEY_BYTES_PASS_TOTAL;
+			break;
+		case XDP_TX:
+			packets_pba = COUNTER_KEY_PACKETS_TX_TOTAL;
+			bytes_pba = COUNTER_KEY_BYTES_TX_TOTAL;
+			break;
+		case XDP_DROP:
+			packets_pba = COUNTER_KEY_PACKETS_DROP_TOTAL;
+			bytes_pba = COUNTER_KEY_BYTES_DROP_TOTAL;
+			break;
+	}
+
+	struct counters_key packets_key = {
+		.af = pctx->af,
+		.pba = packets_pba,
+		.prog_end = pctx->prog_end,
+	};
+
+	struct counters_key bytes_key = {
+		.af = pctx->af,
+		.pba = bytes_pba,
+		.prog_end = pctx->prog_end,
+	};
+
+	inc_counter(packets_key, 1);
+	inc_counter(bytes_key, bytes);
+
+	return 0;
+}
+
+#define is_ipv6 (pctx->af == COUNTER_KEY_AF_IPV6)
+static __always_inline int handle_packet(struct xdp_md *ctx, struct packet_context *pctx) {
+	void *data_end = (void *)(long)ctx->data_end;
+	void *data     = (void *)(long)ctx->data;
+	pctx->af = COUNTER_KEY_AF_UNKNOWN;
+	pctx->prog_end = COUNTER_KEY_END_UNSPECIFIED;
+
+	struct ethhdr *eth = data;
+	if ((void *)(eth + 1) > data_end) {
+		return XDP_PASS;
+	}
+
+	struct iphdr *ip;
+	struct ipv6hdr *ip6;
+	struct udphdr *udp;
+
+	int validate_udp_csum = 0;
+	if (eth->h_proto == bpf_htons(ETH_P_IP)) {
+		pctx->af = COUNTER_KEY_AF_IPV4;
+		ip = (void *)(eth + 1);
+		if ((void *)(ip + 1) > data_end) {
+			return XDP_PASS;
+		}
+
+		if (ip->ihl != 5 ||
+			ip->version != 4 ||
+			ip->protocol != IPPROTO_UDP ||
+			(ip->frag_off & bpf_htons(IP_MF | IP_OFFSET)) != 0) {
+			return XDP_PASS;
+		}
+
+		// validate ipv4 header checksum
+		__u32 cs_unfolded = bpf_csum_diff(0, 0, (void *)ip, sizeof(*ip), 0);
+		__u16 cs = csum_fold_flip(cs_unfolded);
+		if (cs != 0) {
+			pctx->prog_end = COUNTER_KEY_END_INVALID_IP_CSUM;
+			return XDP_PASS;
+		}
+
+		if (bpf_ntohs(ip->tot_len) != data_end - (void *)ip) {
+			return XDP_PASS;
+		}
+
+		udp = (void *)(ip + 1);
+		if ((void *)(udp + 1) > data_end) {
+			return XDP_PASS;
+		}
+
+		if (udp->check != 0) {
+			// https://datatracker.ietf.org/doc/html/rfc768#page-3
+			// If the computed  checksum  is zero,  it is transmitted  as all
+			// ones (the equivalent  in one's complement  arithmetic).   An all
+			// zero  transmitted checksum  value means that the transmitter
+			// generated  no checksum  (for debugging or for higher level
+			// protocols that don't care).
+			validate_udp_csum = 1;
+		}
+	} else if (eth->h_proto == bpf_htons(ETH_P_IPV6)) {
+		pctx->af = COUNTER_KEY_AF_IPV6;
+		ip6 = (void *)(eth + 1);
+		if ((void *)(ip6 + 1) > data_end) {
+			return XDP_PASS;
+		}
+
+		if (ip6->version != 6 || ip6->nexthdr != IPPROTO_UDP) {
+			return XDP_PASS;
+		}
+
+		udp = (void *)(ip6 + 1);
+		if ((void *)(udp + 1) > data_end) {
+			return XDP_PASS;
+		}
+
+		if (bpf_ntohs(ip6->payload_len) != data_end - (void *)udp) {
+			return XDP_PASS;
+		}
+
+		// https://datatracker.ietf.org/doc/html/rfc8200#page-28
+		// Unlike IPv4, the default behavior when UDP packets are
+		// originated by an IPv6 node is that the UDP checksum is not
+		// optional.  That is, whenever originating a UDP packet, an IPv6
+		// node must compute a UDP checksum over the packet and the
+		// pseudo-header, and, if that computation yields a result of
+		// zero, it must be changed to hex FFFF for placement in the UDP
+		// header.  IPv6 receivers must discard UDP packets containing a
+		// zero checksum and should log the error.
+		validate_udp_csum = 1;
+	} else {
+		return XDP_PASS;
+	}
+
+	__u32 config_key = 0;
+	struct config *c = bpf_map_lookup_elem(&config_map, &config_key);
+	if (!c) {
+		return XDP_PASS;
+	}
+
+	if (bpf_ntohs(udp->len) != data_end - (void *)udp) {
+		return XDP_PASS;
+	}
+
+	if (bpf_ntohs(udp->dest) != c->dst_port) {
+		pctx->prog_end = COUNTER_KEY_END_NOT_STUN_PORT;
+		return XDP_PASS;
+	}
+
+	if (c->drop_stun) {
+		pctx->prog_end = COUNTER_KEY_END_DROP_STUN;
+		return XDP_DROP;
+	}
+
+	if (validate_udp_csum) {
+		__u16 cs;
+		__u32 pseudo_sum;
+		if (is_ipv6) {
+			pseudo_sum = pseudo_sum_ipv6(ip6, udp->len);
+			cs = csum_const_size(pseudo_sum, udp, data_end, MAX_UDP_LEN_IPV6);
+		} else {
+			pseudo_sum = pseudo_sum_ipv4(ip, udp->len);
+			cs = csum_const_size(pseudo_sum, udp, data_end, MAX_UDP_LEN_IPV4);
+		}
+		if (cs != 0) {
+			pctx->prog_end = COUNTER_KEY_END_INVALID_UDP_CSUM;
+			return XDP_PASS;
+		}
+	}
+
+	struct stunreq *req = (void *)(udp + 1);
+	if ((void *)(req + 1) > data_end) {
+		return XDP_PASS;
+	}
+
+	if (req->type != bpf_htons(STUN_BINDING_REQUEST)) {
+		return XDP_PASS;
+	}
+	if (bpf_ntohl(req->magic) != STUN_MAGIC) {
+		return XDP_PASS;
+	}
+
+	void *attrs = (void *)(req + 1);
+	__u16 attrs_len = ((char *)data_end) - ((char *)attrs);
+	if (bpf_ntohs(req->length) != attrs_len) {
+		return XDP_PASS;
+	}
+
+	struct stunattr *sa = attrs;
+	if ((void *)(sa + 1) > data_end) {
+		return XDP_PASS;
+	}
+
+	// Assume the order and contents of attributes. We *could* loop through
+	// them, but parsing their lengths and performing arithmetic against the
+	// packet pointer is more pain than it's worth. Bounds checks are invisible
+	// to the verifier in certain circumstances where things move from registers
+	// to the stack and/or compilation optimizations remove them entirely. There
+	// have only ever been two attributes included by the client, and we are
+	// only interested in one of them, anyway. Verify the software attribute,
+	// but ignore the fingerprint attribute as it's only useful where STUN is
+	// multiplexed with other traffic on the same port/socket, which is not the
+	// case here.
+	void *attr_data = (void *)(sa + 1);
+	if (bpf_ntohs(sa->length) != 8 || bpf_ntohs(sa->num) != STUN_ATTR_SW) {
+		pctx->prog_end = COUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR;
+		return XDP_PASS;
+	}
+	if (attr_data + 8 > data_end) {
+		return XDP_PASS;
+	}
+	char want_sw[] = {0x74, 0x61, 0x69, 0x6c, 0x6e, 0x6f, 0x64, 0x65}; // tailnode
+	char *got_sw = attr_data;
+	for (int j = 0; j < 8; j++) {
+		if (got_sw[j] != want_sw[j]) {
+			pctx->prog_end = COUNTER_KEY_END_INVALID_SW_ATTR_VAL;
+			return XDP_PASS;
+		}
+	}
+
+	// Begin transforming packet into a STUN_BINDING_RESPONSE. From here
+	// onwards we return XDP_ABORTED instead of XDP_PASS when transformations or
+	// bounds checks fail as it would be nonsensical to pass a mangled packet
+	// through to the kernel, and we may be interested in debugging via
+	// tracepoint.
+
+	// Set success response and new length. Magic cookie and txid remain the
+	// same.
+	req->type = bpf_htons(STUN_BINDING_RESPONSE);
+	if (is_ipv6) {
+		req->length = bpf_htons(sizeof(struct stunattr) + sizeof(struct stunxor6));
+	} else {
+		req->length = bpf_htons(sizeof(struct stunattr) + sizeof(struct stunxor));
+	}
+
+	// Set attr type. Length remains unchanged, but set it again for future
+	// safety reasons.
+	sa->num = bpf_htons(STUN_ATTR_XOR_MAPPED_ADDR);
+	if (is_ipv6) {
+		sa->length = bpf_htons(sizeof(struct stunxor6));
+	} else {
+		sa->length = bpf_htons(sizeof(struct stunxor));
+	}
+
+	struct stunxor *xor;
+	struct stunxor6 *xor6;
+
+	// Adjust tail and reset header pointers.
+	int adjust_tail_by;
+	if (is_ipv6) {
+		xor6 = attr_data;
+		adjust_tail_by = (void *)(xor6 + 1) - data_end;
+	} else {
+		xor = attr_data;
+		adjust_tail_by = (void *)(xor + 1) - data_end;
+	}
+	if (bpf_xdp_adjust_tail(ctx, adjust_tail_by)) {
+		return XDP_ABORTED;
+	}
+	data_end = (void *)(long)ctx->data_end;
+	data = (void *)(long)ctx->data;
+	eth = data;
+	if ((void *)(eth + 1) > data_end) {
+		return XDP_ABORTED;
+	}
+	if (is_ipv6) {
+		ip6 = (void *)(eth + 1);
+		if ((void *)(ip6 + 1) > data_end) {
+			return XDP_ABORTED;
+		}
+		udp = (void *)(ip6 + 1);
+		if ((void *)(udp + 1) > data_end) {
+			return XDP_ABORTED;
+		}
+	} else {
+		ip = (void *)(eth + 1);
+		if ((void *)(ip + 1) > data_end) {
+			return XDP_ABORTED;
+		}
+		udp = (void *)(ip + 1);
+		if ((void *)(udp + 1) > data_end) {
+			return XDP_ABORTED;
+		}
+	}
+	req = (void *)(udp + 1);
+	if ((void *)(req + 1) > data_end) {
+		return XDP_ABORTED;
+	}
+	sa = (void *)(req + 1);
+	if ((void *)(sa + 1) > data_end) {
+		return XDP_ABORTED;
+	}
+
+	// Set attr data.
+	if (is_ipv6) {
+		xor6 = (void *)(sa + 1);
+		if ((void *)(xor6 + 1) > data_end) {
+			return XDP_ABORTED;
+		}
+		xor6->unused = 0x00; // unused byte
+		xor6->family = 0x02;
+		xor6->port = udp->source ^ bpf_htons(STUN_MAGIC_FOR_PORT_XOR);
+		xor6->addr[0] = ip6->saddr.in6_u.u6_addr32[0] ^ bpf_htonl(STUN_MAGIC);
+		for (int i = 1; i < 4; i++) {
+			// All three are __be32, no endianness flips.
+			xor6->addr[i] = ip6->saddr.in6_u.u6_addr32[i] ^ req->txid[i-1];
+		}
+	} else {
+		xor = (void *)(sa + 1);
+		if ((void *)(xor + 1) > data_end) {
+			return XDP_ABORTED;
+		}
+		xor->unused = 0x00; // unused byte
+		xor->family = 0x01;
+		xor->port = udp->source ^ bpf_htons(STUN_MAGIC_FOR_PORT_XOR);
+		xor->addr = ip->saddr ^ bpf_htonl(STUN_MAGIC);
+	}
+
+	// Flip ethernet header source and destination address.
+	__u8 eth_tmp[ETH_ALEN];
+	__builtin_memcpy(eth_tmp, eth->h_source, ETH_ALEN);
+	__builtin_memcpy(eth->h_source, eth->h_dest, ETH_ALEN);
+	__builtin_memcpy(eth->h_dest, eth_tmp, ETH_ALEN);
+
+	// Flip ip header source and destination address.
+	if (is_ipv6) {
+		struct in6_addr ip_tmp = ip6->saddr;
+		ip6->saddr = ip6->daddr;
+		ip6->daddr = ip_tmp;
+	} else {
+		__be32 ip_tmp = ip->saddr;
+		ip->saddr = ip->daddr;
+		ip->daddr = ip_tmp;
+	}
+
+	// Flip udp header source and destination ports;
+	__be16 port_tmp = udp->source;
+	udp->source = udp->dest;
+	udp->dest = port_tmp;
+
+	// Update total length, TTL, and checksum.
+	__u32 cs = 0;
+	if (is_ipv6) {
+		if ((void *)(ip6 +1) > data_end) {
+			return XDP_ABORTED;
+		}
+		__u16 payload_len = data_end - (void *)(ip6 + 1);
+		ip6->payload_len = bpf_htons(payload_len);
+		ip6->hop_limit = IPDEFTTL;
+	} else {
+		__u16 tot_len = data_end - (void *)ip;
+		ip->tot_len = bpf_htons(tot_len);
+		ip->ttl = IPDEFTTL;
+		ip->check = 0;
+		cs = bpf_csum_diff(0, 0, (void *)ip, sizeof(*ip), cs);
+		ip->check = csum_fold_flip(cs);
+	}
+
+	// Avoid dynamic length math against the packet pointer, which is just a big
+	// verifier headache. Instead sizeof() all the things.
+	int to_csum_len = sizeof(*udp) + sizeof(*req) + sizeof(*sa);
+	// Update udp header length and checksum.
+	if (is_ipv6) {
+		to_csum_len += sizeof(*xor6);
+		udp = (void *)(ip6 + 1);
+		if ((void *)(udp +1) > data_end) {
+			return XDP_ABORTED;
+		}
+		__u16 udp_len = data_end - (void *)udp;
+		udp->len = bpf_htons(udp_len);
+		udp->check = 0;
+		cs = pseudo_sum_ipv6(ip6, udp->len);
+	} else {
+		to_csum_len += sizeof(*xor);
+		udp = (void *)(ip + 1);
+		if ((void *)(udp +1) > data_end) {
+			return XDP_ABORTED;
+		}
+		__u16 udp_len = data_end - (void *)udp;
+		udp->len = bpf_htons(udp_len);
+		udp->check = 0;
+		cs = pseudo_sum_ipv4(ip, udp->len);
+	}
+	if ((void *)udp + to_csum_len > data_end) {
+		return XDP_ABORTED;
+	}
+	cs = bpf_csum_diff(0, 0, (void*)udp, to_csum_len, cs);
+	udp->check = csum_fold_flip(cs);
+	return XDP_TX;
+}
+#undef is_ipv6
+
+SEC("xdp")
+int xdp_prog_func(struct xdp_md *ctx) {
+	struct packet_context pctx = {
+		.af = COUNTER_KEY_AF_UNKNOWN,
+		.prog_end = COUNTER_KEY_END_UNSPECIFIED,
+	};
+	int action = XDP_PASS;
+	action = handle_packet(ctx, &pctx);
+	handle_counters(ctx, action, &pctx);
+	return action;
+}

derp/xdp/xdp.go

@@ -0,0 +1,45 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package xdp
+
+// XDPAttachFlags represents how XDP program will be attached to interface. This
+// is a mirror of cilium/ebpf/link.AttachFlags, without pulling it in for
+// non-Linux.
+type XDPAttachFlags uint32
+
+const (
+	// XDPDriverFallbackGenericMode attempts XDPDriverMode, and falls back to
+	// XDPGenericMode if the driver does not support XDP.
+	XDPDriverFallbackGenericMode = 0
+)
+
+const (
+	// XDPGenericMode (SKB) links XDP BPF program for drivers which do
+	// not yet support native XDP.
+	XDPGenericMode XDPAttachFlags = 1 << (iota + 1)
+	// XDPDriverMode links XDP BPF program into the driver’s receive path.
+	XDPDriverMode
+	// XDPOffloadMode offloads the entire XDP BPF program into hardware.
+	XDPOffloadMode
+)
+
+// STUNServerConfig represents the configuration of a STUNServer.
+type STUNServerConfig struct {
+	DeviceName  string
+	DstPort     int
+	AttachFlags XDPAttachFlags
+	// Return XDP verifier errors in their entirety. This is a multiline error
+	// that can be very long. Full verifier errors are primarily useful during
+	// development, but should be mostly unexpected in a production environment.
+	FullVerifierErr bool
+}
+
+type STUNServerOption interface {
+	apply(*stunServerOptions)
+}
+
+type stunServerOptions struct {
+	//lint:ignore U1000 used in xdp_linux_test.go
+	noAttach bool
+}

derp/xdp/xdp_default.go

@@ -0,0 +1,36 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !linux
+
+package xdp
+
+import (
+	"errors"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// STUNServer is unimplemented on these platforms, see xdp_linux.go.
+type STUNServer struct {
+}
+
+func NewSTUNServer(config *STUNServerConfig, opts ...STUNServerOption) (*STUNServer, error) {
+	return nil, errors.New("unimplemented on this GOOS")
+}
+
+func (s *STUNServer) Close() error {
+	return errors.New("unimplemented on this GOOS")
+}
+
+func (s *STUNServer) Describe(descCh chan<- *prometheus.Desc) {}
+
+func (s *STUNServer) Collect(metricCh chan<- prometheus.Metric) {}
+
+func (s *STUNServer) SetDropSTUN(v bool) error {
+	return errors.New("unimplemented on this GOOS")
+}
+
+func (s *STUNServer) GetDropSTUN() bool {
+	return true
+}

derp/xdp/xdp_linux.go

@@ -0,0 +1,311 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package xdp
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"net"
+	"sync"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/link"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type config -type counters_key -type counter_key_af -type counter_key_packets_bytes_action -type counter_key_prog_end bpf xdp.c -- -I headers
+
+// STUNServer manages loading and unloading of an eBPF XDP program that serves
+// the STUN protocol. It exports statistics for the XDP program via its
+// implementation of the prometheus.Collector interface.
+type STUNServer struct {
+	mu       sync.Mutex
+	objs     *bpfObjects
+	metrics  *stunServerMetrics
+	dstPort  int
+	dropSTUN bool
+}
+
+//lint:ignore U1000 used in xdp_linux_test.go, which has a build tag
+type noAttachOption struct{}
+
+//lint:ignore u1000 Used in xdp_linux_test.go, which has a build tag
+func (n noAttachOption) apply(opts *stunServerOptions) {
+	opts.noAttach = true
+}
+
+func (s *STUNServerConfig) validate() error {
+	if len(s.DeviceName) < 1 {
+		return errors.New("DeviceName is unspecified")
+	}
+	if s.DstPort < 0 || s.DstPort > math.MaxUint16 {
+		return errors.New("DstPort is outside of uint16 bounds")
+	}
+	return nil
+}
+
+// NewSTUNServer returns an instance of a STUNServer that has attached the STUN
+// XDP program to the netdev and destination port specified by config.
+func NewSTUNServer(config *STUNServerConfig, opts ...STUNServerOption) (*STUNServer, error) {
+	o := &stunServerOptions{}
+	for _, opt := range opts {
+		opt.apply(o)
+	}
+	err := config.validate()
+	if err != nil {
+		return nil, fmt.Errorf("invalid config: %v", err)
+	}
+	objs := new(bpfObjects)
+	err = loadBpfObjects(objs, nil)
+	if err != nil {
+		var ve *ebpf.VerifierError
+		if config.FullVerifierErr && errors.As(err, &ve) {
+			err = fmt.Errorf("verifier error: %+v", ve)
+		}
+		return nil, fmt.Errorf("error loading XDP program: %w", err)
+	}
+	server := &STUNServer{
+		objs:    objs,
+		metrics: newSTUNServerMetrics(),
+		dstPort: config.DstPort,
+	}
+	var key uint32
+	xdpConfig := &bpfConfig{
+		DstPort: uint16(config.DstPort),
+	}
+	err = objs.ConfigMap.Put(key, xdpConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error loading config in eBPF map: %w", err)
+	}
+	if o.noAttach {
+		return server, nil
+	}
+	iface, err := net.InterfaceByName(config.DeviceName)
+	if err != nil {
+		return nil, fmt.Errorf("error finding device: %w", err)
+	}
+	_, err = link.AttachXDP(link.XDPOptions{
+		Program:   objs.XdpProgFunc,
+		Interface: iface.Index,
+		Flags:     link.XDPAttachFlags(config.AttachFlags),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error attaching XDP program to dev: %w", err)
+	}
+	return server, nil
+}
+
+// Close unloads the XDP program and associated maps.
+func (s *STUNServer) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.objs.Close()
+}
+
+type stunServerMetrics struct {
+	last     map[bpfCountersKey]uint64
+	registry *prometheus.Registry
+	packets  *prometheus.CounterVec
+	bytes    *prometheus.CounterVec
+}
+
+func newSTUNServerMetrics() *stunServerMetrics {
+	last := make(map[bpfCountersKey]uint64)
+	registry := prometheus.NewRegistry()
+	packets := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "xdp",
+		Subsystem: "stun_server",
+		Name:      "packets_total",
+	}, []string{addressFamilyKey, xdpOutcomeKey, progEndKey})
+	bytes := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "xdp",
+		Subsystem: "stun_server",
+		Name:      "bytes_total",
+	}, []string{addressFamilyKey, xdpOutcomeKey, progEndKey})
+	registry.MustRegister(packets, bytes)
+	return &stunServerMetrics{
+		last:     last,
+		registry: registry,
+		packets:  packets,
+		bytes:    bytes,
+	}
+}
+
+const (
+	xdpOutcomeKey = "xdp_outcome"
+	progEndKey    = "prog_end"
+)
+
+const (
+	xdpOutcomePass    = "pass"
+	xdpOutcomeAborted = "aborted"
+	xdpOutcomeDrop    = "drop"
+	xdpOutcomeTX      = "tx"
+)
+
+func sum(vals []uint64) uint64 {
+	var s uint64
+	for _, v := range vals {
+		s += v
+	}
+	return s
+}
+
+const (
+	addressFamilyKey = "address_family"
+)
+
+const (
+	addressFamilyUnknown = "unknown"
+	addressFamilyIPv4    = "ipv4"
+	addressFamilyIPv6    = "ipv6"
+)
+
+var (
+	// TODO(jwhited): go generate these maps or equivalent switch logic behind bpf2go
+	pbaToOutcomeLV = map[bpfCounterKeyPacketsBytesAction]string{
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL:    xdpOutcomePass,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL:      xdpOutcomePass,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL: xdpOutcomeAborted,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL:   xdpOutcomeAborted,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL:      xdpOutcomeTX,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL:        xdpOutcomeTX,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL:    xdpOutcomeDrop,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL:      xdpOutcomeDrop,
+	}
+
+	progEndLV = map[bpfCounterKeyProgEnd]string{
+		bpfCounterKeyProgEndCOUNTER_KEY_END_UNSPECIFIED:                "unspecified",
+		bpfCounterKeyProgEndCOUNTER_KEY_END_UNEXPECTED_FIRST_STUN_ATTR: "unexpected_first_stun_attr",
+		bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_UDP_CSUM:           "invalid_udp_csum",
+		bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_IP_CSUM:            "invalid_ip_csum",
+		bpfCounterKeyProgEndCOUNTER_KEY_END_NOT_STUN_PORT:              "not_stun_port",
+		bpfCounterKeyProgEndCOUNTER_KEY_END_INVALID_SW_ATTR_VAL:        "invalid_sw_attr_val",
+		bpfCounterKeyProgEndCOUNTER_KEY_END_DROP_STUN:                  "drop_stun",
+	}
+
+	packetCounterKeys = map[bpfCounterKeyPacketsBytesAction]bool{
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_PASS_TOTAL:    true,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_ABORTED_TOTAL: true,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_TX_TOTAL:      true,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_DROP_TOTAL:    true,
+	}
+
+	//lint:ignore U1000 used in xdp_linux_test.go, which has a build tag
+	bytesCounterKeys = map[bpfCounterKeyPacketsBytesAction]bool{
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_PASS_TOTAL:    true,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_ABORTED_TOTAL: true,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_TX_TOTAL:      true,
+		bpfCounterKeyPacketsBytesActionCOUNTER_KEY_BYTES_DROP_TOTAL:    true,
+	}
+)
+
+// increase returns the difference between "from" and "to" assuming they
+// originated from the same counter gathered at different times, i.e. "from"
+// was incremented by a non-negative value into "to". In the case of wraps
+// increase returns the difference between "to" and zero.
+func increase(from, to uint64) uint64 {
+	if to >= from {
+		return to - from
+	}
+	return to
+}
+
+func (s *stunServerMetrics) updateFromMapKV(key bpfCountersKey, vals []uint64) error {
+	if key.Unused != 0 ||
+		key.Af >= uint8(bpfCounterKeyAfCOUNTER_KEY_AF_LEN) ||
+		key.Pba >= uint8(bpfCounterKeyPacketsBytesActionCOUNTER_KEY_PACKETS_BYTES_ACTION_LEN) ||
+		key.ProgEnd >= uint8(bpfCounterKeyProgEndCOUNTER_KEY_END_LEN) {
+		return fmt.Errorf("unexpected counter key: %+v", key)
+	}
+	previousAllCPUs := s.last[key]
+	allCPUs := sum(vals)
+	s.last[key] = allCPUs
+	inc := increase(previousAllCPUs, allCPUs)
+	if inc == 0 {
+		return nil
+	}
+	var af string
+	switch key.Af {
+	case uint8(bpfCounterKeyAfCOUNTER_KEY_AF_UNKNOWN):
+		af = addressFamilyUnknown
+	case uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV4):
+		af = addressFamilyIPv4
+	case uint8(bpfCounterKeyAfCOUNTER_KEY_AF_IPV6):
+		af = addressFamilyIPv6
+	}
+	labels := prometheus.Labels{
+		addressFamilyKey: af,
+		xdpOutcomeKey:    pbaToOutcomeLV[bpfCounterKeyPacketsBytesAction(key.Pba)],
+		progEndKey:       progEndLV[bpfCounterKeyProgEnd(key.ProgEnd)],
+	}
+	var metric *prometheus.CounterVec
+	if packetCounterKeys[bpfCounterKeyPacketsBytesAction(key.Pba)] {
+		metric = s.packets
+	} else {
+		metric = s.bytes
+	}
+	metric.With(labels).Add(float64(inc))
+	return nil
+}
+
+// Describe is part of the implementation of prometheus.Collector.
+func (s *STUNServer) Describe(descCh chan<- *prometheus.Desc) {
+	s.metrics.registry.Describe(descCh)
+}
+
+// Collect is part of the implementation of prometheus.Collector.
+func (s *STUNServer) Collect(metricCh chan<- prometheus.Metric) {
+	err := s.updateMetrics()
+	if err != nil {
+		log.Printf("xdp: error collecting metrics: %v", err)
+	}
+	s.metrics.registry.Collect(metricCh)
+}
+
+func (s *STUNServer) SetDropSTUN(v bool) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	dropSTUN := 0
+	if v {
+		dropSTUN = 1
+	}
+	xdpConfig := &bpfConfig{
+		DstPort:  uint16(s.dstPort),
+		DropStun: uint16(dropSTUN),
+	}
+	var key uint32
+	err := s.objs.ConfigMap.Put(key, xdpConfig)
+	if err == nil {
+		s.dropSTUN = v
+	}
+	return err
+}
+
+func (s *STUNServer) GetDropSTUN() bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.dropSTUN
+}
+
+func (s *STUNServer) updateMetrics() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	iter := s.objs.CountersMap.Iterate()
+	var key bpfCountersKey
+	numCPU, err := ebpf.PossibleCPU()
+	if err != nil {
+		return err
+	}
+	vals := make([]uint64, numCPU)
+	for iter.Next(&key, &vals) {
+		err := s.metrics.updateFromMapKV(key, vals)
+		if err != nil {
+			return err
+		}
+	}
+	return iter.Err()
+}

drive/driveimpl/remote_impl.go

@@ -167,7 +167,7 @@ func (s *FileSystemForRemote) buildChild(share *drive.Share) *compositedav.Child
 			return fmt.Sprintf("http://%s/%s/%s", hex.EncodeToString([]byte(share.Name)), secretToken, url.PathEscape(share.Name)), nil
 		},
 		Transport: &http.Transport{
-			Dial: func(_, shareAddr string) (net.Conn, error) {
+			DialContext: func(ctx context.Context, _, shareAddr string) (net.Conn, error) {
 				shareNameHex, _, err := net.SplitHostPort(shareAddr)
 				if err != nil {
 					return nil, fmt.Errorf("unable to parse share address %v: %w", shareAddr, err)
@@ -188,10 +188,11 @@ func (s *FileSystemForRemote) buildChild(share *drive.Share) *compositedav.Child
 				_, err = netip.ParseAddrPort(addr)
 				if err == nil {
 					// this is a regular network address, dial normally
-					return net.Dial("tcp", addr)
+					var std net.Dialer
+					return std.DialContext(ctx, "tcp", addr)
 				}
 				// assume this is a safesocket address
-				return safesocket.Connect(addr)
+				return safesocket.ConnectContext(ctx, addr)
 			},
 		},
 	}

flake.nix

@@ -120,4 +120,4 @@
   in
     flake-utils.lib.eachDefaultSystem (system: flakeForSystem nixpkgs system);
 }
-# nix-direnv cache busting line: sha256-KLJibt2Yv4WMxcmKHs+3e0sjVWFc9Fg0ak4sLoAocA0=
+# nix-direnv cache busting line: sha256-ye8puuEDd/CRSy/AHrtLdKVxVASJAdpt6bW3jU2OUvw=

go.mod

@@ -5,6 +5,7 @@ go 1.22.0
 require (
 	filippo.io/mkcert v1.4.4
 	fybrik.io/crdoc v0.6.3
+	github.com/Masterminds/squirrel v1.5.4
 	github.com/akutz/memconn v0.1.0
 	github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa
 	github.com/andybalholm/brotli v1.1.0
@@ -15,6 +16,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.33.0
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.44.7
 	github.com/bramvdbogaerde/go-scp v1.4.0
+	github.com/cilium/ebpf v0.15.0
 	github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/creack/pty v1.1.21
@@ -23,17 +25,19 @@ require (
 	github.com/dave/patsy v0.0.0-20210517141501-957256f50cba
 	github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa
 	github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e
+	github.com/distribution/reference v0.6.0
 	github.com/djherbis/times v1.6.0
 	github.com/dsnet/try v0.0.3
 	github.com/evanw/esbuild v0.19.11
 	github.com/frankban/quicktest v1.14.6
-	github.com/fxamacker/cbor/v2 v2.5.0
+	github.com/fxamacker/cbor/v2 v2.6.0
 	github.com/gaissmai/bart v0.4.1
 	github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0
 	github.com/go-logr/zapr v1.3.0
 	github.com/go-ole/go-ole v1.3.0
 	github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
+	github.com/golang/snappy v0.0.4
 	github.com/golangci/golangci-lint v1.52.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.18.0
@@ -62,15 +66,16 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.6
 	github.com/prometheus-community/pro-bing v0.4.0
-	github.com/prometheus/client_golang v1.18.0
-	github.com/prometheus/common v0.46.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/common v0.48.0
+	github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff
 	github.com/safchain/ethtool v0.3.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/studio-b12/gowebdav v0.9.0
 	github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41
-	github.com/tailscale/golang-x-crypto v0.0.0-20240108194725-7ce1f622c780
+	github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4
 	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
 	github.com/tailscale/mkctr v0.0.0-20240102155253-bf50773ba734
@@ -89,28 +94,29 @@ require (
 	go.uber.org/zap v1.26.0
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
-	golang.org/x/crypto v0.21.0
+	golang.org/x/crypto v0.24.0
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
-	golang.org/x/mod v0.14.0
-	golang.org/x/net v0.23.0
+	golang.org/x/mod v0.18.0
+	golang.org/x/net v0.26.0
 	golang.org/x/oauth2 v0.16.0
-	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.18.0
-	golang.org/x/term v0.18.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/sys v0.21.0
+	golang.org/x/term v0.21.0
 	golang.org/x/time v0.5.0
-	golang.org/x/tools v0.17.0
+	golang.org/x/tools v0.22.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard/windows v0.5.3
 	gopkg.in/square/go-jose.v2 v2.6.0
 	gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3
 	honnef.co/go/tools v0.4.6
-	k8s.io/api v0.29.1
-	k8s.io/apimachinery v0.29.1
-	k8s.io/apiserver v0.29.1
-	k8s.io/client-go v0.29.1
+	k8s.io/api v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/apiserver v0.30.1
+	k8s.io/client-go v0.30.1
+	modernc.org/sqlite v1.29.10
 	nhooyr.io/websocket v1.8.10
-	sigs.k8s.io/controller-runtime v0.16.2
-	sigs.k8s.io/controller-tools v0.13.0
+	sigs.k8s.io/controller-runtime v0.18.4
+	sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab
 	sigs.k8s.io/yaml v1.4.0
 	software.sslmate.com/src/go-pkcs12 v0.4.0
 )
@@ -121,9 +127,21 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14 // indirect
 	github.com/dave/brenda v1.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	modernc.org/libc v1.49.3 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 )
 
 require (
@@ -179,7 +197,7 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/curioswitch/go-reassign v0.2.0 // indirect
 	github.com/daixiang0/gci v0.10.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/denis-tingaikin/go-header v0.4.3 // indirect
 	github.com/docker/cli v25.0.0+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
@@ -190,8 +208,8 @@ require (
 	github.com/esimonov/ifshort v1.0.4 // indirect
 	github.com/ettle/strcase v0.1.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.8.1 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/firefart/nonamedreturns v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0
@@ -215,7 +233,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
 	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
 	github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect
@@ -276,7 +294,7 @@ require (
 	github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
-	github.com/mdlayher/socket v0.5.0 // indirect
+	github.com/mdlayher/socket v0.5.0
 	github.com/mgechev/revive v1.3.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -299,7 +317,7 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/polyfloyd/go-errorlint v1.4.1 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
@@ -359,7 +377,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20240119083558-1b970713d09a // indirect
 	golang.org/x/image v0.15.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
@@ -369,10 +387,9 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 	howett.net/plist v1.0.0 // indirect
-	k8s.io/apiextensions-apiserver v0.29.1 // indirect
-	k8s.io/component-base v0.29.1 // indirect
+	k8s.io/apiextensions-apiserver v0.30.1 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240117194847-208609032b15 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	mvdan.cc/gofumpt v0.5.0 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect

go.mod.sri

@@ -1 +1 @@
-sha256-KLJibt2Yv4WMxcmKHs+3e0sjVWFc9Fg0ak4sLoAocA0=
+sha256-ye8puuEDd/CRSy/AHrtLdKVxVASJAdpt6bW3jU2OUvw=

go.sum

@@ -75,6 +75,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
+github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
@@ -204,8 +206,8 @@ github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8/go.mod h1:gakxgyXa
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4=
-github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM=
+github.com/cilium/ebpf v0.15.0 h1:7NxJhNiBT3NG8pZJ3c+yfrVdHY8ScgKD27sScgjLMMk=
+github.com/cilium/ebpf v0.15.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
@@ -240,14 +242,17 @@ github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3
 github.com/dave/patsy v0.0.0-20210517141501-957256f50cba h1:1o36L4EKbZzazMk8iGC4kXpVnZ6TPxR2mZ9qVKjNNAs=
 github.com/dave/patsy v0.0.0-20210517141501-957256f50cba/go.mod h1:qfR88CgEGLoiqDaE+xxDCi5QA5v4vUoW0UCX2Nd5Tlc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk=
 github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ=
 github.com/denis-tingaikin/go-header v0.4.3 h1:tEaZKAlqql6SKCY++utLmkPLd6K8IBM20Ha7UVm+mtU=
 github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c=
 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q=
 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
 github.com/docker/cli v25.0.0+incompatible h1:zaimaQdnX7fYWFqzN88exE9LDEvRslexpFowZBX6GoQ=
@@ -260,6 +265,8 @@ github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqY
 github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
 github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
 github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU=
@@ -278,12 +285,12 @@ github.com/ettle/strcase v0.1.1 h1:htFueZyVeE1XNnMEfbqp5r67qAN/4r6ya1ysq8Q+Zcw=
 github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.8.1 h1:iPEdwg0XayoS+E7Mth9JxwUtOgyVxnDTXHtKhZPlZxA=
-github.com/evanphx/json-patch/v5 v5.8.1/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/evanw/esbuild v0.19.11 h1:mbPO1VJ/df//jjUd+p/nRLYCpizXxXb2w/zZMShxa2k=
 github.com/evanw/esbuild v0.19.11/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/firefart/nonamedreturns v1.0.4 h1:abzI1p7mAEPYuR4A+VLKn4eNDOycjYo2phmY9sfv40Y=
@@ -292,8 +299,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
-github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
+github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA=
+github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
 github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
 github.com/gaissmai/bart v0.4.1 h1:G1t58voWkNmT47lBDawH5QhtTDsdqRIO+ftq5x4P9Ls=
@@ -335,6 +342,8 @@ github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdX
 github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
 github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8=
 github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
@@ -398,8 +407,10 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
@@ -463,8 +474,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/rpmpack v0.5.0 h1:L16KZ3QvkFGpYhmp23iQip+mx1X39foEsqszjMNBm8A=
 github.com/google/rpmpack v0.5.0/go.mod h1:uqVAUVQLq8UY2hCDfmJ/+rtO3aw7qyhc90rCVEabEfI=
@@ -511,6 +522,8 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU=
@@ -610,6 +623,10 @@ github.com/kunwardeep/paralleltest v1.0.6 h1:FCKYMF1OF2+RveWlABsdnmsvJrei5aoyZoa
 github.com/kunwardeep/paralleltest v1.0.6/go.mod h1:Y0Y0XISdZM5IKm3TREQMZ6iteqn1YuwCsJO/0kL9Zes=
 github.com/kyoh86/exportloopref v0.1.11 h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ=
 github.com/kyoh86/exportloopref v0.1.11/go.mod h1:qkV4UF1zGl6EkF1ox8L5t9SwyeBAZ3qLMd6up458uqA=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
+github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
+github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
 github.com/ldez/gomoddirectives v0.2.3 h1:y7MBaisZVDYmKvt9/l1mjNCiSA1BVn34U0ObUcJwlhA=
 github.com/ldez/gomoddirectives v0.2.3/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0=
 github.com/ldez/tagliatelle v0.5.0 h1:epgfuYt9v0CG3fms0pEgIMNPuFf/LpPIfjk4kyqSioo=
@@ -682,6 +699,8 @@ github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81
 github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE=
 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nishanths/exhaustive v0.10.0 h1:BMznKAcVa9WOoLq/kTGp4NJOJSMwEpcpjFNAVRfPlSo=
@@ -696,10 +715,10 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
-github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
-github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
+github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
+github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
+github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU=
@@ -729,8 +748,9 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
 github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polyfloyd/go-errorlint v1.4.1 h1:r8ru5FhXSn34YU1GJDOuoJv2LdsQkPmK325EOpPMJlM=
 github.com/polyfloyd/go-errorlint v1.4.1/go.mod h1:k6fU/+fQe38ednoZS51T7gSIGQW1y94d6TkSr35OzH8=
 github.com/prometheus-community/pro-bing v0.4.0 h1:YMbv+i08gQz97OZZBwLyvmmQEEzyfyrrjEaAchdy3R4=
@@ -740,8 +760,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
-github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -752,8 +772,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
-github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@@ -761,6 +781,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff h1:X1Tly81aZ22DA1fxBdfvR3iw8+yFoUBUHMEd+AX/ZXI=
+github.com/prometheus/prometheus v0.49.2-0.20240125131847-c3b8ef1694ff/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU=
 github.com/quasilyte/go-ruleguard v0.3.19 h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc=
 github.com/quasilyte/go-ruleguard v0.3.19/go.mod h1:lHSn69Scl48I7Gt9cX3VrbsZYvYiBYszZOZW4A+oTEw=
 github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo=
@@ -769,6 +791,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl
 github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
 github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs=
 github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -874,8 +898,8 @@ github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 h1:Gzfnfk2TWrk8
 github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55/go.mod h1:4k4QO+dQ3R5FofL+SanAUZe+/QfeK0+OIuwDIRu2vSg=
 github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41 h1:/V2rCMMWcsjYaYO2MeovLw+ClP63OtXgCF2Y1eb8+Ns=
 github.com/tailscale/goexpect v0.0.0-20210902213824-6e8c725cea41/go.mod h1:/roCdA6gg6lQyw/Oz6gIIGu3ggJKYhF+WC/AQReE5XQ=
-github.com/tailscale/golang-x-crypto v0.0.0-20240108194725-7ce1f622c780 h1:U0J2CUrrTcc2wmr9tSLYEo+USfwNikRRsmxVLD4eZ7E=
-github.com/tailscale/golang-x-crypto v0.0.0-20240108194725-7ce1f622c780/go.mod h1:ikbF+YT089eInTp9f2vmvy4+ZVnW5hzX1q2WknxSprQ=
+github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4 h1:rXZGgEa+k2vJM8xT0PoSKfVXwFGPQ3z3CJfmnHJkZZw=
+github.com/tailscale/golang-x-crypto v0.0.0-20240604161659-3fde5e568aa4/go.mod h1:ikbF+YT089eInTp9f2vmvy4+ZVnW5hzX1q2WknxSprQ=
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
 github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw=
@@ -964,8 +988,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
-go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
@@ -988,8 +1012,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1038,8 +1062,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1084,8 +1108,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1111,8 +1135,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1179,8 +1203,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1189,8 +1213,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1205,8 +1229,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1285,8 +1309,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
 golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
+golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1435,24 +1459,48 @@ honnef.co/go/tools v0.4.6 h1:oFEHCKeID7to/3autwsWfnuv69j3NsfcXbvJKuIcep8=
 honnef.co/go/tools v0.4.6/go.mod h1:+rnGS1THNh8zMwnd2oVOTL9QF6vmfyG6ZXBULae2uc0=
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
-k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw=
-k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ=
-k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw=
-k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU=
-k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc=
-k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
-k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g=
-k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw=
-k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
-k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
-k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw=
-k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc=
+k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY=
+k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM=
+k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws=
+k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4=
+k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
+k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8=
+k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo=
+k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
+k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kube-openapi v0.0.0-20240117194847-208609032b15 h1:m6dl1pkxz3HuE2mP9MUYPCCGyy6IIFlv/vTlLBDxIwA=
-k8s.io/kube-openapi v0.0.0-20240117194847-208609032b15/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk=
+modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA=
+modernc.org/ccgo/v4 v4.16.0/go.mod h1:dkNyWIjFrVIZ68DTo36vHK+6/ShBn4ysU61So6PIqCI=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
+modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg=
+modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
+modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
+modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
+modernc.org/sqlite v1.29.10 h1:3u93dz83myFnMilBGCOLbr+HjklS6+5rJLx4q86RDAg=
+modernc.org/sqlite v1.29.10/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E=
 mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
@@ -1466,10 +1514,10 @@ nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU=
-sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU=
-sigs.k8s.io/controller-tools v0.13.0 h1:NfrvuZ4bxyolhDBt/rCZhDnx3M2hzlhgo5n3Iv2RykI=
-sigs.k8s.io/controller-tools v0.13.0/go.mod h1:5vw3En2NazbejQGCeWKRrE7q4P+CW8/klfVqP8QZkgA=
+sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
+sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
+sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab h1:Fq4VD28nejtsijBNTeRRy9Tt3FVwq+o6NB7fIxja8uY=
+sigs.k8s.io/controller-tools v0.15.1-0.20240618033008-7824932b0cab/go.mod h1:egedX5jq2KrZ3A2zaOz3e2DSsh5BhFyyjvNcBRIQel8=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=

go.toolchain.rev

@@ -1 +1 @@
-467a489ae3c080d80f4cfdd05f2aa08cb44c9d6a
+4d101c0f2d2a234b8902bfff5fadb16070201f0a

gokrazy/.gitignore

@@ -0,0 +1,2 @@
+tsapp.img
+go.work

gokrazy/Makefile

@@ -0,0 +1,8 @@
+help:
+	echo "See Makefile"
+
+image:
+	go run build.go --build
+
+qemu: image
+	qemu-system-x86_64 -m 1G -drive file=tsapp.img,format=raw -boot d -netdev user,id=user.0 -device virtio-net-pci,netdev=user.0 -serial mon:stdio -audio none

gokrazy/README.md

@@ -0,0 +1,76 @@
+# Tailscale Appliance Gokrazy Image
+
+This is (as of 2024-06-02) a **WORK IN PROGRESS** (pre-alpha) experiment to
+package Tailscale as a [Gokrazy](https://gokrazy.org/) appliance image
+for use on both VMs (AWS, GCP, Azure, Proxmox, ...) and Rasperry Pis.
+
+See https://github.com/tailscale/tailscale/issues/1866
+
+## Overview
+
+It makes a ~70MB image (about the same size as
+`tailscale-setup-full-1.66.4.exe` and smaller than the combined
+Tailscale Android APK) that combines the Linux kernel and Tailscale
+and that's it. Nothing written in C. (except optional busybox for
+debugging) So no operating system to maintain. Gokrazy has three
+partitions: two read-only ones (one active at a time, the other for
+updates for the next boot) and one optional stateful, writable
+partition that survives upgrades (`/perm/`)
+
+Initial bootstrap configuration of this appliance will be over either
+serial or configuration files (auth keys, subnet routes, etc) baked into
+the image (for Raspberry Pis) or in cloud-init/user-data (for AWS, etc).
+As of 2024-06-02, AWS user-data config files work.
+
+## Quick start
+
+Install dependencies:
+```
+$ brew install qemu e2fsprogs
+```
+
+Build + launch:
+```
+$ make qemu
+```
+
+That puts serial on stdio. To exit the serial console and escape to
+the qemu monitor, type `Ctrl-a c`. Then type `quit` in the monitor to
+quit.
+
+## Building
+
+`make image` to build just the image (`tsapp.img`), without uploading it.
+
+## UTM
+
+You can also use UTM, but the qemu path above is easier.
+For UTM, see the [UTM instructions](UTM.md).
+
+## AWS
+
+### Build an AMI
+
+`go run build.go --bucket=your-S3-temp-bucket` to build an AMI. Make
+sure your "aws" command is in your path and has access.
+
+### Creating an instance
+
+When creating an instance, you need a Nitro machine type to get a
+virtual serial console. Notably, that means the `t2.*` instance types
+that AWS pushes as a free option are not new enough. Use `t3.*` at least.
+
+As of 2024-06-02 this builder tool only supports x86_64 (arm64 should
+be trivial and will come soon), so don't use a Graviton machine type.
+
+To connect to the serial console, you can either use the web console, or
+use the CLI like:
+
+```
+$ aws ec2-instance-connect send-serial-console-ssh-public-key --instance-id i-0b4a0eabc43629f13 --serial-port 0 --ssh-public-key file:///your/home/.ssh/id_ed25519.pub --region us-west-2
+{
+    "RequestId": "a93b0ea3-9ff9-45d5-b8ed-b1e70ccc0410",
+    "Success": true
+}
+$ ssh i-0b4a0eabc43629f13.port0@serial-console.ec2-instance-connect.us-west-2.aws 
+```

gokrazy/UTM.md

@@ -0,0 +1,50 @@
+# tsapp on UTM
+
+qemu from homebrew is recommended for tsapp development.
+See the [main README](README.md) for details.
+
+If you don't want to use qemu, this documents a way to use UTM on
+macOS for tsapp development. It's not as quick of an edit-run-test
+iteration cycle, but this is how:
+
+* Create new VM, choose "Emulate" (for now) and not "Virtualize"
+* Pick "Linux" as the operating system
+* For "Boot ISO Image", select the built `tsapp.img`
+* Architecture: `x86_64` (for now; arm64 later)
+* System: `Standard PC (...) (q35)`
+* Memory: 1024 MB is fine for testing
+* CPUs: Default
+* Storage size: 3GB
+* Shared Directory: none. Continue.
+* Summary: check "Open VM Settings"
+* Network: Emulated Network Card: `virtio-net-pci`
+* Display: Emulated Display Card: `virtio-vga` (not that there's much to see)
+* Drives: delete all disks
+* Drives: New... Interface `VirtIO`, Import ... find `tsapp.img` again. Save.
+* Devices: New... Serial. Mode: Psuedo-TTY Device, Target: Automatic Serial Device.
+
+Once created & the `img` is imported once, UTM converts it to qcow2 format
+under `$HOME/Library/Containers/com.utmapp.UTM/Data/Documents/Tsapp.utm/Data/tsapp.qcow2`.
+
+To update it, stop the VM, then:
+
+```
+qemu-img convert -f raw -O qcow2 tsapp.img tsapp.qcow2 && \
+  mv tsapp.qcow2 $HOME/Library/Containers/com.utmapp.UTM/Data/Documents/Tsapp.utm/Data/tsapp.qcow2
+```
+
+To attach to its serial:
+
+```
+% /Applications/UTM.app/Contents/MacOS/utmctl list
+UUID                                 Status   Name
+C0DE927B-F426-4ABA-A6E7-E30AA429371F started  Tsapp
+
+% % /Applications/UTM.app/Contents/MacOS/utmctl attach C0DE927B-F426-4ABA-A6E7-E30AA429371F
+WARNING: attach command is not implemented yet!
+PTTY: /dev/ttys017
+
+% screen /dev/ttys017
+```
+
+(Then `Ctrl-a K` to kill screen session)

gokrazy/build.go

@@ -0,0 +1,274 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// This program builds the Tailscale Appliance Gokrazy image.
+//
+// As of 2024-06-02 this is a exploratory work in progress and is
+// not intended for serious use.
+//
+// Tracking issue is https://github.com/tailscale/tailscale/issues/1866
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"time"
+)
+
+var (
+	bucket = flag.String("bucket", "tskrazy-import", "S3 bucket to upload disk image to while making AMI")
+	build  = flag.Bool("build", false, "if true, just build locally and stop, without uploading")
+)
+
+func findMkfsExt4() (string, error) {
+	tries := []string{
+		"/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4",
+		"/sbin/mkfs.ext4",
+	}
+	for _, p := range tries {
+		if _, err := os.Stat(p); err == nil {
+			return p, nil
+		}
+	}
+	p, err := exec.LookPath("mkfs.ext4")
+	if err == nil {
+		return p, nil
+	}
+	if runtime.GOOS == "darwin" {
+		return "", errors.New("no mkfs.ext4 found; run `brew install e2fsprogs`")
+	}
+	return "", errors.New("No mkfs.ext4 found on system")
+}
+
+func main() {
+	flag.Parse()
+
+	if err := buildImage(); err != nil {
+		log.Fatalf("build image: %v", err)
+	}
+	if *build {
+		log.Printf("built. stopping.")
+		return
+	}
+
+	if err := copyToS3(); err != nil {
+		log.Fatalf("copy to S3: %v", err)
+	}
+
+	importTask, err := startImportSnapshot()
+	if err != nil {
+		log.Fatalf("start import snapshot: %v", err)
+	}
+	snapID, err := waitForImportSnapshot(importTask)
+	if err != nil {
+		log.Fatalf("waitForImportSnapshot(%v): %v", importTask, err)
+	}
+	log.Printf("snap ID: %v", snapID)
+
+	ami, err := makeAMI(fmt.Sprintf("tsapp-%d", time.Now().Unix()), snapID)
+	if err != nil {
+		log.Fatalf("makeAMI: %v", err)
+	}
+	log.Printf("made AMI: %v", ami)
+}
+
+func buildImage() error {
+	mkfs, err := findMkfsExt4()
+	if err != nil {
+		return err
+	}
+
+	dir, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	if fi, err := os.Stat(filepath.Join(dir, "tsapp")); err != nil || !fi.IsDir() {
+		return fmt.Errorf("in wrong directorg %v; no tsapp subdirectory found", dir)
+	}
+	// Build the tsapp.img
+	var buf bytes.Buffer
+	cmd := exec.Command("go", "run",
+		"-exec=env GOOS=linux GOARCH=amd64 ",
+		"github.com/gokrazy/tools/cmd/gok",
+		"--parent_dir="+dir,
+		"--instance=tsapp",
+		"overwrite",
+		"--full", "tsapp.img",
+		"--target_storage_bytes=1258299392")
+	cmd.Stdout = io.MultiWriter(os.Stdout, &buf)
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	// gok overwrite emits a line of text saying how to run mkfs.ext4
+	// to create the ext4 /perm filesystem. Parse that and run it.
+	// The regexp is tight to avoid matching if the command changes,
+	// to force us to check it's still correct/safe. But it shouldn't
+	// change on its own because we pin the gok version in our go.mod.
+	//
+	// TODO(bradfitz): emit this in a machine-readable way from gok.
+	rx := regexp.MustCompile(`(?m)/mkfs.ext4 (-F) (-E) (offset=\d+) (\S+) (\d+)\s*?$`)
+	m := rx.FindStringSubmatch(buf.String())
+	if m == nil {
+		return fmt.Errorf("found no ext4 instructions in output")
+	}
+
+	log.Printf("Running %s %q ...", mkfs, m[1:])
+	out, err := exec.Command(mkfs, m[1:]...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error running %v: %v, %s", mkfs, err, out)
+	}
+	log.Printf("Success.")
+
+	return nil
+}
+
+func copyToS3() error {
+	cmd := exec.Command("aws", "s3", "cp", "tsapp.img", "s3://"+*bucket+"/")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func startImportSnapshot() (importTaskID string, err error) {
+	out, err := exec.Command("aws", "ec2", "import-snapshot", "--disk-container", "Url=s3://"+*bucket+"/tsappp.img").CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("import snapshot: %v: %s", err, out)
+	}
+	var resp struct {
+		ImportTaskID string `json:"ImportTaskId"`
+	}
+	/*
+		{
+			"ImportTaskId": "import-snap-0d2d72622b4359567",
+			"SnapshotTaskDetail": {
+				"DiskImageSize": 0.0,
+				"Progress": "0",
+				"Status": "active",
+				"StatusMessage": "pending",
+				"Url": "s3://tskrazy-import/tskrazy.img"
+			},
+			"Tags": []
+		}
+	*/
+	if err := json.Unmarshal(out, &resp); err != nil {
+		return "", fmt.Errorf("unmarshal response: %v: %s", err, out)
+	}
+	return resp.ImportTaskID, nil
+}
+
+/*
+% aws ec2 describe-import-snapshot-tasks --import-task-ids import-snap-0d2d72622b4359567
+{
+    "ImportSnapshotTasks": [
+        {
+            "ImportTaskId": "import-snap-0d2d72622b4359567",
+            "SnapshotTaskDetail": {
+                "DiskImageSize": 1258299392.0,
+                "Format": "RAW",
+                "SnapshotId": "snap-053efd3539d787927",
+                "Status": "completed",
+                "Url": "s3://tskrazy-import/tskrazy.img",
+                "UserBucket": {
+                    "S3Bucket": "tskrazy-import",
+                    "S3Key": "tskrazy.img"
+                }
+            },
+            "Tags": []
+        }
+    ]
+}
+*/
+
+func waitForImportSnapshot(importTaskID string) (snapID string, err error) {
+	for {
+		out, err := exec.Command("aws", "ec2", "describe-import-snapshot-tasks", "--import-task-ids", importTaskID).CombinedOutput()
+		if err != nil {
+			return "", fmt.Errorf("describe import snapshot tasks: %v: %s", err, out)
+		}
+
+		var resp struct {
+			ImportSnapshotTasks []struct {
+				SnapshotTaskDetail struct {
+					SnapshotID string `json:"SnapshotId"`
+					Status     string `json:"Status"`
+				} `json:"SnapshotTaskDetail"`
+			} `json:"ImportSnapshotTasks"`
+		}
+		if err := json.Unmarshal(out, &resp); err != nil {
+			return "", fmt.Errorf("unmarshal response: %v: %s", err, out)
+		}
+		if len(resp.ImportSnapshotTasks) > 0 {
+			first := &resp.ImportSnapshotTasks[0]
+			if first.SnapshotTaskDetail.Status == "completed" {
+				return first.SnapshotTaskDetail.SnapshotID, nil
+			}
+		}
+		log.Printf("Still waiting; got: %s", out)
+		time.Sleep(5 * time.Second)
+
+		// TODO(bradfitz): percentage bar?
+		// Looks like:
+		/* 2024/05/14 13:03:21 Still waiting; got: {
+		    "ImportSnapshotTasks": [
+		        {
+		            "ImportTaskId": "import-snap-0232251d0fbcb33fd",
+		            "SnapshotTaskDetail": {
+		                "DiskImageSize": 1258299392.0,
+		                "Format": "RAW",
+		                "Progress": "32",
+		                "Status": "active",
+		                "StatusMessage": "validated",
+		                "Url": "s3://tskrazy-import/tskrazy.img",
+		                "UserBucket": {
+		                    "S3Bucket": "tskrazy-import",
+		                    "S3Key": "tskrazy.img"
+		                }
+		            },
+		            "Tags": []
+		        }
+		    ]
+		}*/
+	}
+}
+
+func makeAMI(name, ebsSnapID string) (ami string, err error) {
+	out, err := exec.Command("aws", "ec2", "register-image",
+		"--name", name,
+		"--architecture", "x86_64",
+		"--root-device-name", "/dev/sda",
+		"--ena-support",
+		"--imds-support", "v2.0",
+		"--boot-mode", "uefi-preferred",
+		"--block-device-mappings", "DeviceName=/dev/sda,Ebs={SnapshotId="+ebsSnapID+"}").CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("register image: %v: %s", err, out)
+	}
+	/*
+		On success:
+		{
+		    "ImageId": "ami-052e1538166886ad2"
+		}
+	*/
+	var resp struct {
+		ImageID string `json:"ImageId"`
+	}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		return "", fmt.Errorf("unmarshal response: %v: %s", err, out)
+	}
+	if resp.ImageID == "" {
+		return "", fmt.Errorf("empty image ID in response: %s", out)
+	}
+	return resp.ImageID, nil
+}