ipn, tstime : add opt in rate limiting for netmap updates on the IPN bus

updates tailscale/corp#24553

Adds opt-in rate limiting to limit netmap updates to, at most, one every
3 seconds when the client includes the NotifyRateLimitNetmaps option
in the ipn bus watcher opts.   This should mitigate issues with excessive
memory and CPU usage in clients on large, busy tailnets.

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

.github/workflows/codeql-analysis.yml

@@ -49,13 +49,13 @@ jobs:
 
     # Install a more recent Go that understands modern go.mod content.
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@6db8d6351fd0be61f9ed8ebd12ccd35dcec51fea # v3.26.11
+      uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1

.github/workflows/golangci-lint.yml

@@ -25,7 +25,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
         with:
           go-version-file: go.mod
           cache: false

.github/workflows/test.yml

@@ -80,7 +80,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -153,13 +153,13 @@ jobs:
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
     - name: Install Go
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
       with:
         go-version-file: go.mod
         cache: false
 
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -260,7 +260,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -319,7 +319,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -367,7 +367,7 @@ jobs:
     - name: checkout
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Restore Cache
-      uses: actions/cache@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0
+      uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
       with:
         # Note: unlike the other setups, this is only grabbing the mod download
         # cache, rather than the whole mod directory, as the download cache
@@ -461,7 +461,7 @@ jobs:
       run: |
         echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
     - name: upload crash
-      uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
       if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
       with:
         name: artifacts

cmd/containerboot/main.go

@@ -102,7 +102,6 @@ import (
 	"net/netip"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -731,7 +730,6 @@ func tailscaledConfigFilePath() string {
 		}
 		cv, err := kubeutils.CapVerFromFileName(e.Name())
 		if err != nil {
-			log.Printf("skipping file %q in tailscaled config directory %q: %v", e.Name(), dir, err)
 			continue
 		}
 		if cv > maxCompatVer && cv <= tailcfg.CurrentCapabilityVersion {
@@ -739,8 +737,9 @@ func tailscaledConfigFilePath() string {
 		}
 	}
 	if maxCompatVer == -1 {
-		log.Fatalf("no tailscaled config file found in %q for current capability version %q", dir, tailcfg.CurrentCapabilityVersion)
+		log.Fatalf("no tailscaled config file found in %q for current capability version %d", dir, tailcfg.CurrentCapabilityVersion)
 	}
-	log.Printf("Using tailscaled config file %q for capability version %q", maxCompatVer, tailcfg.CurrentCapabilityVersion)
-	return path.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	filePath := filepath.Join(dir, kubeutils.TailscaledConfigFileName(maxCompatVer))
+	log.Printf("Using tailscaled config file %q to match current capability version %d", filePath, tailcfg.CurrentCapabilityVersion)
+	return filePath
 }

cmd/k8s-operator/operator_test.go

@@ -1388,7 +1388,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		parentType:      "svc",
 		hostname:        "default-test",
 		clusterTargetIP: "10.20.30.40",
-		confFileHash:    "362360188dac62bca8013c8134929fed8efd84b1f410c00873d14a05709b5647",
+		confFileHash:    "a67b5ad3ff605531c822327e8f1a23dd0846e1075b722c13402f7d5d0ba32ba2",
 		app:             kubetypes.AppIngressProxy,
 	}
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
@@ -1399,7 +1399,7 @@ func TestTailscaledConfigfileHash(t *testing.T) {
 		mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
 	})
 	o.hostname = "another-test"
-	o.confFileHash = "20db57cfabc3fc6490f6bb1dc85994e61d255cdfa2a56abb0141736e59f263ef"
+	o.confFileHash = "888a993ebee20ad6be99623b45015339de117946850cf1252bede0b570e04293"
 	expectReconciled(t, sr, "default", "test")
 	expectEqual(t, fc, expectedSTS(t, fc, o), nil)
 }

cmd/k8s-operator/sts.go

@@ -521,11 +521,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_KUBE_SECRET",
 			Value: proxySecret,
 		},
-		corev1.EnvVar{
-			// Old tailscaled config key is still used for backwards compatibility.
-			Name:  "EXPERIMENTAL_TS_CONFIGFILE_PATH",
-			Value: "/etc/tsconfig/tailscaled",
-		},
 		corev1.EnvVar{
 			// New style is in the form of cap-<capability-version>.hujson.
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
@@ -789,15 +784,9 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) {
 	return origConf.AuthKey, nil
 }
 
-// tailscaledConfig takes a proxy config, a newly generated auth key if
-// generated and a Secret with the previous proxy state and auth key and
-// returns tailscaled configuration and a hash of that configuration.
-//
-// As of 2024-05-09 it also returns legacy tailscaled config without the
-// later added NoStatefulFilter field to support proxies older than cap95.
-// TODO (irbekrm): remove the legacy config once we no longer need to support
-// versions older than cap94,
-// https://tailscale.com/kb/1236/kubernetes-operator#operator-and-proxies
+// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
+// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that
+// configuration.
 func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
 	conf := &ipn.ConfigVAlpha{
 		Version:             "alpha0",
@@ -846,10 +835,6 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
 	// AppConnector config option is only understood by clients of capver 107 and newer.
 	conf.AppConnector = nil
 	capVerConfigs[95] = *conf
-
-	// StatefulFiltering is only understood by clients of capver 95 and newer.
-	conf.NoStatefulFiltering.Clear()
-	capVerConfigs[94] = *conf
 	return capVerConfigs, nil
 }
 

cmd/k8s-operator/testutils_test.go

@@ -71,7 +71,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			{Name: "TS_USERSPACE", Value: "false"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 		},
 		SecurityContext: &corev1.SecurityContext{
@@ -230,7 +229,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "TS_USERSPACE", Value: "true"},
 			{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: opts.secretName},
-			{Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH", Value: "/etc/tsconfig/tailscaled"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
 			{Name: "TS_INTERNAL_APP", Value: opts.app},
@@ -404,12 +402,6 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
 	if err != nil {
 		t.Fatalf("error marshalling tailscaled config")
 	}
-	conf.NoStatefulFiltering.Clear()
-	b, err := json.Marshal(conf)
-	if err != nil {
-		t.Fatalf("error marshalling tailscaled config")
-	}
-	mak.Set(&s.StringData, "tailscaled", string(b))
 	mak.Set(&s.StringData, "cap-95.hujson", string(bn))
 	mak.Set(&s.StringData, "cap-107.hujson", string(bnn))
 	labels := map[string]string{
@@ -662,18 +654,6 @@ func removeTargetPortsFromSvc(svc *corev1.Service) {
 func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
 	return func(secret *corev1.Secret) {
 		t.Helper()
-		if len(secret.StringData["tailscaled"]) != 0 {
-			conf := &ipn.ConfigVAlpha{}
-			if err := json.Unmarshal([]byte(secret.StringData["tailscaled"]), conf); err != nil {
-				t.Fatalf("error unmarshalling 'tailscaled' contents: %v", err)
-			}
-			conf.AuthKey = nil
-			b, err := json.Marshal(conf)
-			if err != nil {
-				t.Fatalf("error marshalling updated 'tailscaled' config: %v", err)
-			}
-			mak.Set(&secret.StringData, "tailscaled", string(b))
-		}
 		if len(secret.StringData["cap-95.hujson"]) != 0 {
 			conf := &ipn.ConfigVAlpha{}
 			if err := json.Unmarshal([]byte(secret.StringData["cap-95.hujson"]), conf); err != nil {

cmd/tailscale/cli/cli.go

@@ -93,8 +93,13 @@ func Run(args []string) (err error) {
 
 	args = CleanUpArgs(args)
 
-	if len(args) == 1 && (args[0] == "-V" || args[0] == "--version") {
-		args = []string{"version"}
+	if len(args) == 1 {
+		switch args[0] {
+		case "-V", "--version":
+			args = []string{"version"}
+		case "help":
+			args = []string{"--help"}
+		}
 	}
 
 	var warnOnce sync.Once

cmd/tailscale/cli/cli_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"net/netip"
 	"reflect"
 	"strings"
@@ -1480,3 +1481,33 @@ func TestParseNLArgs(t *testing.T) {
 		})
 	}
 }
+
+func TestHelpAlias(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	tstest.Replace[io.Writer](t, &Stdout, &stdout)
+	tstest.Replace[io.Writer](t, &Stderr, &stderr)
+
+	gotExit0 := false
+	defer func() {
+		if !gotExit0 {
+			t.Error("expected os.Exit(0) to be called")
+			return
+		}
+		if !strings.Contains(stderr.String(), "SUBCOMMANDS") {
+			t.Errorf("expected help output to contain SUBCOMMANDS; got stderr=%q; stdout=%q", stderr.String(), stdout.String())
+		}
+	}()
+	defer func() {
+		if e := recover(); e != nil {
+			if strings.Contains(fmt.Sprint(e), "unexpected call to os.Exit(0)") {
+				gotExit0 = true
+			} else {
+				t.Errorf("unexpected panic: %v", e)
+			}
+		}
+	}()
+	err := Run([]string{"help"})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}

control/controlclient/direct.go

@@ -564,12 +564,6 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 	case opt.URL != "":
 		// Nothing.
 	case regen || persist.PrivateNodeKey.IsZero():
-		if regen {
-			c.logf("TEST: need to regenerate")
-		} else {
-			c.logf("TEST: private node key is zero, persist is %v", persist)
-			c.logf("TEST: private node key is zero, persist is %v", persist)
-		}
 		c.logf("Generating a new nodekey.")
 		persist.OldPrivateNodeKey = persist.PrivateNodeKey
 		tryingNewKey = key.NewNode()

control/controlclient/noise.go

@@ -17,7 +17,6 @@ import (
 
 	"golang.org/x/net/http2"
 	"tailscale.com/control/controlhttp"
-	"tailscale.com/envknob"
 	"tailscale.com/health"
 	"tailscale.com/internal/noiseconn"
 	"tailscale.com/net/dnscache"
@@ -30,7 +29,6 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/multierr"
 	"tailscale.com/util/singleflight"
-	"tailscale.com/util/testenv"
 )
 
 // NoiseClient provides a http.Client to connect to tailcontrol over
@@ -107,11 +105,6 @@ type NoiseOpts struct {
 	DialPlan func() *tailcfg.ControlDialPlan
 }
 
-// controlIsPlaintext is whether we should assume that the controlplane is only accessible
-// over plaintext HTTP (as the first hop, before the ts2021 encryption begins).
-// This is used by some tests which don't have a real TLS certificate.
-var controlIsPlaintext = envknob.RegisterBool("TS_CONTROL_IS_PLAINTEXT_HTTP")
-
 // NewNoiseClient returns a new noiseClient for the provided server and machine key.
 // serverURL is of the form https://<host>:<port> (no trailing slash).
 //
@@ -129,7 +122,7 @@ func NewNoiseClient(opts NoiseOpts) (*NoiseClient, error) {
 		if u.Scheme == "http" {
 			httpPort = port
 			httpsPort = "443"
-			if (testenv.InTest() || controlIsPlaintext()) && (u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost") {
+			if u.Hostname() == "127.0.0.1" || u.Hostname() == "localhost" {
 				httpsPort = ""
 			}
 		} else {

go.toolchain.rev

@@ -1 +1 @@
-bf15628b759344c6fc7763795a405ba65b8be5d7
+96578f73d04e1a231fa2a495ad3fa97747785bc6

ipn/backend.go

@@ -73,6 +73,15 @@ const (
 	NotifyInitialOutgoingFiles // if set, the first Notify message (sent immediately) will contain the current Taildrop OutgoingFiles
 
 	NotifyInitialHealthState // if set, the first Notify message (sent immediately) will contain the current health.State of the client
+
+	NotifyRateLimitNetmaps // if set, rate limit netmap updates to once every DefaultNetmapRateLimit seconds
+)
+
+const (
+	// This is the minimum time between netmap updates when NotifyRateLimitNetmaps is included in the Notify opts.
+	// 3 seconds should be sufficient to avoid flooding the UI with netmap updates on large/chatty tailnets without
+	// causing noticable issues with the UI being out of date.
+	DefaultNetmapRateLimit = time.Duration(3 * time.Second)
 )
 
 // Notify is a communication from a backend (e.g. tailscaled) to a frontend

ipn/ipnlocal/local.go

@@ -82,6 +82,7 @@ import (
 	"tailscale.com/tka"
 	"tailscale.com/tsd"
 	"tailscale.com/tstime"
+	"tailscale.com/tstime/rate"
 	"tailscale.com/types/appctype"
 	"tailscale.com/types/dnstype"
 	"tailscale.com/types/empty"
@@ -370,6 +371,16 @@ type LocalBackend struct {
 	// backend is healthy and captive portal detection is not required
 	// (sending false).
 	needsCaptiveDetection chan bool
+
+	// netmapRateLimiter rate limits netmap updates to to the IPN bus.
+	// It should be nil if the ipn bus options do not include the rate limiting flag.
+	// It is automatically created via setNetmapRateLimit.
+	netmapRateLimiter *rate.Limiter
+
+	// deferredNetmapCancel is used to cancel deferred netmap updates which
+	// were initially blocked due to rate limiting.  We always attempt to send the latest
+	// netmap once the rate limiter allows it, discarding any pending netmaps.
+	deferredNetmapCancel context.CancelFunc
 }
 
 // HealthTracker returns the health tracker for the backend.
@@ -475,6 +486,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
 		captiveCtx:            captiveCtx,
 		captiveCancel:         nil, // so that we start checkCaptivePortalLoop when Running
 		needsCaptiveDetection: make(chan bool),
+		deferredNetmapCancel:  nil,
 	}
 	mConn.SetNetInfoCallback(b.setNetInfo)
 
@@ -965,6 +977,11 @@ func (b *LocalBackend) Shutdown() {
 	if b.notifyCancel != nil {
 		b.notifyCancel()
 	}
+
+	if b.deferredNetmapCancel != nil {
+		b.deferredNetmapCancel()
+	}
+
 	b.mu.Unlock()
 	b.webClientShutdown()
 
@@ -1591,8 +1608,7 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control
 
 		// Update the DERP map in the health package, which uses it for health notifications
 		b.health.SetDERPMap(st.NetMap.DERPMap)
-
-		b.send(ipn.Notify{NetMap: st.NetMap})
+		b.sendNetmap(st.NetMap)
 	}
 	if st.URL != "" {
 		b.logf("Received auth URL: %.20v...", st.URL)
@@ -1677,20 +1693,91 @@ func applySysPolicy(prefs *ipn.Prefs) (anyChange bool) {
 	return anyChange
 }
 
+// setNetmapRateLimit Sets the minimum interval between netmap updates on the IPN Bus (in seconds)
+// If interval is 0 or negative, the rate limiter is disabled.  Netmap rate limiting is
+// disabled by default
+// b.mu must be held
+func (b *LocalBackend) setNetmapRateLimit(interval time.Duration) {
+	if interval > 0 {
+		b.netmapRateLimiter = rate.NewLimiter(rate.Every(interval), 1)
+	} else {
+		b.netmapRateLimiter = nil
+	}
+}
+
+// sendNetmap sends a netmap update to the IPN bus respecting the rate limiter.  This function
+// should be used for all netmap updates on the IPN bus unless there is some critical reason that
+// a netmap update be sent immediately.
+//
+// A non-nil channel will be returned if the netmap update was deferred due to rate limiting.  The channel will be closed
+// when the netmap update is handled.  true or false will be sent to the channel to indicate whether
+// or not the netmap was sent or cancelled respectively.  A nil return value indicates that the netmap
+// was sent immediately.  The returned value is primarily useful for testing and you can safely ignore
+// it and just call this method at will.
+func (b *LocalBackend) sendNetmap(nm *netmap.NetworkMap) chan bool {
+	notify := ipn.Notify{NetMap: nm}
+
+	b.mu.Lock()
+
+	// Cancel all pending netmap updates, they're stale and we have something newer
+	if b.deferredNetmapCancel != nil {
+		b.deferredNetmapCancel()
+	}
+
+	// No rate limiter?  Send it.
+	// Rate limiter allows the send?  Send it.
+	if b.netmapRateLimiter == nil || b.netmapRateLimiter.Allow() {
+		b.mu.Unlock()
+		b.send(notify)
+		return nil
+	}
+
+	// We're rate limited.  Defer the netmap update
+	var ctx context.Context
+	ctx, cancel := context.WithCancel(b.ctx)
+	b.deferredNetmapCancel = cancel
+	// The rate limiter is set to Limit() events per second.  Convert that back to
+	// the time interval we need to wait
+	delay := b.netmapRateLimiter.Delay()
+	b.mu.Unlock()
+
+	c := make(chan bool)
+
+	// Send the netmap update once the rate limiter allows it
+	go func() {
+		select {
+		case <-time.After(delay):
+			b.send(notify)
+			c <- true
+		case <-ctx.Done():
+			c <- false
+		}
+		close(c)
+	}()
+	return c
+}
+
 var _ controlclient.NetmapDeltaUpdater = (*LocalBackend)(nil)
 
-// UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater.
 func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) {
 	if !b.MagicConn().UpdateNetmapDelta(muts) {
 		return false
 	}
 
-	var notify *ipn.Notify // non-nil if we need to send a Notify
+	// Will be sent if non nil
+	var netmap *netmap.NetworkMap
+	// Will send an empty notify if true and netmap is nil (for tests - see below)
+	var sendEmpty = false
+
 	defer func() {
-		if notify != nil {
+		if netmap != nil {
+			b.sendNetmap(netmap)
+		} else if sendEmpty {
+			notify := new(ipn.Notify)
 			b.send(*notify)
 		}
 	}()
+
 	unlock := b.lockAndGetUnlock()
 	defer unlock()
 	if !b.updateNetmapDeltaLocked(muts) {
@@ -1712,13 +1799,14 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo
 		slices.SortFunc(nm.Peers, func(a, b tailcfg.NodeView) int {
 			return cmp.Compare(a.ID(), b.ID())
 		})
-		notify = &ipn.Notify{NetMap: nm}
+		netmap = nm
 	} else if testenv.InTest() {
 		// In tests, send an empty Notify as a wake-up so end-to-end
 		// integration tests in another repo can check on the status of
 		// LocalBackend after processing deltas.
-		notify = new(ipn.Notify)
+		sendEmpty = true
 	}
+
 	return true
 }
 
@@ -1995,7 +2083,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	defer unlock()
 
 	if opts.UpdatePrefs != nil {
-		log.Printf("TESTPREFS: update prefs non-nil")
 		if err := b.checkPrefsLocked(opts.UpdatePrefs); err != nil {
 			return err
 		}
@@ -2062,10 +2149,6 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 	}
 
 	prefs := b.pm.CurrentPrefs()
-	log.Printf("TESTPREFS persistent prefs: %v", prefs.Persist())
-	if s := prefs.Persist().AsStruct(); s != nil {
-		log.Printf("TESTPREFS persistent prefs private key is %v", s.PrivateNodeKey)
-	}
 	wantRunning := prefs.WantRunning()
 	if wantRunning {
 		if err := b.initMachineKeyLocked(); err != nil {
@@ -2754,6 +2837,12 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 		cancel:    cancel,
 	}
 	mak.Set(&b.notifyWatchers, sessionID, session)
+	if mask&ipn.NotifyRateLimitNetmaps != 0 {
+		b.setNetmapRateLimit(ipn.DefaultNetmapRateLimit)
+	} else {
+		b.setNetmapRateLimit(0)
+	}
+
 	b.mu.Unlock()
 
 	defer func() {
@@ -4994,6 +5083,9 @@ func (b *LocalBackend) enterStateLockedOnEntry(newState ipn.State, unlock unlock
 		if authURL == "" {
 			systemd.Status("Stopped; run 'tailscale up' to log in")
 		}
+		if b.deferredNetmapCancel != nil {
+			b.deferredNetmapCancel()
+		}
 	case ipn.Starting, ipn.NeedsMachineAuth:
 		b.authReconfig()
 		// Needed so that UpdateEndpoints can run
@@ -5006,7 +5098,9 @@ func (b *LocalBackend) enterStateLockedOnEntry(newState ipn.State, unlock unlock
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrStrs, " "))
 	case ipn.NoState:
-		// Do nothing.
+		if b.deferredNetmapCancel != nil {
+			b.deferredNetmapCancel()
+		}
 	default:
 		b.logf("[unexpected] unknown newState %#v", newState)
 	}
@@ -6783,7 +6877,6 @@ func (b *LocalBackend) CurrentProfile() ipn.LoginProfile {
 
 // NewProfile creates and switches to the new profile.
 func (b *LocalBackend) NewProfile() error {
-	log.Printf("TESTPREFS: NewProfile LB")
 	unlock := b.lockAndGetUnlock()
 	defer unlock()
 

ipn/ipnlocal/local_test.go

@@ -572,6 +572,164 @@ func TestSetUseExitNodeEnabled(t *testing.T) {
 	}
 }
 
+func TestNetmapRateLimiting(t *testing.T) {
+	b := new(LocalBackend)
+	var cancel context.CancelFunc
+	b.ctx, cancel = context.WithCancel(context.Background())
+	b.logf = t.Logf
+	b.setNetmapRateLimit(time.Duration(100 * time.Millisecond))
+
+	if b.netmapRateLimiter == nil {
+		t.Fatalf("no netmapRateLimiter")
+	}
+
+	now := time.Now()
+
+	b.netMap = new(netmap.NetworkMap)
+	if g := b.sendNetmap(b.netMap); g != nil {
+		t.Errorf("First should be immediately sent immediately")
+	}
+
+	// We just sent a netmap, so these should all be rate limited. c1 should get cancelled.
+	// c2 should be cancelled.  c3 should be sent after 100ms.
+	c1 := b.sendNetmap(b.netMap)
+	c2 := b.sendNetmap(b.netMap)
+
+	// Let's spam a bunch more we won't track, just for fun
+	for i := 0; i < 10; i++ {
+		if g := b.sendNetmap(b.netMap); g == nil {
+			t.Errorf("should have been deferred")
+		}
+	}
+
+	// This is our last netmap send. It should be deferred and sent after 100ms.
+	c3 := b.sendNetmap(b.netMap)
+
+	// The first onnetmape should be cancelled
+	select {
+	case sent := <-c1:
+		if sent {
+			t.Errorf("Second netmap update was not cacncelled; sent got %v, want %v", sent, false)
+		}
+	}
+
+	// The second netmap should be cancelled
+	select {
+	case sent := <-c2:
+		if sent {
+			t.Errorf("Second netmap update was not cacncelled; got %v, sent want %v", sent, false)
+		}
+	}
+
+	// The last netmap should be sent after about 100ms
+	select {
+	case sent := <-c3:
+		if !sent {
+			t.Errorf("Fourth netmap update was deferred but not sent; sent got %v, want %v", sent, true)
+		}
+	}
+
+	elapsed := time.Since(now)
+	if elapsed < 90*time.Millisecond {
+		t.Errorf("elapsed time %v is too short", elapsed)
+	}
+	if elapsed > 110*time.Millisecond {
+		t.Errorf("elapsed time %v is too long", elapsed)
+	}
+
+	// The rate limiter should be reset at this point and the next netmap should be sent immediately.
+	if g := b.sendNetmap(b.netMap); g != nil {
+		t.Errorf("netmap should be immediately sent immediately")
+	}
+
+	// We're rate limited - becuase we just sent a netmap.
+	// Lower the rate limit and make sure we can send again once the rate limit is up.
+	b.setNetmapRateLimit(time.Duration(10 * time.Millisecond))
+	time.Sleep(12 * time.Millisecond)
+	if g := b.sendNetmap(b.netMap); g != nil {
+		t.Errorf("netmap should be immediately sent immediately")
+	}
+
+	// Check to make sure the cancellation function is properly set and does what it's
+	// supposed to do.
+	c4 := b.sendNetmap(b.netMap)
+	b.deferredNetmapCancel()
+	select {
+	case sent := <-c4:
+		if sent {
+			t.Errorf("Fourth netmap should have been cancelled; sent got %v, want %v", sent, true)
+		}
+	}
+
+	cancel()
+}
+
+func TestNetmapDeferral(t *testing.T) {
+	b := new(LocalBackend)
+	var cancel context.CancelFunc
+	b.ctx, cancel = context.WithCancel(context.Background())
+	b.logf = t.Logf
+
+	w := 40 * time.Millisecond
+
+	// Ensure that a deferred netmap gets sent with the correct delay
+	b.setNetmapRateLimit(time.Duration(w))
+	start := time.Now()
+	b.sendNetmap(b.netMap)
+
+	// Snooze for 20ms
+	time.Sleep(20 * time.Millisecond)
+
+	// This one should be deferred and sent after ~40-20ms
+	c := b.sendNetmap(b.netMap)
+	select {
+	case sent := <-c:
+		if !sent {
+			t.Errorf("Fourth netmap update was deferred but not sent; sent got %v, want %v", sent, true)
+		}
+	}
+
+	slop := 5 * time.Millisecond
+	g := time.Since(start) * time.Millisecond
+
+	// The difference between the elapsed time and the expected time should be within the slop
+	// and our total time should always be slightly greater than the expected time.
+	if w-g > slop || w > g {
+		t.Errorf("elapsed time is too incorrect w:%v g:%v", w, g)
+	}
+
+	cancel()
+}
+
+func TestNetmapNoRateLimiting(t *testing.T) {
+	b := new(LocalBackend)
+	var cancel context.CancelFunc
+	b.ctx, cancel = context.WithCancel(context.Background())
+	b.logf = t.Logf
+
+	// A zero rate limit means send-at-will
+	b.setNetmapRateLimit(0)
+
+	b.netMap = new(netmap.NetworkMap)
+
+	for i := 0; i < 10; i++ {
+		if g := b.sendNetmap(b.netMap); g != nil {
+			t.Errorf("should be immediately sent immediately")
+		}
+	}
+
+	// A negative rate limit also means send-at-will
+	b.setNetmapRateLimit(-1)
+
+	for i := 0; i < 10; i++ {
+		if g := b.sendNetmap(b.netMap); g != nil {
+			t.Errorf("should be immediately sent immediately")
+		}
+	}
+
+	cancel()
+}
+
 func TestFileTargets(t *testing.T) {
 	b := new(LocalBackend)
 	_, err := b.FileTargets()

ipn/ipnlocal/profiles.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"runtime"
 	"slices"
 	"strings"
@@ -204,7 +203,6 @@ func (pm *profileManager) setUnattendedModeAsConfigured() error {
 
 // Reset unloads the current profile, if any.
 func (pm *profileManager) Reset() {
-	log.Printf("TESTPREFS: Reset")
 	pm.currentUserID = ""
 	pm.NewProfile()
 }
@@ -217,7 +215,6 @@ func (pm *profileManager) Reset() {
 // is logged into so that we can keep track of things like their domain name
 // across user switches to disambiguate the same account but a different tailnet.
 func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile) error {
-	log.Printf("TESTPREFS: SetPrefs with prefs %v", prefsIn)
 	cp := pm.currentProfile
 	if persist := prefsIn.Persist(); !persist.Valid() || persist.NodeID() == "" || persist.UserProfile().LoginName == "" {
 		// We don't know anything about this profile, so ignore it for now.
@@ -226,7 +223,6 @@ func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile)
 
 	// Check if we already have an existing profile that matches the user/node.
 	if existing := pm.findMatchingProfiles(prefsIn); len(existing) > 0 {
-		log.Printf("TESTPREFS: SetPrefs found existing profile")
 		// We already have a profile for this user/node we should reuse it. Also
 		// cleanup any other duplicate profiles.
 		cp = existing[0]
@@ -234,7 +230,6 @@ func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile)
 		for _, p := range existing {
 			// Clear the state.
 			if err := pm.store.WriteState(p.Key, nil); err != nil {
-				log.Printf("TESTPREFS: SetPrefs found existing profile, error writing state: %v", err)
 				// We couldn't delete the state, so keep the profile around.
 				continue
 			}
@@ -242,8 +237,6 @@ func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile)
 			// in [profileManager.setProfilePrefs] below.
 			delete(pm.knownProfiles, p.ID)
 		}
-	} else {
-		log.Printf("TESTPREFS: SetPrefs not found existing profile")
 	}
 	pm.currentProfile = cp
 	if err := pm.SetProfilePrefs(cp, prefsIn, np); err != nil {
@@ -334,7 +327,6 @@ func newUnusedID(knownProfiles map[ipn.ProfileID]*ipn.LoginProfile) (ipn.Profile
 // profile, such as verifying the caller's access rights or checking
 // if another profile for the same node already exists.
 func (pm *profileManager) setProfilePrefsNoPermCheck(profile *ipn.LoginProfile, clonedPrefs ipn.PrefsView) error {
-	log.Printf("TESTPREFS: setProfilePrefsNoPerm")
 	isCurrentProfile := pm.currentProfile == profile
 	if isCurrentProfile {
 		pm.prefs = clonedPrefs
@@ -431,7 +423,6 @@ func (pm *profileManager) profilePrefs(p *ipn.LoginProfile) (ipn.PrefsView, erro
 // If the profile exists but is not accessible to the current user, it returns an [errProfileAccessDenied].
 // If the profile does not exist, it returns an [errProfileNotFound].
 func (pm *profileManager) SwitchProfile(id ipn.ProfileID) error {
-	log.Printf("TESTPREFS: SwitchProfile")
 	metricSwitchProfile.Add(1)
 
 	kp, ok := pm.knownProfiles[id]
@@ -459,7 +450,6 @@ func (pm *profileManager) SwitchProfile(id ipn.ProfileID) error {
 // It creates a new one and switches to it if the current user does not have a default profile,
 // or returns an error if the default profile is inaccessible or could not be loaded.
 func (pm *profileManager) SwitchToDefaultProfile() error {
-	log.Printf("TESTPREFS: SwitchToDefault")
 	if id := pm.DefaultUserProfileID(pm.currentUserID); id != "" {
 		return pm.SwitchProfile(id)
 	}
@@ -557,7 +547,6 @@ func (pm *profileManager) DeleteProfile(id ipn.ProfileID) error {
 }
 
 func (pm *profileManager) deleteCurrentProfile() error {
-	log.Printf("TESTPREFS: deleteCurrent")
 	if err := pm.checkProfileAccess(pm.currentProfile); err != nil {
 		return err
 	}
@@ -638,7 +627,6 @@ func (pm *profileManager) NewProfile() {
 // NewProfileForUser is like [profileManager.NewProfile], but it switches to the
 // specified user and sets that user as the profile owner for the new profile.
 func (pm *profileManager) NewProfileForUser(uid ipn.WindowsUserID) {
-	log.Printf("TESTPREFS: NewProfileForUser")
 	pm.currentUserID = uid
 
 	metricNewProfile.Add(1)
@@ -653,7 +641,6 @@ func (pm *profileManager) NewProfileForUser(uid ipn.WindowsUserID) {
 // newly created profile immediately. It returns the newly created profile on success,
 // or an error on failure.
 func (pm *profileManager) newProfileWithPrefs(uid ipn.WindowsUserID, prefs ipn.PrefsView, switchNow bool) (*ipn.LoginProfile, error) {
-	log.Printf("TESTPREFS: newProfileWithPrefs")
 	metricNewProfile.Add(1)
 
 	profile := &ipn.LoginProfile{LocalUserID: uid}
@@ -746,7 +733,6 @@ func newProfileManagerWithGOOS(store ipn.StateStore, logf logger.Logf, ht *healt
 	if err != nil {
 		return nil, err
 	}
-	log.Printf("TESTPREFS: newProfileWithGOOS with state key %v", stateKey)
 
 	knownProfiles, err := readKnownProfiles(store)
 	if err != nil {
@@ -762,15 +748,12 @@ func newProfileManagerWithGOOS(store ipn.StateStore, logf logger.Logf, ht *healt
 	}
 
 	if stateKey != "" {
-		log.Printf("TESTPREFS: state key %v exists", stateKey)
 		for _, v := range knownProfiles {
-			log.Printf("TESTPREFS: state key %v exists looking at matching profile %s", stateKey, v)
 			if v.Key == stateKey {
 				pm.currentProfile = v
 			}
 		}
 		if pm.currentProfile == nil {
-			log.Printf("TESTPREFS: current profile is nil")
 			if suf, ok := strings.CutPrefix(string(stateKey), "user-"); ok {
 				pm.currentUserID = ipn.WindowsUserID(suf)
 			}
@@ -793,14 +776,12 @@ func newProfileManagerWithGOOS(store ipn.StateStore, logf logger.Logf, ht *healt
 		// uid passed in from the unix tests. The uid's used for Windows tests
 		// and runtime must be valid Windows security identifier structures.
 	} else if len(knownProfiles) == 0 && goos != "windows" && runtime.GOOS != "windows" {
-		log.Printf("TESTPREFS: no known profiles")
 		// No known profiles, try a migration.
 		pm.dlogf("no known profiles; trying to migrate from legacy prefs")
 		if _, err := pm.migrateFromLegacyPrefs(pm.currentUserID, true); err != nil {
 			return nil, err
 		}
 	} else {
-		log.Printf("TESTPREFS: newProfileWithGOOS new profile")
 		pm.NewProfile()
 	}
 

ipn/store/kubestore/store_kube.go

@@ -7,7 +7,6 @@ package kubestore
 import (
 	"context"
 	"fmt"
-	"log"
 	"net"
 	"os"
 	"strings"
@@ -143,7 +142,6 @@ func (s *Store) loadState() error {
 		}
 		return err
 	}
-	log.Printf("TEST: kube store: got secret: %#+v", secret.Data)
 	s.memory.LoadFromMap(secret.Data)
 	return nil
 }

ipn/store/mem/store_mem.go

@@ -7,7 +7,6 @@ package mem
 import (
 	"bytes"
 	"encoding/json"
-	"log"
 	"sync"
 
 	xmaps "golang.org/x/exp/maps"
@@ -33,21 +32,18 @@ func (s *Store) String() string { return "mem.Store" }
 // ReadState implements the StateStore interface.
 // It returns ipn.ErrStateNotExist if the state does not exist.
 func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
-	log.Printf("TEST: ReadState key %v ", id)
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	bs, ok := s.cache[id]
 	if !ok {
 		return nil, ipn.ErrStateNotExist
 	}
-	log.Printf("TEST: ReadState key %v val %v", id, string(bs))
 	return bs, nil
 }
 
 // WriteState implements the StateStore interface.
 // It never returns an error.
 func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
-	log.Printf("TEST: WriteState key %v ", id)
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.cache == nil {
@@ -61,12 +57,10 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) error {
 // Any existing content is cleared, and the provided map is
 // copied into the cache.
 func (s *Store) LoadFromMap(m map[string][]byte) {
-	log.Printf("Store: LoadFromMap")
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	xmaps.Clear(s.cache)
 	for k, v := range m {
-		log.Printf("TEST: setting state key %v %+#v", k, string(v))
 		mak.Set(&s.cache, ipn.StateKey(k), v)
 	}
 	return

k8s-operator/utils.go

@@ -32,9 +32,6 @@ type Records struct {
 // TailscaledConfigFileName returns a tailscaled config file name in
 // format expected by containerboot for the given CapVer.
 func TailscaledConfigFileName(cap tailcfg.CapabilityVersion) string {
-	if cap < 95 {
-		return "tailscaled"
-	}
 	return fmt.Sprintf("cap-%v.hujson", cap)
 }
 

tstime/rate/rate.go

@@ -56,6 +56,29 @@ func NewLimiter(r Limit, b int) *Limiter {
 	return &Limiter{limit: r, burst: float64(b)}
 }
 
+// Limit returns the maximum overall event rate.
+func (lim *Limiter) Limit() Limit {
+	return lim.limit
+}
+
+// Delay returns the approximate minimum duration before sufficient tokens
+// will be available to permit another event.
+func (lim *Limiter) Delay() time.Duration {
+	lim.mu.Lock()
+	defer lim.mu.Unlock()
+
+	// Calculate the new number of tokens available due to the passage of time.
+	elapsed := mono.Now().Sub(lim.last)
+	tokens := lim.tokens + float64(lim.limit)*elapsed.Seconds()
+	if tokens > lim.burst {
+		tokens = lim.burst
+	}
+
+	// Calculate the time until the next token is available.
+	wait := time.Duration((1-tokens)/float64(lim.limit)*1e9) * time.Nanosecond
+	return wait
+}
+
 // Allow reports whether an event may happen now.
 func (lim *Limiter) Allow() bool {
 	return lim.allow(mono.Now())

tstime/rate/rate_test.go

@@ -145,6 +145,31 @@ func TestSimultaneousRequests(t *testing.T) {
 	}
 }
 
+func TestDelay(t *testing.T) {
+	lim := NewLimiter(Every(1*time.Second), 1)
+	// We'll allow for 10 ms of slop to avoid flakiness.
+	// w should always be just slightly greater than d
+	slop := int64(10)
+
+	lim.Allow()
+
+	d := lim.Delay().Milliseconds()
+	w := int64(1000)
+
+	if w-d > slop || d > w {
+		t.Errorf("Delay() = %v want 1000", d)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+	w = 950
+	d = lim.Delay().Milliseconds()
+
+	// ~50 milliseconds will have passed,
+	if w-d > slop || d > w {
+		t.Errorf("Delay() = %v want 950", d)
+	}
+}
+
 func BenchmarkAllowN(b *testing.B) {
 	lim := NewLimiter(Every(1*time.Second), 1)
 	now := mono.Now()

words/scales.txt

@@ -391,3 +391,11 @@ godzilla
 sirius
 vector
 cherimoya
+shilling
+kettle
+kitchen
+fahrenheit
+rankine
+piano
+ruler
+scoville