When deleting a domain, only remove a route if the route is no longer used in routeInfo.

1. Patch prefs handler doesn't add the remote routes to pref if the pref update is also turning appconnector off.
2. Unadvertise routes when StoreRoutes is on but appConnector is turning off.
3. When appConnector is turning on, add a task to recreate and reload routeInfo from store to the queue
3. When shouldStore is changed, remove the state saved for routeInfo (Set to nil)

appc/appconnector.go

@@ -18,6 +18,8 @@ import (
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/appc/routeinfo"
+	"tailscale.com/ipn"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
 	"tailscale.com/util/dnsname"
@@ -34,6 +36,10 @@ type RouteAdvertiser interface {
 
 	// UnadvertiseRoute removes any matching route advertisements.
 	UnadvertiseRoute(...netip.Prefix) error
+
+	// Store/ReadRouteInfo persists and retreives RouteInfo to stable storage
+	StoreRouteInfo(*routeinfo.RouteInfo) error
+	ReadRouteInfo() (*routeinfo.RouteInfo, error)
 }
 
 // AppConnector is an implementation of an AppConnector that performs
@@ -62,15 +68,27 @@ type AppConnector struct {
 	// wildcards is the list of domain strings that match subdomains.
 	wildcards []string
 
+	// the in memory copy of all the routes that's advertised
+	routeInfo *routeinfo.RouteInfo
+
 	// queue provides ordering for update operations
 	queue execqueue.ExecQueue
+
+	// whether this tailscaled should persist routes. Storing RouteInfo enables the app connector
+	// to forget routes when appropriate and should make routes smaller. While we are verifying that
+	// writing the RouteInfo to StateStore is a good solution (and doesn't for example cause writes
+	// that are too frequent or too large) use a controlknob to manage this flag.
+	ShouldStoreRoutes bool
 }
 
 // NewAppConnector creates a new AppConnector.
-func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser) *AppConnector {
+func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, shouldStoreRoutes bool) *AppConnector {
+	// TODO(fran) if !shouldStoreRoutes we probably want to try and clean up any stored routes
 	return &AppConnector{
-		logf:            logger.WithPrefix(logf, "appc: "),
-		routeAdvertiser: routeAdvertiser,
+		logf:              logger.WithPrefix(logf, "appc: "),
+		routeAdvertiser:   routeAdvertiser,
+		ShouldStoreRoutes: shouldStoreRoutes,
+		routeInfo:         nil,
 	}
 }
 
@@ -84,6 +102,57 @@ func (e *AppConnector) UpdateDomainsAndRoutes(domains []string, routes []netip.P
 	})
 }
 
+func (e *AppConnector) RouteInfo() *routeinfo.RouteInfo {
+	if e.routeInfo == nil {
+		ret, err := e.routeAdvertiser.ReadRouteInfo()
+		if err != nil {
+			if err != ipn.ErrStateNotExist {
+				e.logf("Unsuccessful Read RouteInfo: ", err)
+			}
+			return routeinfo.NewRouteInfo()
+		}
+		return ret
+	}
+	return e.routeInfo
+}
+
+// RecreateRouteInfoFromStore searches from presist store for existing routeInfo for current profile,
+// then update local routes of routeInfo in store as the current advertised routes.
+func (e *AppConnector) RecreateRouteInfoFromStore(localRoutes []netip.Prefix) {
+	e.queue.Add(func() {
+		ri := e.RouteInfo()
+		ri.Local = localRoutes
+		err := e.routeAdvertiser.StoreRouteInfo(ri)
+		if err != nil {
+			e.logf("Appc recreate routeInfo: Error updating routeInfo in store: ", err)
+		}
+		err = e.routeAdvertiser.AdvertiseRoute(ri.Routes(false, true, true)...)
+		if err != nil {
+			e.logf("Appc recreate routeInfo: Error advertise routes: ", err)
+		}
+		e.routeInfo = ri
+	})
+}
+
+// UpdateRouteInfo updates routeInfo value of the AppConnector and
+// stores the routeInfo value to presist store.
+func (e *AppConnector) UpdateRouteInfo(ri *routeinfo.RouteInfo) {
+	err := e.routeAdvertiser.StoreRouteInfo(ri)
+	if err != nil {
+		e.logf("Appc: Failed to remove routeInfo from store: ", err)
+	}
+	e.routeInfo = ri
+}
+
+func (e *AppConnector) UnadvertiseRemoteRoutes() {
+	e.queue.Add(func() {
+		toRemove := e.RouteInfo().Routes(false, true, true)
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise routes %v: %v", toRemove, err)
+		}
+	})
+}
+
 // UpdateDomains asynchronously replaces the current set of configured domains
 // with the supplied set of domains. Domains must not contain a trailing dot,
 // and should be lower case. If the domain contains a leading '*' label it
@@ -106,6 +175,13 @@ func (e *AppConnector) updateDomains(domains []string) {
 
 	var oldDomains map[string][]netip.Addr
 	oldDomains, e.domains = e.domains, make(map[string][]netip.Addr, len(domains))
+	var oldDiscovered map[string]*routeinfo.DatedRoutes
+	var routeInfo *routeinfo.RouteInfo
+	shouldStoreRoutes := e.ShouldStoreRoutes
+	if shouldStoreRoutes {
+		routeInfo = e.RouteInfo()
+		oldDiscovered, routeInfo.Discovered = routeInfo.Discovered, make(map[string]*routeinfo.DatedRoutes, len(domains))
+	}
 	e.wildcards = e.wildcards[:0]
 	for _, d := range domains {
 		d = strings.ToLower(d)
@@ -118,6 +194,10 @@ func (e *AppConnector) updateDomains(domains []string) {
 		}
 		e.domains[d] = oldDomains[d]
 		delete(oldDomains, d)
+		if shouldStoreRoutes {
+			routeInfo.Discovered[d] = oldDiscovered[d]
+			delete(oldDiscovered, d)
+		}
 	}
 
 	// Ensure that still-live wildcards addresses are preserved as well.
@@ -129,6 +209,39 @@ func (e *AppConnector) updateDomains(domains []string) {
 			}
 		}
 	}
+	if shouldStoreRoutes {
+		for d, dr := range oldDiscovered {
+			for _, wc := range e.wildcards {
+				if dnsname.HasSuffix(d, wc) {
+					routeInfo.Discovered[d] = dr
+					delete(oldDiscovered, d)
+					break
+				}
+			}
+		}
+	}
+
+	if shouldStoreRoutes {
+		e.UpdateRouteInfo(routeInfo)
+		// every domain left in oldDiscovered won't be in e.domains
+		// routes can be unadvertised if it's not in local, control, or new discovered
+		currentRoutes := routeInfo.Routes(true, true, true)
+		slices.SortFunc(currentRoutes, comparePrefix)
+		currentRoutes = slices.Compact(currentRoutes)
+		for domainName, domainsRoutes := range oldDiscovered {
+			if domainsRoutes != nil {
+				toRemove := []netip.Prefix{}
+				for _, route := range domainsRoutes.RoutesSlice() {
+					_, ok := slices.BinarySearchFunc(currentRoutes, route, comparePrefix)
+					if !ok {
+						toRemove = append(toRemove, route)
+					}
+				}
+				e.logf("unadvertising %d routes for domain: %s", len(toRemove), domainName)
+				e.scheduleUnadvertisement(domainName, toRemove...)
+			}
+		}
+	}
 	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
 }
 
@@ -145,12 +258,42 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 		return
 	}
 
+	var toRemove []netip.Prefix
+	var routeInfo *routeinfo.RouteInfo
+	var err error
+	if e.ShouldStoreRoutes {
+		routeInfo, err = e.routeAdvertiser.ReadRouteInfo()
+		if err != nil {
+			if err != ipn.ErrStateNotExist {
+				e.logf("Appc: Unsuccessful Read RouteInfo: ", err)
+			}
+			routeInfo = routeinfo.NewRouteInfo()
+		}
+		oldControl := routeInfo.Control
+		routeInfo.Control = routes
+		e.routeInfo = routeInfo
+		e.routeAdvertiser.StoreRouteInfo(e.routeInfo)
+		oldOtherRoutes := routeInfo.Routes(true, false, true)
+		for _, ipp := range oldControl {
+			if slices.Contains(routes, ipp) {
+				continue
+			}
+			// unadvertise the prefix if the prefix is not recorded from other source.
+			if !slices.Contains(oldOtherRoutes, ipp) {
+				toRemove = append(toRemove, ipp)
+			}
+		}
+
+		if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
+			e.logf("failed to unadvertise old routes: %v: %v", routes, err)
+		}
+	}
 	if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
 		e.logf("failed to advertise routes: %v: %v", routes, err)
 		return
 	}
 
-	var toRemove []netip.Prefix
+	toRemove = toRemove[:0]
 
 nextRoute:
 	for _, r := range routes {
@@ -168,7 +311,6 @@ nextRoute:
 	if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
 		e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
 	}
-
 	e.controlRoutes = routes
 }
 
@@ -305,6 +447,11 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 		}
 
 		e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
+		if e.ShouldStoreRoutes && len(toAdvertise) != 0 {
+			routeInfo := e.RouteInfo()
+			routeInfo.AddRoutesInDiscoveredForDomain(domain, toAdvertise)
+			e.UpdateRouteInfo(routeInfo)
+		}
 		e.scheduleAdvertisement(domain, toAdvertise...)
 	}
 }
@@ -383,6 +530,27 @@ func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Pref
 	})
 }
 
+func (e *AppConnector) scheduleUnadvertisement(domain string, routes ...netip.Prefix) {
+	e.queue.Add(func() {
+		if err := e.routeAdvertiser.UnadvertiseRoute(routes...); err != nil {
+			e.logf("failed to unadvertise routes for %s: %v: %v", domain, routes, err)
+			return
+		}
+		e.mu.Lock()
+		defer e.mu.Unlock()
+
+		for _, route := range routes {
+			if !route.IsSingleIP() {
+				continue
+			}
+			addr := route.Addr()
+
+			//e.deleteDomainAddrLocked(domain, addr)
+			e.logf("[v2] unadvertised route for %v: %v", domain, addr)
+		}
+	})
+}
+
 // hasDomainAddrLocked returns true if the address has been observed in a
 // resolution of domain.
 func (e *AppConnector) hasDomainAddrLocked(domain string, addr netip.Addr) bool {
@@ -400,3 +568,7 @@ func (e *AppConnector) addDomainAddrLocked(domain string, addr netip.Addr) {
 func compareAddr(l, r netip.Addr) int {
 	return l.Compare(r)
 }
+
+func comparePrefix(i, j netip.Prefix) int {
+	return i.Addr().Compare(j.Addr())
+}

appc/appconnector_test.go

@@ -9,17 +9,88 @@ import (
 	"reflect"
 	"slices"
 	"testing"
+	"time"
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
 
-func TestUpdateDomains(t *testing.T) {
+func TestUpdateDomainsWithPresistStore(t *testing.T) {
+
 	ctx := context.Background()
-	a := NewAppConnector(t.Logf, nil)
+	a := NewAppConnector(t.Logf, nil, true)
+	TestRouteAdvertiser := &appctest.RouteCollector{}
+	a.routeAdvertiser = TestRouteAdvertiser
+	prefixControl := netip.MustParsePrefix("192.0.0.8/32")
+	prefixLocal := netip.MustParsePrefix("192.0.0.9/32")
+	prefixDiscovered := netip.MustParsePrefix("192.0.0.10/32")
+	prefixShouldUnadvertise := netip.MustParsePrefix("192.0.0.11/32")
+	ri := routeinfo.NewRouteInfo()
+	ri.Control = []netip.Prefix{prefixControl}
+	ri.Local = []netip.Prefix{prefixLocal}
+	a.UpdateRouteInfo(ri)
+	a.UpdateDomains([]string{"example.com", "test.com"})
+
+	a.Wait(ctx)
+	if got, want := a.Domains().AsSlice(), []string{"example.com", "test.com"}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+	if _, hasKey := a.routeInfo.Discovered["example.com"]; !hasKey {
+		t.Errorf("Discovered did not record the test domain.")
+	}
+
+	now := time.Now()
+	testRoutes := make(map[netip.Prefix]time.Time)
+	testRoutes[prefixDiscovered] = now
+	exampleRoutes := make(map[netip.Prefix]time.Time)
+	exampleRoutes[prefixShouldUnadvertise] = now
+	exampleRoutes[prefixControl] = now
+	exampleRoutes[prefixLocal] = now
+	exampleRoutes[prefixDiscovered] = now
+	ri.Discovered["test.com"] = &routeinfo.DatedRoutes{Routes: testRoutes, LastCleanup: now}
+	ri.Discovered["example.com"] = &routeinfo.DatedRoutes{Routes: exampleRoutes, LastCleanup: now}
+	a.UpdateRouteInfo(ri)
+	TestRouteAdvertiser.SetRoutes(ri.Discovered["example.com"].RoutesSlice())
+
+	addrControl := netip.MustParseAddr("192.0.0.8")
+	addrLocal := netip.MustParseAddr("192.0.0.9")
+	addrDiscovered := netip.MustParseAddr("192.0.0.10")
+	addrShouldUnadvertise := netip.MustParseAddr("192.0.0.11")
+	a.domains["example.com"] = append(a.domains["example.com"], []netip.Addr{addrControl, addrLocal, addrDiscovered, addrShouldUnadvertise}...)
+	a.domains["test.com"] = append(a.domains["test.com"], addrDiscovered)
+	a.UpdateDomains([]string{"example.com", "test.com"})
+	a.Wait(ctx)
+
+	if got, want := a.domains["example.com"], []netip.Addr{addrControl, addrLocal, addrDiscovered, addrShouldUnadvertise}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+	// domains are explicitly downcased on set.
+	a.UpdateDomains([]string{"UP.EXAMPLE.COM", "test.com"})
+	a.Wait(ctx)
+	if got, want := xmaps.Keys(a.domains), []string{"up.example.com", "test.com"}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+	if got, want := TestRouteAdvertiser.RemovedRoutes(), []netip.Prefix{prefixShouldUnadvertise}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+	if got, want := TestRouteAdvertiser.Routes(), []netip.Prefix{prefixControl, prefixLocal, prefixDiscovered}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+}
+
+func TestUpdateDomainsWithoutPresistStore(t *testing.T) {
+	ctx := context.Background()
+	a := NewAppConnector(t.Logf, nil, false)
+	a.routeAdvertiser = &appctest.RouteCollector{}
+	a.UpdateRouteInfo(routeinfo.NewRouteInfo())
 	a.UpdateDomains([]string{"example.com"})
 
 	a.Wait(ctx)
@@ -45,166 +116,216 @@ func TestUpdateDomains(t *testing.T) {
 }
 
 func TestUpdateRoutes(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"*.example.com"})
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		a.updateDomains([]string{"*.example.com"})
 
-	// This route should be collapsed into the range
-	a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
-	a.Wait(ctx)
+		// This route should be collapsed into the range
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		a.Wait(ctx)
 
-	if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
-		t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
+			t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		}
+
+		// This route should not be collapsed or removed
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		a.Wait(ctx)
+
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
+
+		if shouldStore {
+			wantRouteInfoControlRoutes := routes
+			if !slices.Equal(a.routeInfo.Control, wantRouteInfoControlRoutes) {
+				t.Fatalf("got %v, want %v (shouldStore=%t)", a.routeInfo.Control, wantRouteInfoControlRoutes, shouldStore)
+			}
+		} else {
+			if a.routeInfo != nil {
+				t.Fatalf("got %v, want %v (shouldStore=%t)", a.routeInfo.Control, nil, shouldStore)
+			}
+		}
+
+		slices.SortFunc(rc.Routes(), prefixCompare)
+		rc.SetRoutes(slices.Compact(rc.Routes()))
+		slices.SortFunc(routes, prefixCompare)
+
+		// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
+		}
+
+		// Ensure that the contained /32 is removed, replaced by the /24.
+		wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
+}
 
-	// This route should not be collapsed or removed
-	a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
-	a.Wait(ctx)
+func TestUpdateRoutesNotUnadvertiseRoutesFromOtherSources(t *testing.T) {
+	for _, shouldStore := range []bool{true, false} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		testRi := routeinfo.NewRouteInfo()
+		testRi.Local = append(testRi.Local, netip.MustParsePrefix("192.0.2.0/24"))
+		a.routeInfo = testRi
+		rc.StoreRouteInfo(testRi)
 
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
-	a.updateRoutes(routes)
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
+		wantRouteInfoControlRoutes := []netip.Prefix{}
+		if shouldStore {
+			wantRouteInfoControlRoutes = routes
+		}
+		if !slices.Equal(a.routeInfo.Control, wantRouteInfoControlRoutes) {
+			t.Fatalf("got %v, want %v (shouldStore=%t)", a.routeInfo.Control, wantRouteInfoControlRoutes, shouldStore)
+		}
 
-	slices.SortFunc(rc.Routes(), prefixCompare)
-	rc.SetRoutes(slices.Compact(rc.Routes()))
-	slices.SortFunc(routes, prefixCompare)
+		routes2 := []netip.Prefix{netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes2)
 
-	// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
-	}
-
-	// Ensure that the contained /32 is removed, replaced by the /24.
-	wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
-	if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
-		t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		wantRemoved := []netip.Prefix{}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
-	rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
-	a.updateRoutes(routes)
+	for _, shouldStore := range []bool{true, false} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
+		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+		a.updateRoutes(routes)
 
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		}
 	}
 }
 
 func TestDomainRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(context.Background())
+	for _, shouldStore := range []bool{true, false} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(context.Background())
 
-	want := map[string][]netip.Addr{
-		"example.com": {netip.MustParseAddr("192.0.0.8")},
-	}
+		want := map[string][]netip.Addr{
+			"example.com": {netip.MustParseAddr("192.0.0.8")},
+		}
 
-	if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
-		t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
+			t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		}
 	}
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
 
-	// a has no domains configured, so it should not advertise any routes
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a has no domains configured, so it should not advertise any routes
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// matches a routed domain.
-	a.updateDomains([]string{"www.example.com", "example.com"})
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// matches a routed domain.
+		a.updateDomains([]string{"www.example.com", "example.com"})
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// even if only found in the middle of the chain
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// even if only found in the middle of the chain
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// don't re-advertise routes that have already been advertised
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
+		// don't re-advertise routes that have already been advertised
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
 
-	// don't advertise addresses that are already in a control provided route
-	pfx := netip.MustParsePrefix("192.0.2.0/24")
-	a.updateRoutes([]netip.Prefix{pfx})
-	wantRoutes = append(wantRoutes, pfx)
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
-	if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
-		t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		// don't advertise addresses that are already in a control provided route
+		pfx := netip.MustParsePrefix("192.0.2.0/24")
+		a.updateRoutes([]netip.Prefix{pfx})
+		wantRoutes = append(wantRoutes, pfx)
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
+		if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
+			t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		}
 	}
 }
 
 func TestWildcardDomains(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
 
-	a.updateDomains([]string{"*.example.com"})
-	a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
-		t.Errorf("routes: got %v; want %v", got, want)
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com"})
+		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
+			t.Errorf("routes: got %v; want %v", got, want)
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if _, ok := a.domains["foo.example.com"]; !ok {
-		t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if _, ok := a.domains["foo.example.com"]; !ok {
+			t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if len(a.wildcards) != 1 {
-		t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if len(a.wildcards) != 1 {
+			t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		}
 	}
 }
 

appc/appctest/appctest.go

@@ -6,12 +6,16 @@ package appctest
 import (
 	"net/netip"
 	"slices"
+
+	"tailscale.com/appc/routeinfo"
+	"tailscale.com/ipn"
 )
 
 // RouteCollector is a test helper that collects the list of routes advertised
 type RouteCollector struct {
 	routes        []netip.Prefix
 	removedRoutes []netip.Prefix
+	routeInfo     *routeinfo.RouteInfo
 }
 
 func (rc *RouteCollector) AdvertiseRoute(pfx ...netip.Prefix) error {
@@ -32,6 +36,18 @@ func (rc *RouteCollector) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 	return nil
 }
 
+func (rc *RouteCollector) StoreRouteInfo(ri *routeinfo.RouteInfo) error {
+	rc.routeInfo = ri
+	return nil
+}
+
+func (rc *RouteCollector) ReadRouteInfo() (*routeinfo.RouteInfo, error) {
+	if rc.routeInfo == nil {
+		return nil, ipn.ErrStateNotExist
+	}
+	return rc.routeInfo, nil
+}
+
 // RemovedRoutes returns the list of routes that were removed.
 func (rc *RouteCollector) RemovedRoutes() []netip.Prefix {
 	return rc.removedRoutes

appc/routeinfo/routeinfo.go

@@ -0,0 +1,87 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package routeinfo
+
+import (
+	"net/netip"
+	"time"
+)
+
+type RouteInfo struct {
+	// routes set with --advertise-routes
+	Local []netip.Prefix
+	// routes from the 'routes' section of an app connector acl
+	Control []netip.Prefix
+	// routes discovered by observing dns lookups for configured domains
+	Discovered map[string]*DatedRoutes
+}
+
+func NewRouteInfo() *RouteInfo {
+	discovered := make(map[string]*DatedRoutes)
+	return &RouteInfo{
+		Local:      []netip.Prefix{},
+		Control:    []netip.Prefix{},
+		Discovered: discovered,
+	}
+}
+
+// RouteInfo.Routes returns a slice containing all the routes stored from the wanted resources.
+func (ri *RouteInfo) Routes(local, control, discovered bool) []netip.Prefix {
+	var ret []netip.Prefix
+	if local {
+		ret = ri.Local
+	}
+	if control && len(ret) == 0 {
+		ret = ri.Control
+	} else if control {
+		ret = append(ret, ri.Control...)
+	}
+
+	if discovered {
+		for _, dr := range ri.Discovered {
+			if dr != nil {
+				ret = append(ret, dr.RoutesSlice()...)
+			}
+		}
+	}
+	return ret
+}
+
+type DatedRoutes struct {
+	// routes discovered for a domain, and when they were last seen in a dns query
+	Routes map[netip.Prefix]time.Time
+	// the time at which we last expired old routes
+	LastCleanup time.Time
+}
+
+func (dr *DatedRoutes) RoutesSlice() []netip.Prefix {
+	var routes []netip.Prefix
+	for k := range dr.Routes {
+		routes = append(routes, k)
+	}
+	return routes
+}
+
+func (r *RouteInfo) AddRoutesInDiscoveredForDomain(domain string, addrs []netip.Prefix) {
+	dr, hasKey := r.Discovered[domain]
+	if !hasKey || dr == nil || dr.Routes == nil {
+		newDatedRoutes := &DatedRoutes{make(map[netip.Prefix]time.Time), time.Now()}
+		newDatedRoutes.addAddrsToDatedRoute(addrs)
+		r.Discovered[domain] = newDatedRoutes
+		return
+	}
+
+	// kevin comment: we won't see any existing routes here because know addrs are filtered.
+	currentRoutes := r.Discovered[domain]
+	currentRoutes.addAddrsToDatedRoute(addrs)
+	r.Discovered[domain] = currentRoutes
+	return
+}
+
+func (d *DatedRoutes) addAddrsToDatedRoute(addrs []netip.Prefix) {
+	time := time.Now()
+	for _, addr := range addrs {
+		d.Routes[addr] = time
+	}
+}

control/controlknobs/controlknobs.go

@@ -72,6 +72,10 @@ type Knobs struct {
 	// ProbeUDPLifetime is whether the node should probe UDP path lifetime on
 	// the tail end of an active direct connection in magicsock.
 	ProbeUDPLifetime atomic.Bool
+
+	// AppCStoreRoutes is whether the node should store RouteInfo to StateStore
+	// if it's an app connector.
+	AppCStoreRoutes atomic.Bool
 }
 
 // UpdateFromNodeAttributes updates k (if non-nil) based on the provided self
@@ -96,6 +100,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 		forceNfTables                 = has(tailcfg.NodeAttrLinuxMustUseNfTables)
 		seamlessKeyRenewal            = has(tailcfg.NodeAttrSeamlessKeyRenewal)
 		probeUDPLifetime              = has(tailcfg.NodeAttrProbeUDPLifetime)
+		appCStoreRoutes               = has(tailcfg.NodeAttrStoreAppCRoutes)
 	)
 
 	if has(tailcfg.NodeAttrOneCGNATEnable) {
@@ -118,6 +123,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 	k.LinuxForceNfTables.Store(forceNfTables)
 	k.SeamlessKeyRenewal.Store(seamlessKeyRenewal)
 	k.ProbeUDPLifetime.Store(probeUDPLifetime)
+	k.AppCStoreRoutes.Store(appCStoreRoutes)
 }
 
 // AsDebugJSON returns k as something that can be marshalled with json.Marshal
@@ -141,5 +147,6 @@ func (k *Knobs) AsDebugJSON() map[string]any {
 		"LinuxForceNfTables":            k.LinuxForceNfTables.Load(),
 		"SeamlessKeyRenewal":            k.SeamlessKeyRenewal.Load(),
 		"ProbeUDPLifetime":              k.ProbeUDPLifetime.Load(),
+		"AppCStoreRoutes":               k.AppCStoreRoutes.Load(),
 	}
 }

ipn/ipnlocal/local.go

@@ -34,6 +34,7 @@ import (
 	xmaps "golang.org/x/exp/maps"
 	"gvisor.dev/gvisor/pkg/tcpip"
 	"tailscale.com/appc"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/control/controlclient"
@@ -3143,6 +3144,29 @@ func (b *LocalBackend) checkFunnelEnabledLocked(p *ipn.Prefs) error {
 	return nil
 }
 
+func (b *LocalBackend) PatchPrefsHandler(mp *ipn.MaskedPrefs) (ipn.PrefsView, error) {
+	// we believe that for the purpose of figuring out advertisedRoutes setPrefsLockedOnEntry is _only_ called when
+	// up or set is used on the tailscale cli _not_ when we calculate the new advertisedRoutes field.
+	if b.appConnector != nil && b.appConnector.ShouldStoreRoutes && mp.AdvertiseRoutesSet {
+		routeInfo := b.appConnector.RouteInfo()
+		curRoutes := routeInfo.Routes(false, true, true)
+		routeInfo.Local = mp.AdvertiseRoutes
+		b.appConnector.UpdateRouteInfo(routeInfo)
+		// When b.appConnector != nil, AppConnectorSet = true means
+		// The appConnector is turned off, in this case we should not
+		// append the remote routes to mp.AdvertiseRoutes. Appc will be
+		// set to nil first and unadvertise remote routes, but these remote routes
+		// will then be advertised again when the prefs are sent.
+		if !mp.AppConnectorSet {
+			curRoutes = append(curRoutes, mp.AdvertiseRoutes...)
+			slices.SortFunc(curRoutes, func(i, j netip.Prefix) int { return i.Addr().Compare(j.Addr()) })
+			curRoutes = slices.Compact(curRoutes)
+			mp.AdvertiseRoutes = curRoutes
+		}
+	}
+	return b.EditPrefs(mp)
+}
+
 func (b *LocalBackend) EditPrefs(mp *ipn.MaskedPrefs) (ipn.PrefsView, error) {
 	b.mu.Lock()
 	if mp.EggSet {
@@ -3512,13 +3536,27 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i
 		}
 	}()
 
+	shouldAppCStoreRoutesHasChanged := false
+	shouldAppCStoreRoutes := b.ControlKnobs().AppCStoreRoutes.Load()
+	if b.appConnector != nil {
+		shouldAppCStoreRoutesHasChanged = b.appConnector.ShouldStoreRoutes != shouldAppCStoreRoutes
+	}
+
 	if !prefs.AppConnector().Advertise {
+		if b.appConnector != nil && shouldAppCStoreRoutes {
+			b.appConnector.UnadvertiseRemoteRoutes()
+		}
 		b.appConnector = nil
 		return
 	}
 
-	if b.appConnector == nil {
-		b.appConnector = appc.NewAppConnector(b.logf, b)
+	if b.appConnector == nil || shouldAppCStoreRoutesHasChanged {
+		b.appConnector = appc.NewAppConnector(b.logf, b, shouldAppCStoreRoutes)
+		if shouldAppCStoreRoutes {
+			b.appConnector.RecreateRouteInfoFromStore(prefs.AsStruct().AdvertiseRoutes)
+		} else if shouldAppCStoreRoutesHasChanged && !shouldAppCStoreRoutes {
+			b.appConnector.UpdateRouteInfo(nil)
+		}
 	}
 	if nm == nil {
 		return
@@ -6015,6 +6053,49 @@ func (b *LocalBackend) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 	return err
 }
 
+// namespace a key with the profile manager's current profile key, if any
+func namespaceKeyForCurrentProfile(pm *profileManager, key ipn.StateKey) ipn.StateKey {
+	return pm.CurrentProfile().Key + "||" + key
+}
+
+const routeInfoStateStoreKey ipn.StateKey = "_routeInfo"
+
+// StoreRouteInfo implements the appc.RouteAdvertiser interface. It stores
+// RouteInfo to StateStore per profile.
+func (b *LocalBackend) StoreRouteInfo(ri *routeinfo.RouteInfo) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.pm.CurrentProfile().ID == "" {
+		return nil
+	}
+	key := namespaceKeyForCurrentProfile(b.pm, routeInfoStateStoreKey)
+	bs, err := json.Marshal(ri)
+	if err != nil {
+		return err
+	}
+	return b.pm.WriteState(key, bs)
+}
+
+// ReadRouteInfo implements the appc.RouteAdvertiser interface. It reads
+// RouteInfo from StateStore per profile.
+func (b *LocalBackend) ReadRouteInfo() (*routeinfo.RouteInfo, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.pm.CurrentProfile().ID == "" {
+		return &routeinfo.RouteInfo{}, nil
+	}
+	key := namespaceKeyForCurrentProfile(b.pm, routeInfoStateStoreKey)
+	bs, err := b.pm.Store().ReadState(key)
+	ri := &routeinfo.RouteInfo{}
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(bs, ri); err != nil {
+		return nil, err
+	}
+	return ri, nil
+}
+
 // seamlessRenewalEnabled reports whether seamless key renewals are enabled
 // (i.e. we saw our self node with the SeamlessKeyRenewal attr in a netmap).
 // This enables beta functionality of renewing node keys without breaking

ipn/ipnlocal/local_test.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
@@ -1152,13 +1153,15 @@ func TestDNSConfigForNetmapForExitNodeConfigs(t *testing.T) {
 }
 
 func TestOfferingAppConnector(t *testing.T) {
-	b := newTestBackend(t)
-	if b.OfferingAppConnector() {
-		t.Fatal("unexpected offering app connector")
-	}
-	b.appConnector = appc.NewAppConnector(t.Logf, nil)
-	if !b.OfferingAppConnector() {
-		t.Fatal("unexpected not offering app connector")
+	for _, shouldStore := range []bool{true, false} {
+		b := newTestBackend(t)
+		if b.OfferingAppConnector() {
+			t.Fatal("unexpected offering app connector")
+		}
+		b.appConnector = appc.NewAppConnector(t.Logf, nil, shouldStore)
+		if !b.OfferingAppConnector() {
+			t.Fatal("unexpected not offering app connector")
+		}
 	}
 }
 
@@ -1203,21 +1206,23 @@ func TestRouterAdvertiserIgnoresContainedRoutes(t *testing.T) {
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	b := newTestBackend(t)
+	for _, shouldStore := range []bool{true, false} {
+		b := newTestBackend(t)
 
-	// ensure no error when no app connector is configured
-	b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		// ensure no error when no app connector is configured
+		b.ObserveDNSResponse(dnsResponse("example.com.", []string{"192.0.0.8"}))
 
-	rc := &appctest.RouteCollector{}
-	b.appConnector = appc.NewAppConnector(t.Logf, rc)
-	b.appConnector.UpdateDomains([]string{"example.com"})
-	b.appConnector.Wait(context.Background())
+		rc := &appctest.RouteCollector{}
+		b.appConnector = appc.NewAppConnector(t.Logf, rc, shouldStore)
+		b.appConnector.UpdateDomains([]string{"example.com"})
+		b.appConnector.Wait(context.Background())
 
-	b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	b.appConnector.Wait(context.Background())
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Fatalf("got routes %v, want %v", rc.Routes(), wantRoutes)
+		b.ObserveDNSResponse(dnsResponse("example.com.", []string{"192.0.0.8"}))
+		b.appConnector.Wait(context.Background())
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Fatalf("got routes %v, want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 
@@ -1366,38 +1371,41 @@ func routesEqual(t *testing.T, a, b map[dnsname.FQDN][]*dnstype.Resolver) bool {
 }
 
 // dnsResponse is a test helper that creates a DNS response buffer for the given domain and address
-func dnsResponse(domain, address string) []byte {
-	addr := netip.MustParseAddr(address)
+// func dnsResponse(domain, address string) []byte {
+func dnsResponse(domain string, addresses []string) []byte {
 	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{})
 	b.EnableCompression()
 	b.StartAnswers()
-	switch addr.BitLen() {
-	case 32:
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName(domain),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: addr.As4(),
-			},
-		)
-	case 128:
-		b.AAAAResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName(domain),
-				Type:  dnsmessage.TypeAAAA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AAAAResource{
-				AAAA: addr.As16(),
-			},
-		)
-	default:
-		panic("invalid address length")
+	for _, address := range addresses {
+		addr := netip.MustParseAddr(address)
+		switch addr.BitLen() {
+		case 32:
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName(domain),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: addr.As4(),
+				},
+			)
+		case 128:
+			b.AAAAResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName(domain),
+					Type:  dnsmessage.TypeAAAA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AAAAResource{
+					AAAA: addr.As16(),
+				},
+			)
+		default:
+			panic("invalid address length")
+		}
 	}
 	return must.Get(b.Finish())
 }
@@ -2507,3 +2515,479 @@ func TestValidPopBrowserURL(t *testing.T) {
 		})
 	}
 }
+
+func TestReadWriteRouteInfo(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+	now := time.Now()
+	discovered := make(map[string]*routeinfo.DatedRoutes)
+	routes := make(map[netip.Prefix]time.Time)
+	routes[prefix3] = now
+	discovered["example.com"] = &routeinfo.DatedRoutes{
+		LastCleanup: now,
+		Routes:      routes,
+	}
+	b := newTestBackend(t)
+	ri := routeinfo.RouteInfo{
+		Local:      []netip.Prefix{prefix1},
+		Control:    []netip.Prefix{prefix2},
+		Discovered: discovered,
+	}
+	if err := b.StoreRouteInfo(&ri); err != nil {
+		t.Fatal(err)
+	}
+	readRi, err := b.ReadRouteInfo()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(readRi.Local) != 1 || len(readRi.Control) != 1 || len(readRi.Discovered) != 1 {
+		t.Fatal("read Ri expected to be same shape as ri")
+	}
+	if readRi.Local[0] != ri.Local[0] {
+		t.Fatalf("wanted %v, got %v", ri.Local[0], readRi.Local[0])
+	}
+	if readRi.Control[0] != ri.Control[0] {
+		t.Fatalf("wanted %v, got %v", ri.Control[0], readRi.Control[0])
+	}
+	dr := readRi.Discovered["example.com"]
+	if dr.LastCleanup.Compare(now) != 0 {
+		t.Fatalf("wanted %v, got %v", now, dr.LastCleanup)
+	}
+	if len(dr.Routes) != 1 {
+		t.Fatalf("read Ri expected to be same shape as ri")
+	}
+	if dr.Routes[prefix3].Compare(routes[prefix3]) != 0 {
+		t.Fatalf("wanted %v, got %v", routes[prefix3], dr.Routes[prefix3])
+	}
+}
+
+func TestPatchPrefsHandlerWithPresistStore(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+	mp := new(ipn.MaskedPrefs)
+	mp.AdvertiseRoutesSet = true
+	mp.AdvertiseRoutes = []netip.Prefix{prefix1}
+
+	b := newTestBackend(t)
+	testAppConnector := appc.NewAppConnector(t.Logf, b, true)
+	b.appConnector = testAppConnector
+	b.ControlKnobs().AppCStoreRoutes.Store(true)
+
+	b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AppConnector: ipn.AppConnectorPrefs{
+				Advertise: true,
+			},
+		},
+		AppConnectorSet: true,
+	})
+
+	appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"],
+		"routes":[%s]
+	}`
+	appCfg := fmt.Sprintf(appCfgTmpl, `"example.com"`, `"1.2.3.5/32"`)
+
+	reconfigWithAppCfg := func(appCfg string) {
+		b.netMap.SelfNode = (&tailcfg.Node{
+			Name: "example.ts.net",
+			Tags: []string{"tag:example"},
+			CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+				"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+			}),
+		}).View()
+
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		b.appConnector.Wait(context.Background())
+	}
+
+	reconfigWithAppCfg(appCfg)
+
+	now := time.Now()
+	ri := routeinfo.NewRouteInfo()
+	ri.Control = []netip.Prefix{prefix2}
+	discovered := make(map[string]*routeinfo.DatedRoutes)
+	routes := make(map[netip.Prefix]time.Time)
+	routes[prefix3] = now
+	discovered["example.com"] = &routeinfo.DatedRoutes{
+		LastCleanup: now,
+		Routes:      routes,
+	}
+	ri.Discovered = discovered
+
+	b.appConnector.UpdateRouteInfo(ri)
+	b.AdvertiseRoute([]netip.Prefix{prefix3}...)
+
+	prefView, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView.AdvertiseRoutes().Len() != 3 {
+		t.Fatalf("wanted %d, got %d", 3, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("New prefix was not advertised")
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+	//Check if route is stored in Appc/Appc.routeAdvertiser
+	storedRouteInfo, _ := b.ReadRouteInfo()
+	if len(storedRouteInfo.Local) != 1 {
+		t.Fatalf("wanted %d, got %d", 1, len(storedRouteInfo.Local))
+	}
+	if !slices.Contains(storedRouteInfo.Local, prefix1) {
+		t.Fatalf("New local route not stored.")
+	}
+
+	//Check if the routes in control and discovered are presisted.
+	routesShouldPresist := storedRouteInfo.Routes(false, true, true)
+	if len(routesShouldPresist) != 2 {
+		t.Fatalf("wanted %d, got %d", 2, len(routesShouldPresist))
+	}
+	if !slices.Contains(routesShouldPresist, prefix2) || !slices.Contains(routesShouldPresist, prefix3) {
+		t.Fatalf("Pre-existed routes not presisted.")
+	}
+
+	//Patch again with no route, see if prefix1 is removed/ prefix2, prefix3 presists.
+	mp.AdvertiseRoutes = []netip.Prefix{}
+	prefView2, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView2.AdvertiseRoutes().Len() != 2 {
+		t.Fatalf("wanted %d, got %d", 2, prefView.AdvertiseRoutes().Len())
+	}
+
+	if slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("Local route was not removed")
+	}
+
+	if !slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+}
+
+func TestPatchPrefsHandlerWithoutPresistStore(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+
+	mp := new(ipn.MaskedPrefs)
+	mp.AdvertiseRoutesSet = true
+	mp.AdvertiseRoutes = []netip.Prefix{prefix1}
+
+	b := newTestBackend(t)
+	testAppConnector := appc.NewAppConnector(t.Logf, b, false)
+	b.appConnector = testAppConnector
+	b.ControlKnobs().AppCStoreRoutes.Store(false)
+
+	b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AppConnector: ipn.AppConnectorPrefs{
+				Advertise: true,
+			},
+		},
+		AppConnectorSet: true,
+	})
+
+	appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"],
+		"routes":[%s]
+	}`
+	appCfg := fmt.Sprintf(appCfgTmpl, `"example.com"`, `"1.2.3.5/32"`)
+
+	reconfigWithAppCfg := func(appCfg string) {
+		b.netMap.SelfNode = (&tailcfg.Node{
+			Name: "example.ts.net",
+			Tags: []string{"tag:example"},
+			CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+				"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+			}),
+		}).View()
+
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		b.appConnector.Wait(context.Background())
+	}
+	reconfigWithAppCfg(appCfg)
+
+	b.AdvertiseRoute([]netip.Prefix{prefix3}...)
+
+	testAppConnector.RouteInfo()
+
+	prefView, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView.AdvertiseRoutes().Len() != 1 {
+		t.Fatalf("wanted %d, got %d", 1, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("New prefix was not advertised")
+	}
+
+	if slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix2) || slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are still advertised.")
+	}
+
+	//Check if route is stored in Appc/Appc.routeAdvertiser
+	storedRouteInfo, _ := b.ReadRouteInfo()
+	if storedRouteInfo != nil {
+		t.Fatalf("wanted nil, got %v", storedRouteInfo)
+	}
+
+	//Patch again with no route, see if prefix1 is removed/ prefix2, prefix3 presists.
+	mp.AdvertiseRoutes = []netip.Prefix{}
+	prefView2, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView2.AdvertiseRoutes().Len() != 0 {
+		t.Fatalf("wanted %d, got %d", 0, prefView.AdvertiseRoutes().Len())
+	}
+
+	if slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("Local route was not removed")
+	}
+
+	if slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix2) || slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are still advertised.")
+	}
+
+}
+
+func TestFran(t *testing.T) {
+	prefixSortFunc := func(i, j netip.Prefix) int {
+		return i.Addr().Compare(j.Addr())
+	}
+	epIP2 := netip.MustParsePrefix("192.1.0.9/32")
+	explicitlyAdvertisedRoutes := []netip.Prefix{netip.MustParsePrefix("192.1.0.8/32"), epIP2}
+	oneRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32"), netip.MustParsePrefix("192.0.0.16/32")}
+	twoRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.10/32"), netip.MustParsePrefix("192.0.0.18/32"), epIP2}
+
+	for _, shouldStore := range []bool{true, false} {
+		b := newTestBackend(t)
+		b.ControlKnobs().AppCStoreRoutes.Store(shouldStore)
+		// make b an app connector
+		b.EditPrefs(&ipn.MaskedPrefs{
+			Prefs: ipn.Prefs{
+				AppConnector: ipn.AppConnectorPrefs{
+					Advertise: true,
+				},
+				AdvertiseRoutes: explicitlyAdvertisedRoutes,
+			},
+			AppConnectorSet:    true,
+			AdvertiseRoutesSet: true,
+		})
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+
+		appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"]
+	}`
+		appCfg1Domain := fmt.Sprintf(appCfgTmpl, `"one.com"`)
+		appCfg2Domains := fmt.Sprintf(appCfgTmpl, `"one.com", "two.com"`)
+
+		reconfigWithAppCfg := func(appCfg string) {
+			b.netMap.SelfNode = (&tailcfg.Node{
+				Name: "example.ts.net",
+				Tags: []string{"tag:example"},
+				CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+					"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+				}),
+			}).View()
+
+			b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+			b.appConnector.Wait(context.Background())
+		}
+
+		// set the app connector to watch 2 domains
+		reconfigWithAppCfg(appCfg2Domains)
+
+		// when the app connector observes dns for the domains it adds routes
+		afterTwoRoutes := append([]netip.Prefix{}, explicitlyAdvertisedRoutes...)
+		afterTwoRoutes = append(afterTwoRoutes, oneRoutes...)
+		afterTwoRoutes = append(afterTwoRoutes, twoRoutes...)
+		slices.SortFunc(afterTwoRoutes, prefixSortFunc)
+		afterTwoRoutes = slices.Compact(afterTwoRoutes)
+
+		afterOneRoutes := append([]netip.Prefix{}, explicitlyAdvertisedRoutes...)
+		afterOneRoutes = append(afterOneRoutes, oneRoutes...)
+		slices.SortFunc(afterOneRoutes, prefixSortFunc)
+		afterOneRoutes = slices.Compact(afterOneRoutes)
+
+		for _, tst := range []struct {
+			domain    string
+			route     []string
+			wantAfter []netip.Prefix
+		}{
+			// learns the route for one.com
+			{domain: "one.com", route: []string{"192.0.0.8", "192.0.0.16"}, wantAfter: afterOneRoutes},
+			// doesn't care about example.com, so still just the route for one.com
+			{domain: "example.com", route: []string{"192.0.0.9", "192.0.0.17"}, wantAfter: afterOneRoutes},
+			// learns the route for two.com as well
+			{domain: "two.com", route: []string{"192.0.0.10", "192.0.0.18", "192.1.0.9"}, wantAfter: afterTwoRoutes},
+		} {
+			b.ObserveDNSResponse(dnsResponse(tst.domain+".", tst.route))
+			b.appConnector.Wait(context.Background())
+			routesNow := b.pm.prefs.AdvertiseRoutes().AsSlice()
+			slices.SortFunc(routesNow, prefixSortFunc)
+			if !slices.Equal(routesNow, tst.wantAfter) {
+				t.Fatalf("after dns response for %s got routes %v, want %v", tst.domain, routesNow, tst.wantAfter)
+			}
+		}
+
+		// reconfigure the app connector to observe fewer domains
+		reconfigWithAppCfg(appCfg1Domain)
+		routesNow := b.pm.prefs.AdvertiseRoutes().AsSlice()
+		wantRoutes := afterOneRoutes
+		if !shouldStore {
+			wantRoutes = afterTwoRoutes
+		}
+		slices.SortFunc(routesNow, prefixSortFunc)
+		if !slices.Equal(routesNow, wantRoutes) {
+			t.Fatalf("after removing two.com (shouldStore=%t), got %v, want %v", shouldStore, routesNow, wantRoutes)
+		}
+
+		ac := b.appConnector
+		// heck forget about being an app connector
+		b.EditPrefs(&ipn.MaskedPrefs{
+			Prefs: ipn.Prefs{
+				AppConnector: ipn.AppConnectorPrefs{
+					Advertise: false,
+				},
+			},
+			AppConnectorSet: true,
+		})
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		ac.Wait(context.Background())
+
+		routesNow = b.pm.prefs.AdvertiseRoutes().AsSlice()
+		wantRoutes = explicitlyAdvertisedRoutes
+		if !shouldStore {
+			wantRoutes = afterTwoRoutes
+		}
+		slices.SortFunc(routesNow, prefixSortFunc)
+		if !slices.Equal(routesNow, wantRoutes) {
+			t.Fatalf("after becoming not an app connector got routes %v, want %v", routesNow, wantRoutes)
+		}
+	}
+}
+
+func TestPatchPrefsHandlerNoDuplicateAdvertisement(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+	mp := new(ipn.MaskedPrefs)
+	mp.AdvertiseRoutesSet = true
+	mp.AdvertiseRoutes = []netip.Prefix{prefix1}
+
+	b := newTestBackend(t)
+	testAppConnector := appc.NewAppConnector(t.Logf, b, true)
+	b.appConnector = testAppConnector
+	b.ControlKnobs().AppCStoreRoutes.Store(true)
+
+	b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AppConnector: ipn.AppConnectorPrefs{
+				Advertise: true,
+			},
+		},
+		AppConnectorSet: true,
+	})
+
+	appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"],
+		"routes":[%s]
+	}`
+	appCfg := fmt.Sprintf(appCfgTmpl, `"example.com"`, `"1.2.3.4/32", "1.2.3.5/32"`)
+
+	reconfigWithAppCfg := func(appCfg string) {
+		b.netMap.SelfNode = (&tailcfg.Node{
+			Name: "example.ts.net",
+			Tags: []string{"tag:example"},
+			CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+				"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+			}),
+		}).View()
+
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		b.appConnector.Wait(context.Background())
+	}
+
+	reconfigWithAppCfg(appCfg)
+
+	now := time.Now()
+	ri := routeinfo.NewRouteInfo()
+	ri.Control = []netip.Prefix{prefix1, prefix2}
+	discovered := make(map[string]*routeinfo.DatedRoutes)
+	routes := make(map[netip.Prefix]time.Time)
+	routes[prefix3] = now
+	discovered["example.com"] = &routeinfo.DatedRoutes{
+		LastCleanup: now,
+		Routes:      routes,
+	}
+	ri.Discovered = discovered
+
+	b.appConnector.UpdateRouteInfo(ri)
+	b.AdvertiseRoute([]netip.Prefix{prefix3}...)
+
+	prefView, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView.AdvertiseRoutes().Len() != 3 {
+		t.Fatalf("wanted %d, got %d", 3, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("New prefix was not advertised")
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+	//Patch again with no route, see if prefix1 is removed/ prefix2, prefix3 presists.
+	mp.AdvertiseRoutes = []netip.Prefix{}
+	prefView2, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView2.AdvertiseRoutes().Len() != 3 {
+		t.Fatalf("wanted %d, got %d", 3, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("Local route was not removed")
+	}
+
+	if !slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+}

ipn/ipnlocal/peerapi_test.go

@@ -687,185 +687,191 @@ func TestPeerAPIReplyToDNSQueries(t *testing.T) {
 }
 
 func TestPeerAPIPrettyReplyCNAME(t *testing.T) {
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{true, false} {
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:     eng,
-			pm:    pm,
-			store: pm.Store(),
-			// configure as an app connector just to enable the API.
-			appConnector: appc.NewAppConnector(t.Logf, &appctest.RouteCollector{}),
-		},
-	}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:     eng,
+				pm:    pm,
+				store: pm.Store(),
+				// configure as an app connector just to enable the API.
+				appConnector: appc.NewAppConnector(t.Logf, &appctest.RouteCollector{}, shouldStore),
+			},
+		}
 
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.CNAMEResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("www.example.com."),
-				Type:  dnsmessage.TypeCNAME,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.CNAMEResource{
-				CNAME: dnsmessage.MustNewName("example.com."),
-			},
-		)
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.CNAMEResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("www.example.com."),
+					Type:  dnsmessage.TypeCNAME,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.CNAMEResource{
+					CNAME: dnsmessage.MustNewName("example.com."),
+				},
+			)
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	var addrs []string
-	json.NewDecoder(w.Body).Decode(&addrs)
-	if len(addrs) == 0 {
-		t.Fatalf("no addresses returned")
-	}
-	for _, addr := range addrs {
-		netip.MustParseAddr(addr)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		var addrs []string
+		json.NewDecoder(w.Body).Decode(&addrs)
+		if len(addrs) == 0 {
+			t.Fatalf("no addresses returned")
+		}
+		for _, addr := range addrs {
+			netip.MustParseAddr(addr)
+		}
 	}
 }
 
 func TestPeerAPIReplyToDNSQueriesAreObserved(t *testing.T) {
-	ctx := context.Background()
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	rc := &appctest.RouteCollector{}
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:            eng,
-			pm:           pm,
-			store:        pm.Store(),
-			appConnector: appc.NewAppConnector(t.Logf, rc),
-		},
-	}
-	h.ps.b.appConnector.UpdateDomains([]string{"example.com"})
-	h.ps.b.appConnector.Wait(ctx)
-
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
+		rc := &appctest.RouteCollector{}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:            eng,
+				pm:           pm,
+				store:        pm.Store(),
+				appConnector: appc.NewAppConnector(t.Logf, rc, shouldStore),
 			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		}
+		h.ps.b.appConnector.UpdateDomains([]string{"example.com"})
+		h.ps.b.appConnector.Wait(ctx)
 
-	if !h.ps.b.OfferingAppConnector() {
-		t.Fatal("expecting to be offering app connector")
-	}
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	h.ps.b.appConnector.Wait(ctx)
+		if !h.ps.b.OfferingAppConnector() {
+			t.Fatal("expecting to be offering app connector")
+		}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		h.ps.b.appConnector.Wait(ctx)
+
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 
 func TestPeerAPIReplyToDNSQueriesAreObservedWithCNAMEFlattening(t *testing.T) {
-	ctx := context.Background()
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	rc := &appctest.RouteCollector{}
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:            eng,
-			pm:           pm,
-			store:        pm.Store(),
-			appConnector: appc.NewAppConnector(t.Logf, rc),
-		},
-	}
-	h.ps.b.appConnector.UpdateDomains([]string{"www.example.com"})
-	h.ps.b.appConnector.Wait(ctx)
-
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.CNAMEResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("www.example.com."),
-				Type:  dnsmessage.TypeCNAME,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
+		rc := &appctest.RouteCollector{}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:            eng,
+				pm:           pm,
+				store:        pm.Store(),
+				appConnector: appc.NewAppConnector(t.Logf, rc, shouldStore),
 			},
-			dnsmessage.CNAMEResource{
-				CNAME: dnsmessage.MustNewName("example.com."),
-			},
-		)
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		}
+		h.ps.b.appConnector.UpdateDomains([]string{"www.example.com"})
+		h.ps.b.appConnector.Wait(ctx)
 
-	if !h.ps.b.OfferingAppConnector() {
-		t.Fatal("expecting to be offering app connector")
-	}
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.CNAMEResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("www.example.com."),
+					Type:  dnsmessage.TypeCNAME,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.CNAMEResource{
+					CNAME: dnsmessage.MustNewName("example.com."),
+				},
+			)
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	h.ps.b.appConnector.Wait(ctx)
+		if !h.ps.b.OfferingAppConnector() {
+			t.Fatal("expecting to be offering app connector")
+		}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		h.ps.b.appConnector.Wait(ctx)
+
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 

ipn/localapi/localapi.go

@@ -1376,7 +1376,7 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		var err error
-		prefs, err = h.b.EditPrefs(mp)
+		prefs, err = h.b.PatchPrefsHandler(mp)
 		if err != nil {
 			w.Header().Set("Content-Type", "application/json")
 			w.WriteHeader(http.StatusBadRequest)

tailcfg/tailcfg.go

@@ -2230,6 +2230,9 @@ const (
 
 	// NodeAttrDisableWebClient disables using the web client.
 	NodeAttrDisableWebClient NodeCapability = "disable-web-client"
+
+	// NodeAttrStoreAppCRoutes enables storing app connector routes persistently.
+	NodeAttrStoreAppCRoutes NodeCapability = "store-appc-routes"
 )
 
 // SetDNSRequest is a request to add a DNS record.