add debug cmd for routeInfo

1. Patch prefs handler doesn't add the remote routes to pref if the pref update is also turning appconnector off.
2. Unadvertise routes when StoreRoutes is on but appConnector is turning off.
3. When appConnector is turning on, add a task to recreate and reload routeInfo from store to the queue
3. When shouldStore is changed, remove the state saved for routeInfo (Set to nil)

appc/appconnector.go

@@ -18,6 +18,8 @@ import (
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
+	"tailscale.com/appc/routeinfo"
+	"tailscale.com/ipn"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/views"
 	"tailscale.com/util/dnsname"
@@ -28,12 +30,11 @@ import (
 // RouteAdvertiser is an interface that allows the AppConnector to advertise
 // newly discovered routes that need to be served through the AppConnector.
 type RouteAdvertiser interface {
-	// AdvertiseRoute adds one or more route advertisements skipping any that
-	// are already advertised.
-	AdvertiseRoute(...netip.Prefix) error
+	AdvertiseRouteInfo(*routeinfo.RouteInfo)
 
-	// UnadvertiseRoute removes any matching route advertisements.
-	UnadvertiseRoute(...netip.Prefix) error
+	// Store/ReadRouteInfo persists and retreives RouteInfo to stable storage
+	StoreRouteInfo(*routeinfo.RouteInfo) error
+	ReadRouteInfo() (*routeinfo.RouteInfo, error)
 }
 
 // AppConnector is an implementation of an AppConnector that performs
@@ -54,23 +55,35 @@ type AppConnector struct {
 
 	// domains is a map of lower case domain names with no trailing dot, to an
 	// ordered list of resolved IP addresses.
-	domains map[string][]netip.Addr
+	// domains map[string][]netip.Addr
 
 	// controlRoutes is the list of routes that were last supplied by control.
-	controlRoutes []netip.Prefix
+	// controlRoutes []netip.Prefix
 
 	// wildcards is the list of domain strings that match subdomains.
-	wildcards []string
+	// wildcards []string
+
+	// the in memory copy of all the routes that's advertised
+	routeInfo *routeinfo.RouteInfo
 
 	// queue provides ordering for update operations
 	queue execqueue.ExecQueue
+
+	// whether this tailscaled should persist routes. Storing RouteInfo enables the app connector
+	// to forget routes when appropriate and should make routes smaller. While we are verifying that
+	// writing the RouteInfo to StateStore is a good solution (and doesn't for example cause writes
+	// that are too frequent or too large) use a controlknob to manage this flag.
+	ShouldStoreRoutes bool
 }
 
 // NewAppConnector creates a new AppConnector.
-func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser) *AppConnector {
+func NewAppConnector(logf logger.Logf, routeAdvertiser RouteAdvertiser, shouldStoreRoutes bool) *AppConnector {
+	// TODO(fran) if !shouldStoreRoutes we probably want to try and clean up any stored routes
 	return &AppConnector{
-		logf:            logger.WithPrefix(logf, "appc: "),
-		routeAdvertiser: routeAdvertiser,
+		logf:              logger.WithPrefix(logf, "appc: "),
+		routeAdvertiser:   routeAdvertiser,
+		ShouldStoreRoutes: shouldStoreRoutes,
+		routeInfo:         nil,
 	}
 }
 
@@ -84,6 +97,59 @@ func (e *AppConnector) UpdateDomainsAndRoutes(domains []string, routes []netip.P
 	})
 }
 
+func (e *AppConnector) RouteInfo() *routeinfo.RouteInfo {
+	// e.mu.Lock()
+	// defer e.mu.Unlock()
+	if e.routeInfo == nil {
+		ret, err := e.routeAdvertiser.ReadRouteInfo()
+		if err != nil {
+			if err != ipn.ErrStateNotExist {
+				e.logf("Unsuccessful Read RouteInfo: ", err)
+			}
+			return routeinfo.NewRouteInfo()
+		}
+		return ret
+	}
+	return e.routeInfo
+}
+
+// RecreateRouteInfoFromStore searches from presist store for existing routeInfo for current profile,
+// then update local routes of routeInfo in store as the current advertised routes.
+func (e *AppConnector) RecreateRouteInfoFromStore() {
+	e.queue.Add(func() {
+		e.mu.Lock()
+		defer e.mu.Unlock()
+		ri := e.RouteInfo()
+		err := e.routeAdvertiser.StoreRouteInfo(ri)
+		if err != nil {
+			e.logf("Appc recreate routeInfo: Error updating routeInfo in store: ", err)
+		}
+		e.routeAdvertiser.AdvertiseRouteInfo(ri)
+		if err != nil {
+			e.logf("Appc recreate routeInfo: Error advertise routes: ", err)
+		}
+		e.routeInfo = ri
+	})
+}
+
+// UpdateRouteInfo updates routeInfo value of the AppConnector and
+// stores the routeInfo value to presist store.
+func (e *AppConnector) UpdateRouteInfo(ri *routeinfo.RouteInfo) {
+	err := e.routeAdvertiser.StoreRouteInfo(ri)
+	if err != nil {
+		e.logf("Appc: Failed to remove routeInfo from store: ", err)
+	}
+	e.routeInfo = ri
+}
+
+func (e *AppConnector) UnadvertiseRemoteRoutes() {
+	e.queue.Add(func() {
+		e.mu.Lock()
+		defer e.mu.Unlock()
+		e.routeAdvertiser.AdvertiseRouteInfo(nil)
+	})
+}
+
 // UpdateDomains asynchronously replaces the current set of configured domains
 // with the supplied set of domains. Domains must not contain a trailing dot,
 // and should be lower case. If the domain contains a leading '*' label it
@@ -104,32 +170,45 @@ func (e *AppConnector) updateDomains(domains []string) {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	var oldDomains map[string][]netip.Addr
-	oldDomains, e.domains = e.domains, make(map[string][]netip.Addr, len(domains))
-	e.wildcards = e.wildcards[:0]
+	var oldDiscovered map[string]*routeinfo.DatedRoutes
+	var routeInfo *routeinfo.RouteInfo
+	shouldStoreRoutes := e.ShouldStoreRoutes
+
+	routeInfo = e.RouteInfo()
+	oldDiscovered, routeInfo.Discovered = routeInfo.Discovered, make(map[string]*routeinfo.DatedRoutes, len(domains))
+
+	routeInfo.Wildcards = routeInfo.Wildcards[:0]
 	for _, d := range domains {
 		d = strings.ToLower(d)
 		if len(d) == 0 {
 			continue
 		}
 		if strings.HasPrefix(d, "*.") {
-			e.wildcards = append(e.wildcards, d[2:])
+			routeInfo.Wildcards = append(routeInfo.Wildcards, d[2:])
 			continue
 		}
-		e.domains[d] = oldDomains[d]
-		delete(oldDomains, d)
+		routeInfo.Discovered[d] = oldDiscovered[d]
+		delete(oldDiscovered, d)
 	}
 
-	// Ensure that still-live wildcards addresses are preserved as well.
-	for d, addrs := range oldDomains {
-		for _, wc := range e.wildcards {
+	for d, dr := range oldDiscovered {
+		for _, wc := range routeInfo.Wildcards {
 			if dnsname.HasSuffix(d, wc) {
-				e.domains[d] = addrs
+				routeInfo.Discovered[d] = dr
+				delete(oldDiscovered, d)
 				break
 			}
 		}
 	}
-	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
+
+	if shouldStoreRoutes {
+		e.UpdateRouteInfo(routeInfo)
+	} else {
+		e.routeInfo = routeInfo
+	}
+	e.scheduleAdvertiseRouteInfo(e.RouteInfo())
+
+	e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.RouteInfo().Discovered), routeInfo.Wildcards)
 }
 
 // updateRoutes merges the supplied routes into the currently configured routes. The routes supplied
@@ -141,35 +220,19 @@ func (e *AppConnector) updateRoutes(routes []netip.Prefix) {
 	defer e.mu.Unlock()
 
 	// If there was no change since the last update, no work to do.
-	if slices.Equal(e.controlRoutes, routes) {
+	if slices.Equal(e.RouteInfo().Control, routes) {
 		return
 	}
 
-	if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
-		e.logf("failed to advertise routes: %v: %v", routes, err)
-		return
+	var routeInfo *routeinfo.RouteInfo
+	e.routeInfo.Control = routes
+	if e.ShouldStoreRoutes {
+		routeInfo = e.RouteInfo()
+		routeInfo.Control = routes
+		e.routeAdvertiser.StoreRouteInfo(e.routeInfo)
 	}
 
-	var toRemove []netip.Prefix
-
-nextRoute:
-	for _, r := range routes {
-		for _, addr := range e.domains {
-			for _, a := range addr {
-				if r.Contains(a) && netip.PrefixFrom(a, a.BitLen()) != r {
-					pfx := netip.PrefixFrom(a, a.BitLen())
-					toRemove = append(toRemove, pfx)
-					continue nextRoute
-				}
-			}
-		}
-	}
-
-	if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
-		e.logf("failed to unadvertise routes: %v: %v", toRemove, err)
-	}
-
-	e.controlRoutes = routes
+	e.routeAdvertiser.AdvertiseRouteInfo(e.routeInfo)
 }
 
 // Domains returns the currently configured domain list.
@@ -177,7 +240,7 @@ func (e *AppConnector) Domains() views.Slice[string] {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	return views.SliceOf(xmaps.Keys(e.domains))
+	return views.SliceOf(xmaps.Keys(e.RouteInfo().Discovered))
 }
 
 // DomainRoutes returns a map of domains to resolved IP
@@ -186,12 +249,7 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
-	drCopy := make(map[string][]netip.Addr)
-	for k, v := range e.domains {
-		drCopy[k] = append(drCopy[k], v...)
-	}
-
-	return drCopy
+	return e.routeInfo.DomainRoutes()
 }
 
 // ObserveDNSResponse is a callback invoked by the DNS resolver when a DNS
@@ -287,6 +345,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 	e.mu.Lock()
 	defer e.mu.Unlock()
 
+	routeInfo := e.RouteInfo()
 	for domain, addrs := range addressRecords {
 		domain, isRouted := e.findRoutedDomainLocked(domain, cnameChain)
 
@@ -297,16 +356,18 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 
 		// advertise each address we have learned for the routed domain, that
 		// was not already known.
-		var toAdvertise []netip.Prefix
+		var domainPrefixs []netip.Prefix
 		for _, addr := range addrs {
-			if !e.isAddrKnownLocked(domain, addr) {
-				toAdvertise = append(toAdvertise, netip.PrefixFrom(addr, addr.BitLen()))
-			}
+			domainPrefixs = append(domainPrefixs, netip.PrefixFrom(addr, addr.BitLen()))
 		}
 
-		e.logf("[v2] observed new routes for %s: %s", domain, toAdvertise)
-		e.scheduleAdvertisement(domain, toAdvertise...)
+		e.logf("[v2] observed new routes for %s: %s", domain, domainPrefixs)
+		routeInfo.AddRoutesInDiscoveredForDomain(domain, domainPrefixs)
+		if e.ShouldStoreRoutes {
+			e.UpdateRouteInfo(routeInfo)
+		}
 	}
+	e.scheduleAdvertiseRouteInfo(e.RouteInfo())
 }
 
 // starting from the given domain that resolved to an address, find it, or any
@@ -317,15 +378,15 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) {
 func (e *AppConnector) findRoutedDomainLocked(domain string, cnameChain map[string]string) (string, bool) {
 	var isRouted bool
 	for {
-		_, isRouted = e.domains[domain]
+		_, isRouted = e.RouteInfo().Discovered[domain]
 		if isRouted {
 			break
 		}
 
 		// match wildcard domains
-		for _, wc := range e.wildcards {
+		for _, wc := range e.RouteInfo().Wildcards {
 			if dnsname.HasSuffix(domain, wc) {
-				e.domains[domain] = nil
+				e.routeInfo.Discovered[domain] = nil
 				isRouted = true
 				break
 			}
@@ -340,63 +401,10 @@ func (e *AppConnector) findRoutedDomainLocked(domain string, cnameChain map[stri
 	return domain, isRouted
 }
 
-// isAddrKnownLocked returns true if the address is known to be associated with
-// the given domain. Known domain tables are updated for covered routes to speed
-// up future matches.
-// e.mu must be held.
-func (e *AppConnector) isAddrKnownLocked(domain string, addr netip.Addr) bool {
-	if e.hasDomainAddrLocked(domain, addr) {
-		return true
-	}
-	for _, route := range e.controlRoutes {
-		if route.Contains(addr) {
-			// record the new address associated with the domain for faster matching in subsequent
-			// requests and for diagnostic records.
-			e.addDomainAddrLocked(domain, addr)
-			return true
-		}
-	}
-	return false
-}
-
-// scheduleAdvertisement schedules an advertisement of the given address
-// associated with the given domain.
-func (e *AppConnector) scheduleAdvertisement(domain string, routes ...netip.Prefix) {
+func (e *AppConnector) scheduleAdvertiseRouteInfo(ri *routeinfo.RouteInfo) {
 	e.queue.Add(func() {
-		if err := e.routeAdvertiser.AdvertiseRoute(routes...); err != nil {
-			e.logf("failed to advertise routes for %s: %v: %v", domain, routes, err)
-			return
-		}
 		e.mu.Lock()
 		defer e.mu.Unlock()
-
-		for _, route := range routes {
-			if !route.IsSingleIP() {
-				continue
-			}
-			addr := route.Addr()
-			if !e.hasDomainAddrLocked(domain, addr) {
-				e.addDomainAddrLocked(domain, addr)
-				e.logf("[v2] advertised route for %v: %v", domain, addr)
-			}
-		}
+		e.routeAdvertiser.AdvertiseRouteInfo(ri)
 	})
 }
-
-// hasDomainAddrLocked returns true if the address has been observed in a
-// resolution of domain.
-func (e *AppConnector) hasDomainAddrLocked(domain string, addr netip.Addr) bool {
-	_, ok := slices.BinarySearchFunc(e.domains[domain], addr, compareAddr)
-	return ok
-}
-
-// addDomainAddrLocked adds the address to the list of addresses resolved for
-// domain and ensures the list remains sorted. Does not attempt to deduplicate.
-func (e *AppConnector) addDomainAddrLocked(domain string, addr netip.Addr) {
-	e.domains[domain] = append(e.domains[domain], addr)
-	slices.SortFunc(e.domains[domain], compareAddr)
-}
-
-func compareAddr(l, r netip.Addr) int {
-	return l.Compare(r)
-}

appc/appconnector_test.go

@@ -9,17 +9,88 @@ import (
 	"reflect"
 	"slices"
 	"testing"
+	"time"
 
 	xmaps "golang.org/x/exp/maps"
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 )
 
-func TestUpdateDomains(t *testing.T) {
+func TestUpdateDomainsWithPresistStore(t *testing.T) {
+
 	ctx := context.Background()
-	a := NewAppConnector(t.Logf, nil)
+	a := NewAppConnector(t.Logf, nil, true)
+	TestRouteAdvertiser := &appctest.RouteCollector{}
+	a.routeAdvertiser = TestRouteAdvertiser
+	prefixControl := netip.MustParsePrefix("192.0.0.8/32")
+	prefixLocal := netip.MustParsePrefix("192.0.0.9/32")
+	prefixDiscovered := netip.MustParsePrefix("192.0.0.10/32")
+	prefixShouldUnadvertise := netip.MustParsePrefix("192.0.0.11/32")
+	ri := routeinfo.NewRouteInfo()
+	ri.Control = []netip.Prefix{prefixControl}
+	ri.Local = []netip.Prefix{prefixLocal}
+	a.UpdateRouteInfo(ri)
+	a.UpdateDomains([]string{"example.com", "test.com"})
+
+	a.Wait(ctx)
+	if got, want := a.Domains().AsSlice(), []string{"example.com", "test.com"}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+	if _, hasKey := a.routeInfo.Discovered["example.com"]; !hasKey {
+		t.Errorf("Discovered did not record the test domain.")
+	}
+
+	now := time.Now()
+	testRoutes := make(map[netip.Prefix]time.Time)
+	testRoutes[prefixDiscovered] = now
+	exampleRoutes := make(map[netip.Prefix]time.Time)
+	exampleRoutes[prefixShouldUnadvertise] = now
+	exampleRoutes[prefixControl] = now
+	exampleRoutes[prefixLocal] = now
+	exampleRoutes[prefixDiscovered] = now
+	ri.Discovered["test.com"] = &routeinfo.DatedRoutes{Routes: testRoutes, LastCleanup: now}
+	ri.Discovered["example.com"] = &routeinfo.DatedRoutes{Routes: exampleRoutes, LastCleanup: now}
+	a.UpdateRouteInfo(ri)
+	TestRouteAdvertiser.SetRoutes(ri.Discovered["example.com"].RoutesSlice())
+
+	addrControl := netip.MustParseAddr("192.0.0.8")
+	addrLocal := netip.MustParseAddr("192.0.0.9")
+	addrDiscovered := netip.MustParseAddr("192.0.0.10")
+	addrShouldUnadvertise := netip.MustParseAddr("192.0.0.11")
+	a.domains["example.com"] = append(a.domains["example.com"], []netip.Addr{addrControl, addrLocal, addrDiscovered, addrShouldUnadvertise}...)
+	a.domains["test.com"] = append(a.domains["test.com"], addrDiscovered)
+	a.UpdateDomains([]string{"example.com", "test.com"})
+	a.Wait(ctx)
+
+	if got, want := a.domains["example.com"], []netip.Addr{addrControl, addrLocal, addrDiscovered, addrShouldUnadvertise}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+	// domains are explicitly downcased on set.
+	a.UpdateDomains([]string{"UP.EXAMPLE.COM", "test.com"})
+	a.Wait(ctx)
+	if got, want := xmaps.Keys(a.domains), []string{"up.example.com", "test.com"}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+	if got, want := TestRouteAdvertiser.RemovedRoutes(), []netip.Prefix{prefixShouldUnadvertise}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+	if got, want := TestRouteAdvertiser.Routes(), []netip.Prefix{prefixControl, prefixLocal, prefixDiscovered}; !slices.Equal(got, want) {
+		t.Errorf("got %v; want %v", got, want)
+	}
+
+}
+
+func TestUpdateDomainsWithoutPresistStore(t *testing.T) {
+	ctx := context.Background()
+	a := NewAppConnector(t.Logf, nil, false)
+	a.routeAdvertiser = &appctest.RouteCollector{}
+	a.UpdateRouteInfo(routeinfo.NewRouteInfo())
 	a.UpdateDomains([]string{"example.com"})
 
 	a.Wait(ctx)
@@ -45,166 +116,216 @@ func TestUpdateDomains(t *testing.T) {
 }
 
 func TestUpdateRoutes(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"*.example.com"})
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		a.updateDomains([]string{"*.example.com"})
 
-	// This route should be collapsed into the range
-	a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
-	a.Wait(ctx)
+		// This route should be collapsed into the range
+		a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
+		a.Wait(ctx)
 
-	if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
-		t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
+			t.Fatalf("got %v, want %v", rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		}
+
+		// This route should not be collapsed or removed
+		a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
+		a.Wait(ctx)
+
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
+
+		if shouldStore {
+			wantRouteInfoControlRoutes := routes
+			if !slices.Equal(a.routeInfo.Control, wantRouteInfoControlRoutes) {
+				t.Fatalf("got %v, want %v (shouldStore=%t)", a.routeInfo.Control, wantRouteInfoControlRoutes, shouldStore)
+			}
+		} else {
+			if a.routeInfo != nil {
+				t.Fatalf("got %v, want %v (shouldStore=%t)", a.routeInfo.Control, nil, shouldStore)
+			}
+		}
+
+		slices.SortFunc(rc.Routes(), prefixCompare)
+		rc.SetRoutes(slices.Compact(rc.Routes()))
+		slices.SortFunc(routes, prefixCompare)
+
+		// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
+		}
+
+		// Ensure that the contained /32 is removed, replaced by the /24.
+		wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
+}
 
-	// This route should not be collapsed or removed
-	a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
-	a.Wait(ctx)
+func TestUpdateRoutesNotUnadvertiseRoutesFromOtherSources(t *testing.T) {
+	for _, shouldStore := range []bool{true, false} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		testRi := routeinfo.NewRouteInfo()
+		testRi.Local = append(testRi.Local, netip.MustParsePrefix("192.0.2.0/24"))
+		a.routeInfo = testRi
+		rc.StoreRouteInfo(testRi)
 
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
-	a.updateRoutes(routes)
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes)
+		wantRouteInfoControlRoutes := []netip.Prefix{}
+		if shouldStore {
+			wantRouteInfoControlRoutes = routes
+		}
+		if !slices.Equal(a.routeInfo.Control, wantRouteInfoControlRoutes) {
+			t.Fatalf("got %v, want %v (shouldStore=%t)", a.routeInfo.Control, wantRouteInfoControlRoutes, shouldStore)
+		}
 
-	slices.SortFunc(rc.Routes(), prefixCompare)
-	rc.SetRoutes(slices.Compact(rc.Routes()))
-	slices.SortFunc(routes, prefixCompare)
+		routes2 := []netip.Prefix{netip.MustParsePrefix("192.0.0.1/32")}
+		a.updateRoutes(routes2)
 
-	// Ensure that the non-matching /32 is preserved, even though it's in the domains table.
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Errorf("added routes: got %v, want %v", rc.Routes(), routes)
-	}
-
-	// Ensure that the contained /32 is removed, replaced by the /24.
-	wantRemoved := []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}
-	if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
-		t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		wantRemoved := []netip.Prefix{}
+		if !slices.EqualFunc(rc.RemovedRoutes(), wantRemoved, prefixEqual) {
+			t.Fatalf("unexpected removed routes: %v", rc.RemovedRoutes())
+		}
 	}
 }
 
 func TestUpdateRoutesUnadvertisesContainedRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
-	rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
-	routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
-	a.updateRoutes(routes)
+	for _, shouldStore := range []bool{true, false} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		mak.Set(&a.domains, "example.com", []netip.Addr{netip.MustParseAddr("192.0.2.1")})
+		rc.SetRoutes([]netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")})
+		routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")}
+		a.updateRoutes(routes)
 
-	if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
-		t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		if !slices.EqualFunc(routes, rc.Routes(), prefixEqual) {
+			t.Fatalf("got %v, want %v", rc.Routes(), routes)
+		}
 	}
 }
 
 func TestDomainRoutes(t *testing.T) {
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(context.Background())
+	for _, shouldStore := range []bool{true, false} {
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(context.Background())
 
-	want := map[string][]netip.Addr{
-		"example.com": {netip.MustParseAddr("192.0.0.8")},
-	}
+		want := map[string][]netip.Addr{
+			"example.com": {netip.MustParseAddr("192.0.0.8")},
+		}
 
-	if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
-		t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		if got := a.DomainRoutes(); !reflect.DeepEqual(got, want) {
+			t.Fatalf("DomainRoutes: got %v, want %v", got, want)
+		}
 	}
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
 
-	// a has no domains configured, so it should not advertise any routes
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a has no domains configured, so it should not advertise any routes
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
 
-	a.updateDomains([]string{"example.com"})
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"example.com"})
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// matches a routed domain.
-	a.updateDomains([]string{"www.example.com", "example.com"})
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// matches a routed domain.
+		a.updateDomains([]string{"www.example.com", "example.com"})
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// a CNAME record chain should result in a route being added if the chain
-	// even if only found in the middle of the chain
-	a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
-	a.Wait(ctx)
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		// a CNAME record chain should result in a route being added if the chain
+		// even if only found in the middle of the chain
+		a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
+		a.Wait(ctx)
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
+		wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
 
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
-		t.Errorf("got %v; want %v", got, want)
-	}
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
+			t.Errorf("got %v; want %v", got, want)
+		}
 
-	// don't re-advertise routes that have already been advertised
-	a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
+		// don't re-advertise routes that have already been advertised
+		a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
 
-	// don't advertise addresses that are already in a control provided route
-	pfx := netip.MustParsePrefix("192.0.2.0/24")
-	a.updateRoutes([]netip.Prefix{pfx})
-	wantRoutes = append(wantRoutes, pfx)
-	a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
-	a.Wait(ctx)
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
-	}
-	if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
-		t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		// don't advertise addresses that are already in a control provided route
+		pfx := netip.MustParsePrefix("192.0.2.0/24")
+		a.updateRoutes([]netip.Prefix{pfx})
+		wantRoutes = append(wantRoutes, pfx)
+		a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
+		a.Wait(ctx)
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
+		}
+		if !slices.Contains(a.domains["example.com"], netip.MustParseAddr("192.0.2.1")) {
+			t.Errorf("missing %v from %v", "192.0.2.1", a.domains["exmaple.com"])
+		}
 	}
 }
 
 func TestWildcardDomains(t *testing.T) {
-	ctx := context.Background()
-	rc := &appctest.RouteCollector{}
-	a := NewAppConnector(t.Logf, rc)
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		rc := &appctest.RouteCollector{}
+		a := NewAppConnector(t.Logf, rc, shouldStore)
 
-	a.updateDomains([]string{"*.example.com"})
-	a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
-	a.Wait(ctx)
-	if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
-		t.Errorf("routes: got %v; want %v", got, want)
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com"})
+		a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
+		a.Wait(ctx)
+		if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
+			t.Errorf("routes: got %v; want %v", got, want)
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if _, ok := a.domains["foo.example.com"]; !ok {
-		t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
-	}
-	if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
-		t.Errorf("wildcards: got %v; want %v", got, want)
-	}
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if _, ok := a.domains["foo.example.com"]; !ok {
+			t.Errorf("expected foo.example.com to be preserved in domains due to wildcard")
+		}
+		if got, want := a.wildcards, []string{"example.com"}; !slices.Equal(got, want) {
+			t.Errorf("wildcards: got %v; want %v", got, want)
+		}
 
-	// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
-	a.updateDomains([]string{"*.example.com", "example.com"})
-	if len(a.wildcards) != 1 {
-		t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		// There was an early regression where the wildcard domain was added repeatedly, this guards against that.
+		a.updateDomains([]string{"*.example.com", "example.com"})
+		if len(a.wildcards) != 1 {
+			t.Errorf("expected only one wildcard domain, got %v", a.wildcards)
+		}
 	}
 }
 

appc/appctest/appctest.go

@@ -6,12 +6,16 @@ package appctest
 import (
 	"net/netip"
 	"slices"
+
+	"tailscale.com/appc/routeinfo"
+	"tailscale.com/ipn"
 )
 
 // RouteCollector is a test helper that collects the list of routes advertised
 type RouteCollector struct {
 	routes        []netip.Prefix
 	removedRoutes []netip.Prefix
+	routeInfo     *routeinfo.RouteInfo
 }
 
 func (rc *RouteCollector) AdvertiseRoute(pfx ...netip.Prefix) error {
@@ -32,6 +36,18 @@ func (rc *RouteCollector) UnadvertiseRoute(toRemove ...netip.Prefix) error {
 	return nil
 }
 
+func (rc *RouteCollector) StoreRouteInfo(ri *routeinfo.RouteInfo) error {
+	rc.routeInfo = ri
+	return nil
+}
+
+func (rc *RouteCollector) ReadRouteInfo() (*routeinfo.RouteInfo, error) {
+	if rc.routeInfo == nil {
+		return nil, ipn.ErrStateNotExist
+	}
+	return rc.routeInfo, nil
+}
+
 // RemovedRoutes returns the list of routes that were removed.
 func (rc *RouteCollector) RemovedRoutes() []netip.Prefix {
 	return rc.removedRoutes

appc/routeinfo/routeinfo.go

@@ -0,0 +1,105 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package routeinfo
+
+import (
+	"net/netip"
+	"time"
+)
+
+type RouteInfo struct {
+	// routes from the 'routes' section of an app connector acl
+	Control []netip.Prefix
+	// routes discovered by observing dns lookups for configured domains
+	Discovered map[string]*DatedRoutes
+
+	Wildcards []string
+}
+
+func NewRouteInfo() *RouteInfo {
+	discovered := make(map[string]*DatedRoutes)
+	return &RouteInfo{
+		Control:    []netip.Prefix{},
+		Discovered: discovered,
+		Wildcards:  []string{},
+	}
+}
+
+// RouteInfo.Routes returns a slice containing all the routes stored from the wanted resources.
+func (ri *RouteInfo) Routes(control, discovered bool) []netip.Prefix {
+	if ri == nil {
+		return []netip.Prefix{}
+	}
+	var ret []netip.Prefix
+	if control && len(ret) == 0 {
+		ret = ri.Control
+	} else if control {
+		ret = append(ret, ri.Control...)
+	}
+
+	if discovered {
+		for _, dr := range ri.Discovered {
+			if dr != nil {
+				ret = append(ret, dr.RoutesSlice()...)
+			}
+		}
+	}
+	return ret
+}
+
+func (ri *RouteInfo) DomainRoutes() map[string][]netip.Addr {
+	drCopy := make(map[string][]netip.Addr)
+	for k, v := range ri.Discovered {
+		drCopy[k] = append(drCopy[k], v.AddrsSlice()...)
+	}
+	return drCopy
+}
+
+type DatedRoutes struct {
+	// routes discovered for a domain, and when they were last seen in a dns query
+	Routes map[netip.Prefix]time.Time
+	// the time at which we last expired old routes
+	LastCleanup time.Time
+}
+
+func (dr *DatedRoutes) RoutesSlice() []netip.Prefix {
+	var routes []netip.Prefix
+	for k := range dr.Routes {
+		routes = append(routes, k)
+	}
+	return routes
+}
+
+func (dr *DatedRoutes) AddrsSlice() []netip.Addr {
+	var routes []netip.Addr
+	for k := range dr.Routes {
+		if k.IsSingleIP() {
+			routes = append(routes, k.Addr())
+		}
+	}
+	return routes
+}
+
+func (r *RouteInfo) AddRoutesInDiscoveredForDomain(domain string, addrs []netip.Prefix) {
+	dr, hasKey := r.Discovered[domain]
+	if !hasKey || dr == nil || dr.Routes == nil {
+		newDatedRoutes := &DatedRoutes{make(map[netip.Prefix]time.Time), time.Now()}
+		newDatedRoutes.addAddrsToDatedRoute(addrs)
+		r.Discovered[domain] = newDatedRoutes
+		return
+	}
+
+	// kevin comment: we won't see any existing routes here because know addrs are filtered.
+	currentRoutes := r.Discovered[domain]
+	currentRoutes.addAddrsToDatedRoute(addrs)
+	r.Discovered[domain] = currentRoutes
+	return
+}
+
+func (d *DatedRoutes) addAddrsToDatedRoute(addrs []netip.Prefix) {
+	time := time.Now()
+	for _, addr := range addrs {
+		d.Routes[addr] = time
+	}
+}

client/tailscale/localclient.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"go4.org/mem"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/drive"
 	"tailscale.com/envknob"
@@ -722,6 +723,18 @@ func (lc *LocalClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) {
 	return &p, nil
 }
 
+func (lc *LocalClient) GetRouteInfo(ctx context.Context) (*routeinfo.RouteInfo, error) {
+	body, err := lc.get200(ctx, "/localapi/v0/routeInfo")
+	if err != nil {
+		return nil, err
+	}
+	var p routeinfo.RouteInfo
+	if err := json.Unmarshal(body, &p); err != nil {
+		return nil, fmt.Errorf("invalid prefs JSON: %w", err)
+	}
+	return &p, nil
+}
+
 func (lc *LocalClient) EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) {
 	body, err := lc.send(ctx, "PATCH", "/localapi/v0/prefs", http.StatusOK, jsonBody(mp))
 	if err != nil {

cmd/tailscale/cli/debug.go

@@ -316,6 +316,12 @@ var debugCmd = &ffcli.Command{
 				return fs
 			})(),
 		},
+		{
+			Name:       "appc-routes",
+			ShortUsage: "tailscale debug appc-routes",
+			Exec:       runDebugAppc,
+			ShortHelp:  "Prints the routes the node is advertising if it is running as an appConnector. ",
+		},
 	},
 }
 
@@ -1115,3 +1121,14 @@ func runDebugDialTypes(ctx context.Context, args []string) error {
 	fmt.Printf("%s", body)
 	return nil
 }
+
+func runDebugAppc(ctx context.Context, args []string) error {
+	prefs, err := localClient.GetRouteInfo(ctx)
+	if err != nil {
+		return err
+	}
+	j, _ := json.MarshalIndent(prefs, "", "\t")
+	outln(string(j))
+
+	return nil
+}

control/controlknobs/controlknobs.go

@@ -72,6 +72,10 @@ type Knobs struct {
 	// ProbeUDPLifetime is whether the node should probe UDP path lifetime on
 	// the tail end of an active direct connection in magicsock.
 	ProbeUDPLifetime atomic.Bool
+
+	// AppCStoreRoutes is whether the node should store RouteInfo to StateStore
+	// if it's an app connector.
+	AppCStoreRoutes atomic.Bool
 }
 
 // UpdateFromNodeAttributes updates k (if non-nil) based on the provided self
@@ -96,6 +100,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 		forceNfTables                 = has(tailcfg.NodeAttrLinuxMustUseNfTables)
 		seamlessKeyRenewal            = has(tailcfg.NodeAttrSeamlessKeyRenewal)
 		probeUDPLifetime              = has(tailcfg.NodeAttrProbeUDPLifetime)
+		appCStoreRoutes               = has(tailcfg.NodeAttrStoreAppCRoutes)
 	)
 
 	if has(tailcfg.NodeAttrOneCGNATEnable) {
@@ -118,6 +123,7 @@ func (k *Knobs) UpdateFromNodeAttributes(capMap tailcfg.NodeCapMap) {
 	k.LinuxForceNfTables.Store(forceNfTables)
 	k.SeamlessKeyRenewal.Store(seamlessKeyRenewal)
 	k.ProbeUDPLifetime.Store(probeUDPLifetime)
+	k.AppCStoreRoutes.Store(appCStoreRoutes)
 }
 
 // AsDebugJSON returns k as something that can be marshalled with json.Marshal
@@ -141,5 +147,6 @@ func (k *Knobs) AsDebugJSON() map[string]any {
 		"LinuxForceNfTables":            k.LinuxForceNfTables.Load(),
 		"SeamlessKeyRenewal":            k.SeamlessKeyRenewal.Load(),
 		"ProbeUDPLifetime":              k.ProbeUDPLifetime.Load(),
+		"AppCStoreRoutes":               k.AppCStoreRoutes.Load(),
 	}
 }

ipn/ipnlocal/local.go

@@ -35,6 +35,7 @@ import (
 	xmaps "golang.org/x/exp/maps"
 	"gvisor.dev/gvisor/pkg/tcpip"
 	"tailscale.com/appc"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/control/controlclient"
@@ -3555,13 +3556,27 @@ func (b *LocalBackend) reconfigAppConnectorLocked(nm *netmap.NetworkMap, prefs i
 		}
 	}()
 
+	shouldAppCStoreRoutesHasChanged := false
+	shouldAppCStoreRoutes := b.ControlKnobs().AppCStoreRoutes.Load()
+	if b.appConnector != nil {
+		shouldAppCStoreRoutesHasChanged = b.appConnector.ShouldStoreRoutes != shouldAppCStoreRoutes
+	}
+
 	if !prefs.AppConnector().Advertise {
+		if b.appConnector != nil && shouldAppCStoreRoutes {
+			b.appConnector.UnadvertiseRemoteRoutes()
+		}
 		b.appConnector = nil
 		return
 	}
 
-	if b.appConnector == nil {
-		b.appConnector = appc.NewAppConnector(b.logf, b)
+	if b.appConnector == nil || shouldAppCStoreRoutesHasChanged {
+		b.appConnector = appc.NewAppConnector(b.logf, b, shouldAppCStoreRoutes)
+		if shouldAppCStoreRoutes {
+			b.appConnector.RecreateRouteInfoFromStore()
+		} else if shouldAppCStoreRoutesHasChanged && !shouldAppCStoreRoutes {
+			b.appConnector.UpdateRouteInfo(nil)
+		}
 	}
 	if nm == nil {
 		return
@@ -3615,7 +3630,6 @@ func (b *LocalBackend) authReconfig() {
 	// If the current node is an app connector, ensure the app connector machine is started
 	b.reconfigAppConnectorLocked(nm, prefs)
 	b.mu.Unlock()
-
 	if blocked {
 		b.logf("[v1] authReconfig: blocked, skipping.")
 		return
@@ -3642,7 +3656,6 @@ func (b *LocalBackend) authReconfig() {
 			flags &^= netmap.AllowSubnetRoutes
 		}
 	}
-
 	// Keep the dialer updated about whether we're supposed to use
 	// an exit node's DNS server (so SOCKS5/HTTP outgoing dials
 	// can use it for name resolution)
@@ -3657,8 +3670,8 @@ func (b *LocalBackend) authReconfig() {
 		b.logf("wgcfg: %v", err)
 		return
 	}
-
 	oneCGNATRoute := shouldUseOneCGNATRoute(b.logf, b.sys.ControlKnobs(), version.OS())
+
 	rcfg := b.routerConfig(cfg, prefs, oneCGNATRoute)
 
 	err = b.e.Reconfig(cfg, rcfg, dcfg)
@@ -4166,9 +4179,13 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs ipn.PrefsView, oneC
 		netfilterKind = prefs.NetfilterKind()
 	}
 
+	b.mu.Lock()
+	toAdvertise := b.appConnector.RouteInfo().Routes(true, true)
+	b.mu.Unlock()
+	toAdvertise = append(toAdvertise, prefs.AdvertiseRoutes().AsSlice()...)
 	rs := &router.Config{
 		LocalAddrs:       unmapIPPrefixes(cfg.Addresses),
-		SubnetRoutes:     unmapIPPrefixes(prefs.AdvertiseRoutes().AsSlice()),
+		SubnetRoutes:     unmapIPPrefixes(toAdvertise),
 		SNATSubnetRoutes: !prefs.NoSNAT(),
 		NetfilterMode:    prefs.NetfilterMode(),
 		Routes:           peerRoutes(b.logf, cfg.Peers, singleRouteThreshold),
@@ -4252,7 +4269,13 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip
 	if h := prefs.Hostname(); h != "" {
 		hi.Hostname = h
 	}
-	hi.RoutableIPs = prefs.AdvertiseRoutes().AsSlice()
+
+	var routableIPs []netip.Prefix
+	if b.appConnector != nil {
+		routableIPs = b.appConnector.RouteInfo().Routes(true, true)
+	}
+	routableIPs = append(routableIPs, prefs.AdvertiseRoutes().AsSlice()...)
+	hi.RoutableIPs = routableIPs
 	hi.RequestTags = prefs.AdvertiseTags().AsSlice()
 	hi.ShieldsUp = prefs.ShieldsUp()
 	hi.AllowsUpdate = envknob.AllowsRemoteUpdate() || prefs.AutoUpdate().Apply.EqualBool(true)
@@ -6075,6 +6098,16 @@ func (b *LocalBackend) StreamDebugCapture(ctx context.Context, w io.Writer) erro
 	return nil
 }
 
+func (b *LocalBackend) AppcRouteInfo() *routeinfo.RouteInfo {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.appConnector == nil {
+		return nil
+	}
+	ri := b.appConnector.RouteInfo()
+	return ri
+}
+
 func (b *LocalBackend) GetPeerEndpointChanges(ctx context.Context, ip netip.Addr) ([]magicsock.EndpointChange, error) {
 	pip, ok := b.e.PeerForIP(ip)
 	if !ok {
@@ -6174,38 +6207,50 @@ var ErrDisallowedAutoRoute = errors.New("route is not allowed")
 // AdvertiseRoute implements the appc.RouteAdvertiser interface. It sets a new
 // route advertisement if one is not already present in the existing routes.
 // If the route is disallowed, ErrDisallowedAutoRoute is returned.
-func (b *LocalBackend) AdvertiseRoute(ipps ...netip.Prefix) error {
-	finalRoutes := b.Prefs().AdvertiseRoutes().AsSlice()
-	newRoutes := false
+func (b *LocalBackend) AdvertiseRouteInfo(ri *routeinfo.RouteInfo) {
+	b.mu.Lock()
+	pref := b.pm.CurrentPrefs()
+	newRoutes := pref.AdvertiseRoutes().AsSlice()
+	oldHi := b.hostinfo
+	oldRoutes := oldHi.RoutableIPs
+	newHi := oldHi.Clone()
+	if newHi == nil {
+		newHi = new(tailcfg.Hostinfo)
+	}
+	routeInfoRoutes := ri.Routes(true, true)
 
-	for _, ipp := range ipps {
+	for _, ipp := range routeInfoRoutes {
 		if !allowedAutoRoute(ipp) {
 			continue
 		}
-		if slices.Contains(finalRoutes, ipp) {
+		if slices.Contains(newRoutes, ipp) {
 			continue
 		}
 
 		// If the new prefix is already contained by existing routes, skip it.
-		if coveredRouteRangeNoDefault(finalRoutes, ipp) {
+		if coveredRouteRangeNoDefault(newRoutes, ipp) {
 			continue
 		}
 
-		finalRoutes = append(finalRoutes, ipp)
-		newRoutes = true
+		newRoutes = append(newRoutes, ipp)
 	}
 
-	if !newRoutes {
-		return nil
+	slices.SortFunc(oldRoutes, comparePrefix)
+	slices.SortFunc(newRoutes, comparePrefix)
+
+	if slices.CompareFunc(oldRoutes, newRoutes, comparePrefix) != 0 {
+		return
 	}
 
-	_, err := b.EditPrefs(&ipn.MaskedPrefs{
-		Prefs: ipn.Prefs{
-			AdvertiseRoutes: finalRoutes,
-		},
-		AdvertiseRoutesSet: true,
-	})
-	return err
+	newHi.RoutableIPs = newRoutes
+	b.hostinfo = newHi
+
+	if !oldHi.Equal(newHi) {
+		b.doSetHostinfoFilterServices()
+	}
+	b.mu.Unlock()
+
+	b.authReconfig()
 }
 
 // coveredRouteRangeNoDefault checks if a route is already included in a slice of
@@ -6228,26 +6273,47 @@ func coveredRouteRangeNoDefault(finalRoutes []netip.Prefix, ipp netip.Prefix) bo
 	return false
 }
 
-// UnadvertiseRoute implements the appc.RouteAdvertiser interface. It removes
-// a route advertisement if one is present in the existing routes.
-func (b *LocalBackend) UnadvertiseRoute(toRemove ...netip.Prefix) error {
-	currentRoutes := b.Prefs().AdvertiseRoutes().AsSlice()
-	finalRoutes := currentRoutes[:0]
+// namespace a key with the profile manager's current profile key, if any
+func namespaceKeyForCurrentProfile(pm *profileManager, key ipn.StateKey) ipn.StateKey {
+	return pm.CurrentProfile().Key + "||" + key
+}
 
-	for _, ipp := range currentRoutes {
-		if slices.Contains(toRemove, ipp) {
-			continue
-		}
-		finalRoutes = append(finalRoutes, ipp)
+const routeInfoStateStoreKey ipn.StateKey = "_routeInfo"
+
+// StoreRouteInfo implements the appc.RouteAdvertiser interface. It stores
+// RouteInfo to StateStore per profile.
+func (b *LocalBackend) StoreRouteInfo(ri *routeinfo.RouteInfo) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.pm.CurrentProfile().ID == "" {
+		return nil
 	}
+	key := namespaceKeyForCurrentProfile(b.pm, routeInfoStateStoreKey)
+	bs, err := json.Marshal(ri)
+	if err != nil {
+		return err
+	}
+	return b.pm.WriteState(key, bs)
+}
 
-	_, err := b.EditPrefs(&ipn.MaskedPrefs{
-		Prefs: ipn.Prefs{
-			AdvertiseRoutes: finalRoutes,
-		},
-		AdvertiseRoutesSet: true,
-	})
-	return err
+// ReadRouteInfo implements the appc.RouteAdvertiser interface. It reads
+// RouteInfo from StateStore per profile.
+func (b *LocalBackend) ReadRouteInfo() (*routeinfo.RouteInfo, error) {
+	// b.mu.Lock()
+	// defer b.mu.Unlock()
+	if b.pm.CurrentProfile().ID == "" {
+		return &routeinfo.RouteInfo{}, nil
+	}
+	key := namespaceKeyForCurrentProfile(b.pm, routeInfoStateStoreKey)
+	bs, err := b.pm.Store().ReadState(key)
+	ri := &routeinfo.RouteInfo{}
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(bs, ri); err != nil {
+		return nil, err
+	}
+	return ri, nil
 }
 
 // seamlessRenewalEnabled reports whether seamless key renewals are enabled
@@ -6295,3 +6361,7 @@ func mayDeref[T any](p *T) (v T) {
 	}
 	return *p
 }
+
+func comparePrefix(i, j netip.Prefix) int {
+	return i.Addr().Compare(j.Addr())
+}

ipn/ipnlocal/local_test.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/net/dns/dnsmessage"
 	"tailscale.com/appc"
 	"tailscale.com/appc/appctest"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/drive"
 	"tailscale.com/drive/driveimpl"
@@ -1252,13 +1253,15 @@ func TestDNSConfigForNetmapForExitNodeConfigs(t *testing.T) {
 }
 
 func TestOfferingAppConnector(t *testing.T) {
-	b := newTestBackend(t)
-	if b.OfferingAppConnector() {
-		t.Fatal("unexpected offering app connector")
-	}
-	b.appConnector = appc.NewAppConnector(t.Logf, nil)
-	if !b.OfferingAppConnector() {
-		t.Fatal("unexpected not offering app connector")
+	for _, shouldStore := range []bool{true, false} {
+		b := newTestBackend(t)
+		if b.OfferingAppConnector() {
+			t.Fatal("unexpected offering app connector")
+		}
+		b.appConnector = appc.NewAppConnector(t.Logf, nil, shouldStore)
+		if !b.OfferingAppConnector() {
+			t.Fatal("unexpected not offering app connector")
+		}
 	}
 }
 
@@ -1303,21 +1306,23 @@ func TestRouterAdvertiserIgnoresContainedRoutes(t *testing.T) {
 }
 
 func TestObserveDNSResponse(t *testing.T) {
-	b := newTestBackend(t)
+	for _, shouldStore := range []bool{true, false} {
+		b := newTestBackend(t)
 
-	// ensure no error when no app connector is configured
-	b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
+		// ensure no error when no app connector is configured
+		b.ObserveDNSResponse(dnsResponse("example.com.", []string{"192.0.0.8"}))
 
-	rc := &appctest.RouteCollector{}
-	b.appConnector = appc.NewAppConnector(t.Logf, rc)
-	b.appConnector.UpdateDomains([]string{"example.com"})
-	b.appConnector.Wait(context.Background())
+		rc := &appctest.RouteCollector{}
+		b.appConnector = appc.NewAppConnector(t.Logf, rc, shouldStore)
+		b.appConnector.UpdateDomains([]string{"example.com"})
+		b.appConnector.Wait(context.Background())
 
-	b.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
-	b.appConnector.Wait(context.Background())
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Fatalf("got routes %v, want %v", rc.Routes(), wantRoutes)
+		b.ObserveDNSResponse(dnsResponse("example.com.", []string{"192.0.0.8"}))
+		b.appConnector.Wait(context.Background())
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Fatalf("got routes %v, want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 
@@ -1466,38 +1471,41 @@ func routesEqual(t *testing.T, a, b map[dnsname.FQDN][]*dnstype.Resolver) bool {
 }
 
 // dnsResponse is a test helper that creates a DNS response buffer for the given domain and address
-func dnsResponse(domain, address string) []byte {
-	addr := netip.MustParseAddr(address)
+// func dnsResponse(domain, address string) []byte {
+func dnsResponse(domain string, addresses []string) []byte {
 	b := dnsmessage.NewBuilder(nil, dnsmessage.Header{})
 	b.EnableCompression()
 	b.StartAnswers()
-	switch addr.BitLen() {
-	case 32:
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName(domain),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: addr.As4(),
-			},
-		)
-	case 128:
-		b.AAAAResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName(domain),
-				Type:  dnsmessage.TypeAAAA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AAAAResource{
-				AAAA: addr.As16(),
-			},
-		)
-	default:
-		panic("invalid address length")
+	for _, address := range addresses {
+		addr := netip.MustParseAddr(address)
+		switch addr.BitLen() {
+		case 32:
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName(domain),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: addr.As4(),
+				},
+			)
+		case 128:
+			b.AAAAResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName(domain),
+					Type:  dnsmessage.TypeAAAA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AAAAResource{
+					AAAA: addr.As16(),
+				},
+			)
+		default:
+			panic("invalid address length")
+		}
 	}
 	return must.Get(b.Finish())
 }
@@ -2634,3 +2642,478 @@ func (b *LocalBackend) SetPrefsForTest(newp *ipn.Prefs) {
 	defer unlock()
 	b.setPrefsLockedOnEntry(newp, unlock)
 }
+func TestReadWriteRouteInfo(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+	now := time.Now()
+	discovered := make(map[string]*routeinfo.DatedRoutes)
+	routes := make(map[netip.Prefix]time.Time)
+	routes[prefix3] = now
+	discovered["example.com"] = &routeinfo.DatedRoutes{
+		LastCleanup: now,
+		Routes:      routes,
+	}
+	b := newTestBackend(t)
+	ri := routeinfo.RouteInfo{
+		Local:      []netip.Prefix{prefix1},
+		Control:    []netip.Prefix{prefix2},
+		Discovered: discovered,
+	}
+	if err := b.StoreRouteInfo(&ri); err != nil {
+		t.Fatal(err)
+	}
+	readRi, err := b.ReadRouteInfo()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(readRi.Local) != 1 || len(readRi.Control) != 1 || len(readRi.Discovered) != 1 {
+		t.Fatal("read Ri expected to be same shape as ri")
+	}
+	if readRi.Local[0] != ri.Local[0] {
+		t.Fatalf("wanted %v, got %v", ri.Local[0], readRi.Local[0])
+	}
+	if readRi.Control[0] != ri.Control[0] {
+		t.Fatalf("wanted %v, got %v", ri.Control[0], readRi.Control[0])
+	}
+	dr := readRi.Discovered["example.com"]
+	if dr.LastCleanup.Compare(now) != 0 {
+		t.Fatalf("wanted %v, got %v", now, dr.LastCleanup)
+	}
+	if len(dr.Routes) != 1 {
+		t.Fatalf("read Ri expected to be same shape as ri")
+	}
+	if dr.Routes[prefix3].Compare(routes[prefix3]) != 0 {
+		t.Fatalf("wanted %v, got %v", routes[prefix3], dr.Routes[prefix3])
+	}
+}
+
+func TestPatchPrefsHandlerWithPresistStore(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+	mp := new(ipn.MaskedPrefs)
+	mp.AdvertiseRoutesSet = true
+	mp.AdvertiseRoutes = []netip.Prefix{prefix1}
+
+	b := newTestBackend(t)
+	testAppConnector := appc.NewAppConnector(t.Logf, b, true)
+	b.appConnector = testAppConnector
+	b.ControlKnobs().AppCStoreRoutes.Store(true)
+
+	b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AppConnector: ipn.AppConnectorPrefs{
+				Advertise: true,
+			},
+		},
+		AppConnectorSet: true,
+	})
+
+	appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"],
+		"routes":[%s]
+	}`
+	appCfg := fmt.Sprintf(appCfgTmpl, `"example.com"`, `"1.2.3.5/32"`)
+
+	reconfigWithAppCfg := func(appCfg string) {
+		b.netMap.SelfNode = (&tailcfg.Node{
+			Name: "example.ts.net",
+			Tags: []string{"tag:example"},
+			CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+				"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+			}),
+		}).View()
+
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		b.appConnector.Wait(context.Background())
+	}
+
+	reconfigWithAppCfg(appCfg)
+
+	now := time.Now()
+	ri := routeinfo.NewRouteInfo()
+	ri.Control = []netip.Prefix{prefix2}
+	discovered := make(map[string]*routeinfo.DatedRoutes)
+	routes := make(map[netip.Prefix]time.Time)
+	routes[prefix3] = now
+	discovered["example.com"] = &routeinfo.DatedRoutes{
+		LastCleanup: now,
+		Routes:      routes,
+	}
+	ri.Discovered = discovered
+
+	b.appConnector.UpdateRouteInfo(ri)
+	b.AdvertiseRoute([]netip.Prefix{prefix3}...)
+
+	prefView, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView.AdvertiseRoutes().Len() != 3 {
+		t.Fatalf("wanted %d, got %d", 3, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("New prefix was not advertised")
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+	//Check if route is stored in Appc/Appc.routeAdvertiser
+	storedRouteInfo, _ := b.ReadRouteInfo()
+	if len(storedRouteInfo.Local) != 1 {
+		t.Fatalf("wanted %d, got %d", 1, len(storedRouteInfo.Local))
+	}
+	if !slices.Contains(storedRouteInfo.Local, prefix1) {
+		t.Fatalf("New local route not stored.")
+	}
+
+	//Check if the routes in control and discovered are presisted.
+	routesShouldPresist := storedRouteInfo.Routes(false, true, true)
+	if len(routesShouldPresist) != 2 {
+		t.Fatalf("wanted %d, got %d", 2, len(routesShouldPresist))
+	}
+	if !slices.Contains(routesShouldPresist, prefix2) || !slices.Contains(routesShouldPresist, prefix3) {
+		t.Fatalf("Pre-existed routes not presisted.")
+	}
+
+	//Patch again with no route, see if prefix1 is removed/ prefix2, prefix3 presists.
+	mp.AdvertiseRoutes = []netip.Prefix{}
+	prefView2, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView2.AdvertiseRoutes().Len() != 2 {
+		t.Fatalf("wanted %d, got %d", 2, prefView.AdvertiseRoutes().Len())
+	}
+
+	if slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("Local route was not removed")
+	}
+
+	if !slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+}
+
+func TestPatchPrefsHandlerWithoutPresistStore(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+
+	mp := new(ipn.MaskedPrefs)
+	mp.AdvertiseRoutesSet = true
+	mp.AdvertiseRoutes = []netip.Prefix{prefix1}
+
+	b := newTestBackend(t)
+	testAppConnector := appc.NewAppConnector(t.Logf, b, false)
+	b.appConnector = testAppConnector
+	b.ControlKnobs().AppCStoreRoutes.Store(false)
+
+	b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AppConnector: ipn.AppConnectorPrefs{
+				Advertise: true,
+			},
+		},
+		AppConnectorSet: true,
+	})
+
+	appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"],
+		"routes":[%s]
+	}`
+	appCfg := fmt.Sprintf(appCfgTmpl, `"example.com"`, `"1.2.3.5/32"`)
+
+	reconfigWithAppCfg := func(appCfg string) {
+		b.netMap.SelfNode = (&tailcfg.Node{
+			Name: "example.ts.net",
+			Tags: []string{"tag:example"},
+			CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+				"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+			}),
+		}).View()
+
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		b.appConnector.Wait(context.Background())
+	}
+	reconfigWithAppCfg(appCfg)
+
+	b.AdvertiseRoute([]netip.Prefix{prefix3}...)
+
+	testAppConnector.RouteInfo()
+
+	prefView, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView.AdvertiseRoutes().Len() != 1 {
+		t.Fatalf("wanted %d, got %d", 1, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("New prefix was not advertised")
+	}
+
+	if slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix2) || slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are still advertised.")
+	}
+
+	//Check if route is stored in Appc/Appc.routeAdvertiser
+	storedRouteInfo, _ := b.ReadRouteInfo()
+	if storedRouteInfo != nil {
+		t.Fatalf("wanted nil, got %v", storedRouteInfo)
+	}
+
+	//Patch again with no route, see if prefix1 is removed/ prefix2, prefix3 presists.
+	mp.AdvertiseRoutes = []netip.Prefix{}
+	prefView2, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView2.AdvertiseRoutes().Len() != 0 {
+		t.Fatalf("wanted %d, got %d", 0, prefView.AdvertiseRoutes().Len())
+	}
+
+	if slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("Local route was not removed")
+	}
+
+	if slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix2) || slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are still advertised.")
+	}
+
+}
+
+func TestFran(t *testing.T) {
+	prefixSortFunc := func(i, j netip.Prefix) int {
+		return i.Addr().Compare(j.Addr())
+	}
+	epIP2 := netip.MustParsePrefix("192.1.0.9/32")
+	explicitlyAdvertisedRoutes := []netip.Prefix{netip.MustParsePrefix("192.1.0.8/32"), epIP2}
+	oneRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32"), netip.MustParsePrefix("192.0.0.16/32")}
+	twoRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.10/32"), netip.MustParsePrefix("192.0.0.18/32"), epIP2}
+
+	for _, shouldStore := range []bool{true, false} {
+		b := newTestBackend(t)
+		b.ControlKnobs().AppCStoreRoutes.Store(shouldStore)
+		// make b an app connector
+		b.EditPrefs(&ipn.MaskedPrefs{
+			Prefs: ipn.Prefs{
+				AppConnector: ipn.AppConnectorPrefs{
+					Advertise: true,
+				},
+				AdvertiseRoutes: explicitlyAdvertisedRoutes,
+			},
+			AppConnectorSet:    true,
+			AdvertiseRoutesSet: true,
+		})
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+
+		appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"]
+	}`
+		appCfg1Domain := fmt.Sprintf(appCfgTmpl, `"one.com"`)
+		appCfg2Domains := fmt.Sprintf(appCfgTmpl, `"one.com", "two.com"`)
+
+		reconfigWithAppCfg := func(appCfg string) {
+			b.netMap.SelfNode = (&tailcfg.Node{
+				Name: "example.ts.net",
+				Tags: []string{"tag:example"},
+				CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+					"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+				}),
+			}).View()
+
+			b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+			b.appConnector.Wait(context.Background())
+		}
+
+		// set the app connector to watch 2 domains
+		reconfigWithAppCfg(appCfg2Domains)
+
+		// when the app connector observes dns for the domains it adds routes
+		afterTwoRoutes := append([]netip.Prefix{}, explicitlyAdvertisedRoutes...)
+		afterTwoRoutes = append(afterTwoRoutes, oneRoutes...)
+		afterTwoRoutes = append(afterTwoRoutes, twoRoutes...)
+		slices.SortFunc(afterTwoRoutes, prefixSortFunc)
+		afterTwoRoutes = slices.Compact(afterTwoRoutes)
+
+		afterOneRoutes := append([]netip.Prefix{}, explicitlyAdvertisedRoutes...)
+		afterOneRoutes = append(afterOneRoutes, oneRoutes...)
+		slices.SortFunc(afterOneRoutes, prefixSortFunc)
+		afterOneRoutes = slices.Compact(afterOneRoutes)
+
+		for _, tst := range []struct {
+			domain    string
+			route     []string
+			wantAfter []netip.Prefix
+		}{
+			// learns the route for one.com
+			{domain: "one.com", route: []string{"192.0.0.8", "192.0.0.16"}, wantAfter: afterOneRoutes},
+			// doesn't care about example.com, so still just the route for one.com
+			{domain: "example.com", route: []string{"192.0.0.9", "192.0.0.17"}, wantAfter: afterOneRoutes},
+			// learns the route for two.com as well
+			{domain: "two.com", route: []string{"192.0.0.10", "192.0.0.18", "192.1.0.9"}, wantAfter: afterTwoRoutes},
+		} {
+			b.ObserveDNSResponse(dnsResponse(tst.domain+".", tst.route))
+			b.appConnector.Wait(context.Background())
+			routesNow := b.pm.prefs.AdvertiseRoutes().AsSlice()
+			slices.SortFunc(routesNow, prefixSortFunc)
+			if !slices.Equal(routesNow, tst.wantAfter) {
+				t.Fatalf("after dns response for %s got routes %v, want %v", tst.domain, routesNow, tst.wantAfter)
+			}
+		}
+
+		// reconfigure the app connector to observe fewer domains
+		reconfigWithAppCfg(appCfg1Domain)
+		routesNow := b.pm.prefs.AdvertiseRoutes().AsSlice()
+		wantRoutes := afterOneRoutes
+		if !shouldStore {
+			wantRoutes = afterTwoRoutes
+		}
+		slices.SortFunc(routesNow, prefixSortFunc)
+		if !slices.Equal(routesNow, wantRoutes) {
+			t.Fatalf("after removing two.com (shouldStore=%t), got %v, want %v", shouldStore, routesNow, wantRoutes)
+		}
+
+		ac := b.appConnector
+		// heck forget about being an app connector
+		b.EditPrefs(&ipn.MaskedPrefs{
+			Prefs: ipn.Prefs{
+				AppConnector: ipn.AppConnectorPrefs{
+					Advertise: false,
+				},
+			},
+			AppConnectorSet: true,
+		})
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		ac.Wait(context.Background())
+
+		routesNow = b.pm.prefs.AdvertiseRoutes().AsSlice()
+		wantRoutes = explicitlyAdvertisedRoutes
+		if !shouldStore {
+			wantRoutes = afterTwoRoutes
+		}
+		slices.SortFunc(routesNow, prefixSortFunc)
+		if !slices.Equal(routesNow, wantRoutes) {
+			t.Fatalf("after becoming not an app connector got routes %v, want %v", routesNow, wantRoutes)
+		}
+	}
+}
+
+func TestPatchPrefsHandlerNoDuplicateAdvertisement(t *testing.T) {
+	// test can read what's written
+	prefix1 := netip.MustParsePrefix("1.2.3.4/32")
+	prefix2 := netip.MustParsePrefix("1.2.3.5/32")
+	prefix3 := netip.MustParsePrefix("1.2.3.6/32")
+	mp := new(ipn.MaskedPrefs)
+	mp.AdvertiseRoutesSet = true
+	mp.AdvertiseRoutes = []netip.Prefix{prefix1}
+
+	b := newTestBackend(t)
+	testAppConnector := appc.NewAppConnector(t.Logf, b, true)
+	b.appConnector = testAppConnector
+	b.ControlKnobs().AppCStoreRoutes.Store(true)
+
+	b.EditPrefs(&ipn.MaskedPrefs{
+		Prefs: ipn.Prefs{
+			AppConnector: ipn.AppConnectorPrefs{
+				Advertise: true,
+			},
+		},
+		AppConnectorSet: true,
+	})
+
+	appCfgTmpl := `{
+		"name": "example",
+		"domains": [%s],
+		"connectors": ["tag:example"],
+		"routes":[%s]
+	}`
+	appCfg := fmt.Sprintf(appCfgTmpl, `"example.com"`, `"1.2.3.4/32", "1.2.3.5/32"`)
+
+	reconfigWithAppCfg := func(appCfg string) {
+		b.netMap.SelfNode = (&tailcfg.Node{
+			Name: "example.ts.net",
+			Tags: []string{"tag:example"},
+			CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{
+				"tailscale.com/app-connectors": {tailcfg.RawMessage(appCfg)},
+			}),
+		}).View()
+
+		b.reconfigAppConnectorLocked(b.netMap, b.pm.prefs)
+		b.appConnector.Wait(context.Background())
+	}
+
+	reconfigWithAppCfg(appCfg)
+
+	now := time.Now()
+	ri := routeinfo.NewRouteInfo()
+	ri.Control = []netip.Prefix{prefix1, prefix2}
+	discovered := make(map[string]*routeinfo.DatedRoutes)
+	routes := make(map[netip.Prefix]time.Time)
+	routes[prefix3] = now
+	discovered["example.com"] = &routeinfo.DatedRoutes{
+		LastCleanup: now,
+		Routes:      routes,
+	}
+	ri.Discovered = discovered
+
+	b.appConnector.UpdateRouteInfo(ri)
+	b.AdvertiseRoute([]netip.Prefix{prefix3}...)
+
+	prefView, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView.AdvertiseRoutes().Len() != 3 {
+		t.Fatalf("wanted %d, got %d", 3, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("New prefix was not advertised")
+	}
+
+	if !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+	//Patch again with no route, see if prefix1 is removed/ prefix2, prefix3 presists.
+	mp.AdvertiseRoutes = []netip.Prefix{}
+	prefView2, err := b.PatchPrefsHandler(mp)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	if prefView2.AdvertiseRoutes().Len() != 3 {
+		t.Fatalf("wanted %d, got %d", 3, prefView.AdvertiseRoutes().Len())
+	}
+
+	if !slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix1) {
+		t.Fatalf("Local route was not removed")
+	}
+
+	if !slices.Contains(prefView2.AdvertiseRoutes().AsSlice(), prefix2) || !slices.Contains(prefView.AdvertiseRoutes().AsSlice(), prefix3) {
+		t.Fatalf("Old prefixes are no longer advertised.")
+	}
+
+}

ipn/ipnlocal/peerapi_test.go

@@ -687,185 +687,191 @@ func TestPeerAPIReplyToDNSQueries(t *testing.T) {
 }
 
 func TestPeerAPIPrettyReplyCNAME(t *testing.T) {
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{true, false} {
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:     eng,
-			pm:    pm,
-			store: pm.Store(),
-			// configure as an app connector just to enable the API.
-			appConnector: appc.NewAppConnector(t.Logf, &appctest.RouteCollector{}),
-		},
-	}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:     eng,
+				pm:    pm,
+				store: pm.Store(),
+				// configure as an app connector just to enable the API.
+				appConnector: appc.NewAppConnector(t.Logf, &appctest.RouteCollector{}, shouldStore),
+			},
+		}
 
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.CNAMEResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("www.example.com."),
-				Type:  dnsmessage.TypeCNAME,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.CNAMEResource{
-				CNAME: dnsmessage.MustNewName("example.com."),
-			},
-		)
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.CNAMEResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("www.example.com."),
+					Type:  dnsmessage.TypeCNAME,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.CNAMEResource{
+					CNAME: dnsmessage.MustNewName("example.com."),
+				},
+			)
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	var addrs []string
-	json.NewDecoder(w.Body).Decode(&addrs)
-	if len(addrs) == 0 {
-		t.Fatalf("no addresses returned")
-	}
-	for _, addr := range addrs {
-		netip.MustParseAddr(addr)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		var addrs []string
+		json.NewDecoder(w.Body).Decode(&addrs)
+		if len(addrs) == 0 {
+			t.Fatalf("no addresses returned")
+		}
+		for _, addr := range addrs {
+			netip.MustParseAddr(addr)
+		}
 	}
 }
 
 func TestPeerAPIReplyToDNSQueriesAreObserved(t *testing.T) {
-	ctx := context.Background()
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	rc := &appctest.RouteCollector{}
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:            eng,
-			pm:           pm,
-			store:        pm.Store(),
-			appConnector: appc.NewAppConnector(t.Logf, rc),
-		},
-	}
-	h.ps.b.appConnector.UpdateDomains([]string{"example.com"})
-	h.ps.b.appConnector.Wait(ctx)
-
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
+		rc := &appctest.RouteCollector{}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:            eng,
+				pm:           pm,
+				store:        pm.Store(),
+				appConnector: appc.NewAppConnector(t.Logf, rc, shouldStore),
 			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		}
+		h.ps.b.appConnector.UpdateDomains([]string{"example.com"})
+		h.ps.b.appConnector.Wait(ctx)
 
-	if !h.ps.b.OfferingAppConnector() {
-		t.Fatal("expecting to be offering app connector")
-	}
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	h.ps.b.appConnector.Wait(ctx)
+		if !h.ps.b.OfferingAppConnector() {
+			t.Fatal("expecting to be offering app connector")
+		}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		h.ps.b.appConnector.Wait(ctx)
+
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 
 func TestPeerAPIReplyToDNSQueriesAreObservedWithCNAMEFlattening(t *testing.T) {
-	ctx := context.Background()
-	var h peerAPIHandler
-	h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
+	for _, shouldStore := range []bool{true, false} {
+		ctx := context.Background()
+		var h peerAPIHandler
+		h.remoteAddr = netip.MustParseAddrPort("100.150.151.152:12345")
 
-	rc := &appctest.RouteCollector{}
-	eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
-	pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
-	h.ps = &peerAPIServer{
-		b: &LocalBackend{
-			e:            eng,
-			pm:           pm,
-			store:        pm.Store(),
-			appConnector: appc.NewAppConnector(t.Logf, rc),
-		},
-	}
-	h.ps.b.appConnector.UpdateDomains([]string{"www.example.com"})
-	h.ps.b.appConnector.Wait(ctx)
-
-	h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
-		b.CNAMEResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("www.example.com."),
-				Type:  dnsmessage.TypeCNAME,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
+		rc := &appctest.RouteCollector{}
+		eng, _ := wgengine.NewFakeUserspaceEngine(logger.Discard, 0)
+		pm := must.Get(newProfileManager(new(mem.Store), t.Logf))
+		h.ps = &peerAPIServer{
+			b: &LocalBackend{
+				e:            eng,
+				pm:           pm,
+				store:        pm.Store(),
+				appConnector: appc.NewAppConnector(t.Logf, rc, shouldStore),
 			},
-			dnsmessage.CNAMEResource{
-				CNAME: dnsmessage.MustNewName("example.com."),
-			},
-		)
-		b.AResource(
-			dnsmessage.ResourceHeader{
-				Name:  dnsmessage.MustNewName("example.com."),
-				Type:  dnsmessage.TypeA,
-				Class: dnsmessage.ClassINET,
-				TTL:   0,
-			},
-			dnsmessage.AResource{
-				A: [4]byte{192, 0, 0, 8},
-			},
-		)
-	}}
-	f := filter.NewAllowAllForTest(logger.Discard)
-	h.ps.b.setFilter(f)
+		}
+		h.ps.b.appConnector.UpdateDomains([]string{"www.example.com"})
+		h.ps.b.appConnector.Wait(ctx)
 
-	if !h.ps.b.OfferingAppConnector() {
-		t.Fatal("expecting to be offering app connector")
-	}
-	if !h.replyToDNSQueries() {
-		t.Errorf("unexpectedly deny; wanted to be a DNS server")
-	}
+		h.ps.resolver = &fakeResolver{build: func(b *dnsmessage.Builder) {
+			b.CNAMEResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("www.example.com."),
+					Type:  dnsmessage.TypeCNAME,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.CNAMEResource{
+					CNAME: dnsmessage.MustNewName("example.com."),
+				},
+			)
+			b.AResource(
+				dnsmessage.ResourceHeader{
+					Name:  dnsmessage.MustNewName("example.com."),
+					Type:  dnsmessage.TypeA,
+					Class: dnsmessage.ClassINET,
+					TTL:   0,
+				},
+				dnsmessage.AResource{
+					A: [4]byte{192, 0, 0, 8},
+				},
+			)
+		}}
+		f := filter.NewAllowAllForTest(logger.Discard)
+		h.ps.b.setFilter(f)
 
-	w := httptest.NewRecorder()
-	h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
-	if w.Code != http.StatusOK {
-		t.Errorf("unexpected status code: %v", w.Code)
-	}
-	h.ps.b.appConnector.Wait(ctx)
+		if !h.ps.b.OfferingAppConnector() {
+			t.Fatal("expecting to be offering app connector")
+		}
+		if !h.replyToDNSQueries() {
+			t.Errorf("unexpectedly deny; wanted to be a DNS server")
+		}
 
-	wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
-	if !slices.Equal(rc.Routes(), wantRoutes) {
-		t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		w := httptest.NewRecorder()
+		h.handleDNSQuery(w, httptest.NewRequest("GET", "/dns-query?q=www.example.com.", nil))
+		if w.Code != http.StatusOK {
+			t.Errorf("unexpected status code: %v", w.Code)
+		}
+		h.ps.b.appConnector.Wait(ctx)
+
+		wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
+		if !slices.Equal(rc.Routes(), wantRoutes) {
+			t.Errorf("got %v; want %v", rc.Routes(), wantRoutes)
+		}
 	}
 }
 

ipn/localapi/localapi.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"tailscale.com/appc/routeinfo"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/clientupdate"
 	"tailscale.com/drive"
@@ -114,6 +115,7 @@ var handler = map[string]localAPIHandler{
 	"query-feature":               (*Handler).serveQueryFeature,
 	"reload-config":               (*Handler).reloadConfig,
 	"reset-auth":                  (*Handler).serveResetAuth,
+	"routeInfo":                   (*Handler).serveRouteInfo,
 	"serve-config":                (*Handler).serveServeConfig,
 	"set-dns":                     (*Handler).serveSetDNS,
 	"set-expiry-sooner":           (*Handler).serveSetExpirySooner,
@@ -1387,6 +1389,26 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) {
 	e.Encode(prefs)
 }
 
+func (h *Handler) serveRouteInfo(w http.ResponseWriter, r *http.Request) {
+	if !h.PermitRead {
+		http.Error(w, "routeInfo access denied", http.StatusForbidden)
+		return
+	}
+	if r.Method != "GET" {
+		http.Error(w, "unsupported method", http.StatusMethodNotAllowed)
+		return
+	}
+
+	var ri *routeinfo.RouteInfo
+
+	ri = h.b.AppcRouteInfo()
+	w.Header().Set("Content-Type", "application/json")
+	e := json.NewEncoder(w)
+	e.SetIndent("", "\t")
+	e.Encode(ri)
+
+}
+
 type resJSON struct {
 	Error string `json:",omitempty"`
 }

tailcfg/tailcfg.go

@@ -2234,6 +2234,8 @@ const (
 
 	// NodeAttrExitDstNetworkFlowLog enables exit node destinations in network flow logs.
 	NodeAttrExitDstNetworkFlowLog NodeCapability = "exit-dst-network-flow-log"
+	// NodeAttrStoreAppCRoutes enables storing app connector routes persistently.
+	NodeAttrStoreAppCRoutes NodeCapability = "store-appc-routes"
 )
 
 // SetDNSRequest is a request to add a DNS record.

wgengine/watchdog.go

@@ -39,7 +39,7 @@ func NewWatchdog(e Engine) Engine {
 		wrap:     e,
 		logf:     log.Printf,
 		fatalf:   log.Fatalf,
-		maxWait:  45 * time.Second,
+		maxWait:  45 * time.Minute,
 		inFlight: make(map[inFlightKey]time.Time),
 	}
 }