net/dns: add debug envknob to enable dual stack MagicDNS

Updates #15404

Change-Id: Ic754cc54113b1660b7071b40babb9d3c0e25b2e1
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

ALPINE.txt

@@ -1 +1 @@
-3.19
+3.18

Dockerfile

@@ -62,10 +62,8 @@ RUN GOARCH=$TARGETARCH go install -ldflags="\
       -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \
       -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot
 
-FROM alpine:3.19
+FROM alpine:3.18
 RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables
-RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables
-RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables
 
 COPY --from=build-env /go/bin/* /usr/local/bin/
 # For compat with the previous run.sh, although ideally you should be

Dockerfile.base

@@ -1,12 +1,5 @@
 # Copyright (c) Tailscale Inc & AUTHORS
 # SPDX-License-Identifier: BSD-3-Clause
 
-FROM alpine:3.19
-RUN apk add --no-cache ca-certificates iptables iptables-legacy iproute2 ip6tables iputils
-# Alpine 3.19 replaces legacy iptables with nftables based implementation.  We
-# can't be certain that all hosts that run Tailscale containers currently
-# suppport nftables, so link back to legacy for backwards compatibility reasons.
-# TODO(irbekrm): add some way how to determine if we still run on nodes that
-# don't support nftables, so that we can eventually remove these symlinks.
-RUN rm /sbin/iptables && ln -s /sbin/iptables-legacy /sbin/iptables
-RUN rm /sbin/ip6tables && ln -s /sbin/ip6tables-legacy /sbin/ip6tables
+FROM alpine:3.18
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables iputils

VERSION.txt

@@ -1 +1 @@
-1.83.0
+1.81.0

build_docker.sh

@@ -16,7 +16,7 @@ eval "$(./build_dist.sh shellvars)"
 
 DEFAULT_TARGET="client"
 DEFAULT_TAGS="v${VERSION_SHORT},v${VERSION_MINOR}"
-DEFAULT_BASE="tailscale/alpine-base:3.19"
+DEFAULT_BASE="tailscale/alpine-base:3.18"
 # Set a few pre-defined OCI annotations. The source annotation is used by tools such as Renovate that scan the linked
 # Github repo to find release notes for any new image tags. Note that for official Tailscale images the default
 # annotations defined here will be overriden by release scripts that call this script.

cmd/containerboot/certs.go

@@ -60,9 +60,6 @@ func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig)
 		if _, exists := cm.certLoops[domain]; !exists {
 			cancelCtx, cancel := context.WithCancel(ctx)
 			mak.Set(&cm.certLoops, domain, cancel)
-			// Note that most of the issuance anyway happens
-			// serially because the cert client has a shared lock
-			// that's held during any issuance.
 			cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) })
 		}
 	}
@@ -119,13 +116,7 @@ func (cm *certManager) runCertLoop(ctx context.Context, domain string) {
 			// issuance endpoint that explicitly only triggers
 			// issuance and stores certs in the relevant store, but
 			// does not return certs to the caller?
-
-			// An issuance holds a shared lock, so we need to avoid
-			// a situation where other services cannot issue certs
-			// because a single one is holding the lock.
-			ctxT, cancel := context.WithTimeout(ctx, time.Second*300)
-			defer cancel()
-			_, _, err := cm.lc.CertPair(ctxT, domain)
+			_, _, err := cm.lc.CertPair(ctx, domain)
 			if err != nil {
 				log.Printf("error refreshing certificate for %s: %v", domain, err)
 			}

cmd/derper/cert.go

@@ -128,17 +128,16 @@ func (m *manualCertManager) TLSConfig() *tls.Config {
 }
 
 func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-	// if hi.ServerName != m.hostname && !m.noHostname {
-	// 	return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
-	// }
+	if hi.ServerName != m.hostname && !m.noHostname {
+		return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
+	}
 
 	// Return a shallow copy of the cert so the caller can append to its
 	// Certificate field.
-	// certCopy := new(tls.Certificate)
-	// *certCopy = *m.cert
-	// certCopy.Certificate = certCopy.Certificate[:len(certCopy.Certificate):len(certCopy.Certificate)]
-	// return certCopy, nil
-	return m.cert, nil
+	certCopy := new(tls.Certificate)
+	*certCopy = *m.cert
+	certCopy.Certificate = certCopy.Certificate[:len(certCopy.Certificate):len(certCopy.Certificate)]
+	return certCopy, nil
 }
 
 func (m *manualCertManager) HTTPHandler(fallback http.Handler) http.Handler {

cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml

@@ -103,7 +103,7 @@ spec:
                     pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
                 type:
                   description: |-
-                    Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
                     Type is immutable once a ProxyGroup is created.
                   type: string
                   enum:

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -2876,7 +2876,7 @@ spec:
                                 type: array
                             type:
                                 description: |-
-                                    Type of the ProxyGroup proxies. Currently the only supported type is egress.
+                                    Type of the ProxyGroup proxies. Supported types are egress and ingress.
                                     Type is immutable once a ProxyGroup is created.
                                 enum:
                                     - egress

cmd/k8s-operator/ingress-for-pg.go

@@ -49,11 +49,10 @@ const (
 	// FinalizerNamePG is the finalizer used by the IngressPGReconciler
 	FinalizerNamePG = "tailscale.com/ingress-pg-finalizer"
 
+	indexIngressProxyGroup = ".metadata.annotations.ingress-proxy-group"
 	// annotationHTTPEndpoint can be used to configure the Ingress to expose an HTTP endpoint to tailnet (as
 	// well as the default HTTPS endpoint).
 	annotationHTTPEndpoint = "tailscale.com/http-endpoint"
-
-	labelDomain = "tailscale.com/domain"
 )
 
 var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
@@ -242,7 +241,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 		return false, nil
 	}
 	// 3. Ensure that TLS Secret and RBAC exists
-	if err := r.ensureCertResources(ctx, pgName, dnsName, ing); err != nil {
+	if err := r.ensureCertResources(ctx, pgName, dnsName); err != nil {
 		return false, fmt.Errorf("error ensuring cert resources: %w", err)
 	}
 
@@ -339,11 +338,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 
 	// 5. Update tailscaled's AdvertiseServices config, which should add the VIPService
 	// IPs to the ProxyGroup Pods' AllowedIPs in the next netmap update if approved.
-	mode := serviceAdvertisementHTTPS
-	if isHTTPEndpointEnabled(ing) {
-		mode = serviceAdvertisementHTTPAndHTTPS
-	}
-	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, mode, logger); err != nil {
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg.Name, serviceName, true, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config: %w", err)
 	}
 
@@ -359,17 +354,11 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 	case 0:
 		ing.Status.LoadBalancer.Ingress = nil
 	default:
-		var ports []networkingv1.IngressPortStatus
-		hasCerts, err := r.hasCerts(ctx, serviceName)
-		if err != nil {
-			return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err)
-		}
-		// If TLS certs have not been issued (yet), do not set port 443.
-		if hasCerts {
-			ports = append(ports, networkingv1.IngressPortStatus{
+		ports := []networkingv1.IngressPortStatus{
+			{
 				Protocol: "TCP",
 				Port:     443,
-			})
+			},
 		}
 		if isHTTPEndpointEnabled(ing) {
 			ports = append(ports, networkingv1.IngressPortStatus{
@@ -377,14 +366,9 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin
 				Port:     80,
 			})
 		}
-		// Set Ingress status hostname only if either port 443 or 80 is advertised.
-		var hostname string
-		if len(ports) != 0 {
-			hostname = dnsName
-		}
 		ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
 			{
-				Hostname: hostname,
+				Hostname: dnsName,
 				Ports:    ports,
 			},
 		}
@@ -445,7 +429,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG
 			}
 
 			// Make sure the VIPService is not advertised in tailscaled or serve config.
-			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, serviceAdvertisementOff, logger); err != nil {
+			if err = r.maybeUpdateAdvertiseServicesConfig(ctx, proxyGroupName, vipServiceName, false, logger); err != nil {
 				return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 			}
 			_, ok := cfg.Services[vipServiceName]
@@ -528,7 +512,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string,
 	}
 
 	// 4. Unadvertise the VIPService in tailscaled config.
-	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, serviceAdvertisementOff, logger); err != nil {
+	if err = r.maybeUpdateAdvertiseServicesConfig(ctx, pg, serviceName, false, logger); err != nil {
 		return false, fmt.Errorf("failed to update tailscaled config services: %w", err)
 	}
 
@@ -725,16 +709,8 @@ func isHTTPEndpointEnabled(ing *networkingv1.Ingress) bool {
 	return ing.Annotations[annotationHTTPEndpoint] == "enabled"
 }
 
-// serviceAdvertisementMode describes the desired state of a VIPService.
-type serviceAdvertisementMode int
-
-const (
-	serviceAdvertisementOff          serviceAdvertisementMode = iota // Should not be advertised
-	serviceAdvertisementHTTPS                                        // Port 443 should be advertised
-	serviceAdvertisementHTTPAndHTTPS                                 // Both ports 80 and 443 should be advertised
-)
-
-func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) {
+func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, shouldBeAdvertised bool, logger *zap.SugaredLogger) (err error) {
+	logger.Debugf("Updating ProxyGroup tailscaled configs to advertise service %q: %v", serviceName, shouldBeAdvertised)
 
 	// Get all config Secrets for this ProxyGroup.
 	secrets := &corev1.SecretList{}
@@ -742,21 +718,6 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con
 		return fmt.Errorf("failed to list config Secrets: %w", err)
 	}
 
-	// Verify that TLS cert for the VIPService has been successfully issued
-	// before attempting to advertise the service.
-	// This is so that in multi-cluster setups where some Ingresses succeed
-	// to issue certs and some do not (rate limits), clients are not pinned
-	// to a backend that is not able to serve HTTPS.
-	// The only exception is Ingresses with an HTTP endpoint enabled - if an
-	// Ingress has an HTTP endpoint enabled, it will be advertised even if the
-	// TLS cert is not yet provisioned.
-	hasCert, err := a.hasCerts(ctx, serviceName)
-	if err != nil {
-		return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err)
-	}
-	shouldBeAdvertised := (mode == serviceAdvertisementHTTPAndHTTPS) ||
-		(mode == serviceAdvertisementHTTPS && hasCert) // if we only expose port 443 and don't have certs (yet), do not advertise
-
 	for _, secret := range secrets.Items {
 		var updated bool
 		for fileName, confB := range secret.Data {
@@ -909,8 +870,8 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool {
 // (domain) is a valid Kubernetes resource name.
 // https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99
 // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
-func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string, ing *networkingv1.Ingress) error {
-	secret := certSecret(pgName, r.tsNamespace, domain, ing)
+func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string) error {
+	secret := certSecret(pgName, r.tsNamespace, domain)
 	if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil {
 		return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err)
 	}
@@ -1005,14 +966,9 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding
 
 // certSecret creates a Secret that will store the TLS certificate and private
 // key for the given domain. Domain must be a valid Kubernetes resource name.
-func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret {
+func certSecret(pgName, namespace, domain string) *corev1.Secret {
 	labels := certResourceLabels(pgName, domain)
 	labels[kubetypes.LabelSecretType] = "certs"
-	// Labels that let us identify the Ingress resource lets us reconcile
-	// the Ingress when the TLS Secret is updated (for example, when TLS
-	// certs have been provisioned).
-	labels[LabelParentName] = ing.Name
-	labels[LabelParentNamespace] = ing.Namespace
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -1033,9 +989,9 @@ func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *co
 
 func certResourceLabels(pgName, domain string) map[string]string {
 	return map[string]string{
-		kubetypes.LabelManaged: "true",
-		labelProxyGroup:        pgName,
-		labelDomain:            domain,
+		kubetypes.LabelManaged:      "true",
+		"tailscale.com/proxy-group": pgName,
+		"tailscale.com/domain":      domain,
 	}
 }
 
@@ -1048,28 +1004,3 @@ func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg
 	}
 	return s + "." + tcd, nil
 }
-
-// hasCerts checks if the TLS Secret for the given service has non-zero cert and key data.
-func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) {
-	domain, err := r.dnsNameForService(ctx, svc)
-	if err != nil {
-		return false, fmt.Errorf("failed to get DNS name for service: %w", err)
-	}
-	secret := &corev1.Secret{}
-	err = r.Get(ctx, client.ObjectKey{
-		Namespace: r.tsNamespace,
-		Name:      domain,
-	}, secret)
-
-	if err != nil {
-		if apierrors.IsNotFound(err) {
-			return false, nil
-		}
-		return false, fmt.Errorf("failed to get TLS Secret: %w", err)
-	}
-
-	cert := secret.Data[corev1.TLSCertKey]
-	key := secret.Data[corev1.TLSPrivateKeyKey]
-
-	return len(cert) > 0 && len(key) > 0, nil
-}

cmd/k8s-operator/ingress-for-pg_test.go

@@ -31,7 +31,6 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
-	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/ptr"
 )
@@ -60,7 +59,7 @@ func TestIngressPGReconciler(t *testing.T) {
 				},
 			},
 			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"my-svc"}},
+				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
 			},
 		},
 	}
@@ -68,14 +67,12 @@ func TestIngressPGReconciler(t *testing.T) {
 
 	// Verify initial reconciliation
 	expectReconciled(t, ingPGR, "default", "test-ingress")
-	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
-	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
-	// Verify that Role and RoleBinding have been created for the first Ingress.
-	// Do not verify the cert Secret as that was already verified implicitly above.
+	// Verify cert resources were created for the first Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-svc.ts.net"))
 	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net"))
 	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net"))
 
@@ -130,13 +127,11 @@ func TestIngressPGReconciler(t *testing.T) {
 
 	// Verify second Ingress reconciliation
 	expectReconciled(t, ingPGR, "default", "my-other-ingress")
-	populateTLSSecret(context.Background(), fc, "test-pg", "my-other-svc.ts.net")
-	expectReconciled(t, ingPGR, "default", "my-other-ingress")
 	verifyServeConfig(t, fc, "svc:my-other-svc", false)
 	verifyVIPService(t, ft, "svc:my-other-svc", []string{"443"})
 
-	// Verify that Role and RoleBinding have been created for the first Ingress.
-	// Do not verify the cert Secret as that was already verified implicitly above.
+	// Verify cert resources were created for the second Ingress
+	expectEqual(t, fc, certSecret("test-pg", "operator-ns", "my-other-svc.ts.net"))
 	expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net"))
 	expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net"))
 
@@ -236,7 +231,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
 				},
 			},
 			TLS: []networkingv1.IngressTLS{
-				{Hosts: []string{"my-svc"}},
+				{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
 			},
 		},
 	}
@@ -244,19 +239,15 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) {
 
 	// Verify initial reconciliation
 	expectReconciled(t, ingPGR, "default", "test-ingress")
-	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
-	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyServeConfig(t, fc, "svc:my-svc", false)
 	verifyVIPService(t, ft, "svc:my-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:my-svc"})
 
 	// Update the Ingress hostname and make sure the original VIPService is deleted.
 	mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
-		ing.Spec.TLS[0].Hosts[0] = "updated-svc"
+		ing.Spec.TLS[0].Hosts[0] = "updated-svc.tailnetxyz.ts.net"
 	})
 	expectReconciled(t, ingPGR, "default", "test-ingress")
-	populateTLSSecret(context.Background(), fc, "test-pg", "updated-svc.ts.net")
-	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyServeConfig(t, fc, "svc:updated-svc", false)
 	verifyVIPService(t, ft, "svc:updated-svc", []string{"443"})
 	verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"})
@@ -477,8 +468,6 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) {
 
 	// Verify initial reconciliation with HTTP enabled
 	expectReconciled(t, ingPGR, "default", "test-ingress")
-	populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net")
-	expectReconciled(t, ingPGR, "default", "test-ingress")
 	verifyVIPService(t, ft, "svc:my-svc", []string{"80", "443"})
 	verifyServeConfig(t, fc, "svc:my-svc", true)
 
@@ -622,7 +611,6 @@ func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantH
 }
 
 func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) {
-	t.Helper()
 	var expected string
 	if expectedServices != nil {
 		expectedServicesJSON, err := json.Marshal(expectedServices)
@@ -816,28 +804,3 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) {
 		t.Errorf("incorrect owner refs after deletion\ngot:  %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs)
 	}
 }
-
-func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error {
-	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      domain,
-			Namespace: "operator-ns",
-			Labels: map[string]string{
-				kubetypes.LabelManaged:    "true",
-				labelProxyGroup:           pgName,
-				labelDomain:               domain,
-				kubetypes.LabelSecretType: "certs",
-			},
-		},
-		Type: corev1.SecretTypeTLS,
-		Data: map[string][]byte{
-			corev1.TLSCertKey:       []byte("fake-cert"),
-			corev1.TLSPrivateKeyKey: []byte("fake-key"),
-		},
-	}
-
-	_, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) {
-		s.Data = secret.Data
-	})
-	return err
-}

cmd/k8s-operator/operator.go

@@ -9,6 +9,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"regexp"
@@ -39,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/client/local"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/hostinfo"
 	"tailscale.com/ipn"
@@ -331,6 +333,40 @@ func runReconcilers(opts reconcilerOpts) {
 	if err != nil {
 		startlog.Fatalf("could not create ingress reconciler: %v", err)
 	}
+	lc, err := opts.tsServer.LocalClient()
+	if err != nil {
+		startlog.Fatalf("could not get local client: %v", err)
+	}
+	id, err := id(context.Background(), lc)
+	if err != nil {
+		startlog.Fatalf("error determining stable ID of the operator's Tailscale device: %v", err)
+	}
+	ingressProxyGroupFilter := handler.EnqueueRequestsFromMapFunc(ingressesFromIngressProxyGroup(mgr.GetClient(), opts.log))
+	err = builder.
+		ControllerManagedBy(mgr).
+		For(&networkingv1.Ingress{}).
+		Named("ingress-pg-reconciler").
+		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(ingressesFromPGStateSecret(mgr.GetClient(), startlog))).
+		Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter).
+		Complete(&HAIngressReconciler{
+			recorder:    eventRecorder,
+			tsClient:    opts.tsClient,
+			tsnetServer: opts.tsServer,
+			defaultTags: strings.Split(opts.proxyTags, ","),
+			Client:      mgr.GetClient(),
+			logger:      opts.log.Named("ingress-pg-reconciler"),
+			lc:          lc,
+			operatorID:  id,
+			tsNamespace: opts.tailscaleNamespace,
+		})
+	if err != nil {
+		startlog.Fatalf("could not create ingress-pg-reconciler: %v", err)
+	}
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyGroup, indexPGIngresses); err != nil {
+		startlog.Fatalf("failed setting up indexer for HA Ingresses: %v", err)
+	}
+
 	connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector"))
 	// If a ProxyClassChanges, enqueue all Connectors that have
 	// .spec.proxyClass set to the name of this ProxyClass.
@@ -1003,6 +1039,45 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile.
 	return reqs
 }
 
+func ingressesFromPGStateSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		secret, ok := o.(*corev1.Secret)
+		if !ok {
+			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
+			return nil
+		}
+		if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" {
+			return nil
+		}
+		if secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" {
+			return nil
+		}
+		if secret.ObjectMeta.Labels[kubetypes.LabelSecretType] != "state" {
+			return nil
+		}
+		pgName, ok := secret.ObjectMeta.Labels[LabelParentName]
+		if !ok {
+			return nil
+		}
+
+		ingList := &networkingv1.IngressList{}
+		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pgName}); err != nil {
+			logger.Infof("error listing Ingresses, skipping a reconcile for event on Secret %s: %v", secret.Name, err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, ing := range ingList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: ing.Namespace,
+					Name:      ing.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all
 // user-created ExternalName Services that should be exposed on this ProxyGroup.
 func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
@@ -1033,6 +1108,36 @@ func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger)
 	}
 }
 
+// ingressesFromIngressProxyGroup is an event handler for ingress ProxyGroups. It returns reconcile requests for all
+// user-created Ingresses that should be exposed on this ProxyGroup.
+func ingressesFromIngressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		pg, ok := o.(*tsapi.ProxyGroup)
+		if !ok {
+			logger.Infof("[unexpected] ProxyGroup handler triggered for an object that is not a ProxyGroup")
+			return nil
+		}
+		if pg.Spec.Type != tsapi.ProxyGroupTypeIngress {
+			return nil
+		}
+		ingList := &networkingv1.IngressList{}
+		if err := cl.List(ctx, ingList, client.MatchingFields{indexIngressProxyGroup: pg.Name}); err != nil {
+			logger.Infof("error listing Ingresses: %v, skipping a reconcile for event on ProxyGroup %s", err, pg.Name)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, svc := range ingList.Items {
+			reqs = append(reqs, reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: svc.Namespace,
+					Name:      svc.Name,
+				},
+			})
+		}
+		return reqs
+	}
+}
+
 // epsFromExternalNameService is an event handler for ExternalName Services that define a Tailscale egress service that
 // should be exposed on a ProxyGroup. It returns reconcile requests for EndpointSlices created for this Service.
 func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns string) handler.MapFunc {
@@ -1153,7 +1258,63 @@ func indexEgressServices(o client.Object) []string {
 	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
 }
 
+// indexPGIngresses adds a local index to a cached Tailscale Ingresses meant to be exposed on a ProxyGroup. The index is
+// used a list filter.
+func indexPGIngresses(o client.Object) []string {
+	if !hasProxyGroupAnnotation(o) {
+		return nil
+	}
+	return []string{o.GetAnnotations()[AnnotationProxyGroup]}
+}
+
+// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service
+// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation,
+// the associated Ingress gets reconciled.
+func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		ingList := networkingv1.IngressList{}
+		if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil {
+			logger.Debugf("error listing Ingresses: %v", err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0)
+		for _, ing := range ingList.Items {
+			if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
+				continue
+			}
+			if !hasProxyGroupAnnotation(&ing) {
+				continue
+			}
+			if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
+				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
+			}
+			for _, rule := range ing.Spec.Rules {
+				if rule.HTTP == nil {
+					continue
+				}
+				for _, path := range rule.HTTP.Paths {
+					if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() {
+						reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
+					}
+				}
+			}
+		}
+		return reqs
+	}
+}
+
 func hasProxyGroupAnnotation(obj client.Object) bool {
 	ing := obj.(*networkingv1.Ingress)
 	return ing.Annotations[AnnotationProxyGroup] != ""
 }
+
+func id(ctx context.Context, lc *local.Client) (string, error) {
+	st, err := lc.StatusWithoutPeers(ctx)
+	if err != nil {
+		return "", fmt.Errorf("error getting tailscale status: %w", err)
+	}
+	if st.Self == nil {
+		return "", fmt.Errorf("unexpected: device's status does not contain node's metadata")
+	}
+	return string(st.Self.ID), nil
+}

cmd/natc/natc.go

@@ -94,24 +94,18 @@ func main() {
 		}
 		ignoreDstTable.Insert(pfx, true)
 	}
-	var (
-		v4Prefixes    []netip.Prefix
-		numV4DNSAddrs int
-	)
+	var v4Prefixes []netip.Prefix
 	for _, s := range strings.Split(*v4PfxStr, ",") {
 		p := netip.MustParsePrefix(strings.TrimSpace(s))
 		if p.Masked() != p {
 			log.Fatalf("v4 prefix %v is not a masked prefix", p)
 		}
 		v4Prefixes = append(v4Prefixes, p)
-		numIPs := 1 << (32 - p.Bits())
-		numV4DNSAddrs += numIPs
 	}
 	if len(v4Prefixes) == 0 {
 		log.Fatalf("no v4 prefixes specified")
 	}
 	dnsAddr := v4Prefixes[0].Addr()
-	numV4DNSAddrs -= 1 // Subtract the dnsAddr allocated above.
 	ts := &tsnet.Server{
 		Hostname: *hostname,
 	}
@@ -159,13 +153,12 @@ func main() {
 	}
 
 	c := &connector{
-		ts:            ts,
-		lc:            lc,
-		dnsAddr:       dnsAddr,
-		v4Ranges:      v4Prefixes,
-		numV4DNSAddrs: numV4DNSAddrs,
-		v6ULA:         ula(uint16(*siteID)),
-		ignoreDsts:    ignoreDstTable,
+		ts:         ts,
+		lc:         lc,
+		dnsAddr:    dnsAddr,
+		v4Ranges:   v4Prefixes,
+		v6ULA:      ula(uint16(*siteID)),
+		ignoreDsts: ignoreDstTable,
 	}
 	c.run(ctx)
 }
@@ -184,11 +177,6 @@ type connector struct {
 	// v4Ranges is the list of IPv4 ranges to advertise and assign addresses from.
 	// These are masked prefixes.
 	v4Ranges []netip.Prefix
-
-	// numV4DNSAddrs is the total size of the IPv4 ranges in addresses, minus the
-	// dnsAddr allocation.
-	numV4DNSAddrs int
-
 	// v6ULA is the ULA prefix used by the app connector to assign IPv6 addresses.
 	v6ULA netip.Prefix
 
@@ -514,7 +502,6 @@ type perPeerState struct {
 	mu           sync.Mutex
 	domainToAddr map[string][]netip.Addr
 	addrToDomain *bart.Table[string]
-	numV4Allocs  int
 }
 
 // domainForIP returns the domain name assigned to the given IP address and
@@ -560,25 +547,17 @@ func (ps *perPeerState) isIPUsedLocked(ip netip.Addr) bool {
 
 // unusedIPv4Locked returns an unused IPv4 address from the available ranges.
 func (ps *perPeerState) unusedIPv4Locked() netip.Addr {
-	// All addresses have been allocated.
-	if ps.numV4Allocs >= ps.c.numV4DNSAddrs {
-		return netip.Addr{}
-	}
-
 	// TODO: skip ranges that have been exhausted
-	// TODO: implement a much more efficient algorithm for finding unused IPs,
-	// this is fairly crazy.
-	for {
-		for _, r := range ps.c.v4Ranges {
-			ip := randV4(r)
-			if !r.Contains(ip) {
-				panic("error: randV4 returned invalid address")
-			}
+	for _, r := range ps.c.v4Ranges {
+		ip := randV4(r)
+		for r.Contains(ip) {
 			if !ps.isIPUsedLocked(ip) && ip != ps.c.dnsAddr {
 				return ip
 			}
+			ip = ip.Next()
 		}
 	}
+	return netip.Addr{}
 }
 
 // randV4 returns a random IPv4 address within the given prefix.
@@ -604,7 +583,6 @@ func (ps *perPeerState) assignAddrsLocked(domain string) []netip.Addr {
 	if !v4.IsValid() {
 		return nil
 	}
-	ps.numV4Allocs++
 	as16 := ps.c.v6ULA.Addr().As16()
 	as4 := v4.As4()
 	copy(as16[12:], as4[:])

cmd/natc/natc_test.go

@@ -4,8 +4,6 @@
 package main
 
 import (
-	"errors"
-	"fmt"
 	"net/netip"
 	"slices"
 	"testing"
@@ -227,10 +225,9 @@ func TestDNSResponse(t *testing.T) {
 
 func TestPerPeerState(t *testing.T) {
 	c := &connector{
-		v4Ranges:      []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
-		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
-		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
-		numV4DNSAddrs: (1<<(32-24) - 1),
+		v4Ranges: []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
+		v6ULA:    netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		dnsAddr:  netip.MustParseAddr("100.64.1.1"),
 	}
 
 	ps := &perPeerState{c: c}
@@ -331,10 +328,9 @@ func TestIgnoreDestination(t *testing.T) {
 
 func TestConnectorGenerateDNSResponse(t *testing.T) {
 	c := &connector{
-		v4Ranges:      []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
-		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
-		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
-		numV4DNSAddrs: (1<<(32-24) - 1),
+		v4Ranges: []netip.Prefix{netip.MustParsePrefix("100.64.1.0/24")},
+		v6ULA:    netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
+		dnsAddr:  netip.MustParseAddr("100.64.1.1"),
 	}
 
 	req := &dnsmessage.Message{
@@ -367,63 +363,3 @@ func TestConnectorGenerateDNSResponse(t *testing.T) {
 		t.Errorf("generateDNSResponse() responses differ between calls")
 	}
 }
-
-func TestIPPoolExhaustion(t *testing.T) {
-	smallPrefix := netip.MustParsePrefix("100.64.1.0/30") // Only 4 IPs: .0, .1, .2, .3
-	c := &connector{
-		v6ULA:         netip.MustParsePrefix("fd7a:115c:a1e0:a99c:0001::/80"),
-		v4Ranges:      []netip.Prefix{smallPrefix},
-		dnsAddr:       netip.MustParseAddr("100.64.1.0"),
-		numV4DNSAddrs: 3,
-	}
-
-	ps := &perPeerState{c: c}
-
-	assignedIPs := make(map[netip.Addr]string)
-
-	domains := []string{"a.example.com", "b.example.com", "c.example.com", "d.example.com"}
-
-	var errs []error
-
-	for i := 0; i < 5; i++ {
-		for _, domain := range domains {
-			addrs, err := ps.ipForDomain(domain)
-			if err != nil {
-				errs = append(errs, fmt.Errorf("failed to get IP for domain %q: %w", domain, err))
-				continue
-			}
-
-			for _, addr := range addrs {
-				if d, ok := assignedIPs[addr]; ok {
-					if d != domain {
-						t.Errorf("IP %s reused for domain %q, previously assigned to %q", addr, domain, d)
-					}
-				} else {
-					assignedIPs[addr] = domain
-				}
-			}
-		}
-	}
-
-	for addr, domain := range assignedIPs {
-		if addr.Is4() && !smallPrefix.Contains(addr) {
-			t.Errorf("IP %s for domain %q not in expected range %s", addr, domain, smallPrefix)
-		}
-		if addr.Is6() && !c.v6ULA.Contains(addr) {
-			t.Errorf("IP %s for domain %q not in expected range %s", addr, domain, c.v6ULA)
-		}
-		if addr == c.dnsAddr {
-			t.Errorf("IP %s for domain %q is the reserved DNS address", addr, domain)
-		}
-	}
-
-	// expect one error for each iteration with the 4th domain
-	if len(errs) != 5 {
-		t.Errorf("Expected 5 errors, got %d: %v", len(errs), errs)
-	}
-	for _, err := range errs {
-		if !errors.Is(err, ErrNoIPsAvailable) {
-			t.Errorf("generateDNSResponse() error = %v, want ErrNoIPsAvailable", err)
-		}
-	}
-}

ipn/store/kubestore/store_kube.go

@@ -143,6 +143,15 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
 	if err := dnsname.ValidHostname(domain); err != nil {
 		return fmt.Errorf("invalid domain name %q: %w", domain, err)
 	}
+	defer func() {
+		// TODO(irbekrm): a read between these two separate writes would
+		// get a mismatched cert and key.  Allow writing both cert and
+		// key to the memory store in a single, lock-protected operation.
+		if err == nil {
+			s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
+			s.memory.WriteState(ipn.StateKey(domain+".key"), key)
+		}
+	}()
 	secretName := s.secretName
 	data := map[string][]byte{
 		domain + ".crt": cert,
@@ -157,32 +166,19 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
 			keyTLSKey:  key,
 		}
 	}
-	if err := s.updateSecret(data, secretName); err != nil {
-		return fmt.Errorf("error writing TLS cert and key to Secret: %w", err)
-	}
-	// TODO(irbekrm): certs for write replicas are currently not
-	// written to memory to avoid out of sync memory state after
-	// Ingress resources have been recreated.  This means that TLS
-	// certs for write replicas are retrieved from the Secret on
-	// each HTTPS request.  This is a temporary solution till we
-	// implement a Secret watch.
-	if s.certShareMode != "rw" {
-		s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
-		s.memory.WriteState(ipn.StateKey(domain+".key"), key)
-	}
-	return nil
+	return s.updateSecret(data, secretName)
 }
 
 // ReadTLSCertAndKey reads a TLS cert and key from memory or from a
 // domain-specific Secret. It first checks the in-memory store, if not found in
 // memory and running cert store in read-only mode, looks up a Secret.
-// Note that write replicas of HA Ingress always retrieve TLS certs from Secrets.
 func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
 	if err := dnsname.ValidHostname(domain); err != nil {
 		return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
 	}
 	certKey := domain + ".crt"
 	keyKey := domain + ".key"
+
 	cert, err = s.memory.ReadState(ipn.StateKey(certKey))
 	if err == nil {
 		key, err = s.memory.ReadState(ipn.StateKey(keyKey))
@@ -190,12 +186,16 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
 			return cert, key, nil
 		}
 	}
-	if s.certShareMode == "" {
+	if s.certShareMode != "ro" {
 		return nil, nil, ipn.ErrStateNotExist
 	}
+	// If we are in cert share read only mode, it is possible that a write
+	// replica just issued the TLS cert for this DNS name and it has not
+	// been loaded to store yet, so check the Secret.
 
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
+
 	secret, err := s.client.GetSecret(ctx, domain)
 	if err != nil {
 		if kubeclient.IsNotFoundErr(err) {
@@ -212,18 +212,9 @@ func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
 	}
 	// TODO(irbekrm): a read between these two separate writes would
 	// get a mismatched cert and key.  Allow writing both cert and
-	// key to the memory store in a single, lock-protected operation.
-	//
-	// TODO(irbekrm): currently certs for write replicas of HA Ingress get
-	// retrieved from the cluster Secret on each HTTPS request to avoid a
-	// situation when after Ingress recreation stale certs are read from
-	// memory.
-	// Fix this by watching Secrets to ensure that memory store gets updated
-	// when Secrets are deleted.
-	if s.certShareMode == "ro" {
-		s.memory.WriteState(ipn.StateKey(certKey), cert)
-		s.memory.WriteState(ipn.StateKey(keyKey), key)
-	}
+	// key to the memory store in a single lock-protected operation.
+	s.memory.WriteState(ipn.StateKey(certKey), cert)
+	s.memory.WriteState(ipn.StateKey(keyKey), key)
 	return cert, key, nil
 }
 

ipn/store/kubestore/store_kube_test.go

@@ -201,6 +201,10 @@ func TestWriteTLSCertAndKey(t *testing.T) {
 				"tls.crt": []byte(testCert),
 				"tls.key": []byte(testKey),
 			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
 		},
 		{
 			name: "cert_share_mode_write_update_existing",
@@ -215,6 +219,10 @@ func TestWriteTLSCertAndKey(t *testing.T) {
 				"tls.crt": []byte(testCert),
 				"tls.key": []byte(testKey),
 			},
+			wantMemoryStore: map[ipn.StateKey][]byte{
+				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
+				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
+			},
 		},
 		{
 			name: "update_existing",
@@ -359,7 +367,7 @@ func TestReadTLSCertAndKey(t *testing.T) {
 		wantMemoryStore map[ipn.StateKey][]byte
 	}{
 		{
-			name: "found_in_memory",
+			name: "found",
 			memoryStore: map[ipn.StateKey][]byte{
 				"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
 				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
@@ -373,7 +381,7 @@ func TestReadTLSCertAndKey(t *testing.T) {
 			},
 		},
 		{
-			name:    "not_found_in_memory",
+			name:    "not_found",
 			domain:  testDomain,
 			wantErr: ipn.ErrStateNotExist,
 		},
@@ -392,17 +400,6 @@ func TestReadTLSCertAndKey(t *testing.T) {
 				"my-app.tailnetxyz.ts.net.key": []byte(testKey),
 			},
 		},
-		{
-			name:          "cert_share_rw_mode_found_in_secret",
-			certShareMode: "rw",
-			domain:        testDomain,
-			secretData: map[string][]byte{
-				"tls.crt": []byte(testCert),
-				"tls.key": []byte(testKey),
-			},
-			wantCert: []byte(testCert),
-			wantKey:  []byte(testKey),
-		},
 		{
 			name:          "cert_share_ro_mode_found_in_memory",
 			certShareMode: "ro",

k8s-operator/api.md

@@ -600,7 +600,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Currently the only supported type is egress.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress] <br />Type: string <br /> |
+| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.<br />Type is immutable once a ProxyGroup is created. |  | Enum: [egress ingress] <br />Type: string <br /> |
 | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].<br />If you specify custom tags here, make sure you also make the operator<br />an owner of these tags.<br />See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.<br />Tags cannot be changed once a ProxyGroup device has been created.<br />Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. |  | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$` <br />Type: string <br /> |
 | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.<br />Defaults to 2. |  | Minimum: 0 <br /> |
 | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created<br />by the ProxyGroup. Each device will have the integer number from its<br />StatefulSet pod appended to this prefix to form the full hostname.<br />HostnamePrefix can contain lower case letters, numbers and dashes, it<br />must not start with a dash and must be between 1 and 62 characters long. |  | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$` <br />Type: string <br /> |

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -48,7 +48,7 @@ type ProxyGroupList struct {
 }
 
 type ProxyGroupSpec struct {
-	// Type of the ProxyGroup proxies. Currently the only supported type is egress.
+	// Type of the ProxyGroup proxies. Supported types are egress and ingress.
 	// Type is immutable once a ProxyGroup is created.
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable"
 	Type ProxyGroupType `json:"type"`

net/dns/config.go

@@ -10,6 +10,8 @@ import (
 	"net/netip"
 	"sort"
 
+	"tailscale.com/control/controlknobs"
+	"tailscale.com/envknob"
 	"tailscale.com/net/dns/publicdns"
 	"tailscale.com/net/dns/resolver"
 	"tailscale.com/net/tsaddr"
@@ -47,11 +49,28 @@ type Config struct {
 	OnlyIPv6 bool
 }
 
-func (c *Config) serviceIP() netip.Addr {
+var magicDNSDualStack = envknob.RegisterBool("TS_DEBUG_MAGIC_DNS_DUAL_STACK")
+
+// serviceIPs returns the list of service IPs where MagicDNS is reachable.
+//
+// The provided knobs may be nil.
+func (c *Config) serviceIPs(knobs *controlknobs.Knobs) []netip.Addr {
 	if c.OnlyIPv6 {
-		return tsaddr.TailscaleServiceIPv6()
+		return []netip.Addr{tsaddr.TailscaleServiceIPv6()}
 	}
-	return tsaddr.TailscaleServiceIP()
+
+	// TODO(bradfitz,mikeodr,raggi): include IPv6 here too; tailscale/tailscale#15404
+	// And add a controlknobs knob to disable dual stack.
+	//
+	// For now, opt-in for testing.
+	if magicDNSDualStack() {
+		return []netip.Addr{
+			tsaddr.TailscaleServiceIP(),
+			tsaddr.TailscaleServiceIPv6(),
+		}
+	}
+
+	return []netip.Addr{tsaddr.TailscaleServiceIP()}
 }
 
 // WriteToBufioWriter write a debug version of c for logs to w, omitting

net/dns/manager.go

@@ -307,7 +307,7 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 		// through quad-100.
 		rcfg.Routes = routes
 		rcfg.Routes["."] = cfg.DefaultResolvers
-		ocfg.Nameservers = []netip.Addr{cfg.serviceIP()}
+		ocfg.Nameservers = cfg.serviceIPs(m.knobs)
 		return rcfg, ocfg, nil
 	}
 
@@ -345,7 +345,7 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig
 	// or routes + MagicDNS, or just MagicDNS, or on an OS that cannot
 	// split-DNS. Install a split config pointing at quad-100.
 	rcfg.Routes = routes
-	ocfg.Nameservers = []netip.Addr{cfg.serviceIP()}
+	ocfg.Nameservers = cfg.serviceIPs(m.knobs)
 
 	var baseCfg *OSConfig // base config; non-nil if/when known
 

tsnet/tsnet.go

@@ -505,11 +505,6 @@ func (s *Server) start() (reterr error) {
 			// directory and hostname when they're not supplied. But we can fall
 			// back to "tsnet" as well.
 			exe = "tsnet"
-		case "ios":
-			// When compiled as a framework (via TailscaleKit in libtailscale),
-			// os.Executable() returns an error, so fall back to "tsnet" there
-			// too.
-			exe = "tsnet"
 		default:
 			return err
 		}