Experimental

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/containerboot/main.go

@@ -106,6 +106,7 @@ func main() {
 		Hostname:          defaultEnv("TS_HOSTNAME", ""),
 		Routes:            defaultEnvPointer("TS_ROUTES"),
 		ServeConfigPath:   defaultEnv("TS_SERVE_CONFIG", ""),
+		ConfigFilePath:    defaultEnv("TS_CONFIGFILE_PATH", ""),
 		ProxyTo:           defaultEnv("TS_DEST_IP", ""),
 		TailnetTargetIP:   defaultEnv("TS_TAILNET_TARGET_IP", ""),
 		TailnetTargetFQDN: defaultEnv("TS_TAILNET_TARGET_FQDN", ""),
@@ -293,13 +294,13 @@ authLoop:
 	ctx, cancel := contextWithExitSignalWatch()
 	defer cancel()
 
-	if cfg.AuthOnce {
-		// Now that we are authenticated, we can set/reset any of the
-		// settings that we need to.
-		if err := tailscaleSet(ctx, cfg); err != nil {
-			log.Fatalf("failed to auth tailscale: %v", err)
-		}
-	}
+	// if cfg.AuthOnce {
+	// 	// Now that we are authenticated, we can set/reset any of the
+	// 	// settings that we need to.
+	// 	// if err := tailscaleSet(ctx, cfg); err != nil {
+	// 	// 	log.Fatalf("failed to auth tailscale: %v", err)
+	// 	// }
+	// }
 
 	if cfg.ServeConfigPath != "" {
 		// Remove any serve config that may have been set by a previous run of
@@ -313,10 +314,10 @@ authLoop:
 		// We were told to only auth once, so any secret-bound
 		// authkey is no longer needed. We don't strictly need to
 		// wipe it, but it's good hygiene.
-		log.Printf("Deleting authkey from kube secret")
-		if err := deleteAuthKey(ctx, cfg.KubeSecret); err != nil {
-			log.Fatalf("deleting authkey from kube secret: %v", err)
-		}
+		// log.Printf("Deleting authkey from kube secret")
+		// if err := deleteAuthKey(ctx, cfg.KubeSecret); err != nil {
+		// 	log.Fatalf("deleting authkey from kube secret: %v", err)
+		// }
 	}
 
 	w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
@@ -637,40 +638,49 @@ func tailscaledArgs(cfg *settings) []string {
 	if cfg.DaemonExtraArgs != "" {
 		args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
 	}
+	if cfg.ConfigFilePath != "" {
+		args = append(args, "--config="+cfg.ConfigFilePath)
+	}
 	return args
 }
 
 // tailscaleUp uses cfg to run 'tailscale up' everytime containerboot starts, or
 // if TS_AUTH_ONCE is set, only the first time containerboot starts.
 func tailscaleUp(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "up"}
-	if cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
+	command := "up"
+	if cfg.ConfigFilePath != "" {
+		command = "login"
 	}
+	args := []string{"--socket=" + cfg.Socket, command}
 	if cfg.AuthKey != "" {
 		args = append(args, "--authkey="+cfg.AuthKey)
 	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
+	if cfg.ConfigFilePath == "" {
+		if cfg.AcceptDNS {
+			args = append(args, "--accept-dns=true")
+		} else {
+			args = append(args, "--accept-dns=false")
+		}
+		// --advertise-routes can be passed an empty string to configure a
+		// device (that might have previously advertised subnet routes) to not
+		// advertise any routes. Respect an empty string passed by a user and
+		// use it to explicitly unset the routes.
+		if cfg.Routes != nil {
+			args = append(args, "--advertise-routes="+*cfg.Routes)
+		}
+		if cfg.Hostname != "" {
+			args = append(args, "--hostname="+cfg.Hostname)
+		}
+		if cfg.ExtraArgs != "" {
+			args = append(args, strings.Fields(cfg.ExtraArgs)...)
+		}
 	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	if cfg.ExtraArgs != "" {
-		args = append(args, strings.Fields(cfg.ExtraArgs)...)
-	}
-	log.Printf("Running 'tailscale up'")
+	log.Printf("Running 'tailscale %s' with args %#+v", command, args)
 	cmd := exec.CommandContext(ctx, "tailscale", args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale up failed: %v", err)
+		log.Printf("tailscale up with args %#+v failed: %v", args, err)
 	}
 	return nil
 }
@@ -678,32 +688,35 @@ func tailscaleUp(ctx context.Context, cfg *settings) error {
 // tailscaleSet uses cfg to run 'tailscale set' to set any known configuration
 // options that are passed in via environment variables. This is run after the
 // node is in Running state and only if TS_AUTH_ONCE is set.
-func tailscaleSet(ctx context.Context, cfg *settings) error {
-	args := []string{"--socket=" + cfg.Socket, "set"}
-	if cfg.AcceptDNS {
-		args = append(args, "--accept-dns=true")
-	} else {
-		args = append(args, "--accept-dns=false")
-	}
-	// --advertise-routes can be passed an empty string to configure a
-	// device (that might have previously advertised subnet routes) to not
-	// advertise any routes. Respect an empty string passed by a user and
-	// use it to explicitly unset the routes.
-	if cfg.Routes != nil {
-		args = append(args, "--advertise-routes="+*cfg.Routes)
-	}
-	if cfg.Hostname != "" {
-		args = append(args, "--hostname="+cfg.Hostname)
-	}
-	log.Printf("Running 'tailscale set'")
-	cmd := exec.CommandContext(ctx, "tailscale", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("tailscale set failed: %v", err)
-	}
-	return nil
-}
+// func tailscaleSet(ctx context.Context, cfg *settings) error {
+// 	args := []string{"--socket=" + cfg.Socket, "set"}
+// 	// TODO: fix
+// 	if cfg.ConfigFilePath == "" {
+// 		if cfg.AcceptDNS {
+// 			args = append(args, "--accept-dns=true")
+// 		} else {
+// 			args = append(args, "--accept-dns=false")
+// 		}
+// 		// --advertise-routes can be passed an empty string to configure a
+// 		// device (that might have previously advertised subnet routes) to not
+// 		// advertise any routes. Respect an empty string passed by a user and
+// 		// use it to explicitly unset the routes.
+// 		if cfg.Routes != nil {
+// 			args = append(args, "--advertise-routes="+*cfg.Routes)
+// 		}
+// 		if cfg.Hostname != "" {
+// 			args = append(args, "--hostname="+cfg.Hostname)
+// 		}
+// 	}
+// 	log.Printf("Running 'tailscale set'")
+// 	cmd := exec.CommandContext(ctx, "tailscale", args...)
+// 	cmd.Stdout = os.Stdout
+// 	cmd.Stderr = os.Stderr
+// 	if err := cmd.Run(); err != nil {
+// 		return fmt.Errorf("tailscale set failed: %v", err)
+// 	}
+// 	return nil
+// }
 
 // ensureTunFile checks that /dev/net/tun exists, creating it if
 // missing.
@@ -888,6 +901,7 @@ type settings struct {
 	AuthOnce           bool
 	Root               string
 	KubernetesCanPatch bool
+	ConfigFilePath     string
 }
 
 // defaultEnv returns the value of the given envvar name, or defVal if

cmd/k8s-operator/connector.go

@@ -7,10 +7,10 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/netip"
 	"slices"
-	"strings"
 	"sync"
 	"time"
 
@@ -25,25 +25,26 @@ import (
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"tailscale.com/ipn"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/net/netutil"
 	"tailscale.com/tstime"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/util/set"
 )
 
 const (
-	reasonSubnetRouterCreationFailed    = "SubnetRouterCreationFailed"
-	reasonSubnetRouterCreated           = "SubnetRouterCreated"
-	reasonSubnetRouterCleanupFailed     = "SubnetRouterCleanupFailed"
-	reasonSubnetRouterCleanupInProgress = "SubnetRouterCleanupInProgress"
-	reasonSubnetRouterInvalid           = "SubnetRouterInvalid"
+	reasonConnectorCreationFailed = "ConnectorCreationFailed"
 
-	messageSubnetRouterCreationFailed = "Failed creating subnet router for routes %s: %v"
-	messageSubnetRouterInvalid        = "Subnet router is invalid: %v"
-	messageSubnetRouterCreated        = "Created subnet router for routes %s"
-	messageSubnetRouterCleanupFailed  = "Failed cleaning up subnet router resources: %v"
-	msgSubnetRouterCleanupInProgress  = "SubnetRouterCleanupInProgress"
+	reasonConnectorCreated           = "ConnectorCreated"
+	reasonConnectorCleanupFailed     = "ConnectorCleanupFailed"
+	reasonConnectorCleanupInProgress = "ConnectorCleanupInProgress"
+	reasonConnectorInvalid           = "ConnectorInvalid"
+
+	messageConnectorCreationFailed   = "Failed creating Connector: %v"
+	messageConnectorInvalid          = "Connector is invalid: %v"
+	messageSubnetRouterCleanupFailed = "Failed cleaning up Connector resources: %v"
 
 	shortRequeue = time.Second * 5
 )
@@ -61,42 +62,39 @@ type ConnectorReconciler struct {
 
 	mu sync.Mutex // protects following
 
-	// subnetRouters tracks the subnet routers managed by this Tailscale
-	// Operator instance.
-	subnetRouters set.Slice[types.UID]
+	connectors set.Slice[types.UID] // for connectors gauge
 }
 
 var (
-	// gaugeIngressResources tracks the number of subnet routers that we're
-	// currently managing.
-	gaugeSubnetRouterResources = clientmetric.NewGauge("k8s_subnet_router_resources")
+	// gaugeConnectorResources tracks the number of Connectors currently managed by this operator instance
+	gaugeConnectorResources = clientmetric.NewGauge("k8s_connector_resources")
 )
 
 func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
-	logger := a.logger.With("connector", req.Name)
+	logger := a.logger.With("Connector", req.Name)
 	logger.Debugf("starting reconcile")
 	defer logger.Debugf("reconcile finished")
 
 	cn := new(tsapi.Connector)
 	err = a.Get(ctx, req.NamespacedName, cn)
 	if apierrors.IsNotFound(err) {
-		logger.Debugf("connector not found, assuming it was deleted")
+		logger.Debugf("Connector not found, assuming it was deleted")
 		return reconcile.Result{}, nil
 	} else if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to get tailscale.com Connector: %w", err)
 	}
 	if !cn.DeletionTimestamp.IsZero() {
-		logger.Debugf("connector is being deleted or should not be exposed, cleaning up components")
+		logger.Debugf("Connector is being deleted or should not be exposed, cleaning up resources")
 		ix := xslices.Index(cn.Finalizers, FinalizerName)
 		if ix < 0 {
 			logger.Debugf("no finalizer, nothing to do")
 			return reconcile.Result{}, nil
 		}
 
-		if done, err := a.maybeCleanupSubnetRouter(ctx, logger, cn); err != nil {
+		if done, err := a.maybeCleanupConnector(ctx, logger, cn); err != nil {
 			return reconcile.Result{}, err
 		} else if !done {
-			logger.Debugf("cleanup not finished, will retry...")
+			logger.Debugf("Connector resource cleanup not yet finished, will retry...")
 			return reconcile.Result{RequeueAfter: shortRequeue}, nil
 		}
 
@@ -104,21 +102,20 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		if err := a.Update(ctx, cn); err != nil {
 			return reconcile.Result{}, err
 		}
-		logger.Infof("connector resources cleaned up")
+		logger.Infof("Connector resources cleaned up")
 		return reconcile.Result{}, nil
 	}
 
+	var (
+		reason, message string
+		readyStatus     metav1.ConditionStatus
+	)
+
 	oldCnStatus := cn.Status.DeepCopy()
 	defer func() {
-		if cn.Status.SubnetRouter == nil {
-			tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, metav1.ConditionUnknown, "", "", cn.Generation, a.clock, logger)
-		} else if cn.Status.SubnetRouter.Ready == metav1.ConditionTrue {
-			tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonSubnetRouterCreated, reasonSubnetRouterCreated, cn.Generation, a.clock, logger)
-		} else {
-			tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, metav1.ConditionFalse, cn.Status.SubnetRouter.Reason, cn.Status.SubnetRouter.Reason, cn.Generation, a.clock, logger)
-		}
+		tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, readyStatus, reason, message, cn.Generation, a.clock, logger)
 		if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
-			// an error encountered here should get returned by the Reconcile function
+			// An error encountered here should get returned by the Reconcile function.
 			if updateErr := a.Client.Status().Update(ctx, cn); updateErr != nil {
 				err = updateErr
 			}
@@ -130,67 +127,138 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
 		// because once the finalizer is in place this block gets skipped. So,
 		// this is a nice place to tell the operator that the high level,
 		// multi-reconcile operation is underway.
-		logger.Infof("ensuring connector is set up")
+		logger.Infof("ensuring Connector is set up")
 		cn.Finalizers = append(cn.Finalizers, FinalizerName)
 		if err := a.Update(ctx, cn); err != nil {
 			err = fmt.Errorf("failed to add finalizer: %w", err)
 			logger.Errorf("error adding finalizer: %v", err)
+			reason = reasonConnectorCreationFailed
+			message = fmt.Sprintf(messageConnectorCreationFailed, err)
+			readyStatus = metav1.ConditionFalse
 			return reconcile.Result{}, err
 		}
 	}
 
-	// A Connector with unset .spec.subnetRouter and unset
-	// cn.spec.subnetRouter.Routes will be rejected at apply time (because
-	// these fields are set as required by our CRD validation). This check
-	// is here for if our CRD validation breaks unnoticed we don't crash the
-	// operator with nil pointer exception.
-	if cn.Spec.SubnetRouter == nil || len(cn.Spec.SubnetRouter.Routes) < 1 {
+	if err := a.validate(cn); err != nil {
+		logger.Errorf("error validating Connector spec: %w", err)
+		reason = reasonConnectorInvalid
+		message = fmt.Sprintf(messageConnectorInvalid, err)
+		readyStatus = metav1.ConditionFalse
+		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorInvalid, message)
 		return reconcile.Result{}, nil
 	}
 
-	if err := validateSubnetRouter(*cn.Spec.SubnetRouter); err != nil {
-		msg := fmt.Sprintf(messageSubnetRouterInvalid, err)
-		cn.Status.SubnetRouter = &tsapi.SubnetRouterStatus{
-			Ready:   metav1.ConditionFalse,
-			Reason:  reasonSubnetRouterInvalid,
-			Message: msg,
+	if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil {
+		logger.Errorf("error creating Connector resources: %w", err)
+		reason = reasonConnectorCreationFailed
+		message = fmt.Sprintf(messageConnectorCreationFailed, err)
+		readyStatus = metav1.ConditionFalse
+		a.recorder.Eventf(cn, corev1.EventTypeWarning, reason, message)
+	} else {
+		logger.Info("Connector resources synced")
+		reason = reasonConnectorCreated
+		message = reasonConnectorCreated
+		readyStatus = metav1.ConditionTrue
+		cn.Status.IsExitNode = cn.Spec.IsExitNode
+		if cn.Spec.SubnetRouter != nil {
+			cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.Routes.Stringify()
+		} else {
+			cn.Status.SubnetRoutes = ""
 		}
-		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonSubnetRouterInvalid, msg)
-		return reconcile.Result{}, nil
 	}
-
-	var sb strings.Builder
-	sb.WriteString(string(cn.Spec.SubnetRouter.Routes[0]))
-	for _, r := range cn.Spec.SubnetRouter.Routes[1:] {
-		sb.WriteString(fmt.Sprintf(",%s", r))
-	}
-	cidrsS := sb.String()
-	logger.Debugf("ensuring a subnet router is deployed")
-	err = a.maybeProvisionSubnetRouter(ctx, logger, cn, cidrsS)
-	if err != nil {
-		msg := fmt.Sprintf(messageSubnetRouterCreationFailed, cidrsS, err)
-		cn.Status.SubnetRouter = &tsapi.SubnetRouterStatus{
-			Ready:   metav1.ConditionFalse,
-			Reason:  reasonSubnetRouterCreationFailed,
-			Message: msg,
-		}
-		a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonSubnetRouterCreationFailed, msg)
-		return reconcile.Result{}, err
-	}
-	cn.Status.SubnetRouter = &tsapi.SubnetRouterStatus{
-		Routes:  cidrsS,
-		Ready:   metav1.ConditionTrue,
-		Reason:  reasonSubnetRouterCreated,
-		Message: fmt.Sprintf(messageSubnetRouterCreated, cidrsS),
-	}
-	return reconcile.Result{}, nil
+	return reconcile.Result{}, err
 }
 
-func (a *ConnectorReconciler) maybeCleanupSubnetRouter(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
-	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "subnetrouter")); err != nil {
-		return false, fmt.Errorf("failed to cleanup: %w", err)
+// maybeProvisionConnector ensures that any new resources required for this
+// Connector instance are deployed to the cluster.
+func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) error {
+	hostname := cn.Name + "-connector"
+	if cn.Spec.Hostname != "" {
+		hostname = string(cn.Spec.Hostname)
+	}
+	crl := childResourceLabels(cn.Name, a.tsnamespace, "connector")
+	sts := &tailscaleSTSConfig{
+		ParentResourceName:  cn.Name,
+		ParentResourceUID:   string(cn.UID),
+		Hostname:            hostname,
+		ChildResourceLabels: crl,
+		Tags:                cn.Spec.Tags.Stringify(),
+		Connector: &connector{
+			isExitNode: cn.Spec.IsExitNode,
+		},
+	}
+
+	if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.Routes) > 0 {
+		sts.Connector.routes = cn.Spec.SubnetRouter.Routes.Stringify()
+	}
+
+	a.mu.Lock()
+	a.connectors.Add(cn.UID)
+	gaugeConnectorResources.Set(int64(a.connectors.Len()))
+	a.mu.Unlock()
+
+	_, err := a.ssr.Provision(ctx, logger, sts)
+	return err
+}
+
+func (a *tailscaleSTSReconciler) tsConfigCM(ctx context.Context, name, namespace string, logger *zap.SugaredLogger, sts *tailscaleSTSConfig) error {
+	confFile, err := confFile(sts)
+	if err != nil {
+		return fmt.Errorf("error provisioning config: %v", err)
+	}
+
+	jsonBytes, err := json.Marshal(confFile)
+	if err != nil {
+		return fmt.Errorf("error marshaling config file: %v", err)
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    sts.ChildResourceLabels,
+		},
+		Data: map[string]string{
+			"tailscaled": string(jsonBytes),
+		},
+	}
+	_, err = createOrUpdate(ctx, a.Client, namespace, cm, func(config *corev1.ConfigMap) { config.Labels = cm.Labels; config.Data = cm.Data })
+	if err != nil {
+		return fmt.Errorf("error creating a ConfigMap: %v", err)
+	}
+	return nil
+}
+
+func confFile(sts *tailscaleSTSConfig) (*ipn.ConfigVAlpha, error) {
+	var (
+		routes []netip.Prefix
+		err    error
+	)
+	if sts.Connector != nil {
+		routes, err = netutil.CalcAdvertiseRoutes(sts.Connector.routes, sts.Connector.isExitNode)
+		if err != nil {
+			return nil, fmt.Errorf("error calculating routes: %v", err)
+		}
+	}
+	conf := &ipn.ConfigVAlpha{
+		Version:         "alpha0",
+		AdvertiseRoutes: routes,
+		AcceptDNS:       "false",
+		Hostname:        &sts.Hostname,
+		// Not sure how to log in if it's locked?
+		Locked: "false",
+	}
+	// fix - don't put the key there
+	if sts.key != "" {
+		conf.AuthKey = &sts.key
+	}
+	return conf, nil
+}
+
+func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
+	if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector")); err != nil {
+		return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
 	} else if !done {
-		logger.Debugf("cleanup not done yet, waiting for next reconcile")
+		logger.Debugf("Connector cleanup not done yet, waiting for next reconcile")
 		return false, nil
 	}
 
@@ -198,42 +266,31 @@ func (a *ConnectorReconciler) maybeCleanupSubnetRouter(ctx context.Context, logg
 	// exactly once at the very end of cleanup, because the final step of
 	// cleanup removes the tailscale finalizer, which will make all future
 	// reconciles exit early.
-	logger.Infof("cleaned up subnet router")
+	logger.Infof("cleaned up Connector resources")
 	a.mu.Lock()
 	defer a.mu.Unlock()
-	a.subnetRouters.Remove(cn.UID)
-	gaugeSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
+	a.connectors.Remove(cn.UID)
+	gaugeConnectorResources.Set(int64(a.connectors.Len()))
 	return true, nil
 }
 
-// maybeProvisionSubnetRouter maybe deploys subnet router that exposes a subset of cluster cidrs to the tailnet
-func (a *ConnectorReconciler) maybeProvisionSubnetRouter(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector, cidrs string) error {
-	if cn.Spec.SubnetRouter == nil || len(cn.Spec.SubnetRouter.Routes) < 1 {
+func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
+	// Connector fields are already validated at apply time with CEL validation
+	// on custom resource fields. The checks here are a backup in case the
+	// CEL validation breaks without us noticing.
+	if !(cn.Spec.SubnetRouter != nil || cn.Spec.IsExitNode) {
+		return errors.New("invalid Connector spec- a Connector must be either expose subnet routes or act as exit node (or both)")
+	}
+	if cn.Spec.SubnetRouter == nil {
 		return nil
 	}
-	a.mu.Lock()
-	a.subnetRouters.Add(cn.UID)
-	gaugeSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
-	a.mu.Unlock()
-
-	crl := childResourceLabels(cn.Name, a.tsnamespace, "subnetrouter")
-	hostname := hostnameForSubnetRouter(cn)
-	sts := &tailscaleSTSConfig{
-		ParentResourceName:  cn.Name,
-		ParentResourceUID:   string(cn.UID),
-		Hostname:            hostname,
-		ChildResourceLabels: crl,
-		Routes:              cidrs,
-	}
-	for _, tag := range cn.Spec.SubnetRouter.Tags {
-		sts.Tags = append(sts.Tags, string(tag))
-	}
-
-	_, err := a.ssr.Provision(ctx, logger, sts)
-
-	return err
+	return validateSubnetRouter(cn.Spec.SubnetRouter)
 }
-func validateSubnetRouter(sb tsapi.SubnetRouter) error {
+
+func validateSubnetRouter(sb *tsapi.SubnetRouter) error {
+	if len(sb.Routes) < 1 {
+		return errors.New("invalid subnet router spec: no routes defined")
+	}
 	var err error
 	for _, route := range sb.Routes {
 		pfx, e := netip.ParsePrefix(string(route))
@@ -247,13 +304,3 @@ func validateSubnetRouter(sb tsapi.SubnetRouter) error {
 	}
 	return err
 }
-
-func hostnameForSubnetRouter(cn *tsapi.Connector) string {
-	if cn.Spec.SubnetRouter == nil {
-		return ""
-	}
-	if cn.Spec.SubnetRouter.Hostname != "" {
-		return string(cn.Spec.SubnetRouter.Hostname)
-	}
-	return cn.Name + "-" + "subnetrouter"
-}

cmd/k8s-operator/connector_test.go

@@ -7,6 +7,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"go.uber.org/zap"
@@ -21,6 +22,8 @@ import (
 )
 
 func TestConnector(t *testing.T) {
+	// Create a Connector that defines a Tailscale node that advertises
+	// 10.40.0.0/14 route and acts as an exit node.
 	cn := &tsapi.Connector{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "test",
@@ -34,6 +37,7 @@ func TestConnector(t *testing.T) {
 			SubnetRouter: &tsapi.SubnetRouter{
 				Routes: []tsapi.Route{"10.40.0.0/14"},
 			},
+			IsExitNode: true,
 		},
 	}
 	fc := fake.NewClientBuilder().
@@ -48,7 +52,6 @@ func TestConnector(t *testing.T) {
 	}
 
 	cl := tstest.NewClock(tstest.ClockOpts{})
-	// Create a Connector with a subnet router definition
 	cr := &ConnectorReconciler{
 		Client: fc,
 		ssr: &tailscaleSTSReconciler{
@@ -63,26 +66,54 @@ func TestConnector(t *testing.T) {
 	}
 
 	expectReconciled(t, cr, "", "test")
-	fullName, shortName := findGenName(t, fc, "", "test", "subnetrouter")
+	fullName, shortName := findGenName(t, fc, "", "test", "connector")
 
-	expectEqual(t, fc, expectedSecret(fullName, "", "subnetrouter"))
-	expectEqual(t, fc, expectedConnectorSTS(shortName, fullName, "10.40.0.0/14"))
+	expectEqual(t, fc, expectedSecret(fullName, "", "connector"))
+	opts := connectorSTSOpts{
+		connectorName: "test",
+		stsName:       shortName,
+		secretName:    fullName,
+		routes:        "10.40.0.0/14",
+		isExitNode:    true,
+	}
+	expectEqual(t, fc, expectedConnectorSTS(opts))
 
-	// Add another CIDR
+	// Add another route to be advertised.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
 		conn.Spec.SubnetRouter.Routes = []tsapi.Route{"10.40.0.0/14", "10.44.0.0/20"}
 	})
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedConnectorSTS(shortName, fullName, "10.40.0.0/14,10.44.0.0/20"))
+	opts.routes = "10.40.0.0/14,10.44.0.0/20"
 
-	// Remove a CIDR
+	expectEqual(t, fc, expectedConnectorSTS(opts))
+
+	// Remove a route.
 	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
 		conn.Spec.SubnetRouter.Routes = []tsapi.Route{"10.44.0.0/20"}
 	})
 	expectReconciled(t, cr, "", "test")
-	expectEqual(t, fc, expectedConnectorSTS(shortName, fullName, "10.44.0.0/20"))
+	opts.routes = "10.44.0.0/20"
+	expectEqual(t, fc, expectedConnectorSTS(opts))
 
-	// Delete the Connector
+	// Remove the subnet router.
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.SubnetRouter = nil
+	})
+	expectReconciled(t, cr, "", "test")
+	opts.routes = ""
+	expectEqual(t, fc, expectedConnectorSTS(opts))
+
+	// Re-add the subnet router.
+	mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
+		conn.Spec.SubnetRouter = &tsapi.SubnetRouter{
+			Routes: []tsapi.Route{"10.44.0.0/20"},
+		}
+	})
+	expectReconciled(t, cr, "", "test")
+	opts.routes = "10.44.0.0/20"
+	expectEqual(t, fc, expectedConnectorSTS(opts))
+
+	// Delete the Connector.
 	if err = fc.Delete(context.Background(), cn); err != nil {
 		t.Fatalf("error deleting Connector: %v", err)
 	}
@@ -93,22 +124,85 @@ func TestConnector(t *testing.T) {
 	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
 	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 
+	// Create a Connector that advertises a route and is not an exit node.
+	cn = &tsapi.Connector{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test",
+			UID:  types.UID("1234-UID"),
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       tsapi.ConnectorKind,
+			APIVersion: "tailscale.io/v1alpha1",
+		},
+		Spec: tsapi.ConnectorSpec{
+			SubnetRouter: &tsapi.SubnetRouter{
+				Routes: []tsapi.Route{"10.40.0.0/14"},
+			},
+		},
+	}
+	mustCreate(t, fc, cn)
+	expectReconciled(t, cr, "", "test")
+	fullName, shortName = findGenName(t, fc, "", "test", "connector")
+
+	expectEqual(t, fc, expectedSecret(fullName, "", "connector"))
+	opts = connectorSTSOpts{
+		connectorName: "test",
+		stsName:       shortName,
+		secretName:    fullName,
+		routes:        "10.40.0.0/14",
+		isExitNode:    false,
+	}
+	expectEqual(t, fc, expectedConnectorSTS(opts))
+
+	// Delete the Connector.
+	if err = fc.Delete(context.Background(), cn); err != nil {
+		t.Fatalf("error deleting Connector: %v", err)
+	}
+
+	expectRequeue(t, cr, "", "test")
+	expectReconciled(t, cr, "", "test")
+
+	expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
+	expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
 }
 
-func expectedConnectorSTS(stsName, secretName, routes string) *appsv1.StatefulSet {
-	return &appsv1.StatefulSet{
+type connectorSTSOpts struct {
+	stsName       string
+	secretName    string
+	connectorName string
+	hostname      string
+	routes        string
+	isExitNode    bool
+}
+
+func expectedConnectorSTS(opts connectorSTSOpts) *appsv1.StatefulSet {
+	var hostname string
+	if opts.hostname != "" {
+		hostname = opts.hostname
+	} else {
+		hostname = opts.connectorName + "-connector"
+	}
+	containerEnv := []corev1.EnvVar{
+		{Name: "TS_USERSPACE", Value: "false"},
+		{Name: "TS_AUTH_ONCE", Value: "true"},
+		{Name: "TS_KUBE_SECRET", Value: opts.secretName},
+		{Name: "TS_HOSTNAME", Value: hostname},
+		{Name: "TS_EXTRA_ARGS", Value: fmt.Sprintf("--advertise-exit-node=%v", opts.isExitNode)},
+		{Name: "TS_ROUTES", Value: opts.routes},
+	}
+	sts := &appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "StatefulSet",
 			APIVersion: "apps/v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      stsName,
+			Name:      opts.stsName,
 			Namespace: "operator-ns",
 			Labels: map[string]string{
 				"tailscale.com/managed":              "true",
 				"tailscale.com/parent-resource":      "test",
 				"tailscale.com/parent-resource-ns":   "",
-				"tailscale.com/parent-resource-type": "subnetrouter",
+				"tailscale.com/parent-resource-type": "connector",
 			},
 		},
 		Spec: appsv1.StatefulSetSpec{
@@ -116,13 +210,13 @@ func expectedConnectorSTS(stsName, secretName, routes string) *appsv1.StatefulSe
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{"app": "1234-UID"},
 			},
-			ServiceName: stsName,
+			ServiceName: opts.stsName,
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					DeletionGracePeriodSeconds: ptr.To[int64](10),
 					Labels:                     map[string]string{"app": "1234-UID"},
 					Annotations: map[string]string{
-						"tailscale.com/operator-last-set-hostname": "test-subnetrouter",
+						"tailscale.com/operator-last-set-hostname": hostname,
 					},
 				},
 				Spec: corev1.PodSpec{
@@ -142,13 +236,7 @@ func expectedConnectorSTS(stsName, secretName, routes string) *appsv1.StatefulSe
 						{
 							Name:  "tailscale",
 							Image: "tailscale/tailscale",
-							Env: []corev1.EnvVar{
-								{Name: "TS_USERSPACE", Value: "false"},
-								{Name: "TS_AUTH_ONCE", Value: "true"},
-								{Name: "TS_KUBE_SECRET", Value: secretName},
-								{Name: "TS_HOSTNAME", Value: "test-subnetrouter"},
-								{Name: "TS_ROUTES", Value: routes},
-							},
+							Env:   containerEnv,
 							SecurityContext: &corev1.SecurityContext{
 								Capabilities: &corev1.Capabilities{
 									Add: []corev1.Capability{"NET_ADMIN"},
@@ -161,4 +249,5 @@ func expectedConnectorSTS(stsName, secretName, routes string) *appsv1.StatefulSe
 			},
 		},
 	}
+	return sts
 }

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -21,6 +21,9 @@ rules:
 - apiGroups: ["tailscale.com"]
   resources: ["connectors", "connectors/status"]
   verbs: ["get", "list", "watch", "update"]
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["*"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml

@@ -16,11 +16,15 @@ spec:
   scope: Cluster
   versions:
     - additionalPrinterColumns:
-        - description: Cluster CIDR ranges exposed to tailnet via subnet router
-          jsonPath: .status.subnetRouter.routes
+        - description: CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance.
+          jsonPath: .status.subnetRoutes
           name: SubnetRoutes
           type: string
-        - description: Status of the components deployed by the connector
+        - description: Whether this Connector instance defines an exit node.
+          jsonPath: .status.isExitNode
+          name: IsExitNode
+          type: string
+        - description: Status of the deployed Connector resources.
           jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
           name: Status
           type: string
@@ -40,38 +44,39 @@ spec:
             metadata:
               type: object
             spec:
-              description: Desired state of the Connector resource.
+              description: ConnectorSpec describes the desired Tailscale component.
               type: object
-              required:
-                - subnetRouter
               properties:
+                hostname:
+                  description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname is defaulted to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
+                  type: string
+                  pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
+                isExitNode:
+                  description: IsExitNode defines whether the Connector node should act as a Tailscale exit node. Defaults to false. https://tailscale.com/kb/1103/exit-nodes
+                  type: boolean
                 subnetRouter:
-                  description: SubnetRouter configures a Tailscale subnet router to be deployed in the cluster. If unset no subnet router will be deployed. https://tailscale.com/kb/1019/subnets/
+                  description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
                   type: object
                   required:
                     - routes
                   properties:
-                    hostname:
-                      description: Hostname is the tailnet hostname that should be assigned to the subnet router node. If unset hostname is defaulted to <connector name>-subnetrouter. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
-                      type: string
-                      pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
                     routes:
                       description: Routes refer to in-cluster CIDRs that the subnet router should make available. Route values must be strings that represent a valid IPv4 or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. https://tailscale.com/kb/1201/4via6-subnets/
                       type: array
                       items:
                         type: string
                         format: cidr
-                    tags:
-                      description: Tags that the Tailscale node will be tagged with. If you want the subnet router to be autoapproved, you can configure Tailscale ACLs to autoapprove the subnetrouter's CIDRs for these tags. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes Defaults to tag:k8s. If you specify custom tags here, you must also make tag:k8s-operator owner of the custom tag. See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
-                      type: array
-                      items:
-                        type: string
-                        pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
-                  x-kubernetes-validations:
-                    - rule: has(self.tags) == has(oldSelf.tags)
-                      message: Subnetrouter tags cannot be changed. Delete and redeploy the Connector if you need to change it.
+                tags:
+                  description: Tags that the Tailscale node will be tagged with. Defaults to [tag:k8s]. To autoapprove the subnet routes or exit node defined by a Connector, you can configure Tailscale ACLs to give these tags the necessary permissions. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes If you specify custom tags here, you must also make the operator an owner of these tags. See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector node has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                  type: array
+                  items:
+                    type: string
+                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+              x-kubernetes-validations:
+                - rule: has(self.subnetRouter) || self.isExitNode == true
+                  message: A Connector needs to be either an exit node or a subnet router, or both.
             status:
-              description: Status of the Connector. This is set and managed by the Tailscale operator.
+              description: ConnectorStatus describes the status of the Connector. This is set and managed by the Tailscale operator.
               type: object
               properties:
                 conditions:
@@ -107,27 +112,12 @@ spec:
                   x-kubernetes-list-map-keys:
                     - type
                   x-kubernetes-list-type: map
-                subnetRouter:
-                  description: SubnetRouter status is the current status of a subnet router
-                  type: object
-                  required:
-                    - message
-                    - ready
-                    - reason
-                    - routes
-                  properties:
-                    message:
-                      description: Message is a more verbose reason for the current subnet router status
-                      type: string
-                    ready:
-                      description: Ready is the ready status of the subnet router
-                      type: string
-                    reason:
-                      description: Reason is the reason for the subnet router status
-                      type: string
-                    routes:
-                      description: Routes are the CIDRs currently exposed via subnet router
-                      type: string
+                isExitNode:
+                  description: IsExitNode is set to true if the Connector acts as an exit node.
+                  type: boolean
+                subnetRoutes:
+                  description: SubnetRoutes are the routes currently exposed to tailnet via this Connector instance.
+                  type: string
       served: true
       storage: true
       subresources:

cmd/k8s-operator/deploy/examples/connector.yaml

@@ -0,0 +1,19 @@
+# Before applying ensure that the operator owns tag:prod.
+# https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+# To set up autoapproval set tag:prod as approver for 10.40.0.0/14 route and exit node.
+# Otherwise approve it manually in Machines panel once the
+# ts-prod Tailscale node has been created.
+# See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes
+apiVersion: tailscale.com/v1alpha1
+kind: Connector
+metadata:
+  name: prod
+spec:
+  tags:
+  - "tag:prod"
+  hostname: ts-prod
+  subnetRouter:
+    routes:
+    - "10.40.0.0/14"
+    - "192.168.0.0/14"
+  isExitNode: true

cmd/k8s-operator/deploy/examples/subnetrouter.yaml

@@ -1,17 +0,0 @@
-# Before applyong this ensure that the operator is owner of tag:subnet.
-# https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
-# To set up autoapproval set tag:subnet as approver for 10.40.0.0/14 route
-# otherwise you will need to approve it manually in control panel once the
-# subnet router has been created.
-# https://tailscale.com/kb/1019/subnets/#advertise-subnet-routes
-apiVersion: tailscale.com/v1alpha1
-kind: Connector
-metadata:
-  name: exposepods
-spec:
-  subnetRouter:
-    routes:
-    - "10.40.0.0/14"
-    tags:
-    - "tag:subnet"
-    hostname: pods-subnetrouter

cmd/k8s-operator/operator.go

@@ -9,7 +9,6 @@ package main
 
 import (
 	"context"
-	"os"
 	"regexp"
 	"strings"
 	"time"
@@ -17,7 +16,6 @@ import (
 	"github.com/go-logr/zapr"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"golang.org/x/oauth2/clientcredentials"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -41,6 +39,7 @@ import (
 	"tailscale.com/tsnet"
 	"tailscale.com/tstime"
 	"tailscale.com/types/logger"
+	"tailscale.com/util/must"
 	"tailscale.com/version"
 )
 
@@ -56,13 +55,13 @@ func main() {
 	tailscale.I_Acknowledge_This_API_Is_Unstable = true
 
 	var (
-		tsNamespace       = defaultEnv("OPERATOR_NAMESPACE", "")
-		tslogging         = defaultEnv("OPERATOR_LOGGING", "info")
-		image             = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
-		priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
-		tags              = defaultEnv("PROXY_TAGS", "tag:k8s")
-		tsFirewallMode    = defaultEnv("PROXY_FIREWALL_MODE", "")
-		tsEnableConnector = defaultBool("ENABLE_CONNECTOR", false)
+		// tsNamespace       = defaultEnv("OPERATOR_NAMESPACE", "")
+		tslogging = defaultEnv("OPERATOR_LOGGING", "info")
+		// image             = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest")
+		// priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "")
+		// tags              = defaultEnv("PROXY_TAGS", "tag:k8s")
+		// tsFirewallMode    = defaultEnv("PROXY_FIREWALL_MODE", "")
+		// tsEnableConnector = defaultBool("ENABLE_CONNECTOR", false)
 	)
 
 	var opts []kzap.Opts
@@ -80,20 +79,22 @@ func main() {
 	// The operator can run either as a plain operator or it can
 	// additionally act as api-server proxy
 	// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
-	mode := parseAPIProxyMode()
+	// mode := parseAPIProxyMode()
+	mode := apiserverProxyModeNoAuth
 	if mode == apiserverProxyModeDisabled {
 		hostinfo.SetApp("k8s-operator")
 	} else {
 		hostinfo.SetApp("k8s-operator-proxy")
 	}
 
-	s, tsClient := initTSNet(zlog)
+	s, _ := initTSNet(zlog)
 	defer s.Close()
-	restConfig := config.GetConfigOrDie()
+	restConfig := must.Get(config.GetConfigWithContext(""))
 	maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
+	select {}
 	// TODO (irbekrm): gather the reconciler options into an opts struct
 	// rather than passing a million of them in one by one.
-	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tsEnableConnector)
+	// runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tsEnableConnector)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -101,31 +102,31 @@ func main() {
 // with Tailscale.
 func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
 	var (
-		clientIDPath     = defaultEnv("CLIENT_ID_FILE", "")
-		clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "")
-		hostname         = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator")
-		kubeSecret       = defaultEnv("OPERATOR_SECRET", "")
-		operatorTags     = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator")
+		// clientIDPath     = defaultEnv("CLIENT_ID_FILE", "")
+		// clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "")
+		hostname   = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator")
+		kubeSecret = defaultEnv("OPERATOR_SECRET", "")
+		// operatorTags     = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator")
 	)
 	startlog := zlog.Named("startup")
-	if clientIDPath == "" || clientSecretPath == "" {
-		startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set")
-	}
-	clientID, err := os.ReadFile(clientIDPath)
-	if err != nil {
-		startlog.Fatalf("reading client ID %q: %v", clientIDPath, err)
-	}
-	clientSecret, err := os.ReadFile(clientSecretPath)
-	if err != nil {
-		startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err)
-	}
-	credentials := clientcredentials.Config{
-		ClientID:     string(clientID),
-		ClientSecret: string(clientSecret),
-		TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
-	}
-	tsClient := tailscale.NewClient("-", nil)
-	tsClient.HTTPClient = credentials.Client(context.Background())
+	// if clientIDPath == "" || clientSecretPath == "" {
+	// 	startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set")
+	// }
+	// clientID, err := os.ReadFile(clientIDPath)
+	// if err != nil {
+	// 	startlog.Fatalf("reading client ID %q: %v", clientIDPath, err)
+	// }
+	// clientSecret, err := os.ReadFile(clientSecretPath)
+	// if err != nil {
+	// 	startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err)
+	// }
+	// credentials := clientcredentials.Config{
+	// 	ClientID:     string(clientID),
+	// 	ClientSecret: string(clientSecret),
+	// 	TokenURL:     "https://login.tailscale.com/api/v2/oauth/token",
+	// }
+	// tsClient := tailscale.NewClient("-", nil)
+	// tsClient.HTTPClient = credentials.Client(context.Background())
 
 	s := &tsnet.Server{
 		Hostname: hostname,
@@ -163,21 +164,22 @@ waitOnline:
 			if loginDone {
 				break
 			}
-			caps := tailscale.KeyCapabilities{
-				Devices: tailscale.KeyDeviceCapabilities{
-					Create: tailscale.KeyDeviceCreateCapabilities{
-						Reusable:      false,
-						Preauthorized: true,
-						Tags:          strings.Split(operatorTags, ","),
-					},
-				},
-			}
-			authkey, _, err := tsClient.CreateKey(ctx, caps)
-			if err != nil {
-				startlog.Fatalf("creating operator authkey: %v", err)
-			}
+			// caps := tailscale.KeyCapabilities{
+			// 	Devices: tailscale.KeyDeviceCapabilities{
+			// 		Create: tailscale.KeyDeviceCreateCapabilities{
+			// 			Reusable:      false,
+			// 			Preauthorized: true,
+			// 			Tags:          strings.Split(operatorTags, ","),
+			// 		},
+			// 	},
+			// }
+			// authkey := ""
+			// authkey, _, err := tsClient.CreateKey(ctx, caps)
+			// if err != nil {
+			// 	startlog.Fatalf("creating operator authkey: %v", err)
+			// }
 			if err := lc.Start(ctx, ipn.Options{
-				AuthKey: authkey,
+				// AuthKey: authkey,
 			}); err != nil {
 				startlog.Fatalf("starting tailscale: %v", err)
 			}
@@ -196,7 +198,7 @@ waitOnline:
 		}
 		time.Sleep(time.Second)
 	}
-	return s, tsClient
+	return s, nil
 }
 
 // runReconcilers starts the controller-runtime manager and registers the

cmd/k8s-operator/proxy.go

@@ -6,16 +6,25 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httputil"
+	"net/textproto"
 	"net/url"
 	"os"
+	"path"
 	"strings"
+	"sync"
+	"time"
 
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/transport"
@@ -87,9 +96,9 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config,
 		return
 	}
 	startlog := zlog.Named("launchAPIProxy")
-	if mode == apiserverProxyModeNoAuth {
-		restConfig = rest.AnonymousClientConfig(restConfig)
-	}
+	// if mode == apiserverProxyModeNoAuth {
+	// 	restConfig = rest.AnonymousClientConfig(restConfig)
+	// }
 	cfg, err := restConfig.TransportConfig()
 	if err != nil {
 		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
@@ -108,15 +117,89 @@ func maybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config,
 	if err != nil {
 		startlog.Fatalf("could not get rest.TransportConfig(): %v", err)
 	}
-	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode)
+	go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host)
 }
 
 // apiserverProxy is an http.Handler that authenticates requests using the Tailscale
 // LocalAPI and then proxies them to the Kubernetes API.
 type apiserverProxy struct {
-	log *zap.SugaredLogger
-	lc  *tailscale.LocalClient
-	rp  *httputil.ReverseProxy
+	log  *zap.SugaredLogger
+	lc   *tailscale.LocalClient
+	rp   *httputil.ReverseProxy
+	mode apiServerProxyMode
+
+	upstreamURL    *url.URL
+	upstreamClient *http.Client
+}
+
+// Hop-by-hop headers. These are removed when sent to the backend.
+// As of RFC 7230, hop-by-hop headers are required to appear in the
+// Connection header field. These are the headers defined by the
+// obsoleted RFC 2616 (section 13.5.1) and are used for backward
+// compatibility.
+var hopHeaders = []string{
+	"Connection",
+	"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
+	"Keep-Alive",
+	"Proxy-Authenticate",
+	"Proxy-Authorization",
+	"Te",      // canonicalized version of "TE"
+	"Trailer", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522
+	"Transfer-Encoding",
+	"Upgrade",
+}
+
+// removeHopByHopHeaders removes hop-by-hop headers.
+func removeHopByHopHeaders(h http.Header) {
+	// RFC 7230, section 6.1: Remove headers listed in the "Connection" header.
+	for _, f := range h["Connection"] {
+		for _, sf := range strings.Split(f, ",") {
+			if sf = textproto.TrimString(sf); sf != "" {
+				h.Del(sf)
+			}
+		}
+	}
+	// RFC 2616, section 13.5.1: Remove a set of known hop-by-hop headers.
+	// This behavior is superseded by the RFC 7230 Connection header, but
+	// preserve it for backwards compatibility.
+	for _, f := range hopHeaders {
+		h.Del(f)
+	}
+}
+
+func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
+	// Replace the URL with the Kubernetes APIServer.
+
+	r.URL.Scheme = h.upstreamURL.Scheme
+	r.URL.Host = h.upstreamURL.Host
+	if h.mode == apiserverProxyModeNoAuth {
+		// If we are not providing authentication, then we are just
+		// proxying to the Kubernetes API, so we don't need to do
+		// anything else.
+		return
+	}
+
+	// We want to proxy to the Kubernetes API, but we want to use
+	// the caller's identity to do so. We do this by impersonating
+	// the caller using the Kubernetes User Impersonation feature:
+	// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
+
+	// Out of paranoia, remove all authentication headers that might
+	// have been set by the client.
+	r.Header.Del("Authorization")
+	r.Header.Del("Impersonate-Group")
+	r.Header.Del("Impersonate-User")
+	r.Header.Del("Impersonate-Uid")
+	for k := range r.Header {
+		if strings.HasPrefix(k, "Impersonate-Extra-") {
+			r.Header.Del(k)
+		}
+	}
+
+	// Now add the impersonation headers that we want.
+	if err := addImpersonationHeaders(r, h.log); err != nil {
+		panic("failed to add impersonation headers: " + err.Error())
+	}
 }
 
 func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -127,7 +210,84 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	counterNumRequestsProxied.Add(1)
-	h.rp.ServeHTTP(w, addWhoIsToRequest(r, who))
+	r = addWhoIsToRequest(r, who)
+	if r.Method != "POST" || path.Base(r.URL.Path) != "exec" { // also check for pod
+		h.rp.ServeHTTP(w, r)
+		return
+	}
+	// hj := w.(http.Hijacker)
+	// reqConn, brw, err := hj.Hijack()
+	// if err != nil {
+	// 	return
+	// }
+	// defer reqConn.Close()
+	// if err := brw.Flush(); err != nil {
+	// 	return
+	// }
+	// reqConn = netutil.NewDrainBufConn(reqConn, brw.Reader)
+	// respConn, err := net.Dial("tcp", h.upstreamURL.Host)
+	// if err != nil {
+	// 	h.log.Errorf("failed to dial upstream: %v", err)
+	// 	return
+	// }
+	// defer respConn.Close()
+
+	req2 := r.Clone(r.Context())
+	h.addImpersonationHeadersAsRequired(req2)
+
+	req2.Body = io.NopCloser(io.TeeReader(r.Body, os.Stdout))
+	defer r.Body.Close()
+
+	h.rp.ServeHTTP(&teeResponseWriter{
+		ResponseWriter: w,
+		hj:             w.(http.Hijacker),
+		multiWriter:    io.MultiWriter(os.Stdout, w),
+	}, req2)
+}
+
+type teeResponseWriter struct {
+	http.ResponseWriter
+	hj          http.Hijacker
+	multiWriter io.Writer
+}
+
+func (w *teeResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	reqConn, brw, err := w.hj.Hijack()
+	if err != nil {
+		return nil, nil, err
+	}
+	f, err := os.OpenFile("/tmp/recording.cast", os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, nil, err
+	}
+	r := &recording{
+		start:    time.Now(),
+		failOpen: true,
+		out:      f,
+	}
+	lc := &loggingConn{Conn: reqConn, lw: &loggingWriter{
+		r:                   r,
+		recordingFailedOpen: false,
+	}}
+
+	ch := CastHeader{
+		Version:   2,
+		Timestamp: r.start.Unix(),
+	}
+	j, err := json.Marshal(ch)
+	if err != nil {
+		return nil, nil, err
+	}
+	j = append(j, '\n')
+	if _, err := f.Write(j); err != nil {
+		return nil, nil, err
+	}
+
+	return lc, brw, nil
+}
+
+func (w *teeResponseWriter) Write(b []byte) (int, error) {
+	return w.multiWriter.Write(b)
 }
 
 // runAPIServerProxy runs an HTTP server that authenticates requests using the
@@ -144,7 +304,7 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 //     are passed through to the Kubernetes API.
 //
 // It never returns.
-func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode) {
+func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode, host string) {
 	if mode == apiserverProxyModeDisabled {
 		return
 	}
@@ -152,7 +312,7 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 	if err != nil {
 		log.Fatalf("could not listen on :443: %v", err)
 	}
-	u, err := url.Parse(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
+	u, err := url.Parse(host)
 	if err != nil {
 		log.Fatalf("runAPIServerProxy: failed to parse URL %v", err)
 	}
@@ -162,45 +322,16 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 		log.Fatalf("could not get local client: %v", err)
 	}
 	ap := &apiserverProxy{
-		log: log,
-		lc:  lc,
-		rp: &httputil.ReverseProxy{
-			Rewrite: func(r *httputil.ProxyRequest) {
-				// Replace the URL with the Kubernetes APIServer.
-
-				r.Out.URL.Scheme = u.Scheme
-				r.Out.URL.Host = u.Host
-				if mode == apiserverProxyModeNoAuth {
-					// If we are not providing authentication, then we are just
-					// proxying to the Kubernetes API, so we don't need to do
-					// anything else.
-					return
-				}
-
-				// We want to proxy to the Kubernetes API, but we want to use
-				// the caller's identity to do so. We do this by impersonating
-				// the caller using the Kubernetes User Impersonation feature:
-				// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation
-
-				// Out of paranoia, remove all authentication headers that might
-				// have been set by the client.
-				r.Out.Header.Del("Authorization")
-				r.Out.Header.Del("Impersonate-Group")
-				r.Out.Header.Del("Impersonate-User")
-				r.Out.Header.Del("Impersonate-Uid")
-				for k := range r.Out.Header {
-					if strings.HasPrefix(k, "Impersonate-Extra-") {
-						r.Out.Header.Del(k)
-					}
-				}
-
-				// Now add the impersonation headers that we want.
-				if err := addImpersonationHeaders(r.Out, log); err != nil {
-					panic("failed to add impersonation headers: " + err.Error())
-				}
-			},
-			Transport: rt,
+		log:         log,
+		lc:          lc,
+		mode:        mode,
+		upstreamURL: u,
+	}
+	ap.rp = &httputil.ReverseProxy{
+		Rewrite: func(pr *httputil.ProxyRequest) {
+			ap.addImpersonationHeadersAsRequired(pr.Out)
 		},
+		Transport: rt,
 	}
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
@@ -285,3 +416,151 @@ func addImpersonationHeaders(r *http.Request, log *zap.SugaredLogger) error {
 	}
 	return nil
 }
+
+// CastHeader is the header of an asciinema file.
+type CastHeader struct {
+	// Version is the asciinema file format version.
+	Version int `json:"version"`
+
+	// Width is the terminal width in characters.
+	// It is non-zero for Pty sessions.
+	Width int `json:"width"`
+
+	// Height is the terminal height in characters.
+	// It is non-zero for Pty sessions.
+	Height int `json:"height"`
+
+	// Timestamp is the unix timestamp of when the recording started.
+	Timestamp int64 `json:"timestamp"`
+
+	// Env is the environment variables of the session.
+	// Only "TERM" is set (2023-03-22).
+	Env map[string]string `json:"env"`
+
+	// Command is the command that was executed.
+	// Typically empty for shell sessions.
+	Command string `json:"command,omitempty"`
+
+	// Tailscale-specific fields:
+	// SrcNode is the FQDN of the node originating the connection.
+	// It is also the MagicDNS name for the node.
+	// It does not have a trailing dot.
+	// e.g. "host.tail-scale.ts.net"
+	SrcNode string `json:"srcNode"`
+
+	// SrcNodeID is the node ID of the node originating the connection.
+	SrcNodeID tailcfg.StableNodeID `json:"srcNodeID"`
+
+	// SrcNodeTags is the list of tags on the node originating the connection (if any).
+	SrcNodeTags []string `json:"srcNodeTags,omitempty"`
+
+	// SrcNodeUserID is the user ID of the node originating the connection (if not tagged).
+	SrcNodeUserID tailcfg.UserID `json:"srcNodeUserID,omitempty"` // if not tagged
+
+	// SrcNodeUser is the LoginName of the node originating the connection (if not tagged).
+	SrcNodeUser string `json:"srcNodeUser,omitempty"`
+
+	// SSHUser is the username as presented by the client.
+	SSHUser string `json:"sshUser"` // as presented by the client
+
+	// LocalUser is the effective username on the server.
+	LocalUser string `json:"localUser"`
+
+	// ConnectionID uniquely identifies a connection made to the SSH server.
+	// It may be shared across multiple sessions over the same connection in
+	// case of SSH multiplexing.
+	ConnectionID string `json:"connectionID"`
+}
+
+// loggingWriter is an io.Writer wrapper that writes first an
+// asciinema JSON cast format recording line, and then writes to w.
+type loggingWriter struct {
+	r *recording
+
+	// recordingFailedOpen specifies whether we've failed to write to
+	// r.out and should stop trying. It is set to true if we fail to write
+	// to r.out and r.failOpen is set.
+	recordingFailedOpen bool
+}
+
+func (w *loggingWriter) Write(p []byte) (n int, err error) {
+	if w.recordingFailedOpen {
+		return 0, nil
+	}
+	j, err := json.Marshal([]any{
+		time.Since(w.r.start).Seconds(),
+		"o",
+		string(p),
+	})
+	if err != nil {
+		return 0, err
+	}
+	j = append(j, '\n')
+	if err := w.writeCastLine(j); err != nil {
+		if !w.r.failOpen {
+			return 0, err
+		}
+		w.recordingFailedOpen = true
+	}
+	return len(p), nil
+}
+
+func (w *loggingWriter) writeCastLine(j []byte) error {
+	w.r.mu.Lock()
+	defer w.r.mu.Unlock()
+	if w.r.out == nil {
+		return errors.New("logger closed")
+	}
+	_, err := w.r.out.Write(j)
+	if err != nil {
+		return fmt.Errorf("logger Write: %w", err)
+	}
+	return nil
+}
+
+type loggingConn struct {
+	mu     sync.Mutex // guards writes to r.out
+	closed bool
+	net.Conn
+	lw *loggingWriter
+}
+
+func (c *loggingConn) Write(b []byte) (int, error) {
+	n, err := c.Conn.Write(b)
+	c.lw.Write(b[:n])
+	return n, err
+}
+
+func (c *loggingConn) Close() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.closed {
+		return nil
+	}
+	c.closed = true
+	c.lw.r.Close()
+	return c.Conn.Close()
+}
+
+// recording is the state for an SSH session recording.
+type recording struct {
+	start time.Time
+
+	// failOpen specifies whether the session should be allowed to
+	// continue if writing to the recording fails.
+	failOpen bool
+
+	mu  sync.Mutex // guards writes to, close of out
+	out io.WriteCloser
+}
+
+func (r *recording) Close() error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.out == nil {
+		return nil
+	}
+	err := r.out.Close()
+	r.out = nil
+	return err
+}

cmd/k8s-operator/sts.go

@@ -11,6 +11,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"strings"
@@ -59,6 +60,7 @@ const (
 	podAnnotationLastSetHostname          = "tailscale.com/operator-last-set-hostname"
 	podAnnotationLastSetTailnetTargetIP   = "tailscale.com/operator-last-set-ts-tailnet-target-ip"
 	podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
+	podAnnotationLastSetConfigFileHash    = "tailscale.com/operator-last-set-config-file-hash"
 )
 
 type tailscaleSTSConfig struct {
@@ -79,9 +81,19 @@ type tailscaleSTSConfig struct {
 	Hostname string
 	Tags     []string // if empty, use defaultTags
 
-	// Routes is a list of CIDRs to pass via --advertise-routes flag
-	// Should only be set if this is config for subnetRouter
-	Routes string
+	// Connector specifies a configuration of a Connector instance if that's
+	// what this StatefulSet should be created for.
+	Connector *connector
+
+	// temp fix whilst prototyping, remove
+	key string
+}
+
+type connector struct {
+	// routes is a list of subnet routes that this Connector should expose.
+	routes string
+	// isExitNode defines whether this Connector should act as an exit node.
+	isExitNode bool
 }
 
 type tailscaleSTSReconciler struct {
@@ -116,10 +128,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
 		return nil, fmt.Errorf("failed to reconcile headless service: %w", err)
 	}
 
-	secretName, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
+	secretName, key, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
 	}
+	sts.key = key
 	_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
@@ -234,7 +247,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
 	return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
 }
 
-func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, error) {
+func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (string, string, error) {
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			// Hardcode a -0 suffix so that in future, if we support
@@ -250,22 +263,23 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
 		orig = secret.DeepCopy()
 	} else if !apierrors.IsNotFound(err) {
-		return "", err
+		return "", "", err
 	}
 
+	var authKey string
 	if orig == nil {
 		// Secret doesn't exist yet, create one. Initially it contains
 		// only the Tailscale authkey, but once Tailscale starts it'll
 		// also store the daemon state.
 		sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
 		if err != nil {
-			return "", err
+			return "", "", err
 		}
 		if sts != nil {
 			// StatefulSet exists, so we have already created the secret.
 			// If the secret is missing, they should delete the StatefulSet.
 			logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
-			return "", nil
+			return "", "", nil
 		}
 		// Create API Key secret which is going to be used by the statefulset
 		// to authenticate with Tailscale.
@@ -276,28 +290,32 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *
 		}
 		authKey, err := a.newAuthKey(ctx, tags)
 		if err != nil {
-			return "", err
+			return "", "", err
 		}
 
 		mak.Set(&secret.StringData, "authkey", authKey)
+	} else {
+		authKey = string(orig.Data["authkey"])
 	}
 	if stsC.ServeConfig != nil {
 		j, err := json.Marshal(stsC.ServeConfig)
 		if err != nil {
-			return "", err
+			return "", "", err
 		}
 		mak.Set(&secret.StringData, "serve-config", string(j))
 	}
 	if orig != nil {
+		log.Printf("Patching existing secret %s", secret.Name)
 		if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
-			return "", err
+			return "", "", err
 		}
 	} else {
 		if err := a.Create(ctx, secret); err != nil {
-			return "", err
+			return "", authKey, err
 		}
+		log.Printf("Created secret %s", secret.Name)
 	}
-	return secret.Name, nil
+	return secret.Name, authKey, nil
 }
 
 // DeviceInfo returns the device ID and hostname for the Tailscale device
@@ -382,6 +400,8 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_HOSTNAME",
 			Value: sts.Hostname,
 		})
+
+	var configFileHash string
 	if sts.ClusterTargetIP != "" {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "TS_DEST_IP",
@@ -419,19 +439,51 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 				},
 			},
 		})
-	} else if len(sts.Routes) > 0 {
+	} else if sts.Connector != nil {
+		// TODO: definitely not the right place for this
+		var err error
+		configFileHash, err = a.tsConfigCM(ctx, headlessSvc.Name, a.operatorNamespace, logger, sts)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create configmap: %w", err)
+		}
 		container.Env = append(container.Env, corev1.EnvVar{
-			Name:  "TS_ROUTES",
-			Value: sts.Routes,
+			Name:  "TS_CONFIGFILE_PATH",
+			Value: "/tsconfig/tailscaled",
 		})
+		ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+			Name: "configfile",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: headlessSvc.Name,
+					},
+				},
+			},
+		})
+		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+			Name:      "configfile",
+			MountPath: "/tsconfig",
+		})
+		if sts.key != "" {
 
+		}
+		// We need to provide these env vars even if the values are empty to
+		// ensure that a transition from a Connector with a defined subnet
+		// router or exit node to one without succeeds.
+		// container.Env = append(container.Env, corev1.EnvVar{
+		// 	Name:  "TS_EXTRA_ARGS",
+		// 	Value: fmt.Sprintf("--advertise-exit-node=%v", sts.Connector.isExitNode),
+		// })
+		// container.Env = append(container.Env, corev1.EnvVar{
+		// 	Name:  "TS_ROUTES",
+		// 	Value: sts.Connector.routes,
+		// })
 	}
 	if a.tsFirewallMode != "" {
 		container.Env = append(container.Env, corev1.EnvVar{
 			Name:  "TS_DEBUG_FIREWALL_MODE",
 			Value: a.tsFirewallMode,
-		},
-		)
+		})
 	}
 	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:      headlessSvc.Name,
@@ -461,6 +513,9 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 	if sts.TailnetTargetFQDN != "" {
 		ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetFQDN] = sts.TailnetTargetFQDN
 	}
+	if configFileHash != "" {
+		ss.Spec.Template.Annotations[podAnnotationLastSetTailnetTargetFQDN] = configFileHash
+	}
 	ss.Spec.Template.Labels = map[string]string{
 		"app": sts.ParentResourceUID,
 	}

cmd/tailscaled/tailscaled.go

@@ -332,11 +332,13 @@ func run() (err error) {
 	// Parse config, if specified, to fail early if it's invalid.
 	var conf *conffile.Config
 	if args.confFile != "" {
+		logf("loading config file")
 		conf, err = conffile.Load(args.confFile)
 		if err != nil {
 			return fmt.Errorf("error reading config file: %w", err)
 		}
 		sys.InitialConfig = conf
+		logf("loaded initial config: %#+v", string(sys.InitialConfig.Raw))
 	}
 
 	var netMon *netmon.Monitor

ipn/ipnlocal/local.go

@@ -341,6 +341,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
 	}
 
 	if sys.InitialConfig != nil {
+		log.Printf("Found initial config")
 		p := pm.CurrentPrefs().AsStruct()
 		mp, err := sys.InitialConfig.Parsed.ToPrefs()
 		if err != nil {

ipn/ipnlocal/profiles.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"math/rand"
 	"runtime"
 	"slices"
@@ -209,9 +210,11 @@ func init() {
 // is logged into so that we can keep track of things like their domain name
 // across user switches to disambiguate the same account but a different tailnet.
 func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile) error {
+	log.Printf("set prefs")
 	prefs := prefsIn.AsStruct()
 	newPersist := prefs.Persist
 	if newPersist == nil || newPersist.NodeID == "" || newPersist.UserProfile.LoginName == "" {
+		log.Printf("prefs: ignore profile")
 		// We don't know anything about this profile, so ignore it for now.
 		return pm.setPrefsLocked(prefs.View())
 	}
@@ -220,6 +223,7 @@ func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile)
 		up.DisplayName = up.LoginName
 	}
 	cp := pm.currentProfile
+	log.Printf("current profile: %v", cp.UserProfile)
 	// Check if we already have an existing profile that matches the user/node.
 	if existing := pm.findMatchingProfiles(prefs); len(existing) > 0 {
 		// We already have a profile for this user/node we should reuse it. Also
@@ -256,12 +260,15 @@ func (pm *profileManager) SetPrefs(prefsIn ipn.PrefsView, np ipn.NetworkProfile)
 	pm.knownProfiles[cp.ID] = cp
 	pm.currentProfile = cp
 	if err := pm.writeKnownProfiles(); err != nil {
+		log.Printf("error writing known profiles")
 		return err
 	}
 	if err := pm.setAsUserSelectedProfileLocked(); err != nil {
+		log.Printf("error locking profile")
 		return err
 	}
 	if err := pm.setPrefsLocked(prefs.View()); err != nil {
+		log.Printf("error viewing prefs")
 		return err
 	}
 	return nil

k8s-operator/apis/v1alpha1/types_connector.go

@@ -6,6 +6,9 @@
 package v1alpha1
 
 import (
+	"fmt"
+	"strings"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -17,17 +20,19 @@ var ConnectorKind = "Connector"
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster,shortName=cn
-// +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRouter.routes`,description="Cluster CIDR ranges exposed to tailnet via subnet router"
-// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the components deployed by the connector"
+// +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRoutes`,description="CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance."
+// +kubebuilder:printcolumn:name="IsExitNode",type="string",JSONPath=`.status.isExitNode`,description="Whether this Connector instance defines an exit node."
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the deployed Connector resources."
 
 type Connector struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	// Desired state of the Connector resource.
+	// ConnectorSpec describes the desired Tailscale component.
 	Spec ConnectorSpec `json:"spec"`
 
-	// Status of the Connector. This is set and managed by the Tailscale operator.
+	// ConnectorStatus describes the status of the Connector. This is set
+	// and managed by the Tailscale operator.
 	// +optional
 	Status ConnectorStatus `json:"status"`
 }
@@ -41,40 +46,69 @@ type ConnectorList struct {
 	Items []Connector `json:"items"`
 }
 
-// ConnectorSpec defines the desired state of a ConnectorSpec.
+// ConnectorSpec describes a Tailscale node to be deployed in the cluster.
+// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || self.isExitNode == true",message="A Connector needs to be either an exit node or a subnet router, or both."
 type ConnectorSpec struct {
-	// SubnetRouter configures a Tailscale subnet router to be deployed in
-	// the cluster. If unset no subnet router will be deployed.
+	// Tags that the Tailscale node will be tagged with.
+	// Defaults to [tag:k8s].
+	// To autoapprove the subnet routes or exit node defined by a Connector,
+	// you can configure Tailscale ACLs to give these tags the necessary
+	// permissions.
+	// See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes
+	// If you specify custom tags here, you must also make the operator an owner of these tags.
+	// See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+	// Tags cannot be changed once a Connector node has been created.
+	// Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+	// +optional
+	Tags Tags `json:"tags,omitempty"`
+	// Hostname is the tailnet hostname that should be assigned to the
+	// Connector node. If unset, hostname is defaulted to <connector
+	// name>-connector. Hostname can contain lower case letters, numbers and
+	// dashes, it must not start or end with a dash and must be between 2
+	// and 63 characters long.
+	// +optional
+	Hostname Hostname `json:"hostname,omitempty"`
+	// SubnetRouter defines subnet routes that the Connector node should
+	// expose to tailnet. If unset, none are exposed.
 	// https://tailscale.com/kb/1019/subnets/
+	// +optional
 	SubnetRouter *SubnetRouter `json:"subnetRouter"`
+	// IsExitNode defines whether the Connector node should act as a
+	// Tailscale exit node. Defaults to false.
+	// https://tailscale.com/kb/1103/exit-nodes
+	// +optional
+	IsExitNode bool `json:"isExitNode"`
 }
 
-// SubnetRouter describes a subnet router.
-// +kubebuilder:validation:XValidation:rule="has(self.tags) == has(oldSelf.tags)",message="Subnetrouter tags cannot be changed. Delete and redeploy the Connector if you need to change it."
+// SubnetRouter defines subnet routes that should be exposed to tailnet via a
+// Connector node.
 type SubnetRouter struct {
 	// Routes refer to in-cluster CIDRs that the subnet router should make
 	// available. Route values must be strings that represent a valid IPv4
 	// or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes.
 	// https://tailscale.com/kb/1201/4via6-subnets/
-	Routes []Route `json:"routes"`
-	// Tags that the Tailscale node will be tagged with. If you want the
-	// subnet router to be autoapproved, you can configure Tailscale ACLs to
-	// autoapprove the subnetrouter's CIDRs for these tags.
-	// See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes
-	// Defaults to tag:k8s.
-	// If you specify custom tags here, you must also make tag:k8s-operator owner of the custom tag.
-	// See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
-	// Tags cannot be changed once a Connector has been created.
-	// Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
-	// +optional
-	Tags []Tag `json:"tags,omitempty"`
-	// Hostname is the tailnet hostname that should be assigned to the
-	// subnet router node. If unset hostname is defaulted to <connector
-	// name>-subnetrouter. Hostname can contain lower case letters, numbers
-	// and dashes, it must not start or end with a dash and must be between
-	// 2 and 63 characters long.
-	// +optional
-	Hostname Hostname `json:"hostname,omitempty"`
+	Routes Routes `json:"routes"`
+}
+
+type Tags []Tag
+
+func (tags Tags) Stringify() []string {
+	stringTags := make([]string, len(tags))
+	for i, t := range tags {
+		stringTags[i] = string(t)
+	}
+	return stringTags
+}
+
+type Routes []Route
+
+func (routes Routes) Stringify() string {
+	var sb strings.Builder
+	sb.WriteString(string(routes[0]))
+	for _, r := range routes[1:] {
+		sb.WriteString(fmt.Sprintf(",%s", r))
+	}
+	return sb.String()
 }
 
 // +kubebuilder:validation:Type=string
@@ -91,28 +125,19 @@ type Hostname string
 
 // ConnectorStatus defines the observed state of the Connector.
 type ConnectorStatus struct {
-
 	// List of status conditions to indicate the status of the Connector.
 	// Known condition types are `ConnectorReady`.
 	// +listType=map
 	// +listMapKey=type
 	// +optional
 	Conditions []ConnectorCondition `json:"conditions"`
-	// SubnetRouter status is the current status of a subnet router
+	// SubnetRoutes are the routes currently exposed to tailnet via this
+	// Connector instance.
 	// +optional
-	SubnetRouter *SubnetRouterStatus `json:"subnetRouter"`
-}
-
-// SubnetRouter status is the current status of a subnet router if deployed
-type SubnetRouterStatus struct {
-	// Routes are the CIDRs currently exposed via subnet router
-	Routes string `json:"routes"`
-	// Ready is the ready status of the subnet router
-	Ready metav1.ConditionStatus `json:"ready"`
-	// Reason is the reason for the subnet router status
-	Reason string `json:"reason"`
-	// Message is a more verbose reason for the current subnet router status
-	Message string `json:"message"`
+	SubnetRoutes string `json:"subnetRoutes"`
+	// IsExitNode is set to true if the Connector acts as an exit node.
+	// +optional
+	IsExitNode bool `json:"isExitNode"`
 }
 
 // ConnectorCondition contains condition information for a Connector.
@@ -147,7 +172,7 @@ type ConnectorCondition struct {
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 }
 
-// ConnectorConditionType represents a Connector condition type
+// ConnectorConditionType represents a Connector condition type.
 type ConnectorConditionType string
 
 const (

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -92,6 +92,11 @@ func (in *ConnectorList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) {
 	*out = *in
+	if in.Tags != nil {
+		in, out := &in.Tags, &out.Tags
+		*out = make(Tags, len(*in))
+		copy(*out, *in)
+	}
 	if in.SubnetRouter != nil {
 		in, out := &in.SubnetRouter, &out.SubnetRouter
 		*out = new(SubnetRouter)
@@ -119,11 +124,6 @@ func (in *ConnectorStatus) DeepCopyInto(out *ConnectorStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	if in.SubnetRouter != nil {
-		in, out := &in.SubnetRouter, &out.SubnetRouter
-		*out = new(SubnetRouterStatus)
-		**out = **in
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorStatus.
@@ -136,17 +136,31 @@ func (in *ConnectorStatus) DeepCopy() *ConnectorStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in Routes) DeepCopyInto(out *Routes) {
+	{
+		in := &in
+		*out = make(Routes, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Routes.
+func (in Routes) DeepCopy() Routes {
+	if in == nil {
+		return nil
+	}
+	out := new(Routes)
+	in.DeepCopyInto(out)
+	return *out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) {
 	*out = *in
 	if in.Routes != nil {
 		in, out := &in.Routes, &out.Routes
-		*out = make([]Route, len(*in))
-		copy(*out, *in)
-	}
-	if in.Tags != nil {
-		in, out := &in.Tags, &out.Tags
-		*out = make([]Tag, len(*in))
+		*out = make(Routes, len(*in))
 		copy(*out, *in)
 	}
 }
@@ -162,16 +176,20 @@ func (in *SubnetRouter) DeepCopy() *SubnetRouter {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *SubnetRouterStatus) DeepCopyInto(out *SubnetRouterStatus) {
-	*out = *in
+func (in Tags) DeepCopyInto(out *Tags) {
+	{
+		in := &in
+		*out = make(Tags, len(*in))
+		copy(*out, *in)
+	}
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetRouterStatus.
-func (in *SubnetRouterStatus) DeepCopy() *SubnetRouterStatus {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tags.
+func (in Tags) DeepCopy() Tags {
 	if in == nil {
 		return nil
 	}
-	out := new(SubnetRouterStatus)
+	out := new(Tags)
 	in.DeepCopyInto(out)
-	return out
+	return *out
 }