cmd/k8s-operator,ipn: prototype- user provisioned certs

Allow users to pass their own certs instead of minting LE ones each time

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -35,6 +35,9 @@ spec:
       - name: oauth
         secret:
           secretName: operator-oauth
+      - name: tls-certs
+        secret:
+          secretName: tls-certs
       containers:
         - name: operator
           {{- with .Values.operatorConfig.securityContext }}
@@ -74,10 +77,17 @@ spec:
               value: "{{ .Values.apiServerProxyConfig.mode }}"
             - name: PROXY_FIREWALL_MODE
               value: {{ .Values.proxyConfig.firewallMode }}
+            - name: TLS_CERT_PATH
+              value: /tls/tls.crt
+            - name: TLS_KEY_PATH
+              value: /tls/tls.key
           volumeMounts:
           - name: oauth
             mountPath: /oauth
             readOnly: true
+          - name: tls-certs
+            mountPath: /tls
+            readOnly: true
       {{- with .Values.operatorConfig.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -174,6 +174,10 @@ spec:
                       value: "false"
                     - name: PROXY_FIREWALL_MODE
                       value: auto
+                    - name: TLS_CERT_PATH
+                      value: /tls/tls.crt
+                    - name: TLS_KEY_PATH
+                      value: /tls/tls.key
                   image: tailscale/k8s-operator:unstable
                   imagePullPolicy: Always
                   name: operator
@@ -181,6 +185,9 @@ spec:
                     - mountPath: /oauth
                       name: oauth
                       readOnly: true
+                    - mountPath: /tls
+                      name: tls-certs
+                      readOnly: true
             nodeSelector:
                 kubernetes.io/os: linux
             serviceAccountName: operator
@@ -188,3 +195,6 @@ spec:
                 - name: oauth
                   secret:
                     secretName: operator-oauth
+                - name: tls-certs
+                  secret:
+                    secretName: tls-certs

cmd/k8s-operator/deploy/manifests/proxy.yaml

@@ -34,3 +34,11 @@ spec:
             capabilities:
               add:
                 - NET_ADMIN
+          volumeMounts:
+          - name: tls-certs
+            mountPath: /tls
+            readOnly: true
+      volumes:
+      - name: tls-certs
+        secret:
+          secretName: tls-certs

cmd/k8s-operator/deploy/manifests/userspace-proxy.yaml

@@ -22,3 +22,11 @@ spec:
               value: "true"
             - name: TS_AUTH_ONCE
               value: "true"
+          volumeMounts:
+          - name: tls-certs
+            mountPath: /tls
+            readOnly: true
+      volumes:
+      - name: tls-certs
+        secret:
+          secretName: tls-certs

cmd/k8s-operator/ingress.go

@@ -38,6 +38,10 @@ type IngressReconciler struct {
 	// managedIngresses is a set of all ingress resources that we're currently
 	// managing. This is only used for metrics.
 	managedIngresses set.Slice[types.UID]
+
+	// TODO: configure this for each ingress individually instead perhaps
+	tlsCertPath string
+	tlsKeyPath  string
 }
 
 var (
@@ -144,6 +148,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 			},
 		},
 	}
+	if a.tlsCertPath != "" && a.tlsKeyPath != "" {
+		sc.Web[magic443].TLSCertPath = a.tlsCertPath
+		sc.Web[magic443].TLSKeyPath = a.tlsKeyPath
+	}
 	if opt.Bool(ing.Annotations[AnnotationFunnel]).EqualBool(true) {
 		sc.AllowFunnel = map[ipn.HostPort]bool{
 			magic443: true,

cmd/k8s-operator/operator.go

@@ -63,6 +63,8 @@ func main() {
 		tags              = defaultEnv("PROXY_TAGS", "tag:k8s")
 		tsFirewallMode    = defaultEnv("PROXY_FIREWALL_MODE", "")
 		tsEnableConnector = defaultBool("ENABLE_CONNECTOR", false)
+		tlsCertPath       = defaultEnv("TLS_CERT_PATH", "")
+		tlsKeyPath        = defaultEnv("TLS_KEY_PATH", "")
 	)
 
 	var opts []kzap.Opts
@@ -93,7 +95,7 @@ func main() {
 	maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
 	// TODO (irbekrm): gather the reconciler options into an opts struct
 	// rather than passing a million of them in one by one.
-	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tsEnableConnector)
+	runReconcilers(zlog, s, tsNamespace, restConfig, tsClient, image, priorityClassName, tags, tsFirewallMode, tlsCertPath, tlsKeyPath, tsEnableConnector)
 }
 
 // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
@@ -201,7 +203,7 @@ waitOnline:
 
 // runReconcilers starts the controller-runtime manager and registers the
 // ServiceReconciler. It blocks forever.
-func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode string, enableConnector bool) {
+func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string, restConfig *rest.Config, tsClient *tailscale.Client, image, priorityClassName, tags, tsFirewallMode, tlsCertPath, tlsKeyPath string, enableConnector bool) {
 	var (
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 	)
@@ -269,10 +271,12 @@ func runReconcilers(zlog *zap.SugaredLogger, s *tsnet.Server, tsNamespace string
 		Watches(&corev1.Secret{}, ingressChildFilter).
 		Watches(&corev1.Service{}, ingressChildFilter).
 		Complete(&IngressReconciler{
-			ssr:      ssr,
-			recorder: eventRecorder,
-			Client:   mgr.GetClient(),
-			logger:   zlog.Named("ingress-reconciler"),
+			ssr:         ssr,
+			recorder:    eventRecorder,
+			Client:      mgr.GetClient(),
+			logger:      zlog.Named("ingress-reconciler"),
+			tlsCertPath: tlsCertPath,
+			tlsKeyPath:  tlsKeyPath,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create controller: %v", err)

cmd/k8s-operator/proxy.go

@@ -145,6 +145,8 @@ func (h *apiserverProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 //
 // It never returns.
 func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode apiServerProxyMode) {
+	proxyCertPath := os.Getenv("TLS_CERT_PATH")
+	proxyKeyPath := os.Getenv("TLS_KEY_PATH")
 	if mode == apiserverProxyModeDisabled {
 		return
 	}
@@ -202,11 +204,37 @@ func runAPIServerProxy(s *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLo
 			Transport: rt,
 		},
 	}
+	certGetter := lc.GetCertificate
+	if proxyCertPath != "" && proxyKeyPath != "" {
+		log.Infof("Using cert path: %v, key path: %v", proxyCertPath, proxyKeyPath)
+
+		certGetter = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			// TODO: check that the path actually contains a valid cert for the requested SNI
+			keyData, err := os.ReadFile(proxyKeyPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read keyPath: %w", err)
+			}
+
+			certData, err := os.ReadFile(proxyCertPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read certPath: %w", err)
+			}
+
+			cert, err := tls.X509KeyPair(certData, keyData)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse TLS cert and key: %v", err)
+			}
+			return &cert, nil
+		}
+	} else {
+		log.Info("Will be provisioning certs for tailscale")
+	}
+	// GetCertificate func(*ClientHelloInfo) (*Certificate, error)
 	hs := &http.Server{
 		// Kubernetes uses SPDY for exec and port-forward, however SPDY is
 		// incompatible with HTTP/2; so disable HTTP/2 in the proxy.
 		TLSConfig: &tls.Config{
-			GetCertificate: lc.GetCertificate,
+			GetCertificate: certGetter,
 			NextProtos:     []string{"http/1.1"},
 		},
 		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),

ipn/ipn_clone.go

@@ -154,5 +154,7 @@ func (src *WebServerConfig) Clone() *WebServerConfig {
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _WebServerConfigCloneNeedsRegeneration = WebServerConfig(struct {
-	Handlers map[string]*HTTPHandler
+	Handlers    map[string]*HTTPHandler
+	TLSCertPath string
+	TLSKeyPath  string
 }{})

ipn/ipn_view.go

@@ -365,8 +365,12 @@ func (v WebServerConfigView) Handlers() views.MapFn[string, *HTTPHandler, HTTPHa
 		return t.View()
 	})
 }
+func (v WebServerConfigView) TLSCertPath() string { return v.ж.TLSCertPath }
+func (v WebServerConfigView) TLSKeyPath() string  { return v.ж.TLSKeyPath }
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _WebServerConfigViewNeedsRegeneration = WebServerConfig(struct {
-	Handlers map[string]*HTTPHandler
+	Handlers    map[string]*HTTPHandler
+	TLSCertPath string
+	TLSKeyPath  string
 }{})

ipn/ipnlocal/serve.go

@@ -868,10 +868,29 @@ func (b *LocalBackend) getTLSServeCertForPort(port uint16) func(hi *tls.ClientHe
 		if hi == nil || hi.ServerName == "" {
 			return nil, errors.New("no SNI ServerName")
 		}
-		_, ok := b.webServerConfig(hi.ServerName, port)
+		cfg, ok := b.webServerConfig(hi.ServerName, port)
 		if !ok {
 			return nil, errors.New("no webserver configured for name/port")
 		}
+		if cfg.AsStruct().TLSCertPath != "" && cfg.AsStruct().TLSKeyPath != "" {
+			// TODO: check that the cert is actually right for the domain of that webServerConfig
+			// TODO: verify these paths in a reliable way
+			keyData, err := os.ReadFile(cfg.AsStruct().TLSKeyPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read keyPath: %w", err)
+			}
+
+			certData, err := os.ReadFile(cfg.AsStruct().TLSCertPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read certPath: %w", err)
+			}
+
+			cert, err := tls.X509KeyPair(certData, keyData)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse TLS cert and key: %v", err)
+			}
+			return &cert, nil
+		}
 
 		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 		defer cancel()

ipn/serve.go

@@ -94,6 +94,9 @@ type FunnelConn struct {
 // WebServerConfig describes a web server's configuration.
 type WebServerConfig struct {
 	Handlers map[string]*HTTPHandler // mountPoint => handler
+	// TODO: put these two into a single struct with some validation functions
+	TLSCertPath string `json:",omitempty"`  // a filesystem location containing TLS cert
+	TLSKeyPath  string `json:",omitempty:"` // a filesystem location containing TLS key
 }
 
 // TCPPortHandler describes what to do when handling a TCP