Wip

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -22,7 +22,7 @@ rules:
   resources: ["ingressclasses"]
   verbs: ["get", "list", "watch"]
 - apiGroups: ["tailscale.com"]
-  resources: ["connectors", "connectors/status"]
+  resources: ["connectors", "connectors/status", "proxyclasses", "proxyclasses/status"]
   verbs: ["get", "list", "watch", "update"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cmd/k8s-operator/deploy/examples/proxclass.yaml

@@ -0,0 +1,12 @@
+apiVersion: tailscale.com/v1alpha1
+kind: ProxyClass
+metadata:
+  name: prod
+spec:
+  statefulSet:
+    pod:
+     spec:
+       nodeSelector:
+          beta.kubernetes.io/os: "linux"
+       imagePullSecrets:
+       - name: "foo"

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -186,6 +186,8 @@ rules:
       resources:
         - connectors
         - connectors/status
+        - proxyclasses
+        - proxyclasses/status
       verbs:
         - get
         - list

cmd/k8s-operator/sts.go

@@ -22,11 +22,13 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apiserver/pkg/storage/names"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
 	"tailscale.com/client/tailscale"
 	"tailscale.com/ipn"
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/net/netutil"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -52,6 +54,9 @@ const (
 	//MagicDNS name of tailnet node.
 	AnnotationTailnetTargetFQDN = "tailscale.com/tailnet-fqdn"
 
+	// Users can set this on a Service or Ingress. This prototype only looks at Services
+	AnnotationProxyClass = "tailscale.com/proxy-class"
+
 	// Annotations settable by users on ingresses.
 	AnnotationFunnel = "tailscale.com/funnel"
 
@@ -87,6 +92,8 @@ type tailscaleSTSConfig struct {
 	// Connector specifies a configuration of a Connector instance if that's
 	// what this StatefulSet should be created for.
 	Connector *connector
+
+	ProxyClass string
 }
 
 type connector struct {
@@ -397,7 +404,44 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			}
 		}
 	}
-	container := &ss.Spec.Template.Spec.Containers[0]
+	// if proxyclass is set
+	// get the proxy class
+	// get the pod template thing from there
+	// how to merge?
+	if sts.ProxyClass != "" {
+		proxyClass := &tsapi.ProxyClass{}
+		if err := a.Get(ctx, client.ObjectKeyFromObject(proxyClass), proxyClass); err != nil {
+			return nil, fmt.Errorf("failed to get ProxyClass: %w", err)
+		}
+		// only look at pod spec for this prototype
+		if proxyClass.Spec.StatefulSet != nil && proxyClass.Spec.StatefulSet.Pod != nil && proxyClass.Spec.StatefulSet.Pod.Spec != nil {
+			overlay := proxyClass.Spec.StatefulSet.Pod.Spec
+			// hack to use merge functionality from apimachinery -
+			// we do want the array patching behaviour. We can deal
+			// with marshaling/unmarshaling later (maybe just move
+			// to SSA in which case we can apply the patch bytes as
+			// is)
+			overlayBytes, err := json.Marshal(overlay)
+			if err != nil {
+				return nil, fmt.Errorf("error marshaling overlay pod template: %w", err)
+			}
+			origBytes, err := json.Marshal(&ss.Spec.Template.Spec)
+			if err != nil {
+				return nil, fmt.Errorf("error marshaling original pod spec: %w", err)
+			}
+			mergedBytes, err := strategicpatch.CreateTwoWayMergePatch(origBytes, overlayBytes, &corev1.PodSpec{})
+			if err != nil {
+				return nil, fmt.Errorf("error ")
+			}
+			modifiedSpec := &corev1.PodSpec{}
+			if err := json.Unmarshal(mergedBytes, modifiedSpec); err != nil {
+				return nil, fmt.Errorf("error unmarshaling modified pod spec: %w", err)
+			}
+			ss.Spec.Template.Spec = *modifiedSpec
+		}
+
+	}
+	container := &ss.Spec.Template.Spec.Containers[0] // instead get the one that's named 'tailscale'
 	container.Image = a.proxyImage
 	ss.ObjectMeta = metav1.ObjectMeta{
 		Name:      headlessSvc.Name,

cmd/k8s-operator/svc.go

@@ -183,6 +183,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
 		Hostname:            hostname,
 		Tags:                tags,
 		ChildResourceLabels: crl,
+		ProxyClass:          svc.GetAnnotations()[AnnotationProxyClass], // nil?
 	}
 
 	a.mu.Lock()

k8s-operator/apis/v1alpha1/register.go

@@ -49,7 +49,7 @@ func init() {
 
 // Adds the list of known types to api.Scheme.
 func addKnownTypes(scheme *runtime.Scheme) error {
-	scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{})
+	scheme.AddKnownTypes(SchemeGroupVersion, &Connector{}, &ConnectorList{}, &ProxyClass{}, &ProxyClassList{})
 
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

k8s-operator/apis/v1alpha1/types_proxyclass.go

@@ -0,0 +1,75 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var ProxyClassKind = "ProxyClass"
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=pc
+
+type ProxyClass struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ProxyClassSpec `json:"spec"`
+
+	// This would need status if we do any validation in operator. Ideally I
+	// would like to validate with kubebuilder annots/CEL only
+	// +optional
+	// Status ProxyClassStatus `json:"status"`
+}
+
+// +kubebuilder:object:root=true
+
+type ProxyClassList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []ProxyClass `json:"items"`
+}
+
+type ProxyClassSpec struct {
+	// +optional
+	Service `json:"service,omitempty"`
+	// +optional
+	StatefulSet *StatefulSet `json:"statefulSet,omitempty"`
+}
+
+// Configuration for the headless Service, not actually used in this prototype,
+// but is here to better illustrate the API structure
+type Service struct {
+	Labels map[string]string `json:"labels,omitempty"`
+}
+
+type StatefulSet struct {
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// +optional
+	Pod *Pod `json:"pod,omitempty"`
+}
+
+type Pod struct {
+	// Or should we just sync statefulset.labels, statefulset.annotations?
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// I don't want to embed the full PodTemplate as that contains a bunch
+	// of fields that shouldn't be touched (namespace, type meta etc)
+	// Here do a bunch of CEL validations (Host namespace should not be set etc)
+	// merge containers?
+	// merge init containers?
+	// +optional
+	Spec *corev1.PodSpec `json:"spec,omitempty"`
+}

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -8,6 +8,7 @@
 package v1alpha1
 
 import (
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -136,6 +137,119 @@ func (in *ConnectorStatus) DeepCopy() *ConnectorStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Pod) DeepCopyInto(out *Pod) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Spec != nil {
+		in, out := &in.Spec, &out.Spec
+		*out = new(v1.PodSpec)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod.
+func (in *Pod) DeepCopy() *Pod {
+	if in == nil {
+		return nil
+	}
+	out := new(Pod)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClass) DeepCopyInto(out *ProxyClass) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClass.
+func (in *ProxyClass) DeepCopy() *ProxyClass {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClass)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProxyClass) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClassList) DeepCopyInto(out *ProxyClassList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProxyClass, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassList.
+func (in *ProxyClassList) DeepCopy() *ProxyClassList {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClassList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ProxyClassList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) {
+	*out = *in
+	in.Service.DeepCopyInto(&out.Service)
+	if in.StatefulSet != nil {
+		in, out := &in.StatefulSet, &out.StatefulSet
+		*out = new(StatefulSet)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec.
+func (in *ProxyClassSpec) DeepCopy() *ProxyClassSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ProxyClassSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in Routes) DeepCopyInto(out *Routes) {
 	{
@@ -155,6 +269,62 @@ func (in Routes) DeepCopy() Routes {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Service) DeepCopyInto(out *Service) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Service.
+func (in *Service) DeepCopy() *Service {
+	if in == nil {
+		return nil
+	}
+	out := new(Service)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSet) DeepCopyInto(out *StatefulSet) {
+	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Pod != nil {
+		in, out := &in.Pod, &out.Pod
+		*out = new(Pod)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSet.
+func (in *StatefulSet) DeepCopy() *StatefulSet {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSet)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SubnetRouter) DeepCopyInto(out *SubnetRouter) {
 	*out = *in