cmd/k8s-operator/deploy/manifests: update generated static manifest

make kube-generate-all

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/deploy/chart/values.yaml

@@ -9,11 +9,15 @@ oauth: {}
   # clientSecret: ""
 
 # enableConnector determines whether the operator should reconcile
-# connector.tailscale.com custom resources. If set to true you have to install
-# connector CRD in a separate step.
-# You can do so by running 'kubectl apply -f ./cmd/k8s-operator/deploy/crds'.
+# connector.tailscale.com custom resources.
 enableConnector: "false"
 
+# installCRDs determines whether tailscale.com CRDs should be installed as part
+# of chart installation. We do not use Helm's CRD installation mechanism as that
+# does not allow for upgrading CRDs.
+# https://helm.sh/docs/chart_best_practices/custom_resource_definitions/
+installCRDs: "true"
+
 operatorConfig:
   image:
     repo: tailscale/k8s-operator

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -27,6 +27,132 @@ metadata:
     name: proxies
     namespace: tailscale
 ---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+    annotations:
+        controller-gen.kubebuilder.io/version: v0.13.0
+    name: connectors.tailscale.com
+spec:
+    group: tailscale.com
+    names:
+        kind: Connector
+        listKind: ConnectorList
+        plural: connectors
+        shortNames:
+            - cn
+        singular: connector
+    scope: Cluster
+    versions:
+        - additionalPrinterColumns:
+            - description: CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance.
+              jsonPath: .status.subnetRoutes
+              name: SubnetRoutes
+              type: string
+            - description: Whether this Connector instance defines an exit node.
+              jsonPath: .status.isExitNode
+              name: IsExitNode
+              type: string
+            - description: Status of the deployed Connector resources.
+              jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
+              name: Status
+              type: string
+          name: v1alpha1
+          schema:
+            openAPIV3Schema:
+                properties:
+                    apiVersion:
+                        description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+                        type: string
+                    kind:
+                        description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                        type: string
+                    metadata:
+                        type: object
+                    spec:
+                        description: ConnectorSpec describes the desired Tailscale component.
+                        properties:
+                            exitNode:
+                                description: ExitNode defines whether the Connector node should act as a Tailscale exit node. Defaults to false. https://tailscale.com/kb/1103/exit-nodes
+                                type: boolean
+                            hostname:
+                                description: Hostname is the tailnet hostname that should be assigned to the Connector node. If unset, hostname defaults to <connector name>-connector. Hostname can contain lower case letters, numbers and dashes, it must not start or end with a dash and must be between 2 and 63 characters long.
+                                pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
+                                type: string
+                            subnetRouter:
+                                description: SubnetRouter defines subnet routes that the Connector node should expose to tailnet. If unset, none are exposed. https://tailscale.com/kb/1019/subnets/
+                                properties:
+                                    advertiseRoutes:
+                                        description: AdvertiseRoutes refer to CIDRs that the subnet router should make available. Route values must be strings that represent a valid IPv4 or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. https://tailscale.com/kb/1201/4via6-subnets/
+                                        items:
+                                            format: cidr
+                                            type: string
+                                        minItems: 1
+                                        type: array
+                                required:
+                                    - advertiseRoutes
+                                type: object
+                            tags:
+                                description: Tags that the Tailscale node will be tagged with. Defaults to [tag:k8s]. To autoapprove the subnet routes or exit node defined by a Connector, you can configure Tailscale ACLs to give these tags the necessary permissions. See https://tailscale.com/kb/1018/acls/#auto-approvers-for-routes-and-exit-nodes. If you specify custom tags here, you must also make the operator an owner of these tags. See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. Tags cannot be changed once a Connector node has been created. Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+                                items:
+                                    pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
+                                    type: string
+                                type: array
+                        type: object
+                        x-kubernetes-validations:
+                            - message: A Connector needs to be either an exit node or a subnet router, or both.
+                              rule: has(self.subnetRouter) || self.exitNode == true
+                    status:
+                        description: ConnectorStatus describes the status of the Connector. This is set and managed by the Tailscale operator.
+                        properties:
+                            conditions:
+                                description: List of status conditions to indicate the status of the Connector. Known condition types are `ConnectorReady`.
+                                items:
+                                    description: ConnectorCondition contains condition information for a Connector.
+                                    properties:
+                                        lastTransitionTime:
+                                            description: LastTransitionTime is the timestamp corresponding to the last status change of this condition.
+                                            format: date-time
+                                            type: string
+                                        message:
+                                            description: Message is a human readable description of the details of the last transition, complementing reason.
+                                            type: string
+                                        observedGeneration:
+                                            description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Connector.
+                                            format: int64
+                                            type: integer
+                                        reason:
+                                            description: Reason is a brief machine readable explanation for the condition's last transition.
+                                            type: string
+                                        status:
+                                            description: Status of the condition, one of ('True', 'False', 'Unknown').
+                                            type: string
+                                        type:
+                                            description: Type of the condition, known values are (`SubnetRouterReady`).
+                                            type: string
+                                    required:
+                                        - status
+                                        - type
+                                    type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                    - type
+                                x-kubernetes-list-type: map
+                            isExitNode:
+                                description: IsExitNode is set to true if the Connector acts as an exit node.
+                                type: boolean
+                            subnetRoutes:
+                                description: SubnetRoutes are the routes currently exposed to tailnet via this Connector instance.
+                                type: string
+                        type: object
+                required:
+                    - spec
+                type: object
+          served: true
+          storage: true
+          subresources:
+            status: {}
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

cmd/k8s-operator/generate/main.go

@@ -18,15 +18,51 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	operatorDeploymentFilesPath = "cmd/k8s-operator/deploy"
+	crdPath                     = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
+	helmTemplatesPath           = operatorDeploymentFilesPath + "/chart/templates"
+	crdTemplatePath             = helmTemplatesPath + "/connectors.yaml"
+
+	helmConditionalStart = "{{ if and .Values.installCRDs -}}\n"
+	helmConditionalEnd   = "{{- end -}}"
+)
+
 func main() {
+	if len(os.Args) < 2 {
+		log.Fatalf("usage ./generate [staticmanifests|helmcrd]")
+	}
 	repoRoot := "../../"
-	cmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
+	switch os.Args[1] {
+	case "helmcrd":
+		log.Print("Adding Connector CRD to Helm templates")
+		if err := generate(repoRoot); err != nil {
+			log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+		}
+		if err := cleanup(repoRoot); err != nil {
+			log.Fatalf("error cleaning up generated resources")
+		}
+	case "staticmanifests":
+		break
+	default:
+		log.Fatalf("unknown command %s, known commands are 'staticmanifests', 'helmcrd'", os.Args[1])
+	}
+	log.Print("Templating Helm chart contents")
+	if err := generate(repoRoot); err != nil {
+		log.Fatalf("error adding Connector CRD to Helm templates: %v", err)
+	}
+	defer func() {
+		if err := cleanup(repoRoot); err != nil {
+			log.Fatalf("error cleaning up generated resources")
+		}
+	}()
+	helmTmplCmd := exec.Command("./tool/helm", "template", "operator", "./cmd/k8s-operator/deploy/chart",
 		"--namespace=tailscale")
-	cmd.Dir = repoRoot
+	helmTmplCmd.Dir = repoRoot
 	var out bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
+	helmTmplCmd.Stdout = &out
+	helmTmplCmd.Stderr = os.Stderr
+	if err := helmTmplCmd.Run(); err != nil {
 		log.Fatalf("error templating helm manifests: %v", err)
 	}
 
@@ -54,7 +90,6 @@ func main() {
 		if err != nil {
 			log.Fatalf("failed read from input data: %v", err)
 		}
-
 		bytes, err := yaml.Marshal(document)
 		if err != nil {
 			log.Fatalf("failed to marshal YAML document: %v", err)
@@ -72,3 +107,35 @@ func main() {
 		log.Fatalf("error writing new file: %v", err)
 	}
 }
+
+func generate(baseDir string) error {
+	log.Print("Placing Connector CRD into Helm templates..")
+	chartBytes, err := os.ReadFile(filepath.Join(baseDir, crdPath))
+	if err != nil {
+		return fmt.Errorf("error reading CRD contents: %w", err)
+	}
+	// Place a new temporary Helm template file with the templated CRD
+	// contents into Helm templates.
+	file, err := os.Create(filepath.Join(baseDir, crdTemplatePath))
+	if err != nil {
+		return fmt.Errorf("error creating CRD template file: %w", err)
+	}
+	if _, err := file.Write([]byte(helmConditionalStart)); err != nil {
+		return fmt.Errorf("error writing helm if statement start: %w", err)
+	}
+	if _, err := file.Write(chartBytes); err != nil {
+		return fmt.Errorf("error writing chart bytes: %w", err)
+	}
+	if _, err := file.Write([]byte(helmConditionalEnd)); err != nil {
+		return fmt.Errorf("error writing helm if-statement end: %w", err)
+	}
+	return nil
+}
+
+func cleanup(baseDir string) error {
+	log.Print("Cleaning up CRD from Helm templates")
+	if err := os.Remove(filepath.Join(baseDir, crdTemplatePath)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error cleaning up CRD template: %w", err)
+	}
+	return nil
+}

cmd/k8s-operator/generate/main_test.go

@@ -0,0 +1,68 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9 && !windows
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func Test_generate(t *testing.T) {
+	base, err := os.Getwd()
+	base = filepath.Join(base, "../../../")
+	if err != nil {
+		t.Fatalf("error getting current working directory: %v", err)
+	}
+	defer cleanup(base)
+	if err := generate(base); err != nil {
+		t.Fatalf("CRD template generation: %v", err)
+	}
+
+	tempDir := t.TempDir()
+	helmCLIPath := filepath.Join(base, "tool/helm")
+	helmChartTemplatesPath := filepath.Join(base, "cmd/k8s-operator/deploy/chart")
+	helmPackageCmd := exec.Command(helmCLIPath, "package", helmChartTemplatesPath, "--destination", tempDir, "--version", "0.0.1")
+	helmPackageCmd.Stderr = os.Stderr
+	helmPackageCmd.Stdout = os.Stdout
+	if err := helmPackageCmd.Run(); err != nil {
+		t.Fatalf("error packaging Helm chart: %v", err)
+	}
+	helmPackagePath := filepath.Join(tempDir, "tailscale-operator-0.0.1.tgz")
+	helmLintCmd := exec.Command(helmCLIPath, "lint", helmPackagePath)
+	helmLintCmd.Stderr = os.Stderr
+	helmLintCmd.Stdout = os.Stdout
+	if err := helmLintCmd.Run(); err != nil {
+		t.Fatalf("Helm chart linter failed: %v", err)
+	}
+
+	// Test that default Helm install contains the CRD
+	installContentsWithCRD := bytes.NewBuffer([]byte{})
+	helmTemplateWithCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath)
+	helmTemplateWithCRDCmd.Stderr = os.Stderr
+	helmTemplateWithCRDCmd.Stdout = installContentsWithCRD
+	if err := helmTemplateWithCRDCmd.Run(); err != nil {
+		t.Fatalf("templating Helm chart with CRDs failed: %v", err)
+	}
+	if !strings.Contains(installContentsWithCRD.String(), "name: connectors.tailscale.com") {
+		t.Errorf("CRD not found in default chart install")
+	}
+
+	// Test that CRD can be excluded from Helm chart install
+	installContentsWithoutCRD := bytes.NewBuffer([]byte{})
+	helmTemplateWithoutCRDCmd := exec.Command(helmCLIPath, "template", helmPackagePath, "--set", "installCRDs=false")
+	helmTemplateWithoutCRDCmd.Stderr = os.Stderr
+	helmTemplateWithoutCRDCmd.Stdout = installContentsWithoutCRD
+	if err := helmTemplateWithoutCRDCmd.Run(); err != nil {
+		t.Fatalf("templating Helm chart without CRDs failed: %v", err)
+	}
+	if strings.Contains(installContentsWithoutCRD.String(), "name: connectors.tailscale.com") {
+		t.Errorf("CRD found in chart install that should not contain a CRD")
+	}
+}

cmd/k8s-operator/operator.go

@@ -45,7 +45,7 @@ import (
 )
 
 // Generate static manifests for deploying Tailscale operator on Kubernetes from the operator's Helm chart.
-//go:generate go run tailscale.com/cmd/k8s-operator/generate
+//go:generate go run tailscale.com/cmd/k8s-operator/generate staticmanifests
 
 // Generate Connector CustomResourceDefinition yaml from its Go types.
 //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen crd schemapatch:manifests=./deploy/crds output:dir=./deploy/crds paths=../../k8s-operator/apis/...