Compare commits

..

1 Commits

Author SHA1 Message Date
Brad Fitzpatrick
c084e3f6ec net/netcheck: respect DERPRegion.Avoid on initial probe plan too
As found by @jwhited/@raggi.

Updates #8603
Updates #13969
Updates tailscale/corp#24697

Change-Id: I32bb412a06e46a5fc154d87147e75363cf0d5407
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
2024-11-20 10:11:51 -08:00
348 changed files with 4689 additions and 17231 deletions

View File

@@ -49,13 +49,13 @@ jobs:
# Install a more recent Go that understands modern go.mod content.
- name: Install Go
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
with:
go-version-file: go.mod
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1

View File

@@ -25,13 +25,14 @@ jobs:
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
- uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
with:
go-version-file: go.mod
cache: false
- name: golangci-lint
uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
# Note: this is the 'v6.1.0' tag as of 2024-08-21
uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86
with:
version: v1.60

View File

@@ -24,13 +24,13 @@ jobs:
- name: Post to slack
if: failure() && github.event_name == 'schedule'
uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
env:
SLACK_BOT_TOKEN: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
with:
method: chat.postMessage
token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }}
channel-id: 'C05PXRM304B'
payload: |
{
"channel": "C05PXRM304B",
"blocks": [
{
"type": "section",

View File

@@ -6,13 +6,11 @@ on:
- "main"
paths:
- scripts/installer.sh
- .github/workflows/installer.yml
pull_request:
branches:
- "*"
paths:
- scripts/installer.sh
- .github/workflows/installer.yml
jobs:
test:
@@ -31,9 +29,10 @@ jobs:
- "debian:stable-slim"
- "debian:testing-slim"
- "debian:sid-slim"
- "ubuntu:18.04"
- "ubuntu:20.04"
- "ubuntu:22.04"
- "ubuntu:24.04"
- "ubuntu:23.04"
- "elementary/docker:stable"
- "elementary/docker:unstable"
- "parrotsec/core:lts-amd64"
@@ -49,7 +48,7 @@ jobs:
- "opensuse/leap:latest"
- "opensuse/tumbleweed:latest"
- "archlinux:latest"
- "alpine:3.21"
- "alpine:3.14"
- "alpine:latest"
- "alpine:edge"
deps:
@@ -59,6 +58,10 @@ jobs:
# Check a few images with wget rather than curl.
- { image: "debian:oldstable-slim", deps: "wget" }
- { image: "debian:sid-slim", deps: "wget" }
- { image: "ubuntu:23.04", deps: "wget" }
# Ubuntu 16.04 also needs apt-transport-https installed.
- { image: "ubuntu:16.04", deps: "curl apt-transport-https" }
- { image: "ubuntu:16.04", deps: "wget apt-transport-https" }
runs-on: ubuntu-latest
container:
image: ${{ matrix.image }}

View File

@@ -80,7 +80,7 @@ jobs:
- name: checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Restore Cache
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
# Note: unlike the other setups, this is only grabbing the mod download
# cache, rather than the whole mod directory, as the download cache
@@ -153,13 +153,13 @@ jobs:
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Install Go
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
with:
go-version-file: go.mod
cache: false
- name: Restore Cache
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
# Note: unlike the other setups, this is only grabbing the mod download
# cache, rather than the whole mod directory, as the download cache
@@ -260,7 +260,7 @@ jobs:
- name: checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Restore Cache
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
# Note: unlike the other setups, this is only grabbing the mod download
# cache, rather than the whole mod directory, as the download cache
@@ -313,19 +313,13 @@ jobs:
# AIX
- goos: aix
goarch: ppc64
# Solaris
- goos: solaris
goarch: amd64
# illumos
- goos: illumos
goarch: amd64
runs-on: ubuntu-22.04
steps:
- name: checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Restore Cache
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
# Note: unlike the other setups, this is only grabbing the mod download
# cache, rather than the whole mod directory, as the download cache
@@ -373,7 +367,7 @@ jobs:
- name: checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: Restore Cache
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
# Note: unlike the other setups, this is only grabbing the mod download
# cache, rather than the whole mod directory, as the download cache
@@ -467,7 +461,7 @@ jobs:
run: |
echo "artifacts_path=$(realpath .)" >> $GITHUB_ENV
- name: upload crash
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: steps.run.outcome != 'success' && steps.build.outcome == 'success'
with:
name: artifacts
@@ -569,10 +563,8 @@ jobs:
# By having the job always run, but skipping its only step as needed, we
# let the CI output collapse nicely in PRs.
if: failure() && github.event_name == 'push'
uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
with:
webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
webhook-type: incoming-webhook
payload: |
{
"attachments": [{
@@ -584,6 +576,9 @@ jobs:
"color": "danger"
}]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
check_mergeability:
if: always()

View File

@@ -36,7 +36,7 @@ jobs:
private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }}
- name: Send pull request
uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6
uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
with:
token: ${{ steps.generate-token.outputs.token }}
author: Flakes Updater <noreply+flakes-updater@tailscale.com>

View File

@@ -35,7 +35,7 @@ jobs:
- name: Send pull request
id: pull-request
uses: peter-evans/create-pull-request@67ccf781d68cd99b580ae25a5c18a1cc84ffff1f #v7.0.6
uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f #v7.0.5
with:
token: ${{ steps.generate-token.outputs.token }}
author: OSS Updater <noreply+oss-updater@tailscale.com>

View File

@@ -100,7 +100,7 @@ publishdevoperator: ## Build and publish k8s-operator image to location specifie
@test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1)
@test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1)
@test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1)
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh
TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=operator ./build_docker.sh
publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO}
@test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1)
@@ -116,6 +116,7 @@ sshintegrationtest: ## Run the SSH integration tests in various Docker container
GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \
echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \
echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \
echo "Testing on ubuntu:mantic" && docker build --build-arg="BASE=ubuntu:mantic" -t ssh-ubuntu-mantic ssh/tailssh/testcontainers && \
echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \
echo "Testing on alpine:latest" && docker build --build-arg="BASE=alpine:latest" -t ssh-alpine-latest ssh/tailssh/testcontainers

View File

@@ -72,7 +72,7 @@ Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin)
`Signed-off-by` lines in commits.
See `git log` for our commit message style. It's basically the same as
[Go's style](https://go.dev/wiki/CommitMessage).
[Go's style](https://github.com/golang/go/wiki/CommitMessage).
## About Us

View File

@@ -1 +1 @@
1.79.0
1.77.0

View File

@@ -18,6 +18,7 @@ import (
"sync"
"time"
xmaps "golang.org/x/exp/maps"
"golang.org/x/net/dns/dnsmessage"
"tailscale.com/types/logger"
"tailscale.com/types/views"
@@ -290,11 +291,11 @@ func (e *AppConnector) updateDomains(domains []string) {
}
}
if err := e.routeAdvertiser.UnadvertiseRoute(toRemove...); err != nil {
e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", slicesx.MapKeys(oldDomains), toRemove, err)
e.logf("failed to unadvertise routes on domain removal: %v: %v: %v", xmaps.Keys(oldDomains), toRemove, err)
}
}
e.logf("handling domains: %v and wildcards: %v", slicesx.MapKeys(e.domains), e.wildcards)
e.logf("handling domains: %v and wildcards: %v", xmaps.Keys(e.domains), e.wildcards)
}
// updateRoutes merges the supplied routes into the currently configured routes. The routes supplied
@@ -353,7 +354,7 @@ func (e *AppConnector) Domains() views.Slice[string] {
e.mu.Lock()
defer e.mu.Unlock()
return views.SliceOf(slicesx.MapKeys(e.domains))
return views.SliceOf(xmaps.Keys(e.domains))
}
// DomainRoutes returns a map of domains to resolved IP
@@ -374,13 +375,13 @@ func (e *AppConnector) DomainRoutes() map[string][]netip.Addr {
// response is being returned over the PeerAPI. The response is parsed and
// matched against the configured domains, if matched the routeAdvertiser is
// advised to advertise the discovered route.
func (e *AppConnector) ObserveDNSResponse(res []byte) error {
func (e *AppConnector) ObserveDNSResponse(res []byte) {
var p dnsmessage.Parser
if _, err := p.Start(res); err != nil {
return err
return
}
if err := p.SkipAllQuestions(); err != nil {
return err
return
}
// cnameChain tracks a chain of CNAMEs for a given query in order to reverse
@@ -399,12 +400,12 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) error {
break
}
if err != nil {
return err
return
}
if h.Class != dnsmessage.ClassINET {
if err := p.SkipAnswer(); err != nil {
return err
return
}
continue
}
@@ -413,7 +414,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) error {
case dnsmessage.TypeCNAME, dnsmessage.TypeA, dnsmessage.TypeAAAA:
default:
if err := p.SkipAnswer(); err != nil {
return err
return
}
continue
@@ -427,7 +428,7 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) error {
if h.Type == dnsmessage.TypeCNAME {
res, err := p.CNAMEResource()
if err != nil {
return err
return
}
cname := strings.TrimSuffix(strings.ToLower(res.CNAME.String()), ".")
if len(cname) == 0 {
@@ -441,20 +442,20 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) error {
case dnsmessage.TypeA:
r, err := p.AResource()
if err != nil {
return err
return
}
addr := netip.AddrFrom4(r.A)
mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
case dnsmessage.TypeAAAA:
r, err := p.AAAAResource()
if err != nil {
return err
return
}
addr := netip.AddrFrom16(r.AAAA)
mak.Set(&addressRecords, domain, append(addressRecords[domain], addr))
default:
if err := p.SkipAnswer(); err != nil {
return err
return
}
continue
}
@@ -485,7 +486,6 @@ func (e *AppConnector) ObserveDNSResponse(res []byte) error {
e.scheduleAdvertisement(domain, toAdvertise...)
}
}
return nil
}
// starting from the given domain that resolved to an address, find it, or any

View File

@@ -11,13 +11,13 @@ import (
"testing"
"time"
xmaps "golang.org/x/exp/maps"
"golang.org/x/net/dns/dnsmessage"
"tailscale.com/appc/appctest"
"tailscale.com/tstest"
"tailscale.com/util/clientmetric"
"tailscale.com/util/mak"
"tailscale.com/util/must"
"tailscale.com/util/slicesx"
)
func fakeStoreRoutes(*RouteInfo) error { return nil }
@@ -50,7 +50,7 @@ func TestUpdateDomains(t *testing.T) {
// domains are explicitly downcased on set.
a.UpdateDomains([]string{"UP.EXAMPLE.COM"})
a.Wait(ctx)
if got, want := slicesx.MapKeys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
if got, want := xmaps.Keys(a.domains), []string{"up.example.com"}; !slices.Equal(got, want) {
t.Errorf("got %v; want %v", got, want)
}
}
@@ -69,9 +69,7 @@ func TestUpdateRoutes(t *testing.T) {
a.updateDomains([]string{"*.example.com"})
// This route should be collapsed into the range
if err := a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("a.example.com.", "192.0.2.1"))
a.Wait(ctx)
if !slices.Equal(rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.2.1/32")}) {
@@ -79,9 +77,7 @@ func TestUpdateRoutes(t *testing.T) {
}
// This route should not be collapsed or removed
if err := a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("b.example.com.", "192.0.0.1"))
a.Wait(ctx)
routes := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.0.0.1/32")}
@@ -134,9 +130,7 @@ func TestDomainRoutes(t *testing.T) {
a = NewAppConnector(t.Logf, rc, nil, nil)
}
a.updateDomains([]string{"example.com"})
if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
a.Wait(context.Background())
want := map[string][]netip.Addr{
@@ -161,9 +155,7 @@ func TestObserveDNSResponse(t *testing.T) {
}
// a has no domains configured, so it should not advertise any routes
if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
if got, want := rc.Routes(), ([]netip.Prefix)(nil); !slices.Equal(got, want) {
t.Errorf("got %v; want %v", got, want)
}
@@ -171,9 +163,7 @@ func TestObserveDNSResponse(t *testing.T) {
wantRoutes := []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}
a.updateDomains([]string{"example.com"})
if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.0.8"))
a.Wait(ctx)
if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
t.Errorf("got %v; want %v", got, want)
@@ -182,9 +172,7 @@ func TestObserveDNSResponse(t *testing.T) {
// a CNAME record chain should result in a route being added if the chain
// matches a routed domain.
a.updateDomains([]string{"www.example.com", "example.com"})
if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com.")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.9", "www.example.com.", "chain.example.com.", "example.com."))
a.Wait(ctx)
wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.9/32"))
if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -193,9 +181,7 @@ func TestObserveDNSResponse(t *testing.T) {
// a CNAME record chain should result in a route being added if the chain
// even if only found in the middle of the chain
if err := a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org.")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsCNAMEResponse("192.0.0.10", "outside.example.org.", "www.example.com.", "example.org."))
a.Wait(ctx)
wantRoutes = append(wantRoutes, netip.MustParsePrefix("192.0.0.10/32"))
if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
@@ -204,18 +190,14 @@ func TestObserveDNSResponse(t *testing.T) {
wantRoutes = append(wantRoutes, netip.MustParsePrefix("2001:db8::1/128"))
if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
a.Wait(ctx)
if got, want := rc.Routes(), wantRoutes; !slices.Equal(got, want) {
t.Errorf("got %v; want %v", got, want)
}
// don't re-advertise routes that have already been advertised
if err := a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("example.com.", "2001:db8::1"))
a.Wait(ctx)
if !slices.Equal(rc.Routes(), wantRoutes) {
t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -225,9 +207,7 @@ func TestObserveDNSResponse(t *testing.T) {
pfx := netip.MustParsePrefix("192.0.2.0/24")
a.updateRoutes([]netip.Prefix{pfx})
wantRoutes = append(wantRoutes, pfx)
if err := a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("example.com.", "192.0.2.1"))
a.Wait(ctx)
if !slices.Equal(rc.Routes(), wantRoutes) {
t.Errorf("rc.Routes(): got %v; want %v", rc.Routes(), wantRoutes)
@@ -250,9 +230,7 @@ func TestWildcardDomains(t *testing.T) {
}
a.updateDomains([]string{"*.example.com"})
if err := a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8")); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
a.ObserveDNSResponse(dnsResponse("foo.example.com.", "192.0.0.8"))
a.Wait(ctx)
if got, want := rc.Routes(), []netip.Prefix{netip.MustParsePrefix("192.0.0.8/32")}; !slices.Equal(got, want) {
t.Errorf("routes: got %v; want %v", got, want)
@@ -460,16 +438,10 @@ func TestUpdateDomainRouteRemoval(t *testing.T) {
// adding domains doesn't immediately cause any routes to be advertised
assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
for _, res := range [][]byte{
dnsResponse("a.example.com.", "1.2.3.1"),
dnsResponse("a.example.com.", "1.2.3.2"),
dnsResponse("b.example.com.", "1.2.3.3"),
dnsResponse("b.example.com.", "1.2.3.4"),
} {
if err := a.ObserveDNSResponse(res); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
}
a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.3"))
a.ObserveDNSResponse(dnsResponse("b.example.com.", "1.2.3.4"))
a.Wait(ctx)
// observing dns responses causes routes to be advertised
assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})
@@ -515,16 +487,10 @@ func TestUpdateWildcardRouteRemoval(t *testing.T) {
// adding domains doesn't immediately cause any routes to be advertised
assertRoutes("update domains", []netip.Prefix{}, []netip.Prefix{})
for _, res := range [][]byte{
dnsResponse("a.example.com.", "1.2.3.1"),
dnsResponse("a.example.com.", "1.2.3.2"),
dnsResponse("1.b.example.com.", "1.2.3.3"),
dnsResponse("2.b.example.com.", "1.2.3.4"),
} {
if err := a.ObserveDNSResponse(res); err != nil {
t.Errorf("ObserveDNSResponse: %v", err)
}
}
a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.1"))
a.ObserveDNSResponse(dnsResponse("a.example.com.", "1.2.3.2"))
a.ObserveDNSResponse(dnsResponse("1.b.example.com.", "1.2.3.3"))
a.ObserveDNSResponse(dnsResponse("2.b.example.com.", "1.2.3.4"))
a.Wait(ctx)
// observing dns responses causes routes to be advertised
assertRoutes("observed dns", prefixes("1.2.3.1/32", "1.2.3.2/32", "1.2.3.3/32", "1.2.3.4/32"), []netip.Prefix{})

View File

@@ -15,9 +15,8 @@ import (
)
// WriteFile writes data to filename+some suffix, then renames it into filename.
// The perm argument is ignored on Windows, but if the target filename already
// exists then the target file's attributes and ACLs are preserved. If the target
// filename already exists but is not a regular file, WriteFile returns an error.
// The perm argument is ignored on Windows. If the target filename already
// exists but is not a regular file, WriteFile returns an error.
func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
fi, err := os.Stat(filename)
if err == nil && !fi.Mode().IsRegular() {
@@ -48,5 +47,5 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) {
if err := f.Close(); err != nil {
return err
}
return rename(tmpName, filename)
return os.Rename(tmpName, filename)
}

View File

@@ -1,14 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !windows
package atomicfile
import (
"os"
)
func rename(srcFile, destFile string) error {
return os.Rename(srcFile, destFile)
}

View File

@@ -1,33 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package atomicfile
import (
"os"
"golang.org/x/sys/windows"
)
func rename(srcFile, destFile string) error {
// Use replaceFile when possible to preserve the original file's attributes and ACLs.
if err := replaceFile(destFile, srcFile); err == nil || err != windows.ERROR_FILE_NOT_FOUND {
return err
}
// destFile doesn't exist. Just do a normal rename.
return os.Rename(srcFile, destFile)
}
func replaceFile(destFile, srcFile string) error {
destFile16, err := windows.UTF16PtrFromString(destFile)
if err != nil {
return err
}
srcFile16, err := windows.UTF16PtrFromString(srcFile)
if err != nil {
return err
}
return replaceFileW(destFile16, srcFile16, nil, 0, nil, nil)
}

View File

@@ -1,146 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package atomicfile
import (
"os"
"testing"
"unsafe"
"golang.org/x/sys/windows"
)
var _SECURITY_RESOURCE_MANAGER_AUTHORITY = windows.SidIdentifierAuthority{[6]byte{0, 0, 0, 0, 0, 9}}
// makeRandomSID generates a SID derived from a v4 GUID.
// This is basically the same algorithm used by browser sandboxes for generating
// random SIDs.
func makeRandomSID() (*windows.SID, error) {
guid, err := windows.GenerateGUID()
if err != nil {
return nil, err
}
rids := *((*[4]uint32)(unsafe.Pointer(&guid)))
var pSID *windows.SID
if err := windows.AllocateAndInitializeSid(&_SECURITY_RESOURCE_MANAGER_AUTHORITY, 4, rids[0], rids[1], rids[2], rids[3], 0, 0, 0, 0, &pSID); err != nil {
return nil, err
}
defer windows.FreeSid(pSID)
// Make a copy that lives on the Go heap
return pSID.Copy()
}
func getExistingFileSD(name string) (*windows.SECURITY_DESCRIPTOR, error) {
const infoFlags = windows.DACL_SECURITY_INFORMATION
return windows.GetNamedSecurityInfo(name, windows.SE_FILE_OBJECT, infoFlags)
}
func getExistingFileDACL(name string) (*windows.ACL, error) {
sd, err := getExistingFileSD(name)
if err != nil {
return nil, err
}
dacl, _, err := sd.DACL()
return dacl, err
}
func addDenyACEForRandomSID(dacl *windows.ACL) (*windows.ACL, error) {
randomSID, err := makeRandomSID()
if err != nil {
return nil, err
}
randomSIDTrustee := windows.TRUSTEE{nil, windows.NO_MULTIPLE_TRUSTEE,
windows.TRUSTEE_IS_SID, windows.TRUSTEE_IS_UNKNOWN,
windows.TrusteeValueFromSID(randomSID)}
entries := []windows.EXPLICIT_ACCESS{
{
windows.GENERIC_ALL,
windows.DENY_ACCESS,
windows.NO_INHERITANCE,
randomSIDTrustee,
},
}
return windows.ACLFromEntries(entries, dacl)
}
func setExistingFileDACL(name string, dacl *windows.ACL) error {
return windows.SetNamedSecurityInfo(name, windows.SE_FILE_OBJECT,
windows.DACL_SECURITY_INFORMATION, nil, nil, dacl, nil)
}
// makeOrigFileWithCustomDACL creates a new, temporary file with a custom
// DACL that we can check for later. It returns the name of the temporary
// file and the security descriptor for the file in SDDL format.
func makeOrigFileWithCustomDACL() (name, sddl string, err error) {
f, err := os.CreateTemp("", "foo*.tmp")
if err != nil {
return "", "", err
}
name = f.Name()
if err := f.Close(); err != nil {
return "", "", err
}
f = nil
defer func() {
if err != nil {
os.Remove(name)
}
}()
dacl, err := getExistingFileDACL(name)
if err != nil {
return "", "", err
}
// Add a harmless, deny-only ACE for a random SID that isn't used for anything
// (but that we can check for later).
dacl, err = addDenyACEForRandomSID(dacl)
if err != nil {
return "", "", err
}
if err := setExistingFileDACL(name, dacl); err != nil {
return "", "", err
}
sd, err := getExistingFileSD(name)
if err != nil {
return "", "", err
}
return name, sd.String(), nil
}
func TestPreserveSecurityInfo(t *testing.T) {
// Make a test file with a custom ACL.
origFileName, want, err := makeOrigFileWithCustomDACL()
if err != nil {
t.Fatalf("makeOrigFileWithCustomDACL returned %v", err)
}
t.Cleanup(func() {
os.Remove(origFileName)
})
if err := WriteFile(origFileName, []byte{}, 0); err != nil {
t.Fatalf("WriteFile returned %v", err)
}
// We expect origFileName's security descriptor to be unchanged despite
// the WriteFile call.
sd, err := getExistingFileSD(origFileName)
if err != nil {
t.Fatalf("getExistingFileSD(%q) returned %v", origFileName, err)
}
if got := sd.String(); got != want {
t.Errorf("security descriptor comparison failed: got %q, want %q", got, want)
}
}

View File

@@ -1,8 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package atomicfile
//go:generate go run golang.org/x/sys/windows/mkwinsyscall -output zsyscall_windows.go mksyscall.go
//sys replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) [int32(failretval)==0] = kernel32.ReplaceFileW

View File

@@ -1,52 +0,0 @@
// Code generated by 'go generate'; DO NOT EDIT.
package atomicfile
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
// TODO: add more here, after collecting data on the common
// error values see on Windows. (perhaps when running
// all.bat?)
return e
}
var (
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
procReplaceFileW = modkernel32.NewProc("ReplaceFileW")
)
func replaceFileW(replaced *uint16, replacement *uint16, backup *uint16, flags uint32, exclude unsafe.Pointer, reserved unsafe.Pointer) (err error) {
r1, _, e1 := syscall.Syscall6(procReplaceFileW.Addr(), 6, uintptr(unsafe.Pointer(replaced)), uintptr(unsafe.Pointer(replacement)), uintptr(unsafe.Pointer(backup)), uintptr(flags), uintptr(exclude), uintptr(reserved))
if int32(r1) == 0 {
err = errnoErr(e1)
}
return
}

View File

@@ -37,7 +37,7 @@ while [ "$#" -gt 1 ]; do
--extra-small)
shift
ldflags="$ldflags -w -s"
tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion,ts_omit_ssh,ts_omit_wakeonlan"
tags="${tags:+$tags,}ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube,ts_omit_completion"
;;
--box)
shift

View File

@@ -54,7 +54,7 @@ case "$TARGET" in
--annotations="${ANNOTATIONS}" \
/usr/local/bin/containerboot
;;
k8s-operator)
operator)
DEFAULT_REPOS="tailscale/k8s-operator"
REPOS="${REPOS:-${DEFAULT_REPOS}}"
go run github.com/tailscale/mkctr \

View File

@@ -1,319 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build cgo || !darwin
package systray
import (
"bytes"
"context"
"image"
"image/color"
"image/png"
"sync"
"time"
"fyne.io/systray"
"github.com/fogleman/gg"
)
// tsLogo represents the Tailscale logo displayed as the systray icon.
type tsLogo struct {
// dots represents the state of the 3x3 dot grid in the logo.
// A 0 represents a gray dot, any other value is a white dot.
dots [9]byte
// dotMask returns an image mask to be used when rendering the logo dots.
dotMask func(dc *gg.Context, borderUnits int, radius int) *image.Alpha
// overlay is called after the dots are rendered to draw an additional overlay.
overlay func(dc *gg.Context, borderUnits int, radius int)
}
var (
// disconnected is all gray dots
disconnected = tsLogo{dots: [9]byte{
0, 0, 0,
0, 0, 0,
0, 0, 0,
}}
// connected is the normal Tailscale logo
connected = tsLogo{dots: [9]byte{
0, 0, 0,
1, 1, 1,
0, 1, 0,
}}
// loading is a special tsLogo value that is not meant to be rendered directly,
// but indicates that the loading animation should be shown.
loading = tsLogo{dots: [9]byte{'l', 'o', 'a', 'd', 'i', 'n', 'g'}}
// loadingIcons are shown in sequence as an animated loading icon.
loadingLogos = []tsLogo{
{dots: [9]byte{
0, 1, 1,
1, 0, 1,
0, 0, 1,
}},
{dots: [9]byte{
0, 1, 1,
0, 0, 1,
0, 1, 0,
}},
{dots: [9]byte{
0, 1, 1,
0, 0, 0,
0, 0, 1,
}},
{dots: [9]byte{
0, 0, 1,
0, 1, 0,
0, 0, 0,
}},
{dots: [9]byte{
0, 1, 0,
0, 0, 0,
0, 0, 0,
}},
{dots: [9]byte{
0, 0, 0,
0, 0, 1,
0, 0, 0,
}},
{dots: [9]byte{
0, 0, 0,
0, 0, 0,
0, 0, 0,
}},
{dots: [9]byte{
0, 0, 1,
0, 0, 0,
0, 0, 0,
}},
{dots: [9]byte{
0, 0, 0,
0, 0, 0,
1, 0, 0,
}},
{dots: [9]byte{
0, 0, 0,
0, 0, 0,
1, 1, 0,
}},
{dots: [9]byte{
0, 0, 0,
1, 0, 0,
1, 1, 0,
}},
{dots: [9]byte{
0, 0, 0,
1, 1, 0,
0, 1, 0,
}},
{dots: [9]byte{
0, 0, 0,
1, 1, 0,
0, 1, 1,
}},
{dots: [9]byte{
0, 0, 0,
1, 1, 1,
0, 0, 1,
}},
{dots: [9]byte{
0, 1, 0,
0, 1, 1,
1, 0, 1,
}},
}
// exitNodeOnline is the Tailscale logo with an additional arrow overlay in the corner.
exitNodeOnline = tsLogo{
dots: [9]byte{
0, 0, 0,
1, 1, 1,
0, 1, 0,
},
// draw an arrow mask in the bottom right corner with a reasonably thick line width.
dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
bu, r := float64(borderUnits), float64(radius)
x1 := r * (bu + 3.5)
y := r * (bu + 7)
x2 := x1 + (r * 5)
mc := gg.NewContext(dc.Width(), dc.Height())
mc.DrawLine(x1, y, x2, y) // arrow center line
mc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
mc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
mc.SetLineWidth(r * 3)
mc.Stroke()
return mc.AsMask()
},
// draw an arrow in the bottom right corner over the masked area.
overlay: func(dc *gg.Context, borderUnits int, radius int) {
bu, r := float64(borderUnits), float64(radius)
x1 := r * (bu + 3.5)
y := r * (bu + 7)
x2 := x1 + (r * 5)
dc.DrawLine(x1, y, x2, y) // arrow center line
dc.DrawLine(x2-(1.5*r), y-(1.5*r), x2, y) // top of arrow tip
dc.DrawLine(x2-(1.5*r), y+(1.5*r), x2, y) // bottom of arrow tip
dc.SetColor(fg)
dc.SetLineWidth(r)
dc.Stroke()
},
}
// exitNodeOffline is the Tailscale logo with a red "x" in the corner.
exitNodeOffline = tsLogo{
dots: [9]byte{
0, 0, 0,
1, 1, 1,
0, 1, 0,
},
// Draw a square that hides the four dots in the bottom right corner,
dotMask: func(dc *gg.Context, borderUnits int, radius int) *image.Alpha {
bu, r := float64(borderUnits), float64(radius)
x := r * (bu + 3)
mc := gg.NewContext(dc.Width(), dc.Height())
mc.DrawRectangle(x, x, r*6, r*6)
mc.Fill()
return mc.AsMask()
},
// draw a red "x" over the bottom right corner.
overlay: func(dc *gg.Context, borderUnits int, radius int) {
bu, r := float64(borderUnits), float64(radius)
x1 := r * (bu + 4)
x2 := x1 + (r * 3.5)
dc.DrawLine(x1, x1, x2, x2) // top-left to bottom-right stroke
dc.DrawLine(x1, x2, x2, x1) // bottom-left to top-right stroke
dc.SetColor(red)
dc.SetLineWidth(r)
dc.Stroke()
},
}
)
var (
bg = color.NRGBA{0, 0, 0, 255}
fg = color.NRGBA{255, 255, 255, 255}
gray = color.NRGBA{255, 255, 255, 102}
red = color.NRGBA{229, 111, 74, 255}
)
// render returns a PNG image of the logo.
func (logo tsLogo) render() *bytes.Buffer {
const borderUnits = 1
return logo.renderWithBorder(borderUnits)
}
// renderWithBorder returns a PNG image of the logo with the specified border width.
// One border unit is equal to the radius of a tailscale logo dot.
func (logo tsLogo) renderWithBorder(borderUnits int) *bytes.Buffer {
const radius = 25
dim := radius * (8 + borderUnits*2)
dc := gg.NewContext(dim, dim)
dc.DrawRectangle(0, 0, float64(dim), float64(dim))
dc.SetColor(bg)
dc.Fill()
if logo.dotMask != nil {
mask := logo.dotMask(dc, borderUnits, radius)
dc.SetMask(mask)
dc.InvertMask()
}
for y := 0; y < 3; y++ {
for x := 0; x < 3; x++ {
px := (borderUnits + 1 + 3*x) * radius
py := (borderUnits + 1 + 3*y) * radius
col := fg
if logo.dots[y*3+x] == 0 {
col = gray
}
dc.DrawCircle(float64(px), float64(py), radius)
dc.SetColor(col)
dc.Fill()
}
}
if logo.overlay != nil {
dc.ResetClip()
logo.overlay(dc, borderUnits, radius)
}
b := bytes.NewBuffer(nil)
png.Encode(b, dc.Image())
return b
}
// setAppIcon renders logo and sets it as the systray icon.
func setAppIcon(icon tsLogo) {
if icon.dots == loading.dots {
startLoadingAnimation()
} else {
stopLoadingAnimation()
systray.SetIcon(icon.render().Bytes())
}
}
var (
loadingMu sync.Mutex // protects loadingCancel
// loadingCancel stops the loading animation in the systray icon.
// This is nil if the animation is not currently active.
loadingCancel func()
)
// startLoadingAnimation starts the animated loading icon in the system tray.
// The animation continues until [stopLoadingAnimation] is called.
// If the loading animation is already active, this func does nothing.
func startLoadingAnimation() {
loadingMu.Lock()
defer loadingMu.Unlock()
if loadingCancel != nil {
// loading icon already displayed
return
}
ctx := context.Background()
ctx, loadingCancel = context.WithCancel(ctx)
go func() {
t := time.NewTicker(500 * time.Millisecond)
var i int
for {
select {
case <-ctx.Done():
return
case <-t.C:
systray.SetIcon(loadingLogos[i].render().Bytes())
i++
if i >= len(loadingLogos) {
i = 0
}
}
}
}()
}
// stopLoadingAnimation stops the animated loading icon in the system tray.
// If the loading animation is not currently active, this func does nothing.
func stopLoadingAnimation() {
loadingMu.Lock()
defer loadingMu.Unlock()
if loadingCancel != nil {
loadingCancel()
loadingCancel = nil
}
}

View File

@@ -1,712 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build cgo || !darwin
// Package systray provides a minimal Tailscale systray application.
package systray
import (
"context"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
"runtime"
"slices"
"strings"
"sync"
"syscall"
"time"
"fyne.io/systray"
"github.com/atotto/clipboard"
dbus "github.com/godbus/dbus/v5"
"github.com/toqueteos/webbrowser"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
"tailscale.com/tailcfg"
"tailscale.com/util/slicesx"
"tailscale.com/util/stringsx"
)
var (
// newMenuDelay is the amount of time to sleep after creating a new menu,
// but before adding items to it. This works around a bug in some dbus implementations.
newMenuDelay time.Duration
// if true, treat all mullvad exit node countries as single-city.
// Instead of rendering a submenu with cities, just select the highest-priority peer.
hideMullvadCities bool
)
// Run starts the systray menu and blocks until the menu exits.
func (menu *Menu) Run() {
menu.updateState()
// exit cleanly on SIGINT and SIGTERM
go func() {
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
select {
case <-interrupt:
menu.onExit()
case <-menu.bgCtx.Done():
}
}()
go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1)
systray.Run(menu.onReady, menu.onExit)
}
// Menu represents the systray menu, its items, and the current Tailscale state.
type Menu struct {
mu sync.Mutex // protects the entire Menu
lc tailscale.LocalClient
status *ipnstate.Status
curProfile ipn.LoginProfile
allProfiles []ipn.LoginProfile
bgCtx context.Context // ctx for background tasks not involving menu item clicks
bgCancel context.CancelFunc
// Top-level menu items
connect *systray.MenuItem
disconnect *systray.MenuItem
self *systray.MenuItem
exitNodes *systray.MenuItem
more *systray.MenuItem
quit *systray.MenuItem
rebuildCh chan struct{} // triggers a menu rebuild
accountsCh chan ipn.ProfileID
exitNodeCh chan tailcfg.StableNodeID // ID of selected exit node
eventCancel context.CancelFunc // cancel eventLoop
notificationIcon *os.File // icon used for desktop notifications
}
func (menu *Menu) init() {
if menu.bgCtx != nil {
// already initialized
return
}
menu.rebuildCh = make(chan struct{}, 1)
menu.accountsCh = make(chan ipn.ProfileID)
menu.exitNodeCh = make(chan tailcfg.StableNodeID)
// dbus wants a file path for notification icons, so copy to a temp file.
menu.notificationIcon, _ = os.CreateTemp("", "tailscale-systray.png")
io.Copy(menu.notificationIcon, connected.renderWithBorder(3))
menu.bgCtx, menu.bgCancel = context.WithCancel(context.Background())
go menu.watchIPNBus()
}
func init() {
if runtime.GOOS != "linux" {
// so far, these tweaks are only needed on Linux
return
}
desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP"))
switch desktop {
case "gnome":
// GNOME expands submenus downward in the main menu, rather than flyouts to the side.
// Either as a result of that or another limitation, there seems to be a maximum depth of submenus.
// Mullvad countries that have a city submenu are not being rendered, and so can't be selected.
// Handle this by simply treating all mullvad countries as single-city and select the best peer.
hideMullvadCities = true
case "kde":
// KDE doesn't need a delay, and actually won't render submenus
// if we delay for more than about 400µs.
newMenuDelay = 0
default:
// Add a slight delay to ensure the menu is created before adding items.
//
// Systray implementations that use libdbusmenu sometimes process messages out of order,
// resulting in errors such as:
// (waybar:153009): LIBDBUSMENU-GTK-WARNING **: 18:07:11.551: Children but no menu, someone's been naughty with their 'children-display' property: 'submenu'
//
// See also: https://github.com/fyne-io/systray/issues/12
newMenuDelay = 10 * time.Millisecond
}
}
// onReady is called by the systray package when the menu is ready to be built.
func (menu *Menu) onReady() {
log.Printf("starting")
setAppIcon(disconnected)
menu.rebuild()
}
// updateState updates the Menu state from the Tailscale local client.
func (menu *Menu) updateState() {
menu.mu.Lock()
defer menu.mu.Unlock()
menu.init()
var err error
menu.status, err = menu.lc.Status(menu.bgCtx)
if err != nil {
log.Print(err)
}
menu.curProfile, menu.allProfiles, err = menu.lc.ProfileStatus(menu.bgCtx)
if err != nil {
log.Print(err)
}
}
// rebuild the systray menu based on the current Tailscale state.
//
// We currently rebuild the entire menu because it is not easy to update the existing menu.
// You cannot iterate over the items in a menu, nor can you remove some items like separators.
// So for now we rebuild the whole thing, and can optimize this later if needed.
func (menu *Menu) rebuild() {
menu.mu.Lock()
defer menu.mu.Unlock()
menu.init()
if menu.eventCancel != nil {
menu.eventCancel()
}
ctx := context.Background()
ctx, menu.eventCancel = context.WithCancel(ctx)
systray.ResetMenu()
menu.connect = systray.AddMenuItem("Connect", "")
menu.disconnect = systray.AddMenuItem("Disconnect", "")
menu.disconnect.Hide()
systray.AddSeparator()
// delay to prevent race setting icon on first start
time.Sleep(newMenuDelay)
// Set systray menu icon and title.
// Also adjust connect/disconnect menu items if needed.
var backendState string
if menu.status != nil {
backendState = menu.status.BackendState
}
switch backendState {
case ipn.Running.String():
if menu.status.ExitNodeStatus != nil && !menu.status.ExitNodeStatus.ID.IsZero() {
if menu.status.ExitNodeStatus.Online {
setTooltip("Using exit node")
setAppIcon(exitNodeOnline)
} else {
setTooltip("Exit node offline")
setAppIcon(exitNodeOffline)
}
} else {
setTooltip(fmt.Sprintf("Connected to %s", menu.status.CurrentTailnet.Name))
setAppIcon(connected)
}
menu.connect.SetTitle("Connected")
menu.connect.Disable()
menu.disconnect.Show()
menu.disconnect.Enable()
case ipn.Starting.String():
setTooltip("Connecting")
setAppIcon(loading)
default:
setTooltip("Disconnected")
setAppIcon(disconnected)
}
account := "Account"
if pt := profileTitle(menu.curProfile); pt != "" {
account = pt
}
accounts := systray.AddMenuItem(account, "")
setRemoteIcon(accounts, menu.curProfile.UserProfile.ProfilePicURL)
time.Sleep(newMenuDelay)
for _, profile := range menu.allProfiles {
title := profileTitle(profile)
var item *systray.MenuItem
if profile.ID == menu.curProfile.ID {
item = accounts.AddSubMenuItemCheckbox(title, "", true)
} else {
item = accounts.AddSubMenuItem(title, "")
}
setRemoteIcon(item, profile.UserProfile.ProfilePicURL)
onClick(ctx, item, func(ctx context.Context) {
select {
case <-ctx.Done():
case menu.accountsCh <- profile.ID:
}
})
}
if menu.status != nil && menu.status.Self != nil && len(menu.status.Self.TailscaleIPs) > 0 {
title := fmt.Sprintf("This Device: %s (%s)", menu.status.Self.HostName, menu.status.Self.TailscaleIPs[0])
menu.self = systray.AddMenuItem(title, "")
} else {
menu.self = systray.AddMenuItem("This Device: not connected", "")
menu.self.Disable()
}
systray.AddSeparator()
menu.rebuildExitNodeMenu(ctx)
if menu.status != nil {
menu.more = systray.AddMenuItem("More settings", "")
onClick(ctx, menu.more, func(_ context.Context) {
webbrowser.Open("http://100.100.100.100/")
})
}
menu.quit = systray.AddMenuItem("Quit", "Quit the app")
menu.quit.Enable()
go menu.eventLoop(ctx)
}
// profileTitle returns the title string for a profile menu item.
func profileTitle(profile ipn.LoginProfile) string {
title := profile.Name
if profile.NetworkProfile.DomainName != "" {
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
// windows and mac don't support multi-line menu
title += " (" + profile.NetworkProfile.DomainName + ")"
} else {
title += "\n" + profile.NetworkProfile.DomainName
}
}
return title
}
var (
cacheMu sync.Mutex
httpCache = map[string][]byte{} // URL => response body
)
// setRemoteIcon sets the icon for menu to the specified remote image.
// Remote images are fetched as needed and cached.
func setRemoteIcon(menu *systray.MenuItem, urlStr string) {
if menu == nil || urlStr == "" {
return
}
cacheMu.Lock()
b, ok := httpCache[urlStr]
if !ok {
resp, err := http.Get(urlStr)
if err == nil && resp.StatusCode == http.StatusOK {
b, _ = io.ReadAll(resp.Body)
httpCache[urlStr] = b
resp.Body.Close()
}
}
cacheMu.Unlock()
if len(b) > 0 {
menu.SetIcon(b)
}
}
// setTooltip sets the tooltip text for the systray icon.
func setTooltip(text string) {
if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
systray.SetTooltip(text)
} else {
// on Linux, SetTitle actually sets the tooltip
systray.SetTitle(text)
}
}
// eventLoop is the main event loop for handling click events on menu items
// and responding to Tailscale state changes.
// This method does not return until ctx.Done is closed.
func (menu *Menu) eventLoop(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case <-menu.rebuildCh:
menu.updateState()
menu.rebuild()
case <-menu.connect.ClickedCh:
_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
Prefs: ipn.Prefs{
WantRunning: true,
},
WantRunningSet: true,
})
if err != nil {
log.Printf("error connecting: %v", err)
}
case <-menu.disconnect.ClickedCh:
_, err := menu.lc.EditPrefs(ctx, &ipn.MaskedPrefs{
Prefs: ipn.Prefs{
WantRunning: false,
},
WantRunningSet: true,
})
if err != nil {
log.Printf("error disconnecting: %v", err)
}
case <-menu.self.ClickedCh:
menu.copyTailscaleIP(menu.status.Self)
case id := <-menu.accountsCh:
if err := menu.lc.SwitchProfile(ctx, id); err != nil {
log.Printf("error switching to profile ID %v: %v", id, err)
}
case exitNode := <-menu.exitNodeCh:
if exitNode.IsZero() {
log.Print("disable exit node")
if err := menu.lc.SetUseExitNode(ctx, false); err != nil {
log.Printf("error disabling exit node: %v", err)
}
} else {
log.Printf("enable exit node: %v", exitNode)
mp := &ipn.MaskedPrefs{
Prefs: ipn.Prefs{
ExitNodeID: exitNode,
},
ExitNodeIDSet: true,
}
if _, err := menu.lc.EditPrefs(ctx, mp); err != nil {
log.Printf("error setting exit node: %v", err)
}
}
case <-menu.quit.ClickedCh:
systray.Quit()
}
}
}
// onClick registers a click handler for a menu item.
func onClick(ctx context.Context, item *systray.MenuItem, fn func(ctx context.Context)) {
go func() {
for {
select {
case <-ctx.Done():
return
case <-item.ClickedCh:
fn(ctx)
}
}
}()
}
// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
// This method does not return.
func (menu *Menu) watchIPNBus() {
for {
if err := menu.watchIPNBusInner(); err != nil {
log.Println(err)
if errors.Is(err, context.Canceled) {
// If the context got canceled, we will never be able to
// reconnect to IPN bus, so exit the process.
log.Fatalf("watchIPNBus: %v", err)
}
}
// If our watch connection breaks, wait a bit before reconnecting. No
// reason to spam the logs if e.g. tailscaled is restarting or goes
// down.
time.Sleep(3 * time.Second)
}
}
func (menu *Menu) watchIPNBusInner() error {
watcher, err := menu.lc.WatchIPNBus(menu.bgCtx, ipn.NotifyNoPrivateKeys)
if err != nil {
return fmt.Errorf("watching ipn bus: %w", err)
}
defer watcher.Close()
for {
select {
case <-menu.bgCtx.Done():
return nil
default:
n, err := watcher.Next()
if err != nil {
return fmt.Errorf("ipnbus error: %w", err)
}
var rebuild bool
if n.State != nil {
log.Printf("new state: %v", n.State)
rebuild = true
}
if n.Prefs != nil {
rebuild = true
}
if rebuild {
menu.rebuildCh <- struct{}{}
}
}
}
}
// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
// and sends a notification with the copied value.
func (menu *Menu) copyTailscaleIP(device *ipnstate.PeerStatus) {
if device == nil || len(device.TailscaleIPs) == 0 {
return
}
name := strings.Split(device.DNSName, ".")[0]
ip := device.TailscaleIPs[0].String()
err := clipboard.WriteAll(ip)
if err != nil {
log.Printf("clipboard error: %v", err)
}
menu.sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
}
// sendNotification sends a desktop notification with the given title and content.
func (menu *Menu) sendNotification(title, content string) {
conn, err := dbus.SessionBus()
if err != nil {
log.Printf("dbus: %v", err)
return
}
timeout := 3 * time.Second
obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
menu.notificationIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
if call.Err != nil {
log.Printf("dbus: %v", call.Err)
}
}
func (menu *Menu) rebuildExitNodeMenu(ctx context.Context) {
if menu.status == nil {
return
}
status := menu.status
menu.exitNodes = systray.AddMenuItem("Exit Nodes", "")
time.Sleep(newMenuDelay)
// register a click handler for a menu item to set nodeID as the exit node.
setExitNodeOnClick := func(item *systray.MenuItem, nodeID tailcfg.StableNodeID) {
onClick(ctx, item, func(ctx context.Context) {
select {
case <-ctx.Done():
case menu.exitNodeCh <- nodeID:
}
})
}
noExitNodeMenu := menu.exitNodes.AddSubMenuItemCheckbox("None", "", status.ExitNodeStatus == nil)
setExitNodeOnClick(noExitNodeMenu, "")
// Show recommended exit node if available.
if status.Self.CapMap.Contains(tailcfg.NodeAttrSuggestExitNodeUI) {
sugg, err := menu.lc.SuggestExitNode(ctx)
if err == nil {
title := "Recommended: "
if loc := sugg.Location; loc.Valid() && loc.Country() != "" {
flag := countryFlag(loc.CountryCode())
title += fmt.Sprintf("%s %s: %s", flag, loc.Country(), loc.City())
} else {
title += strings.Split(sugg.Name, ".")[0]
}
menu.exitNodes.AddSeparator()
rm := menu.exitNodes.AddSubMenuItemCheckbox(title, "", false)
setExitNodeOnClick(rm, sugg.ID)
if status.ExitNodeStatus != nil && sugg.ID == status.ExitNodeStatus.ID {
rm.Check()
}
}
}
// Add tailnet exit nodes if present.
var tailnetExitNodes []*ipnstate.PeerStatus
for _, ps := range status.Peer {
if ps.ExitNodeOption && ps.Location == nil {
tailnetExitNodes = append(tailnetExitNodes, ps)
}
}
if len(tailnetExitNodes) > 0 {
menu.exitNodes.AddSeparator()
menu.exitNodes.AddSubMenuItem("Tailnet Exit Nodes", "").Disable()
for _, ps := range status.Peer {
if !ps.ExitNodeOption || ps.Location != nil {
continue
}
name := strings.Split(ps.DNSName, ".")[0]
if !ps.Online {
name += " (offline)"
}
sm := menu.exitNodes.AddSubMenuItemCheckbox(name, "", false)
if !ps.Online {
sm.Disable()
}
if status.ExitNodeStatus != nil && ps.ID == status.ExitNodeStatus.ID {
sm.Check()
}
setExitNodeOnClick(sm, ps.ID)
}
}
// Add mullvad exit nodes if present.
var mullvadExitNodes mullvadPeers
if status.Self.CapMap.Contains("mullvad") {
mullvadExitNodes = newMullvadPeers(status)
}
if len(mullvadExitNodes.countries) > 0 {
menu.exitNodes.AddSeparator()
menu.exitNodes.AddSubMenuItem("Location-based Exit Nodes", "").Disable()
mullvadMenu := menu.exitNodes.AddSubMenuItemCheckbox("Mullvad VPN", "", false)
for _, country := range mullvadExitNodes.sortedCountries() {
flag := countryFlag(country.code)
countryMenu := mullvadMenu.AddSubMenuItemCheckbox(flag+" "+country.name, "", false)
// single-city country, no submenu
if len(country.cities) == 1 || hideMullvadCities {
setExitNodeOnClick(countryMenu, country.best.ID)
if status.ExitNodeStatus != nil {
for _, city := range country.cities {
for _, ps := range city.peers {
if status.ExitNodeStatus.ID == ps.ID {
mullvadMenu.Check()
countryMenu.Check()
}
}
}
}
continue
}
// multi-city country, build submenu with "best available" option and cities.
time.Sleep(newMenuDelay)
bm := countryMenu.AddSubMenuItemCheckbox("Best Available", "", false)
setExitNodeOnClick(bm, country.best.ID)
countryMenu.AddSeparator()
for _, city := range country.sortedCities() {
cityMenu := countryMenu.AddSubMenuItemCheckbox(city.name, "", false)
setExitNodeOnClick(cityMenu, city.best.ID)
if status.ExitNodeStatus != nil {
for _, ps := range city.peers {
if status.ExitNodeStatus.ID == ps.ID {
mullvadMenu.Check()
countryMenu.Check()
cityMenu.Check()
}
}
}
}
}
}
// TODO: "Allow Local Network Access" and "Run Exit Node" menu items
}
// mullvadPeers contains all mullvad peer nodes, sorted by country and city.
type mullvadPeers struct {
countries map[string]*mvCountry // country code (uppercase) => country
}
// sortedCountries returns countries containing mullvad nodes, sorted by name.
func (mp mullvadPeers) sortedCountries() []*mvCountry {
countries := slicesx.MapValues(mp.countries)
slices.SortFunc(countries, func(a, b *mvCountry) int {
return stringsx.CompareFold(a.name, b.name)
})
return countries
}
type mvCountry struct {
code string
name string
best *ipnstate.PeerStatus // highest priority peer in the country
cities map[string]*mvCity // city code => city
}
// sortedCities returns cities containing mullvad nodes, sorted by name.
func (mc *mvCountry) sortedCities() []*mvCity {
cities := slicesx.MapValues(mc.cities)
slices.SortFunc(cities, func(a, b *mvCity) int {
return stringsx.CompareFold(a.name, b.name)
})
return cities
}
// countryFlag takes a 2-character ASCII string and returns the corresponding emoji flag.
// It returns the empty string on error.
func countryFlag(code string) string {
if len(code) != 2 {
return ""
}
runes := make([]rune, 0, 2)
for i := range 2 {
b := code[i] | 32 // lowercase
if b < 'a' || b > 'z' {
return ""
}
// https://en.wikipedia.org/wiki/Regional_indicator_symbol
runes = append(runes, 0x1F1E6+rune(b-'a'))
}
return string(runes)
}
type mvCity struct {
name string
best *ipnstate.PeerStatus // highest priority peer in the city
peers []*ipnstate.PeerStatus
}
func newMullvadPeers(status *ipnstate.Status) mullvadPeers {
countries := make(map[string]*mvCountry)
for _, ps := range status.Peer {
if !ps.ExitNodeOption || ps.Location == nil {
continue
}
loc := ps.Location
country, ok := countries[loc.CountryCode]
if !ok {
country = &mvCountry{
code: loc.CountryCode,
name: loc.Country,
cities: make(map[string]*mvCity),
}
countries[loc.CountryCode] = country
}
city, ok := countries[loc.CountryCode].cities[loc.CityCode]
if !ok {
city = &mvCity{
name: loc.City,
}
countries[loc.CountryCode].cities[loc.CityCode] = city
}
city.peers = append(city.peers, ps)
if city.best == nil || ps.Location.Priority > city.best.Location.Priority {
city.best = ps
}
if country.best == nil || ps.Location.Priority > country.best.Location.Priority {
country.best = ps
}
}
return mullvadPeers{countries}
}
// onExit is called by the systray package when the menu is exiting.
func (menu *Menu) onExit() {
log.Printf("exiting")
if menu.bgCancel != nil {
menu.bgCancel()
}
if menu.eventCancel != nil {
menu.eventCancel()
}
os.Remove(menu.notificationIcon.Name())
}

View File

@@ -1,7 +1,7 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build go1.22
//go:build go1.19
package tailscale
@@ -62,12 +62,6 @@ type LocalClient struct {
// machine's tailscaled or equivalent. If nil, a default is used.
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
// Transport optionally specifies an alternate [http.RoundTripper]
// used to execute HTTP requests. If nil, a default [http.Transport] is used,
// potentially with custom dialing logic from [Dial].
// It is primarily used for testing.
Transport http.RoundTripper
// Socket specifies an alternate path to the local Tailscale socket.
// If empty, a platform-specific default is used.
Socket string
@@ -135,9 +129,9 @@ func (lc *LocalClient) DoLocalRequest(req *http.Request) (*http.Response, error)
req.Header.Set("Tailscale-Cap", strconv.Itoa(int(tailcfg.CurrentCapabilityVersion)))
lc.tsClientOnce.Do(func() {
lc.tsClient = &http.Client{
Transport: cmp.Or(lc.Transport, http.RoundTripper(
&http.Transport{DialContext: lc.dialer()}),
),
Transport: &http.Transport{
DialContext: lc.dialer(),
},
}
})
if !lc.OmitAuth {
@@ -499,17 +493,6 @@ func (lc *LocalClient) DebugAction(ctx context.Context, action string) error {
return nil
}
// DebugActionBody invokes a debug action with a body parameter, such as
// "debug-force-prefer-derp".
// These are development tools and subject to change or removal over time.
func (lc *LocalClient) DebugActionBody(ctx context.Context, action string, rbody io.Reader) error {
body, err := lc.send(ctx, "POST", "/localapi/v0/debug?action="+url.QueryEscape(action), 200, rbody)
if err != nil {
return fmt.Errorf("error %w: %s", err, body)
}
return nil
}
// DebugResultJSON invokes a debug action and returns its result as something JSON-able.
// These are development tools and subject to change or removal over time.
func (lc *LocalClient) DebugResultJSON(ctx context.Context, action string) (any, error) {

View File

@@ -89,8 +89,8 @@ type Server struct {
type ServerMode string
const (
// LoginServerMode serves a read-only login client for logging a
// node into a tailnet, and viewing a read-only interface of the
// LoginServerMode serves a readonly login client for logging a
// node into a tailnet, and viewing a readonly interface of the
// node's current Tailscale settings.
//
// In this mode, API calls are authenticated via platform auth.
@@ -110,7 +110,7 @@ const (
// This mode restricts the app to only being assessible over Tailscale,
// and API calls are authenticated via browser sessions associated with
// the source's Tailscale identity. If the source browser does not have
// a valid session, a read-only version of the app is displayed.
// a valid session, a readonly version of the app is displayed.
ManageServerMode ServerMode = "manage"
)
@@ -695,16 +695,16 @@ func (s *Server) serveAPIAuth(w http.ResponseWriter, r *http.Request) {
switch {
case sErr != nil && errors.Is(sErr, errNotUsingTailscale):
s.lc.IncrementCounter(r.Context(), "web_client_viewing_local", 1)
resp.Authorized = false // restricted to the read-only view
resp.Authorized = false // restricted to the readonly view
case sErr != nil && errors.Is(sErr, errNotOwner):
s.lc.IncrementCounter(r.Context(), "web_client_viewing_not_owner", 1)
resp.Authorized = false // restricted to the read-only view
resp.Authorized = false // restricted to the readonly view
case sErr != nil && errors.Is(sErr, errTaggedLocalSource):
s.lc.IncrementCounter(r.Context(), "web_client_viewing_local_tag", 1)
resp.Authorized = false // restricted to the read-only view
resp.Authorized = false // restricted to the readonly view
case sErr != nil && errors.Is(sErr, errTaggedRemoteSource):
s.lc.IncrementCounter(r.Context(), "web_client_viewing_remote_tag", 1)
resp.Authorized = false // restricted to the read-only view
resp.Authorized = false // restricted to the readonly view
case sErr != nil && !errors.Is(sErr, errNoSession):
// Any other error.
http.Error(w, sErr.Error(), http.StatusInternalServerError)
@@ -804,8 +804,8 @@ type nodeData struct {
DeviceName string
TailnetName string // TLS cert name
DomainName string
IPv4 netip.Addr
IPv6 netip.Addr
IPv4 string
IPv6 string
OS string
IPNVersion string
@@ -864,14 +864,10 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
return
}
filterRules, _ := s.lc.DebugPacketFilterRules(r.Context())
ipv4, ipv6 := s.selfNodeAddresses(r, st)
data := &nodeData{
ID: st.Self.ID,
Status: st.BackendState,
DeviceName: strings.Split(st.Self.DNSName, ".")[0],
IPv4: ipv4,
IPv6: ipv6,
OS: st.Self.OS,
IPNVersion: strings.Split(st.Version, "-")[0],
Profile: st.User[st.Self.UserID],
@@ -891,6 +887,10 @@ func (s *Server) serveGetNodeData(w http.ResponseWriter, r *http.Request) {
ACLAllowsAnyIncomingTraffic: s.aclsAllowAccess(filterRules),
}
ipv4, ipv6 := s.selfNodeAddresses(r, st)
data.IPv4 = ipv4.String()
data.IPv6 = ipv6.String()
if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn && data.URLPrefix == "" {
// X-Ingress-Path is the path prefix in use for Home Assistant
// https://developers.home-assistant.io/docs/add-ons/presentation#ingress

View File

@@ -18,12 +18,12 @@ var (
)
func usage() {
fmt.Fprint(os.Stderr, `
fmt.Fprintf(os.Stderr, `
usage: addlicense -file FILE <subcommand args...>
`[1:])
flag.PrintDefaults()
fmt.Fprint(os.Stderr, `
fmt.Fprintf(os.Stderr, `
addlicense adds a Tailscale license to the beginning of file.
It is intended for use with 'go generate', so it also runs a subcommand,

View File

@@ -1,131 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// checkmetrics validates that all metrics in the tailscale client-metrics
// are documented in a given path or URL.
package main
import (
"context"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/http/httptest"
"os"
"strings"
"time"
"tailscale.com/ipn/store/mem"
"tailscale.com/tsnet"
"tailscale.com/tstest/integration/testcontrol"
"tailscale.com/util/httpm"
)
var (
kbPath = flag.String("kb-path", "", "filepath to the client-metrics knowledge base")
kbUrl = flag.String("kb-url", "", "URL to the client-metrics knowledge base page")
)
func main() {
flag.Parse()
if *kbPath == "" && *kbUrl == "" {
log.Fatalf("either -kb-path or -kb-url must be set")
}
var control testcontrol.Server
ts := httptest.NewServer(&control)
defer ts.Close()
td, err := os.MkdirTemp("", "testcontrol")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(td)
// tsnet is used not used as a Tailscale client, but as a way to
// boot up Tailscale, have all the metrics registered, and then
// verifiy that all the metrics are documented.
tsn := &tsnet.Server{
Dir: td,
Store: new(mem.Store),
UserLogf: log.Printf,
Ephemeral: true,
ControlURL: ts.URL,
}
if err := tsn.Start(); err != nil {
log.Fatal(err)
}
defer tsn.Close()
log.Printf("checking that all metrics are documented, looking for: %s", tsn.Sys().UserMetricsRegistry().MetricNames())
if *kbPath != "" {
kb, err := readKB(*kbPath)
if err != nil {
log.Fatalf("reading kb: %v", err)
}
missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
if len(missing) > 0 {
log.Fatalf("found undocumented metrics in %q: %v", *kbPath, missing)
}
}
if *kbUrl != "" {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
kb, err := getKB(ctx, *kbUrl)
if err != nil {
log.Fatalf("getting kb: %v", err)
}
missing := undocumentedMetrics(kb, tsn.Sys().UserMetricsRegistry().MetricNames())
if len(missing) > 0 {
log.Fatalf("found undocumented metrics in %q: %v", *kbUrl, missing)
}
}
}
func readKB(path string) (string, error) {
b, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("reading file: %w", err)
}
return string(b), nil
}
func getKB(ctx context.Context, url string) (string, error) {
req, err := http.NewRequestWithContext(ctx, httpm.GET, url, nil)
if err != nil {
return "", fmt.Errorf("creating request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", fmt.Errorf("getting kb page: %w", err)
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
b, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("reading body: %w", err)
}
return string(b), nil
}
func undocumentedMetrics(b string, metrics []string) []string {
var missing []string
for _, metric := range metrics {
if !strings.Contains(b, metric) {
missing = append(missing, metric)
}
}
return missing
}

View File

@@ -7,6 +7,7 @@ package main
import (
"log"
"net"
"net/http"
"sync"
)
@@ -22,29 +23,29 @@ type healthz struct {
func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.Lock()
defer h.Unlock()
if h.hasAddrs {
w.Write([]byte("ok"))
} else {
http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable)
http.Error(w, "node currently has no tailscale IPs", http.StatusInternalServerError)
}
}
func (h *healthz) update(healthy bool) {
h.Lock()
defer h.Unlock()
if h.hasAddrs != healthy {
log.Println("Setting healthy", healthy)
}
h.hasAddrs = healthy
}
// healthHandlers registers a simple health handler at /healthz.
// A containerized tailscale instance is considered healthy if
// runHealthz runs a simple HTTP health endpoint on /healthz, listening on the
// provided address. A containerized tailscale instance is considered healthy if
// it has at least one tailnet IP address.
func healthHandlers(mux *http.ServeMux) *healthz {
h := &healthz{}
mux.Handle("GET /healthz", h)
return h
func runHealthz(addr string, h *healthz) {
lis, err := net.Listen("tcp", addr)
if err != nil {
log.Fatalf("error listening on the provided health endpoint address %q: %v", addr, err)
}
mux := http.NewServeMux()
mux.Handle("/healthz", h)
log.Printf("Running healthcheck endpoint at %s/healthz", addr)
hs := &http.Server{Handler: mux}
go func() {
if err := hs.Serve(lis); err != nil {
log.Fatalf("failed running health endpoint: %v", err)
}
}()
}

View File

@@ -9,56 +9,30 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"net/netip"
"os"
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
)
// kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use
// this rather than any of the upstream Kubernetes client libaries to avoid extra imports.
type kubeClient struct {
kubeclient.Client
stateSecret string
canPatch bool // whether the client has permissions to patch Kubernetes Secrets
}
func newKubeClient(root string, stateSecret string) (*kubeClient, error) {
if root != "/" {
// If we are running in a test, we need to set the root path to the fake
// service account directory.
kubeclient.SetRootPathForTesting(root)
}
var err error
kc, err := kubeclient.New("tailscale-container")
if err != nil {
return nil, fmt.Errorf("Error creating kube client: %w", err)
}
if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
// Derive the API server address from the environment variables
// Used to set http server in tests, or optionally enabled by flag
kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
}
return &kubeClient{Client: kc, stateSecret: stateSecret}, nil
}
// storeDeviceID writes deviceID to 'device_id' data field of the client's state Secret.
func (kc *kubeClient) storeDeviceID(ctx context.Context, deviceID tailcfg.StableNodeID) error {
// storeDeviceID writes deviceID to 'device_id' data field of the named
// Kubernetes Secret.
func storeDeviceID(ctx context.Context, secretName string, deviceID tailcfg.StableNodeID) error {
s := &kubeapi.Secret{
Data: map[string][]byte{
kubetypes.KeyDeviceID: []byte(deviceID),
"device_id": []byte(deviceID),
},
}
return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
}
// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields 'device_ips', 'device_fqdn' of client's
// state Secret.
func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, addresses []netip.Prefix) error {
// storeDeviceEndpoints writes device's tailnet IPs and MagicDNS name to fields
// 'device_ips', 'device_fqdn' of the named Kubernetes Secret.
func storeDeviceEndpoints(ctx context.Context, secretName string, fqdn string, addresses []netip.Prefix) error {
var ips []string
for _, addr := range addresses {
ips = append(ips, addr.Addr().String())
@@ -70,28 +44,16 @@ func (kc *kubeClient) storeDeviceEndpoints(ctx context.Context, fqdn string, add
s := &kubeapi.Secret{
Data: map[string][]byte{
kubetypes.KeyDeviceFQDN: []byte(fqdn),
kubetypes.KeyDeviceIPs: deviceIPs,
"device_fqdn": []byte(fqdn),
"device_ips": deviceIPs,
},
}
return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
}
// storeHTTPSEndpoint writes an HTTPS endpoint exposed by this device via 'tailscale serve' to the client's state
// Secret. In practice this will be the same value that gets written to 'device_fqdn', but this should only be called
// when the serve config has been successfully set up.
func (kc *kubeClient) storeHTTPSEndpoint(ctx context.Context, ep string) error {
s := &kubeapi.Secret{
Data: map[string][]byte{
kubetypes.KeyHTTPSEndpoint: []byte(ep),
},
}
return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
return kc.StrategicMergePatchSecret(ctx, secretName, s, "tailscale-container")
}
// deleteAuthKey deletes the 'authkey' field of the given kube
// secret. No-op if there is no authkey in the secret.
func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
func deleteAuthKey(ctx context.Context, secretName string) error {
// m is a JSON Patch data structure, see https://jsonpatch.com/ or RFC 6902.
m := []kubeclient.JSONPatch{
{
@@ -99,7 +61,7 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
Path: "/data/authkey",
},
}
if err := kc.JSONPatchResource(ctx, kc.stateSecret, kubeclient.TypeSecrets, m); err != nil {
if err := kc.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
if s, ok := err.(*kubeapi.Status); ok && s.Code == http.StatusUnprocessableEntity {
// This is kubernetes-ese for "the field you asked to
// delete already doesn't exist", aka no-op.
@@ -110,19 +72,22 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error {
return nil
}
// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale
// state Secret.
// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container.
func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error {
capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
d := map[string][]byte{
kubetypes.KeyCapVer: []byte(capVerS),
var kc kubeclient.Client
func initKubeClient(root string) {
if root != "/" {
// If we are running in a test, we need to set the root path to the fake
// service account directory.
kubeclient.SetRootPathForTesting(root)
}
if podUID != "" {
d[kubetypes.KeyPodUID] = []byte(podUID)
var err error
kc, err = kubeclient.New("tailscale-container")
if err != nil {
log.Fatalf("Error creating kube client: %v", err)
}
s := &kubeapi.Secret{
Data: d,
if (root != "/") || os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
// Derive the API server address from the environment variables
// Used to set http server in tests, or optionally enabled by flag
kc.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
}
return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container")
}

View File

@@ -21,7 +21,7 @@ func TestSetupKube(t *testing.T) {
cfg *settings
wantErr bool
wantCfg *settings
kc *kubeClient
kc kubeclient.Client
}{
{
name: "TS_AUTHKEY set, state Secret exists",
@@ -29,14 +29,14 @@ func TestSetupKube(t *testing.T) {
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return nil, nil
},
}},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
@@ -48,14 +48,14 @@ func TestSetupKube(t *testing.T) {
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, true, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return nil, &kubeapi.Status{Code: 404}
},
}},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
@@ -67,14 +67,14 @@ func TestSetupKube(t *testing.T) {
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return nil, &kubeapi.Status{Code: 404}
},
}},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
@@ -87,14 +87,14 @@ func TestSetupKube(t *testing.T) {
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return nil, &kubeapi.Status{Code: 403}
},
}},
},
wantCfg: &settings{
AuthKey: "foo",
KubeSecret: "foo",
@@ -111,11 +111,11 @@ func TestSetupKube(t *testing.T) {
AuthKey: "foo",
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, errors.New("broken")
},
}},
},
wantErr: true,
},
{
@@ -127,14 +127,14 @@ func TestSetupKube(t *testing.T) {
wantCfg: &settings{
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, true, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return nil, &kubeapi.Status{Code: 404}
},
}},
},
},
{
// Interactive login using URL in Pod logs
@@ -145,28 +145,28 @@ func TestSetupKube(t *testing.T) {
wantCfg: &settings{
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{}, nil
},
}},
},
},
{
name: "TS_AUTHKEY not set, state Secret contains auth key, we do not have RBAC to patch it",
cfg: &settings{
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return false, false, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
},
}},
},
wantCfg: &settings{
KubeSecret: "foo",
},
@@ -177,14 +177,14 @@ func TestSetupKube(t *testing.T) {
cfg: &settings{
KubeSecret: "foo",
},
kc: &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{
kc: &kubeclient.FakeClient{
CheckSecretPermissionsImpl: func(context.Context, string) (bool, bool, error) {
return true, false, nil
},
GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{Data: map[string][]byte{"authkey": []byte("foo")}}, nil
},
}},
},
wantCfg: &settings{
KubeSecret: "foo",
AuthKey: "foo",
@@ -194,9 +194,9 @@ func TestSetupKube(t *testing.T) {
}
for _, tt := range tests {
kc := tt.kc
kc = tt.kc
t.Run(tt.name, func(t *testing.T) {
if err := tt.cfg.setupKube(context.Background(), kc); (err != nil) != tt.wantErr {
if err := tt.cfg.setupKube(context.Background()); (err != nil) != tt.wantErr {
t.Errorf("settings.setupKube() error = %v, wantErr %v", err, tt.wantErr)
}
if diff := cmp.Diff(*tt.cfg, *tt.wantCfg); diff != "" {

View File

@@ -52,17 +52,11 @@
// ${TS_CERT_DOMAIN}, it will be replaced with the value of the available FQDN.
// It cannot be used in conjunction with TS_DEST_IP. The file is watched for changes,
// and will be re-applied when it changes.
// - TS_HEALTHCHECK_ADDR_PORT: deprecated, use TS_ENABLE_HEALTH_CHECK instead and optionally
// set TS_LOCAL_ADDR_PORT. Will be removed in 1.82.0.
// - TS_LOCAL_ADDR_PORT: the address and port to serve local metrics and health
// check endpoints if enabled via TS_ENABLE_METRICS and/or TS_ENABLE_HEALTH_CHECK.
// Defaults to [::]:9002, serving on all available interfaces.
// - TS_ENABLE_METRICS: if true, a metrics endpoint will be served at /metrics on
// the address specified by TS_LOCAL_ADDR_PORT. See https://tailscale.com/kb/1482/client-metrics
// for more information on the metrics exposed.
// - TS_ENABLE_HEALTH_CHECK: if true, a health check endpoint will be served at /healthz on
// the address specified by TS_LOCAL_ADDR_PORT. The health endpoint will return 200
// OK if this node has at least one tailnet IP address, otherwise returns 503.
// - TS_HEALTHCHECK_ADDR_PORT: if specified, an HTTP health endpoint will be
// served at /healthz at the provided address, which should be in form [<address>]:<port>.
// If not set, no health check will be run. If set to :<port>, addr will default to 0.0.0.0
// The health endpoint will return 200 OK if this node has at least one tailnet IP address,
// otherwise returns 503.
// NB: the health criteria might change in the future.
// - TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR: if specified, a path to a
// directory that containers tailscaled config in file. The config file needs to be
@@ -105,7 +99,6 @@ import (
"log"
"math"
"net"
"net/http"
"net/netip"
"os"
"os/signal"
@@ -121,7 +114,6 @@ import (
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
kubeutils "tailscale.com/k8s-operator"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/types/logger"
"tailscale.com/types/ptr"
@@ -168,13 +160,9 @@ func main() {
bootCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
var kc *kubeClient
if cfg.InKubernetes {
kc, err = newKubeClient(cfg.Root, cfg.KubeSecret)
if err != nil {
log.Fatalf("error initializing kube client: %v", err)
}
if err := cfg.setupKube(bootCtx, kc); err != nil {
initKubeClient(cfg.Root)
if err := cfg.setupKube(bootCtx); err != nil {
log.Fatalf("error setting up for running on Kubernetes: %v", err)
}
}
@@ -190,34 +178,6 @@ func main() {
}
defer killTailscaled()
var healthCheck *healthz
if cfg.HealthCheckAddrPort != "" {
mux := http.NewServeMux()
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
healthCheck = healthHandlers(mux)
close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
defer close()
}
if cfg.localMetricsEnabled() || cfg.localHealthEnabled() {
mux := http.NewServeMux()
if cfg.localMetricsEnabled() {
log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort)
metricsHandlers(mux, client, cfg.DebugAddrPort)
}
if cfg.localHealthEnabled() {
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
healthCheck = healthHandlers(mux)
}
close := runHTTPServer(mux, cfg.LocalAddrPort)
defer close()
}
if cfg.EnableForwardingOptimizations {
if err := client.SetUDPGROForwarding(bootCtx); err != nil {
log.Printf("[unexpected] error enabling UDP GRO forwarding: %v", err)
@@ -324,18 +284,12 @@ authLoop:
}
}
// Remove any serve config and advertised HTTPS endpoint that may have been set by a previous run of
// containerboot, but only if we're providing a new one.
if cfg.ServeConfigPath != "" {
log.Printf("serve proxy: unsetting previous config")
// Remove any serve config that may have been set by a previous run of
// containerboot, but only if we're providing a new one.
if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil {
log.Fatalf("failed to unset serve config: %v", err)
}
if hasKubeStateStore(cfg) {
if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil {
log.Fatalf("failed to update HTTPS endpoint in tailscale state: %v", err)
}
}
}
if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) {
@@ -343,28 +297,16 @@ authLoop:
// authkey is no longer needed. We don't strictly need to
// wipe it, but it's good hygiene.
log.Printf("Deleting authkey from kube secret")
if err := kc.deleteAuthKey(ctx); err != nil {
if err := deleteAuthKey(ctx, cfg.KubeSecret); err != nil {
log.Fatalf("deleting authkey from kube secret: %v", err)
}
}
if hasKubeStateStore(cfg) {
if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil {
log.Fatalf("storing capability version and UID: %v", err)
}
}
w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState)
if err != nil {
log.Fatalf("rewatching tailscaled for updates after auth: %v", err)
}
// If tailscaled config was read from a mounted file, watch the file for updates and reload.
cfgWatchErrChan := make(chan error)
if cfg.TailscaledConfigFilePath != "" {
go watchTailscaledConfigChanges(ctx, cfg.TailscaledConfigFilePath, client, cfgWatchErrChan)
}
var (
startupTasksDone = false
currentIPs deephash.Sum // tailscale IPs assigned to device
@@ -379,9 +321,12 @@ authLoop:
certDomain = new(atomic.Pointer[string])
certDomainChanged = make(chan bool, 1)
triggerWatchServeConfigChanges sync.Once
h = &healthz{} // http server for the healthz endpoint
healthzRunner = sync.OnceFunc(func() { runHealthz(cfg.HealthCheckAddrPort, h) })
)
if cfg.ServeConfigPath != "" {
go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client)
}
var nfr linuxfw.NetfilterRunner
if isL3Proxy(cfg) {
nfr, err = newNetfilterRunner(log.Printf)
@@ -458,8 +403,6 @@ runLoop:
break runLoop
case err := <-errChan:
log.Fatalf("failed to read from tailscaled: %v", err)
case err := <-cfgWatchErrChan:
log.Fatalf("failed to watch tailscaled config: %v", err)
case n := <-notifyChan:
if n.State != nil && *n.State != ipn.Running {
// Something's gone wrong and we've left the authenticated state.
@@ -484,7 +427,7 @@ runLoop:
// fails.
deviceID := n.NetMap.SelfNode.StableID()
if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceID, &deviceID) {
if err := kc.storeDeviceID(ctx, n.NetMap.SelfNode.StableID()); err != nil {
if err := storeDeviceID(ctx, cfg.KubeSecret, n.NetMap.SelfNode.StableID()); err != nil {
log.Fatalf("storing device ID in Kubernetes Secret: %v", err)
}
}
@@ -557,11 +500,8 @@ runLoop:
resetTimer(false)
backendAddrs = newBackendAddrs
}
if cfg.ServeConfigPath != "" {
cd := certDomainFromNetmap(n.NetMap)
if cd == "" {
cd = kubetypes.ValueNoHTTPS
}
if cfg.ServeConfigPath != "" && len(n.NetMap.DNS.CertDomains) != 0 {
cd := n.NetMap.DNS.CertDomains[0]
prev := certDomain.Swap(ptr.To(cd))
if prev == nil || *prev != cd {
select {
@@ -603,21 +543,17 @@ runLoop:
// TODO (irbekrm): instead of using the IP and FQDN, have some other mechanism for the proxy signal that it is 'Ready'.
deviceEndpoints := []any{n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses()}
if hasKubeStateStore(cfg) && deephash.Update(&currentDeviceEndpoints, &deviceEndpoints) {
if err := kc.storeDeviceEndpoints(ctx, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
if err := storeDeviceEndpoints(ctx, cfg.KubeSecret, n.NetMap.SelfNode.Name(), n.NetMap.SelfNode.Addresses().AsSlice()); err != nil {
log.Fatalf("storing device IPs and FQDN in Kubernetes Secret: %v", err)
}
}
if healthCheck != nil {
healthCheck.update(len(addrs) != 0)
if cfg.HealthCheckAddrPort != "" {
h.Lock()
h.hasAddrs = len(addrs) != 0
h.Unlock()
healthzRunner()
}
if cfg.ServeConfigPath != "" {
triggerWatchServeConfigChanges.Do(func() {
go watchServeConfigChanges(ctx, cfg.ServeConfigPath, certDomainChanged, certDomain, client, kc)
})
}
if egressSvcsNotify != nil {
egressSvcsNotify <- n
}
@@ -807,22 +743,3 @@ func tailscaledConfigFilePath() string {
log.Printf("Using tailscaled config file %q to match current capability version %d", filePath, tailcfg.CurrentCapabilityVersion)
return filePath
}
func runHTTPServer(mux *http.ServeMux, addr string) (close func() error) {
ln, err := net.Listen("tcp", addr)
if err != nil {
log.Fatalf("failed to listen on addr %q: %v", addr, err)
}
srv := &http.Server{Handler: mux}
go func() {
if err := srv.Serve(ln); err != nil {
log.Fatalf("failed running server: %v", err)
}
}()
return func() error {
err := srv.Shutdown(context.Background())
return errors.Join(err, ln.Close())
}
}

View File

@@ -31,7 +31,6 @@ import (
"github.com/google/go-cmp/cmp"
"golang.org/x/sys/unix"
"tailscale.com/ipn"
"tailscale.com/kube/egressservices"
"tailscale.com/tailcfg"
"tailscale.com/tstest"
"tailscale.com/types/netmap"
@@ -58,16 +57,6 @@ func TestContainerBoot(t *testing.T) {
if err != nil {
t.Fatalf("error unmarshaling tailscaled config: %v", err)
}
serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}}
serveConfBytes, err := json.Marshal(serveConf)
if err != nil {
t.Fatalf("error unmarshaling serve config: %v", err)
}
egressSvcsCfg := egressservices.Configs{"foo": {TailnetTarget: egressservices.TailnetTarget{FQDN: "foo.tailnetxyx.ts.net"}}}
egressSvcsCfgBytes, err := json.Marshal(egressSvcsCfg)
if err != nil {
t.Fatalf("error unmarshaling egress services config: %v", err)
}
dirs := []string{
"var/lib",
@@ -84,16 +73,14 @@ func TestContainerBoot(t *testing.T) {
}
}
files := map[string][]byte{
"usr/bin/tailscaled": fakeTailscaled,
"usr/bin/tailscale": fakeTailscale,
"usr/bin/iptables": fakeTailscale,
"usr/bin/ip6tables": fakeTailscale,
"dev/net/tun": []byte(""),
"proc/sys/net/ipv4/ip_forward": []byte("0"),
"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
"etc/tailscaled/cap-95.hujson": tailscaledConfBytes,
"etc/tailscaled/serve-config.json": serveConfBytes,
"etc/tailscaled/egress-services-config.json": egressSvcsCfgBytes,
"usr/bin/tailscaled": fakeTailscaled,
"usr/bin/tailscale": fakeTailscale,
"usr/bin/iptables": fakeTailscale,
"usr/bin/ip6tables": fakeTailscale,
"dev/net/tun": []byte(""),
"proc/sys/net/ipv4/ip_forward": []byte("0"),
"proc/sys/net/ipv6/conf/all/forwarding": []byte("0"),
"etc/tailscaled/cap-95.hujson": tailscaledConfBytes,
}
resetFiles := func() {
for path, content := range files {
@@ -114,26 +101,6 @@ func TestContainerBoot(t *testing.T) {
argFile := filepath.Join(d, "args")
runningSockPath := filepath.Join(d, "tmp/tailscaled.sock")
var localAddrPort, healthAddrPort int
for _, p := range []*int{&localAddrPort, &healthAddrPort} {
ln, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("Failed to open listener: %v", err)
}
if err := ln.Close(); err != nil {
t.Fatalf("Failed to close listener: %v", err)
}
port := ln.Addr().(*net.TCPAddr).Port
*p = port
}
metricsURL := func(port int) string {
return fmt.Sprintf("http://127.0.0.1:%d/metrics", port)
}
healthURL := func(port int) string {
return fmt.Sprintf("http://127.0.0.1:%d/healthz", port)
}
capver := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion)
type phase struct {
// If non-nil, send this IPN bus notification (and remember it as the
@@ -152,8 +119,6 @@ func TestContainerBoot(t *testing.T) {
// WantFatalLog is the fatal log message we expect from containerboot.
// If set for a phase, the test will finish on that phase.
WantFatalLog string
EndpointStatuses map[string]int
}
runningNotify := &ipn.Notify{
State: ptr.To(ipn.Running),
@@ -182,11 +147,6 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
},
// No metrics or health by default.
EndpointStatuses: map[string]int{
metricsURL(9002): -1,
healthURL(9002): -1,
},
},
{
Notify: runningNotify,
@@ -493,11 +453,10 @@ func TestContainerBoot(t *testing.T) {
{
Notify: runningNotify,
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver,
"authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
},
},
},
@@ -587,10 +546,9 @@ func TestContainerBoot(t *testing.T) {
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false",
},
WantKubeSecret: map[string]string{
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver,
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
},
},
},
@@ -617,11 +575,10 @@ func TestContainerBoot(t *testing.T) {
{
Notify: runningNotify,
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver,
"authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
},
},
{
@@ -636,11 +593,10 @@ func TestContainerBoot(t *testing.T) {
},
},
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
"device_fqdn": "new-name.test.ts.net",
"device_id": "newID",
"device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver,
"authkey": "tskey-key",
"device_fqdn": "new-name.test.ts.net",
"device_id": "newID",
"device_ips": `["100.64.0.1"]`,
},
},
},
@@ -744,199 +700,6 @@ func TestContainerBoot(t *testing.T) {
},
},
},
{
Name: "metrics_enabled",
Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort),
"TS_ENABLE_METRICS": "true",
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
},
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200,
healthURL(localAddrPort): -1,
},
}, {
Notify: runningNotify,
},
},
},
{
Name: "health_enabled",
Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort),
"TS_ENABLE_HEALTH_CHECK": "true",
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
},
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): -1,
healthURL(localAddrPort): 503, // Doesn't start passing until the next phase.
},
}, {
Notify: runningNotify,
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): -1,
healthURL(localAddrPort): 200,
},
},
},
},
{
Name: "metrics_and_health_on_same_port",
Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort),
"TS_ENABLE_METRICS": "true",
"TS_ENABLE_HEALTH_CHECK": "true",
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
},
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200,
healthURL(localAddrPort): 503, // Doesn't start passing until the next phase.
},
}, {
Notify: runningNotify,
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200,
healthURL(localAddrPort): 200,
},
},
},
},
{
Name: "local_metrics_and_deprecated_health",
Env: map[string]string{
"TS_LOCAL_ADDR_PORT": fmt.Sprintf("[::]:%d", localAddrPort),
"TS_ENABLE_METRICS": "true",
"TS_HEALTHCHECK_ADDR_PORT": fmt.Sprintf("[::]:%d", healthAddrPort),
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false",
},
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200,
healthURL(healthAddrPort): 503, // Doesn't start passing until the next phase.
},
}, {
Notify: runningNotify,
EndpointStatuses: map[string]int{
metricsURL(localAddrPort): 200,
healthURL(healthAddrPort): 200,
},
},
},
},
{
Name: "serve_config_no_kube",
Env: map[string]string{
"TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"),
"TS_AUTHKEY": "tskey-key",
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
},
},
{
Notify: runningNotify,
},
},
},
{
Name: "serve_config_kube",
Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
"TS_SERVE_CONFIG": filepath.Join(d, "etc/tailscaled/serve-config.json"),
},
KubeSecret: map[string]string{
"authkey": "tskey-key",
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
},
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
},
},
{
Notify: runningNotify,
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
"https_endpoint": "no-https",
"tailscale_capver": capver,
},
},
},
},
{
Name: "egress_svcs_config_kube",
Env: map[string]string{
"KUBERNETES_SERVICE_HOST": kube.Host,
"KUBERNETES_SERVICE_PORT_HTTPS": kube.Port,
"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
},
KubeSecret: map[string]string{
"authkey": "tskey-key",
},
Phases: []phase{
{
WantCmds: []string{
"/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking",
"/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key",
},
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
},
},
{
Notify: runningNotify,
WantKubeSecret: map[string]string{
"authkey": "tskey-key",
"device_fqdn": "test-node.test.ts.net",
"device_id": "myID",
"device_ips": `["100.64.0.1"]`,
"tailscale_capver": capver,
},
},
},
},
{
Name: "egress_svcs_config_no_kube",
Env: map[string]string{
"TS_EGRESS_SERVICES_CONFIG_PATH": filepath.Join(d, "etc/tailscaled/egress-services-config.json"),
"TS_AUTHKEY": "tskey-key",
},
Phases: []phase{
{
WantFatalLog: "TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes",
},
},
},
}
for _, test := range tests {
@@ -1033,26 +796,7 @@ func TestContainerBoot(t *testing.T) {
return nil
})
if err != nil {
t.Fatalf("phase %d: %v", i, err)
}
for url, want := range p.EndpointStatuses {
err := tstest.WaitFor(2*time.Second, func() error {
resp, err := http.Get(url)
if err != nil && want != -1 {
return fmt.Errorf("GET %s: %v", url, err)
}
if want > 0 && resp.StatusCode != want {
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("GET %s, want %d, got %d\n%s", url, want, resp.StatusCode, string(body))
}
return nil
})
if err != nil {
t.Fatalf("phase %d: %v", i, err)
}
t.Fatal(err)
}
}
waitLogLine(t, 2*time.Second, cbOut, "Startup complete, waiting for shutdown signal")
@@ -1211,12 +955,6 @@ func (l *localAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
panic(fmt.Sprintf("unsupported method %q", r.Method))
}
case "/localapi/v0/usermetrics":
if r.Method != "GET" {
panic(fmt.Sprintf("unsupported method %q", r.Method))
}
w.Write([]byte("fake metrics"))
return
default:
panic(fmt.Sprintf("unsupported path %q", r.URL.Path))
}

View File

@@ -1,79 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"fmt"
"io"
"net/http"
"tailscale.com/client/tailscale"
"tailscale.com/client/tailscale/apitype"
)
// metrics is a simple metrics HTTP server, if enabled it forwards requests to
// the tailscaled's LocalAPI usermetrics endpoint at /localapi/v0/usermetrics.
type metrics struct {
debugEndpoint string
lc *tailscale.LocalClient
}
func proxy(w http.ResponseWriter, r *http.Request, url string, do func(*http.Request) (*http.Response, error)) {
req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("failed to construct request: %s", err), http.StatusInternalServerError)
return
}
req.Header = r.Header.Clone()
resp, err := do(req)
if err != nil {
http.Error(w, fmt.Sprintf("failed to proxy request: %s", err), http.StatusInternalServerError)
return
}
defer resp.Body.Close()
for key, val := range resp.Header {
for _, v := range val {
w.Header().Add(key, v)
}
}
w.WriteHeader(resp.StatusCode)
if _, err := io.Copy(w, resp.Body); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
func (m *metrics) handleMetrics(w http.ResponseWriter, r *http.Request) {
localAPIURL := "http://" + apitype.LocalAPIHost + "/localapi/v0/usermetrics"
proxy(w, r, localAPIURL, m.lc.DoLocalRequest)
}
func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) {
if m.debugEndpoint == "" {
http.Error(w, "debug endpoint not configured", http.StatusNotFound)
return
}
debugURL := "http://" + m.debugEndpoint + r.URL.Path
proxy(w, r, debugURL, http.DefaultClient.Do)
}
// metricsHandlers registers a simple HTTP metrics handler at /metrics, forwarding
// requests to tailscaled's /localapi/v0/usermetrics API.
//
// In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug
// endpoint if configured to ease migration for a breaking change serving user
// metrics instead of debug metrics on the "metrics" port.
func metricsHandlers(mux *http.ServeMux, lc *tailscale.LocalClient, debugAddrPort string) {
m := &metrics{
lc: lc,
debugEndpoint: debugAddrPort,
}
mux.HandleFunc("GET /metrics", m.handleMetrics)
mux.HandleFunc("/debug/", m.handleDebug) // TODO(tomhjp): Remove for 1.82.0 release.
}

View File

@@ -19,8 +19,6 @@ import (
"github.com/fsnotify/fsnotify"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
"tailscale.com/kube/kubetypes"
"tailscale.com/types/netmap"
)
// watchServeConfigChanges watches path for changes, and when it sees one, reads
@@ -28,21 +26,21 @@ import (
// applies it to lc. It exits when ctx is canceled. cdChanged is a channel that
// is written to when the certDomain changes, causing the serve config to be
// re-read and applied.
func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient, kc *kubeClient) {
func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan bool, certDomainAtomic *atomic.Pointer[string], lc *tailscale.LocalClient) {
if certDomainAtomic == nil {
panic("certDomainAtomic must not be nil")
panic("cd must not be nil")
}
var tickChan <-chan time.Time
var eventChan <-chan fsnotify.Event
if w, err := fsnotify.NewWatcher(); err != nil {
log.Printf("serve proxy: failed to create fsnotify watcher, timer-only mode: %v", err)
log.Printf("failed to create fsnotify watcher, timer-only mode: %v", err)
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
tickChan = ticker.C
} else {
defer w.Close()
if err := w.Add(filepath.Dir(path)); err != nil {
log.Fatalf("serve proxy: failed to add fsnotify watch: %v", err)
log.Fatalf("failed to add fsnotify watch: %v", err)
}
eventChan = w.Events
}
@@ -61,72 +59,24 @@ func watchServeConfigChanges(ctx context.Context, path string, cdChanged <-chan
// k8s handles these mounts. So just re-read the file and apply it
// if it's changed.
}
if certDomain == "" {
continue
}
sc, err := readServeConfig(path, certDomain)
if err != nil {
log.Fatalf("serve proxy: failed to read serve config: %v", err)
}
if sc == nil {
log.Printf("serve proxy: no serve config at %q, skipping", path)
continue
log.Fatalf("failed to read serve config: %v", err)
}
if prevServeConfig != nil && reflect.DeepEqual(sc, prevServeConfig) {
continue
}
if err := updateServeConfig(ctx, sc, certDomain, lc); err != nil {
log.Fatalf("serve proxy: error updating serve config: %v", err)
}
if kc != nil && kc.canPatch {
if err := kc.storeHTTPSEndpoint(ctx, certDomain); err != nil {
log.Fatalf("serve proxy: error storing HTTPS endpoint: %v", err)
}
log.Printf("Applying serve config")
if err := lc.SetServeConfig(ctx, sc); err != nil {
log.Fatalf("failed to set serve config: %v", err)
}
prevServeConfig = sc
}
}
func certDomainFromNetmap(nm *netmap.NetworkMap) string {
if len(nm.DNS.CertDomains) == 0 {
return ""
}
return nm.DNS.CertDomains[0]
}
// localClient is a subset of tailscale.LocalClient that can be mocked for testing.
type localClient interface {
SetServeConfig(context.Context, *ipn.ServeConfig) error
}
func updateServeConfig(ctx context.Context, sc *ipn.ServeConfig, certDomain string, lc localClient) error {
if !isValidHTTPSConfig(certDomain, sc) {
return nil
}
log.Printf("serve proxy: applying serve config")
return lc.SetServeConfig(ctx, sc)
}
func isValidHTTPSConfig(certDomain string, sc *ipn.ServeConfig) bool {
if certDomain == kubetypes.ValueNoHTTPS && hasHTTPSEndpoint(sc) {
log.Printf(
`serve proxy: this node is configured as a proxy that exposes an HTTPS endpoint to tailnet,
(perhaps a Kubernetes operator Ingress proxy) but it is not able to issue TLS certs, so this will likely not work.
To make it work, ensure that HTTPS is enabled for your tailnet, see https://tailscale.com/kb/1153/enabling-https for more details.`)
return false
}
return true
}
func hasHTTPSEndpoint(cfg *ipn.ServeConfig) bool {
if cfg == nil {
return false
}
for _, tcpCfg := range cfg.TCP {
if tcpCfg.HTTPS {
return true
}
}
return false
}
// readServeConfig reads the ipn.ServeConfig from path, replacing
// ${TS_CERT_DOMAIN} with certDomain.
func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
@@ -135,17 +85,8 @@ func readServeConfig(path, certDomain string) (*ipn.ServeConfig, error) {
}
j, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
// Serve config can be provided by users as well as the Kubernetes Operator (for its proxies). User-provided
// config could be empty for reasons.
if len(j) == 0 {
log.Printf("serve proxy: serve config file is empty, skipping")
return nil, nil
}
j = bytes.ReplaceAll(j, []byte("${TS_CERT_DOMAIN}"), []byte(certDomain))
var sc ipn.ServeConfig
if err := json.Unmarshal(j, &sc); err != nil {

View File

@@ -1,267 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux
package main
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
"tailscale.com/kube/kubetypes"
)
func TestUpdateServeConfig(t *testing.T) {
tests := []struct {
name string
sc *ipn.ServeConfig
certDomain string
wantCall bool
}{
{
name: "no_https_no_cert_domain",
sc: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
80: {HTTP: true},
},
},
certDomain: kubetypes.ValueNoHTTPS, // tailnet has HTTPS disabled
wantCall: true, // should set serve config as it doesn't have HTTPS endpoints
},
{
name: "https_with_cert_domain",
sc: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
443: {HTTPS: true},
},
Web: map[ipn.HostPort]*ipn.WebServerConfig{
"${TS_CERT_DOMAIN}:443": {
Handlers: map[string]*ipn.HTTPHandler{
"/": {Proxy: "http://10.0.1.100:8080"},
},
},
},
},
certDomain: "test-node.tailnet.ts.net",
wantCall: true,
},
{
name: "https_without_cert_domain",
sc: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
443: {HTTPS: true},
},
},
certDomain: kubetypes.ValueNoHTTPS,
wantCall: false, // incorrect configuration- should not set serve config
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fakeLC := &fakeLocalClient{}
err := updateServeConfig(context.Background(), tt.sc, tt.certDomain, fakeLC)
if err != nil {
t.Errorf("updateServeConfig() error = %v", err)
}
if fakeLC.setServeCalled != tt.wantCall {
t.Errorf("SetServeConfig() called = %v, want %v", fakeLC.setServeCalled, tt.wantCall)
}
})
}
}
func TestReadServeConfig(t *testing.T) {
tests := []struct {
name string
gotSC string
certDomain string
wantSC *ipn.ServeConfig
wantErr bool
}{
{
name: "empty_file",
},
{
name: "valid_config_with_cert_domain_placeholder",
gotSC: `{
"TCP": {
"443": {
"HTTPS": true
}
},
"Web": {
"${TS_CERT_DOMAIN}:443": {
"Handlers": {
"/api": {
"Proxy": "https://10.2.3.4/api"
}}}}}`,
certDomain: "example.com",
wantSC: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
443: {
HTTPS: true,
},
},
Web: map[ipn.HostPort]*ipn.WebServerConfig{
ipn.HostPort("example.com:443"): {
Handlers: map[string]*ipn.HTTPHandler{
"/api": {
Proxy: "https://10.2.3.4/api",
},
},
},
},
},
},
{
name: "valid_config_for_http_proxy",
gotSC: `{
"TCP": {
"80": {
"HTTP": true
}
}}`,
wantSC: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
80: {
HTTP: true,
},
},
},
},
{
name: "config_without_cert_domain",
gotSC: `{
"TCP": {
"443": {
"HTTPS": true
}
},
"Web": {
"localhost:443": {
"Handlers": {
"/api": {
"Proxy": "https://10.2.3.4/api"
}}}}}`,
certDomain: "",
wantErr: false,
wantSC: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
443: {
HTTPS: true,
},
},
Web: map[ipn.HostPort]*ipn.WebServerConfig{
ipn.HostPort("localhost:443"): {
Handlers: map[string]*ipn.HTTPHandler{
"/api": {
Proxy: "https://10.2.3.4/api",
},
},
},
},
},
},
{
name: "invalid_json",
gotSC: "invalid json",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "serve-config.json")
if err := os.WriteFile(path, []byte(tt.gotSC), 0644); err != nil {
t.Fatal(err)
}
got, err := readServeConfig(path, tt.certDomain)
if (err != nil) != tt.wantErr {
t.Errorf("readServeConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !cmp.Equal(got, tt.wantSC) {
t.Errorf("readServeConfig() diff (-got +want):\n%s", cmp.Diff(got, tt.wantSC))
}
})
}
}
type fakeLocalClient struct {
*tailscale.LocalClient
setServeCalled bool
}
func (m *fakeLocalClient) SetServeConfig(ctx context.Context, cfg *ipn.ServeConfig) error {
m.setServeCalled = true
return nil
}
func TestHasHTTPSEndpoint(t *testing.T) {
tests := []struct {
name string
cfg *ipn.ServeConfig
want bool
}{
{
name: "nil_config",
cfg: nil,
want: false,
},
{
name: "empty_config",
cfg: &ipn.ServeConfig{},
want: false,
},
{
name: "no_https_endpoints",
cfg: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
80: {
HTTPS: false,
},
},
},
want: false,
},
{
name: "has_https_endpoint",
cfg: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
443: {
HTTPS: true,
},
},
},
want: true,
},
{
name: "mixed_endpoints",
cfg: &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
80: {HTTPS: false},
443: {HTTPS: true},
},
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := hasHTTPSEndpoint(tt.cfg)
if got != tt.want {
t.Errorf("hasHTTPSEndpoint() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -67,12 +67,7 @@ type settings struct {
PodIP string
PodIPv4 string
PodIPv6 string
PodUID string
HealthCheckAddrPort string
LocalAddrPort string
MetricsEnabled bool
HealthCheckEnabled bool
DebugAddrPort string
EgressSvcsCfgPath string
}
@@ -103,12 +98,7 @@ func configFromEnv() (*settings, error) {
PodIP: defaultEnv("POD_IP", ""),
EnableForwardingOptimizations: defaultBool("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS", false),
HealthCheckAddrPort: defaultEnv("TS_HEALTHCHECK_ADDR_PORT", ""),
LocalAddrPort: defaultEnv("TS_LOCAL_ADDR_PORT", "[::]:9002"),
MetricsEnabled: defaultBool("TS_ENABLE_METRICS", false),
HealthCheckEnabled: defaultBool("TS_ENABLE_HEALTH_CHECK", false),
DebugAddrPort: defaultEnv("TS_DEBUG_ADDR_PORT", ""),
EgressSvcsCfgPath: defaultEnv("TS_EGRESS_SERVICES_CONFIG_PATH", ""),
PodUID: defaultEnv("POD_UID", ""),
}
podIPs, ok := os.LookupEnv("POD_IPS")
if ok {
@@ -181,34 +171,17 @@ func (s *settings) validate() error {
return errors.New("TS_EXPERIMENTAL_ENABLE_FORWARDING_OPTIMIZATIONS is not supported in userspace mode")
}
if s.HealthCheckAddrPort != "" {
log.Printf("[warning] TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0. Please use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT instead.")
if _, err := netip.ParseAddrPort(s.HealthCheckAddrPort); err != nil {
return fmt.Errorf("error parsing TS_HEALTHCHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
return fmt.Errorf("error parsing TS_HEALTH_CHECK_ADDR_PORT value %q: %w", s.HealthCheckAddrPort, err)
}
}
if s.localMetricsEnabled() || s.localHealthEnabled() {
if _, err := netip.ParseAddrPort(s.LocalAddrPort); err != nil {
return fmt.Errorf("error parsing TS_LOCAL_ADDR_PORT value %q: %w", s.LocalAddrPort, err)
}
}
if s.DebugAddrPort != "" {
if _, err := netip.ParseAddrPort(s.DebugAddrPort); err != nil {
return fmt.Errorf("error parsing TS_DEBUG_ADDR_PORT value %q: %w", s.DebugAddrPort, err)
}
}
if s.HealthCheckEnabled && s.HealthCheckAddrPort != "" {
return errors.New("TS_HEALTHCHECK_ADDR_PORT is deprecated and will be removed in 1.82.0, use TS_ENABLE_HEALTH_CHECK and optionally TS_LOCAL_ADDR_PORT")
}
if s.EgressSvcsCfgPath != "" && !(s.InKubernetes && s.KubeSecret != "") {
return errors.New("TS_EGRESS_SERVICES_CONFIG_PATH is only supported for Tailscale running on Kubernetes")
}
return nil
}
// setupKube is responsible for doing any necessary configuration and checks to
// ensure that tailscale state storage and authentication mechanism will work on
// Kubernetes.
func (cfg *settings) setupKube(ctx context.Context, kc *kubeClient) error {
func (cfg *settings) setupKube(ctx context.Context) error {
if cfg.KubeSecret == "" {
return nil
}
@@ -217,7 +190,6 @@ func (cfg *settings) setupKube(ctx context.Context, kc *kubeClient) error {
return fmt.Errorf("some Kubernetes permissions are missing, please check your RBAC configuration: %v", err)
}
cfg.KubernetesCanPatch = canPatch
kc.canPatch = canPatch
s, err := kc.GetSecret(ctx, cfg.KubeSecret)
if err != nil {
@@ -300,14 +272,6 @@ func hasKubeStateStore(cfg *settings) bool {
return cfg.InKubernetes && cfg.KubernetesCanPatch && cfg.KubeSecret != ""
}
func (cfg *settings) localMetricsEnabled() bool {
return cfg.LocalAddrPort != "" && cfg.MetricsEnabled
}
func (cfg *settings) localHealthEnabled() bool {
return cfg.LocalAddrPort != "" && cfg.HealthCheckEnabled
}
// defaultEnv returns the value of the given envvar name, or defVal if
// unset.
func defaultEnv(name, defVal string) string {

View File

@@ -13,13 +13,10 @@ import (
"log"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"syscall"
"time"
"github.com/fsnotify/fsnotify"
"tailscale.com/client/tailscale"
)
@@ -93,12 +90,6 @@ func tailscaledArgs(cfg *settings) []string {
if cfg.TailscaledConfigFilePath != "" {
args = append(args, "--config="+cfg.TailscaledConfigFilePath)
}
// Once enough proxy versions have been released for all the supported
// versions to understand this cfg setting, the operator can stop
// setting TS_TAILSCALED_EXTRA_ARGS for the debug flag.
if cfg.DebugAddrPort != "" && !strings.Contains(cfg.DaemonExtraArgs, cfg.DebugAddrPort) {
args = append(args, "--debug="+cfg.DebugAddrPort)
}
if cfg.DaemonExtraArgs != "" {
args = append(args, strings.Fields(cfg.DaemonExtraArgs)...)
}
@@ -169,70 +160,3 @@ func tailscaleSet(ctx context.Context, cfg *settings) error {
}
return nil
}
func watchTailscaledConfigChanges(ctx context.Context, path string, lc *tailscale.LocalClient, errCh chan<- error) {
var (
tickChan <-chan time.Time
tailscaledCfgDir = filepath.Dir(path)
prevTailscaledCfg []byte
)
w, err := fsnotify.NewWatcher()
if err != nil {
log.Printf("tailscaled config watch: failed to create fsnotify watcher, timer-only mode: %v", err)
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
tickChan = ticker.C
} else {
defer w.Close()
if err := w.Add(tailscaledCfgDir); err != nil {
errCh <- fmt.Errorf("failed to add fsnotify watch: %w", err)
return
}
}
b, err := os.ReadFile(path)
if err != nil {
errCh <- fmt.Errorf("error reading configfile: %w", err)
return
}
prevTailscaledCfg = b
// kubelet mounts Secrets to Pods using a series of symlinks, one of
// which is <mount-dir>/..data that Kubernetes recommends consumers to
// use if they need to monitor changes
// https://github.com/kubernetes/kubernetes/blob/v1.28.1/pkg/volume/util/atomic_writer.go#L39-L61
const kubeletMountedCfg = "..data"
toWatch := filepath.Join(tailscaledCfgDir, kubeletMountedCfg)
for {
select {
case <-ctx.Done():
return
case err := <-w.Errors:
errCh <- fmt.Errorf("watcher error: %w", err)
return
case <-tickChan:
case event := <-w.Events:
if event.Name != toWatch {
continue
}
}
b, err := os.ReadFile(path)
if err != nil {
errCh <- fmt.Errorf("error reading configfile: %w", err)
return
}
// For some proxy types the mounted volume also contains tailscaled state and other files. We
// don't want to reload config unnecessarily on unrelated changes to these files.
if reflect.DeepEqual(b, prevTailscaledCfg) {
continue
}
prevTailscaledCfg = b
log.Printf("tailscaled config watch: ensuring that config is up to date")
ok, err := lc.ReloadConfig(ctx)
if err != nil {
errCh <- fmt.Errorf("error reloading tailscaled config: %w", err)
return
}
if ok {
log.Printf("tailscaled config watch: config was reloaded")
}
}
}

View File

@@ -20,10 +20,10 @@ import (
)
func BenchmarkHandleBootstrapDNS(b *testing.B) {
tstest.Replace(b, bootstrapDNS, "log.tailscale.com,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
tstest.Replace(b, bootstrapDNS, "log.tailscale.io,login.tailscale.com,controlplane.tailscale.com,login.us.tailscale.com")
refreshBootstrapDNS()
w := new(bitbucketResponseWriter)
req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.com"), nil)
req, _ := http.NewRequest("GET", "https://localhost/bootstrap-dns?q="+url.QueryEscape("log.tailscale.io"), nil)
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(b *testing.PB) {
@@ -63,7 +63,7 @@ func TestUnpublishedDNS(t *testing.T) {
nettest.SkipIfNoNetwork(t)
const published = "login.tailscale.com"
const unpublished = "log.tailscale.com"
const unpublished = "log.tailscale.io"
prev1, prev2 := *bootstrapDNS, *unpublishedDNS
*bootstrapDNS = published
@@ -119,18 +119,18 @@ func TestUnpublishedDNSEmptyList(t *testing.T) {
unpublishedDNSCache.Store(&dnsEntryMap{
IPs: map[string][]net.IP{
"log.tailscale.com": {},
"log.tailscale.io": {},
"controlplane.tailscale.com": {net.IPv4(1, 2, 3, 4)},
},
Percent: map[string]float64{
"log.tailscale.com": 1.0,
"log.tailscale.io": 1.0,
"controlplane.tailscale.com": 1.0,
},
})
t.Run("CacheMiss", func(t *testing.T) {
// One domain in map but empty, one not in map at all
for _, q := range []string{"log.tailscale.com", "login.tailscale.com"} {
for _, q := range []string{"log.tailscale.io", "login.tailscale.com"} {
resetMetrics()
ips := getBootstrapDNS(t, q)

View File

@@ -8,7 +8,6 @@ import (
"crypto/x509"
"errors"
"fmt"
"net"
"net/http"
"path/filepath"
"regexp"
@@ -54,9 +53,8 @@ func certProviderByCertMode(mode, dir, hostname string) (certProvider, error) {
}
type manualCertManager struct {
cert *tls.Certificate
hostname string // hostname or IP address of server
noHostname bool // whether hostname is an IP address
cert *tls.Certificate
hostname string
}
// NewManualCertManager returns a cert provider which read certificate by given hostname on create.
@@ -76,11 +74,7 @@ func NewManualCertManager(certdir, hostname string) (certProvider, error) {
if err := x509Cert.VerifyHostname(hostname); err != nil {
return nil, fmt.Errorf("cert invalid for hostname %q: %w", hostname, err)
}
return &manualCertManager{
cert: &cert,
hostname: hostname,
noHostname: net.ParseIP(hostname) != nil,
}, nil
return &manualCertManager{cert: &cert, hostname: hostname}, nil
}
func (m *manualCertManager) TLSConfig() *tls.Config {
@@ -94,7 +88,7 @@ func (m *manualCertManager) TLSConfig() *tls.Config {
}
func (m *manualCertManager) getCertificate(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
if hi.ServerName != m.hostname && !m.noHostname {
if hi.ServerName != m.hostname {
return nil, fmt.Errorf("cert mismatch with hostname: %q", hi.ServerName)
}

View File

@@ -1,97 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package main
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net"
"os"
"path/filepath"
"testing"
"time"
)
// Verify that in --certmode=manual mode, we can use a bare IP address
// as the --hostname and that GetCertificate will return it.
func TestCertIP(t *testing.T) {
dir := t.TempDir()
const hostname = "1.2.3.4"
priv, err := ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
if err != nil {
t.Fatal(err)
}
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
t.Fatal(err)
}
ip := net.ParseIP(hostname)
if ip == nil {
t.Fatalf("invalid IP address %q", hostname)
}
template := &x509.Certificate{
SerialNumber: serialNumber,
Subject: pkix.Name{
Organization: []string{"Tailscale Test Corp"},
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(30 * 24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
IPAddresses: []net.IP{ip},
}
derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
if err != nil {
t.Fatal(err)
}
certOut, err := os.Create(filepath.Join(dir, hostname+".crt"))
if err != nil {
t.Fatal(err)
}
if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
t.Fatalf("Failed to write data to cert.pem: %v", err)
}
if err := certOut.Close(); err != nil {
t.Fatalf("Error closing cert.pem: %v", err)
}
keyOut, err := os.OpenFile(filepath.Join(dir, hostname+".key"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
t.Fatal(err)
}
privBytes, err := x509.MarshalPKCS8PrivateKey(priv)
if err != nil {
t.Fatalf("Unable to marshal private key: %v", err)
}
if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
t.Fatalf("Failed to write data to key.pem: %v", err)
}
if err := keyOut.Close(); err != nil {
t.Fatalf("Error closing key.pem: %v", err)
}
cp, err := certProviderByCertMode("manual", dir, hostname)
if err != nil {
t.Fatal(err)
}
back, err := cp.TLSConfig().GetCertificate(&tls.ClientHelloInfo{
ServerName: "", // no SNI
})
if err != nil {
t.Fatalf("GetCertificate: %v", err)
}
if back == nil {
t.Fatalf("GetCertificate returned nil")
}
}

View File

@@ -27,7 +27,9 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
L github.com/google/nftables/expr from github.com/google/nftables+
L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+
L github.com/google/nftables/xt from github.com/google/nftables/expr+
github.com/google/uuid from tailscale.com/util/fastuuid
github.com/hdevalence/ed25519consensus from tailscale.com/tka
L github.com/josharian/native from github.com/mdlayher/netlink+
L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon
L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink
L 💣 github.com/mdlayher/netlink from github.com/google/nftables+
@@ -35,11 +37,11 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
L github.com/mdlayher/netlink/nltest from github.com/google/nftables
L 💣 github.com/mdlayher/socket from github.com/mdlayher/netlink
💣 github.com/mitchellh/go-ps from tailscale.com/safesocket
github.com/munnerz/goautoneg from github.com/prometheus/common/expfmt
💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz
github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+
LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus
LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs
@@ -85,7 +87,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
google.golang.org/protobuf/runtime/protoimpl from github.com/prometheus/client_model/go+
google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+
tailscale.com from tailscale.com/version
💣 tailscale.com/atomicfile from tailscale.com/cmd/derper+
tailscale.com/atomicfile from tailscale.com/cmd/derper+
tailscale.com/client/tailscale from tailscale.com/derp
tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale
tailscale.com/derp from tailscale.com/cmd/derper+
@@ -99,7 +101,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
tailscale.com/ipn/ipnstate from tailscale.com/client/tailscale+
tailscale.com/kube/kubetypes from tailscale.com/envknob
tailscale.com/metrics from tailscale.com/cmd/derper+
tailscale.com/net/bakedroots from tailscale.com/net/tlsdial
tailscale.com/net/dnscache from tailscale.com/derp/derphttp
tailscale.com/net/ktimeout from tailscale.com/cmd/derper
tailscale.com/net/netaddr from tailscale.com/ipn+
@@ -151,6 +152,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
💣 tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting
L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics
tailscale.com/util/dnsname from tailscale.com/hostinfo+
tailscale.com/util/fastuuid from tailscale.com/tsweb
💣 tailscale.com/util/hashx from tailscale.com/util/deephash
tailscale.com/util/httpm from tailscale.com/client/tailscale
tailscale.com/util/lineiter from tailscale.com/hostinfo+
@@ -158,7 +160,6 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
tailscale.com/util/mak from tailscale.com/health+
tailscale.com/util/multierr from tailscale.com/health+
tailscale.com/util/nocasemaps from tailscale.com/types/ipproto
tailscale.com/util/rands from tailscale.com/tsweb
tailscale.com/util/set from tailscale.com/derp+
tailscale.com/util/singleflight from tailscale.com/net/dnscache
tailscale.com/util/slicesx from tailscale.com/cmd/derper+
@@ -204,7 +205,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
golang.org/x/net/proxy from tailscale.com/net/netns
D golang.org/x/net/route from net+
golang.org/x/sync/errgroup from github.com/mdlayher/socket+
golang.org/x/sys/cpu from golang.org/x/crypto/argon2+
golang.org/x/sys/cpu from github.com/josharian/native+
LD golang.org/x/sys/unix from github.com/google/nftables+
W golang.org/x/sys/windows from github.com/dblohm7/wingoes+
W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+
@@ -243,6 +244,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
crypto/tls from golang.org/x/crypto/acme+
crypto/x509 from crypto/tls+
crypto/x509/pkix from crypto/x509+
database/sql/driver from github.com/google/uuid
embed from crypto/internal/nistec+
encoding from encoding/json+
encoding/asn1 from crypto/x509+
@@ -265,7 +267,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
html/template from tailscale.com/cmd/derper
io from bufio+
io/fs from crypto/x509+
L io/ioutil from github.com/mitchellh/go-ps+
io/ioutil from github.com/mitchellh/go-ps+
iter from maps+
log from expvar+
log/internal from log
@@ -274,7 +276,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
math/big from crypto/dsa+
math/bits from compress/flate+
math/rand from github.com/mdlayher/netlink+
math/rand/v2 from internal/concurrent+
math/rand/v2 from tailscale.com/util/fastuuid+
mime from github.com/prometheus/common/expfmt+
mime/multipart from net/http
mime/quotedprintable from mime/multipart

View File

@@ -58,12 +58,12 @@ var (
configPath = flag.String("c", "", "config file path")
certMode = flag.String("certmode", "letsencrypt", "mode for getting a cert. possible options: manual, letsencrypt")
certDir = flag.String("certdir", tsweb.DefaultCertDir("derper-certs"), "directory to store LetsEncrypt certs, if addr's port is :443")
hostname = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443. When --certmode=manual, this can be an IP address to avoid SNI checks")
hostname = flag.String("hostname", "derp.tailscale.com", "LetsEncrypt host name, if addr's port is :443")
runSTUN = flag.Bool("stun", true, "whether to run a STUN server. It will bind to the same IP (if any) as the --addr flag value.")
runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.")
meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.")
meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.")
meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list")
bootstrapDNS = flag.String("bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns")
unpublishedDNS = flag.String("unpublished-bootstrap-dns-names", "", "optional comma-separated list of hostnames to make available at /bootstrap-dns and not publish in the list. If an entry contains a slash, the second part names a DNS record to poll for its TXT record with a `0` to `100` value for rollout percentage.")
verifyClients = flag.Bool("verify-clients", false, "verify clients to this DERP server through a local tailscaled instance.")
@@ -77,8 +77,6 @@ var (
tcpKeepAlive = flag.Duration("tcp-keepalive-time", 10*time.Minute, "TCP keepalive time")
// tcpUserTimeout is intentionally short, so that hung connections are cleaned up promptly. DERPs should be nearby users.
tcpUserTimeout = flag.Duration("tcp-user-timeout", 15*time.Second, "TCP user timeout")
// tcpWriteTimeout is the timeout for writing to client TCP connections. It does not apply to mesh connections.
tcpWriteTimeout = flag.Duration("tcp-write-timeout", derp.DefaultTCPWiteTimeout, "TCP write timeout; 0 results in no timeout being set on writes")
)
var (
@@ -175,7 +173,6 @@ func main() {
s.SetVerifyClient(*verifyClients)
s.SetVerifyClientURL(*verifyClientURL)
s.SetVerifyClientURLFailOpen(*verifyFailOpen)
s.SetTCPWriteTimeout(*tcpWriteTimeout)
if *meshPSKFile != "" {
b, err := os.ReadFile(*meshPSKFile)

View File

@@ -6,6 +6,7 @@ package main
import (
"bytes"
"context"
"fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -108,7 +109,6 @@ func TestDeps(t *testing.T) {
"gvisor.dev/gvisor/pkg/tcpip/header": "https://github.com/tailscale/tailscale/issues/9756",
"tailscale.com/net/packet": "not needed in derper",
"github.com/gaissmai/bart": "not needed in derper",
"database/sql/driver": "not needed in derper", // previously came in via github.com/google/uuid
},
}.Check(t)
}
@@ -137,4 +137,5 @@ func TestTemplate(t *testing.T) {
if !strings.Contains(str, "Debug info") {
t.Error("Output is missing debug info")
}
fmt.Println(buf.String())
}

View File

@@ -10,6 +10,7 @@ import (
"log"
"net"
"strings"
"time"
"tailscale.com/derp"
"tailscale.com/derp/derphttp"
@@ -24,28 +25,15 @@ func startMesh(s *derp.Server) error {
if !s.HasMeshKey() {
return errors.New("--mesh-with requires --mesh-psk-file")
}
for _, hostTuple := range strings.Split(*meshWith, ",") {
if err := startMeshWithHost(s, hostTuple); err != nil {
for _, host := range strings.Split(*meshWith, ",") {
if err := startMeshWithHost(s, host); err != nil {
return err
}
}
return nil
}
func startMeshWithHost(s *derp.Server, hostTuple string) error {
var host string
var dialHost string
hostParts := strings.Split(hostTuple, "/")
if len(hostParts) > 2 {
return fmt.Errorf("too many components in host tuple %q", hostTuple)
}
host = hostParts[0]
if len(hostParts) == 2 {
dialHost = hostParts[1]
} else {
dialHost = hostParts[0]
}
func startMeshWithHost(s *derp.Server, host string) error {
logf := logger.WithPrefix(log.Printf, fmt.Sprintf("mesh(%q): ", host))
netMon := netmon.NewStatic() // good enough for cmd/derper; no need for netns fanciness
c, err := derphttp.NewClient(s.PrivateKey(), "https://"+host+"/derp", logf, netMon)
@@ -55,20 +43,31 @@ func startMeshWithHost(s *derp.Server, hostTuple string) error {
c.MeshKey = s.MeshKey()
c.WatchConnectionChanges = true
logf("will dial %q for %q", dialHost, host)
if dialHost != host {
// For meshed peers within a region, connect via VPC addresses.
c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
var d net.Dialer
c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
_, port, err := net.SplitHostPort(addr)
if err != nil {
logf("failed to split %q: %v", addr, err)
return nil, err
var r net.Resolver
if base, ok := strings.CutSuffix(host, ".tailscale.com"); ok && port == "443" {
subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
vpcHost := base + "-vpc.tailscale.com"
ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
if len(ips) > 0 {
vpcAddr := net.JoinHostPort(ips[0].String(), port)
c, err := d.DialContext(subCtx, network, vpcAddr)
if err == nil {
log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
return c, nil
}
log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
}
dialAddr := net.JoinHostPort(dialHost, port)
logf("dialing %q instead of %q", dialAddr, addr)
return d.DialContext(ctx, network, dialAddr)
})
}
}
return d.DialContext(ctx, network, addr)
})
add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) }
remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) }

View File

@@ -18,21 +18,18 @@ import (
)
var (
derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
versionFlag = flag.Bool("version", false, "print version and exit")
listen = flag.String("listen", ":8030", "HTTP listen address")
probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
spread = flag.Bool("spread", true, "whether to spread probing over time")
interval = flag.Duration("interval", 15*time.Second, "probe interval")
meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
bwTUNIPv4Address = flag.String("bw-tun-ipv4-addr", "", "if specified, bandwidth probes will be performed over a TUN device at this address in order to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP; we will use a /30 subnet including this IP address")
qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second")
qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings")
regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed")
derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map")
versionFlag = flag.Bool("version", false, "print version and exit")
listen = flag.String("listen", ":8030", "HTTP listen address")
probeOnce = flag.Bool("once", false, "probe once and print results, then exit; ignores the listen flag")
spread = flag.Bool("spread", true, "whether to spread probing over time")
interval = flag.Duration("interval", 15*time.Second, "probe interval")
meshInterval = flag.Duration("mesh-interval", 15*time.Second, "mesh probe interval")
stunInterval = flag.Duration("stun-interval", 15*time.Second, "STUN probe interval")
tlsInterval = flag.Duration("tls-interval", 15*time.Second, "TLS probe interval")
bwInterval = flag.Duration("bw-interval", 0, "bandwidth probe interval (0 = no bandwidth probing)")
bwSize = flag.Int64("bw-probe-size-bytes", 1_000_000, "bandwidth probe size")
regionCode = flag.String("region-code", "", "probe only this region (e.g. 'lax'); if left blank, all regions will be probed")
)
func main() {
@@ -47,13 +44,12 @@ func main() {
prober.WithMeshProbing(*meshInterval),
prober.WithSTUNProbing(*stunInterval),
prober.WithTLSProbing(*tlsInterval),
prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout),
}
if *bwInterval > 0 {
opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address))
opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize))
}
if *regionCodeOrID != "" {
opts = append(opts, prober.WithRegionCodeOrID(*regionCodeOrID))
if *regionCode != "" {
opts = append(opts, prober.WithRegion(*regionCode))
}
dp, err := prober.DERP(p, *derpMapURL, opts...)
if err != nil {
@@ -110,7 +106,7 @@ func getOverallStatus(p *prober.Prober) (o overallStatus) {
// Do not show probes that have not finished yet.
continue
}
if i.Status == prober.ProbeStatusSucceeded {
if i.Result {
o.addGoodf("%s: %s", p, i.Latency)
} else {
o.addBadf("%s: %s", p, i.Error)

View File

@@ -46,6 +46,7 @@ func main() {
ClientID: clientID,
ClientSecret: clientSecret,
TokenURL: baseURL + "/api/v2/oauth/token",
Scopes: []string{"device"},
}
ctx := context.Background()

View File

@@ -58,8 +58,8 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
}
if cache.PrevETag == "" {
log.Println("no previous etag found, assuming the latest control etag")
cache.PrevETag = controlEtag
log.Println("no previous etag found, assuming local file is correct and recording that")
cache.PrevETag = localEtag
}
log.Printf("control: %s", controlEtag)
@@ -105,8 +105,8 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
}
if cache.PrevETag == "" {
log.Println("no previous etag found, assuming the latest control etag")
cache.PrevETag = controlEtag
log.Println("no previous etag found, assuming local file is correct and recording that")
cache.PrevETag = localEtag
}
log.Printf("control: %s", controlEtag)
@@ -148,8 +148,8 @@ func getChecksums(cache *Cache, client *http.Client, tailnet, apiKey string) fun
}
if cache.PrevETag == "" {
log.Println("no previous etag found, assuming control etag")
cache.PrevETag = Shuck(controlEtag)
log.Println("no previous etag found, assuming local file is correct and recording that")
cache.PrevETag = Shuck(localEtag)
}
log.Printf("control: %s", controlEtag)

View File

@@ -10,7 +10,6 @@ import (
"fmt"
"net/netip"
"slices"
"strings"
"sync"
"time"
@@ -36,7 +35,6 @@ import (
const (
reasonConnectorCreationFailed = "ConnectorCreationFailed"
reasonConnectorCreating = "ConnectorCreating"
reasonConnectorCreated = "ConnectorCreated"
reasonConnectorInvalid = "ConnectorInvalid"
@@ -115,7 +113,7 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
setStatus := func(cn *tsapi.Connector, _ tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetConnectorCondition(cn, tsapi.ConnectorReady, status, reason, message, cn.Generation, a.clock, logger)
var updateErr error
if !apiequality.Semantic.DeepEqual(oldCnStatus, &cn.Status) {
if !apiequality.Semantic.DeepEqual(oldCnStatus, cn.Status) {
// An error encountered here should get returned by the Reconcile function.
updateErr = a.Client.Status().Update(ctx, cn)
}
@@ -136,24 +134,17 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
}
if err := a.validate(cn); err != nil {
logger.Errorf("error validating Connector spec: %w", err)
message := fmt.Sprintf(messageConnectorInvalid, err)
a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorInvalid, message)
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reasonConnectorInvalid, message)
}
if err = a.maybeProvisionConnector(ctx, logger, cn); err != nil {
reason := reasonConnectorCreationFailed
logger.Errorf("error creating Connector resources: %w", err)
message := fmt.Sprintf(messageConnectorCreationFailed, err)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
reason = reasonConnectorCreating
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
err = nil
logger.Info(message)
} else {
a.recorder.Eventf(cn, corev1.EventTypeWarning, reason, message)
}
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reason, message)
a.recorder.Eventf(cn, corev1.EventTypeWarning, reasonConnectorCreationFailed, message)
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionFalse, reasonConnectorCreationFailed, message)
}
logger.Info("Connector resources synced")
@@ -198,7 +189,6 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
isExitNode: cn.Spec.ExitNode,
},
ProxyClassName: proxyClass,
proxyType: proxyTypeConnector,
}
if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
@@ -243,27 +233,27 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
return err
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
_, tsHost, ips, err := a.ssr.DeviceInfo(ctx, crl)
if err != nil {
return err
}
if dev == nil || dev.hostname == "" {
logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth")
if tsHost == "" {
logger.Debugf("no Tailscale hostname known yet, waiting for connector pod to finish auth")
// No hostname yet. Wait for the connector pod to auth.
cn.Status.TailnetIPs = nil
cn.Status.Hostname = ""
return nil
}
cn.Status.TailnetIPs = dev.ips
cn.Status.Hostname = dev.hostname
cn.Status.TailnetIPs = ips
cn.Status.Hostname = tsHost
return nil
}
func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil {
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector")); err != nil {
return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
} else if !done {
logger.Debugf("Connector cleanup not done yet, waiting for next reconcile")

View File

@@ -79,8 +79,8 @@ func TestConnector(t *testing.T) {
subnetRoutes: "10.40.0.0/14",
app: kubetypes.AppConnector,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Connector status should get updated with the IP/hostname info when available.
const hostname = "foo.tailnetxyz.ts.net"
@@ -106,7 +106,7 @@ func TestConnector(t *testing.T) {
opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Remove a route.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -114,7 +114,7 @@ func TestConnector(t *testing.T) {
})
opts.subnetRoutes = "10.44.0.0/20"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Remove the subnet router.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -122,7 +122,7 @@ func TestConnector(t *testing.T) {
})
opts.subnetRoutes = ""
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Re-add the subnet router.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -132,7 +132,7 @@ func TestConnector(t *testing.T) {
})
opts.subnetRoutes = "10.44.0.0/20"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Delete the Connector.
if err = fc.Delete(context.Background(), cn); err != nil {
@@ -175,8 +175,8 @@ func TestConnector(t *testing.T) {
hostname: "test-connector",
app: kubetypes.AppConnector,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Add an exit node.
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -184,7 +184,7 @@ func TestConnector(t *testing.T) {
})
opts.isExitNode = true
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Delete the Connector.
if err = fc.Delete(context.Background(), cn); err != nil {
@@ -203,7 +203,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar"},
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
@@ -261,8 +261,8 @@ func TestConnectorWithProxyClass(t *testing.T) {
subnetRoutes: "10.40.0.0/14",
app: kubetypes.AppConnector,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 2. Update Connector to specify a ProxyClass. ProxyClass is not yet
// ready, so its configuration is NOT applied to the Connector
@@ -271,7 +271,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
conn.Spec.ProxyClass = "custom-metadata"
})
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 3. ProxyClass is set to Ready by proxy-class reconciler. Connector
// get reconciled and configuration from the ProxyClass is applied to
@@ -286,7 +286,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
})
opts.proxyClass = pc.Name
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 4. Connector.spec.proxyClass field is unset, Connector gets
// reconciled and configuration from the ProxyClass is removed from the
@@ -296,7 +296,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
})
opts.proxyClass = ""
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
}
func TestConnectorWithAppConnector(t *testing.T) {
@@ -351,8 +351,8 @@ func TestConnectorWithAppConnector(t *testing.T) {
app: kubetypes.AppConnector,
isAppConnector: true,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// Connector's ready condition should be set to true
cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
@@ -364,7 +364,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
Reason: reasonConnectorCreated,
Message: reasonConnectorCreated,
}}
expectEqual(t, fc, cn)
expectEqual(t, fc, cn, nil)
// 2. Connector with invalid app connector routes has status set to invalid
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
@@ -379,7 +379,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
Reason: reasonConnectorInvalid,
Message: "Connector is invalid: route 1.2.3.4/5 has non-address bits set; expected 0.0.0.0/5",
}}
expectEqual(t, fc, cn)
expectEqual(t, fc, cn, nil)
// 3. Connector with valid app connnector routes becomes ready
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {

View File

@@ -94,7 +94,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/evanphx/json-patch/v5 from sigs.k8s.io/controller-runtime/pkg/client
github.com/evanphx/json-patch/v5/internal/json from github.com/evanphx/json-patch/v5
💣 github.com/fsnotify/fsnotify from sigs.k8s.io/controller-runtime/pkg/certwatcher
github.com/fxamacker/cbor/v2 from tailscale.com/tka+
github.com/fxamacker/cbor/v2 from tailscale.com/tka
github.com/gaissmai/bart from tailscale.com/net/ipset+
github.com/go-json-experiment/json from tailscale.com/types/opt+
github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json/internal/jsonflags+
@@ -110,11 +110,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/go-openapi/jsonpointer from github.com/go-openapi/jsonreference
github.com/go-openapi/jsonreference from k8s.io/kube-openapi/pkg/internal+
github.com/go-openapi/jsonreference/internal from github.com/go-openapi/jsonreference
💣 github.com/go-openapi/swag from github.com/go-openapi/jsonpointer+
github.com/go-openapi/swag from github.com/go-openapi/jsonpointer+
L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns
💣 github.com/gogo/protobuf/proto from k8s.io/api/admission/v1+
github.com/gogo/protobuf/sortkeys from k8s.io/api/admission/v1+
github.com/golang/groupcache/lru from tailscale.com/net/dnscache
github.com/golang/groupcache/lru from k8s.io/client-go/tools/record+
github.com/golang/protobuf/proto from k8s.io/client-go/discovery+
github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+
github.com/google/gnostic-models/compiler from github.com/google/gnostic-models/openapiv2+
@@ -140,12 +140,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/gorilla/securecookie from github.com/gorilla/csrf
github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+
L 💣 github.com/illarion/gonotify/v2 from tailscale.com/net/dns
L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/feature/tap
github.com/imdario/mergo from k8s.io/client-go/tools/clientcmd
L github.com/insomniacslk/dhcp/dhcpv4 from tailscale.com/net/tstun
L github.com/insomniacslk/dhcp/iana from github.com/insomniacslk/dhcp/dhcpv4
L github.com/insomniacslk/dhcp/interfaces from github.com/insomniacslk/dhcp/dhcpv4
L github.com/insomniacslk/dhcp/rfc1035label from github.com/insomniacslk/dhcp/dhcpv4
L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm
github.com/josharian/intern from github.com/mailru/easyjson/jlexer
L github.com/josharian/native from github.com/mdlayher/netlink+
L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon
L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink
💣 github.com/json-iterator/go from sigs.k8s.io/structured-merge-diff/v4/fieldpath+
@@ -156,7 +158,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
github.com/kortschak/wol from tailscale.com/feature/wakeonlan
github.com/kortschak/wol from tailscale.com/ipn/ipnlocal
github.com/mailru/easyjson/buffer from github.com/mailru/easyjson/jwriter
💣 github.com/mailru/easyjson/jlexer from github.com/go-openapi/swag
github.com/mailru/easyjson/jwriter from github.com/go-openapi/swag
@@ -170,7 +172,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
💣 github.com/mitchellh/go-ps from tailscale.com/safesocket
github.com/modern-go/concurrent from github.com/json-iterator/go
💣 github.com/modern-go/reflect2 from github.com/json-iterator/go
github.com/munnerz/goautoneg from k8s.io/kube-openapi/pkg/handler3+
github.com/munnerz/goautoneg from k8s.io/kube-openapi/pkg/handler3
github.com/opencontainers/go-digest from github.com/distribution/reference
L github.com/pierrec/lz4/v4 from github.com/u-root/uio/uio
L github.com/pierrec/lz4/v4/internal/lz4block from github.com/pierrec/lz4/v4+
@@ -185,6 +187,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/prometheus/client_golang/prometheus/promhttp from sigs.k8s.io/controller-runtime/pkg/metrics/server+
github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+
LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus
LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs
@@ -222,6 +225,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+
github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device
💣 github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+
github.com/tcnksm/go-httpstat from tailscale.com/net/netcheck
L github.com/u-root/uio/rand from github.com/insomniacslk/dhcp/dhcpv4
L github.com/u-root/uio/uio from github.com/insomniacslk/dhcp/dhcpv4+
L github.com/vishvananda/netns from github.com/tailscale/netlink+
@@ -248,7 +252,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
google.golang.org/protobuf/internal/descopts from google.golang.org/protobuf/internal/filedesc+
google.golang.org/protobuf/internal/detrand from google.golang.org/protobuf/internal/descfmt+
google.golang.org/protobuf/internal/editiondefaults from google.golang.org/protobuf/internal/filedesc+
google.golang.org/protobuf/internal/editionssupport from google.golang.org/protobuf/reflect/protodesc
google.golang.org/protobuf/internal/encoding/defval from google.golang.org/protobuf/internal/encoding/tag+
google.golang.org/protobuf/internal/encoding/messageset from google.golang.org/protobuf/encoding/prototext+
google.golang.org/protobuf/internal/encoding/tag from google.golang.org/protobuf/internal/impl
@@ -274,8 +277,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
google.golang.org/protobuf/types/gofeaturespb from google.golang.org/protobuf/reflect/protodesc
google.golang.org/protobuf/types/known/anypb from github.com/google/gnostic-models/compiler+
google.golang.org/protobuf/types/known/timestamppb from github.com/prometheus/client_golang/prometheus+
gopkg.in/evanphx/json-patch.v4 from k8s.io/client-go/testing
gopkg.in/inf.v0 from k8s.io/apimachinery/pkg/api/resource
gopkg.in/yaml.v2 from k8s.io/kube-openapi/pkg/util/proto+
gopkg.in/yaml.v3 from github.com/go-openapi/swag+
gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/buffer+
gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/buffer
@@ -302,7 +305,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
gvisor.dev/gvisor/pkg/tcpip/network/internal/ip from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+
gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/feature/tap+
gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/net/tstun+
gvisor.dev/gvisor/pkg/tcpip/network/ipv6 from tailscale.com/wgengine/netstack+
gvisor.dev/gvisor/pkg/tcpip/ports from gvisor.dev/gvisor/pkg/tcpip/stack+
gvisor.dev/gvisor/pkg/tcpip/seqnum from gvisor.dev/gvisor/pkg/tcpip/header+
@@ -344,7 +347,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/api/certificates/v1alpha1 from k8s.io/client-go/applyconfigurations/certificates/v1alpha1+
k8s.io/api/certificates/v1beta1 from k8s.io/client-go/applyconfigurations/certificates/v1beta1+
k8s.io/api/coordination/v1 from k8s.io/client-go/applyconfigurations/coordination/v1+
k8s.io/api/coordination/v1alpha2 from k8s.io/client-go/applyconfigurations/coordination/v1alpha2+
k8s.io/api/coordination/v1beta1 from k8s.io/client-go/applyconfigurations/coordination/v1beta1+
k8s.io/api/core/v1 from k8s.io/api/apps/v1+
k8s.io/api/discovery/v1 from k8s.io/client-go/applyconfigurations/discovery/v1+
@@ -367,8 +369,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/api/rbac/v1 from k8s.io/client-go/applyconfigurations/rbac/v1+
k8s.io/api/rbac/v1alpha1 from k8s.io/client-go/applyconfigurations/rbac/v1alpha1+
k8s.io/api/rbac/v1beta1 from k8s.io/client-go/applyconfigurations/rbac/v1beta1+
k8s.io/api/resource/v1alpha3 from k8s.io/client-go/applyconfigurations/resource/v1alpha3+
k8s.io/api/resource/v1beta1 from k8s.io/client-go/applyconfigurations/resource/v1beta1+
k8s.io/api/resource/v1alpha2 from k8s.io/client-go/applyconfigurations/resource/v1alpha2+
k8s.io/api/scheduling/v1 from k8s.io/client-go/applyconfigurations/scheduling/v1+
k8s.io/api/scheduling/v1alpha1 from k8s.io/client-go/applyconfigurations/scheduling/v1alpha1+
k8s.io/api/scheduling/v1beta1 from k8s.io/client-go/applyconfigurations/scheduling/v1beta1+
@@ -377,16 +378,14 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/api/storage/v1beta1 from k8s.io/client-go/applyconfigurations/storage/v1beta1+
k8s.io/api/storagemigration/v1alpha1 from k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1+
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 from sigs.k8s.io/controller-runtime/pkg/webhook/conversion+
💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 from sigs.k8s.io/controller-runtime/pkg/webhook/conversion
k8s.io/apimachinery/pkg/api/equality from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
k8s.io/apimachinery/pkg/api/errors from k8s.io/apimachinery/pkg/util/managedfields/internal+
k8s.io/apimachinery/pkg/api/meta from k8s.io/apimachinery/pkg/api/validation+
k8s.io/apimachinery/pkg/api/meta/testrestmapper from k8s.io/client-go/testing
k8s.io/apimachinery/pkg/api/resource from k8s.io/api/autoscaling/v1+
k8s.io/apimachinery/pkg/api/validation from k8s.io/apimachinery/pkg/util/managedfields/internal+
💣 k8s.io/apimachinery/pkg/apis/meta/internalversion from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme from k8s.io/client-go/metadata
k8s.io/apimachinery/pkg/apis/meta/internalversion/validation from k8s.io/client-go/util/watchlist
💣 k8s.io/apimachinery/pkg/apis/meta/v1 from k8s.io/api/admission/v1+
k8s.io/apimachinery/pkg/apis/meta/v1/unstructured from k8s.io/apimachinery/pkg/runtime/serializer/versioning+
k8s.io/apimachinery/pkg/apis/meta/v1/validation from k8s.io/apimachinery/pkg/api/validation+
@@ -398,9 +397,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/apimachinery/pkg/runtime from k8s.io/api/admission/v1+
k8s.io/apimachinery/pkg/runtime/schema from k8s.io/api/admission/v1+
k8s.io/apimachinery/pkg/runtime/serializer from k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme+
k8s.io/apimachinery/pkg/runtime/serializer/cbor from k8s.io/client-go/dynamic+
k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
k8s.io/apimachinery/pkg/runtime/serializer/cbor/internal/modes from k8s.io/apimachinery/pkg/runtime/serializer/cbor+
k8s.io/apimachinery/pkg/runtime/serializer/json from k8s.io/apimachinery/pkg/runtime/serializer+
k8s.io/apimachinery/pkg/runtime/serializer/protobuf from k8s.io/apimachinery/pkg/runtime/serializer
k8s.io/apimachinery/pkg/runtime/serializer/recognizer from k8s.io/apimachinery/pkg/runtime/serializer+
@@ -452,7 +448,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/applyconfigurations/certificates/v1alpha1 from k8s.io/client-go/kubernetes/typed/certificates/v1alpha1
k8s.io/client-go/applyconfigurations/certificates/v1beta1 from k8s.io/client-go/kubernetes/typed/certificates/v1beta1
k8s.io/client-go/applyconfigurations/coordination/v1 from k8s.io/client-go/kubernetes/typed/coordination/v1
k8s.io/client-go/applyconfigurations/coordination/v1alpha2 from k8s.io/client-go/kubernetes/typed/coordination/v1alpha2
k8s.io/client-go/applyconfigurations/coordination/v1beta1 from k8s.io/client-go/kubernetes/typed/coordination/v1beta1
k8s.io/client-go/applyconfigurations/core/v1 from k8s.io/client-go/applyconfigurations/apps/v1+
k8s.io/client-go/applyconfigurations/discovery/v1 from k8s.io/client-go/kubernetes/typed/discovery/v1
@@ -477,8 +472,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/applyconfigurations/rbac/v1 from k8s.io/client-go/kubernetes/typed/rbac/v1
k8s.io/client-go/applyconfigurations/rbac/v1alpha1 from k8s.io/client-go/kubernetes/typed/rbac/v1alpha1
k8s.io/client-go/applyconfigurations/rbac/v1beta1 from k8s.io/client-go/kubernetes/typed/rbac/v1beta1
k8s.io/client-go/applyconfigurations/resource/v1alpha3 from k8s.io/client-go/kubernetes/typed/resource/v1alpha3
k8s.io/client-go/applyconfigurations/resource/v1beta1 from k8s.io/client-go/kubernetes/typed/resource/v1beta1
k8s.io/client-go/applyconfigurations/resource/v1alpha2 from k8s.io/client-go/kubernetes/typed/resource/v1alpha2
k8s.io/client-go/applyconfigurations/scheduling/v1 from k8s.io/client-go/kubernetes/typed/scheduling/v1
k8s.io/client-go/applyconfigurations/scheduling/v1alpha1 from k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1
k8s.io/client-go/applyconfigurations/scheduling/v1beta1 from k8s.io/client-go/kubernetes/typed/scheduling/v1beta1
@@ -488,80 +482,8 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1
k8s.io/client-go/discovery from k8s.io/client-go/applyconfigurations/meta/v1+
k8s.io/client-go/dynamic from sigs.k8s.io/controller-runtime/pkg/cache/internal+
k8s.io/client-go/features from k8s.io/client-go/tools/cache+
k8s.io/client-go/gentype from k8s.io/client-go/kubernetes/typed/admissionregistration/v1+
k8s.io/client-go/informers from k8s.io/client-go/tools/leaderelection
k8s.io/client-go/informers/admissionregistration from k8s.io/client-go/informers
k8s.io/client-go/informers/admissionregistration/v1 from k8s.io/client-go/informers/admissionregistration
k8s.io/client-go/informers/admissionregistration/v1alpha1 from k8s.io/client-go/informers/admissionregistration
k8s.io/client-go/informers/admissionregistration/v1beta1 from k8s.io/client-go/informers/admissionregistration
k8s.io/client-go/informers/apiserverinternal from k8s.io/client-go/informers
k8s.io/client-go/informers/apiserverinternal/v1alpha1 from k8s.io/client-go/informers/apiserverinternal
k8s.io/client-go/informers/apps from k8s.io/client-go/informers
k8s.io/client-go/informers/apps/v1 from k8s.io/client-go/informers/apps
k8s.io/client-go/informers/apps/v1beta1 from k8s.io/client-go/informers/apps
k8s.io/client-go/informers/apps/v1beta2 from k8s.io/client-go/informers/apps
k8s.io/client-go/informers/autoscaling from k8s.io/client-go/informers
k8s.io/client-go/informers/autoscaling/v1 from k8s.io/client-go/informers/autoscaling
k8s.io/client-go/informers/autoscaling/v2 from k8s.io/client-go/informers/autoscaling
k8s.io/client-go/informers/autoscaling/v2beta1 from k8s.io/client-go/informers/autoscaling
k8s.io/client-go/informers/autoscaling/v2beta2 from k8s.io/client-go/informers/autoscaling
k8s.io/client-go/informers/batch from k8s.io/client-go/informers
k8s.io/client-go/informers/batch/v1 from k8s.io/client-go/informers/batch
k8s.io/client-go/informers/batch/v1beta1 from k8s.io/client-go/informers/batch
k8s.io/client-go/informers/certificates from k8s.io/client-go/informers
k8s.io/client-go/informers/certificates/v1 from k8s.io/client-go/informers/certificates
k8s.io/client-go/informers/certificates/v1alpha1 from k8s.io/client-go/informers/certificates
k8s.io/client-go/informers/certificates/v1beta1 from k8s.io/client-go/informers/certificates
k8s.io/client-go/informers/coordination from k8s.io/client-go/informers
k8s.io/client-go/informers/coordination/v1 from k8s.io/client-go/informers/coordination
k8s.io/client-go/informers/coordination/v1alpha2 from k8s.io/client-go/informers/coordination
k8s.io/client-go/informers/coordination/v1beta1 from k8s.io/client-go/informers/coordination
k8s.io/client-go/informers/core from k8s.io/client-go/informers
k8s.io/client-go/informers/core/v1 from k8s.io/client-go/informers/core
k8s.io/client-go/informers/discovery from k8s.io/client-go/informers
k8s.io/client-go/informers/discovery/v1 from k8s.io/client-go/informers/discovery
k8s.io/client-go/informers/discovery/v1beta1 from k8s.io/client-go/informers/discovery
k8s.io/client-go/informers/events from k8s.io/client-go/informers
k8s.io/client-go/informers/events/v1 from k8s.io/client-go/informers/events
k8s.io/client-go/informers/events/v1beta1 from k8s.io/client-go/informers/events
k8s.io/client-go/informers/extensions from k8s.io/client-go/informers
k8s.io/client-go/informers/extensions/v1beta1 from k8s.io/client-go/informers/extensions
k8s.io/client-go/informers/flowcontrol from k8s.io/client-go/informers
k8s.io/client-go/informers/flowcontrol/v1 from k8s.io/client-go/informers/flowcontrol
k8s.io/client-go/informers/flowcontrol/v1beta1 from k8s.io/client-go/informers/flowcontrol
k8s.io/client-go/informers/flowcontrol/v1beta2 from k8s.io/client-go/informers/flowcontrol
k8s.io/client-go/informers/flowcontrol/v1beta3 from k8s.io/client-go/informers/flowcontrol
k8s.io/client-go/informers/internalinterfaces from k8s.io/client-go/informers+
k8s.io/client-go/informers/networking from k8s.io/client-go/informers
k8s.io/client-go/informers/networking/v1 from k8s.io/client-go/informers/networking
k8s.io/client-go/informers/networking/v1alpha1 from k8s.io/client-go/informers/networking
k8s.io/client-go/informers/networking/v1beta1 from k8s.io/client-go/informers/networking
k8s.io/client-go/informers/node from k8s.io/client-go/informers
k8s.io/client-go/informers/node/v1 from k8s.io/client-go/informers/node
k8s.io/client-go/informers/node/v1alpha1 from k8s.io/client-go/informers/node
k8s.io/client-go/informers/node/v1beta1 from k8s.io/client-go/informers/node
k8s.io/client-go/informers/policy from k8s.io/client-go/informers
k8s.io/client-go/informers/policy/v1 from k8s.io/client-go/informers/policy
k8s.io/client-go/informers/policy/v1beta1 from k8s.io/client-go/informers/policy
k8s.io/client-go/informers/rbac from k8s.io/client-go/informers
k8s.io/client-go/informers/rbac/v1 from k8s.io/client-go/informers/rbac
k8s.io/client-go/informers/rbac/v1alpha1 from k8s.io/client-go/informers/rbac
k8s.io/client-go/informers/rbac/v1beta1 from k8s.io/client-go/informers/rbac
k8s.io/client-go/informers/resource from k8s.io/client-go/informers
k8s.io/client-go/informers/resource/v1alpha3 from k8s.io/client-go/informers/resource
k8s.io/client-go/informers/resource/v1beta1 from k8s.io/client-go/informers/resource
k8s.io/client-go/informers/scheduling from k8s.io/client-go/informers
k8s.io/client-go/informers/scheduling/v1 from k8s.io/client-go/informers/scheduling
k8s.io/client-go/informers/scheduling/v1alpha1 from k8s.io/client-go/informers/scheduling
k8s.io/client-go/informers/scheduling/v1beta1 from k8s.io/client-go/informers/scheduling
k8s.io/client-go/informers/storage from k8s.io/client-go/informers
k8s.io/client-go/informers/storage/v1 from k8s.io/client-go/informers/storage
k8s.io/client-go/informers/storage/v1alpha1 from k8s.io/client-go/informers/storage
k8s.io/client-go/informers/storage/v1beta1 from k8s.io/client-go/informers/storage
k8s.io/client-go/informers/storagemigration from k8s.io/client-go/informers
k8s.io/client-go/informers/storagemigration/v1alpha1 from k8s.io/client-go/informers/storagemigration
k8s.io/client-go/kubernetes from k8s.io/client-go/tools/leaderelection/resourcelock+
k8s.io/client-go/features from k8s.io/client-go/tools/cache
k8s.io/client-go/kubernetes from k8s.io/client-go/tools/leaderelection/resourcelock
k8s.io/client-go/kubernetes/scheme from k8s.io/client-go/discovery+
k8s.io/client-go/kubernetes/typed/admissionregistration/v1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1 from k8s.io/client-go/kubernetes
@@ -585,7 +507,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/kubernetes/typed/certificates/v1alpha1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/certificates/v1beta1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/coordination/v1 from k8s.io/client-go/kubernetes+
k8s.io/client-go/kubernetes/typed/coordination/v1alpha2 from k8s.io/client-go/kubernetes+
k8s.io/client-go/kubernetes/typed/coordination/v1beta1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/core/v1 from k8s.io/client-go/kubernetes+
k8s.io/client-go/kubernetes/typed/discovery/v1 from k8s.io/client-go/kubernetes
@@ -608,8 +529,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/kubernetes/typed/rbac/v1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/rbac/v1beta1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/resource/v1alpha3 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/resource/v1beta1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/resource/v1alpha2 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/scheduling/v1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 from k8s.io/client-go/kubernetes
@@ -617,56 +537,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/kubernetes/typed/storage/v1alpha1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/storage/v1beta1 from k8s.io/client-go/kubernetes
k8s.io/client-go/kubernetes/typed/storagemigration/v1alpha1 from k8s.io/client-go/kubernetes
k8s.io/client-go/listers from k8s.io/client-go/listers/admissionregistration/v1+
k8s.io/client-go/listers/admissionregistration/v1 from k8s.io/client-go/informers/admissionregistration/v1
k8s.io/client-go/listers/admissionregistration/v1alpha1 from k8s.io/client-go/informers/admissionregistration/v1alpha1
k8s.io/client-go/listers/admissionregistration/v1beta1 from k8s.io/client-go/informers/admissionregistration/v1beta1
k8s.io/client-go/listers/apiserverinternal/v1alpha1 from k8s.io/client-go/informers/apiserverinternal/v1alpha1
k8s.io/client-go/listers/apps/v1 from k8s.io/client-go/informers/apps/v1
k8s.io/client-go/listers/apps/v1beta1 from k8s.io/client-go/informers/apps/v1beta1
k8s.io/client-go/listers/apps/v1beta2 from k8s.io/client-go/informers/apps/v1beta2
k8s.io/client-go/listers/autoscaling/v1 from k8s.io/client-go/informers/autoscaling/v1
k8s.io/client-go/listers/autoscaling/v2 from k8s.io/client-go/informers/autoscaling/v2
k8s.io/client-go/listers/autoscaling/v2beta1 from k8s.io/client-go/informers/autoscaling/v2beta1
k8s.io/client-go/listers/autoscaling/v2beta2 from k8s.io/client-go/informers/autoscaling/v2beta2
k8s.io/client-go/listers/batch/v1 from k8s.io/client-go/informers/batch/v1
k8s.io/client-go/listers/batch/v1beta1 from k8s.io/client-go/informers/batch/v1beta1
k8s.io/client-go/listers/certificates/v1 from k8s.io/client-go/informers/certificates/v1
k8s.io/client-go/listers/certificates/v1alpha1 from k8s.io/client-go/informers/certificates/v1alpha1
k8s.io/client-go/listers/certificates/v1beta1 from k8s.io/client-go/informers/certificates/v1beta1
k8s.io/client-go/listers/coordination/v1 from k8s.io/client-go/informers/coordination/v1
k8s.io/client-go/listers/coordination/v1alpha2 from k8s.io/client-go/informers/coordination/v1alpha2
k8s.io/client-go/listers/coordination/v1beta1 from k8s.io/client-go/informers/coordination/v1beta1
k8s.io/client-go/listers/core/v1 from k8s.io/client-go/informers/core/v1
k8s.io/client-go/listers/discovery/v1 from k8s.io/client-go/informers/discovery/v1
k8s.io/client-go/listers/discovery/v1beta1 from k8s.io/client-go/informers/discovery/v1beta1
k8s.io/client-go/listers/events/v1 from k8s.io/client-go/informers/events/v1
k8s.io/client-go/listers/events/v1beta1 from k8s.io/client-go/informers/events/v1beta1
k8s.io/client-go/listers/extensions/v1beta1 from k8s.io/client-go/informers/extensions/v1beta1
k8s.io/client-go/listers/flowcontrol/v1 from k8s.io/client-go/informers/flowcontrol/v1
k8s.io/client-go/listers/flowcontrol/v1beta1 from k8s.io/client-go/informers/flowcontrol/v1beta1
k8s.io/client-go/listers/flowcontrol/v1beta2 from k8s.io/client-go/informers/flowcontrol/v1beta2
k8s.io/client-go/listers/flowcontrol/v1beta3 from k8s.io/client-go/informers/flowcontrol/v1beta3
k8s.io/client-go/listers/networking/v1 from k8s.io/client-go/informers/networking/v1
k8s.io/client-go/listers/networking/v1alpha1 from k8s.io/client-go/informers/networking/v1alpha1
k8s.io/client-go/listers/networking/v1beta1 from k8s.io/client-go/informers/networking/v1beta1
k8s.io/client-go/listers/node/v1 from k8s.io/client-go/informers/node/v1
k8s.io/client-go/listers/node/v1alpha1 from k8s.io/client-go/informers/node/v1alpha1
k8s.io/client-go/listers/node/v1beta1 from k8s.io/client-go/informers/node/v1beta1
k8s.io/client-go/listers/policy/v1 from k8s.io/client-go/informers/policy/v1
k8s.io/client-go/listers/policy/v1beta1 from k8s.io/client-go/informers/policy/v1beta1
k8s.io/client-go/listers/rbac/v1 from k8s.io/client-go/informers/rbac/v1
k8s.io/client-go/listers/rbac/v1alpha1 from k8s.io/client-go/informers/rbac/v1alpha1
k8s.io/client-go/listers/rbac/v1beta1 from k8s.io/client-go/informers/rbac/v1beta1
k8s.io/client-go/listers/resource/v1alpha3 from k8s.io/client-go/informers/resource/v1alpha3
k8s.io/client-go/listers/resource/v1beta1 from k8s.io/client-go/informers/resource/v1beta1
k8s.io/client-go/listers/scheduling/v1 from k8s.io/client-go/informers/scheduling/v1
k8s.io/client-go/listers/scheduling/v1alpha1 from k8s.io/client-go/informers/scheduling/v1alpha1
k8s.io/client-go/listers/scheduling/v1beta1 from k8s.io/client-go/informers/scheduling/v1beta1
k8s.io/client-go/listers/storage/v1 from k8s.io/client-go/informers/storage/v1
k8s.io/client-go/listers/storage/v1alpha1 from k8s.io/client-go/informers/storage/v1alpha1
k8s.io/client-go/listers/storage/v1beta1 from k8s.io/client-go/informers/storage/v1beta1
k8s.io/client-go/listers/storagemigration/v1alpha1 from k8s.io/client-go/informers/storagemigration/v1alpha1
k8s.io/client-go/metadata from sigs.k8s.io/controller-runtime/pkg/cache/internal+
k8s.io/client-go/openapi from k8s.io/client-go/discovery
k8s.io/client-go/pkg/apis/clientauthentication from k8s.io/client-go/pkg/apis/clientauthentication/install+
@@ -678,7 +548,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/rest from k8s.io/client-go/discovery+
k8s.io/client-go/rest/watch from k8s.io/client-go/rest
k8s.io/client-go/restmapper from sigs.k8s.io/controller-runtime/pkg/client/apiutil
k8s.io/client-go/testing from k8s.io/client-go/gentype
k8s.io/client-go/tools/auth from k8s.io/client-go/tools/clientcmd
k8s.io/client-go/tools/cache from sigs.k8s.io/controller-runtime/pkg/cache+
k8s.io/client-go/tools/cache/synctrack from k8s.io/client-go/tools/cache
@@ -695,14 +564,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/client-go/tools/record/util from k8s.io/client-go/tools/record
k8s.io/client-go/tools/reference from k8s.io/client-go/kubernetes/typed/core/v1+
k8s.io/client-go/transport from k8s.io/client-go/plugin/pkg/client/auth/exec+
k8s.io/client-go/util/apply from k8s.io/client-go/dynamic+
k8s.io/client-go/util/cert from k8s.io/client-go/rest+
k8s.io/client-go/util/connrotation from k8s.io/client-go/plugin/pkg/client/auth/exec+
k8s.io/client-go/util/consistencydetector from k8s.io/client-go/dynamic+
k8s.io/client-go/util/flowcontrol from k8s.io/client-go/kubernetes+
k8s.io/client-go/util/homedir from k8s.io/client-go/tools/clientcmd
k8s.io/client-go/util/keyutil from k8s.io/client-go/util/cert
k8s.io/client-go/util/watchlist from k8s.io/client-go/dynamic+
k8s.io/client-go/util/workqueue from k8s.io/client-go/transport+
k8s.io/klog/v2 from k8s.io/apimachinery/pkg/api/meta+
k8s.io/klog/v2/internal/buffer from k8s.io/klog/v2
@@ -723,12 +589,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
k8s.io/utils/buffer from k8s.io/client-go/tools/cache
k8s.io/utils/clock from k8s.io/apimachinery/pkg/util/cache+
k8s.io/utils/clock/testing from k8s.io/client-go/util/flowcontrol
k8s.io/utils/internal/third_party/forked/golang/golang-lru from k8s.io/utils/lru
k8s.io/utils/internal/third_party/forked/golang/net from k8s.io/utils/net
k8s.io/utils/lru from k8s.io/client-go/tools/record
k8s.io/utils/net from k8s.io/apimachinery/pkg/util/net+
k8s.io/utils/pointer from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
k8s.io/utils/ptr from k8s.io/client-go/tools/cache+
k8s.io/utils/strings/slices from k8s.io/apimachinery/pkg/labels
k8s.io/utils/trace from k8s.io/client-go/tools/cache
sigs.k8s.io/controller-runtime/pkg/builder from tailscale.com/cmd/k8s-operator
sigs.k8s.io/controller-runtime/pkg/cache from sigs.k8s.io/controller-runtime/pkg/cluster+
@@ -761,12 +626,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
sigs.k8s.io/controller-runtime/pkg/metrics from sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics+
sigs.k8s.io/controller-runtime/pkg/metrics/server from sigs.k8s.io/controller-runtime/pkg/manager
sigs.k8s.io/controller-runtime/pkg/predicate from sigs.k8s.io/controller-runtime/pkg/builder+
sigs.k8s.io/controller-runtime/pkg/ratelimiter from sigs.k8s.io/controller-runtime/pkg/controller+
sigs.k8s.io/controller-runtime/pkg/reconcile from sigs.k8s.io/controller-runtime/pkg/builder+
sigs.k8s.io/controller-runtime/pkg/recorder from sigs.k8s.io/controller-runtime/pkg/leaderelection+
sigs.k8s.io/controller-runtime/pkg/source from sigs.k8s.io/controller-runtime/pkg/builder+
sigs.k8s.io/controller-runtime/pkg/webhook from sigs.k8s.io/controller-runtime/pkg/manager
sigs.k8s.io/controller-runtime/pkg/webhook/admission from sigs.k8s.io/controller-runtime/pkg/builder+
sigs.k8s.io/controller-runtime/pkg/webhook/admission/metrics from sigs.k8s.io/controller-runtime/pkg/webhook/admission
sigs.k8s.io/controller-runtime/pkg/webhook/conversion from sigs.k8s.io/controller-runtime/pkg/builder
sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics from sigs.k8s.io/controller-runtime/pkg/webhook+
sigs.k8s.io/json from k8s.io/apimachinery/pkg/runtime/serializer/json+
@@ -777,10 +642,10 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
sigs.k8s.io/structured-merge-diff/v4/typed from k8s.io/apimachinery/pkg/util/managedfields+
sigs.k8s.io/structured-merge-diff/v4/value from k8s.io/apimachinery/pkg/runtime+
sigs.k8s.io/yaml from k8s.io/apimachinery/pkg/runtime/serializer/json+
sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml+
sigs.k8s.io/yaml/goyaml.v2 from sigs.k8s.io/yaml
tailscale.com from tailscale.com/version
tailscale.com/appc from tailscale.com/ipn/ipnlocal
💣 tailscale.com/atomicfile from tailscale.com/ipn+
tailscale.com/atomicfile from tailscale.com/ipn+
tailscale.com/client/tailscale from tailscale.com/client/web+
tailscale.com/client/tailscale/apitype from tailscale.com/client/tailscale+
tailscale.com/client/web from tailscale.com/ipn/ipnlocal
@@ -801,10 +666,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/drive from tailscale.com/client/tailscale+
tailscale.com/envknob from tailscale.com/client/tailscale+
tailscale.com/envknob/featureknob from tailscale.com/client/web+
tailscale.com/feature from tailscale.com/feature/wakeonlan+
tailscale.com/feature/condregister from tailscale.com/tsnet
L tailscale.com/feature/tap from tailscale.com/feature/condregister
tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister
tailscale.com/health from tailscale.com/control/controlclient+
tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal
tailscale.com/hostinfo from tailscale.com/client/web+
@@ -839,7 +700,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/logtail/backoff from tailscale.com/control/controlclient+
tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+
tailscale.com/metrics from tailscale.com/derp+
tailscale.com/net/bakedroots from tailscale.com/net/tlsdial+
tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+
tailscale.com/net/connstats from tailscale.com/net/tstun+
tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+
@@ -958,6 +818,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
tailscale.com/util/systemd from tailscale.com/control/controlclient+
tailscale.com/util/testenv from tailscale.com/control/controlclient+
tailscale.com/util/truncate from tailscale.com/logtail
tailscale.com/util/uniq from tailscale.com/ipn/ipnlocal+
tailscale.com/util/usermetric from tailscale.com/health+
tailscale.com/util/vizerror from tailscale.com/tailcfg+
💣 tailscale.com/util/winutil from tailscale.com/clientupdate+
@@ -1018,7 +879,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
golang.org/x/oauth2/clientcredentials from tailscale.com/cmd/k8s-operator
golang.org/x/oauth2/internal from golang.org/x/oauth2+
golang.org/x/sync/errgroup from github.com/mdlayher/socket+
golang.org/x/sys/cpu from github.com/tailscale/certstore+
golang.org/x/sys/cpu from github.com/josharian/native+
LD golang.org/x/sys/unix from github.com/fsnotify/fsnotify+
W golang.org/x/sys/windows from github.com/dblohm7/wingoes+
W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+

View File

@@ -35,13 +35,9 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
- name: oauth
{{- with .Values.oauthSecretVolume }}
{{- toYaml . | nindent 10 }}
{{- else }}
secret:
secretName: operator-oauth
{{- end }}
- name: oauth
secret:
secretName: operator-oauth
containers:
- name: operator
{{- with .Values.operatorConfig.securityContext }}
@@ -85,14 +81,6 @@ spec:
- name: PROXY_DEFAULT_CLASS
value: {{ .Values.proxyConfig.defaultProxyClass }}
{{- end }}
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
{{- with .Values.operatorConfig.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}

View File

@@ -1,4 +1,3 @@
{{- if .Values.ingressClass.enabled }}
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
@@ -7,4 +6,3 @@ metadata:
spec:
controller: tailscale.com/ts-ingress # controller name currently can not be changed
# parameters: {} # currently no parameters are supported
{{- end }}

View File

@@ -6,10 +6,6 @@ kind: ServiceAccount
metadata:
name: operator
namespace: {{ .Release.Namespace }}
{{- with .Values.operatorConfig.serviceAccountAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
@@ -34,10 +30,6 @@ rules:
- apiGroups: ["tailscale.com"]
resources: ["recorders", "recorders/status"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "list", "watch"]
resourceNames: ["servicemonitors.monitoring.coreos.com"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
@@ -73,9 +65,6 @@ rules:
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles", "rolebindings"]
verbs: ["get", "create", "patch", "update", "list", "watch"]
- apiGroups: ["monitoring.coreos.com"]
resources: ["servicemonitors"]
verbs: ["get", "list", "update", "create", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding

View File

@@ -3,26 +3,11 @@
# Operator oauth credentials. If set a Kubernetes Secret with the provided
# values will be created in the operator namespace. If unset a Secret named
# operator-oauth must be precreated or oauthSecretVolume needs to be adjusted.
# This block will be overridden by oauthSecretVolume, if set.
# operator-oauth must be precreated.
oauth: {}
# clientId: ""
# clientSecret: ""
# Secret volume.
# If set it defines the volume the oauth secrets will be mounted from.
# The volume needs to contain two files named `client_id` and `client_secret`.
# If unset the volume will reference the Secret named operator-oauth.
# This block will override the oauth block.
oauthSecretVolume: {}
# csi:
# driver: secrets-store.csi.k8s.io
# readOnly: true
# volumeAttributes:
# secretProviderClass: tailscale-oauth
#
## NAME is pre-defined!
# installCRDs determines whether tailscale.com CRDs should be installed as part
# of chart installation. We do not use Helm's CRD installation mechanism as that
# does not allow for upgrading CRDs.
@@ -55,9 +40,6 @@ operatorConfig:
podAnnotations: {}
podLabels: {}
serviceAccountAnnotations: {}
# eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/tailscale-operator-role
tolerations: []
affinity: {}
@@ -72,9 +54,6 @@ operatorConfig:
# - name: EXTRA_VAR2
# value: "value2"
# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
ingressClass:
enabled: true
# proxyConfig contains configuraton that will be applied to any ingress/egress
# proxies created by the operator.

View File

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: connectors.tailscale.com
spec:
group: tailscale.com

View File

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: dnsconfigs.tailscale.com
spec:
group: tailscale.com

View File

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: proxyclasses.tailscale.com
spec:
group: tailscale.com
@@ -73,45 +73,9 @@ spec:
enable:
description: |-
Setting enable to true will make the proxy serve Tailscale metrics
at <pod-ip>:9002/metrics.
A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
serve the metrics at <service-ip>:9002/metrics.
In 1.78.x and 1.80.x, this field also serves as the default value for
.spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
fields will independently default to false.
at <pod-ip>:9001/debug/metrics.
Defaults to false.
type: boolean
serviceMonitor:
description: |-
Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
The ingested metrics for each Service monitor will have labels to identify the proxy:
ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
job: ts_<proxy type>_[<parent namespace>]_<parent_name>
type: object
required:
- enable
properties:
enable:
description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
type: boolean
labels:
description: |-
Labels to add to the ServiceMonitor.
Labels must be valid Kubernetes labels.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
additionalProperties:
type: string
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
x-kubernetes-validations:
- rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)'
message: ServiceMonitor can only be enabled if metrics are enabled
statefulSet:
description: |-
Configuration parameters for the proxy's StatefulSet. Tailscale
@@ -143,8 +107,6 @@ spec:
type: object
additionalProperties:
type: string
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
pod:
description: Configuration for the proxy Pod.
type: object
@@ -428,7 +390,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -443,7 +405,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -600,7 +562,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -615,7 +577,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -773,7 +735,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -788,7 +750,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -945,7 +907,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -960,7 +922,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -1074,8 +1036,6 @@ spec:
type: object
additionalProperties:
type: string
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
nodeName:
description: |-
Proxy Pod's node name.
@@ -1174,32 +1134,6 @@ spec:
Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
seLinuxChangePolicy:
description: |-
seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
Valid values are "MountOption" and "Recursive".
"Recursive" means relabeling of all files on all Pod volumes by the container runtime.
This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
"MountOption" mounts all eligible Pod volumes with `-o context` mount option.
This requires all Pods that share the same volume to use the same SELinux label.
It is not possible to share the same volume among privileged and unprivileged Pods.
Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
CSIDriver instance. Other volumes are always re-labelled recursively.
"MountOption" value is allowed only when SELinuxMount feature gate is enabled.
If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
and "Recursive" for all other volumes.
This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
Note that this field cannot be set when spec.os.name is windows.
type: string
seLinuxOptions:
description: |-
The SELinux context to be applied to all containers.
@@ -1248,28 +1182,18 @@ spec:
type: string
supplementalGroups:
description: |-
A list of groups applied to the first process run in each container, in
addition to the container's primary GID and fsGroup (if specified). If
the SupplementalGroupsPolicy feature is enabled, the
supplementalGroupsPolicy field determines whether these are in addition
to or instead of any group memberships defined in the container image.
If unspecified, no additional groups are added, though group memberships
defined in the container image may still be used, depending on the
supplementalGroupsPolicy field.
A list of groups applied to the first process run in each container, in addition
to the container's primary GID, the fsGroup (if specified), and group memberships
defined in the container image for the uid of the container process. If unspecified,
no additional groups are added to any container. Note that group memberships
defined in the container image for the uid of the container process are still effective,
even if they are not included in this list.
Note that this field cannot be set when spec.os.name is windows.
type: array
items:
type: integer
format: int64
x-kubernetes-list-type: atomic
supplementalGroupsPolicy:
description: |-
Defines how supplemental groups of the first container processes are calculated.
Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
(Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
and the container runtime must implement support for this feature.
Note that this field cannot be set when spec.os.name is windows.
type: string
sysctls:
description: |-
Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -1325,25 +1249,6 @@ spec:
description: Configuration for the proxy container running tailscale.
type: object
properties:
debug:
description: |-
Configuration for enabling extra debug information in the container.
Not recommended for production use.
type: object
properties:
enable:
description: |-
Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
9001 is a container port named "debug". The endpoints and their responses
may change in backwards incompatible ways in the future, and should not
be considered stable.
In 1.78.x and 1.80.x, this setting will default to the value of
.spec.metrics.enable, and requests to the "metrics" port matching the
mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
this setting will default to false, and no requests will be proxied.
type: boolean
env:
description: |-
List of environment variables to set in the container.
@@ -1425,12 +1330,6 @@ spec:
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
@@ -1461,12 +1360,11 @@ spec:
securityContext:
description: |-
Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
Security context specified here will override the security context by the operator.
By default the operator:
- sets 'privileged: true' for the init container
- set NET_ADMIN capability for tailscale container for proxies that
are created for Services or Connector.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
type: object
properties:
@@ -1535,7 +1433,7 @@ spec:
procMount:
description: |-
procMount denotes the type of proc mount to use for the containers.
The default value is Default which uses the container runtime defaults for
The default is DefaultProcMount which uses the container runtime defaults for
readonly paths and masked paths.
This requires the ProcMountType feature flag to be enabled.
Note that this field cannot be set when spec.os.name is windows.
@@ -1655,25 +1553,6 @@ spec:
description: Configuration for the proxy init container that enables forwarding.
type: object
properties:
debug:
description: |-
Configuration for enabling extra debug information in the container.
Not recommended for production use.
type: object
properties:
enable:
description: |-
Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
9001 is a container port named "debug". The endpoints and their responses
may change in backwards incompatible ways in the future, and should not
be considered stable.
In 1.78.x and 1.80.x, this setting will default to the value of
.spec.metrics.enable, and requests to the "metrics" port matching the
mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
this setting will default to false, and no requests will be proxied.
type: boolean
env:
description: |-
List of environment variables to set in the container.
@@ -1755,12 +1634,6 @@ spec:
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
@@ -1791,12 +1664,11 @@ spec:
securityContext:
description: |-
Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
Security context specified here will override the security context by the operator.
By default the operator:
- sets 'privileged: true' for the init container
- set NET_ADMIN capability for tailscale container for proxies that
are created for Services or Connector.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
type: object
properties:
@@ -1865,7 +1737,7 @@ spec:
procMount:
description: |-
procMount denotes the type of proc mount to use for the containers.
The default value is Default which uses the container runtime defaults for
The default is DefaultProcMount which uses the container runtime defaults for
readonly paths and masked paths.
This requires the ProcMountType feature flag to be enabled.
Note that this field cannot be set when spec.os.name is windows.

View File

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: proxygroups.tailscale.com
spec:
group: tailscale.com
@@ -20,24 +20,9 @@ spec:
jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
name: Status
type: string
- description: ProxyGroup type.
jsonPath: .spec.type
name: Type
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
ProxyGroup defines a set of Tailscale devices that will act as proxies.
Currently only egress ProxyGroups are supported.
Use the tailscale.com/proxy-group annotation on a Service to specify that
the egress proxy should be implemented by a ProxyGroup instead of a single
dedicated proxy. In addition to running a highly available set of proxies,
ProxyGroup also allows for serving many annotated Services from a single
set of proxies to minimise resource consumption.
More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
type: object
required:
- spec
@@ -88,7 +73,6 @@ spec:
Defaults to 2.
type: integer
format: int32
minimum: 0
tags:
description: |-
Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
@@ -102,16 +86,10 @@ spec:
type: string
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
type:
description: |-
Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type is immutable once a ProxyGroup is created.
description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
type: string
enum:
- egress
- ingress
x-kubernetes-validations:
- rule: self == oldSelf
message: ProxyGroup type is immutable
status:
description: |-
ProxyGroupStatus describes the status of the ProxyGroup resources. This is

View File

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: recorders.tailscale.com
spec:
group: tailscale.com
@@ -27,12 +27,6 @@ spec:
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
Recorder defines a tsrecorder device for recording SSH sessions. By default,
it will store recordings in a local ephemeral volume. If you want to persist
recordings, you can configure an S3-compatible API for storage.
More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
type: object
required:
- spec
@@ -372,7 +366,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -387,7 +381,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -544,7 +538,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -559,7 +553,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -717,7 +711,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -732,7 +726,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -889,7 +883,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -904,7 +898,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
type: array
items:
type: string
@@ -1066,12 +1060,6 @@ spec:
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
@@ -1171,7 +1159,7 @@ spec:
procMount:
description: |-
procMount denotes the type of proc mount to use for the containers.
The default value is Default which uses the container runtime defaults for
The default is DefaultProcMount which uses the container runtime defaults for
readonly paths and masked paths.
This requires the ProcMountType feature flag to be enabled.
Note that this field cannot be set when spec.os.name is windows.
@@ -1407,32 +1395,6 @@ spec:
Note that this field cannot be set when spec.os.name is windows.
type: integer
format: int64
seLinuxChangePolicy:
description: |-
seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
Valid values are "MountOption" and "Recursive".
"Recursive" means relabeling of all files on all Pod volumes by the container runtime.
This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
"MountOption" mounts all eligible Pod volumes with `-o context` mount option.
This requires all Pods that share the same volume to use the same SELinux label.
It is not possible to share the same volume among privileged and unprivileged Pods.
Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
CSIDriver instance. Other volumes are always re-labelled recursively.
"MountOption" value is allowed only when SELinuxMount feature gate is enabled.
If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
and "Recursive" for all other volumes.
This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
Note that this field cannot be set when spec.os.name is windows.
type: string
seLinuxOptions:
description: |-
The SELinux context to be applied to all containers.
@@ -1481,28 +1443,18 @@ spec:
type: string
supplementalGroups:
description: |-
A list of groups applied to the first process run in each container, in
addition to the container's primary GID and fsGroup (if specified). If
the SupplementalGroupsPolicy feature is enabled, the
supplementalGroupsPolicy field determines whether these are in addition
to or instead of any group memberships defined in the container image.
If unspecified, no additional groups are added, though group memberships
defined in the container image may still be used, depending on the
supplementalGroupsPolicy field.
A list of groups applied to the first process run in each container, in addition
to the container's primary GID, the fsGroup (if specified), and group memberships
defined in the container image for the uid of the container process. If unspecified,
no additional groups are added to any container. Note that group memberships
defined in the container image for the uid of the container process are still effective,
even if they are not included in this list.
Note that this field cannot be set when spec.os.name is windows.
type: array
items:
type: integer
format: int64
x-kubernetes-list-type: atomic
supplementalGroupsPolicy:
description: |-
Defines how supplemental groups of the first container processes are calculated.
Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
(Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
and the container runtime must implement support for this feature.
Note that this field cannot be set when spec.os.name is windows.
type: string
sysctls:
description: |-
Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported

View File

@@ -31,7 +31,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: connectors.tailscale.com
spec:
group: tailscale.com
@@ -294,7 +294,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: dnsconfigs.tailscale.com
spec:
group: tailscale.com
@@ -476,7 +476,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: proxyclasses.tailscale.com
spec:
group: tailscale.com
@@ -540,48 +540,12 @@ spec:
enable:
description: |-
Setting enable to true will make the proxy serve Tailscale metrics
at <pod-ip>:9002/metrics.
A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
serve the metrics at <service-ip>:9002/metrics.
In 1.78.x and 1.80.x, this field also serves as the default value for
.spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
fields will independently default to false.
at <pod-ip>:9001/debug/metrics.
Defaults to false.
type: boolean
serviceMonitor:
description: |-
Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
The ingested metrics for each Service monitor will have labels to identify the proxy:
ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
job: ts_<proxy type>_[<parent namespace>]_<parent_name>
properties:
enable:
description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
type: boolean
labels:
additionalProperties:
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
type: string
description: |-
Labels to add to the ServiceMonitor.
Labels must be valid Kubernetes labels.
https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
type: object
required:
- enable
type: object
required:
- enable
type: object
x-kubernetes-validations:
- message: ServiceMonitor can only be enabled if metrics are enabled
rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)'
statefulSet:
description: |-
Configuration parameters for the proxy's StatefulSet. Tailscale
@@ -602,8 +566,6 @@ spec:
type: object
labels:
additionalProperties:
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
type: string
description: |-
Labels that will be added to the StatefulSet created for the proxy.
@@ -886,7 +848,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -901,7 +863,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1062,7 +1024,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1077,7 +1039,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1231,7 +1193,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1246,7 +1208,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1407,7 +1369,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1422,7 +1384,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -1534,8 +1496,6 @@ spec:
type: array
labels:
additionalProperties:
maxLength: 63
pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$
type: string
description: |-
Labels that will be added to the proxy Pod.
@@ -1641,32 +1601,6 @@ spec:
Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
seLinuxChangePolicy:
description: |-
seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
Valid values are "MountOption" and "Recursive".
"Recursive" means relabeling of all files on all Pod volumes by the container runtime.
This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
"MountOption" mounts all eligible Pod volumes with `-o context` mount option.
This requires all Pods that share the same volume to use the same SELinux label.
It is not possible to share the same volume among privileged and unprivileged Pods.
Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
CSIDriver instance. Other volumes are always re-labelled recursively.
"MountOption" value is allowed only when SELinuxMount feature gate is enabled.
If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
and "Recursive" for all other volumes.
This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
Note that this field cannot be set when spec.os.name is windows.
type: string
seLinuxOptions:
description: |-
The SELinux context to be applied to all containers.
@@ -1715,28 +1649,18 @@ spec:
type: object
supplementalGroups:
description: |-
A list of groups applied to the first process run in each container, in
addition to the container's primary GID and fsGroup (if specified). If
the SupplementalGroupsPolicy feature is enabled, the
supplementalGroupsPolicy field determines whether these are in addition
to or instead of any group memberships defined in the container image.
If unspecified, no additional groups are added, though group memberships
defined in the container image may still be used, depending on the
supplementalGroupsPolicy field.
A list of groups applied to the first process run in each container, in addition
to the container's primary GID, the fsGroup (if specified), and group memberships
defined in the container image for the uid of the container process. If unspecified,
no additional groups are added to any container. Note that group memberships
defined in the container image for the uid of the container process are still effective,
even if they are not included in this list.
Note that this field cannot be set when spec.os.name is windows.
items:
format: int64
type: integer
type: array
x-kubernetes-list-type: atomic
supplementalGroupsPolicy:
description: |-
Defines how supplemental groups of the first container processes are calculated.
Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
(Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
and the container runtime must implement support for this feature.
Note that this field cannot be set when spec.os.name is windows.
type: string
sysctls:
description: |-
Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -1792,25 +1716,6 @@ spec:
tailscaleContainer:
description: Configuration for the proxy container running tailscale.
properties:
debug:
description: |-
Configuration for enabling extra debug information in the container.
Not recommended for production use.
properties:
enable:
description: |-
Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
9001 is a container port named "debug". The endpoints and their responses
may change in backwards incompatible ways in the future, and should not
be considered stable.
In 1.78.x and 1.80.x, this setting will default to the value of
.spec.metrics.enable, and requests to the "metrics" port matching the
mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
this setting will default to false, and no requests will be proxied.
type: boolean
type: object
env:
description: |-
List of environment variables to set in the container.
@@ -1887,12 +1792,6 @@ spec:
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
@@ -1928,12 +1827,11 @@ spec:
securityContext:
description: |-
Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
Security context specified here will override the security context by the operator.
By default the operator:
- sets 'privileged: true' for the init container
- set NET_ADMIN capability for tailscale container for proxies that
are created for Services or Connector.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
properties:
allowPrivilegeEscalation:
@@ -2001,7 +1899,7 @@ spec:
procMount:
description: |-
procMount denotes the type of proc mount to use for the containers.
The default value is Default which uses the container runtime defaults for
The default is DefaultProcMount which uses the container runtime defaults for
readonly paths and masked paths.
This requires the ProcMountType feature flag to be enabled.
Note that this field cannot be set when spec.os.name is windows.
@@ -2122,25 +2020,6 @@ spec:
tailscaleInitContainer:
description: Configuration for the proxy init container that enables forwarding.
properties:
debug:
description: |-
Configuration for enabling extra debug information in the container.
Not recommended for production use.
properties:
enable:
description: |-
Enable tailscaled's HTTP pprof endpoints at <pod-ip>:9001/debug/pprof/
and internal debug metrics endpoint at <pod-ip>:9001/debug/metrics, where
9001 is a container port named "debug". The endpoints and their responses
may change in backwards incompatible ways in the future, and should not
be considered stable.
In 1.78.x and 1.80.x, this setting will default to the value of
.spec.metrics.enable, and requests to the "metrics" port matching the
mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x,
this setting will default to false, and no requests will be proxied.
type: boolean
type: object
env:
description: |-
List of environment variables to set in the container.
@@ -2217,12 +2096,6 @@ spec:
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
@@ -2258,12 +2131,11 @@ spec:
securityContext:
description: |-
Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
Security context specified here will override the security context by the operator.
By default the operator:
- sets 'privileged: true' for the init container
- set NET_ADMIN capability for tailscale container for proxies that
are created for Services or Connector.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context
properties:
allowPrivilegeEscalation:
@@ -2331,7 +2203,7 @@ spec:
procMount:
description: |-
procMount denotes the type of proc mount to use for the containers.
The default value is Default which uses the container runtime defaults for
The default is DefaultProcMount which uses the container runtime defaults for
readonly paths and masked paths.
This requires the ProcMountType feature flag to be enabled.
Note that this field cannot be set when spec.os.name is windows.
@@ -2765,7 +2637,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: proxygroups.tailscale.com
spec:
group: tailscale.com
@@ -2783,24 +2655,9 @@ spec:
jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason
name: Status
type: string
- description: ProxyGroup type.
jsonPath: .spec.type
name: Type
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
ProxyGroup defines a set of Tailscale devices that will act as proxies.
Currently only egress ProxyGroups are supported.
Use the tailscale.com/proxy-group annotation on a Service to specify that
the egress proxy should be implemented by a ProxyGroup instead of a single
dedicated proxy. In addition to running a highly available set of proxies,
ProxyGroup also allows for serving many annotated Services from a single
set of proxies to minimise resource consumption.
More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress
properties:
apiVersion:
description: |-
@@ -2844,7 +2701,6 @@ spec:
Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2.
format: int32
minimum: 0
type: integer
tags:
description: |-
@@ -2859,16 +2715,10 @@ spec:
type: string
type: array
type:
description: |-
Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type is immutable once a ProxyGroup is created.
description: Type of the ProxyGroup proxies. Currently the only supported type is egress.
enum:
- egress
- ingress
type: string
x-kubernetes-validations:
- message: ProxyGroup type is immutable
rule: self == oldSelf
required:
- type
type: object
@@ -2975,7 +2825,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.17.0
controller-gen.kubebuilder.io/version: v0.15.1-0.20240618033008-7824932b0cab
name: recorders.tailscale.com
spec:
group: tailscale.com
@@ -3000,12 +2850,6 @@ spec:
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
Recorder defines a tsrecorder device for recording SSH sessions. By default,
it will store recordings in a local ephemeral volume. If you want to persist
recordings, you can configure an S3-compatible API for storage.
More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder
properties:
apiVersion:
description: |-
@@ -3329,7 +3173,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3344,7 +3188,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3505,7 +3349,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3520,7 +3364,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3674,7 +3518,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3689,7 +3533,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3850,7 +3694,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both matchLabelKeys and labelSelector.
Also, matchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -3865,7 +3709,7 @@ spec:
pod labels will be ignored. The default value is empty.
The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.
Also, mismatchLabelKeys cannot be set when labelSelector isn't set.
This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).
This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
items:
type: string
type: array
@@ -4027,12 +3871,6 @@ spec:
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
@@ -4136,7 +3974,7 @@ spec:
procMount:
description: |-
procMount denotes the type of proc mount to use for the containers.
The default value is Default which uses the container runtime defaults for
The default is DefaultProcMount which uses the container runtime defaults for
readonly paths and masked paths.
This requires the ProcMountType feature flag to be enabled.
Note that this field cannot be set when spec.os.name is windows.
@@ -4373,32 +4211,6 @@ spec:
Note that this field cannot be set when spec.os.name is windows.
format: int64
type: integer
seLinuxChangePolicy:
description: |-
seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.
It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.
Valid values are "MountOption" and "Recursive".
"Recursive" means relabeling of all files on all Pod volumes by the container runtime.
This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.
"MountOption" mounts all eligible Pod volumes with `-o context` mount option.
This requires all Pods that share the same volume to use the same SELinux label.
It is not possible to share the same volume among privileged and unprivileged Pods.
Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes
whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their
CSIDriver instance. Other volumes are always re-labelled recursively.
"MountOption" value is allowed only when SELinuxMount feature gate is enabled.
If not specified and SELinuxMount feature gate is enabled, "MountOption" is used.
If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes
and "Recursive" for all other volumes.
This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.
All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.
Note that this field cannot be set when spec.os.name is windows.
type: string
seLinuxOptions:
description: |-
The SELinux context to be applied to all containers.
@@ -4447,28 +4259,18 @@ spec:
type: object
supplementalGroups:
description: |-
A list of groups applied to the first process run in each container, in
addition to the container's primary GID and fsGroup (if specified). If
the SupplementalGroupsPolicy feature is enabled, the
supplementalGroupsPolicy field determines whether these are in addition
to or instead of any group memberships defined in the container image.
If unspecified, no additional groups are added, though group memberships
defined in the container image may still be used, depending on the
supplementalGroupsPolicy field.
A list of groups applied to the first process run in each container, in addition
to the container's primary GID, the fsGroup (if specified), and group memberships
defined in the container image for the uid of the container process. If unspecified,
no additional groups are added to any container. Note that group memberships
defined in the container image for the uid of the container process are still effective,
even if they are not included in this list.
Note that this field cannot be set when spec.os.name is windows.
items:
format: int64
type: integer
type: array
x-kubernetes-list-type: atomic
supplementalGroupsPolicy:
description: |-
Defines how supplemental groups of the first container processes are calculated.
Valid values are "Merge" and "Strict". If not specified, "Merge" is used.
(Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled
and the container runtime must implement support for this feature.
Note that this field cannot be set when spec.os.name is windows.
type: string
sysctls:
description: |-
Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported
@@ -4801,16 +4603,6 @@ rules:
- list
- watch
- update
- apiGroups:
- apiextensions.k8s.io
resourceNames:
- servicemonitors.monitoring.coreos.com
resources:
- customresourcedefinitions
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
@@ -4891,16 +4683,6 @@ rules:
- update
- list
- watch
- apiGroups:
- monitoring.coreos.com
resources:
- servicemonitors
verbs:
- get
- list
- update
- create
- delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
@@ -5001,14 +4783,6 @@ spec:
value: "false"
- name: PROXY_FIREWALL_MODE
value: auto
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
image: tailscale/k8s-operator:unstable
imagePullPolicy: Always
name: operator

View File

@@ -39,4 +39,6 @@ spec:
fieldRef:
fieldPath: metadata.uid
securityContext:
privileged: true
capabilities:
add:
- NET_ADMIN

View File

@@ -10,7 +10,6 @@ import (
"encoding/json"
"fmt"
"slices"
"strings"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
@@ -99,15 +98,7 @@ func (dnsRR *dnsRecordsReconciler) Reconcile(ctx context.Context, req reconcile.
return reconcile.Result{}, nil
}
if err := dnsRR.maybeProvision(ctx, headlessSvc, logger); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
return reconcile.Result{}, dnsRR.maybeProvision(ctx, headlessSvc, logger)
}
// maybeProvision ensures that dnsrecords ConfigMap contains a record for the

View File

@@ -1,108 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"fmt"
"net/http"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
kube "tailscale.com/k8s-operator"
"tailscale.com/tstest"
)
// See [TestMain] for test requirements.
func TestIngress(t *testing.T) {
if tsClient == nil {
t.Skip("TestIngress requires credentials for a tailscale client")
}
ctx := context.Background()
cfg := config.GetConfigOrDie()
cl, err := client.New(cfg, client.Options{})
if err != nil {
t.Fatal(err)
}
// Apply nginx
createAndCleanup(t, ctx, cl, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "nginx",
Namespace: "default",
Labels: map[string]string{
"app.kubernetes.io/name": "nginx",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx",
},
},
},
})
// Apply service to expose it as ingress
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress",
Namespace: "default",
Annotations: map[string]string{
"tailscale.com/expose": "true",
},
},
Spec: corev1.ServiceSpec{
Selector: map[string]string{
"app.kubernetes.io/name": "nginx",
},
Ports: []corev1.ServicePort{
{
Name: "http",
Protocol: "TCP",
Port: 80,
},
},
},
}
createAndCleanup(t, ctx, cl, svc)
// TODO: instead of timing out only when test times out, cancel context after 60s or so.
if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) {
maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")}
if err := get(ctx, cl, maybeReadySvc); err != nil {
return false, err
}
isReady := kube.SvcIsReady(maybeReadySvc)
if isReady {
t.Log("Service is ready")
}
return isReady, nil
}); err != nil {
t.Fatalf("error waiting for the Service to become Ready: %v", err)
}
var resp *http.Response
if err := tstest.WaitFor(time.Second*60, func() error {
// TODO(tomhjp): Get the tailnet DNS name from the associated secret instead.
// If we are not the first tailnet node with the requested name, we'll get
// a -N suffix.
resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name))
if err != nil {
return err
}
return nil
}); err != nil {
t.Fatalf("error trying to reach service: %v", err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status: %v; response body s", resp.StatusCode)
}
}

View File

@@ -1,194 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"errors"
"fmt"
"log"
"os"
"slices"
"strings"
"testing"
"github.com/go-logr/zapr"
"github.com/tailscale/hujson"
"go.uber.org/zap/zapcore"
"golang.org/x/oauth2/clientcredentials"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log"
kzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
"tailscale.com/client/tailscale"
)
const (
e2eManagedComment = "// This is managed by the k8s-operator e2e tests"
)
var (
tsClient *tailscale.Client
testGrants = map[string]string{
"test-proxy": `{
"src": ["tag:e2e-test-proxy"],
"dst": ["tag:k8s-operator"],
"app": {
"tailscale.com/cap/kubernetes": [{
"impersonate": {
"groups": ["ts:e2e-test-proxy"],
},
}],
},
}`,
}
)
// This test suite is currently not run in CI.
// It requires some setup not handled by this code:
// - Kubernetes cluster with tailscale operator installed
// - Current kubeconfig context set to connect to that cluster (directly, no operator proxy)
// - Operator installed with --set apiServerProxyConfig.mode="true"
// - ACLs that define tag:e2e-test-proxy tag. TODO(tomhjp): Can maybe replace this prereq onwards with an API key
// - OAuth client ID and secret in TS_API_CLIENT_ID and TS_API_CLIENT_SECRET env
// - OAuth client must have auth_keys and policy_file write for tag:e2e-test-proxy tag
func TestMain(m *testing.M) {
code, err := runTests(m)
if err != nil {
log.Fatal(err)
}
os.Exit(code)
}
func runTests(m *testing.M) (int, error) {
zlog := kzap.NewRaw([]kzap.Opts{kzap.UseDevMode(true), kzap.Level(zapcore.DebugLevel)}...).Sugar()
logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
tailscale.I_Acknowledge_This_API_Is_Unstable = true
if clientID := os.Getenv("TS_API_CLIENT_ID"); clientID != "" {
cleanup, err := setupClientAndACLs()
if err != nil {
return 0, err
}
defer func() {
err = errors.Join(err, cleanup())
}()
}
return m.Run(), nil
}
func setupClientAndACLs() (cleanup func() error, _ error) {
ctx := context.Background()
credentials := clientcredentials.Config{
ClientID: os.Getenv("TS_API_CLIENT_ID"),
ClientSecret: os.Getenv("TS_API_CLIENT_SECRET"),
TokenURL: "https://login.tailscale.com/api/v2/oauth/token",
Scopes: []string{"auth_keys", "policy_file"},
}
tsClient = tailscale.NewClient("-", nil)
tsClient.HTTPClient = credentials.Client(ctx)
if err := patchACLs(ctx, tsClient, func(acls *hujson.Value) {
for test, grant := range testGrants {
deleteTestGrants(test, acls)
addTestGrant(test, grant, acls)
}
}); err != nil {
return nil, err
}
return func() error {
return patchACLs(ctx, tsClient, func(acls *hujson.Value) {
for test := range testGrants {
deleteTestGrants(test, acls)
}
})
}, nil
}
func patchACLs(ctx context.Context, tsClient *tailscale.Client, patchFn func(*hujson.Value)) error {
acls, err := tsClient.ACLHuJSON(ctx)
if err != nil {
return err
}
hj, err := hujson.Parse([]byte(acls.ACL))
if err != nil {
return err
}
patchFn(&hj)
hj.Format()
acls.ACL = hj.String()
if _, err := tsClient.SetACLHuJSON(ctx, *acls, true); err != nil {
return err
}
return nil
}
func addTestGrant(test, grant string, acls *hujson.Value) error {
v, err := hujson.Parse([]byte(grant))
if err != nil {
return err
}
// Add the managed comment to the first line of the grant object contents.
v.Value.(*hujson.Object).Members[0].Name.BeforeExtra = hujson.Extra(fmt.Sprintf("%s: %s\n", e2eManagedComment, test))
if err := acls.Patch([]byte(fmt.Sprintf(`[{"op": "add", "path": "/grants/-", "value": %s}]`, v.String()))); err != nil {
return err
}
return nil
}
func deleteTestGrants(test string, acls *hujson.Value) error {
grants := acls.Find("/grants")
var patches []string
for i, g := range grants.Value.(*hujson.Array).Elements {
members := g.Value.(*hujson.Object).Members
if len(members) == 0 {
continue
}
comment := strings.TrimSpace(string(members[0].Name.BeforeExtra))
if name, found := strings.CutPrefix(comment, e2eManagedComment+": "); found && name == test {
patches = append(patches, fmt.Sprintf(`{"op": "remove", "path": "/grants/%d"}`, i))
}
}
// Remove in reverse order so we don't affect the found indices as we mutate.
slices.Reverse(patches)
if err := acls.Patch([]byte(fmt.Sprintf("[%s]", strings.Join(patches, ",")))); err != nil {
return err
}
return nil
}
func objectMeta(namespace, name string) metav1.ObjectMeta {
return metav1.ObjectMeta{
Namespace: namespace,
Name: name,
}
}
func createAndCleanup(t *testing.T, ctx context.Context, cl client.Client, obj client.Object) {
t.Helper()
if err := cl.Create(ctx, obj); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := cl.Delete(ctx, obj); err != nil {
t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
}
})
}
func get(ctx context.Context, cl client.Client, obj client.Object) error {
return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)
}

View File

@@ -1,156 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package e2e
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"tailscale.com/client/tailscale"
"tailscale.com/tsnet"
"tailscale.com/tstest"
)
// See [TestMain] for test requirements.
func TestProxy(t *testing.T) {
if tsClient == nil {
t.Skip("TestProxy requires credentials for a tailscale client")
}
ctx := context.Background()
cfg := config.GetConfigOrDie()
cl, err := client.New(cfg, client.Options{})
if err != nil {
t.Fatal(err)
}
// Create role and role binding to allow a group we'll impersonate to do stuff.
createAndCleanup(t, ctx, cl, &rbacv1.Role{
ObjectMeta: objectMeta("tailscale", "read-secrets"),
Rules: []rbacv1.PolicyRule{{
APIGroups: []string{""},
Verbs: []string{"get"},
Resources: []string{"secrets"},
}},
})
createAndCleanup(t, ctx, cl, &rbacv1.RoleBinding{
ObjectMeta: objectMeta("tailscale", "read-secrets"),
Subjects: []rbacv1.Subject{{
Kind: "Group",
Name: "ts:e2e-test-proxy",
}},
RoleRef: rbacv1.RoleRef{
Kind: "Role",
Name: "read-secrets",
},
})
// Get operator host name from kube secret.
operatorSecret := corev1.Secret{
ObjectMeta: objectMeta("tailscale", "operator"),
}
if err := get(ctx, cl, &operatorSecret); err != nil {
t.Fatal(err)
}
// Connect to tailnet with test-specific tag so we can use the
// [testGrants] ACLs when connecting to the API server proxy
ts := tsnetServerWithTag(t, ctx, "tag:e2e-test-proxy")
proxyCfg := &rest.Config{
Host: fmt.Sprintf("https://%s:443", hostNameFromOperatorSecret(t, operatorSecret)),
Dial: ts.Dial,
}
proxyCl, err := client.New(proxyCfg, client.Options{})
if err != nil {
t.Fatal(err)
}
// Expect success.
allowedSecret := corev1.Secret{
ObjectMeta: objectMeta("tailscale", "operator"),
}
// Wait for up to a minute the first time we use the proxy, to give it time
// to provision the TLS certs.
if err := tstest.WaitFor(time.Second*60, func() error {
return get(ctx, proxyCl, &allowedSecret)
}); err != nil {
t.Fatal(err)
}
// Expect forbidden.
forbiddenSecret := corev1.Secret{
ObjectMeta: objectMeta("default", "operator"),
}
if err := get(ctx, proxyCl, &forbiddenSecret); err == nil || !apierrors.IsForbidden(err) {
t.Fatalf("expected forbidden error fetching secret from default namespace: %s", err)
}
}
func tsnetServerWithTag(t *testing.T, ctx context.Context, tag string) *tsnet.Server {
caps := tailscale.KeyCapabilities{
Devices: tailscale.KeyDeviceCapabilities{
Create: tailscale.KeyDeviceCreateCapabilities{
Reusable: false,
Preauthorized: true,
Ephemeral: true,
Tags: []string{tag},
},
},
}
authKey, authKeyMeta, err := tsClient.CreateKey(ctx, caps)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := tsClient.DeleteKey(ctx, authKeyMeta.ID); err != nil {
t.Errorf("error deleting auth key: %s", err)
}
})
ts := &tsnet.Server{
Hostname: "test-proxy",
Ephemeral: true,
Dir: t.TempDir(),
AuthKey: authKey,
}
_, err = ts.Up(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := ts.Close(); err != nil {
t.Errorf("error shutting down tsnet.Server: %s", err)
}
})
return ts
}
func hostNameFromOperatorSecret(t *testing.T, s corev1.Secret) string {
profiles := map[string]any{}
if err := json.Unmarshal(s.Data["_profiles"], &profiles); err != nil {
t.Fatal(err)
}
key, ok := strings.CutPrefix(string(s.Data["_current-profile"]), "profile-")
if !ok {
t.Fatal(string(s.Data["_current-profile"]))
}
profile, ok := profiles[key]
if !ok {
t.Fatal(profiles)
}
return ((profile.(map[string]any))["Name"]).(string)
}

View File

@@ -112,7 +112,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
Terminating: pointer.ToBool(false),
},
})
expectEqual(t, fc, eps)
expectEqual(t, fc, eps, nil)
})
t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
_, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1
@@ -122,7 +122,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
})
expectReconciled(t, er, "operator-ns", "foo")
eps.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, eps)
expectEqual(t, fc, eps, nil)
})
}

View File

@@ -64,7 +64,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
oldStatus := svc.Status.DeepCopy()
defer func() {
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, st, reason, msg, esrr.clock, l)
if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
err = errors.Join(err, esrr.Status().Update(ctx, svc))
}
}()

View File

@@ -67,24 +67,24 @@ func TestEgressServiceReadiness(t *testing.T) {
setClusterNotReady(egressSvc, cl, zl.Sugar())
t.Run("endpointslice_does_not_exist", func(t *testing.T) {
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // not ready
expectEqual(t, fc, egressSvc, nil) // not ready
})
t.Run("proxy_group_does_not_exist", func(t *testing.T) {
mustCreate(t, fc, eps)
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // still not ready
expectEqual(t, fc, egressSvc, nil) // still not ready
})
t.Run("proxy_group_not_ready", func(t *testing.T) {
mustCreate(t, fc, pg)
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // still not ready
expectEqual(t, fc, egressSvc, nil) // still not ready
})
t.Run("no_ready_replicas", func(t *testing.T) {
setPGReady(pg, cl, zl.Sugar())
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
p.Status = pg.Status
})
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
for i := range pgReplicas(pg) {
p := pod(pg, i)
mustCreate(t, fc, p)
@@ -94,7 +94,7 @@ func TestEgressServiceReadiness(t *testing.T) {
}
expectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
expectEqual(t, fc, egressSvc) // still not ready
expectEqual(t, fc, egressSvc, nil) // still not ready
})
t.Run("one_ready_replica", func(t *testing.T) {
setEndpointForReplica(pg, 0, eps)
@@ -103,7 +103,7 @@ func TestEgressServiceReadiness(t *testing.T) {
})
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), 1)
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // partially ready
expectEqual(t, fc, egressSvc, nil) // partially ready
})
t.Run("all_replicas_ready", func(t *testing.T) {
for i := range pgReplicas(pg) {
@@ -114,7 +114,7 @@ func TestEgressServiceReadiness(t *testing.T) {
})
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc) // ready
expectEqual(t, fc, egressSvc, nil) // ready
})
}

View File

@@ -51,12 +51,12 @@ const (
labelSvcType = "tailscale.com/svc-type" // ingress or egress
typeEgress = "egress"
// maxPorts is the maximum number of ports that can be exposed on a
// container. In practice this will be ports in range [10000 - 11000). The
// container. In practice this will be ports in range [3000 - 4000). The
// high range should make it easier to distinguish container ports from
// the tailnet target ports for debugging purposes (i.e when reading
// netfilter rules). The limit of 1000 is somewhat arbitrary, the
// netfilter rules). The limit of 10000 is somewhat arbitrary, the
// assumption is that this would not be hit in practice.
maxPorts = 1000
maxPorts = 10000
indexEgressProxyGroup = ".metadata.annotations.egress-proxy-group"
)
@@ -123,7 +123,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
oldStatus := svc.Status.DeepCopy()
defer func() {
if !apiequality.Semantic.DeepEqual(oldStatus, &svc.Status) {
if !apiequality.Semantic.DeepEqual(oldStatus, svc.Status) {
err = errors.Join(err, esr.Status().Update(ctx, svc))
}
}()
@@ -136,8 +136,9 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
}
if !slices.Contains(svc.Finalizers, FinalizerName) {
l.Infof("configuring tailnet service") // logged exactly once
svc.Finalizers = append(svc.Finalizers, FinalizerName)
if err := esr.updateSvcSpec(ctx, svc); err != nil {
if err := esr.Update(ctx, svc); err != nil {
err := fmt.Errorf("failed to add finalizer: %w", err)
r := svcConfiguredReason(svc, false, l)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcConfigured, metav1.ConditionFalse, r, err.Error(), esr.clock, l)
@@ -156,15 +157,7 @@ func (esr *egressSvcsReconciler) Reconcile(ctx context.Context, req reconcile.Re
return res, err
}
if err := esr.maybeProvision(ctx, svc, l); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
l.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
}
}
return res, nil
return res, esr.maybeProvision(ctx, svc, l)
}
func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1.Service, l *zap.SugaredLogger) (err error) {
@@ -205,7 +198,7 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
if svc.Spec.ExternalName != clusterIPSvcFQDN {
l.Infof("Configuring ExternalName Service to point to ClusterIP Service %s", clusterIPSvcFQDN)
svc.Spec.ExternalName = clusterIPSvcFQDN
if err = esr.updateSvcSpec(ctx, svc); err != nil {
if err = esr.Update(ctx, svc); err != nil {
err = fmt.Errorf("error updating ExternalName Service: %w", err)
return err
}
@@ -229,15 +222,6 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
found := false
for _, wantsPM := range svc.Spec.Ports {
if wantsPM.Port == pm.Port && strings.EqualFold(string(wantsPM.Protocol), string(pm.Protocol)) {
// We don't use the port name to distinguish this port internally, but Kubernetes
// require that, for Service ports with more than one name each port is uniquely named.
// So we can always pick the port name from the ExternalName Service as at this point we
// know that those are valid names because Kuberentes already validated it once. Note
// that users could have changed an unnamed port to a named port and might have changed
// port names- this should still work.
// https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services
// See also https://github.com/tailscale/tailscale/issues/13406#issuecomment-2507230388
clusterIPSvc.Spec.Ports[i].Name = wantsPM.Name
found = true
break
}
@@ -262,7 +246,7 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
if !found {
// Calculate a free port to expose on container and add
// a new PortMap to the ClusterIP Service.
if usedPorts.Len() >= maxPorts {
if usedPorts.Len() == maxPorts {
// TODO(irbekrm): refactor to avoid extra reconciles here. Low priority as in practice,
// the limit should not be hit.
return nil, false, fmt.Errorf("unable to allocate additional ports on ProxyGroup %s, %d ports already used. Create another ProxyGroup or open an issue if you believe this is unexpected.", proxyGroupName, maxPorts)
@@ -495,6 +479,13 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, err
}
if !tsoperator.ProxyGroupIsReady(pg) {
l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
}
if violations := validateEgressService(svc, pg); len(violations) > 0 {
msg := fmt.Sprintf("invalid egress Service: %s", strings.Join(violations, ", "))
esr.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg)
@@ -503,13 +494,6 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
}
if !tsoperator.ProxyGroupIsReady(pg) {
l.Infof("ProxyGroup %s is not ready, waiting...", proxyGroupName)
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l)
tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured)
return false, nil
}
l.Debugf("egress service is valid")
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionTrue, reasonEgressSvcValid, reasonEgressSvcValid, esr.clock, l)
return true, nil
@@ -556,13 +540,13 @@ func svcNameBase(s string) string {
}
}
// unusedPort returns a port in range [10000 - 11000). The caller must ensure that
// usedPorts does not contain all ports in range [10000 - 11000).
// unusedPort returns a port in range [3000 - 4000). The caller must ensure that
// usedPorts does not contain all ports in range [3000 - 4000).
func unusedPort(usedPorts sets.Set[int32]) int32 {
foundFreePort := false
var suggestPort int32
for !foundFreePort {
suggestPort = rand.Int32N(maxPorts) + 10000
suggestPort = rand.Int32N(maxPorts) + 3000
if !usedPorts.Has(suggestPort) {
foundFreePort = true
}
@@ -730,13 +714,3 @@ func epsPortsFromSvc(svc *corev1.Service) (ep []discoveryv1.EndpointPort) {
}
return ep
}
// updateSvcSpec ensures that the given Service's spec is updated in cluster, but the local Service object still retains
// the not-yet-applied status.
// TODO(irbekrm): once we do SSA for these patch updates, this will no longer be needed.
func (esr *egressSvcsReconciler) updateSvcSpec(ctx context.Context, svc *corev1.Service) error {
st := svc.Status.DeepCopy()
err := esr.Update(ctx, svc)
svc.Status = *st
return err
}

View File

@@ -96,7 +96,7 @@ func TestTailscaleEgressServices(t *testing.T) {
expectReconciled(t, esr, "default", "test")
// Service should have EgressSvcValid condition set to Unknown.
svc.Status.Conditions = []metav1.Condition{condition(tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, clock)}
expectEqual(t, fc, svc)
expectEqual(t, fc, svc, nil)
})
t.Run("proxy_group_ready", func(t *testing.T) {
@@ -105,40 +105,28 @@ func TestTailscaleEgressServices(t *testing.T) {
condition(tsapi.ProxyGroupReady, metav1.ConditionTrue, "", "", clock),
}
})
expectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_retain_one_unnamed_port", func(t *testing.T) {
svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80}}
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
// Quirks of the fake client.
mustUpdateStatus(t, fc, "default", "test", func(svc *corev1.Service) {
svc.Status.Conditions = []metav1.Condition{}
})
expectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_add_two_named_ports", func(t *testing.T) {
svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}}
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
})
expectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_add_udp_port", func(t *testing.T) {
svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Port: 53, Protocol: "UDP", Name: "dns"})
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
})
expectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
})
t.Run("service_change_protocol", func(t *testing.T) {
svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}, {Port: 53, Protocol: "TCP", Name: "tcp_dns"}}
mustUpdate(t, fc, "default", "test", func(s *corev1.Service) {
s.Spec.Ports = svc.Spec.Ports
})
expectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
// Verify that a ClusterIP Service has been created.
name := findGenNameForEgressSvcResources(t, fc, svc)
expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
// Verify that an EndpointSlice has been created.
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc), nil)
// Verify that ConfigMap contains configuration for the new egress service.
mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
r := svcConfiguredReason(svc, true, zl.Sugar())
// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
// CluterIP Service.
svc.Status.Conditions = []metav1.Condition{
condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
}
svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
expectEqual(t, fc, svc, nil)
})
t.Run("delete_external_name_service", func(t *testing.T) {
@@ -155,29 +143,6 @@ func TestTailscaleEgressServices(t *testing.T) {
})
}
func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) {
expectReconciled(t, esr, "default", "test")
// Verify that a ClusterIP Service has been created.
name := findGenNameForEgressSvcResources(t, fc, svc)
expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
// Verify that an EndpointSlice has been created.
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc))
// Verify that ConfigMap contains configuration for the new egress service.
mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm)
r := svcConfiguredReason(svc, true, zl.Sugar())
// Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the
// CluterIP Service.
svc.Status.Conditions = []metav1.Condition{
condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "EgressSvcValid", "EgressSvcValid", clock),
condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock),
}
svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name)
expectEqual(t, fc, svc)
}
func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition {
return metav1.Condition{
Type: string(typ),

View File

@@ -1,569 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"slices"
"strings"
"sync"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
"tailscale.com/tailcfg"
"tailscale.com/util/clientmetric"
"tailscale.com/util/dnsname"
"tailscale.com/util/mak"
"tailscale.com/util/set"
)
const (
serveConfigKey = "serve-config.json"
VIPSvcOwnerRef = "tailscale.com/k8s-operator:owned-by:%s"
// FinalizerNamePG is the finalizer used by the IngressPGReconciler
FinalizerNamePG = "tailscale.com/ingress-pg-finalizer"
)
var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGResourceCount)
// IngressPGReconciler is a controller that reconciles Tailscale Ingresses should be exposed on an ingress ProxyGroup
// (in HA mode).
type IngressPGReconciler struct {
client.Client
recorder record.EventRecorder
logger *zap.SugaredLogger
tsClient tsClient
tsnetServer tsnetServer
tsNamespace string
lc localClient
defaultTags []string
mu sync.Mutex // protects following
// managedIngresses is a set of all ingress resources that we're currently
// managing. This is only used for metrics.
managedIngresses set.Slice[types.UID]
}
// Reconcile reconciles Ingresses that should be exposed over Tailscale in HA mode (on a ProxyGroup). It looks at all
// Ingresses with tailscale.com/proxy-group annotation. For each such Ingress, it ensures that a VIPService named after
// the hostname of the Ingress exists and is up to date. It also ensures that the serve config for the ingress
// ProxyGroup is updated to route traffic for the VIPService to the Ingress's backend Services.
// When an Ingress is deleted or unexposed, the VIPService and the associated serve config are cleaned up.
// Ingress hostname change also results in the VIPService for the previous hostname being cleaned up and a new VIPService
// being created for the new hostname.
func (a *IngressPGReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
logger := a.logger.With("Ingress", req.NamespacedName)
logger.Debugf("starting reconcile")
defer logger.Debugf("reconcile finished")
ing := new(networkingv1.Ingress)
err = a.Get(ctx, req.NamespacedName, ing)
if apierrors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
logger.Debugf("Ingress not found, assuming it was deleted")
return res, nil
} else if err != nil {
return res, fmt.Errorf("failed to get Ingress: %w", err)
}
// hostname is the name of the VIPService that will be created for this Ingress as well as the first label in
// the MagicDNS name of the Ingress.
hostname := hostnameForIngress(ing)
logger = logger.With("hostname", hostname)
if !ing.DeletionTimestamp.IsZero() || !a.shouldExpose(ing) {
return res, a.maybeCleanup(ctx, hostname, ing, logger)
}
if err := a.maybeProvision(ctx, hostname, ing, logger); err != nil {
return res, fmt.Errorf("failed to provision: %w", err)
}
return res, nil
}
// maybeProvision ensures that the VIPService and serve config for the Ingress are created or updated.
func (a *IngressPGReconciler) maybeProvision(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
if err := validateIngressClass(ctx, a.Client); err != nil {
logger.Infof("error validating tailscale IngressClass: %v.", err)
return nil
}
// Get and validate ProxyGroup readiness
pgName := ing.Annotations[AnnotationProxyGroup]
if pgName == "" {
logger.Infof("[unexpected] no ProxyGroup annotation, skipping VIPService provisioning")
return nil
}
pg := &tsapi.ProxyGroup{}
if err := a.Get(ctx, client.ObjectKey{Name: pgName}, pg); err != nil {
if apierrors.IsNotFound(err) {
logger.Infof("ProxyGroup %q does not exist", pgName)
return nil
}
return fmt.Errorf("getting ProxyGroup %q: %w", pgName, err)
}
if !tsoperator.ProxyGroupIsReady(pg) {
// TODO(irbekrm): we need to reconcile ProxyGroup Ingresses on ProxyGroup changes to not miss the status update
// in this case.
logger.Infof("ProxyGroup %q is not ready", pgName)
return nil
}
// Validate Ingress configuration
if err := a.validateIngress(ing, pg); err != nil {
logger.Infof("invalid Ingress configuration: %v", err)
a.recorder.Event(ing, corev1.EventTypeWarning, "InvalidIngressConfiguration", err.Error())
return nil
}
if !IsHTTPSEnabledOnTailnet(a.tsnetServer) {
a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
}
logger = logger.With("proxy-group", pg)
if !slices.Contains(ing.Finalizers, FinalizerNamePG) {
// This log line is printed exactly once during initial provisioning,
// because once the finalizer is in place this block gets skipped. So,
// this is a nice place to tell the operator that the high level,
// multi-reconcile operation is underway.
logger.Infof("exposing Ingress over tailscale")
ing.Finalizers = append(ing.Finalizers, FinalizerNamePG)
if err := a.Update(ctx, ing); err != nil {
return fmt.Errorf("failed to add finalizer: %w", err)
}
a.mu.Lock()
a.managedIngresses.Add(ing.UID)
gaugePGIngressResources.Set(int64(a.managedIngresses.Len()))
a.mu.Unlock()
}
// 1. Ensure that if Ingress' hostname has changed, any VIPService resources corresponding to the old hostname
// are cleaned up.
// In practice, this function will ensure that any VIPServices that are associated with the provided ProxyGroup
// and no longer owned by an Ingress are cleaned up. This is fine- it is not expensive and ensures that in edge
// cases (a single update changed both hostname and removed ProxyGroup annotation) the VIPService is more likely
// to be (eventually) removed.
if err := a.maybeCleanupProxyGroup(ctx, pgName, logger); err != nil {
return fmt.Errorf("failed to cleanup VIPService resources for ProxyGroup: %w", err)
}
// 2. Ensure that there isn't a VIPService with the same hostname already created and not owned by this Ingress.
// TODO(irbekrm): perhaps in future we could have record names being stored on VIPServices. I am not certain if
// there might not be edge cases (custom domains, etc?) where attempting to determine the DNS name of the
// VIPService in this way won't be incorrect.
tcd, err := a.tailnetCertDomain(ctx)
if err != nil {
return fmt.Errorf("error determining DNS name base: %w", err)
}
dnsName := hostname + "." + tcd
existingVIPSvc, err := a.tsClient.getVIPServiceByName(ctx, hostname)
// TODO(irbekrm): here and when creating the VIPService, verify if the error is not terminal (and therefore
// should not be reconciled). For example, if the hostname is already a hostname of a Tailscale node, the GET
// here will fail.
if err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound {
return fmt.Errorf("error getting VIPService %q: %w", hostname, err)
}
}
if existingVIPSvc != nil && !isVIPServiceForIngress(existingVIPSvc, ing) {
logger.Infof("VIPService %q for MagicDNS name %q already exists, but is not owned by this Ingress. Please delete it manually and recreate this Ingress to proceed or create an Ingress for a different MagicDNS name", hostname, dnsName)
a.recorder.Event(ing, corev1.EventTypeWarning, "ConflictingVIPServiceExists", fmt.Sprintf("VIPService %q for MagicDNS name %q already exists, but is not owned by this Ingress. Please delete it manually to proceed or create an Ingress for a different MagicDNS name", hostname, dnsName))
return nil
}
// 3. Ensure that the serve config for the ProxyGroup contains the VIPService
cm, cfg, err := a.proxyGroupServeConfig(ctx, pgName)
if err != nil {
return fmt.Errorf("error getting ingress serve config: %w", err)
}
if cm == nil {
logger.Infof("no ingress serve config ConfigMap found, unable to update serve config. Ensure that ProxyGroup is healthy.")
return nil
}
ep := ipn.HostPort(fmt.Sprintf("%s:443", dnsName))
handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, dnsName, logger)
if err != nil {
return fmt.Errorf("failed to get handlers for ingress: %w", err)
}
ingCfg := &ipn.ServiceConfig{
TCP: map[uint16]*ipn.TCPPortHandler{
443: {
HTTPS: true,
},
},
Web: map[ipn.HostPort]*ipn.WebServerConfig{
ep: {
Handlers: handlers,
},
},
}
serviceName := tailcfg.ServiceName("svc:" + hostname)
var gotCfg *ipn.ServiceConfig
if cfg != nil && cfg.Services != nil {
gotCfg = cfg.Services[serviceName]
}
if !reflect.DeepEqual(gotCfg, ingCfg) {
logger.Infof("Updating serve config")
mak.Set(&cfg.Services, serviceName, ingCfg)
cfgBytes, err := json.Marshal(cfg)
if err != nil {
return fmt.Errorf("error marshaling serve config: %w", err)
}
mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
if err := a.Update(ctx, cm); err != nil {
return fmt.Errorf("error updating serve config: %w", err)
}
}
// 4. Ensure that the VIPService exists and is up to date.
tags := a.defaultTags
if tstr, ok := ing.Annotations[AnnotationTags]; ok {
tags = strings.Split(tstr, ",")
}
vipSvc := &VIPService{
Name: hostname,
Tags: tags,
Ports: []string{"443"}, // always 443 for Ingress
Comment: fmt.Sprintf(VIPSvcOwnerRef, ing.UID),
}
if existingVIPSvc != nil {
vipSvc.Addrs = existingVIPSvc.Addrs
}
if existingVIPSvc == nil || !reflect.DeepEqual(vipSvc.Tags, existingVIPSvc.Tags) {
logger.Infof("Ensuring VIPService %q exists and is up to date", hostname)
if err := a.tsClient.createOrUpdateVIPServiceByName(ctx, vipSvc); err != nil {
logger.Infof("error creating VIPService: %v", err)
return fmt.Errorf("error creating VIPService: %w", err)
}
}
// 5. Update Ingress status
oldStatus := ing.Status.DeepCopy()
// TODO(irbekrm): once we have ingress ProxyGroup, we can determine if instances are ready to route traffic to the VIPService
ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
{
Hostname: dnsName,
Ports: []networkingv1.IngressPortStatus{
{
Protocol: "TCP",
Port: 443,
},
},
},
}
if apiequality.Semantic.DeepEqual(oldStatus, ing.Status) {
return nil
}
if err := a.Status().Update(ctx, ing); err != nil {
return fmt.Errorf("failed to update Ingress status: %w", err)
}
return nil
}
// maybeCleanupProxyGroup ensures that if an Ingress hostname has changed, any VIPService resources created for the
// Ingress' ProxyGroup corresponding to the old hostname are cleaned up. A run of this function will ensure that any
// VIPServices that are associated with the provided ProxyGroup and no longer owned by an Ingress are cleaned up.
func (a *IngressPGReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyGroupName string, logger *zap.SugaredLogger) error {
// Get serve config for the ProxyGroup
cm, cfg, err := a.proxyGroupServeConfig(ctx, proxyGroupName)
if err != nil {
return fmt.Errorf("getting serve config: %w", err)
}
if cfg == nil {
return nil // ProxyGroup does not have any VIPServices
}
ingList := &networkingv1.IngressList{}
if err := a.List(ctx, ingList); err != nil {
return fmt.Errorf("listing Ingresses: %w", err)
}
serveConfigChanged := false
// For each VIPService in serve config...
for vipHostname := range cfg.Services {
// ...check if there is currently an Ingress with this hostname
found := false
for _, i := range ingList.Items {
ingressHostname := hostnameForIngress(&i)
if ingressHostname == vipHostname.WithoutPrefix() {
found = true
break
}
}
if !found {
logger.Infof("VIPService %q is not owned by any Ingress, cleaning up", vipHostname)
svc, err := a.getVIPService(ctx, vipHostname.WithoutPrefix(), logger)
if err != nil {
errResp := &tailscale.ErrResponse{}
if errors.As(err, &errResp) && errResp.Status == http.StatusNotFound {
delete(cfg.Services, vipHostname)
serveConfigChanged = true
continue
}
return err
}
if isVIPServiceForAnyIngress(svc) {
logger.Infof("cleaning up orphaned VIPService %q", vipHostname)
if err := a.tsClient.deleteVIPServiceByName(ctx, vipHostname.WithoutPrefix()); err != nil {
errResp := &tailscale.ErrResponse{}
if !errors.As(err, &errResp) || errResp.Status != http.StatusNotFound {
return fmt.Errorf("deleting VIPService %q: %w", vipHostname, err)
}
}
}
delete(cfg.Services, vipHostname)
serveConfigChanged = true
}
}
if serveConfigChanged {
cfgBytes, err := json.Marshal(cfg)
if err != nil {
return fmt.Errorf("marshaling serve config: %w", err)
}
mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
if err := a.Update(ctx, cm); err != nil {
return fmt.Errorf("updating serve config: %w", err)
}
}
return nil
}
// maybeCleanup ensures that any resources, such as a VIPService created for this Ingress, are cleaned up when the
// Ingress is being deleted or is unexposed.
func (a *IngressPGReconciler) maybeCleanup(ctx context.Context, hostname string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
logger.Debugf("Ensuring any resources for Ingress are cleaned up")
ix := slices.Index(ing.Finalizers, FinalizerNamePG)
if ix < 0 {
logger.Debugf("no finalizer, nothing to do")
a.mu.Lock()
defer a.mu.Unlock()
a.managedIngresses.Remove(ing.UID)
gaugePGIngressResources.Set(int64(a.managedIngresses.Len()))
return nil
}
// 1. Check if there is a VIPService created for this Ingress.
pg := ing.Annotations[AnnotationProxyGroup]
cm, cfg, err := a.proxyGroupServeConfig(ctx, pg)
if err != nil {
return fmt.Errorf("error getting ProxyGroup serve config: %w", err)
}
serviceName := tailcfg.ServiceName("svc:" + hostname)
// VIPService is always first added to serve config and only then created in the Tailscale API, so if it is not
// found in the serve config, we can assume that there is no VIPService. TODO(irbekrm): once we have ingress
// ProxyGroup, we will probably add currently exposed VIPServices to its status. At that point, we can use the
// status rather than checking the serve config each time.
if cfg == nil || cfg.Services == nil || cfg.Services[serviceName] == nil {
return nil
}
logger.Infof("Ensuring that VIPService %q configuration is cleaned up", hostname)
// 2. Delete the VIPService.
if err := a.deleteVIPServiceIfExists(ctx, hostname, ing, logger); err != nil {
return fmt.Errorf("error deleting VIPService: %w", err)
}
// 3. Remove the VIPService from the serve config for the ProxyGroup.
logger.Infof("Removing VIPService %q from serve config for ProxyGroup %q", hostname, pg)
delete(cfg.Services, serviceName)
cfgBytes, err := json.Marshal(cfg)
if err != nil {
return fmt.Errorf("error marshaling serve config: %w", err)
}
mak.Set(&cm.BinaryData, serveConfigKey, cfgBytes)
if err := a.Update(ctx, cm); err != nil {
return fmt.Errorf("error updating ConfigMap %q: %w", cm.Name, err)
}
if err := a.deleteFinalizer(ctx, ing, logger); err != nil {
return fmt.Errorf("failed to remove finalizer: %w", err)
}
a.mu.Lock()
defer a.mu.Unlock()
a.managedIngresses.Remove(ing.UID)
gaugePGIngressResources.Set(int64(a.managedIngresses.Len()))
return nil
}
func (a *IngressPGReconciler) deleteFinalizer(ctx context.Context, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
found := false
ing.Finalizers = slices.DeleteFunc(ing.Finalizers, func(f string) bool {
found = true
return f == FinalizerNamePG
})
if !found {
return nil
}
logger.Debug("ensure %q finalizer is removed", FinalizerNamePG)
if err := a.Update(ctx, ing); err != nil {
return fmt.Errorf("failed to remove finalizer %q: %w", FinalizerNamePG, err)
}
return nil
}
func pgIngressCMName(pg string) string {
return fmt.Sprintf("%s-ingress-config", pg)
}
func (a *IngressPGReconciler) proxyGroupServeConfig(ctx context.Context, pg string) (cm *corev1.ConfigMap, cfg *ipn.ServeConfig, err error) {
name := pgIngressCMName(pg)
cm = &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: a.tsNamespace,
},
}
if err := a.Get(ctx, client.ObjectKeyFromObject(cm), cm); err != nil && !apierrors.IsNotFound(err) {
return nil, nil, fmt.Errorf("error retrieving ingress serve config ConfigMap %s: %v", name, err)
}
if apierrors.IsNotFound(err) {
return nil, nil, nil
}
cfg = &ipn.ServeConfig{}
if len(cm.BinaryData[serveConfigKey]) != 0 {
if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
return nil, nil, fmt.Errorf("error unmarshaling ingress serve config %v: %w", cm.BinaryData[serveConfigKey], err)
}
}
return cm, cfg, nil
}
type localClient interface {
StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error)
}
// tailnetCertDomain returns the base domain (TCD) of the current tailnet.
func (a *IngressPGReconciler) tailnetCertDomain(ctx context.Context) (string, error) {
st, err := a.lc.StatusWithoutPeers(ctx)
if err != nil {
return "", fmt.Errorf("error getting tailscale status: %w", err)
}
return st.CurrentTailnet.MagicDNSSuffix, nil
}
// shouldExpose returns true if the Ingress should be exposed over Tailscale in HA mode (on a ProxyGroup)
func (a *IngressPGReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
isTSIngress := ing != nil &&
ing.Spec.IngressClassName != nil &&
*ing.Spec.IngressClassName == tailscaleIngressClassName
pgAnnot := ing.Annotations[AnnotationProxyGroup]
return isTSIngress && pgAnnot != ""
}
func (a *IngressPGReconciler) getVIPService(ctx context.Context, hostname string, logger *zap.SugaredLogger) (*VIPService, error) {
svc, err := a.tsClient.getVIPServiceByName(ctx, hostname)
if err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status != http.StatusNotFound {
logger.Infof("error getting VIPService %q: %v", hostname, err)
return nil, fmt.Errorf("error getting VIPService %q: %w", hostname, err)
}
}
return svc, nil
}
func isVIPServiceForIngress(svc *VIPService, ing *networkingv1.Ingress) bool {
if svc == nil || ing == nil {
return false
}
return strings.EqualFold(svc.Comment, fmt.Sprintf(VIPSvcOwnerRef, ing.UID))
}
func isVIPServiceForAnyIngress(svc *VIPService) bool {
if svc == nil {
return false
}
return strings.HasPrefix(svc.Comment, "tailscale.com/k8s-operator:owned-by:")
}
// validateIngress validates that the Ingress is properly configured.
// Currently validates:
// - Any tags provided via tailscale.com/tags annotation are valid Tailscale ACL tags
// - The derived hostname is a valid DNS label
// - The referenced ProxyGroup exists and is of type 'ingress'
// - Ingress' TLS block is invalid
func (a *IngressPGReconciler) validateIngress(ing *networkingv1.Ingress, pg *tsapi.ProxyGroup) error {
var errs []error
// Validate tags if present
if tstr, ok := ing.Annotations[AnnotationTags]; ok {
tags := strings.Split(tstr, ",")
for _, tag := range tags {
tag = strings.TrimSpace(tag)
if err := tailcfg.CheckTag(tag); err != nil {
errs = append(errs, fmt.Errorf("tailscale.com/tags annotation contains invalid tag %q: %w", tag, err))
}
}
}
// Validate TLS configuration
if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) {
errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS))
}
// Validate that the hostname will be a valid DNS label
hostname := hostnameForIngress(ing)
if err := dnsname.ValidLabel(hostname); err != nil {
errs = append(errs, fmt.Errorf("invalid hostname %q: %w. Ensure that the hostname is a valid DNS label", hostname, err))
}
// Validate ProxyGroup type
if pg.Spec.Type != tsapi.ProxyGroupTypeIngress {
errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q",
pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress))
}
// Validate ProxyGroup readiness
if !tsoperator.ProxyGroupIsReady(pg) {
errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name))
}
return errors.Join(errs...)
}
// deleteVIPServiceIfExists attempts to delete the VIPService if it exists and is owned by the given Ingress.
func (a *IngressPGReconciler) deleteVIPServiceIfExists(ctx context.Context, name string, ing *networkingv1.Ingress, logger *zap.SugaredLogger) error {
svc, err := a.getVIPService(ctx, name, logger)
if err != nil {
return fmt.Errorf("error getting VIPService: %w", err)
}
// isVIPServiceForIngress handles nil svc, so we don't need to check it here
if !isVIPServiceForIngress(svc, ing) {
return nil
}
logger.Infof("Deleting VIPService %q", name)
if err = a.tsClient.deleteVIPServiceByName(ctx, name); err != nil {
return fmt.Errorf("error deleting VIPService: %w", err)
}
return nil
}

View File

@@ -1,337 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"encoding/json"
"testing"
"slices"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/types/ptr"
)
func TestIngressPGReconciler(t *testing.T) {
tsIngressClass := &networkingv1.IngressClass{
ObjectMeta: metav1.ObjectMeta{Name: "tailscale"},
Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"},
}
// Pre-create the ProxyGroup
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg",
Generation: 1,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
},
}
// Pre-create the ConfigMap for the ProxyGroup
pgConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg-ingress-config",
Namespace: "operator-ns",
},
BinaryData: map[string][]byte{
"serve-config.json": []byte(`{"Services":{}}`),
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, pgConfigMap, tsIngressClass).
WithStatusSubresource(pg).
Build()
mustUpdateStatus(t, fc, "", pg.Name, func(pg *tsapi.ProxyGroup) {
pg.Status.Conditions = []metav1.Condition{
{
Type: string(tsapi.ProxyGroupReady),
Status: metav1.ConditionTrue,
ObservedGeneration: 1,
},
}
})
ft := &fakeTSClient{}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
lc := &fakeLocalClient{
status: &ipnstate.Status{
CurrentTailnet: &ipnstate.TailnetStatus{
MagicDNSSuffix: "ts.net",
},
},
}
ingPGR := &IngressPGReconciler{
Client: fc,
tsClient: ft,
tsnetServer: fakeTsnetServer,
defaultTags: []string{"tag:k8s"},
tsNamespace: "operator-ns",
logger: zl.Sugar(),
recorder: record.NewFakeRecorder(10),
lc: lc,
}
// Test 1: Default tags
ing := &networkingv1.Ingress{
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
"tailscale.com/proxy-group": "test-pg",
},
},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To("tailscale"),
DefaultBackend: &networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: "test",
Port: networkingv1.ServiceBackendPort{
Number: 8080,
},
},
},
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"my-svc.tailnetxyz.ts.net"}},
},
},
}
mustCreate(t, fc, ing)
// Verify initial reconciliation
expectReconciled(t, ingPGR, "default", "test-ingress")
// Get and verify the ConfigMap was updated
cm := &corev1.ConfigMap{}
if err := fc.Get(context.Background(), types.NamespacedName{
Name: "test-pg-ingress-config",
Namespace: "operator-ns",
}, cm); err != nil {
t.Fatalf("getting ConfigMap: %v", err)
}
cfg := &ipn.ServeConfig{}
if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
t.Fatalf("unmarshaling serve config: %v", err)
}
if cfg.Services["svc:my-svc"] == nil {
t.Error("expected serve config to contain VIPService configuration")
}
// Verify VIPService uses default tags
vipSvc, err := ft.getVIPServiceByName(context.Background(), "my-svc")
if err != nil {
t.Fatalf("getting VIPService: %v", err)
}
if vipSvc == nil {
t.Fatal("VIPService not created")
}
wantTags := []string{"tag:k8s"} // default tags
if !slices.Equal(vipSvc.Tags, wantTags) {
t.Errorf("incorrect VIPService tags: got %v, want %v", vipSvc.Tags, wantTags)
}
// Test 2: Custom tags
mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) {
ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test"
})
expectReconciled(t, ingPGR, "default", "test-ingress")
// Verify VIPService uses custom tags
vipSvc, err = ft.getVIPServiceByName(context.Background(), "my-svc")
if err != nil {
t.Fatalf("getting VIPService: %v", err)
}
if vipSvc == nil {
t.Fatal("VIPService not created")
}
wantTags = []string{"tag:custom", "tag:test"} // custom tags only
gotTags := slices.Clone(vipSvc.Tags)
slices.Sort(gotTags)
slices.Sort(wantTags)
if !slices.Equal(gotTags, wantTags) {
t.Errorf("incorrect VIPService tags: got %v, want %v", gotTags, wantTags)
}
// Delete the Ingress and verify cleanup
if err := fc.Delete(context.Background(), ing); err != nil {
t.Fatalf("deleting Ingress: %v", err)
}
expectReconciled(t, ingPGR, "default", "test-ingress")
// Verify the ConfigMap was cleaned up
cm = &corev1.ConfigMap{}
if err := fc.Get(context.Background(), types.NamespacedName{
Name: "test-pg-ingress-config",
Namespace: "operator-ns",
}, cm); err != nil {
t.Fatalf("getting ConfigMap: %v", err)
}
cfg = &ipn.ServeConfig{}
if err := json.Unmarshal(cm.BinaryData[serveConfigKey], cfg); err != nil {
t.Fatalf("unmarshaling serve config: %v", err)
}
if len(cfg.Services) > 0 {
t.Error("serve config not cleaned up")
}
}
func TestValidateIngress(t *testing.T) {
baseIngress := &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress",
Namespace: "default",
},
}
readyProxyGroup := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pg",
Generation: 1,
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
},
Status: tsapi.ProxyGroupStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ProxyGroupReady),
Status: metav1.ConditionTrue,
ObservedGeneration: 1,
},
},
},
}
tests := []struct {
name string
ing *networkingv1.Ingress
pg *tsapi.ProxyGroup
wantErr string
}{
{
name: "valid_ingress_with_hostname",
ing: &networkingv1.Ingress{
ObjectMeta: baseIngress.ObjectMeta,
Spec: networkingv1.IngressSpec{
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"test.example.com"}},
},
},
},
pg: readyProxyGroup,
},
{
name: "valid_ingress_with_default_hostname",
ing: baseIngress,
pg: readyProxyGroup,
},
{
name: "invalid_tags",
ing: &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: baseIngress.Name,
Namespace: baseIngress.Namespace,
Annotations: map[string]string{
AnnotationTags: "tag:invalid!",
},
},
},
pg: readyProxyGroup,
wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes",
},
{
name: "multiple_TLS_entries",
ing: &networkingv1.Ingress{
ObjectMeta: baseIngress.ObjectMeta,
Spec: networkingv1.IngressSpec{
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"test1.example.com"}},
{Hosts: []string{"test2.example.com"}},
},
},
},
pg: readyProxyGroup,
wantErr: "Ingress contains invalid TLS block [{[test1.example.com] } {[test2.example.com] }]: only a single TLS entry with a single host is allowed",
},
{
name: "multiple_hosts_in_TLS_entry",
ing: &networkingv1.Ingress{
ObjectMeta: baseIngress.ObjectMeta,
Spec: networkingv1.IngressSpec{
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"test1.example.com", "test2.example.com"}},
},
},
},
pg: readyProxyGroup,
wantErr: "Ingress contains invalid TLS block [{[test1.example.com test2.example.com] }]: only a single TLS entry with a single host is allowed",
},
{
name: "wrong_proxy_group_type",
ing: baseIngress,
pg: &tsapi.ProxyGroup{
ObjectMeta: readyProxyGroup.ObjectMeta,
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupType("foo"),
},
Status: readyProxyGroup.Status,
},
wantErr: "ProxyGroup \"test-pg\" is of type \"foo\" but must be of type \"ingress\"",
},
{
name: "proxy_group_not_ready",
ing: baseIngress,
pg: &tsapi.ProxyGroup{
ObjectMeta: readyProxyGroup.ObjectMeta,
Spec: readyProxyGroup.Spec,
Status: tsapi.ProxyGroupStatus{
Conditions: []metav1.Condition{
{
Type: string(tsapi.ProxyGroupReady),
Status: metav1.ConditionFalse,
ObservedGeneration: 1,
},
},
},
},
wantErr: "ProxyGroup \"test-pg\" is not ready",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &IngressPGReconciler{}
err := r.validateIngress(tt.ing, tt.pg)
if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) {
t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@@ -26,7 +26,6 @@ import (
"tailscale.com/kube/kubetypes"
"tailscale.com/types/opt"
"tailscale.com/util/clientmetric"
"tailscale.com/util/mak"
"tailscale.com/util/set"
)
@@ -59,7 +58,7 @@ var (
)
func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
logger := a.logger.With("Ingress", req.NamespacedName)
logger := a.logger.With("ingress-ns", req.Namespace, "ingress-name", req.Name)
logger.Debugf("starting reconcile")
defer logger.Debugf("reconcile finished")
@@ -77,15 +76,7 @@ func (a *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
return reconcile.Result{}, a.maybeCleanup(ctx, logger, ing)
}
if err := a.maybeProvision(ctx, logger, ing); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
return reconcile.Result{}, a.maybeProvision(ctx, logger, ing)
}
func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
@@ -99,7 +90,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
return nil
}
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil {
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress")); err != nil {
return fmt.Errorf("failed to cleanup: %w", err)
} else if !done {
logger.Debugf("cleanup not done yet, waiting for next reconcile")
@@ -129,8 +120,9 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
// This function adds a finalizer to ing, ensuring that we can handle orderly
// deprovisioning later.
func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error {
if err := validateIngressClass(ctx, a.Client); err != nil {
if err := a.validateIngressClass(ctx); err != nil {
logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err)
}
if !slices.Contains(ing.Finalizers, FinalizerName) {
// This log line is printed exactly once during initial provisioning,
@@ -159,7 +151,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
gaugeIngressResources.Set(int64(a.managedIngresses.Len()))
a.mu.Unlock()
if !IsHTTPSEnabledOnTailnet(a.ssr.tsnetServer) {
if !a.ssr.IsHTTPSEnabledOnTailnet() {
a.recorder.Event(ing, corev1.EventTypeWarning, "HTTPSNotEnabled", "HTTPS is not enabled on the tailnet; ingress may not work")
}
@@ -185,16 +177,73 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
}
web := sc.Web[magic443]
addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
if b == nil {
return
}
if b.Service == nil {
a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
return
}
var svc corev1.Service
if err := a.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
return
}
if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
return
}
var port int32
if b.Service.Port.Name != "" {
for _, p := range svc.Spec.Ports {
if p.Name == b.Service.Port.Name {
port = p.Port
break
}
}
} else {
port = b.Service.Port.Number
}
if port == 0 {
a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
return
}
proto := "http://"
if port == 443 || b.Service.Port.Name == "https" {
proto = "https+insecure://"
}
web.Handlers[path] = &ipn.HTTPHandler{
Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
}
}
addIngressBackend(ing.Spec.DefaultBackend, "/")
var tlsHost string // hostname or FQDN or empty
if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
tlsHost = ing.Spec.TLS[0].Hosts[0]
}
handlers, err := handlersForIngress(ctx, ing, a.Client, a.recorder, tlsHost, logger)
if err != nil {
return fmt.Errorf("failed to get handlers for ingress: %w", err)
for _, rule := range ing.Spec.Rules {
// Host is optional, but if it's present it must match the TLS host
// otherwise we ignore the rule.
if rule.Host != "" && rule.Host != tlsHost {
a.recorder.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
continue
}
for _, p := range rule.HTTP.Paths {
// Send a warning if folks use Exact path type - to make
// it easier for us to support Exact path type matching
// in the future if needed.
// https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types
if *p.PathType == networkingv1.PathTypeExact {
msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future."
logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg))
a.recorder.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg)
}
addIngressBackend(&p.Backend, p.Path)
}
}
web.Handlers = handlers
if len(web.Handlers) == 0 {
logger.Warn("Ingress contains no valid backends")
a.recorder.Eventf(ing, corev1.EventTypeWarning, "NoValidBackends", "no valid backends")
@@ -206,7 +255,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
if tstr, ok := ing.Annotations[AnnotationTags]; ok {
tags = strings.Split(tstr, ",")
}
hostname := hostnameForIngress(ing)
hostname := ing.Namespace + "-" + ing.Name + "-ingress"
if tlsHost != "" {
hostname, _, _ = strings.Cut(tlsHost, ".")
}
sts := &tailscaleSTSConfig{
Hostname: hostname,
@@ -216,7 +268,6 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Tags: tags,
ChildResourceLabels: crl,
ProxyClassName: proxyClass,
proxyType: proxyTypeIngressResource,
}
if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {
@@ -227,12 +278,12 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return fmt.Errorf("failed to provision: %w", err)
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
_, tsHost, _, err := a.ssr.DeviceInfo(ctx, crl)
if err != nil {
return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err)
return fmt.Errorf("failed to get device ID: %w", err)
}
if dev == nil || dev.ingressDNSName == "" {
logger.Debugf("no Ingress DNS name known yet, waiting for proxy Pod initialize and start serving Ingress")
if tsHost == "" {
logger.Debugf("no Tailscale hostname known yet, waiting for proxy pod to finish auth")
// No hostname yet. Wait for the proxy pod to auth.
ing.Status.LoadBalancer.Ingress = nil
if err := a.Status().Update(ctx, ing); err != nil {
@@ -241,10 +292,10 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
logger.Debugf("setting ingress hostname to %q", tsHost)
ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
{
Hostname: dev.ingressDNSName,
Hostname: tsHost,
Ports: []networkingv1.IngressPortStatus{
{
Protocol: "TCP",
@@ -262,106 +313,28 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool {
return ing != nil &&
ing.Spec.IngressClassName != nil &&
*ing.Spec.IngressClassName == tailscaleIngressClassName &&
ing.Annotations[AnnotationProxyGroup] == ""
*ing.Spec.IngressClassName == tailscaleIngressClassName
}
// validateIngressClass attempts to validate that 'tailscale' IngressClass
// included in Tailscale installation manifests exists and has not been modified
// to attempt to enable features that we do not support.
func validateIngressClass(ctx context.Context, cl client.Client) error {
func (a *IngressReconciler) validateIngressClass(ctx context.Context) error {
ic := &networkingv1.IngressClass{
ObjectMeta: metav1.ObjectMeta{
Name: tailscaleIngressClassName,
},
}
if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) {
return errors.New("'tailscale' IngressClass not found in cluster.")
if err := a.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) {
return errors.New("Tailscale IngressClass not found in cluster. Latest installation manifests include a tailscale IngressClass - please update")
} else if err != nil {
return fmt.Errorf("error retrieving 'tailscale' IngressClass: %w", err)
}
if ic.Spec.Controller != tailscaleIngressControllerName {
return fmt.Errorf("'tailscale' Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName)
return fmt.Errorf("Tailscale Ingress class controller name %s does not match tailscale Ingress controller name %s. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ic.Spec.Controller, tailscaleIngressControllerName)
}
if ic.GetAnnotations()[ingressClassDefaultAnnotation] != "" {
return fmt.Errorf("%s annotation is set on 'tailscale' IngressClass, but Tailscale Ingress controller does not support default Ingress class. Ensure that you are using 'tailscale' IngressClass from latest Tailscale installation manifests", ingressClassDefaultAnnotation)
}
return nil
}
func handlersForIngress(ctx context.Context, ing *networkingv1.Ingress, cl client.Client, rec record.EventRecorder, tlsHost string, logger *zap.SugaredLogger) (handlers map[string]*ipn.HTTPHandler, err error) {
addIngressBackend := func(b *networkingv1.IngressBackend, path string) {
if b == nil {
return
}
if b.Service == nil {
rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q is missing service", path)
return
}
var svc corev1.Service
if err := cl.Get(ctx, types.NamespacedName{Namespace: ing.Namespace, Name: b.Service.Name}, &svc); err != nil {
rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "failed to get service %q for path %q: %v", b.Service.Name, path, err)
return
}
if svc.Spec.ClusterIP == "" || svc.Spec.ClusterIP == "None" {
rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid ClusterIP", path)
return
}
var port int32
if b.Service.Port.Name != "" {
for _, p := range svc.Spec.Ports {
if p.Name == b.Service.Port.Name {
port = p.Port
break
}
}
} else {
port = b.Service.Port.Number
}
if port == 0 {
rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "backend for path %q has invalid port", path)
return
}
proto := "http://"
if port == 443 || b.Service.Port.Name == "https" {
proto = "https+insecure://"
}
mak.Set(&handlers, path, &ipn.HTTPHandler{
Proxy: proto + svc.Spec.ClusterIP + ":" + fmt.Sprint(port) + path,
})
}
addIngressBackend(ing.Spec.DefaultBackend, "/")
for _, rule := range ing.Spec.Rules {
// Host is optional, but if it's present it must match the TLS host
// otherwise we ignore the rule.
if rule.Host != "" && rule.Host != tlsHost {
rec.Eventf(ing, corev1.EventTypeWarning, "InvalidIngressBackend", "rule with host %q ignored, unsupported", rule.Host)
continue
}
for _, p := range rule.HTTP.Paths {
// Send a warning if folks use Exact path type - to make
// it easier for us to support Exact path type matching
// in the future if needed.
// https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types
if *p.PathType == networkingv1.PathTypeExact {
msg := "Exact path type strict matching is currently not supported and requests will be routed as for Prefix path type. This behaviour might change in the future."
logger.Warnf(fmt.Sprintf("Unsupported Path type exact for path %s. %s", p.Path, msg))
rec.Eventf(ing, corev1.EventTypeWarning, "UnsupportedPathTypeExact", msg)
}
addIngressBackend(&p.Backend, p.Path)
}
}
return handlers, nil
}
// hostnameForIngress returns the hostname for an Ingress resource.
// If the Ingress has TLS configured with a host, it returns the first component of that host.
// Otherwise, it returns a hostname derived from the Ingress name and namespace.
func hostnameForIngress(ing *networkingv1.Ingress) string {
if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && len(ing.Spec.TLS[0].Hosts) > 0 {
h := ing.Spec.TLS[0].Hosts[0]
hostname, _, _ := strings.Cut(h, ".")
return hostname
}
return ing.Namespace + "-" + ing.Name + "-ingress"
}

View File

@@ -12,7 +12,6 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -103,9 +102,9 @@ func TestTailscaleIngress(t *testing.T) {
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
// 2. Ingress status gets updated with ingress proxy's MagicDNS name
// once that becomes available.
@@ -120,7 +119,7 @@ func TestTailscaleIngress(t *testing.T) {
{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
},
}
expectEqual(t, fc, ing)
expectEqual(t, fc, ing, nil)
// 3. Resources get created for Ingress that should allow forwarding
// cluster traffic
@@ -129,7 +128,7 @@ func TestTailscaleIngress(t *testing.T) {
})
opts.shouldEnableForwardingClusterTrafficViaIngress = true
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 4. Resources get cleaned up when Ingress class is unset
mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) {
@@ -142,160 +141,12 @@ func TestTailscaleIngress(t *testing.T) {
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)
}
func TestTailscaleIngressHostname(t *testing.T) {
tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
fc := fake.NewFakeClient(tsIngressClass)
ft := &fakeTSClient{}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
ingR := &IngressReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
tsnetServer: fakeTsnetServer,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
// 1. Resources get created for regular Ingress
ing := &networkingv1.Ingress{
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To("tailscale"),
DefaultBackend: &networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: "test",
Port: networkingv1.ServiceBackendPort{
Number: 8080,
},
},
},
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"default-test"}},
},
},
}
mustCreate(t, fc, ing)
mustCreate(t, fc, &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
},
Spec: corev1.ServiceSpec{
ClusterIP: "1.2.3.4",
Ports: []corev1.ServicePort{{
Port: 8080,
Name: "http"},
},
},
})
expectReconciled(t, ingR, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
mustCreate(t, fc, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fullName,
Namespace: "operator-ns",
UID: "test-uid",
},
})
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "ingress",
hostname: "default-test",
app: kubetypes.AppIngressResource,
}
serveConfig := &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
// 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set
mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
mak.Set(&secret.Data, "device_id", []byte("1234"))
mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
mak.Set(&secret.Data, "pod_uid", []byte("test-uid"))
mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
})
expectReconciled(t, ingR, "default", "test")
ing.Finalizers = append(ing.Finalizers, "tailscale.com/finalizer")
expectEqual(t, fc, ing)
// 3. Ingress proxy with capability version >= 110 advertises HTTPS endpoint
mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
mak.Set(&secret.Data, "device_id", []byte("1234"))
mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
mak.Set(&secret.Data, "pod_uid", []byte("test-uid"))
mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
mak.Set(&secret.Data, "https_endpoint", []byte("foo.tailnetxyz.ts.net"))
})
expectReconciled(t, ingR, "default", "test")
ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
Ingress: []networkingv1.IngressLoadBalancerIngress{
{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
},
}
expectEqual(t, fc, ing)
// 4. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint ready
mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
mak.Set(&secret.Data, "device_id", []byte("1234"))
mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
mak.Set(&secret.Data, "pod_uid", []byte("test-uid"))
mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
mak.Set(&secret.Data, "https_endpoint", []byte("no-https"))
})
expectReconciled(t, ingR, "default", "test")
ing.Status.LoadBalancer.Ingress = nil
expectEqual(t, fc, ing)
// 5. Ingress proxy's state has https_endpoints set, but its capver is not matching Pod UID (downgrade)
mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) {
mak.Set(&secret.Data, "device_id", []byte("1234"))
mak.Set(&secret.Data, "tailscale_capver", []byte("110"))
mak.Set(&secret.Data, "pod_uid", []byte("not-the-right-uid"))
mak.Set(&secret.Data, "device_fqdn", []byte("foo.tailnetxyz.ts.net"))
mak.Set(&secret.Data, "https_endpoint", []byte("bar.tailnetxyz.ts.net"))
})
ing.Status.LoadBalancer = networkingv1.IngressLoadBalancerStatus{
Ingress: []networkingv1.IngressLoadBalancerIngress{
{Hostname: "foo.tailnetxyz.ts.net", Ports: []networkingv1.IngressPortStatus{{Port: 443, Protocol: "TCP"}}},
},
}
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, ing)
}
func TestTailscaleIngressWithProxyClass(t *testing.T) {
// Setup
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "custom-metadata"},
Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar"},
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
@@ -383,9 +234,9 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
}
opts.serveConfig = serveConfig
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"))
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
// 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet
// ready, so proxy resource configuration does not change.
@@ -393,7 +244,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata")
})
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
// 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get
// reconciled and configuration from the ProxyClass is applied to the
@@ -408,7 +259,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
})
expectReconciled(t, ingR, "default", "test")
opts.proxyClass = pc.Name
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
// 4. tailscale.com/proxy-class label is removed from the Ingress, the
// Ingress gets reconciled and the custom ProxyClass configuration is
@@ -418,145 +269,5 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
})
expectReconciled(t, ingR, "default", "test")
opts.proxyClass = ""
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs)
}
func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
Spec: tsapi.ProxyClassSpec{},
Status: tsapi.ProxyClassStatus{
Conditions: []metav1.Condition{{
Status: metav1.ConditionTrue,
Type: string(tsapi.ProxyClassReady),
ObservedGeneration: 1,
}}},
}
ing := &networkingv1.Ingress{
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
// The apiserver is supposed to set the UID, but the fake client
// doesn't. So, set it explicitly because other code later depends
// on it being set.
UID: types.UID("1234-UID"),
Labels: map[string]string{
"tailscale.com/proxy-class": "metrics",
},
},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To("tailscale"),
DefaultBackend: &networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: "test",
Port: networkingv1.ServiceBackendPort{
Number: 8080,
},
},
},
TLS: []networkingv1.IngressTLS{
{Hosts: []string{"default-test"}},
},
},
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
},
Spec: corev1.ServiceSpec{
ClusterIP: "1.2.3.4",
Ports: []corev1.ServicePort{{
Port: 8080,
Name: "http"},
},
},
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, tsIngressClass, ing, svc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
ingR := &IngressReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
tsnetServer: fakeTsnetServer,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
}
expectReconciled(t, ingR, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
serveConfig := &ipn.ServeConfig{
TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
}
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
tailscaleNamespace: "operator-ns",
parentType: "ingress",
hostname: "default-test",
app: kubetypes.AppIngressResource,
namespaced: true,
proxyType: proxyTypeIngressResource,
serveConfig: serveConfig,
resourceVersion: "1",
}
// 1. Enable metrics- expect metrics Service to be created
mustUpdate(t, fc, "", "metrics", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics = &tsapi.Metrics{Enable: true}
})
opts.enableMetrics = true
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedMetricsService(opts))
// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true, Labels: tsapi.Labels{"foo": "bar"}}
})
expectReconciled(t, ingR, "default", "test")
expectEqual(t, fc, expectedMetricsService(opts))
// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
mustCreate(t, fc, crd)
expectReconciled(t, ingR, "default", "test")
opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
expectEqual(t, fc, expectedMetricsService(opts))
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 4. Update ServiceMonitor CRD and reconcile- ServiceMonitor should get updated
mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics.ServiceMonitor.Labels = nil
})
expectReconciled(t, ingR, "default", "test")
opts.serviceMonitorLabels = nil
opts.resourceVersion = "2"
expectEqual(t, fc, expectedMetricsService(opts))
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 5. Disable metrics - metrics resources should get deleted.
mustUpdate(t, fc, pc.Namespace, pc.Name, func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics = nil
})
expectReconciled(t, ingR, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(shortName))
// ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here.
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
}

View File

@@ -1,295 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"context"
"fmt"
"reflect"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
)
const (
labelMetricsTarget = "tailscale.com/metrics-target"
// These labels get transferred from the metrics Service to the ingested Prometheus metrics.
labelPromProxyType = "ts_proxy_type"
labelPromProxyParentName = "ts_proxy_parent_name"
labelPromProxyParentNamespace = "ts_proxy_parent_namespace"
labelPromJob = "ts_prom_job"
serviceMonitorCRD = "servicemonitors.monitoring.coreos.com"
)
// ServiceMonitor contains a subset of fields of servicemonitors.monitoring.coreos.com Custom Resource Definition.
// Duplicating it here allows us to avoid importing prometheus-operator library.
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L40
type ServiceMonitor struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata"`
Spec ServiceMonitorSpec `json:"spec"`
}
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L55
type ServiceMonitorSpec struct {
// Endpoints defines the endpoints to be scraped on the selected Service(s).
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L82
Endpoints []ServiceMonitorEndpoint `json:"endpoints"`
// JobLabel is the label on the Service whose value will become the value of the Prometheus job label for the metrics ingested via this ServiceMonitor.
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L66
JobLabel string `json:"jobLabel"`
// NamespaceSelector selects the namespace of Service(s) that this ServiceMonitor allows to scrape.
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L88
NamespaceSelector ServiceMonitorNamespaceSelector `json:"namespaceSelector,omitempty"`
// Selector is the label selector for Service(s) that this ServiceMonitor allows to scrape.
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L85
Selector metav1.LabelSelector `json:"selector"`
// TargetLabels are labels on the selected Service that should be applied as Prometheus labels to the ingested metrics.
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L72
TargetLabels []string `json:"targetLabels"`
}
// ServiceMonitorNamespaceSelector selects namespaces in which Prometheus operator will attempt to find Services for
// this ServiceMonitor.
// https://github.com/prometheus-operator/prometheus-operator/blob/bb4514e0d5d69f20270e29cfd4ad39b87865ccdf/pkg/apis/monitoring/v1/servicemonitor_types.go#L88
type ServiceMonitorNamespaceSelector struct {
MatchNames []string `json:"matchNames,omitempty"`
}
// ServiceMonitorEndpoint defines an endpoint of Service to scrape. We only define port here. Prometheus by default
// scrapes /metrics path, which is what we want.
type ServiceMonitorEndpoint struct {
// Port is the name of the Service port that Prometheus will scrape.
Port string `json:"port,omitempty"`
}
func reconcileMetricsResources(ctx context.Context, logger *zap.SugaredLogger, opts *metricsOpts, pc *tsapi.ProxyClass, cl client.Client) error {
if opts.proxyType == proxyTypeEgress {
// Metrics are currently not being enabled for standalone egress proxies.
return nil
}
if pc == nil || pc.Spec.Metrics == nil || !pc.Spec.Metrics.Enable {
return maybeCleanupMetricsResources(ctx, opts, cl)
}
metricsSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: metricsResourceName(opts.proxyStsName),
Namespace: opts.tsNamespace,
Labels: metricsResourceLabels(opts),
},
Spec: corev1.ServiceSpec{
Selector: opts.proxyLabels,
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}},
},
}
var err error
metricsSvc, err = createOrUpdate(ctx, cl, opts.tsNamespace, metricsSvc, func(svc *corev1.Service) {
svc.Spec.Ports = metricsSvc.Spec.Ports
svc.Spec.Selector = metricsSvc.Spec.Selector
})
if err != nil {
return fmt.Errorf("error ensuring metrics Service: %w", err)
}
crdExists, err := hasServiceMonitorCRD(ctx, cl)
if err != nil {
return fmt.Errorf("error verifying that %q CRD exists: %w", serviceMonitorCRD, err)
}
if !crdExists {
return nil
}
if pc.Spec.Metrics.ServiceMonitor == nil || !pc.Spec.Metrics.ServiceMonitor.Enable {
return maybeCleanupServiceMonitor(ctx, cl, opts.proxyStsName, opts.tsNamespace)
}
logger.Infof("ensuring ServiceMonitor for metrics Service %s/%s", metricsSvc.Namespace, metricsSvc.Name)
svcMonitor, err := newServiceMonitor(metricsSvc, pc.Spec.Metrics.ServiceMonitor)
if err != nil {
return fmt.Errorf("error creating ServiceMonitor: %w", err)
}
// We don't use createOrUpdate here because that does not work with unstructured types.
existing := svcMonitor.DeepCopy()
err = cl.Get(ctx, client.ObjectKeyFromObject(metricsSvc), existing)
if apierrors.IsNotFound(err) {
if err := cl.Create(ctx, svcMonitor); err != nil {
return fmt.Errorf("error creating ServiceMonitor: %w", err)
}
return nil
}
if err != nil {
return fmt.Errorf("error getting ServiceMonitor: %w", err)
}
// Currently, we only update labels on the ServiceMonitor as those are the only values that can change.
if !reflect.DeepEqual(existing.GetLabels(), svcMonitor.GetLabels()) {
existing.SetLabels(svcMonitor.GetLabels())
if err := cl.Update(ctx, existing); err != nil {
return fmt.Errorf("error updating ServiceMonitor: %w", err)
}
}
return nil
}
// maybeCleanupMetricsResources ensures that any metrics resources created for a proxy are deleted. Only metrics Service
// gets deleted explicitly because the ServiceMonitor has Service's owner reference, so gets garbage collected
// automatically.
func maybeCleanupMetricsResources(ctx context.Context, opts *metricsOpts, cl client.Client) error {
sel := metricsSvcSelector(opts.proxyLabels, opts.proxyType)
return cl.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(opts.tsNamespace), client.MatchingLabels(sel))
}
// maybeCleanupServiceMonitor cleans up any ServiceMonitor created for the named proxy StatefulSet.
func maybeCleanupServiceMonitor(ctx context.Context, cl client.Client, stsName, ns string) error {
smName := metricsResourceName(stsName)
sm := serviceMonitorTemplate(smName, ns)
u, err := serviceMonitorToUnstructured(sm)
if err != nil {
return fmt.Errorf("error building ServiceMonitor: %w", err)
}
err = cl.Get(ctx, types.NamespacedName{Name: smName, Namespace: ns}, u)
if apierrors.IsNotFound(err) {
return nil // nothing to do
}
if err != nil {
return fmt.Errorf("error verifying if ServiceMonitor %s/%s exists: %w", ns, stsName, err)
}
return cl.Delete(ctx, u)
}
// newServiceMonitor takes a metrics Service created for a proxy and constructs and returns a ServiceMonitor for that
// proxy that can be applied to the kube API server.
// The ServiceMonitor is returned as Unstructured type - this allows us to avoid importing prometheus-operator API server client/schema.
func newServiceMonitor(metricsSvc *corev1.Service, spec *tsapi.ServiceMonitor) (*unstructured.Unstructured, error) {
sm := serviceMonitorTemplate(metricsSvc.Name, metricsSvc.Namespace)
sm.ObjectMeta.Labels = metricsSvc.Labels
if spec != nil && len(spec.Labels) > 0 {
sm.ObjectMeta.Labels = mergeMapKeys(sm.ObjectMeta.Labels, spec.Labels.Parse())
}
sm.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(metricsSvc, corev1.SchemeGroupVersion.WithKind("Service"))}
sm.Spec = ServiceMonitorSpec{
Selector: metav1.LabelSelector{MatchLabels: metricsSvc.Labels},
Endpoints: []ServiceMonitorEndpoint{{
Port: "metrics",
}},
NamespaceSelector: ServiceMonitorNamespaceSelector{
MatchNames: []string{metricsSvc.Namespace},
},
JobLabel: labelPromJob,
TargetLabels: []string{
labelPromProxyParentName,
labelPromProxyParentNamespace,
labelPromProxyType,
},
}
return serviceMonitorToUnstructured(sm)
}
// serviceMonitorToUnstructured takes a ServiceMonitor and converts it to Unstructured type that can be used by the c/r
// client in Kubernetes API server calls.
func serviceMonitorToUnstructured(sm *ServiceMonitor) (*unstructured.Unstructured, error) {
contents, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sm)
if err != nil {
return nil, fmt.Errorf("error converting ServiceMonitor to Unstructured: %w", err)
}
u := &unstructured.Unstructured{}
u.SetUnstructuredContent(contents)
u.SetGroupVersionKind(sm.GroupVersionKind())
return u, nil
}
// metricsResourceName returns name for metrics Service and ServiceMonitor for a proxy StatefulSet.
func metricsResourceName(stsName string) string {
// Maximum length of StatefulSet name if 52 chars, so this is fine.
return fmt.Sprintf("%s-metrics", stsName)
}
// metricsResourceLabels constructs labels that will be applied to metrics Service and metrics ServiceMonitor for a
// proxy.
func metricsResourceLabels(opts *metricsOpts) map[string]string {
lbls := map[string]string{
LabelManaged: "true",
labelMetricsTarget: opts.proxyStsName,
labelPromProxyType: opts.proxyType,
labelPromProxyParentName: opts.proxyLabels[LabelParentName],
}
// Include namespace label for proxies created for a namespaced type.
if isNamespacedProxyType(opts.proxyType) {
lbls[labelPromProxyParentNamespace] = opts.proxyLabels[LabelParentNamespace]
}
lbls[labelPromJob] = promJobName(opts)
return lbls
}
// promJobName constructs the value of the Prometheus job label that will apply to all metrics for a ServiceMonitor.
func promJobName(opts *metricsOpts) string {
// Include parent resource namespace for proxies created for namespaced types.
if opts.proxyType == proxyTypeIngressResource || opts.proxyType == proxyTypeIngressService {
return fmt.Sprintf("ts_%s_%s_%s", opts.proxyType, opts.proxyLabels[LabelParentNamespace], opts.proxyLabels[LabelParentName])
}
return fmt.Sprintf("ts_%s_%s", opts.proxyType, opts.proxyLabels[LabelParentName])
}
// metricsSvcSelector returns the minimum label set to uniquely identify a metrics Service for a proxy.
func metricsSvcSelector(proxyLabels map[string]string, proxyType string) map[string]string {
sel := map[string]string{
labelPromProxyType: proxyType,
labelPromProxyParentName: proxyLabels[LabelParentName],
}
// Include namespace label for proxies created for a namespaced type.
if isNamespacedProxyType(proxyType) {
sel[labelPromProxyParentNamespace] = proxyLabels[LabelParentNamespace]
}
return sel
}
// serviceMonitorTemplate returns a base ServiceMonitor type that, when converted to Unstructured, is a valid type that
// can be used in kube API server calls via the c/r client.
func serviceMonitorTemplate(name, ns string) *ServiceMonitor {
return &ServiceMonitor{
TypeMeta: metav1.TypeMeta{
Kind: "ServiceMonitor",
APIVersion: "monitoring.coreos.com/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns,
},
}
}
type metricsOpts struct {
proxyStsName string // name of StatefulSet for proxy
tsNamespace string // namespace in which Tailscale is installed
proxyLabels map[string]string // labels of the proxy StatefulSet
proxyType string
}
func isNamespacedProxyType(typ string) bool {
return typ == proxyTypeIngressResource || typ == proxyTypeIngressService
}
func mergeMapKeys(a, b map[string]string) map[string]string {
m := make(map[string]string, len(a)+len(b))
for key, val := range b {
m[key] = val
}
for key, val := range a {
m[key] = val
}
return m
}

View File

@@ -9,7 +9,6 @@ import (
"context"
"fmt"
"slices"
"strings"
"sync"
_ "embed"
@@ -87,7 +86,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
return reconcile.Result{}, nil
}
logger.Info("Cleaning up DNSConfig resources")
if err := a.maybeCleanup(&dnsCfg); err != nil {
if err := a.maybeCleanup(ctx, &dnsCfg, logger); err != nil {
logger.Errorf("error cleaning up reconciler resource: %v", err)
return res, err
}
@@ -101,9 +100,9 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
oldCnStatus := dnsCfg.Status.DeepCopy()
setStatus := func(dnsCfg *tsapi.DNSConfig, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
setStatus := func(dnsCfg *tsapi.DNSConfig, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetDNSConfigCondition(dnsCfg, tsapi.NameserverReady, status, reason, message, dnsCfg.Generation, a.clock, logger)
if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) {
if !apiequality.Semantic.DeepEqual(oldCnStatus, dnsCfg.Status) {
// An error encountered here should get returned by the Reconcile function.
if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil {
err = errors.Wrap(err, updateErr.Error())
@@ -119,7 +118,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
msg := "invalid cluster configuration: more than one tailscale.com/dnsconfigs found. Please ensure that no more than one is created."
logger.Error(msg)
a.recorder.Event(&dnsCfg, corev1.EventTypeWarning, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
setStatus(&dnsCfg, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonMultipleDNSConfigsPresent, messageMultipleDNSConfigsPresent)
}
if !slices.Contains(dnsCfg.Finalizers, FinalizerName) {
@@ -128,16 +127,11 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
if err := a.Update(ctx, &dnsCfg); err != nil {
msg := fmt.Sprintf(messageNameserverCreationFailed, err)
logger.Error(msg)
return setStatus(&dnsCfg, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionFalse, reasonNameserverCreationFailed, msg)
}
}
if err := a.maybeProvision(ctx, &dnsCfg, logger); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
return reconcile.Result{}, nil
} else {
return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
}
return reconcile.Result{}, fmt.Errorf("error provisioning nameserver resources: %w", err)
}
a.mu.Lock()
@@ -155,7 +149,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ
dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{
IP: ip,
}
return setStatus(&dnsCfg, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated)
return setStatus(&dnsCfg, tsapi.NameserverReady, metav1.ConditionTrue, reasonNameserverCreated, reasonNameserverCreated)
}
logger.Info("nameserver Service does not have an IP address allocated, waiting...")
return reconcile.Result{}, nil
@@ -194,7 +188,7 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa
// maybeCleanup removes DNSConfig from being tracked. The cluster resources
// created, will be automatically garbage collected as they are owned by the
// DNSConfig.
func (a *NameserverReconciler) maybeCleanup(dnsCfg *tsapi.DNSConfig) error {
func (a *NameserverReconciler) maybeCleanup(ctx context.Context, dnsCfg *tsapi.DNSConfig, logger *zap.SugaredLogger) error {
a.mu.Lock()
a.managedNameservers.Remove(dnsCfg.UID)
a.mu.Unlock()

View File

@@ -69,7 +69,7 @@ func TestNameserverReconciler(t *testing.T) {
wantsDeploy.Namespace = "tailscale"
labels := nameserverResourceLabels("test", "tailscale")
wantsDeploy.ObjectMeta.Labels = labels
expectEqual(t, fc, wantsDeploy)
expectEqual(t, fc, wantsDeploy, nil)
// Verify that DNSConfig advertizes the nameserver's Service IP address,
// has the ready status condition and tailscale finalizer.
@@ -88,7 +88,7 @@ func TestNameserverReconciler(t *testing.T) {
Message: reasonNameserverCreated,
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
})
expectEqual(t, fc, dnsCfg)
expectEqual(t, fc, dnsCfg, nil)
// // Verify that nameserver image gets updated to match DNSConfig spec.
mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) {
@@ -96,7 +96,7 @@ func TestNameserverReconciler(t *testing.T) {
})
expectReconciled(t, nr, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2"
expectEqual(t, fc, wantsDeploy)
expectEqual(t, fc, wantsDeploy, nil)
// Verify that when another actor sets ConfigMap data, it does not get
// overwritten by nameserver reconciler.
@@ -114,7 +114,7 @@ func TestNameserverReconciler(t *testing.T) {
TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"},
Data: map[string]string{"records.json": string(bs)},
}
expectEqual(t, fc, wantCm)
expectEqual(t, fc, wantCm, nil)
// Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset,
// the nameserver image defaults to tailscale/k8s-nameserver:unstable.
@@ -123,5 +123,5 @@ func TestNameserverReconciler(t *testing.T) {
})
expectReconciled(t, nr, "", "test")
wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable"
expectEqual(t, fc, wantsDeploy)
expectEqual(t, fc, wantsDeploy, nil)
}

View File

@@ -18,16 +18,14 @@ import (
"github.com/go-logr/zapr"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"golang.org/x/oauth2/clientcredentials"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
networkingv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
toolscache "k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -87,15 +85,6 @@ func main() {
zlog := kzap.NewRaw(opts...).Sugar()
logf.SetLogger(zapr.NewLogger(zlog.Desugar()))
if tsNamespace == "" {
const namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
b, err := os.ReadFile(namespaceFile)
if err != nil {
zlog.Fatalf("Could not get operator namespace from OPERATOR_NAMESPACE environment variable or default projected volume: %v", err)
}
tsNamespace = strings.TrimSpace(string(b))
}
// The operator can run either as a plain operator or it can
// additionally act as api-server proxy
// https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy.
@@ -106,14 +95,14 @@ func main() {
hostinfo.SetApp(kubetypes.AppAPIServerProxy)
}
s, tsc := initTSNet(zlog)
s, tsClient := initTSNet(zlog)
defer s.Close()
restConfig := config.GetConfigOrDie()
maybeLaunchAPIServerProxy(zlog, restConfig, s, mode)
rOpts := reconcilerOpts{
log: zlog,
tsServer: s,
tsClient: tsc,
tsClient: tsClient,
tailscaleNamespace: tsNamespace,
restConfig: restConfig,
proxyImage: image,
@@ -129,7 +118,7 @@ func main() {
// initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the
// CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate
// with Tailscale.
func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) {
func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, *tailscale.Client) {
var (
clientIDPath = defaultEnv("CLIENT_ID_FILE", "")
clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "")
@@ -141,10 +130,23 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) {
if clientIDPath == "" || clientSecretPath == "" {
startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set")
}
tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath)
clientID, err := os.ReadFile(clientIDPath)
if err != nil {
startlog.Fatalf("error creating Tailscale client: %v", err)
startlog.Fatalf("reading client ID %q: %v", clientIDPath, err)
}
clientSecret, err := os.ReadFile(clientSecretPath)
if err != nil {
startlog.Fatalf("reading client secret %q: %v", clientSecretPath, err)
}
credentials := clientcredentials.Config{
ClientID: string(clientID),
ClientSecret: string(clientSecret),
TokenURL: "https://login.tailscale.com/api/v2/oauth/token",
}
tsClient := tailscale.NewClient("-", nil)
tsClient.UserAgent = "tailscale-k8s-operator"
tsClient.HTTPClient = credentials.Client(context.Background())
s := &tsnet.Server{
Hostname: hostname,
Logf: zlog.Named("tailscaled").Debugf,
@@ -197,7 +199,7 @@ waitOnline:
},
},
}
authkey, _, err := tsc.CreateKey(ctx, caps)
authkey, _, err := tsClient.CreateKey(ctx, caps)
if err != nil {
startlog.Fatalf("creating operator authkey: %v", err)
}
@@ -221,7 +223,7 @@ waitOnline:
}
time.Sleep(time.Second)
}
return s, tsc
return s, tsClient
}
// runReconcilers starts the controller-runtime manager and registers the
@@ -237,29 +239,21 @@ func runReconcilers(opts reconcilerOpts) {
nsFilter := cache.ByObject{
Field: client.InNamespace(opts.tailscaleNamespace).AsSelector(),
}
// We watch the ServiceMonitor CRD to ensure that reconcilers are re-triggered if user's workflows result in the
// ServiceMonitor CRD applied after some of our resources that define ServiceMonitor creation. This selector
// ensures that we only watch the ServiceMonitor CRD and that we don't cache full contents of it.
serviceMonitorSelector := cache.ByObject{
Field: fields.SelectorFromSet(fields.Set{"metadata.name": serviceMonitorCRD}),
Transform: crdTransformer(startlog),
}
mgrOpts := manager.Options{
// TODO (irbekrm): stricter filtering what we watch/cache/call
// reconcilers on. c/r by default starts a watch on any
// resources that we GET via the controller manager's client.
Cache: cache.Options{
ByObject: map[client.Object]cache.ByObject{
&corev1.Secret{}: nsFilter,
&corev1.ServiceAccount{}: nsFilter,
&corev1.Pod{}: nsFilter,
&corev1.ConfigMap{}: nsFilter,
&appsv1.StatefulSet{}: nsFilter,
&appsv1.Deployment{}: nsFilter,
&discoveryv1.EndpointSlice{}: nsFilter,
&rbacv1.Role{}: nsFilter,
&rbacv1.RoleBinding{}: nsFilter,
&apiextensionsv1.CustomResourceDefinition{}: serviceMonitorSelector,
&corev1.Secret{}: nsFilter,
&corev1.ServiceAccount{}: nsFilter,
&corev1.Pod{}: nsFilter,
&corev1.ConfigMap{}: nsFilter,
&appsv1.StatefulSet{}: nsFilter,
&appsv1.Deployment{}: nsFilter,
&discoveryv1.EndpointSlice{}: nsFilter,
&rbacv1.Role{}: nsFilter,
&rbacv1.RoleBinding{}: nsFilter,
},
},
Scheme: tsapi.GlobalScheme,
@@ -315,7 +309,6 @@ func runReconcilers(opts reconcilerOpts) {
err = builder.
ControllerManagedBy(mgr).
For(&networkingv1.Ingress{}).
Named("ingress-reconciler").
Watches(&appsv1.StatefulSet{}, ingressChildFilter).
Watches(&corev1.Secret{}, ingressChildFilter).
Watches(&corev1.Service{}, svcHandlerForIngress).
@@ -330,28 +323,6 @@ func runReconcilers(opts reconcilerOpts) {
if err != nil {
startlog.Fatalf("could not create ingress reconciler: %v", err)
}
lc, err := opts.tsServer.LocalClient()
if err != nil {
startlog.Fatalf("could not get local client: %v", err)
}
err = builder.
ControllerManagedBy(mgr).
For(&networkingv1.Ingress{}).
Named("ingress-pg-reconciler").
Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))).
Complete(&IngressPGReconciler{
recorder: eventRecorder,
tsClient: opts.tsClient,
tsnetServer: opts.tsServer,
defaultTags: strings.Split(opts.proxyTags, ","),
Client: mgr.GetClient(),
logger: opts.log.Named("ingress-pg-reconciler"),
lc: lc,
tsNamespace: opts.tailscaleNamespace,
})
if err != nil {
startlog.Fatalf("could not create ingress-pg-reconciler: %v", err)
}
connectorFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("connector"))
// If a ProxyClassChanges, enqueue all Connectors that have
@@ -359,7 +330,6 @@ func runReconcilers(opts reconcilerOpts) {
proxyClassFilterForConnector := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForConnector(mgr.GetClient(), startlog))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.Connector{}).
Named("connector-reconciler").
Watches(&appsv1.StatefulSet{}, connectorFilter).
Watches(&corev1.Secret{}, connectorFilter).
Watches(&tsapi.ProxyClass{}, proxyClassFilterForConnector).
@@ -379,7 +349,6 @@ func runReconcilers(opts reconcilerOpts) {
nameserverFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("nameserver"))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.DNSConfig{}).
Named("nameserver-reconciler").
Watches(&appsv1.Deployment{}, nameserverFilter).
Watches(&corev1.ConfigMap{}, nameserverFilter).
Watches(&corev1.Service{}, nameserverFilter).
@@ -453,14 +422,8 @@ func runReconcilers(opts reconcilerOpts) {
startlog.Fatalf("could not create egress EndpointSlices reconciler: %v", err)
}
// ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that
// define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get
// reconciled if the CRD is applied at a later point.
serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.ProxyClass{}).
Named("proxyclass-reconciler").
Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter).
Complete(&ProxyClassReconciler{
Client: mgr.GetClient(),
recorder: eventRecorder,
@@ -503,7 +466,6 @@ func runReconcilers(opts reconcilerOpts) {
recorderFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.Recorder{})
err = builder.ControllerManagedBy(mgr).
For(&tsapi.Recorder{}).
Named("recorder-reconciler").
Watches(&appsv1.StatefulSet{}, recorderFilter).
Watches(&corev1.ServiceAccount{}, recorderFilter).
Watches(&corev1.Secret{}, recorderFilter).
@@ -521,14 +483,12 @@ func runReconcilers(opts reconcilerOpts) {
startlog.Fatalf("could not create Recorder reconciler: %v", err)
}
// ProxyGroup reconciler.
// Recorder reconciler.
ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{})
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.ProxyGroup{}).
Named("proxygroup-reconciler").
Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter).
Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter).
Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
@@ -560,7 +520,7 @@ func runReconcilers(opts reconcilerOpts) {
type reconcilerOpts struct {
log *zap.SugaredLogger
tsServer *tsnet.Server
tsClient tsClient
tsClient *tailscale.Client
tailscaleNamespace string // namespace in which operator resources will be deployed
restConfig *rest.Config // config for connecting to the kube API server
proxyImage string // <proxy-image-repo>:<proxy-image-tag>
@@ -685,6 +645,12 @@ func dnsRecordsReconcilerIngressHandler(ns string, isDefaultLoadBalancer bool, c
}
}
type tsClient interface {
CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
DeleteDevice(ctx context.Context, nodeStableID string) error
}
func isManagedResource(o client.Object) bool {
ls := o.GetLabels()
return ls[LabelManaged] == "true"
@@ -820,10 +786,6 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl
if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
return nil
}
if hasProxyGroupAnnotation(&ing) {
// We don't want to reconcile backend Services for Ingresses for ProxyGroups.
continue
}
if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
}
@@ -1056,49 +1018,6 @@ func epsFromExternalNameService(cl client.Client, logger *zap.SugaredLogger, ns
}
}
// proxyClassesWithServiceMonitor returns an event handler that, given that the event is for the Prometheus
// ServiceMonitor CRD, returns all ProxyClasses that define that a ServiceMonitor should be created.
func proxyClassesWithServiceMonitor(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
crd, ok := o.(*apiextensionsv1.CustomResourceDefinition)
if !ok {
logger.Debugf("[unexpected] ServiceMonitor CRD handler received an object that is not a CustomResourceDefinition")
return nil
}
if crd.Name != serviceMonitorCRD {
logger.Debugf("[unexpected] ServiceMonitor CRD handler received an unexpected CRD %q", crd.Name)
return nil
}
pcl := &tsapi.ProxyClassList{}
if err := cl.List(ctx, pcl); err != nil {
logger.Debugf("[unexpected] error listing ProxyClasses: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, pc := range pcl.Items {
if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable {
reqs = append(reqs, reconcile.Request{
NamespacedName: types.NamespacedName{Namespace: pc.Namespace, Name: pc.Name},
})
}
}
return reqs
}
}
// crdTransformer gets called before a CRD is stored to c/r cache, it removes the CRD spec to reduce memory consumption.
func crdTransformer(log *zap.SugaredLogger) toolscache.TransformFunc {
return func(o any) (any, error) {
crd, ok := o.(*apiextensionsv1.CustomResourceDefinition)
if !ok {
log.Infof("[unexpected] CRD transformer called for a non-CRD type")
return crd, nil
}
crd.Spec = apiextensionsv1.CustomResourceDefinitionSpec{}
return crd, nil
}
}
// indexEgressServices adds a local index to a cached Tailscale egress Services meant to be exposed on a ProxyGroup. The
// index is used a list filter.
func indexEgressServices(o client.Object) []string {
@@ -1107,44 +1026,3 @@ func indexEgressServices(o client.Object) []string {
}
return []string{o.GetAnnotations()[AnnotationProxyGroup]}
}
// serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service
// associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation,
// the associated Ingress gets reconciled.
func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
ingList := networkingv1.IngressList{}
if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil {
logger.Debugf("error listing Ingresses: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0)
for _, ing := range ingList.Items {
if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName {
continue
}
if !hasProxyGroupAnnotation(&ing) {
continue
}
if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil && ing.Spec.DefaultBackend.Service.Name == o.GetName() {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
}
for _, rule := range ing.Spec.Rules {
if rule.HTTP == nil {
continue
}
for _, path := range rule.HTTP.Paths {
if path.Backend.Service != nil && path.Backend.Service.Name == o.GetName() {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)})
}
}
}
}
return reqs
}
}
func hasProxyGroupAnnotation(obj client.Object) bool {
ing := obj.(*networkingv1.Ingress)
return ing.Annotations[AnnotationProxyGroup] != ""
}

View File

@@ -16,7 +16,6 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
@@ -106,7 +105,7 @@ func TestLoadBalancerClass(t *testing.T) {
}},
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Delete the misconfiguration so the proxy starts getting created on the
// next reconcile.
@@ -128,9 +127,9 @@ func TestLoadBalancerClass(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
want.Annotations = nil
want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"}
@@ -143,7 +142,7 @@ func TestLoadBalancerClass(t *testing.T) {
Message: "no Tailscale hostname known yet, waiting for proxy pod to finish auth",
}},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@@ -169,7 +168,7 @@ func TestLoadBalancerClass(t *testing.T) {
},
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Turn the service back into a ClusterIP service, which should make the
// operator clean up.
@@ -206,7 +205,7 @@ func TestLoadBalancerClass(t *testing.T) {
Type: corev1.ServiceTypeClusterIP,
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestTailnetTargetFQDNAnnotation(t *testing.T) {
@@ -266,9 +265,9 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
app: kubetypes.AppEgressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
@@ -288,10 +287,10 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, want, nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
// Change the tailscale-target-fqdn annotation which should update the
// StatefulSet
@@ -378,9 +377,9 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
app: kubetypes.AppEgressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
@@ -400,10 +399,10 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, want, nil)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
// Change the tailscale-target-ip annotation which should update the
// StatefulSet
@@ -501,7 +500,7 @@ func TestTailnetTargetIPAnnotation_IPCouldNotBeParsed(t *testing.T) {
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
@@ -572,7 +571,7 @@ func TestTailnetTargetIPAnnotation_InvalidIP(t *testing.T) {
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestAnnotations(t *testing.T) {
@@ -629,9 +628,9 @@ func TestAnnotations(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
@@ -650,7 +649,7 @@ func TestAnnotations(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Turn the service back into a ClusterIP service, which should make the
// operator clean up.
@@ -678,7 +677,7 @@ func TestAnnotations(t *testing.T) {
Type: corev1.ServiceTypeClusterIP,
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestAnnotationIntoLB(t *testing.T) {
@@ -735,9 +734,9 @@ func TestAnnotationIntoLB(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, since it would have normally happened at
@@ -769,7 +768,7 @@ func TestAnnotationIntoLB(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Remove Tailscale's annotation, and at the same time convert the service
// into a tailscale LoadBalancer.
@@ -780,8 +779,8 @@ func TestAnnotationIntoLB(t *testing.T) {
})
expectReconciled(t, sr, "default", "test")
// None of the proxy machinery should have changed...
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
// ... but the service should have a LoadBalancer status.
want = &corev1.Service{
@@ -810,7 +809,7 @@ func TestAnnotationIntoLB(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestLBIntoAnnotation(t *testing.T) {
@@ -865,9 +864,9 @@ func TestLBIntoAnnotation(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
// Normally the Tailscale proxy pod would come up here and write its info
// into the secret. Simulate that, then verify reconcile again and verify
@@ -907,7 +906,7 @@ func TestLBIntoAnnotation(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Turn the service back into a ClusterIP service, but also add the
// tailscale annotation.
@@ -926,8 +925,8 @@ func TestLBIntoAnnotation(t *testing.T) {
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
@@ -947,7 +946,7 @@ func TestLBIntoAnnotation(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestCustomHostname(t *testing.T) {
@@ -1005,9 +1004,9 @@ func TestCustomHostname(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, o))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, o), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
want := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
@@ -1027,7 +1026,7 @@ func TestCustomHostname(t *testing.T) {
Conditions: proxyCreatedCondition(clock),
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
// Turn the service back into a ClusterIP service, which should make the
// operator clean up.
@@ -1058,7 +1057,7 @@ func TestCustomHostname(t *testing.T) {
Type: corev1.ServiceTypeClusterIP,
},
}
expectEqual(t, fc, want)
expectEqual(t, fc, want, nil)
}
func TestCustomPriorityClassName(t *testing.T) {
@@ -1118,7 +1117,7 @@ func TestCustomPriorityClassName(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
}
func TestProxyClassForService(t *testing.T) {
@@ -1130,7 +1129,7 @@ func TestProxyClassForService(t *testing.T) {
AcceptRoutes: true,
},
StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar"},
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"bar.io/foo": "some-val"},
Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}},
}
@@ -1186,9 +1185,9 @@ func TestProxyClassForService(t *testing.T) {
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 2. The Service gets updated with tailscale.com/proxy-class label
// pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not
@@ -1197,8 +1196,8 @@ func TestProxyClassForService(t *testing.T) {
mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata")
})
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
// 3. ProxyClass is set to Ready, the Service gets reconciled by the
// services-reconciler and the customization from the ProxyClass is
@@ -1213,7 +1212,7 @@ func TestProxyClassForService(t *testing.T) {
})
opts.proxyClass = pc.Name
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t))
// 4. tailscale.com/proxy-class label is removed from the Service, the
@@ -1224,7 +1223,7 @@ func TestProxyClassForService(t *testing.T) {
})
opts.proxyClass = ""
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
}
func TestDefaultLoadBalancer(t *testing.T) {
@@ -1270,7 +1269,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
o := configOpts{
stsName: shortName,
secretName: fullName,
@@ -1280,7 +1279,8 @@ func TestDefaultLoadBalancer(t *testing.T) {
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
}
func TestProxyFirewallMode(t *testing.T) {
@@ -1336,7 +1336,7 @@ func TestProxyFirewallMode(t *testing.T) {
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation)
}
func TestTailscaledConfigfileHash(t *testing.T) {
@@ -1378,7 +1378,6 @@ func TestTailscaledConfigfileHash(t *testing.T) {
},
})
expectReconciled(t, sr, "default", "test")
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
@@ -1389,10 +1388,10 @@ func TestTailscaledConfigfileHash(t *testing.T) {
parentType: "svc",
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
confFileHash: "848bff4b5ba83ac999e6984c8464e597156daba961ae045e7dbaef606d54ab5e",
confFileHash: "a67b5ad3ff605531c822327e8f1a23dd0846e1075b722c13402f7d5d0ba32ba2",
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, o), nil)
// 2. Hostname gets changed, configfile is updated and a new hash value
// is produced.
@@ -1400,9 +1399,9 @@ func TestTailscaledConfigfileHash(t *testing.T) {
mak.Set(&svc.Annotations, AnnotationHostname, "another-test")
})
o.hostname = "another-test"
o.confFileHash = "d4cc13f09f55f4f6775689004f9a466723325b84d2b590692796bfe22aeaa389"
o.confFileHash = "888a993ebee20ad6be99623b45015339de117946850cf1252bede0b570e04293"
expectReconciled(t, sr, "default", "test")
expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs)
expectEqual(t, fc, expectedSTS(t, fc, o), nil)
}
func Test_isMagicDNSName(t *testing.T) {
tests := []struct {
@@ -1680,9 +1679,9 @@ func Test_authKeyRemoval(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 2. Apply update to the Secret that imitates the proxy setting device_id.
s := expectedSecret(t, fc, opts)
@@ -1694,7 +1693,7 @@ func Test_authKeyRemoval(t *testing.T) {
expectReconciled(t, sr, "default", "test")
opts.shouldRemoveAuthKey = true
opts.secretExtraData = map[string][]byte{"device_id": []byte("dkkdi4CNTRL")}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
}
func Test_externalNameService(t *testing.T) {
@@ -1754,9 +1753,9 @@ func Test_externalNameService(t *testing.T) {
app: kubetypes.AppIngressProxy,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
// 2. Change the ExternalName and verify that changes get propagated.
mustUpdate(t, sr, "default", "test", func(s *corev1.Service) {
@@ -1764,107 +1763,7 @@ func Test_externalNameService(t *testing.T) {
})
expectReconciled(t, sr, "default", "test")
opts.clusterTargetDNS = "bar.com"
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs)
}
func Test_metricsResourceCreation(t *testing.T) {
pc := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
Spec: tsapi.ProxyClassSpec{},
Status: tsapi.ProxyClassStatus{
Conditions: []metav1.Condition{{
Status: metav1.ConditionTrue,
Type: string(tsapi.ProxyClassReady),
ObservedGeneration: 1,
}}},
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Labels: map[string]string{LabelProxyClass: "metrics"},
},
Spec: corev1.ServiceSpec{
ClusterIP: "10.20.30.40",
Type: corev1.ServiceTypeLoadBalancer,
LoadBalancerClass: ptr.To("tailscale"),
},
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pc, svc).
WithStatusSubresource(pc).
Build()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
clock := tstest.NewClock(tstest.ClockOpts{})
sr := &ServiceReconciler{
Client: fc,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
operatorNamespace: "operator-ns",
},
logger: zl.Sugar(),
clock: clock,
}
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
stsName: shortName,
secretName: fullName,
namespace: "default",
parentType: "svc",
tailscaleNamespace: "operator-ns",
hostname: "default-test",
namespaced: true,
proxyType: proxyTypeIngressService,
app: kubetypes.AppIngressProxy,
resourceVersion: "1",
}
// 1. Enable metrics- expect metrics Service to be created
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec = tsapi.ProxyClassSpec{Metrics: &tsapi.Metrics{Enable: true}}
})
expectReconciled(t, sr, "default", "test")
opts.enableMetrics = true
expectEqual(t, fc, expectedMetricsService(opts))
// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
})
expectReconciled(t, sr, "default", "test")
// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
mustCreate(t, fc, crd)
expectReconciled(t, sr, "default", "test")
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 4. A change to ServiceMonitor config gets reflected in the ServiceMonitor resource
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar"}
})
expectReconciled(t, sr, "default", "test")
opts.serviceMonitorLabels = tsapi.Labels{"foo": "bar"}
opts.resourceVersion = "2"
expectEqual(t, fc, expectedMetricsService(opts))
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
// 5. Disable metrics- expect metrics Service to be deleted
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
pc.Spec.Metrics = nil
})
expectReconciled(t, sr, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", metricsResourceName(opts.stsName))
// ServiceMonitor gets garbage collected when Service gets deleted (it has OwnerReference of the Service
// object). We cannot test this using the fake client.
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
}
func toFQDN(t *testing.T, s string) dnsname.FQDN {

View File

@@ -311,7 +311,7 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) {
// Now add the impersonation headers that we want.
if err := addImpersonationHeaders(r, h.log); err != nil {
log.Print("failed to add impersonation headers: ", err.Error())
log.Printf("failed to add impersonation headers: " + err.Error())
}
}

View File

@@ -15,7 +15,6 @@ import (
dockerref "github.com/distribution/reference"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apivalidation "k8s.io/apimachinery/pkg/api/validation"
@@ -96,14 +95,14 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
pcr.mu.Unlock()
oldPCStatus := pc.Status.DeepCopy()
if errs := pcr.validate(ctx, pc); errs != nil {
if errs := pcr.validate(pc); errs != nil {
msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error())
pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg)
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger)
} else {
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, pc.Generation, pcr.clock, logger)
}
if !apiequality.Semantic.DeepEqual(oldPCStatus, &pc.Status) {
if !apiequality.Semantic.DeepEqual(oldPCStatus, pc.Status) {
if err := pcr.Client.Status().Update(ctx, pc); err != nil {
logger.Errorf("error updating ProxyClass status: %v", err)
return reconcile.Result{}, err
@@ -112,10 +111,10 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re
return reconcile.Result{}, nil
}
func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) {
func (pcr *ProxyClassReconciler) validate(pc *tsapi.ProxyClass) (violations field.ErrorList) {
if sts := pc.Spec.StatefulSet; sts != nil {
if len(sts.Labels) > 0 {
if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil {
if errs := metavalidation.ValidateLabels(sts.Labels, field.NewPath(".spec.statefulSet.labels")); errs != nil {
violations = append(violations, errs...)
}
}
@@ -126,7 +125,7 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
}
if pod := sts.Pod; pod != nil {
if len(pod.Labels) > 0 {
if errs := metavalidation.ValidateLabels(pod.Labels.Parse(), field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
if errs := metavalidation.ValidateLabels(pod.Labels, field.NewPath(".spec.statefulSet.pod.labels")); errs != nil {
violations = append(violations, errs...)
}
}
@@ -161,28 +160,9 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "image"), tc.Image, err.Error()))
}
}
if tc.Debug != nil {
violations = append(violations, field.TypeInvalid(field.NewPath("spec", "statefulSet", "pod", "tailscaleInitContainer", "debug"), tc.Debug, "debug settings cannot be configured on the init container"))
}
}
}
}
if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && pc.Spec.Metrics.ServiceMonitor.Enable {
found, err := hasServiceMonitorCRD(ctx, pcr.Client)
if err != nil {
pcr.logger.Infof("[unexpected]: error retrieving %q CRD: %v", serviceMonitorCRD, err)
// best effort validation - don't error out here
} else if !found {
msg := fmt.Sprintf("ProxyClass defines that a ServiceMonitor custom resource should be created, but %q CRD was not found", serviceMonitorCRD)
violations = append(violations, field.TypeInvalid(field.NewPath("spec", "metrics", "serviceMonitor"), "enable", msg))
}
}
if pc.Spec.Metrics != nil && pc.Spec.Metrics.ServiceMonitor != nil && len(pc.Spec.Metrics.ServiceMonitor.Labels) > 0 {
if errs := metavalidation.ValidateLabels(pc.Spec.Metrics.ServiceMonitor.Labels.Parse(), field.NewPath(".spec.metrics.serviceMonitor.labels")); errs != nil {
violations = append(violations, errs...)
}
}
// We do not validate embedded fields (security context, resource
// requirements etc) as we inherit upstream validation for those fields.
// Invalid values would get rejected by upstream validations at apply
@@ -190,16 +170,6 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl
return violations
}
func hasServiceMonitorCRD(ctx context.Context, cl client.Client) (bool, error) {
sm := &apiextensionsv1.CustomResourceDefinition{}
if err := cl.Get(ctx, types.NamespacedName{Name: serviceMonitorCRD}, sm); apierrors.IsNotFound(err) {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
// maybeCleanup removes tailscale.com finalizer and ensures that the ProxyClass
// is no longer counted towards k8s_proxyclass_resources.
func (pcr *ProxyClassReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, pc *tsapi.ProxyClass) error {

View File

@@ -8,12 +8,10 @@
package main
import (
"context"
"testing"
"time"
"go.uber.org/zap"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
@@ -36,10 +34,10 @@ func TestProxyClass(t *testing.T) {
},
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar", "xyz1234": "abc567"},
Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"},
Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
Pod: &tsapi.Pod{
Labels: tsapi.Labels{"foo": "bar", "xyz1234": "abc567"},
Labels: map[string]string{"foo": "bar", "xyz1234": "abc567"},
Annotations: map[string]string{"foo.io/bar": "{'key': 'val1232'}"},
TailscaleContainer: &tsapi.Container{
Env: []tsapi.Env{{Name: "FOO", Value: "BAR"}},
@@ -78,7 +76,7 @@ func TestProxyClass(t *testing.T) {
LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)},
})
expectEqual(t, fc, pc)
expectEqual(t, fc, pc, nil)
// 2. A ProxyClass resource with invalid labels gets its status updated to Invalid with an error message.
pc.Spec.StatefulSet.Labels["foo"] = "?!someVal"
@@ -88,7 +86,7 @@ func TestProxyClass(t *testing.T) {
expectReconciled(t, pcr, "", "test")
msg := `ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: "?!someVal": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
expectEqual(t, fc, pc, nil)
expectedEvent := "Warning ProxyClassInvalid ProxyClass is not valid: .spec.statefulSet.labels: Invalid value: \"?!someVal\": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')"
expectEvents(t, fr, []string{expectedEvent})
@@ -102,7 +100,7 @@ func TestProxyClass(t *testing.T) {
expectReconciled(t, pcr, "", "test")
msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
expectEqual(t, fc, pc, nil)
expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
expectEvents(t, fr, []string{expectedEvent})
@@ -121,7 +119,7 @@ func TestProxyClass(t *testing.T) {
expectReconciled(t, pcr, "", "test")
msg = `ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
expectEqual(t, fc, pc, nil)
expectedEvent = `Warning ProxyClassInvalid ProxyClass is not valid: spec.statefulSet.pod.tailscaleInitContainer.image: Invalid value: "FOO bar": invalid reference format: repository name (library/FOO bar) must be lowercase`
expectEvents(t, fr, []string{expectedEvent})
@@ -136,95 +134,4 @@ func TestProxyClass(t *testing.T) {
"Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."}
expectReconciled(t, pcr, "", "test")
expectEvents(t, fr, expectedEvents)
// 6. A ProxyClass with ServiceMonitor enabled and in a cluster that has not ServiceMonitor CRD is invalid
pc.Spec.Metrics = &tsapi.Metrics{Enable: true, ServiceMonitor: &tsapi.ServiceMonitor{Enable: true}}
mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec = pc.Spec
})
expectReconciled(t, pcr, "", "test")
msg = `ProxyClass is not valid: spec.metrics.serviceMonitor: Invalid value: "enable": ProxyClass defines that a ServiceMonitor custom resource should be created, but "servicemonitors.monitoring.coreos.com" CRD was not found`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
expectedEvent = "Warning ProxyClassInvalid " + msg
expectEvents(t, fr, []string{expectedEvent})
// 7. A ProxyClass with ServiceMonitor enabled and in a cluster that does have the ServiceMonitor CRD is valid
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
mustCreate(t, fc, crd)
expectReconciled(t, pcr, "", "test")
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
// 7. A ProxyClass with invalid ServiceMonitor labels gets its status updated to Invalid with an error message.
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar!"}
mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels
})
expectReconciled(t, pcr, "", "test")
msg = `ProxyClass is not valid: .spec.metrics.serviceMonitor.labels: Invalid value: "bar!": a valid label must be an empty string or consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyValue', or 'my_value', or '12345', regex used for validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?')`
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
// 8. A ProxyClass with valid ServiceMonitor labels gets its status updated to Valid.
pc.Spec.Metrics.ServiceMonitor.Labels = tsapi.Labels{"foo": "bar", "xyz1234": "abc567", "empty": "", "onechar": "a"}
mustUpdate(t, fc, "", "test", func(proxyClass *tsapi.ProxyClass) {
proxyClass.Spec.Metrics.ServiceMonitor.Labels = pc.Spec.Metrics.ServiceMonitor.Labels
})
expectReconciled(t, pcr, "", "test")
tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionTrue, reasonProxyClassValid, reasonProxyClassValid, 0, cl, zl.Sugar())
expectEqual(t, fc, pc)
}
func TestValidateProxyClass(t *testing.T) {
for name, tc := range map[string]struct {
pc *tsapi.ProxyClass
valid bool
}{
"empty": {
valid: true,
pc: &tsapi.ProxyClass{},
},
"debug_enabled_for_main_container": {
valid: true,
pc: &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Pod: &tsapi.Pod{
TailscaleContainer: &tsapi.Container{
Debug: &tsapi.Debug{
Enable: true,
},
},
},
},
},
},
},
"debug_enabled_for_init_container": {
valid: false,
pc: &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Pod: &tsapi.Pod{
TailscaleInitContainer: &tsapi.Container{
Debug: &tsapi.Debug{
Enable: true,
},
},
},
},
},
},
},
} {
t.Run(name, func(t *testing.T) {
pcr := &ProxyClassReconciler{}
err := pcr.validate(context.Background(), tc.pc)
valid := err == nil
if valid != tc.valid {
t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err)
}
})
}
}

View File

@@ -12,7 +12,6 @@ import (
"fmt"
"net/http"
"slices"
"strings"
"sync"
"github.com/pkg/errors"
@@ -46,15 +45,9 @@ const (
reasonProxyGroupReady = "ProxyGroupReady"
reasonProxyGroupCreating = "ProxyGroupCreating"
reasonProxyGroupInvalid = "ProxyGroupInvalid"
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
)
var (
gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount)
)
var gaugeProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount)
// ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition.
type ProxyGroupReconciler struct {
@@ -71,9 +64,8 @@ type ProxyGroupReconciler struct {
tsFirewallMode string
defaultProxyClass string
mu sync.Mutex // protects following
egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge
ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge
mu sync.Mutex // protects following
proxyGroups set.Slice[types.UID] // for proxygroups gauge
}
func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@@ -118,7 +110,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
oldPGStatus := pg.Status.DeepCopy()
setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) {
if !apiequality.Semantic.DeepEqual(oldPGStatus, pg.Status) {
// An error encountered here should get returned by the Reconcile function.
if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil {
err = errors.Wrap(err, updateErr.Error())
@@ -174,17 +166,9 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
if err = r.maybeProvision(ctx, pg, proxyClass); err != nil {
reason := reasonProxyGroupCreationFailed
msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
reason = reasonProxyGroupCreating
msg = fmt.Sprintf("optimistic lock error, retrying: %s", err)
err = nil
logger.Info(msg)
} else {
r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg)
}
return setStatusReady(pg, metav1.ConditionFalse, reason, msg)
err = fmt.Errorf("error provisioning ProxyGroup resources: %w", err)
r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error())
return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error())
}
desiredReplicas := int(pgReplicas(pg))
@@ -207,7 +191,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ
func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error {
logger := r.logger(pg.Name)
r.mu.Lock()
r.ensureAddedToGaugeForProxyGroup(pg)
r.proxyGroups.Add(pg.UID)
gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
r.mu.Unlock()
cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass)
@@ -258,67 +243,22 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro
existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
}); err != nil {
return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err)
return fmt.Errorf("error provisioning ConfigMap: %w", err)
}
}
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
cm := pgIngressCM(pg, r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) {
existing.ObjectMeta.Labels = cm.ObjectMeta.Labels
existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences
}); err != nil {
return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err)
}
}
ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode)
ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, cfgHash)
if err != nil {
return fmt.Errorf("error generating StatefulSet spec: %w", err)
}
ss = applyProxyClassToStatefulSet(proxyClass, ss, nil, logger)
capver, err := r.capVerForPG(ctx, pg, logger)
if err != nil {
return fmt.Errorf("error getting device info: %w", err)
}
updateSS := func(s *appsv1.StatefulSet) {
// This is a temporary workaround to ensure that egress ProxyGroup proxies with capver older than 110
// are restarted when tailscaled configfile contents have changed.
// This workaround ensures that:
// 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110.
// 2. Proxies above capver are not unnecessarily restarted when the configfile contents change.
// 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid
// unnecessary pod restarts that could result in an update loop where capver cannot be determined for a
// restarting Pod and the hash is re-added again.
// Note that this workaround is only applied to egress ProxyGroups, because ingress ProxyGroup was added after capver 110.
// Note also that the hash annotation is only set on updates, not creation, because if the StatefulSet is
// being created, there is no need for a restart.
// TODO(irbekrm): remove this in 1.84.
hash := cfgHash
if capver >= 110 {
hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash]
}
s.Spec = ss.Spec
if hash != "" && pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash)
}
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) {
s.ObjectMeta.Labels = ss.ObjectMeta.Labels
s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations
s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences
}
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil {
s.Spec = ss.Spec
}); err != nil {
return fmt.Errorf("error provisioning StatefulSet: %w", err)
}
mo := &metricsOpts{
tsNamespace: r.tsNamespace,
proxyStsName: pg.Name,
proxyLabels: pgLabels(pg.Name, nil),
proxyType: "proxygroup",
}
if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil {
return fmt.Errorf("error reconciling metrics resources: %w", err)
}
if err := r.cleanupDanglingResources(ctx, pg); err != nil {
return fmt.Errorf("error cleaning up dangling resources: %w", err)
@@ -387,17 +327,10 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy
}
}
mo := &metricsOpts{
proxyLabels: pgLabels(pg.Name, nil),
tsNamespace: r.tsNamespace,
proxyType: "proxygroup"}
if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil {
return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
}
logger.Infof("cleaned up ProxyGroup resources")
r.mu.Lock()
r.ensureRemovedFromGaugeForProxyGroup(pg)
r.proxyGroups.Remove(pg.UID)
gaugeProxyGroupResources.Set(int64(r.proxyGroups.Len()))
r.mu.Unlock()
return true, nil
}
@@ -507,32 +440,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p
return configSHA256Sum, nil
}
// ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup
// is created. r.mu must be held.
func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGroup) {
switch pg.Spec.Type {
case tsapi.ProxyGroupTypeEgress:
r.egressProxyGroups.Add(pg.UID)
case tsapi.ProxyGroupTypeIngress:
r.ingressProxyGroups.Add(pg.UID)
}
gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
}
// ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the
// ProxyGroup is deleted. r.mu must be held.
func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.ProxyGroup) {
switch pg.Spec.Type {
case tsapi.ProxyGroupTypeEgress:
r.egressProxyGroups.Remove(pg.UID)
case tsapi.ProxyGroupTypeIngress:
r.ingressProxyGroups.Remove(pg.UID)
}
gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len()))
gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len()))
}
func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
conf := &ipn.ConfigVAlpha{
Version: "alpha0",
@@ -543,7 +450,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32
}
if pg.Spec.HostnamePrefix != "" {
conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx))
conf.Hostname = ptr.To(fmt.Sprintf("%s%d", pg.Spec.HostnamePrefix, idx))
}
if shouldAcceptRoutes(class) {
@@ -600,19 +507,12 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr
continue
}
nm := nodeMetadata{
metadata = append(metadata, nodeMetadata{
ordinal: ordinal,
stateSecret: &secret,
tsID: id,
dnsName: dnsName,
}
pod := &corev1.Pod{}
if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) {
return nil, err
} else if err == nil {
nm.podUID = string(pod.UID)
}
metadata = append(metadata, nm)
})
}
return metadata, nil
@@ -644,29 +544,6 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.Prox
type nodeMetadata struct {
ordinal int
stateSecret *corev1.Secret
// podUID is the UID of the current Pod or empty if the Pod does not exist.
podUID string
tsID tailcfg.StableNodeID
dnsName string
}
// capVerForPG returns best effort capability version for the given ProxyGroup. It attempts to find it by looking at the
// Secret + Pod for the replica with ordinal 0. Returns -1 if it is not possible to determine the capability version
// (i.e there is no Pod yet).
func (r *ProxyGroupReconciler) capVerForPG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (tailcfg.CapabilityVersion, error) {
metas, err := r.getNodeMetadata(ctx, pg)
if err != nil {
return -1, fmt.Errorf("error getting node metadata: %w", err)
}
if len(metas) == 0 {
return -1, nil
}
dev, err := deviceInfo(metas[0].stateSecret, metas[0].podUID, logger)
if err != nil {
return -1, fmt.Errorf("error getting device info: %w", err)
}
if dev == nil {
return -1, nil
}
return dev.capver, nil
tsID tailcfg.StableNodeID
dnsName string
}

View File

@@ -21,7 +21,7 @@ import (
// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
// applied over the top after.
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string) (*appsv1.StatefulSet, error) {
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode, cfgHash string) (*appsv1.StatefulSet, error) {
ss := new(appsv1.StatefulSet)
if err := yaml.Unmarshal(proxyYaml, &ss); err != nil {
return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err)
@@ -53,13 +53,12 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
Namespace: namespace,
Labels: pgLabels(pg.Name, nil),
DeletionGracePeriodSeconds: ptr.To[int64](10),
Annotations: map[string]string{
podAnnotationLastSetConfigFileHash: cfgHash,
},
}
tmpl.Spec.ServiceAccountName = pg.Name
tmpl.Spec.InitContainers[0].Image = image
proxyConfigVolName := pgEgressCMName(pg.Name)
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress {
proxyConfigVolName = pgIngressCMName(pg.Name)
}
tmpl.Spec.Volumes = func() []corev1.Volume {
var volumes []corev1.Volume
for i := range pgReplicas(pg) {
@@ -73,16 +72,18 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
})
}
volumes = append(volumes, corev1.Volume{
Name: proxyConfigVolName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: proxyConfigVolName,
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
volumes = append(volumes, corev1.Volume{
Name: pgEgressCMName(pg.Name),
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: pgEgressCMName(pg.Name),
},
},
},
},
})
})
}
return volumes
}()
@@ -104,11 +105,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
})
}
mounts = append(mounts, corev1.VolumeMount{
Name: proxyConfigVolName,
MountPath: "/etc/proxies",
ReadOnly: true,
})
if pg.Spec.Type == tsapi.ProxyGroupTypeEgress {
mounts = append(mounts, corev1.VolumeMount{
Name: pgEgressCMName(pg.Name),
MountPath: "/etc/proxies",
ReadOnly: true,
})
}
return mounts
}()
@@ -135,6 +138,10 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
Value: "/etc/tsconfig/$(POD_NAME)",
},
{
Name: "TS_INTERNAL_APP",
Value: kubetypes.AppProxyGroupEgress,
},
}
if tsFirewallMode != "" {
@@ -148,22 +155,9 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
envs = append(envs, corev1.EnvVar{
Name: "TS_EGRESS_SERVICES_CONFIG_PATH",
Value: fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices),
},
corev1.EnvVar{
Name: "TS_INTERNAL_APP",
Value: kubetypes.AppProxyGroupEgress,
},
)
} else { // ingress
envs = append(envs, corev1.EnvVar{
Name: "TS_INTERNAL_APP",
Value: kubetypes.AppProxyGroupIngress,
},
corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
Value: fmt.Sprintf("/etc/proxies/%s", serveConfigKey),
})
})
}
return append(c.Env, envs...)
}()
@@ -268,16 +262,6 @@ func pgEgressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
},
}
}
func pgIngressCM(pg *tsapi.ProxyGroup, namespace string) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: pgIngressCMName(pg.Name),
Namespace: namespace,
Labels: pgLabels(pg.Name, nil),
OwnerReferences: pgOwnerReference(pg),
},
}
}
func pgSecretLabels(pgName, typ string) map[string]string {
return pgLabels(pgName, map[string]string{

View File

@@ -17,7 +17,6 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -25,11 +24,8 @@ import (
"tailscale.com/client/tailscale"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
)
const testProxyImage = "tailscale/tailscale:test"
@@ -56,9 +52,6 @@ func TestProxyGroup(t *testing.T) {
Name: "test",
Finalizers: []string{"tailscale.com/finalizer"},
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeEgress,
},
}
fc := fake.NewClientBuilder().
@@ -83,20 +76,12 @@ func TestProxyGroup(t *testing.T) {
l: zl.Sugar(),
clock: cl,
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
opts := configOpts{
proxyType: "proxygroup",
stsName: pg.Name,
parentType: "proxygroup",
tailscaleNamespace: "tailscale",
resourceVersion: "1",
}
t.Run("proxyclass_not_ready", func(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, false, "")
})
@@ -117,12 +102,12 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, "")
if expected := 1; reconciler.egressProxyGroups.Len() != expected {
t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
if expected := 1; reconciler.proxyGroups.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.proxyGroups.Len())
}
expectProxyGroupResources(t, fc, pg, true, "")
expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
keyReq := tailscale.KeyCapabilities{
Devices: tailscale.KeyDeviceCapabilities{
Create: tailscale.KeyDeviceCreateCapabilities{
@@ -153,7 +138,7 @@ func TestProxyGroup(t *testing.T) {
},
}
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
})
@@ -164,7 +149,7 @@ func TestProxyGroup(t *testing.T) {
})
expectReconciled(t, reconciler, "", pg.Name)
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
addNodeIDToStateSecrets(t, fc, pg)
@@ -174,7 +159,7 @@ func TestProxyGroup(t *testing.T) {
Hostname: "hostname-nodeid-2",
TailnetIPs: []string{"1.2.3.4", "::1"},
})
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
})
@@ -187,7 +172,7 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device.
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, true, initialCfgHash)
})
@@ -201,31 +186,10 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
expectEqual(t, fc, pg)
expectEqual(t, fc, pg, nil)
expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74")
})
t.Run("enable_metrics", func(t *testing.T) {
pc.Spec.Metrics = &tsapi.Metrics{Enable: true}
mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
p.Spec = pc.Spec
})
expectReconciled(t, reconciler, "", pg.Name)
expectEqual(t, fc, expectedMetricsService(opts))
})
t.Run("enable_service_monitor_no_crd", func(t *testing.T) {
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) {
p.Spec.Metrics = pc.Spec.Metrics
})
expectReconciled(t, reconciler, "", pg.Name)
})
t.Run("create_crd_expect_service_monitor", func(t *testing.T) {
mustCreate(t, fc, crd)
expectReconciled(t, reconciler, "", pg.Name)
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
})
t.Run("delete_and_cleanup", func(t *testing.T) {
if err := fc.Delete(context.Background(), pg); err != nil {
t.Fatal(err)
@@ -233,195 +197,37 @@ func TestProxyGroup(t *testing.T) {
expectReconciled(t, reconciler, "", pg.Name)
expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name)
if expected := 0; reconciler.egressProxyGroups.Len() != expected {
t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len())
expectMissing[tsapi.Recorder](t, fc, "", pg.Name)
if expected := 0; reconciler.proxyGroups.Len() != expected {
t.Fatalf("expected %d ProxyGroups, got %d", expected, reconciler.proxyGroups.Len())
}
// 2 nodes should get deleted as part of the scale down, and then finally
// the first node gets deleted with the ProxyGroup cleanup.
if diff := cmp.Diff(tsClient.deleted, []string{"nodeid-1", "nodeid-2", "nodeid-0"}); diff != "" {
t.Fatalf("unexpected deleted devices (-got +want):\n%s", diff)
}
expectMissing[corev1.Service](t, reconciler, "tailscale", metricsResourceName(pg.Name))
// The fake client does not clean up objects whose owner has been
// deleted, so we can't test for the owned resources getting deleted.
})
}
func TestProxyGroupTypes(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
Build()
zl, _ := zap.NewDevelopment()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
proxyImage: testProxyImage,
Client: fc,
l: zl.Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
}
t.Run("egress_type", func(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-egress",
UID: "test-egress-uid",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeEgress,
Replicas: ptr.To[int32](0),
},
}
if err := fc.Create(context.Background(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 0, 1)
sts := &appsv1.StatefulSet{}
if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
t.Fatalf("failed to get StatefulSet: %v", err)
}
verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress)
verifyEnvVar(t, sts, "TS_EGRESS_SERVICES_CONFIG_PATH", fmt.Sprintf("/etc/proxies/%s", egressservices.KeyEgressServices))
// Verify that egress configuration has been set up.
cm := &corev1.ConfigMap{}
cmName := fmt.Sprintf("%s-egress-config", pg.Name)
if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil {
t.Fatalf("failed to get ConfigMap: %v", err)
}
expectedVolumes := []corev1.Volume{
{
Name: cmName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: cmName,
},
},
},
},
}
expectedVolumeMounts := []corev1.VolumeMount{
{
Name: cmName,
MountPath: "/etc/proxies",
ReadOnly: true,
},
}
if diff := cmp.Diff(expectedVolumes, sts.Spec.Template.Spec.Volumes); diff != "" {
t.Errorf("unexpected volumes (-want +got):\n%s", diff)
}
if diff := cmp.Diff(expectedVolumeMounts, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" {
t.Errorf("unexpected volume mounts (-want +got):\n%s", diff)
}
})
t.Run("ingress_type", func(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress",
UID: "test-ingress-uid",
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
Replicas: ptr.To[int32](0),
},
}
if err := fc.Create(context.Background(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 1, 1)
sts := &appsv1.StatefulSet{}
if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
t.Fatalf("failed to get StatefulSet: %v", err)
}
verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
// Verify ConfigMap volume mount
cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
expectedVolume := corev1.Volume{
Name: cmName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: cmName,
},
},
},
}
expectedVolumeMount := corev1.VolumeMount{
Name: cmName,
MountPath: "/etc/proxies",
ReadOnly: true,
}
if diff := cmp.Diff([]corev1.Volume{expectedVolume}, sts.Spec.Template.Spec.Volumes); diff != "" {
t.Errorf("unexpected volumes (-want +got):\n%s", diff)
}
if diff := cmp.Diff([]corev1.VolumeMount{expectedVolumeMount}, sts.Spec.Template.Spec.Containers[0].VolumeMounts); diff != "" {
t.Errorf("unexpected volume mounts (-want +got):\n%s", diff)
}
})
}
func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) {
t.Helper()
if r.ingressProxyGroups.Len() != wantIngress {
t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len())
}
if r.egressProxyGroups.Len() != wantEgress {
t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len())
}
}
func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) {
t.Helper()
for _, env := range sts.Spec.Template.Spec.Containers[0].Env {
if env.Name == name {
if env.Value != expectedValue {
t.Errorf("expected %s=%s, got %s", name, expectedValue, env.Value)
}
return
}
}
t.Errorf("%s environment variable not found", name)
}
func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string) {
t.Helper()
role := pgRole(pg, tsNamespace)
roleBinding := pgRoleBinding(pg, tsNamespace)
serviceAccount := pgServiceAccount(pg, tsNamespace)
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto")
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", cfgHash)
if err != nil {
t.Fatal(err)
}
statefulSet.Annotations = defaultProxyClassAnnotations
if cfgHash != "" {
mak.Set(&statefulSet.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, cfgHash)
}
if shouldExist {
expectEqual(t, fc, role)
expectEqual(t, fc, roleBinding)
expectEqual(t, fc, serviceAccount)
expectEqual(t, fc, statefulSet, removeResourceReqs)
expectEqual(t, fc, role, nil)
expectEqual(t, fc, roleBinding, nil)
expectEqual(t, fc, serviceAccount, nil)
expectEqual(t, fc, statefulSet, nil)
} else {
expectMissing[rbacv1.Role](t, fc, role.Namespace, role.Name)
expectMissing[rbacv1.RoleBinding](t, fc, roleBinding.Namespace, roleBinding.Name)

View File

@@ -15,7 +15,6 @@ import (
"net/http"
"os"
"slices"
"strconv"
"strings"
"go.uber.org/zap"
@@ -95,12 +94,6 @@ const (
podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn"
// podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents.
podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash"
proxyTypeEgress = "egress_service"
proxyTypeIngressService = "ingress_service"
proxyTypeIngressResource = "ingress_resource"
proxyTypeConnector = "connector"
proxyTypeProxyGroup = "proxygroup"
)
var (
@@ -129,8 +122,6 @@ type tailscaleSTSConfig struct {
Hostname string
Tags []string // if empty, use defaultTags
proxyType string
// Connector specifies a configuration of a Connector instance if that's
// what this StatefulSet should be created for.
Connector *connector
@@ -172,8 +163,8 @@ func (sts tailscaleSTSReconciler) validate() error {
}
// IsHTTPSEnabledOnTailnet reports whether HTTPS is enabled on the tailnet.
func IsHTTPSEnabledOnTailnet(tsnetServer tsnetServer) bool {
return len(tsnetServer.CertDomains()) > 0
func (a *tailscaleSTSReconciler) IsHTTPSEnabledOnTailnet() bool {
return len(a.tsnetServer.CertDomains()) > 0
}
// Provision ensures that the StatefulSet for the given service is running and
@@ -198,30 +189,22 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
}
sts.ProxyClass = proxyClass
secretName, tsConfigHash, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
secretName, tsConfigHash, configs, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
if err != nil {
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
}
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash)
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash, configs)
if err != nil {
return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
}
mo := &metricsOpts{
proxyStsName: hsvc.Name,
tsNamespace: hsvc.Namespace,
proxyLabels: hsvc.Labels,
proxyType: sts.proxyType,
}
if err = reconcileMetricsResources(ctx, logger, mo, sts.ProxyClass, a.Client); err != nil {
return nil, fmt.Errorf("failed to ensure metrics resources: %w", err)
}
return hsvc, nil
}
// Cleanup removes all resources associated that were created by Provision with
// the given labels. It returns true when all resources have been removed,
// otherwise it returns false and the caller should retry later.
func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string, typ string) (done bool, _ error) {
func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.SugaredLogger, labels map[string]string) (done bool, _ error) {
// Need to delete the StatefulSet first, and delete it with foreground
// cascading deletion. That way, the pod that's writing to the Secret will
// stop running before we start looking at the Secret's contents, and
@@ -247,21 +230,21 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
return false, nil
}
dev, err := a.DeviceInfo(ctx, labels, logger)
id, _, _, err := a.DeviceInfo(ctx, labels)
if err != nil {
return false, fmt.Errorf("getting device info: %w", err)
}
if dev != nil && dev.id != "" {
logger.Debugf("deleting device %s from control", string(dev.id))
if err := a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
if id != "" {
logger.Debugf("deleting device %s from control", string(id))
if err := a.tsClient.DeleteDevice(ctx, string(id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(id))
} else {
return false, fmt.Errorf("deleting device: %w", err)
}
} else {
logger.Debugf("device %s deleted from control", string(dev.id))
logger.Debugf("device %s deleted from control", string(id))
}
}
@@ -274,14 +257,6 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
return false, err
}
}
mo := &metricsOpts{
proxyLabels: labels,
tsNamespace: a.operatorNamespace,
proxyType: typ,
}
if err := maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
}
return true, nil
}
@@ -437,75 +412,44 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
return string(sanitizedBytes)
}
// DeviceInfo returns the device ID, hostname, IPs and capver for the Tailscale device that acts as an operator proxy.
// It retrieves info from a Kubernetes Secret labeled with the provided labels. Capver is cross-validated against the
// Pod to ensure that it is the currently running Pod that set the capver. If the Pod or the Secret does not exist, the
// returned capver is -1. Either of device ID, hostname and IPs can be empty string if not found in the Secret.
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) {
// DeviceInfo returns the device ID, hostname and IPs for the Tailscale device
// that acts as an operator proxy. It retrieves info from a Kubernetes Secret
// labeled with the provided labels.
// Either of device ID, hostname and IPs can be empty string if not found in the Secret.
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
if err != nil {
return dev, err
return "", "", nil, err
}
if sec == nil {
return dev, nil
return "", "", nil, nil
}
podUID := ""
pod := new(corev1.Pod)
if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) {
return dev, err
} else if err == nil {
podUID = string(pod.ObjectMeta.UID)
}
return deviceInfo(sec, podUID, logger)
return deviceInfo(sec)
}
// device contains tailscale state of a proxy device as gathered from its tailscale state Secret.
type device struct {
id tailcfg.StableNodeID // device's stable ID
hostname string // MagicDNS name of the device
ips []string // Tailscale IPs of the device
// ingressDNSName is the L7 Ingress DNS name. In practice this will be the same value as hostname, but only set
// when the device has been configured to serve traffic on it via 'tailscale serve'.
ingressDNSName string
capver tailcfg.CapabilityVersion
}
func deviceInfo(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) (dev *device, err error) {
id := tailcfg.StableNodeID(sec.Data[kubetypes.KeyDeviceID])
func deviceInfo(sec *corev1.Secret) (id tailcfg.StableNodeID, hostname string, ips []string, err error) {
id = tailcfg.StableNodeID(sec.Data["device_id"])
if id == "" {
return dev, nil
return "", "", nil, nil
}
dev = &device{id: id}
// Kubernetes chokes on well-formed FQDNs with the trailing dot, so we have
// to remove it.
dev.hostname = strings.TrimSuffix(string(sec.Data[kubetypes.KeyDeviceFQDN]), ".")
if dev.hostname == "" {
hostname = strings.TrimSuffix(string(sec.Data["device_fqdn"]), ".")
if hostname == "" {
// Device ID gets stored and retrieved in a different flow than
// FQDN and IPs. A device that acts as Kubernetes operator
// proxy, but whose route setup has failed might have a device
// proxy, but whose route setup has failed might have an device
// ID, but no FQDN/IPs. If so, return the ID, to allow the
// operator to clean up such devices.
return dev, nil
return id, "", nil, nil
}
dev.ingressDNSName = dev.hostname
pcv := proxyCapVer(sec, podUID, log)
dev.capver = pcv
// TODO(irbekrm): we fall back to using the hostname field to determine Ingress's hostname to ensure backwards
// compatibility. In 1.82 we can remove this fallback mechanism.
if pcv >= 109 {
dev.ingressDNSName = strings.TrimSuffix(string(sec.Data[kubetypes.KeyHTTPSEndpoint]), ".")
if strings.EqualFold(dev.ingressDNSName, kubetypes.ValueNoHTTPS) {
dev.ingressDNSName = ""
}
}
if rawDeviceIPs, ok := sec.Data[kubetypes.KeyDeviceIPs]; ok {
ips := make([]string, 0)
if rawDeviceIPs, ok := sec.Data["device_ips"]; ok {
if err := json.Unmarshal(rawDeviceIPs, &ips); err != nil {
return nil, err
return "", "", nil, err
}
dev.ips = ips
}
return dev, nil
return id, hostname, ips, nil
}
func newAuthKey(ctx context.Context, tsClient tsClient, tags []string) (string, error) {
@@ -532,7 +476,7 @@ var proxyYaml []byte
//go:embed deploy/manifests/userspace-proxy.yaml
var userspaceProxyYaml []byte
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) {
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string, configs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) (*appsv1.StatefulSet, error) {
ss := new(appsv1.StatefulSet)
if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -589,6 +533,8 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
Value: "true",
})
}
// Configure containeboot to run tailscaled with a configfile read from the state Secret.
mak.Set(&ss.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, tsConfigHash)
configVolume := corev1.Volume{
Name: "tailscaledconfig",
@@ -658,12 +604,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
},
})
}
dev, err := a.DeviceInfo(ctx, sts.ChildResourceLabels, logger)
if err != nil {
return nil, fmt.Errorf("failed to get device info: %w", err)
}
app, err := appInfoForProxy(sts)
if err != nil {
// No need to error out if now or in future we end up in a
@@ -682,25 +622,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger)
}
updateSS := func(s *appsv1.StatefulSet) {
// This is a temporary workaround to ensure that proxies with capver older than 110
// are restarted when tailscaled configfile contents have changed.
// This workaround ensures that:
// 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110.
// 2. Proxies above capver are not unnecessarily restarted when the configfile contents change.
// 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid
// unnecessary pod restarts that could result in an update loop where capver cannot be determined for a
// restarting Pod and the hash is re-added again.
// Note that the hash annotation is only set on updates not creation, because if the StatefulSet is
// being created, there is no need for a restart.
// TODO(irbekrm): remove this in 1.84.
hash := tsConfigHash
if dev != nil && dev.capver >= 110 {
hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash]
}
s.Spec = ss.Spec
if hash != "" {
mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash)
}
s.ObjectMeta.Labels = ss.Labels
s.ObjectMeta.Annotations = ss.Annotations
}
@@ -744,42 +666,24 @@ func mergeStatefulSetLabelsOrAnnots(current, custom map[string]string, managed [
return custom
}
func debugSetting(pc *tsapi.ProxyClass) bool {
if pc == nil ||
pc.Spec.StatefulSet == nil ||
pc.Spec.StatefulSet.Pod == nil ||
pc.Spec.StatefulSet.Pod.TailscaleContainer == nil ||
pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug == nil {
// This default will change to false in 1.82.0.
return pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
}
return pc.Spec.StatefulSet.Pod.TailscaleContainer.Debug.Enable
}
func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, stsCfg *tailscaleSTSConfig, logger *zap.SugaredLogger) *appsv1.StatefulSet {
if pc == nil || ss == nil {
return ss
}
metricsEnabled := pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable
debugEnabled := debugSetting(pc)
if metricsEnabled || debugEnabled {
isEgress := stsCfg != nil && (stsCfg.TailnetTargetFQDN != "" || stsCfg.TailnetTargetIP != "")
isForwardingL7Ingress := stsCfg != nil && stsCfg.ForwardClusterTrafficViaL7IngressProxy
if isEgress {
if stsCfg != nil && pc.Spec.Metrics != nil && pc.Spec.Metrics.Enable {
if stsCfg.TailnetTargetFQDN == "" && stsCfg.TailnetTargetIP == "" && !stsCfg.ForwardClusterTrafficViaL7IngressProxy {
enableMetrics(ss)
} else if stsCfg.ForwardClusterTrafficViaL7IngressProxy {
// TODO (irbekrm): fix this
// For Ingress proxies that have been configured with
// tailscale.com/experimental-forward-cluster-traffic-via-ingress
// annotation, all cluster traffic is forwarded to the
// Ingress backend(s).
logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for egress proxies.")
} else if isForwardingL7Ingress {
logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
} else {
// TODO (irbekrm): fix this
// For egress proxies, currently all cluster traffic is forwarded to the tailnet target.
logger.Info("ProxyClass specifies that metrics should be enabled, but this is currently not supported for Ingress proxies that accept cluster traffic.")
} else {
enableEndpoints(ss, metricsEnabled, debugEnabled)
}
}
@@ -788,7 +692,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
}
// Update StatefulSet metadata.
if wantsSSLabels := pc.Spec.StatefulSet.Labels.Parse(); len(wantsSSLabels) > 0 {
if wantsSSLabels := pc.Spec.StatefulSet.Labels; len(wantsSSLabels) > 0 {
ss.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.ObjectMeta.Labels, wantsSSLabels, tailscaleManagedLabels)
}
if wantsSSAnnots := pc.Spec.StatefulSet.Annotations; len(wantsSSAnnots) > 0 {
@@ -800,7 +704,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
return ss
}
wantsPod := pc.Spec.StatefulSet.Pod
if wantsPodLabels := wantsPod.Labels.Parse(); len(wantsPodLabels) > 0 {
if wantsPodLabels := wantsPod.Labels; len(wantsPodLabels) > 0 {
ss.Spec.Template.ObjectMeta.Labels = mergeStatefulSetLabelsOrAnnots(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, tailscaleManagedLabels)
}
if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 {
@@ -857,58 +761,16 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet,
return ss
}
func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) {
func enableMetrics(ss *appsv1.StatefulSet) {
for i, c := range ss.Spec.Template.Spec.Containers {
if c.Name == "tailscale" {
if debug {
ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
// Serve tailscaled's debug metrics on on
// <pod-ip>:9001/debug/metrics. If we didn't specify Pod IP
// here, the proxy would, in some cases, also listen to its
// Tailscale IP- we don't want folks to start relying on this
// side-effect as a feature.
corev1.EnvVar{
Name: "TS_DEBUG_ADDR_PORT",
Value: "$(POD_IP):9001",
},
// TODO(tomhjp): Can remove this env var once 1.76.x is no
// longer supported.
corev1.EnvVar{
Name: "TS_TAILSCALED_EXTRA_ARGS",
Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
},
)
ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
corev1.ContainerPort{
Name: "debug",
Protocol: "TCP",
ContainerPort: 9001,
},
)
}
if metrics {
ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env,
// Serve client metrics on <pod-ip>:9002/metrics.
corev1.EnvVar{
Name: "TS_LOCAL_ADDR_PORT",
Value: "$(POD_IP):9002",
},
corev1.EnvVar{
Name: "TS_ENABLE_METRICS",
Value: "true",
},
)
ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports,
corev1.ContainerPort{
Name: "metrics",
Protocol: "TCP",
ContainerPort: 9002,
},
)
}
// Serve metrics on on <pod-ip>:9001/debug/metrics. If
// we didn't specify Pod IP here, the proxy would, in
// some cases, also listen to its Tailscale IP- we don't
// want folks to start relying on this side-effect as a
// feature.
ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
ss.Spec.Template.Spec.Containers[i].Ports = append(ss.Spec.Template.Spec.Containers[i].Ports, corev1.ContainerPort{Name: "metrics", Protocol: "TCP", HostPort: 9001, ContainerPort: 9001})
break
}
}
@@ -932,10 +794,17 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
AcceptRoutes: "false", // AcceptRoutes defaults to true
Locked: "false",
Hostname: &stsC.Hostname,
NoStatefulFiltering: "true", // Explicitly enforce default value, see #14216
NoStatefulFiltering: "false",
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
}
// For egress proxies only, we need to ensure that stateful filtering is
// not in place so that traffic from cluster can be forwarded via
// Tailscale IPs.
// TODO (irbekrm): set it to true always as this is now the default in core.
if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
conf.NoStatefulFiltering = "true"
}
if stsC.Connector != nil {
routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
if err != nil {
@@ -1138,24 +1007,3 @@ func nameForService(svc *corev1.Service) string {
func isValidFirewallMode(m string) bool {
return m == "auto" || m == "nftables" || m == "iptables"
}
// proxyCapVer accepts a proxy state Secret and UID of the current proxy Pod returns the capability version of the
// tailscale running in that Pod. This is best effort - if the capability version can not (currently) be determined, it
// returns -1.
func proxyCapVer(sec *corev1.Secret, podUID string, log *zap.SugaredLogger) tailcfg.CapabilityVersion {
if sec == nil || podUID == "" {
return tailcfg.CapabilityVersion(-1)
}
if len(sec.Data[kubetypes.KeyCapVer]) == 0 || len(sec.Data[kubetypes.KeyPodUID]) == 0 {
return tailcfg.CapabilityVersion(-1)
}
capVer, err := strconv.Atoi(string(sec.Data[kubetypes.KeyCapVer]))
if err != nil {
log.Infof("[unexpected]: unexpected capability version in proxy's state Secret, expected an integer, got %q", string(sec.Data[kubetypes.KeyCapVer]))
return tailcfg.CapabilityVersion(-1)
}
if !strings.EqualFold(podUID, string(sec.Data[kubetypes.KeyPodUID])) {
return tailcfg.CapabilityVersion(-1)
}
return tailcfg.CapabilityVersion(capVer)
}

View File

@@ -61,10 +61,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
proxyClassAllOpts := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar"},
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"foo.io/bar": "foo"},
Pod: &tsapi.Pod{
Labels: tsapi.Labels{"bar": "foo"},
Labels: map[string]string{"bar": "foo"},
Annotations: map[string]string{"bar.io/foo": "foo"},
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: ptr.To(int64(0)),
@@ -116,36 +116,21 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
proxyClassJustLabels := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
StatefulSet: &tsapi.StatefulSet{
Labels: tsapi.Labels{"foo": "bar"},
Labels: map[string]string{"foo": "bar"},
Annotations: map[string]string{"foo.io/bar": "foo"},
Pod: &tsapi.Pod{
Labels: tsapi.Labels{"bar": "foo"},
Labels: map[string]string{"bar": "foo"},
Annotations: map[string]string{"bar.io/foo": "foo"},
},
},
},
}
proxyClassWithMetricsDebug := func(metrics bool, debug *bool) *tsapi.ProxyClass {
return &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
Metrics: &tsapi.Metrics{Enable: metrics},
StatefulSet: func() *tsapi.StatefulSet {
if debug == nil {
return nil
}
return &tsapi.StatefulSet{
Pod: &tsapi.Pod{
TailscaleContainer: &tsapi.Container{
Debug: &tsapi.Debug{Enable: *debug},
},
},
}
}(),
},
}
proxyClassMetrics := &tsapi.ProxyClass{
Spec: tsapi.ProxyClassSpec{
Metrics: &tsapi.Metrics{Enable: true},
},
}
var userspaceProxySS, nonUserspaceProxySS appsv1.StatefulSet
if err := yaml.Unmarshal(userspaceProxyYaml, &userspaceProxySS); err != nil {
t.Fatalf("unmarshaling userspace proxy template: %v", err)
@@ -175,9 +160,9 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
// 1. Test that a ProxyClass with all fields set gets correctly applied
// to a Statefulset built from non-userspace proxy template.
wantSS := nonUserspaceProxySS.DeepCopy()
updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
@@ -199,28 +184,28 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
gotSS := applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
t.Fatalf("Unexpected result applying ProxyClass with all fields set to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
}
// 2. Test that a ProxyClass with custom labels and annotations for
// StatefulSet and Pod set gets correctly applied to a Statefulset built
// from non-userspace proxy template.
wantSS = nonUserspaceProxySS.DeepCopy()
updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for non-userspace proxy (-got +want):\n%s", diff)
}
// 3. Test that a ProxyClass with all fields set gets correctly applied
// to a Statefulset built from a userspace proxy template.
wantSS = userspaceProxySS.DeepCopy()
updateMap(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassAllOpts.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassAllOpts.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassAllOpts.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassAllOpts.Spec.StatefulSet.Pod.Annotations
wantSS.Spec.Template.Spec.SecurityContext = proxyClassAllOpts.Spec.StatefulSet.Pod.SecurityContext
wantSS.Spec.Template.Spec.ImagePullSecrets = proxyClassAllOpts.Spec.StatefulSet.Pod.ImagePullSecrets
@@ -236,61 +221,36 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) {
wantSS.Spec.Template.Spec.Containers[0].Image = "ghcr.io/my-repo/tailscale:v0.01testsomething"
gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
t.Fatalf("Unexpected result applying ProxyClass with all options to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
}
// 4. Test that a ProxyClass with custom labels and annotations gets correctly applied
// to a Statefulset built from a userspace proxy template.
wantSS = userspaceProxySS.DeepCopy()
updateMap(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels.Parse())
updateMap(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels.Parse()
wantSS.ObjectMeta.Labels = mergeMapKeys(wantSS.ObjectMeta.Labels, proxyClassJustLabels.Spec.StatefulSet.Labels)
wantSS.ObjectMeta.Annotations = mergeMapKeys(wantSS.ObjectMeta.Annotations, proxyClassJustLabels.Spec.StatefulSet.Annotations)
wantSS.Spec.Template.Labels = proxyClassJustLabels.Spec.StatefulSet.Pod.Labels
wantSS.Spec.Template.Annotations = proxyClassJustLabels.Spec.StatefulSet.Pod.Annotations
gotSS = applyProxyClassToStatefulSet(proxyClassJustLabels, userspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
t.Fatalf("Unexpected result applying ProxyClass with custom labels and annotations to a StatefulSet for a userspace proxy (-got +want):\n%s", diff)
}
// 5. Metrics enabled defaults to enabling both metrics and debug.
// 5. Test that a ProxyClass with metrics enabled gets correctly applied to a StatefulSet.
wantSS = nonUserspaceProxySS.DeepCopy()
wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"},
)
wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{
{Name: "debug", Protocol: "TCP", ContainerPort: 9001},
{Name: "metrics", Protocol: "TCP", ContainerPort: 9002},
}
gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, nil), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(POD_IP):9001"})
wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9001, HostPort: 9001}}
gotSS = applyProxyClassToStatefulSet(proxyClassMetrics, nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
t.Fatalf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
}
}
// 6. Enable _just_ metrics by explicitly disabling debug.
wantSS = nonUserspaceProxySS.DeepCopy()
wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
corev1.EnvVar{Name: "TS_LOCAL_ADDR_PORT", Value: "$(POD_IP):9002"},
corev1.EnvVar{Name: "TS_ENABLE_METRICS", Value: "true"},
)
wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "metrics", Protocol: "TCP", ContainerPort: 9002}}
gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(true, ptr.To(false)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
}
// 7. Enable _just_ debug without metrics.
wantSS = nonUserspaceProxySS.DeepCopy()
wantSS.Spec.Template.Spec.Containers[0].Env = append(wantSS.Spec.Template.Spec.Containers[0].Env,
corev1.EnvVar{Name: "TS_DEBUG_ADDR_PORT", Value: "$(POD_IP):9001"},
corev1.EnvVar{Name: "TS_TAILSCALED_EXTRA_ARGS", Value: "--debug=$(TS_DEBUG_ADDR_PORT)"},
)
wantSS.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{{Name: "debug", Protocol: "TCP", ContainerPort: 9001}}
gotSS = applyProxyClassToStatefulSet(proxyClassWithMetricsDebug(false, ptr.To(true)), nonUserspaceProxySS.DeepCopy(), new(tailscaleSTSConfig), zl.Sugar())
if diff := cmp.Diff(gotSS, wantSS); diff != "" {
t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff)
func mergeMapKeys(a, b map[string]string) map[string]string {
for key, val := range b {
a[key] = val
}
return a
}
func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
@@ -384,10 +344,3 @@ func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) {
})
}
}
// updateMap updates map a with the values from map b.
func updateMap(a, b map[string]string) {
for key, val := range b {
a[key] = val
}
}

View File

@@ -121,15 +121,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
return reconcile.Result{}, a.maybeCleanup(ctx, logger, svc)
}
if err := a.maybeProvision(ctx, logger, svc); err != nil {
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
logger.Infof("optimistic lock error, retrying: %s", err)
} else {
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
return reconcile.Result{}, a.maybeProvision(ctx, logger, svc)
}
// maybeCleanup removes any existing resources related to serving svc over tailscale.
@@ -139,7 +131,7 @@ func (a *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
oldSvcStatus := svc.Status.DeepCopy()
defer func() {
if !apiequality.Semantic.DeepEqual(oldSvcStatus, &svc.Status) {
if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
// An error encountered here should get returned by the Reconcile function.
err = errors.Join(err, a.Client.Status().Update(ctx, svc))
}
@@ -160,12 +152,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
return nil
}
proxyTyp := proxyTypeEgress
if a.shouldExpose(svc) {
proxyTyp = proxyTypeIngressService
}
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc"), proxyTyp); err != nil {
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(svc.Name, svc.Namespace, "svc")); err != nil {
return fmt.Errorf("failed to cleanup: %w", err)
} else if !done {
logger.Debugf("cleanup not done yet, waiting for next reconcile")
@@ -204,7 +191,7 @@ func (a *ServiceReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service) (err error) {
oldSvcStatus := svc.Status.DeepCopy()
defer func() {
if !apiequality.Semantic.DeepEqual(oldSvcStatus, &svc.Status) {
if !apiequality.Semantic.DeepEqual(oldSvcStatus, svc.Status) {
// An error encountered here should get returned by the Reconcile function.
err = errors.Join(err, a.Client.Status().Update(ctx, svc))
}
@@ -269,10 +256,6 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
ChildResourceLabels: crl,
ProxyClassName: proxyClass,
}
sts.proxyType = proxyTypeEgress
if a.shouldExpose(svc) {
sts.proxyType = proxyTypeIngressService
}
a.mu.Lock()
if a.shouldExposeClusterIP(svc) {
@@ -328,11 +311,11 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
_, tsHost, tsIPs, err := a.ssr.DeviceInfo(ctx, crl)
if err != nil {
return fmt.Errorf("failed to get device ID: %w", err)
}
if dev == nil || dev.hostname == "" {
if tsHost == "" {
msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
logger.Debug(msg)
// No hostname yet. Wait for the proxy pod to auth.
@@ -341,9 +324,9 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
logger.Debugf("setting Service LoadBalancer status to %q, %s", dev.hostname, strings.Join(dev.ips, ", "))
logger.Debugf("setting Service LoadBalancer status to %q, %s", tsHost, strings.Join(tsIPs, ", "))
ingress := []corev1.LoadBalancerIngress{
{Hostname: dev.hostname},
{Hostname: tsHost},
}
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
if err != nil {
@@ -351,7 +334,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
return errors.New(msg)
}
for _, ip := range dev.ips {
for _, ip := range tsIPs {
addr, err := netip.ParseAddr(ip)
if err != nil {
continue

View File

@@ -8,8 +8,6 @@ package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/netip"
"reflect"
"strings"
@@ -23,14 +21,12 @@ import (
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
@@ -43,10 +39,7 @@ type configOpts struct {
secretName string
hostname string
namespace string
tailscaleNamespace string
namespaced bool
parentType string
proxyType string
priorityClassName string
firewallMode string
tailnetTargetIP string
@@ -63,10 +56,6 @@ type configOpts struct {
app string
shouldRemoveAuthKey bool
secretExtraData map[string][]byte
resourceVersion string
enableMetrics bool
serviceMonitorLabels tsapi.Labels
}
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -87,7 +76,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
},
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
ImagePullPolicy: "Always",
}
@@ -97,7 +88,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
Value: "true",
})
}
var annots map[string]string
annots := make(map[string]string)
var volumes []corev1.Volume
volumes = []corev1.Volume{
{
@@ -115,7 +106,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
MountPath: "/etc/tsconfig",
}}
if opts.confFileHash != "" {
mak.Set(&annots, "tailscale.com/operator-last-set-config-file-hash", opts.confFileHash)
annots["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash
}
if opts.firewallMode != "" {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
@@ -124,13 +115,13 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
})
}
if opts.tailnetTargetIP != "" {
mak.Set(&annots, "tailscale.com/operator-last-set-ts-tailnet-target-ip", opts.tailnetTargetIP)
annots["tailscale.com/operator-last-set-ts-tailnet-target-ip"] = opts.tailnetTargetIP
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_TAILNET_TARGET_IP",
Value: opts.tailnetTargetIP,
})
} else if opts.tailnetTargetFQDN != "" {
mak.Set(&annots, "tailscale.com/operator-last-set-ts-tailnet-target-fqdn", opts.tailnetTargetFQDN)
annots["tailscale.com/operator-last-set-ts-tailnet-target-fqdn"] = opts.tailnetTargetFQDN
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_TAILNET_TARGET_FQDN",
Value: opts.tailnetTargetFQDN,
@@ -141,13 +132,13 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
Name: "TS_DEST_IP",
Value: opts.clusterTargetIP,
})
mak.Set(&annots, "tailscale.com/operator-last-set-cluster-ip", opts.clusterTargetIP)
annots["tailscale.com/operator-last-set-cluster-ip"] = opts.clusterTargetIP
} else if opts.clusterTargetDNS != "" {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_EXPERIMENTAL_DEST_DNS_NAME",
Value: opts.clusterTargetDNS,
})
mak.Set(&annots, "tailscale.com/operator-last-set-cluster-dns-name", opts.clusterTargetDNS)
annots["tailscale.com/operator-last-set-cluster-dns-name"] = opts.clusterTargetDNS
}
if opts.serveConfig != nil {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
@@ -161,29 +152,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
Name: "TS_INTERNAL_APP",
Value: opts.app,
})
if opts.enableMetrics {
tsContainer.Env = append(tsContainer.Env,
corev1.EnvVar{
Name: "TS_DEBUG_ADDR_PORT",
Value: "$(POD_IP):9001"},
corev1.EnvVar{
Name: "TS_TAILSCALED_EXTRA_ARGS",
Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
},
corev1.EnvVar{
Name: "TS_LOCAL_ADDR_PORT",
Value: "$(POD_IP):9002",
},
corev1.EnvVar{
Name: "TS_ENABLE_METRICS",
Value: "true",
},
)
tsContainer.Ports = append(tsContainer.Ports,
corev1.ContainerPort{Name: "debug", ContainerPort: 9001, Protocol: "TCP"},
corev1.ContainerPort{Name: "metrics", ContainerPort: 9002, Protocol: "TCP"},
)
}
ss := &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
Kind: "StatefulSet",
@@ -275,29 +243,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"},
},
}
if opts.enableMetrics {
tsContainer.Env = append(tsContainer.Env,
corev1.EnvVar{
Name: "TS_DEBUG_ADDR_PORT",
Value: "$(POD_IP):9001"},
corev1.EnvVar{
Name: "TS_TAILSCALED_EXTRA_ARGS",
Value: "--debug=$(TS_DEBUG_ADDR_PORT)",
},
corev1.EnvVar{
Name: "TS_LOCAL_ADDR_PORT",
Value: "$(POD_IP):9002",
},
corev1.EnvVar{
Name: "TS_ENABLE_METRICS",
Value: "true",
},
)
tsContainer.Ports = append(tsContainer.Ports, corev1.ContainerPort{
Name: "debug", ContainerPort: 9001, Protocol: "TCP"},
corev1.ContainerPort{Name: "metrics", ContainerPort: 9002, Protocol: "TCP"},
)
}
volumes := []corev1.Volume{
{
Name: "tailscaledconfig",
@@ -392,90 +337,6 @@ func expectedHeadlessService(name string, parentType string) *corev1.Service {
}
}
func expectedMetricsService(opts configOpts) *corev1.Service {
labels := metricsLabels(opts)
selector := map[string]string{
"tailscale.com/managed": "true",
"tailscale.com/parent-resource": "test",
"tailscale.com/parent-resource-type": opts.parentType,
}
if opts.namespaced {
selector["tailscale.com/parent-resource-ns"] = opts.namespace
}
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: metricsResourceName(opts.stsName),
Namespace: opts.tailscaleNamespace,
Labels: labels,
},
Spec: corev1.ServiceSpec{
Selector: selector,
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{{Protocol: "TCP", Port: 9002, Name: "metrics"}},
},
}
}
func metricsLabels(opts configOpts) map[string]string {
promJob := fmt.Sprintf("ts_%s_default_test", opts.proxyType)
if !opts.namespaced {
promJob = fmt.Sprintf("ts_%s_test", opts.proxyType)
}
labels := map[string]string{
"tailscale.com/managed": "true",
"tailscale.com/metrics-target": opts.stsName,
"ts_prom_job": promJob,
"ts_proxy_type": opts.proxyType,
"ts_proxy_parent_name": "test",
}
if opts.namespaced {
labels["ts_proxy_parent_namespace"] = "default"
}
return labels
}
func expectedServiceMonitor(t *testing.T, opts configOpts) *unstructured.Unstructured {
t.Helper()
smLabels := metricsLabels(opts)
if len(opts.serviceMonitorLabels) != 0 {
smLabels = mergeMapKeys(smLabels, opts.serviceMonitorLabels.Parse())
}
name := metricsResourceName(opts.stsName)
sm := &ServiceMonitor{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: opts.tailscaleNamespace,
Labels: smLabels,
ResourceVersion: opts.resourceVersion,
OwnerReferences: []metav1.OwnerReference{{APIVersion: "v1", Kind: "Service", Name: name, BlockOwnerDeletion: ptr.To(true), Controller: ptr.To(true)}},
},
TypeMeta: metav1.TypeMeta{
Kind: "ServiceMonitor",
APIVersion: "monitoring.coreos.com/v1",
},
Spec: ServiceMonitorSpec{
Selector: metav1.LabelSelector{MatchLabels: metricsLabels(opts)},
Endpoints: []ServiceMonitorEndpoint{{
Port: "metrics",
}},
NamespaceSelector: ServiceMonitorNamespaceSelector{
MatchNames: []string{opts.tailscaleNamespace},
},
JobLabel: "ts_prom_job",
TargetLabels: []string{
"ts_proxy_parent_name",
"ts_proxy_parent_namespace",
"ts_proxy_type",
},
},
}
u, err := serviceMonitorToUnstructured(sm)
if err != nil {
t.Fatalf("error converting ServiceMonitor to unstructured: %v", err)
}
return u
}
func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Secret {
t.Helper()
s := &corev1.Secret{
@@ -492,14 +353,13 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
mak.Set(&s.StringData, "serve-config", string(serveConfigBs))
}
conf := &ipn.ConfigVAlpha{
Version: "alpha0",
AcceptDNS: "false",
Hostname: &opts.hostname,
Locked: "false",
AuthKey: ptr.To("secret-authkey"),
AcceptRoutes: "false",
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
NoStatefulFiltering: "true",
Version: "alpha0",
AcceptDNS: "false",
Hostname: &opts.hostname,
Locked: "false",
AuthKey: ptr.To("secret-authkey"),
AcceptRoutes: "false",
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
}
if opts.proxyClass != "" {
t.Logf("applying configuration from ProxyClass %s", opts.proxyClass)
@@ -531,6 +391,11 @@ func expectedSecret(t *testing.T, cl client.Client, opts configOpts) *corev1.Sec
routes = append(routes, prefix)
}
}
if opts.tailnetTargetFQDN != "" || opts.tailnetTargetIP != "" {
conf.NoStatefulFiltering = "true"
} else {
conf.NoStatefulFiltering = "false"
}
conf.AdvertiseRoutes = routes
bnn, err := json.Marshal(conf)
if err != nil {
@@ -620,7 +485,7 @@ func mustUpdateStatus[T any, O ptrObject[T]](t *testing.T, client client.Client,
// modify func to ensure that they are removed from the cluster object and the
// object passed as 'want'. If no such modifications are needed, you can pass
// nil in place of the modify function.
func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifiers ...func(O)) {
func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want O, modifier func(O)) {
t.Helper()
got := O(new(T))
if err := client.Get(context.Background(), types.NamespacedName{
@@ -634,7 +499,7 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
// so just remove it from both got and want.
got.SetResourceVersion("")
want.SetResourceVersion("")
for _, modifier := range modifiers {
if modifier != nil {
modifier(want)
modifier(got)
}
@@ -643,29 +508,13 @@ func expectEqual[T any, O ptrObject[T]](t *testing.T, client client.Client, want
}
}
func expectEqualUnstructured(t *testing.T, client client.Client, want *unstructured.Unstructured) {
t.Helper()
got := &unstructured.Unstructured{}
got.SetGroupVersionKind(want.GroupVersionKind())
if err := client.Get(context.Background(), types.NamespacedName{
Name: want.GetName(),
Namespace: want.GetNamespace(),
}, got); err != nil {
t.Fatalf("getting %q: %v", want.GetName(), err)
}
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("unexpected contents of Unstructured (-got +want):\n%s", diff)
}
}
func expectMissing[T any, O ptrObject[T]](t *testing.T, client client.Client, ns, name string) {
t.Helper()
obj := O(new(T))
err := client.Get(context.Background(), types.NamespacedName{
if err := client.Get(context.Background(), types.NamespacedName{
Name: name,
Namespace: ns,
}, obj)
if !apierrors.IsNotFound(err) {
}, obj); !apierrors.IsNotFound(err) {
t.Fatalf("%s %s/%s unexpectedly present, wanted missing", reflect.TypeOf(obj).Elem().Name(), ns, name)
}
}
@@ -739,7 +588,6 @@ type fakeTSClient struct {
sync.Mutex
keyRequests []tailscale.KeyCapabilities
deleted []string
vipServices map[string]*VIPService
}
type fakeTSNetServer struct {
certDomains []string
@@ -797,21 +645,12 @@ func (c *fakeTSClient) Deleted() []string {
// change to the configfile contents).
func removeHashAnnotation(sts *appsv1.StatefulSet) {
delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash)
if len(sts.Spec.Template.Annotations) == 0 {
sts.Spec.Template.Annotations = nil
}
}
func removeResourceReqs(sts *appsv1.StatefulSet) {
if sts != nil {
sts.Spec.Template.Spec.Resources = nil
}
}
func removeTargetPortsFromSvc(svc *corev1.Service) {
newPorts := make([]corev1.ServicePort, 0)
for _, p := range svc.Spec.Ports {
newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port, Name: p.Name})
newPorts = append(newPorts, corev1.ServicePort{Protocol: p.Protocol, Port: p.Port})
}
svc.Spec.Ports = newPorts
}
@@ -845,50 +684,3 @@ func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
}
}
}
func (c *fakeTSClient) getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) {
c.Lock()
defer c.Unlock()
if c.vipServices == nil {
return nil, &tailscale.ErrResponse{Status: http.StatusNotFound}
}
svc, ok := c.vipServices[name]
if !ok {
return nil, &tailscale.ErrResponse{Status: http.StatusNotFound}
}
return svc, nil
}
func (c *fakeTSClient) createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error {
c.Lock()
defer c.Unlock()
if c.vipServices == nil {
c.vipServices = make(map[string]*VIPService)
}
c.vipServices[svc.Name] = svc
return nil
}
func (c *fakeTSClient) deleteVIPServiceByName(ctx context.Context, name string) error {
c.Lock()
defer c.Unlock()
if c.vipServices != nil {
delete(c.vipServices, name)
}
return nil
}
type fakeLocalClient struct {
status *ipnstate.Status
}
func (f *fakeLocalClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) {
if f.status == nil {
return &ipnstate.Status{
Self: &ipnstate.PeerStatus{
DNSName: "test-node.test.ts.net.",
},
}, nil
}
return f.status, nil
}

View File

@@ -1,185 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build !plan9
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"golang.org/x/oauth2/clientcredentials"
"tailscale.com/client/tailscale"
"tailscale.com/util/httpm"
)
// defaultTailnet is a value that can be used in Tailscale API calls instead of tailnet name to indicate that the API
// call should be performed on the default tailnet for the provided credentials.
const (
defaultTailnet = "-"
defaultBaseURL = "https://api.tailscale.com"
)
func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (tsClient, error) {
clientID, err := os.ReadFile(clientIDPath)
if err != nil {
return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err)
}
clientSecret, err := os.ReadFile(clientSecretPath)
if err != nil {
return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err)
}
credentials := clientcredentials.Config{
ClientID: string(clientID),
ClientSecret: string(clientSecret),
TokenURL: "https://login.tailscale.com/api/v2/oauth/token",
}
c := tailscale.NewClient(defaultTailnet, nil)
c.UserAgent = "tailscale-k8s-operator"
c.HTTPClient = credentials.Client(ctx)
tsc := &tsClientImpl{
Client: c,
baseURL: defaultBaseURL,
tailnet: defaultTailnet,
}
return tsc, nil
}
type tsClient interface {
CreateKey(ctx context.Context, caps tailscale.KeyCapabilities) (string, *tailscale.Key, error)
Device(ctx context.Context, deviceID string, fields *tailscale.DeviceFieldsOpts) (*tailscale.Device, error)
DeleteDevice(ctx context.Context, nodeStableID string) error
getVIPServiceByName(ctx context.Context, name string) (*VIPService, error)
createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error
deleteVIPServiceByName(ctx context.Context, name string) error
}
type tsClientImpl struct {
*tailscale.Client
baseURL string
tailnet string
}
// VIPService is a Tailscale VIPService with Tailscale API JSON representation.
type VIPService struct {
// Name is the leftmost label of the DNS name of the VIP service.
// Name is required.
Name string `json:"name,omitempty"`
// Addrs are the IP addresses of the VIP Service. There are two addresses:
// the first is IPv4 and the second is IPv6.
// When creating a new VIP Service, the IP addresses are optional: if no
// addresses are specified then they will be selected. If an IPv4 address is
// specified at index 0, then that address will attempt to be used. An IPv6
// address can not be specified upon creation.
Addrs []string `json:"addrs,omitempty"`
// Comment is an optional text string for display in the admin panel.
Comment string `json:"comment,omitempty"`
// Ports are the ports of a VIPService that will be configured via Tailscale serve config.
// If set, any node wishing to advertise this VIPService must have this port configured via Tailscale serve.
Ports []string `json:"ports,omitempty"`
// Tags are optional ACL tags that will be applied to the VIPService.
Tags []string `json:"tags,omitempty"`
}
// GetVIPServiceByName retrieves a VIPService by its name. It returns 404 if the VIPService is not found.
func (c *tsClientImpl) getVIPServiceByName(ctx context.Context, name string) (*VIPService, error) {
path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(name))
req, err := http.NewRequestWithContext(ctx, httpm.GET, path, nil)
if err != nil {
return nil, fmt.Errorf("error creating new HTTP request: %w", err)
}
b, resp, err := c.sendRequest(req)
if err != nil {
return nil, fmt.Errorf("error making Tailsale API request: %w", err)
}
// If status code was not successful, return the error.
// TODO: Change the check for the StatusCode to include other 2XX success codes.
if resp.StatusCode != http.StatusOK {
return nil, handleErrorResponse(b, resp)
}
svc := &VIPService{}
if err := json.Unmarshal(b, svc); err != nil {
return nil, err
}
return svc, nil
}
// CreateOrUpdateVIPServiceByName creates or updates a VIPService by its name. Caller must ensure that, if the
// VIPService already exists, the VIPService is fetched first to ensure that any auto-allocated IP addresses are not
// lost during the update. If the VIPService was created without any IP addresses explicitly set (so that they were
// auto-allocated by Tailscale) any subsequent request to this function that does not set any IP addresses will error.
func (c *tsClientImpl) createOrUpdateVIPServiceByName(ctx context.Context, svc *VIPService) error {
data, err := json.Marshal(svc)
if err != nil {
return err
}
path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(svc.Name))
req, err := http.NewRequestWithContext(ctx, httpm.PUT, path, bytes.NewBuffer(data))
if err != nil {
return fmt.Errorf("error creating new HTTP request: %w", err)
}
b, resp, err := c.sendRequest(req)
if err != nil {
return fmt.Errorf("error making Tailscale API request: %w", err)
}
// If status code was not successful, return the error.
// TODO: Change the check for the StatusCode to include other 2XX success codes.
if resp.StatusCode != http.StatusOK {
return handleErrorResponse(b, resp)
}
return nil
}
// DeleteVIPServiceByName deletes a VIPService by its name. It returns an error if the VIPService
// does not exist or if the deletion fails.
func (c *tsClientImpl) deleteVIPServiceByName(ctx context.Context, name string) error {
path := fmt.Sprintf("%s/api/v2/tailnet/%s/vip-services/by-name/%s", c.baseURL, c.tailnet, url.PathEscape(name))
req, err := http.NewRequestWithContext(ctx, httpm.DELETE, path, nil)
if err != nil {
return fmt.Errorf("error creating new HTTP request: %w", err)
}
b, resp, err := c.sendRequest(req)
if err != nil {
return fmt.Errorf("error making Tailscale API request: %w", err)
}
// If status code was not successful, return the error.
if resp.StatusCode != http.StatusOK {
return handleErrorResponse(b, resp)
}
return nil
}
// sendRequest add the authentication key to the request and sends it. It
// receives the response and reads up to 10MB of it.
func (c *tsClientImpl) sendRequest(req *http.Request) ([]byte, *http.Response, error) {
resp, err := c.Do(req)
if err != nil {
return nil, resp, fmt.Errorf("error actually doing request: %w", err)
}
defer resp.Body.Close()
// Read response
b, err := io.ReadAll(resp.Body)
if err != nil {
err = fmt.Errorf("error reading response body: %v", err)
}
return b, resp, err
}
// handleErrorResponse decodes the error message from the server and returns
// an ErrResponse from it.
func handleErrorResponse(b []byte, resp *http.Response) error {
var errResp tailscale.ErrResponse
if err := json.Unmarshal(b, &errResp); err != nil {
return err
}
errResp.Status = resp.StatusCode
return errResp
}

View File

@@ -11,7 +11,6 @@ import (
"fmt"
"net/http"
"slices"
"strings"
"sync"
"github.com/pkg/errors"
@@ -39,7 +38,6 @@ import (
const (
reasonRecorderCreationFailed = "RecorderCreationFailed"
reasonRecorderCreating = "RecorderCreating"
reasonRecorderCreated = "RecorderCreated"
reasonRecorderInvalid = "RecorderInvalid"
@@ -104,7 +102,7 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
oldTSRStatus := tsr.Status.DeepCopy()
setStatusReady := func(tsr *tsapi.Recorder, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) {
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, status, reason, message, tsr.Generation, r.clock, logger)
if !apiequality.Semantic.DeepEqual(oldTSRStatus, &tsr.Status) {
if !apiequality.Semantic.DeepEqual(oldTSRStatus, tsr.Status) {
// An error encountered here should get returned by the Reconcile function.
if updateErr := r.Client.Status().Update(ctx, tsr); updateErr != nil {
err = errors.Wrap(err, updateErr.Error())
@@ -121,28 +119,23 @@ func (r *RecorderReconciler) Reconcile(ctx context.Context, req reconcile.Reques
logger.Infof("ensuring Recorder is set up")
tsr.Finalizers = append(tsr.Finalizers, FinalizerName)
if err := r.Update(ctx, tsr); err != nil {
logger.Errorf("error adding finalizer: %w", err)
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, reasonRecorderCreationFailed)
}
}
if err := r.validate(tsr); err != nil {
logger.Errorf("error validating Recorder spec: %w", err)
message := fmt.Sprintf("Recorder is invalid: %s", err)
r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderInvalid, message)
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderInvalid, message)
}
if err = r.maybeProvision(ctx, tsr); err != nil {
reason := reasonRecorderCreationFailed
logger.Errorf("error creating Recorder resources: %w", err)
message := fmt.Sprintf("failed creating Recorder: %s", err)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
reason = reasonRecorderCreating
message = fmt.Sprintf("optimistic lock error, retrying: %s", err)
err = nil
logger.Info(message)
} else {
r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
}
return setStatusReady(tsr, metav1.ConditionFalse, reason, message)
r.recorder.Eventf(tsr, corev1.EventTypeWarning, reasonRecorderCreationFailed, message)
return setStatusReady(tsr, metav1.ConditionFalse, reasonRecorderCreationFailed, message)
}
logger.Info("Recorder resources synced")

View File

@@ -130,15 +130,6 @@ func tsrRole(tsr *tsapi.Recorder, namespace string) *rbacv1.Role {
fmt.Sprintf("%s-0", tsr.Name), // Contains the node state.
},
},
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{
"get",
"create",
"patch",
},
},
},
}
}
@@ -212,14 +203,6 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar {
},
},
},
{
Name: "POD_UID",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.uid",
},
},
},
{
Name: "TS_STATE",
Value: "kube:$(POD_NAME)",

View File

@@ -57,7 +57,7 @@ func TestRecorder(t *testing.T) {
msg := "Recorder is invalid: must either enable UI or use S3 storage to ensure recordings are accessible"
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionFalse, reasonRecorderInvalid, msg, 0, cl, zl.Sugar())
expectEqual(t, fc, tsr)
expectEqual(t, fc, tsr, nil)
if expected := 0; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
@@ -76,7 +76,7 @@ func TestRecorder(t *testing.T) {
expectReconciled(t, reconciler, "", tsr.Name)
tsoperator.SetRecorderCondition(tsr, tsapi.RecorderReady, metav1.ConditionTrue, reasonRecorderCreated, reasonRecorderCreated, 0, cl, zl.Sugar())
expectEqual(t, fc, tsr)
expectEqual(t, fc, tsr, nil)
if expected := 1; reconciler.recorders.Len() != expected {
t.Fatalf("expected %d recorders, got %d", expected, reconciler.recorders.Len())
}
@@ -112,7 +112,7 @@ func TestRecorder(t *testing.T) {
URL: "https://test-0.example.ts.net",
},
}
expectEqual(t, fc, tsr)
expectEqual(t, fc, tsr, nil)
})
t.Run("delete the Recorder and observe cleanup", func(t *testing.T) {
@@ -145,12 +145,12 @@ func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recor
statefulSet := tsrStatefulSet(tsr, tsNamespace)
if shouldExist {
expectEqual(t, fc, auth)
expectEqual(t, fc, state)
expectEqual(t, fc, role)
expectEqual(t, fc, roleBinding)
expectEqual(t, fc, serviceAccount)
expectEqual(t, fc, statefulSet, removeResourceReqs)
expectEqual(t, fc, auth, nil)
expectEqual(t, fc, state, nil)
expectEqual(t, fc, role, nil)
expectEqual(t, fc, roleBinding, nil)
expectEqual(t, fc, serviceAccount, nil)
expectEqual(t, fc, statefulSet, nil)
} else {
expectMissing[corev1.Secret](t, fc, auth.Namespace, auth.Name)
expectMissing[corev1.Secret](t, fc, state.Namespace, state.Name)

View File

@@ -5,40 +5,24 @@
package main
import (
"flag"
"log"
"net"
"os"
"strconv"
"time"
"tailscale.com/net/stun"
)
func main() {
log.SetFlags(0)
var host string
if len(os.Args) < 2 || len(os.Args) > 3 {
log.Fatalf("usage: %s <hostname> [port]", os.Args[0])
}
host := os.Args[1]
port := "3478"
var readTimeout time.Duration
flag.DurationVar(&readTimeout, "timeout", 3*time.Second, "response wait timeout")
flag.Parse()
values := flag.Args()
if len(values) < 1 || len(values) > 2 {
log.Printf("usage: %s <hostname> [port]", os.Args[0])
flag.PrintDefaults()
os.Exit(1)
} else {
for i, value := range values {
switch i {
case 0:
host = value
case 1:
port = value
}
}
if len(os.Args) == 3 {
port = os.Args[2]
}
_, err := strconv.ParseUint(port, 10, 16)
if err != nil {
@@ -62,10 +46,6 @@ func main() {
log.Fatal(err)
}
err = c.SetReadDeadline(time.Now().Add(readTimeout))
if err != nil {
log.Fatal(err)
}
var buf [1024]byte
n, raddr, err := c.ReadFromUDPAddrPort(buf[:])
if err != nil {

View File

@@ -8,11 +8,12 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+
github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+
github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+
github.com/munnerz/goautoneg from github.com/prometheus/common/expfmt
github.com/google/uuid from tailscale.com/util/fastuuid
💣 github.com/prometheus/client_golang/prometheus from tailscale.com/tsweb/promvarz
github.com/prometheus/client_golang/prometheus/internal from github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_model/go from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/expfmt from github.com/prometheus/client_golang/prometheus+
github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg from github.com/prometheus/common/expfmt
github.com/prometheus/common/model from github.com/prometheus/client_golang/prometheus+
LD github.com/prometheus/procfs from github.com/prometheus/client_golang/prometheus
LD github.com/prometheus/procfs/internal/fs from github.com/prometheus/procfs
@@ -55,7 +56,6 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
tailscale.com/net/stun from tailscale.com/net/stunserver
tailscale.com/net/stunserver from tailscale.com/cmd/stund
tailscale.com/net/tsaddr from tailscale.com/tsweb
tailscale.com/syncs from tailscale.com/metrics
tailscale.com/tailcfg from tailscale.com/version
tailscale.com/tsweb from tailscale.com/cmd/stund
tailscale.com/tsweb/promvarz from tailscale.com/tsweb
@@ -74,10 +74,9 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
tailscale.com/util/ctxkey from tailscale.com/tsweb+
L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics
tailscale.com/util/dnsname from tailscale.com/tailcfg
tailscale.com/util/fastuuid from tailscale.com/tsweb
tailscale.com/util/lineiter from tailscale.com/version/distro
tailscale.com/util/mak from tailscale.com/syncs
tailscale.com/util/nocasemaps from tailscale.com/types/ipproto
tailscale.com/util/rands from tailscale.com/tsweb
tailscale.com/util/slicesx from tailscale.com/tailcfg
tailscale.com/util/vizerror from tailscale.com/tailcfg+
tailscale.com/version from tailscale.com/envknob+
@@ -134,6 +133,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
crypto/tls from net/http+
crypto/x509 from crypto/tls
crypto/x509/pkix from crypto/x509
database/sql/driver from github.com/google/uuid
embed from crypto/internal/nistec+
encoding from encoding/json+
encoding/asn1 from crypto/x509+
@@ -155,6 +155,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
html from net/http/pprof+
io from bufio+
io/fs from crypto/x509+
io/ioutil from google.golang.org/protobuf/internal/impl
iter from maps+
log from expvar+
log/internal from log
@@ -163,7 +164,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar
math/big from crypto/dsa+
math/bits from compress/flate+
math/rand from math/big+
math/rand/v2 from internal/concurrent+
math/rand/v2 from tailscale.com/util/fastuuid+
mime from github.com/prometheus/common/expfmt+
mime/multipart from net/http
mime/quotedprintable from mime/multipart

220
cmd/systray/logo.go Normal file
View File

@@ -0,0 +1,220 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build cgo || !darwin
package main
import (
"bytes"
"context"
"image/color"
"image/png"
"sync"
"time"
"fyne.io/systray"
"github.com/fogleman/gg"
)
// tsLogo represents the state of the 3x3 dot grid in the Tailscale logo.
// A 0 represents a gray dot, any other value is a white dot.
type tsLogo [9]byte
var (
// disconnected is all gray dots
disconnected = tsLogo{
0, 0, 0,
0, 0, 0,
0, 0, 0,
}
// connected is the normal Tailscale logo
connected = tsLogo{
0, 0, 0,
1, 1, 1,
0, 1, 0,
}
// loading is a special tsLogo value that is not meant to be rendered directly,
// but indicates that the loading animation should be shown.
loading = tsLogo{'l', 'o', 'a', 'd', 'i', 'n', 'g'}
// loadingIcons are shown in sequence as an animated loading icon.
loadingLogos = []tsLogo{
{
0, 1, 1,
1, 0, 1,
0, 0, 1,
},
{
0, 1, 1,
0, 0, 1,
0, 1, 0,
},
{
0, 1, 1,
0, 0, 0,
0, 0, 1,
},
{
0, 0, 1,
0, 1, 0,
0, 0, 0,
},
{
0, 1, 0,
0, 0, 0,
0, 0, 0,
},
{
0, 0, 0,
0, 0, 1,
0, 0, 0,
},
{
0, 0, 0,
0, 0, 0,
0, 0, 0,
},
{
0, 0, 1,
0, 0, 0,
0, 0, 0,
},
{
0, 0, 0,
0, 0, 0,
1, 0, 0,
},
{
0, 0, 0,
0, 0, 0,
1, 1, 0,
},
{
0, 0, 0,
1, 0, 0,
1, 1, 0,
},
{
0, 0, 0,
1, 1, 0,
0, 1, 0,
},
{
0, 0, 0,
1, 1, 0,
0, 1, 1,
},
{
0, 0, 0,
1, 1, 1,
0, 0, 1,
},
{
0, 1, 0,
0, 1, 1,
1, 0, 1,
},
}
)
var (
black = color.NRGBA{0, 0, 0, 255}
white = color.NRGBA{255, 255, 255, 255}
gray = color.NRGBA{255, 255, 255, 102}
)
// render returns a PNG image of the logo.
func (logo tsLogo) render() *bytes.Buffer {
const radius = 25
const borderUnits = 1
dim := radius * (8 + borderUnits*2)
dc := gg.NewContext(dim, dim)
dc.DrawRectangle(0, 0, float64(dim), float64(dim))
dc.SetColor(black)
dc.Fill()
for y := 0; y < 3; y++ {
for x := 0; x < 3; x++ {
px := (borderUnits + 1 + 3*x) * radius
py := (borderUnits + 1 + 3*y) * radius
col := white
if logo[y*3+x] == 0 {
col = gray
}
dc.DrawCircle(float64(px), float64(py), radius)
dc.SetColor(col)
dc.Fill()
}
}
b := bytes.NewBuffer(nil)
png.Encode(b, dc.Image())
return b
}
// setAppIcon renders logo and sets it as the systray icon.
func setAppIcon(icon tsLogo) {
if icon == loading {
startLoadingAnimation()
} else {
stopLoadingAnimation()
systray.SetIcon(icon.render().Bytes())
}
}
var (
loadingMu sync.Mutex // protects loadingCancel
// loadingCancel stops the loading animation in the systray icon.
// This is nil if the animation is not currently active.
loadingCancel func()
)
// startLoadingAnimation starts the animated loading icon in the system tray.
// The animation continues until [stopLoadingAnimation] is called.
// If the loading animation is already active, this func does nothing.
func startLoadingAnimation() {
loadingMu.Lock()
defer loadingMu.Unlock()
if loadingCancel != nil {
// loading icon already displayed
return
}
ctx := context.Background()
ctx, loadingCancel = context.WithCancel(ctx)
go func() {
t := time.NewTicker(500 * time.Millisecond)
var i int
for {
select {
case <-ctx.Done():
return
case <-t.C:
systray.SetIcon(loadingLogos[i].render().Bytes())
i++
if i >= len(loadingLogos) {
i = 0
}
}
}
}()
}
// stopLoadingAnimation stops the animated loading icon in the system tray.
// If the loading animation is not currently active, this func does nothing.
func stopLoadingAnimation() {
loadingMu.Lock()
defer loadingMu.Unlock()
if loadingCancel != nil {
loadingCancel()
loadingCancel = nil
}
}

View File

@@ -3,13 +3,256 @@
//go:build cgo || !darwin
// systray is a minimal Tailscale systray application.
// The systray command is a minimal Tailscale systray application for Linux.
package main
import (
"tailscale.com/client/systray"
"context"
"errors"
"fmt"
"io"
"log"
"os"
"strings"
"sync"
"time"
"fyne.io/systray"
"github.com/atotto/clipboard"
dbus "github.com/godbus/dbus/v5"
"github.com/toqueteos/webbrowser"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
"tailscale.com/ipn/ipnstate"
)
var (
localClient tailscale.LocalClient
chState chan ipn.State // tailscale state changes
appIcon *os.File
)
func main() {
new(systray.Menu).Run()
systray.Run(onReady, onExit)
}
// Menu represents the systray menu, its items, and the current Tailscale state.
type Menu struct {
mu sync.Mutex // protects the entire Menu
status *ipnstate.Status
connect *systray.MenuItem
disconnect *systray.MenuItem
self *systray.MenuItem
more *systray.MenuItem
quit *systray.MenuItem
eventCancel func() // cancel eventLoop
}
func onReady() {
log.Printf("starting")
ctx := context.Background()
setAppIcon(disconnected)
// dbus wants a file path for notification icons, so copy to a temp file.
appIcon, _ = os.CreateTemp("", "tailscale-systray.png")
io.Copy(appIcon, connected.render())
chState = make(chan ipn.State, 1)
status, err := localClient.Status(ctx)
if err != nil {
log.Print(err)
}
menu := new(Menu)
menu.rebuild(status)
go watchIPNBus(ctx)
}
// rebuild the systray menu based on the current Tailscale state.
//
// We currently rebuild the entire menu because it is not easy to update the existing menu.
// You cannot iterate over the items in a menu, nor can you remove some items like separators.
// So for now we rebuild the whole thing, and can optimize this later if needed.
func (menu *Menu) rebuild(status *ipnstate.Status) {
menu.mu.Lock()
defer menu.mu.Unlock()
if menu.eventCancel != nil {
menu.eventCancel()
}
menu.status = status
systray.ResetMenu()
menu.connect = systray.AddMenuItem("Connect", "")
menu.disconnect = systray.AddMenuItem("Disconnect", "")
menu.disconnect.Hide()
systray.AddSeparator()
if status != nil && status.Self != nil {
title := fmt.Sprintf("This Device: %s (%s)", status.Self.HostName, status.Self.TailscaleIPs[0])
menu.self = systray.AddMenuItem(title, "")
}
systray.AddSeparator()
menu.more = systray.AddMenuItem("More settings", "")
menu.more.Enable()
menu.quit = systray.AddMenuItem("Quit", "Quit the app")
menu.quit.Enable()
ctx := context.Background()
ctx, menu.eventCancel = context.WithCancel(ctx)
go menu.eventLoop(ctx)
}
// eventLoop is the main event loop for handling click events on menu items
// and responding to Tailscale state changes.
// This method does not return until ctx.Done is closed.
func (menu *Menu) eventLoop(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case state := <-chState:
switch state {
case ipn.Running:
setAppIcon(loading)
status, err := localClient.Status(ctx)
if err != nil {
log.Printf("error getting tailscale status: %v", err)
}
menu.rebuild(status)
setAppIcon(connected)
menu.connect.SetTitle("Connected")
menu.connect.Disable()
menu.disconnect.Show()
menu.disconnect.Enable()
case ipn.NoState, ipn.Stopped:
menu.connect.SetTitle("Connect")
menu.connect.Enable()
menu.disconnect.Hide()
setAppIcon(disconnected)
case ipn.Starting:
setAppIcon(loading)
}
case <-menu.connect.ClickedCh:
_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
Prefs: ipn.Prefs{
WantRunning: true,
},
WantRunningSet: true,
})
if err != nil {
log.Print(err)
continue
}
case <-menu.disconnect.ClickedCh:
_, err := localClient.EditPrefs(ctx, &ipn.MaskedPrefs{
Prefs: ipn.Prefs{
WantRunning: false,
},
WantRunningSet: true,
})
if err != nil {
log.Printf("disconnecting: %v", err)
continue
}
case <-menu.self.ClickedCh:
copyTailscaleIP(menu.status.Self)
case <-menu.more.ClickedCh:
webbrowser.Open("http://100.100.100.100/")
case <-menu.quit.ClickedCh:
systray.Quit()
}
}
}
// watchIPNBus subscribes to the tailscale event bus and sends state updates to chState.
// This method does not return.
func watchIPNBus(ctx context.Context) {
for {
if err := watchIPNBusInner(ctx); err != nil {
log.Println(err)
if errors.Is(err, context.Canceled) {
// If the context got canceled, we will never be able to
// reconnect to IPN bus, so exit the process.
log.Fatalf("watchIPNBus: %v", err)
}
}
// If our watch connection breaks, wait a bit before reconnecting. No
// reason to spam the logs if e.g. tailscaled is restarting or goes
// down.
time.Sleep(3 * time.Second)
}
}
func watchIPNBusInner(ctx context.Context) error {
watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
if err != nil {
return fmt.Errorf("watching ipn bus: %w", err)
}
defer watcher.Close()
for {
select {
case <-ctx.Done():
return nil
default:
n, err := watcher.Next()
if err != nil {
return fmt.Errorf("ipnbus error: %w", err)
}
if n.State != nil {
chState <- *n.State
log.Printf("new state: %v", n.State)
}
}
}
}
// copyTailscaleIP copies the first Tailscale IP of the given device to the clipboard
// and sends a notification with the copied value.
func copyTailscaleIP(device *ipnstate.PeerStatus) {
if device == nil || len(device.TailscaleIPs) == 0 {
return
}
name := strings.Split(device.DNSName, ".")[0]
ip := device.TailscaleIPs[0].String()
err := clipboard.WriteAll(ip)
if err != nil {
log.Printf("clipboard error: %v", err)
}
sendNotification(fmt.Sprintf("Copied Address for %v", name), ip)
}
// sendNotification sends a desktop notification with the given title and content.
func sendNotification(title, content string) {
conn, err := dbus.SessionBus()
if err != nil {
log.Printf("dbus: %v", err)
return
}
timeout := 3 * time.Second
obj := conn.Object("org.freedesktop.Notifications", "/org/freedesktop/Notifications")
call := obj.Call("org.freedesktop.Notifications.Notify", 0, "Tailscale", uint32(0),
appIcon.Name(), title, content, []string{}, map[string]dbus.Variant{}, int32(timeout.Milliseconds()))
if call.Err != nil {
log.Printf("dbus: %v", call.Err)
}
}
func onExit() {
log.Printf("exiting")
os.Remove(appIcon.Name())
}

View File

@@ -21,21 +21,23 @@ var advertiseArgs struct {
// TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode
// envknob is not needed.
func advertiseCmd() *ffcli.Command {
var advertiseCmd = &ffcli.Command{
Name: "advertise",
ShortUsage: "tailscale advertise --services=<services>",
ShortHelp: "Advertise this node as a destination for a service",
Exec: runAdvertise,
FlagSet: (func() *flag.FlagSet {
fs := newFlagSet("advertise")
fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")")
return fs
})(),
}
func maybeAdvertiseCmd() []*ffcli.Command {
if !envknob.UseWIPCode() {
return nil
}
return &ffcli.Command{
Name: "advertise",
ShortUsage: "tailscale advertise --services=<services>",
ShortHelp: "Advertise this node as a destination for a service",
Exec: runAdvertise,
FlagSet: (func() *flag.FlagSet {
fs := newFlagSet("advertise")
fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")")
return fs
})(),
}
return []*ffcli.Command{advertiseCmd}
}
func runAdvertise(ctx context.Context, args []string) error {
@@ -66,7 +68,7 @@ func parseServiceNames(servicesArg string) ([]string, error) {
if servicesArg != "" {
services = strings.Split(servicesArg, ",")
for _, svc := range services {
err := tailcfg.ServiceName(svc).Validate()
err := tailcfg.CheckServiceName(svc)
if err != nil {
return nil, fmt.Errorf("service %q: %s", svc, err)
}

View File

@@ -25,7 +25,6 @@ import (
"tailscale.com/cmd/tailscale/cli/ffcomplete"
"tailscale.com/envknob"
"tailscale.com/paths"
"tailscale.com/util/slicesx"
"tailscale.com/version/distro"
)
@@ -85,9 +84,9 @@ var localClient = tailscale.LocalClient{
// Run runs the CLI. The args do not include the binary name.
func Run(args []string) (err error) {
if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 && len(args) == 0 {
// We're running on gokrazy and the user did not specify 'up'.
// Don't run the tailscale CLI and spam logs with usage; just exit.
if runtime.GOOS == "linux" && os.Getenv("GOKRAZY_FIRST_START") == "1" && distro.Get() == distro.Gokrazy && os.Getppid() == 1 {
// We're running on gokrazy and it's the first start.
// Don't run the tailscale CLI as a service; just exit.
// See https://gokrazy.org/development/process-interface/
os.Exit(0)
}
@@ -183,14 +182,14 @@ For help on subcommands, add --help after: "tailscale status --help".
This CLI is still under active development. Commands and flags will
change in the future.
`),
Subcommands: nonNilCmds(
Subcommands: append([]*ffcli.Command{
upCmd,
downCmd,
setCmd,
loginCmd,
logoutCmd,
switchCmd,
configureCmd(),
configureCmd,
syspolicyCmd,
netcheckCmd,
ipCmd,
@@ -215,9 +214,7 @@ change in the future.
debugCmd,
driveCmd,
idTokenCmd,
advertiseCmd(),
configureHostCmd(),
),
}, maybeAdvertiseCmd()...),
FlagSet: rootfs,
Exec: func(ctx context.Context, args []string) error {
if len(args) > 0 {
@@ -227,6 +224,10 @@ change in the future.
},
}
if runtime.GOOS == "linux" && distro.Get() == distro.Synology {
rootCmd.Subcommands = append(rootCmd.Subcommands, configureHostCmd)
}
walkCommands(rootCmd, func(w cmdWalk) bool {
if w.UsageFunc == nil {
w.UsageFunc = usageFunc
@@ -238,10 +239,6 @@ change in the future.
return rootCmd
}
func nonNilCmds(cmds ...*ffcli.Command) []*ffcli.Command {
return slicesx.AppendNonzero(cmds[:0], cmds)
}
func fatalf(format string, a ...any) {
if Fatalf != nil {
Fatalf(format, a...)

View File

@@ -17,7 +17,6 @@ import (
qt "github.com/frankban/quicktest"
"github.com/google/go-cmp/cmp"
"github.com/peterbourgon/ff/v3/ffcli"
"tailscale.com/envknob"
"tailscale.com/health/healthmsg"
"tailscale.com/ipn"
@@ -602,19 +601,6 @@ func TestCheckForAccidentalSettingReverts(t *testing.T) {
goos: "linux",
want: "",
},
{
name: "losing_posture_checking",
flags: []string{"--accept-dns"},
curPrefs: &ipn.Prefs{
ControlURL: ipn.DefaultControlURL,
WantRunning: false,
CorpDNS: true,
PostureChecking: true,
NetfilterMode: preftype.NetfilterOn,
NoStatefulFiltering: opt.NewBool(true),
},
want: accidentalUpPrefix + " --accept-dns --posture-checking",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -1059,7 +1045,6 @@ func TestUpdatePrefs(t *testing.T) {
NoSNATSet: true,
NoStatefulFilteringSet: true,
OperatorUserSet: true,
PostureCheckingSet: true,
RouteAllSet: true,
RunSSHSet: true,
ShieldsUpSet: true,
@@ -1526,45 +1511,3 @@ func TestHelpAlias(t *testing.T) {
t.Fatalf("Run: %v", err)
}
}
func TestDocs(t *testing.T) {
root := newRootCmd()
check := func(t *testing.T, c *ffcli.Command) {
shortVerb, _, ok := strings.Cut(c.ShortHelp, " ")
if !ok || shortVerb == "" {
t.Errorf("couldn't find verb+space in ShortHelp")
} else {
if strings.HasSuffix(shortVerb, ".") {
t.Errorf("ShortHelp shouldn't end in period; got %q", c.ShortHelp)
}
if b := shortVerb[0]; b >= 'a' && b <= 'z' {
t.Errorf("ShortHelp should start with upper-case letter; got %q", c.ShortHelp)
}
if strings.HasSuffix(shortVerb, "s") && shortVerb != "Does" {
t.Errorf("verb %q ending in 's' is unexpected, from %q", shortVerb, c.ShortHelp)
}
}
name := t.Name()
wantPfx := strings.ReplaceAll(strings.TrimPrefix(name, "TestDocs/"), "/", " ")
switch name {
case "TestDocs/tailscale/completion/bash",
"TestDocs/tailscale/completion/zsh":
wantPfx = "" // special-case exceptions
}
if !strings.HasPrefix(c.ShortUsage, wantPfx) {
t.Errorf("ShortUsage should start with %q; got %q", wantPfx, c.ShortUsage)
}
}
var walk func(t *testing.T, c *ffcli.Command)
walk = func(t *testing.T, c *ffcli.Command) {
t.Run(c.Name, func(t *testing.T) {
check(t, c)
for _, sub := range c.Subcommands {
walk(t, sub)
}
})
}
walk(t, root)
}

View File

@@ -20,31 +20,33 @@ import (
"tailscale.com/version"
)
func configureKubeconfigCmd() *ffcli.Command {
return &ffcli.Command{
Name: "kubeconfig",
ShortHelp: "[ALPHA] Connect to a Kubernetes cluster using a Tailscale Auth Proxy",
ShortUsage: "tailscale configure kubeconfig <hostname-or-fqdn>",
LongHelp: strings.TrimSpace(`
func init() {
configureCmd.Subcommands = append(configureCmd.Subcommands, configureKubeconfigCmd)
}
var configureKubeconfigCmd = &ffcli.Command{
Name: "kubeconfig",
ShortHelp: "[ALPHA] Connect to a Kubernetes cluster using a Tailscale Auth Proxy",
ShortUsage: "tailscale configure kubeconfig <hostname-or-fqdn>",
LongHelp: strings.TrimSpace(`
Run this command to configure kubectl to connect to a Kubernetes cluster over Tailscale.
The hostname argument should be set to the Tailscale hostname of the peer running as an auth proxy in the cluster.
See: https://tailscale.com/s/k8s-auth-proxy
`),
FlagSet: (func() *flag.FlagSet {
fs := newFlagSet("kubeconfig")
return fs
})(),
Exec: runConfigureKubeconfig,
}
FlagSet: (func() *flag.FlagSet {
fs := newFlagSet("kubeconfig")
return fs
})(),
Exec: runConfigureKubeconfig,
}
// kubeconfigPath returns the path to the kubeconfig file for the current user.
func kubeconfigPath() (string, error) {
if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" {
if version.IsSandboxedMacOS() {
return "", errors.New("cannot read $KUBECONFIG on GUI builds of the macOS client: this requires the open-source tailscaled distribution")
return "", errors.New("$KUBECONFIG is incompatible with the App Store version")
}
var out string
for _, out = range filepath.SplitList(kubeconfig) {

Some files were not shown because too many files have changed in this diff Show More