net/netcheck: change DERP HTTPS latency measurement to TCP connect time

HTTPS and ICMP latency are fallbacks when STUN doesn't work. The lowest
value wins. When ICMP doesn't work you just get HTTPS. HTTPS latency was
way more variable than it needed to be as we were measuring "server
processing time" (inclusive of TLS, HTTPS) instead of just TCP connect
time.

We don't need to perform an HTTPS request, we really just need a TCP
ping, but aiming for a simpler change to start.

Updates tailscale/corp#22114

Signed-off-by: Jordan Whited <jordan@tailscale.com>

cmd/derper/derper.go

@@ -237,7 +237,7 @@ func main() {
 		tsweb.AddBrowserHeaders(w)
 		io.WriteString(w, "User-agent: *\nDisallow: /\n")
 	}))
-	mux.Handle("/generate_204", http.HandlerFunc(serveNoContent))
+	mux.Handle("/generate_204", http.HandlerFunc(derphttp.ServeNoContent))
 	debug := tsweb.Debugger(mux)
 	debug.KV("TLS hostname", *hostname)
 	debug.KV("Mesh key", s.HasMeshKey())
@@ -337,7 +337,7 @@ func main() {
 		if *httpPort > -1 {
 			go func() {
 				port80mux := http.NewServeMux()
-				port80mux.HandleFunc("/generate_204", serveNoContent)
+				port80mux.HandleFunc("/generate_204", derphttp.ServeNoContent)
 				port80mux.Handle("/", certManager.HTTPHandler(tsweb.Port80Handler{Main: mux}))
 				port80srv := &http.Server{
 					Addr:        net.JoinHostPort(listenHost, fmt.Sprintf("%d", *httpPort)),
@@ -378,31 +378,6 @@ func main() {
 	}
 }
 
-const (
-	noContentChallengeHeader = "X-Tailscale-Challenge"
-	noContentResponseHeader  = "X-Tailscale-Response"
-)
-
-// For captive portal detection
-func serveNoContent(w http.ResponseWriter, r *http.Request) {
-	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
-		badChar := strings.IndexFunc(challenge, func(r rune) bool {
-			return !isChallengeChar(r)
-		}) != -1
-		if len(challenge) <= 64 && !badChar {
-			w.Header().Set(noContentResponseHeader, "response "+challenge)
-		}
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func isChallengeChar(c rune) bool {
-	// Semi-randomly chosen as a limited set of valid characters
-	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-		('0' <= c && c <= '9') ||
-		c == '.' || c == '-' || c == '_'
-}
-
 var validProdHostname = regexp.MustCompile(`^derp([^.]*)\.tailscale\.com\.?$`)
 
 func prodAutocertHostPolicy(_ context.Context, host string) error {

cmd/derper/derper_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"tailscale.com/derp/derphttp"
 	"tailscale.com/tstest/deptest"
 )
 
@@ -76,20 +77,20 @@ func TestNoContent(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
 			if tt.input != "" {
-				req.Header.Set(noContentChallengeHeader, tt.input)
+				req.Header.Set(derphttp.NoContentChallengeHeader, tt.input)
 			}
 			w := httptest.NewRecorder()
-			serveNoContent(w, req)
+			derphttp.ServeNoContent(w, req)
 			resp := w.Result()
 
 			if tt.want == "" {
-				if h, found := resp.Header[noContentResponseHeader]; found {
+				if h, found := resp.Header[derphttp.NoContentResponseHeader]; found {
 					t.Errorf("got %+v; expected no response header", h)
 				}
 				return
 			}
 
-			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+			if got := resp.Header.Get(derphttp.NoContentResponseHeader); got != tt.want {
 				t.Errorf("got %q; want %q", got, tt.want)
 			}
 		})

cmd/gitops-pusher/gitops-pusher.go

@@ -28,19 +28,20 @@ import (
 )
 
 var (
-	rootFlagSet  = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
-	policyFname  = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
-	cacheFname   = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
-	timeout      = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
-	githubSyntax = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
-	apiServer    = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	rootFlagSet       = flag.NewFlagSet("gitops-pusher", flag.ExitOnError)
+	policyFname       = rootFlagSet.String("policy-file", "./policy.hujson", "filename for policy file")
+	cacheFname        = rootFlagSet.String("cache-file", "./version-cache.json", "filename for the previous known version hash")
+	timeout           = rootFlagSet.Duration("timeout", 5*time.Minute, "timeout for the entire CI run")
+	githubSyntax      = rootFlagSet.Bool("github-syntax", true, "use GitHub Action error syntax (https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)")
+	apiServer         = rootFlagSet.String("api-server", "api.tailscale.com", "API server to contact")
+	failOnManualEdits = rootFlagSet.Bool("fail-on-manual-edits", false, "fail if manual edits to the ACLs in the admin panel are detected; when set to false (the default) only a warning is printed")
 )
 
-func modifiedExternallyError() {
+func modifiedExternallyError() error {
 	if *githubSyntax {
-		fmt.Printf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.\n", *policyFname)
+		return fmt.Errorf("::warning file=%s,line=1,col=1,title=Policy File Modified Externally::The policy file was modified externally in the admin console.", *policyFname)
 	} else {
-		fmt.Printf("The policy file was modified externally in the admin console.\n")
+		return fmt.Errorf("The policy file was modified externally in the admin console.")
 	}
 }
 
@@ -65,16 +66,22 @@ func apply(cache *Cache, client *http.Client, tailnet, apiKey string) func(conte
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			cache.PrevETag = localEtag
 			log.Println("no update needed, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := applyNewACL(ctx, client, tailnet, apiKey, *policyFname, controlEtag); err != nil {
 			return err
 		}
@@ -106,15 +113,21 @@ func test(cache *Cache, client *http.Client, tailnet, apiKey string) func(contex
 		log.Printf("local:   %s", localEtag)
 		log.Printf("cache:   %s", cache.PrevETag)
 
-		if cache.PrevETag != controlEtag {
-			modifiedExternallyError()
-		}
-
 		if controlEtag == localEtag {
 			log.Println("no updates found, doing nothing")
 			return nil
 		}
 
+		if cache.PrevETag != controlEtag {
+			if err := modifiedExternallyError(); err != nil {
+				if *failOnManualEdits {
+					return err
+				} else {
+					fmt.Println(err)
+				}
+			}
+		}
+
 		if err := testNewACLs(ctx, client, tailnet, apiKey, *policyFname); err != nil {
 			return err
 		}

cmd/tsidp/tsidp.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/rsa"
@@ -16,6 +17,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -25,6 +27,7 @@ import (
 	"net/netip"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
@@ -35,6 +38,7 @@ import (
 	"tailscale.com/client/tailscale"
 	"tailscale.com/client/tailscale/apitype"
 	"tailscale.com/envknob"
+	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tsnet"
@@ -44,13 +48,22 @@ import (
 	"tailscale.com/util/mak"
 	"tailscale.com/util/must"
 	"tailscale.com/util/rands"
+	"tailscale.com/version"
 )
 
+// ctxConn is a key to look up a net.Conn stored in an HTTP request's context.
+type ctxConn struct{}
+
+// funnelClientsFile is the file where client IDs and secrets for OIDC clients
+// accessing the IDP over Funnel are persisted.
+const funnelClientsFile = "oidc-funnel-clients.json"
+
 var (
 	flagVerbose            = flag.Bool("verbose", false, "be verbose")
 	flagPort               = flag.Int("port", 443, "port to listen on")
 	flagLocalPort          = flag.Int("local-port", -1, "allow requests from localhost")
 	flagUseLocalTailscaled = flag.Bool("use-local-tailscaled", false, "use local tailscaled instead of tsnet")
+	flagFunnel             = flag.Bool("funnel", false, "use Tailscale Funnel to make tsidp available on the public internet")
 )
 
 func main() {
@@ -61,9 +74,11 @@ func main() {
 	}
 
 	var (
-		lc  *tailscale.LocalClient
-		st  *ipnstate.Status
-		err error
+		lc          *tailscale.LocalClient
+		st          *ipnstate.Status
+		err         error
+		watcherChan chan error
+		cleanup     func()
 
 		lns []net.Listener
 	)
@@ -90,6 +105,18 @@ func main() {
 		if !anySuccess {
 			log.Fatalf("failed to listen on any of %v", st.TailscaleIPs)
 		}
+
+		// tailscaled needs to be setting an HTTP header for funneled requests
+		// that older versions don't provide.
+		// TODO(naman): is this the correct check?
+		if *flagFunnel && !version.AtLeast(st.Version, "1.71.0") {
+			log.Fatalf("Local tailscaled not new enough to support -funnel. Update Tailscale or use tsnet mode.")
+		}
+		cleanup, watcherChan, err = serveOnLocalTailscaled(ctx, lc, st, uint16(*flagPort), *flagFunnel)
+		if err != nil {
+			log.Fatalf("could not serve on local tailscaled: %v", err)
+		}
+		defer cleanup()
 	} else {
 		ts := &tsnet.Server{
 			Hostname: "idp",
@@ -105,7 +132,15 @@ func main() {
 		if err != nil {
 			log.Fatalf("getting local client: %v", err)
 		}
-		ln, err := ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		var ln net.Listener
+		if *flagFunnel {
+			if err := ipn.CheckFunnelAccess(uint16(*flagPort), st.Self); err != nil {
+				log.Fatalf("%v", err)
+			}
+			ln, err = ts.ListenFunnel("tcp", fmt.Sprintf(":%d", *flagPort))
+		} else {
+			ln, err = ts.ListenTLS("tcp", fmt.Sprintf(":%d", *flagPort))
+		}
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -113,13 +148,26 @@ func main() {
 	}
 
 	srv := &idpServer{
-		lc: lc,
+		lc:          lc,
+		funnel:      *flagFunnel,
+		localTSMode: *flagUseLocalTailscaled,
 	}
 	if *flagPort != 443 {
 		srv.serverURL = fmt.Sprintf("https://%s:%d", strings.TrimSuffix(st.Self.DNSName, "."), *flagPort)
 	} else {
 		srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, "."))
 	}
+	if *flagFunnel {
+		f, err := os.Open(funnelClientsFile)
+		if err == nil {
+			srv.funnelClients = make(map[string]*funnelClient)
+			if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil {
+				log.Fatalf("could not parse %s: %v", funnelClientsFile, err)
+			}
+		} else if !errors.Is(err, os.ErrNotExist) {
+			log.Fatalf("could not open %s: %v", funnelClientsFile, err)
+		}
+	}
 
 	log.Printf("Running tsidp at %s ...", srv.serverURL)
 
@@ -134,35 +182,129 @@ func main() {
 	}
 
 	for _, ln := range lns {
-		go http.Serve(ln, srv)
+		server := http.Server{
+			Handler: srv,
+			ConnContext: func(ctx context.Context, c net.Conn) context.Context {
+				return context.WithValue(ctx, ctxConn{}, c)
+			},
+		}
+		go server.Serve(ln)
 	}
-	select {}
+	// need to catch os.Interrupt, otherwise deferred cleanup code doesn't run
+	exitChan := make(chan os.Signal, 1)
+	signal.Notify(exitChan, os.Interrupt)
+	select {
+	case <-exitChan:
+		log.Printf("interrupt, exiting")
+		return
+	case <-watcherChan:
+		if errors.Is(err, io.EOF) || errors.Is(err, context.Canceled) {
+			log.Printf("watcher closed, exiting")
+			return
+		}
+		log.Fatalf("watcher error: %v", err)
+		return
+	}
+}
+
+// serveOnLocalTailscaled starts a serve session using an already-running
+// tailscaled instead of starting a fresh tsnet server, making something
+// listening on clientDNSName:dstPort accessible over serve/funnel.
+func serveOnLocalTailscaled(ctx context.Context, lc *tailscale.LocalClient, st *ipnstate.Status, dstPort uint16, shouldFunnel bool) (cleanup func(), watcherChan chan error, err error) {
+	// In order to support funneling out in local tailscaled mode, we need
+	// to add a serve config to forward the listeners we bound above and
+	// allow those forwarders to be funneled out.
+	sc, err := lc.GetServeConfig(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get serve config: %v", err)
+	}
+	if sc == nil {
+		sc = new(ipn.ServeConfig)
+	}
+
+	// We watch the IPN bus just to get a session ID. The session expires
+	// when we stop watching the bus, and that auto-deletes the foreground
+	// serve/funnel configs we are creating below.
+	watcher, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialState|ipn.NotifyNoPrivateKeys)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not set up ipn bus watcher: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+	n, err := watcher.Next()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get initial state from ipn bus watcher: %v", err)
+	}
+	if n.SessionID == "" {
+		err = fmt.Errorf("missing sessionID in ipn.Notify")
+		return nil, nil, err
+	}
+	watcherChan = make(chan error)
+	go func() {
+		for {
+			_, err = watcher.Next()
+			if err != nil {
+				watcherChan <- err
+				return
+			}
+		}
+	}()
+
+	// Create a foreground serve config that gets cleaned up when tsidp
+	// exits and the session ID associated with this config is invalidated.
+	foregroundSc := new(ipn.ServeConfig)
+	mak.Set(&sc.Foreground, n.SessionID, foregroundSc)
+	serverURL := strings.TrimSuffix(st.Self.DNSName, ".")
+	fmt.Printf("setting funnel for %s:%v\n", serverURL, dstPort)
+
+	foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel)
+	foregroundSc.SetWebHandler(&ipn.HTTPHandler{
+		Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))),
+	}, serverURL, uint16(*flagPort), "/", true)
+	err = lc.SetServeConfig(ctx, sc)
+	if err != nil {
+		return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err)
+	}
+
+	return func() { watcher.Close() }, watcherChan, nil
 }
 
 type idpServer struct {
 	lc          *tailscale.LocalClient
 	loopbackURL string
 	serverURL   string // "https://foo.bar.ts.net"
+	funnel      bool
+	localTSMode bool
 
 	lazyMux        lazy.SyncValue[*http.ServeMux]
 	lazySigningKey lazy.SyncValue[*signingKey]
 	lazySigner     lazy.SyncValue[jose.Signer]
 
-	mu          sync.Mutex              // guards the fields below
-	code        map[string]*authRequest // keyed by random hex
-	accessToken map[string]*authRequest // keyed by random hex
+	mu            sync.Mutex               // guards the fields below
+	code          map[string]*authRequest  // keyed by random hex
+	accessToken   map[string]*authRequest  // keyed by random hex
+	funnelClients map[string]*funnelClient // keyed by client ID
 }
 
 type authRequest struct {
 	// localRP is true if the request is from a relying party running on the
-	// same machine as the idp server. It is mutually exclusive with rpNodeID.
+	// same machine as the idp server. It is mutually exclusive with rpNodeID
+	// and funnelRP.
 	localRP bool
 
 	// rpNodeID is the NodeID of the relying party (who requested the auth, such
 	// as Proxmox or Synology), not the user node who is being authenticated. It
-	// is mutually exclusive with localRP.
+	// is mutually exclusive with localRP and funnelRP.
 	rpNodeID tailcfg.NodeID
 
+	// funnelRP is non-nil if the request is from a relying party outside the
+	// tailnet, via Tailscale Funnel. It is mutually exclusive with rpNodeID
+	// and localRP.
+	funnelRP *funnelClient
+
 	// clientID is the "client_id" sent in the authorized request.
 	clientID string
 
@@ -181,9 +323,12 @@ type authRequest struct {
 	validTill time.Time
 }
 
-func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string, lc *tailscale.LocalClient) error {
+// allowRelyingParty validates that a relying party identified either by a
+// known remoteAddr or a valid client ID/secret pair is allowed to proceed
+// with the authorization flow associated with this authRequest.
+func (ar *authRequest) allowRelyingParty(r *http.Request, lc *tailscale.LocalClient) error {
 	if ar.localRP {
-		ra, err := netip.ParseAddrPort(remoteAddr)
+		ra, err := netip.ParseAddrPort(r.RemoteAddr)
 		if err != nil {
 			return err
 		}
@@ -192,7 +337,18 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 		}
 		return nil
 	}
-	who, err := lc.WhoIs(ctx, remoteAddr)
+	if ar.funnelRP != nil {
+		clientID, clientSecret, ok := r.BasicAuth()
+		if !ok {
+			clientID = r.FormValue("client_id")
+			clientSecret = r.FormValue("client_secret")
+		}
+		if ar.funnelRP.ID != clientID || ar.funnelRP.Secret != clientSecret {
+			return fmt.Errorf("tsidp: invalid client credentials")
+		}
+		return nil
+	}
+	who, err := lc.WhoIs(r.Context(), r.RemoteAddr)
 	if err != nil {
 		return fmt.Errorf("tsidp: error getting WhoIs: %w", err)
 	}
@@ -203,24 +359,60 @@ func (ar *authRequest) allowRelyingParty(ctx context.Context, remoteAddr string,
 }
 
 func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
-	who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr)
+	// This URL is visited by the user who is being authenticated. If they are
+	// visiting the URL over Funnel, that means they are not part of the
+	// tailnet that they are trying to be authenticated for.
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	uq := r.URL.Query()
+
+	redirectURI := uq.Get("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must specify redirect_uri", http.StatusBadRequest)
+		return
+	}
+
+	var remoteAddr string
+	if s.localTSMode {
+		// in local tailscaled mode, the local tailscaled is forwarding us
+		// HTTP requests, so reading r.RemoteAddr will just get us our own
+		// address.
+		remoteAddr = r.Header.Get("X-Forwarded-For")
+	} else {
+		remoteAddr = r.RemoteAddr
+	}
+	who, err := s.lc.WhoIs(r.Context(), remoteAddr)
 	if err != nil {
 		log.Printf("Error getting WhoIs: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	uq := r.URL.Query()
-
 	code := rands.HexString(32)
 	ar := &authRequest{
 		nonce:       uq.Get("nonce"),
 		remoteUser:  who,
-		redirectURI: uq.Get("redirect_uri"),
+		redirectURI: redirectURI,
 		clientID:    uq.Get("client_id"),
 	}
 
-	if r.URL.Path == "/authorize/localhost" {
+	if r.URL.Path == "/authorize/funnel" {
+		s.mu.Lock()
+		c, ok := s.funnelClients[ar.clientID]
+		s.mu.Unlock()
+		if !ok {
+			http.Error(w, "tsidp: invalid client ID", http.StatusBadRequest)
+			return
+		}
+		if ar.redirectURI != c.RedirectURI {
+			http.Error(w, "tsidp: redirect_uri mismatch", http.StatusBadRequest)
+			return
+		}
+		ar.funnelRP = c
+	} else if r.URL.Path == "/authorize/localhost" {
 		ar.localRP = true
 	} else {
 		var ok bool
@@ -237,8 +429,10 @@ func (s *idpServer) authorize(w http.ResponseWriter, r *http.Request) {
 
 	q := make(url.Values)
 	q.Set("code", code)
-	q.Set("state", uq.Get("state"))
-	u := uq.Get("redirect_uri") + "?" + q.Encode()
+	if state := uq.Get("state"); state != "" {
+		q.Set("state", state)
+	}
+	u := redirectURI + "?" + q.Encode()
 	log.Printf("Redirecting to %q", u)
 
 	http.Redirect(w, r, u, http.StatusFound)
@@ -251,6 +445,7 @@ func (s *idpServer) newMux() *http.ServeMux {
 	mux.HandleFunc("/authorize/", s.authorize)
 	mux.HandleFunc("/userinfo", s.serveUserInfo)
 	mux.HandleFunc("/token", s.serveToken)
+	mux.HandleFunc("/clients/", s.serveClients)
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
 			io.WriteString(w, "<html><body><h1>Tailscale OIDC IdP</h1>")
@@ -284,11 +479,6 @@ func (s *idpServer) serveUserInfo(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: invalid token", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
-		log.Printf("Error allowing relying party: %v", err)
-		http.Error(w, err.Error(), http.StatusForbidden)
-		return
-	}
 
 	if ar.validTill.Before(time.Now()) {
 		http.Error(w, "tsidp: token expired", http.StatusBadRequest)
@@ -348,7 +538,7 @@ func (s *idpServer) serveToken(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "tsidp: code not found", http.StatusBadRequest)
 		return
 	}
-	if err := ar.allowRelyingParty(r.Context(), r.RemoteAddr, s.lc); err != nil {
+	if err := ar.allowRelyingParty(r, s.lc); err != nil {
 		log.Printf("Error allowing relying party: %v", err)
 		http.Error(w, err.Error(), http.StatusForbidden)
 		return
@@ -581,7 +771,9 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 	var authorizeEndpoint string
 	rpEndpoint := s.serverURL
-	if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
+	if isFunnelRequest(r) {
+		authorizeEndpoint = fmt.Sprintf("%s/authorize/funnel", s.serverURL)
+	} else if who, err := s.lc.WhoIs(r.Context(), r.RemoteAddr); err == nil {
 		authorizeEndpoint = fmt.Sprintf("%s/authorize/%d", s.serverURL, who.Node.ID)
 	} else if ap.Addr().IsLoopback() {
 		rpEndpoint = s.loopbackURL
@@ -611,6 +803,148 @@ func (s *idpServer) serveOpenIDConfig(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// funnelClient represents an OIDC client/relying party that is accessing the
+// IDP over Funnel.
+type funnelClient struct {
+	ID          string `json:"client_id"`
+	Secret      string `json:"client_secret,omitempty"`
+	Name        string `json:"name,omitempty"`
+	RedirectURI string `json:"redirect_uri"`
+}
+
+// /clients is a privileged endpoint that allows the visitor to create new
+// Funnel-capable OIDC clients, so it is only accessible over the tailnet.
+func (s *idpServer) serveClients(w http.ResponseWriter, r *http.Request) {
+	if isFunnelRequest(r) {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	path := strings.TrimPrefix(r.URL.Path, "/clients/")
+
+	if path == "new" {
+		s.serveNewClient(w, r)
+		return
+	}
+
+	if path == "" {
+		s.serveGetClientsList(w, r)
+		return
+	}
+
+	s.mu.Lock()
+	c, ok := s.funnelClients[path]
+	s.mu.Unlock()
+	if !ok {
+		http.Error(w, "tsidp: not found", http.StatusNotFound)
+		return
+	}
+
+	switch r.Method {
+	case "DELETE":
+		s.serveDeleteClient(w, r, path)
+	case "GET":
+		json.NewEncoder(w).Encode(&funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	default:
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *idpServer) serveNewClient(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	redirectURI := r.FormValue("redirect_uri")
+	if redirectURI == "" {
+		http.Error(w, "tsidp: must provide redirect_uri", http.StatusBadRequest)
+		return
+	}
+	clientID := rands.HexString(32)
+	clientSecret := rands.HexString(64)
+	newClient := funnelClient{
+		ID:          clientID,
+		Secret:      clientSecret,
+		Name:        r.FormValue("name"),
+		RedirectURI: redirectURI,
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.funnelClients, clientID, &newClient)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// delete the new client to avoid inconsistent state between memory
+		// and disk
+		delete(s.funnelClients, clientID)
+		return
+	}
+	json.NewEncoder(w).Encode(newClient)
+}
+
+func (s *idpServer) serveGetClientsList(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	redactedClients := make([]funnelClient, 0, len(s.funnelClients))
+	for _, c := range s.funnelClients {
+		redactedClients = append(redactedClients, funnelClient{
+			ID:          c.ID,
+			Name:        c.Name,
+			Secret:      "",
+			RedirectURI: c.RedirectURI,
+		})
+	}
+	s.mu.Unlock()
+	json.NewEncoder(w).Encode(redactedClients)
+}
+
+func (s *idpServer) serveDeleteClient(w http.ResponseWriter, r *http.Request, clientID string) {
+	if r.Method != "DELETE" {
+		http.Error(w, "tsidp: method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.funnelClients == nil {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	if _, ok := s.funnelClients[clientID]; !ok {
+		http.Error(w, "tsidp: client not found", http.StatusNotFound)
+		return
+	}
+	deleted := s.funnelClients[clientID]
+	delete(s.funnelClients, clientID)
+	if err := s.storeFunnelClientsLocked(); err != nil {
+		log.Printf("could not write funnel clients db: %v", err)
+		http.Error(w, "tsidp: could not write funnel clients to db", http.StatusInternalServerError)
+		// restore the deleted value to avoid inconsistent state between memory
+		// and disk
+		s.funnelClients[clientID] = deleted
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// storeFunnelClientsLocked writes the current mapping of OIDC client ID/secret
+// pairs for RPs that access the IDP over funnel. s.mu must be held while
+// calling this.
+func (s *idpServer) storeFunnelClientsLocked() error {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(s.funnelClients); err != nil {
+		return err
+	}
+	return os.WriteFile(funnelClientsFile, buf.Bytes(), 0600)
+}
+
 const (
 	minimumRSAKeySize = 2048
 )
@@ -700,3 +1034,24 @@ func parseID[T ~int64](input string) (_ T, ok bool) {
 	}
 	return T(i), true
 }
+
+// isFunnelRequest checks if an HTTP request is coming over Tailscale Funnel.
+func isFunnelRequest(r *http.Request) bool {
+	// If we're funneling through the local tailscaled, it will set this HTTP
+	// header.
+	if r.Header.Get("Tailscale-Funnel-Request") != "" {
+		return true
+	}
+
+	// If the funneled connection is from tsnet, then the net.Conn will be of
+	// type ipn.FunnelConn.
+	netConn := r.Context().Value(ctxConn{})
+	// if the conn is wrapped inside TLS, unwrap it
+	if tlsConn, ok := netConn.(*tls.Conn); ok {
+		netConn = tlsConn.NetConn()
+	}
+	if _, ok := netConn.(*ipn.FunnelConn); ok {
+		return true
+	}
+	return false
+}

cmd/tta/tta.go

@@ -0,0 +1,159 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The tta server is the Tailscale Test Agent.
+//
+// It runs on each Tailscale node being integration tested and permits the test
+// harness to control the node. It connects out to the test drver (rather than
+// accepting any TCP connections inbound, which might be blocked depending on
+// the scenario being tested) and then the test driver turns the TCP connection
+// around and sends request back.
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/util/set"
+	"tailscale.com/version/distro"
+)
+
+var (
+	driverAddr = flag.String("driver", "test-driver.tailscale:8008", "address of the test driver; by default we use the DNS name test-driver.tailscale which is special cased in the emulated network's DNS server")
+)
+
+type chanListener <-chan net.Conn
+
+func serveCmd(w http.ResponseWriter, cmd string, args ...string) {
+	if distro.Get() == distro.Gokrazy && !strings.Contains(cmd, "/") {
+		cmd = "/user/" + cmd
+	}
+	out, err := exec.Command(cmd, args...).CombinedOutput()
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	if err != nil {
+		w.Header().Set("Exec-Err", err.Error())
+		w.WriteHeader(500)
+	}
+	w.Write(out)
+}
+
+func main() {
+	if distro.Get() == distro.Gokrazy {
+		cmdLine, _ := os.ReadFile("/proc/cmdline")
+		if !bytes.Contains(cmdLine, []byte("tailscale-tta=1")) {
+			// "Exiting immediately with status code 0 when the
+			// GOKRAZY_FIRST_START=1 environment variable is set means “don’t
+			// start the program on boot”"
+			return
+		}
+	}
+	flag.Parse()
+	log.Printf("Tailscale Test Agent running.")
+
+	var mux http.ServeMux
+	var hs http.Server
+	hs.Handler = &mux
+	var (
+		stMu   sync.Mutex
+		newSet = set.Set[net.Conn]{} // conns in StateNew
+	)
+	needConnCh := make(chan bool, 1)
+	hs.ConnState = func(c net.Conn, s http.ConnState) {
+		stMu.Lock()
+		defer stMu.Unlock()
+		switch s {
+		case http.StateNew:
+			newSet.Add(c)
+		case http.StateClosed:
+			newSet.Delete(c)
+		}
+		if len(newSet) == 0 {
+			select {
+			case needConnCh <- true:
+			default:
+			}
+		}
+	}
+	conns := make(chan net.Conn, 1)
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "TTA\n")
+		return
+	})
+	mux.HandleFunc("/up", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "up", "--auth-key=test")
+	})
+	mux.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
+		serveCmd(w, "tailscale", "status", "--json")
+	})
+	mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) {
+		target := r.FormValue("target")
+		cmd := exec.Command("tailscale", "ping", target)
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.(http.Flusher).Flush()
+		cmd.Stdout = w
+		cmd.Stderr = w
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintf(w, "error: %v\n", err)
+		}
+	})
+	go hs.Serve(chanListener(conns))
+
+	var lastErr string
+	needConnCh <- true
+	for {
+		<-needConnCh
+		c, err := connect()
+		log.Printf("Connect: %v", err)
+		if err != nil {
+			s := err.Error()
+			if s != lastErr {
+				log.Printf("Connect failure: %v", s)
+			}
+			lastErr = s
+			time.Sleep(time.Second)
+			continue
+		}
+		conns <- c
+
+		time.Sleep(time.Second)
+	}
+}
+
+func connect() (net.Conn, error) {
+	c, err := net.Dial("tcp", *driverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+func (cl chanListener) Accept() (net.Conn, error) {
+	c, ok := <-cl
+	if !ok {
+		return nil, errors.New("closed")
+	}
+	return c, nil
+}
+
+func (cl chanListener) Close() error {
+	return nil
+}
+
+func (cl chanListener) Addr() net.Addr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("52.0.0.34"), // TS..DR(iver)
+		Port: 123,
+	}
+}

cmd/vnet/run-krazy.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "Type 'C-a c' to enter monitor; q to quit."
+
+set -eux
+qemu-system-x86_64 -M microvm,isa-serial=off \
+    -m 1G \
+    -nodefaults -no-user-config -nographic \
+    -kernel $HOME/src/github.com/tailscale/gokrazy-kernel/vmlinuz \
+    -append "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-dd02023b0001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet tailscale-tta=1" \
+    -drive id=blk0,file=$HOME/src/tailscale.com/gokrazy/tsapp.img,format=raw \
+    -device virtio-blk-device,drive=blk0 \
+    -netdev stream,id=net0,addr.type=unix,addr.path=/tmp/qemu.sock \
+    -device virtio-serial-device \
+    -device virtio-net-device,netdev=net0,mac=52:cc:cc:cc:cc:00 \
+    -chardev stdio,id=virtiocon0,mux=on \
+    -device virtconsole,chardev=virtiocon0 \
+    -mon chardev=virtiocon0,mode=readline \
+    -audio none
+

cmd/vnet/vnet-main.go

@@ -0,0 +1,101 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// The vnet binary runs a virtual network stack in userspace for qemu instances
+// to connect to and simulate various network conditions.
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+	"os"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+var (
+	listen  = flag.String("listen", "/tmp/qemu.sock", "path to listen on")
+	nat     = flag.String("nat", "easy", "type of NAT to use")
+	portmap = flag.Bool("portmap", false, "enable portmapping")
+	dgram   = flag.Bool("dgram", false, "enable datagram mode; for use with macOS Hypervisor.Framework and VZFileHandleNetworkDeviceAttachment")
+)
+
+func main() {
+	flag.Parse()
+
+	if _, err := os.Stat(*listen); err == nil {
+		os.Remove(*listen)
+	}
+
+	var srv net.Listener
+	var err error
+	var conn *net.UnixConn
+	if *dgram {
+		addr, err := net.ResolveUnixAddr("unixgram", *listen)
+		if err != nil {
+			log.Fatalf("ResolveUnixAddr: %v", err)
+		}
+		conn, err = net.ListenUnixgram("unixgram", addr)
+		if err != nil {
+			log.Fatalf("ListenUnixgram: %v", err)
+		}
+		defer conn.Close()
+	} else {
+		srv, err = net.Listen("unix", *listen)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var c vnet.Config
+	node1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.NAT(*nat)))
+	c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", vnet.NAT(*nat)))
+	if *portmap {
+		node1.Network().AddService(vnet.NATPMP)
+	}
+
+	s, err := vnet.New(&c)
+	if err != nil {
+		log.Fatalf("newServer: %v", err)
+	}
+
+	if err := s.PopulateDERPMapIPs(); err != nil {
+		log.Printf("warning: ignoring failure to populate DERP map: %v", err)
+	}
+
+	s.WriteStartingBanner(os.Stdout)
+
+	go func() {
+		getStatus := func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			st, err := s.NodeStatus(ctx, node1)
+			if err != nil {
+				log.Printf("NodeStatus: %v", err)
+				return
+			}
+			log.Printf("NodeStatus: %q", st)
+		}
+		for {
+			time.Sleep(5 * time.Second)
+			getStatus()
+		}
+	}()
+
+	if conn != nil {
+		s.ServeUnixConn(conn, vnet.ProtocolUnixDGRAM)
+		return
+	}
+
+	for {
+		c, err := srv.Accept()
+		if err != nil {
+			log.Printf("Accept: %v", err)
+			continue
+		}
+		go s.ServeUnixConn(c.(*net.UnixConn), vnet.ProtocolQEMU)
+	}
+}

derp/derphttp/derphttp_server.go

@@ -18,6 +18,7 @@ import (
 // following its HTTP request.
 const fastStartHeader = "Derp-Fast-Start"
 
+// Handler returns an http.Handler to be mounted at /derp, serving s.
 func Handler(s *derp.Server) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// These are installed both here and in cmd/derper. The check here
@@ -79,3 +80,29 @@ func ProbeHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "bogus probe method", http.StatusMethodNotAllowed)
 	}
 }
+
+// ServeNoContent generates the /generate_204 response used by Tailscale's
+// captive portal detection.
+func ServeNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(NoContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(NoContentResponseHeader, "response "+challenge)
+		}
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
+const (
+	NoContentChallengeHeader = "X-Tailscale-Challenge"
+	NoContentResponseHeader  = "X-Tailscale-Response"
+)

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/golangci/golangci-lint v1.52.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.18.0
+	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806
 	github.com/google/uuid v1.6.0
 	github.com/goreleaser/nfpm/v2 v2.33.1

go.sum

@@ -477,6 +477,8 @@ github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2 h1:CVuJwN34x4xM2aT4sIKhmeib40NeBPhRihNjQmpJsA4=
 github.com/google/goterm v0.0.0-20200907032337-555d40f16ae2/go.mod h1:nOFQdrUlIlx6M6ODdSpBj1NVA+VgLC6kmw60mkw34H4=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=

go.toolchain.rev

@@ -1 +1 @@
-2f152a4eff5875655a9a84fce8f8d329f8d9a321
+22ef9eb38e9a2d21b4a45f7adc75addb05f3efb8

gokrazy/tsapp/builddir/tailscale.com/go.sum

@@ -122,8 +122,12 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
 github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1 h1:ycpNCSYwzZ7x4G4ioPNtKQmIY0G/3o4pVf8wCZq6blY=
 github.com/tailscale/wireguard-go v0.0.0-20240705152531-2f5d148bcfe1/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98 h1:RNpJrXfI5u6e+uzyIzvmnXbhmhdRkVf//90sMBH3lso=
+github.com/tailscale/wireguard-go v0.0.0-20240731203015-71393c576b98/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
 github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek=
+github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
 github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ=
@@ -170,6 +174,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3 h1:/8/t5pz/mgdRXhYOIeqqYhFAQLE4DDGegc0Y4ZjyFJM=
 gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
+gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
 k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q=
 k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc=
 k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k=

gokrazy/tsapp/config.json

@@ -1,16 +1,21 @@
 {
     "Hostname": "tsapp",
-    "Update": { "NoPassword": true },
+    "Update": {
+        "NoPassword": true
+    },
     "SerialConsole": "ttyS0,115200",
     "Packages": [
         "github.com/gokrazy/serial-busybox",
         "github.com/gokrazy/breakglass",
         "tailscale.com/cmd/tailscale",
-        "tailscale.com/cmd/tailscaled"
+        "tailscale.com/cmd/tailscaled",
+        "tailscale.com/cmd/tta"
     ],
     "PackageConfig": {
         "github.com/gokrazy/breakglass": {
-            "CommandLineFlags": [ "-authorized_keys=ec2" ]
+            "CommandLineFlags": [
+                "-authorized_keys=ec2"
+            ]
         },
         "tailscale.com/cmd/tailscale": {
             "ExtraFilePaths": {
@@ -21,4 +26,4 @@
     "KernelPackage": "github.com/tailscale/gokrazy-kernel",
     "FirmwarePackage": "github.com/tailscale/gokrazy-kernel",
     "InternalCompatibilityFlags": {}
-}
+}

ipn/ipnlocal/local.go

@@ -3781,7 +3781,7 @@ func (b *LocalBackend) TCPHandlerForDst(src, dst netip.AddrPort) (handler func(c
 			return nil
 		}, opts
 	}
-	if handler := b.tcpHandlerForServe(dst.Port(), src); handler != nil {
+	if handler := b.tcpHandlerForServe(dst.Port(), src, nil); handler != nil {
 		return handler, opts
 	}
 	return nil, nil

ipn/ipnlocal/serve.go

@@ -56,6 +56,16 @@ var serveHTTPContextKey ctxkey.Key[*serveHTTPContext]
 type serveHTTPContext struct {
 	SrcAddr  netip.AddrPort
 	DestPort uint16
+
+	// provides funnel-specific context, nil if not funneled
+	Funnel *funnelFlow
+}
+
+// funnelFlow represents a funneled connection initiated via IngressPeer
+// to Host.
+type funnelFlow struct {
+	Host        string
+	IngressPeer tailcfg.NodeView
 }
 
 // localListener is the state of host-level net.Listen for a specific (Tailscale IP, port)
@@ -91,7 +101,7 @@ func (b *LocalBackend) newServeListener(ctx context.Context, ap netip.AddrPort,
 
 		handler: func(conn net.Conn) error {
 			srcAddr := conn.RemoteAddr().(*net.TCPAddr).AddrPort()
-			handler := b.tcpHandlerForServe(ap.Port(), srcAddr)
+			handler := b.tcpHandlerForServe(ap.Port(), srcAddr, nil)
 			if handler == nil {
 				b.logf("[unexpected] local-serve: no handler for %v to port %v", srcAddr, ap.Port())
 				conn.Close()
@@ -382,7 +392,7 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 		return
 	}
 
-	_, port, err := net.SplitHostPort(string(target))
+	host, port, err := net.SplitHostPort(string(target))
 	if err != nil {
 		logf("got ingress conn for bad target %q; rejecting", target)
 		sendRST()
@@ -407,9 +417,10 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 			return
 		}
 	}
-	// TODO(bradfitz): pass ingressPeer etc in context to tcpHandlerForServe,
-	// extend serveHTTPContext or similar.
-	handler := b.tcpHandlerForServe(dport, srcAddr)
+	handler := b.tcpHandlerForServe(dport, srcAddr, &funnelFlow{
+		Host:        host,
+		IngressPeer: ingressPeer,
+	})
 	if handler == nil {
 		logf("[unexpected] no matching ingress serve handler for %v to port %v", srcAddr, dport)
 		sendRST()
@@ -424,8 +435,9 @@ func (b *LocalBackend) HandleIngressTCPConn(ingressPeer tailcfg.NodeView, target
 }
 
 // tcpHandlerForServe returns a handler for a TCP connection to be served via
-// the ipn.ServeConfig.
-func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort) (handler func(net.Conn) error) {
+// the ipn.ServeConfig. The funnelFlow can be nil if this is not a funneled
+// connection.
+func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort, f *funnelFlow) (handler func(net.Conn) error) {
 	b.mu.Lock()
 	sc := b.serveConfig
 	b.mu.Unlock()
@@ -444,6 +456,7 @@ func (b *LocalBackend) tcpHandlerForServe(dport uint16, srcAddr netip.AddrPort)
 			Handler: http.HandlerFunc(b.serveWebHandler),
 			BaseContext: func(_ net.Listener) context.Context {
 				return serveHTTPContextKey.WithValue(context.Background(), &serveHTTPContext{
+					Funnel:   f,
 					SrcAddr:  srcAddr,
 					DestPort: dport,
 				})
@@ -712,15 +725,20 @@ func (b *LocalBackend) addTailscaleIdentityHeaders(r *httputil.ProxyRequest) {
 	r.Out.Header.Del("Tailscale-User-Login")
 	r.Out.Header.Del("Tailscale-User-Name")
 	r.Out.Header.Del("Tailscale-User-Profile-Pic")
+	r.Out.Header.Del("Tailscale-Funnel-Request")
 	r.Out.Header.Del("Tailscale-Headers-Info")
 
 	c, ok := serveHTTPContextKey.ValueOk(r.Out.Context())
 	if !ok {
 		return
 	}
+	if c.Funnel != nil {
+		r.Out.Header.Set("Tailscale-Funnel-Request", "?1")
+		return
+	}
 	node, user, ok := b.WhoIs("tcp", c.SrcAddr)
 	if !ok {
-		return // traffic from outside of Tailnet (funneled)
+		return // traffic from outside of Tailnet (funneled or local machine)
 	}
 	if node.IsTagged() {
 		// 2023-06-14: Not setting identity headers for tagged nodes.

net/dns/resolver/forwarder.go

@@ -6,6 +6,8 @@ package resolver
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -498,9 +500,10 @@ var (
 func (f *forwarder) send(ctx context.Context, fq *forwardQuery, rr resolverAndDelay) (ret []byte, err error) {
 	if verboseDNSForward() {
 		id := forwarderCount.Add(1)
-		f.logf("forwarder.send(%q) [%d] ...", rr.name.Addr, id)
+		domain, typ, _ := nameFromQuery(fq.packet)
+		f.logf("forwarder.send(%q, %d, %v, %d) [%d] ...", rr.name.Addr, fq.txid, typ, len(domain), id)
 		defer func() {
-			f.logf("forwarder.send(%q) [%d] = %v, %v", rr.name.Addr, id, len(ret), err)
+			f.logf("forwarder.send(%q, %d, %v, %d) [%d] = %v, %v", rr.name.Addr, fq.txid, typ, len(domain), id, len(ret), err)
 		}()
 	}
 	if strings.HasPrefix(rr.name.Addr, "http://") {
@@ -866,7 +869,7 @@ type forwardQuery struct {
 // node DNS proxy queries), otherwise f.resolvers is used.
 func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, responseChan chan<- packet, resolvers ...resolverAndDelay) error {
 	metricDNSFwd.Add(1)
-	domain, err := nameFromQuery(query.bs)
+	domain, typ, err := nameFromQuery(query.bs)
 	if err != nil {
 		metricDNSFwdErrorName.Add(1)
 		return err
@@ -943,6 +946,12 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 	}
 	defer fq.closeOnCtxDone.Close()
 
+	if verboseDNSForward() {
+		domainSha256 := sha256.Sum256([]byte(domain))
+		domainSig := base64.RawStdEncoding.EncodeToString(domainSha256[:3])
+		f.logf("request(%d, %v, %d, %s) %d...", fq.txid, typ, len(domain), domainSig, len(fq.packet))
+	}
+
 	resc := make(chan []byte, 1) // it's fine buffered or not
 	errc := make(chan error, 1)  // it's fine buffered or not too
 	for i := range resolvers {
@@ -982,6 +991,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 				metricDNSFwdErrorContext.Add(1)
 				return fmt.Errorf("waiting to send response: %w", ctx.Err())
 			case responseChan <- packet{v, query.family, query.addr}:
+				if verboseDNSForward() {
+					f.logf("response(%d, %v, %d) = %d, nil", fq.txid, typ, len(domain), len(v))
+				}
 				metricDNSFwdSuccess.Add(1)
 				f.health.SetHealthy(dnsForwarderFailing)
 				return nil
@@ -1009,6 +1021,9 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 						}
 						f.health.SetUnhealthy(dnsForwarderFailing, health.Args{health.ArgDNSServers: strings.Join(resolverAddrs, ",")})
 					case responseChan <- res:
+						if verboseDNSForward() {
+							f.logf("forwarder response(%d, %v, %d) = %d, %v", fq.txid, typ, len(domain), len(res.bs), firstErr)
+						}
 					}
 				}
 				return firstErr
@@ -1037,24 +1052,28 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo
 var initListenConfig func(_ *net.ListenConfig, _ *netmon.Monitor, tunName string) error
 
 // nameFromQuery extracts the normalized query name from bs.
-func nameFromQuery(bs []byte) (dnsname.FQDN, error) {
+func nameFromQuery(bs []byte) (dnsname.FQDN, dns.Type, error) {
 	var parser dns.Parser
 
 	hdr, err := parser.Start(bs)
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
 	if hdr.Response {
-		return "", errNotQuery
+		return "", 0, errNotQuery
 	}
 
 	q, err := parser.Question()
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
 
 	n := q.Name.Data[:q.Name.Length]
-	return dnsname.ToFQDN(rawNameToLower(n))
+	fqdn, err := dnsname.ToFQDN(rawNameToLower(n))
+	if err != nil {
+		return "", 0, err
+	}
+	return fqdn, q.Type, nil
 }
 
 // nxDomainResponse returns an NXDomain DNS reply for the provided request.

net/dns/resolver/forwarder_test.go

@@ -201,7 +201,7 @@ func BenchmarkNameFromQuery(b *testing.B) {
 	b.ResetTimer()
 	b.ReportAllocs()
 	for range b.N {
-		_, err := nameFromQuery(msg)
+		_, _, err := nameFromQuery(msg)
 		if err != nil {
 			b.Fatal(err)
 		}

net/netcheck/netcheck.go

@@ -1108,9 +1108,9 @@ func (c *Client) measureHTTPSLatency(ctx context.Context, reg *tailcfg.DERPRegio
 	}
 	result.End(c.timeNow())
 
-	// TODO: decide best timing heuristic here.
-	// Maybe the server should return the tcpinfo_rtt?
-	return result.ServerProcessing, ip, nil
+	// TODO(jwhited): consider simplified TCP RTT. We don't need HTTPS or TLS
+	//  involvement above.
+	return result.TCPConnection, ip, nil
 }
 
 func (c *Client) measureAllICMPLatency(ctx context.Context, rs *reportState, need []*tailcfg.DERPRegion) error {

prober/prober.go

@@ -496,7 +496,8 @@ func (p *Prober) RunHandler(w http.ResponseWriter, r *http.Request) error {
 		return nil
 	}
 
-	stats := fmt.Sprintf("Previous runs: success rate %d%%, median latency %v",
+	stats := fmt.Sprintf("Last %d probes: success rate %d%%, median latency %v\n",
+		len(prevInfo.RecentResults),
 		int(prevInfo.RecentSuccessRatio()*100), prevInfo.RecentMedianLatency())
 	if err != nil {
 		return tsweb.Error(respStatus, fmt.Sprintf("Probe failed: %s\n%s", err.Error(), stats), err)

prober/status.go

@@ -86,7 +86,7 @@ func (p *Prober) StatusHandler(opts ...statusHandlerOpt) tsweb.ReturnHandlerFunc
 			}
 			s := probeStatus{ProbeInfo: info}
 			if !info.End.IsZero() {
-				s.TimeSinceLast = time.Since(info.End)
+				s.TimeSinceLast = time.Since(info.End).Truncate(time.Second)
 			}
 			for textTpl, urlTpl := range params.probeLinks {
 				text, err := renderTemplate(textTpl, info)

prober/status.html

@@ -71,12 +71,12 @@
     <table class="sortable">
         <thead><tr>
             <th>Name</th>
-            <th>Class & Labels</th>
+            <th>Probe Class & Labels</th>
             <th>Interval</th>
-            <th>Result</th>
+            <th>Last Attempt</th>
             <th>Success</th>
             <th>Latency</th>
-            <th>Error</th>
+            <th>Last Error</th>
         </tr></thead>
         <tbody>
         {{range $name, $probeInfo := .Probes}}
@@ -100,8 +100,8 @@
             <td>{{$probeInfo.Interval}}</td>
             <td data-sort="{{$probeInfo.TimeSinceLast.Milliseconds}}">
                 {{if $probeInfo.TimeSinceLast}}
-                    {{$probeInfo.TimeSinceLast.String}}<br/>
-                    <span class="small">{{$probeInfo.End}}</span>
+                    {{$probeInfo.TimeSinceLast.String}} ago<br/>
+                    <span class="small">{{$probeInfo.End.Format "2006-01-02T15:04:05Z07:00"}}</span>
                 {{else}}
                     Never
                 {{end}}

tstest/natlab/vnet/conf.go

@@ -0,0 +1,217 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"cmp"
+	"fmt"
+	"net/netip"
+	"slices"
+
+	"tailscale.com/util/set"
+)
+
+// Note: the exported Node and Network are the configuration types;
+// the unexported node and network are the runtime types that are actually
+// used once the server is created.
+
+// Config is the requested state of the natlab virtual network.
+//
+// The zero value is a valid empty configuration. Call AddNode
+// and AddNetwork to methods on the returned Node and Network
+// values to modify the config before calling NewServer.
+// Once the NewServer is called, Config is no longer used.
+type Config struct {
+	nodes    []*Node
+	networks []*Network
+}
+
+// AddNode creates a new node in the world.
+//
+// The opts may be of the following types:
+//   - *Network: zero, one, or more networks to add this node to
+//   - TODO: more
+//
+// On an error or unknown opt type, AddNode returns a
+// node with a carried error that gets returned later.
+func (c *Config) AddNode(opts ...any) *Node {
+	num := len(c.nodes)
+	n := &Node{
+		mac: MAC{0x52, 0xcc, 0xcc, 0xcc, 0xcc, byte(num)}, // 52=TS then 0xcc for ccclient
+	}
+	c.nodes = append(c.nodes, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case *Network:
+			if !slices.Contains(o.nodes, n) {
+				o.nodes = append(o.nodes, n)
+			}
+			n.nets = append(n.nets, o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNode option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// AddNetwork add a new network.
+//
+// The opts may be of the following types:
+//   - string IP address, for the network's WAN IP (if any)
+//   - string netip.Prefix, for the network's LAN IP (defaults to 192.168.0.0/24)
+//   - NAT, the type of NAT to use
+//   - NetworkService, a service to add to the network
+//
+// On an error or unknown opt type, AddNetwork returns a
+// network with a carried error that gets returned later.
+func (c *Config) AddNetwork(opts ...any) *Network {
+	num := len(c.networks)
+	n := &Network{
+		mac: MAC{0x52, 0xee, 0xee, 0xee, 0xee, byte(num)}, // 52=TS then 0xee for 'etwork
+	}
+	c.networks = append(c.networks, n)
+	for _, o := range opts {
+		switch o := o.(type) {
+		case string:
+			if ip, err := netip.ParseAddr(o); err == nil {
+				n.wanIP = ip
+			} else if ip, err := netip.ParsePrefix(o); err == nil {
+				n.lanIP = ip
+			} else {
+				if n.err == nil {
+					n.err = fmt.Errorf("unknown string option %q", o)
+				}
+			}
+		case NAT:
+			n.natType = o
+		case NetworkService:
+			n.AddService(o)
+		default:
+			if n.err == nil {
+				n.err = fmt.Errorf("unknown AddNetwork option type %T", o)
+			}
+		}
+	}
+	return n
+}
+
+// Node is the configuration of a node in the virtual network.
+type Node struct {
+	err error
+	n   *node // nil until NewServer called
+
+	// TODO(bradfitz): this is halfway converted to supporting multiple NICs
+	// but not done. We need a MAC-per-Network.
+
+	mac  MAC
+	nets []*Network
+}
+
+// Network returns the first network this node is connected to,
+// or nil if none.
+func (n *Node) Network() *Network {
+	if len(n.nets) == 0 {
+		return nil
+	}
+	return n.nets[0]
+}
+
+// Network is the configuration of a network in the virtual network.
+type Network struct {
+	mac     MAC // MAC address of the router/gateway
+	natType NAT
+
+	wanIP netip.Addr
+	lanIP netip.Prefix
+	nodes []*Node
+
+	svcs set.Set[NetworkService]
+
+	// ...
+	err error // carried error
+}
+
+// NetworkService is a service that can be added to a network.
+type NetworkService string
+
+const (
+	NATPMP NetworkService = "NAT-PMP"
+	PCP    NetworkService = "PCP"
+	UPnP   NetworkService = "UPnP"
+)
+
+// AddService adds a network service (such as port mapping protocols) to a
+// network.
+func (n *Network) AddService(s NetworkService) {
+	if n.svcs == nil {
+		n.svcs = set.Of(s)
+	} else {
+		n.svcs.Add(s)
+	}
+}
+
+// initFromConfig initializes the server from the previous calls
+// to NewNode and NewNetwork and returns an error if
+// there were any configuration issues.
+func (s *Server) initFromConfig(c *Config) error {
+	netOfConf := map[*Network]*network{}
+	for _, conf := range c.networks {
+		if conf.err != nil {
+			return conf.err
+		}
+		if !conf.lanIP.IsValid() {
+			conf.lanIP = netip.MustParsePrefix("192.168.0.0/24")
+		}
+		n := &network{
+			s:         s,
+			mac:       conf.mac,
+			portmap:   conf.svcs.Contains(NATPMP), // TODO: expand network.portmap
+			wanIP:     conf.wanIP,
+			lanIP:     conf.lanIP,
+			nodesByIP: map[netip.Addr]*node{},
+		}
+		netOfConf[conf] = n
+		s.networks.Add(n)
+		if _, ok := s.networkByWAN[conf.wanIP]; ok {
+			return fmt.Errorf("two networks have the same WAN IP %v; Anycast not (yet?) supported", conf.wanIP)
+		}
+		s.networkByWAN[conf.wanIP] = n
+	}
+	for _, conf := range c.nodes {
+		if conf.err != nil {
+			return conf.err
+		}
+		n := &node{
+			mac: conf.mac,
+			net: netOfConf[conf.Network()],
+		}
+		conf.n = n
+		if _, ok := s.nodeByMAC[n.mac]; ok {
+			return fmt.Errorf("two nodes have the same MAC %v", n.mac)
+		}
+		s.nodes = append(s.nodes, n)
+		s.nodeByMAC[n.mac] = n
+
+		// Allocate a lanIP for the node. Use the network's CIDR and use final
+		// octet 101 (for first node), 102, etc. The node number comes from the
+		// last octent of the MAC address (0-based)
+		ip4 := n.net.lanIP.Addr().As4()
+		ip4[3] = 101 + n.mac[5]
+		n.lanIP = netip.AddrFrom4(ip4)
+		n.net.nodesByIP[n.lanIP] = n
+	}
+
+	// Now that nodes are populated, set up NAT:
+	for _, conf := range c.networks {
+		n := netOfConf[conf]
+		natType := cmp.Or(conf.natType, EasyNAT)
+		if err := n.InitNAT(natType); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tstest/natlab/vnet/conf_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import "testing"
+
+func TestConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		setup   func(*Config)
+		wantErr string
+	}{
+		{
+			name: "simple",
+			setup: func(c *Config) {
+				c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", EasyNAT, NATPMP))
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", HardNAT))
+			},
+		},
+		{
+			name: "indirect",
+			setup: func(c *Config) {
+				n1 := c.AddNode(c.AddNetwork("2.1.1.1", "192.168.1.1/24", HardNAT))
+				n1.Network().AddService(NATPMP)
+				c.AddNode(c.AddNetwork("2.2.2.2", "10.2.0.1/16", NAT("hard")))
+			},
+		},
+		{
+			name: "multi-node-in-net",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+		},
+		{
+			name: "dup-wan-ip",
+			setup: func(c *Config) {
+				c.AddNetwork("2.1.1.1", "192.168.1.1/24")
+				c.AddNetwork("2.1.1.1", "10.2.0.1/16")
+			},
+			wantErr: "two networks have the same WAN IP 2.1.1.1; Anycast not (yet?) supported",
+		},
+		{
+			name: "one-to-one-nat-with-multiple-nodes",
+			setup: func(c *Config) {
+				net1 := c.AddNetwork("2.1.1.1", "192.168.1.1/24", One2OneNAT)
+				c.AddNode(net1)
+				c.AddNode(net1)
+			},
+			wantErr: "error creating NAT type \"one2one\" for network 2.1.1.1: can't use one2one NAT type on networks other than single-node networks",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var c Config
+			tt.setup(&c)
+			_, err := New(&c)
+			if err == nil {
+				if tt.wantErr == "" {
+					return
+				}
+				t.Fatalf("got success; wanted error %q", tt.wantErr)
+			}
+			if err.Error() != tt.wantErr {
+				t.Fatalf("got error %q; want %q", err, tt.wantErr)
+			}
+		})
+	}
+}

tstest/natlab/vnet/nat.go

@@ -0,0 +1,239 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+import (
+	"errors"
+	"math/rand/v2"
+	"net/netip"
+	"time"
+
+	"tailscale.com/util/mak"
+)
+
+const (
+	One2OneNAT NAT = "one2one"
+	EasyNAT    NAT = "easy"
+	HardNAT    NAT = "hard"
+)
+
+// IPPool is the interface that a NAT implementation uses to get information
+// about a network.
+//
+// Outside of tests, this is typically a *network.
+type IPPool interface {
+	// WANIP returns the primary WAN IP address.
+	//
+	// TODO: add another method for networks with multiple WAN IP addresses.
+	WANIP() netip.Addr
+
+	// SoleLanIP reports whether this network has a sole LAN client
+	// and if so, its IP address.
+	SoleLANIP() (_ netip.Addr, ok bool)
+
+	// TODO: port availability stuff for interacting with portmapping
+}
+
+// newTableFunc is a constructor for a NAT table.
+// The provided IPPool is typically (outside of tests) a *network.
+type newTableFunc func(IPPool) (NATTable, error)
+
+// NAT is a type of NAT that's known to natlab.
+//
+// For example, "easy" for Linux-style NAT, "hard" for FreeBSD-style NAT, etc.
+type NAT string
+
+// natTypes are the known NAT types.
+var natTypes = map[NAT]newTableFunc{}
+
+// registerNATType registers a NAT type.
+func registerNATType(name NAT, f newTableFunc) {
+	if _, ok := natTypes[name]; ok {
+		panic("duplicate NAT type: " + name)
+	}
+	natTypes[name] = f
+}
+
+// NATTable is what a NAT implementation is expected to do.
+//
+// This project tests Tailscale as it faces various combinations various NAT
+// implementations (e.g. Linux easy style NAT vs FreeBSD hard/endpoint dependent
+// NAT vs Cloud 1:1 NAT, etc)
+//
+// Implementations of NATTable need not handle concurrency; the natlab serializes
+// all calls into a NATTable.
+//
+// The provided `at` value will typically be time.Now, except for tests.
+// Implementations should not use real time and should only compare
+// previously provided time values.
+type NATTable interface {
+	// PickOutgoingSrc returns the source address to use for an outgoing packet.
+	//
+	// The result should either be invalid (to drop the packet) or a WAN (not
+	// private) IP address.
+	//
+	// Typically, the src is a LAN source IP address, but it might also be a WAN
+	// IP address if the packet is being forwarded for a source machine that has
+	// a public IP address.
+	PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort)
+
+	// PickIncomingDst returns the destination address to use for an incoming
+	// packet. The incoming src address is always a public WAN IP.
+	//
+	// The result should either be invalid (to drop the packet) or the IP
+	// address of a machine on the local network address, usually a private
+	// LAN IP.
+	PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort)
+}
+
+// oneToOneNAT is a 1:1 NAT, like a typical EC2 VM.
+type oneToOneNAT struct {
+	lanIP netip.Addr
+	wanIP netip.Addr
+}
+
+func init() {
+	registerNATType(One2OneNAT, func(p IPPool) (NATTable, error) {
+		lanIP, ok := p.SoleLANIP()
+		if !ok {
+			return nil, errors.New("can't use one2one NAT type on networks other than single-node networks")
+		}
+		return &oneToOneNAT{lanIP: lanIP, wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *oneToOneNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	return netip.AddrPortFrom(n.wanIP, src.Port())
+}
+
+func (n *oneToOneNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	return netip.AddrPortFrom(n.lanIP, dst.Port())
+}
+
+type hardKeyOut struct {
+	lanIP netip.Addr
+	dst   netip.AddrPort
+}
+
+type hardKeyIn struct {
+	wanPort uint16
+	src     netip.AddrPort
+}
+
+type portMappingAndTime struct {
+	port uint16
+	at   time.Time
+}
+
+type lanAddrAndTime struct {
+	lanAddr netip.AddrPort
+	at      time.Time
+}
+
+// hardNAT is an "Endpoint Dependent" NAT, like FreeBSD/pfSense/OPNsense.
+// This is shown as "MappingVariesByDestIP: true" by netcheck, and what
+// Tailscale calls "Hard NAT".
+type hardNAT struct {
+	wanIP netip.Addr
+
+	out map[hardKeyOut]portMappingAndTime
+	in  map[hardKeyIn]lanAddrAndTime
+}
+
+func init() {
+	registerNATType(HardNAT, func(p IPPool) (NATTable, error) {
+		return &hardNAT{wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *hardNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	ko := hardKeyOut{src.Addr(), dst}
+	if pm, ok := n.out[ko]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// No existing mapping exists. Create one.
+
+	// TODO: clean up old expired mappings
+
+	// Instead of proper data structures that would be efficient, we instead
+	// just loop a bunch and look for a free port. This project is only used
+	// by tests and doesn't care about performance, this is good enough.
+	for {
+		port := rand.N(uint16(32<<10)) + 32<<10 // pick some "ephemeral" port
+		ki := hardKeyIn{wanPort: port, src: dst}
+		if _, ok := n.in[ki]; ok {
+			// Port already in use.
+			continue
+		}
+		mak.Set(&n.in, ki, lanAddrAndTime{lanAddr: src, at: at})
+		mak.Set(&n.out, ko, portMappingAndTime{port: port, at: at})
+		return netip.AddrPortFrom(n.wanIP, port)
+	}
+}
+
+func (n *hardNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	ki := hardKeyIn{wanPort: dst.Port(), src: src}
+	if pm, ok := n.in[ki]; ok {
+		// Existing flow.
+		return pm.lanAddr
+	}
+	return netip.AddrPort{} // drop; no mapping
+}
+
+// easyNAT is an "Endpoint Independent" NAT, like Linux and most home routers
+// (many of which are Linux).
+//
+// This is shown as "MappingVariesByDestIP: false" by netcheck, and what
+// Tailscale calls "Easy NAT".
+//
+// Unlike Linux, this implementation is capped at 32k entries and doesn't resort
+// to other allocation strategies when all 32k WAN ports are taken.
+type easyNAT struct {
+	wanIP netip.Addr
+	out   map[netip.AddrPort]portMappingAndTime
+	in    map[uint16]lanAddrAndTime
+}
+
+func init() {
+	registerNATType(EasyNAT, func(p IPPool) (NATTable, error) {
+		return &easyNAT{wanIP: p.WANIP()}, nil
+	})
+}
+
+func (n *easyNAT) PickOutgoingSrc(src, dst netip.AddrPort, at time.Time) (wanSrc netip.AddrPort) {
+	if pm, ok := n.out[src]; ok {
+		// Existing flow.
+		// TODO: bump timestamp
+		return netip.AddrPortFrom(n.wanIP, pm.port)
+	}
+
+	// Loop through all 32k high (ephemeral) ports, starting at a random
+	// position and looping back around to the start.
+	start := rand.N(uint16(32 << 10))
+	for off := range uint16(32 << 10) {
+		port := 32<<10 + (start+off)%(32<<10)
+		if _, ok := n.in[port]; !ok {
+			wanAddr := netip.AddrPortFrom(n.wanIP, port)
+
+			// Found a free port.
+			mak.Set(&n.out, src, portMappingAndTime{port: port, at: at})
+			mak.Set(&n.in, port, lanAddrAndTime{lanAddr: src, at: at})
+			return wanAddr
+		}
+	}
+	return netip.AddrPort{} // failed to allocate a mapping; TODO: fire an alert?
+}
+
+func (n *easyNAT) PickIncomingDst(src, dst netip.AddrPort, at time.Time) (lanDst netip.AddrPort) {
+	if dst.Addr() != n.wanIP {
+		return netip.AddrPort{} // drop; not for us. shouldn't happen if natlabd routing isn't broken.
+	}
+	return n.in[dst.Port()].lanAddr
+}