syncs: add Map (#6260)

Map is a concurrent safe map that is a trivial wrapper
over a Go map and a sync.RWMutex.

It is optimized for use-cases where the entries change often,
which is the opposite use-case of what sync.Map is optimized for.

The API is patterned off of sync.Map, but made generic.

Signed-off-by: Joe Tsai <joetsai@digital-static.net>

VERSION.txt

@@ -1 +1 @@
-1.31.0
+1.32.2

client/tailscale/localclient.go

@@ -276,6 +276,12 @@ type BugReportOpts struct {
 	// Diagnose specifies whether to print additional diagnostic information to
 	// the logs when generating this bugreport.
 	Diagnose bool
+
+	// Record specifies, if non-nil, whether to perform a bugreport
+	// "recording"–generating an initial log marker, then waiting for
+	// this channel to be closed before finishing the request, which
+	// generates another log marker.
+	Record <-chan struct{}
 }
 
 // BugReportWithOpts logs and returns a log marker that can be shared by the
@@ -284,16 +290,40 @@ type BugReportOpts struct {
 // The opts type specifies options to pass to the Tailscale daemon when
 // generating this bug report.
 func (lc *LocalClient) BugReportWithOpts(ctx context.Context, opts BugReportOpts) (string, error) {
-	var qparams url.Values
+	qparams := make(url.Values)
 	if opts.Note != "" {
 		qparams.Set("note", opts.Note)
 	}
 	if opts.Diagnose {
 		qparams.Set("diagnose", "true")
 	}
+	if opts.Record != nil {
+		qparams.Set("record", "true")
+	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var requestBody io.Reader
+	if opts.Record != nil {
+		pr, pw := io.Pipe()
+		requestBody = pr
+
+		// This goroutine waits for the 'Record' channel to be closed,
+		// and then closes the write end of our pipe to unblock the
+		// reader.
+		go func() {
+			defer pw.Close()
+			select {
+			case <-opts.Record:
+			case <-ctx.Done():
+			}
+		}()
+	}
+
+	// lc.send might block if opts.Record != nil; see above.
 	uri := fmt.Sprintf("/localapi/v0/bugreport?%s", qparams.Encode())
-	body, err := lc.send(ctx, "POST", uri, 200, nil)
+	body, err := lc.send(ctx, "POST", uri, 200, requestBody)
 	if err != nil {
 		return "", err
 	}

cmd/derper/depaware.txt

@@ -47,6 +47,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp
         tailscale.com/net/tsaddr                                     from tailscale.com/ipn+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
+        tailscale.com/net/wsconn                                     from tailscale.com/cmd/derper+
         tailscale.com/paths                                          from tailscale.com/client/tailscale
         tailscale.com/safesocket                                     from tailscale.com/client/tailscale
         tailscale.com/syncs                                          from tailscale.com/cmd/derper+
@@ -74,6 +75,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa
         tailscale.com/util/dnsname                                   from tailscale.com/hostinfo+
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/lineread                                  from tailscale.com/hostinfo+
+        tailscale.com/util/mak                                       from tailscale.com/syncs
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
    L    tailscale.com/util/strs                                      from tailscale.com/hostinfo
    W 💣 tailscale.com/util/winutil                                   from tailscale.com/hostinfo+

cmd/derper/derper.go

@@ -325,11 +325,31 @@ func main() {
 	}
 }
 
+const (
+	noContentChallengeHeader = "X-Tailscale-Challenge"
+	noContentResponseHeader  = "X-Tailscale-Response"
+)
+
 // For captive portal detection
 func serveNoContent(w http.ResponseWriter, r *http.Request) {
+	if challenge := r.Header.Get(noContentChallengeHeader); challenge != "" {
+		badChar := strings.IndexFunc(challenge, func(r rune) bool {
+			return !isChallengeChar(r)
+		}) != -1
+		if len(challenge) <= 64 && !badChar {
+			w.Header().Set(noContentResponseHeader, "response "+challenge)
+		}
+	}
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func isChallengeChar(c rune) bool {
+	// Semi-randomly chosen as a limited set of valid characters
+	return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9') ||
+		c == '.' || c == '-' || c == '_'
+}
+
 // probeHandler is the endpoint that js/wasm clients hit to measure
 // DERP latency, since they can't do UDP STUN queries.
 func probeHandler(w http.ResponseWriter, r *http.Request) {

cmd/derper/derper_test.go

@@ -7,6 +7,9 @@ package main
 import (
 	"context"
 	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"tailscale.com/net/stun"
@@ -67,3 +70,57 @@ func BenchmarkServerSTUN(b *testing.B) {
 	}
 
 }
+
+func TestNoContent(t *testing.T) {
+	testCases := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			name: "no challenge",
+		},
+		{
+			name:  "valid challenge",
+			input: "input",
+			want:  "response input",
+		},
+		{
+			name:  "invalid challenge",
+			input: "foo\x00bar",
+			want:  "",
+		},
+		{
+			name:  "whitespace invalid challenge",
+			input: "foo bar",
+			want:  "",
+		},
+		{
+			name:  "long challenge",
+			input: strings.Repeat("x", 65),
+			want:  "",
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			req, _ := http.NewRequest("GET", "https://localhost/generate_204", nil)
+			if tt.input != "" {
+				req.Header.Set(noContentChallengeHeader, tt.input)
+			}
+			w := httptest.NewRecorder()
+			serveNoContent(w, req)
+			resp := w.Result()
+
+			if tt.want == "" {
+				if h, found := resp.Header[noContentResponseHeader]; found {
+					t.Errorf("got %+v; expected no response header", h)
+				}
+				return
+			}
+
+			if got := resp.Header.Get(noContentResponseHeader); got != tt.want {
+				t.Errorf("got %q; want %q", got, tt.want)
+			}
+		})
+	}
+}

cmd/derper/websocket.go

@@ -13,6 +13,7 @@ import (
 
 	"nhooyr.io/websocket"
 	"tailscale.com/derp"
+	"tailscale.com/net/wsconn"
 )
 
 var counterWebSocketAccepts = expvar.NewInt("derp_websocket_accepts")
@@ -50,7 +51,7 @@ func addWebSocketSupport(s *derp.Server, base http.Handler) http.Handler {
 			return
 		}
 		counterWebSocketAccepts.Add(1)
-		wc := websocket.NetConn(r.Context(), c, websocket.MessageBinary)
+		wc := wsconn.NetConn(r.Context(), c, websocket.MessageBinary)
 		brw := bufio.NewReadWriter(bufio.NewReader(wc), bufio.NewWriter(wc))
 		s.Accept(r.Context(), wc, brw, r.RemoteAddr)
 	})

cmd/netlogfmt/main.go

@@ -0,0 +1,387 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// netlogfmt parses a stream of JSON log messages from stdin and
+// formats the network traffic logs produced by "tailscale.com/wgengine/netlog"
+// according to the schema in "tailscale.com/types/netlogtype.Message"
+// in a more humanly readable format.
+//
+// Example usage:
+//
+//	$ cat netlog.json | go run tailscale.com/cmd/netlogfmt
+//	=========================================================================================
+//	NodeID: n123456CNTRL
+//	Logged: 2022-10-13T20:23:10.165Z
+//	Window: 2022-10-13T20:23:09.644Z (5s)
+//	---------------------------------------------------  Tx[P/s]  Tx[B/s]  Rx[P/s]    Rx[B/s]
+//	VirtualTraffic:                                       16.80    1.64Ki   11.20      1.03Ki
+//	    TCP:    100.109.51.95:22 -> 100.85.80.41:42912    16.00    1.59Ki   10.40   1008.84
+//	    TCP: 100.109.51.95:21291 -> 100.107.177.2:53133    0.40   27.60      0.40     24.20
+//	    TCP: 100.109.51.95:21291 -> 100.107.177.2:53134    0.40   23.40      0.40     24.20
+//	PhysicalTraffic:                                      16.80    2.32Ki   11.20      1.48Ki
+//	                100.85.80.41 -> 192.168.0.101:41641   16.00    2.23Ki   10.40      1.40Ki
+//	               100.107.177.2 -> 192.168.0.100:41641    0.80   83.20      0.80     83.20
+//	=========================================================================================
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"net/netip"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dsnet/try"
+	jsonv2 "github.com/go-json-experiment/json"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+	"tailscale.com/logtail"
+	"tailscale.com/types/netlogtype"
+	"tailscale.com/util/must"
+)
+
+var (
+	resolveNames = flag.Bool("resolve-names", false, "convert tailscale IP addresses to hostnames; must also specify --api-key and --tailnet-id")
+	apiKey       = flag.String("api-key", "", "API key to query the Tailscale API with; see https://login.tailscale.com/admin/settings/keys")
+	tailnetName  = flag.String("tailnet-name", "", "tailnet domain name to lookup devices in; see https://login.tailscale.com/admin/settings/general")
+)
+
+var namesByAddr map[netip.Addr]string
+
+func main() {
+	flag.Parse()
+	if *resolveNames {
+		namesByAddr = mustMakeNamesByAddr()
+	}
+
+	// The logic handles a stream of arbitrary JSON.
+	// So long as a JSON object seems like a network log message,
+	// then this will unmarshal and print it.
+	if err := processStream(os.Stdin); err != nil {
+		if err == io.EOF {
+			return
+		}
+		log.Fatalf("processStream: %v", err)
+	}
+}
+
+func processStream(r io.Reader) (err error) {
+	defer try.Handle(&err)
+	dec := jsonv2.NewDecoder(os.Stdin)
+	for {
+		processValue(dec)
+	}
+}
+
+func processValue(dec *jsonv2.Decoder) {
+	switch dec.PeekKind() {
+	case '[':
+		processArray(dec)
+	case '{':
+		processObject(dec)
+	default:
+		try.E(dec.SkipValue())
+	}
+}
+
+func processArray(dec *jsonv2.Decoder) {
+	try.E1(dec.ReadToken()) // parse '['
+	for dec.PeekKind() != ']' {
+		processValue(dec)
+	}
+	try.E1(dec.ReadToken()) // parse ']'
+}
+
+func processObject(dec *jsonv2.Decoder) {
+	var hasTraffic bool
+	var rawMsg []byte
+	try.E1(dec.ReadToken()) // parse '{'
+	for dec.PeekKind() != '}' {
+		// Capture any members that could belong to a network log message.
+		switch name := try.E1(dec.ReadToken()); name.String() {
+		case "virtualTraffic", "subnetTraffic", "exitTraffic", "physicalTraffic":
+			hasTraffic = true
+			fallthrough
+		case "logtail", "nodeId", "logged", "start", "end":
+			if len(rawMsg) == 0 {
+				rawMsg = append(rawMsg, '{')
+			} else {
+				rawMsg = append(rawMsg[:len(rawMsg)-1], ',')
+			}
+			rawMsg = append(append(append(rawMsg, '"'), name.String()...), '"')
+			rawMsg = append(rawMsg, ':')
+			rawMsg = append(rawMsg, try.E1(dec.ReadValue())...)
+			rawMsg = append(rawMsg, '}')
+		default:
+			processValue(dec)
+		}
+	}
+	try.E1(dec.ReadToken()) // parse '}'
+
+	// If this appears to be a network log message, then unmarshal and print it.
+	if hasTraffic {
+		var msg message
+		try.E(jsonv2.Unmarshal(rawMsg, &msg))
+		printMessage(msg)
+	}
+}
+
+type message struct {
+	Logtail struct {
+		ID     logtail.PublicID `json:"id"`
+		Logged time.Time        `json:"server_time"`
+	} `json:"logtail"`
+	Logged time.Time `json:"logged"`
+	netlogtype.Message
+}
+
+func printMessage(msg message) {
+	// Construct a table of network traffic per connection.
+	rows := [][7]string{{3: "Tx[P/s]", 4: "Tx[B/s]", 5: "Rx[P/s]", 6: "Rx[B/s]"}}
+	duration := msg.End.Sub(msg.Start)
+	addRows := func(heading string, traffic []netlogtype.ConnectionCounts) {
+		if len(traffic) == 0 {
+			return
+		}
+		slices.SortFunc(traffic, func(x, y netlogtype.ConnectionCounts) bool {
+			nx := x.TxPackets + x.TxBytes + x.RxPackets + x.RxBytes
+			ny := y.TxPackets + y.TxBytes + y.RxPackets + y.RxBytes
+			return nx > ny
+		})
+		var sum netlogtype.Counts
+		for _, cc := range traffic {
+			sum = sum.Add(cc.Counts)
+		}
+		rows = append(rows, [7]string{
+			0: heading + ":",
+			3: formatSI(float64(sum.TxPackets) / duration.Seconds()),
+			4: formatIEC(float64(sum.TxBytes) / duration.Seconds()),
+			5: formatSI(float64(sum.RxPackets) / duration.Seconds()),
+			6: formatIEC(float64(sum.RxBytes) / duration.Seconds()),
+		})
+		if len(traffic) == 1 && traffic[0].Connection.IsZero() {
+			return // this is already a summary counts
+		}
+		formatAddrPort := func(a netip.AddrPort) string {
+			if !a.IsValid() {
+				return ""
+			}
+			if name, ok := namesByAddr[a.Addr()]; ok {
+				if a.Port() == 0 {
+					return name
+				}
+				return name + ":" + strconv.Itoa(int(a.Port()))
+			}
+			if a.Port() == 0 {
+				return a.Addr().String()
+			}
+			return a.String()
+		}
+		for _, cc := range traffic {
+			row := [7]string{
+				0: "    ",
+				1: formatAddrPort(cc.Src),
+				2: formatAddrPort(cc.Dst),
+				3: formatSI(float64(cc.TxPackets) / duration.Seconds()),
+				4: formatIEC(float64(cc.TxBytes) / duration.Seconds()),
+				5: formatSI(float64(cc.RxPackets) / duration.Seconds()),
+				6: formatIEC(float64(cc.RxBytes) / duration.Seconds()),
+			}
+			if cc.Proto > 0 {
+				row[0] += cc.Proto.String() + ":"
+			}
+			rows = append(rows, row)
+		}
+	}
+	addRows("VirtualTraffic", msg.VirtualTraffic)
+	addRows("SubnetTraffic", msg.SubnetTraffic)
+	addRows("ExitTraffic", msg.ExitTraffic)
+	addRows("PhysicalTraffic", msg.PhysicalTraffic)
+
+	// Compute the maximum width of each field.
+	var maxWidths [7]int
+	for _, row := range rows {
+		for i, col := range row {
+			if maxWidths[i] < len(col) && !(i == 0 && !strings.HasPrefix(col, " ")) {
+				maxWidths[i] = len(col)
+			}
+		}
+	}
+	var maxSum int
+	for _, n := range maxWidths {
+		maxSum += n
+	}
+
+	// Output a table of network traffic per connection.
+	line := make([]byte, 0, maxSum+len(" ")+len(" -> ")+4*len("  "))
+	line = appendRepeatByte(line, '=', cap(line))
+	fmt.Println(string(line))
+	if !msg.Logtail.ID.IsZero() {
+		fmt.Printf("LogID:  %s\n", msg.Logtail.ID)
+	}
+	if msg.NodeID != "" {
+		fmt.Printf("NodeID: %s\n", msg.NodeID)
+	}
+	formatTime := func(t time.Time) string {
+		return t.In(time.Local).Format("2006-01-02 15:04:05.000")
+	}
+	switch {
+	case !msg.Logged.IsZero():
+		fmt.Printf("Logged: %s\n", formatTime(msg.Logged))
+	case !msg.Logtail.Logged.IsZero():
+		fmt.Printf("Logged: %s\n", formatTime(msg.Logtail.Logged))
+	}
+	fmt.Printf("Window: %s (%0.3fs)\n", formatTime(msg.Start), duration.Seconds())
+	for i, row := range rows {
+		line = line[:0]
+		isHeading := !strings.HasPrefix(row[0], " ")
+		for j, col := range row {
+			if isHeading && j == 0 {
+				col = "" // headings will be printed later
+			}
+			switch j {
+			case 0, 2: // left justified
+				line = append(line, col...)
+				line = appendRepeatByte(line, ' ', maxWidths[j]-len(col))
+			case 1, 3, 4, 5, 6: // right justified
+				line = appendRepeatByte(line, ' ', maxWidths[j]-len(col))
+				line = append(line, col...)
+			}
+			switch j {
+			case 0:
+				line = append(line, " "...)
+			case 1:
+				if row[1] == "" && row[2] == "" {
+					line = append(line, "    "...)
+				} else {
+					line = append(line, " -> "...)
+				}
+			case 2, 3, 4, 5:
+				line = append(line, "  "...)
+			}
+		}
+		switch {
+		case i == 0: // print dashed-line table heading
+			line = appendRepeatByte(line[:0], '-', maxWidths[0]+len(" ")+maxWidths[1]+len(" -> ")+maxWidths[2])[:cap(line)]
+		case isHeading:
+			copy(line[:], row[0])
+		}
+		fmt.Println(string(line))
+	}
+}
+
+func mustMakeNamesByAddr() map[netip.Addr]string {
+	switch {
+	case *apiKey == "":
+		log.Fatalf("--api-key must be specified with --resolve-names")
+	case *tailnetName == "":
+		log.Fatalf("--tailnet must be specified with --resolve-names")
+	}
+
+	// Query the Tailscale API for a list of devices in the tailnet.
+	const apiURL = "https://api.tailscale.com/api/v2"
+	req := must.Get(http.NewRequest("GET", apiURL+"/tailnet/"+*tailnetName+"/devices", nil))
+	req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*apiKey+":")))
+	resp := must.Get(http.DefaultClient.Do(req))
+	defer resp.Body.Close()
+	b := must.Get(io.ReadAll(resp.Body))
+	if resp.StatusCode != 200 {
+		log.Fatalf("http: %v: %s", http.StatusText(resp.StatusCode), b)
+	}
+
+	// Unmarshal the API response.
+	var m struct {
+		Devices []struct {
+			Name  string       `json:"name"`
+			Addrs []netip.Addr `json:"addresses"`
+		} `json:"devices"`
+	}
+	must.Do(json.Unmarshal(b, &m))
+
+	// Construct a unique mapping of Tailscale IP addresses to hostnames.
+	// For brevity, we start with the first segment of the name and
+	// use more segments until we find the shortest prefix that is unique
+	// for all names in the tailnet.
+	seen := make(map[string]bool)
+	namesByAddr := make(map[netip.Addr]string)
+retry:
+	for i := 0; i < 10; i++ {
+		maps.Clear(seen)
+		maps.Clear(namesByAddr)
+		for _, d := range m.Devices {
+			name := fieldPrefix(d.Name, i)
+			if seen[name] {
+				continue retry
+			}
+			seen[name] = true
+			for _, a := range d.Addrs {
+				namesByAddr[a] = name
+			}
+		}
+		return namesByAddr
+	}
+	panic("unable to produce unique mapping of address to names")
+}
+
+// fieldPrefix returns the first n number of dot-separated segments.
+//
+// Example:
+//
+//	fieldPrefix("foo.bar.baz", 0) returns ""
+//	fieldPrefix("foo.bar.baz", 1) returns "foo"
+//	fieldPrefix("foo.bar.baz", 2) returns "foo.bar"
+//	fieldPrefix("foo.bar.baz", 3) returns "foo.bar.baz"
+//	fieldPrefix("foo.bar.baz", 4) returns "foo.bar.baz"
+func fieldPrefix(s string, n int) string {
+	s0 := s
+	for i := 0; i < n && len(s) > 0; i++ {
+		if j := strings.IndexByte(s, '.'); j >= 0 {
+			s = s[j+1:]
+		} else {
+			s = ""
+		}
+	}
+	return strings.TrimSuffix(s0[:len(s0)-len(s)], ".")
+}
+
+func appendRepeatByte(b []byte, c byte, n int) []byte {
+	for i := 0; i < n; i++ {
+		b = append(b, c)
+	}
+	return b
+}
+
+func formatSI(n float64) string {
+	switch n := math.Abs(n); {
+	case n < 1e3:
+		return fmt.Sprintf("%0.2f ", n/(1e0))
+	case n < 1e6:
+		return fmt.Sprintf("%0.2fk", n/(1e3))
+	case n < 1e9:
+		return fmt.Sprintf("%0.2fM", n/(1e6))
+	default:
+		return fmt.Sprintf("%0.2fG", n/(1e9))
+	}
+}
+
+func formatIEC(n float64) string {
+	switch n := math.Abs(n); {
+	case n < 1<<10:
+		return fmt.Sprintf("%0.2f  ", n/(1<<0))
+	case n < 1<<20:
+		return fmt.Sprintf("%0.2fKi", n/(1<<10))
+	case n < 1<<30:
+		return fmt.Sprintf("%0.2fMi", n/(1<<20))
+	default:
+		return fmt.Sprintf("%0.2fGi", n/(1<<30))
+	}
+}

cmd/tailscale/cli/bugreport.go

@@ -41,29 +41,44 @@ func runBugReport(ctx context.Context, args []string) error {
 	default:
 		return errors.New("unknown arguments")
 	}
-	logMarker, err := localClient.BugReportWithOpts(ctx, tailscale.BugReportOpts{
+	opts := tailscale.BugReportOpts{
 		Note:     note,
 		Diagnose: bugReportArgs.diagnose,
-	})
-	if err != nil {
-		return err
 	}
-
-	if bugReportArgs.record {
-		outln("The initial bugreport is below; please reproduce your issue and then press Enter...")
-	}
-
-	outln(logMarker)
-
-	if bugReportArgs.record {
-		fmt.Scanln()
-
-		logMarker, err := localClient.BugReportWithOpts(ctx, tailscale.BugReportOpts{})
+	if !bugReportArgs.record {
+		// Simple, non-record case
+		logMarker, err := localClient.BugReportWithOpts(ctx, opts)
 		if err != nil {
 			return err
 		}
 		outln(logMarker)
-		outln("Please provide both bugreport markers above to the support team or GitHub issue.")
+		return nil
 	}
+
+	// Recording; run the request in the background
+	done := make(chan struct{})
+	opts.Record = done
+
+	type bugReportResp struct {
+		marker string
+		err    error
+	}
+	resCh := make(chan bugReportResp, 1)
+	go func() {
+		m, err := localClient.BugReportWithOpts(ctx, opts)
+		resCh <- bugReportResp{m, err}
+	}()
+
+	outln("Recording started; please reproduce your issue and then press Enter...")
+	fmt.Scanln()
+	close(done)
+	res := <-resCh
+
+	if res.err != nil {
+		return res.err
+	}
+
+	outln(res.marker)
+	outln("Please provide both bugreport markers above to the support team or GitHub issue.")
 	return nil
 }

cmd/tailscale/cli/debug.go

@@ -42,7 +42,7 @@ var debugCmd = &ffcli.Command{
 	FlagSet: (func() *flag.FlagSet {
 		fs := newFlagSet("debug")
 		fs.StringVar(&debugArgs.file, "file", "", "get, delete:NAME, or NAME")
-		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-sec seconds and write it to this file; - for stdout")
+		fs.StringVar(&debugArgs.cpuFile, "cpu-profile", "", "if non-empty, grab a CPU profile for --profile-seconds seconds and write it to this file; - for stdout")
 		fs.StringVar(&debugArgs.memFile, "mem-profile", "", "if non-empty, grab a memory profile and write it to this file; - for stdout")
 		fs.IntVar(&debugArgs.cpuSec, "profile-seconds", 15, "number of seconds to run a CPU profile for, when --cpu-profile is non-empty")
 		return fs

cmd/tailscale/cli/up.go

@@ -501,7 +501,7 @@ func runUp(ctx context.Context, args []string) (retErr error) {
 		fatalf("%s", err)
 	}
 	if justEditMP != nil {
-		justEditMP.EggSet = true
+		justEditMP.EggSet = egg
 		_, err := localClient.EditPrefs(ctx, justEditMP)
 		return err
 	}

cmd/tailscale/depaware.txt

@@ -70,6 +70,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/net/tlsdial                                    from tailscale.com/derp/derphttp+
         tailscale.com/net/tsaddr                                     from tailscale.com/net/interfaces+
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/derp/derphttp+
+        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/paths                                          from tailscale.com/cmd/tailscale/cli+
         tailscale.com/safesocket                                     from tailscale.com/cmd/tailscale/cli+
         tailscale.com/syncs                                          from tailscale.com/net/netcheck+
@@ -99,7 +100,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
    W    tailscale.com/util/endian                                    from tailscale.com/net/netns
         tailscale.com/util/groupmember                               from tailscale.com/cmd/tailscale/cli
         tailscale.com/util/lineread                                  from tailscale.com/net/interfaces+
-        tailscale.com/util/mak                                       from tailscale.com/net/netcheck
+        tailscale.com/util/mak                                       from tailscale.com/net/netcheck+
         tailscale.com/util/multierr                                  from tailscale.com/control/controlhttp
         tailscale.com/util/singleflight                              from tailscale.com/net/dnscache
    L    tailscale.com/util/strs                                      from tailscale.com/hostinfo

cmd/tailscaled/depaware.txt

@@ -241,6 +241,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
      💣 tailscale.com/net/tshttpproxy                                from tailscale.com/control/controlclient+
         tailscale.com/net/tstun                                      from tailscale.com/net/dns+
         tailscale.com/net/tunstats                                   from tailscale.com/net/tstun
+        tailscale.com/net/wsconn                                     from tailscale.com/control/controlhttp+
         tailscale.com/paths                                          from tailscale.com/ipn/ipnlocal+
         tailscale.com/portlist                                       from tailscale.com/ipn/ipnlocal
         tailscale.com/safesocket                                     from tailscale.com/client/tailscale+
@@ -261,6 +262,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/control/controlbase+
         tailscale.com/types/logger                                   from tailscale.com/control/controlclient+
+        tailscale.com/types/netlogtype                               from tailscale.com/net/tstun+
         tailscale.com/types/netmap                                   from tailscale.com/control/controlclient+
         tailscale.com/types/nettype                                  from tailscale.com/wgengine/magicsock+
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
@@ -297,6 +299,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
      💣 tailscale.com/wgengine/magicsock                             from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/monitor                               from tailscale.com/control/controlclient+
+        tailscale.com/wgengine/netlog                                from tailscale.com/wgengine
         tailscale.com/wgengine/netstack                              from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/router                                from tailscale.com/ipn/ipnlocal+
         tailscale.com/wgengine/wgcfg                                 from tailscale.com/ipn/ipnlocal+

control/controlhttp/client_js.go

@@ -13,6 +13,7 @@ import (
 
 	"nhooyr.io/websocket"
 	"tailscale.com/control/controlbase"
+	"tailscale.com/net/wsconn"
 )
 
 // Variant of Dial that tunnels the request over WebSockets, since we cannot do
@@ -51,7 +52,7 @@ func (d *Dialer) Dial(ctx context.Context) (*controlbase.Conn, error) {
 	if err != nil {
 		return nil, err
 	}
-	netConn := websocket.NetConn(context.Background(), wsConn, websocket.MessageBinary)
+	netConn := wsconn.NetConn(context.Background(), wsConn, websocket.MessageBinary)
 	cbConn, err := cont(ctx, netConn)
 	if err != nil {
 		netConn.Close()

control/controlhttp/http_test.go

@@ -459,13 +459,26 @@ func TestDialPlan(t *testing.T) {
 
 	const (
 		testProtocolVersion = 1
-
-		// We need consistent ports for each address; these are chosen
-		// randomly and we hope that they won't conflict during this test.
-		httpPort  = "40080"
-		httpsPort = "40443"
 	)
 
+	getRandomPort := func() string {
+		ln, err := net.Listen("tcp", ":0")
+		if err != nil {
+			t.Fatalf("net.Listen: %v", err)
+		}
+		defer ln.Close()
+		_, port, err := net.SplitHostPort(ln.Addr().String())
+		if err != nil {
+			t.Fatal(err)
+		}
+		return port
+	}
+
+	// We need consistent ports for each address; these are chosen
+	// randomly and we hope that they won't conflict during this test.
+	httpPort := getRandomPort()
+	httpsPort := getRandomPort()
+
 	makeHandler := func(t *testing.T, name string, host netip.Addr, wrap func(http.Handler) http.Handler) {
 		done := make(chan struct{})
 		t.Cleanup(func() {

control/controlhttp/server.go

@@ -14,6 +14,7 @@ import (
 	"nhooyr.io/websocket"
 	"tailscale.com/control/controlbase"
 	"tailscale.com/net/netutil"
+	"tailscale.com/net/wsconn"
 	"tailscale.com/types/key"
 )
 
@@ -111,7 +112,7 @@ func acceptWebsocket(ctx context.Context, w http.ResponseWriter, r *http.Request
 		return nil, fmt.Errorf("decoding base64 handshake parameter: %v", err)
 	}
 
-	conn := websocket.NetConn(ctx, c, websocket.MessageBinary)
+	conn := wsconn.NetConn(ctx, c, websocket.MessageBinary)
 	nc, err := controlbase.Server(ctx, conn, private, init)
 	if err != nil {
 		conn.Close()

derp/derphttp/derphttp_client.go

@@ -96,7 +96,7 @@ func NewRegionClient(privateKey key.NodePrivate, logf logger.Logf, getRegion fun
 	return c
 }
 
-// NewNetcheckClient returns a Client that's only able to have its DialRegion method called.
+// NewNetcheckClient returns a Client that's only able to have its DialRegionTLS method called.
 // It's used by the netcheck package.
 func NewNetcheckClient(logf logger.Logf) *Client {
 	return &Client{logf: logf}
@@ -985,7 +985,9 @@ func (c *Client) isClosed() bool {
 // Close closes the client. It will not automatically reconnect after
 // being closed.
 func (c *Client) Close() error {
-	c.cancelCtx() // not in lock, so it can cancel Connect, which holds mu
+	if c.cancelCtx != nil {
+		c.cancelCtx() // not in lock, so it can cancel Connect, which holds mu
+	}
 
 	c.mu.Lock()
 	defer c.mu.Unlock()

derp/derphttp/websocket.go

@@ -13,6 +13,7 @@ import (
 	"net"
 
 	"nhooyr.io/websocket"
+	"tailscale.com/net/wsconn"
 )
 
 func init() {
@@ -28,6 +29,6 @@ func dialWebsocket(ctx context.Context, urlStr string) (net.Conn, error) {
 		return nil, err
 	}
 	log.Printf("websocket: connected to %v", urlStr)
-	netConn := websocket.NetConn(context.Background(), c, websocket.MessageBinary)
+	netConn := wsconn.NetConn(context.Background(), c, websocket.MessageBinary)
 	return netConn, nil
 }

go.mod

@@ -17,9 +17,11 @@ require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/creack/pty v1.1.17
 	github.com/dave/jennifer v1.4.1
+	github.com/dsnet/try v0.0.3
 	github.com/evanw/esbuild v0.14.53
 	github.com/frankban/quicktest v1.14.0
 	github.com/fxamacker/cbor/v2 v2.4.0
+	github.com/go-json-experiment/json v0.0.0-20221017203807-c5ed296b8c92
 	github.com/go-ole/go-ole v1.2.6
 	github.com/godbus/dbus/v5 v5.0.6
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
@@ -66,7 +68,7 @@ require (
 	golang.zx2c4.com/wireguard v0.0.0-20220904105730-b51010ba13f0
 	golang.zx2c4.com/wireguard/windows v0.5.3
 	gvisor.dev/gvisor v0.0.0-20220817001344-846276b3dbc5
-	honnef.co/go/tools v0.4.0-0.dev.0.20220404092545-59d7a2877f83
+	honnef.co/go/tools v0.4.0-0.dev.0.20220517111757-f4a2f64ce238
 	inet.af/peercred v0.0.0-20210906144145-0893ea02156a
 	inet.af/wf v0.0.0-20220728202103-50d96caab2f6
 	nhooyr.io/websocket v1.8.7

go.sum

@@ -261,6 +261,8 @@ github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405c
 github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
 github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
+github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
+github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
@@ -328,6 +330,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-json-experiment/json v0.0.0-20221017203807-c5ed296b8c92 h1:eoE7yxLELqDQVlHGoYYxXLFZqF8NcdOnrukTm4ObJaY=
+github.com/go-json-experiment/json v0.0.0-20221017203807-c5ed296b8c92/go.mod h1:I+I5/LT2lLP0eZsBNaVDrOrYASx9h7o7mRHmy+535/A=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -1829,8 +1833,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
 honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
-honnef.co/go/tools v0.4.0-0.dev.0.20220404092545-59d7a2877f83 h1:lZ9GIYaU+o5+X6ST702I/Ntyq9Y2oIMZ42rBQpem64A=
-honnef.co/go/tools v0.4.0-0.dev.0.20220404092545-59d7a2877f83/go.mod h1:vlRD9XErLMGT+mDuofSr0mMMquscM/1nQqtRSsh6m70=
+honnef.co/go/tools v0.4.0-0.dev.0.20220517111757-f4a2f64ce238 h1:8Vr1KP9OTjoKQSSeLefzibQgDV4s2ujJElKHqMi7nsA=
+honnef.co/go/tools v0.4.0-0.dev.0.20220517111757-f4a2f64ce238/go.mod h1:DCQzo6aCmhYDJH+We7BIU38vNvVkaOKa6s57pewKdvI=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=

ipn/ipnlocal/local.go

@@ -198,6 +198,14 @@ type LocalBackend struct {
 	// dialPlan is any dial plan that we've received from the control
 	// server during a previous connection; it is cleared on logout.
 	dialPlan atomic.Pointer[tailcfg.ControlDialPlan]
+
+	// tkaSyncLock is used to make tkaSyncIfNeeded an exclusive
+	// section. This is needed to stop two map-responses in quick succession
+	// from racing each other through TKA sync logic / RPCs.
+	//
+	// tkaSyncLock MUST be taken before mu (or inversely, mu must not be held
+	// at the moment that tkaSyncLock is taken).
+	tkaSyncLock sync.Mutex
 }
 
 // clientGen is a func that creates a control plane client.
@@ -355,6 +363,21 @@ func (b *LocalBackend) SetComponentDebugLogging(component string, until time.Tim
 	return nil
 }
 
+// GetComponentDebugLogging gets the time that component's debug logging is
+// enabled until, or the zero time if component's time is not currently
+// enabled.
+func (b *LocalBackend) GetComponentDebugLogging(component string) time.Time {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	now := time.Now()
+	ls := b.componentLogUntil[component]
+	if ls.until.IsZero() || ls.until.Before(now) {
+		return time.Time{}
+	}
+	return ls.until
+}
+
 // Dialer returns the backend's dialer.
 func (b *LocalBackend) Dialer() *tsdial.Dialer {
 	return b.dialer
@@ -775,9 +798,12 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		}
 	}
 	if st.NetMap != nil {
-		if err := b.tkaSyncIfNeededLocked(st.NetMap); err != nil {
+		b.mu.Unlock() // respect locking rules for tkaSyncIfNeeded
+		if err := b.tkaSyncIfNeeded(st.NetMap); err != nil {
 			b.logf("[v1] TKA sync error: %v", err)
 		}
+		b.mu.Lock()
+
 		if !envknob.TKASkipSignatureCheck() {
 			b.tkaFilterNetmapLocked(st.NetMap)
 		}
@@ -2321,7 +2347,7 @@ func (b *LocalBackend) doSetHostinfoFilterServices(hi *tailcfg.Hostinfo) {
 	}
 	peerAPIServices := b.peerAPIServicesLocked()
 	if b.egg {
-		peerAPIServices = append(peerAPIServices, tailcfg.Service{Proto: "egg"})
+		peerAPIServices = append(peerAPIServices, tailcfg.Service{Proto: "egg", Port: 1})
 	}
 	b.mu.Unlock()
 

ipn/ipnlocal/network-lock.go

@@ -39,6 +39,8 @@ type tkaState struct {
 
 // tkaFilterNetmapLocked checks the signatures on each node key, dropping
 // nodes from the netmap who's signature does not verify.
+//
+// b.mu must be held.
 func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 	if !envknob.UseWIPCode() {
 		return // Feature-flag till network-lock is in Alpha.
@@ -70,7 +72,7 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 	nm.Peers = peers
 }
 
-// tkaSyncIfNeededLocked examines TKA info reported from the control plane,
+// tkaSyncIfNeeded examines TKA info reported from the control plane,
 // performing the steps necessary to synchronize local tka state.
 //
 // There are 4 scenarios handled here:
@@ -85,13 +87,19 @@ func (b *LocalBackend) tkaFilterNetmapLocked(nm *netmap.NetworkMap) {
 //   - Everything up to date: All other cases.
 //     ∴ no action necessary.
 //
-// b.mu must be held. b.mu will be stepped out of (and back in) during network
-// RPCs.
-func (b *LocalBackend) tkaSyncIfNeededLocked(nm *netmap.NetworkMap) error {
+// tkaSyncIfNeeded immediately takes b.takeSyncLock which is held throughout,
+// and may take b.mu as required.
+func (b *LocalBackend) tkaSyncIfNeeded(nm *netmap.NetworkMap) error {
 	if !envknob.UseWIPCode() {
 		// If the feature flag is not enabled, pretend we don't exist.
 		return nil
 	}
+
+	b.tkaSyncLock.Lock() // take tkaSyncLock to make this function an exclusive section.
+	defer b.tkaSyncLock.Unlock()
+	b.mu.Lock() // take mu to protect access to synchronized fields.
+	defer b.mu.Unlock()
+
 	ourNodeKey := b.prefs.Persist.PrivateNodeKey.Public()
 
 	isEnabled := b.tka != nil
@@ -158,6 +166,8 @@ func toSyncOffer(head string, ancestors []string) (tka.SyncOffer, error) {
 // tkaSyncLocked synchronizes TKA state with control. b.mu must be held
 // and tka must be initialized. b.mu will be stepped out of (and back into)
 // during network RPCs.
+//
+// b.mu must be held.
 func (b *LocalBackend) tkaSyncLocked(ourNodeKey key.NodePublic) error {
 	offer, err := b.tka.authority.SyncOffer(b.tka.storage)
 	if err != nil {

ipn/ipnlocal/network-lock_test.go

@@ -127,12 +127,10 @@ func TestTKAEnablementFlow(t *testing.T) {
 		},
 	}
 
-	b.mu.Lock()
-	err = b.tkaSyncIfNeededLocked(&netmap.NetworkMap{
+	err = b.tkaSyncIfNeeded(&netmap.NetworkMap{
 		TKAEnabled: true,
 		TKAHead:    a1.Head(),
 	})
-	b.mu.Unlock()
 	if err != nil {
 		t.Errorf("tkaSyncIfNeededLocked() failed: %v", err)
 	}
@@ -228,12 +226,10 @@ func TestTKADisablementFlow(t *testing.T) {
 
 	// Test that the wrong disablement secret does not shut down the authority.
 	returnWrongSecret = true
-	b.mu.Lock()
-	err = b.tkaSyncIfNeededLocked(&netmap.NetworkMap{
+	err = b.tkaSyncIfNeeded(&netmap.NetworkMap{
 		TKAEnabled: false,
 		TKAHead:    authority.Head(),
 	})
-	b.mu.Unlock()
 	if err != nil {
 		t.Errorf("tkaSyncIfNeededLocked() failed: %v", err)
 	}
@@ -243,12 +239,10 @@ func TestTKADisablementFlow(t *testing.T) {
 
 	// Test the correct disablement secret shuts down the authority.
 	returnWrongSecret = false
-	b.mu.Lock()
-	err = b.tkaSyncIfNeededLocked(&netmap.NetworkMap{
+	err = b.tkaSyncIfNeeded(&netmap.NetworkMap{
 		TKAEnabled: false,
 		TKAHead:    authority.Head(),
 	})
-	b.mu.Unlock()
 	if err != nil {
 		t.Errorf("tkaSyncIfNeededLocked() failed: %v", err)
 	}
@@ -468,12 +462,10 @@ func TestTKASync(t *testing.T) {
 			}
 
 			// Finally, lets trigger a sync.
-			b.mu.Lock()
-			err = b.tkaSyncIfNeededLocked(&netmap.NetworkMap{
+			err = b.tkaSyncIfNeeded(&netmap.NetworkMap{
 				TKAEnabled: true,
 				TKAHead:    controlAuthority.Head(),
 			})
-			b.mu.Unlock()
 			if err != nil {
 				t.Errorf("tkaSyncIfNeededLocked() failed: %v", err)
 			}

ipn/localapi/localapi.go

@@ -232,12 +232,16 @@ func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logMarker := fmt.Sprintf("BUG-%v-%v-%v", h.backendLogID, time.Now().UTC().Format("20060102150405Z"), randHex(8))
-	if envknob.NoLogsNoSupport() {
-		logMarker = "BUG-NO-LOGS-NO-SUPPORT-this-node-has-had-its-logging-disabled"
+	logMarker := func() string {
+		return fmt.Sprintf("BUG-%v-%v-%v", h.backendLogID, time.Now().UTC().Format("20060102150405Z"), randHex(8))
 	}
-	h.logf("user bugreport: %s", logMarker)
-	if note := r.FormValue("note"); len(note) > 0 {
+	if envknob.NoLogsNoSupport() {
+		logMarker = func() string { return "BUG-NO-LOGS-NO-SUPPORT-this-node-has-had-its-logging-disabled" }
+	}
+
+	startMarker := logMarker()
+	h.logf("user bugreport: %s", startMarker)
+	if note := r.URL.Query().Get("note"); len(note) > 0 {
 		h.logf("user bugreport note: %s", note)
 	}
 	hi, _ := json.Marshal(hostinfo.New())
@@ -247,11 +251,62 @@ func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) {
 	} else {
 		h.logf("user bugreport health: ok")
 	}
-	if defBool(r.FormValue("diagnose"), false) {
+	if defBool(r.URL.Query().Get("diagnose"), false) {
 		h.b.Doctor(r.Context(), logger.WithPrefix(h.logf, "diag: "))
 	}
 	w.Header().Set("Content-Type", "text/plain")
-	fmt.Fprintln(w, logMarker)
+	fmt.Fprintln(w, startMarker)
+
+	// Nothing else to do if we're not in record mode; we wrote the marker
+	// above, so we can just finish our response now.
+	if !defBool(r.URL.Query().Get("record"), false) {
+		return
+	}
+
+	until := time.Now().Add(12 * time.Hour)
+
+	var changed map[string]bool
+	for _, component := range []string{"magicsock"} {
+		if h.b.GetComponentDebugLogging(component).IsZero() {
+			if err := h.b.SetComponentDebugLogging(component, until); err != nil {
+				h.logf("bugreport: error setting component %q logging: %v", component, err)
+				continue
+			}
+
+			mak.Set(&changed, component, true)
+		}
+	}
+	defer func() {
+		for component := range changed {
+			h.b.SetComponentDebugLogging(component, time.Time{})
+		}
+	}()
+
+	// NOTE(andrew): if we have anything else we want to do while recording
+	// a bugreport, we can add it here.
+
+	// Read from the client; this will also return when the client closes
+	// the connection.
+	var buf [1]byte
+	_, err := r.Body.Read(buf[:])
+
+	switch {
+	case err == nil:
+		// good
+	case errors.Is(err, io.EOF):
+		// good
+	case errors.Is(err, io.ErrUnexpectedEOF):
+		// this happens when Ctrl-C'ing the tailscale client; don't
+		// bother logging an error
+	default:
+		// Log but continue anyway.
+		h.logf("user bugreport: error reading body: %v", err)
+	}
+
+	// Generate another log marker and return it to the client.
+	endMarker := logMarker()
+	h.logf("user bugreport end: %s", endMarker)
+	fmt.Fprintln(w, endMarker)
 }
 
 func (h *Handler) serveWhoIs(w http.ResponseWriter, r *http.Request) {

kube/client.go

@@ -100,7 +100,9 @@ func (c *Client) secretURL(name string) string {
 }
 
 func getError(resp *http.Response) error {
-	if resp.StatusCode == 200 {
+	if resp.StatusCode == 200 || resp.StatusCode == 201 {
+		// These are the only success codes returned by the Kubernetes API.
+		// https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#http-status-codes
 		return nil
 	}
 	st := &Status{}

net/dnscache/dnscache.go

@@ -276,6 +276,11 @@ func (r *Resolver) lookupIP(host string) (ip, ip6 netip.Addr, allIPs []netip.Add
 		return netip.Addr{}, netip.Addr{}, nil, fmt.Errorf("no IPs for %q found", host)
 	}
 
+        // Unmap everything; LookupNetIP can return mapped addresses (see #5698)
+	for i := range ips {
+		ips[i] = ips[i].Unmap()
+	}
+
 	have4 := false
 	for _, ipa := range ips {
 		if ipa.Is4() {

net/interfaces/interfaces_bsd.go

@@ -2,11 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// This might work on other BSDs, but only tested on FreeBSD.
-// Originally a fork of interfaces_darwin.go with slightly different flags.
+// Common code for FreeBSD and Darwin. This might also work on other
+// BSD systems (e.g. OpenBSD) but has not been tested.
 
-//go:build freebsd
-// +build freebsd
+//go:build darwin || freebsd
+// +build darwin freebsd
 
 package interfaces
 
@@ -37,11 +37,6 @@ func defaultRoute() (d DefaultRouteDetails, err error) {
 	return d, nil
 }
 
-// fetchRoutingTable calls route.FetchRIB, fetching NET_RT_DUMP.
-func fetchRoutingTable() (rib []byte, err error) {
-	return route.FetchRIB(syscall.AF_UNSPEC, unix.NET_RT_DUMP, 0)
-}
-
 func DefaultRouteInterfaceIndex() (int, error) {
 	// $ netstat -nr
 	// Routing tables
@@ -61,35 +56,20 @@ func DefaultRouteInterfaceIndex() (int, error) {
 	if err != nil {
 		return 0, fmt.Errorf("route.FetchRIB: %w", err)
 	}
-	msgs, err := route.ParseRIB(unix.NET_RT_IFLIST, rib)
+	msgs, err := parseRoutingTable(rib)
 	if err != nil {
 		return 0, fmt.Errorf("route.ParseRIB: %w", err)
 	}
-	indexSeen := map[int]int{} // index => count
 	for _, m := range msgs {
 		rm, ok := m.(*route.RouteMessage)
 		if !ok {
 			continue
 		}
-		const RTF_GATEWAY = 0x2
-		const RTF_IFSCOPE = 0x1000000
-		if rm.Flags&RTF_GATEWAY == 0 {
-			continue
-		}
-		if rm.Flags&RTF_IFSCOPE != 0 {
-			continue
-		}
-		indexSeen[rm.Index]++
-	}
-	if len(indexSeen) == 0 {
-		return 0, errors.New("no gateway index found")
-	}
-	if len(indexSeen) == 1 {
-		for idx := range indexSeen {
-			return idx, nil
+		if isDefaultGateway(rm) {
+			return rm.Index, nil
 		}
 	}
-	return 0, fmt.Errorf("ambiguous gateway interfaces found: %v", indexSeen)
+	return 0, errors.New("no gateway index found")
 }
 
 func init() {
@@ -102,7 +82,7 @@ func likelyHomeRouterIPBSDFetchRIB() (ret netip.Addr, ok bool) {
 		log.Printf("routerIP/FetchRIB: %v", err)
 		return ret, false
 	}
-	msgs, err := route.ParseRIB(unix.NET_RT_IFLIST, rib)
+	msgs, err := parseRoutingTable(rib)
 	if err != nil {
 		log.Printf("routerIP/ParseRIB: %v", err)
 		return ret, false
@@ -112,26 +92,59 @@ func likelyHomeRouterIPBSDFetchRIB() (ret netip.Addr, ok bool) {
 		if !ok {
 			continue
 		}
-		const RTF_IFSCOPE = 0x1000000
-		if rm.Flags&unix.RTF_GATEWAY == 0 {
+		if !isDefaultGateway(rm) {
 			continue
 		}
-		if rm.Flags&RTF_IFSCOPE != 0 {
+
+		gw, ok := rm.Addrs[unix.RTAX_GATEWAY].(*route.Inet4Addr)
+		if !ok {
 			continue
 		}
-		if len(rm.Addrs) > unix.RTAX_GATEWAY {
-			dst4, ok := rm.Addrs[unix.RTAX_DST].(*route.Inet4Addr)
-			if !ok || dst4.IP != ([4]byte{0, 0, 0, 0}) {
-				// Expect 0.0.0.0 as DST field.
-				continue
-			}
-			gw, ok := rm.Addrs[unix.RTAX_GATEWAY].(*route.Inet4Addr)
-			if !ok {
-				continue
-			}
-			return netaddr.IPv4(gw.IP[0], gw.IP[1], gw.IP[2], gw.IP[3]), true
-		}
+		return netaddr.IPv4(gw.IP[0], gw.IP[1], gw.IP[2], gw.IP[3]), true
 	}
 
 	return ret, false
 }
+
+var v4default = [4]byte{0, 0, 0, 0}
+var v6default = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+func isDefaultGateway(rm *route.RouteMessage) bool {
+	if rm.Flags&unix.RTF_GATEWAY == 0 {
+		return false
+	}
+	// Defined locally because FreeBSD does not have unix.RTF_IFSCOPE.
+	const RTF_IFSCOPE = 0x1000000
+	if rm.Flags&RTF_IFSCOPE != 0 {
+		return false
+	}
+
+	// Addrs is [RTAX_DST, RTAX_GATEWAY, RTAX_NETMASK, ...]
+	if len(rm.Addrs) <= unix.RTAX_NETMASK {
+		return false
+	}
+
+	dst := rm.Addrs[unix.RTAX_DST]
+	netmask := rm.Addrs[unix.RTAX_NETMASK]
+	if dst == nil || netmask == nil {
+		return false
+	}
+
+	if dst.Family() == syscall.AF_INET && netmask.Family() == syscall.AF_INET {
+		dstAddr, dstOk := dst.(*route.Inet4Addr)
+		nmAddr, nmOk := netmask.(*route.Inet4Addr)
+		if dstOk && nmOk && dstAddr.IP == v4default && nmAddr.IP == v4default {
+			return true
+		}
+	}
+
+	if dst.Family() == syscall.AF_INET6 && netmask.Family() == syscall.AF_INET6 {
+		dstAddr, dstOk := dst.(*route.Inet6Addr)
+		nmAddr, nmOk := netmask.(*route.Inet6Addr)
+		if dstOk && nmOk && dstAddr.IP == v6default && nmAddr.IP == v6default {
+			return true
+		}
+	}
+
+	return false
+}

net/interfaces/interfaces_darwin.go

@@ -5,128 +5,16 @@
 package interfaces
 
 import (
-	"errors"
-	"fmt"
-	"log"
-	"net"
-	"net/netip"
 	"syscall"
 
 	"golang.org/x/net/route"
-	"golang.org/x/sys/unix"
-	"tailscale.com/net/netaddr"
 )
 
-func defaultRoute() (d DefaultRouteDetails, err error) {
-	idx, err := DefaultRouteInterfaceIndex()
-	if err != nil {
-		return d, err
-	}
-	iface, err := net.InterfaceByIndex(idx)
-	if err != nil {
-		return d, err
-	}
-	d.InterfaceName = iface.Name
-	d.InterfaceIndex = idx
-	return d, nil
-}
-
 // fetchRoutingTable calls route.FetchRIB, fetching NET_RT_DUMP2.
 func fetchRoutingTable() (rib []byte, err error) {
 	return route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_DUMP2, 0)
 }
 
-func DefaultRouteInterfaceIndex() (int, error) {
-	// $ netstat -nr
-	// Routing tables
-	// Internet:
-	// Destination        Gateway            Flags        Netif Expire
-	// default            10.0.0.1           UGSc           en0         <-- want this one
-	// default            10.0.0.1           UGScI          en1
-
-	// From man netstat:
-	// U       RTF_UP           Route usable
-	// G       RTF_GATEWAY      Destination requires forwarding by intermediary
-	// S       RTF_STATIC       Manually added
-	// c       RTF_PRCLONING    Protocol-specified generate new routes on use
-	// I       RTF_IFSCOPE      Route is associated with an interface scope
-
-	rib, err := fetchRoutingTable()
-	if err != nil {
-		return 0, fmt.Errorf("route.FetchRIB: %w", err)
-	}
-	msgs, err := route.ParseRIB(syscall.NET_RT_IFLIST2, rib)
-	if err != nil {
-		return 0, fmt.Errorf("route.ParseRIB: %w", err)
-	}
-	indexSeen := map[int]int{} // index => count
-	for _, m := range msgs {
-		rm, ok := m.(*route.RouteMessage)
-		if !ok {
-			continue
-		}
-		const RTF_GATEWAY = 0x2
-		const RTF_IFSCOPE = 0x1000000
-		if rm.Flags&RTF_GATEWAY == 0 {
-			continue
-		}
-		if rm.Flags&RTF_IFSCOPE != 0 {
-			continue
-		}
-		indexSeen[rm.Index]++
-	}
-	if len(indexSeen) == 0 {
-		return 0, errors.New("no gateway index found")
-	}
-	if len(indexSeen) == 1 {
-		for idx := range indexSeen {
-			return idx, nil
-		}
-	}
-	return 0, fmt.Errorf("ambiguous gateway interfaces found: %v", indexSeen)
-}
-
-func init() {
-	likelyHomeRouterIP = likelyHomeRouterIPDarwinFetchRIB
-}
-
-func likelyHomeRouterIPDarwinFetchRIB() (ret netip.Addr, ok bool) {
-	rib, err := fetchRoutingTable()
-	if err != nil {
-		log.Printf("routerIP/FetchRIB: %v", err)
-		return ret, false
-	}
-	msgs, err := route.ParseRIB(syscall.NET_RT_IFLIST2, rib)
-	if err != nil {
-		log.Printf("routerIP/ParseRIB: %v", err)
-		return ret, false
-	}
-	for _, m := range msgs {
-		rm, ok := m.(*route.RouteMessage)
-		if !ok {
-			continue
-		}
-		const RTF_GATEWAY = 0x2
-		const RTF_IFSCOPE = 0x1000000
-		if rm.Flags&RTF_GATEWAY == 0 {
-			continue
-		}
-		if rm.Flags&RTF_IFSCOPE != 0 {
-			continue
-		}
-		if len(rm.Addrs) > unix.RTAX_GATEWAY {
-			dst4, ok := rm.Addrs[unix.RTAX_DST].(*route.Inet4Addr)
-			if !ok || dst4.IP != ([4]byte{0, 0, 0, 0}) {
-				// Expect 0.0.0.0 as DST field.
-				continue
-			}
-			gw, ok := rm.Addrs[unix.RTAX_GATEWAY].(*route.Inet4Addr)
-			if !ok {
-				continue
-			}
-			return netaddr.IPv4(gw.IP[0], gw.IP[1], gw.IP[2], gw.IP[3]), true
-		}
-	}
-
-	return ret, false
+func parseRoutingTable(rib []byte) ([]route.Message, error) {
+	return route.ParseRIB(syscall.NET_RT_IFLIST2, rib)
 }

net/interfaces/interfaces_darwin_test.go

@@ -16,18 +16,32 @@ import (
 )
 
 func TestLikelyHomeRouterIPSyscallExec(t *testing.T) {
-	syscallIP, syscallOK := likelyHomeRouterIPDarwinFetchRIB()
-	netstatIP, netstatOK := likelyHomeRouterIPDarwinExec()
+	syscallIP, syscallOK := likelyHomeRouterIPBSDFetchRIB()
+	netstatIP, netstatIf, netstatOK := likelyHomeRouterIPDarwinExec()
+
 	if syscallOK != netstatOK || syscallIP != netstatIP {
 		t.Errorf("syscall() = %v, %v, netstat = %v, %v",
 			syscallIP, syscallOK,
 			netstatIP, netstatOK,
 		)
 	}
+
+	if !syscallOK {
+		return
+	}
+
+	def, err := defaultRoute()
+	if err != nil {
+		t.Errorf("defaultRoute() error: %v", err)
+	}
+
+	if def.InterfaceName != netstatIf {
+		t.Errorf("syscall default route interface %s differs from netstat %s", def.InterfaceName, netstatIf)
+	}
 }
 
 /*
-Parse out 10.0.0.1 from:
+Parse out 10.0.0.1 and en0 from:
 
 $ netstat -r -n -f inet
 Routing tables
@@ -40,12 +54,12 @@ default            link#14            UCSI         utun2
 10.0.0.1/32        link#4             UCS            en0      !
 ...
 */
-func likelyHomeRouterIPDarwinExec() (ret netip.Addr, ok bool) {
+func likelyHomeRouterIPDarwinExec() (ret netip.Addr, netif string, ok bool) {
 	if version.IsMobile() {
 		// Don't try to do subprocesses on iOS. Ends up with log spam like:
 		// kernel: "Sandbox: IPNExtension(86580) deny(1) process-fork"
 		// This is why we have likelyHomeRouterIPDarwinSyscall.
-		return ret, false
+		return ret, "", false
 	}
 	cmd := exec.Command("/usr/sbin/netstat", "-r", "-n", "-f", "inet")
 	stdout, err := cmd.StdoutPipe()
@@ -64,22 +78,26 @@ func likelyHomeRouterIPDarwinExec() (ret netip.Addr, ok bool) {
 			return nil
 		}
 		f = mem.AppendFields(f[:0], line)
-		if len(f) < 3 || !f[0].EqualString("default") {
+		if len(f) < 4 || !f[0].EqualString("default") {
 			return nil
 		}
-		ipm, flagsm := f[1], f[2]
+		ipm, flagsm, netifm := f[1], f[2], f[3]
 		if !mem.Contains(flagsm, mem.S("G")) {
 			return nil
 		}
+		if mem.Contains(flagsm, mem.S("I")) {
+			return nil
+		}
 		ip, err := netip.ParseAddr(string(mem.Append(nil, ipm)))
 		if err == nil && ip.IsPrivate() {
 			ret = ip
+			netif = netifm.StringCopy()
 			// We've found what we're looking for.
 			return errStopReadingNetstatTable
 		}
 		return nil
 	})
-	return ret, ret.IsValid()
+	return ret, netif, ret.IsValid()
 }
 
 func TestFetchRoutingTable(t *testing.T) {

net/interfaces/interfaces_freebsd.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This might work on other BSDs, but only tested on FreeBSD.
+
+//go:build freebsd
+// +build freebsd
+
+package interfaces
+
+import (
+	"syscall"
+
+	"golang.org/x/net/route"
+	"golang.org/x/sys/unix"
+)
+
+// fetchRoutingTable calls route.FetchRIB, fetching NET_RT_DUMP.
+func fetchRoutingTable() (rib []byte, err error) {
+	return route.FetchRIB(syscall.AF_UNSPEC, unix.NET_RT_DUMP, 0)
+}
+
+func parseRoutingTable(rib []byte) ([]route.Message, error) {
+	return route.ParseRIB(syscall.NET_RT_IFLIST, rib)
+}

net/netcheck/netcheck.go

@@ -1105,6 +1105,9 @@ func (c *Client) checkCaptivePortal(ctx context.Context, dm *tailcfg.DERPMap, pr
 			}
 			rids = append(rids, id)
 		}
+		if len(rids) == 0 {
+			return false, nil
+		}
 		preferredDERP = rids[rand.Intn(len(rids))]
 	}
 
@@ -1113,13 +1116,20 @@ func (c *Client) checkCaptivePortal(ctx context.Context, dm *tailcfg.DERPMap, pr
 	if err != nil {
 		return false, err
 	}
+
+	chal := "tailscale " + node.HostName
+	req.Header.Set("X-Tailscale-Challenge", chal)
 	r, err := noRedirectClient.Do(req)
 	if err != nil {
 		return false, err
 	}
-	c.logf("[v2] checkCaptivePortal url=%q status_code=%d", req.URL.String(), r.StatusCode)
+	defer r.Body.Close()
 
-	return r.StatusCode != 204, nil
+	expectedResponse := "response " + chal
+	validResponse := r.Header.Get("X-Tailscale-Response") == expectedResponse
+
+	c.logf("[v2] checkCaptivePortal url=%q status_code=%d valid_response=%v", req.URL.String(), r.StatusCode, validResponse)
+	return r.StatusCode != 204 || !validResponse, nil
 }
 
 // runHTTPOnlyChecks is the netcheck done by environments that can

net/tstun/wrap.go

@@ -22,7 +22,6 @@ import (
 	"golang.zx2c4.com/wireguard/tun"
 	"gvisor.dev/gvisor/pkg/tcpip/stack"
 	"tailscale.com/disco"
-	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/packet"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tunstats"
@@ -31,6 +30,7 @@ import (
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/netlogtype"
 	"tailscale.com/util/clientmetric"
 	"tailscale.com/wgengine/filter"
 )
@@ -844,14 +844,16 @@ func (t *Wrapper) Unwrap() tun.Device {
 }
 
 // SetStatisticsEnabled enables per-connections packet counters.
-// ExtractStatistics must be called periodically to avoid unbounded memory use.
+// Disabling statistics gathering does not reset the counters.
+// ExtractStatistics must be called to reset the counters and
+// be periodically called while enabled to avoid unbounded memory use.
 func (t *Wrapper) SetStatisticsEnabled(enable bool) {
 	t.stats.enabled.Store(enable)
 }
 
 // ExtractStatistics extracts and resets the counters for all active connections.
 // It must be called periodically otherwise the memory used is unbounded.
-func (t *Wrapper) ExtractStatistics() map[flowtrack.Tuple]tunstats.Counts {
+func (t *Wrapper) ExtractStatistics() map[netlogtype.Connection]netlogtype.Counts {
 	return t.stats.Extract()
 }
 

net/tstun/wrap_test.go

@@ -19,15 +19,14 @@ import (
 	"go4.org/netipx"
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"tailscale.com/disco"
-	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/netaddr"
 	"tailscale.com/net/packet"
-	"tailscale.com/net/tunstats"
 	"tailscale.com/tstest"
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/netlogtype"
 	"tailscale.com/wgengine/filter"
 )
 
@@ -379,17 +378,17 @@ func TestFilter(t *testing.T) {
 			}
 
 			got := tun.ExtractStatistics()
-			want := map[flowtrack.Tuple]tunstats.Counts{}
+			want := map[netlogtype.Connection]netlogtype.Counts{}
 			if !tt.drop {
 				var p packet.Parsed
 				p.Decode(tt.data)
 				switch tt.dir {
 				case in:
-					tuple := flowtrack.Tuple{Proto: ipproto.UDP, Src: p.Dst, Dst: p.Src}
-					want[tuple] = tunstats.Counts{RxPackets: 1, RxBytes: uint64(len(tt.data))}
+					conn := netlogtype.Connection{Proto: ipproto.UDP, Src: p.Dst, Dst: p.Src}
+					want[conn] = netlogtype.Counts{RxPackets: 1, RxBytes: uint64(len(tt.data))}
 				case out:
-					tuple := flowtrack.Tuple{Proto: ipproto.UDP, Src: p.Src, Dst: p.Dst}
-					want[tuple] = tunstats.Counts{TxPackets: 1, TxBytes: uint64(len(tt.data))}
+					conn := netlogtype.Connection{Proto: ipproto.UDP, Src: p.Src, Dst: p.Dst}
+					want[conn] = netlogtype.Counts{TxPackets: 1, TxBytes: uint64(len(tt.data))}
 				}
 			}
 			if !reflect.DeepEqual(got, want) {

net/tunstats/stats.go

@@ -9,8 +9,8 @@ package tunstats
 import (
 	"sync"
 
-	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/packet"
+	"tailscale.com/types/netlogtype"
 )
 
 // Statistics maintains counters for every connection.
@@ -18,36 +18,19 @@ import (
 // The zero value is ready for use.
 type Statistics struct {
 	mu sync.Mutex
-	m  map[flowtrack.Tuple]Counts
-}
-
-// Counts are statistics about a particular connection.
-type Counts struct {
-	TxPackets uint64 `json:"txPkts,omitempty"`
-	TxBytes   uint64 `json:"txBytes,omitempty"`
-	RxPackets uint64 `json:"rxPkts,omitempty"`
-	RxBytes   uint64 `json:"rxBytes,omitempty"`
-}
-
-// Add adds the counts from both c1 and c2.
-func (c1 Counts) Add(c2 Counts) Counts {
-	c1.TxPackets += c2.TxPackets
-	c1.TxBytes += c2.TxBytes
-	c1.RxPackets += c2.RxPackets
-	c1.RxBytes += c2.RxBytes
-	return c1
+	m  map[netlogtype.Connection]netlogtype.Counts
 }
 
 // UpdateTx updates the counters for a transmitted IP packet
 // The source and destination of the packet directly correspond with
-// the source and destination in flowtrack.Tuple.
+// the source and destination in netlogtype.Connection.
 func (s *Statistics) UpdateTx(b []byte) {
 	s.update(b, false)
 }
 
 // UpdateRx updates the counters for a received IP packet.
 // The source and destination of the packet are inverted with respect to
-// the source and destination in flowtrack.Tuple.
+// the source and destination in netlogtype.Connection.
 func (s *Statistics) UpdateRx(b []byte) {
 	s.update(b, true)
 }
@@ -55,17 +38,17 @@ func (s *Statistics) UpdateRx(b []byte) {
 func (s *Statistics) update(b []byte, receive bool) {
 	var p packet.Parsed
 	p.Decode(b)
-	tuple := flowtrack.Tuple{Proto: p.IPProto, Src: p.Src, Dst: p.Dst}
+	conn := netlogtype.Connection{Proto: p.IPProto, Src: p.Src, Dst: p.Dst}
 	if receive {
-		tuple.Src, tuple.Dst = tuple.Dst, tuple.Src
+		conn.Src, conn.Dst = conn.Dst, conn.Src
 	}
 
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.m == nil {
-		s.m = make(map[flowtrack.Tuple]Counts)
+		s.m = make(map[netlogtype.Connection]netlogtype.Counts)
 	}
-	cnts := s.m[tuple]
+	cnts := s.m[conn]
 	if receive {
 		cnts.RxPackets++
 		cnts.RxBytes += uint64(len(b))
@@ -73,15 +56,15 @@ func (s *Statistics) update(b []byte, receive bool) {
 		cnts.TxPackets++
 		cnts.TxBytes += uint64(len(b))
 	}
-	s.m[tuple] = cnts
+	s.m[conn] = cnts
 }
 
 // Extract extracts and resets the counters for all active connections.
 // It must be called periodically otherwise the memory used is unbounded.
-func (s *Statistics) Extract() map[flowtrack.Tuple]Counts {
+func (s *Statistics) Extract() map[netlogtype.Connection]netlogtype.Counts {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	m := s.m
-	s.m = make(map[flowtrack.Tuple]Counts)
+	s.m = make(map[netlogtype.Connection]netlogtype.Counts)
 	return m
 }

net/tunstats/stats_test.go

@@ -15,8 +15,8 @@ import (
 	"time"
 
 	qt "github.com/frankban/quicktest"
-	"tailscale.com/net/flowtrack"
 	"tailscale.com/types/ipproto"
+	"tailscale.com/types/netlogtype"
 )
 
 func testPacketV4(proto ipproto.Proto, srcAddr, dstAddr [4]byte, srcPort, dstPort, size uint16) (out []byte) {
@@ -48,17 +48,17 @@ func TestConcurrent(t *testing.T) {
 	c := qt.New(t)
 
 	var stats Statistics
-	var wants []map[flowtrack.Tuple]Counts
-	gots := make([]map[flowtrack.Tuple]Counts, runtime.NumCPU())
+	var wants []map[netlogtype.Connection]netlogtype.Counts
+	gots := make([]map[netlogtype.Connection]netlogtype.Counts, runtime.NumCPU())
 	var group sync.WaitGroup
 	for i := range gots {
 		group.Add(1)
 		go func(i int) {
 			defer group.Done()
-			gots[i] = make(map[flowtrack.Tuple]Counts)
+			gots[i] = make(map[netlogtype.Connection]netlogtype.Counts)
 			rn := rand.New(rand.NewSource(time.Now().UnixNano()))
 			var p []byte
-			var t flowtrack.Tuple
+			var t netlogtype.Connection
 			for j := 0; j < 1000; j++ {
 				delay := rn.Intn(10000)
 				if p == nil || rn.Intn(64) == 0 {
@@ -72,7 +72,7 @@ func TestConcurrent(t *testing.T) {
 					dstPort := uint16(rand.Intn(16))
 					size := uint16(64 + rand.Intn(1024))
 					p = testPacketV4(proto, srcAddr.As4(), dstAddr.As4(), srcPort, dstPort, size)
-					t = flowtrack.Tuple{Proto: proto, Src: netip.AddrPortFrom(srcAddr, srcPort), Dst: netip.AddrPortFrom(dstAddr, dstPort)}
+					t = netlogtype.Connection{Proto: proto, Src: netip.AddrPortFrom(srcAddr, srcPort), Dst: netip.AddrPortFrom(dstAddr, dstPort)}
 				}
 				t2 := t
 				receive := rn.Intn(2) == 0
@@ -102,17 +102,17 @@ func TestConcurrent(t *testing.T) {
 	group.Wait()
 	wants = append(wants, stats.Extract())
 
-	got := make(map[flowtrack.Tuple]Counts)
-	want := make(map[flowtrack.Tuple]Counts)
+	got := make(map[netlogtype.Connection]netlogtype.Counts)
+	want := make(map[netlogtype.Connection]netlogtype.Counts)
 	mergeMaps(got, gots...)
 	mergeMaps(want, wants...)
 	c.Assert(got, qt.DeepEquals, want)
 }
 
-func mergeMaps(dst map[flowtrack.Tuple]Counts, srcs ...map[flowtrack.Tuple]Counts) {
+func mergeMaps(dst map[netlogtype.Connection]netlogtype.Counts, srcs ...map[netlogtype.Connection]netlogtype.Counts) {
 	for _, src := range srcs {
-		for tuple, cnts := range src {
-			dst[tuple] = dst[tuple].Add(cnts)
+		for conn, cnts := range src {
+			dst[conn] = dst[conn].Add(cnts)
 		}
 	}
 }

net/wsconn/wsconn.go

@@ -0,0 +1,213 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package wsconn contains an adapter type that turns
+// a websocket connection into a net.Conn. It a temporary fork of the
+// netconn.go file from the nhooyr.io/websocket package while we wait for
+// https://github.com/nhooyr/websocket/pull/350 to be merged.
+package wsconn
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"nhooyr.io/websocket"
+)
+
+// NetConn converts a *websocket.Conn into a net.Conn.
+//
+// It's for tunneling arbitrary protocols over WebSockets.
+// Few users of the library will need this but it's tricky to implement
+// correctly and so provided in the library.
+// See https://github.com/nhooyr/websocket/issues/100.
+//
+// Every Write to the net.Conn will correspond to a message write of
+// the given type on *websocket.Conn.
+//
+// The passed ctx bounds the lifetime of the net.Conn. If cancelled,
+// all reads and writes on the net.Conn will be cancelled.
+//
+// If a message is read that is not of the correct type, the connection
+// will be closed with StatusUnsupportedData and an error will be returned.
+//
+// Close will close the *websocket.Conn with StatusNormalClosure.
+//
+// When a deadline is hit, the connection will be closed. This is
+// different from most net.Conn implementations where only the
+// reading/writing goroutines are interrupted but the connection is kept alive.
+//
+// The Addr methods will return a mock net.Addr that returns "websocket" for Network
+// and "websocket/unknown-addr" for String.
+//
+// A received StatusNormalClosure or StatusGoingAway close frame will be translated to
+// io.EOF when reading.
+func NetConn(ctx context.Context, c *websocket.Conn, msgType websocket.MessageType) net.Conn {
+	nc := &netConn{
+		c:       c,
+		msgType: msgType,
+	}
+
+	var writeCancel context.CancelFunc
+	nc.writeContext, writeCancel = context.WithCancel(ctx)
+	nc.writeTimer = time.AfterFunc(math.MaxInt64, func() {
+		nc.afterWriteDeadline.Store(true)
+		if nc.writing.Load() {
+			writeCancel()
+		}
+	})
+	if !nc.writeTimer.Stop() {
+		<-nc.writeTimer.C
+	}
+
+	var readCancel context.CancelFunc
+	nc.readContext, readCancel = context.WithCancel(ctx)
+	nc.readTimer = time.AfterFunc(math.MaxInt64, func() {
+		nc.afterReadDeadline.Store(true)
+		if nc.reading.Load() {
+			readCancel()
+		}
+	})
+	if !nc.readTimer.Stop() {
+		<-nc.readTimer.C
+	}
+
+	return nc
+}
+
+type netConn struct {
+	c       *websocket.Conn
+	msgType websocket.MessageType
+
+	writeTimer         *time.Timer
+	writeContext       context.Context
+	writing            atomic.Bool
+	afterWriteDeadline atomic.Bool
+
+	readTimer         *time.Timer
+	readContext       context.Context
+	reading           atomic.Bool
+	afterReadDeadline atomic.Bool
+
+	readMu sync.Mutex
+	eofed  bool
+	reader io.Reader
+}
+
+var _ net.Conn = &netConn{}
+
+func (c *netConn) Close() error {
+	return c.c.Close(websocket.StatusNormalClosure, "")
+}
+
+func (c *netConn) Write(p []byte) (int, error) {
+	if c.afterWriteDeadline.Load() {
+		return 0, os.ErrDeadlineExceeded
+	}
+
+	if swapped := c.writing.CompareAndSwap(false, true); !swapped {
+		panic("Concurrent writes not allowed")
+	}
+	defer c.writing.Store(false)
+
+	err := c.c.Write(c.writeContext, c.msgType, p)
+	if err != nil {
+		return 0, err
+	}
+
+	return len(p), nil
+}
+
+func (c *netConn) Read(p []byte) (int, error) {
+	if c.afterReadDeadline.Load() {
+		return 0, os.ErrDeadlineExceeded
+	}
+
+	c.readMu.Lock()
+	defer c.readMu.Unlock()
+	if swapped := c.reading.CompareAndSwap(false, true); !swapped {
+		panic("Concurrent reads not allowed")
+	}
+	defer c.reading.Store(false)
+
+	if c.eofed {
+		return 0, io.EOF
+	}
+
+	if c.reader == nil {
+		typ, r, err := c.c.Reader(c.readContext)
+		if err != nil {
+			switch websocket.CloseStatus(err) {
+			case websocket.StatusNormalClosure, websocket.StatusGoingAway:
+				c.eofed = true
+				return 0, io.EOF
+			}
+			return 0, err
+		}
+		if typ != c.msgType {
+			err := fmt.Errorf("unexpected frame type read (expected %v): %v", c.msgType, typ)
+			c.c.Close(websocket.StatusUnsupportedData, err.Error())
+			return 0, err
+		}
+		c.reader = r
+	}
+
+	n, err := c.reader.Read(p)
+	if err == io.EOF {
+		c.reader = nil
+		err = nil
+	}
+	return n, err
+}
+
+type websocketAddr struct {
+}
+
+func (a websocketAddr) Network() string {
+	return "websocket"
+}
+
+func (a websocketAddr) String() string {
+	return "websocket/unknown-addr"
+}
+
+func (c *netConn) RemoteAddr() net.Addr {
+	return websocketAddr{}
+}
+
+func (c *netConn) LocalAddr() net.Addr {
+	return websocketAddr{}
+}
+
+func (c *netConn) SetDeadline(t time.Time) error {
+	c.SetWriteDeadline(t)
+	c.SetReadDeadline(t)
+	return nil
+}
+
+func (c *netConn) SetWriteDeadline(t time.Time) error {
+	if t.IsZero() {
+		c.writeTimer.Stop()
+	} else {
+		c.writeTimer.Reset(time.Until(t))
+	}
+	c.afterWriteDeadline.Store(false)
+	return nil
+}
+
+func (c *netConn) SetReadDeadline(t time.Time) error {
+	if t.IsZero() {
+		c.readTimer.Stop()
+	} else {
+		c.readTimer.Reset(time.Until(t))
+	}
+	c.afterReadDeadline.Store(false)
+	return nil
+}

syncs/syncs.go

@@ -7,7 +7,10 @@ package syncs
 
 import (
 	"context"
+	"sync"
 	"sync/atomic"
+
+	"tailscale.com/util/mak"
 )
 
 // ClosedChan returns a channel that's already closed.
@@ -152,3 +155,66 @@ func (s Semaphore) TryAcquire() bool {
 func (s Semaphore) Release() {
 	<-s.c
 }
+
+// Map is a Go map protected by a [sync.RWMutex].
+// It is preferred over [sync.Map] for maps with entries that change
+// at a relatively high frequency.
+// This must not be shallow copied.
+type Map[K comparable, V any] struct {
+	mu sync.RWMutex
+	m  map[K]V
+}
+
+func (m *Map[K, V]) Load(key K) (value V, ok bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	value, ok = m.m[key]
+	return value, ok
+}
+
+func (m *Map[K, V]) Store(key K, value V) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	mak.Set(&m.m, key, value)
+}
+
+func (m *Map[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) {
+	if actual, loaded = m.Load(key); loaded {
+		return actual, loaded
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	actual, loaded = m.m[key]
+	if !loaded {
+		actual = value
+		mak.Set(&m.m, key, value)
+	}
+	return actual, loaded
+}
+
+func (m *Map[K, V]) LoadAndDelete(key K) (value V, loaded bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	value, loaded = m.m[key]
+	if loaded {
+		delete(m.m, key)
+	}
+	return value, loaded
+}
+
+func (m *Map[K, V]) Delete(key K) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	delete(m.m, key)
+}
+
+func (m *Map[K, V]) Range(f func(key K, value V) bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	for k, v := range m.m {
+		if !f(k, v) {
+			return
+		}
+	}
+}

syncs/syncs_test.go

@@ -6,7 +6,10 @@ package syncs
 
 import (
 	"context"
+	"sync"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestWaitGroupChan(t *testing.T) {
@@ -73,3 +76,66 @@ func TestSemaphore(t *testing.T) {
 	s.Release()
 	s.Release()
 }
+
+func TestMap(t *testing.T) {
+	var m Map[string, int]
+	if v, ok := m.Load("noexist"); v != 0 || ok {
+		t.Errorf(`Load("noexist") = (%v, %v), want (0, false)`, v, ok)
+	}
+	m.Store("one", 1)
+	if v, ok := m.LoadOrStore("one", -1); v != 1 || !ok {
+		t.Errorf(`LoadOrStore("one", 1) = (%v, %v), want (1, true)`, v, ok)
+	}
+	if v, ok := m.Load("one"); v != 1 || !ok {
+		t.Errorf(`Load("one") = (%v, %v), want (1, true)`, v, ok)
+	}
+	if v, ok := m.LoadOrStore("two", 2); v != 2 || ok {
+		t.Errorf(`LoadOrStore("two", 2) = (%v, %v), want (2, false)`, v, ok)
+	}
+	got := map[string]int{}
+	want := map[string]int{"one": 1, "two": 2}
+	m.Range(func(k string, v int) bool {
+		got[k] = v
+		return true
+	})
+	if d := cmp.Diff(got, want); d != "" {
+		t.Errorf("Range mismatch (-got +want):\n%s", d)
+	}
+	if v, ok := m.LoadAndDelete("two"); v != 2 || !ok {
+		t.Errorf(`LoadAndDelete("two) = (%v, %v), want (2, true)`, v, ok)
+	}
+	if v, ok := m.LoadAndDelete("two"); v != 0 || ok {
+		t.Errorf(`LoadAndDelete("two) = (%v, %v), want (0, false)`, v, ok)
+	}
+	m.Delete("one")
+	m.Delete("noexist")
+	got = map[string]int{}
+	want = map[string]int{}
+	m.Range(func(k string, v int) bool {
+		got[k] = v
+		return true
+	})
+	if d := cmp.Diff(got, want); d != "" {
+		t.Errorf("Range mismatch (-got +want):\n%s", d)
+	}
+
+	t.Run("LoadOrStore", func(t *testing.T) {
+		var m Map[string, string]
+		var wg sync.WaitGroup
+		wg.Add(2)
+		var ok1, ok2 bool
+		go func() {
+			defer wg.Done()
+			_, ok1 = m.LoadOrStore("", "")
+		}()
+		go func() {
+			defer wg.Done()
+			_, ok2 = m.LoadOrStore("", "")
+		}()
+		wg.Wait()
+
+		if ok1 == ok2 {
+			t.Errorf("exactly one LoadOrStore should load")
+		}
+	})
+}

tsnet/example/tshello/tshello.go

@@ -29,13 +29,20 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	defer ln.Close()
+
+	lc, err := s.LocalClient()
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	if *addr == ":443" {
 		ln = tls.NewListener(ln, &tls.Config{
 			GetCertificate: tailscale.GetCertificate,
 		})
 	}
 	log.Fatal(http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		who, err := tailscale.WhoIs(r.Context(), r.RemoteAddr)
+		who, err := lc.WhoIs(r.Context(), r.RemoteAddr)
 		if err != nil {
 			http.Error(w, err.Error(), 500)
 			return

types/netlogtype/netlogtype.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package netlogtype defines types for network logging.
+package netlogtype
+
+import (
+	"net/netip"
+	"time"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/ipproto"
+)
+
+// TODO(joetsai): Remove "omitempty" if "omitzero" is ever supported in both
+// the v1 and v2 "json" packages.
+
+// Message is the log message that captures network traffic.
+type Message struct {
+	NodeID tailcfg.StableNodeID `json:"nodeId" cbor:"0,keyasint"` // e.g., "n123456CNTRL"
+
+	Start time.Time `json:"start" cbor:"12,keyasint"` // inclusive
+	End   time.Time `json:"end"   cbor:"13,keyasint"` // inclusive
+
+	VirtualTraffic  []ConnectionCounts `json:"virtualTraffic,omitempty"  cbor:"14,keyasint,omitempty"`
+	SubnetTraffic   []ConnectionCounts `json:"subnetTraffic,omitempty"   cbor:"15,keyasint,omitempty"`
+	ExitTraffic     []ConnectionCounts `json:"exitTraffic,omitempty"     cbor:"16,keyasint,omitempty"`
+	PhysicalTraffic []ConnectionCounts `json:"physicalTraffic,omitempty" cbor:"17,keyasint,omitempty"`
+}
+
+const (
+	maxJSONConnCounts = `{` + maxJSONConn + `,` + maxJSONCounts + `}`
+	maxJSONConn       = `"proto":` + maxJSONProto + `,"src":` + maxJSONAddrPort + `,"dst":` + maxJSONAddrPort
+	maxJSONProto      = `255`
+	maxJSONAddrPort   = `"[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535"`
+	maxJSONCounts     = `"txPkts":` + maxJSONCount + `,"txBytes":` + maxJSONCount + `,"rxPkts":` + maxJSONCount + `,"rxBytes":` + maxJSONCount
+	maxJSONCount      = `18446744073709551615`
+
+	// MaxConnectionCountsJSONSize is the maximum size of a ConnectionCounts
+	// when it is serialized as JSON, assuming no superfluous whitespace.
+	// It does not include the trailing comma that often appears when
+	// this object is nested within an array.
+	// It assumes that netip.Addr never has IPv6 zones.
+	MaxConnectionCountsJSONSize = len(maxJSONConnCounts)
+
+	maxCBORConnCounts = "\xbf" + maxCBORConn + maxCBORCounts + "\xff"
+	maxCBORConn       = "\x00" + maxCBORProto + "\x01" + maxCBORAddrPort + "\x02" + maxCBORAddrPort
+	maxCBORProto      = "\x18\xff"
+	maxCBORAddrPort   = "\x52\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+	maxCBORCounts     = "\x0c" + maxCBORCount + "\x0d" + maxCBORCount + "\x0e" + maxCBORCount + "\x0f" + maxCBORCount
+	maxCBORCount      = "\x1b\xff\xff\xff\xff\xff\xff\xff\xff"
+
+	// MaxConnectionCountsCBORSize is the maximum size of a ConnectionCounts
+	// when it is serialized as CBOR.
+	// It assumes that netip.Addr never has IPv6 zones.
+	MaxConnectionCountsCBORSize = len(maxCBORConnCounts)
+)
+
+// ConnectionCounts is a flattened struct of both a connection and counts.
+type ConnectionCounts struct {
+	Connection
+	Counts
+}
+
+// Connection is a 5-tuple of proto, source and destination IP and port.
+type Connection struct {
+	Proto ipproto.Proto  `json:"proto,omitzero,omitempty" cbor:"0,keyasint,omitempty"`
+	Src   netip.AddrPort `json:"src,omitzero,omitempty"   cbor:"1,keyasint,omitempty"`
+	Dst   netip.AddrPort `json:"dst,omitzero,omitempty"   cbor:"2,keyasint,omitempty"`
+}
+
+func (c Connection) IsZero() bool { return c == Connection{} }
+
+// Counts are statistics about a particular connection.
+type Counts struct {
+	TxPackets uint64 `json:"txPkts,omitzero,omitempty"  cbor:"12,keyasint,omitempty"`
+	TxBytes   uint64 `json:"txBytes,omitzero,omitempty" cbor:"13,keyasint,omitempty"`
+	RxPackets uint64 `json:"rxPkts,omitzero,omitempty"  cbor:"14,keyasint,omitempty"`
+	RxBytes   uint64 `json:"rxBytes,omitzero,omitempty" cbor:"15,keyasint,omitempty"`
+}
+
+func (c Counts) IsZero() bool { return c == Counts{} }
+
+// Add adds the counts from both c1 and c2.
+func (c1 Counts) Add(c2 Counts) Counts {
+	c1.TxPackets += c2.TxPackets
+	c1.TxBytes += c2.TxBytes
+	c1.RxPackets += c2.RxPackets
+	c1.RxBytes += c2.RxBytes
+	return c1
+}

types/netlogtype/netlogtype_test.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netlogtype
+
+import (
+	"encoding/json"
+	"math"
+	"net/netip"
+	"testing"
+
+	"github.com/fxamacker/cbor/v2"
+	"github.com/google/go-cmp/cmp"
+	"tailscale.com/util/must"
+)
+
+func TestMaxSize(t *testing.T) {
+	maxAddr := netip.AddrFrom16([16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255})
+	maxAddrPort := netip.AddrPortFrom(maxAddr, math.MaxUint16)
+	cc := ConnectionCounts{
+		// NOTE: These composite literals are deliberately unkeyed so that
+		// added fields result in a build failure here.
+		// Newly added fields should result in an update to both
+		// MaxConnectionCountsJSONSize and MaxConnectionCountsCBORSize.
+		Connection{math.MaxUint8, maxAddrPort, maxAddrPort},
+		Counts{math.MaxUint64, math.MaxUint64, math.MaxUint64, math.MaxUint64},
+	}
+
+	outJSON := must.Get(json.Marshal(cc))
+	if string(outJSON) != maxJSONConnCounts {
+		t.Errorf("JSON mismatch (-got +want):\n%s", cmp.Diff(string(outJSON), maxJSONConnCounts))
+	}
+
+	outCBOR := must.Get(cbor.Marshal(cc))
+	maxCBORConnCountsAlt := "\xa7" + maxCBORConnCounts[1:len(maxCBORConnCounts)-1] // may use a definite encoding of map
+	if string(outCBOR) != maxCBORConnCounts && string(outCBOR) != maxCBORConnCountsAlt {
+		t.Errorf("CBOR mismatch (-got +want):\n%s", cmp.Diff(string(outCBOR), maxCBORConnCounts))
+	}
+}

types/views/views.go

@@ -216,6 +216,17 @@ func (v IPPrefixSlice) ContainsExitRoutes() bool {
 	return tsaddr.ContainsExitRoutes(v.ж.ж)
 }
 
+// ContainsNonExitSubnetRoutes reports whether v contains Subnet
+// Routes other than ExitNode Routes.
+func (v IPPrefixSlice) ContainsNonExitSubnetRoutes() bool {
+	for i := 0; i < v.Len(); i++ {
+		if v.At(i).Bits() != 0 {
+			return true
+		}
+	}
+	return false
+}
+
 // MarshalJSON implements json.Marshaler.
 func (v IPPrefixSlice) MarshalJSON() ([]byte, error) {
 	return v.ж.MarshalJSON()

wgengine/magicsock/magicsock.go

@@ -52,6 +52,7 @@ import (
 	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/netlogtype"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/nettype"
 	"tailscale.com/util/clientmetric"
@@ -332,6 +333,21 @@ type Conn struct {
 	// port is the preferred port from opts.Port; 0 means auto.
 	port atomic.Uint32
 
+	// stats maintains per-connection counters.
+	// See SetStatisticsEnabled and ExtractStatistics for details.
+	stats struct {
+		enabled atomic.Bool
+
+		// TODO(joetsai): A per-Conn map of connections is easiest to implement.
+		// Since every packet occurs within the context of an endpoint,
+		// we could track the counts within the endpoint itself,
+		// and then merge the results when ExtractStatistics is called.
+		// That would avoid a map lookup for every packet.
+
+		mu sync.Mutex
+		m  map[netlogtype.Connection]netlogtype.Counts
+	}
+
 	// ============================================================
 	// mu guards all following fields; see userspaceEngine lock
 	// ordering rules against the engine. For derphttp, mu must
@@ -1744,6 +1760,9 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *ippEndpointCache,
 		ep = de
 	}
 	ep.noteRecvActivity()
+	if c.stats.enabled.Load() {
+		c.updateStats(ep.nodeAddr, ipp, netlogtype.Counts{RxPackets: 1, RxBytes: uint64(len(b))})
+	}
 	return ep, true
 }
 
@@ -1799,6 +1818,9 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en
 	}
 
 	ep.noteRecvActivity()
+	if c.stats.enabled.Load() {
+		c.updateStats(ep.nodeAddr, ipp, netlogtype.Counts{RxPackets: 1, RxBytes: uint64(dm.n)})
+	}
 	return n, ep
 }
 
@@ -2395,15 +2417,18 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 		ep := &endpoint{
 			c:                 c,
 			publicKey:         n.Key,
+			publicKeyHex:      n.Key.UntypedHexString(),
 			sentPing:          map[stun.TxID]sentPing{},
 			endpointState:     map[netip.AddrPort]*endpointState{},
 			heartbeatDisabled: heartbeatDisabled,
 		}
+		if len(n.Addresses) > 0 {
+			ep.nodeAddr = n.Addresses[0].Addr()
+		}
 		if !n.DiscoKey.IsZero() {
 			ep.discoKey = n.DiscoKey
 			ep.discoShort = n.DiscoKey.ShortString()
 		}
-		ep.wgEndpoint = n.Key.UntypedHexString()
 		ep.initFakeUDPAddr()
 		if debugDisco() { // rather than making a new knob
 			c.logf("magicsock: created endpoint key=%s: disco=%s; %v", n.Key.ShortString(), n.DiscoKey.ShortString(), logger.ArgWriter(func(w *bufio.Writer) {
@@ -3291,6 +3316,39 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 	})
 }
 
+// updateStats updates the statistics counters with the src, dst, and cnts.
+// It is the caller's responsibility to check whether logging is enabled.
+func (c *Conn) updateStats(src netip.Addr, dst netip.AddrPort, cnts netlogtype.Counts) {
+	conn := netlogtype.Connection{Src: netip.AddrPortFrom(src, 0), Dst: dst}
+	c.stats.mu.Lock()
+	defer c.stats.mu.Unlock()
+	mak.Set(&c.stats.m, conn, c.stats.m[conn].Add(cnts))
+}
+
+// SetStatisticsEnabled enables per-connection packet counters.
+// Disabling statistics gathering does not reset the counters.
+// ExtractStatistics must be called to reset the counters and
+// be periodically called while enabled to avoid unbounded memory use.
+func (c *Conn) SetStatisticsEnabled(enable bool) {
+	c.stats.enabled.Store(enable)
+}
+
+// ExtractStatistics extracts and resets the counters for all active connections.
+// It must be called periodically otherwise the memory used is unbounded.
+//
+// The source is always a peer's tailscale IP address,
+// while the destination is the peer's physical IP address and port.
+// As a special case, packets routed through DERP use a destination address
+// of 127.3.3.40 with the port being the DERP region.
+// This node's tailscale IP address never appears in the returned map.
+func (c *Conn) ExtractStatistics() map[netlogtype.Connection]netlogtype.Counts {
+	c.stats.mu.Lock()
+	defer c.stats.mu.Unlock()
+	m := c.stats.m
+	c.stats.m = nil
+	return m
+}
+
 func ippDebugString(ua netip.AddrPort) string {
 	if ua.Addr() == derpMagicIPAddr {
 		return fmt.Sprintf("derp-%d", ua.Port())
@@ -3307,10 +3365,11 @@ type endpoint struct {
 	numStopAndResetAtomic int64
 
 	// These fields are initialized once and never modified.
-	c          *Conn
-	publicKey  key.NodePublic // peer public key (for WireGuard + DERP)
-	fakeWGAddr netip.AddrPort // the UDP address we tell wireguard-go we're using
-	wgEndpoint string         // string from ParseEndpoint, holds a JSON-serialized wgcfg.Endpoints
+	c            *Conn
+	publicKey    key.NodePublic // peer public key (for WireGuard + DERP)
+	publicKeyHex string         // cached output of publicKey.UntypedHexString
+	fakeWGAddr   netip.AddrPort // the UDP address we tell wireguard-go we're using
+	nodeAddr     netip.Addr     // the node's first tailscale address (only used for logging)
 
 	// mu protects all following fields.
 	mu sync.Mutex // Lock ordering: Conn.mu, then endpoint.mu
@@ -3492,7 +3551,7 @@ func (de *endpoint) String() string {
 func (de *endpoint) ClearSrc()           {}
 func (de *endpoint) SrcToString() string { panic("unused") } // unused by wireguard-go
 func (de *endpoint) SrcIP() netip.Addr   { panic("unused") } // unused by wireguard-go
-func (de *endpoint) DstToString() string { return de.wgEndpoint }
+func (de *endpoint) DstToString() string { return de.publicKeyHex }
 func (de *endpoint) DstIP() netip.Addr   { panic("unused") }
 func (de *endpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr) }
 
@@ -3640,11 +3699,19 @@ func (de *endpoint) send(b []byte) error {
 	var err error
 	if udpAddr.IsValid() {
 		_, err = de.c.sendAddr(udpAddr, de.publicKey, b)
+		if err == nil && de.c.stats.enabled.Load() {
+			de.c.updateStats(de.nodeAddr, udpAddr, netlogtype.Counts{TxPackets: 1, TxBytes: uint64(len(b))})
+		}
 	}
 	if derpAddr.IsValid() {
-		if ok, _ := de.c.sendAddr(derpAddr, de.publicKey, b); ok && err != nil {
-			// UDP failed but DERP worked, so good enough:
-			return nil
+		if ok, _ := de.c.sendAddr(derpAddr, de.publicKey, b); ok {
+			if de.c.stats.enabled.Load() {
+				de.c.updateStats(de.nodeAddr, derpAddr, netlogtype.Counts{TxPackets: 1, TxBytes: uint64(len(b))})
+			}
+			if err != nil {
+				// UDP failed but DERP worked, so good enough:
+				return nil
+			}
 		}
 	}
 	return err

wgengine/magicsock/magicsock_test.go

@@ -28,6 +28,7 @@ import (
 	"unsafe"
 
 	"go4.org/mem"
+	"golang.org/x/exp/maps"
 	"golang.zx2c4.com/wireguard/device"
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"tailscale.com/derp"
@@ -42,6 +43,7 @@ import (
 	"tailscale.com/tstest/natlab"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
+	"tailscale.com/types/netlogtype"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/nettype"
 	"tailscale.com/util/cibuild"
@@ -1093,17 +1095,45 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 		}
 	}
 
+	m1.conn.SetStatisticsEnabled(true)
+	m2.conn.SetStatisticsEnabled(true)
+
+	checkStats := func(t *testing.T, m *magicStack, wantConns []netlogtype.Connection) {
+		stats := m.conn.ExtractStatistics()
+		for _, conn := range wantConns {
+			if _, ok := stats[conn]; ok {
+				return
+			}
+		}
+		t.Helper()
+		t.Errorf("missing any connection to %s from %s", wantConns, maps.Keys(stats))
+	}
+
+	addrPort := netip.MustParseAddrPort
+	m1Conns := []netlogtype.Connection{
+		{Src: addrPort("1.0.0.2:0"), Dst: m2.conn.pconn4.LocalAddr().AddrPort()},
+		{Src: addrPort("1.0.0.2:0"), Dst: addrPort("127.3.3.40:1")},
+	}
+	m2Conns := []netlogtype.Connection{
+		{Src: addrPort("1.0.0.1:0"), Dst: m1.conn.pconn4.LocalAddr().AddrPort()},
+		{Src: addrPort("1.0.0.1:0"), Dst: addrPort("127.3.3.40:1")},
+	}
+
 	outerT := t
 	t.Run("ping 1.0.0.1", func(t *testing.T) {
 		setT(t)
 		defer setT(outerT)
 		ping1(t)
+		checkStats(t, m1, m1Conns)
+		checkStats(t, m2, m2Conns)
 	})
 
 	t.Run("ping 1.0.0.2", func(t *testing.T) {
 		setT(t)
 		defer setT(outerT)
 		ping2(t)
+		checkStats(t, m1, m1Conns)
+		checkStats(t, m2, m2Conns)
 	})
 
 	t.Run("ping 1.0.0.2 via SendPacket", func(t *testing.T) {
@@ -1120,6 +1150,8 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 		if err := sendWithTimeout(msg1to2, in, send); err != nil {
 			t.Error(err)
 		}
+		checkStats(t, m1, m1Conns)
+		checkStats(t, m2, m2Conns)
 	})
 
 	t.Run("no-op dev1 reconfig", func(t *testing.T) {
@@ -1130,6 +1162,8 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 		}
 		ping1(t)
 		ping2(t)
+		checkStats(t, m1, m1Conns)
+		checkStats(t, m2, m2Conns)
 	})
 }
 
@@ -1731,9 +1765,6 @@ func (m *peerMap) validate() error {
 		if got := pi.ep.publicKey; got != pub {
 			return fmt.Errorf("byNodeKey[%v].publicKey = %v", pub, got)
 		}
-		if got, want := pi.ep.wgEndpoint, pub.UntypedHexString(); got != want {
-			return fmt.Errorf("byNodeKey[%v].wgEndpoint = %q, want %q", pub, got, want)
-		}
 		if _, ok := seenEps[pi.ep]; ok {
 			return fmt.Errorf("duplicate endpoint present: %v", pi.ep.publicKey)
 		}

wgengine/netlog/logger.go

@@ -0,0 +1,303 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package netlog provides a logger that monitors a TUN device and
+// periodically records any traffic into a log stream.
+package netlog
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/netip"
+	"sync"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	"tailscale.com/logpolicy"
+	"tailscale.com/logtail"
+	"tailscale.com/net/tsaddr"
+	"tailscale.com/smallzstd"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/netlogtype"
+	"tailscale.com/wgengine/router"
+)
+
+// pollPeriod specifies how often to poll for network traffic.
+const pollPeriod = 5 * time.Second
+
+// Device is an abstraction over a tunnel device or a magic socket.
+// *tstun.Wrapper implements this interface.
+// *magicsock.Conn implements this interface.
+type Device interface {
+	SetStatisticsEnabled(bool)
+	ExtractStatistics() map[netlogtype.Connection]netlogtype.Counts
+}
+
+type noopDevice struct{}
+
+func (noopDevice) SetStatisticsEnabled(bool)                                      {}
+func (noopDevice) ExtractStatistics() map[netlogtype.Connection]netlogtype.Counts { return nil }
+
+// Logger logs statistics about every connection.
+// At present, it only logs connections within a tailscale network.
+// Exit node traffic is not logged for privacy reasons.
+// The zero value is ready for use.
+type Logger struct {
+	mu sync.Mutex
+
+	logger *logtail.Logger
+
+	addrs    map[netip.Addr]bool
+	prefixes map[netip.Prefix]bool
+
+	group  errgroup.Group
+	cancel context.CancelFunc
+}
+
+// Running reports whether the logger is running.
+func (nl *Logger) Running() bool {
+	nl.mu.Lock()
+	defer nl.mu.Unlock()
+	return nl.logger != nil
+}
+
+var testClient *http.Client
+
+// Startup starts an asynchronous network logger that monitors
+// statistics for the provided tun and/or sock device.
+//
+// The tun Device captures packets within the tailscale network,
+// where at least one address is a tailscale IP address.
+// The source is always from the perspective of the current node.
+// If one of the other endpoint is not a tailscale IP address,
+// then it suggests the use of a subnet router or exit node.
+// For example, when using a subnet router, the source address is
+// the tailscale IP address of the current node, and
+// the destination address is an IP address within the subnet range.
+// In contrast, when acting as a subnet router, the source address is
+// an IP address within the subnet range, and the destination is a
+// tailscale IP address that initiated the subnet proxy connection.
+// In this case, the node acting as a subnet router is acting on behalf
+// of some remote endpoint within the subnet range.
+// The tun is used to populate the VirtualTraffic, SubnetTraffic,
+// and ExitTraffic fields in Message.
+//
+// The sock Device captures packets at the magicsock layer.
+// The source is always a tailscale IP address and the destination
+// is a non-tailscale IP address to contact for that particular tailscale node.
+// The IP protocol and source port are always zero.
+// The sock is used to populated the PhysicalTraffic field in Message.
+func (nl *Logger) Startup(nodeID tailcfg.StableNodeID, nodeLogID, domainLogID logtail.PrivateID, tun, sock Device) error {
+	nl.mu.Lock()
+	defer nl.mu.Unlock()
+	if nl.logger != nil {
+		return fmt.Errorf("network logger already running for %v", nl.logger.PrivateID().Public())
+	}
+	if tun == nil {
+		tun = noopDevice{}
+	}
+	if sock == nil {
+		sock = noopDevice{}
+	}
+
+	httpc := &http.Client{Transport: logpolicy.NewLogtailTransport(logtail.DefaultHost)}
+	if testClient != nil {
+		httpc = testClient
+	}
+	logger := logtail.NewLogger(logtail.Config{
+		Collection:    "tailtraffic.log.tailscale.io",
+		PrivateID:     nodeLogID,
+		CopyPrivateID: domainLogID,
+		Stderr:        io.Discard,
+		// TODO(joetsai): Set Buffer? Use an in-memory buffer for now.
+		NewZstdEncoder: func() logtail.Encoder {
+			w, err := smallzstd.NewEncoder(nil)
+			if err != nil {
+				panic(err)
+			}
+			return w
+		},
+		HTTPC: httpc,
+
+		// Include process sequence numbers to identify missing samples.
+		IncludeProcID:       true,
+		IncludeProcSequence: true,
+	}, log.Printf)
+	nl.logger = logger
+
+	ctx, cancel := context.WithCancel(context.Background())
+	nl.cancel = cancel
+	nl.group.Go(func() error {
+		tun.SetStatisticsEnabled(true)
+		defer tun.SetStatisticsEnabled(false)
+		tun.ExtractStatistics() // clear out any stale statistics
+
+		sock.SetStatisticsEnabled(true)
+		defer sock.SetStatisticsEnabled(false)
+		sock.ExtractStatistics() // clear out any stale statistics
+
+		start := time.Now()
+		ticker := time.NewTicker(pollPeriod)
+		for {
+			var end time.Time
+			select {
+			case <-ctx.Done():
+				tun.SetStatisticsEnabled(false)
+				end = time.Now()
+			case end = <-ticker.C:
+			}
+
+			// NOTE: tunStats and sockStats will always be slightly out-of-sync.
+			// It is impossible to have an atomic snapshot of statistics
+			// at both layers without a global mutex that spans all layers.
+			tunStats := tun.ExtractStatistics()
+			sockStats := sock.ExtractStatistics()
+			if len(tunStats)+len(sockStats) > 0 {
+				nl.mu.Lock()
+				addrs := nl.addrs
+				prefixes := nl.prefixes
+				nl.mu.Unlock()
+				recordStatistics(logger, nodeID, start, end, tunStats, sockStats, addrs, prefixes)
+			}
+
+			if ctx.Err() != nil {
+				break
+			}
+			start = end.Add(time.Nanosecond)
+		}
+		return nil
+	})
+	return nil
+}
+
+func recordStatistics(logger *logtail.Logger, nodeID tailcfg.StableNodeID, start, end time.Time, tunStats, sockStats map[netlogtype.Connection]netlogtype.Counts, addrs map[netip.Addr]bool, prefixes map[netip.Prefix]bool) {
+	m := netlogtype.Message{NodeID: nodeID, Start: start.UTC(), End: end.UTC()}
+
+	classifyAddr := func(a netip.Addr) (isTailscale, withinRoute bool) {
+		// NOTE: There could be mis-classifications where an address is treated
+		// as a Tailscale IP address because the subnet range overlaps with
+		// the subnet range that Tailscale IP addresses are allocated from.
+		// This should never happen for IPv6, but could happen for IPv4.
+		withinRoute = addrs[a]
+		for p := range prefixes {
+			if p.Contains(a) && p.Bits() > 0 {
+				withinRoute = true
+				break
+			}
+		}
+		return withinRoute && tsaddr.IsTailscaleIP(a), withinRoute && !tsaddr.IsTailscaleIP(a)
+	}
+
+	exitTraffic := make(map[netlogtype.Connection]netlogtype.Counts)
+	for conn, cnts := range tunStats {
+		srcIsTailscaleIP, srcWithinSubnet := classifyAddr(conn.Src.Addr())
+		dstIsTailscaleIP, dstWithinSubnet := classifyAddr(conn.Dst.Addr())
+		switch {
+		case srcIsTailscaleIP && dstIsTailscaleIP:
+			m.VirtualTraffic = append(m.VirtualTraffic, netlogtype.ConnectionCounts{Connection: conn, Counts: cnts})
+		case srcWithinSubnet || dstWithinSubnet:
+			m.SubnetTraffic = append(m.SubnetTraffic, netlogtype.ConnectionCounts{Connection: conn, Counts: cnts})
+		default:
+			const anonymize = true
+			if anonymize {
+				// Only preserve the address if it is a Tailscale IP address.
+				srcOrig, dstOrig := conn.Src, conn.Dst
+				conn = netlogtype.Connection{} // scrub everything by default
+				if srcIsTailscaleIP {
+					conn.Src = netip.AddrPortFrom(srcOrig.Addr(), 0)
+				}
+				if dstIsTailscaleIP {
+					conn.Dst = netip.AddrPortFrom(dstOrig.Addr(), 0)
+				}
+			}
+			exitTraffic[conn] = exitTraffic[conn].Add(cnts)
+		}
+	}
+	for conn, cnts := range exitTraffic {
+		m.ExitTraffic = append(m.ExitTraffic, netlogtype.ConnectionCounts{Connection: conn, Counts: cnts})
+	}
+	for conn, cnts := range sockStats {
+		m.PhysicalTraffic = append(m.PhysicalTraffic, netlogtype.ConnectionCounts{Connection: conn, Counts: cnts})
+	}
+
+	if len(m.VirtualTraffic)+len(m.SubnetTraffic)+len(m.ExitTraffic)+len(m.PhysicalTraffic) > 0 {
+		// TODO(joetsai): Place a hard limit on the size of a network log message.
+		// The log server rejects any payloads above a certain size, so logging
+		// a message that large would cause logtail to be stuck forever trying
+		// and failing to upload the same excessively large payload.
+		//
+		// We should figure out the behavior for handling this. We could split
+		// the message apart so that there are multiple chunks with the same window,
+		// We could also consider reducing the granularity of the data
+		// by dropping port numbers.
+		const maxSize = 256 << 10
+		if b, err := json.Marshal(m); err != nil {
+			logger.Logf("json.Marshal error: %v", err)
+		} else if len(b) > maxSize {
+			logger.Logf("JSON body too large: %dB (virtual:%d subnet:%d exit:%d physical:%d)",
+				len(b), len(m.VirtualTraffic), len(m.SubnetTraffic), len(m.ExitTraffic), len(m.PhysicalTraffic))
+		} else {
+			logger.Logf("%s", b)
+		}
+	}
+}
+
+func makeRouteMaps(cfg *router.Config) (addrs map[netip.Addr]bool, prefixes map[netip.Prefix]bool) {
+	addrs = make(map[netip.Addr]bool)
+	for _, p := range cfg.LocalAddrs {
+		if p.IsSingleIP() {
+			addrs[p.Addr()] = true
+		}
+	}
+	prefixes = make(map[netip.Prefix]bool)
+	insertPrefixes := func(rs []netip.Prefix) {
+		for _, p := range rs {
+			if p.IsSingleIP() {
+				addrs[p.Addr()] = true
+			} else {
+				prefixes[p] = true
+			}
+		}
+	}
+	insertPrefixes(cfg.Routes)
+	insertPrefixes(cfg.SubnetRoutes)
+	return addrs, prefixes
+}
+
+// ReconfigRoutes configures the network logger with updated routes.
+// The cfg is used to classify the types of connections captured by
+// the tun Device passed to Startup.
+func (nl *Logger) ReconfigRoutes(cfg *router.Config) {
+	nl.mu.Lock()
+	defer nl.mu.Unlock()
+	// TODO(joetsai): There is a race where deleted routes are not known at
+	// the time of extraction. We need to keep old routes around for a bit.
+	nl.addrs, nl.prefixes = makeRouteMaps(cfg)
+}
+
+// Shutdown shuts down the network logger.
+// This attempts to flush out all pending log messages.
+// Even if an error is returned, the logger is still shut down.
+func (nl *Logger) Shutdown(ctx context.Context) error {
+	nl.mu.Lock()
+	defer nl.mu.Unlock()
+	if nl.logger == nil {
+		return nil
+	}
+	nl.cancel()
+	nl.mu.Unlock()
+	nl.group.Wait() // do not hold lock while waiting
+	nl.mu.Lock()
+	err := nl.logger.Shutdown(ctx)
+
+	nl.logger = nil
+	nl.addrs = nil
+	nl.prefixes = nil
+	nl.cancel = nil
+	return err
+}

wgengine/netlog/logger_test.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netlog
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"tailscale.com/logtail"
+	"tailscale.com/tstest"
+	"tailscale.com/types/netlogtype"
+	"tailscale.com/util/must"
+	"tailscale.com/wgengine/router"
+)
+
+func init() {
+	testClient = &http.Client{Transport: &roundTripper}
+}
+
+var roundTripper roundTripperFunc
+
+type roundTripperFunc struct {
+	F func(*http.Request) (*http.Response, error)
+}
+
+func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f.F(r)
+}
+
+type fakeDevice struct {
+	toggled int // even => disabled, odd => enabled
+}
+
+func (d *fakeDevice) SetStatisticsEnabled(enable bool) {
+	if enabled := d.toggled%2 == 1; enabled != enable {
+		d.toggled++
+	}
+
+}
+func (fakeDevice) ExtractStatistics() map[netlogtype.Connection]netlogtype.Counts {
+	// TODO(dsnet): Add a test that verifies that statistics are correctly
+	// extracted from the device and uploaded. Unfortunately,
+	// we can't reliably run this test until we fix http://go/oss/5856.
+	return nil
+}
+
+func TestResourceCheck(t *testing.T) {
+	roundTripper.F = func(r *http.Request) (*http.Response, error) {
+		return &http.Response{StatusCode: 200}, nil
+	}
+
+	c := qt.New(t)
+	tstest.ResourceCheck(t)
+	var l Logger
+	var d fakeDevice
+	for i := 0; i < 10; i++ {
+		must.Do(l.Startup("", logtail.PrivateID{}, logtail.PrivateID{}, &d, nil))
+		l.ReconfigRoutes(&router.Config{})
+		must.Do(l.Shutdown(context.Background()))
+		c.Assert(d.toggled, qt.Equals, 2*(i+1))
+	}
+}

wgengine/netstack/netstack.go

@@ -160,6 +160,11 @@ func Create(logf logger.Logf, tundev *tstun.Wrapper, e wgengine.Engine, mc *magi
 		NetworkProtocols:   []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
 		TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
 	})
+	sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default
+	tcpipErr := ipstack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt)
+	if tcpipErr != nil {
+		return nil, fmt.Errorf("could not enable TCP SACK: %v", tcpipErr)
+	}
 	linkEP := channel.New(512, mtu, "")
 	if tcpipProblem := ipstack.CreateNIC(nicID, linkEP); tcpipProblem != nil {
 		return nil, fmt.Errorf("could not create netstack NIC: %v", tcpipProblem)

wgengine/router/ifconfig_windows.go

@@ -24,6 +24,7 @@ import (
 	"tailscale.com/health"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/tsaddr"
+	"tailscale.com/net/tstun"
 	"tailscale.com/util/multierr"
 	"tailscale.com/wgengine/winnet"
 )
@@ -247,7 +248,7 @@ func interfaceFromLUID(luid winipcfg.LUID, flags winipcfg.GAAFlags) (*winipcfg.I
 }
 
 func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
-	const mtu = 0
+	const mtu = tstun.DefaultMTU
 	luid := winipcfg.LUID(tun.LUID())
 	iface, err := interfaceFromLUID(luid,
 		// Issue 474: on early boot, when the network is still

wgengine/router/router_linux.go

@@ -1532,25 +1532,10 @@ func cidrDiff(kind string, old map[netip.Prefix]bool, new []netip.Prefix, add, d
 		ret[cidr] = true
 	}
 
-	var delFail []error
-	for cidr := range old {
-		if newMap[cidr] {
-			continue
-		}
-		if err := del(cidr); err != nil {
-			logf("%s del failed: %v", kind, err)
-			delFail = append(delFail, err)
-		} else {
-			delete(ret, cidr)
-		}
-	}
-	if len(delFail) == 1 {
-		return ret, delFail[0]
-	}
-	if len(delFail) > 0 {
-		return ret, fmt.Errorf("%d delete %s failures; first was: %w", len(delFail), kind, delFail[0])
-	}
-
+	// We want to add before we delete, so that if there is no overlap, we don't
+	// end up in a state where we have no addresses on an interface as that
+	// results in other kernel entities (like routes) pointing to that interface
+	// to also be deleted.
 	var addFail []error
 	for cidr := range newMap {
 		if old[cidr] {
@@ -1571,6 +1556,25 @@ func cidrDiff(kind string, old map[netip.Prefix]bool, new []netip.Prefix, add, d
 		return ret, fmt.Errorf("%d add %s failures; first was: %w", len(addFail), kind, addFail[0])
 	}
 
+	var delFail []error
+	for cidr := range old {
+		if newMap[cidr] {
+			continue
+		}
+		if err := del(cidr); err != nil {
+			logf("%s del failed: %v", kind, err)
+			delFail = append(delFail, err)
+		} else {
+			delete(ret, cidr)
+		}
+	}
+	if len(delFail) == 1 {
+		return ret, delFail[0]
+	}
+	if len(delFail) > 0 {
+		return ret, fmt.Errorf("%d delete %s failures; first was: %w", len(delFail), kind, delFail[0])
+	}
+
 	return ret, nil
 }
 

wgengine/router/router_linux_test.go

@@ -10,6 +10,7 @@ import (
 	"math/rand"
 	"net/netip"
 	"os"
+	"reflect"
 	"sort"
 	"strings"
 	"sync/atomic"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/exp/slices"
 	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/tstest"
 	"tailscale.com/types/logger"
@@ -839,3 +841,84 @@ Usage: busybox [function [arguments]...]
 		t.Errorf("version = %q, want %q", got, want)
 	}
 }
+
+func TestCIDRDiff(t *testing.T) {
+	pfx := func(p ...string) []netip.Prefix {
+		var ret []netip.Prefix
+		for _, s := range p {
+			ret = append(ret, netip.MustParsePrefix(s))
+		}
+		return ret
+	}
+	tests := []struct {
+		old     []netip.Prefix
+		new     []netip.Prefix
+		wantAdd []netip.Prefix
+		wantDel []netip.Prefix
+		final   []netip.Prefix
+	}{
+		{
+			old:     nil,
+			new:     pfx("1.1.1.1/32"),
+			wantAdd: pfx("1.1.1.1/32"),
+			final:   pfx("1.1.1.1/32"),
+		},
+		{
+			old:   pfx("1.1.1.1/32"),
+			new:   pfx("1.1.1.1/32"),
+			final: pfx("1.1.1.1/32"),
+		},
+		{
+			old:     pfx("1.1.1.1/32", "2.3.4.5/32"),
+			new:     pfx("1.1.1.1/32"),
+			wantDel: pfx("2.3.4.5/32"),
+			final:   pfx("1.1.1.1/32"),
+		},
+		{
+			old:     pfx("1.1.1.1/32", "2.3.4.5/32"),
+			new:     pfx("1.0.0.0/32", "3.4.5.6/32"),
+			wantDel: pfx("1.1.1.1/32", "2.3.4.5/32"),
+			wantAdd: pfx("1.0.0.0/32", "3.4.5.6/32"),
+			final:   pfx("1.0.0.0/32", "3.4.5.6/32"),
+		},
+	}
+	for _, tc := range tests {
+		om := make(map[netip.Prefix]bool)
+		for _, p := range tc.old {
+			om[p] = true
+		}
+		var added []netip.Prefix
+		var deleted []netip.Prefix
+		fm, err := cidrDiff("test", om, tc.new, func(p netip.Prefix) error {
+			if len(deleted) > 0 {
+				t.Error("delete called before add")
+			}
+			added = append(added, p)
+			return nil
+		}, func(p netip.Prefix) error {
+			deleted = append(deleted, p)
+			return nil
+		}, t.Logf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		slices.SortFunc(added, func(a, b netip.Prefix) bool { return a.Addr().Less(b.Addr()) })
+		slices.SortFunc(deleted, func(a, b netip.Prefix) bool { return a.Addr().Less(b.Addr()) })
+		if !reflect.DeepEqual(added, tc.wantAdd) {
+			t.Errorf("added = %v, want %v", added, tc.wantAdd)
+		}
+		if !reflect.DeepEqual(deleted, tc.wantDel) {
+			t.Errorf("deleted = %v, want %v", deleted, tc.wantDel)
+		}
+
+		// Check that the final state is correct.
+		if len(fm) != len(tc.final) {
+			t.Fatalf("final state = %v, want %v", fm, tc.final)
+		}
+		for _, p := range tc.final {
+			if !fm[p] {
+				t.Errorf("final state = %v, want %v", fm, tc.final)
+			}
+		}
+	}
+}

wgengine/userspace.go

@@ -6,6 +6,7 @@ package wgengine
 
 import (
 	"bufio"
+	"context"
 	crand "crypto/rand"
 	"errors"
 	"fmt"
@@ -48,6 +49,7 @@ import (
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/magicsock"
 	"tailscale.com/wgengine/monitor"
+	"tailscale.com/wgengine/netlog"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
 	"tailscale.com/wgengine/wgint"
@@ -84,6 +86,10 @@ const (
 // status (as long as there's activity). See docs on its use below.
 const statusPollInterval = 1 * time.Minute
 
+// networkLoggerUploadTimeout is the maximum timeout to wait when
+// shutting down the network logger as it uploads the last network log messages.
+const networkLoggerUploadTimeout = 5 * time.Second
+
 type userspaceEngine struct {
 	logf              logger.Logf
 	wgLogger          *wglog.Logger //a wireguard-go logging wrapper
@@ -145,6 +151,9 @@ type userspaceEngine struct {
 	// value of the ICMP identifer and sequence number concatenated.
 	icmpEchoResponseCallback map[uint32]func()
 
+	// networkLogger logs statistics about network connections.
+	networkLogger netlog.Logger
+
 	// Lock ordering: magicsock.Conn.mu, wgLock, then mu.
 }
 
@@ -872,6 +881,15 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 	if !engineChanged && !routerChanged && listenPort == e.magicConn.LocalPort() && !isSubnetRouterChanged {
 		return ErrNoChanges
 	}
+	newLogIDs := cfg.NetworkLogging
+	oldLogIDs := e.lastCfgFull.NetworkLogging
+	netLogIDsNowValid := !newLogIDs.NodeID.IsZero() && !newLogIDs.DomainID.IsZero()
+	netLogIDsWasValid := !oldLogIDs.NodeID.IsZero() && !oldLogIDs.DomainID.IsZero()
+	netLogIDsChanged := netLogIDsNowValid && netLogIDsWasValid && newLogIDs != oldLogIDs
+	netLogRunning := netLogIDsNowValid && !routerCfg.Equal(&router.Config{})
+	if envknob.NoLogsNoSupport() {
+		netLogRunning = false
+	}
 
 	// TODO(bradfitz,danderson): maybe delete this isDNSIPOverTailscale
 	// field and delete the resolver.ForwardLinkSelector hook and
@@ -921,8 +939,32 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 		return err
 	}
 
+	// Shutdown the network logger because the IDs changed.
+	// Let it be started back up by subsequent logic.
+	if netLogIDsChanged && e.networkLogger.Running() {
+		e.logf("wgengine: Reconfig: shutting down network logger")
+		ctx, cancel := context.WithTimeout(context.Background(), networkLoggerUploadTimeout)
+		defer cancel()
+		if err := e.networkLogger.Shutdown(ctx); err != nil {
+			e.logf("wgengine: Reconfig: error shutting down network logger: %v", err)
+		}
+	}
+
+	// Startup the network logger.
+	// Do this before configuring the router so that we capture initial packets.
+	if netLogRunning && !e.networkLogger.Running() {
+		nid := cfg.NetworkLogging.NodeID
+		tid := cfg.NetworkLogging.DomainID
+		e.logf("wgengine: Reconfig: starting up network logger (node:%s tailnet:%s)", nid.Public(), tid.Public())
+		if err := e.networkLogger.Startup(cfg.NodeID, nid, tid, e.tundev, e.magicConn); err != nil {
+			e.logf("wgengine: Reconfig: error starting up network logger: %v", err)
+		}
+		e.networkLogger.ReconfigRoutes(routerCfg)
+	}
+
 	if routerChanged {
 		e.logf("wgengine: Reconfig: configuring router")
+		e.networkLogger.ReconfigRoutes(routerCfg)
 		err := e.router.Set(routerCfg)
 		health.SetRouterHealth(err)
 		if err != nil {
@@ -939,6 +981,18 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 		}
 	}
 
+	// Shutdown the network logger.
+	// Do this after configuring the router so that we capture final packets.
+	// This attempts to flush out any log messages and may block.
+	if !netLogRunning && e.networkLogger.Running() {
+		e.logf("wgengine: Reconfig: shutting down network logger")
+		ctx, cancel := context.WithTimeout(context.Background(), networkLoggerUploadTimeout)
+		defer cancel()
+		if err := e.networkLogger.Shutdown(ctx); err != nil {
+			e.logf("wgengine: Reconfig: error shutting down network logger: %v", err)
+		}
+	}
+
 	if isSubnetRouterChanged && e.birdClient != nil {
 		e.logf("wgengine: Reconfig: configuring BIRD")
 		var err error
@@ -1092,6 +1146,12 @@ func (e *userspaceEngine) Close() {
 		e.birdClient.Close()
 	}
 	close(e.waitCh)
+
+	ctx, cancel := context.WithTimeout(context.Background(), networkLoggerUploadTimeout)
+	defer cancel()
+	if err := e.networkLogger.Shutdown(ctx); err != nil {
+		e.logf("wgengine: Close: error shutting down network logger: %v", err)
+	}
 }
 
 func (e *userspaceEngine) Wait() {

wgengine/wgcfg/config.go

@@ -9,6 +9,7 @@ import (
 	"net/netip"
 
 	"tailscale.com/logtail"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
 
@@ -18,6 +19,7 @@ import (
 // It only supports the set of things Tailscale uses.
 type Config struct {
 	Name       string
+	NodeID     tailcfg.StableNodeID
 	PrivateKey key.NodePrivate
 	Addresses  []netip.Prefix
 	MTU        uint16

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -62,6 +62,7 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 
 	// Setup log IDs for data plane audit logging.
 	if nm.SelfNode != nil {
+		cfg.NodeID = nm.SelfNode.StableID
 		canNetworkLog := slices.Contains(nm.SelfNode.Capabilities, tailcfg.CapabilityDataPlaneAuditLogs)
 		if canNetworkLog && nm.SelfNode.DataPlaneAuditLogID != "" && nm.DomainAuditLogID != "" {
 			nodeID, errNode := logtail.ParsePrivateID(nm.SelfNode.DataPlaneAuditLogID)

wgengine/wgcfg/wgcfg_clone.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 
 	"tailscale.com/logtail"
+	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
 
@@ -33,6 +34,7 @@ func (src *Config) Clone() *Config {
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _ConfigCloneNeedsRegeneration = Config(struct {
 	Name           string
+	NodeID         tailcfg.StableNodeID
 	PrivateKey     key.NodePrivate
 	Addresses      []netip.Prefix
 	MTU            uint16

wgengine/wgengine.go

@@ -79,7 +79,7 @@ type Engine interface {
 	Reconfig(*wgcfg.Config, *router.Config, *dns.Config, *tailcfg.Debug) error
 
 	// PeerForIP returns the node to which the provided IP routes,
-	// if any. If none is found, (nil, nil) is returned.
+	// if any. If none is found, (nil, false) is returned.
 	PeerForIP(netip.Addr) (_ PeerForIP, ok bool)
 
 	// GetFilter returns the current packet filter, if any.