net/portmapper: fix UPnP probing, work against all ports

Prior to Tailscale 1.12 it detected UPnP on any port.
Starting with Tailscale 1.11.x, it stopped detecting UPnP on all ports.

Then start plumbing its discovered Location header port number to the
code that was assuming port 5000.

Fixes #2109

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

Dockerfile

@@ -42,8 +42,7 @@ FROM golang:1.16-alpine AS build-env
 
 WORKDIR /go/src/tailscale
 
-COPY go.mod .
-COPY go.sum .
+COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .

VERSION.txt

@@ -1 +1 @@
-1.12.2
+1.13.0

cmd/derper/mesh.go

@@ -9,7 +9,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"net"
 	"strings"
+	"time"
 
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -39,6 +41,34 @@ func startMeshWithHost(s *derp.Server, host string) error {
 		return err
 	}
 	c.MeshKey = s.MeshKey()
+
+	// For meshed peers within a region, connect via VPC addresses.
+	c.SetURLDialer(func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var d net.Dialer
+		var r net.Resolver
+		if port == "443" && strings.HasSuffix(host, ".tailscale.com") {
+			base := strings.TrimSuffix(host, ".tailscale.com")
+			subCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
+			defer cancel()
+			vpcHost := base + "-vpc.tailscale.com"
+			ips, _ := r.LookupIP(subCtx, "ip", vpcHost)
+			if len(ips) > 0 {
+				vpcAddr := net.JoinHostPort(ips[0].String(), port)
+				c, err := d.DialContext(subCtx, network, vpcAddr)
+				if err == nil {
+					log.Printf("connected to %v (%v) instead of %v", vpcHost, ips[0], base)
+					return c, nil
+				}
+				log.Printf("failed to connect to %v (%v): %v; trying non-VPC route", vpcHost, ips[0], err)
+			}
+		}
+		return d.DialContext(ctx, network, addr)
+	})
+
 	add := func(k key.Public) { s.AddPacketForwarder(k, c) }
 	remove := func(k key.Public) { s.RemovePacketForwarder(k, c) }
 	go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove)

cmd/tailscale/cli/status.go

@@ -23,6 +23,7 @@ import (
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/util/dnsname"
 )
 
@@ -193,7 +194,7 @@ func runStatus(ctx context.Context, args []string) error {
 //
 // TODO: have the server report this bool instead.
 func peerActive(ps *ipnstate.PeerStatus) bool {
-	return !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+	return !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 }
 
 func dnsOrQuoteHostname(st *ipnstate.Status, ps *ipnstate.PeerStatus) string {

cmd/tailscale/cli/up.go

@@ -70,13 +70,13 @@ func newUpFlagSet(goos string, upArgs *upArgsT) *flag.FlagSet {
 	upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", false, "accept routes advertised by other Tailscale nodes")
 	upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel")
 	upf.BoolVar(&upArgs.singleRoutes, "host-routes", true, "install host routes to other Tailscale nodes")
-	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic")
+	upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale IP of the exit node for internet traffic, or empty string to not use an exit node")
 	upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node")
 	upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections")
 	upf.StringVar(&upArgs.advertiseTags, "advertise-tags", "", "comma-separated ACL tags to request; each must start with \"tag:\" (e.g. \"tag:eng,tag:montreal,tag:ssh\")")
 	upf.StringVar(&upArgs.authKey, "authkey", "", "node authorization key")
 	upf.StringVar(&upArgs.hostname, "hostname", "", "hostname to use instead of the one provided by the OS")
-	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\")")
+	upf.StringVar(&upArgs.advertiseRoutes, "advertise-routes", "", "routes to advertise to other nodes (comma-separated, e.g. \"10.0.0.0/8,192.168.0.0/24\") or empty string to not advertise routes")
 	upf.BoolVar(&upArgs.advertiseDefaultRoute, "advertise-exit-node", false, "offer to be an exit node for internet traffic for the tailnet")
 	if safesocket.GOOSUsesPeerCreds(goos) {
 		upf.StringVar(&upArgs.opUser, "operator", "", "Unix username to allow to operate on tailscaled without sudo")

cmd/tailscale/depaware.txt

@@ -49,6 +49,8 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/cmd/tailscale/cli+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/cmd/tailscale/cli+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/ipn
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+
         tailscale.com/types/key                                      from tailscale.com/derp+

cmd/tailscaled/debug.go

@@ -14,18 +14,23 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/httptrace"
 	"net/url"
 	"os"
+	"strings"
 	"time"
 
+	"inet.af/netaddr"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/ipn"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/portmapper"
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/monitor"
 )
 
@@ -33,6 +38,7 @@ var debugArgs struct {
 	monitor   bool
 	getURL    string
 	derpCheck string
+	portmap   bool
 }
 
 var debugModeFunc = debugMode // so it can be addressable
@@ -40,6 +46,7 @@ var debugModeFunc = debugMode // so it can be addressable
 func debugMode(args []string) error {
 	fs := flag.NewFlagSet("debug", flag.ExitOnError)
 	fs.BoolVar(&debugArgs.monitor, "monitor", false, "If true, run link monitor forever. Precludes all other options.")
+	fs.BoolVar(&debugArgs.portmap, "portmap", false, "If true, run portmap debugging. Precludes all other options.")
 	fs.StringVar(&debugArgs.getURL, "get-url", "", "If non-empty, fetch provided URL.")
 	fs.StringVar(&debugArgs.derpCheck, "derp", "", "if non-empty, test a DERP ping via named region code")
 	if err := fs.Parse(args); err != nil {
@@ -55,6 +62,9 @@ func debugMode(args []string) error {
 	if debugArgs.monitor {
 		return runMonitor(ctx)
 	}
+	if debugArgs.portmap {
+		return debugPortmap(ctx)
+	}
 	if debugArgs.getURL != "" {
 		return getURL(ctx, debugArgs.getURL)
 	}
@@ -191,3 +201,83 @@ func checkDerp(ctx context.Context, derpRegion string) error {
 	log.Printf("ok")
 	return err
 }
+
+func debugPortmap(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	portmapper.VerboseLogs = true
+
+	done := make(chan bool, 1)
+
+	var c *portmapper.Client
+	logf := log.Printf
+	c = portmapper.NewClient(logger.WithPrefix(logf, "portmapper: "), func() {
+		logf("portmapping changed.")
+		logf("have mapping: %v", c.HaveMapping())
+
+		if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+			logf("cb: mapping: %v", ext)
+			select {
+			case done <- true:
+			default:
+			}
+			return
+		}
+		logf("cb: no mapping")
+	})
+	linkMon, err := monitor.New(logger.WithPrefix(logf, "monitor: "))
+	if err != nil {
+		return err
+	}
+
+	gatewayAndSelfIP := func() (gw, self netaddr.IP, ok bool) {
+		if v := os.Getenv("TS_DEBUG_GW_SELF"); strings.Contains(v, "/") {
+			i := strings.Index(v, "/")
+			gw = netaddr.MustParseIP(v[:i])
+			self = netaddr.MustParseIP(v[i+1:])
+			return gw, self, true
+		}
+		return linkMon.GatewayAndSelfIP()
+	}
+
+	c.SetGatewayLookupFunc(gatewayAndSelfIP)
+
+	gw, selfIP, ok := gatewayAndSelfIP()
+	if !ok {
+		logf("no gateway or self IP; %v", linkMon.InterfaceState())
+		return nil
+	}
+	logf("gw=%v; self=%v", gw, selfIP)
+
+	res, err := c.Probe(ctx)
+	if err != nil {
+		return fmt.Errorf("Probe: %v", err)
+	}
+	logf("Probe: %+v", res)
+
+	if !res.PCP && !res.PMP && !res.UPnP {
+		logf("no portmapping services available")
+		return nil
+	}
+
+	uc, err := net.ListenPacket("udp", "0.0.0.0:0")
+	if err != nil {
+		return err
+	}
+	defer uc.Close()
+	c.SetLocalPort(uint16(uc.LocalAddr().(*net.UDPAddr).Port))
+
+	if ext, ok := c.GetCachedMappingOrStartCreatingOne(); ok {
+		logf("mapping: %v", ext)
+	} else {
+		logf("no mapping")
+	}
+
+	select {
+	case <-done:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

cmd/tailscaled/depaware.txt

@@ -130,6 +130,8 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
+     💣 tailscale.com/tstime/mono                                    from tailscale.com/net/tstun+
+        tailscale.com/tstime/rate                                    from tailscale.com/wgengine/filter
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
         tailscale.com/types/flagtype                                 from tailscale.com/cmd/tailscaled
         tailscale.com/types/ipproto                                  from tailscale.com/net/flowtrack+

cmd/tailscaled/tailscaled.go

@@ -210,6 +210,9 @@ func run() error {
 	logf = logger.RateLimitedFn(logf, 5*time.Second, 5, 100)
 
 	if args.cleanup {
+		if os.Getenv("TS_PLEASE_PANIC") != "" {
+			panic("TS_PLEASE_PANIC asked us to panic")
+		}
 		dns.Cleanup(logf, args.tunname)
 		router.Cleanup(logf, args.tunname)
 		return nil

cmd/tsshd/tsshd.go

@@ -29,8 +29,8 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/creack/pty"
 	"github.com/gliderlabs/ssh"
-	"github.com/kr/pty"
 	gossh "golang.org/x/crypto/ssh"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"

derp/derp_server.go

@@ -167,7 +167,7 @@ type PacketForwarder interface {
 // Conn is the subset of the underlying net.Conn the DERP Server needs.
 // It is a defined type so that non-net connections can be used.
 type Conn interface {
-	io.Closer
+	io.WriteCloser
 
 	// The *Deadline methods follow the semantics of net.Conn.
 
@@ -470,8 +470,9 @@ func (s *Server) addWatcher(c *sclient) {
 }
 
 func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connNum int64) error {
-	br, bw := brw.Reader, brw.Writer
+	br := brw.Reader
 	nc.SetDeadline(time.Now().Add(10 * time.Second))
+	bw := &lazyBufioWriter{w: nc, lbw: brw.Writer}
 	if err := s.sendServerKey(bw); err != nil {
 		return fmt.Errorf("send server key: %v", err)
 	}
@@ -534,7 +535,7 @@ func (s *Server) accept(nc Conn, brw *bufio.ReadWriter, remoteAddr string, connN
 	}
 	defer s.unregisterClient(c)
 
-	err = s.sendServerInfo(bw, clientKey)
+	err = s.sendServerInfo(c.bw, clientKey)
 	if err != nil {
 		return fmt.Errorf("send server info: %v", err)
 	}
@@ -737,7 +738,7 @@ func (c *sclient) handleFrameSendPacket(ft frameType, fl uint32) error {
 // dropReason is why we dropped a DERP frame.
 type dropReason int
 
-//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go stringer -type=dropReason -trimprefix=dropReason
+//go:generate go run tailscale.com/cmd/addlicense -year 2021 -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason
 
 const (
 	dropReasonUnknownDest      dropReason = iota // unknown destination pubkey
@@ -846,18 +847,20 @@ func (s *Server) verifyClient(clientKey key.Public, info *clientInfo) error {
 	return nil
 }
 
-func (s *Server) sendServerKey(bw *bufio.Writer) error {
+func (s *Server) sendServerKey(lw *lazyBufioWriter) error {
 	buf := make([]byte, 0, len(magic)+len(s.publicKey))
 	buf = append(buf, magic...)
 	buf = append(buf, s.publicKey[:]...)
-	return writeFrame(bw, frameServerKey, buf)
+	err := writeFrame(lw.bw(), frameServerKey, buf)
+	lw.Flush() // redundant (no-op) flush to release bufio.Writer
+	return err
 }
 
 type serverInfo struct {
 	Version int `json:"version,omitempty"`
 }
 
-func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
+func (s *Server) sendServerInfo(bw *lazyBufioWriter, clientKey key.Public) error {
 	var nonce [24]byte
 	if _, err := crand.Read(nonce[:]); err != nil {
 		return err
@@ -868,7 +871,7 @@ func (s *Server) sendServerInfo(bw *bufio.Writer, clientKey key.Public) error {
 	}
 
 	msgbox := box.Seal(nil, msg, &nonce, clientKey.B32(), s.privateKey.B32())
-	if err := writeFrameHeader(bw, frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
+	if err := writeFrameHeader(bw.bw(), frameServerInfo, nonceLen+uint32(len(msgbox))); err != nil {
 		return err
 	}
 	if _, err := bw.Write(nonce[:]); err != nil {
@@ -1002,7 +1005,7 @@ type sclient struct {
 	preferred   bool
 
 	// Owned by sender, not thread-safe.
-	bw *bufio.Writer
+	bw *lazyBufioWriter
 
 	// Guarded by s.mu
 	//
@@ -1164,14 +1167,14 @@ func (c *sclient) setWriteDeadline() {
 // sendKeepAlive sends a keep-alive frame, without flushing.
 func (c *sclient) sendKeepAlive() error {
 	c.setWriteDeadline()
-	return writeFrameHeader(c.bw, frameKeepAlive, 0)
+	return writeFrameHeader(c.bw.bw(), frameKeepAlive, 0)
 }
 
 // sendPeerGone sends a peerGone frame, without flushing.
 func (c *sclient) sendPeerGone(peer key.Public) error {
 	c.s.peerGoneFrames.Add(1)
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerGone, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerGone, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1181,7 +1184,7 @@ func (c *sclient) sendPeerGone(peer key.Public) error {
 // sendPeerPresent sends a peerPresent frame, without flushing.
 func (c *sclient) sendPeerPresent(peer key.Public) error {
 	c.setWriteDeadline()
-	if err := writeFrameHeader(c.bw, framePeerPresent, keyLen); err != nil {
+	if err := writeFrameHeader(c.bw.bw(), framePeerPresent, keyLen); err != nil {
 		return err
 	}
 	_, err := c.bw.Write(peer[:])
@@ -1254,11 +1257,11 @@ func (c *sclient) sendPacket(srcKey key.Public, contents []byte) (err error) {
 	if withKey {
 		pktLen += len(srcKey)
 	}
-	if err = writeFrameHeader(c.bw, frameRecvPacket, uint32(pktLen)); err != nil {
+	if err = writeFrameHeader(c.bw.bw(), frameRecvPacket, uint32(pktLen)); err != nil {
 		return err
 	}
 	if withKey {
-		err := writePublicKey(c.bw, &srcKey)
+		err := writePublicKey(c.bw.bw(), &srcKey)
 		if err != nil {
 			return err
 		}
@@ -1573,3 +1576,45 @@ func (s *Server) ServeDebugTraffic(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(minTimeBetweenLogs)
 	}
 }
+
+var bufioWriterPool = &sync.Pool{
+	New: func() interface{} {
+		return bufio.NewWriterSize(ioutil.Discard, 2<<10)
+	},
+}
+
+// lazyBufioWriter is a bufio.Writer-like wrapping writer that lazily
+// allocates its actual bufio.Writer from a sync.Pool, releasing it to
+// the pool upon flush.
+//
+// We do this to reduce memory overhead; most DERP connections are
+// idle and the idle bufio.Writers were 30% of overall memory usage.
+type lazyBufioWriter struct {
+	w   io.Writer     // underlying
+	lbw *bufio.Writer // lazy; nil means it needs an associated buffer
+}
+
+func (w *lazyBufioWriter) bw() *bufio.Writer {
+	if w.lbw == nil {
+		w.lbw = bufioWriterPool.Get().(*bufio.Writer)
+		w.lbw.Reset(w.w)
+	}
+	return w.lbw
+}
+
+func (w *lazyBufioWriter) Available() int { return w.bw().Available() }
+
+func (w *lazyBufioWriter) Write(p []byte) (int, error) { return w.bw().Write(p) }
+
+func (w *lazyBufioWriter) Flush() error {
+	if w.lbw == nil {
+		return nil
+	}
+	err := w.lbw.Flush()
+
+	w.lbw.Reset(ioutil.Discard)
+	bufioWriterPool.Put(w.lbw)
+	w.lbw = nil
+
+	return err
+}

derp/derphttp/derphttp_client.go

@@ -54,6 +54,7 @@ type Client struct {
 
 	privateKey key.Private
 	logf       logger.Logf
+	dialer     func(ctx context.Context, network, addr string) (net.Conn, error)
 
 	// Either url or getRegion is non-nil:
 	url       *url.URL
@@ -363,10 +364,22 @@ func (c *Client) connect(ctx context.Context, caller string) (client *derp.Clien
 	return c.client, c.connGen, nil
 }
 
+// SetURLDialer sets the dialer to use for dialing URLs.
+// This dialer is only use for clients created with NewClient, not NewRegionClient.
+// If unset or nil, the default dialer is used.
+//
+// The primary use for this is the derper mesh mode to connect to each
+// other over a VPC network.
+func (c *Client) SetURLDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) {
+	c.dialer = dialer
+}
+
 func (c *Client) dialURL(ctx context.Context) (net.Conn, error) {
 	host := c.url.Hostname()
+	if c.dialer != nil {
+		return c.dialer(ctx, "tcp", net.JoinHostPort(host, urlPort(c.url)))
+	}
 	hostOrIP := host
-
 	dialer := netns.NewDialer()
 
 	if c.DNSCache != nil {

go.mod

@@ -7,7 +7,8 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go v1.38.52
 	github.com/coreos/go-iptables v0.6.0
-	github.com/dave/jennifer v1.4.1 // indirect
+	github.com/creack/pty v1.1.9
+	github.com/dave/jennifer v1.4.1
 	github.com/frankban/quicktest v1.13.0
 	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
@@ -17,11 +18,10 @@ require (
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.1.2
 	github.com/goreleaser/nfpm v1.10.3
-	github.com/iancoleman/strcase v0.2.0 // indirect
+	github.com/iancoleman/strcase v0.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/klauspost/compress v1.12.2
-	github.com/kr/pty v1.1.8
 	github.com/mdlayher/netlink v1.4.1
 	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
 	github.com/miekg/dns v1.1.42
@@ -31,8 +31,8 @@ require (
 	github.com/pkg/sftp v1.13.0
 	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 // indirect
-	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 // indirect
+	github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05
+	github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174

go.sum

@@ -82,7 +82,6 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
@@ -359,8 +358,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -584,12 +581,8 @@ github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3 h1:fEubocuQkrl
 github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3/go.mod h1:2P+hpOwd53e7JMX/L4f3VXkv1G+33ES6IWZSrkIeWNs=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw=
 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/tailscale/goupnp v1.0.1-0.20210629174436-7df6c9efe30c h1:F+pROyGPs+9wdB7jBPHr9IZEF8SKj9YUCFFShnyLNZM=
-github.com/tailscale/goupnp v1.0.1-0.20210629174436-7df6c9efe30c/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
-github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683 h1:ZXmZQuVebYllEJL/dpttpIDGx723ezC5GJkHIu0YKrM=
-github.com/tailscale/goupnp v1.0.1-0.20210629175715-39c5a55db683/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
-github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2 h1:AIJ8AF9O7jBmCwilP0ydwJMIzW5dw48Us8f3hLJhYBY=
-github.com/tailscale/goupnp v1.0.1-0.20210710010003-1cf2d718bbb2/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05 h1:4chzWmimtJPxRs2O36yuGRW3f9SYV+bMTTvMBI0EKio=
+github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy9JzfWMJf1H5UJW2ip33/d4YkoKN0r67yKH1mG8=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2 h1:reREUgl2FG+o7YCsrZB8XLjnuKv5hEIWtnOdAbRAXZI=
 github.com/tailscale/hujson v0.0.0-20200924210142-dde312d0d6a2/go.mod h1:STqf+YV0ADdzk4ejtXFsGqDpATP9JoL0OB+hiFQbkdE=
 github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
@@ -698,7 +691,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -745,7 +737,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
@@ -964,8 +955,6 @@ honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzE
 honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ=
 honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 inet.af/netaddr v0.0.0-20210515010201-ad03edc7c841/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3 h1:RlarOdsmOUCCvy7Xm1JchJIGuQsuKwD/Lo1bjYmfuQI=
-inet.af/netaddr v0.0.0-20210602152128-50f8686885e3/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1 h1:mxmfTV6kjXTlFqqFETnG9FQZzNFc6AKunZVAgQ3b7WA=
 inet.af/netaddr v0.0.0-20210721214506-ce7a8ad02cc1/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 inet.af/netstack v0.0.0-20210622165351-29b14ebc044e h1:z11NK94NQcI3DA+a3pUC/2dRYTph1kPX6B0FnCaMDzk=

ipn/ipnlocal/local.go

@@ -984,6 +984,44 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 	tsaddr.TailscaleULARange(),
 }
 
+// internalAndExternalInterfaces splits interface routes into "internal"
+// and "external" sets. Internal routes are those of virtual ethernet
+// network interfaces used by guest VMs and containers, such as WSL and
+// Docker.
+//
+// Given that "internal" routes don't leave the device, we choose to
+// trust them more, allowing access to them when an Exit Node is enabled.
+func internalAndExternalInterfaces() (internal, external []netaddr.IPPrefix, err error) {
+	if err := interfaces.ForeachInterfaceAddress(func(iface interfaces.Interface, pfx netaddr.IPPrefix) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
+			return
+		}
+		if pfx.IsSingleIP() {
+			return
+		}
+		if runtime.GOOS == "windows" {
+			// Windows Hyper-V prefixes all MAC addresses with 00:15:5d.
+			// https://docs.microsoft.com/en-us/troubleshoot/windows-server/virtualization/default-limit-256-dynamic-mac-addresses
+			//
+			// This includes WSL2 vEthernet.
+			// Importantly: by default WSL2 /etc/resolv.conf points to
+			// a stub resolver running on the host vEthernet IP.
+			// So enabling exit nodes with the default tailnet
+			// configuration breaks WSL2 DNS without this.
+			mac := iface.Interface.HardwareAddr
+			if len(mac) == 6 && mac[0] == 0x00 && mac[1] == 0x15 && mac[2] == 0x5d {
+				internal = append(internal, pfx)
+				return
+			}
+		}
+		external = append(external, pfx)
+	}); err != nil {
+		return nil, nil, err
+	}
+
+	return internal, external, nil
+}
+
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
@@ -2119,18 +2157,21 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		if !default6 {
 			rs.Routes = append(rs.Routes, ipv6Default)
 		}
+		internalIPs, externalIPs, err := internalAndExternalInterfaces()
+		if err != nil {
+			b.logf("failed to discover interface ips: %v", err)
+		}
 		if runtime.GOOS == "linux" || runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
-			// Only allow local lan access on linux machines for now.
-			ips, _, err := interfaceRoutes()
-			if err != nil {
-				b.logf("failed to discover interface ips: %v", err)
-			}
+			rs.LocalRoutes = internalIPs // unconditionally allow access to guest VM networks
 			if prefs.ExitNodeAllowLANAccess {
-				rs.LocalRoutes = ips.Prefixes()
+				rs.LocalRoutes = append(rs.LocalRoutes, externalIPs...)
+				if len(externalIPs) != 0 {
+					b.logf("allowing exit node access to internal IPs: %v", internalIPs)
+				}
 			} else {
 				// Explicitly add routes to the local network so that we do not
 				// leak any traffic.
-				rs.Routes = append(rs.Routes, ips.Prefixes()...)
+				rs.Routes = append(rs.Routes, externalIPs...)
 			}
 		}
 	}
@@ -2743,17 +2784,18 @@ func (b *LocalBackend) CheckIPForwarding() error {
 		return nil
 	}
 
+	const suffix = "\nSubnet routes won't work without IP forwarding.\nSee https://tailscale.com/kb/1104/enable-ip-forwarding/"
 	for _, key := range keys {
 		bs, err := exec.Command("sysctl", "-n", key).Output()
 		if err != nil {
-			return fmt.Errorf("couldn't check %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't check %s (%v)%s", key, err, suffix)
 		}
 		on, err := strconv.ParseBool(string(bytes.TrimSpace(bs)))
 		if err != nil {
-			return fmt.Errorf("couldn't parse %s (%v).\nSubnet routes won't work without IP forwarding.", key, err)
+			return fmt.Errorf("couldn't parse %s (%v)%s.", key, err, suffix)
 		}
 		if !on {
-			return fmt.Errorf("%s is disabled. Subnet routes won't work.", key)
+			return fmt.Errorf("%s is disabled.%s", key, suffix)
 		}
 	}
 	return nil

ipn/ipnstate/ipnstate.go

@@ -20,6 +20,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/util/dnsname"
 )
@@ -90,7 +91,7 @@ type PeerStatus struct {
 	RxBytes       int64
 	TxBytes       int64
 	Created       time.Time // time registered with tailcontrol
-	LastWrite     time.Time // time last packet sent
+	LastWrite     mono.Time // time last packet sent
 	LastSeen      time.Time // last seen to tailcontrol
 	LastHandshake time.Time // with local wireguard
 	KeepAlive     bool
@@ -320,7 +321,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 	f("<tr><th>Peer</th><th>OS</th><th>Node</th><th>Owner</th><th>Rx</th><th>Tx</th><th>Activity</th><th>Connection</th></tr>\n")
 	f("</thead>\n<tbody>\n")
 
-	now := time.Now()
+	now := mono.Now()
 
 	var peers []*PeerStatus
 	for _, peer := range st.Peers() {
@@ -378,7 +379,7 @@ table tbody tr:nth-child(even) td { background-color: #f5f5f5; }
 		f("<td>")
 
 		// TODO: let server report this active bool instead
-		active := !ps.LastWrite.IsZero() && time.Since(ps.LastWrite) < 2*time.Minute
+		active := !ps.LastWrite.IsZero() && mono.Since(ps.LastWrite) < 2*time.Minute
 		if active {
 			if ps.Relay != "" && ps.CurAddr == "" {
 				f("relay <b>%s</b>", html.EscapeString(ps.Relay))

logpolicy/logpolicy.go

@@ -187,6 +187,10 @@ func runningUnderSystemd() bool {
 	return false
 }
 
+func redirectStderrToLogPanics() bool {
+	return runningUnderSystemd() || os.Getenv("TS_PLEASE_PANIC") != ""
+}
+
 // tryFixLogStateLocation is a temporary fixup for
 // https://github.com/tailscale/tailscale/issues/247 . We accidentally
 // wrote logging state files to /, and then later to $CACHE_DIRECTORY
@@ -435,9 +439,14 @@ func New(collection string) *Policy {
 		c.HTTPC = &http.Client{Transport: newLogtailTransport(u.Host)}
 	}
 
-	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{})
+	filchBuf, filchErr := filch.New(filepath.Join(dir, cmdName), filch.Options{
+		ReplaceStderr: redirectStderrToLogPanics(),
+	})
 	if filchBuf != nil {
 		c.Buffer = filchBuf
+		if filchBuf.OrigStderr != nil {
+			c.Stderr = filchBuf.OrigStderr
+		}
 	}
 	lw := logtail.NewLogger(c, log.Printf)
 	log.SetFlags(0) // other logflags are set on console, not here

logtail/logtail.go

@@ -118,9 +118,10 @@ type Logger struct {
 	bo             *backoff.Backoff
 	zstdEncoder    Encoder
 	uploadCancel   func()
+	explainedRaw   bool
 
 	shutdownStart chan struct{} // closed when shutdown begins
-	shutdownDone  chan struct{} // closd when shutdown complete
+	shutdownDone  chan struct{} // closed when shutdown complete
 }
 
 // SetVerbosityLevel controls the verbosity level that should be
@@ -230,6 +231,14 @@ func (l *Logger) drainPending() (res []byte) {
 			// outside of the logtail logger. Encode it.
 			// Do not add a client time, as it could have been
 			// been written a long time ago.
+			if !l.explainedRaw {
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: *** Lines prefixed with RAW-STDERR below bypassed logtail and probably come from a previous run of the program\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR: ***\n")
+				fmt.Fprintf(l.stderr, "RAW-STDERR:\n")
+				l.explainedRaw = true
+			}
+			fmt.Fprintf(l.stderr, "RAW-STDERR: %s", b)
 			b = l.encodeText(b, true)
 		}
 

net/dns/manager_windows.go

@@ -293,7 +293,7 @@ func (m windowsManager) SetDNS(cfg OSConfig) error {
 		}
 
 		t0 = time.Now()
-		m.logf("running ipconfig /registerdns ...")
+		m.logf("running ipconfig /flushdns ...")
 		cmd = exec.Command("ipconfig", "/flushdns")
 		cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true}
 		err = cmd.Run()

net/dns/resolver/doh_test.go

@@ -44,7 +44,9 @@ func TestDoH(t *testing.T) {
 		t.Fatal("no known DoH")
 	}
 
-	f := new(forwarder)
+	f := &forwarder{
+		dohSem: make(chan struct{}, 10),
+	}
 
 	for ip := range knownDoH {
 		t.Run(ip.String(), func(t *testing.T) {

net/dns/resolver/forwarder_test.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolver
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"inet.af/netaddr"
+)
+
+func (rr resolverAndDelay) String() string {
+	return fmt.Sprintf("%v+%v", rr.ipp, rr.startDelay)
+}
+
+func TestResolversWithDelays(t *testing.T) {
+	// query
+	q := func(ss ...string) (ipps []netaddr.IPPort) {
+		for _, s := range ss {
+			ipps = append(ipps, netaddr.MustParseIPPort(s))
+		}
+		return
+	}
+	// output
+	o := func(ss ...string) (rr []resolverAndDelay) {
+		for _, s := range ss {
+			var d time.Duration
+			if i := strings.Index(s, "+"); i != -1 {
+				var err error
+				d, err = time.ParseDuration(s[i+1:])
+				if err != nil {
+					panic(fmt.Sprintf("parsing duration in %q: %v", s, err))
+				}
+				s = s[:i]
+			}
+			rr = append(rr, resolverAndDelay{
+				ipp:        netaddr.MustParseIPPort(s),
+				startDelay: d,
+			})
+		}
+		return
+	}
+
+	tests := []struct {
+		name string
+		in   []netaddr.IPPort
+		want []resolverAndDelay
+	}{
+		{
+			name: "unknown-no-delays",
+			in:   q("1.2.3.4:53", "2.3.4.5:53"),
+			want: o("1.2.3.4:53", "2.3.4.5:53"),
+		},
+		{
+			name: "google-all-ipv4",
+			in:   q("8.8.8.8:53", "8.8.4.4:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms"),
+		},
+		{
+			name: "google-only-ipv6",
+			in:   q("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53+200ms"),
+		},
+		{
+			name: "google-all-four",
+			in:   q("8.8.8.8:53", "8.8.4.4:53", "[2001:4860:4860::8888]:53", "[2001:4860:4860::8844]:53"),
+			want: o("8.8.8.8:53", "8.8.4.4:53+200ms", "[2001:4860:4860::8888]:53+2.5s", "[2001:4860:4860::8844]:53+2.7s"),
+		},
+		{
+			name: "quad9-one-v4-one-v6",
+			in:   q("9.9.9.9:53", "[2620:fe::fe]:53"),
+			want: o("9.9.9.9:53", "[2620:fe::fe]:53+200ms"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := resolversWithDelays(tt.in)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+
+}

net/dns/resolver/tsdns_test.go

@@ -931,8 +931,11 @@ func TestAllocs(t *testing.T) {
 		query []byte
 		want  int
 	}{
-		// Name lowercasing and response slice created by dns.NewBuilder.
-		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 2},
+		// Name lowercasing, response slice created by dns.NewBuilder,
+		// and closure allocation from go call.
+		// (Closure allocation only happens when using new register ABI,
+		// which is amd64 with Go 1.17, and probably more platforms later.)
+		{"forward", dnspacket("test1.ipn.dev.", dns.TypeA, noEdns), 3},
 		// 3 extra allocs in rdnsNameToIPv4 and one in marshalPTRRecord (dns.NewName).
 		{"reverse", dnspacket("4.3.2.1.in-addr.arpa.", dns.TypePTR, noEdns), 5},
 	}

net/dnsfallback/dns-fallback-servers.json

@@ -90,18 +90,25 @@
 			"RegionName": "r4",
 			"Nodes": [
 				{
-					"Name": "4a",
+					"Name": "4c",
 					"RegionID": 4,
-					"HostName": "derp4.tailscale.com",
-					"IPv4": "167.172.182.26",
-					"IPv6": "2a03:b0c0:3:e0::36e:9001"
+					"HostName": "derp4c.tailscale.com",
+					"IPv4": "134.122.77.138",
+					"IPv6": "2a03:b0c0:3:d0::1501:6001"
 				},
 				{
-					"Name": "4b",
+					"Name": "4d",
 					"RegionID": 4,
-					"HostName": "derp4b.tailscale.com",
-					"IPv4": "157.230.25.0",
-					"IPv6": "2a03:b0c0:3:e0::58f:3001"
+					"HostName": "derp4d.tailscale.com",
+					"IPv4": "134.122.94.167",
+					"IPv6": "2a03:b0c0:3:d0::1501:b001"
+				},
+				{
+					"Name": "4e",
+					"RegionID": 4,
+					"HostName": "derp4e.tailscale.com",
+					"IPv4": "134.122.74.153",
+					"IPv6": "2a03:b0c0:3:d0::29:9001"
 				}
 			]
 		},
@@ -152,19 +159,26 @@
 			"RegionCode": "r8",
 			"RegionName": "r8",
 			"Nodes": [
-				{
-					"Name": "8a",
-					"RegionID": 8,
-					"HostName": "derp8.tailscale.com",
-					"IPv4": "167.71.139.179",
-					"IPv6": "2a03:b0c0:1:e0::3cc:e001"
-				},
 				{
 					"Name": "8b",
 					"RegionID": 8,
 					"HostName": "derp8b.tailscale.com",
 					"IPv4": "46.101.74.201",
 					"IPv6": "2a03:b0c0:1:d0::ec1:e001"
+				},
+				{
+					"Name": "8c",
+					"RegionID": 8,
+					"HostName": "derp8c.tailscale.com",
+					"IPv4": "206.189.16.32",
+					"IPv6": "2a03:b0c0:1:d0::e1f:4001"
+				},
+				{
+					"Name": "8d",
+					"RegionID": 8,
+					"HostName": "derp8d.tailscale.com",
+					"IPv4": "178.62.44.132",
+					"IPv6": "2a03:b0c0:1:d0::e08:e001"
 				}
 			]
 		},

net/interfaces/interfaces.go

@@ -134,7 +134,7 @@ func LocalAddresses() (regular, loopback []netaddr.IP, err error) {
 					// but their OS supports IPv6 so they have an fe80::
 					// address. We don't want to report all of those
 					// IPv6 LL to Control.
-				} else if ip.Is6() && tsaddr.IsULA(ip) {
+				} else if ip.Is6() && ip.IsPrivate() {
 					// Google Cloud Run uses NAT with IPv6 Unique
 					// Local Addresses to provide IPv6 connectivity.
 					ula6 = append(ula6, ip)
@@ -548,7 +548,7 @@ func isUsableV4(ip netaddr.IP) bool {
 // (fc00::/7) are in some environments used with address translation.
 func isUsableV6(ip netaddr.IP) bool {
 	return v6Global1.Contains(ip) ||
-		(tsaddr.IsULA(ip) && !tsaddr.TailscaleULARange().Contains(ip))
+		(ip.Is6() && ip.IsPrivate() && !tsaddr.TailscaleULARange().Contains(ip))
 }
 
 var (

net/interfaces/interfaces_test.go

@@ -58,6 +58,8 @@ func TestIsUsableV6(t *testing.T) {
 		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
 		{"Link Local", "fe80::1", false},
 		{"Global", "2602::1", true},
+		{"IPv4 public", "192.0.2.1", false},
+		{"IPv4 private", "192.168.1.1", false},
 	}
 
 	for _, test := range tests {

net/portmapper/disabled_stubs.go

@@ -15,10 +15,6 @@ import (
 
 type upnpClient interface{}
 
-func getUPnPClient(ctx context.Context, gw netaddr.IP) (upnpClient, error) {
-	return nil, nil
-}
-
 func (c *Client) getUPnPPortMapping(
 	ctx context.Context,
 	gw netaddr.IP,

net/portmapper/portmapper.go

@@ -14,9 +14,11 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"sync"
 	"time"
 
+	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/net/netns"
@@ -62,8 +64,11 @@ type Client struct {
 	pmpPubIPTime time.Time  // time pmpPubIP last verified
 	pmpLastEpoch uint32
 
-	pcpSawTime  time.Time // time we last saw PCP was available
-	uPnPSawTime time.Time // time we last saw UPnP was available
+	pcpSawTime time.Time // time we last saw PCP was available
+
+	uPnPSawTime    time.Time         // time we last saw UPnP was available
+	uPnPMeta       uPnPDiscoResponse // Location header from UPnP UDP discovery response
+	uPnPHTTPClient *http.Client      // nil until needed
 
 	localPort uint16
 
@@ -560,27 +565,9 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	defer cancel()
 	defer closeCloserOnContextDone(ctx, uc)()
 
-	if c.sawUPnPRecently() {
-		res.UPnP = true
-	} else {
-		hasUPnP := make(chan bool, 1)
-		defer func() {
-			res.UPnP = <-hasUPnP
-		}()
-		go func() {
-			client, err := getUPnPClient(ctx, gw)
-			if err == nil && client != nil {
-				hasUPnP <- true
-				c.mu.Lock()
-				c.uPnPSawTime = time.Now()
-				c.mu.Unlock()
-			}
-			close(hasUPnP)
-		}()
-	}
-
 	pcpAddr := netaddr.IPPortFrom(gw, pcpPort).UDPAddr()
 	pmpAddr := netaddr.IPPortFrom(gw, pmpPort).UDPAddr()
+	upnpAddr := netaddr.IPPortFrom(gw, upnpPort).UDPAddr()
 
 	// Don't send probes to services that we recently learned (for
 	// the same gw/myIP) are available. See
@@ -595,11 +582,16 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	} else {
 		uc.WriteTo(pcpAnnounceRequest(myIP), pcpAddr)
 	}
+	if c.sawUPnPRecently() {
+		res.UPnP = true
+	} else {
+		uc.WriteTo(uPnPPacket, upnpAddr)
+	}
 
 	buf := make([]byte, 1500)
 	pcpHeard := false // true when we get any PCP response
 	for {
-		if pcpHeard && res.PMP {
+		if pcpHeard && res.PMP && res.UPnP {
 			// Nothing more to discover.
 			return res, nil
 		}
@@ -612,6 +604,19 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 		}
 		port := addr.(*net.UDPAddr).Port
 		switch port {
+		case upnpPort:
+			if mem.Contains(mem.B(buf[:n]), mem.S(":InternetGatewayDevice:")) {
+				meta, err := parseUPnPDiscoResponse(buf[:n])
+				if err != nil {
+					c.logf("unrecognized UPnP discovery response; ignoring")
+				}
+				// log.Printf("UPnP reply %+v, %q", meta, buf[:n])
+				res.UPnP = true
+				c.mu.Lock()
+				c.uPnPSawTime = time.Now()
+				c.uPnPMeta = meta
+				c.mu.Unlock()
+			}
 		case pcpPort: // same as pmpPort
 			if pres, ok := parsePCPResponse(buf[:n]); ok {
 				if pres.OpCode == pcpOpReply|pcpOpAnnounce {
@@ -724,3 +729,13 @@ func parsePCPResponse(b []byte) (res pcpResponse, ok bool) {
 }
 
 var pmpReqExternalAddrPacket = []byte{0, 0} // version 0, opcode 0 = "Public address request"
+
+const (
+	upnpPort = 1900
+)
+
+var uPnPPacket = []byte("M-SEARCH * HTTP/1.1\r\n" +
+	"HOST: 239.255.255.250:1900\r\n" +
+	"ST: ssdp:all\r\n" +
+	"MAN: \"ssdp:discover\"\r\n" +
+	"MX: 2\r\n\r\n")

net/portmapper/upnp.go

@@ -8,17 +8,27 @@
 package portmapper
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"fmt"
+	"log"
 	"math/rand"
+	"net/http"
 	"net/url"
 	"time"
 
+	"github.com/tailscale/goupnp"
 	"github.com/tailscale/goupnp/dcps/internetgateway2"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlknobs"
+	"tailscale.com/net/netns"
 )
 
+// VerboseLogs controls verbose debug logging.
+// It exists for use by "tailscaled debug --portmap".
+var VerboseLogs bool
+
 // References:
 //
 // WANIP Connection v2: http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v2-Service.pdf
@@ -44,7 +54,8 @@ func (u *upnpMapping) Release(ctx context.Context) {
 }
 
 // upnpClient is an interface over the multiple different clients exported by goupnp,
-// exposing the functions we need for portmapping. They are auto-generated from XML-specs.
+// exposing the functions we need for portmapping. Those clients are auto-generated from XML-specs,
+// which is why they're not very idiomatic.
 type upnpClient interface {
 	AddPortMapping(
 		ctx context.Context,
@@ -77,7 +88,7 @@ type upnpClient interface {
 		// greater than 0. From the spec, it appears if it is set to 0, it will switch to using
 		// 604800 seconds, but not sure why this is desired. The recommended time is 3600 seconds.
 		leaseDurationSec uint32,
-	) (err error)
+	) error
 
 	DeletePortMapping(ctx context.Context, remoteHost string, externalPort uint16, protocol string) error
 	GetExternalIPAddress(ctx context.Context) (externalIPAddress string, err error)
@@ -92,6 +103,8 @@ const tsPortMappingDesc = "tailscale-portmap"
 // behavior of calling AddPortMapping with port = 0 to specify a wildcard port.
 // It returns the new external port (which may not be identical to the external port specified),
 // or an error.
+//
+// TODO(bradfitz): also returned the actual lease duration obtained. and check it regularly.
 func addAnyPortMapping(
 	ctx context.Context,
 	upnp upnpClient,
@@ -130,51 +143,76 @@ func addAnyPortMapping(
 	return externalPort, err
 }
 
-// getUPnPClients gets a client for interfacing with UPnP, ignoring the underlying protocol for
+// getUPnPClient gets a client for interfacing with UPnP, ignoring the underlying protocol for
 // now.
 // Adapted from https://github.com/huin/goupnp/blob/master/GUIDE.md.
-func getUPnPClient(ctx context.Context, gw netaddr.IP) (upnpClient, error) {
+//
+// The gw is the detected gateway.
+//
+// The meta is the most recently parsed UDP discovery packet response
+// from the Internet Gateway Device.
+//
+// The provided ctx is not retained in the returned upnpClient, but
+// its associated HTTP client is (if set via goupnp.WithHTTPClient).
+func getUPnPClient(ctx context.Context, gw netaddr.IP, meta uPnPDiscoResponse) (upnpClient, error) {
 	if controlknobs.DisableUPnP() {
 		return nil, nil
 	}
-	ctx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
-	defer cancel()
-	// Attempt to connect over the multiple available connection types concurrently,
-	// returning the fastest.
 
-	// TODO(jknodt): this url seems super brittle? maybe discovery is better but this is faster
-	u, err := url.Parse(fmt.Sprintf("http://%s:5000/rootDesc.xml", gw))
+	if meta.Location == "" {
+		return nil, nil
+	}
+
+	if VerboseLogs {
+		log.Printf("fetching %v", meta.Location)
+	}
+	u, err := url.Parse(meta.Location)
 	if err != nil {
 		return nil, err
 	}
 
-	clients := make(chan upnpClient, 3)
-	go func() {
-		var err error
-		ip1Clients, err := internetgateway2.NewWANIPConnection1ClientsByURL(ctx, u)
-		if err == nil && len(ip1Clients) > 0 {
-			clients <- ip1Clients[0]
-		}
-	}()
-	go func() {
-		ip2Clients, err := internetgateway2.NewWANIPConnection2ClientsByURL(ctx, u)
-		if err == nil && len(ip2Clients) > 0 {
-			clients <- ip2Clients[0]
-		}
-	}()
-	go func() {
-		ppp1Clients, err := internetgateway2.NewWANPPPConnection1ClientsByURL(ctx, u)
-		if err == nil && len(ppp1Clients) > 0 {
-			clients <- ppp1Clients[0]
-		}
-	}()
-
-	select {
-	case client := <-clients:
-		return client, nil
-	case <-ctx.Done():
-		return nil, ctx.Err()
+	ipp, err := netaddr.ParseIPPort(u.Host)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected host %q in %q", u.Host, meta.Location)
 	}
+	if ipp.IP() != gw {
+		return nil, fmt.Errorf("UPnP discovered root %q does not match gateway IP %v; ignoring UPnP",
+			meta.Location, gw)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+	defer cancel()
+
+	// This part does a network fetch.
+	root, err := goupnp.DeviceByURL(ctx, u)
+	if err != nil {
+		return nil, err
+	}
+
+	// These parts don't do a network fetch.
+	// Pick the best service type available.
+	if cc, _ := internetgateway2.NewWANIPConnection2ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANIPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	if cc, _ := internetgateway2.NewWANPPPConnection1ClientsFromRootDevice(ctx, root, u); len(cc) > 0 {
+		return cc[0], nil
+	}
+	return nil, nil
+}
+
+func (c *Client) upnpHTTPClientLocked() *http.Client {
+	if c.uPnPHTTPClient == nil {
+		c.uPnPHTTPClient = &http.Client{
+			Transport: &http.Transport{
+				DialContext:     netns.NewDialer().DialContext,
+				IdleConnTimeout: 2 * time.Second, // LAN is cheap
+			},
+		}
+	}
+	return c.uPnPHTTPClient
 }
 
 // getUPnPPortMapping attempts to create a port-mapping over the UPnP protocol. On success,
@@ -199,11 +237,17 @@ func (c *Client) getUPnPPortMapping(
 	var err error
 	c.mu.Lock()
 	oldMapping, ok := c.mapping.(*upnpMapping)
+	meta := c.uPnPMeta
+	httpClient := c.upnpHTTPClientLocked()
 	c.mu.Unlock()
 	if ok && oldMapping != nil {
 		client = oldMapping.client
 	} else {
-		client, err = getUPnPClient(ctx, gw)
+		ctx := goupnp.WithHTTPClient(ctx, httpClient)
+		client, err = getUPnPClient(ctx, gw, meta)
+		if VerboseLogs {
+			log.Printf("getUPnPClient: %T, %v", client, err)
+		}
 		if err != nil {
 			return netaddr.IPPort{}, false
 		}
@@ -221,11 +265,17 @@ func (c *Client) getUPnPPortMapping(
 		internal.IP().String(),
 		time.Second*pmpMapLifetimeSec,
 	)
+	if VerboseLogs {
+		log.Printf("addAnyPortMapping: %v, %v", newPort, err)
+	}
 	if err != nil {
 		return netaddr.IPPort{}, false
 	}
 	// TODO cache this ip somewhere?
 	extIP, err := client.GetExternalIPAddress(ctx)
+	if VerboseLogs {
+		log.Printf("client.GetExternalIPAddress: %v, %v", extIP, err)
+	}
 	if err != nil {
 		// TODO this doesn't seem right
 		return netaddr.IPPort{}, false
@@ -246,3 +296,18 @@ func (c *Client) getUPnPPortMapping(
 	c.localPort = newPort
 	return upnp.external, true
 }
+
+type uPnPDiscoResponse struct {
+	Location string
+}
+
+// parseUPnPDiscoResponse parses a UPnP HTTP-over-UDP discovery response.
+func parseUPnPDiscoResponse(body []byte) (uPnPDiscoResponse, error) {
+	var r uPnPDiscoResponse
+	res, err := http.ReadResponse(bufio.NewReaderSize(bytes.NewReader(body), 128), nil)
+	if err != nil {
+		return r, err
+	}
+	r.Location = res.Header.Get("Location")
+	return r, nil
+}

net/portmapper/upnp_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package portmapper
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+
+	"inet.af/netaddr"
+)
+
+// Google Wifi
+const (
+	googleWifiUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nUSN: uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece::urn:schemas-upnp-org:device:InternetGatewayDevice:2\r\nEXT:\r\nSERVER: Linux/5.4.0-1034-gcp UPnP/1.1 MiniUPnPd/1.9\r\nLOCATION: http://192.168.86.1:5000/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1\r\nBOOTID.UPNP.ORG: 1\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	googleWifiRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0"><specVersion><major>1</major><minor>0</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:2</deviceType><friendlyName>OnHub</friendlyName><manufacturer>Google</manufacturer><manufacturerURL>http://google.com/</manufacturerURL><modelDescription>Wireless Router</modelDescription><modelName>OnHub</modelName><modelNumber>1</modelNumber><modelURL>https://on.google.com/hub/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ece</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:Layer3Forwarding1</serviceId><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL><SCPDURL>/L3F.xml</SCPDURL></service><service><serviceType>urn:schemas-upnp-org:service:DeviceProtection:1</serviceType><serviceId>urn:upnp-org:serviceId:DeviceProtection1</serviceId><controlURL>/ctl/DP</controlURL><eventSubURL>/evt/DP</eventSubURL><SCPDURL>/DP.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:2</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ecf</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL><SCPDURL>/WANCfg.xml</SCPDURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:2</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210414</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>00000000</serialNumber><UDN>uuid:a9708184-a6c0-413a-bbac-11bcf7e30ec0</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:2</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL><SCPDURL>/WANIPCn.xml</SCPDURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>http://testwifi.here/</presentationURL></device></root>`
+)
+
+// pfSense 2.5.0-RELEASE / FreeBSD 12.2-STABLE
+const (
+	pfSenseUPnPDisco = "HTTP/1.1 200 OK\r\nCACHE-CONTROL: max-age=120\r\nST: urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nUSN: uuid:bee7052b-49e8-3597-b545-55a1e38ac11::urn:schemas-upnp-org:device:InternetGatewayDevice:1\r\nEXT:\r\nSERVER: FreeBSD/12.2-STABLE UPnP/1.1 MiniUPnPd/2.2.1\r\nLOCATION: http://192.168.1.1:2189/rootDesc.xml\r\nOPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n01-NLS: 1627958564\r\nBOOTID.UPNP.ORG: 1627958564\r\nCONFIGID.UPNP.ORG: 1337\r\n\r\n"
+
+	pfSenseRootDescXML = `<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0" configId="1337"><specVersion><major>1</major><minor>1</minor></specVersion><device><deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:1</deviceType><friendlyName>FreeBSD router</friendlyName><manufacturer>FreeBSD</manufacturer><manufacturerURL>http://www.freebsd.org/</manufacturerURL><modelDescription>FreeBSD router</modelDescription><modelName>FreeBSD router</modelName><modelNumber>2.5.0-RELEASE</modelNumber><modelURL>http://www.freebsd.org/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac11</UDN><serviceList><service><serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType><serviceId>urn:upnp-org:serviceId:L3Forwarding1</serviceId><SCPDURL>/L3F.xml</SCPDURL><controlURL>/ctl/L3F</controlURL><eventSubURL>/evt/L3F</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANDevice:1</deviceType><friendlyName>WANDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>WAN Device</modelDescription><modelName>WAN Device</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac12</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType><serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId><SCPDURL>/WANCfg.xml</SCPDURL><controlURL>/ctl/CmnIfCfg</controlURL><eventSubURL>/evt/CmnIfCfg</eventSubURL></service></serviceList><deviceList><device><deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:1</deviceType><friendlyName>WANConnectionDevice</friendlyName><manufacturer>MiniUPnP</manufacturer><manufacturerURL>http://miniupnp.free.fr/</manufacturerURL><modelDescription>MiniUPnP daemon</modelDescription><modelName>MiniUPnPd</modelName><modelNumber>20210205</modelNumber><modelURL>http://miniupnp.free.fr/</modelURL><serialNumber>BEE7052B</serialNumber><UDN>uuid:bee7052b-49e8-3597-b545-55a1e38ac13</UDN><UPC>000000000000</UPC><serviceList><service><serviceType>urn:schemas-upnp-org:service:WANIPConnection:1</serviceType><serviceId>urn:upnp-org:serviceId:WANIPConn1</serviceId><SCPDURL>/WANIPCn.xml</SCPDURL><controlURL>/ctl/IPConn</controlURL><eventSubURL>/evt/IPConn</eventSubURL></service></serviceList></device></deviceList></device></deviceList><presentationURL>https://192.168.1.1/</presentationURL></device></root>`
+)
+
+func TestParseUPnPDiscoResponse(t *testing.T) {
+	tests := []struct {
+		name    string
+		headers string
+		want    uPnPDiscoResponse
+	}{
+		{"google", googleWifiUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.86.1:5000/rootDesc.xml",
+		}},
+		{"pfsense", pfSenseUPnPDisco, uPnPDiscoResponse{
+			Location: "http://192.168.1.1:2189/rootDesc.xml",
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseUPnPDiscoResponse([]byte(tt.headers))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unexpected result:\n got: %+v\nwant: %+v\n", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetUPnPClient(t *testing.T) {
+	tests := []struct {
+		name    string
+		xmlBody string
+		want    string
+	}{
+		{"google", googleWifiRootDescXML, "*internetgateway2.WANIPConnection2"},
+		{"pfsense", pfSenseRootDescXML, "*internetgateway2.WANIPConnection1"},
+		// TODO(bradfitz): find a PPP one in the wild
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.RequestURI == "/rootDesc.xml" {
+					io.WriteString(w, tt.xmlBody)
+					return
+				}
+				http.NotFound(w, r)
+			}))
+			defer ts.Close()
+			gw, _ := netaddr.FromStdIP(ts.Listener.Addr().(*net.TCPAddr).IP)
+			c, err := getUPnPClient(context.Background(), gw, uPnPDiscoResponse{
+				Location: ts.URL + "/rootDesc.xml",
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := fmt.Sprintf("%T", c)
+			if got != tt.want {
+				t.Errorf("got %v; want %v", got, tt.want)
+			}
+		})
+	}
+}

net/tsaddr/tsaddr.go

@@ -105,11 +105,6 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	return netaddr.IPFrom16(ret)
 }
 
-func IsULA(ip netaddr.IP) bool {
-	ulaRange.Do(func() { mustPrefix(&ulaRange.v, "fc00::/7") })
-	return ulaRange.v.Contains(ip)
-}
-
 func mustPrefix(v *netaddr.IPPrefix, prefix string) {
 	var err error
 	*v, err = netaddr.ParseIPPrefix(prefix)

net/tsaddr/tsaddr_test.go

@@ -43,28 +43,6 @@ func TestCGNATRange(t *testing.T) {
 	}
 }
 
-func TestIsUla(t *testing.T) {
-	tests := []struct {
-		name string
-		ip   string
-		want bool
-	}{
-		{"first ULA", "fc00::1", true},
-		{"not ULA", "fb00::1", false},
-		{"Tailscale", "fd7a:115c:a1e0::1", true},
-		{"Cloud Run", "fddf:3978:feb1:d745::1", true},
-		{"zeros", "0000:0000:0000:0000:0000:0000:0000:0000", false},
-		{"Link Local", "fe80::1", false},
-		{"Global", "2602::1", false},
-	}
-
-	for _, test := range tests {
-		if got := IsULA(netaddr.MustParseIP(test.ip)); got != test.want {
-			t.Errorf("IsULA(%s) = %v, want %v", test.name, got, test.want)
-		}
-	}
-}
-
 func TestNewContainsIPFunc(t *testing.T) {
 	f := NewContainsIPFunc([]netaddr.IPPrefix{netaddr.MustParseIPPrefix("10.0.0.0/8")})
 	if f(netaddr.MustParseIP("8.8.8.8")) {

net/tstun/wrap.go

@@ -18,6 +18,7 @@ import (
 	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -64,7 +65,7 @@ type Wrapper struct {
 
 	closeOnce sync.Once
 
-	lastActivityAtomic int64 // unix seconds of last send or receive
+	lastActivityAtomic mono.Time // time of last send or receive
 
 	destIPActivity atomic.Value // of map[netaddr.IP]func()
 
@@ -153,9 +154,10 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 		// a goroutine should not block when setting it, even with no listeners.
 		bufferConsumed: make(chan struct{}, 1),
 		closed:         make(chan struct{}),
-		outbound:       make(chan tunReadResult),
-		eventsUpDown:   make(chan tun.Event),
-		eventsOther:    make(chan tun.Event),
+		// outbound can be unbuffered; the buffer is an optimization.
+		outbound:     make(chan tunReadResult, 1),
+		eventsUpDown: make(chan tun.Event),
+		eventsOther:  make(chan tun.Event),
 		// TODO(dmytro): (highly rate-limited) hexdumps should happen on unknown packets.
 		filterFlags: filter.LogAccepts | filter.LogDrops,
 	}
@@ -164,6 +166,7 @@ func Wrap(logf logger.Logf, tdev tun.Device) *Wrapper {
 	go tun.pumpEvents()
 	// The buffer starts out consumed.
 	tun.bufferConsumed <- struct{}{}
+	tun.noteActivity()
 
 	return tun
 }
@@ -363,16 +366,15 @@ func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
 
 // noteActivity records that there was a read or write at the current time.
 func (t *Wrapper) noteActivity() {
-	atomic.StoreInt64(&t.lastActivityAtomic, time.Now().Unix())
+	t.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 // IdleDuration reports how long it's been since the last read or write to this device.
 //
-// Its value is only accurate to roughly second granularity.
-// If there's never been activity, the duration is since 1970.
+// Its value should only be presumed accurate to roughly 10ms granularity.
+// If there's never been activity, the duration is since the wrapper was created.
 func (t *Wrapper) IdleDuration() time.Duration {
-	sec := atomic.LoadInt64(&t.lastActivityAtomic)
-	return time.Since(time.Unix(sec, 0))
+	return mono.Since(t.lastActivityAtomic.LoadAtomic())
 }
 
 func (t *Wrapper) Read(buf []byte, offset int) (int, error) {

net/tstun/wrap_test.go

@@ -10,13 +10,13 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"unsafe"
 
 	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine/filter"
@@ -335,9 +335,9 @@ func TestFilter(t *testing.T) {
 				// data was actually filtered.
 				// If it stays zero, nothing made it through
 				// to the wrapped TUN.
-				atomic.StoreInt64(&tun.lastActivityAtomic, 0)
+				tun.lastActivityAtomic.StoreAtomic(0)
 				_, err = tun.Write(tt.data, 0)
-				filtered = atomic.LoadInt64(&tun.lastActivityAtomic) == 0
+				filtered = tun.lastActivityAtomic.LoadAtomic() == 0
 			} else {
 				chtun.Outbound <- tt.data
 				n, err = tun.Read(buf[:], 0)
@@ -416,7 +416,7 @@ func TestAtomic64Alignment(t *testing.T) {
 	}
 
 	c := new(Wrapper)
-	atomic.StoreInt64(&c.lastActivityAtomic, 123)
+	c.lastActivityAtomic.StoreAtomic(mono.Now())
 }
 
 func TestPeerAPIBypass(t *testing.T) {

tailcfg/tailcfg.go

@@ -164,6 +164,12 @@ type Node struct {
 	Hostinfo   Hostinfo
 	Created    time.Time
 
+	// PrimaryRoutes are the routes from AllowedIPs that this node
+	// is currently the primary subnet router for, as determined
+	// by the control plane. It does not include the self address
+	// values from Addresses that are in AllowedIPs.
+	PrimaryRoutes []netaddr.IPPrefix `json:",omitempty"`
+
 	// LastSeen is when the node was last online. It is not
 	// updated when Online is true. It is nil if the current
 	// node doesn't have permission to know, or the node
@@ -1142,6 +1148,7 @@ func (n *Node) Equal(n2 *Node) bool {
 		eqBoolPtr(n.Online, n2.Online) &&
 		eqCIDRs(n.Addresses, n2.Addresses) &&
 		eqCIDRs(n.AllowedIPs, n2.AllowedIPs) &&
+		eqCIDRs(n.PrimaryRoutes, n2.PrimaryRoutes) &&
 		eqStrings(n.Endpoints, n2.Endpoints) &&
 		n.DERP == n2.DERP &&
 		n.Hostinfo.Equal(&n2.Hostinfo) &&

tailcfg/tailcfg_clone.go

@@ -49,6 +49,7 @@ func (src *Node) Clone() *Node {
 	dst.AllowedIPs = append(src.AllowedIPs[:0:0], src.AllowedIPs...)
 	dst.Endpoints = append(src.Endpoints[:0:0], src.Endpoints...)
 	dst.Hostinfo = *src.Hostinfo.Clone()
+	dst.PrimaryRoutes = append(src.PrimaryRoutes[:0:0], src.PrimaryRoutes...)
 	if dst.LastSeen != nil {
 		dst.LastSeen = new(time.Time)
 		*dst.LastSeen = *src.LastSeen
@@ -79,6 +80,7 @@ var _NodeNeedsRegeneration = Node(struct {
 	DERP                    string
 	Hostinfo                Hostinfo
 	Created                 time.Time
+	PrimaryRoutes           []netaddr.IPPrefix
 	LastSeen                *time.Time
 	Online                  *bool
 	KeepAlive               bool

tailcfg/tailcfg_test.go

@@ -194,7 +194,8 @@ func TestNodeEqual(t *testing.T) {
 		"ID", "StableID", "Name", "User", "Sharer",
 		"Key", "KeyExpiry", "Machine", "DiscoKey",
 		"Addresses", "AllowedIPs", "Endpoints", "DERP", "Hostinfo",
-		"Created", "LastSeen", "Online", "KeepAlive", "MachineAuthorized",
+		"Created", "PrimaryRoutes",
+		"LastSeen", "Online", "KeepAlive", "MachineAuthorized",
 		"Capabilities",
 		"ComputedName", "computedHostIfDifferent", "ComputedNameWithHost",
 	}

tstest/integration/integration_test.go

@@ -84,6 +84,40 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+func TestCollectPanic(t *testing.T) {
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n := newTestNode(t, env)
+
+	cmd := exec.Command(n.env.Binaries.Daemon, "--cleanup")
+	cmd.Env = append(os.Environ(),
+		"TS_PLEASE_PANIC=1",
+		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
+	)
+	got, _ := cmd.CombinedOutput() // we expect it to fail, ignore err
+	t.Logf("initial run: %s", got)
+
+	// Now we run it again, and on start, it will upload the logs to logcatcher.
+	cmd = exec.Command(n.env.Binaries.Daemon, "--cleanup")
+	cmd.Env = append(os.Environ(), "TS_LOG_TARGET="+n.env.LogCatcherServer.URL)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("cleanup failed: %v: %q", err, out)
+	}
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		const sub = `panic`
+		if !n.env.LogCatcher.logsContains(mem.S(sub)) {
+			return fmt.Errorf("log catcher didn't see %#q; got %s", sub, n.env.LogCatcher.logsString())
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
 // test Issue 2321: Start with UpdatePrefs should save prefs to disk
 func TestStateSavedOnStart(t *testing.T) {
 	t.Parallel()

tstest/integration/tailscaled_deps_test_darwin.go

@@ -41,6 +41,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_freebsd.go

@@ -39,6 +39,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_linux.go

@@ -39,6 +39,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_openbsd.go

@@ -39,6 +39,7 @@ import (
 	_ "tailscale.com/logpolicy"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstest/integration/tailscaled_deps_test_windows.go

@@ -45,6 +45,7 @@ import (
 	_ "tailscale.com/logtail/backoff"
 	_ "tailscale.com/net/dns"
 	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/portmapper"
 	_ "tailscale.com/net/socks5/tssocks"
 	_ "tailscale.com/net/tshttpproxy"
 	_ "tailscale.com/net/tstun"

tstime/mono/mono.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package mono provides fast monotonic time.
+// On most platforms, mono.Now is about 2x faster than time.Now.
+// However, time.Now is really fast, and nicer to use.
+//
+// For almost all purposes, you should use time.Now.
+//
+// Package mono exists because we get the current time multiple
+// times per network packet, at which point it makes a
+// measurable difference.
+package mono
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+	_ "unsafe" // for go:linkname
+)
+
+// Time is the number of nanoseconds elapsed since an unspecified reference start time.
+type Time int64
+
+// Now returns the current monotonic time.
+func Now() Time {
+	// On a newly started machine, the monotonic clock might be very near zero.
+	// Thus mono.Time(0).Before(mono.Now.Add(-time.Minute)) might yield true.
+	// The corresponding package time expression never does, if the wall clock is correct.
+	// Preserve this correspondence by increasing the "base" monotonic clock by a fair amount.
+	const baseOffset int64 = 1 << 55 // approximately 10,000 hours in nanoseconds
+	return Time(now() + baseOffset)
+}
+
+// Since returns the time elapsed since t.
+func Since(t Time) time.Duration {
+	return time.Duration(Now() - t)
+}
+
+// Sub returns t-n, the duration from n to t.
+func (t Time) Sub(n Time) time.Duration {
+	return time.Duration(t - n)
+}
+
+// Add returns t+d.
+func (t Time) Add(d time.Duration) Time {
+	return t + Time(d)
+}
+
+// After reports t > n, whether t is after n.
+func (t Time) After(n Time) bool {
+	return t > n
+}
+
+// After reports t < n, whether t is before n.
+func (t Time) Before(n Time) bool {
+	return t < n
+}
+
+// IsZero reports whether t == 0.
+func (t Time) IsZero() bool {
+	return t == 0
+}
+
+// StoreAtomic does an atomic store *t = new.
+func (t *Time) StoreAtomic(new Time) {
+	atomic.StoreInt64((*int64)(t), int64(new))
+}
+
+// LoadAtomic does an atomic load *t.
+func (t *Time) LoadAtomic() Time {
+	return Time(atomic.LoadInt64((*int64)(t)))
+}
+
+//go:linkname now runtime.nanotime1
+func now() int64
+
+// baseWall and baseMono are a pair of almost-identical times used to correlate a Time with a wall time.
+var (
+	baseWall time.Time
+	baseMono Time
+)
+
+func init() {
+	baseWall = time.Now()
+	baseMono = Now()
+}
+
+// String prints t, including an estimated equivalent wall clock.
+// This is best-effort only, for rough debugging purposes only.
+// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
+// Even in the best of circumstances, it may vary by a few milliseconds.
+func (t Time) String() string {
+	return fmt.Sprintf("mono.Time(ns=%d, estimated wall=%v)", int64(t), baseWall.Add(t.Sub(baseMono)).Truncate(0))
+}
+
+// MarshalJSON formats t for JSON as if it were a time.Time.
+// We format Time this way for backwards-compatibility.
+// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
+// Since t is a monotonic time, it can vary from the actual wall clock by arbitrary amounts.
+// Even in the best of circumstances, it may vary by a few milliseconds.
+func (t Time) MarshalJSON() ([]byte, error) {
+	var tt time.Time
+	if !t.IsZero() {
+		tt = baseWall.Add(t.Sub(baseMono)).Truncate(0)
+	}
+	return tt.MarshalJSON()
+}
+
+// UnmarshalJSON sets t according to data.
+// This is best-effort only. Time does not survive a MarshalJSON/UnmarshalJSON round trip unchanged.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	var tt time.Time
+	err := tt.UnmarshalJSON(data)
+	if err != nil {
+		return err
+	}
+	*t = Now().Add(-time.Since(tt))
+	return nil
+}

tstime/mono/mono_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package mono
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNow(t *testing.T) {
+	start := Now()
+	time.Sleep(100 * time.Millisecond)
+	if elapsed := Since(start); elapsed < 100*time.Millisecond {
+		t.Errorf("short sleep: %v elapsed, want min %v", elapsed, 100*time.Millisecond)
+	}
+}
+
+func BenchmarkMonoNow(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Now()
+	}
+}
+
+func BenchmarkTimeNow(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		time.Now()
+	}
+}

tstime/rate/rate.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a modified, simplified version of code from golang.org/x/time/rate.
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package rate provides a rate limiter.
+package rate
+
+import (
+	"sync"
+	"time"
+
+	"tailscale.com/tstime/mono"
+)
+
+// Limit defines the maximum frequency of some events.
+// Limit is represented as number of events per second.
+// A zero Limit is invalid.
+type Limit float64
+
+// Every converts a minimum time interval between events to a Limit.
+func Every(interval time.Duration) Limit {
+	if interval <= 0 {
+		panic("invalid interval")
+	}
+	return 1 / Limit(interval.Seconds())
+}
+
+// A Limiter controls how frequently events are allowed to happen.
+// It implements a "token bucket" of size b, initially full and refilled
+// at rate r tokens per second.
+// Informally, in any large enough time interval, the Limiter limits the
+// rate to r tokens per second, with a maximum burst size of b events.
+// See https://en.wikipedia.org/wiki/Token_bucket for more about token buckets.
+// Use NewLimiter to create non-zero Limiters.
+type Limiter struct {
+	limit  Limit
+	burst  float64
+	mu     sync.Mutex // protects following fields
+	tokens float64    // number of tokens currently in bucket
+	last   mono.Time  // the last time the limiter's tokens field was updated
+}
+
+// NewLimiter returns a new Limiter that allows events up to rate r and permits
+// bursts of at most b tokens.
+func NewLimiter(r Limit, b int) *Limiter {
+	if b < 1 {
+		panic("bad burst, must be at least 1")
+	}
+	return &Limiter{limit: r, burst: float64(b)}
+}
+
+// AllowN reports whether an event may happen now.
+func (lim *Limiter) Allow() bool {
+	return lim.allow(mono.Now())
+}
+
+func (lim *Limiter) allow(now mono.Time) bool {
+	lim.mu.Lock()
+	defer lim.mu.Unlock()
+
+	// If time has moved backwards, look around awkwardly and pretend nothing happened.
+	if now.Before(lim.last) {
+		lim.last = now
+	}
+
+	// Calculate the new number of tokens available due to the passage of time.
+	elapsed := now.Sub(lim.last)
+	tokens := lim.tokens + float64(lim.limit)*elapsed.Seconds()
+	if tokens > lim.burst {
+		tokens = lim.burst
+	}
+
+	// Consume a token.
+	tokens--
+
+	// Update state.
+	ok := tokens >= 0
+	if ok {
+		lim.last = now
+		lim.tokens = tokens
+	}
+	return ok
+}

tstime/rate/rate_test.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This is a modified, simplified version of code from golang.org/x/time/rate.
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.7
+// +build go1.7
+
+package rate
+
+import (
+	"context"
+	"math"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"tailscale.com/tstime/mono"
+)
+
+func closeEnough(a, b Limit) bool {
+	return (math.Abs(float64(a)/float64(b)) - 1.0) < 1e-9
+}
+
+func TestEvery(t *testing.T) {
+	cases := []struct {
+		interval time.Duration
+		lim      Limit
+	}{
+		{1 * time.Nanosecond, Limit(1e9)},
+		{1 * time.Microsecond, Limit(1e6)},
+		{1 * time.Millisecond, Limit(1e3)},
+		{10 * time.Millisecond, Limit(100)},
+		{100 * time.Millisecond, Limit(10)},
+		{1 * time.Second, Limit(1)},
+		{2 * time.Second, Limit(0.5)},
+		{time.Duration(2.5 * float64(time.Second)), Limit(0.4)},
+		{4 * time.Second, Limit(0.25)},
+		{10 * time.Second, Limit(0.1)},
+		{time.Duration(math.MaxInt64), Limit(1e9 / float64(math.MaxInt64))},
+	}
+	for _, tc := range cases {
+		lim := Every(tc.interval)
+		if !closeEnough(lim, tc.lim) {
+			t.Errorf("Every(%v) = %v want %v", tc.interval, lim, tc.lim)
+		}
+	}
+}
+
+const (
+	d = 100 * time.Millisecond
+)
+
+var (
+	t0 = mono.Now()
+	t1 = t0.Add(time.Duration(1) * d)
+	t2 = t0.Add(time.Duration(2) * d)
+	t3 = t0.Add(time.Duration(3) * d)
+	t4 = t0.Add(time.Duration(4) * d)
+	t5 = t0.Add(time.Duration(5) * d)
+	t9 = t0.Add(time.Duration(9) * d)
+)
+
+type allow struct {
+	t  mono.Time
+	ok bool
+}
+
+func run(t *testing.T, lim *Limiter, allows []allow) {
+	t.Helper()
+	for i, allow := range allows {
+		ok := lim.allow(allow.t)
+		if ok != allow.ok {
+			t.Errorf("step %d: lim.AllowN(%v) = %v want %v",
+				i, allow.t, ok, allow.ok)
+		}
+	}
+}
+
+func TestLimiterBurst1(t *testing.T) {
+	run(t, NewLimiter(10, 1), []allow{
+		{t0, true},
+		{t0, false},
+		{t0, false},
+		{t1, true},
+		{t1, false},
+		{t1, false},
+		{t2, true},
+		{t2, false},
+	})
+}
+
+func TestLimiterJumpBackwards(t *testing.T) {
+	run(t, NewLimiter(10, 3), []allow{
+		{t1, true}, // start at t1
+		{t0, true}, // jump back to t0, two tokens remain
+		{t0, true},
+		{t0, false},
+		{t0, false},
+		{t1, true}, // got a token
+		{t1, false},
+		{t1, false},
+		{t2, true}, // got another token
+		{t2, false},
+		{t2, false},
+	})
+}
+
+// Ensure that tokensFromDuration doesn't produce
+// rounding errors by truncating nanoseconds.
+// See golang.org/issues/34861.
+func TestLimiter_noTruncationErrors(t *testing.T) {
+	if !NewLimiter(0.7692307692307693, 1).Allow() {
+		t.Fatal("expected true")
+	}
+}
+
+func TestSimultaneousRequests(t *testing.T) {
+	const (
+		limit       = 1
+		burst       = 5
+		numRequests = 15
+	)
+	var (
+		wg    sync.WaitGroup
+		numOK = uint32(0)
+	)
+
+	// Very slow replenishing bucket.
+	lim := NewLimiter(limit, burst)
+
+	// Tries to take a token, atomically updates the counter and decreases the wait
+	// group counter.
+	f := func() {
+		defer wg.Done()
+		if ok := lim.Allow(); ok {
+			atomic.AddUint32(&numOK, 1)
+		}
+	}
+
+	wg.Add(numRequests)
+	for i := 0; i < numRequests; i++ {
+		go f()
+	}
+	wg.Wait()
+	if numOK != burst {
+		t.Errorf("numOK = %d, want %d", numOK, burst)
+	}
+}
+
+func TestLongRunningQPS(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	if runtime.GOOS == "openbsd" {
+		t.Skip("low resolution time.Sleep invalidates test (golang.org/issue/14183)")
+		return
+	}
+
+	// The test runs for a few seconds executing many requests and then checks
+	// that overall number of requests is reasonable.
+	const (
+		limit = 100
+		burst = 100
+	)
+	var numOK = int32(0)
+
+	lim := NewLimiter(limit, burst)
+
+	var wg sync.WaitGroup
+	f := func() {
+		if ok := lim.Allow(); ok {
+			atomic.AddInt32(&numOK, 1)
+		}
+		wg.Done()
+	}
+
+	start := time.Now()
+	end := start.Add(5 * time.Second)
+	for time.Now().Before(end) {
+		wg.Add(1)
+		go f()
+
+		// This will still offer ~500 requests per second, but won't consume
+		// outrageous amount of CPU.
+		time.Sleep(2 * time.Millisecond)
+	}
+	wg.Wait()
+	elapsed := time.Since(start)
+	ideal := burst + (limit * float64(elapsed) / float64(time.Second))
+
+	// We should never get more requests than allowed.
+	if want := int32(ideal + 1); numOK > want {
+		t.Errorf("numOK = %d, want %d (ideal %f)", numOK, want, ideal)
+	}
+	// We should get very close to the number of requests allowed.
+	if want := int32(0.999 * ideal); numOK < want {
+		t.Errorf("numOK = %d, want %d (ideal %f)", numOK, want, ideal)
+	}
+}
+
+type request struct {
+	t   time.Time
+	n   int
+	act time.Time
+	ok  bool
+}
+
+// dFromDuration converts a duration to a multiple of the global constant d
+func dFromDuration(dur time.Duration) int {
+	// Adding a millisecond to be swallowed by the integer division
+	// because we don't care about small inaccuracies
+	return int((dur + time.Millisecond) / d)
+}
+
+// dSince returns multiples of d since t0
+func dSince(t mono.Time) int {
+	return dFromDuration(t.Sub(t0))
+}
+
+type wait struct {
+	name   string
+	ctx    context.Context
+	n      int
+	delay  int // in multiples of d
+	nilErr bool
+}
+
+func BenchmarkAllowN(b *testing.B) {
+	lim := NewLimiter(Every(1*time.Second), 1)
+	now := mono.Now()
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			lim.allow(now)
+		}
+	})
+}

util/deephash/deephash.go

@@ -21,7 +21,6 @@ import (
 	"hash"
 	"math"
 	"reflect"
-	"strconv"
 	"sync"
 	"time"
 	"unsafe"
@@ -38,21 +37,15 @@ type hasher struct {
 	visitStack visitStack
 }
 
-// newHasher initializes a new hasher, for use by hasherPool.
-func newHasher() *hasher {
-	h := &hasher{h: sha256.New()}
-	h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
-	return h
-}
-
-// setBufioWriter switches the bufio writer to w after flushing
-// any output to the old one. It then also returns the old one, so
-// the caller can switch back to it.
-func (h *hasher) setBufioWriter(w *bufio.Writer) (old *bufio.Writer) {
-	old = h.bw
-	old.Flush()
-	h.bw = w
-	return old
+func (h *hasher) reset() {
+	if h.h == nil {
+		h.h = sha256.New()
+	}
+	if h.bw == nil {
+		h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
+	}
+	h.bw.Flush()
+	h.h.Reset()
 }
 
 // Sum is an opaque checksum type that is comparable.
@@ -60,6 +53,12 @@ type Sum struct {
 	sum [sha256.Size]byte
 }
 
+func (s1 *Sum) xor(s2 Sum) {
+	for i := 0; i < sha256.Size; i++ {
+		s1.sum[i] ^= s2.sum[i]
+	}
+}
+
 func (s Sum) String() string {
 	return hex.EncodeToString(s.sum[:])
 }
@@ -69,34 +68,31 @@ var (
 	seed uint64
 )
 
-// Hash returns the hash of v.
-func (h *hasher) Hash(v interface{}) (hash Sum) {
-	h.bw.Flush()
-	h.h.Reset()
-	once.Do(func() {
-		seed = uint64(time.Now().UnixNano())
-	})
-	h.uint(seed)
-	h.print(reflect.ValueOf(v))
+func (h *hasher) sum() (s Sum) {
 	h.bw.Flush()
 	// Sum into scratch & copy out, as hash.Hash is an interface
 	// so the slice necessarily escapes, and there's no sha256
 	// concrete type exported and we don't want the 'hash' result
 	// parameter to escape to the heap:
-	h.h.Sum(h.scratch[:0])
-	copy(hash.sum[:], h.scratch[:])
-	return
+	copy(s.sum[:], h.h.Sum(h.scratch[:0]))
+	return s
 }
 
 var hasherPool = &sync.Pool{
-	New: func() interface{} { return newHasher() },
+	New: func() interface{} { return new(hasher) },
 }
 
 // Hash returns the hash of v.
-func Hash(v interface{}) Sum {
+func Hash(v interface{}) (s Sum) {
 	h := hasherPool.Get().(*hasher)
 	defer hasherPool.Put(h)
-	return h.Hash(v)
+	h.reset()
+	once.Do(func() {
+		seed = uint64(time.Now().UnixNano())
+	})
+	h.hashUint64(seed)
+	h.hashValue(reflect.ValueOf(v))
+	return h.sum()
 }
 
 // Update sets last to the hash of v and reports whether its value changed.
@@ -116,19 +112,25 @@ type appenderTo interface {
 	AppendTo([]byte) []byte
 }
 
-func (h *hasher) uint(i uint64) {
-	binary.BigEndian.PutUint64(h.scratch[:8], i)
-	h.bw.Write(h.scratch[:8])
+func (h *hasher) hashUint8(i uint8) {
+	h.bw.WriteByte(i)
 }
-
-func (h *hasher) int(i int) {
-	binary.BigEndian.PutUint64(h.scratch[:8], uint64(i))
+func (h *hasher) hashUint16(i uint16) {
+	binary.LittleEndian.PutUint16(h.scratch[:2], i)
+	h.bw.Write(h.scratch[:2])
+}
+func (h *hasher) hashUint32(i uint32) {
+	binary.LittleEndian.PutUint32(h.scratch[:4], i)
+	h.bw.Write(h.scratch[:4])
+}
+func (h *hasher) hashUint64(i uint64) {
+	binary.LittleEndian.PutUint64(h.scratch[:8], i)
 	h.bw.Write(h.scratch[:8])
 }
 
 var uint8Type = reflect.TypeOf(byte(0))
 
-func (h *hasher) print(v reflect.Value) {
+func (h *hasher) hashValue(v reflect.Value) {
 	if !v.IsValid() {
 		return
 	}
@@ -155,33 +157,33 @@ func (h *hasher) print(v reflect.Value) {
 		panic(fmt.Sprintf("unhandled kind %v for type %v", v.Kind(), v.Type()))
 	case reflect.Ptr:
 		if v.IsNil() {
-			w.WriteByte(0) // indicates nil
+			h.hashUint8(0) // indicates nil
 			return
 		}
 
 		// Check for cycle.
 		ptr := pointerOf(v)
 		if idx, ok := h.visitStack.seen(ptr); ok {
-			w.WriteByte(2) // indicates cycle
-			h.uint(uint64(idx))
+			h.hashUint8(2) // indicates cycle
+			h.hashUint64(uint64(idx))
 			return
 		}
 		h.visitStack.push(ptr)
 		defer h.visitStack.pop(ptr)
 
-		w.WriteByte(1) // indicates visiting a pointer
-		h.print(v.Elem())
+		h.hashUint8(1) // indicates visiting a pointer
+		h.hashValue(v.Elem())
 	case reflect.Struct:
 		w.WriteString("struct")
-		h.int(v.NumField())
+		h.hashUint64(uint64(v.NumField()))
 		for i, n := 0, v.NumField(); i < n; i++ {
-			h.int(i)
-			h.print(v.Field(i))
+			h.hashUint64(uint64(i))
+			h.hashValue(v.Field(i))
 		}
 	case reflect.Slice, reflect.Array:
 		vLen := v.Len()
 		if v.Kind() == reflect.Slice {
-			h.int(vLen)
+			h.hashUint64(uint64(vLen))
 		}
 		if v.Type().Elem() == uint8Type && v.CanInterface() {
 			if vLen > 0 && vLen <= scratchSize {
@@ -200,96 +202,91 @@ func (h *hasher) print(v reflect.Value) {
 			// TODO(dsnet): Perform cycle detection for slices,
 			// which is functionally a list of pointers.
 			// See https://github.com/google/go-cmp/blob/402949e8139bb890c71a707b6faf6dd05c92f4e5/cmp/compare.go#L438-L450
-			h.int(i)
-			h.print(v.Index(i))
+			h.hashUint64(uint64(i))
+			h.hashValue(v.Index(i))
 		}
 	case reflect.Interface:
 		if v.IsNil() {
-			w.WriteByte(0) // indicates nil
+			h.hashUint8(0) // indicates nil
 			return
 		}
 		v = v.Elem()
 
-		w.WriteByte(1) // indicates visiting interface value
+		h.hashUint8(1) // indicates visiting interface value
 		h.hashType(v.Type())
-		h.print(v)
+		h.hashValue(v)
 	case reflect.Map:
 		// Check for cycle.
 		ptr := pointerOf(v)
 		if idx, ok := h.visitStack.seen(ptr); ok {
-			w.WriteByte(2) // indicates cycle
-			h.uint(uint64(idx))
+			h.hashUint8(2) // indicates cycle
+			h.hashUint64(uint64(idx))
 			return
 		}
 		h.visitStack.push(ptr)
 		defer h.visitStack.pop(ptr)
 
-		w.WriteByte(1) // indicates visiting a map
+		h.hashUint8(1) // indicates visiting a map
 		h.hashMap(v)
 	case reflect.String:
-		h.int(v.Len())
-		w.WriteString(v.String())
+		s := v.String()
+		h.hashUint64(uint64(len(s)))
+		w.WriteString(s)
 	case reflect.Bool:
-		w.Write(strconv.AppendBool(h.scratch[:0], v.Bool()))
-	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		w.Write(strconv.AppendInt(h.scratch[:0], v.Int(), 10))
-	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		h.uint(v.Uint())
-	case reflect.Float32, reflect.Float64:
-		w.Write(strconv.AppendUint(h.scratch[:0], math.Float64bits(v.Float()), 10))
-	case reflect.Complex64, reflect.Complex128:
-		fmt.Fprintf(w, "%v", v.Complex())
+		if v.Bool() {
+			h.hashUint8(1)
+		} else {
+			h.hashUint8(0)
+		}
+	case reflect.Int8:
+		h.hashUint8(uint8(v.Int()))
+	case reflect.Int16:
+		h.hashUint16(uint16(v.Int()))
+	case reflect.Int32:
+		h.hashUint32(uint32(v.Int()))
+	case reflect.Int64, reflect.Int:
+		h.hashUint64(uint64(v.Int()))
+	case reflect.Uint8:
+		h.hashUint8(uint8(v.Uint()))
+	case reflect.Uint16:
+		h.hashUint16(uint16(v.Uint()))
+	case reflect.Uint32:
+		h.hashUint32(uint32(v.Uint()))
+	case reflect.Uint64, reflect.Uint, reflect.Uintptr:
+		h.hashUint64(uint64(v.Uint()))
+	case reflect.Float32:
+		h.hashUint32(math.Float32bits(float32(v.Float())))
+	case reflect.Float64:
+		h.hashUint64(math.Float64bits(float64(v.Float())))
+	case reflect.Complex64:
+		h.hashUint32(math.Float32bits(real(complex64(v.Complex()))))
+		h.hashUint32(math.Float32bits(imag(complex64(v.Complex()))))
+	case reflect.Complex128:
+		h.hashUint64(math.Float64bits(real(complex128(v.Complex()))))
+		h.hashUint64(math.Float64bits(imag(complex128(v.Complex()))))
 	}
 }
 
 type mapHasher struct {
-	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
-	ebuf [sha256.Size]byte // scratch buffer
-	s256 hash.Hash         // sha256 hash.Hash
-	bw   *bufio.Writer     // to hasher into ebuf
-	val  valueCache        // re-usable values for map iteration
-	iter *reflect.MapIter  // re-usable map iterator
-}
-
-func (mh *mapHasher) Reset() {
-	for i := range mh.xbuf {
-		mh.xbuf[i] = 0
-	}
-}
-
-func (mh *mapHasher) startEntry() {
-	for i := range mh.ebuf {
-		mh.ebuf[i] = 0
-	}
-	mh.bw.Flush()
-	mh.s256.Reset()
-}
-
-func (mh *mapHasher) endEntry() {
-	mh.bw.Flush()
-	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
-		mh.xbuf[i] ^= b
-	}
+	h    hasher
+	val  valueCache      // re-usable values for map iteration
+	iter reflect.MapIter // re-usable map iterator
 }
 
 var mapHasherPool = &sync.Pool{
-	New: func() interface{} {
-		mh := new(mapHasher)
-		mh.s256 = sha256.New()
-		mh.bw = bufio.NewWriter(mh.s256)
-		mh.val = make(valueCache)
-		mh.iter = new(reflect.MapIter)
-		return mh
-	},
+	New: func() interface{} { return new(mapHasher) },
 }
 
 type valueCache map[reflect.Type]reflect.Value
 
-func (c valueCache) get(t reflect.Type) reflect.Value {
-	v, ok := c[t]
+func (c *valueCache) get(t reflect.Type) reflect.Value {
+	v, ok := (*c)[t]
 	if !ok {
 		v = reflect.New(t).Elem()
-		c[t] = v
+		if *c == nil {
+			*c = make(valueCache)
+		}
+		(*c)[t] = v
 	}
 	return v
 }
@@ -301,25 +298,22 @@ func (c valueCache) get(t reflect.Type) reflect.Value {
 func (h *hasher) hashMap(v reflect.Value) {
 	mh := mapHasherPool.Get().(*mapHasher)
 	defer mapHasherPool.Put(mh)
-	mh.Reset()
-	iter := mapIter(mh.iter, v)
-	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
-
-	// Temporarily switch to the map hasher's bufio.Writer.
-	oldw := h.setBufioWriter(mh.bw)
-	defer h.setBufioWriter(oldw)
+	iter := mapIter(&mh.iter, v)
+	defer mapIter(&mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
 
+	var sum Sum
 	k := mh.val.get(v.Type().Key())
 	e := mh.val.get(v.Type().Elem())
+	mh.h.visitStack = h.visitStack // always use the parent's visit stack to avoid cycles
 	for iter.Next() {
 		key := iterKey(iter, k)
 		val := iterVal(iter, e)
-		mh.startEntry()
-		h.print(key)
-		h.print(val)
-		mh.endEntry()
+		mh.h.reset()
+		mh.h.hashValue(key)
+		mh.h.hashValue(val)
+		sum.xor(mh.h.sum())
 	}
-	oldw.Write(mh.xbuf[:])
+	h.bw.Write(append(h.scratch[:0], sum.sum[:]...)) // append into scratch to avoid heap allocation
 }
 
 // visitStack is a stack of pointers visited.
@@ -361,5 +355,5 @@ func (h *hasher) hashType(t reflect.Type) {
 	// that maps reflect.Type to some arbitrary and unique index.
 	// While safer, it requires global state with memory that can never be GC'd.
 	rtypeAddr := reflect.ValueOf(t).Pointer() // address of *reflect.rtype
-	h.uint(uint64(rtypeAddr))
+	h.hashUint64(uint64(rtypeAddr))
 }

util/deephash/deephash_test.go

@@ -9,6 +9,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"math"
 	"reflect"
 	"testing"
 
@@ -31,12 +32,56 @@ func (p appendBytes) AppendTo(b []byte) []byte {
 func TestHash(t *testing.T) {
 	type tuple [2]interface{}
 	type iface struct{ X interface{} }
+	type scalars struct {
+		I8   int8
+		I16  int16
+		I32  int32
+		I64  int64
+		I    int
+		U8   uint8
+		U16  uint16
+		U32  uint32
+		U64  uint64
+		U    uint
+		UP   uintptr
+		F32  float32
+		F64  float64
+		C64  complex64
+		C128 complex128
+	}
 	type MyBool bool
 	type MyHeader tar.Header
 	tests := []struct {
 		in     tuple
 		wantEq bool
 	}{
+		{in: tuple{false, true}, wantEq: false},
+		{in: tuple{true, true}, wantEq: true},
+		{in: tuple{false, false}, wantEq: true},
+		{
+			in: tuple{
+				scalars{-8, -16, -32, -64, -1234, 8, 16, 32, 64, 1234, 5678, 32.32, 64.64, 32 + 32i, 64 + 64i},
+				scalars{-8, -16, -32, -64, -1234, 8, 16, 32, 64, 1234, 5678, 32.32, 64.64, 32 + 32i, 64 + 64i},
+			},
+			wantEq: true,
+		},
+		{in: tuple{scalars{I8: math.MinInt8}, scalars{I8: math.MinInt8 / 2}}, wantEq: false},
+		{in: tuple{scalars{I16: math.MinInt16}, scalars{I16: math.MinInt16 / 2}}, wantEq: false},
+		{in: tuple{scalars{I32: math.MinInt32}, scalars{I32: math.MinInt32 / 2}}, wantEq: false},
+		{in: tuple{scalars{I64: math.MinInt64}, scalars{I64: math.MinInt64 / 2}}, wantEq: false},
+		{in: tuple{scalars{I: -1234}, scalars{I: -1234 / 2}}, wantEq: false},
+		{in: tuple{scalars{U8: math.MaxUint8}, scalars{U8: math.MaxUint8 / 2}}, wantEq: false},
+		{in: tuple{scalars{U16: math.MaxUint16}, scalars{U16: math.MaxUint16 / 2}}, wantEq: false},
+		{in: tuple{scalars{U32: math.MaxUint32}, scalars{U32: math.MaxUint32 / 2}}, wantEq: false},
+		{in: tuple{scalars{U64: math.MaxUint64}, scalars{U64: math.MaxUint64 / 2}}, wantEq: false},
+		{in: tuple{scalars{U: 1234}, scalars{U: 1234 / 2}}, wantEq: false},
+		{in: tuple{scalars{UP: 5678}, scalars{UP: 5678 / 2}}, wantEq: false},
+		{in: tuple{scalars{F32: 32.32}, scalars{F32: math.Nextafter32(32.32, 0)}}, wantEq: false},
+		{in: tuple{scalars{F64: 64.64}, scalars{F64: math.Nextafter(64.64, 0)}}, wantEq: false},
+		{in: tuple{scalars{F32: float32(math.NaN())}, scalars{F32: float32(math.NaN())}}, wantEq: true},
+		{in: tuple{scalars{F64: float64(math.NaN())}, scalars{F64: float64(math.NaN())}}, wantEq: true},
+		{in: tuple{scalars{C64: 32 + 32i}, scalars{C64: complex(math.Nextafter32(32, 0), 32)}}, wantEq: false},
+		{in: tuple{scalars{C128: 64 + 64i}, scalars{C128: complex(math.Nextafter(64, 0), 64)}}, wantEq: false},
 		{in: tuple{[]appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}, []appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}}, wantEq: true},
 		{in: tuple{[]appendBytes{{}, {0, 0, 0, 0, 0, 0, 0, 1}}, []appendBytes{{0, 0, 0, 0, 0, 0, 0, 1}, {}}}, wantEq: false},
 		{in: tuple{iface{MyBool(true)}, iface{MyBool(true)}}, wantEq: true},
@@ -47,9 +92,6 @@ func TestHash(t *testing.T) {
 		{in: tuple{iface{&MyHeader{}}, iface{&tar.Header{}}}, wantEq: false},
 		{in: tuple{iface{[]map[string]MyBool{}}, iface{[]map[string]MyBool{}}}, wantEq: true},
 		{in: tuple{iface{[]map[string]bool{}}, iface{[]map[string]MyBool{}}}, wantEq: false},
-		{in: tuple{false, true}, wantEq: false},
-		{in: tuple{true, true}, wantEq: true},
-		{in: tuple{false, false}, wantEq: true},
 		{
 			in: func() tuple {
 				i1 := 1
@@ -225,10 +267,10 @@ func TestPrintArray(t *testing.T) {
 	var got bytes.Buffer
 	bw := bufio.NewWriter(&got)
 	h := &hasher{bw: bw}
-	h.print(reflect.ValueOf(x))
+	h.hashValue(reflect.ValueOf(x))
 	bw.Flush()
 	const want = "struct" +
-		"\x00\x00\x00\x00\x00\x00\x00\x01" + // 1 field
+		"\x01\x00\x00\x00\x00\x00\x00\x00" + // 1 field
 		"\x00\x00\x00\x00\x00\x00\x00\x00" + // 0th field
 		// the 32 bytes:
 		"\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f"

version/version.go

@@ -8,7 +8,7 @@ package version
 // Long is a full version number for this build, of the form
 // "x.y.z-commithash", or "date.yyyymmdd" if no actual version was
 // provided.
-var Long = "date.20210603"
+var Long = "date.20210727"
 
 // Short is a short version number for this build, of the form
 // "x.y.z", or "date.yyyymmdd" if no actual version was provided.

wgengine/filter/filter.go

@@ -11,10 +11,10 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/rate"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 )

wgengine/filter/filter_test.go

@@ -13,11 +13,11 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"golang.org/x/time/rate"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/rate"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/logger"
 )

wgengine/magicsock/legacy.go

@@ -23,6 +23,7 @@ import (
 	"golang.zx2c4.com/wireguard/tai64n"
 	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/wgkey"
@@ -348,12 +349,12 @@ type addrSet struct {
 	ipPorts []netaddr.IPPort
 
 	// clock, if non-nil, is used in tests instead of time.Now.
-	clock func() time.Time
+	clock func() mono.Time
 	Logf  logger.Logf // must not be nil
 
 	mu sync.Mutex // guards following fields
 
-	lastSend time.Time
+	lastSend mono.Time
 
 	// roamAddr is non-nil if/when we receive a correctly signed
 	// WireGuard packet from an unexpected address. If so, we
@@ -369,10 +370,10 @@ type addrSet struct {
 	curAddr int
 
 	// stopSpray is the time after which we stop spraying packets.
-	stopSpray time.Time
+	stopSpray mono.Time
 
 	// lastSpray is the last time we sprayed a packet.
-	lastSpray time.Time
+	lastSpray mono.Time
 
 	// loggedLogPriMask is a bit field of that tracks whether
 	// we've already logged about receiving a packet from a low
@@ -395,11 +396,11 @@ func (as *addrSet) derpID() int {
 	return 0
 }
 
-func (as *addrSet) timeNow() time.Time {
+func (as *addrSet) timeNow() mono.Time {
 	if as.clock != nil {
 		return as.clock()
 	}
-	return time.Now()
+	return mono.Now()
 }
 
 var noAddr, _ = netaddr.FromStdAddr(net.ParseIP("127.127.127.127"), 127, "")

wgengine/magicsock/magicsock.go

@@ -47,6 +47,7 @@ import (
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstime"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -814,20 +815,20 @@ func (c *Conn) SetNetInfoCallback(fn func(*tailcfg.NetInfo)) {
 	}
 }
 
-// LastRecvActivityOfDisco returns the time we last got traffic from
+// LastRecvActivityOfDisco describes the time we last got traffic from
 // this endpoint (updated every ~10 seconds).
-func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) time.Time {
+func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) string {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	de, ok := c.endpointOfDisco[dk]
 	if !ok {
-		return time.Time{}
+		return "never"
 	}
-	unix := atomic.LoadInt64(&de.lastRecvUnixAtomic)
-	if unix == 0 {
-		return time.Time{}
+	saw := de.lastRecv.LoadAtomic()
+	if saw == 0 {
+		return "never"
 	}
-	return time.Unix(unix, 0)
+	return mono.Since(saw).Round(time.Second).String()
 }
 
 // Ping handles a "tailscale ping" CLI query.
@@ -3126,7 +3127,7 @@ func ippDebugString(ua netaddr.IPPort) string {
 // advertise a DiscoKey and participate in active discovery.
 type discoEndpoint struct {
 	// atomically accessed; declared first for alignment reasons
-	lastRecvUnixAtomic    int64
+	lastRecv              mono.Time
 	numStopAndResetAtomic int64
 
 	// These fields are initialized once and never modified.
@@ -3145,13 +3146,13 @@ type discoEndpoint struct {
 	mu sync.Mutex // Lock ordering: Conn.mu, then discoEndpoint.mu
 
 	heartBeatTimer *time.Timer    // nil when idle
-	lastSend       time.Time      // last time there was outgoing packets sent to this peer (from wireguard-go)
-	lastFullPing   time.Time      // last time we pinged all endpoints
+	lastSend       mono.Time      // last time there was outgoing packets sent to this peer (from wireguard-go)
+	lastFullPing   mono.Time      // last time we pinged all endpoints
 	derpAddr       netaddr.IPPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients)
 
 	bestAddr           addrLatency // best non-DERP path; zero if none
-	bestAddrAt         time.Time   // time best address re-confirmed
-	trustBestAddrUntil time.Time   // time when bestAddr expires
+	bestAddrAt         mono.Time   // time best address re-confirmed
+	trustBestAddrUntil mono.Time   // time when bestAddr expires
 	sentPing           map[stun.TxID]sentPing
 	endpointState      map[netaddr.IPPort]*endpointState
 	isCallMeMaybeEP    map[netaddr.IPPort]bool
@@ -3218,7 +3219,7 @@ type endpointState struct {
 	// all fields guarded by discoEndpoint.mu
 
 	// lastPing is the last (outgoing) ping time.
-	lastPing time.Time
+	lastPing mono.Time
 
 	// lastGotPing, if non-zero, means that this was an endpoint
 	// that we learned about at runtime (from an incoming ping)
@@ -3266,14 +3267,14 @@ const pongHistoryCount = 64
 
 type pongReply struct {
 	latency time.Duration
-	pongAt  time.Time      // when we received the pong
+	pongAt  mono.Time      // when we received the pong
 	from    netaddr.IPPort // the pong's src (usually same as endpoint map key)
 	pongSrc netaddr.IPPort // what they reported they heard
 }
 
 type sentPing struct {
 	to      netaddr.IPPort
-	at      time.Time
+	at      mono.Time
 	timer   *time.Timer // timeout timer
 	purpose discoPingPurpose
 }
@@ -3293,10 +3294,10 @@ func (de *discoEndpoint) initFakeUDPAddr() {
 // endpoint and reports whether it's been at least 10 seconds since the last
 // receive activity (including having never received from this peer before).
 func (de *discoEndpoint) isFirstRecvActivityInAwhile() bool {
-	now := time.Now().Unix()
-	old := atomic.LoadInt64(&de.lastRecvUnixAtomic)
-	if old <= now-10 {
-		atomic.StoreInt64(&de.lastRecvUnixAtomic, now)
+	now := mono.Now()
+	elapsed := now.Sub(de.lastRecv.LoadAtomic())
+	if elapsed > 10*time.Second {
+		de.lastRecv.StoreAtomic(now)
 		return true
 	}
 	return false
@@ -3321,7 +3322,7 @@ func (de *discoEndpoint) DstToBytes() []byte  { return packIPPort(de.fakeWGAddr)
 // addr may be non-zero.
 //
 // de.mu must be held.
-func (de *discoEndpoint) addrForSendLocked(now time.Time) (udpAddr, derpAddr netaddr.IPPort) {
+func (de *discoEndpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netaddr.IPPort) {
 	udpAddr = de.bestAddr.IPPort
 	if udpAddr.IsZero() || now.After(de.trustBestAddrUntil) {
 		// We had a bestAddr but it expired so send both to it
@@ -3344,13 +3345,13 @@ func (de *discoEndpoint) heartbeat() {
 		return
 	}
 
-	if time.Since(de.lastSend) > sessionActiveTimeout {
+	if mono.Since(de.lastSend) > sessionActiveTimeout {
 		// Session's idle. Stop heartbeating.
 		de.c.logf("[v1] magicsock: disco: ending heartbeats for idle session to %v (%v)", de.publicKey.ShortString(), de.discoShort)
 		return
 	}
 
-	now := time.Now()
+	now := mono.Now()
 	udpAddr, _ := de.addrForSendLocked(now)
 	if !udpAddr.IsZero() {
 		// We have a preferred path. Ping that every 2 seconds.
@@ -3368,7 +3369,7 @@ func (de *discoEndpoint) heartbeat() {
 // a better path.
 //
 // de.mu must be held.
-func (de *discoEndpoint) wantFullPingLocked(now time.Time) bool {
+func (de *discoEndpoint) wantFullPingLocked(now mono.Time) bool {
 	if de.bestAddr.IsZero() || de.lastFullPing.IsZero() {
 		return true
 	}
@@ -3385,7 +3386,7 @@ func (de *discoEndpoint) wantFullPingLocked(now time.Time) bool {
 }
 
 func (de *discoEndpoint) noteActiveLocked() {
-	de.lastSend = time.Now()
+	de.lastSend = mono.Now()
 	if de.heartBeatTimer == nil {
 		de.heartBeatTimer = time.AfterFunc(heartbeatInterval, de.heartbeat)
 	}
@@ -3399,7 +3400,7 @@ func (de *discoEndpoint) cliPing(res *ipnstate.PingResult, cb func(*ipnstate.Pin
 
 	de.pendingCLIPings = append(de.pendingCLIPings, pendingCLIPing{res, cb})
 
-	now := time.Now()
+	now := mono.Now()
 	udpAddr, derpAddr := de.addrForSendLocked(now)
 	if !derpAddr.IsZero() {
 		de.startPingLocked(derpAddr, now, pingCLI)
@@ -3419,7 +3420,7 @@ func (de *discoEndpoint) cliPing(res *ipnstate.PingResult, cb func(*ipnstate.Pin
 }
 
 func (de *discoEndpoint) send(b []byte) error {
-	now := time.Now()
+	now := mono.Now()
 
 	de.mu.Lock()
 	udpAddr, derpAddr := de.addrForSendLocked(now)
@@ -3452,7 +3453,7 @@ func (de *discoEndpoint) pingTimeout(txid stun.TxID) {
 	if !ok {
 		return
 	}
-	if debugDisco || de.bestAddr.IsZero() || time.Now().After(de.trustBestAddrUntil) {
+	if debugDisco || de.bestAddr.IsZero() || mono.Now().After(de.trustBestAddrUntil) {
 		de.c.logf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort)
 	}
 	de.removeSentPingLocked(txid, sp)
@@ -3489,7 +3490,7 @@ func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLev
 // discoPingPurpose is the reason why a discovery ping message was sent.
 type discoPingPurpose int
 
-//go:generate go run tailscale.com/cmd/addlicense -year 2020 -file discopingpurpose_string.go stringer -type=discoPingPurpose -trimprefix=ping
+//go:generate go run tailscale.com/cmd/addlicense -year 2020 -file discopingpurpose_string.go go run golang.org/x/tools/cmd/stringer -type=discoPingPurpose -trimprefix=ping
 const (
 	// pingDiscovery means that purpose of a ping was to see if a
 	// path was valid.
@@ -3504,7 +3505,7 @@ const (
 	pingCLI
 )
 
-func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time, purpose discoPingPurpose) {
+func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now mono.Time, purpose discoPingPurpose) {
 	if purpose != pingCLI {
 		st, ok := de.endpointState[ep]
 		if !ok {
@@ -3530,7 +3531,7 @@ func (de *discoEndpoint) startPingLocked(ep netaddr.IPPort, now time.Time, purpo
 	go de.sendDiscoPing(ep, txid, logLevel)
 }
 
-func (de *discoEndpoint) sendPingsLocked(now time.Time, sendCallMeMaybe bool) {
+func (de *discoEndpoint) sendPingsLocked(now mono.Time, sendCallMeMaybe bool) {
 	de.lastFullPing = now
 	var sentAny bool
 	for ep, st := range de.endpointState {
@@ -3652,7 +3653,7 @@ func (de *discoEndpoint) noteConnectivityChange() {
 	de.mu.Lock()
 	defer de.mu.Unlock()
 
-	de.trustBestAddrUntil = time.Time{}
+	de.trustBestAddrUntil = 0
 }
 
 // handlePongConnLocked handles a Pong message (a reply to an earlier ping).
@@ -3670,7 +3671,7 @@ func (de *discoEndpoint) handlePongConnLocked(m *disco.Pong, src netaddr.IPPort)
 	}
 	de.removeSentPingLocked(m.TxID, sp)
 
-	now := time.Now()
+	now := mono.Now()
 	latency := now.Sub(sp.at)
 
 	if !isDerp {
@@ -3822,9 +3823,9 @@ func (de *discoEndpoint) handleCallMeMaybe(m *disco.CallMeMaybe) {
 	// Zero out all the lastPing times to force sendPingsLocked to send new ones,
 	// even if it's been less than 5 seconds ago.
 	for _, st := range de.endpointState {
-		st.lastPing = time.Time{}
+		st.lastPing = 0
 	}
-	de.sendPingsLocked(time.Now(), false)
+	de.sendPingsLocked(mono.Now(), false)
 }
 
 func (de *discoEndpoint) populatePeerStatus(ps *ipnstate.PeerStatus) {
@@ -3837,7 +3838,7 @@ func (de *discoEndpoint) populatePeerStatus(ps *ipnstate.PeerStatus) {
 
 	ps.LastWrite = de.lastSend
 
-	now := time.Now()
+	now := mono.Now()
 	if udpAddr, derpAddr := de.addrForSendLocked(now); !udpAddr.IsZero() && derpAddr.IsZero() {
 		ps.CurAddr = udpAddr.String()
 	}
@@ -3855,13 +3856,13 @@ func (de *discoEndpoint) stopAndReset() {
 
 	// Zero these fields so if the user re-starts the network, the discovery
 	// state isn't a mix of before & after two sessions.
-	de.lastSend = time.Time{}
-	de.lastFullPing = time.Time{}
+	de.lastSend = 0
+	de.lastFullPing = 0
 	de.bestAddr = addrLatency{}
-	de.bestAddrAt = time.Time{}
-	de.trustBestAddrUntil = time.Time{}
+	de.bestAddrAt = 0
+	de.trustBestAddrUntil = 0
 	for _, es := range de.endpointState {
-		es.lastPing = time.Time{}
+		es.lastPing = 0
 	}
 
 	for txid, sp := range de.sentPing {

wgengine/magicsock/magicsock_test.go

@@ -38,6 +38,7 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/tstest"
 	"tailscale.com/tstest/natlab"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
@@ -1146,13 +1147,13 @@ func TestAddrSet(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			faket := time.Unix(0, 0)
+			faket := mono.Time(0)
 			var logBuf bytes.Buffer
 			tt.as.Logf = func(format string, args ...interface{}) {
 				fmt.Fprintf(&logBuf, format, args...)
 				t.Logf(format, args...)
 			}
-			tt.as.clock = func() time.Time { return faket }
+			tt.as.clock = func() mono.Time { return faket }
 			for i, st := range tt.steps {
 				faket = faket.Add(st.advance)
 
@@ -1229,8 +1230,8 @@ func TestDiscoStringLogRace(t *testing.T) {
 func Test32bitAlignment(t *testing.T) {
 	var de discoEndpoint
 
-	if off := unsafe.Offsetof(de.lastRecvUnixAtomic); off%8 != 0 {
-		t.Fatalf("discoEndpoint.lastRecvUnixAtomic is not 8-byte aligned")
+	if off := unsafe.Offsetof(de.lastRecv); off%8 != 0 {
+		t.Fatalf("discoEndpoint.lastRecv is not 8-byte aligned")
 	}
 
 	if !de.isFirstRecvActivityInAwhile() { // verify this doesn't panic on 32-bit

wgengine/pendopen.go

@@ -231,7 +231,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	e.logf("open-conn-track: timeout opening %v to node %v; online=%v, lastRecv=%v",
 		flow, n.Key.ShortString(),
 		online,
-		durFmt(e.magicConn.LastRecvActivityOfDisco(n.DiscoKey)))
+		e.magicConn.LastRecvActivityOfDisco(n.DiscoKey))
 }
 
 func durFmt(t time.Time) string {

wgengine/userspace.go

@@ -36,6 +36,7 @@ import (
 	"tailscale.com/net/tshttpproxy"
 	"tailscale.com/net/tstun"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/ipproto"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
@@ -83,7 +84,7 @@ type userspaceEngine struct {
 	wgLogger          *wglog.Logger //a wireguard-go logging wrapper
 	reqCh             chan struct{}
 	waitCh            chan struct{} // chan is closed when first Close call completes; contrast with closing bool
-	timeNow           func() time.Time
+	timeNow           func() mono.Time
 	tundev            *tstun.Wrapper
 	wgdev             *device.Device
 	router            router.Router
@@ -111,12 +112,12 @@ type userspaceEngine struct {
 	lastEngineSigFull   deephash.Sum // of full wireguard config
 	lastEngineSigTrim   deephash.Sum // of trimmed wireguard config
 	lastDNSConfig       *dns.Config
-	recvActivityAt      map[tailcfg.DiscoKey]time.Time
+	recvActivityAt      map[tailcfg.DiscoKey]mono.Time
 	trimmedDisco        map[tailcfg.DiscoKey]bool // set of disco keys of peers currently excluded from wireguard config
-	sentActivityAt      map[netaddr.IP]*int64     // value is atomic int64 of unixtime
+	sentActivityAt      map[netaddr.IP]*mono.Time // value is accessed atomically
 	destIPActivityFuncs map[netaddr.IP]func()
 	statusBufioReader   *bufio.Reader // reusable for UAPI
-	lastStatusPollTime  time.Time     // last time we polled the engine status
+	lastStatusPollTime  mono.Time     // last time we polled the engine status
 
 	mu                  sync.Mutex         // guards following; see lock order comment below
 	netMap              *netmap.NetworkMap // or nil
@@ -238,7 +239,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	closePool.add(tsTUNDev)
 
 	e := &userspaceEngine{
-		timeNow:        time.Now,
+		timeNow:        mono.Now,
 		logf:           logf,
 		reqCh:          make(chan struct{}, 1),
 		waitCh:         make(chan struct{}),
@@ -566,7 +567,7 @@ func (e *userspaceEngine) noteReceiveActivity(dk tailcfg.DiscoKey) {
 // had a packet sent to or received from it since t.
 //
 // e.wgLock must be held.
-func (e *userspaceEngine) isActiveSince(dk tailcfg.DiscoKey, ip netaddr.IP, t time.Time) bool {
+func (e *userspaceEngine) isActiveSince(dk tailcfg.DiscoKey, ip netaddr.IP, t mono.Time) bool {
 	if e.recvActivityAt[dk].After(t) {
 		return true
 	}
@@ -574,8 +575,7 @@ func (e *userspaceEngine) isActiveSince(dk tailcfg.DiscoKey, ip netaddr.IP, t ti
 	if !ok {
 		return false
 	}
-	unixTime := atomic.LoadInt64(timePtr)
-	return unixTime >= t.Unix()
+	return timePtr.LoadAtomic().After(t)
 }
 
 // discoChanged are the set of peers whose disco keys have changed, implying they've restarted.
@@ -687,7 +687,7 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.Publ
 func (e *userspaceEngine) updateActivityMapsLocked(trackDisco []tailcfg.DiscoKey, trackIPs []netaddr.IP) {
 	// Generate the new map of which discokeys we want to track
 	// receive times for.
-	mr := map[tailcfg.DiscoKey]time.Time{} // TODO: only recreate this if set of keys changed
+	mr := map[tailcfg.DiscoKey]mono.Time{} // TODO: only recreate this if set of keys changed
 	for _, dk := range trackDisco {
 		// Preserve old times in the new map, but also
 		// populate map entries for new trackDisco values with
@@ -699,25 +699,29 @@ func (e *userspaceEngine) updateActivityMapsLocked(trackDisco []tailcfg.DiscoKey
 	e.recvActivityAt = mr
 
 	oldTime := e.sentActivityAt
-	e.sentActivityAt = make(map[netaddr.IP]*int64, len(oldTime))
+	e.sentActivityAt = make(map[netaddr.IP]*mono.Time, len(oldTime))
 	oldFunc := e.destIPActivityFuncs
 	e.destIPActivityFuncs = make(map[netaddr.IP]func(), len(oldFunc))
 
-	updateFn := func(timePtr *int64) func() {
+	updateFn := func(timePtr *mono.Time) func() {
 		return func() {
-			now := e.timeNow().Unix()
-			old := atomic.LoadInt64(timePtr)
+			now := e.timeNow()
+			old := timePtr.LoadAtomic()
 
 			// How long's it been since we last sent a packet?
-			// For our first packet, old is Unix epoch time 0 (1970).
-			elapsedSec := now - old
+			elapsed := now.Sub(old)
+			if old == 0 {
+				// For our first packet, old is 0, which has indeterminate meaning.
+				// Set elapsed to a big number (four score and seven years).
+				elapsed = 762642 * time.Hour
+			}
 
-			if elapsedSec >= int64(packetSendTimeUpdateFrequency/time.Second) {
-				atomic.StoreInt64(timePtr, now)
+			if elapsed >= packetSendTimeUpdateFrequency {
+				timePtr.StoreAtomic(now)
 			}
 			// On a big jump, assume we might no longer be in the wireguard
 			// config and go check.
-			if elapsedSec >= int64(packetSendRecheckWireguardThreshold/time.Second) {
+			if elapsed >= packetSendRecheckWireguardThreshold {
 				e.wgLock.Lock()
 				defer e.wgLock.Unlock()
 				e.maybeReconfigWireguardLocked(nil)
@@ -728,7 +732,7 @@ func (e *userspaceEngine) updateActivityMapsLocked(trackDisco []tailcfg.DiscoKey
 	for _, ip := range trackIPs {
 		timePtr := oldTime[ip]
 		if timePtr == nil {
-			timePtr = new(int64)
+			timePtr = new(mono.Time)
 		}
 		e.sentActivityAt[ip] = timePtr
 

wgengine/userspace_test.go

@@ -9,20 +9,20 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
-	"time"
 
 	"go4.org/mem"
 	"inet.af/netaddr"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
 	"tailscale.com/tailcfg"
+	"tailscale.com/tstime/mono"
 	"tailscale.com/types/key"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
 )
 
 func TestNoteReceiveActivity(t *testing.T) {
-	now := time.Unix(1, 0)
+	now := mono.Time(123456)
 	var logBuf bytes.Buffer
 
 	confc := make(chan bool, 1)
@@ -35,8 +35,8 @@ func TestNoteReceiveActivity(t *testing.T) {
 		}
 	}
 	e := &userspaceEngine{
-		timeNow:        func() time.Time { return now },
-		recvActivityAt: map[tailcfg.DiscoKey]time.Time{},
+		timeNow:        func() mono.Time { return now },
+		recvActivityAt: map[tailcfg.DiscoKey]mono.Time{},
 		logf: func(format string, a ...interface{}) {
 			fmt.Fprintf(&logBuf, format, a...)
 		},
@@ -58,7 +58,7 @@ func TestNoteReceiveActivity(t *testing.T) {
 	}
 
 	// Now track it, but don't mark it trimmed, so shouldn't update.
-	ra[dk] = time.Time{}
+	ra[dk] = 0
 	e.noteReceiveActivity(dk)
 	if len(ra) != 1 {
 		t.Fatalf("unexpected growth in map: now has %d keys; want 1", len(ra))
@@ -114,8 +114,8 @@ func TestUserspaceEngineReconfig(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		wantRecvAt := map[tailcfg.DiscoKey]time.Time{
-			dkFromHex(discoHex): time.Time{},
+		wantRecvAt := map[tailcfg.DiscoKey]mono.Time{
+			dkFromHex(discoHex): 0,
 		}
 		if got := ue.recvActivityAt; !reflect.DeepEqual(got, wantRecvAt) {
 			t.Errorf("wrong recvActivityAt\n got: %v\nwant: %v\n", got, wantRecvAt)