More logging :(

The cyolosecurity fork of certstore did not update its module name and
thus can only be used with a replace directive. This interferes with
installing using `go install` so I created a tailscale fork with an
updated module name.

Signed-off-by: Adrian Dewhurst <adrian@tailscale.com>

cmd/hello/hello.go

@@ -112,13 +112,13 @@ func tailscaleIP(who *apitype.WhoIsResponse) string {
 		return ""
 	}
 	for _, nodeIP := range who.Node.Addresses {
-		if nodeIP.IP.Is4() && nodeIP.IsSingleIP() {
-			return nodeIP.IP.String()
+		if nodeIP.IP().Is4() && nodeIP.IsSingleIP() {
+			return nodeIP.IP().String()
 		}
 	}
 	for _, nodeIP := range who.Node.Addresses {
 		if nodeIP.IsSingleIP() {
-			return nodeIP.IP.String()
+			return nodeIP.IP().String()
 		}
 	}
 	return ""

cmd/tailscale/cli/file.go

@@ -194,7 +194,7 @@ func discoverPeerAPIBase(ctx context.Context, ipStr string) (base string, lastSe
 	for _, ft := range fts {
 		n := ft.Node
 		for _, a := range n.Addresses {
-			if a.IP != ip {
+			if a.IP() != ip {
 				continue
 			}
 			if n.LastSeen != nil {
@@ -301,7 +301,7 @@ func runCpTargets(ctx context.Context, args []string) error {
 		if detail != "" {
 			detail = "\t" + detail
 		}
-		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP, n.ComputedName, detail)
+		fmt.Printf("%s\t%s%s\n", n.Addresses[0].IP(), n.ComputedName, detail)
 	}
 	return nil
 }

cmd/tailscale/cli/ping.go

@@ -64,6 +64,7 @@ var pingArgs struct {
 }
 
 func runPing(ctx context.Context, args []string) error {
+	fmt.Println("runPing")
 	c, bc, ctx, cancel := connect(ctx)
 	defer cancel()
 
@@ -139,6 +140,9 @@ func runPing(ctx context.Context, args []string) error {
 			if !anyPong {
 				return errors.New("no reply")
 			}
+			if pingArgs.untilDirect {
+				return errors.New("direct connection not established")
+			}
 			return nil
 		}
 	}

cmd/tailscale/cli/up.go

@@ -164,10 +164,10 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 		routes = append(routes, r)
 	}
 	sort.Slice(routes, func(i, j int) bool {
-		if routes[i].Bits != routes[j].Bits {
-			return routes[i].Bits < routes[j].Bits
+		if routes[i].Bits() != routes[j].Bits() {
+			return routes[i].Bits() < routes[j].Bits()
 		}
-		return routes[i].IP.Less(routes[j].IP)
+		return routes[i].IP().Less(routes[j].IP())
 	})
 
 	var exitNodeIP netaddr.IP
@@ -230,7 +230,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo
 			warnf("netfilter=nodivert; add iptables calls to ts-* chains manually.")
 		case "off":
 			prefs.NetfilterMode = preftype.NetfilterOff
-			warnf("netfilter=off; configure iptables yourself.")
+			if defaultNetfilterMode() != "off" {
+				warnf("netfilter=off; configure iptables yourself.")
+			}
 		default:
 			return nil, fmt.Errorf("invalid value --netfilter-mode=%q", upArgs.netfilterMode)
 		}
@@ -266,7 +268,7 @@ func runUp(ctx context.Context, args []string) error {
 	}
 
 	if distro.Get() == distro.Synology {
-		notSupported := "not yet supported on Synology; see https://github.com/tailscale/tailscale/issues/451"
+		notSupported := "not supported on Synology; see https://github.com/tailscale/tailscale/issues/1995"
 		if upArgs.acceptRoutes {
 			return errors.New("--accept-routes is " + notSupported)
 		}
@@ -723,10 +725,10 @@ func fmtFlagValueArg(flagName string, val interface{}) string {
 func hasExitNodeRoutes(rr []netaddr.IPPrefix) bool {
 	var v4, v6 bool
 	for _, r := range rr {
-		if r.Bits == 0 {
-			if r.IP.Is4() {
+		if r.Bits() == 0 {
+			if r.IP().Is4() {
 				v4 = true
-			} else if r.IP.Is6() {
+			} else if r.IP().Is6() {
 				v6 = true
 			}
 		}
@@ -743,7 +745,7 @@ func withoutExitNodes(rr []netaddr.IPPrefix) []netaddr.IPPrefix {
 	}
 	var out []netaddr.IPPrefix
 	for _, r := range rr {
-		if r.Bits > 0 {
+		if r.Bits() > 0 {
 			out = append(out, r)
 		}
 	}

cmd/tailscale/cli/web.go

@@ -214,7 +214,8 @@ func webHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		url, err := tailscaleUpForceReauth(r.Context())
 		if err != nil {
-			json.NewEncoder(w).Encode(mi{"error": err})
+			w.WriteHeader(500)
+			json.NewEncoder(w).Encode(mi{"error": err.Error()})
 			return
 		}
 		json.NewEncoder(w).Encode(mi{"url": url})
@@ -320,6 +321,10 @@ func tailscaleUpForceReauth(ctx context.Context) (authURL string, retErr error)
 	})
 	bc.StartLoginInteractive()
 
+	<-pumpCtx.Done() // wait for authURL or complete failure
+	if authURL == "" && retErr == nil {
+		retErr = pumpCtx.Err()
+	}
 	if authURL == "" && retErr == nil {
 		return "", fmt.Errorf("login failed with no backend error message")
 	}

cmd/tailscale/depaware.txt

@@ -1,6 +1,7 @@
 tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/depaware)
 
-   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
         github.com/kballard/go-shellquote                            from tailscale.com/cmd/tailscale/cli
         github.com/peterbourgon/ff/v2                                from github.com/peterbourgon/ff/v2/ffcli
@@ -48,7 +49,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
         tailscale.com/types/opt                                      from tailscale.com/net/netcheck+
         tailscale.com/types/persist                                  from tailscale.com/ipn
         tailscale.com/types/preftype                                 from tailscale.com/cmd/tailscale/cli+
-        tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/ipn+
         tailscale.com/types/wgkey                                    from tailscale.com/types/netmap+
         tailscale.com/util/dnsname                                   from tailscale.com/cmd/tailscale/cli+

cmd/tailscaled/depaware.txt

@@ -1,41 +1,42 @@
 tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/depaware)
 
-   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate
+   W 💣 github.com/alexbrainman/sspi                                 from github.com/alexbrainman/sspi/negotiate+
+   W    github.com/alexbrainman/sspi/internal/common                 from github.com/alexbrainman/sspi/negotiate
    W 💣 github.com/alexbrainman/sspi/negotiate                       from tailscale.com/net/tshttpproxy
    L    github.com/coreos/go-iptables/iptables                       from tailscale.com/wgengine/router
-   W 💣 github.com/github/certstore                                  from tailscale.com/control/controlclient
         github.com/go-multierror/multierror                          from tailscale.com/wgengine/router+
    W 💣 github.com/go-ole/go-ole                                     from github.com/go-ole/go-ole/oleutil+
    W 💣 github.com/go-ole/go-ole/oleutil                             from tailscale.com/wgengine/winnet
    L 💣 github.com/godbus/dbus/v5                                    from tailscale.com/net/dns
+        github.com/golang/snappy                                     from github.com/klauspost/compress/zstd
         github.com/google/btree                                      from inet.af/netstack/tcpip/header+
    L    github.com/josharian/native                                  from github.com/mdlayher/netlink+
    L 💣 github.com/jsimonetti/rtnetlink                              from tailscale.com/wgengine/monitor
    L    github.com/jsimonetti/rtnetlink/internal/unix                from github.com/jsimonetti/rtnetlink
         github.com/klauspost/compress/fse                            from github.com/klauspost/compress/huff0
         github.com/klauspost/compress/huff0                          from github.com/klauspost/compress/zstd
-        github.com/klauspost/compress/snappy                         from github.com/klauspost/compress/zstd
         github.com/klauspost/compress/zstd                           from tailscale.com/smallzstd
         github.com/klauspost/compress/zstd/internal/xxhash           from github.com/klauspost/compress/zstd
    L 💣 github.com/mdlayher/netlink                                  from github.com/jsimonetti/rtnetlink+
    L 💣 github.com/mdlayher/netlink/nlenc                            from github.com/jsimonetti/rtnetlink+
    L    github.com/mdlayher/sdnotify                                 from tailscale.com/util/systemd
-   W    github.com/pkg/errors                                        from github.com/github/certstore
-     💣 github.com/tailscale/wireguard-go/conn                       from github.com/tailscale/wireguard-go/device+
-   W 💣 github.com/tailscale/wireguard-go/conn/winrio                from github.com/tailscale/wireguard-go/conn
-     💣 github.com/tailscale/wireguard-go/device                     from tailscale.com/wgengine+
-     💣 github.com/tailscale/wireguard-go/ipc                        from github.com/tailscale/wireguard-go/device
-   W 💣 github.com/tailscale/wireguard-go/ipc/winpipe                from github.com/tailscale/wireguard-go/ipc
-        github.com/tailscale/wireguard-go/ratelimiter                from github.com/tailscale/wireguard-go/device
-        github.com/tailscale/wireguard-go/replay                     from github.com/tailscale/wireguard-go/device
-        github.com/tailscale/wireguard-go/rwcancel                   from github.com/tailscale/wireguard-go/device+
-        github.com/tailscale/wireguard-go/tai64n                     from github.com/tailscale/wireguard-go/device+
-     💣 github.com/tailscale/wireguard-go/tun                        from github.com/tailscale/wireguard-go/device+
-   W 💣 github.com/tailscale/wireguard-go/tun/wintun                 from github.com/tailscale/wireguard-go/tun+
+   W    github.com/pkg/errors                                        from github.com/tailscale/certstore
+   W 💣 github.com/tailscale/certstore                               from tailscale.com/control/controlclient
         github.com/tcnksm/go-httpstat                                from tailscale.com/net/netcheck
      💣 go4.org/intern                                               from inet.af/netaddr
      💣 go4.org/mem                                                  from tailscale.com/control/controlclient+
         go4.org/unsafe/assume-no-moving-gc                           from go4.org/intern
+     💣 golang.zx2c4.com/wireguard/conn                              from golang.zx2c4.com/wireguard/device+
+   W 💣 golang.zx2c4.com/wireguard/conn/winrio                       from golang.zx2c4.com/wireguard/conn
+     💣 golang.zx2c4.com/wireguard/device                            from tailscale.com/net/tstun+
+     💣 golang.zx2c4.com/wireguard/ipc                               from golang.zx2c4.com/wireguard/device
+   W 💣 golang.zx2c4.com/wireguard/ipc/winpipe                       from golang.zx2c4.com/wireguard/ipc
+        golang.zx2c4.com/wireguard/ratelimiter                       from golang.zx2c4.com/wireguard/device
+        golang.zx2c4.com/wireguard/replay                            from golang.zx2c4.com/wireguard/device
+        golang.zx2c4.com/wireguard/rwcancel                          from golang.zx2c4.com/wireguard/device+
+        golang.zx2c4.com/wireguard/tai64n                            from golang.zx2c4.com/wireguard/device+
+     💣 golang.zx2c4.com/wireguard/tun                               from golang.zx2c4.com/wireguard/device+
+   W 💣 golang.zx2c4.com/wireguard/tun/wintun                        from golang.zx2c4.com/wireguard/tun+
    W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg           from tailscale.com/net/interfaces+
         inet.af/netaddr                                              from tailscale.com/control/controlclient+
      💣 inet.af/netstack/gohacks                                     from inet.af/netstack/state/wire+
@@ -69,6 +70,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         inet.af/netstack/tcpip/transport/udp                         from inet.af/netstack/tcpip/adapters/gonet+
         inet.af/netstack/waiter                                      from inet.af/netstack/tcpip+
         inet.af/peercred                                             from tailscale.com/ipn/ipnserver
+   W 💣 inet.af/wf                                                   from tailscale.com/wf
         rsc.io/goversion/version                                     from tailscale.com/version
         tailscale.com/atomicfile                                     from tailscale.com/ipn+
         tailscale.com/client/tailscale/apitype                       from tailscale.com/ipn/ipnlocal+
@@ -115,7 +117,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/smallzstd                                      from tailscale.com/ipn/ipnserver+
         tailscale.com/syncs                                          from tailscale.com/net/interfaces+
         tailscale.com/tailcfg                                        from tailscale.com/control/controlclient+
-   W 💣 tailscale.com/tempfork/wireguard-windows/firewall            from tailscale.com/cmd/tailscaled
    W    tailscale.com/tsconst                                        from tailscale.com/net/interfaces
         tailscale.com/tstime                                         from tailscale.com/wgengine/magicsock
         tailscale.com/types/empty                                    from tailscale.com/control/controlclient+
@@ -128,7 +129,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/types/opt                                      from tailscale.com/control/controlclient+
         tailscale.com/types/persist                                  from tailscale.com/control/controlclient+
         tailscale.com/types/preftype                                 from tailscale.com/ipn+
-        tailscale.com/types/strbuilder                               from tailscale.com/net/packet
         tailscale.com/types/structs                                  from tailscale.com/control/controlclient+
         tailscale.com/types/wgkey                                    from tailscale.com/control/controlclient+
    L    tailscale.com/util/cmpver                                    from tailscale.com/net/dns
@@ -143,6 +143,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/util/winutil                                   from tailscale.com/logpolicy+
         tailscale.com/version                                        from tailscale.com/cmd/tailscaled+
         tailscale.com/version/distro                                 from tailscale.com/control/controlclient+
+   W    tailscale.com/wf                                             from tailscale.com/cmd/tailscaled
         tailscale.com/wgengine                                       from tailscale.com/cmd/tailscaled+
         tailscale.com/wgengine/filter                                from tailscale.com/control/controlclient+
         tailscale.com/wgengine/magicsock                             from tailscale.com/wgengine+
@@ -154,7 +155,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         tailscale.com/wgengine/wglog                                 from tailscale.com/wgengine
    W 💣 tailscale.com/wgengine/winnet                                from tailscale.com/wgengine/router
         golang.org/x/crypto/blake2b                                  from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/blake2s                                  from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/blake2s                                  from golang.zx2c4.com/wireguard/device+
         golang.org/x/crypto/chacha20                                 from golang.org/x/crypto/chacha20poly1305
         golang.org/x/crypto/chacha20poly1305                         from crypto/tls+
         golang.org/x/crypto/cryptobyte                               from crypto/ecdsa+
@@ -163,7 +164,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/crypto/hkdf                                     from crypto/tls
         golang.org/x/crypto/nacl/box                                 from tailscale.com/control/controlclient+
         golang.org/x/crypto/nacl/secretbox                           from golang.org/x/crypto/nacl/box
-        golang.org/x/crypto/poly1305                                 from github.com/tailscale/wireguard-go/device+
+        golang.org/x/crypto/poly1305                                 from golang.org/x/crypto/chacha20poly1305+
         golang.org/x/crypto/salsa20/salsa                            from golang.org/x/crypto/nacl/box+
         golang.org/x/net/bpf                                         from github.com/mdlayher/netlink+
         golang.org/x/net/dns/dnsmessage                              from net+
@@ -171,15 +172,15 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
         golang.org/x/net/http/httpproxy                              from net/http
         golang.org/x/net/http2/hpack                                 from net/http
         golang.org/x/net/idna                                        from golang.org/x/net/http/httpguts+
-        golang.org/x/net/ipv4                                        from github.com/tailscale/wireguard-go/device
-        golang.org/x/net/ipv6                                        from github.com/tailscale/wireguard-go/device+
+        golang.org/x/net/ipv4                                        from golang.zx2c4.com/wireguard/device
+        golang.org/x/net/ipv6                                        from golang.zx2c4.com/wireguard/device+
         golang.org/x/net/proxy                                       from tailscale.com/net/netns
    D    golang.org/x/net/route                                       from net+
         golang.org/x/sync/errgroup                                   from tailscale.com/derp
         golang.org/x/sync/singleflight                               from tailscale.com/net/dnscache
         golang.org/x/sys/cpu                                         from golang.org/x/crypto/blake2b+
   LD    golang.org/x/sys/unix                                        from github.com/jsimonetti/rtnetlink/internal/unix+
-   W    golang.org/x/sys/windows                                     from github.com/tailscale/wireguard-go/conn+
+   W    golang.org/x/sys/windows                                     from github.com/go-ole/go-ole+
    W    golang.org/x/sys/windows/registry                            from golang.zx2c4.com/wireguard/windows/tunnel/winipcfg+
    W    golang.org/x/sys/windows/svc                                 from tailscale.com/cmd/tailscaled+
    W    golang.org/x/sys/windows/svc/mgr                             from tailscale.com/cmd/tailscaled

cmd/tailscaled/tailscaled.go

@@ -266,7 +266,7 @@ func run() error {
 			if err != nil {
 				return nil, err
 			}
-			if ns != nil && useNetstackForIP(ipp.IP) {
+			if ns != nil && useNetstackForIP(ipp.IP()) {
 				return ns.DialContextTCP(ctx, addr)
 			}
 			var d net.Dialer

cmd/tailscaled/tailscaled_windows.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"net"
 	"os"
 	"time"
 
@@ -32,9 +31,9 @@ import (
 	"tailscale.com/logpolicy"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/tstun"
-	"tailscale.com/tempfork/wireguard-windows/firewall"
 	"tailscale.com/types/logger"
 	"tailscale.com/version"
+	"tailscale.com/wf"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/netstack"
 	"tailscale.com/wgengine/router"
@@ -144,13 +143,13 @@ func beFirewallKillswitch() bool {
 
 	luid, err := winipcfg.LUIDFromGUID(&guid)
 	if err != nil {
-		log.Fatalf("no interface with GUID %q", guid)
+		log.Fatalf("no interface with GUID %q: %v", guid, err)
 	}
 
-	noProtection := false
-	var dnsIPs []net.IP // unused in called code.
 	start := time.Now()
-	firewall.EnableFirewall(uint64(luid), noProtection, dnsIPs)
+	if _, err := wf.New(uint64(luid)); err != nil {
+		log.Fatalf("filewall creation failed: %v", err)
+	}
 	log.Printf("killswitch enabled, took %s", time.Since(start))
 
 	// Block until the monitor goroutine shuts us down.

control/controlclient/auto.go

@@ -7,6 +7,7 @@ package controlclient
 import (
 	"context"
 	"fmt"
+	"log"
 	"sync"
 	"time"
 
@@ -157,6 +158,7 @@ func (c *Auto) Start() {
 //
 // It should be called whenever there's something new to tell the server.
 func (c *Auto) sendNewMapRequest() {
+	log.Println("sendNewMapRequest breakpoint")
 	c.mu.Lock()
 
 	// If we're not already streaming a netmap, or if we're already stuck

control/controlclient/direct.go

@@ -32,6 +32,7 @@ import (
 	"golang.org/x/crypto/nacl/box"
 	"inet.af/netaddr"
 	"tailscale.com/health"
+	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/log/logheap"
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
@@ -66,6 +67,7 @@ type Direct struct {
 	debugFlags             []string
 	keepSharerAndUserSplit bool
 	skipIPForwardingCheck  bool
+	pinger                 Pinger
 
 	mu           sync.Mutex // mutex guards the following fields
 	serverKey    wgkey.Key
@@ -79,6 +81,11 @@ type Direct struct {
 	everEndpoints bool   // whether we've ever had non-empty endpoints
 	localPort     uint16 // or zero to mean auto
 }
+type Pinger interface {
+	// Ping is a request to start a discovery ping with the peer handling
+	// the given IP and then call cb with its ping latency & method.
+	Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.PingResult))
+}
 
 type Options struct {
 	Persist              persist.Persist               // initial persistent data
@@ -94,6 +101,7 @@ type Options struct {
 	HTTPTestClient       *http.Client // optional HTTP client to use (for tests only)
 	DebugFlags           []string     // debug settings to send to control
 	LinkMonitor          *monitor.Mon // optional link monitor
+	Pinger               Pinger
 
 	// KeepSharerAndUserSplit controls whether the client
 	// understands Node.Sharer. If false, the Sharer is mapped to the User.
@@ -165,6 +173,7 @@ func NewDirect(opts Options) (*Direct, error) {
 		keepSharerAndUserSplit: opts.KeepSharerAndUserSplit,
 		linkMon:                opts.LinkMonitor,
 		skipIPForwardingCheck:  opts.SkipIPForwardingCheck,
+		pinger:                 opts.Pinger,
 	}
 	if opts.Hostinfo == nil {
 		c.SetHostinfo(NewHostinfo())
@@ -460,10 +469,10 @@ func (c *Direct) doLogin(ctx context.Context, opt loginOpt) (mustRegen bool, new
 			request.NodeKey.ShortString())
 		return true, "", nil
 	}
-	if persist.Provider == "" {
+	if resp.Login.Provider != "" {
 		persist.Provider = resp.Login.Provider
 	}
-	if persist.LoginName == "" {
+	if resp.Login.LoginName != "" {
 		persist.LoginName = resp.Login.LoginName
 	}
 
@@ -553,6 +562,7 @@ func inTest() bool { return flag.Lookup("test.v") != nil }
 // maxPolls is how many network maps to download; common values are 1
 // or -1 (to keep a long-poll query open to the server).
 func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
+	log.Println("POLLNETMAP BREAKPOINT")
 	return c.sendMapRequest(ctx, maxPolls, cb)
 }
 
@@ -560,6 +570,7 @@ func (c *Direct) PollNetMap(ctx context.Context, maxPolls int, cb func(*netmap.N
 // but does not fetch anything. It returns an error if the server did not return a
 // successful 200 OK response.
 func (c *Direct) SendLiteMapUpdate(ctx context.Context) error {
+	log.Println("SendLiteMapUpdate BREAKPOINT")
 	return c.sendMapRequest(ctx, 1, nil)
 }
 
@@ -571,6 +582,7 @@ const pollTimeout = 120 * time.Second
 // cb nil means to omit peers.
 func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netmap.NetworkMap)) error {
 	c.mu.Lock()
+	log.Println("sendMapRequest ENDPOINT")
 	persist := c.persist
 	serverURL := c.serverURL
 	serverKey := c.serverKey
@@ -761,6 +773,11 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		}
 
 		if pr := resp.PingRequest; pr != nil {
+			// return err
+			log.Println("Ping Triggered")
+			for i := 0; i < 10; i++ {
+				c.CustomPing(&resp)
+			}
 			go answerPing(c.logf, c.httpc, pr)
 		}
 
@@ -859,6 +876,8 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 		c.expiry = &nm.Expiry
 		c.mu.Unlock()
 
+		// log.Println("MAPRESPONSE: ", resp.Node)
+		// c.logf("MAPRESPONSE: %v", resp.Node)
 		cb(nm)
 	}
 	if ctx.Err() != nil {
@@ -1091,7 +1110,7 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 	localIPs := map[netaddr.IP]bool{}
 	for _, addrs := range state.InterfaceIPs {
 		for _, pfx := range addrs {
-			localIPs[pfx.IP] = true
+			localIPs[pfx.IP()] = true
 		}
 	}
 
@@ -1100,10 +1119,10 @@ func ipForwardingBroken(routes []netaddr.IPPrefix, state *interfaces.State) bool
 		// It's possible to advertise a route to one of the local
 		// machine's local IPs. IP forwarding isn't required for this
 		// to work, so we shouldn't warn for such exports.
-		if r.IsSingleIP() && localIPs[r.IP] {
+		if r.IsSingleIP() && localIPs[r.IP()] {
 			continue
 		}
-		if r.IP.Is4() {
+		if r.IP().Is4() {
 			v4Routes = true
 		} else {
 			v6Routes = true
@@ -1211,3 +1230,35 @@ func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<-
 		}
 	}
 }
+
+// Run the ping suite from this client to another one
+// Send the ping results via http to the adminhttp handlers.
+// This is where we hopefully will run the ping suite similar to CLI
+func (c *Direct) CustomPing(mr *tailcfg.MapResponse) bool {
+	log.Printf("Custom Ping Triggered with %d number of peers\n", len(mr.Peers))
+	log.Println("Ping Request: ", mr.PingRequest)
+	log.Println("ALOHA")
+	log.Println("CP PEERLIST : ", mr.Peers, mr.PeersChanged, mr.PeersRemoved, mr.PeerSeenChange)
+	if len(mr.Peers) > 0 {
+		log.Println("Peer data: ", mr.Peers[0].ID)
+	}
+	ip := mr.PingRequest.TestIP
+	log.Println("TestIP : ", ip)
+	start := time.Now()
+	// Run the ping
+	var pingRes *ipnstate.PingResult
+	for i := 1; i <= 10; i++ {
+		log.Println("Ping attempt ", i)
+		go c.pinger.Ping(ip, true, func(res *ipnstate.PingResult) {
+			log.Println("Callback", res, (res.NodeIP))
+			pingRes = res
+		})
+	}
+
+	log.Println("PINGRES", pingRes)
+	duration := time.Since(start)
+	// Send the data to the handler in api.go admin/api/ping
+	log.Printf("Ping operation took %f seconds\n", duration.Seconds())
+
+	return len(mr.Peers) > 0
+}

control/controlclient/direct_test.go

@@ -86,7 +86,7 @@ func TestNewDirect(t *testing.T) {
 func fakeEndpoints(ports ...uint16) (ret []tailcfg.Endpoint) {
 	for _, port := range ports {
 		ret = append(ret, tailcfg.Endpoint{
-			Addr: netaddr.IPPort{Port: port},
+			Addr: netaddr.IPPortFrom(netaddr.IP{}, port),
 		})
 	}
 	return
@@ -103,3 +103,37 @@ func TestNewHostinfo(t *testing.T) {
 	}
 	t.Logf("Got: %s", j)
 }
+
+// Currently not working properly
+func TestPingFromMapResponse(t *testing.T) {
+	hi := NewHostinfo()
+	ni := tailcfg.NetInfo{LinkType: "wired"}
+	hi.NetInfo = &ni
+
+	key, err := wgkey.NewPrivate()
+	if err != nil {
+		t.Error(err)
+	}
+	opts := Options{
+		ServerURL: "https://example.com",
+		Hostinfo:  hi,
+		GetMachinePrivateKey: func() (wgkey.Private, error) {
+			return key, nil
+		},
+	}
+	c, err := NewDirect(opts)
+	if c == nil || err != nil {
+		t.Errorf("Direct not created %w", err)
+	}
+	peers := []*tailcfg.Node{
+		{ID: 1},
+		{ID: 2},
+		{ID: 3},
+	}
+	pingRequest := tailcfg.PingRequest{URL: "localhost:3040", Log: true, PayloadSize: 10}
+	mr := &tailcfg.MapResponse{Peers: peers, Domain: "DumbTest", PingRequest: &pingRequest}
+	if !c.CustomPing(mr) {
+		t.Errorf("Custom ping failed!\n")
+	}
+	t.Log("Successful ping")
+}

control/controlclient/sign_supported.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/github/certstore"
+	"github.com/tailscale/certstore"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/util/winutil"

disco/disco.go

@@ -147,9 +147,9 @@ const epLength = 16 + 2 // 16 byte IP address + 2 byte port
 func (m *CallMeMaybe) AppendMarshal(b []byte) []byte {
 	ret, p := appendMsgHeader(b, TypeCallMeMaybe, v0, epLength*len(m.MyNumber))
 	for _, ipp := range m.MyNumber {
-		a := ipp.IP.As16()
+		a := ipp.IP().As16()
 		copy(p[:], a[:])
-		binary.BigEndian.PutUint16(p[16:], ipp.Port)
+		binary.BigEndian.PutUint16(p[16:], ipp.Port())
 		p = p[epLength:]
 	}
 	return ret
@@ -164,10 +164,9 @@ func parseCallMeMaybe(ver uint8, p []byte) (m *CallMeMaybe, err error) {
 	for len(p) > 0 {
 		var a [16]byte
 		copy(a[:], p)
-		m.MyNumber = append(m.MyNumber, netaddr.IPPort{
-			IP:   netaddr.IPFrom16(a),
-			Port: binary.BigEndian.Uint16(p[16:18]),
-		})
+		m.MyNumber = append(m.MyNumber, netaddr.IPPortFrom(
+			netaddr.IPFrom16(a),
+			binary.BigEndian.Uint16(p[16:18])))
 		p = p[epLength:]
 	}
 	return m, nil
@@ -187,9 +186,9 @@ const pongLen = 12 + 16 + 2
 func (m *Pong) AppendMarshal(b []byte) []byte {
 	ret, d := appendMsgHeader(b, TypePong, v0, pongLen)
 	d = d[copy(d, m.TxID[:]):]
-	ip16 := m.Src.IP.As16()
+	ip16 := m.Src.IP().As16()
 	d = d[copy(d, ip16[:]):]
-	binary.BigEndian.PutUint16(d, m.Src.Port)
+	binary.BigEndian.PutUint16(d, m.Src.Port())
 	return ret
 }
 
@@ -201,10 +200,10 @@ func parsePong(ver uint8, p []byte) (m *Pong, err error) {
 	copy(m.TxID[:], p)
 	p = p[12:]
 
-	m.Src.IP, _ = netaddr.FromStdIP(net.IP(p[:16]))
+	srcIP, _ := netaddr.FromStdIP(net.IP(p[:16]))
 	p = p[16:]
-
-	m.Src.Port = binary.BigEndian.Uint16(p)
+	port := binary.BigEndian.Uint16(p)
+	m.Src = netaddr.IPPortFrom(srcIP, port)
 	return m, nil
 }
 

go.mod

@@ -3,48 +3,44 @@ module tailscale.com
 go 1.16
 
 require (
-	github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5
-	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/coreos/go-iptables v0.4.5
-	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/frankban/quicktest v1.12.1
-	github.com/github/certstore v0.1.0
-	github.com/gliderlabs/ssh v0.2.2
+	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/coreos/go-iptables v0.6.0
+	github.com/frankban/quicktest v1.13.0
+	github.com/tailscale/certstore v0.0.0-20210528134328-066c94b793d3
+	github.com/gliderlabs/ssh v0.3.2
 	github.com/go-multierror/multierror v1.0.2
-	github.com/go-ole/go-ole v1.2.4
-	github.com/godbus/dbus/v5 v5.0.3
+	github.com/go-ole/go-ole v1.2.5
+	github.com/godbus/dbus/v5 v5.0.4
 	github.com/google/go-cmp v0.5.5
-	github.com/goreleaser/nfpm v1.1.10
-	github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b
+	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
+	github.com/goreleaser/nfpm v1.10.3
+	github.com/jsimonetti/rtnetlink v0.0.0-20210409061457-9561dc9288a7
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
-	github.com/klauspost/compress v1.10.10
+	github.com/klauspost/compress v1.12.2
 	github.com/kr/pty v1.1.8
-	github.com/mdlayher/netlink v1.3.2
-	github.com/mdlayher/sdnotify v0.0.0-20200625151349-e4a4f32afc4a
-	github.com/miekg/dns v1.1.30
-	github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3
+	github.com/mdlayher/netlink v1.4.0
+	github.com/mdlayher/sdnotify v0.0.0-20210228150836-ea3ec207d697
+	github.com/miekg/dns v1.1.42
+	github.com/pborman/getopt v1.1.0
 	github.com/peterbourgon/ff/v2 v2.0.0
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027
-	github.com/tailscale/wireguard-go v0.0.0-20210510192616-d1aa5623121d
 	github.com/tcnksm/go-httpstat v0.2.0
 	github.com/toqueteos/webbrowser v1.2.0
 	go4.org/mem v0.0.0-20201119185036-c04c5a6ff174
-	golang.org/x/crypto v0.0.0-20210317152858-513c2a44f670
-	golang.org/x/net v0.0.0-20210510120150-4163338589ed
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
-	golang.org/x/term v0.0.0-20210317153231-de623e64d2a6
+	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea
+	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
-	golang.org/x/tools v0.1.0
-	golang.zx2c4.com/wireguard/windows v0.1.2-0.20201113162609-9b85be97fdf8
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	honnef.co/go/tools v0.1.0
-	inet.af/netaddr v0.0.0-20210511181906-37180328850c
+	golang.org/x/tools v0.1.2
+	golang.zx2c4.com/wireguard v0.0.0-20210525143454-64cb82f2b3f5
+	golang.zx2c4.com/wireguard/windows v0.3.15-0.20210525143335-94c0476d63e3
+	honnef.co/go/tools v0.1.4
+	inet.af/netaddr v0.0.0-20210523191804-d57edf19c517
 	inet.af/netstack v0.0.0-20210317161235-a1bf4e56ef22
-	inet.af/peercred v0.0.0-20210302202138-56e694897155
-	inet.af/wf v0.0.0-20210424212123-eaa011a774a4
+	inet.af/peercred v0.0.0-20210318190834-4259e17bb763
+	inet.af/wf v0.0.0-20210516214145-a5343001b756
 	rsc.io/goversion v1.2.0
 )
-
-replace github.com/github/certstore => github.com/cyolosecurity/certstore v0.0.0-20200922073901-ece7f1d353c2

internal/deephash/deephash.go

@@ -9,26 +9,28 @@ package deephash
 import (
 	"bufio"
 	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"hash"
 	"reflect"
-
-	"inet.af/netaddr"
-	"tailscale.com/tailcfg"
-	"tailscale.com/types/wgkey"
+	"strconv"
+	"sync"
 )
 
-func Hash(v ...interface{}) string {
+func calcHash(v interface{}) string {
 	h := sha256.New()
-	// 64 matches the chunk size in crypto/sha256/sha256.go
-	b := bufio.NewWriterSize(h, 64)
-	Print(b, v)
+	b := bufio.NewWriterSize(h, h.BlockSize())
+	scratch := make([]byte, 0, 128)
+	printTo(b, v, scratch)
 	b.Flush()
-	return fmt.Sprintf("%x", h.Sum(nil))
+	scratch = h.Sum(scratch[:0])
+	hex.Encode(scratch[:cap(scratch)], scratch[:sha256.Size])
+	return string(scratch[:sha256.Size*2])
 }
 
 // UpdateHash sets last to the hash of v and reports whether its value changed.
 func UpdateHash(last *string, v ...interface{}) (changed bool) {
-	sig := Hash(v)
+	sig := calcHash(v)
 	if *last != sig {
 		*last = sig
 		return true
@@ -36,81 +38,30 @@ func UpdateHash(last *string, v ...interface{}) (changed bool) {
 	return false
 }
 
-func Print(w *bufio.Writer, v ...interface{}) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool))
+func printTo(w *bufio.Writer, v interface{}, scratch []byte) {
+	print(w, reflect.ValueOf(v), make(map[uintptr]bool), scratch)
 }
 
-var (
-	netaddrIPType       = reflect.TypeOf(netaddr.IP{})
-	netaddrIPPrefix     = reflect.TypeOf(netaddr.IPPrefix{})
-	wgkeyKeyType        = reflect.TypeOf(wgkey.Key{})
-	wgkeyPrivateType    = reflect.TypeOf(wgkey.Private{})
-	tailcfgDiscoKeyType = reflect.TypeOf(tailcfg.DiscoKey{})
-)
+var appenderToType = reflect.TypeOf((*appenderTo)(nil)).Elem()
 
-func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool) {
+type appenderTo interface {
+	AppendTo([]byte) []byte
+}
+
+// print hashes v into w.
+// It reports whether it was able to do so without hitting a cycle.
+func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
 	if !v.IsValid() {
-		return
+		return true
 	}
 
-	// Special case some common types.
 	if v.CanInterface() {
-		switch v.Type() {
-		case netaddrIPType:
-			var b []byte
-			var err error
-			if v.CanAddr() {
-				x := v.Addr().Interface().(*netaddr.IP)
-				b, err = x.MarshalText()
-			} else {
-				x := v.Interface().(netaddr.IP)
-				b, err = x.MarshalText()
-			}
-			if err == nil {
-				w.Write(b)
-				return
-			}
-		case netaddrIPPrefix:
-			var b []byte
-			var err error
-			if v.CanAddr() {
-				x := v.Addr().Interface().(*netaddr.IPPrefix)
-				b, err = x.MarshalText()
-			} else {
-				x := v.Interface().(netaddr.IPPrefix)
-				b, err = x.MarshalText()
-			}
-			if err == nil {
-				w.Write(b)
-				return
-			}
-		case wgkeyKeyType:
-			if v.CanAddr() {
-				x := v.Addr().Interface().(*wgkey.Key)
-				w.Write(x[:])
-			} else {
-				x := v.Interface().(wgkey.Key)
-				w.Write(x[:])
-			}
-			return
-		case wgkeyPrivateType:
-			if v.CanAddr() {
-				x := v.Addr().Interface().(*wgkey.Private)
-				w.Write(x[:])
-			} else {
-				x := v.Interface().(wgkey.Private)
-				w.Write(x[:])
-			}
-			return
-		case tailcfgDiscoKeyType:
-			if v.CanAddr() {
-				x := v.Addr().Interface().(*tailcfg.DiscoKey)
-				w.Write(x[:])
-			} else {
-				x := v.Interface().(tailcfg.DiscoKey)
-				w.Write(x[:])
-			}
-			return
+		// Use AppendTo methods, if available and cheap.
+		if v.CanAddr() && v.Type().Implements(appenderToType) {
+			a := v.Addr().Interface().(appenderTo)
+			scratch = a.AppendTo(scratch[:0])
+			w.Write(scratch)
+			return true
 		}
 	}
 
@@ -121,43 +72,45 @@ func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool) {
 	case reflect.Ptr:
 		ptr := v.Pointer()
 		if visited[ptr] {
-			return
+			return false
 		}
 		visited[ptr] = true
-		print(w, v.Elem(), visited)
-		return
+		return print(w, v.Elem(), visited, scratch)
 	case reflect.Struct:
+		acyclic = true
 		w.WriteString("struct{\n")
 		for i, n := 0, v.NumField(); i < n; i++ {
 			fmt.Fprintf(w, " [%d]: ", i)
-			print(w, v.Field(i), visited)
+			if !print(w, v.Field(i), visited, scratch) {
+				acyclic = false
+			}
 			w.WriteString("\n")
 		}
 		w.WriteString("}\n")
+		return acyclic
 	case reflect.Slice, reflect.Array:
 		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
 			fmt.Fprintf(w, "%q", v.Interface())
-			return
+			return true
 		}
 		fmt.Fprintf(w, "[%d]{\n", v.Len())
+		acyclic = true
 		for i, ln := 0, v.Len(); i < ln; i++ {
 			fmt.Fprintf(w, " [%d]: ", i)
-			print(w, v.Index(i), visited)
+			if !print(w, v.Index(i), visited, scratch) {
+				acyclic = false
+			}
 			w.WriteString("\n")
 		}
 		w.WriteString("}\n")
+		return acyclic
 	case reflect.Interface:
-		print(w, v.Elem(), visited)
+		return print(w, v.Elem(), visited, scratch)
 	case reflect.Map:
-		sm := newSortedMap(v)
-		fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
-		for i, k := range sm.Key {
-			print(w, k, visited)
-			w.WriteString(": ")
-			print(w, sm.Value[i], visited)
-			w.WriteString("\n")
+		if hashMapAcyclic(w, v, visited, scratch) {
+			return true
 		}
-		w.WriteString("}\n")
+		return hashMapFallback(w, v, visited, scratch)
 	case reflect.String:
 		w.WriteString(v.String())
 	case reflect.Bool:
@@ -165,10 +118,109 @@ func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool) {
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 		fmt.Fprintf(w, "%v", v.Int())
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		fmt.Fprintf(w, "%v", v.Uint())
+		scratch = strconv.AppendUint(scratch[:0], v.Uint(), 10)
+		w.Write(scratch)
 	case reflect.Float32, reflect.Float64:
 		fmt.Fprintf(w, "%v", v.Float())
 	case reflect.Complex64, reflect.Complex128:
 		fmt.Fprintf(w, "%v", v.Complex())
 	}
+	return true
+}
+
+type mapHasher struct {
+	xbuf [sha256.Size]byte // XOR'ed accumulated buffer
+	ebuf [sha256.Size]byte // scratch buffer
+	s256 hash.Hash         // sha256 hash.Hash
+	bw   *bufio.Writer     // to hasher into ebuf
+	val  valueCache        // re-usable values for map iteration
+	iter *reflect.MapIter  // re-usable map iterator
+}
+
+func (mh *mapHasher) Reset() {
+	for i := range mh.xbuf {
+		mh.xbuf[i] = 0
+	}
+}
+
+func (mh *mapHasher) startEntry() {
+	for i := range mh.ebuf {
+		mh.ebuf[i] = 0
+	}
+	mh.bw.Flush()
+	mh.s256.Reset()
+}
+
+func (mh *mapHasher) endEntry() {
+	mh.bw.Flush()
+	for i, b := range mh.s256.Sum(mh.ebuf[:0]) {
+		mh.xbuf[i] ^= b
+	}
+}
+
+var mapHasherPool = &sync.Pool{
+	New: func() interface{} {
+		mh := new(mapHasher)
+		mh.s256 = sha256.New()
+		mh.bw = bufio.NewWriter(mh.s256)
+		mh.val = make(valueCache)
+		mh.iter = new(reflect.MapIter)
+		return mh
+	},
+}
+
+type valueCache map[reflect.Type]reflect.Value
+
+func (c valueCache) get(t reflect.Type) reflect.Value {
+	v, ok := c[t]
+	if !ok {
+		v = reflect.New(t).Elem()
+		c[t] = v
+	}
+	return v
+}
+
+// hashMapAcyclic is the faster sort-free version of map hashing. If
+// it detects a cycle it returns false and guarantees that nothing was
+// written to w.
+func hashMapAcyclic(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	mh := mapHasherPool.Get().(*mapHasher)
+	defer mapHasherPool.Put(mh)
+	mh.Reset()
+	iter := mapIter(mh.iter, v)
+	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+	k := mh.val.get(v.Type().Key())
+	e := mh.val.get(v.Type().Elem())
+	for iter.Next() {
+		key := iterKey(iter, k)
+		val := iterVal(iter, e)
+		mh.startEntry()
+		if !print(mh.bw, key, visited, scratch) {
+			return false
+		}
+		if !print(mh.bw, val, visited, scratch) {
+			return false
+		}
+		mh.endEntry()
+	}
+	w.Write(mh.xbuf[:])
+	return true
+}
+
+func hashMapFallback(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+	acyclic = true
+	sm := newSortedMap(v)
+	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
+	for i, k := range sm.Key {
+		if !print(w, k, visited, scratch) {
+			acyclic = false
+		}
+		w.WriteString(": ")
+		if !print(w, sm.Value[i], visited, scratch) {
+			acyclic = false
+		}
+		w.WriteString("\n")
+	}
+	w.WriteString("}\n")
+	return acyclic
 }

internal/deephash/deephash_test.go

@@ -5,6 +5,10 @@
 package deephash
 
 import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"reflect"
 	"testing"
 
 	"inet.af/netaddr"
@@ -14,15 +18,15 @@ import (
 	"tailscale.com/wgengine/wgcfg"
 )
 
-func TestDeepPrint(t *testing.T) {
+func TestDeepHash(t *testing.T) {
 	// v contains the types of values we care about for our current callers.
 	// Mostly we're just testing that we don't panic on handled types.
 	v := getVal()
 
-	hash1 := Hash(v)
+	hash1 := calcHash(v)
 	t.Logf("hash: %v", hash1)
 	for i := 0; i < 20; i++ {
-		hash2 := Hash(getVal())
+		hash2 := calcHash(getVal())
 		if hash1 != hash2 {
 			t.Error("second hash didn't match")
 		}
@@ -33,7 +37,7 @@ func getVal() []interface{} {
 	return []interface{}{
 		&wgcfg.Config{
 			Name:      "foo",
-			Addresses: []netaddr.IPPrefix{{Bits: 5, IP: netaddr.IPFrom16([16]byte{3: 3})}},
+			Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{3: 3}), 5)},
 			Peers: []wgcfg.Peer{
 				{
 					Endpoints: wgcfg.Endpoints{
@@ -51,14 +55,23 @@ func getVal() []interface{} {
 		map[dnsname.FQDN][]netaddr.IP{
 			dnsname.FQDN("a."): {netaddr.MustParseIP("1.2.3.4"), netaddr.MustParseIP("4.3.2.1")},
 			dnsname.FQDN("b."): {netaddr.MustParseIP("8.8.8.8"), netaddr.MustParseIP("9.9.9.9")},
+			dnsname.FQDN("c."): {netaddr.MustParseIP("6.6.6.6"), netaddr.MustParseIP("7.7.7.7")},
+			dnsname.FQDN("d."): {netaddr.MustParseIP("6.7.6.6"), netaddr.MustParseIP("7.7.7.8")},
+			dnsname.FQDN("e."): {netaddr.MustParseIP("6.8.6.6"), netaddr.MustParseIP("7.7.7.9")},
+			dnsname.FQDN("f."): {netaddr.MustParseIP("6.9.6.6"), netaddr.MustParseIP("7.7.7.0")},
 		},
 		map[dnsname.FQDN][]netaddr.IPPort{
 			dnsname.FQDN("a."): {netaddr.MustParseIPPort("1.2.3.4:11"), netaddr.MustParseIPPort("4.3.2.1:22")},
 			dnsname.FQDN("b."): {netaddr.MustParseIPPort("8.8.8.8:11"), netaddr.MustParseIPPort("9.9.9.9:22")},
+			dnsname.FQDN("c."): {netaddr.MustParseIPPort("8.8.8.8:12"), netaddr.MustParseIPPort("9.9.9.9:23")},
+			dnsname.FQDN("d."): {netaddr.MustParseIPPort("8.8.8.8:13"), netaddr.MustParseIPPort("9.9.9.9:24")},
+			dnsname.FQDN("e."): {netaddr.MustParseIPPort("8.8.8.8:14"), netaddr.MustParseIPPort("9.9.9.9:25")},
 		},
 		map[tailcfg.DiscoKey]bool{
 			{1: 1}: true,
 			{1: 2}: false,
+			{2: 3}: true,
+			{3: 4}: false,
 		},
 	}
 }
@@ -67,6 +80,57 @@ func BenchmarkHash(b *testing.B) {
 	b.ReportAllocs()
 	v := getVal()
 	for i := 0; i < b.N; i++ {
-		Hash(v)
+		calcHash(v)
+	}
+}
+
+func TestHashMapAcyclic(t *testing.T) {
+	m := map[int]string{}
+	for i := 0; i < 100; i++ {
+		m[i] = fmt.Sprint(i)
+	}
+	got := map[string]bool{}
+
+	var buf bytes.Buffer
+	bw := bufio.NewWriter(&buf)
+
+	for i := 0; i < 20; i++ {
+		visited := map[uintptr]bool{}
+		scratch := make([]byte, 0, 64)
+		v := reflect.ValueOf(m)
+		buf.Reset()
+		bw.Reset(&buf)
+		if !hashMapAcyclic(bw, v, visited, scratch) {
+			t.Fatal("returned false")
+		}
+		if got[string(buf.Bytes())] {
+			continue
+		}
+		got[string(buf.Bytes())] = true
+	}
+	if len(got) != 1 {
+		t.Errorf("got %d results; want 1", len(got))
+	}
+}
+
+func BenchmarkHashMapAcyclic(b *testing.B) {
+	b.ReportAllocs()
+	m := map[int]string{}
+	for i := 0; i < 100; i++ {
+		m[i] = fmt.Sprint(i)
+	}
+
+	var buf bytes.Buffer
+	bw := bufio.NewWriter(&buf)
+	visited := map[uintptr]bool{}
+	scratch := make([]byte, 0, 64)
+	v := reflect.ValueOf(m)
+
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		bw.Reset(&buf)
+		if !hashMapAcyclic(bw, v, visited, scratch) {
+			b.Fatal("returned false")
+		}
 	}
 }

internal/deephash/mapiter.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !tailscale_go
+
+package deephash
+
+import "reflect"
+
+// iterKey returns the current iter key.
+// scratch is a re-usable reflect.Value.
+// iterKey may store the iter key in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterKey(iter *reflect.MapIter, _ reflect.Value) reflect.Value {
+	return iter.Key()
+}
+
+// iterVal returns the current iter val.
+// scratch is a re-usable reflect.Value.
+// iterVal may store the iter val in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterVal(iter *reflect.MapIter, _ reflect.Value) reflect.Value {
+	return iter.Value()
+}
+
+// mapIter returns a map iterator for mapVal.
+// scratch is a re-usable reflect.MapIter.
+// mapIter may re-use scratch and return it,
+// or it may allocate and return a new *reflect.MapIter.
+// If mapVal is the zero reflect.Value, mapIter may return nil.
+func mapIter(_ *reflect.MapIter, mapVal reflect.Value) *reflect.MapIter {
+	if !mapVal.IsValid() {
+		return nil
+	}
+	return mapVal.MapRange()
+}

internal/deephash/mapiter_future.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build tailscale_go
+
+package deephash
+
+import "reflect"
+
+// iterKey returns the current iter key.
+// scratch is a re-usable reflect.Value.
+// iterKey may store the iter key in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterKey(iter *reflect.MapIter, scratch reflect.Value) reflect.Value {
+	iter.SetKey(scratch)
+	return scratch
+}
+
+// iterVal returns the current iter val.
+// scratch is a re-usable reflect.Value.
+// iterVal may store the iter val in scratch and return scratch,
+// or it may allocate and return a new reflect.Value.
+func iterVal(iter *reflect.MapIter, scratch reflect.Value) reflect.Value {
+	iter.SetValue(scratch)
+	return scratch
+}
+
+// mapIter returns a map iterator for mapVal.
+// scratch is a re-usable reflect.MapIter.
+// mapIter may re-use scratch and return it,
+// or it may allocate and return a new *reflect.MapIter.
+// If mapVal is the zero reflect.Value, mapIter may return nil.
+func mapIter(scratch *reflect.MapIter, mapVal reflect.Value) *reflect.MapIter {
+	scratch.Reset(mapVal) // always Reset, to allow the caller to avoid pinning memory
+	if !mapVal.IsValid() {
+		// Returning scratch would also be OK.
+		// Do this for consistency with the non-optimized version.
+		return nil
+	}
+	return scratch
+}

ipn/ipnlocal/local.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/http"
 	"os"
@@ -44,11 +45,13 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/netmap"
 	"tailscale.com/types/persist"
+	"tailscale.com/types/preftype"
 	"tailscale.com/types/wgkey"
 	"tailscale.com/util/dnsname"
 	"tailscale.com/util/osshare"
 	"tailscale.com/util/systemd"
 	"tailscale.com/version"
+	"tailscale.com/version/distro"
 	"tailscale.com/wgengine"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
@@ -292,6 +295,7 @@ func (b *LocalBackend) Prefs() *ipn.Prefs {
 // Status returns the latest status of the backend and its
 // sub-components.
 func (b *LocalBackend) Status() *ipnstate.Status {
+	log.Println("Status ENDPOINT")
 	sb := new(ipnstate.StatusBuilder)
 	b.UpdateStatus(sb)
 	return sb.Status()
@@ -356,14 +360,14 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 		var tailAddr4 string
 		var tailscaleIPs = make([]netaddr.IP, 0, len(p.Addresses))
 		for _, addr := range p.Addresses {
-			if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.IP) {
-				if addr.IP.Is4() && tailAddr4 == "" {
+			if addr.IsSingleIP() && tsaddr.IsTailscaleIP(addr.IP()) {
+				if addr.IP().Is4() && tailAddr4 == "" {
 					// The peer struct previously only allowed a single
 					// Tailscale IP address. For compatibility for a few releases starting
 					// with 1.8, keep it pulled out as IPv4-only for a bit.
-					tailAddr4 = addr.IP.String()
+					tailAddr4 = addr.IP().String()
 				}
-				tailscaleIPs = append(tailscaleIPs, addr.IP)
+				tailscaleIPs = append(tailscaleIPs, addr.IP())
 			}
 		}
 		sb.AddPeer(key.Public(p.Key), &ipnstate.PeerStatus{
@@ -390,10 +394,10 @@ func (b *LocalBackend) populatePeerStatusLocked(sb *ipnstate.StatusBuilder) {
 func (b *LocalBackend) WhoIs(ipp netaddr.IPPort) (n *tailcfg.Node, u tailcfg.UserProfile, ok bool) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	n, ok = b.nodeByAddr[ipp.IP]
+	n, ok = b.nodeByAddr[ipp.IP()]
 	if !ok {
 		var ip netaddr.IP
-		if ipp.Port != 0 {
+		if ipp.Port() != 0 {
 			ip, ok = b.e.WhoIsIPPort(ipp)
 		}
 		if !ok {
@@ -434,14 +438,15 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		}
 		return
 	}
-	if st.LoginFinished != nil {
+
+	b.mu.Lock()
+	wasBlocked := b.blocked
+	b.mu.Unlock()
+
+	if st.LoginFinished != nil && wasBlocked {
 		// Auth completed, unblock the engine
 		b.blockEngineUpdates(false)
 		b.authReconfig()
-		b.EditPrefs(&ipn.MaskedPrefs{
-			LoggedOutSet: true,
-			Prefs:        ipn.Prefs{LoggedOut: false},
-		})
 		b.send(ipn.Notify{LoginFinished: &empty.Message{}})
 	}
 
@@ -480,11 +485,15 @@ func (b *LocalBackend) setClientStatus(st controlclient.Status) {
 		b.authURL = st.URL
 		b.authURLSticky = st.URL
 	}
-	if b.state == ipn.NeedsLogin {
-		if !b.prefs.WantRunning {
+	if wasBlocked && st.LoginFinished != nil {
+		// Interactive login finished successfully (URL visited).
+		// After an interactive login, the user always wants
+		// WantRunning.
+		if !b.prefs.WantRunning || b.prefs.LoggedOut {
 			prefsChanged = true
 		}
 		b.prefs.WantRunning = true
+		b.prefs.LoggedOut = false
 	}
 	// Prefs will be written out; this is not safe unless locked or cloned.
 	if prefsChanged {
@@ -547,7 +556,7 @@ func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged
 
 	for _, peer := range nm.Peers {
 		for _, addr := range peer.Addresses {
-			if !addr.IsSingleIP() || addr.IP != b.prefs.ExitNodeIP {
+			if !addr.IsSingleIP() || addr.IP() != b.prefs.ExitNodeIP {
 				continue
 			}
 			// Found the node being referenced, upgrade prefs to
@@ -566,10 +575,18 @@ func (b *LocalBackend) findExitNodeIDLocked(nm *netmap.NetworkMap) (prefsChanged
 func (b *LocalBackend) setWgengineStatus(s *wgengine.Status, err error) {
 	if err != nil {
 		b.logf("wgengine status error: %v", err)
+
+		b.statusLock.Lock()
+		b.statusChanged.Broadcast()
+		b.statusLock.Unlock()
 		return
 	}
 	if s == nil {
 		b.logf("[unexpected] non-error wgengine update with status=nil: %v", s)
+
+		b.statusLock.Lock()
+		b.statusChanged.Broadcast()
+		b.statusLock.Unlock()
 		return
 	}
 
@@ -808,6 +825,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		DiscoPublicKey:       discoPublic,
 		DebugFlags:           debugFlags,
 		LinkMonitor:          b.e.GetLinkMonitor(),
+		Pinger:               b.e,
 
 		// Don't warn about broken Linux IP forwading when
 		// netstack is being used.
@@ -878,7 +896,7 @@ func (b *LocalBackend) updateFilter(netMap *netmap.NetworkMap, prefs *ipn.Prefs)
 	}
 	if prefs != nil {
 		for _, r := range prefs.AdvertiseRoutes {
-			if r.Bits == 0 {
+			if r.Bits() == 0 {
 				// When offering a default route to the world, we
 				// filter out locally reachable LANs, so that the
 				// default route effectively appears to be a "guest
@@ -946,13 +964,13 @@ var removeFromDefaultRoute = []netaddr.IPPrefix{
 func interfaceRoutes() (ips *netaddr.IPSet, hostIPs []netaddr.IP, err error) {
 	var b netaddr.IPSetBuilder
 	if err := interfaces.ForeachInterfaceAddress(func(_ interfaces.Interface, pfx netaddr.IPPrefix) {
-		if tsaddr.IsTailscaleIP(pfx.IP) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
 			return
 		}
 		if pfx.IsSingleIP() {
 			return
 		}
-		hostIPs = append(hostIPs, pfx.IP)
+		hostIPs = append(hostIPs, pfx.IP())
 		b.AddPrefix(pfx)
 	}); err != nil {
 		return nil, nil, err
@@ -1681,9 +1699,45 @@ func (b *LocalBackend) authReconfig() {
 
 	rcfg := b.routerConfig(cfg, uc)
 
-	var dcfg dns.Config
+	dcfg := dns.Config{
+		Routes: map[dnsname.FQDN][]netaddr.IPPort{},
+		Hosts:  map[dnsname.FQDN][]netaddr.IP{},
+	}
+
+	// Populate MagicDNS records. We do this unconditionally so that
+	// quad-100 can always respond to MagicDNS queries, even if the OS
+	// isn't configured to make MagicDNS resolution truly
+	// magic. Details in
+	// https://github.com/tailscale/tailscale/issues/1886.
+	set := func(name string, addrs []netaddr.IPPrefix) {
+		if len(addrs) == 0 || name == "" {
+			return
+		}
+		fqdn, err := dnsname.ToFQDN(name)
+		if err != nil {
+			return // TODO: propagate error?
+		}
+		var ips []netaddr.IP
+		for _, addr := range addrs {
+			// Remove IPv6 addresses for now, as we don't
+			// guarantee that the peer node actually can speak
+			// IPv6 correctly.
+			//
+			// https://github.com/tailscale/tailscale/issues/1152
+			// tracks adding the right capability reporting to
+			// enable AAAA in MagicDNS.
+			if addr.IP().Is6() {
+				continue
+			}
+			ips = append(ips, addr.IP())
+		}
+		dcfg.Hosts[fqdn] = ips
+	}
+	set(nm.Name, nm.Addresses)
+	for _, peer := range nm.Peers {
+		set(peer.Name, peer.Addresses)
+	}
 
-	// If CorpDNS is false, dcfg remains the zero value.
 	if uc.CorpDNS {
 		addDefault := func(resolvers []tailcfg.DNSResolver) {
 			for _, resolver := range resolvers {
@@ -1697,9 +1751,6 @@ func (b *LocalBackend) authReconfig() {
 		}
 
 		addDefault(nm.DNS.Resolvers)
-		if len(nm.DNS.Routes) > 0 {
-			dcfg.Routes = map[dnsname.FQDN][]netaddr.IPPort{}
-		}
 		for suffix, resolvers := range nm.DNS.Routes {
 			fqdn, err := dnsname.ToFQDN(suffix)
 			if err != nil {
@@ -1721,36 +1772,9 @@ func (b *LocalBackend) authReconfig() {
 			}
 			dcfg.SearchDomains = append(dcfg.SearchDomains, fqdn)
 		}
-		set := func(name string, addrs []netaddr.IPPrefix) {
-			if len(addrs) == 0 || name == "" {
-				return
-			}
-			fqdn, err := dnsname.ToFQDN(name)
-			if err != nil {
-				return // TODO: propagate error?
-			}
-			var ips []netaddr.IP
-			for _, addr := range addrs {
-				// Remove IPv6 addresses for now, as we don't
-				// guarantee that the peer node actually can speak
-				// IPv6 correctly.
-				//
-				// https://github.com/tailscale/tailscale/issues/1152
-				// tracks adding the right capability reporting to
-				// enable AAAA in MagicDNS.
-				if addr.IP.Is6() {
-					continue
-				}
-				ips = append(ips, addr.IP)
-			}
-			dcfg.Hosts[fqdn] = ips
-		}
 		if nm.DNS.Proxied { // actually means "enable MagicDNS"
-			dcfg.AuthoritativeSuffixes = magicDNSRootDomains(nm)
-			dcfg.Hosts = map[dnsname.FQDN][]netaddr.IP{}
-			set(nm.Name, nm.Addresses)
-			for _, peer := range nm.Peers {
-				set(peer.Name, peer.Addresses)
+			for _, dom := range magicDNSRootDomains(nm) {
+				dcfg.Routes[dom] = nil // resolve internally with dcfg.Hosts
 			}
 		}
 
@@ -1774,7 +1798,7 @@ func (b *LocalBackend) authReconfig() {
 			//
 			// https://github.com/tailscale/tailscale/issues/1713
 			addDefault(nm.DNS.FallbackResolvers)
-		case len(dcfg.Routes) == 0 && len(dcfg.Hosts) == 0 && len(dcfg.AuthoritativeSuffixes) == 0:
+		case len(dcfg.Routes) == 0:
 			// No settings requiring split DNS, no problem.
 		case version.OS() == "android":
 			// We don't support split DNS at all on Android yet.
@@ -1796,17 +1820,15 @@ func parseResolver(cfg tailcfg.DNSResolver) (netaddr.IPPort, error) {
 	if err != nil {
 		return netaddr.IPPort{}, fmt.Errorf("[unexpected] non-IP resolver %q", cfg.Addr)
 	}
-	return netaddr.IPPort{
-		IP:   ip,
-		Port: 53,
-	}, nil
+	return netaddr.IPPortFrom(ip, 53), nil
 }
 
 // tailscaleVarRoot returns the root directory of Tailscale's writable
 // storage area. (e.g. "/var/lib/tailscale")
 func tailscaleVarRoot() string {
-	if runtime.GOOS == "ios" {
-		dir, _ := paths.IOSSharedDir.Load().(string)
+	switch runtime.GOOS {
+	case "ios", "android":
+		dir, _ := paths.AppSharedDir.Load().(string)
 		return dir
 	}
 	stateFile := paths.DefaultTailscaledStateFile()
@@ -1857,7 +1879,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 	if len(b.netMap.Addresses) == len(b.peerAPIListeners) {
 		allSame := true
 		for i, pln := range b.peerAPIListeners {
-			if pln.ip != b.netMap.Addresses[i].IP {
+			if pln.ip != b.netMap.Addresses[i].IP() {
 				allSame = false
 				break
 			}
@@ -1902,7 +1924,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 		var err error
 		skipListen := i > 0 && isNetstack
 		if !skipListen {
-			ln, err = ps.listen(a.IP, b.prevIfState)
+			ln, err = ps.listen(a.IP(), b.prevIfState)
 			if err != nil {
 				if runtime.GOOS == "windows" {
 					// Expected for now. See Issue 1620.
@@ -1910,13 +1932,13 @@ func (b *LocalBackend) initPeerAPIListener() {
 					// ("peerAPIListeners too low").
 					continue
 				}
-				b.logf("[unexpected] peerapi listen(%q) error: %v", a.IP, err)
+				b.logf("[unexpected] peerapi listen(%q) error: %v", a.IP(), err)
 				continue
 			}
 		}
 		pln := &peerAPIListener{
 			ps: ps,
-			ip: a.IP,
+			ip: a.IP(),
 			ln: ln, // nil for 2nd+ on netstack
 			lb: b,
 		}
@@ -1925,7 +1947,7 @@ func (b *LocalBackend) initPeerAPIListener() {
 		} else {
 			pln.port = ln.Addr().(*net.TCPAddr).Port
 		}
-		pln.urlStr = "http://" + net.JoinHostPort(a.IP.String(), strconv.Itoa(pln.port))
+		pln.urlStr = "http://" + net.JoinHostPort(a.IP().String(), strconv.Itoa(pln.port))
 		b.logf("peerapi: serving on %s", pln.urlStr)
 		go pln.serve()
 		b.peerAPIListeners = append(b.peerAPIListeners, pln)
@@ -1976,14 +1998,14 @@ func peerRoutes(peers []wgcfg.Peer, cgnatThreshold int) (routes []netaddr.IPPref
 		for _, aip := range peer.AllowedIPs {
 			aip = unmapIPPrefix(aip)
 			// Only add the Tailscale IPv6 ULA once, if we see anybody using part of it.
-			if aip.IP.Is6() && aip.IsSingleIP() && tsULA.Contains(aip.IP) {
+			if aip.IP().Is6() && aip.IsSingleIP() && tsULA.Contains(aip.IP()) {
 				if !didULA {
 					didULA = true
 					routes = append(routes, tsULA)
 				}
 				continue
 			}
-			if aip.IsSingleIP() && cgNAT.Contains(aip.IP) {
+			if aip.IsSingleIP() && cgNAT.Contains(aip.IP()) {
 				cgNATIPs = append(cgNATIPs, aip)
 			} else {
 				routes = append(routes, aip)
@@ -2009,6 +2031,11 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		Routes:           peerRoutes(cfg.Peers, 10_000),
 	}
 
+	if distro.Get() == distro.Synology {
+		// Issue 1995: we don't use iptables on Synology.
+		rs.NetfilterMode = preftype.NetfilterOff
+	}
+
 	// Sanity check: we expect the control server to program both a v4
 	// and a v6 default route, if default routing is on. Fill in
 	// blackhole routes appropriately if we're missing some. This is
@@ -2050,16 +2077,13 @@ func (b *LocalBackend) routerConfig(cfg *wgcfg.Config, prefs *ipn.Prefs) *router
 		}
 	}
 
-	rs.Routes = append(rs.Routes, netaddr.IPPrefix{
-		IP:   tsaddr.TailscaleServiceIP(),
-		Bits: 32,
-	})
+	rs.Routes = append(rs.Routes, netaddr.IPPrefixFrom(tsaddr.TailscaleServiceIP(), 32))
 
 	return rs
 }
 
 func unmapIPPrefix(ipp netaddr.IPPrefix) netaddr.IPPrefix {
-	return netaddr.IPPrefix{IP: ipp.IP.Unmap(), Bits: ipp.Bits}
+	return netaddr.IPPrefixFrom(ipp.IP().Unmap(), ipp.Bits())
 }
 
 func unmapIPPrefixes(ippsList ...[]netaddr.IPPrefix) (ret []netaddr.IPPrefix) {
@@ -2113,8 +2137,8 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	if oldState == newState {
 		return
 	}
-	b.logf("Switching ipn state %v -> %v (WantRunning=%v)",
-		oldState, newState, prefs.WantRunning)
+	b.logf("Switching ipn state %v -> %v (WantRunning=%v, nm=%v)",
+		oldState, newState, prefs.WantRunning, netMap != nil)
 	health.SetIPNState(newState.String(), prefs.WantRunning)
 	b.send(ipn.Notify{State: &newState})
 
@@ -2143,7 +2167,7 @@ func (b *LocalBackend) enterState(newState ipn.State) {
 	case ipn.Running:
 		var addrs []string
 		for _, addr := range b.netMap.Addresses {
-			addrs = append(addrs, addr.IP.String())
+			addrs = append(addrs, addr.IP().String())
 		}
 		systemd.Status("Connected; %s; %s", activeLogin, strings.Join(addrs, " "))
 	default:
@@ -2170,13 +2194,14 @@ func (b *LocalBackend) nextState() ipn.State {
 		cc          = b.cc
 		netMap      = b.netMap
 		state       = b.state
+		blocked     = b.blocked
 		wantRunning = b.prefs.WantRunning
 		loggedOut   = b.prefs.LoggedOut
 	)
 	b.mu.Unlock()
 
 	switch {
-	case !wantRunning && !loggedOut && b.hasNodeKey():
+	case !wantRunning && !loggedOut && !blocked && b.hasNodeKey():
 		return ipn.Stopped
 	case netMap == nil:
 		if cc.AuthCantContinue() || loggedOut {
@@ -2410,7 +2435,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 	addNode := func(n *tailcfg.Node) {
 		for _, ipp := range n.Addresses {
 			if ipp.IsSingleIP() {
-				b.nodeByAddr[ipp.IP] = n
+				b.nodeByAddr[ipp.IP()] = n
 			}
 		}
 	}
@@ -2562,9 +2587,9 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 			continue
 		}
 		switch {
-		case a.IP.Is4():
+		case a.IP().Is4():
 			have4 = true
-		case a.IP.Is6():
+		case a.IP().Is6():
 			have6 = true
 		}
 	}
@@ -2580,11 +2605,11 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 	var ipp netaddr.IPPort
 	switch {
 	case have4 && p4 != 0:
-		ipp = netaddr.IPPort{IP: nodeIP(peer, netaddr.IP.Is4), Port: p4}
+		ipp = netaddr.IPPortFrom(nodeIP(peer, netaddr.IP.Is4), p4)
 	case have6 && p6 != 0:
-		ipp = netaddr.IPPort{IP: nodeIP(peer, netaddr.IP.Is6), Port: p6}
+		ipp = netaddr.IPPortFrom(nodeIP(peer, netaddr.IP.Is6), p6)
 	}
-	if ipp.IP.IsZero() {
+	if ipp.IP().IsZero() {
 		return ""
 	}
 	return fmt.Sprintf("http://%v", ipp)
@@ -2592,8 +2617,8 @@ func peerAPIBase(nm *netmap.NetworkMap, peer *tailcfg.Node) string {
 
 func nodeIP(n *tailcfg.Node, pred func(netaddr.IP) bool) netaddr.IP {
 	for _, a := range n.Addresses {
-		if a.IsSingleIP() && pred(a.IP) {
-			return a.IP
+		if a.IsSingleIP() && pred(a.IP()) {
+			return a.IP()
 		}
 	}
 	return netaddr.IP{}

ipn/ipnlocal/local_test.go

@@ -171,7 +171,7 @@ func TestShrinkDefaultRoute(t *testing.T) {
 			out: []string{
 				"fe80::1",
 				"ff00::1",
-				tsaddr.TailscaleULARange().IP.String(),
+				tsaddr.TailscaleULARange().IP().String(),
 			},
 			localIPFn: func(ip netaddr.IP) bool { return !inRemove(ip) && ip.Is6() },
 		},

ipn/ipnlocal/peerapi.go

@@ -510,7 +510,7 @@ func (h *peerAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 <body>
 <h1>Hello, %s (%v)</h1>
 This is my Tailscale device. Your device is %v.
-`, html.EscapeString(who), h.remoteAddr.IP, html.EscapeString(h.peerNode.ComputedName))
+`, html.EscapeString(who), h.remoteAddr.IP(), html.EscapeString(h.peerNode.ComputedName))
 
 	if h.isSelf {
 		fmt.Fprintf(w, "<p>You are the owner of this node.\n")

ipn/ipnlocal/state_test.go

@@ -368,13 +368,11 @@ func TestStateMachine(t *testing.T) {
 	{
 		c.Assert(cc.getCalls(), qt.DeepEquals, []string{"Login"})
 		notifies.drain(0)
-		// BUG: this should immediately set WantRunning to true.
-		// Users don't log in if they don't want to also connect.
-		// (Generally, we're inconsistent about who is supposed to
-		// update Prefs at what time. But the overall philosophy is:
-		// update it when the user's intent changes. This is clearly
-		// at the time the user *requests* Login, not at the time
-		// the login finishes.)
+		// Note: WantRunning isn't true yet. It'll switch to true
+		// after a successful login finishes.
+		// (This behaviour is needed so that b.Login() won't
+		// start connecting to an old account right away, if one
+		// exists when you launch another login.)
 	}
 
 	// Attempted non-interactive login with no key; indicate that
@@ -384,18 +382,16 @@ func TestStateMachine(t *testing.T) {
 	url1 := "http://localhost:1/1"
 	cc.send(nil, url1, false, nil)
 	{
-		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(cc.getCalls(), qt.DeepEquals, []string{})
 
 		// ...but backend eats that notification, because the user
 		// didn't explicitly request interactive login yet, and
 		// we're already in NeedsLogin state.
 		nn := notifies.drain(1)
 
-		// Trying to log in automatically sets WantRunning.
-		// BUG: that should have happened right after Login().
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
 	}
 
 	// Now we'll try an interactive login.
@@ -451,11 +447,12 @@ func TestStateMachine(t *testing.T) {
 	// same time.
 	// The backend should propagate this upward for the UI.
 	t.Logf("\n\nLoginFinished")
-	notifies.expect(2)
+	notifies.expect(3)
 	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user1"
 	cc.send(nil, "", true, &netmap.NetworkMap{})
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(3)
 		// BUG: still too soon for UpdateEndpoints.
 		//
 		// Arguably it makes sense to unpause now, since the machine
@@ -468,15 +465,12 @@ func TestStateMachine(t *testing.T) {
 		// it's visible in the logs)
 		c.Assert([]string{"unpause", "UpdateEndpoints"}, qt.DeepEquals, cc.getCalls())
 		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
-		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[1].State)
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user1")
+		c.Assert(ipn.NeedsMachineAuth, qt.Equals, *nn[2].State)
 	}
 
-	// TODO: check that the logged-in username propagates from control
-	// through to the UI notifications. I think it's used as a hint
-	// for future logins, to pre-fill the username box? Not really sure
-	// how it works.
-
 	// Pretend that the administrator has authorized our machine.
 	t.Logf("\n\nMachineAuthorized")
 	notifies.expect(1)
@@ -581,77 +575,72 @@ func TestStateMachine(t *testing.T) {
 
 	// Let's make the logout succeed.
 	t.Logf("\n\nLogout (async) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		// BUG: WantRunning should be false after manual logout.
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// A second logout should do nothing, since the prefs haven't changed.
 	t.Logf("\n\nLogout2 (async)")
-	notifies.expect(1)
+	notifies.expect(0)
 	b.Logout()
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		// BUG: the backend has already called StartLogout, and we're
 		// still logged out. So it shouldn't call it again.
 		c.Assert([]string{"StartLogout"}, qt.DeepEquals, cc.getCalls())
-		// BUG: Prefs should not change here. Already logged out.
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Let's acknowledge the second logout too.
 	t.Logf("\n\nLogout2 (async) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		// BUG: second logout shouldn't cause WantRunning->true !!
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Try the synchronous logout feature.
 	t.Logf("\n\nLogout3 (sync)")
-	notifies.expect(1)
+	notifies.expect(0)
 	b.LogoutSync(context.Background())
 	// NOTE: This returns as soon as cc.Logout() returns, which is okay
 	// I guess, since that's supposed to be synchronous.
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert([]string{"Logout"}, qt.DeepEquals, cc.getCalls())
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
 	// Generate the third logout event.
 	t.Logf("\n\nLogout3 (sync) - succeed")
-	notifies.expect(1)
+	notifies.expect(0)
 	cc.setAuthBlocked(true)
 	cc.send(nil, "", false, nil)
 	{
-		nn := notifies.drain(1)
+		notifies.drain(0)
 		c.Assert(cc.getCalls(), qt.HasLen, 0)
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		// BUG: third logout shouldn't cause WantRunning->true !!
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(cc.getCalls(), qt.HasLen, 0)
+		c.Assert(b.Prefs().LoggedOut, qt.IsTrue)
+		c.Assert(b.Prefs().WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
 
@@ -669,10 +658,6 @@ func TestStateMachine(t *testing.T) {
 	// happens if the user exits and restarts while logged out.
 	// Note that it's explicitly okay to call b.Start() over and over
 	// again, every time the frontend reconnects.
-	//
-	// BUG: WantRunning is true here (because of the bug above).
-	//  We'll have to adjust the following test's expectations if we
-	//  fix that.
 
 	// TODO: test user switching between statekeys.
 
@@ -691,7 +676,7 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[1].State, qt.Not(qt.IsNil))
 		c.Assert(nn[0].Prefs.LoggedOut, qt.IsTrue)
-		c.Assert(nn[0].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(nn[0].Prefs.WantRunning, qt.IsFalse)
 		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
 		c.Assert(ipn.NeedsLogin, qt.Equals, b.State())
 	}
@@ -703,16 +688,20 @@ func TestStateMachine(t *testing.T) {
 	t.Logf("\n\nLoginFinished3")
 	notifies.expect(3)
 	cc.setAuthBlocked(false)
+	cc.persist.LoginName = "user2"
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
 	{
 		nn := notifies.drain(3)
 		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
-		c.Assert(nn[0].Prefs, qt.Not(qt.IsNil))
-		c.Assert(nn[1].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
 		c.Assert(nn[2].State, qt.Not(qt.IsNil))
-		c.Assert(nn[0].Prefs.LoggedOut, qt.IsFalse)
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user2")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
 		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
 	}
 
@@ -773,6 +762,63 @@ func TestStateMachine(t *testing.T) {
 		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 	}
 
+	// Disconnect.
+	t.Logf("\n\nStop")
+	notifies.expect(2)
+	b.EditPrefs(&ipn.MaskedPrefs{
+		WantRunningSet: true,
+		Prefs:          ipn.Prefs{WantRunning: false},
+	})
+	{
+		nn := notifies.drain(2)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		// BUG: I would expect Prefs to change first, and state after.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(ipn.Stopped, qt.Equals, *nn[0].State)
+	}
+
+	// We want to try logging in as a different user, while Stopped.
+	// First, start the login process (without logging out first).
+	t.Logf("\n\nLoginDifferent")
+	notifies.expect(2)
+	b.StartLoginInteractive()
+	url3 := "http://localhost:1/3"
+	cc.send(nil, url3, false, nil)
+	{
+		nn := notifies.drain(2)
+		// It might seem like WantRunning should switch to true here,
+		// but that would be risky since we already have a valid
+		// user account. It might try to reconnect to the old account
+		// before the new one is ready. So no change yet.
+		c.Assert([]string{"Login", "unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].BrowseToURL, qt.Not(qt.IsNil))
+		c.Assert(nn[1].State, qt.Not(qt.IsNil))
+		c.Assert(*nn[0].BrowseToURL, qt.Equals, url3)
+		c.Assert(ipn.NeedsLogin, qt.Equals, *nn[1].State)
+	}
+
+	// Now, let's say the interactive login completed, using a different
+	// user account than before.
+	t.Logf("\n\nLoginDifferent URL visited")
+	notifies.expect(3)
+	cc.persist.LoginName = "user3"
+	cc.send(nil, "", true, &netmap.NetworkMap{
+		MachineStatus: tailcfg.MachineAuthorized,
+	})
+	{
+		nn := notifies.drain(3)
+		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
+		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
+		c.Assert(nn[1].Prefs, qt.Not(qt.IsNil))
+		c.Assert(nn[2].State, qt.Not(qt.IsNil))
+		// Prefs after finishing the login, so LoginName updated.
+		c.Assert(nn[1].Prefs.Persist.LoginName, qt.Equals, "user3")
+		c.Assert(nn[1].Prefs.LoggedOut, qt.IsFalse)
+		c.Assert(nn[1].Prefs.WantRunning, qt.IsTrue)
+		c.Assert(ipn.Starting, qt.Equals, *nn[2].State)
+	}
+
 	// The last test case is the most common one: restarting when both
 	// logged in and WantRunning.
 	t.Logf("\n\nStart5")
@@ -793,17 +839,18 @@ func TestStateMachine(t *testing.T) {
 
 	// Control server accepts our valid key from before.
 	t.Logf("\n\nLoginFinished5")
-	notifies.expect(2)
+	notifies.expect(1)
 	cc.setAuthBlocked(false)
 	cc.send(nil, "", true, &netmap.NetworkMap{
 		MachineStatus: tailcfg.MachineAuthorized,
 	})
 	{
-		nn := notifies.drain(2)
+		nn := notifies.drain(1)
 		c.Assert([]string{"unpause"}, qt.DeepEquals, cc.getCalls())
-		c.Assert(nn[0].LoginFinished, qt.Not(qt.IsNil))
-		c.Assert(nn[1].State, qt.Not(qt.IsNil))
-		c.Assert(ipn.Starting, qt.Equals, *nn[1].State)
+		// NOTE: No LoginFinished message since no interactive
+		// login was needed.
+		c.Assert(nn[0].State, qt.Not(qt.IsNil))
+		c.Assert(ipn.Starting, qt.Equals, *nn[0].State)
 		// NOTE: No prefs change this time. WantRunning stays true.
 		// We were in Starting in the first place, so that doesn't
 		// change either.

ipn/ipnserver/server.go

@@ -147,7 +147,7 @@ func (s *server) getConnIdentity(c net.Conn) (ci connIdentity, err error) {
 	if err != nil {
 		return ci, fmt.Errorf("parsing local remote: %w", err)
 	}
-	if !la.IP.IsLoopback() || !ra.IP.IsLoopback() {
+	if !la.IP().IsLoopback() || !ra.IP().IsLoopback() {
 		return ci, errors.New("non-loopback connection")
 	}
 	tab, err := netstat.Get()

ipn/message.go

@@ -104,7 +104,9 @@ func NewBackendServer(logf logger.Logf, b Backend, sendNotifyMsg func(Notify)) *
 		b:             b,
 		sendNotifyMsg: sendNotifyMsg,
 	}
-	if sendNotifyMsg != nil {
+	// b may be nil if the BackendServer is being created just to
+	// encapsulate and send an error message.
+	if sendNotifyMsg != nil && b != nil {
 		b.SetNotifyCallback(bs.send)
 	}
 	return bs

ipn/message_test.go

@@ -187,3 +187,17 @@ func TestClientServer(t *testing.T) {
 	})
 	flushUntil(Running)
 }
+
+func TestNilBackend(t *testing.T) {
+	var called *Notify
+	bs := NewBackendServer(t.Logf, nil, func(n Notify) {
+		called = &n
+	})
+	bs.SendErrorMessage("Danger, Will Robinson!")
+	if called == nil {
+		t.Errorf("expect callback to be called, wasn't")
+	}
+	if called.ErrMessage == nil || *called.ErrMessage != "Danger, Will Robinson!" {
+		t.Errorf("callback got wrong error: %v", called.ErrMessage)
+	}
+}

logtail/logtail.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"tailscale.com/logtail/backoff"
@@ -72,7 +73,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 	}
 	l := &Logger{
 		stderr:         cfg.Stderr,
-		stderrLevel:    cfg.StderrLevel,
+		stderrLevel:    int64(cfg.StderrLevel),
 		httpc:          cfg.HTTPC,
 		url:            cfg.BaseURL + "/c/" + cfg.Collection + "/" + cfg.PrivateID.String(),
 		lowMem:         cfg.LowMemory,
@@ -103,7 +104,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 // logging facilities and uploading to a log server.
 type Logger struct {
 	stderr         io.Writer
-	stderrLevel    int
+	stderrLevel    int64 // accessed atomically
 	httpc          *http.Client
 	url            string
 	lowMem         bool
@@ -125,10 +126,8 @@ type Logger struct {
 // SetVerbosityLevel controls the verbosity level that should be
 // written to stderr. 0 is the default (not verbose). Levels 1 or higher
 // are increasingly verbose.
-//
-// It should not be changed concurrently with log writes.
 func (l *Logger) SetVerbosityLevel(level int) {
-	l.stderrLevel = level
+	atomic.StoreInt64(&l.stderrLevel, int64(level))
 }
 
 // SetLinkMonitor sets the optional the link monitor.
@@ -514,7 +513,7 @@ func (l *Logger) Write(buf []byte) (int, error) {
 		return 0, nil
 	}
 	level, buf := parseAndRemoveLogLevel(buf)
-	if l.stderr != nil && l.stderr != ioutil.Discard && level <= l.stderrLevel {
+	if l.stderr != nil && l.stderr != ioutil.Discard && int64(level) <= atomic.LoadInt64(&l.stderrLevel) {
 		if buf[len(buf)-1] == '\n' {
 			l.stderr.Write(buf)
 		} else {

net/dns/config.go

@@ -22,27 +22,26 @@ type Config struct {
 	// for queries that fall within that suffix.
 	// If a query doesn't match any entry in Routes, the
 	// DefaultResolvers are used.
+	// A Routes entry with no resolvers means the route should be
+	// authoritatively answered using the contents of Hosts.
 	Routes map[dnsname.FQDN][]netaddr.IPPort
 	// SearchDomains are DNS suffixes to try when expanding
 	// single-label queries.
 	SearchDomains []dnsname.FQDN
 	// Hosts maps DNS FQDNs to their IPs, which can be a mix of IPv4
 	// and IPv6.
-	// Queries matching entries in Hosts are resolved locally without
-	// recursing off-machine.
+	// Queries matching entries in Hosts are resolved locally by
+	// 100.100.100.100 without leaving the machine.
+	// Adding an entry to Hosts merely creates the record. If you want
+	// it to resolve, you also need to add appropriate routes to
+	// Routes.
 	Hosts map[dnsname.FQDN][]netaddr.IP
-	// AuthoritativeSuffixes is a list of fully-qualified DNS suffixes
-	// for which the in-process Tailscale resolver is authoritative.
-	// Queries for names within AuthoritativeSuffixes can only be
-	// fulfilled by entries in Hosts. Queries with no match in Hosts
-	// return NXDOMAIN.
-	AuthoritativeSuffixes []dnsname.FQDN
 }
 
 // needsAnyResolvers reports whether c requires a resolver to be set
 // at the OS level.
 func (c Config) needsOSResolver() bool {
-	return c.hasDefaultResolvers() || c.hasRoutes() || c.hasHosts()
+	return c.hasDefaultResolvers() || c.hasRoutes()
 }
 
 func (c Config) hasRoutes() bool {
@@ -52,7 +51,7 @@ func (c Config) hasRoutes() bool {
 // hasDefaultResolversOnly reports whether the only resolvers in c are
 // DefaultResolvers.
 func (c Config) hasDefaultResolversOnly() bool {
-	return c.hasDefaultResolvers() && !c.hasRoutes() && !c.hasHosts()
+	return c.hasDefaultResolvers() && !c.hasRoutes()
 }
 
 func (c Config) hasDefaultResolvers() bool {
@@ -63,44 +62,28 @@ func (c Config) hasDefaultResolvers() bool {
 // routes use the same resolvers, or nil if multiple sets of resolvers
 // are specified.
 func (c Config) singleResolverSet() []netaddr.IPPort {
-	var first []netaddr.IPPort
+	var (
+		prev            []netaddr.IPPort
+		prevInitialized bool
+	)
 	for _, resolvers := range c.Routes {
-		if first == nil {
-			first = resolvers
+		if !prevInitialized {
+			prev = resolvers
+			prevInitialized = true
 			continue
 		}
-		if !sameIPPorts(first, resolvers) {
+		if !sameIPPorts(prev, resolvers) {
 			return nil
 		}
 	}
-	return first
+	return prev
 }
 
-// hasHosts reports whether c requires resolution of MagicDNS hosts or
-// domains.
-func (c Config) hasHosts() bool {
-	return len(c.Hosts) > 0 || len(c.AuthoritativeSuffixes) > 0
-}
-
-// matchDomains returns the list of match suffixes needed by Routes,
-// AuthoritativeSuffixes. Hosts is not considered as we assume that
-// they're covered by AuthoritativeSuffixes for now.
+// matchDomains returns the list of match suffixes needed by Routes.
 func (c Config) matchDomains() []dnsname.FQDN {
-	ret := make([]dnsname.FQDN, 0, len(c.Routes)+len(c.AuthoritativeSuffixes))
-	seen := map[dnsname.FQDN]bool{}
-	for _, suffix := range c.AuthoritativeSuffixes {
-		if seen[suffix] {
-			continue
-		}
-		ret = append(ret, suffix)
-		seen[suffix] = true
-	}
+	ret := make([]dnsname.FQDN, 0, len(c.Routes))
 	for suffix := range c.Routes {
-		if seen[suffix] {
-			continue
-		}
 		ret = append(ret, suffix)
-		seen[suffix] = true
 	}
 	sort.Slice(ret, func(i, j int) bool {
 		return ret[i].WithTrailingDot() < ret[j].WithTrailingDot()

net/dns/manager.go

@@ -6,7 +6,6 @@ package dns
 
 import (
 	"runtime"
-	"strings"
 	"time"
 
 	"inet.af/netaddr"
@@ -75,40 +74,40 @@ func (m *Manager) Set(cfg Config) error {
 
 // compileConfig converts cfg into a quad-100 resolver configuration
 // and an OS-level configuration.
-func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
+func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig, err error) {
+	// The internal resolver always gets MagicDNS hosts and
+	// authoritative suffixes, even if we don't propagate MagicDNS to
+	// the OS.
+	rcfg.Hosts = cfg.Hosts
+	routes := map[dnsname.FQDN][]netaddr.IPPort{} // assigned conditionally to rcfg.Routes below.
+	for suffix, resolvers := range cfg.Routes {
+		if len(resolvers) == 0 {
+			rcfg.LocalDomains = append(rcfg.LocalDomains, suffix)
+		} else {
+			routes[suffix] = resolvers
+		}
+	}
+	// Similarly, the OS always gets search paths.
+	ocfg.SearchDomains = cfg.SearchDomains
+
 	// Deal with trivial configs first.
 	switch {
 	case !cfg.needsOSResolver():
 		// Set search domains, but nothing else. This also covers the
 		// case where cfg is entirely zero, in which case these
 		// configs clear all Tailscale DNS settings.
-		return resolver.Config{}, OSConfig{
-			SearchDomains: cfg.SearchDomains,
-		}, nil
+		return rcfg, ocfg, nil
 	case cfg.hasDefaultResolversOnly():
 		// Trivial CorpDNS configuration, just override the OS
 		// resolver.
-		return resolver.Config{}, OSConfig{
-			Nameservers:   toIPsOnly(cfg.DefaultResolvers),
-			SearchDomains: cfg.SearchDomains,
-		}, nil
+		ocfg.Nameservers = toIPsOnly(cfg.DefaultResolvers)
+		return rcfg, ocfg, nil
 	case cfg.hasDefaultResolvers():
 		// Default resolvers plus other stuff always ends up proxying
 		// through quad-100.
-		rcfg := resolver.Config{
-			Routes: map[dnsname.FQDN][]netaddr.IPPort{
-				".": cfg.DefaultResolvers,
-			},
-			Hosts:        cfg.Hosts,
-			LocalDomains: cfg.AuthoritativeSuffixes,
-		}
-		for suffix, resolvers := range cfg.Routes {
-			rcfg.Routes[suffix] = resolvers
-		}
-		ocfg := OSConfig{
-			Nameservers:   []netaddr.IP{tsaddr.TailscaleServiceIP()},
-			SearchDomains: cfg.SearchDomains,
-		}
+		rcfg.Routes = routes
+		rcfg.Routes["."] = cfg.DefaultResolvers
+		ocfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
 		return rcfg, ocfg, nil
 	}
 
@@ -116,8 +115,6 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	// configurations. The possible cases don't return directly any
 	// more, because as a final step we have to handle the case where
 	// the OS can't do split DNS.
-	var rcfg resolver.Config
-	var ocfg OSConfig
 
 	// Workaround for
 	// https://github.com/tailscale/corp/issues/1662. Even though
@@ -135,35 +132,19 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	// This bool is used in a couple of places below to implement this
 	// workaround.
 	isWindows := runtime.GOOS == "windows"
-
-	// The windows check is for
-	// . See also below
-	// for further routing workarounds there.
-	if !cfg.hasHosts() && cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() && !isWindows {
+	if cfg.singleResolverSet() != nil && m.os.SupportsSplitDNS() && !isWindows {
 		// Split DNS configuration requested, where all split domains
 		// go to the same resolvers. We can let the OS do it.
-		return resolver.Config{}, OSConfig{
-			Nameservers:   toIPsOnly(cfg.singleResolverSet()),
-			SearchDomains: cfg.SearchDomains,
-			MatchDomains:  cfg.matchDomains(),
-		}, nil
+		ocfg.Nameservers = toIPsOnly(cfg.singleResolverSet())
+		ocfg.MatchDomains = cfg.matchDomains()
+		return rcfg, ocfg, nil
 	}
 
 	// Split DNS configuration with either multiple upstream routes,
 	// or routes + MagicDNS, or just MagicDNS, or on an OS that cannot
 	// split-DNS. Install a split config pointing at quad-100.
-	rcfg = resolver.Config{
-		Hosts:        cfg.Hosts,
-		LocalDomains: cfg.AuthoritativeSuffixes,
-		Routes:       map[dnsname.FQDN][]netaddr.IPPort{},
-	}
-	for suffix, resolvers := range cfg.Routes {
-		rcfg.Routes[suffix] = resolvers
-	}
-	ocfg = OSConfig{
-		Nameservers:   []netaddr.IP{tsaddr.TailscaleServiceIP()},
-		SearchDomains: cfg.SearchDomains,
-	}
+	rcfg.Routes = routes
+	ocfg.Nameservers = []netaddr.IP{tsaddr.TailscaleServiceIP()}
 
 	// If the OS can't do native split-dns, read out the underlying
 	// resolver config and blend it into our config.
@@ -173,28 +154,7 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 	if !m.os.SupportsSplitDNS() || isWindows {
 		bcfg, err := m.os.GetBaseConfig()
 		if err != nil {
-			// Temporary hack to make OSes where split-DNS isn't fully
-			// implemented yet not completely crap out, but instead
-			// fall back to quad-9 as a hardcoded "backup resolver".
-			//
-			// This codepath currently only triggers when opted into
-			// the split-DNS feature server side, and when at least
-			// one search domain is something within tailscale.com, so
-			// we don't accidentally leak unstable user DNS queries to
-			// quad-9 if we accidentally go down this codepath.
-			canUseHack := false
-			for _, dom := range cfg.SearchDomains {
-				if strings.HasSuffix(dom.WithoutTrailingDot(), ".tailscale.com") {
-					canUseHack = true
-					break
-				}
-			}
-			if !canUseHack {
-				return resolver.Config{}, OSConfig{}, err
-			}
-			bcfg = OSConfig{
-				Nameservers: []netaddr.IP{netaddr.IPv4(9, 9, 9, 9)},
-			}
+			return resolver.Config{}, OSConfig{}, err
 		}
 		rcfg.Routes["."] = toIPPorts(bcfg.Nameservers)
 		ocfg.SearchDomains = append(ocfg.SearchDomains, bcfg.SearchDomains...)
@@ -211,7 +171,7 @@ func (m *Manager) compileConfig(cfg Config) (resolver.Config, OSConfig, error) {
 func toIPsOnly(ipps []netaddr.IPPort) (ret []netaddr.IP) {
 	ret = make([]netaddr.IP, 0, len(ipps))
 	for _, ipp := range ipps {
-		ret = append(ret, ipp.IP)
+		ret = append(ret, ipp.IP())
 	}
 	return ret
 }
@@ -219,7 +179,7 @@ func toIPsOnly(ipps []netaddr.IPPort) (ret []netaddr.IP) {
 func toIPPorts(ips []netaddr.IP) (ret []netaddr.IPPort) {
 	ret = make([]netaddr.IPPort, 0, len(ips))
 	for _, ip := range ips {
-		ret = append(ret, netaddr.IPPort{IP: ip, Port: 53})
+		ret = append(ret, netaddr.IPPortFrom(ip, 53))
 	}
 	return ret
 }

net/dns/manager_test.go

@@ -76,6 +76,20 @@ func TestManager(t *testing.T) {
 				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 		},
+		{
+			// Regression test for https://github.com/tailscale/tailscale/issues/1886
+			name: "hosts-only",
+			in: Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+			rs: resolver.Config{
+				Hosts: hosts(
+					"dave.ts.com.", "1.2.3.4",
+					"bradfitz.ts.com.", "2.3.4.5"),
+			},
+		},
 		{
 			name: "corp",
 			in: Config{
@@ -104,10 +118,10 @@ func TestManager(t *testing.T) {
 			in: Config{
 				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
 				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+				Routes:           upstreams("ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
 			},
 			os: OSConfig{
 				Nameservers:   mustIPs("100.100.100.100"),
@@ -126,10 +140,10 @@ func TestManager(t *testing.T) {
 			in: Config{
 				DefaultResolvers: mustIPPs("1.1.1.1:53", "9.9.9.9:53"),
 				SearchDomains:    fqdns("tailscale.com", "universe.tf"),
+				Routes:           upstreams("ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
 			},
 			split: true,
 			os: OSConfig{
@@ -261,8 +275,8 @@ func TestManager(t *testing.T) {
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				Routes:        upstreams("ts.com", ""),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			bs: OSConfig{
 				Nameservers:   mustIPs("8.8.8.8"),
@@ -286,8 +300,8 @@ func TestManager(t *testing.T) {
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				Routes:        upstreams("ts.com", ""),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			split: true,
 			os: OSConfig{
@@ -305,12 +319,11 @@ func TestManager(t *testing.T) {
 		{
 			name: "routes-magic",
 			in: Config{
-				Routes: upstreams("corp.com", "2.2.2.2:53"),
+				Routes: upstreams("corp.com", "2.2.2.2:53", "ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			bs: OSConfig{
 				Nameservers:   mustIPs("8.8.8.8"),
@@ -333,12 +346,13 @@ func TestManager(t *testing.T) {
 		{
 			name: "routes-magic-split",
 			in: Config{
-				Routes: upstreams("corp.com", "2.2.2.2:53"),
+				Routes: upstreams(
+					"corp.com", "2.2.2.2:53",
+					"ts.com", ""),
 				Hosts: hosts(
 					"dave.ts.com.", "1.2.3.4",
 					"bradfitz.ts.com.", "2.3.4.5"),
-				AuthoritativeSuffixes: fqdns("ts.com"),
-				SearchDomains:         fqdns("tailscale.com", "universe.tf"),
+				SearchDomains: fqdns("tailscale.com", "universe.tf"),
 			},
 			split: true,
 			os: OSConfig{
@@ -368,11 +382,12 @@ func TestManager(t *testing.T) {
 			if err := m.Set(test.in); err != nil {
 				t.Fatalf("m.Set: %v", err)
 			}
-			tr := cmp.Transformer("ipStr", func(ip netaddr.IP) string { return ip.String() })
-			if diff := cmp.Diff(f.OSConfig, test.os, tr, cmpopts.EquateEmpty()); diff != "" {
+			trIP := cmp.Transformer("ipStr", func(ip netaddr.IP) string { return ip.String() })
+			trIPPort := cmp.Transformer("ippStr", func(ipp netaddr.IPPort) string { return ipp.String() })
+			if diff := cmp.Diff(f.OSConfig, test.os, trIP, trIPPort, cmpopts.EquateEmpty()); diff != "" {
 				t.Errorf("wrong OSConfig (-got+want)\n%s", diff)
 			}
-			if diff := cmp.Diff(f.ResolverConfig, test.rs, tr, cmpopts.EquateEmpty()); diff != "" {
+			if diff := cmp.Diff(f.ResolverConfig, test.rs, trIP, trIPPort, cmpopts.EquateEmpty()); diff != "" {
 				t.Errorf("wrong resolver.Config (-got+want)\n%s", diff)
 			}
 		})
@@ -428,7 +443,12 @@ func upstreams(strs ...string) (ret map[dnsname.FQDN][]netaddr.IPPort) {
 	var key dnsname.FQDN
 	ret = map[dnsname.FQDN][]netaddr.IPPort{}
 	for _, s := range strs {
-		if ipp, err := netaddr.ParseIPPort(s); err == nil {
+		if s == "" {
+			if key == "" {
+				panic("IPPort provided before suffix")
+			}
+			ret[key] = nil
+		} else if ipp, err := netaddr.ParseIPPort(s); err == nil {
 			if key == "" {
 				panic("IPPort provided before suffix")
 			}

net/dns/resolver/tsdns_test.go

@@ -433,8 +433,8 @@ func TestDelegateCollision(t *testing.T) {
 		qtype dns.Type
 		addr  netaddr.IPPort
 	}{
-		{"test.site.", dns.TypeA, netaddr.IPPort{IP: netaddr.IPv4(1, 1, 1, 1), Port: 1001}},
-		{"test.site.", dns.TypeAAAA, netaddr.IPPort{IP: netaddr.IPv4(1, 1, 1, 1), Port: 1002}},
+		{"test.site.", dns.TypeA, netaddr.IPPortFrom(netaddr.IPv4(1, 1, 1, 1), 1001)},
+		{"test.site.", dns.TypeAAAA, netaddr.IPPortFrom(netaddr.IPv4(1, 1, 1, 1), 1002)},
 	}
 
 	// packets will have the same dns txid.

net/interfaces/interfaces.go

@@ -195,7 +195,7 @@ func ForeachInterface(fn func(Interface, []netaddr.IPPrefix)) error {
 			}
 		}
 		sort.Slice(pfxs, func(i, j int) bool {
-			return pfxs[i].IP.Less(pfxs[j].IP)
+			return pfxs[i].IP().Less(pfxs[j].IP())
 		})
 		fn(Interface{iface}, pfxs)
 	}
@@ -264,7 +264,7 @@ func (s *State) String() string {
 			fmt.Fprintf(&sb, "%s:[", ifName)
 			needSpace := false
 			for _, pfx := range s.InterfaceIPs[ifName] {
-				if !isInterestingIP(pfx.IP) {
+				if !isInterestingIP(pfx.IP()) {
 					continue
 				}
 				if needSpace {
@@ -367,7 +367,7 @@ func (s *State) AnyInterfaceUp() bool {
 
 func hasTailscaleIP(pfxs []netaddr.IPPrefix) bool {
 	for _, pfx := range pfxs {
-		if tsaddr.IsTailscaleIP(pfx.IP) {
+		if tsaddr.IsTailscaleIP(pfx.IP()) {
 			return true
 		}
 	}
@@ -407,11 +407,11 @@ func GetState() (*State, error) {
 			return
 		}
 		for _, pfx := range pfxs {
-			if pfx.IP.IsLoopback() || pfx.IP.IsLinkLocalUnicast() {
+			if pfx.IP().IsLoopback() || pfx.IP().IsLinkLocalUnicast() {
 				continue
 			}
-			s.HaveV6Global = s.HaveV6Global || isGlobalV6(pfx.IP)
-			s.HaveV4 = s.HaveV4 || pfx.IP.Is4()
+			s.HaveV6Global = s.HaveV6Global || isGlobalV6(pfx.IP())
+			s.HaveV4 = s.HaveV4 || pfx.IP().Is4()
 		}
 	}); err != nil {
 		return nil, err
@@ -447,7 +447,7 @@ func HTTPOfListener(ln net.Listener) string {
 	var goodIP string
 	var privateIP string
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
-		ip := pfx.IP
+		ip := pfx.IP()
 		if isPrivateIP(ip) {
 			if privateIP == "" {
 				privateIP = ip.String()
@@ -484,7 +484,7 @@ func LikelyHomeRouterIP() (gateway, myIP netaddr.IP, ok bool) {
 		return
 	}
 	ForeachInterfaceAddress(func(i Interface, pfx netaddr.IPPrefix) {
-		ip := pfx.IP
+		ip := pfx.IP()
 		if !i.IsUp() || ip.IsZero() || !myIP.IsZero() {
 			return
 		}
@@ -528,7 +528,7 @@ var (
 // isInterestingIP.
 func anyInterestingIP(pfxs []netaddr.IPPrefix) bool {
 	for _, pfx := range pfxs {
-		if isInterestingIP(pfx.IP) {
+		if isInterestingIP(pfx.IP()) {
 			return true
 		}
 	}

net/netcheck/netcheck.go

@@ -625,7 +625,7 @@ func (rs *reportState) stopTimers() {
 func (rs *reportState) addNodeLatency(node *tailcfg.DERPNode, ipp netaddr.IPPort, d time.Duration) {
 	var ipPortStr string
 	if ipp != (netaddr.IPPort{}) {
-		ipPortStr = net.JoinHostPort(ipp.IP.String(), fmt.Sprint(ipp.Port))
+		ipPortStr = net.JoinHostPort(ipp.IP().String(), fmt.Sprint(ipp.Port()))
 	}
 
 	rs.mu.Lock()
@@ -650,13 +650,13 @@ func (rs *reportState) addNodeLatency(node *tailcfg.DERPNode, ipp netaddr.IPPort
 	}
 
 	switch {
-	case ipp.IP.Is6():
+	case ipp.IP().Is6():
 		updateLatency(ret.RegionV6Latency, node.RegionID, d)
 		ret.IPv6 = true
 		ret.GlobalV6 = ipPortStr
 		// TODO: track MappingVariesByDestIP for IPv6
 		// too? Would be sad if so, but who knows.
-	case ipp.IP.Is4():
+	case ipp.IP().Is4():
 		updateLatency(ret.RegionV4Latency, node.RegionID, d)
 		ret.IPv4 = true
 		if rs.gotEP4 == "" {
@@ -1172,7 +1172,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 		if proto == probeIPv6 && ip.Is4() {
 			return nil
 		}
-		return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+		return netaddr.IPPortFrom(ip, uint16(port)).UDPAddr()
 	}
 
 	switch proto {
@@ -1182,7 +1182,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is4() {
 				return nil
 			}
-			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+			return netaddr.IPPortFrom(ip, uint16(port)).UDPAddr()
 		}
 	case probeIPv6:
 		if n.IPv6 != "" {
@@ -1190,7 +1190,7 @@ func (c *Client) nodeAddr(ctx context.Context, n *tailcfg.DERPNode, proto probeP
 			if !ip.Is6() {
 				return nil
 			}
-			return netaddr.IPPort{IP: ip, Port: uint16(port)}.UDPAddr()
+			return netaddr.IPPortFrom(ip, uint16(port)).UDPAddr()
 		}
 	default:
 		return nil

net/netstat/netstat_windows.go

@@ -157,10 +157,9 @@ func ipport4(addr uint32, port uint16) netaddr.IPPort {
 	if !endian.Big {
 		addr = bits.ReverseBytes32(addr)
 	}
-	return netaddr.IPPort{
-		IP:   netaddr.IPv4(byte(addr>>24), byte(addr>>16), byte(addr>>8), byte(addr)),
-		Port: port,
-	}
+	return netaddr.IPPortFrom(
+		netaddr.IPv4(byte(addr>>24), byte(addr>>16), byte(addr>>8), byte(addr)),
+		port)
 }
 
 func ipport6(addr [16]byte, scope uint32, port uint16) netaddr.IPPort {
@@ -169,10 +168,7 @@ func ipport6(addr [16]byte, scope uint32, port uint16) netaddr.IPPort {
 		// TODO: something better here?
 		ip = ip.WithZone(fmt.Sprint(scope))
 	}
-	return netaddr.IPPort{
-		IP:   ip,
-		Port: port,
-	}
+	return netaddr.IPPortFrom(ip, port)
 }
 
 func port(v *uint32) uint16 {

net/packet/packet.go

@@ -12,7 +12,6 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/types/ipproto"
-	"tailscale.com/types/strbuilder"
 )
 
 const unknown = ipproto.Unknown
@@ -62,36 +61,17 @@ func (p *Parsed) String() string {
 		return "Unknown{???}"
 	}
 
-	sb := strbuilder.Get()
-	sb.WriteString(p.IPProto.String())
-	sb.WriteByte('{')
-	writeIPPort(sb, p.Src)
-	sb.WriteString(" > ")
-	writeIPPort(sb, p.Dst)
-	sb.WriteByte('}')
-	return sb.String()
-}
-
-// writeIPPort writes ipp.String() into sb, with fewer allocations.
-//
-// TODO: make netaddr more efficient in this area, and retire this func.
-func writeIPPort(sb *strbuilder.Builder, ipp netaddr.IPPort) {
-	if ipp.IP.Is4() {
-		raw := ipp.IP.As4()
-		sb.WriteUint(uint64(raw[0]))
-		sb.WriteByte('.')
-		sb.WriteUint(uint64(raw[1]))
-		sb.WriteByte('.')
-		sb.WriteUint(uint64(raw[2]))
-		sb.WriteByte('.')
-		sb.WriteUint(uint64(raw[3]))
-		sb.WriteByte(':')
-	} else {
-		sb.WriteByte('[')
-		sb.WriteString(ipp.IP.String()) // TODO: faster?
-		sb.WriteString("]:")
-	}
-	sb.WriteUint(uint64(ipp.Port))
+	// max is the maximum reasonable length of the string we are constructing.
+	// It's OK to overshoot, as the temp buffer is allocated on the stack.
+	const max = len("ICMPv6{[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535 > [ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535}")
+	b := make([]byte, 0, max)
+	b = append(b, p.IPProto.String()...)
+	b = append(b, '{')
+	b = p.Src.AppendTo(b)
+	b = append(b, ' ', '>', ' ')
+	b = p.Dst.AppendTo(b)
+	b = append(b, '}')
+	return string(b)
 }
 
 // Decode extracts data from the packet in b into q.
@@ -142,8 +122,8 @@ func (q *Parsed) decode4(b []byte) {
 	}
 
 	// If it's valid IPv4, then the IP addresses are valid
-	q.Src.IP = netaddr.IPv4(b[12], b[13], b[14], b[15])
-	q.Dst.IP = netaddr.IPv4(b[16], b[17], b[18], b[19])
+	q.Src = q.Src.WithIP(netaddr.IPv4(b[12], b[13], b[14], b[15]))
+	q.Dst = q.Dst.WithIP(netaddr.IPv4(b[16], b[17], b[18], b[19]))
 
 	q.subofs = int((b[0] & 0x0F) << 2)
 	if q.subofs > q.length {
@@ -185,8 +165,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = 0
-			q.Dst.Port = 0
+			q.Src = q.Src.WithPort(0)
+			q.Dst = q.Dst.WithPort(0)
 			q.dataofs = q.subofs + icmp4HeaderLength
 			return
 		case ipproto.IGMP:
@@ -198,8 +178,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-			q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+			q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+			q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 			q.TCPFlags = TCPFlag(sub[13]) & 0x3F
 			headerLength := (sub[12] & 0xF0) >> 2
 			q.dataofs = q.subofs + int(headerLength)
@@ -209,8 +189,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-			q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+			q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+			q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 			q.dataofs = q.subofs + udpHeaderLength
 			return
 		case ipproto.SCTP:
@@ -218,8 +198,8 @@ func (q *Parsed) decode4(b []byte) {
 				q.IPProto = unknown
 				return
 			}
-			q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-			q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+			q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+			q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 			return
 		case ipproto.TSMP:
 			// Inter-tailscale messages.
@@ -265,8 +245,10 @@ func (q *Parsed) decode6(b []byte) {
 
 	// okay to ignore `ok` here, because IPs pulled from packets are
 	// always well-formed stdlib IPs.
-	q.Src.IP, _ = netaddr.FromStdIP(net.IP(b[8:24]))
-	q.Dst.IP, _ = netaddr.FromStdIP(net.IP(b[24:40]))
+	srcIP, _ := netaddr.FromStdIP(net.IP(b[8:24]))
+	dstIP, _ := netaddr.FromStdIP(net.IP(b[24:40]))
+	q.Src = q.Src.WithIP(srcIP)
+	q.Dst = q.Dst.WithIP(dstIP)
 
 	// We don't support any IPv6 extension headers. Don't try to
 	// be clever. Therefore, the IP subprotocol always starts at
@@ -290,16 +272,16 @@ func (q *Parsed) decode6(b []byte) {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = 0
-		q.Dst.Port = 0
+		q.Src = q.Src.WithPort(0)
+		q.Dst = q.Dst.WithPort(0)
 		q.dataofs = q.subofs + icmp6HeaderLength
 	case ipproto.TCP:
 		if len(sub) < tcpHeaderLength {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-		q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+		q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+		q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 		q.TCPFlags = TCPFlag(sub[13]) & 0x3F
 		headerLength := (sub[12] & 0xF0) >> 2
 		q.dataofs = q.subofs + int(headerLength)
@@ -309,16 +291,16 @@ func (q *Parsed) decode6(b []byte) {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-		q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+		q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+		q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 		q.dataofs = q.subofs + udpHeaderLength
 	case ipproto.SCTP:
 		if len(sub) < sctpHeaderLength {
 			q.IPProto = unknown
 			return
 		}
-		q.Src.Port = binary.BigEndian.Uint16(sub[0:2])
-		q.Dst.Port = binary.BigEndian.Uint16(sub[2:4])
+		q.Src = q.Src.WithPort(binary.BigEndian.Uint16(sub[0:2]))
+		q.Dst = q.Dst.WithPort(binary.BigEndian.Uint16(sub[2:4]))
 		return
 	case ipproto.TSMP:
 		// Inter-tailscale messages.
@@ -338,8 +320,8 @@ func (q *Parsed) IP4Header() IP4Header {
 	return IP4Header{
 		IPID:    ipid,
 		IPProto: q.IPProto,
-		Src:     q.Src.IP,
-		Dst:     q.Dst.IP,
+		Src:     q.Src.IP(),
+		Dst:     q.Dst.IP(),
 	}
 }
 
@@ -351,8 +333,8 @@ func (q *Parsed) IP6Header() IP6Header {
 	return IP6Header{
 		IPID:    ipid,
 		IPProto: q.IPProto,
-		Src:     q.Src.IP,
-		Dst:     q.Dst.IP,
+		Src:     q.Src.IP(),
+		Dst:     q.Dst.IP(),
 	}
 }
 
@@ -373,8 +355,8 @@ func (q *Parsed) UDP4Header() UDP4Header {
 	}
 	return UDP4Header{
 		IP4Header: q.IP4Header(),
-		SrcPort:   q.Src.Port,
-		DstPort:   q.Dst.Port,
+		SrcPort:   q.Src.Port(),
+		DstPort:   q.Dst.Port(),
 	}
 }
 

net/packet/packet_test.go

@@ -378,11 +378,9 @@ func TestParsedString(t *testing.T) {
 		})
 	}
 
-	var sink string
 	allocs := testing.AllocsPerRun(1000, func() {
-		sink = tests[0].qdecode.String()
+		sinkString = tests[0].qdecode.String()
 	})
-	_ = sink
 	if allocs != 1 {
 		t.Errorf("allocs = %v; want 1", allocs)
 	}
@@ -532,3 +530,33 @@ func TestMarshalResponse(t *testing.T) {
 		})
 	}
 }
+
+var sinkString string
+
+func BenchmarkString(b *testing.B) {
+	benches := []struct {
+		name string
+		buf  []byte
+	}{
+		{"tcp4", tcp4PacketBuffer},
+		{"tcp6", tcp6RequestBuffer},
+		{"udp4", udp4RequestBuffer},
+		{"udp6", udp6RequestBuffer},
+		{"icmp4", icmp4RequestBuffer},
+		{"icmp6", icmp6PacketBuffer},
+		{"igmp", igmpPacketBuffer},
+		{"unknown", unknownPacketBuffer},
+	}
+
+	for _, bench := range benches {
+		b.Run(bench.name, func(b *testing.B) {
+			b.ReportAllocs()
+			var p Parsed
+			p.Decode(bench.buf)
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				sinkString = p.String()
+			}
+		})
+	}
+}

net/packet/tsmp.go

@@ -14,6 +14,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"log"
 
 	"inet.af/netaddr"
 	"tailscale.com/net/flowtrack"
@@ -143,7 +144,7 @@ func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
 	if len(buf) > maxPacketLength {
 		return errLargePacket
 	}
-	if h.Src.IP.Is4() {
+	if h.Src.IP().Is4() {
 		iph := IP4Header{
 			IPProto: ipproto.TSMP,
 			Src:     h.IPSrc,
@@ -151,7 +152,7 @@ func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
 		}
 		iph.Marshal(buf)
 		buf = buf[ip4HeaderLength:]
-	} else if h.Src.IP.Is6() {
+	} else if h.Src.IP().Is6() {
 		iph := IP6Header{
 			IPProto: ipproto.TSMP,
 			Src:     h.IPSrc,
@@ -165,8 +166,8 @@ func (h TailscaleRejectedHeader) Marshal(buf []byte) error {
 	buf[0] = byte(TSMPTypeRejectedConn)
 	buf[1] = byte(h.Proto)
 	buf[2] = byte(h.Reason)
-	binary.BigEndian.PutUint16(buf[3:5], h.Src.Port)
-	binary.BigEndian.PutUint16(buf[5:7], h.Dst.Port)
+	binary.BigEndian.PutUint16(buf[3:5], h.Src.Port())
+	binary.BigEndian.PutUint16(buf[5:7], h.Dst.Port())
 
 	if h.hasFlags() {
 		var flags byte
@@ -190,10 +191,10 @@ func (pp *Parsed) AsTailscaleRejectedHeader() (h TailscaleRejectedHeader, ok boo
 	h = TailscaleRejectedHeader{
 		Proto:  ipproto.Proto(p[1]),
 		Reason: TailscaleRejectReason(p[2]),
-		IPSrc:  pp.Src.IP,
-		IPDst:  pp.Dst.IP,
-		Src:    netaddr.IPPort{IP: pp.Dst.IP, Port: binary.BigEndian.Uint16(p[3:5])},
-		Dst:    netaddr.IPPort{IP: pp.Src.IP, Port: binary.BigEndian.Uint16(p[5:7])},
+		IPSrc:  pp.Src.IP(),
+		IPDst:  pp.Dst.IP(),
+		Src:    netaddr.IPPortFrom(pp.Dst.IP(), binary.BigEndian.Uint16(p[3:5])),
+		Dst:    netaddr.IPPortFrom(pp.Src.IP(), binary.BigEndian.Uint16(p[5:7])),
 	}
 	if len(p) > 7 {
 		flags := p[7]
@@ -232,6 +233,7 @@ type TSMPPongReply struct {
 // AsTSMPPong returns pp as a TSMPPongReply and whether it is one.
 // The pong.IPHeader field is not populated.
 func (pp *Parsed) AsTSMPPong() (pong TSMPPongReply, ok bool) {
+	log.Println("TSMPPONG")
 	if pp.IPProto != ipproto.TSMP {
 		return
 	}

net/portmapper/portmapper.go

@@ -84,7 +84,7 @@ type pmpMapping struct {
 
 // externalValid reports whether m.external is valid, with both its IP and Port populated.
 func (m *pmpMapping) externalValid() bool {
-	return !m.external.IP.IsZero() && m.external.Port != 0
+	return !m.external.IP().IsZero() && m.external.Port() != 0
 }
 
 // release does a best effort fire-and-forget release of the PMP mapping m.
@@ -94,8 +94,8 @@ func (m *pmpMapping) release() {
 		return
 	}
 	defer uc.Close()
-	pkt := buildPMPRequestMappingPacket(m.internal.Port, m.external.Port, pmpMapLifetimeDelete)
-	uc.WriteTo(pkt, netaddr.IPPort{IP: m.gw, Port: pmpPort}.UDPAddr())
+	pkt := buildPMPRequestMappingPacket(m.internal.Port(), m.external.Port(), pmpMapLifetimeDelete)
+	uc.WriteTo(pkt, netaddr.IPPortFrom(m.gw, pmpPort).UDPAddr())
 }
 
 // NewClient returns a new portmapping client.
@@ -256,7 +256,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	localPort := c.localPort
 	m := &pmpMapping{
 		gw:       gw,
-		internal: netaddr.IPPort{IP: myIP, Port: localPort},
+		internal: netaddr.IPPortFrom(myIP, localPort),
 	}
 
 	// prevPort is the port we had most previously, if any. We try
@@ -271,7 +271,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 			return m.external, nil
 		}
 		// The mapping might still be valid, so just try to renew it.
-		prevPort = m.external.Port
+		prevPort = m.external.Port()
 	}
 
 	// If we just did a Probe (e.g. via netchecker) but didn't
@@ -279,7 +279,7 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	// again. Cuts down latency for most clients.
 	haveRecentPMP := c.sawPMPRecentlyLocked()
 	if haveRecentPMP {
-		m.external.IP = c.pmpPubIP
+		m.external = m.external.WithIP(c.pmpPubIP)
 	}
 	if c.lastProbe.After(now.Add(-5*time.Second)) && !haveRecentPMP {
 		c.mu.Unlock()
@@ -297,11 +297,11 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 	uc.SetReadDeadline(time.Now().Add(portMapServiceTimeout))
 	defer closeCloserOnContextDone(ctx, uc)()
 
-	pmpAddr := netaddr.IPPort{IP: gw, Port: pmpPort}
+	pmpAddr := netaddr.IPPortFrom(gw, pmpPort)
 	pmpAddru := pmpAddr.UDPAddr()
 
 	// Ask for our external address if needed.
-	if m.external.IP.IsZero() {
+	if m.external.IP().IsZero() {
 		if _, err := uc.WriteTo(pmpReqExternalAddrPacket, pmpAddru); err != nil {
 			return netaddr.IPPort{}, err
 		}
@@ -337,10 +337,10 @@ func (c *Client) CreateOrGetMapping(ctx context.Context) (external netaddr.IPPor
 				return netaddr.IPPort{}, NoMappingError{fmt.Errorf("PMP response Op=0x%x,Res=0x%x", pres.OpCode, pres.ResultCode)}
 			}
 			if pres.OpCode == pmpOpReply|pmpOpMapPublicAddr {
-				m.external.IP = pres.PublicAddr
+				m.external = m.external.WithIP(pres.PublicAddr)
 			}
 			if pres.OpCode == pmpOpReply|pmpOpMapUDP {
-				m.external.Port = pres.ExternalPort
+				m.external = m.external.WithPort(pres.ExternalPort)
 				d := time.Duration(pres.MappingValidSeconds) * time.Second
 				d /= 2 // renew in half the time
 				m.useUntil = time.Now().Add(d)
@@ -468,9 +468,9 @@ func (c *Client) Probe(ctx context.Context) (res ProbeResult, err error) {
 	defer cancel()
 	defer closeCloserOnContextDone(ctx, uc)()
 
-	pcpAddr := netaddr.IPPort{IP: gw, Port: pcpPort}.UDPAddr()
-	pmpAddr := netaddr.IPPort{IP: gw, Port: pmpPort}.UDPAddr()
-	upnpAddr := netaddr.IPPort{IP: gw, Port: upnpPort}.UDPAddr()
+	pcpAddr := netaddr.IPPortFrom(gw, pcpPort).UDPAddr()
+	pmpAddr := netaddr.IPPortFrom(gw, pmpPort).UDPAddr()
+	upnpAddr := netaddr.IPPortFrom(gw, upnpPort).UDPAddr()
 
 	// Don't send probes to services that we recently learned (for
 	// the same gw/myIP) are available. See

net/tsaddr/tsaddr.go

@@ -92,7 +92,7 @@ func TailscaleEphemeral6Range() netaddr.IPPrefix {
 // Currently used to work around a Windows limitation when programming
 // IPv6 routes in corner cases.
 func Tailscale4To6Placeholder() netaddr.IP {
-	return Tailscale4To6Range().IP
+	return Tailscale4To6Range().IP()
 }
 
 // Tailscale4To6 returns a Tailscale IPv6 address that maps 1:1 to the
@@ -102,7 +102,7 @@ func Tailscale4To6(ipv4 netaddr.IP) netaddr.IP {
 	if !ipv4.Is4() || !IsTailscaleIP(ipv4) {
 		return netaddr.IP{}
 	}
-	ret := Tailscale4To6Range().IP.As16()
+	ret := Tailscale4To6Range().IP().As16()
 	v4 := ipv4.As4()
 	copy(ret[13:], v4[1:])
 	return netaddr.IPFrom16(ret)
@@ -172,16 +172,16 @@ func NewContainsIPFunc(addrs []netaddr.IPPrefix) func(ip netaddr.IP) bool {
 	// Fast paths for 1 and 2 IPs:
 	if len(addrs) == 1 {
 		a := addrs[0]
-		return func(ip netaddr.IP) bool { return ip == a.IP }
+		return func(ip netaddr.IP) bool { return ip == a.IP() }
 	}
 	if len(addrs) == 2 {
 		a, b := addrs[0], addrs[1]
-		return func(ip netaddr.IP) bool { return ip == a.IP || ip == b.IP }
+		return func(ip netaddr.IP) bool { return ip == a.IP() || ip == b.IP() }
 	}
 	// General case:
 	m := map[netaddr.IP]bool{}
 	for _, a := range addrs {
-		m[a.IP] = true
+		m[a.IP()] = true
 	}
 	return func(ip netaddr.IP) bool { return m[ip] }
 }

net/tstun/fake.go

@@ -8,7 +8,7 @@ import (
 	"io"
 	"os"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 )
 
 type fakeTUN struct {

net/tstun/ifstatus_noop.go

@@ -9,7 +9,7 @@ package tstun
 import (
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 )
 

net/tstun/ifstatus_windows.go

@@ -9,7 +9,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"tailscale.com/types/logger"
 )

net/tstun/tun.go

@@ -13,7 +13,7 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 	"tailscale.com/version/distro"
 )

net/tstun/tun_notwindows.go

@@ -6,7 +6,7 @@
 
 package tstun
 
-import "github.com/tailscale/wireguard-go/tun"
+import "golang.zx2c4.com/wireguard/tun"
 
 func interfaceName(dev tun.Device) (string, error) {
 	return dev.Name()

net/tstun/tun_windows.go

@@ -5,9 +5,9 @@
 package tstun
 
 import (
-	"github.com/tailscale/wireguard-go/tun"
-	"github.com/tailscale/wireguard-go/tun/wintun"
 	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/tun"
+	"golang.zx2c4.com/wireguard/tun/wintun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 )
 

net/tstun/wrap.go

@@ -9,13 +9,14 @@ package tstun
 import (
 	"errors"
 	"io"
+	"log"
 	"os"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/tailscale/wireguard-go/device"
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/types/ipproto"
@@ -276,6 +277,7 @@ func (t *Wrapper) poll() {
 var magicDNSIPPort = netaddr.MustParseIPPort("100.100.100.100:0")
 
 func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
+	log.Println("FILTEROUT")
 	// Fake ICMP echo responses to MagicDNS (100.100.100.100).
 	if p.IsEchoRequest() && p.Dst == magicDNSIPPort {
 		header := p.ICMP4Header()
@@ -352,7 +354,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	p.Decode(buf[offset : offset+n])
 
 	if m, ok := t.destIPActivity.Load().(map[netaddr.IP]func()); ok {
-		if fn := m[p.Dst.IP]; fn != nil {
+		if fn := m[p.Dst.IP()]; fn != nil {
 			fn()
 		}
 	}
@@ -376,6 +378,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 }
 
 func (t *Wrapper) filterIn(buf []byte) filter.Response {
+	log.Println("FILTERIN")
 	p := parsedPacketPool.Get().(*packet.Parsed)
 	defer parsedPacketPool.Put(p)
 	p.Decode(buf)
@@ -412,7 +415,7 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 		p.IPProto == ipproto.TCP &&
 		p.TCPFlags&packet.TCPSyn != 0 &&
 		t.PeerAPIPort != nil {
-		if port, ok := t.PeerAPIPort(p.Dst.IP); ok && port == p.Dst.Port {
+		if port, ok := t.PeerAPIPort(p.Dst.IP()); ok && port == p.Dst.Port() {
 			outcome = filter.Accept
 		}
 	}
@@ -425,8 +428,8 @@ func (t *Wrapper) filterIn(buf []byte) filter.Response {
 		// can show them a rejection history with reasons.
 		if p.IPVersion == 4 && p.IPProto == ipproto.TCP && p.TCPFlags&packet.TCPSyn != 0 && !t.disableTSMPRejected {
 			rj := packet.TailscaleRejectedHeader{
-				IPSrc:  p.Dst.IP,
-				IPDst:  p.Src.IP,
+				IPSrc:  p.Dst.IP(),
+				IPDst:  p.Src.IP(),
 				Src:    p.Src,
 				Dst:    p.Dst,
 				Proto:  p.IPProto,
@@ -532,11 +535,12 @@ func (t *Wrapper) InjectInboundCopy(packet []byte) error {
 }
 
 func (t *Wrapper) injectOutboundPong(pp *packet.Parsed, req packet.TSMPPingRequest) {
+	log.Println("INJECT OUTBOUND")
 	pong := packet.TSMPPongReply{
 		Data: req.Data,
 	}
 	if t.PeerAPIPort != nil {
-		pong.PeerAPIPort, _ = t.PeerAPIPort(pp.Dst.IP)
+		pong.PeerAPIPort, _ = t.PeerAPIPort(pp.Dst.IP())
 	}
 	switch pp.IPVersion {
 	case 4:
@@ -568,8 +572,10 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 	}
 	select {
 	case <-t.closed:
+		log.Println("Closed")
 		return ErrClosed
 	case t.outbound <- packet:
+		log.Println("t.outbound <- packet")
 		return nil
 	}
 }

net/tstun/wrap_test.go

@@ -14,7 +14,7 @@ import (
 	"testing"
 	"unsafe"
 
-	"github.com/tailscale/wireguard-go/tun/tuntest"
+	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/net/packet"
 	"tailscale.com/types/ipproto"
@@ -82,7 +82,7 @@ func nets(nets ...string) (ret []netaddr.IPPrefix) {
 			if ip.Is6() {
 				bits = 128
 			}
-			ret = append(ret, netaddr.IPPrefix{IP: ip, Bits: bits})
+			ret = append(ret, netaddr.IPPrefixFrom(ip, bits))
 		} else {
 			pfx, err := netaddr.ParseIPPrefix(s)
 			if err != nil {

paths/paths.go

@@ -13,9 +13,9 @@ import (
 	"sync/atomic"
 )
 
-// IOSSharedDir is a string set by the iOS app on start
+// AppSharedDir is a string set by the iOS or Android app on start
 // containing a directory we can read/write in.
-var IOSSharedDir atomic.Value
+var AppSharedDir atomic.Value
 
 // DefaultTailscaledSocket returns the path to the tailscaled Unix socket
 // or the empty string if there's no reasonable default.

scripts/installer.sh

@@ -0,0 +1,399 @@
+#!/bin/sh
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# This script detects the current operating system, and installs
+# Tailscale according to that OS's conventions.
+
+set -eu
+
+# All the code is wrapped in a main function that gets called at the
+# bottom of the file, so that a truncated partial download doesn't end
+# up executing half a script.
+main() {
+	# Step 1: detect the current linux distro, version, and packaging system.
+	#
+	# We rely on a combination of 'uname' and /etc/os-release to find
+	# an OS name and version, and from there work out what
+	# installation method we should be using.
+	#
+	# The end result of this step is that the following three
+	# variables are populated, if detection was successful.
+	OS=""
+	VERSION=""
+	PACKAGETYPE=""
+
+	if [ -f /etc/os-release ]; then
+		# /etc/os-release populates a number of shell variables. We care about the following:
+		#  - ID: the short name of the OS (e.g. "debian", "freebsd")
+		#  - VERSION_ID: the numeric release version for the OS, if any (e.g. "18.04")
+		#  - VERSION_CODENAME: the codename of the OS release, if any (e.g. "buster")
+		. /etc/os-release
+		case "$ID" in
+			ubuntu)
+				OS="$ID"
+				VERSION="$VERSION_CODENAME"
+				PACKAGETYPE="apt"
+				;;
+			debian)
+				OS="$ID"
+				VERSION="$VERSION_CODENAME"
+				PACKAGETYPE="apt"
+				;;
+			raspbian)
+				OS="$ID"
+				VERSION="$VERSION_CODENAME"
+				PACKAGETYPE="apt"
+				;;
+			centos)
+				OS="$ID"
+				VERSION="$VERSION_ID"
+				PACKAGETYPE="dnf"
+				if [ "$VERSION" = "7" ]; then
+					PACKAGETYPE="yum"
+				fi
+				;;
+			rhel)
+				OS="$ID"
+				VERSION="$(echo "$VERSION_ID" | cut -f1 -d.)"
+				PACKAGETYPE="dnf"
+				;;
+			fedora)
+				OS="$ID"
+				VERSION=""
+				PACKAGETYPE="dnf"
+				;;
+			amzn)
+				OS="amazon-linux"
+				VERSION="$VERSION_ID"
+				PACKAGETYPE="yum"
+				;;
+			opensuse-leap)
+				OS="opensuse"
+				VERSION="leap/$VERSION_ID"
+				PACKAGETYPE="zypper"
+				;;
+			opensuse-tumbleweed)
+				OS="opensuse"
+				VERSION="tumbleweed"
+				PACKAGETYPE="zypper"
+				;;
+ 			arch)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="pacman"
+				;;
+			manjaro)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="pacman"
+				;;
+			alpine)
+				OS="$ID"
+				VERSION="$VERSION_ID"
+				PACKAGETYPE="apk"
+				;;
+			nixos)
+				echo "Please add Tailscale to your NixOS configuration directly:"
+				echo
+				echo "services.tailscale.enable = true;"
+				exit 1
+				;;
+			void)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="xbps"
+				;;
+			gentoo)
+				OS="$ID"
+				VERSION="" # rolling release
+				PACKAGETYPE="emerge"
+				;;
+			freebsd)
+				OS="$ID"
+				VERSION="$(echo "$VERSION_ID" | cut -f1 -d.)"
+				PACKAGETYPE="pkg"
+				;;
+			# TODO: wsl?
+			# TODO: synology? qnap?
+		esac
+	fi
+
+	# If we failed to detect something through os-release, consult
+	# uname and try to infer things from that.
+	if [ -z "$OS" ]; then
+		if type uname >/dev/null 2>&1; then
+			case "$(uname)" in
+				FreeBSD)
+					# FreeBSD before 12.2 doesn't have
+					# /etc/os-release, so we wouldn't have found it in
+					# the os-release probing above.
+					OS="freebsd"
+					VERSION="$(freebsd-version | cut -f1 -d.)"
+					PACKAGETYPE="pkg"
+					;;
+				OpenBSD)
+					OS="openbsd"
+					VERSION="$(uname -r)"
+					PACKAGETYPE=""
+					;;
+				Darwin)
+					OS="macos"
+					VERSION="$(sw_vers -productVersion | cut -f1-2 -d.)"
+					PACKAGETYPE="appstore"
+					;;
+				Linux)
+					OS="other-linux"
+					VERSION=""
+					PACKAGETYPE=""
+					;;
+			esac
+		fi
+	fi
+
+	# Step 2: having detected an OS we support, is it one of the
+	# versions we support?
+	OS_UNSUPPORTED=
+	case "$OS" in
+		ubuntu)
+			if [ "$VERSION" != "xenial" ] && \
+			   [ "$VERSION" != "bionic" ] && \
+			   [ "$VERSION" != "eoan" ] && \
+			   [ "$VERSION" != "focal" ] && \
+			   [ "$VERSION" != "groovy" ] && \
+			   [ "$VERSION" != "hirsute" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		debian)
+			if [ "$VERSION" != "stretch" ] && \
+			   [ "$VERSION" != "buster" ] && \
+			   [ "$VERSION" != "bullseye" ] && \
+			   [ "$VERSION" != "sid" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		raspbian)
+			if [ "$VERSION" != "buster" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		centos)
+			if [ "$VERSION" != "7" ] && \
+			   [ "$VERSION" != "8" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		rhel)
+			if [ "$VERSION" != "8" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		amazon-linux)
+			if [ "$VERSION" != "2" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+		;;
+		opensuse)
+			if [ "$VERSION" != "leap/15.1" ] && \
+			   [ "$VERSION" != "leap/15.2" ] && \
+			   [ "$VERSION" != "tumbleweed" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+			;;
+		arch)
+			# Rolling release, no version checking needed.
+			;;
+		manjaro)
+			# Rolling release, no version checking needed.
+			;;
+		alpine)
+			# All versions supported, no version checking needed.
+			# TODO: is that true? When was tailscale packaged?
+			;;
+		void)
+			# Rolling release, no version checking needed.
+			;;
+		gentoo)
+			# Rolling release, no version checking needed.
+			;;
+		freebsd)
+			if [ "$VERSION" != "12" ] && \
+			   [ "$VERSION" != "13" ]
+			then
+				OS_UNSUPPORTED=1
+			fi
+			;;
+		openbsd)
+			OS_UNSUPPORTED=1
+			;;
+		macos)
+			# We delegate macOS installation to the app store, it will
+			# perform version checks for us.
+			;;
+		other-linux)
+			OS_UNSUPPORTED=1
+			;;
+		*)
+			OS_UNSUPPORTED=1
+			;;
+	esac
+	if [ "$OS_UNSUPPORTED" = "1" ]; then
+		case "$OS" in
+			other-linux)
+				echo "Couldn't determine what kind of Linux is running."
+				echo "You could try the static binaries at:"
+				echo "https://pkgs.tailscale.com/stable/#static"
+				;;
+			"")
+				echo "Couldn't determine what operating system you're running."
+				;;
+			*)
+				echo "$OS $VERSION isn't supported by this script yet."
+				;;
+		esac
+		echo
+		echo "If you'd like us to support your system better, please email support@tailscale.com"
+		echo "and tell us what OS you're running."
+		echo
+		echo "Please include the following information we gathered from your system:"
+		echo
+		echo "OS=$OS"
+		echo "VERSION=$VERSION"
+		echo "PACKAGETYPE=$PACKAGETYPE"
+		if type uname >/dev/null 2>&1; then
+			echo "UNAME=$(uname -a)"
+		else
+			echo "UNAME="
+		fi
+		echo
+		if [ -f /etc/os-release ]; then
+			cat /etc/os-release
+		else
+			echo "No /etc/os-release"
+		fi
+		exit 1
+	fi
+
+	# Step 3: work out if we can run privileged commands, and if so,
+	# how.
+	CAN_ROOT=
+	SUDO=
+	if [ "$(id -u)" = 0 ]; then
+		CAN_ROOT=1
+		SUDO=""
+	elif type sudo >/dev/null; then
+		CAN_ROOT=1
+		SUDO="sudo"
+	elif type doas >/dev/null; then
+		CAN_ROOT=1
+		SUDO="doas"
+	fi
+	if [ "$CAN_ROOT" != "1" ]; then
+		echo "This installer needs to run commands as root."
+		echo "We tried looking for 'sudo' and 'doas', but couldn't find them."
+		echo "Either re-run this script as root, or set up sudo/doas."
+		exit 1
+	fi
+
+
+	# Step 4: run the installation.
+	echo "Installing Tailscale for $OS $VERSION, using method $PACKAGETYPE"
+	case "$PACKAGETYPE" in
+		apt)
+			# Ideally we want to use curl, but on some installs we
+			# only have wget. Detect and use what's available.
+			CURL=
+			if type curl >/dev/null; then
+				CURL="curl -fsSL"
+			elif type wget >/dev/null; then
+				CURL="wget -q -O-"
+			fi
+			if [ -z "$CURL" ]; then
+				echo "The installer needs either curl or wget to download files."
+				echo "Please install either curl or wget to proceed."
+				exit 1
+			fi
+
+			# TODO: use newfangled per-repo signature scheme
+			set -x
+			$CURL "https://pkgs.tailscale.com/stable/$OS/$VERSION.gpg" | $SUDO apt-key add -
+			$CURL "https://pkgs.tailscale.com/stable/$OS/$VERSION.list" | $SUDO tee /etc/apt/sources.list.d/tailscale.list
+			$SUDO apt-get update
+			$SUDO apt-get install tailscale
+			set +x
+		;;
+		yum)
+			set -x
+			$SUDO yum install yum-utils
+			$SUDO yum-config-manager --add-repo "https://pkgs.tailscale.com/stable/$OS/$VERSION/tailscale.repo"
+			$SUDO yum install tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+		;;
+		dnf)
+			set -x
+			$SUDO dnf config-manager --add-repo "https://pkgs.tailscale.com/stable/$OS/$VERSION/tailscale.repo"
+			$SUDO dnf install tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+		;;
+		zypper)
+			set -x
+			$SUDO zypper ar -g -r "https://pkgs.tailscale.com/stable/$OS/$VERSION/tailscale.repo"
+			$SUDO zypper ref
+			$SUDO zypper in tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+			;;
+		pacman)
+			set -x
+			$SUDO pacman -S tailscale
+			$SUDO systemctl enable --now tailscaled
+			set +x
+			;;
+		apk)
+			set -x
+			$SUDO apk add tailscale
+			$SUDO rc-update add tailscale
+			set +x
+			;;
+		xbps)
+			set -x
+			$SUDO xbps-install tailscale
+			set +x
+			;;
+		emerge)
+			set -x
+			$SUDO emerge net-vpn/tailscale
+			set +x
+			;;
+		appstore)
+			set -x
+			open "https://apps.apple.com/us/app/tailscale/id1475387142"
+			set +x
+			;;
+		*)
+			echo "unexpected: unknown package type $PACKAGETYPE"
+			exit 1
+			;;
+	esac
+
+	echo "Installation complete! Log in to start using Tailscale by running:"
+	echo
+	if [ -z "$SUDO" ]; then
+		echo "tailscale up"
+	else
+		echo "$SUDO tailscale up"
+	fi
+}
+
+main

tailcfg/tailcfg.go

@@ -749,6 +749,9 @@ type MapRequest struct {
 	//     * "minimize-netmap": have control minimize the netmap, removing
 	//       peers that are unreachable per ACLS.
 	DebugFlags []string `json:",omitempty"`
+
+	// Basic boolean field to determine if a Ping is being intitiated
+	Ping bool
 }
 
 // PortRange represents a range of UDP or TCP port numbers.
@@ -884,6 +887,27 @@ type PingRequest struct {
 	// Log is whether to log about this ping in the success case.
 	// For failure cases, the client will log regardless.
 	Log bool `json:",omitempty"`
+
+	Initiator        string // admin@email; "system" (for Tailscale)
+	TestIP           netaddr.IP
+	Types            string // empty means all: TSMP+ICMP+disco
+	StopAfterNDirect int    // 1 means stop on 1st direct ping; 4 means 4 direct pings; 0 means do MaxPings and stop
+	MaxPings         int    // MaxPings total, direct or DERPed
+	PayloadSize      int    // default: 0 extra bytes
+}
+
+// According to https://roamresearch.com/#/app/ts-corp/page/4Bn_Famn2
+// Client can stream responses back via HTTP
+// We will add a struct with the proper fields
+type StreamedPingResult struct {
+	IP      netaddr.IP
+	SeqNum  int     // somewhat redundant with TxID but for clarity
+	SentTo  NodeID  // for exit/subnet relays
+	TxID    string  // N hex bytes random
+	Dir     string  // "in"/"out"
+	Type    string  // ICMP, disco, TSMP, ...
+	Via     string  // "direct", "derp-nyc", ...
+	Seconds float64 // for Dir "in" only
 }
 
 type MapResponse struct {
@@ -1026,11 +1050,16 @@ func (k MachineKey) MarshalText() ([]byte, error)     { return keyMarshalText("m
 func (k MachineKey) HexString() string                { return fmt.Sprintf("%x", k[:]) }
 func (k *MachineKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "mkey:", text) }
 
-func keyMarshalText(prefix string, k [32]byte) []byte {
-	buf := make([]byte, len(prefix)+64)
+func appendKey(base []byte, prefix string, k [32]byte) []byte {
+	ret := append(base, make([]byte, len(prefix)+64)...)
+	buf := ret[len(base):]
 	copy(buf, prefix)
 	hex.Encode(buf[len(prefix):], k[:])
-	return buf
+	return ret
+}
+
+func keyMarshalText(prefix string, k [32]byte) []byte {
+	return appendKey(nil, prefix, k)
 }
 
 func keyUnmarshalText(dst []byte, prefix string, text []byte) error {
@@ -1061,6 +1090,7 @@ func (k DiscoKey) String() string                   { return fmt.Sprintf("discok
 func (k DiscoKey) MarshalText() ([]byte, error)     { return keyMarshalText("discokey:", k), nil }
 func (k *DiscoKey) UnmarshalText(text []byte) error { return keyUnmarshalText(k[:], "discokey:", text) }
 func (k DiscoKey) ShortString() string              { return fmt.Sprintf("d:%x", k[:8]) }
+func (k DiscoKey) AppendTo(b []byte) []byte         { return appendKey(b, "discokey:", k) }
 
 // IsZero reports whether k is the zero value.
 func (k DiscoKey) IsZero() bool { return k == DiscoKey{} }

tailcfg/tailcfg_test.go

@@ -14,6 +14,7 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/types/wgkey"
+	"tailscale.com/version"
 )
 
 func fieldsOf(t reflect.Type) (fields []string) {
@@ -528,3 +529,25 @@ func BenchmarkKeyMarshalText(b *testing.B) {
 		sinkBytes = keyMarshalText("prefix", k)
 	}
 }
+
+func TestAppendKeyAllocs(t *testing.T) {
+	if version.IsRace() {
+		t.Skip("skipping in race detector") // append(b, make([]byte, N)...) not optimized in compiler with race
+	}
+	var k [32]byte
+	n := int(testing.AllocsPerRun(1000, func() {
+		sinkBytes = keyMarshalText("prefix", k)
+	}))
+	if n != 1 {
+		t.Fatalf("allocs = %v; want 1", n)
+	}
+}
+
+func TestDiscoKeyAppend(t *testing.T) {
+	d := DiscoKey{1: 1, 2: 2}
+	got := string(d.AppendTo([]byte("foo")))
+	want := "foodiscokey:0001020000000000000000000000000000000000000000000000000000000000"
+	if got != want {
+		t.Errorf("got %q; want %q", got, want)
+	}
+}

tsnet/example/tshello/tshello.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The tshello server demonstrates how to use Tailscale as a library.
+package main
+
+import (
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+	"strings"
+
+	"tailscale.com/tsnet"
+)
+
+func main() {
+	s := new(tsnet.Server)
+	ln, err := s.Listen("tcp", ":80")
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Fatal(http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		who, ok := s.WhoIs(r.RemoteAddr)
+		if !ok {
+			http.Error(w, "WhoIs failed", 500)
+			return
+		}
+		fmt.Fprintf(w, "<html><body><h1>Hello, world!</h1>\n")
+		fmt.Fprintf(w, "<p>You are <b>%s</b> from <b>%s</b> (%s)</p>",
+			html.EscapeString(who.UserProfile.LoginName),
+			html.EscapeString(firstLabel(who.Node.ComputedName)),
+			r.RemoteAddr)
+	})))
+}
+
+func firstLabel(s string) string {
+	if i := strings.Index(s, "."); i != -1 {
+		return s[:i]
+	}
+	return s
+}

tsnet/tsnet.go

@@ -0,0 +1,274 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tsnet provides Tailscale as a library.
+//
+// It is an experimental work in progress.
+package tsnet
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"inet.af/netaddr"
+	"tailscale.com/client/tailscale/apitype"
+	"tailscale.com/control/controlclient"
+	"tailscale.com/ipn"
+	"tailscale.com/ipn/ipnlocal"
+	"tailscale.com/smallzstd"
+	"tailscale.com/types/logger"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/monitor"
+	"tailscale.com/wgengine/netstack"
+)
+
+// Server is an embedded Tailscale server.
+//
+// Its exported fields may be changed until the first call to Listen.
+type Server struct {
+	// Dir specifies the name of the directory to use for
+	// state. If empty, a directory is selected automatically
+	// under os.UserConfigDir (https://golang.org/pkg/os/#UserConfigDir).
+	// based on the name of the binary.
+	Dir string
+
+	// Hostname is the hostname to present to the control server.
+	// If empty, the binary name is used.l
+	Hostname string
+
+	// Logf, if non-nil, specifies the logger to use. By default,
+	// log.Printf is used.
+	Logf logger.Logf
+
+	initOnce sync.Once
+	initErr  error
+	lb       *ipnlocal.LocalBackend
+	// the state directory
+	dir      string
+	hostname string
+
+	mu        sync.Mutex
+	listeners map[listenKey]*listener
+}
+
+// WhoIs reports the node and user who owns the node with the given
+// address. The addr may be an ip:port (as from an
+// http.Request.RemoteAddr) or just an IP address.
+func (s *Server) WhoIs(addr string) (w *apitype.WhoIsResponse, ok bool) {
+	ipp, err := netaddr.ParseIPPort(addr)
+	if err != nil {
+		ip, err := netaddr.ParseIP(addr)
+		if err != nil {
+			return nil, false
+		}
+		ipp = ipp.WithIP(ip)
+	}
+	n, up, ok := s.lb.WhoIs(ipp)
+	if !ok {
+		return nil, false
+	}
+	return &apitype.WhoIsResponse{
+		Node:        n,
+		UserProfile: &up,
+	}, true
+}
+
+func (s *Server) doInit() {
+	if err := s.start(); err != nil {
+		s.initErr = fmt.Errorf("tsnet: %w", err)
+	}
+}
+
+func (s *Server) start() error {
+	if v, _ := strconv.ParseBool(os.Getenv("TAILSCALE_USE_WIP_CODE")); !v {
+		return errors.New("code disabled without environment variable TAILSCALE_USE_WIP_CODE set true")
+	}
+
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	prog := strings.TrimSuffix(strings.ToLower(filepath.Base(exe)), ".exe")
+
+	s.hostname = s.Hostname
+	if s.hostname == "" {
+		s.hostname = prog
+	}
+
+	s.dir = s.Dir
+	if s.dir == "" {
+		confDir, err := os.UserConfigDir()
+		if err != nil {
+			return err
+		}
+		s.dir = filepath.Join(confDir, "tslib-"+prog)
+		if err := os.MkdirAll(s.dir, 0700); err != nil {
+			return err
+		}
+	}
+	if fi, err := os.Stat(s.dir); err != nil {
+		return err
+	} else if !fi.IsDir() {
+		return fmt.Errorf("%v is not a directory", s.dir)
+	}
+
+	logf := s.Logf
+	if logf == nil {
+		logf = log.Printf
+	}
+
+	// TODO(bradfitz): start logtail? don't use filch, perhaps?
+	// only upload plumbed Logf?
+
+	linkMon, err := monitor.New(logf)
+	if err != nil {
+		return err
+	}
+
+	eng, err := wgengine.NewUserspaceEngine(logf, wgengine.Config{
+		ListenPort:  0,
+		LinkMonitor: linkMon,
+	})
+	if err != nil {
+		return err
+	}
+
+	tunDev, magicConn, ok := eng.(wgengine.InternalsGetter).GetInternals()
+	if !ok {
+		return fmt.Errorf("%T is not a wgengine.InternalsGetter", eng)
+	}
+
+	ns, err := netstack.Create(logf, tunDev, eng, magicConn, false)
+	if err != nil {
+		return fmt.Errorf("netstack.Create: %w", err)
+	}
+	ns.ForwardTCPIn = s.forwardTCP
+	if err := ns.Start(); err != nil {
+		return fmt.Errorf("failed to start netstack: %w", err)
+	}
+
+	statePath := filepath.Join(s.dir, "tailscaled.state")
+	store, err := ipn.NewFileStore(statePath)
+	if err != nil {
+		return err
+	}
+	logid := "tslib-TODO"
+
+	lb, err := ipnlocal.NewLocalBackend(logf, logid, store, eng)
+	if err != nil {
+		return fmt.Errorf("NewLocalBackend: %v", err)
+	}
+	s.lb = lb
+	lb.SetDecompressor(func() (controlclient.Decompressor, error) {
+		return smallzstd.NewDecoder(nil)
+	})
+	prefs := ipn.NewPrefs()
+	prefs.Hostname = s.hostname
+	prefs.WantRunning = true
+	err = lb.Start(ipn.Options{
+		StateKey:    ipn.GlobalDaemonStateKey,
+		UpdatePrefs: prefs,
+	})
+	if err != nil {
+		return fmt.Errorf("starting backend: %w", err)
+	}
+	if os.Getenv("TS_LOGIN") == "1" {
+		s.lb.StartLoginInteractive()
+	}
+	return nil
+}
+
+func (s *Server) forwardTCP(c net.Conn, port uint16) {
+	s.mu.Lock()
+	ln, ok := s.listeners[listenKey{"tcp", "", fmt.Sprint(port)}]
+	s.mu.Unlock()
+	if !ok {
+		c.Close()
+		return
+	}
+	t := time.NewTimer(time.Second)
+	defer t.Stop()
+	select {
+	case ln.conn <- c:
+	case <-t.C:
+		c.Close()
+	}
+}
+
+func (s *Server) Listen(network, addr string) (net.Listener, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, fmt.Errorf("tsnet: %w", err)
+	}
+
+	s.initOnce.Do(s.doInit)
+	if s.initErr != nil {
+		return nil, s.initErr
+	}
+
+	key := listenKey{network, host, port}
+	ln := &listener{
+		s:    s,
+		key:  key,
+		addr: addr,
+
+		conn: make(chan net.Conn),
+	}
+	s.mu.Lock()
+	if s.listeners == nil {
+		s.listeners = map[listenKey]*listener{}
+	}
+	if _, ok := s.listeners[key]; ok {
+		s.mu.Unlock()
+		return nil, fmt.Errorf("tsnet: listener already open for %s, %s", network, addr)
+	}
+	s.listeners[key] = ln
+	s.mu.Unlock()
+	return ln, nil
+}
+
+type listenKey struct {
+	network string
+	host    string
+	port    string
+}
+
+type listener struct {
+	s    *Server
+	key  listenKey
+	addr string
+	conn chan net.Conn
+}
+
+func (ln *listener) Accept() (net.Conn, error) {
+	c, ok := <-ln.conn
+	if !ok {
+		return nil, fmt.Errorf("tsnet: %w", net.ErrClosed)
+	}
+	return c, nil
+}
+
+func (ln *listener) Addr() net.Addr { return addr{ln} }
+func (ln *listener) Close() error {
+	ln.s.mu.Lock()
+	defer ln.s.mu.Unlock()
+	if v, ok := ln.s.listeners[ln.key]; ok && v == ln {
+		delete(ln.s.listeners, ln.key)
+		close(ln.conn)
+	}
+	return nil
+}
+
+type addr struct{ ln *listener }
+
+func (a addr) Network() string { return a.ln.key.network }
+func (a addr) String() string  { return a.ln.addr }

tstest/integration/integration_test.go

@@ -10,6 +10,7 @@ import (
 	crand "crypto/rand"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -21,6 +22,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -41,8 +43,11 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/nettype"
+	"tailscale.com/version"
 )
 
+// var verbose = flag.Bool("verbose", true, "verbose debug logs")
+
 var mainError atomic.Value // of error
 
 func TestMain(m *testing.M) {
@@ -54,14 +59,12 @@ func TestMain(m *testing.M) {
 		fmt.Fprintf(os.Stderr, "FAIL: %v\n", err)
 		os.Exit(1)
 	}
+	time.Sleep(5 * time.Second)
 	os.Exit(0)
 }
 
-func TestIntegration(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("not tested/working on Windows yet")
-	}
-
+func TestOneNodeUp_NoAuth(t *testing.T) {
+	t.Parallel()
 	bins := buildTestBinaries(t)
 
 	env := newTestEnv(t, bins)
@@ -69,8 +72,8 @@ func TestIntegration(t *testing.T) {
 
 	n1 := newTestNode(t, env)
 
-	dcmd := n1.StartDaemon(t)
-	defer dcmd.Process.Kill()
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
 
 	n1.AwaitListening(t)
 
@@ -87,44 +90,141 @@ func TestIntegration(t *testing.T) {
 		t.Error(err)
 	}
 
-	t.Logf("Running up --login-server=%s ...", env.ControlServer.URL)
-	if err := n1.Tailscale("up", "--login-server="+env.ControlServer.URL).Run(); err != nil {
-		t.Fatalf("up: %v", err)
-	}
+	n1.MustUp()
 
 	if d, _ := time.ParseDuration(os.Getenv("TS_POST_UP_SLEEP")); d > 0 {
 		t.Logf("Sleeping for %v to give 'up' time to misbehave (https://github.com/tailscale/tailscale/issues/1840) ...", d)
 		time.Sleep(d)
 	}
 
-	var ip string
-	if err := tstest.WaitFor(20*time.Second, func() error {
-		out, err := n1.Tailscale("ip").Output()
-		if err != nil {
-			return err
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+
+	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
+}
+
+func TestOneNodeUp_Auth(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+	env.Control.RequireAuth = true
+
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+
+	st := n1.MustStatus(t)
+	t.Logf("Status: %s", st.BackendState)
+
+	t.Logf("Running up --login-server=%s ...", env.ControlServer.URL)
+
+	cmd := n1.Tailscale("up", "--login-server="+env.ControlServer.URL)
+	var authCountAtomic int32
+	cmd.Stdout = &authURLParserWriter{fn: func(urlStr string) error {
+		if env.Control.CompleteAuth(urlStr) {
+			atomic.AddInt32(&authCountAtomic, 1)
+			t.Logf("completed auth path %s", urlStr)
+			return nil
+		}
+		err := fmt.Errorf("Failed to complete auth path to %q", urlStr)
+		t.Log(err)
+		return err
+	}}
+	cmd.Stderr = cmd.Stdout
+	if err := cmd.Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
+	t.Logf("Got IP: %v", n1.AwaitIP(t))
+
+	n1.AwaitRunning(t)
+
+	if n := atomic.LoadInt32(&authCountAtomic); n != 1 {
+		t.Errorf("Auth URLs completed = %d; want 1", n)
+	}
+
+	d1.MustCleanShutdown(t)
+
+}
+
+func TestTwoNodes(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		st := n1.MustStatus(t)
+		if len(st.Peer) == 0 {
+			return errors.New("no peers")
+		}
+		if len(st.Peer) > 1 {
+			return fmt.Errorf("got %d peers; want 1", len(st.Peer))
+		}
+		peer := st.Peer[st.Peers()[0]]
+		if peer.ID == st.Self.ID {
+			return errors.New("peer is self")
 		}
-		ip = string(out)
 		return nil
 	}); err != nil {
 		t.Error(err)
 	}
-	t.Logf("Got IP: %v", ip)
 
-	dcmd.Process.Signal(os.Interrupt)
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}
 
-	ps, err := dcmd.Process.Wait()
-	if err != nil {
-		t.Fatalf("tailscaled Wait: %v", err)
+func TestNodeAddressIPFields(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n1.AwaitListening(t)
+	n1.MustUp()
+	n1.AwaitRunning(t)
+
+	testNodes := env.Control.AllNodes()
+
+	if len(testNodes) != 1 {
+		t.Errorf("Expected %d nodes, got %d", 1, len(testNodes))
 	}
-	if ps.ExitCode() != 0 {
-		t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode())
+	node := testNodes[0]
+	if len(node.Addresses) == 0 {
+		t.Errorf("Empty Addresses field in node")
+	}
+	if len(node.AllowedIPs) == 0 {
+		t.Errorf("Empty AllowedIPs field in node")
 	}
 
-	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
-	if err := env.TrafficTrap.Err(); err != nil {
-		t.Errorf("traffic trap: %v", err)
-		t.Logf("logs: %s", env.LogCatcher.logsString())
-	}
+	d1.MustCleanShutdown(t)
 }
 
 // testBinaries are the paths to a tailscaled and tailscale binary.
@@ -139,16 +239,18 @@ type testBinaries struct {
 // if they fail to compile.
 func buildTestBinaries(t testing.TB) *testBinaries {
 	td := t.TempDir()
+	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
 	return &testBinaries{
 		dir:    td,
-		daemon: build(t, td, "tailscale.com/cmd/tailscaled"),
-		cli:    build(t, td, "tailscale.com/cmd/tailscale"),
+		daemon: filepath.Join(td, "tailscaled"+exe()),
+		cli:    filepath.Join(td, "tailscale"+exe()),
 	}
 }
 
 // testEnv contains the test environment (set of servers) used by one
 // or more nodes.
 type testEnv struct {
+	t        testing.TB
 	Binaries *testBinaries
 
 	LogCatcher       *logCatcher
@@ -168,13 +270,20 @@ type testEnv struct {
 //
 // Call Close to shut everything down.
 func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
+	if runtime.GOOS == "windows" {
+		t.Skip("not tested/working on Windows yet")
+	}
 	derpMap, derpShutdown := runDERPAndStun(t, logger.Discard)
 	logc := new(logCatcher)
 	control := &testcontrol.Server{
 		DERPMap: derpMap,
 	}
 	trafficTrap := new(trafficTrap)
+	log.Println("SERVER ATTACHED")
+	log.Println(len(control.PingRequestC))
+	// go func() { control.PingRequestC <- true }()
 	e := &testEnv{
+		t:                 t,
 		Binaries:          bins,
 		LogCatcher:        logc,
 		LogCatcherServer:  httptest.NewServer(logc),
@@ -184,10 +293,16 @@ func newTestEnv(t testing.TB, bins *testBinaries) *testEnv {
 		TrafficTrapServer: httptest.NewServer(trafficTrap),
 		derpShutdown:      derpShutdown,
 	}
+	e.Control.BaseURL = e.ControlServer.URL
 	return e
 }
 
 func (e *testEnv) Close() error {
+	if err := e.TrafficTrap.Err(); err != nil {
+		e.t.Errorf("traffic trap: %v", err)
+		e.t.Logf("logs: %s", e.LogCatcher.logsString())
+	}
+
 	e.LogCatcherServer.Close()
 	e.TrafficTrapServer.Close()
 	e.ControlServer.Close()
@@ -218,9 +333,28 @@ func newTestNode(t *testing.T, env *testEnv) *testNode {
 	}
 }
 
+type Daemon struct {
+	Process *os.Process
+}
+
+func (d *Daemon) Kill() {
+	d.Process.Kill()
+}
+
+func (d *Daemon) MustCleanShutdown(t testing.TB) {
+	d.Process.Signal(os.Interrupt)
+	ps, err := d.Process.Wait()
+	if err != nil {
+		t.Fatalf("tailscaled Wait: %v", err)
+	}
+	if ps.ExitCode() != 0 {
+		t.Errorf("tailscaled ExitCode = %d; want 0", ps.ExitCode())
+	}
+}
+
 // StartDaemon starts the node's tailscaled, failing if it fails to
 // start.
-func (n *testNode) StartDaemon(t testing.TB) *exec.Cmd {
+func (n *testNode) StartDaemon(t testing.TB) *Daemon {
 	cmd := exec.Command(n.env.Binaries.daemon,
 		"--tun=userspace-networking",
 		"--state="+n.stateFile,
@@ -234,7 +368,17 @@ func (n *testNode) StartDaemon(t testing.TB) *exec.Cmd {
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("starting tailscaled: %v", err)
 	}
-	return cmd
+	return &Daemon{
+		Process: cmd.Process,
+	}
+}
+
+func (n *testNode) MustUp() {
+	t := n.env.t
+	t.Logf("Running up --login-server=%s ...", n.env.ControlServer.URL)
+	if err := n.Tailscale("up", "--login-server="+n.env.ControlServer.URL).Run(); err != nil {
+		t.Fatalf("up: %v", err)
+	}
 }
 
 // AwaitListening waits for the tailscaled to be serving local clients
@@ -252,6 +396,40 @@ func (n *testNode) AwaitListening(t testing.TB) {
 	}
 }
 
+func (n *testNode) AwaitIP(t testing.TB) (ips string) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		out, err := n.Tailscale("ip").Output()
+		if err != nil {
+			return err
+		}
+		ips = string(out)
+		return nil
+	}); err != nil {
+		t.Fatalf("awaiting an IP address: %v", err)
+	}
+	if ips == "" {
+		t.Fatalf("returned IP address was blank")
+	}
+	return ips
+}
+
+func (n *testNode) AwaitRunning(t testing.TB) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		st, err := n.Status()
+		if err != nil {
+			return err
+		}
+		if st.BackendState != "Running" {
+			return fmt.Errorf("in state %q", st.BackendState)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failure/timeout waiting for transition to Running status: %v", err)
+	}
+}
+
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
@@ -261,15 +439,23 @@ func (n *testNode) Tailscale(arg ...string) *exec.Cmd {
 	return cmd
 }
 
-func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
-	tb.Helper()
+func (n *testNode) Status() (*ipnstate.Status, error) {
 	out, err := n.Tailscale("status", "--json").CombinedOutput()
 	if err != nil {
-		tb.Fatalf("getting status: %v, %s", err, out)
+		return nil, fmt.Errorf("running tailscale status: %v, %s", err, out)
 	}
 	st := new(ipnstate.Status)
 	if err := json.Unmarshal(out, st); err != nil {
-		tb.Fatalf("parsing status json: %v, from: %s", err, out)
+		return nil, fmt.Errorf("decoding tailscale status JSON: %w", err)
+	}
+	return st, nil
+}
+
+func (n *testNode) MustStatus(tb testing.TB) *ipnstate.Status {
+	tb.Helper()
+	st, err := n.Status()
+	if err != nil {
+		tb.Fatal(err)
 	}
 	return st
 }
@@ -291,21 +477,44 @@ func findGo(t testing.TB) string {
 	} else if !fi.Mode().IsRegular() {
 		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
 	}
-	t.Logf("using go binary %v", goBin)
 	return goBin
 }
 
-func build(t testing.TB, outDir, target string) string {
-	exe := ""
-	if runtime.GOOS == "windows" {
-		exe = ".exe"
+// buildMu limits our use of "go build" to one at a time, so we don't
+// fight Go's built-in caching trying to do the same build concurrently.
+var buildMu sync.Mutex
+
+func build(t testing.TB, outDir string, targets ...string) {
+	buildMu.Lock()
+	defer buildMu.Unlock()
+
+	t0 := time.Now()
+	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
+
+	goBin := findGo(t)
+	cmd := exec.Command(goBin, "install")
+	if version.IsRace() {
+		cmd.Args = append(cmd.Args, "-race")
 	}
-	bin := filepath.Join(outDir, path.Base(target)) + exe
-	errOut, err := exec.Command(findGo(t), "build", "-o", bin, target).CombinedOutput()
-	if err != nil {
-		t.Fatalf("failed to build %v: %v, %s", target, err, errOut)
+	cmd.Args = append(cmd.Args, targets...)
+	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
+	errOut, err := cmd.CombinedOutput()
+	if err == nil {
+		return
 	}
-	return bin
+	if strings.Contains(string(errOut), "when GOBIN is set") {
+		// Fallback slow path for cross-compiled binaries.
+		for _, target := range targets {
+			outFile := filepath.Join(outDir, path.Base(target)+exe())
+			cmd := exec.Command(goBin, "build", "-o", outFile, target)
+			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
+			if errOut, err := cmd.CombinedOutput(); err != nil {
+				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
+			}
+		}
+		return
+	}
+	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
 }
 
 // logCatcher is a minimal logcatcher for the logtail upload client.
@@ -378,6 +587,9 @@ func (lc *logCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	} else {
 		for _, ent := range jreq {
 			fmt.Fprintf(&lc.buf, "%s\n", strings.TrimSpace(ent.Text))
+			if testing.Verbose() {
+				fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(ent.Text))
+			}
 		}
 	}
 	w.WriteHeader(200) // must have no content, but not a 204
@@ -454,3 +666,152 @@ func runDERPAndStun(t testing.TB, logf logger.Logf) (derpMap *tailcfg.DERPMap, c
 
 	return m, cleanup
 }
+
+type authURLParserWriter struct {
+	buf bytes.Buffer
+	fn  func(urlStr string) error
+}
+
+var authURLRx = regexp.MustCompile(`(https?://\S+/auth/\S+)`)
+
+func (w *authURLParserWriter) Write(p []byte) (n int, err error) {
+	n, err = w.buf.Write(p)
+	m := authURLRx.FindSubmatch(w.buf.Bytes())
+	if m != nil {
+		urlStr := string(m[1])
+		w.buf.Reset() // so it's not matched again
+		if err := w.fn(urlStr); err != nil {
+			return 0, err
+		}
+	}
+	return n, err
+}
+
+type panicOnUseTransport struct{}
+
+func (panicOnUseTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	panic("unexpected HTTP request")
+}
+func TestTwoNodePing(t *testing.T) {
+
+	// < --->
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	t.Log("Env :", env.ControlServer.URL)
+	res, err := http.Get(env.ControlServer.URL + "/ping")
+	t.Log("RESPONSE", res)
+	if err != nil {
+		t.Error(err)
+	}
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+	ip1 := n1.AwaitIP(t)
+	ip2 := n2.AwaitIP(t)
+	t.Logf("Node IPs : %s, %s\n", ip1, ip2)
+
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		st := n1.MustStatus(t)
+		t.Log("CURPEER", len(st.Peer))
+		var peers []*ipnstate.PeerStatus
+		for _, peer := range st.Peers() {
+			ps := st.Peer[peer]
+			if ps.ShareeNode {
+				continue
+			}
+			peers = append(peers, ps)
+		}
+		jsonForm, _ := json.MarshalIndent(peers[0], "", " ")
+		t.Log("PeerStatus", string(jsonForm))
+		if len(st.Peer) == 0 {
+			return errors.New("no peers")
+		}
+		if len(st.Peer) > 1 {
+			return fmt.Errorf("got %d peers; want 1", len(st.Peer))
+		}
+		peer := st.Peer[st.Peers()[0]]
+		if peer.ID == st.Self.ID {
+			return errors.New("peer is self")
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}
+
+// Tests if our addPingRequest function works
+func TestAddPingRequest(t *testing.T) {
+
+}
+
+// Test such that we can simulate the ping instead of hardcoding in the map response
+func TestControlSelectivePing(t *testing.T) {
+	t.Parallel()
+	bins := buildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	log.Println("POSTSTARTUP")
+	defer env.Close()
+
+	// Create two nodes:
+	n1 := newTestNode(t, env)
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+
+	n2 := newTestNode(t, env)
+	d2 := n2.StartDaemon(t)
+	defer d2.Kill()
+
+	n1.AwaitListening(t)
+	n2.AwaitListening(t)
+	n1.MustUp()
+	n2.MustUp()
+	n1.AwaitRunning(t)
+	n2.AwaitRunning(t)
+
+	// Wait for server to start serveMap
+	if err := tstest.WaitFor(2*time.Second, func() error {
+		t.Log("ENOUGHTIME")
+		env.Control.AddControlPingRequest()
+		if len(env.Control.PingRequestC) == 0 {
+			return errors.New("failed to add to PingRequestC")
+		}
+		log.Println("CHANNEL LENGTH", len(env.Control.PingRequestC))
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+
+	// Wait for a MapResponse
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		// Simulate the time needed for MapResponse method call.
+		time.Sleep(500 * time.Millisecond)
+		if len(env.Control.PingRequestC) == 1 {
+			t.Error("Expected PingRequestC to be empty")
+		}
+		return nil
+	}); err != nil {
+		t.Error(err)
+	}
+	d1.MustCleanShutdown(t)
+	d2.MustCleanShutdown(t)
+}

tstest/integration/testcontrol/testcontrol.go

@@ -17,6 +17,8 @@ import (
 	"log"
 	"math/rand"
 	"net/http"
+	"net/url"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -34,19 +36,56 @@ import (
 // Server is a control plane server. Its zero value is ready for use.
 // Everything is stored in-memory in one tailnet.
 type Server struct {
-	Logf    logger.Logf      // nil means to use the log package
-	DERPMap *tailcfg.DERPMap // nil means to use prod DERP map
+	Logf         logger.Logf      // nil means to use the log package
+	DERPMap      *tailcfg.DERPMap // nil means to use prod DERP map
+	RequireAuth  bool
+	BaseURL      string // must be set to e.g. "http://127.0.0.1:1234" with no trailing URL
+	Verbose      bool
+	PingRequestC chan bool
 
-	initMuxOnce sync.Once
-	mux         *http.ServeMux
+	initMuxOnce       sync.Once
+	mux               *http.ServeMux
+	initPRchannelOnce sync.Once
 
-	mu      sync.Mutex
-	pubKey  wgkey.Key
-	privKey wgkey.Private
-	nodes   map[tailcfg.NodeKey]*tailcfg.Node
-	users   map[tailcfg.NodeKey]*tailcfg.User
-	logins  map[tailcfg.NodeKey]*tailcfg.Login
-	updates map[tailcfg.NodeID]chan updateType
+	mu            sync.Mutex
+	pubKey        wgkey.Key
+	privKey       wgkey.Private
+	nodes         map[tailcfg.NodeKey]*tailcfg.Node
+	users         map[tailcfg.NodeKey]*tailcfg.User
+	logins        map[tailcfg.NodeKey]*tailcfg.Login
+	updates       map[tailcfg.NodeID]chan updateType
+	authPath      map[string]*AuthPath
+	nodeKeyAuthed map[tailcfg.NodeKey]bool // key => true once authenticated
+}
+
+// NumNodes returns the number of nodes in the testcontrol server.
+//
+// This is useful when connecting a bunch of virtual machines to a testcontrol
+// server to see how many of them connected successfully.
+func (s *Server) NumNodes() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	return len(s.nodes)
+}
+
+type AuthPath struct {
+	nodeKey tailcfg.NodeKey
+
+	closeOnce sync.Once
+	ch        chan struct{}
+	success   bool
+}
+
+func (ap *AuthPath) completeSuccessfully() {
+	ap.success = true
+	close(ap.ch)
+}
+
+// CompleteSuccessfully completes the login path successfully, as if
+// the user did the whole auth dance.
+func (ap *AuthPath) CompleteSuccessfully() {
+	ap.closeOnce.Do(ap.completeSuccessfully)
 }
 
 func (s *Server) logf(format string, a ...interface{}) {
@@ -58,14 +97,26 @@ func (s *Server) logf(format string, a ...interface{}) {
 }
 
 func (s *Server) initMux() {
+	log.Println("Mux inited")
 	s.mux = http.NewServeMux()
 	s.mux.HandleFunc("/", s.serveUnhandled)
 	s.mux.HandleFunc("/key", s.serveKey)
 	s.mux.HandleFunc("/machine/", s.serveMachine)
+	s.mux.HandleFunc("/ping", s.receivePingInfo)
+	s.mux.HandleFunc("/mockpingrequest", s.serveMockPing)
+}
+
+func (s *Server) initPingRequestC() {
+	log.Println("Channel created")
+	s.PingRequestC = make(chan bool, 1)
+	// s.AddControlPingRequest()
+	// log.Println("Channel length : ", len(s.PingRequestC))
 }
 
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log.Println("HTTPSERVE")
 	s.initMuxOnce.Do(s.initMux)
+	s.initPRchannelOnce.Do(s.initPingRequestC)
 	s.mux.ServeHTTP(w, r)
 }
 
@@ -75,6 +126,12 @@ func (s *Server) serveUnhandled(w http.ResponseWriter, r *http.Request) {
 	go panic(fmt.Sprintf("testcontrol.Server received unhandled request: %s", got.Bytes()))
 }
 
+func (s *Server) serveMockPing(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	s.AddControlPingRequest()
+	io.WriteString(w, "A ControlPingRequest has been queued for our next MapResponse.")
+}
+
 func (s *Server) publicKey() wgkey.Key {
 	pub, _ := s.keyPair()
 	return pub
@@ -142,6 +199,28 @@ func (s *Server) Node(nodeKey tailcfg.NodeKey) *tailcfg.Node {
 	return s.nodes[nodeKey].Clone()
 }
 
+func (s *Server) AllNodes() (nodes []*tailcfg.Node) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, n := range s.nodes {
+		nodes = append(nodes, n.Clone())
+	}
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].StableID < nodes[j].StableID
+	})
+	return nodes
+}
+
+// AddControlPingRequest enqueues a bool to PingRequestC.
+// in serveMap this will result to a ControlPingRequest
+// added to the next MapResponse sent to the client
+func (s *Server) AddControlPingRequest() {
+	// Redundant check to avoid errors when called multiple times
+	if len(s.PingRequestC) == 0 {
+		s.PingRequestC <- true
+	}
+}
+
 func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -178,7 +257,58 @@ func (s *Server) getUser(nodeKey tailcfg.NodeKey) (*tailcfg.User, *tailcfg.Login
 	return user, login
 }
 
+// authPathDone returns a close-only struct that's closed when the
+// authPath ("/auth/XXXXXX") has authenticated.
+func (s *Server) authPathDone(authPath string) <-chan struct{} {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if a, ok := s.authPath[authPath]; ok {
+		return a.ch
+	}
+	return nil
+}
+
+func (s *Server) addAuthPath(authPath string, nodeKey tailcfg.NodeKey) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.authPath == nil {
+		s.authPath = map[string]*AuthPath{}
+	}
+	s.authPath[authPath] = &AuthPath{
+		ch:      make(chan struct{}),
+		nodeKey: nodeKey,
+	}
+}
+
+// CompleteAuth marks the provided path or URL (containing
+// "/auth/...")  as successfully authenticated, unblocking any
+// requests blocked on that in serveRegister.
+func (s *Server) CompleteAuth(authPathOrURL string) bool {
+	i := strings.Index(authPathOrURL, "/auth/")
+	if i == -1 {
+		return false
+	}
+	authPath := authPathOrURL[i:]
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	ap, ok := s.authPath[authPath]
+	if !ok {
+		return false
+	}
+	if ap.nodeKey.IsZero() {
+		panic("zero AuthPath.NodeKey")
+	}
+	if s.nodeKeyAuthed == nil {
+		s.nodeKeyAuthed = map[tailcfg.NodeKey]bool{}
+	}
+	s.nodeKeyAuthed[ap.nodeKey] = true
+	ap.CompleteSuccessfully()
+	return true
+}
+
 func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	log.Println("SERVE REGISTER CALLED")
 	var req tailcfg.RegisterRequest
 	if err := s.decode(mkey, r.Body, &req); err != nil {
 		panic(fmt.Sprintf("serveRegister: decode: %v", err))
@@ -189,28 +319,71 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey tail
 	if req.NodeKey.IsZero() {
 		panic("serveRegister: request has zero node key")
 	}
+	if s.Verbose {
+		j, _ := json.MarshalIndent(req, "", "\t")
+		log.Printf("Got %T: %s", req, j)
+	}
+
+	// If this is a followup request, wait until interactive followup URL visit complete.
+	if req.Followup != "" {
+		followupURL, err := url.Parse(req.Followup)
+		if err != nil {
+			panic(err)
+		}
+		doneCh := s.authPathDone(followupURL.Path)
+		select {
+		case <-r.Context().Done():
+			return
+		case <-doneCh:
+		}
+		// TODO(bradfitz): support a side test API to mark an
+		// auth as failued so we can send an error response in
+		// some follow-ups? For now all are successes.
+	}
 
 	user, login := s.getUser(req.NodeKey)
 	s.mu.Lock()
 	if s.nodes == nil {
 		s.nodes = map[tailcfg.NodeKey]*tailcfg.Node{}
 	}
+
+	machineAuthorized := true // TODO: add Server.RequireMachineAuth
+
+	allowedIPs := []netaddr.IPPrefix{
+		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(tailcfg.NodeID(user.ID)>>8), uint8(tailcfg.NodeID(user.ID)))),
+	}
+
 	s.nodes[req.NodeKey] = &tailcfg.Node{
 		ID:                tailcfg.NodeID(user.ID),
 		StableID:          tailcfg.StableNodeID(fmt.Sprintf("TESTCTRL%08x", int(user.ID))),
 		User:              user.ID,
 		Machine:           mkey,
 		Key:               req.NodeKey,
-		MachineAuthorized: true,
+		MachineAuthorized: machineAuthorized,
+		Addresses:         allowedIPs,
+		AllowedIPs:        allowedIPs,
+	}
+	requireAuth := s.RequireAuth
+	if requireAuth && s.nodeKeyAuthed[req.NodeKey] {
+		requireAuth = false
 	}
 	s.mu.Unlock()
 
+	authURL := ""
+	if requireAuth {
+		randHex := make([]byte, 10)
+		crand.Read(randHex)
+		authPath := fmt.Sprintf("/auth/%x", randHex)
+		s.addAuthPath(authPath, req.NodeKey)
+		authURL = s.BaseURL + authPath
+	}
+
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
 		User:              *user,
 		Login:             *login,
 		NodeKeyExpired:    false,
-		MachineAuthorized: true,
-		AuthURL:           "", // all good; TODO(bradfitz): add ways to not start all good.
+		MachineAuthorized: machineAuthorized,
+		AuthURL:           authURL,
 	})
 	if err != nil {
 		go panic(fmt.Sprintf("serveRegister: encode: %v", err))
@@ -239,6 +412,24 @@ func (s *Server) updateLocked(source string, peers []tailcfg.NodeID) {
 	}
 }
 
+// Adds a PingRequest to a MapResponse, we will ping the first peer.
+func (s *Server) addPingRequest(res *tailcfg.MapResponse) error {
+	if len(res.Peers) == 0 {
+		return errors.New("MapResponse has no peers to ping")
+	}
+
+	if len(res.Peers[0].Addresses) == 0 || len(res.Peers[0].AllowedIPs) == 0 {
+		return errors.New("peer has no Addresses or no AllowedIPs")
+	}
+	targetIP := res.Peers[0].AllowedIPs[0].IP()
+	res.PingRequest = &tailcfg.PingRequest{URL: s.BaseURL + "/ping", TestIP: targetIP, Types: "tsmp"}
+	// jsonRes, _ := json.MarshalIndent(res, "", " ")
+	// log.Println("jsonprint", string(jsonRes))
+	// log.Println("respeers", res.Peers)
+	// log.Println("allnodes", s.AllNodes(), res.Node.AllowedIPs)
+	return nil
+}
+
 // sendUpdate sends updateType to dst if dst is non-nil and
 // has capacity.
 func sendUpdate(dst chan<- updateType, updateType updateType) {
@@ -254,7 +445,23 @@ func sendUpdate(dst chan<- updateType, updateType updateType) {
 	}
 }
 
+func (s *Server) UpdateNode(n *tailcfg.Node) (peersToUpdate []tailcfg.NodeID) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if n.Key.IsZero() {
+		panic("zero nodekey")
+	}
+	s.nodes[n.Key] = n.Clone()
+	for _, n2 := range s.nodes {
+		if n.ID != n2.ID {
+			peersToUpdate = append(peersToUpdate, n2.ID)
+		}
+	}
+	return peersToUpdate
+}
+
 func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.MachineKey) {
+	log.Println("SERVEMAP CALLED")
 	ctx := r.Context()
 
 	req := new(tailcfg.MapRequest)
@@ -279,10 +486,8 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 	if !req.ReadOnly {
 		endpoints := filterInvalidIPv6Endpoints(req.Endpoints)
 		node.Endpoints = endpoints
-		// TODO: more
-		// TODO: register node,
-		//s.UpdateEndpoint(mkey, req.NodeKey,
-		// XXX
+		node.DiscoKey = req.DiscoKey
+		peersToUpdate = s.UpdateNode(node)
 	}
 
 	nodeID := node.ID
@@ -309,9 +514,20 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey tailcfg.M
 	streaming := req.Stream && !req.ReadOnly
 	compress := req.Compress != ""
 
+	log.Println("CREATED MAPREQ", *req)
+	log.Println("REQUEST", r)
+	log.Println("REQBODY", r.Body)
 	w.WriteHeader(200)
 	for {
 		res, err := s.MapResponse(req)
+		log.Println("LENGTHER", len(s.PingRequestC))
+		select {
+		case <-s.PingRequestC:
+			log.Println("PINGADD", len(s.PingRequestC))
+			s.addPingRequest(res)
+		default:
+			log.Println("NOTEXIST")
+		}
 		if err != nil {
 			// TODO: log
 			return
@@ -372,6 +588,7 @@ var prodDERPMap = derpmap.Prod()
 //
 // No updates to s are done here.
 func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, err error) {
+	log.Println("MAPREQUEST : ", string(JsonPrint(req)))
 	node := s.Node(req.NodeKey)
 	if node == nil {
 		// node key rotated away (once test server supports that)
@@ -389,10 +606,17 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		CollectServices: "true",
 		PacketFilter:    tailcfg.FilterAllowAll,
 	}
+	for _, p := range s.AllNodes() {
+		if p.StableID != node.StableID {
+			res.Peers = append(res.Peers, p)
+		}
+	}
+
 	res.Node.Addresses = []netaddr.IPPrefix{
 		netaddr.MustParseIPPrefix(fmt.Sprintf("100.64.%d.%d/32", uint8(node.ID>>8), uint8(node.ID))),
 	}
 	res.Node.AllowedIPs = res.Node.Addresses
+
 	return res, nil
 }
 
@@ -506,7 +730,7 @@ func keepClientEndpoint(ep string) bool {
 		// the incoming JSON response.
 		return false
 	}
-	ip := ipp.IP
+	ip := ipp.IP()
 	if ip.Zone() != "" {
 		return false
 	}
@@ -543,3 +767,24 @@ func breakSameNodeMapResponseStreams(req *tailcfg.MapRequest) bool {
 	}
 	return true
 }
+
+// This is where the PUT requests will go
+func (s *Server) receivePingInfo(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "PUT" {
+		log.Println("Received NON PUT request, should panic if this happens after")
+		// panic("Only PUT requests are supported currently")
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	reqBody, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		panic("Failed to read request body")
+	}
+	log.Println("Ping Info Received", string(reqBody))
+	w.WriteHeader(200)
+	io.WriteString(w, "Ping Streamed Back : "+string(reqBody))
+}
+
+func JsonPrint(item interface{}) []byte {
+	res, _ := json.MarshalIndent(item, "", " ")
+	return res
+}

tstest/integration/vms/doc.go

@@ -0,0 +1,7 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package vms does VM-based integration/functional tests by using
+// qemu and a bank of pre-made VM images.
+package vms

tstest/integration/vms/vms_test.go

@@ -0,0 +1,532 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package vms
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"text/template"
+	"time"
+
+	expect "github.com/google/goexpect"
+	"golang.org/x/crypto/ssh"
+	"inet.af/netaddr"
+	"tailscale.com/tstest/integration/testcontrol"
+)
+
+var runVMTests = flag.Bool("run-vm-tests", false, "if set, run expensive (10G+ ram) VM based integration tests")
+
+type Distro struct {
+	name           string // amazon-linux
+	url            string // URL to a qcow2 image
+	sha256sum      string // hex-encoded sha256 sum of contents of URL
+	mem            int    // VM memory in megabytes
+	packageManager string // yum/apt/dnf/zypper
+}
+
+func (d *Distro) InstallPre() string {
+	switch d.packageManager {
+	case "yum":
+		return ` - [ yum, update, gnupg2 ]
+`
+	case "apt":
+		return ` - [ apt-get, update ]
+ - [ apt-get, "-y", install, curl, "apt-transport-https", gnupg2 ]
+`
+	}
+
+	return ""
+}
+
+// fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
+// also validates the sha256 sum from a known good hash.
+func fetchDistro(t *testing.T, resultDistro Distro) {
+	t.Helper()
+
+	cdir, err := os.UserCacheDir()
+	if err != nil {
+		t.Fatalf("can't find cache dir: %v", err)
+	}
+	cdir = filepath.Join(cdir, "tailscale", "vm-test")
+
+	qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
+
+	_, err = os.Stat(qcowPath)
+	if err != nil {
+		t.Logf("downloading distro image %s to %s", resultDistro.url, qcowPath)
+		fout, err := os.Create(qcowPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		resp, err := http.Get(resultDistro.url)
+		if err != nil {
+			t.Fatalf("can't fetch qcow2 for %s (%s): %v", resultDistro.name, resultDistro.url, err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			t.Fatalf("%s replied %s", resultDistro.url, resp.Status)
+		}
+
+		_, err = io.Copy(fout, resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			t.Fatalf("download of %s failed: %v", resultDistro.url, err)
+		}
+
+		err = fout.Close()
+		if err != nil {
+			t.Fatalf("can't close fout: %v", err)
+		}
+
+		fin, err := os.Open(qcowPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		hasher := sha256.New()
+		if _, err := io.Copy(hasher, fin); err != nil {
+			t.Fatal(err)
+		}
+		hash := hex.EncodeToString(hasher.Sum(nil))
+
+		if hash != resultDistro.sha256sum {
+			t.Logf("got:  %q", hash)
+			t.Logf("want: %q", resultDistro.sha256sum)
+			t.Fatal("hash mismatch, someone is doing something nasty")
+		}
+
+		t.Logf("hash check passed (%s)", resultDistro.sha256sum)
+	}
+}
+
+// run runs a command or fails the test.
+func run(t *testing.T, dir, prog string, args ...string) {
+	t.Helper()
+	t.Logf("running: %s %s", prog, strings.Join(args, " "))
+
+	cmd := exec.Command(prog, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Dir = dir
+	if err := cmd.Run(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream VM images
+// pristine and only do our changes on an overlay.
+func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
+	t.Helper()
+
+	cdir, err := os.UserCacheDir()
+	if err != nil {
+		t.Fatalf("can't find cache dir: %v", err)
+	}
+	cdir = filepath.Join(cdir, "tailscale", "vm-test")
+
+	run(t, tdir, "qemu-img", "create",
+		"-f", "qcow2",
+		"-o", "backing_file="+filepath.Join(cdir, "qcow2", d.sha256sum),
+		filepath.Join(tdir, d.name+".qcow2"),
+	)
+}
+
+// mkSeed makes the cloud-init seed ISO that is used to configure a VM with tailscale.
+func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
+	t.Helper()
+
+	dir := filepath.Join(tdir, d.name, "seed")
+	os.MkdirAll(dir, 0700)
+
+	// make meta-data
+	{
+		fout, err := os.Create(filepath.Join(dir, "meta-data"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = template.Must(template.New("meta-data.yaml").Parse(metaDataTemplate)).Execute(fout, struct {
+			ID       string
+			Hostname string
+		}{
+			ID:       "31337",
+			Hostname: d.name,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = fout.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// make user-data
+	{
+		fout, err := os.Create(filepath.Join(dir, "user-data"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = template.Must(template.New("user-data.yaml").Parse(userDataTemplate)).Execute(fout, struct {
+			SSHKey     string
+			HostURL    string
+			Hostname   string
+			Port       int
+			InstallPre string
+		}{
+			SSHKey:     strings.TrimSpace(sshKey),
+			HostURL:    hostURL,
+			Hostname:   d.name,
+			Port:       port,
+			InstallPre: d.InstallPre(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = fout.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	run(t, tdir, "genisoimage",
+		"-output", filepath.Join(dir, "seed.iso"),
+		"-volid", "cidata", "-joliet", "-rock",
+		filepath.Join(dir, "meta-data"),
+		filepath.Join(dir, "user-data"),
+	)
+}
+
+// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction to the
+// testcontrol server. The function it returns is for killing the virtual machine when it
+// is time for it to die.
+func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) func() {
+	t.Helper()
+
+	cdir, err := os.UserCacheDir()
+	if err != nil {
+		t.Fatalf("can't find cache dir: %v", err)
+	}
+	cdir = filepath.Join(cdir, "within", "mkvm")
+	os.MkdirAll(filepath.Join(cdir, "qcow2"), 0755)
+	os.MkdirAll(filepath.Join(cdir, "seed"), 0755)
+
+	port := 23100 + n
+
+	fetchDistro(t, d)
+	mkLayeredQcow(t, tdir, d)
+	mkSeed(t, d, sshKey, hostURL, tdir, port)
+
+	driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
+
+	args := []string{
+		"-machine", "pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off",
+		"-netdev", fmt.Sprintf("user,hostfwd=::%d-:22,id=net0", port),
+		"-device", "virtio-net-pci,netdev=net0,id=net0,mac=8a:28:5c:30:1f:25",
+		"-m", fmt.Sprint(d.mem),
+		"-boot", "c",
+		"-drive", driveArg,
+		"-cdrom", filepath.Join(tdir, d.name, "seed", "seed.iso"),
+		"-vnc", fmt.Sprintf(":%d", n),
+	}
+
+	t.Logf("running: qemu-system-x86_64 %s", strings.Join(args, " "))
+
+	cmd := exec.Command("qemu-system-x86_64", args...)
+	err = cmd.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(time.Second)
+
+	if err := cmd.Process.Signal(syscall.Signal(0)); err != nil {
+		t.Fatal("qemu is not running")
+	}
+
+	return func() {
+		err := cmd.Process.Kill()
+		if err != nil {
+			t.Errorf("can't kill %s (%d): %v", d.name, cmd.Process.Pid, err)
+		}
+	}
+}
+
+// TestVMIntegrationEndToEnd creates a virtual machine with mkvm(1X), installs tailscale on it and then ensures that it connects to the network successfully.
+func TestVMIntegrationEndToEnd(t *testing.T) {
+	if !*runVMTests {
+		t.Skip("not running integration tests (need -run-vm-tests)")
+	}
+
+	if _, err := exec.LookPath("qemu-system-x86_64"); err != nil {
+		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
+		t.Fatalf("missing dependency: %v", err)
+	}
+
+	if _, err := exec.LookPath("genisoimage"); err != nil {
+		t.Logf("hint: nix-shell -p go -p qemu -p cdrkit --run 'go test -v -timeout=60m -run-vm-tests'")
+		t.Fatalf("missing dependency: %v", err)
+	}
+
+	distros := []Distro{
+		{"amazon-linux", "https://cdn.amazonlinux.com/os-images/2.0.20210427.0/kvm/amzn2-kvm-2.0.20210427.0-x86_64.xfs.gpt.qcow2", "6ef9daef32cec69b2d0088626ec96410cd24afc504d57278bbf2f2ba2b7e529b", 512, "yum"},
+		{"centos-7", "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2", "1db30c9c272fb37b00111b93dcebff16c278384755bdbe158559e9c240b73b80", 512, "yum"},
+		{"centos-8", "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", "7ec97062618dc0a7ebf211864abf63629da1f325578868579ee70c495bed3ba0", 768, "dnf"},
+		{"debian-9", "https://cdimage.debian.org/cdimage/openstack/9.13.21-20210511/debian-9.13.21-20210511-openstack-amd64.qcow2", "0667a08e2d947b331aee068db4bbf3a703e03edaf5afa52e23d534adff44b62a", 512, "apt"},
+		{"debian-10", "https://cdimage.debian.org/images/cloud/buster/20210329-591/debian-10-generic-amd64-20210329-591.qcow2", "70c61956095870c4082103d1a7a1cb5925293f8405fc6cb348588ec97e8611b0", 768, "apt"},
+		{"fedora-34", "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", 768, "dnf"},
+		{"opensuse-leap-15.1", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64.qcow2", "3203e256dab5981ca3301408574b63bc522a69972fbe9850b65b54ff44a96e0a", 512, "zypper"},
+		{"opensuse-leap-15.2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper"},
+		{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "ba3ecd281045b5019f0fb11378329a644a41870b77631ea647b128cd07eb804b", 512, "zypper"},
+		{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt"},
+		{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img", "08396cf95c18534a2e3f88289bd92d18eee76f0e75813636b3ab9f1e603816d7", 512, "apt"},
+		{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img", "513158b22ff0f08d0a078d8d60293bcddffdb17094a7809c76c52aba415ecc54", 512, "apt"},
+		{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/current/groovy-server-cloudimg-amd64.img", "e470df72fce4fb8d0ee4ef8af8eed740ee3bf51290515eb42e5c747725e98b6d", 512, "apt"},
+		{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/current/hirsute-server-cloudimg-amd64.img", "7fab8eda0bcf6f8f6e63845ccf1e29de4706e3359c82d3888835093020fe6f05", 512, "apt"},
+	}
+
+	dir := t.TempDir()
+
+	ln, err := net.Listen("tcp", deriveBindhost(t)+":0")
+	if err != nil {
+		t.Fatalf("can't make TCP listener: %v", err)
+	}
+	defer ln.Close()
+	t.Logf("host:port: %s", ln.Addr())
+
+	cs := &testcontrol.Server{}
+
+	var (
+		ipMu  sync.Mutex
+		ipMap = map[string]string{} // SSH port => IP address
+	)
+
+	mux := http.NewServeMux()
+	mux.Handle("/", cs)
+
+	// This handler will let the virtual machines tell the host information about that VM.
+	// This is used to maintain a list of port->IP address mappings that are known to be
+	// working. This allows later steps to connect over SSH. This returns no response to
+	// clients because no response is needed.
+	mux.HandleFunc("/myip/", func(w http.ResponseWriter, r *http.Request) {
+		ipMu.Lock()
+		defer ipMu.Unlock()
+
+		name := path.Base(r.URL.Path)
+		host, _, _ := net.SplitHostPort(r.RemoteAddr)
+		ipMap[name] = host
+		t.Logf("%s: %v", name, host)
+	})
+
+	hs := &http.Server{Handler: mux}
+	go hs.Serve(ln)
+
+	run(t, dir, "ssh-keygen", "-t", "ed25519", "-f", "machinekey", "-N", ``)
+	pubkey, err := os.ReadFile(filepath.Join(dir, "machinekey.pub"))
+	if err != nil {
+		t.Fatalf("can't read ssh key: %v", err)
+	}
+
+	privateKey, err := os.ReadFile(filepath.Join(dir, "machinekey"))
+	if err != nil {
+		t.Fatalf("can't read ssh private key: %v", err)
+	}
+
+	signer, err := ssh.ParsePrivateKey(privateKey)
+	if err != nil {
+		t.Fatalf("can't parse private key: %v", err)
+	}
+
+	loginServer := fmt.Sprintf("http://%s", ln.Addr())
+	t.Logf("loginServer: %s", loginServer)
+
+	cancels := make(chan func(), len(distros))
+
+	t.Run("mkvm", func(t *testing.T) {
+		for n, distro := range distros {
+			n, distro := n, distro
+			t.Run(distro.name, func(t *testing.T) {
+				t.Parallel()
+
+				cancel := mkVM(t, n, distro, string(pubkey), loginServer, dir)
+				cancels <- cancel
+			})
+		}
+	})
+
+	close(cancels)
+	for cancel := range cancels {
+		//lint:ignore SA9001 They do actually get ran
+		defer cancel()
+
+		if len(cancels) == 0 {
+			t.Log("all VMs started")
+			break
+		}
+	}
+
+	t.Run("wait-for-vms", func(t *testing.T) {
+		t.Log("waiting for VMs to register")
+		waiter := time.NewTicker(time.Second)
+		defer waiter.Stop()
+		n := 0
+		for {
+			<-waiter.C
+			ipMu.Lock()
+			if len(ipMap) == len(distros) {
+				ipMu.Unlock()
+				break
+			} else {
+				if n%30 == 0 {
+					t.Logf("ipMap:   %d", len(ipMap))
+					t.Logf("distros: %d", len(distros))
+				}
+			}
+			n++
+			ipMu.Unlock()
+		}
+	})
+
+	ipMu.Lock()
+	defer ipMu.Unlock()
+	t.Run("join-net", func(t *testing.T) {
+		for port := range ipMap {
+			port := port
+			t.Run(port, func(t *testing.T) {
+				config := &ssh.ClientConfig{
+					User:            "ts",
+					Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer), ssh.Password("hunter2")},
+					HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+				}
+
+				cli, err := ssh.Dial("tcp", fmt.Sprintf("127.0.0.1:%s", port), config)
+				if err != nil {
+					t.Fatalf("can't dial 127.0.0.1:%s: %v", port, err)
+				}
+				defer cli.Close()
+
+				t.Parallel()
+				t.Logf("about to ssh into 127.0.0.1:%s", port)
+				timeout := 5 * time.Minute
+
+				e, _, err := expect.SpawnSSH(cli, timeout, expect.Verbose(true), expect.VerboseWriter(os.Stdout))
+				if err != nil {
+					t.Fatalf("%s: can't register a shell session: %v", port, err)
+				}
+				defer e.Close()
+
+				_, _, err = e.Expect(regexp.MustCompile(`(\$|\>)`), timeout)
+				if err != nil {
+					t.Fatalf("%s: can't get a shell: %v", port, err)
+				}
+				t.Logf("got shell for %s", port)
+				err = e.Send(fmt.Sprintf("sudo tailscale up --login-server %s\n", loginServer))
+				if err != nil {
+					t.Fatalf("%s: can't send tailscale up command: %v", port, err)
+				}
+				_, _, err = e.Expect(regexp.MustCompile(`Success.`), timeout)
+				if err != nil {
+					t.Fatalf("can't extract URL: %v", err)
+				}
+			})
+		}
+	})
+
+	if numNodes := cs.NumNodes(); numNodes != len(ipMap) {
+		t.Errorf("wanted %d nodes, got: %d", len(ipMap), numNodes)
+	}
+}
+
+func deriveBindhost(t *testing.T) string {
+	t.Helper()
+
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rex := regexp.MustCompile(`^(eth|enp|wlp|wlan)`)
+
+	for _, iface := range ifaces {
+		t.Logf("found interface %s: %d", iface.Name, iface.Flags&net.FlagUp)
+		if (iface.Flags & net.FlagUp) == 0 {
+			continue
+		}
+
+		if rex.MatchString(iface.Name) {
+			addrs, err := iface.Addrs()
+			if err != nil {
+				t.Fatalf("can't get address for %s: %v", iface.Name, err)
+			}
+
+			for _, addr := range addrs {
+				return netaddr.MustParseIPPrefix(addr.String()).IP().String()
+			}
+		}
+	}
+
+	t.Fatal("can't find a bindhost")
+	return "invalid"
+}
+
+func TestDeriveBindhost(t *testing.T) {
+	t.Log(deriveBindhost(t))
+}
+
+const metaDataTemplate = `instance-id: {{.ID}}
+local-hostname: {{.Hostname}}`
+
+const userDataTemplate = `#cloud-config
+#vim:syntax=yaml
+
+cloud_config_modules:
+ - runcmd
+
+cloud_final_modules:
+ - [users-groups, always]
+ - [scripts-user, once-per-instance]
+
+users:
+  - name: ts
+    plain_text_passwd: hunter2
+    groups: [ wheel ]
+    sudo: [ "ALL=(ALL) NOPASSWD:ALL" ]
+    shell: /bin/sh
+    ssh-authorized-keys:
+    - {{.SSHKey}}
+
+write_files:
+  - path: /etc/cloud/cloud.cfg.d/80_disable_network_after_firstboot.cfg
+    content: |
+      # Disable network configuration after first boot
+      network:
+        config: disabled
+
+runcmd:
+{{.InstallPre}}
+ - [ "sh", "-c", "curl https://raw.githubusercontent.com/tailscale/tailscale/Xe/test-install-script-libvirtd/scripts/installer.sh | sh" ]
+ - [ systemctl, enable, --now, tailscaled.service ]
+ - [ curl, "{{.HostURL}}/myip/{{.Port}}", "-H", "User-Agent: {{.Hostname}}" ]
+`

tstest/natlab/firewall.go

@@ -52,7 +52,7 @@ func (s FirewallType) key(src, dst netaddr.IPPort) fwKey {
 	switch s {
 	case EndpointIndependentFirewall:
 	case AddressDependentFirewall:
-		k.dst.IP = dst.IP
+		k.dst = k.dst.WithIP(dst.IP())
 	case AddressAndPortDependentFirewall:
 		k.dst = dst
 	default:

tstest/natlab/nat.go

@@ -62,7 +62,7 @@ func (t NATType) key(src, dst netaddr.IPPort) natKey {
 	switch t {
 	case EndpointIndependentNAT:
 	case AddressDependentNAT:
-		k.dst.IP = dst.IP
+		k.dst = k.dst.WithIP(dst.IP())
 	case AddressAndPortDependentNAT:
 		k.dst = dst
 	default:
@@ -171,7 +171,7 @@ func (n *SNAT44) HandleIn(p *Packet, iif *Interface) *Packet {
 func (n *SNAT44) HandleForward(p *Packet, iif, oif *Interface) *Packet {
 	switch {
 	case oif == n.ExternalInterface:
-		if p.Src.IP == oif.V4() {
+		if p.Src.IP() == oif.V4() {
 			// Packet already NATed and is just retraversing Forward,
 			// don't touch it again.
 			return p
@@ -237,10 +237,7 @@ func (n *SNAT44) allocateMappedPort() (net.PacketConn, netaddr.IPPort) {
 	if err != nil {
 		panic(fmt.Sprintf("ran out of NAT ports: %v", err))
 	}
-	addr := netaddr.IPPort{
-		IP:   ip,
-		Port: uint16(pc.LocalAddr().(*net.UDPAddr).Port),
-	}
+	addr := netaddr.IPPortFrom(ip, uint16(pc.LocalAddr().(*net.UDPAddr).Port))
 	return pc, addr
 }
 

tstest/natlab/natlab.go

@@ -138,7 +138,7 @@ func (n *Network) allocIPv4(iface *Interface) netaddr.IP {
 		return netaddr.IP{}
 	}
 	if n.lastV4.IsZero() {
-		n.lastV4 = n.Prefix4.IP
+		n.lastV4 = n.Prefix4.IP()
 	}
 	a := n.lastV4.As16()
 	addOne(&a, 15)
@@ -157,7 +157,7 @@ func (n *Network) allocIPv6(iface *Interface) netaddr.IP {
 		return netaddr.IP{}
 	}
 	if n.lastV6.IsZero() {
-		n.lastV6 = n.Prefix6.IP
+		n.lastV6 = n.Prefix6.IP()
 	}
 	a := n.lastV6.As16()
 	addOne(&a, 15)
@@ -183,15 +183,15 @@ func (n *Network) write(p *Packet) (num int, err error) {
 
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	iface, ok := n.machine[p.Dst.IP]
+	iface, ok := n.machine[p.Dst.IP()]
 	if !ok {
 		// If the destination is within the network's authoritative
 		// range, no route to host.
-		if p.Dst.IP.Is4() && n.Prefix4.Contains(p.Dst.IP) {
+		if p.Dst.IP().Is4() && n.Prefix4.Contains(p.Dst.IP()) {
 			p.Trace("no route to %v", p.Dst.IP)
 			return len(p.Payload), nil
 		}
-		if p.Dst.IP.Is6() && n.Prefix6.Contains(p.Dst.IP) {
+		if p.Dst.IP().Is6() && n.Prefix6.Contains(p.Dst.IP()) {
 			p.Trace("no route to %v", p.Dst.IP)
 			return len(p.Payload), nil
 		}
@@ -363,7 +363,7 @@ func (m *Machine) isLocalIP(ip netaddr.IP) bool {
 func (m *Machine) deliverIncomingPacket(p *Packet, iface *Interface) {
 	p.setLocator("mach=%s if=%s", m.Name, iface.name)
 
-	if m.isLocalIP(p.Dst.IP) {
+	if m.isLocalIP(p.Dst.IP()) {
 		m.deliverLocalPacket(p, iface)
 	} else {
 		m.forwardPacket(p, iface)
@@ -391,13 +391,13 @@ func (m *Machine) deliverLocalPacket(p *Packet, iface *Interface) {
 	defer m.mu.Unlock()
 
 	conns := m.conns4
-	if p.Dst.IP.Is6() {
+	if p.Dst.IP().Is6() {
 		conns = m.conns6
 	}
 	possibleDsts := []netaddr.IPPort{
 		p.Dst,
-		netaddr.IPPort{IP: v6unspec, Port: p.Dst.Port},
-		netaddr.IPPort{IP: v4unspec, Port: p.Dst.Port},
+		netaddr.IPPortFrom(v6unspec, p.Dst.Port()),
+		netaddr.IPPortFrom(v4unspec, p.Dst.Port()),
 	}
 	for _, dest := range possibleDsts {
 		c, ok := conns[dest]
@@ -417,7 +417,7 @@ func (m *Machine) deliverLocalPacket(p *Packet, iface *Interface) {
 }
 
 func (m *Machine) forwardPacket(p *Packet, iif *Interface) {
-	oif, err := m.interfaceForIP(p.Dst.IP)
+	oif, err := m.interfaceForIP(p.Dst.IP())
 	if err != nil {
 		p.Trace("%v", err)
 		return
@@ -501,7 +501,7 @@ func (m *Machine) Attach(interfaceName string, n *Network) *Interface {
 		}
 	}
 	sort.Slice(m.routes, func(i, j int) bool {
-		return m.routes[i].prefix.Bits > m.routes[j].prefix.Bits
+		return m.routes[i].prefix.Bits() > m.routes[j].prefix.Bits()
 	})
 
 	return f
@@ -515,33 +515,33 @@ var (
 func (m *Machine) writePacket(p *Packet) (n int, err error) {
 	p.setLocator("mach=%s", m.Name)
 
-	iface, err := m.interfaceForIP(p.Dst.IP)
+	iface, err := m.interfaceForIP(p.Dst.IP())
 	if err != nil {
 		p.Trace("%v", err)
 		return 0, err
 	}
-	origSrcIP := p.Src.IP
+	origSrcIP := p.Src.IP()
 	switch {
-	case p.Src.IP == v4unspec:
+	case p.Src.IP() == v4unspec:
 		p.Trace("assigning srcIP=%s", iface.V4())
-		p.Src.IP = iface.V4()
-	case p.Src.IP == v6unspec:
+		p.Src = p.Src.WithIP(iface.V4())
+	case p.Src.IP() == v6unspec:
 		// v6unspec in Go means "any src, but match address families"
-		if p.Dst.IP.Is6() {
+		if p.Dst.IP().Is6() {
 			p.Trace("assigning srcIP=%s", iface.V6())
-			p.Src.IP = iface.V6()
-		} else if p.Dst.IP.Is4() {
+			p.Src = p.Src.WithIP(iface.V6())
+		} else if p.Dst.IP().Is4() {
 			p.Trace("assigning srcIP=%s", iface.V4())
-			p.Src.IP = iface.V4()
+			p.Src = p.Src.WithIP(iface.V4())
 		}
 	default:
-		if !iface.Contains(p.Src.IP) {
-			err := fmt.Errorf("can't send to %v with src %v on interface %v", p.Dst.IP, p.Src.IP, iface)
+		if !iface.Contains(p.Src.IP()) {
+			err := fmt.Errorf("can't send to %v with src %v on interface %v", p.Dst.IP(), p.Src.IP(), iface)
 			p.Trace("%v", err)
 			return 0, err
 		}
 	}
-	if p.Src.IP.IsZero() {
+	if p.Src.IP().IsZero() {
 		err := fmt.Errorf("no matching address for address family for %v", origSrcIP)
 		p.Trace("%v", err)
 		return 0, err
@@ -602,12 +602,12 @@ func (m *Machine) pickEphemPort() (port uint16, err error) {
 
 func (m *Machine) portInUseLocked(port uint16) bool {
 	for ipp := range m.conns4 {
-		if ipp.Port == port {
+		if ipp.Port() == port {
 			return true
 		}
 	}
 	for ipp := range m.conns6 {
-		if ipp.Port == port {
+		if ipp.Port() == port {
 			return true
 		}
 	}
@@ -617,7 +617,7 @@ func (m *Machine) portInUseLocked(port uint16) bool {
 func (m *Machine) registerConn4(c *conn) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if c.ipp.IP.Is6() && c.ipp.IP != v6unspec {
+	if c.ipp.IP().Is6() && c.ipp.IP() != v6unspec {
 		return fmt.Errorf("registerConn4 got IPv6 %s", c.ipp)
 	}
 	return registerConn(&m.conns4, c)
@@ -632,7 +632,7 @@ func (m *Machine) unregisterConn4(c *conn) {
 func (m *Machine) registerConn6(c *conn) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if c.ipp.IP.Is4() {
+	if c.ipp.IP().Is4() {
 		return fmt.Errorf("registerConn6 got IPv4 %s", c.ipp)
 	}
 	return registerConn(&m.conns6, c)
@@ -707,7 +707,7 @@ func (m *Machine) ListenPacket(ctx context.Context, network, address string) (ne
 			return nil, nil
 		}
 	}
-	ipp := netaddr.IPPort{IP: ip, Port: port}
+	ipp := netaddr.IPPortFrom(ip, port)
 
 	c := &conn{
 		m:   m,

tstest/natlab/natlab_test.go

@@ -49,8 +49,8 @@ func TestSendPacket(t *testing.T) {
 	ifFoo := foo.Attach("eth0", internet)
 	ifBar := bar.Attach("enp0s1", internet)
 
-	fooAddr := netaddr.IPPort{IP: ifFoo.V4(), Port: 123}
-	barAddr := netaddr.IPPort{IP: ifBar.V4(), Port: 456}
+	fooAddr := netaddr.IPPortFrom(ifFoo.V4(), 123)
+	barAddr := netaddr.IPPortFrom(ifBar.V4(), 456)
 
 	ctx := context.Background()
 	fooPC, err := foo.ListenPacket(ctx, "udp4", fooAddr.String())
@@ -111,10 +111,10 @@ func TestMultiNetwork(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	clientAddr := netaddr.IPPort{IP: ifClient.V4(), Port: 123}
-	natLANAddr := netaddr.IPPort{IP: ifNATLAN.V4(), Port: 456}
-	natWANAddr := netaddr.IPPort{IP: ifNATWAN.V4(), Port: 456}
-	serverAddr := netaddr.IPPort{IP: ifServer.V4(), Port: 789}
+	clientAddr := netaddr.IPPortFrom(ifClient.V4(), 123)
+	natLANAddr := netaddr.IPPortFrom(ifNATLAN.V4(), 456)
+	natWANAddr := netaddr.IPPortFrom(ifNATWAN.V4(), 456)
+	serverAddr := netaddr.IPPortFrom(ifServer.V4(), 789)
 
 	const msg1, msg2 = "hello", "world"
 	if _, err := natPC.WriteTo([]byte(msg1), clientAddr.UDPAddr()); err != nil {
@@ -154,8 +154,8 @@ type trivialNAT struct {
 }
 
 func (n *trivialNAT) HandleIn(p *Packet, iface *Interface) *Packet {
-	if iface == n.wanIf && p.Dst.IP == n.wanIf.V4() {
-		p.Dst.IP = n.clientIP
+	if iface == n.wanIf && p.Dst.IP() == n.wanIf.V4() {
+		p.Dst = p.Dst.WithIP(n.clientIP)
 	}
 	return p
 }
@@ -167,13 +167,13 @@ func (n trivialNAT) HandleOut(p *Packet, iface *Interface) *Packet {
 func (n *trivialNAT) HandleForward(p *Packet, iif, oif *Interface) *Packet {
 	// Outbound from LAN -> apply NAT, continue
 	if iif == n.lanIf && oif == n.wanIf {
-		if p.Src.IP == n.clientIP {
-			p.Src.IP = n.wanIf.V4()
+		if p.Src.IP() == n.clientIP {
+			p.Src = p.Src.WithIP(n.wanIf.V4())
 		}
 		return p
 	}
 	// Return traffic to LAN, allow if right dst.
-	if iif == n.wanIf && oif == n.lanIf && p.Dst.IP == n.clientIP {
+	if iif == n.wanIf && oif == n.lanIf && p.Dst.IP() == n.clientIP {
 		return p
 	}
 	// Else drop.
@@ -216,7 +216,7 @@ func TestPacketHandler(t *testing.T) {
 	}
 
 	const msg = "some message"
-	serverAddr := netaddr.IPPort{IP: ifServer.V4(), Port: 456}
+	serverAddr := netaddr.IPPortFrom(ifServer.V4(), 456)
 	if _, err := clientPC.WriteTo([]byte(msg), serverAddr.UDPAddr()); err != nil {
 		t.Fatal(err)
 	}
@@ -230,7 +230,7 @@ func TestPacketHandler(t *testing.T) {
 	if string(buf) != msg {
 		t.Errorf("read %q; want %q", buf, msg)
 	}
-	mappedAddr := netaddr.IPPort{IP: ifNATWAN.V4(), Port: 123}
+	mappedAddr := netaddr.IPPortFrom(ifNATWAN.V4(), 123)
 	if addr.String() != mappedAddr.String() {
 		t.Errorf("addr = %q; want %q", addr, mappedAddr)
 	}

tsweb/jsonhandler.go

@@ -89,3 +89,64 @@ func (fn JSONHandlerFunc) ServeHTTPReturn(w http.ResponseWriter, r *http.Request
 	w.Write(b)
 	return err
 }
+
+// TODO() Set this function such that chunk encoding works
+// Currently the same thing with chunking headers set.
+func (fn JSONHandlerFunc) ServeHTTPChunkEncodingReturn(w http.ResponseWriter, r *http.Request) error {
+	w.Header().Set("Connection", "Keep-Alive")
+	w.Header().Set("Transfer-Encoding", "chunked")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	var resp *response
+	status, data, err := fn(r)
+	if err != nil {
+		if werr, ok := err.(HTTPError); ok {
+			resp = &response{
+				Status: "error",
+				Error:  werr.Msg,
+				Data:   data,
+			}
+			// Unwrap the HTTPError here because we are communicating with
+			// the client in this handler. We don't want the wrapping
+			// ReturnHandler to do it too.
+			err = werr.Err
+			if werr.Msg != "" {
+				err = fmt.Errorf("%s: %w", werr.Msg, err)
+			}
+			// take status from the HTTPError to encourage error handling in one location
+			if status != 0 && status != werr.Code {
+				err = fmt.Errorf("[unexpected] non-zero status that does not match HTTPError status, status: %d, HTTPError.code: %d: %w", status, werr.Code, err)
+			}
+			status = werr.Code
+		} else {
+			status = http.StatusInternalServerError
+			resp = &response{
+				Status: "error",
+				Error:  "internal server error",
+			}
+		}
+	} else if status == 0 {
+		status = http.StatusInternalServerError
+		resp = &response{
+			Status: "error",
+			Error:  "internal server error",
+		}
+	} else if err == nil {
+		resp = &response{
+			Status: "success",
+			Data:   data,
+		}
+	}
+	b, jerr := json.Marshal(resp)
+	if jerr != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(`{"status":"error","error":"json marshal error"}`))
+		if err != nil {
+			return fmt.Errorf("%w, and then we could not respond: %v", err, jerr)
+		}
+		return jerr
+	}
+
+	w.WriteHeader(status)
+	w.Write(b)
+	return err
+}

types/netmap/netmap_test.go

@@ -250,7 +250,7 @@ func TestConciseDiffFrom(t *testing.T) {
 						DERP:       "127.3.3.40:2",
 						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
 						DiscoKey:   testDiscoKey("f00f00f00f"),
-						AllowedIPs: []netaddr.IPPrefix{{IP: netaddr.IPv4(100, 102, 103, 104), Bits: 32}},
+						AllowedIPs: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(100, 102, 103, 104), 32)},
 					},
 				},
 			},
@@ -263,7 +263,7 @@ func TestConciseDiffFrom(t *testing.T) {
 						DERP:       "127.3.3.40:2",
 						Endpoints:  []string{"192.168.0.100:41641", "1.1.1.1:41641"},
 						DiscoKey:   testDiscoKey("ba4ba4ba4b"),
-						AllowedIPs: []netaddr.IPPrefix{{IP: netaddr.IPv4(100, 102, 103, 104), Bits: 32}},
+						AllowedIPs: []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(100, 102, 103, 104), 32)},
 					},
 				},
 			},

types/strbuilder/strbuilder.go

@@ -1,74 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package strbuilder defines a string builder type that allocates
-// less than the standard library's strings.Builder by using a
-// sync.Pool, so it doesn't matter if the compiler can't prove that
-// the builder doesn't escape into the fmt package, etc.
-package strbuilder
-
-import (
-	"bytes"
-	"strconv"
-	"sync"
-)
-
-var pool = sync.Pool{
-	New: func() interface{} { return new(Builder) },
-}
-
-type Builder struct {
-	bb      bytes.Buffer
-	scratch [20]byte // long enough for MinInt64, MaxUint64
-	locked  bool     // in pool, not for use
-}
-
-// Get returns a new or reused string Builder.
-func Get() *Builder {
-	b := pool.Get().(*Builder)
-	b.bb.Reset()
-	b.locked = false
-	return b
-}
-
-// String both returns the Builder's string, and returns the builder
-// to the pool.
-func (b *Builder) String() string {
-	if b.locked {
-		panic("String called twiced on Builder")
-	}
-	s := b.bb.String()
-	b.locked = true
-	pool.Put(b)
-	return s
-}
-
-func (b *Builder) WriteByte(v byte) error {
-	return b.bb.WriteByte(v)
-}
-
-func (b *Builder) WriteString(s string) (int, error) {
-	return b.bb.WriteString(s)
-}
-
-func (b *Builder) Write(p []byte) (int, error) {
-	return b.bb.Write(p)
-}
-
-func (b *Builder) WriteInt(v int64) {
-	b.Write(strconv.AppendInt(b.scratch[:0], v, 10))
-}
-
-func (b *Builder) WriteUint(v uint64) {
-	b.Write(strconv.AppendUint(b.scratch[:0], v, 10))
-}
-
-// Grow grows the buffer's capacity, if necessary, to guarantee space
-// for another n bytes. After Grow(n), at least n bytes can be written
-// to the buffer without another allocation. If n is negative, Grow
-// will panic. If the buffer can't grow it will panic with
-// ErrTooLarge.
-func (b *Builder) Grow(n int) {
-	b.bb.Grow(n)
-}

types/strbuilder/strbuilder_test.go

@@ -1,52 +0,0 @@
-// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package strbuilder
-
-import (
-	"math"
-	"testing"
-)
-
-func TestBuilder(t *testing.T) {
-	const want = "Hello, world 123 -456!"
-	bang := []byte("!")
-	var got string
-	allocs := testing.AllocsPerRun(1000, func() {
-		sb := Get()
-		sb.WriteString("Hello, world ")
-		sb.WriteUint(123)
-		sb.WriteByte(' ')
-		sb.WriteInt(-456)
-		sb.Write(bang)
-		got = sb.String()
-	})
-	if got != want {
-		t.Errorf("got %q; want %q", got, want)
-	}
-	if allocs != 1 {
-		t.Errorf("allocs = %v; want 1", allocs)
-	}
-}
-
-// Verifies scratch buf is large enough.
-func TestIntBounds(t *testing.T) {
-	const want = "-9223372036854775808 9223372036854775807 18446744073709551615"
-	var got string
-	allocs := testing.AllocsPerRun(1000, func() {
-		sb := Get()
-		sb.WriteInt(math.MinInt64)
-		sb.WriteByte(' ')
-		sb.WriteInt(math.MaxInt64)
-		sb.WriteByte(' ')
-		sb.WriteUint(math.MaxUint64)
-		got = sb.String()
-	})
-	if got != want {
-		t.Errorf("got %q; want %q", got, want)
-	}
-	if allocs != 1 {
-		t.Errorf("allocs = %v; want 1", allocs)
-	}
-}

types/wgkey/key.go

@@ -10,7 +10,6 @@
 package wgkey
 
 import (
-	"bytes"
 	"crypto/rand"
 	"crypto/subtle"
 	"encoding/base64"
@@ -72,10 +71,11 @@ func ParsePrivateHex(v string) (Private, error) {
 	return pk, nil
 }
 
-func (k Key) Base64() string    { return base64.StdEncoding.EncodeToString(k[:]) }
-func (k Key) String() string    { return k.ShortString() }
-func (k Key) HexString() string { return hex.EncodeToString(k[:]) }
-func (k Key) Equal(k2 Key) bool { return subtle.ConstantTimeCompare(k[:], k2[:]) == 1 }
+func (k Key) Base64() string           { return base64.StdEncoding.EncodeToString(k[:]) }
+func (k Key) String() string           { return k.ShortString() }
+func (k Key) HexString() string        { return hex.EncodeToString(k[:]) }
+func (k Key) Equal(k2 Key) bool        { return subtle.ConstantTimeCompare(k[:], k2[:]) == 1 }
+func (k Key) AppendTo(b []byte) []byte { return appendKey(b, "", k) }
 
 func (k *Key) ShortString() string {
 	// The goal here is to generate "[" + base64.StdEncoding.EncodeToString(k[:])[:5] + "]".
@@ -178,13 +178,17 @@ func (k *Private) Public() Key {
 	return (Key)(p)
 }
 
-func (k Private) MarshalText() ([]byte, error) {
-	// TODO(josharian): use encoding/hex instead?
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, `privkey:%x`, k[:])
-	return buf.Bytes(), nil
+func appendKey(base []byte, prefix string, k [32]byte) []byte {
+	ret := append(base, make([]byte, len(prefix)+64)...)
+	buf := ret[len(base):]
+	copy(buf, prefix)
+	hex.Encode(buf[len(prefix):], k[:])
+	return ret
 }
 
+func (k Private) MarshalText() ([]byte, error) { return appendKey(nil, "privkey:", k), nil }
+func (k Private) AppendTo(b []byte) []byte     { return appendKey(b, "privkey:", k) }
+
 func (k *Private) UnmarshalText(b []byte) error {
 	s := string(b)
 	if !strings.HasPrefix(s, `privkey:`) {

version/race.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build race
+
+package version
+
+// IsRace reports whether the current binary was built with the Go
+// race detector enabled.
+func IsRace() bool { return true }

version/race_off.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !race
+
+package version
+
+// IsRace reports whether the current binary was built with the Go
+// race detector enabled.
+func IsRace() bool { return false }

wf/firewall.go

@@ -218,7 +218,7 @@ func (f *Firewall) UpdatePermittedRoutes(newRoutes []netaddr.IPPrefix) error {
 			},
 		}
 		var p protocol
-		if r.IP.Is4() {
+		if r.IP().Is4() {
 			p = protocolV4
 		} else {
 			p = protocolV6

wgengine/bench/bench.go

@@ -87,7 +87,7 @@ func main() {
 	}
 
 	logf("initialized ok.")
-	traf.Start(Addr1.IP, Addr2.IP, PayloadSize+ICMPMinSize, 0)
+	traf.Start(Addr1.IP(), Addr2.IP(), PayloadSize+ICMPMinSize, 0)
 
 	var cur, prev Snapshot
 	var pps int64

wgengine/bench/bench_test.go

@@ -78,7 +78,7 @@ func runOnce(b *testing.B, setup SetupFunc, payload int) {
 	logf("initialized. (n=%v)", b.N)
 	b.SetBytes(int64(payload))
 
-	traf.Start(Addr1.IP, Addr2.IP, payload, int64(b.N))
+	traf.Start(Addr1.IP(), Addr2.IP(), payload, int64(b.N))
 
 	var cur, prev Snapshot
 	var pps int64

wgengine/bench/wg.go

@@ -12,7 +12,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 
 	"tailscale.com/net/dns"

wgengine/filter/filter.go

@@ -98,8 +98,8 @@ const (
 // everything. Use in tests only, as it permits some kinds of spoofing
 // attacks to reach the OS network stack.
 func NewAllowAllForTest(logf logger.Logf) *Filter {
-	any4 := netaddr.IPPrefix{IP: netaddr.IPv4(0, 0, 0, 0), Bits: 0}
-	any6 := netaddr.IPPrefix{IP: netaddr.IPFrom16([16]byte{}), Bits: 0}
+	any4 := netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0)
+	any6 := netaddr.IPPrefixFrom(netaddr.IPFrom16([16]byte{}), 0)
 	ms := []Match{
 		{
 			Srcs: []netaddr.IPPrefix{any4},
@@ -185,12 +185,12 @@ func matchesFamily(ms matches, keep func(netaddr.IP) bool) matches {
 		var retm Match
 		retm.IPProto = m.IPProto
 		for _, src := range m.Srcs {
-			if keep(src.IP) {
+			if keep(src.IP()) {
 				retm.Srcs = append(retm.Srcs, src)
 			}
 		}
 		for _, dst := range m.Dsts {
-			if keep(dst.Net.IP) {
+			if keep(dst.Net.IP()) {
 				retm.Dsts = append(retm.Dsts, dst)
 			}
 		}
@@ -266,12 +266,10 @@ func (f *Filter) CheckTCP(srcIP, dstIP netaddr.IP, dstPort uint16) Response {
 	default:
 		panic("unreachable")
 	}
-	pkt.Src.IP = srcIP
-	pkt.Dst.IP = dstIP
+	pkt.Src = netaddr.IPPortFrom(srcIP, 0)
+	pkt.Dst = netaddr.IPPortFrom(dstIP, dstPort)
 	pkt.IPProto = ipproto.TCP
 	pkt.TCPFlags = packet.TCPSyn
-	pkt.Src.Port = 0
-	pkt.Dst.Port = dstPort
 
 	return f.RunIn(pkt, 0)
 }
@@ -321,7 +319,7 @@ func (f *Filter) runIn4(q *packet.Parsed) (r Response, why string) {
 	// A compromised peer could try to send us packets for
 	// destinations we didn't explicitly advertise. This check is to
 	// prevent that.
-	if !f.local.Contains(q.Dst.IP) {
+	if !f.local.Contains(q.Dst.IP()) {
 		return Drop, "destination not allowed"
 	}
 
@@ -378,7 +376,7 @@ func (f *Filter) runIn6(q *packet.Parsed) (r Response, why string) {
 	// A compromised peer could try to send us packets for
 	// destinations we didn't explicitly advertise. This check is to
 	// prevent that.
-	if !f.local.Contains(q.Dst.IP) {
+	if !f.local.Contains(q.Dst.IP()) {
 		return Drop, "destination not allowed"
 	}
 
@@ -480,11 +478,11 @@ func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) Response {
 		return Drop
 	}
 
-	if q.Dst.IP.IsMulticast() {
+	if q.Dst.IP().IsMulticast() {
 		f.logRateLimit(rf, q, dir, Drop, "multicast")
 		return Drop
 	}
-	if q.Dst.IP.IsLinkLocalUnicast() && q.Dst.IP != gcpDNSAddr {
+	if q.Dst.IP().IsLinkLocalUnicast() && q.Dst.IP() != gcpDNSAddr {
 		f.logRateLimit(rf, q, dir, Drop, "link-local-unicast")
 		return Drop
 	}
@@ -506,7 +504,7 @@ func (f *Filter) pre(q *packet.Parsed, rf RunFlags, dir direction) Response {
 
 // loggingAllowed reports whether p can appear in logs at all.
 func (f *Filter) loggingAllowed(p *packet.Parsed) bool {
-	return f.logIPs.Contains(p.Src.IP) && f.logIPs.Contains(p.Dst.IP)
+	return f.logIPs.Contains(p.Src.IP()) && f.logIPs.Contains(p.Dst.IP())
 }
 
 // omitDropLogging reports whether packet p, which has already been
@@ -518,5 +516,5 @@ func omitDropLogging(p *packet.Parsed, dir direction) bool {
 		return false
 	}
 
-	return p.Dst.IP.IsMulticast() || (p.Dst.IP.IsLinkLocalUnicast() && p.Dst.IP != gcpDNSAddr) || p.IPProto == ipproto.IGMP
+	return p.Dst.IP().IsMulticast() || (p.Dst.IP().IsLinkLocalUnicast() && p.Dst.IP() != gcpDNSAddr) || p.IPProto == ipproto.IGMP
 }

wgengine/filter/filter_test.go

@@ -120,9 +120,9 @@ func TestFilter(t *testing.T) {
 		if test.p.IPProto == ipproto.TCP {
 			var got Response
 			if test.p.IPVersion == 4 {
-				got = acl.CheckTCP(test.p.Src.IP, test.p.Dst.IP, test.p.Dst.Port)
+				got = acl.CheckTCP(test.p.Src.IP(), test.p.Dst.IP(), test.p.Dst.Port())
 			} else {
-				got = acl.CheckTCP(test.p.Src.IP, test.p.Dst.IP, test.p.Dst.Port)
+				got = acl.CheckTCP(test.p.Src.IP(), test.p.Dst.IP(), test.p.Dst.Port())
 			}
 			if test.want != got {
 				t.Errorf("#%d CheckTCP got=%v want=%v packet:%v", i, got, test.want, test.p)
@@ -254,7 +254,9 @@ func TestParseIPSet(t *testing.T) {
 			}
 			t.Errorf("parseIPSet(%q, %v) error: %v; want error %q", tt.host, tt.bits, err, tt.wantErr)
 		}
-		if diff := cmp.Diff(got, tt.want, cmp.Comparer(func(a, b netaddr.IP) bool { return a == b })); diff != "" {
+		compareIP := cmp.Comparer(func(a, b netaddr.IP) bool { return a == b })
+		compareIPPrefix := cmp.Comparer(func(a, b netaddr.IPPrefix) bool { return a == b })
+		if diff := cmp.Diff(got, tt.want, compareIP, compareIPPrefix); diff != "" {
 			t.Errorf("parseIPSet(%q, %v) = %s; want %s", tt.host, tt.bits, got, tt.want)
 			continue
 		}
@@ -425,10 +427,10 @@ func TestLoggingPrivacy(t *testing.T) {
 	f.logIPs = logB.IPSet()
 
 	var (
-		ts4       = netaddr.IPPort{IP: tsaddr.CGNATRange().IP.Next(), Port: 1234}
-		internet4 = netaddr.IPPort{IP: netaddr.MustParseIP("8.8.8.8"), Port: 1234}
-		ts6       = netaddr.IPPort{IP: tsaddr.TailscaleULARange().IP.Next(), Port: 1234}
-		internet6 = netaddr.IPPort{IP: netaddr.MustParseIP("2001::1"), Port: 1234}
+		ts4       = netaddr.IPPortFrom(tsaddr.CGNATRange().IP().Next(), 1234)
+		internet4 = netaddr.IPPortFrom(netaddr.MustParseIP("8.8.8.8"), 1234)
+		ts6       = netaddr.IPPortFrom(tsaddr.TailscaleULARange().IP().Next(), 1234)
+		internet6 = netaddr.IPPortFrom(netaddr.MustParseIP("2001::1"), 1234)
 	)
 
 	tests := []struct {
@@ -545,10 +547,8 @@ func parsed(proto ipproto.Proto, src, dst string, sport, dport uint16) packet.Pa
 	var ret packet.Parsed
 	ret.Decode(dummyPacket)
 	ret.IPProto = proto
-	ret.Src.IP = sip
-	ret.Src.Port = sport
-	ret.Dst.IP = dip
-	ret.Dst.Port = dport
+	ret.Src = netaddr.IPPortFrom(sip, sport)
+	ret.Dst = netaddr.IPPortFrom(dip, dport)
 	ret.TCPFlags = packet.TCPSyn
 
 	if sip.Is4() {
@@ -674,7 +674,7 @@ func nets(nets ...string) (ret []netaddr.IPPrefix) {
 			if ip.Is6() {
 				bits = 128
 			}
-			ret = append(ret, netaddr.IPPrefix{IP: ip, Bits: bits})
+			ret = append(ret, netaddr.IPPrefixFrom(ip, bits))
 		} else {
 			pfx, err := netaddr.ParseIPPrefix(s)
 			if err != nil {

wgengine/filter/match.go

@@ -85,14 +85,14 @@ func (ms matches) match(q *packet.Parsed) bool {
 		if !protoInList(q.IPProto, m.IPProto) {
 			continue
 		}
-		if !ipInList(q.Src.IP, m.Srcs) {
+		if !ipInList(q.Src.IP(), m.Srcs) {
 			continue
 		}
 		for _, dst := range m.Dsts {
-			if !dst.Net.Contains(q.Dst.IP) {
+			if !dst.Net.Contains(q.Dst.IP()) {
 				continue
 			}
-			if !dst.Ports.contains(q.Dst.Port) {
+			if !dst.Ports.contains(q.Dst.Port()) {
 				continue
 			}
 			return true
@@ -103,11 +103,11 @@ func (ms matches) match(q *packet.Parsed) bool {
 
 func (ms matches) matchIPsOnly(q *packet.Parsed) bool {
 	for _, m := range ms {
-		if !ipInList(q.Src.IP, m.Srcs) {
+		if !ipInList(q.Src.IP(), m.Srcs) {
 			continue
 		}
 		for _, dst := range m.Dsts {
-			if dst.Net.Contains(q.Dst.IP) {
+			if dst.Net.Contains(q.Dst.IP()) {
 				return true
 			}
 		}

wgengine/filter/tailcfg.go

@@ -99,8 +99,8 @@ func parseIPSet(arg string, bits *int) ([]netaddr.IPPrefix, error) {
 	if arg == "*" {
 		// User explicitly requested wildcard.
 		return []netaddr.IPPrefix{
-			{IP: zeroIP4, Bits: 0},
-			{IP: zeroIP6, Bits: 0},
+			netaddr.IPPrefixFrom(zeroIP4, 0),
+			netaddr.IPPrefixFrom(zeroIP6, 0),
 		}, nil
 	}
 	if strings.Contains(arg, "/") {
@@ -124,7 +124,7 @@ func parseIPSet(arg string, bits *int) ([]netaddr.IPPrefix, error) {
 		if err != nil {
 			return nil, err
 		}
-		r := netaddr.IPRange{From: ip1, To: ip2}
+		r := netaddr.IPRangeFrom(ip1, ip2)
 		if !r.Valid() {
 			return nil, fmt.Errorf("invalid IP range %q", arg)
 		}
@@ -141,5 +141,5 @@ func parseIPSet(arg string, bits *int) ([]netaddr.IPPrefix, error) {
 		}
 		bits8 = uint8(*bits)
 	}
-	return []netaddr.IPPrefix{{IP: ip, Bits: bits8}}, nil
+	return []netaddr.IPPrefix{netaddr.IPPrefixFrom(ip, bits8)}, nil
 }

wgengine/magicsock/legacy.go

@@ -16,11 +16,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/tailscale/wireguard-go/conn"
-	"github.com/tailscale/wireguard-go/tai64n"
 	"golang.org/x/crypto/blake2s"
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/crypto/poly1305"
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/tai64n"
 	"inet.af/netaddr"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/types/key"
@@ -62,7 +62,7 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs wgcfg.IPPortSet,
 
 	// Add entries to c.addrsByUDP.
 	for _, ipp := range a.ipPorts {
-		if ipp.IP == derpMagicIPAddr {
+		if ipp.IP() == derpMagicIPAddr {
 			continue
 		}
 		c.addrsByUDP[ipp] = a
@@ -70,7 +70,7 @@ func (c *Conn) createLegacyEndpointLocked(pk key.Public, addrs wgcfg.IPPortSet,
 
 	// Remove previous c.addrsByUDP entries that are no longer in the new set.
 	for _, ipp := range oldIPP {
-		if ipp.IP != derpMagicIPAddr && c.addrsByUDP[ipp] != a {
+		if ipp.IP() != derpMagicIPAddr && c.addrsByUDP[ipp] != a {
 			delete(c.addrsByUDP, ipp)
 		}
 	}
@@ -388,8 +388,8 @@ type addrSet struct {
 // derpID returns this addrSet's home DERP node, or 0 if none is found.
 func (as *addrSet) derpID() int {
 	for _, ua := range as.ipPorts {
-		if ua.IP == derpMagicIPAddr {
-			return int(ua.Port)
+		if ua.IP() == derpMagicIPAddr {
+			return int(ua.Port())
 		}
 	}
 	return 0
@@ -428,7 +428,7 @@ func (a *addrSet) DstToString() string {
 	return a.rawdst
 }
 func (a *addrSet) DstIP() net.IP {
-	return a.dst().IP.IPAddr().IP // TODO: add netaddr accessor to cut an alloc here?
+	return a.dst().IP().IPAddr().IP // TODO: add netaddr accessor to cut an alloc here?
 }
 func (a *addrSet) SrcIP() net.IP       { return nil }
 func (a *addrSet) SrcToString() string { return "" }
@@ -437,7 +437,7 @@ func (a *addrSet) ClearSrc()           {}
 // updateDst records receipt of a packet from new. This is used to
 // potentially update the transmit address used for this addrSet.
 func (a *addrSet) updateDst(new netaddr.IPPort) error {
-	if new.IP == derpMagicIPAddr {
+	if new.IP() == derpMagicIPAddr {
 		// Never consider DERP addresses as a viable candidate for
 		// either curAddr or roamAddr. It's only ever a last resort
 		// choice, never a preferred choice.
@@ -539,7 +539,7 @@ func (as *addrSet) populatePeerStatus(ps *ipnstate.PeerStatus) {
 
 	ps.LastWrite = as.lastSend
 	for i, ua := range as.ipPorts {
-		if ua.IP == derpMagicIPAddr {
+		if ua.IP() == derpMagicIPAddr {
 			continue
 		}
 		uaStr := ua.String()

wgengine/magicsock/magicsock.go

@@ -15,6 +15,7 @@ import (
 	"errors"
 	"fmt"
 	"hash/fnv"
+	"log"
 	"math"
 	"math/rand"
 	"net"
@@ -27,9 +28,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/tailscale/wireguard-go/conn"
 	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/time/rate"
+	"golang.zx2c4.com/wireguard/conn"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/derp"
@@ -824,6 +825,7 @@ func (c *Conn) LastRecvActivityOfDisco(dk tailcfg.DiscoKey) time.Time {
 
 // Ping handles a "tailscale ping" CLI query.
 func (c *Conn) Ping(peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnstate.PingResult)) {
+	log.Println("CLIPING")
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if c.privateKey.IsZero() {
@@ -832,7 +834,7 @@ func (c *Conn) Ping(peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnst
 		return
 	}
 	if len(peer.Addresses) > 0 {
-		res.NodeIP = peer.Addresses[0].IP.String()
+		res.NodeIP = peer.Addresses[0].IP().String()
 	}
 	res.NodeName = peer.Name // prefer DNS name
 	if res.NodeName == "" {
@@ -878,11 +880,11 @@ func (c *Conn) Ping(peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnst
 // c.mu must be held
 func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep netaddr.IPPort) {
 	res.LatencySeconds = latency.Seconds()
-	if ep.IP != derpMagicIPAddr {
+	if ep.IP() != derpMagicIPAddr {
 		res.Endpoint = ep.String()
 		return
 	}
-	regionID := int(ep.Port)
+	regionID := int(ep.Port())
 	res.DERPRegionID = regionID
 	if c.derpMap != nil {
 		if dr, ok := c.derpMap.Regions[regionID]; ok {
@@ -965,7 +967,7 @@ func (c *Conn) goDerpConnect(node int) {
 	if node == 0 {
 		return
 	}
-	go c.derpWriteChanOfAddr(netaddr.IPPort{IP: derpMagicIPAddr, Port: uint16(node)}, key.Public{})
+	go c.derpWriteChanOfAddr(netaddr.IPPortFrom(derpMagicIPAddr, uint16(node)), key.Public{})
 }
 
 // determineEndpoints returns the machine's endpoint addresses. It
@@ -1037,7 +1039,7 @@ func (c *Conn) determineEndpoints(ctx context.Context) ([]tailcfg.Endpoint, erro
 			ips = loopback
 		}
 		for _, ip := range ips {
-			addAddr(netaddr.IPPort{IP: ip, Port: uint16(localAddr.Port)}, tailcfg.EndpointLocal)
+			addAddr(netaddr.IPPortFrom(ip, uint16(localAddr.Port)), tailcfg.EndpointLocal)
 		}
 	} else {
 		// Our local endpoint is bound to a particular address.
@@ -1169,7 +1171,7 @@ func (c *Conn) sendUDPStd(addr *net.UDPAddr, b []byte) (sent bool, err error) {
 // IPv6 address when the local machine doesn't have IPv6 support
 // returns (false, nil); it's not an error, but nothing was sent.
 func (c *Conn) sendAddr(addr netaddr.IPPort, pubKey key.Public, b []byte) (sent bool, err error) {
-	if addr.IP != derpMagicIPAddr {
+	if addr.IP() != derpMagicIPAddr {
 		return c.sendUDP(addr, b)
 	}
 
@@ -1211,10 +1213,10 @@ const bufferedDerpWritesBeforeDrop = 32
 // If peer is non-zero, it can be used to find an active reverse
 // path, without using addr.
 func (c *Conn) derpWriteChanOfAddr(addr netaddr.IPPort, peer key.Public) chan<- derpWriteRequest {
-	if addr.IP != derpMagicIPAddr {
+	if addr.IP() != derpMagicIPAddr {
 		return nil
 	}
-	regionID := int(addr.Port)
+	regionID := int(addr.Port())
 
 	if c.networkDown() {
 		return nil
@@ -1402,7 +1404,7 @@ func (c *Conn) runDerpReader(ctx context.Context, derpFakeAddr netaddr.IPPort, d
 	}
 
 	didCopy := make(chan struct{}, 1)
-	regionID := int(derpFakeAddr.Port)
+	regionID := int(derpFakeAddr.Port())
 	res := derpReadResult{regionID: regionID}
 	var pkt derp.ReceivedPacket
 	res.copyBuf = func(dst []byte) int {
@@ -1676,7 +1678,7 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep con
 		return 0, nil
 	}
 
-	ipp := netaddr.IPPort{IP: derpMagicIPAddr, Port: uint16(regionID)}
+	ipp := netaddr.IPPortFrom(derpMagicIPAddr, uint16(regionID))
 	if c.handleDiscoMessage(b[:n], ipp) {
 		return 0, nil
 	}
@@ -1922,7 +1924,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netaddr.IPPort) (isDiscoMsg bo
 		}
 		de.handlePongConnLocked(dm, src)
 	case *disco.CallMeMaybe:
-		if src.IP != derpMagicIPAddr {
+		if src.IP() != derpMagicIPAddr {
 			// CallMeMaybe messages should only come via DERP.
 			c.logf("[unexpected] CallMeMaybe packets should only come via DERP")
 			return
@@ -2212,6 +2214,7 @@ func nodesEqual(x, y []*tailcfg.Node) bool {
 // conditionally sent to SetDERPMap instead.
 func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) {
 	c.mu.Lock()
+	log.Println("NETMAP being set")
 	defer c.mu.Unlock()
 
 	if c.netMap != nil && nodesEqual(c.netMap.Peers, nm.Peers) {
@@ -2722,7 +2725,7 @@ func (c *Conn) resetEndpointStates() {
 
 // packIPPort packs an IPPort into the form wanted by WireGuard.
 func packIPPort(ua netaddr.IPPort) []byte {
-	ip := ua.IP.Unmap()
+	ip := ua.IP().Unmap()
 	a := ip.As16()
 	ipb := a[:]
 	if ip.Is4() {
@@ -2730,8 +2733,8 @@ func packIPPort(ua netaddr.IPPort) []byte {
 	}
 	b := make([]byte, 0, len(ipb)+2)
 	b = append(b, ipb...)
-	b = append(b, byte(ua.Port))
-	b = append(b, byte(ua.Port>>8))
+	b = append(b, byte(ua.Port()))
+	b = append(b, byte(ua.Port()>>8))
 	return b
 }
 
@@ -2972,15 +2975,15 @@ func peerShort(k key.Public) string {
 }
 
 func sbPrintAddr(sb *strings.Builder, a netaddr.IPPort) {
-	is6 := a.IP.Is6()
+	is6 := a.IP().Is6()
 	if is6 {
 		sb.WriteByte('[')
 	}
-	fmt.Fprintf(sb, "%s", a.IP)
+	fmt.Fprintf(sb, "%s", a.IP())
 	if is6 {
 		sb.WriteByte(']')
 	}
-	fmt.Fprintf(sb, ":%d", a.Port)
+	fmt.Fprintf(sb, ":%d", a.Port())
 }
 
 func (c *Conn) derpRegionCodeOfAddrLocked(ipPort string) string {
@@ -3017,15 +3020,15 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 			if !addr.IsSingleIP() {
 				continue
 			}
-			sb.AddTailscaleIP(addr.IP)
+			sb.AddTailscaleIP(addr.IP())
 			// TailAddr previously only allowed for a
 			// single Tailscale IP. For compatibility for
 			// a couple releases starting with 1.8, keep
 			// that field pulled out separately.
-			if addr.IP.Is4() {
-				tailAddr4 = addr.IP.String()
+			if addr.IP().Is4() {
+				tailAddr4 = addr.IP().String()
 			}
-			tailscaleIPs = append(tailscaleIPs, addr.IP)
+			tailscaleIPs = append(tailscaleIPs, addr.IP())
 		}
 	}
 
@@ -3084,8 +3087,8 @@ func (c *Conn) UpdateStatus(sb *ipnstate.StatusBuilder) {
 }
 
 func ippDebugString(ua netaddr.IPPort) string {
-	if ua.IP == derpMagicIPAddr {
-		return fmt.Sprintf("derp-%d", ua.Port)
+	if ua.IP() == derpMagicIPAddr {
+		return fmt.Sprintf("derp-%d", ua.Port())
 	}
 	return ua.String()
 }
@@ -3254,10 +3257,7 @@ func (de *discoEndpoint) initFakeUDPAddr() {
 	addr[0] = 0xfd
 	addr[1] = 0x00
 	binary.BigEndian.PutUint64(addr[2:], uint64(reflect.ValueOf(de).Pointer()))
-	de.fakeWGAddr = netaddr.IPPort{
-		IP:   netaddr.IPFrom16(addr),
-		Port: 12345,
-	}
+	de.fakeWGAddr = netaddr.IPPortFrom(netaddr.IPFrom16(addr), 12345)
 }
 
 // isFirstRecvActivityInAwhile notes that receive activity has occured for this
@@ -3393,6 +3393,7 @@ func (de *discoEndpoint) send(b []byte) error {
 	now := time.Now()
 
 	de.mu.Lock()
+	log.Println("Discosend")
 	udpAddr, derpAddr := de.addrForSendLocked(now)
 	if udpAddr.IsZero() || now.After(de.trustBestAddrUntil) {
 		de.sendPingsLocked(now, true)
@@ -3451,7 +3452,9 @@ func (de *discoEndpoint) removeSentPingLocked(txid stun.TxID, sp sentPing) {
 // The caller (startPingLocked) should've already been recorded the ping in
 // sentPing and set up the timer.
 func (de *discoEndpoint) sendDiscoPing(ep netaddr.IPPort, txid stun.TxID, logLevel discoLogLevel) {
+	log.Println("sendDiscoPing")
 	sent, _ := de.sendDiscoMessage(ep, &disco.Ping{TxID: [12]byte(txid)}, logLevel)
+	log.Println(sent)
 	if !sent {
 		de.forgetPing(txid)
 	}
@@ -3630,9 +3633,10 @@ func (de *discoEndpoint) noteConnectivityChange() {
 // It should be called with the Conn.mu held.
 func (de *discoEndpoint) handlePongConnLocked(m *disco.Pong, src netaddr.IPPort) {
 	de.mu.Lock()
+	log.Println("Disco Reached")
 	defer de.mu.Unlock()
 
-	isDerp := src.IP == derpMagicIPAddr
+	isDerp := src.IP() == derpMagicIPAddr
 
 	sp, ok := de.sentPing[m.TxID]
 	if !ok {
@@ -3708,13 +3712,13 @@ func betterAddr(a, b addrLatency) bool {
 	if a.IsZero() {
 		return false
 	}
-	if a.IP.Is6() && b.IP.Is4() {
+	if a.IP().Is6() && b.IP().Is4() {
 		// Prefer IPv6 for being a bit more robust, as long as
 		// the latencies are roughly equivalent.
 		if a.latency/10*9 < b.latency {
 			return true
 		}
-	} else if a.IP.Is4() && b.IP.Is6() {
+	} else if a.IP().Is4() && b.IP().Is6() {
 		if betterAddr(b, a) {
 			return false
 		}
@@ -3754,7 +3758,7 @@ func (de *discoEndpoint) handleCallMeMaybe(m *disco.CallMeMaybe) {
 	}
 	var newEPs []netaddr.IPPort
 	for _, ep := range m.MyNumber {
-		if ep.IP.Is6() && ep.IP.IsLinkLocalUnicast() {
+		if ep.IP().Is6() && ep.IP().IsLinkLocalUnicast() {
 			// We send these out, but ignore them for now.
 			// TODO: teach the ping code to ping on all interfaces
 			// for these.

wgengine/magicsock/magicsock_test.go

@@ -26,9 +26,9 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/tailscale/wireguard-go/device"
-	"github.com/tailscale/wireguard-go/tun/tuntest"
 	"golang.org/x/crypto/nacl/box"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun/tuntest"
 	"inet.af/netaddr"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
@@ -252,13 +252,13 @@ func meshStacks(logf logger.Logf, ms []*magicStack) (cleanup func()) {
 		nm := &netmap.NetworkMap{
 			PrivateKey: me.privateKey,
 			NodeKey:    tailcfg.NodeKey(me.privateKey.Public()),
-			Addresses:  []netaddr.IPPrefix{{IP: netaddr.IPv4(1, 0, 0, byte(myIdx+1)), Bits: 32}},
+			Addresses:  []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(1, 0, 0, byte(myIdx+1)), 32)},
 		}
 		for i, peer := range ms {
 			if i == myIdx {
 				continue
 			}
-			addrs := []netaddr.IPPrefix{{IP: netaddr.IPv4(1, 0, 0, byte(i+1)), Bits: 32}}
+			addrs := []netaddr.IPPrefix{netaddr.IPPrefixFrom(netaddr.IPv4(1, 0, 0, byte(i+1)), 32)}
 			peer := &tailcfg.Node{
 				ID:         tailcfg.NodeID(i + 1),
 				Name:       fmt.Sprintf("node%d", i+1),
@@ -433,7 +433,7 @@ func TestPickDERPFallback(t *testing.T) {
 	// But move if peers are elsewhere.
 	const otherNode = 789
 	c.addrsByKey = map[key.Public]*addrSet{
-		{1}: {ipPorts: []netaddr.IPPort{{IP: derpMagicIPAddr, Port: otherNode}}},
+		{1}: {ipPorts: []netaddr.IPPort{netaddr.IPPortFrom(derpMagicIPAddr, otherNode)}},
 	}
 	if got := c.pickDERPFallback(); got != otherNode {
 		t.Errorf("didn't join peers: got %v; want %v", got, someNode)
@@ -887,8 +887,8 @@ func testTwoDevicePing(t *testing.T, d *devices) {
 	defer m2.Close()
 
 	addrs := []netaddr.IPPort{
-		{IP: d.m1IP, Port: m1.conn.LocalPort()},
-		{IP: d.m2IP, Port: m2.conn.LocalPort()},
+		netaddr.IPPortFrom(d.m1IP, m1.conn.LocalPort()),
+		netaddr.IPPortFrom(d.m2IP, m2.conn.LocalPort()),
 	}
 	cfgs := makeConfigs(t, addrs)
 
@@ -1555,7 +1555,7 @@ func TestEndpointSetsEqual(t *testing.T) {
 	s := func(ports ...uint16) (ret []tailcfg.Endpoint) {
 		for _, port := range ports {
 			ret = append(ret, tailcfg.Endpoint{
-				Addr: netaddr.IPPort{Port: port},
+				Addr: netaddr.IPPortFrom(netaddr.IP{}, port),
 			})
 		}
 		return

wgengine/mem_ios.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package wgengine
+
+import (
+	"golang.zx2c4.com/wireguard/device"
+)
+
+// iOS has a very restrictive memory limit on network extensions.
+// Reduce the maximum amount of memory that wireguard-go can allocate
+// to avoid getting killed.
+
+func init() {
+	device.QueueStagedSize = 64
+	device.QueueOutboundSize = 64
+	device.QueueInboundSize = 64
+	device.QueueHandshakeSize = 64
+	device.PreallocatedBuffersPerPool = 64
+}

wgengine/monitor/monitor_linux.go

@@ -130,11 +130,11 @@ func netaddrIP(std net.IP) netaddr.IP {
 
 func netaddrIPPrefix(std net.IP, bits uint8) netaddr.IPPrefix {
 	ip, _ := netaddr.FromStdIP(std)
-	return netaddr.IPPrefix{IP: ip, Bits: bits}
+	return netaddr.IPPrefixFrom(ip, bits)
 }
 
 func condNetAddrPrefix(ipp netaddr.IPPrefix) string {
-	if ipp.IP.IsZero() {
+	if ipp.IP().IsZero() {
 		return ""
 	}
 	return ipp.String()
@@ -157,7 +157,7 @@ type newRouteMessage struct {
 const tsTable = 52
 
 func (m *newRouteMessage) ignore() bool {
-	return m.Table == tsTable || tsaddr.IsTailscaleIP(m.Dst.IP)
+	return m.Table == tsTable || tsaddr.IsTailscaleIP(m.Dst.IP())
 }
 
 // newAddrMessage is a message for a new address being added.

wgengine/netstack/netstack.go

@@ -48,6 +48,12 @@ const debugNetstack = false
 // and implements wgengine.FakeImpl to act as a userspace network
 // stack when Tailscale is running in fake mode.
 type Impl struct {
+	// ForwardTCPIn, if non-nil, handles forwarding an inbound TCP
+	// connection.
+	// TODO(bradfitz): provide mechanism for tsnet to reject a
+	// port other than accepting it and closing it.
+	ForwardTCPIn func(c net.Conn, port uint16)
+
 	ipstack     *stack.Stack
 	linkEP      *channel.Endpoint
 	tundev      *tstun.Wrapper
@@ -173,7 +179,7 @@ func DNSMapFromNetworkMap(nm *netmap.NetworkMap) DNSMap {
 	suffix := nm.MagicDNSSuffix()
 
 	if nm.Name != "" && len(nm.Addresses) > 0 {
-		ip := nm.Addresses[0].IP
+		ip := nm.Addresses[0].IP()
 		ret[strings.TrimRight(nm.Name, ".")] = ip
 		if dnsname.HasSuffix(nm.Name, suffix) {
 			ret[dnsname.TrimSuffix(nm.Name, suffix)] = ip
@@ -181,7 +187,7 @@ func DNSMapFromNetworkMap(nm *netmap.NetworkMap) DNSMap {
 	}
 	for _, p := range nm.Peers {
 		if p.Name != "" && len(p.Addresses) > 0 {
-			ip := p.Addresses[0].IP
+			ip := p.Addresses[0].IP()
 			ret[strings.TrimRight(p.Name, ".")] = ip
 			if dnsname.HasSuffix(p.Name, suffix) {
 				ret[dnsname.TrimSuffix(p.Name, suffix)] = ip
@@ -221,8 +227,8 @@ func (ns *Impl) removeSubnetAddress(ip netaddr.IP) {
 
 func ipPrefixToAddressWithPrefix(ipp netaddr.IPPrefix) tcpip.AddressWithPrefix {
 	return tcpip.AddressWithPrefix{
-		Address:   tcpip.Address(ipp.IP.IPAddr().IP),
-		PrefixLen: int(ipp.Bits),
+		Address:   tcpip.Address(ipp.IP().IPAddr().IP),
+		PrefixLen: int(ipp.Bits()),
 	}
 }
 
@@ -316,7 +322,7 @@ func (m DNSMap) Resolve(ctx context.Context, addr string) (netaddr.IPPort, error
 	// Try MagicDNS first, else otherwise a real DNS lookup.
 	ip := m[host]
 	if !ip.IsZero() {
-		return netaddr.IPPort{IP: ip, Port: uint16(port16)}, nil
+		return netaddr.IPPortFrom(ip, uint16(port16)), nil
 	}
 
 	// No MagicDNS name so try real DNS.
@@ -329,7 +335,7 @@ func (m DNSMap) Resolve(ctx context.Context, addr string) (netaddr.IPPort, error
 		return netaddr.IPPort{}, fmt.Errorf("DNS lookup returned no results for %q", host)
 	}
 	ip, _ = netaddr.FromStdIP(ips[0])
-	return netaddr.IPPort{IP: ip, Port: uint16(port16)}, nil
+	return netaddr.IPPortFrom(ip, uint16(port16)), nil
 }
 
 func (ns *Impl) DialContextTCP(ctx context.Context, addr string) (*gonet.TCPConn, error) {
@@ -343,11 +349,11 @@ func (ns *Impl) DialContextTCP(ctx context.Context, addr string) (*gonet.TCPConn
 	}
 	remoteAddress := tcpip.FullAddress{
 		NIC:  nicID,
-		Addr: tcpip.Address(remoteIPPort.IP.IPAddr().IP),
-		Port: remoteIPPort.Port,
+		Addr: tcpip.Address(remoteIPPort.IP().IPAddr().IP),
+		Port: remoteIPPort.Port(),
 	}
 	var ipType tcpip.NetworkProtocolNumber
-	if remoteIPPort.IP.Is4() {
+	if remoteIPPort.IP().Is4() {
 		ipType = ipv4.ProtocolNumber
 	} else {
 		ipType = ipv6.ProtocolNumber
@@ -389,7 +395,7 @@ func (ns *Impl) isLocalIP(ip netaddr.IP) bool {
 }
 
 func (ns *Impl) injectInbound(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
-	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP) {
+	if ns.onlySubnets && ns.isLocalIP(p.Dst.IP()) {
 		// In hybrid ("only subnets") mode, bail out early if
 		// the traffic is destined for an actual Tailscale
 		// address. The real host OS interface will handle it.
@@ -441,11 +447,15 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) {
 		r.Complete(true)
 		return
 	}
+	r.Complete(false)
+	c := gonet.NewTCPConn(&wq, ep)
+	if ns.ForwardTCPIn != nil {
+		ns.ForwardTCPIn(c, reqDetails.LocalPort)
+		return
+	}
 	if isTailscaleIP {
 		dialAddr = tcpip.Address(net.ParseIP("127.0.0.1")).To4()
 	}
-	r.Complete(false)
-	c := gonet.NewTCPConn(&wq, ep)
 	ns.forwardTCP(c, &wq, dialAddr, reqDetails.LocalPort)
 }
 

wgengine/pendopen.go

@@ -115,8 +115,8 @@ func (e *userspaceEngine) trackOpenPostFilterOut(pp *packet.Parsed, t *tstun.Wra
 	// Don't start timers tracking those. They won't succeed anyway. Avoids log spam
 	// like:
 	//    open-conn-track: timeout opening (100.115.73.60:52501 => 17.125.252.5:443); no associated peer node
-	if runtime.GOOS == "ios" && flow.Dst.Port == 443 && !tsaddr.IsTailscaleIP(flow.Dst.IP) {
-		if _, err := e.peerForIP(flow.Dst.IP); err != nil {
+	if runtime.GOOS == "ios" && flow.Dst.Port() == 443 && !tsaddr.IsTailscaleIP(flow.Dst.IP()) {
+		if _, err := e.peerForIP(flow.Dst.IP()); err != nil {
 			return
 		}
 	}
@@ -156,7 +156,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	}
 
 	// Diagnose why it might've timed out.
-	n, err := e.peerForIP(flow.Dst.IP)
+	n, err := e.peerForIP(flow.Dst.IP())
 	if err != nil {
 		e.logf("open-conn-track: timeout opening %v; peerForIP: %v", flow, err)
 		return
@@ -193,7 +193,7 @@ func (e *userspaceEngine) onOpenTimeout(flow flowtrack.Tuple) {
 	if ps == nil {
 		onlyZeroRoute := true // whether peerForIP returned n only because its /0 route matched
 		for _, r := range n.AllowedIPs {
-			if r.Bits != 0 && r.Contains(flow.Dst.IP) {
+			if r.Bits() != 0 && r.Contains(flow.Dst.IP()) {
 				onlyZeroRoute = false
 				break
 			}

wgengine/router/ifconfig_windows.go

@@ -17,8 +17,8 @@ import (
 
 	"github.com/go-multierror/multierror"
 	ole "github.com/go-ole/go-ole"
-	"github.com/tailscale/wireguard-go/tun"
 	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"inet.af/netaddr"
 	"tailscale.com/health"
@@ -324,16 +324,16 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 	var firstGateway6 *net.IP
 	addresses := make([]*net.IPNet, 0, len(cfg.LocalAddrs))
 	for _, addr := range cfg.LocalAddrs {
-		if (addr.IP.Is4() && ipif4 == nil) || (addr.IP.Is6() && ipif6 == nil) {
+		if (addr.IP().Is4() && ipif4 == nil) || (addr.IP().Is6() && ipif6 == nil) {
 			// Can't program addresses for disabled protocol.
 			continue
 		}
 		ipnet := addr.IPNet()
 		addresses = append(addresses, ipnet)
 		gateway := ipnet.IP
-		if addr.IP.Is4() && firstGateway4 == nil {
+		if addr.IP().Is4() && firstGateway4 == nil {
 			firstGateway4 = &gateway
-		} else if addr.IP.Is6() && firstGateway6 == nil {
+		} else if addr.IP().Is6() && firstGateway6 == nil {
 			firstGateway6 = &gateway
 		}
 	}
@@ -342,12 +342,12 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 	foundDefault4 := false
 	foundDefault6 := false
 	for _, route := range cfg.Routes {
-		if (route.IP.Is4() && ipif4 == nil) || (route.IP.Is6() && ipif6 == nil) {
+		if (route.IP().Is4() && ipif4 == nil) || (route.IP().Is6() && ipif6 == nil) {
 			// Can't program routes for disabled protocol.
 			continue
 		}
 
-		if route.IP.Is6() && firstGateway6 == nil {
+		if route.IP().Is6() && firstGateway6 == nil {
 			// Windows won't let us set IPv6 routes without having an
 			// IPv6 local address set. However, when we've configured
 			// a default route, we want to forcibly grab IPv6 traffic
@@ -357,16 +357,16 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 			ipnet := &net.IPNet{tsaddr.Tailscale4To6Placeholder().IPAddr().IP, net.CIDRMask(128, 128)}
 			addresses = append(addresses, ipnet)
 			firstGateway6 = &ipnet.IP
-		} else if route.IP.Is4() && firstGateway4 == nil {
+		} else if route.IP().Is4() && firstGateway4 == nil {
 			// TODO: do same dummy behavior as v6?
 			return errors.New("due to a Windows limitation, one cannot have interface routes without an interface address")
 		}
 
 		ipn := route.IPNet()
 		var gateway net.IP
-		if route.IP.Is4() {
+		if route.IP().Is4() {
 			gateway = *firstGateway4
-		} else if route.IP.Is6() {
+		} else if route.IP().Is6() {
 			gateway = *firstGateway6
 		}
 		r := winipcfg.RouteData{
@@ -385,13 +385,13 @@ func configureInterface(cfg *Config, tun *tun.NativeTun) (retErr error) {
 			// then the interface's IP won't be pingable.
 			continue
 		}
-		if route.IP.Is4() {
-			if route.Bits == 0 {
+		if route.IP().Is4() {
+			if route.Bits() == 0 {
 				foundDefault4 = true
 			}
 			r.NextHop = *firstGateway4
-		} else if route.IP.Is6() {
-			if route.Bits == 0 {
+		} else if route.IP().Is6() {
+			if route.Bits() == 0 {
 				foundDefault6 = true
 			}
 			r.NextHop = *firstGateway6
@@ -760,11 +760,8 @@ func filterRoutes(routes []*winipcfg.RouteData, dontDelete []netaddr.IPPrefix) [
 		if nr.IsSingleIP() {
 			continue
 		}
-		lastIP := nr.Range().To
-		ddm[netaddr.IPPrefix{
-			IP:   lastIP,
-			Bits: lastIP.BitLen(),
-		}] = true
+		lastIP := nr.Range().To()
+		ddm[netaddr.IPPrefixFrom(lastIP, lastIP.BitLen())] = true
 	}
 	filtered := make([]*winipcfg.RouteData, 0, len(routes))
 	for _, r := range routes {

wgengine/router/router.go

@@ -7,7 +7,7 @@
 package router
 
 import (
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/types/logger"
 	"tailscale.com/types/preftype"

wgengine/router/router_darwin.go

@@ -5,7 +5,7 @@
 package router
 
 import (
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 )
 

wgengine/router/router_default.go

@@ -7,7 +7,7 @@
 package router
 
 import (
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 )
 

wgengine/router/router_freebsd.go

@@ -5,7 +5,7 @@
 package router
 
 import (
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/types/logger"
 )
 

wgengine/router/router_linux.go

@@ -16,7 +16,7 @@ import (
 
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/go-multierror/multierror"
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
@@ -360,7 +360,7 @@ func (r *linuxRouter) setNetfilterMode(mode preftype.NetfilterMode) error {
 	}
 
 	for cidr := range r.addrs {
-		if err := r.addLoopbackRule(cidr.IP); err != nil {
+		if err := r.addLoopbackRule(cidr.IP()); err != nil {
 			return err
 		}
 	}
@@ -372,13 +372,13 @@ func (r *linuxRouter) setNetfilterMode(mode preftype.NetfilterMode) error {
 // address is already assigned to the interface, or if the addition
 // fails.
 func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error {
-	if !r.v6Available && addr.IP.Is6() {
+	if !r.v6Available && addr.IP().Is6() {
 		return nil
 	}
 	if err := r.cmd.run("ip", "addr", "add", addr.String(), "dev", r.tunname); err != nil {
 		return fmt.Errorf("adding address %q to tunnel interface: %w", addr, err)
 	}
-	if err := r.addLoopbackRule(addr.IP); err != nil {
+	if err := r.addLoopbackRule(addr.IP()); err != nil {
 		return err
 	}
 	return nil
@@ -388,10 +388,10 @@ func (r *linuxRouter) addAddress(addr netaddr.IPPrefix) error {
 // the address is not assigned to the interface, or if the removal
 // fails.
 func (r *linuxRouter) delAddress(addr netaddr.IPPrefix) error {
-	if !r.v6Available && addr.IP.Is6() {
+	if !r.v6Available && addr.IP().Is6() {
 		return nil
 	}
-	if err := r.delLoopbackRule(addr.IP); err != nil {
+	if err := r.delLoopbackRule(addr.IP()); err != nil {
 		return err
 	}
 	if err := r.cmd.run("ip", "addr", "del", addr.String(), "dev", r.tunname); err != nil {
@@ -463,7 +463,7 @@ func (r *linuxRouter) addThrowRoute(cidr netaddr.IPPrefix) error {
 }
 
 func (r *linuxRouter) addRouteDef(routeDef []string, cidr netaddr.IPPrefix) error {
-	if !r.v6Available && cidr.IP.Is6() {
+	if !r.v6Available && cidr.IP().Is6() {
 		return nil
 	}
 	args := append([]string{"ip", "route", "add"}, routeDef...)
@@ -490,7 +490,7 @@ func (r *linuxRouter) delThrowRoute(cidr netaddr.IPPrefix) error {
 }
 
 func (r *linuxRouter) delRouteDef(routeDef []string, cidr netaddr.IPPrefix) error {
-	if !r.v6Available && cidr.IP.Is6() {
+	if !r.v6Available && cidr.IP().Is6() {
 		return nil
 	}
 	args := append([]string{"ip", "route", "del"}, routeDef...)
@@ -520,7 +520,7 @@ func dashFam(ip netaddr.IP) string {
 }
 
 func (r *linuxRouter) hasRoute(routeDef []string, cidr netaddr.IPPrefix) (bool, error) {
-	args := append([]string{"ip", dashFam(cidr.IP), "route", "show"}, routeDef...)
+	args := append([]string{"ip", dashFam(cidr.IP()), "route", "show"}, routeDef...)
 	if r.ipRuleAvailable {
 		args = append(args, "table", tailscaleRouteTable)
 	}

wgengine/router/router_linux_test.go

@@ -16,7 +16,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 )
 

wgengine/router/router_openbsd.go

@@ -10,7 +10,7 @@ import (
 	"log"
 	"os/exec"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/types/logger"
 )
@@ -69,11 +69,11 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 	localAddr4 := netaddr.IPPrefix{}
 	localAddr6 := netaddr.IPPrefix{}
 	for _, addr := range cfg.LocalAddrs {
-		if addr.IP.Is4() {
+		if addr.IP().Is4() {
 			numIPv4++
 			localAddr4 = addr
 		}
-		if addr.IP.Is6() {
+		if addr.IP().Is6() {
 			numIPv6++
 			localAddr6 = addr
 		}
@@ -98,7 +98,7 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 
 			routedel := []string{"route", "-q", "-n",
 				"del", "-inet", r.local4.String(),
-				"-iface", r.local4.IP.String()}
+				"-iface", r.local4.IP().String()}
 			if out, err := cmd(routedel...).CombinedOutput(); err != nil {
 				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
 				if errq == nil {
@@ -120,7 +120,7 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 
 			routeadd := []string{"route", "-q", "-n",
 				"add", "-inet", localAddr4.String(),
-				"-iface", localAddr4.IP.String()}
+				"-iface", localAddr4.IP().String()}
 			if out, err := cmd(routeadd...).CombinedOutput(); err != nil {
 				r.logf("route add failed: %v: %v\n%s", routeadd, err, out)
 				if errq == nil {
@@ -134,7 +134,7 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 		// in https://github.com/tailscale/tailscale/issues/1307 we made
 		// FreeBSD use a /48 for IPv6 addresses, which is nice because we
 		// don't need to additionally add routing entries. Do that here too.
-		localAddr6 = netaddr.IPPrefix{localAddr6.IP, 48}
+		localAddr6 = netaddr.IPPrefixFrom(localAddr6.IP(), 48)
 	}
 
 	if localAddr6 != r.local6 {
@@ -171,10 +171,10 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 		if _, keep := newRoutes[route]; !keep {
 			net := route.IPNet()
 			nip := net.IP.Mask(net.Mask)
-			nstr := fmt.Sprintf("%v/%d", nip, route.Bits)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Bits())
 			routedel := []string{"route", "-q", "-n",
 				"del", "-inet", nstr,
-				"-iface", localAddr4.IP.String()}
+				"-iface", localAddr4.IP().String()}
 			out, err := cmd(routedel...).CombinedOutput()
 			if err != nil {
 				r.logf("route del failed: %v: %v\n%s", routedel, err, out)
@@ -188,10 +188,10 @@ func (r *openbsdRouter) Set(cfg *Config) error {
 		if _, exists := r.routes[route]; !exists {
 			net := route.IPNet()
 			nip := net.IP.Mask(net.Mask)
-			nstr := fmt.Sprintf("%v/%d", nip, route.Bits)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Bits())
 			routeadd := []string{"route", "-q", "-n",
 				"add", "-inet", nstr,
-				"-iface", localAddr4.IP.String()}
+				"-iface", localAddr4.IP().String()}
 			out, err := cmd(routeadd...).CombinedOutput()
 			if err != nil {
 				r.logf("addr add failed: %v: %v\n%s", routeadd, err, out)

wgengine/router/router_userspace_bsd.go

@@ -12,7 +12,7 @@ import (
 	"os/exec"
 	"runtime"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/types/logger"
@@ -87,7 +87,7 @@ func (r *userspaceBSDRouter) Up() error {
 }
 
 func inet(p netaddr.IPPrefix) string {
-	if p.IP.Is6() {
+	if p.IP().Is6() {
 		return "inet6"
 	}
 	return "inet"
@@ -116,15 +116,15 @@ func (r *userspaceBSDRouter) Set(cfg *Config) (reterr error) {
 	}
 	for _, addr := range r.addrsToAdd(cfg.LocalAddrs) {
 		var arg []string
-		if runtime.GOOS == "freebsd" && addr.IP.Is6() && addr.Bits == 128 {
+		if runtime.GOOS == "freebsd" && addr.IP().Is6() && addr.Bits() == 128 {
 			// FreeBSD rejects tun addresses of the form fc00::1/128 -> fc00::1,
 			// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=218508
 			// Instead add our whole /48, which works because we use a /48 route.
 			// Full history: https://github.com/tailscale/tailscale/issues/1307
-			tmp := netaddr.IPPrefix{IP: addr.IP, Bits: 48}
+			tmp := netaddr.IPPrefixFrom(addr.IP(), 48)
 			arg = []string{"ifconfig", r.tunname, inet(tmp), tmp.String()}
 		} else {
-			arg = []string{"ifconfig", r.tunname, inet(addr), addr.String(), addr.IP.String()}
+			arg = []string{"ifconfig", r.tunname, inet(addr), addr.String(), addr.IP().String()}
 		}
 		out, err := cmd(arg...).CombinedOutput()
 		if err != nil {
@@ -148,7 +148,7 @@ func (r *userspaceBSDRouter) Set(cfg *Config) (reterr error) {
 		if _, keep := newRoutes[route]; !keep {
 			net := route.IPNet()
 			nip := net.IP.Mask(net.Mask)
-			nstr := fmt.Sprintf("%v/%d", nip, route.Bits)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Bits())
 			del := "del"
 			if version.OS() == "macOS" {
 				del = "delete"
@@ -168,7 +168,7 @@ func (r *userspaceBSDRouter) Set(cfg *Config) (reterr error) {
 		if _, exists := r.routes[route]; !exists {
 			net := route.IPNet()
 			nip := net.IP.Mask(net.Mask)
-			nstr := fmt.Sprintf("%v/%d", nip, route.Bits)
+			nstr := fmt.Sprintf("%v/%d", nip, route.Bits())
 			routeadd := []string{"route", "-q", "-n",
 				"add", "-" + inet(route), nstr,
 				"-iface", r.tunname}

wgengine/router/router_windows.go

@@ -16,8 +16,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/tailscale/wireguard-go/tun"
 	"golang.org/x/sys/windows"
+	"golang.zx2c4.com/wireguard/tun"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 	"inet.af/netaddr"
 	"tailscale.com/logtail/backoff"
@@ -91,7 +91,7 @@ func (r *winRouter) Set(cfg *Config) error {
 
 func hasDefaultRoute(routes []netaddr.IPPrefix) bool {
 	for _, route := range routes {
-		if route.Bits == 0 {
+		if route.Bits() == 0 {
 			return true
 		}
 	}

wgengine/userspace.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"reflect"
 	"runtime"
@@ -20,9 +21,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/tailscale/wireguard-go/device"
-	"github.com/tailscale/wireguard-go/tun"
 	"go4.org/mem"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/control/controlclient"
 	"tailscale.com/health"
@@ -307,6 +308,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 
 	e.wgLogger = wglog.NewLogger(logf)
 	e.tundev.OnTSMPPongReceived = func(pong packet.TSMPPongReply) {
+		log.Println("PONGReceived")
 		e.mu.Lock()
 		defer e.mu.Unlock()
 		cb := e.pongCallback[pong.Data]
@@ -369,6 +371,7 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 
 // echoRespondToAll is an inbound post-filter responding to all echo requests.
 func echoRespondToAll(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
+	log.Println("ECHO respond to all")
 	if p.IsEchoRequest() {
 		header := p.ICMP4Header()
 		header.ToResponse()
@@ -399,7 +402,7 @@ func (e *userspaceEngine) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper)
 		isLocalAddr, ok := e.isLocalAddr.Load().(func(netaddr.IP) bool)
 		if !ok {
 			e.logf("[unexpected] e.isLocalAddr was nil, can't check for loopback packet")
-		} else if isLocalAddr(p.Dst.IP) {
+		} else if isLocalAddr(p.Dst.IP()) {
 			// macOS NetworkExtension directs packets destined to the
 			// tunnel's local IP address into the tunnel, instead of
 			// looping back within the kernel network stack. We have to
@@ -415,7 +418,7 @@ func (e *userspaceEngine) handleLocalPackets(p *packet.Parsed, t *tstun.Wrapper)
 
 // handleDNS is an outbound pre-filter resolving Tailscale domains.
 func (e *userspaceEngine) handleDNS(p *packet.Parsed, t *tstun.Wrapper) filter.Response {
-	if p.Dst.IP == magicDNSIP && p.Dst.Port == magicDNSPort && p.IPProto == ipproto.UDP {
+	if p.Dst.IP() == magicDNSIP && p.Dst.Port() == magicDNSPort && p.IPProto == ipproto.UDP {
 		err := e.dns.EnqueueRequest(append([]byte(nil), p.Payload()...), p.Src)
 		if err != nil {
 			e.logf("dns: enqueue: %v", err)
@@ -440,10 +443,10 @@ func (e *userspaceEngine) pollResolver() {
 		h := packet.UDP4Header{
 			IP4Header: packet.IP4Header{
 				Src: magicDNSIP,
-				Dst: to.IP,
+				Dst: to.IP(),
 			},
 			SrcPort: magicDNSPort,
-			DstPort: to.Port,
+			DstPort: to.Port(),
 		}
 		hlen := h.Len()
 
@@ -620,8 +623,8 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.Publ
 		trackDisco = append(trackDisco, dk)
 		recentlyActive := false
 		for _, cidr := range p.AllowedIPs {
-			trackIPs = append(trackIPs, cidr.IP)
-			recentlyActive = recentlyActive || e.isActiveSince(dk, cidr.IP, activeCutoff)
+			trackIPs = append(trackIPs, cidr.IP())
+			recentlyActive = recentlyActive || e.isActiveSince(dk, cidr.IP(), activeCutoff)
 		}
 		if recentlyActive {
 			min.Peers = append(min.Peers, *p)
@@ -633,7 +636,7 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked(discoChanged map[key.Publ
 		}
 	}
 
-	if !deephash.UpdateHash(&e.lastEngineSigTrim, min, trimmedDisco, trackDisco, trackIPs) {
+	if !deephash.UpdateHash(&e.lastEngineSigTrim, &min, trimmedDisco, trackDisco, trackIPs) {
 		// No changes
 		return nil
 	}
@@ -1087,6 +1090,7 @@ func (e *userspaceEngine) SetDERPMap(dm *tailcfg.DERPMap) {
 }
 
 func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) {
+	log.Println("SETNETMAP")
 	e.magicConn.SetNetworkMap(nm)
 	e.mu.Lock()
 	e.netMap = nm
@@ -1123,15 +1127,18 @@ func (e *userspaceEngine) UpdateStatus(sb *ipnstate.StatusBuilder) {
 }
 
 func (e *userspaceEngine) Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.PingResult)) {
+	log.Println("Userspace Ping Called")
 	res := &ipnstate.PingResult{IP: ip.String()}
 	peer, err := e.peerForIP(ip)
 	if err != nil {
+		log.Println(err)
 		e.logf("ping(%v): %v", ip, err)
 		res.Err = err.Error()
 		cb(res)
 		return
 	}
 	if peer == nil {
+		log.Println("No peer moment")
 		e.logf("ping(%v): no matching peer", ip)
 		res.Err = "no matching peer"
 		cb(res)
@@ -1139,6 +1146,7 @@ func (e *userspaceEngine) Ping(ip netaddr.IP, useTSMP bool, cb func(*ipnstate.Pi
 	}
 	pingType := "disco"
 	if useTSMP {
+		log.Println("TSMPSELECTED")
 		pingType = "TSMP"
 	}
 	e.logf("ping(%v): sending %v ping to %v %v ...", ip, pingType, peer.Key.ShortString(), peer.ComputedName)
@@ -1156,8 +1164,8 @@ func (e *userspaceEngine) mySelfIPMatchingFamily(dst netaddr.IP) (src netaddr.IP
 		return netaddr.IP{}, errors.New("no netmap")
 	}
 	for _, a := range e.netMap.Addresses {
-		if a.IsSingleIP() && a.IP.BitLen() == dst.BitLen() {
-			return a.IP, nil
+		if a.IsSingleIP() && a.IP().BitLen() == dst.BitLen() {
+			return a.IP(), nil
 		}
 	}
 	if len(e.netMap.Addresses) == 0 {
@@ -1167,8 +1175,10 @@ func (e *userspaceEngine) mySelfIPMatchingFamily(dst netaddr.IP) (src netaddr.IP
 }
 
 func (e *userspaceEngine) sendTSMPPing(ip netaddr.IP, peer *tailcfg.Node, res *ipnstate.PingResult, cb func(*ipnstate.PingResult)) {
+	log.Println("TSMPcheck")
 	srcIP, err := e.mySelfIPMatchingFamily(ip)
 	if err != nil {
+		log.Println("TSMPcheckerror")
 		res.Err = err.Error()
 		cb(res)
 		return
@@ -1190,12 +1200,17 @@ func (e *userspaceEngine) sendTSMPPing(ip netaddr.IP, peer *tailcfg.Node, res *i
 
 	var data [8]byte
 	crand.Read(data[:])
+	log.Println("CRAND CHECKlog")
+	fmt.Println("CRAND CHECKfmt")
 
 	expireTimer := time.AfterFunc(10*time.Second, func() {
+		log.Println("CHECKEXPIRE")
 		e.setTSMPPongCallback(data, nil)
 	})
+	log.Println("TIMECHECK")
 	t0 := time.Now()
 	e.setTSMPPongCallback(data, func(pong packet.TSMPPongReply) {
+		log.Println("ping cb called")
 		expireTimer.Stop()
 		d := time.Since(t0)
 		res.LatencySeconds = d.Seconds()
@@ -1208,22 +1223,32 @@ func (e *userspaceEngine) sendTSMPPing(ip netaddr.IP, peer *tailcfg.Node, res *i
 	var tsmpPayload [9]byte
 	tsmpPayload[0] = byte(packet.TSMPTypePing)
 	copy(tsmpPayload[1:], data[:])
+	log.Println("PAYLOADCHECK")
 
 	tsmpPing := packet.Generate(iph, tsmpPayload[:])
+	log.Println("BEFOREPACKET", tsmpPing)
+	log.Println("PACKETGEN", *res, res.LatencySeconds)
 	e.tundev.InjectOutbound(tsmpPing)
+	log.Println("TUNDEVINJECT")
 }
 
 func (e *userspaceEngine) setTSMPPongCallback(data [8]byte, cb func(packet.TSMPPongReply)) {
+	log.Println("Ponger2nolock", data)
 	e.mu.Lock()
+	log.Println("Ponger2", e.pongCallback == nil, cb == nil)
 	defer e.mu.Unlock()
 	if e.pongCallback == nil {
+		log.Println("pongCallback nil")
 		e.pongCallback = map[[8]byte]func(packet.TSMPPongReply){}
 	}
 	if cb == nil {
+		log.Println("DELETEoccur")
 		delete(e.pongCallback, data)
 	} else {
+		log.Println("Callbackset")
 		e.pongCallback[data] = cb
 	}
+	log.Println("PONGCALLBACKMAP", data, e.pongCallback)
 }
 
 func (e *userspaceEngine) RegisterIPPortIdentity(ipport netaddr.IPPort, tsIP netaddr.IP) {
@@ -1291,17 +1316,25 @@ func (e *userspaceEngine) peerForIP(ip netaddr.IP) (n *tailcfg.Node, err error)
 	// Check for exact matches before looking for subnet matches.
 	var bestInNMPrefix netaddr.IPPrefix
 	var bestInNM *tailcfg.Node
+	log.Println("Scan starting : ", len(nm.Peers), nm.Addresses)
 	for _, p := range nm.Peers {
+		log.Println("peerp", p.Addresses, p.AllowedIPs, p.ID)
 		for _, a := range p.Addresses {
-			if a.IP == ip && a.IsSingleIP() && tsaddr.IsTailscaleIP(ip) {
+			log.Println("paddr", a)
+			if a.IP() == ip && a.IsSingleIP() && tsaddr.IsTailscaleIP(ip) {
+				log.Println("Foundp")
 				return p, nil
+			} else {
+				// log.Println("Failure : ", a.IP(), a.IsSingleIP(), tsaddr.IsTailscaleIP(ip))
 			}
 		}
+		log.Println("ALLOW : ", p.AllowedIPs)
+		bestInNM = p
 		for _, cidr := range p.AllowedIPs {
 			if !cidr.Contains(ip) {
 				continue
 			}
-			if bestInNMPrefix.IsZero() || cidr.Bits > bestInNMPrefix.Bits {
+			if bestInNMPrefix.IsZero() || cidr.Bits() > bestInNMPrefix.Bits() {
 				bestInNMPrefix = cidr
 				bestInNM = p
 			}
@@ -1310,6 +1343,7 @@ func (e *userspaceEngine) peerForIP(ip netaddr.IP) (n *tailcfg.Node, err error)
 
 	e.wgLock.Lock()
 	defer e.wgLock.Unlock()
+	log.Println("Scanpoint2")
 
 	// TODO(bradfitz): this is O(n peers). Add ART to netaddr?
 	var best netaddr.IPPrefix
@@ -1319,7 +1353,7 @@ func (e *userspaceEngine) peerForIP(ip netaddr.IP) (n *tailcfg.Node, err error)
 			if !cidr.Contains(ip) {
 				continue
 			}
-			if best.IsZero() || cidr.Bits > best.Bits {
+			if best.IsZero() || cidr.Bits() > best.Bits() {
 				best = cidr
 				bestKey = tailcfg.NodeKey(p.PublicKey)
 			}
@@ -1334,10 +1368,13 @@ func (e *userspaceEngine) peerForIP(ip netaddr.IP) (n *tailcfg.Node, err error)
 			}
 		}
 	}
+	log.Println("Scanpoint3")
 	if bestInNM == nil {
+		log.Println("Scanpoint4")
 		return nil, nil
 	}
-	if bestInNMPrefix.Bits == 0 {
+	if bestInNMPrefix.Bits() == 0 {
+		log.Println("Scanpoint5")
 		return nil, errors.New("exit node found but not enabled")
 	}
 	return nil, fmt.Errorf("node %q found, but not using its %v route", bestInNM.ComputedNameWithHost, bestInNMPrefix)

wgengine/userspace_ext_test.go

@@ -7,7 +7,7 @@ package wgengine_test
 import (
 	"testing"
 
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/tun"
 	"tailscale.com/net/tstun"
 	"tailscale.com/types/logger"
 	"tailscale.com/wgengine"

wgengine/userspace_test.go

@@ -102,7 +102,7 @@ func TestUserspaceEngineReconfig(t *testing.T) {
 			Peers: []wgcfg.Peer{
 				{
 					AllowedIPs: []netaddr.IPPrefix{
-						{IP: netaddr.IPv4(100, 100, 99, 1), Bits: 32},
+						netaddr.IPPrefixFrom(netaddr.IPv4(100, 100, 99, 1), 32),
 					},
 					Endpoints: wgcfg.Endpoints{DiscoKey: dkFromHex(discoHex)},
 				},

wgengine/wgcfg/device.go

@@ -8,7 +8,7 @@ import (
 	"io"
 	"sort"
 
-	"github.com/tailscale/wireguard-go/device"
+	"golang.zx2c4.com/wireguard/device"
 	"tailscale.com/types/logger"
 )
 

wgengine/wgcfg/device_test.go

@@ -16,9 +16,9 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/tailscale/wireguard-go/conn"
-	"github.com/tailscale/wireguard-go/device"
-	"github.com/tailscale/wireguard-go/tun"
+	"golang.zx2c4.com/wireguard/conn"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 	"inet.af/netaddr"
 	"tailscale.com/types/wgkey"
 )

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -37,7 +37,7 @@ func nodeDebugName(n *tailcfg.Node) string {
 // cidrIsSubnet reports whether cidr is a non-default-route subnet
 // exported by node that is not one of its own self addresses.
 func cidrIsSubnet(node *tailcfg.Node, cidr netaddr.IPPrefix) bool {
-	if cidr.Bits == 0 {
+	if cidr.Bits() == 0 {
 		return false
 	}
 	if !cidr.IsSingleIP() {
@@ -93,7 +93,7 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 		}
 		didExitNodeWarn := false
 		for _, allowedIP := range peer.AllowedIPs {
-			if allowedIP.Bits == 0 && peer.StableID != exitNode {
+			if allowedIP.Bits() == 0 && peer.StableID != exitNode {
 				if didExitNodeWarn {
 					// Don't log about both the IPv4 /0 and IPv6 /0.
 					continue
@@ -104,11 +104,11 @@ func WGCfg(nm *netmap.NetworkMap, logf logger.Logf, flags netmap.WGConfigFlags,
 				}
 				fmt.Fprintf(skippedUnselected, "%q (%v)", nodeDebugName(peer), peer.Key.ShortString())
 				continue
-			} else if allowedIP.IsSingleIP() && tsaddr.IsTailscaleIP(allowedIP.IP) && (flags&netmap.AllowSingleHosts) == 0 {
+			} else if allowedIP.IsSingleIP() && tsaddr.IsTailscaleIP(allowedIP.IP()) && (flags&netmap.AllowSingleHosts) == 0 {
 				if skippedIPs.Len() > 0 {
 					skippedIPs.WriteString(", ")
 				}
-				fmt.Fprintf(skippedIPs, "%v from %q (%v)", allowedIP.IP, nodeDebugName(peer), peer.Key.ShortString())
+				fmt.Fprintf(skippedIPs, "%v from %q (%v)", allowedIP.IP(), nodeDebugName(peer), peer.Key.ShortString())
 				continue
 			} else if cidrIsSubnet(peer, allowedIP) {
 				if (flags & netmap.AllowSubnetRoutes) == 0 {