normal control is working with hardcoded mapresponses

Signed-off-by: Simeng He <simeng@tailscale.com>

.github/workflows/xe-experimental-vm-test.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@v1
 
       - name: Download VM Images
-        run: go test ./tstest/integration/vms -run-vm-tests -run=Download -timeout=60m
+        run: go test ./tstest/integration/vms -run-vm-tests -run=Download -timeout=60m -no-s3
         env:
           XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
 

build_docker.sh

@@ -25,7 +25,7 @@
 
 set -eu
 
-eval $(./version/version.sh)
+eval $(./build_dist.sh shellvars)
 
 docker build \
   --build-arg VERSION_LONG=$VERSION_LONG \

cmd/tailscale/cli/web.go

@@ -60,6 +60,14 @@ var webCmd = &ffcli.Command{
 	ShortUsage: "web [flags]",
 	ShortHelp:  "Run a web server for controlling Tailscale",
 
+	LongHelp: strings.TrimSpace(`
+"tailscale web" runs a webserver for controlling the Tailscale daemon.
+
+It's primarily intended for use on Synology, QNAP, and other
+NAS devices where a web interface is the natural place to control
+Tailscale, as opposed to a CLI or a native app.
+`),
+
 	FlagSet: (func() *flag.FlagSet {
 		webf := flag.NewFlagSet("web", flag.ExitOnError)
 		webf.StringVar(&webArgs.listen, "listen", "localhost:8088", "listen address; use port 0 for automatic")

control/controlclient/direct.go

@@ -777,8 +777,25 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			health.GotStreamedMapResponse()
 		}
 
-		if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+		fmt.Println("Before Ping", resp.PingRequest, c.isUniquePingRequest(resp.PingRequest))
+		fmt.Println("Peers :", resp.Peers, resp.PeersChanged, resp.PeersRemoved)
+		netmap := sess.netmapForResponse(&resp)
+		fmt.Println("Early Netmap : ", netmap.Peers)
+		if len(resp.PeersChanged) > 0 {
+			fmt.Printf("PEER INFO: %+v\n", resp.PeersChanged[0])
+		}
+		// if pr := resp.PingRequest; pr != nil && c.isUniquePingRequest(pr) {
+		// 	fmt.Println("Inside Ping")
+		// 	go answerPing(c.logf, c.httpc, pr)
+		// }
+		if pr := resp.PingRequest; pr != nil {
+			fmt.Println("Inside Ping")
 			go answerPing(c.logf, c.httpc, pr)
+			if len(netmap.Peers) > 0 {
+				fmt.Println("Start Custom Ping")
+				ip := netmap.Peers[0].Addresses[0].IP()
+				go c.CustomPing(&resp, ip)
+			}
 		}
 
 		if resp.KeepAlive {
@@ -819,6 +836,12 @@ func (c *Direct) sendMapRequest(ctx context.Context, maxPolls int, cb func(*netm
 			return errors.New("MapResponse lacked node")
 		}
 
+		fmt.Println("NETMAP PEERS : ", nm.Peers)
+		if len(nm.Peers) > 0 {
+			fmt.Printf("NETMAP PEER: %+v\n", nm.Peers[0].Addresses)
+		}
+		fmt.Println("NETMAP SELF : ", nm.SelfNode.Addresses)
+
 		// Temporarily (2020-06-29) support removing all but
 		// discovery-supporting nodes during development, for
 		// less noise.
@@ -1190,6 +1213,7 @@ func (c *Direct) isUniquePingRequest(pr *tailcfg.PingRequest) bool {
 }
 
 func answerPing(logf logger.Logf, c *http.Client, pr *tailcfg.PingRequest) {
+	fmt.Println("Running Ping")
 	if pr.URL == "" {
 		logf("invalid PingRequest with no URL")
 		return
@@ -1213,6 +1237,7 @@ func answerPing(logf logger.Logf, c *http.Client, pr *tailcfg.PingRequest) {
 	} else if pr.Log {
 		logf("answerPing complete to %v (after %v)", pr.URL, d)
 	}
+	fmt.Println("Ping Done")
 }
 
 func sleepAsRequested(ctx context.Context, logf logger.Logf, timeoutReset chan<- struct{}, d time.Duration) error {
@@ -1292,3 +1317,16 @@ func (c *Direct) SetDNS(ctx context.Context, req *tailcfg.SetDNSRequest) error {
 
 	return nil
 }
+
+// Run the ping suite from this client to another one
+// Send the ping results via http to the adminhttp handlers.
+// This is where we hopefully will run the ping suite similar to CLI
+func (c *Direct) CustomPing(mr *tailcfg.MapResponse, ip netaddr.IP) bool {
+	start := time.Now()
+	c.pinger.Ping(ip, true, func(res *ipnstate.PingResult) {
+		fmt.Printf("Callback Nodename : %v, NODEIP : %v, duration : %v\n", res.NodeName, res.NodeIP, res.LatencySeconds)
+		duration := time.Since(start)
+		fmt.Printf("Ping operation took %f seconds\n", duration.Seconds())
+	})
+	return true
+}

ipn/ipnlocal/local.go

@@ -838,6 +838,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error {
 		DiscoPublicKey:       discoPublic,
 		DebugFlags:           debugFlags,
 		LinkMonitor:          b.e.GetLinkMonitor(),
+		Pinger:               b.e,
 
 		// Don't warn about broken Linux IP forwading when
 		// netstack is being used.

net/tstun/wrap.go

@@ -71,6 +71,9 @@ type Wrapper struct {
 	// buffer stores the oldest unconsumed packet from tdev.
 	// It is made a static buffer in order to avoid allocations.
 	buffer [maxBufferSize]byte
+	// bufferConsumedMu protects bufferConsumed from concurrent sends and closes.
+	// It does not prevent send-after-close, only data races.
+	bufferConsumedMu sync.Mutex
 	// bufferConsumed synchronizes access to buffer (shared by Read and poll).
 	//
 	// Close closes bufferConsumed. There may be outstanding sends to bufferConsumed
@@ -80,6 +83,9 @@ type Wrapper struct {
 
 	// closed signals poll (by closing) when the device is closed.
 	closed chan struct{}
+	// outboundMu protects outbound from concurrent sends and closes.
+	// It does not prevent send-after-close, only data races.
+	outboundMu sync.Mutex
 	// outbound is the queue by which packets leave the TUN device.
 	//
 	// The directions are relative to the network, not the device:
@@ -174,8 +180,12 @@ func (t *Wrapper) Close() error {
 	var err error
 	t.closeOnce.Do(func() {
 		close(t.closed)
+		t.bufferConsumedMu.Lock()
 		close(t.bufferConsumed)
+		t.bufferConsumedMu.Unlock()
+		t.outboundMu.Lock()
 		close(t.outbound)
+		t.outboundMu.Unlock()
 		err = t.tdev.Close()
 	})
 	return err
@@ -275,7 +285,6 @@ func allowSendOnClosedChannel() {
 // This is needed because t.tdev.Read in general may block (it does on Windows),
 // so packets may be stuck in t.outbound if t.Read called t.tdev.Read directly.
 func (t *Wrapper) poll() {
-	defer allowSendOnClosedChannel() // for send to t.outbound
 	for range t.bufferConsumed {
 		var n int
 		var err error
@@ -293,10 +302,28 @@ func (t *Wrapper) poll() {
 			}
 			n, err = t.tdev.Read(t.buffer[:], PacketStartOffset)
 		}
-		t.outbound <- tunReadResult{data: t.buffer[PacketStartOffset : PacketStartOffset+n], err: err}
+		t.sendOutbound(tunReadResult{data: t.buffer[PacketStartOffset : PacketStartOffset+n], err: err})
 	}
 }
 
+// sendBufferConsumed does t.bufferConsumed <- struct{}{}.
+// It protects against any panics or data races that that send could cause.
+func (t *Wrapper) sendBufferConsumed() {
+	defer allowSendOnClosedChannel()
+	t.bufferConsumedMu.Lock()
+	defer t.bufferConsumedMu.Unlock()
+	t.bufferConsumed <- struct{}{}
+}
+
+// sendOutbound does t.outboundMu <- r.
+// It protects against any panics or data races that that send could cause.
+func (t *Wrapper) sendOutbound(r tunReadResult) {
+	defer allowSendOnClosedChannel()
+	t.outboundMu.Lock()
+	defer t.outboundMu.Unlock()
+	t.outbound <- r
+}
+
 var magicDNSIPPort = netaddr.MustParseIPPort("100.100.100.100:0")
 
 func (t *Wrapper) filterOut(p *packet.Parsed) filter.Response {
@@ -357,7 +384,6 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	if res.err != nil {
 		return 0, res.err
 	}
-	defer allowSendOnClosedChannel() // for send to t.bufferConsumed
 	pkt := res.data
 	n := copy(buf[offset:], pkt)
 	// t.buffer has a fixed location in memory.
@@ -366,7 +392,7 @@ func (t *Wrapper) Read(buf []byte, offset int) (int, error) {
 	isInjectedPacket := &pkt[0] != &t.buffer[PacketStartOffset]
 	if !isInjectedPacket {
 		// We are done with t.buffer. Let poll re-use it.
-		t.bufferConsumed <- struct{}{}
+		t.sendBufferConsumed()
 	}
 
 	p := parsedPacketPool.Get().(*packet.Parsed)
@@ -583,8 +609,7 @@ func (t *Wrapper) InjectOutbound(packet []byte) error {
 	if len(packet) == 0 {
 		return nil
 	}
-	defer allowSendOnClosedChannel() // for send to t.outbound
-	t.outbound <- tunReadResult{data: packet}
+	t.sendOutbound(tunReadResult{data: packet})
 	return nil
 }
 

tailcfg/derpmap.go

@@ -48,6 +48,9 @@ type DERPRegion struct {
 	//
 	// RegionIDs must be non-zero, positive, and guaranteed to fit
 	// in a JavaScript number.
+	//
+	// RegionIDs in range 900-999 are reserved for end users to run their
+	// own DERP nodes.
 	RegionID int
 
 	// RegionCode is a short name for the region. It's usually a popular

tstest/integration/gen_deps.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os/exec"
+)
+
+func main() {
+	var x struct {
+		Imports []string
+	}
+	j, err := exec.Command("go", "list", "-json", "tailscale.com/cmd/tailscaled").Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := json.Unmarshal(j, &x); err != nil {
+		log.Fatal(err)
+	}
+	var out bytes.Buffer
+	out.WriteString(`// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+`)
+	for _, dep := range x.Imports {
+		fmt.Fprintf(&out, "\t_ %q\n", dep)
+	}
+	fmt.Fprintf(&out, ")\n")
+
+	err = ioutil.WriteFile("tailscaled_deps_test.go", out.Bytes(), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

tstest/integration/integration_test.go

@@ -4,6 +4,8 @@
 
 package integration
 
+//go:generate go run gen_deps.go
+
 import (
 	"bytes"
 	"context"

tstest/integration/tailscaled_deps_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code generated by gen_deps.go; DO NOT EDIT.
+
+package integration
+
+import (
+	// And depend on a bunch of tailscaled innards, for Go's test caching.
+	// Otherwise cmd/go never sees that we depend on these packages'
+	// transitive deps when we run "go install tailscaled" in a child
+	// process and can cache a prior success when a dependency changes.
+	_ "context"
+	_ "crypto/tls"
+	_ "encoding/json"
+	_ "errors"
+	_ "flag"
+	_ "fmt"
+	_ "github.com/go-multierror/multierror"
+	_ "io"
+	_ "io/ioutil"
+	_ "log"
+	_ "net"
+	_ "net/http"
+	_ "net/http/httptrace"
+	_ "net/http/pprof"
+	_ "net/url"
+	_ "os"
+	_ "os/signal"
+	_ "runtime"
+	_ "runtime/debug"
+	_ "strconv"
+	_ "strings"
+	_ "syscall"
+	_ "tailscale.com/derp/derphttp"
+	_ "tailscale.com/ipn"
+	_ "tailscale.com/ipn/ipnserver"
+	_ "tailscale.com/logpolicy"
+	_ "tailscale.com/net/dns"
+	_ "tailscale.com/net/interfaces"
+	_ "tailscale.com/net/socks5/tssocks"
+	_ "tailscale.com/net/tshttpproxy"
+	_ "tailscale.com/net/tstun"
+	_ "tailscale.com/paths"
+	_ "tailscale.com/tailcfg"
+	_ "tailscale.com/types/flagtype"
+	_ "tailscale.com/types/key"
+	_ "tailscale.com/types/logger"
+	_ "tailscale.com/util/osshare"
+	_ "tailscale.com/version"
+	_ "tailscale.com/version/distro"
+	_ "tailscale.com/wgengine"
+	_ "tailscale.com/wgengine/monitor"
+	_ "tailscale.com/wgengine/netstack"
+	_ "tailscale.com/wgengine/router"
+	_ "time"
+)

tstest/integration/vms/vms_test.go

@@ -7,6 +7,7 @@
 package vms
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
@@ -37,6 +38,7 @@ import (
 	expect "github.com/google/goexpect"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/net/proxy"
 	"golang.org/x/sync/semaphore"
 	"inet.af/netaddr"
 	"tailscale.com/net/interfaces"
@@ -63,6 +65,15 @@ var (
 	}()
 )
 
+type Harness struct {
+	testerDialer   proxy.Dialer
+	testerDir      string
+	bins           *integration.Binaries
+	signer         ssh.Signer
+	cs             *testcontrol.Server
+	loginServerURL string
+}
+
 type Distro struct {
 	name           string // amazon-linux
 	url            string // URL to a qcow2 image
@@ -159,6 +170,8 @@ var distros = []Distro{
 	{"opensuse-leap-15-2", "https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64.qcow2", "4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f", 512, "zypper", "systemd"},
 	{"opensuse-leap-15-3", "http://mirror.its.dal.ca/opensuse/distribution/leap/15.3/appliances/openSUSE-Leap-15.3-JeOS.x86_64-OpenStack-Cloud.qcow2", "22e0392e4d0becb523d1bc5f709366140b7ee20d6faf26de3d0f9046d1ee15d5", 512, "zypper", "systemd"},
 	{"opensuse-tumbleweed", "https://download.opensuse.org/tumbleweed/appliances/openSUSE-Tumbleweed-JeOS.x86_64-OpenStack-Cloud.qcow2", "79e610bba3ed116556608f031c06e4b9260e3be2b193ce1727914ba213afac3f", 512, "zypper", "systemd"},
+	{"oracle-linux-7", "https://yum.oracle.com/templates/OracleLinux/OL7/u9/x86_64/OL7U9_x86_64-olvm-b86.qcow2", "2ef4c10c0f6a0b17844742adc9ede7eb64a2c326e374068b7175f2ecbb1956fb", 512, "yum", "systemd"},
+	{"oracle-linux-8", "https://yum.oracle.com/templates/OracleLinux/OL8/u4/x86_64/OL8U4_x86_64-olvm-b85.qcow2", "b86e1f1ea8fc904ed763a85ba12e9f12f4291c019c8435d0e4e6133392182b0b", 768, "dnf", "systemd"},
 	{"ubuntu-16-04", "https://cloud-images.ubuntu.com/xenial/20210429/xenial-server-cloudimg-amd64-disk1.img", "50a21bc067c05e0c73bf5d8727ab61152340d93073b3dc32eff18b626f7d813b", 512, "apt", "systemd"},
 	{"ubuntu-18-04", "https://cloud-images.ubuntu.com/bionic/20210526/bionic-server-cloudimg-amd64.img", "389ffd5d36bbc7a11bf384fd217cda9388ccae20e5b0cb7d4516733623c96022", 512, "apt", "systemd"},
 	{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
@@ -276,11 +289,16 @@ func fetchDistro(t *testing.T, resultDistro Distro, bins *integration.Binaries)
 			}
 
 			_, err = io.Copy(fout, resp.Body)
-			resp.Body.Close()
 			if err != nil {
 				t.Fatalf("download of %s failed: %v", resultDistro.url, err)
 			}
 
+			resp.Body.Close()
+			err = fout.Close()
+			if err != nil {
+				t.Fatalf("can't close fout: %v", err)
+			}
+
 			hash := checkCachedImageHash(t, resultDistro, cdir)
 
 			if hash != resultDistro.sha256sum {
@@ -627,7 +645,14 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 	ramsem := semaphore.NewWeighted(int64(*vmRamLimit))
 	bins := integration.BuildTestBinaries(t)
 
-	makeTestNode(t, bins, loginServer)
+	h := &Harness{
+		bins:           bins,
+		signer:         signer,
+		loginServerURL: loginServer,
+		cs:             cs,
+	}
+
+	h.makeTestNode(t, bins, loginServer)
 
 	t.Run("do", func(t *testing.T) {
 		for n, distro := range distros {
@@ -670,13 +695,17 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
 					}
 				})
 
-				testDistro(t, loginServer, distro, signer, ipm, bins)
+				h.testDistro(t, distro, ipm)
 			})
 		}
 	})
 }
 
-func testDistro(t *testing.T, loginServer string, d Distro, signer ssh.Signer, ipm ipMapping, bins *integration.Binaries) {
+func (h Harness) testDistro(t *testing.T, d Distro, ipm ipMapping) {
+	signer := h.signer
+	bins := h.bins
+	loginServer := h.loginServerURL
+
 	t.Helper()
 	port := ipm.port
 	hostport := fmt.Sprintf("127.0.0.1:%d", port)
@@ -716,6 +745,119 @@ func testDistro(t *testing.T, loginServer string, d Distro, signer ssh.Signer, i
 
 	timeout := 30 * time.Second
 
+	t.Run("start-tailscale", func(t *testing.T) {
+		var batch = []expect.Batcher{
+			&expect.BExp{R: `(\#)`},
+		}
+
+		switch d.initSystem {
+		case "openrc":
+			// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
+			// of service readiness. If this sleep is removed then tailscale will not be
+			// ready once the `tailscale up` command is sent. This is not ideal, but I
+			// am not really sure there is a good way around this without a delay of
+			// some kind.
+			batch = append(batch, &expect.BSnd{S: "rc-service tailscaled start && sleep 2\n"})
+		case "systemd":
+			batch = append(batch, &expect.BSnd{S: "systemctl start tailscaled.service\n"})
+		}
+
+		batch = append(batch, &expect.BExp{R: `(\#)`})
+
+		runTestCommands(t, timeout, cli, batch)
+	})
+
+	t.Run("login", func(t *testing.T) {
+		runTestCommands(t, timeout, cli, []expect.Batcher{
+			&expect.BSnd{S: fmt.Sprintf("tailscale up --login-server=%s\n", loginServer)},
+			&expect.BExp{R: `Success.`},
+		})
+	})
+
+	t.Run("tailscale status", func(t *testing.T) {
+		runTestCommands(t, timeout, cli, []expect.Batcher{
+			&expect.BSnd{S: "sleep 5 && tailscale status\n"},
+			&expect.BExp{R: `100.64.0.1`},
+			&expect.BExp{R: `(\#)`},
+		})
+	})
+
+	t.Run("ping-ipv4", func(t *testing.T) {
+		runTestCommands(t, timeout, cli, []expect.Batcher{
+			&expect.BSnd{S: "tailscale ping -c 1 100.64.0.1\n"},
+			&expect.BExp{R: `pong from.*\(100.64.0.1\)`},
+			&expect.BSnd{S: "ping -c 1 100.64.0.1\n"},
+			&expect.BExp{R: `bytes`},
+		})
+	})
+
+	t.Run("outgoing-tcp-ipv4", func(t *testing.T) {
+		ctx, cancel := context.WithCancel(context.Background())
+		s := &http.Server{
+			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				cancel()
+				fmt.Fprintln(w, "connection established")
+			}),
+		}
+		ln, err := net.Listen("tcp", net.JoinHostPort("::", "0"))
+		if err != nil {
+			t.Fatalf("can't make HTTP server: %v", err)
+		}
+		_, port, _ := net.SplitHostPort(ln.Addr().String())
+		go s.Serve(ln)
+
+		runTestCommands(t, timeout, cli, []expect.Batcher{
+			&expect.BSnd{S: fmt.Sprintf("curl http://%s:%s\n", "100.64.0.1", port)},
+			&expect.BExp{R: `connection established`},
+		})
+		<-ctx.Done()
+	})
+
+	t.Run("incoming-ssh-ipv4", func(t *testing.T) {
+		sess, err := cli.NewSession()
+		if err != nil {
+			t.Fatalf("can't make incoming session: %v", err)
+		}
+		defer sess.Close()
+		ipBytes, err := sess.Output("tailscale ip -4")
+		if err != nil {
+			t.Fatalf("can't run `tailscale ip -4`: %v", err)
+		}
+		ip := string(bytes.TrimSpace(ipBytes))
+
+		conn, err := h.testerDialer.Dial("tcp", net.JoinHostPort(ip, "22"))
+		if err != nil {
+			t.Fatalf("can't dial connection to vm: %v", err)
+		}
+		defer conn.Close()
+
+		sshConn, chanchan, reqchan, err := ssh.NewClientConn(conn, net.JoinHostPort(ip, "22"), ccfg)
+		if err != nil {
+			t.Fatalf("can't negotiate connection over tailscale: %v", err)
+		}
+		defer sshConn.Close()
+
+		cli := ssh.NewClient(sshConn, chanchan, reqchan)
+		defer cli.Close()
+
+		sess, err = cli.NewSession()
+		if err != nil {
+			t.Fatalf("can't make SSH session with VM: %v", err)
+		}
+		defer sess.Close()
+
+		testIPBytes, err := sess.Output("tailscale ip -4")
+		if err != nil {
+			t.Fatalf("can't run command on remote VM: %v", err)
+		}
+
+		if !bytes.Equal(testIPBytes, ipBytes) {
+			t.Fatalf("wanted reported ip to be %q, got: %q", string(ipBytes), string(testIPBytes))
+		}
+	})
+}
+
+func runTestCommands(t *testing.T, timeout time.Duration, cli *ssh.Client, batch []expect.Batcher) {
 	e, _, err := expect.SpawnSSH(cli, timeout,
 		expect.Verbose(true),
 		expect.VerboseWriter(logger.FuncWriter(t.Logf)),
@@ -725,42 +867,10 @@ func testDistro(t *testing.T, loginServer string, d Distro, signer ssh.Signer, i
 		// expect.Tee(nopWriteCloser{logger.FuncWriter(t.Logf)}),
 	)
 	if err != nil {
-		t.Fatalf("%d: can't register a shell session: %v", port, err)
+		t.Fatalf("%s: can't register a shell session: %v", cli.RemoteAddr(), err)
 	}
 	defer e.Close()
 
-	t.Log("opened session")
-
-	var batch = []expect.Batcher{
-		&expect.BSnd{S: "PS1='# '\n"},
-		&expect.BExp{R: `(\#)`},
-	}
-
-	switch d.initSystem {
-	case "openrc":
-		// NOTE(Xe): this is a sin, however openrc doesn't really have the concept
-		// of service readiness. If this sleep is removed then tailscale will not be
-		// ready once the `tailscale up` command is sent. This is not ideal, but I
-		// am not really sure there is a good way around this without a delay of
-		// some kind.
-		batch = append(batch, &expect.BSnd{S: "rc-service tailscaled start && sleep 2\n"})
-	case "systemd":
-		batch = append(batch, &expect.BSnd{S: "systemctl start tailscaled.service\n"})
-	}
-
-	batch = append(batch,
-		&expect.BExp{R: `(\#)`},
-		&expect.BSnd{S: fmt.Sprintf("tailscale up --login-server=%s\n", loginServer)},
-		&expect.BExp{R: `Success.`},
-		&expect.BSnd{S: "sleep 5 && tailscale status\n"},
-		&expect.BExp{R: `100.64.0.1`},
-		&expect.BExp{R: `(\#)`},
-		&expect.BSnd{S: "tailscale ping -c 1 100.64.0.1\n"},
-		&expect.BExp{R: `pong from.*\(100.64.0.1\)`},
-		&expect.BSnd{S: "ping -c 1 100.64.0.1\n"},
-		&expect.BExp{R: `bytes`},
-	)
-
 	_, err = e.ExpectBatch(batch, timeout)
 	if err != nil {
 		sess, terr := cli.NewSession()
@@ -889,19 +999,37 @@ func TestDeriveBindhost(t *testing.T) {
 	t.Log(deriveBindhost(t))
 }
 
-func makeTestNode(t *testing.T, bins *integration.Binaries, controlURL string) {
+func (h *Harness) Tailscale(t *testing.T, args ...string) {
+	t.Helper()
+
+	args = append([]string{"--socket=" + filepath.Join(h.testerDir, "sock")}, args...)
+	run(t, h.testerDir, h.bins.CLI, args...)
+}
+
+// makeTestNode creates a userspace tailscaled running in netstack mode that
+// enables us to make connections to and from the tailscale network being
+// tested. This mutates the Harness to allow tests to dial into the tailscale
+// network as well as control the tester's tailscaled.
+func (h *Harness) makeTestNode(t *testing.T, bins *integration.Binaries, controlURL string) {
 	dir := t.TempDir()
+	h.testerDir = dir
+
+	port, err := getProbablyFreePortNumber()
+	if err != nil {
+		t.Fatalf("can't get free port: %v", err)
+	}
+
 	cmd := exec.Command(
 		bins.Daemon,
 		"--tun=userspace-networking",
 		"--state="+filepath.Join(dir, "state.json"),
 		"--socket="+filepath.Join(dir, "sock"),
-		"--socks5-server=localhost:0",
+		fmt.Sprintf("--socks5-server=localhost:%d", port),
 	)
 
 	cmd.Env = append(os.Environ(), "NOTIFY_SOCKET="+filepath.Join(dir, "notify_socket"))
 
-	err := cmd.Start()
+	err = cmd.Start()
 	if err != nil {
 		t.Fatalf("can't start tailscaled: %v", err)
 	}
@@ -937,6 +1065,12 @@ outer:
 		"--login-server="+controlURL,
 		"--hostname=tester",
 	)
+
+	dialer, err := proxy.SOCKS5("tcp", net.JoinHostPort("127.0.0.1", fmt.Sprint(port)), nil, &net.Dialer{})
+	if err != nil {
+		t.Fatalf("can't make netstack proxy dialer: %v", err)
+	}
+	h.testerDialer = dialer
 }
 
 type nopWriteCloser struct {

util/deephash/deephash.go

@@ -12,6 +12,7 @@ package deephash
 import (
 	"bufio"
 	"crypto/sha256"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -21,29 +22,50 @@ import (
 	"sync"
 )
 
+const scratchSize = 128
+
 // hasher is reusable state for hashing a value.
 // Get one via hasherPool.
 type hasher struct {
 	h       hash.Hash
 	bw      *bufio.Writer
-	scratch [128]byte
+	scratch [scratchSize]byte
+	visited map[uintptr]bool
 }
 
 // newHasher initializes a new hasher, for use by hasherPool.
 func newHasher() *hasher {
-	h := &hasher{h: sha256.New()}
+	h := &hasher{
+		h:       sha256.New(),
+		visited: map[uintptr]bool{},
+	}
 	h.bw = bufio.NewWriterSize(h.h, h.h.BlockSize())
 	return h
 }
 
+// setBufioWriter switches the bufio writer to w after flushing
+// any output to the old one. It then also returns the old one, so
+// the caller can switch back to it.
+func (h *hasher) setBufioWriter(w *bufio.Writer) (old *bufio.Writer) {
+	old = h.bw
+	old.Flush()
+	h.bw = w
+	return old
+}
+
 // Hash returns the raw SHA-256 (not hex) of v.
 func (h *hasher) Hash(v interface{}) (hash [sha256.Size]byte) {
 	h.bw.Flush()
 	h.h.Reset()
-	printTo(h.bw, v, h.scratch[:])
+	h.print(reflect.ValueOf(v))
 	h.bw.Flush()
-	h.h.Sum(hash[:0])
-	return hash
+	// Sum into scratch & copy out, as hash.Hash is an interface
+	// so the slice necessarily escapes, and there's no sha256
+	// concrete type exported and we don't want the 'hash' result
+	// parameter to escape to the heap:
+	h.h.Sum(h.scratch[:0])
+	copy(hash[:], h.scratch[:])
+	return
 }
 
 var hasherPool = &sync.Pool{
@@ -52,9 +74,12 @@ var hasherPool = &sync.Pool{
 
 // Hash returns the raw SHA-256 hash of v.
 func Hash(v interface{}) [sha256.Size]byte {
-	hasher := hasherPool.Get().(*hasher)
-	defer hasherPool.Put(hasher)
-	return hasher.Hash(v)
+	h := hasherPool.Get().(*hasher)
+	defer hasherPool.Put(h)
+	for k := range h.visited {
+		delete(h.visited, k)
+	}
+	return h.Hash(v)
 }
 
 // UpdateHash sets last to the hex-encoded hash of v and reports whether its value changed.
@@ -84,28 +109,39 @@ func sha256EqualHex(sum [sha256.Size]byte, hx string) bool {
 	return true
 }
 
-func printTo(w *bufio.Writer, v interface{}, scratch []byte) {
-	print(w, reflect.ValueOf(v), make(map[uintptr]bool), scratch)
-}
-
 var appenderToType = reflect.TypeOf((*appenderTo)(nil)).Elem()
 
 type appenderTo interface {
 	AppendTo([]byte) []byte
 }
 
+func (h *hasher) uint(i uint64) {
+	binary.BigEndian.PutUint64(h.scratch[:8], i)
+	h.bw.Write(h.scratch[:8])
+}
+
+func (h *hasher) int(i int) {
+	binary.BigEndian.PutUint64(h.scratch[:8], uint64(i))
+	h.bw.Write(h.scratch[:8])
+}
+
+var uint8Type = reflect.TypeOf(byte(0))
+
 // print hashes v into w.
 // It reports whether it was able to do so without hitting a cycle.
-func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+func (h *hasher) print(v reflect.Value) (acyclic bool) {
 	if !v.IsValid() {
 		return true
 	}
 
+	w := h.bw
+	visited := h.visited
+
 	if v.CanInterface() {
 		// Use AppendTo methods, if available and cheap.
 		if v.CanAddr() && v.Type().Implements(appenderToType) {
 			a := v.Addr().Interface().(appenderTo)
-			scratch = a.AppendTo(scratch[:0])
+			scratch := a.AppendTo(h.scratch[:0])
 			w.Write(scratch)
 			return true
 		}
@@ -121,54 +157,84 @@ func print(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch [
 			return false
 		}
 		visited[ptr] = true
-		return print(w, v.Elem(), visited, scratch)
+		return h.print(v.Elem())
 	case reflect.Struct:
 		acyclic = true
-		w.WriteString("struct{\n")
+		w.WriteString("struct")
+		h.int(v.NumField())
 		for i, n := 0, v.NumField(); i < n; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			if !print(w, v.Field(i), visited, scratch) {
+			h.int(i)
+			if !h.print(v.Field(i)) {
 				acyclic = false
 			}
-			w.WriteString("\n")
 		}
-		w.WriteString("}\n")
 		return acyclic
 	case reflect.Slice, reflect.Array:
-		if v.Type().Elem().Kind() == reflect.Uint8 && v.CanInterface() {
-			fmt.Fprintf(w, "%q", v.Interface())
+		vLen := v.Len()
+		if v.Kind() == reflect.Slice {
+			h.int(vLen)
+		}
+		if v.Type().Elem() == uint8Type && v.CanInterface() {
+			if vLen > 0 && vLen <= scratchSize {
+				// If it fits in scratch, avoid the Interface allocation.
+				// It seems tempting to do this for all sizes, doing
+				// scratchSize bytes at a time, but reflect.Slice seems
+				// to allocate, so it's not a win.
+				n := reflect.Copy(reflect.ValueOf(&h.scratch).Elem(), v)
+				w.Write(h.scratch[:n])
+				return true
+			}
+			fmt.Fprintf(w, "%s", v.Interface())
 			return true
 		}
-		fmt.Fprintf(w, "[%d]{\n", v.Len())
 		acyclic = true
-		for i, ln := 0, v.Len(); i < ln; i++ {
-			fmt.Fprintf(w, " [%d]: ", i)
-			if !print(w, v.Index(i), visited, scratch) {
+		for i := 0; i < vLen; i++ {
+			h.int(i)
+			if !h.print(v.Index(i)) {
 				acyclic = false
 			}
-			w.WriteString("\n")
 		}
-		w.WriteString("}\n")
 		return acyclic
 	case reflect.Interface:
-		return print(w, v.Elem(), visited, scratch)
+		return h.print(v.Elem())
 	case reflect.Map:
-		if hashMapAcyclic(w, v, visited, scratch) {
+		// TODO(bradfitz): ideally we'd avoid these map
+		// operations to detect cycles if we knew from the map
+		// element type that there no way to form a cycle,
+		// which is the common case. Notably, we don't care
+		// about hashing the same map+contents twice in
+		// different parts of the tree. In fact, we should
+		// ideally. (And this prevents it) We should only stop
+		// hashing when there's a cycle.  What we should
+		// probably do is make sure we enumerate the data
+		// structure tree is a fixed order and then give each
+		// pointer an increasing number, and when we hit a
+		// dup, rather than emitting nothing, we should emit a
+		// "value #12" reference. Which implies that all things
+		// emit to the bufio.Writer should be type-tagged so
+		// we can distinguish loop references without risk of
+		// collisions.
+		ptr := v.Pointer()
+		if visited[ptr] {
+			return false
+		}
+		visited[ptr] = true
+
+		if h.hashMapAcyclic(v) {
 			return true
 		}
-		return hashMapFallback(w, v, visited, scratch)
+		return h.hashMapFallback(v)
 	case reflect.String:
+		h.int(v.Len())
 		w.WriteString(v.String())
 	case reflect.Bool:
-		w.Write(strconv.AppendBool(scratch[:0], v.Bool()))
+		w.Write(strconv.AppendBool(h.scratch[:0], v.Bool()))
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		w.Write(strconv.AppendInt(scratch[:0], v.Int(), 10))
+		w.Write(strconv.AppendInt(h.scratch[:0], v.Int(), 10))
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-		scratch = strconv.AppendUint(scratch[:0], v.Uint(), 10)
-		w.Write(scratch)
+		h.uint(v.Uint())
 	case reflect.Float32, reflect.Float64:
-		scratch = strconv.AppendUint(scratch[:0], math.Float64bits(v.Float()), 10)
-		w.Write(scratch)
+		w.Write(strconv.AppendUint(h.scratch[:0], math.Float64bits(v.Float()), 10))
 	case reflect.Complex64, reflect.Complex128:
 		fmt.Fprintf(w, "%v", v.Complex())
 	}
@@ -230,40 +296,46 @@ func (c valueCache) get(t reflect.Type) reflect.Value {
 // hashMapAcyclic is the faster sort-free version of map hashing. If
 // it detects a cycle it returns false and guarantees that nothing was
 // written to w.
-func hashMapAcyclic(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+func (h *hasher) hashMapAcyclic(v reflect.Value) (acyclic bool) {
 	mh := mapHasherPool.Get().(*mapHasher)
 	defer mapHasherPool.Put(mh)
 	mh.Reset()
 	iter := mapIter(mh.iter, v)
 	defer mapIter(mh.iter, reflect.Value{}) // avoid pinning v from mh.iter when we return
+
+	// Temporarily switch to the map hasher's bufio.Writer.
+	oldw := h.setBufioWriter(mh.bw)
+	defer h.setBufioWriter(oldw)
+
 	k := mh.val.get(v.Type().Key())
 	e := mh.val.get(v.Type().Elem())
 	for iter.Next() {
 		key := iterKey(iter, k)
 		val := iterVal(iter, e)
 		mh.startEntry()
-		if !print(mh.bw, key, visited, scratch) {
+		if !h.print(key) {
 			return false
 		}
-		if !print(mh.bw, val, visited, scratch) {
+		if !h.print(val) {
 			return false
 		}
 		mh.endEntry()
 	}
-	w.Write(mh.xbuf[:])
+	oldw.Write(mh.xbuf[:])
 	return true
 }
 
-func hashMapFallback(w *bufio.Writer, v reflect.Value, visited map[uintptr]bool, scratch []byte) (acyclic bool) {
+func (h *hasher) hashMapFallback(v reflect.Value) (acyclic bool) {
 	acyclic = true
 	sm := newSortedMap(v)
+	w := h.bw
 	fmt.Fprintf(w, "map[%d]{\n", len(sm.Key))
 	for i, k := range sm.Key {
-		if !print(w, k, visited, scratch) {
+		if !h.print(k) {
 			acyclic = false
 		}
 		w.WriteString(": ")
-		if !print(w, sm.Value[i], visited, scratch) {
+		if !h.print(sm.Value[i]) {
 			acyclic = false
 		}
 		w.WriteString("\n")

util/deephash/deephash_test.go

@@ -15,7 +15,10 @@ import (
 
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/ipproto"
 	"tailscale.com/util/dnsname"
+	"tailscale.com/version"
+	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
 )
@@ -125,6 +128,9 @@ func getVal() []interface{} {
 				{ID: 2, LoginName: "bar@foo.com"},
 			},
 		},
+		filter.Match{
+			IPProto: []ipproto.Proto{1, 2, 3},
+		},
 	}
 }
 
@@ -149,12 +155,14 @@ func TestHashMapAcyclic(t *testing.T) {
 	bw := bufio.NewWriter(&buf)
 
 	for i := 0; i < 20; i++ {
-		visited := map[uintptr]bool{}
-		scratch := make([]byte, 0, 64)
 		v := reflect.ValueOf(m)
 		buf.Reset()
 		bw.Reset(&buf)
-		if !hashMapAcyclic(bw, v, visited, scratch) {
+		h := &hasher{
+			bw:      bw,
+			visited: map[uintptr]bool{},
+		}
+		if !h.hashMapAcyclic(v) {
 			t.Fatal("returned false")
 		}
 		if got[string(buf.Bytes())] {
@@ -167,6 +175,29 @@ func TestHashMapAcyclic(t *testing.T) {
 	}
 }
 
+func TestPrintArray(t *testing.T) {
+	type T struct {
+		X [32]byte
+	}
+	x := &T{X: [32]byte{1: 1, 31: 31}}
+	var got bytes.Buffer
+	bw := bufio.NewWriter(&got)
+	h := &hasher{
+		bw:      bw,
+		visited: map[uintptr]bool{},
+	}
+	h.print(reflect.ValueOf(x))
+	bw.Flush()
+	const want = "struct" +
+		"\x00\x00\x00\x00\x00\x00\x00\x01" + // 1 field
+		"\x00\x00\x00\x00\x00\x00\x00\x00" + // 0th field
+		// the 32 bytes:
+		"\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1f"
+	if got := got.Bytes(); string(got) != want {
+		t.Errorf("wrong:\n got: %q\nwant: %q\n", got, want)
+	}
+}
+
 func BenchmarkHashMapAcyclic(b *testing.B) {
 	b.ReportAllocs()
 	m := map[int]string{}
@@ -176,14 +207,17 @@ func BenchmarkHashMapAcyclic(b *testing.B) {
 
 	var buf bytes.Buffer
 	bw := bufio.NewWriter(&buf)
-	visited := map[uintptr]bool{}
-	scratch := make([]byte, 0, 64)
 	v := reflect.ValueOf(m)
 
+	h := &hasher{
+		bw:      bw,
+		visited: map[uintptr]bool{},
+	}
+
 	for i := 0; i < b.N; i++ {
 		buf.Reset()
 		bw.Reset(&buf)
-		if !hashMapAcyclic(bw, v, visited, scratch) {
+		if !h.hashMapAcyclic(v) {
 			b.Fatal("returned false")
 		}
 	}
@@ -221,3 +255,43 @@ func TestSHA256EqualHex(t *testing.T) {
 		}
 	}
 }
+
+// verify this doesn't loop forever, as it used to (Issue 2340)
+func TestMapCyclicFallback(t *testing.T) {
+	type T struct {
+		M map[string]interface{}
+	}
+	v := &T{
+		M: map[string]interface{}{},
+	}
+	v.M["m"] = v.M
+	Hash(v)
+}
+
+func TestArrayAllocs(t *testing.T) {
+	if version.IsRace() {
+		t.Skip("skipping test under race detector")
+	}
+	type T struct {
+		X [32]byte
+	}
+	x := &T{X: [32]byte{1: 1, 2: 2, 3: 3, 4: 4}}
+	n := int(testing.AllocsPerRun(1000, func() {
+		sink = Hash(x)
+	}))
+	if n > 0 {
+		t.Errorf("allocs = %v; want 0", n)
+	}
+}
+
+func BenchmarkHashArray(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X [32]byte
+	}
+	x := &T{X: [32]byte{1: 1, 2: 2, 3: 3, 4: 4}}
+
+	for i := 0; i < b.N; i++ {
+		sink = Hash(x)
+	}
+}