Use Netroute (#25)

Use OS routing table as a hint when dialing

go.mod

@@ -2,6 +2,7 @@ module github.com/libp2p/go-reuseport-transport
 
 require (
 	github.com/ipfs/go-log v0.0.1
+	github.com/libp2p/go-netroute v0.1.2
 	github.com/libp2p/go-reuseport v0.0.1
 	github.com/multiformats/go-multiaddr v0.2.1
 	github.com/multiformats/go-multiaddr-net v0.1.3

go.sum

@@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
+github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyFSs7UnsU=
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
 github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc=
@@ -10,8 +12,11 @@ github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/libp2p/go-netroute v0.1.2 h1:UHhB35chwgvcRI392znJA3RCBtZ3MpE3ahNCN5MR4Xg=
+github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk=
 github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw=
 github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
+github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k=
 github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
@@ -79,12 +84,14 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXT
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e h1:ZytStCyV048ZqDsWHiYDdoI2Vd4msMcrDECFxS+tL9c=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

multidialer.go

@@ -5,12 +5,15 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+
+	"github.com/libp2p/go-netroute"
 )
 
 type multiDialer struct {
-	loopback    []*net.TCPAddr
-	unspecified []*net.TCPAddr
-	global      *net.TCPAddr
+	listeningAddresses []*net.TCPAddr
+	loopback           []*net.TCPAddr
+	unspecified        []*net.TCPAddr
+	fallback           net.TCPAddr
 }
 
 func (d *multiDialer) Dial(network, addr string) (net.Conn, error) {
@@ -24,87 +27,64 @@ func randAddr(addrs []*net.TCPAddr) *net.TCPAddr {
 	return nil
 }
 
+// DialContext dials a target addr.
+// Dialing preference is
+// * If there is a listener on the local interface the OS expects to use to route towards addr, use that.
+// * If there is a listener on a loopback address, addr is loopback, use that.
+// * If there is a listener on an undefined address (0.0.0.0 or ::), use that.
+// * Use the fallback IP specified during construction, with a port that's already being listened on, if one exists.
 func (d *multiDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
 	tcpAddr, err := net.ResolveTCPAddr(network, addr)
 	if err != nil {
 		return nil, err
 	}
-
-	// We pick the source *port* based on the following algorithm.
-	//
-	// 1. If we're dialing loopback, choose a source-port in order of
-	//    preference:
-	//   1. A port in-use by an explicit loopback listener.
-	//   2. A port in-use by a listener on an unspecified address (must
-	//      also be listening on localhost).
-	//   3. A port in-use by a listener on a global address. We don't have
-	//      any other better options (other than picking a random port).
-	// 2. If we're dialing a global address, choose a source-port in order
-	//    of preference:
-	//   1. A port in-use by a listener on an unspecified address (the most
-	//      general case).
-	//   2. A port in-use by a listener on the global address.
-	// 3. Fail on link-local dials (go-ipfs currently forbids this and I
-	//    figured we could try lifting this restriction later).
-	//
-	//
-	// Note: We *always* dial from the unspecified address (regardless of
-	// the port we pick). In the future, we could use netlink (on Linux) to
-	// figure out the right source address but we're going to punt on that.
-
 	ip := tcpAddr.IP
-	source := d.global
-	switch {
-	case ip.IsLoopback():
-		switch {
-		case len(d.loopback) > 0:
-			source = randAddr(d.loopback)
-		case len(d.unspecified) > 0:
-			source = randAddr(d.unspecified)
-		}
-	case ip.IsGlobalUnicast():
-		switch {
-		case len(d.unspecified) > 0:
-			source = randAddr(d.unspecified)
+	if !ip.IsLoopback() && !ip.IsGlobalUnicast() {
+		return nil, fmt.Errorf("undialable IP: %s", ip)
+	}
+
+	if router, err := netroute.New(); err == nil {
+		if _, _, preferredSrc, err := router.Route(ip); err == nil {
+			for _, optAddr := range d.listeningAddresses {
+				if optAddr.IP.Equal(preferredSrc) {
+					return reuseDial(ctx, optAddr, network, addr)
+				}
+			}
 		}
-	default:
-		return nil, fmt.Errorf("undialable IP: %s", tcpAddr.IP)
 	}
-	return reuseDial(ctx, source, network, addr)
+
+	if ip.IsLoopback() && len(d.loopback) > 0 {
+		return reuseDial(ctx, randAddr(d.loopback), network, addr)
+	}
+	if len(d.unspecified) == 0 {
+		return reuseDial(ctx, &d.fallback, network, addr)
+	}
+
+	return reuseDial(ctx, randAddr(d.unspecified), network, addr)
 }
 
-func newMultiDialer(unspec net.IP, listeners map[*listener]struct{}) dialer {
-	m := new(multiDialer)
+func newMultiDialer(unspec net.IP, listeners map[*listener]struct{}) (m dialer) {
+	addrs := make([]*net.TCPAddr, 0)
+	loopback := make([]*net.TCPAddr, 0)
+	unspecified := make([]*net.TCPAddr, 0)
+	existingPort := 0
+
 	for l := range listeners {
-		laddr := l.Addr().(*net.TCPAddr)
-		switch {
-		case laddr.IP.IsLoopback():
-			m.loopback = append(m.loopback, laddr)
-		case laddr.IP.IsGlobalUnicast():
-			// Different global ports? Crap.
-			//
-			// The *proper* way to deal with this is to, e.g., use
-			// netlink to figure out which source address we would
-			// normally use to dial a destination address and then
-			// pick one of the ports we're listening on on that
-			// source address. However, this is a pain in the ass.
-			//
-			// Instead, we're just going to always dial from the
-			// unspecified address with the first global port we
-			// find.
-			//
-			// TODO: Port priority? Addr priority?
-			if m.global == nil {
-				m.global = &net.TCPAddr{
-					IP:   unspec,
-					Port: laddr.Port,
-				}
-			} else {
-				log.Warning("listening on external interfaces on multiple ports, will dial from %d, not %s", m.global, laddr)
-			}
-		case laddr.IP.IsUnspecified():
-			m.unspecified = append(m.unspecified, laddr)
+		addr := l.Addr().(*net.TCPAddr)
+		addrs = append(addrs, addr)
+		if addr.IP.IsLoopback() {
+			loopback = append(loopback, addr)
+		} else if addr.IP.IsGlobalUnicast() && existingPort == 0 {
+			existingPort = addr.Port
+		} else if addr.IP.IsUnspecified() {
+			unspecified = append(unspecified, addr)
 		}
 	}
-	return m
+	m = &multiDialer{
+		listeningAddresses: addrs,
+		loopback:           loopback,
+		unspecified:        unspecified,
+		fallback:           net.TCPAddr{IP: unspec, Port: existingPort},
+	}
+	return
 }

transport_test.go

@@ -38,11 +38,12 @@ func init() {
 	}
 }
 
-func acceptOne(t *testing.T, listener manet.Listener) <-chan struct{} {
+func acceptOne(t *testing.T, listener manet.Listener) <-chan interface{} {
 	t.Helper()
-	done := make(chan struct{})
+	done := make(chan interface{}, 1)
 	go func() {
 		defer close(done)
+		done <- nil
 		c, err := listener.Accept()
 		if err != nil {
 			t.Error(err)
@@ -50,6 +51,7 @@ func acceptOne(t *testing.T, listener manet.Listener) <-chan struct{} {
 		}
 		c.Close()
 	}()
+	<-done
 	return done
 }
 
@@ -72,7 +74,7 @@ func dialOne(t *testing.T, tr *Transport, listener manet.Listener, expected ...i
 			return port
 		}
 	}
-	t.Errorf("dialed from %d, expected to dial from one of %v", port, expected)
+	t.Errorf("dialed %s from %v. expected to dial from port %v", listener.Multiaddr(), c.LocalAddr(), expected)
 	return 0
 }
 
@@ -127,10 +129,12 @@ func TestGlobalPreferenceV4(t *testing.T) {
 		t.Skip("no global IPv4 addresses configured")
 		return
 	}
+	t.Logf("when listening on %v, should prefer %v over %v", loopbackV4, loopbackV4, globalV4)
 	testPrefer(t, loopbackV4, loopbackV4, globalV4)
+	t.Logf("when listening on %v, should prefer %v over %v", loopbackV4, unspecV4, globalV4)
 	testPrefer(t, loopbackV4, unspecV4, globalV4)
 
-	testPrefer(t, globalV4, unspecV4, globalV4)
+	t.Logf("when listening on %v, should prefer %v over %v", globalV4, unspecV4, loopbackV4)
 	testPrefer(t, globalV4, unspecV4, loopbackV4)
 }
 
@@ -142,7 +146,6 @@ func TestGlobalPreferenceV6(t *testing.T) {
 	testPrefer(t, loopbackV6, loopbackV6, globalV6)
 	testPrefer(t, loopbackV6, unspecV6, globalV6)
 
-	testPrefer(t, globalV6, unspecV6, globalV6)
 	testPrefer(t, globalV6, unspecV6, loopbackV6)
 }