bitswap first working commit!

bitswap/bitswap.go

@@ -67,6 +67,7 @@ func NewBitSwap(p *peer.Peer, net swarm.Network, d ds.Datastore, r routing.IpfsR
 		routing:   r.(*dht.IpfsDHT),
 		meschan:   net.GetChannel(swarm.PBWrapper_BITSWAP),
 		haltChan:  make(chan struct{}),
+		listener:  swarm.NewMesListener(),
 	}
 
 	go bs.handleMessages()
@@ -90,7 +91,7 @@ func (bs *BitSwap) GetBlock(k u.Key, timeout time.Duration) (
 				ledger := bs.GetLedger(pr.Key())
 				blk, err := bs.getBlock(k, pr, tleft)
 				if err != nil {
-					u.PErr("%v\n", err)
+					u.PErr("getBlock returned: %v\n", err)
 					return
 				}
 				// NOTE: this credits everyone who sends us a block,
@@ -106,6 +107,7 @@ func (bs *BitSwap) GetBlock(k u.Key, timeout time.Duration) (
 
 	select {
 	case blkdata := <-valchan:
+		close(valchan)
 		return blocks.NewBlock(blkdata)
 	case <-after:
 		return nil, u.ErrTimeout
@@ -113,6 +115,7 @@ func (bs *BitSwap) GetBlock(k u.Key, timeout time.Duration) (
 }
 
 func (bs *BitSwap) getBlock(k u.Key, p *peer.Peer, timeout time.Duration) ([]byte, error) {
+	u.DOut("[%s] getBlock '%s' from [%s]\n", bs.peer.ID.Pretty(), k.Pretty(), p.ID.Pretty())
 	//
 	mes := new(PBMessage)
 	mes.Id = proto.Uint64(swarm.GenerateMessageID())
@@ -161,6 +164,7 @@ func (bs *BitSwap) handleMessages() {
 			}
 			if pmes.GetResponse() {
 				bs.listener.Respond(pmes.GetId(), mes)
+				continue
 			}
 
 			switch pmes.GetType() {
@@ -176,16 +180,20 @@ func (bs *BitSwap) handleMessages() {
 }
 
 func (bs *BitSwap) handleGetBlock(p *peer.Peer, pmes *PBMessage) {
+	u.DOut("handleGetBlock.\n")
 	ledger := bs.GetLedger(p.Key())
 
+	u.DOut("finding [%s] in datastore.\n", u.Key(pmes.GetKey()).Pretty())
 	idata, err := bs.datastore.Get(ds.NewKey(pmes.GetKey()))
 	if err != nil {
+		u.PErr("handleGetBlock datastore returned: %v\n", err)
 		if err == ds.ErrNotFound {
 			return
 		}
-		u.PErr("%v\n", err)
 		return
 	}
+
+	u.DOut("found value!\n")
 	data, ok := idata.([]byte)
 	if !ok {
 		u.PErr("Failed casting data from datastore.")
@@ -193,13 +201,18 @@ func (bs *BitSwap) handleGetBlock(p *peer.Peer, pmes *PBMessage) {
 	}
 
 	if ledger.ShouldSend() {
+		u.DOut("Sending value back!\n")
 		resp := &Message{
 			Value:    data,
 			Response: true,
 			ID:       pmes.GetId(),
+			Type:     PBMessage_GET_BLOCK,
+			Success:  true,
 		}
 		bs.meschan.Outgoing <- swarm.NewMessage(p, resp.ToProtobuf())
 		ledger.SentBytes(uint64(len(data)))
+	} else {
+		u.DOut("Ledger decided not to send anything...\n")
 	}
 }
 
@@ -210,6 +223,7 @@ func (bs *BitSwap) GetLedger(k u.Key) *Ledger {
 	}
 
 	l = new(Ledger)
+	l.Strategy = StandardStrategy
 	l.Partner = peer.ID(k)
 	bs.partners[k] = l
 	return l

bitswap/ledger.go

 package bitswap
 
 import (
-	"math/rand"
-
 	peer "github.com/jbenet/go-ipfs/peer"
 	u "github.com/jbenet/go-ipfs/util"
 
@@ -26,13 +24,15 @@ type Ledger struct {
 
 	// WantList is a (bounded, small) set of keys that Partner desires.
 	WantList KeySet
+
+	Strategy StrategyFunc
 }
 
 // LedgerMap lists Ledgers by their Partner key.
 type LedgerMap map[u.Key]*Ledger
 
 func (l *Ledger) ShouldSend() bool {
-	return rand.Float64() <= probabilitySend(l.Accounting.Value())
+	return l.Strategy(l.Accounting)
 }
 
 func (l *Ledger) SentBytes(n uint64) {

bitswap/message.go

@@ -6,6 +6,7 @@ import (
 )
 
 type Message struct {
+	Type     PBMessage_MessageType
 	ID       uint64
 	Response bool
 	Key      u.Key
@@ -16,6 +17,7 @@ type Message struct {
 func (m *Message) ToProtobuf() *PBMessage {
 	pmes := new(PBMessage)
 	pmes.Id = &m.ID
+	pmes.Type = &m.Type
 	if m.Response {
 		pmes.Response = proto.Bool(true)
 	}

bitswap/strategy.go

@@ -2,8 +2,19 @@ package bitswap
 
 import (
 	"math"
+	"math/rand"
 )
 
+type StrategyFunc func(debtRatio) bool
+
+func StandardStrategy(db debtRatio) bool {
+	return rand.Float64() <= probabilitySend(db.Value())
+}
+
+func YesManStrategy(db debtRatio) bool {
+	return true
+}
+
 func probabilitySend(ratio float64) float64 {
 	x := 1 + math.Exp(6-3*ratio)
 	y := 1 / x

blockservice/blockservice.go

@@ -2,6 +2,7 @@ package blockservice
 
 import (
 	"fmt"
+	"time"
 
 	ds "github.com/jbenet/datastore.go"
 	bitswap "github.com/jbenet/go-ipfs/bitswap"
@@ -19,18 +20,27 @@ type BlockService struct {
 }
 
 // NewBlockService creates a BlockService with given datastore instance.
-func NewBlockService(d ds.Datastore) (*BlockService, error) {
+func NewBlockService(d ds.Datastore, rem *bitswap.BitSwap) (*BlockService, error) {
 	if d == nil {
 		return nil, fmt.Errorf("BlockService requires valid datastore")
 	}
-	return &BlockService{Datastore: d}, nil
+	if rem == nil {
+		return nil, fmt.Errorf("BlockService requires a valid bitswap")
+	}
+	return &BlockService{Datastore: d, Remote: rem}, nil
 }
 
 // AddBlock adds a particular block to the service, Putting it into the datastore.
 func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) {
 	k := b.Key()
 	dsk := ds.NewKey(string(k))
-	return k, s.Datastore.Put(dsk, b.Data)
+	u.DOut("storing [%s] in datastore\n", k.Pretty())
+	err := s.Datastore.Put(dsk, b.Data)
+	if err != nil {
+		return k, err
+	}
+	err = s.Remote.HaveBlock(b.Key())
+	return k, err
 }
 
 // GetBlock retrieves a particular block from the service,
@@ -38,17 +48,22 @@ func (s *BlockService) AddBlock(b *blocks.Block) (u.Key, error) {
 func (s *BlockService) GetBlock(k u.Key) (*blocks.Block, error) {
 	dsk := ds.NewKey(string(k))
 	datai, err := s.Datastore.Get(dsk)
-	if err != nil {
-		return nil, err
+	if err == nil {
+		bdata, ok := datai.([]byte)
+		if !ok {
+			return nil, fmt.Errorf("data associated with %s is not a []byte", k)
+		}
+		return &blocks.Block{
+			Multihash: mh.Multihash(k),
+			Data:      bdata,
+		}, nil
+	} else if err == ds.ErrNotFound {
+		blk, err := s.Remote.GetBlock(k, time.Second*5)
+		if err != nil {
+			return nil, err
+		}
+		return blk, nil
+	} else {
+		return nil, u.ErrNotFound
 	}
-
-	data, ok := datai.([]byte)
-	if !ok {
-		return nil, fmt.Errorf("data associated with %s is not a []byte", k)
-	}
-
-	return &blocks.Block{
-		Multihash: mh.Multihash(k),
-		Data:      data,
-	}, nil
 }

core/core.go

@@ -59,7 +59,7 @@ func NewIpfsNode(cfg *config.Config) (*IpfsNode, error) {
 		return nil, err
 	}
 
-	bs, err := bserv.NewBlockService(d)
+	bs, err := bserv.NewBlockService(d, nil)
 	if err != nil {
 		return nil, err
 	}

routing/dht/dht.go

@@ -53,11 +53,11 @@ type IpfsDHT struct {
 }
 
 // NewDHT creates a new DHT object with the given peer as the 'local' host
-func NewDHT(p *peer.Peer, net swarm.Network) *IpfsDHT {
+func NewDHT(p *peer.Peer, net swarm.Network, dstore ds.Datastore) *IpfsDHT {
 	dht := new(IpfsDHT)
 	dht.network = net
 	dht.netChan = net.GetChannel(swarm.PBWrapper_DHT_MESSAGE)
-	dht.datastore = ds.NewMapDatastore()
+	dht.datastore = dstore
 	dht.self = p
 	dht.providers = NewProviderManager()
 	dht.shutdown = make(chan struct{})
@@ -322,6 +322,7 @@ type providerInfo struct {
 
 func (dht *IpfsDHT) handleAddProvider(p *peer.Peer, pmes *PBDHTMessage) {
 	key := u.Key(pmes.GetKey())
+	u.DOut("[%s] Adding [%s] as a provider for '%s'\n", dht.self.ID.Pretty(), p.ID.Pretty(), peer.ID(key).Pretty())
 	dht.providers.AddProvider(key, p)
 }
 
@@ -615,12 +616,8 @@ func (dht *IpfsDHT) addPeerList(key u.Key, peers []*PBDHTMessage_PBPeer) []*peer
 		p := dht.network.Find(u.Key(prov.GetId()))
 		if p == nil {
 			u.DOut("given provider %s was not in our network already.\n", peer.ID(prov.GetId()).Pretty())
-			maddr, err := ma.NewMultiaddr(prov.GetAddr())
-			if err != nil {
-				u.PErr("error connecting to new peer: %s\n", err)
-				continue
-			}
-			p, err = dht.network.GetConnection(peer.ID(prov.GetId()), maddr)
+			var err error
+			p, err = dht.peerFromInfo(prov)
 			if err != nil {
 				u.PErr("error connecting to new peer: %s\n", err)
 				continue
@@ -631,3 +628,12 @@ func (dht *IpfsDHT) addPeerList(key u.Key, peers []*PBDHTMessage_PBPeer) []*peer
 	}
 	return provArr
 }
+
+func (dht *IpfsDHT) peerFromInfo(pbp *PBDHTMessage_PBPeer) (*peer.Peer, error) {
+	maddr, err := ma.NewMultiaddr(pbp.GetAddr())
+	if err != nil {
+		return nil, err
+	}
+
+	return dht.network.GetConnection(peer.ID(pbp.GetId()), maddr)
+}

routing/dht/dht_test.go

@@ -3,6 +3,7 @@ package dht
 import (
 	"testing"
 
+	ds "github.com/jbenet/datastore.go"
 	peer "github.com/jbenet/go-ipfs/peer"
 	swarm "github.com/jbenet/go-ipfs/swarm"
 	u "github.com/jbenet/go-ipfs/util"
@@ -37,7 +38,7 @@ func setupDHTS(n int, t *testing.T) ([]*ma.Multiaddr, []*peer.Peer, []*IpfsDHT)
 		if err != nil {
 			t.Fatal(err)
 		}
-		d := NewDHT(peers[i], net)
+		d := NewDHT(peers[i], net, ds.NewMapDatastore())
 		dhts = append(dhts, d)
 		d.Start()
 	}
@@ -69,14 +70,14 @@ func TestPing(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	dhtA := NewDHT(peerA, neta)
+	dhtA := NewDHT(peerA, neta, ds.NewMapDatastore())
 
 	netb := swarm.NewSwarm(peerB)
 	err = netb.Listen()
 	if err != nil {
 		t.Fatal(err)
 	}
-	dhtB := NewDHT(peerB, netb)
+	dhtB := NewDHT(peerB, netb, ds.NewMapDatastore())
 
 	dhtA.Start()
 	dhtB.Start()
@@ -120,14 +121,14 @@ func TestValueGetSet(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	dhtA := NewDHT(peerA, neta)
+	dhtA := NewDHT(peerA, neta, ds.NewMapDatastore())
 
 	netb := swarm.NewSwarm(peerB)
 	err = netb.Listen()
 	if err != nil {
 		t.Fatal(err)
 	}
-	dhtB := NewDHT(peerB, netb)
+	dhtB := NewDHT(peerB, netb, ds.NewMapDatastore())
 
 	dhtA.Start()
 	dhtB.Start()

routing/dht/ext_test.go

@@ -7,6 +7,7 @@ import (
 
 	"code.google.com/p/goprotobuf/proto"
 
+	ds "github.com/jbenet/datastore.go"
 	peer "github.com/jbenet/go-ipfs/peer"
 	swarm "github.com/jbenet/go-ipfs/swarm"
 	u "github.com/jbenet/go-ipfs/util"
@@ -89,7 +90,7 @@ func TestGetFailures(t *testing.T) {
 	local := new(peer.Peer)
 	local.ID = peer.ID("test_peer")
 
-	d := NewDHT(local, fn)
+	d := NewDHT(local, fn, ds.NewMapDatastore())
 
 	other := &peer.Peer{ID: peer.ID("other_peer")}
 
@@ -177,7 +178,7 @@ func TestNotFound(t *testing.T) {
 	local := new(peer.Peer)
 	local.ID = peer.ID("test_peer")
 
-	d := NewDHT(local, fn)
+	d := NewDHT(local, fn, ds.NewMapDatastore())
 	d.Start()
 
 	var ps []*peer.Peer
@@ -239,7 +240,7 @@ func TestLessThanKResponses(t *testing.T) {
 	local := new(peer.Peer)
 	local.ID = peer.ID("test_peer")
 
-	d := NewDHT(local, fn)
+	d := NewDHT(local, fn, ds.NewMapDatastore())
 	d.Start()
 
 	var ps []*peer.Peer

routing/dht/routing.go

@@ -63,7 +63,7 @@ func (dht *IpfsDHT) GetValue(key u.Key, timeout time.Duration) ([]byte, error) {
 	val, err := dht.getLocal(key)
 	if err == nil {
 		ll.Success = true
-		u.DOut("Found local, returning.")
+		u.DOut("Found local, returning.\n")
 		return val, nil
 	}
 
@@ -218,6 +218,9 @@ func (dht *IpfsDHT) FindProvidersAsync(key u.Key, count int, timeout time.Durati
 //TODO: this function could also be done asynchronously
 func (dht *IpfsDHT) addPeerListAsync(k u.Key, peers []*PBDHTMessage_PBPeer, ps *peerSet, count int, out chan *peer.Peer) {
 	for _, pbp := range peers {
+		if peer.ID(pbp.GetId()).Equal(dht.self.ID) {
+			continue
+		}
 		maddr, err := ma.NewMultiaddr(pbp.GetAddr())
 		if err != nil {
 			u.PErr("%v\n", err)
@@ -256,11 +259,14 @@ func (dht *IpfsDHT) FindProviders(key u.Key, timeout time.Duration) ([]*peer.Pee
 			return nil, err
 		}
 		if pmes.GetSuccess() {
+			u.DOut("Got providers back from findProviders call!\n")
 			provs := dht.addPeerList(key, pmes.GetPeers())
 			ll.Success = true
 			return provs, nil
 		}
 
+		u.DOut("Didnt get providers, just closer peers.\n")
+
 		closer := pmes.GetPeers()
 		if len(closer) == 0 {
 			level++
@@ -337,7 +343,7 @@ func (dht *IpfsDHT) FindPeer(id peer.ID, timeout time.Duration) (*peer.Peer, err
 // Ping a peer, log the time it took
 func (dht *IpfsDHT) Ping(p *peer.Peer, timeout time.Duration) error {
 	// Thoughts: maybe this should accept an ID and do a peer lookup?
-	u.DOut("Enter Ping.")
+	u.DOut("Enter Ping.\n")
 
 	pmes := Message{ID: swarm.GenerateMessageID(), Type: PBDHTMessage_PING}
 	mes := swarm.NewMessage(p, pmes.ToProtobuf())