implement recursive indirect blocks

improve efficiency of multilayered indirect blocks

clean up tests

panic cleanup

clean up logic, improve readability

add final root node to the dagservice upon creation

importer: simplified dag generation

test: updated hashes using latest code

@whyrusleeping this is why the sharness tests
were failing: the hashes are added manually to
make sure our generation doesn't change.

cleanup after CR

fix merkledag tests

fix small block generation (no subblocks!)

merkledag.go

@@ -295,12 +295,12 @@ func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{}
 
 // FindLinks searches this nodes links for the given key,
 // returns the indexes of any links pointing to it
-func FindLinks(n *Node, k u.Key) []int {
+func FindLinks(n *Node, k u.Key, start int) []int {
 	var out []int
 	keybytes := []byte(k)
-	for i, lnk := range n.Links {
+	for i, lnk := range n.Links[start:] {
 		if bytes.Equal([]byte(lnk.Hash), keybytes) {
-			out = append(out, i)
+			out = append(out, i+start)
 		}
 	}
 	return out
@@ -330,7 +330,7 @@ func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
 				log.Error("Got back bad block!")
 				break
 			}
-			is := FindLinks(root, blk.Key())
+			is := FindLinks(root, blk.Key(), next)
 			for _, i := range is {
 				nodes[i] = nd
 			}

merkledag_test.go

@@ -8,14 +8,40 @@ import (
 	"sync"
 	"testing"
 
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+	dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
+	bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
 	blockservice "github.com/jbenet/go-ipfs/blockservice"
+	bserv "github.com/jbenet/go-ipfs/blockservice"
+	offline "github.com/jbenet/go-ipfs/exchange/offline"
 	imp "github.com/jbenet/go-ipfs/importer"
 	chunk "github.com/jbenet/go-ipfs/importer/chunk"
 	. "github.com/jbenet/go-ipfs/merkledag"
+	"github.com/jbenet/go-ipfs/pin"
 	uio "github.com/jbenet/go-ipfs/unixfs/io"
 	u "github.com/jbenet/go-ipfs/util"
 )
 
+type dagservAndPinner struct {
+	ds DAGService
+	mp pin.ManualPinner
+}
+
+func getDagservAndPinner(t *testing.T) dagservAndPinner {
+	db := ds.NewMapDatastore()
+	bs := bstore.NewBlockstore(dssync.MutexWrap(db))
+	blockserv, err := bserv.New(bs, offline.Exchange(bs))
+	if err != nil {
+		t.Fatal(err)
+	}
+	dserv := NewDAGService(blockserv)
+	mpin := pin.NewPinner(db, dserv).GetManual()
+	return dagservAndPinner{
+		ds: dserv,
+		mp: mpin,
+	}
+}
+
 func TestNode(t *testing.T) {
 
 	n1 := &Node{Data: []byte("beep")}
@@ -66,16 +92,6 @@ func TestNode(t *testing.T) {
 	printn("beep boop", n3)
 }
 
-func makeTestDag(t *testing.T) *Node {
-	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
-	spl := &chunk.SizeSplitter{512}
-	root, err := imp.NewDagFromReaderWithSplitter(read, spl)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return root
-}
-
 type devZero struct{}
 
 func (_ devZero) Read(b []byte) (int, error) {
@@ -85,38 +101,37 @@ func (_ devZero) Read(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func makeZeroDag(t *testing.T) *Node {
-	read := io.LimitReader(devZero{}, 1024*32)
-	spl := &chunk.SizeSplitter{512}
-	root, err := imp.NewDagFromReaderWithSplitter(read, spl)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return root
-}
-
 func TestBatchFetch(t *testing.T) {
-	root := makeTestDag(t)
-	runBatchFetchTest(t, root)
+	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
+	runBatchFetchTest(t, read)
 }
 
 func TestBatchFetchDupBlock(t *testing.T) {
-	root := makeZeroDag(t)
-	runBatchFetchTest(t, root)
+	read := io.LimitReader(devZero{}, 1024*32)
+	runBatchFetchTest(t, read)
 }
 
-func runBatchFetchTest(t *testing.T, root *Node) {
+func runBatchFetchTest(t *testing.T, read io.Reader) {
 	var dagservs []DAGService
 	for _, bsi := range blockservice.Mocks(t, 5) {
 		dagservs = append(dagservs, NewDAGService(bsi))
 	}
+
+	spl := &chunk.SizeSplitter{512}
+
+	root, err := imp.BuildDagFromReader(read, dagservs[0], nil, spl)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	t.Log("finished setup.")
 
-	read, err := uio.NewDagReader(root, nil)
+	dagr, err := uio.NewDagReader(root, dagservs[0])
 	if err != nil {
 		t.Fatal(err)
 	}
-	expected, err := ioutil.ReadAll(read)
+
+	expected, err := ioutil.ReadAll(dagr)
 	if err != nil {
 		t.Fatal(err)
 	}