implement mark and sweep GC

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

dont GC blocks used by pinner

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

comment GC algo

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

add lock to blockstore to prevent GC from eating wanted blocks

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

improve FetchGraph

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

separate interfaces for blockstore and GCBlockstore

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

reintroduce indirect pinning, add enumerateChildren dag method

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

mod/dagmodifier.go

@@ -11,7 +11,6 @@ import (
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	key "github.com/ipfs/go-ipfs/blocks/key"
-	imp "github.com/ipfs/go-ipfs/importer"
 	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	help "github.com/ipfs/go-ipfs/importer/helpers"
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
@@ -266,10 +265,6 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
 	for i, bs := range f.GetBlocksizes() {
 		// We found the correct child to write into
 		if cur+bs > offset {
-			// Unpin block
-			ckey := key.Key(node.Links[i].Hash)
-			dm.mp.RemovePinWithMode(ckey, pin.Indirect)
-
 			child, err := node.Links[i].GetNode(dm.ctx, dm.dagserv)
 			if err != nil {
 				return "", false, err
@@ -279,9 +274,6 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
 				return "", false, err
 			}
 
-			// pin the new node
-			dm.mp.PinWithMode(k, pin.Indirect)
-
 			offset += bs
 			node.Links[i].Hash = mh.Multihash(k)
 
@@ -310,7 +302,6 @@ func (dm *DagModifier) appendData(node *mdag.Node, blks <-chan []byte, errs <-ch
 	dbp := &help.DagBuilderParams{
 		Dagserv:  dm.dagserv,
 		Maxlinks: help.DefaultLinksPerBlock,
-		NodeCB:   imp.BasicPinnerCB(dm.mp),
 	}
 
 	return trickle.TrickleAppend(dm.ctx, node, dbp.New(blks, errs))

mod/dagmodifier_test.go

@@ -19,6 +19,7 @@ import (
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 	mdag "github.com/ipfs/go-ipfs/merkledag"
 	pin "github.com/ipfs/go-ipfs/pin"
+	gc "github.com/ipfs/go-ipfs/pin/gc"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 	u "github.com/ipfs/go-ipfs/util"
@@ -36,7 +37,7 @@ func getMockDagServ(t testing.TB) (mdag.DAGService, pin.Pinner) {
 	return dserv, pin.NewPinner(tsds, dserv)
 }
 
-func getMockDagServAndBstore(t testing.TB) (mdag.DAGService, blockstore.Blockstore, pin.Pinner) {
+func getMockDagServAndBstore(t testing.TB) (mdag.DAGService, blockstore.GCBlockstore, pin.Pinner) {
 	dstore := ds.NewMapDatastore()
 	tsds := sync.MutexWrap(dstore)
 	bstore := blockstore.NewBlockstore(tsds)
@@ -47,7 +48,7 @@ func getMockDagServAndBstore(t testing.TB) (mdag.DAGService, blockstore.Blocksto
 
 func getNode(t testing.TB, dserv mdag.DAGService, size int64, pinner pin.Pinner) ([]byte, *mdag.Node) {
 	in := io.LimitReader(u.NewTimeSeededRand(), size)
-	node, err := imp.BuildTrickleDagFromReader(dserv, sizeSplitterGen(500)(in), imp.BasicPinnerCB(pinner))
+	node, err := imp.BuildTrickleDagFromReader(dserv, sizeSplitterGen(500)(in))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -469,22 +470,17 @@ func TestSparseWrite(t *testing.T) {
 	}
 }
 
-func basicGC(t *testing.T, bs blockstore.Blockstore, pins pin.Pinner) {
+func basicGC(t *testing.T, bs blockstore.GCBlockstore, pins pin.Pinner) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel() // in case error occurs during operation
-	keychan, err := bs.AllKeysChan(ctx)
+	out, err := gc.GC(ctx, bs, pins)
 	if err != nil {
 		t.Fatal(err)
 	}
-	for k := range keychan { // rely on AllKeysChan to close chan
-		if !pins.IsPinned(k) {
-			err := bs.DeleteBlock(k)
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
+	for range out {
 	}
 }
+
 func TestCorrectPinning(t *testing.T) {
 	dserv, bstore, pins := getMockDagServAndBstore(t)
 	b, n := getNode(t, dserv, 50000, pins)
@@ -566,14 +562,6 @@ func TestCorrectPinning(t *testing.T) {
 		t.Fatal("Incorrect node recursively pinned")
 	}
 
-	indirpins := pins.IndirectKeys()
-	children := enumerateChildren(t, nd, dserv)
-	// TODO this is not true if the contents happen to be identical
-	if len(indirpins) != len(children) {
-		t.Log(len(indirpins), len(children))
-		t.Fatal("Incorrect number of indirectly pinned blocks")
-	}
-
 }
 
 func enumerateChildren(t *testing.T, nd *mdag.Node, ds mdag.DAGService) []key.Key {