fix for #1008 and other pinning fixes

This commit adds a new set of sharness tests for pinning, and addresses
bugs that were pointed out by said tests.

test/sharness: added more pinning tests

Pinning is currently broken. See issue #1051. This commit introduces
a few more pinning tests. These are by no means exhaustive, but
definitely surface the present problems going on. I believe these
tests are correct, but not sure. Pushing them as failing so that
pinning is fixed in this PR.

make pinning and merkledag.Get take contexts

improve 'add' commands usage of pinning

FIXUP: fix 'pin lists look good'

ipfs-pin-stat simple script to help check pinning

This is a simple shell script to help check pinning.

We ought to strive towards making adding commands this easy.
The http api is great and powerful, but our setup right now
gets in the way. Perhaps we can clean up that area.

updated t0081-repo-pinning

- fixed a couple bugs with the tests
- made it a bit clearer (still a lot going on)
- the remaining tests are correct and highlight a problem with
  pinning. Namely, that recursive pinning is buggy. At least:
  towards the end of the test, $HASH_DIR4 and $HASH_FILE4 should
  be pinned indirectly, but they're not. And thus get gc-ed out.
  There may be other problems too.

cc @whyrusleeping

fix grep params for context deadline check

fix bugs in pin and pin tests

check for block local before checking recursive pin

merkledag.go

@@ -4,7 +4,6 @@ package merkledag
 import (
 	"fmt"
 	"sync"
-	"time"
 
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	blocks "github.com/ipfs/go-ipfs/blocks"
@@ -19,7 +18,7 @@ var ErrNotFound = fmt.Errorf("merkledag: not found")
 type DAGService interface {
 	Add(*Node) (u.Key, error)
 	AddRecursive(*Node) error
-	Get(u.Key) (*Node, error)
+	Get(context.Context, u.Key) (*Node, error)
 	Remove(*Node) error
 
 	// GetDAG returns, in order, all the single leve child
@@ -83,17 +82,11 @@ func (n *dagService) AddRecursive(nd *Node) error {
 }
 
 // Get retrieves a node from the dagService, fetching the block in the BlockService
-func (n *dagService) Get(k u.Key) (*Node, error) {
+func (n *dagService) Get(ctx context.Context, k u.Key) (*Node, error) {
 	if n == nil {
 		return nil, fmt.Errorf("dagService is nil")
 	}
 
-	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
-	defer cancel()
-	// we shouldn't use an arbitrary timeout here.
-	// since Get doesnt take in a context yet, we give a large upper bound.
-	// think of an http request. we want it to go on as long as the client requests it.
-
 	b, err := n.Blocks.GetBlock(ctx, k)
 	if err != nil {
 		return nil, err
@@ -134,7 +127,7 @@ func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{}
 				return
 			}
 
-			nd, err := lnk.GetNode(serv)
+			nd, err := lnk.GetNode(ctx, serv)
 			if err != nil {
 				log.Debug(err)
 				return

merkledag_test.go

@@ -190,7 +190,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
-			first, err := dagservs[i].Get(k)
+			first, err := dagservs[i].Get(context.Background(), k)
 			if err != nil {
 				t.Fatal(err)
 			}

node.go

@@ -3,6 +3,8 @@ package merkledag
 import (
 	"fmt"
 
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
 	mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
 	u "github.com/ipfs/go-ipfs/util"
 )
@@ -77,12 +79,12 @@ func MakeLink(n *Node) (*Link, error) {
 }
 
 // GetNode returns the MDAG Node that this link points to
-func (l *Link) GetNode(serv DAGService) (*Node, error) {
+func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
 	if l.Node != nil {
 		return l.Node, nil
 	}
 
-	return serv.Get(u.Key(l.Hash))
+	return serv.Get(ctx, u.Key(l.Hash))
 }
 
 // AddNodeLink adds a link to another node.

traverse/traverse.go

@@ -3,6 +3,9 @@ package traverse
 
 import (
 	"errors"
+	"time"
+
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	mdag "github.com/ipfs/go-ipfs/merkledag"
 )
@@ -64,7 +67,10 @@ func (t *traversal) callFunc(next State) error {
 func (t *traversal) getNode(link *mdag.Link) (*mdag.Node, error) {
 
 	getNode := func(l *mdag.Link) (*mdag.Node, error) {
-		next, err := l.GetNode(t.opts.DAG)
+		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+		defer cancel()
+
+		next, err := l.GetNode(ctx, t.opts.DAG)
 		if err != nil {
 			return nil, err
 		}