blocks: AllKeys + tests

blockstore.go

@@ -6,12 +6,17 @@ import (
 	"errors"
 
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+	dsns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
+	dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
 	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
 
 	blocks "github.com/jbenet/go-ipfs/blocks"
 	u "github.com/jbenet/go-ipfs/util"
 )
 
+// BlockPrefix namespaces blockstore datastores
+var BlockPrefix = ds.NewKey("blocks")
+
 var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
 
 var ErrNotFound = errors.New("blockstore: block not found")
@@ -22,16 +27,20 @@ type Blockstore interface {
 	Has(u.Key) (bool, error)
 	Get(u.Key) (*blocks.Block, error)
 	Put(*blocks.Block) error
+	AllKeys(offset int, limit int) ([]u.Key, error)
 }
 
 func NewBlockstore(d ds.ThreadSafeDatastore) Blockstore {
+	dd := dsns.Wrap(d, BlockPrefix)
 	return &blockstore{
-		datastore: d,
+		datastore: dd,
 	}
 }
 
 type blockstore struct {
-	datastore ds.ThreadSafeDatastore
+	datastore ds.Datastore
+	// cant be ThreadSafeDatastore cause namespace.Datastore doesnt support it.
+	// we do check it on `NewBlockstore` though.
 }
 
 func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) {
@@ -67,3 +76,25 @@ func (bs *blockstore) Has(k u.Key) (bool, error) {
 func (s *blockstore) DeleteBlock(k u.Key) error {
 	return s.datastore.Delete(k.DsKey())
 }
+
+// AllKeys runs a query for keys from the blockstore.
+// this is very simplistic, in the future, take dsq.Query as a param?
+// if offset and limit are 0, they are ignored.
+func (bs *blockstore) AllKeys(offset int, limit int) ([]u.Key, error) {
+	var keys []u.Key
+
+	// TODO make async inside ds/leveldb.Query
+	// KeysOnly, because that would be _a lot_ of data.
+	q := dsq.Query{KeysOnly: true, Offset: offset, Limit: limit}
+	res, err := bs.datastore.Query(q)
+	if err != nil {
+		return nil, err
+	}
+
+	for e := range res.Entries() {
+		// need to convert to u.Key using u.KeyFromDsKey.
+		k := u.KeyFromDsKey(ds.NewKey(e.Key))
+		keys = append(keys, k)
+	}
+	return keys, nil
+}

blockstore_test.go

@@ -2,6 +2,7 @@ package blockstore
 
 import (
 	"bytes"
+	"fmt"
 	"testing"
 
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@@ -41,11 +42,49 @@ func TestPutThenGetBlock(t *testing.T) {
 	}
 }
 
+func TestAllKeys(t *testing.T) {
+	bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+	N := 100
+
+	keys := make([]u.Key, N)
+	for i := 0; i < N; i++ {
+		block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
+		err := bs.Put(block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		keys[i] = block.Key()
+	}
+
+	keys2, err := bs.AllKeys(0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// for _, k2 := range keys2 {
+	// 	t.Log("found ", k2.Pretty())
+	// }
+
+	expectMatches(t, keys, keys2)
+
+	keys3, err := bs.AllKeys(N/3, N/3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, k3 := range keys3 {
+		t.Log("found ", k3.Pretty())
+	}
+	if len(keys3) != N/3 {
+		t.Errorf("keys3 should be: %d != %d", N/3, len(keys3))
+	}
+
+}
+
 func TestValueTypeMismatch(t *testing.T) {
 	block := blocks.NewBlock([]byte("some data"))
 
 	datastore := ds.NewMapDatastore()
-	datastore.Put(block.Key().DsKey(), "data that isn't a block!")
+	k := BlockPrefix.Child(block.Key().DsKey())
+	datastore.Put(k, "data that isn't a block!")
 
 	blockstore := NewBlockstore(ds_sync.MutexWrap(datastore))
 
@@ -54,3 +93,21 @@ func TestValueTypeMismatch(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func expectMatches(t *testing.T, expect, actual []u.Key) {
+
+	if len(expect) != len(actual) {
+		t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual))
+	}
+	for _, ek := range expect {
+		found := false
+		for _, ak := range actual {
+			if ek == ak {
+				found = true
+			}
+		}
+		if !found {
+			t.Error("expected key not found: ", ek)
+		}
+	}
+}

write_cache.go

@@ -43,3 +43,7 @@ func (w *writecache) Put(b *blocks.Block) error {
 	w.cache.Add(b.Key(), struct{}{})
 	return w.blockstore.Put(b)
 }
+
+func (w *writecache) AllKeys(offset int, limit int) ([]u.Key, error) {
+	return w.blockstore.AllKeys(offset, limit)
+}