From da976a5f2159151859e6de4b70e96a50023b6496 Mon Sep 17 00:00:00 2001
From: Juan Batiz-Benet <juan@benet.ai>
Date: Fri, 9 Jan 2015 17:39:56 -0800
Subject: [PATCH] blocks: AllKeys + tests
---
blocks/blockstore/blockstore.go | 35 ++++++++++++++++-
blocks/blockstore/blockstore_test.go | 59 +++++++++++++++++++++++++++-
blocks/blockstore/write_cache.go | 4 ++
util/key.go | 9 ++++-
4 files changed, 103 insertions(+), 4 deletions(-)
diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go
index e203ffc50..6132d155e 100644
--- a/blocks/blockstore/blockstore.go
+++ b/blocks/blockstore/blockstore.go
@@ -6,12 +6,17 @@ import (
"errors"
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+ dsns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
+ dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
blocks "github.com/jbenet/go-ipfs/blocks"
u "github.com/jbenet/go-ipfs/util"
)
+// BlockPrefix namespaces blockstore datastores
+var BlockPrefix = ds.NewKey("blocks")
+
var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
var ErrNotFound = errors.New("blockstore: block not found")
@@ -22,16 +27,20 @@ type Blockstore interface {
Has(u.Key) (bool, error)
Get(u.Key) (*blocks.Block, error)
Put(*blocks.Block) error
+ AllKeys(offset int, limit int) ([]u.Key, error)
}
func NewBlockstore(d ds.ThreadSafeDatastore) Blockstore {
+ dd := dsns.Wrap(d, BlockPrefix)
return &blockstore{
- datastore: d,
+ datastore: dd,
}
}
type blockstore struct {
- datastore ds.ThreadSafeDatastore
+ datastore ds.Datastore
+ // cant be ThreadSafeDatastore cause namespace.Datastore doesnt support it.
+ // we do check it on `NewBlockstore` though.
}
func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) {
@@ -67,3 +76,25 @@ func (bs *blockstore) Has(k u.Key) (bool, error) {
func (s *blockstore) DeleteBlock(k u.Key) error {
return s.datastore.Delete(k.DsKey())
}
+
+// AllKeys runs a query for keys from the blockstore.
+// this is very simplistic, in the future, take dsq.Query as a param?
+// if offset and limit are 0, they are ignored.
+func (bs *blockstore) AllKeys(offset int, limit int) ([]u.Key, error) {
+ var keys []u.Key
+
+ // TODO make async inside ds/leveldb.Query
+ // KeysOnly, because that would be _a lot_ of data.
+ q := dsq.Query{KeysOnly: true, Offset: offset, Limit: limit}
+ res, err := bs.datastore.Query(q)
+ if err != nil {
+ return nil, err
+ }
+
+ for e := range res.Entries() {
+ // need to convert to u.Key using u.KeyFromDsKey.
+ k := u.KeyFromDsKey(ds.NewKey(e.Key))
+ keys = append(keys, k)
+ }
+ return keys, nil
+}
diff --git a/blocks/blockstore/blockstore_test.go b/blocks/blockstore/blockstore_test.go
index 00edf61ab..a80ce8337 100644
--- a/blocks/blockstore/blockstore_test.go
+++ b/blocks/blockstore/blockstore_test.go
@@ -2,6 +2,7 @@ package blockstore
import (
"bytes"
+ "fmt"
"testing"
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@@ -41,11 +42,49 @@ func TestPutThenGetBlock(t *testing.T) {
}
}
+func TestAllKeys(t *testing.T) {
+ bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+ N := 100
+
+ keys := make([]u.Key, N)
+ for i := 0; i < N; i++ {
+ block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
+ err := bs.Put(block)
+ if err != nil {
+ t.Fatal(err)
+ }
+ keys[i] = block.Key()
+ }
+
+ keys2, err := bs.AllKeys(0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // for _, k2 := range keys2 {
+ // t.Log("found ", k2.Pretty())
+ // }
+
+ expectMatches(t, keys, keys2)
+
+ keys3, err := bs.AllKeys(N/3, N/3)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, k3 := range keys3 {
+ t.Log("found ", k3.Pretty())
+ }
+ if len(keys3) != N/3 {
+ t.Errorf("keys3 should be: %d != %d", N/3, len(keys3))
+ }
+
+}
+
func TestValueTypeMismatch(t *testing.T) {
block := blocks.NewBlock([]byte("some data"))
datastore := ds.NewMapDatastore()
- datastore.Put(block.Key().DsKey(), "data that isn't a block!")
+ k := BlockPrefix.Child(block.Key().DsKey())
+ datastore.Put(k, "data that isn't a block!")
blockstore := NewBlockstore(ds_sync.MutexWrap(datastore))
@@ -54,3 +93,21 @@ func TestValueTypeMismatch(t *testing.T) {
t.Fatal(err)
}
}
+
+func expectMatches(t *testing.T, expect, actual []u.Key) {
+
+ if len(expect) != len(actual) {
+ t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual))
+ }
+ for _, ek := range expect {
+ found := false
+ for _, ak := range actual {
+ if ek == ak {
+ found = true
+ }
+ }
+ if !found {
+ t.Error("expected key not found: ", ek)
+ }
+ }
+}
diff --git a/blocks/blockstore/write_cache.go b/blocks/blockstore/write_cache.go
index b46d05846..da9a0a01d 100644
--- a/blocks/blockstore/write_cache.go
+++ b/blocks/blockstore/write_cache.go
@@ -43,3 +43,7 @@ func (w *writecache) Put(b *blocks.Block) error {
w.cache.Add(b.Key(), struct{}{})
return w.blockstore.Put(b)
}
+
+func (w *writecache) AllKeys(offset int, limit int) ([]u.Key, error) {
+ return w.blockstore.AllKeys(offset, limit)
+}
diff --git a/util/key.go b/util/key.go
index eca1255b5..445181b75 100644
--- a/util/key.go
+++ b/util/key.go
@@ -71,7 +71,7 @@ func (k *Key) Loggable() map[string]interface{} {
// KeyFromDsKey returns a Datastore key
func KeyFromDsKey(dsk ds.Key) Key {
- return Key(dsk.BaseNamespace())
+ return Key(dsk.String()[1:])
}
// B58KeyConverter -- for KeyTransform datastores
@@ -131,3 +131,10 @@ func XOR(a, b []byte) []byte {
}
return c
}
+
+// KeySlice is used for sorting Keys
+type KeySlice []Key
+
+func (es KeySlice) Len() int { return len(es) }
+func (es KeySlice) Swap(i, j int) { es[i], es[j] = es[j], es[i] }
+func (es KeySlice) Less(i, j int) bool { return es[i] < es[j] }
--
GitLab