godeps: updated datastore

Godeps/Godeps.json

@@ -64,7 +64,7 @@
 		},
 		{
 			"ImportPath": "github.com/jbenet/datastore.go",
-			"Rev": "e89f0511689bb2d0608496e15491f241842de085"
+			"Rev": "e7d6f7cb9e3c207a04c5397c449d10a6f9d403a0"
 		},
 		{
 			"ImportPath": "github.com/jbenet/go-base58",

Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Godeps.json

+{
+	"ImportPath": "github.com/jbenet/datastore.go",
+	"GoVersion": "go1.3.1",
+	"Packages": [
+		"./..."
+	],
+	"Deps": [
+		{
+			"ImportPath": "code.google.com/p/go-uuid/uuid",
+			"Comment": "null-12",
+			"Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9"
+		},
+		{
+			"ImportPath": "code.google.com/p/snappy-go/snappy",
+			"Comment": "null-15",
+			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
+		},
+		{
+			"ImportPath": "github.com/codahale/blake2",
+			"Rev": "3fa823583afba430e8fc7cdbcc670dbf90bfacc4"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/golang-lru",
+			"Rev": "4dfff096c4973178c8f35cf6dd1a732a0a139370"
+		},
+		{
+			"ImportPath": "github.com/mattbaird/elastigo/api",
+			"Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d"
+		},
+		{
+			"ImportPath": "github.com/mattbaird/elastigo/core",
+			"Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d"
+		},
+		{
+			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
+			"Rev": "9bca75c48d6c31becfbb127702b425e7226052e3"
+		},
+		{
+			"ImportPath": "gopkg.in/check.v1",
+			"Rev": "91ae5f88a67b14891cfd43895b01164f6c120420"
+		}
+	]
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/Godeps/Readme

+This directory tree is generated automatically by godep.
+
+Please do not edit.
+
+See https://github.com/tools/godep for more information.

Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds.go

 package datastore
 
-import (
-	"log"
-)
+import "log"
 
 // Here are some basic datastore implementations.
 
-// MapDatastore uses a standard Go map for internal storage.
 type keyMap map[Key]interface{}
+
+// MapDatastore uses a standard Go map for internal storage.
 type MapDatastore struct {
 	values keyMap
 }
 
+// NewMapDatastore constructs a MapDatastore
 func NewMapDatastore() (d *MapDatastore) {
 	return &MapDatastore{
 		values: keyMap{},
 	}
 }
 
+// Put implements Datastore.Put
 func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
 	d.values[key] = value
 	return nil
 }
 
+// Get implements Datastore.Get
 func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
 	val, found := d.values[key]
 	if !found {
@@ -31,19 +33,22 @@ func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
 	return val, nil
 }
 
+// Has implements Datastore.Has
 func (d *MapDatastore) Has(key Key) (exists bool, err error) {
 	_, found := d.values[key]
 	return found, nil
 }
 
+// Delete implements Datastore.Delete
 func (d *MapDatastore) Delete(key Key) (err error) {
 	delete(d.values, key)
 	return nil
 }
 
+// KeyList implements Datastore.KeyList
 func (d *MapDatastore) KeyList() ([]Key, error) {
 	var keys []Key
-	for k, _ := range d.values {
+	for k := range d.values {
 		keys = append(keys, k)
 	}
 	return keys, nil
@@ -54,26 +59,32 @@ func (d *MapDatastore) KeyList() ([]Key, error) {
 type NullDatastore struct {
 }
 
+// NewNullDatastore constructs a null datastoe
 func NewNullDatastore() *NullDatastore {
 	return &NullDatastore{}
 }
 
+// Put implements Datastore.Put
 func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
 	return nil
 }
 
+// Get implements Datastore.Get
 func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
 	return nil, nil
 }
 
+// Has implements Datastore.Has
 func (d *NullDatastore) Has(key Key) (exists bool, err error) {
 	return false, nil
 }
 
+// Delete implements Datastore.Delete
 func (d *NullDatastore) Delete(key Key) (err error) {
 	return nil
 }
 
+// KeyList implements Datastore.KeyList
 func (d *NullDatastore) KeyList() ([]Key, error) {
 	return nil, nil
 }
@@ -81,38 +92,56 @@ func (d *NullDatastore) KeyList() ([]Key, error) {
 // LogDatastore logs all accesses through the datastore.
 type LogDatastore struct {
 	Name  string
-	Child Datastore
+	child Datastore
+}
+
+// Shim is a datastore which has a child.
+type Shim interface {
+	Datastore
+
+	Children() []Datastore
 }
 
-func NewLogDatastore(ds Datastore, name string) *LogDatastore {
+// NewLogDatastore constructs a log datastore.
+func NewLogDatastore(ds Datastore, name string) Shim {
 	if len(name) < 1 {
 		name = "LogDatastore"
 	}
-	return &LogDatastore{Name: name, Child: ds}
+	return &LogDatastore{Name: name, child: ds}
+}
+
+// Children implements Shim
+func (d *LogDatastore) Children() []Datastore {
+	return []Datastore{d.child}
 }
 
+// Put implements Datastore.Put
 func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
-	log.Printf("%s: Put %s", d.Name, key)
+	log.Printf("%s: Put %s\n", d.Name, key)
 	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
-	return d.Child.Put(key, value)
+	return d.child.Put(key, value)
 }
 
+// Get implements Datastore.Get
 func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
-	log.Printf("%s: Get %s", d.Name, key)
-	return d.Child.Get(key)
+	log.Printf("%s: Get %s\n", d.Name, key)
+	return d.child.Get(key)
 }
 
+// Has implements Datastore.Has
 func (d *LogDatastore) Has(key Key) (exists bool, err error) {
-	log.Printf("%s: Has %s", d.Name, key)
-	return d.Child.Has(key)
+	log.Printf("%s: Has %s\n", d.Name, key)
+	return d.child.Has(key)
 }
 
+// Delete implements Datastore.Delete
 func (d *LogDatastore) Delete(key Key) (err error) {
-	log.Printf("%s: Delete %s", d.Name, key)
-	return d.Child.Delete(key)
+	log.Printf("%s: Delete %s\n", d.Name, key)
+	return d.child.Delete(key)
 }
 
+// KeyList implements Datastore.KeyList
 func (d *LogDatastore) KeyList() ([]Key, error) {
-	log.Printf("%s: Get KeyList.", d.Name)
-	return d.Child.KeyList()
+	log.Printf("%s: Get KeyList\n", d.Name)
+	return d.child.KeyList()
 }

Godeps/_workspace/src/github.com/jbenet/datastore.go/basic_ds_test.go

+package datastore_test
+
+import (
+	. "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type BasicSuite struct{}
+
+var _ = Suite(&BasicSuite{})

Godeps/_workspace/src/github.com/jbenet/datastore.go/datastore.go

@@ -5,7 +5,7 @@ import (
 )
 
 /*
-A Datastore represents storage for any key-value pair.
+Datastore represents storage for any key-value pair.
 
 Datastores are general enough to be backed by all kinds of different storage:
 in-memory caches, databases, a remote datastore, flat files on disk, etc.
@@ -27,7 +27,6 @@ and thus it should behave predictably and handle exceptional conditions with
 proper error reporting. Thus, all Datastore calls may return errors, which
 should be checked by callers.
 */
-
 type Datastore interface {
 	// Put stores the object `value` named by `key`.
 	//
@@ -53,20 +52,27 @@ type Datastore interface {
 	// Delete removes the value for given `key`.
 	Delete(key Key) (err error)
 
-	// Returns a list of keys in the datastore
+	// KeyList returns a list of keys in the datastore
 	KeyList() ([]Key, error)
 }
 
+// ThreadSafeDatastore is an interface that all threadsafe datastore should
+// implement to leverage type safety checks.
+type ThreadSafeDatastore interface {
+	Datastore
+	IsThreadSafe()
+}
+
 // Errors
 
 // ErrNotFound is returned by Get, Has, and Delete when a datastore does not
 // map the given key to a value.
-var ErrNotFound = errors.New("datastore: key not found.")
+var ErrNotFound = errors.New("datastore: key not found")
 
 // ErrInvalidType is returned by Put when a given value is incopatible with
 // the type the datastore supports. This means a conversion (or serialization)
 // is needed beforehand.
-var ErrInvalidType = errors.New("datastore: invalid type error.")
+var ErrInvalidType = errors.New("datastore: invalid type error")
 
 // GetBackedHas provides a default Datastore.Has implementation.
 // It exists so Datastore.Has implementations can use it, like so:

Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs.go

+package fs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// Datastore uses a standard Go map for internal storage.
+type Datastore struct {
+	path string
+}
+
+// NewDatastore returns a new fs Datastore at given `path`
+func NewDatastore(path string) (ds.Datastore, error) {
+	if !isDir(path) {
+		return nil, fmt.Errorf("Failed to find directory at: %v (file? perms?)", path)
+	}
+
+	return &Datastore{path: path}, nil
+}
+
+// KeyFilename returns the filename associated with `key`
+func (d *Datastore) KeyFilename(key ds.Key) string {
+	return filepath.Join(d.path, key.String(), ".dsobject")
+}
+
+// Put stores the given value.
+func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
+
+	// TODO: maybe use io.Readers/Writers?
+	// r, err := dsio.CastAsReader(value)
+	// if err != nil {
+	// 	return err
+	// }
+
+	val, ok := value.([]byte)
+	if !ok {
+		return ds.ErrInvalidType
+	}
+
+	fn := d.KeyFilename(key)
+
+	// mkdirall above.
+	err = os.MkdirAll(filepath.Dir(fn), 0755)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(fn, val, 0666)
+}
+
+// Get returns the value for given key
+func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
+	fn := d.KeyFilename(key)
+	if !isFile(fn) {
+		return nil, ds.ErrNotFound
+	}
+
+	return ioutil.ReadFile(fn)
+}
+
+// Has returns whether the datastore has a value for a given key
+func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
+	return ds.GetBackedHas(d, key)
+}
+
+// Delete removes the value for given key
+func (d *Datastore) Delete(key ds.Key) (err error) {
+	fn := d.KeyFilename(key)
+	if !isFile(fn) {
+		return ds.ErrNotFound
+	}
+
+	return os.Remove(fn)
+}
+
+// KeyList returns a list of all keys in the datastore
+func (d *Datastore) KeyList() ([]ds.Key, error) {
+
+	keys := []ds.Key{}
+
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		// remove ds path prefix
+		if strings.HasPrefix(path, d.path) {
+			path = path[len(d.path):]
+		}
+
+		if !info.IsDir() {
+			key := ds.NewKey(path)
+			keys = append(keys, key)
+		}
+		return nil
+	}
+
+	filepath.Walk(d.path, walkFn)
+	return keys, nil
+}
+
+// isDir returns whether given path is a directory
+func isDir(path string) bool {
+	finfo, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	return finfo.IsDir()
+}
+
+// isFile returns whether given path is a file
+func isFile(path string) bool {
+	finfo, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	return !finfo.IsDir()
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/fs/fs_test.go

+package fs_test
+
+import (
+	"bytes"
+	"testing"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	fs "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/fs"
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type DSSuite struct {
+	dir string
+	ds  ds.Datastore
+}
+
+var _ = Suite(&DSSuite{})
+
+func (ks *DSSuite) SetUpTest(c *C) {
+	ks.dir = c.MkDir()
+	ks.ds, _ = fs.NewDatastore(ks.dir)
+}
+
+func (ks *DSSuite) TestOpen(c *C) {
+	_, err := fs.NewDatastore("/tmp/foo/bar/baz")
+	c.Assert(err, Not(Equals), nil)
+
+	// setup ds
+	_, err = fs.NewDatastore(ks.dir)
+	c.Assert(err, Equals, nil)
+}
+
+func (ks *DSSuite) TestBasic(c *C) {
+
+	keys := strsToKeys([]string{
+		"foo",
+		"foo/bar",
+		"foo/bar/baz",
+		"foo/barb",
+		"foo/bar/bazb",
+		"foo/bar/baz/barb",
+	})
+
+	for _, k := range keys {
+		err := ks.ds.Put(k, []byte(k.String()))
+		c.Check(err, Equals, nil)
+	}
+
+	for _, k := range keys {
+		v, err := ks.ds.Get(k)
+		c.Check(err, Equals, nil)
+		c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true)
+	}
+}
+
+func strsToKeys(strs []string) []ds.Key {
+	keys := make([]ds.Key, len(strs))
+	for i, s := range strs {
+		keys[i] = ds.NewKey(s)
+	}
+	return keys
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/io/io.go

+package leveldb
+
+import (
+	"bytes"
+	"io"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// CastAsReader does type assertions to find the type of a value and attempts
+// to turn it into an io.Reader. If not possible, will return ds.ErrInvalidType
+func CastAsReader(value interface{}) (io.Reader, error) {
+	switch v := value.(type) {
+	case io.Reader:
+		return v, nil
+
+	case []byte:
+		return bytes.NewReader(v), nil
+
+	case string:
+		return bytes.NewReader([]byte(v)), nil
+
+	default:
+		return nil, ds.ErrInvalidType
+	}
+}
+
+// // CastAsWriter does type assertions to find the type of a value and attempts
+// // to turn it into an io.Writer. If not possible, will return ds.ErrInvalidType
+// func CastAsWriter(value interface{}) (err error) {
+// 	switch v := value.(type) {
+// 	case io.Reader:
+// 		return v, nil
+//
+// 	case []byte:
+// 		return bytes.NewReader(v), nil
+//
+// 	case string:
+// 		return bytes.NewReader([]byte(v)), nil
+//
+// 	default:
+// 		return nil, ds.ErrInvalidType
+// 	}
+// }

Godeps/_workspace/src/github.com/jbenet/datastore.go/key_test.go

@@ -2,12 +2,13 @@ package datastore_test
 
 import (
 	"bytes"
-	. "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
-	. "launchpad.net/gocheck"
 	"math/rand"
 	"path"
 	"strings"
 	"testing"
+
+	. "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	. "gopkg.in/check.v1"
 )
 
 // Hook up gocheck into the "go test" runner.

Godeps/_workspace/src/github.com/jbenet/datastore.go/leveldb/datastore.go

@@ -13,7 +13,7 @@ type Datastore struct {
 
 type Options opt.Options
 
-func NewDatastore(path string, opts *Options) (*Datastore, error) {
+func NewDatastore(path string, opts *Options) (ds.ThreadSafeDatastore, error) {
 	var nopts opt.Options
 	if opts != nil {
 		nopts = opt.Options(*opts)
@@ -76,3 +76,5 @@ func (d *Datastore) KeyList() ([]ds.Key, error) {
 func (d *Datastore) Close() (err error) {
 	return d.DB.Close()
 }
+
+func (d *Datastore) IsThreadSafe() {}

Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore.go

+package lru
+
+import (
+	"errors"
+
+	lru "github.com/hashicorp/golang-lru"
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// Datastore uses golang-lru for internal storage.
+type Datastore struct {
+	cache *lru.Cache
+}
+
+// NewDatastore constructs a new LRU Datastore with given capacity.
+func NewDatastore(capacity int) (*Datastore, error) {
+	cache, err := lru.New(capacity)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Datastore{cache: cache}, nil
+}
+
+// Put stores the object `value` named by `key`.
+func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
+	d.cache.Add(key, value)
+	return nil
+}
+
+// Get retrieves the object `value` named by `key`.
+func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
+	val, ok := d.cache.Get(key)
+	if !ok {
+		return nil, ds.ErrNotFound
+	}
+	return val, nil
+}
+
+// Has returns whether the `key` is mapped to a `value`.
+func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
+	return ds.GetBackedHas(d, key)
+}
+
+// Delete removes the value for given `key`.
+func (d *Datastore) Delete(key ds.Key) (err error) {
+	d.cache.Remove(key)
+	return nil
+}
+
+// KeyList returns a list of keys in the datastore
+func (d *Datastore) KeyList() ([]ds.Key, error) {
+	return nil, errors.New("KeyList not implemented.")
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/lru/datastore_test.go

+package lru_test
+
+import (
+	"strconv"
+	"testing"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	lru "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/lru"
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type DSSuite struct{}
+
+var _ = Suite(&DSSuite{})
+
+func (ks *DSSuite) TestBasic(c *C) {
+	var size = 1000
+
+	d, err := lru.NewDatastore(size)
+	c.Check(err, Equals, nil)
+
+	for i := 0; i < size; i++ {
+		err := d.Put(ds.NewKey(strconv.Itoa(i)), i)
+		c.Check(err, Equals, nil)
+	}
+
+	for i := 0; i < size; i++ {
+		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
+		c.Check(j, Equals, i)
+		c.Check(err, Equals, nil)
+	}
+
+	for i := 0; i < size; i++ {
+		err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i)
+		c.Check(err, Equals, nil)
+	}
+
+	for i := 0; i < size; i++ {
+		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
+		c.Check(j, Equals, nil)
+		c.Check(err, Equals, ds.ErrNotFound)
+	}
+
+	for i := 0; i < size; i++ {
+		j, err := d.Get(ds.NewKey(strconv.Itoa(i + size)))
+		c.Check(j, Equals, i)
+		c.Check(err, Equals, nil)
+	}
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/query.go

+package datastore
+
+// type KeyIterator struct {
+//   HasNext() bool
+//   Next() interface{}
+// }
+
+// type Query struct {
+// }
+
+/*
+QueryDatastores support a Query interface. Queries are used to support
+searching for values (beyond simple key-based `Get`s).
+*/
+// type QueryDatastore interface {
+//   // Query returns an Iterator of Keys whose Values match criteria
+//   // expressed in `query`.
+//   Query(Query) (iter Iterator, err error)
+// }

Godeps/_workspace/src/github.com/jbenet/datastore.go/sync/sync.go

+package sync
+
+import (
+	"sync"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+)
+
+// MutexDatastore contains a child datastire and a mutex.
+// used for coarse sync
+type MutexDatastore struct {
+	sync.RWMutex
+
+	child ds.Datastore
+}
+
+// MutexWrap constructs a datastore with a coarse lock around
+// the entire datastore, for every single operation
+func MutexWrap(d ds.Datastore) ds.ThreadSafeDatastore {
+	return &MutexDatastore{child: d}
+}
+
+// Children implements Shim
+func (d *MutexDatastore) Children() []ds.Datastore {
+	return []ds.Datastore{d.child}
+}
+
+// IsThreadSafe implements ThreadSafeDatastore
+func (d *MutexDatastore) IsThreadSafe() {}
+
+// Put implements Datastore.Put
+func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) {
+	d.Lock()
+	defer d.Unlock()
+	return d.child.Put(key, value)
+}
+
+// Get implements Datastore.Get
+func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) {
+	d.RLock()
+	defer d.RUnlock()
+	return d.child.Get(key)
+}
+
+// Has implements Datastore.Has
+func (d *MutexDatastore) Has(key ds.Key) (exists bool, err error) {
+	d.RLock()
+	defer d.RUnlock()
+	return d.child.Has(key)
+}
+
+// Delete implements Datastore.Delete
+func (d *MutexDatastore) Delete(key ds.Key) (err error) {
+	d.Lock()
+	defer d.Unlock()
+	return d.child.Delete(key)
+}
+
+// KeyList implements Datastore.KeyList
+func (d *MutexDatastore) KeyList() ([]ds.Key, error) {
+	d.RLock()
+	defer d.RUnlock()
+	return d.child.KeyList()
+}