Commit 4a973ddc authored by Steven Allen's avatar Steven Allen

switch the datastore to using bytes

None of our actual datastores support any other types and the cast noise gets
really annoying after a while.
parent e80704d5
......@@ -13,7 +13,7 @@ type Datastore struct {
child ds.Batching
// TODO: discuss making ds.Batch implement the full ds.Datastore interface
buffer map[ds.Key]interface{}
buffer map[ds.Key][]byte
maxBufferEntries int
}
......@@ -23,7 +23,7 @@ type Datastore struct {
func NewAutoBatching(d ds.Batching, size int) *Datastore {
return &Datastore{
child: d,
buffer: make(map[ds.Key]interface{}),
buffer: make(map[ds.Key][]byte),
maxBufferEntries: size,
}
}
......@@ -36,7 +36,7 @@ func (d *Datastore) Delete(k ds.Key) error {
}
// Get retrieves a value given a key.
func (d *Datastore) Get(k ds.Key) (interface{}, error) {
func (d *Datastore) Get(k ds.Key) ([]byte, error) {
val, ok := d.buffer[k]
if ok {
return val, nil
......@@ -46,7 +46,7 @@ func (d *Datastore) Get(k ds.Key) (interface{}, error) {
}
// Put stores a key/value.
func (d *Datastore) Put(k ds.Key, val interface{}) error {
func (d *Datastore) Put(k ds.Key, val []byte) error {
d.buffer[k] = val
if len(d.buffer) > d.maxBufferEntries {
return d.Flush()
......@@ -68,7 +68,7 @@ func (d *Datastore) Flush() error {
}
}
// clear out buffer
d.buffer = make(map[ds.Key]interface{})
d.buffer = make(map[ds.Key][]byte)
return b.Commit()
}
......
package autobatch
import (
"bytes"
"fmt"
"testing"
......@@ -11,7 +12,7 @@ func TestBasicPuts(t *testing.T) {
d := NewAutoBatching(ds.NewMapDatastore(), 16)
k := ds.NewKey("test")
v := "hello world"
v := []byte("hello world")
err := d.Put(k, v)
if err != nil {
......@@ -23,7 +24,7 @@ func TestBasicPuts(t *testing.T) {
t.Fatal(err)
}
if out != v {
if !bytes.Equal(out, v) {
t.Fatal("wasnt the same! ITS NOT THE SAME")
}
}
......@@ -36,7 +37,7 @@ func TestFlushing(t *testing.T) {
for i := 0; i < 16; i++ {
keys = append(keys, ds.NewKey(fmt.Sprintf("test%d", i)))
}
v := "hello world"
v := []byte("hello world")
for _, k := range keys {
err := d.Put(k, v)
......@@ -62,8 +63,8 @@ func TestFlushing(t *testing.T) {
t.Fatal(err)
}
if val != v {
t.Fatal(err)
if !bytes.Equal(val, v) {
t.Fatal("wrong value")
}
}
}
......@@ -9,28 +9,26 @@ import (
// Here are some basic datastore implementations.
type keyMap map[Key]interface{}
// MapDatastore uses a standard Go map for internal storage.
type MapDatastore struct {
values keyMap
values map[Key][]byte
}
// NewMapDatastore constructs a MapDatastore
func NewMapDatastore() (d *MapDatastore) {
return &MapDatastore{
values: keyMap{},
values: make(map[Key][]byte),
}
}
// Put implements Datastore.Put
func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
func (d *MapDatastore) Put(key Key, value []byte) (err error) {
d.values[key] = value
return nil
}
// Get implements Datastore.Get
func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
func (d *MapDatastore) Get(key Key) (value []byte, err error) {
val, found := d.values[key]
if !found {
return nil, ErrNotFound
......@@ -83,12 +81,12 @@ func NewNullDatastore() *NullDatastore {
}
// Put implements Datastore.Put
func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
func (d *NullDatastore) Put(key Key, value []byte) (err error) {
return nil
}
// Get implements Datastore.Get
func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
func (d *NullDatastore) Get(key Key) (value []byte, err error) {
return nil, ErrNotFound
}
......@@ -142,14 +140,14 @@ func (d *LogDatastore) Children() []Datastore {
}
// Put implements Datastore.Put
func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
func (d *LogDatastore) Put(key Key, value []byte) (err error) {
log.Printf("%s: Put %s\n", d.Name, key)
// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
return d.child.Put(key, value)
}
// Get implements Datastore.Get
func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
func (d *LogDatastore) Get(key Key) (value []byte, err error) {
log.Printf("%s: Get %s\n", d.Name, key)
return d.child.Get(key)
}
......@@ -207,7 +205,7 @@ func (d *LogDatastore) Batch() (Batch, error) {
}
// Put implements Batch.Put
func (d *LogBatch) Put(key Key, value interface{}) (err error) {
func (d *LogBatch) Put(key Key, value []byte) (err error) {
log.Printf("%s: BatchPut %s\n", d.Name, key)
// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
return d.child.Put(key, value)
......
......@@ -3,7 +3,7 @@ package datastore
// basicBatch implements the transaction interface for datastores who do
// not have any sort of underlying transactional support
type basicBatch struct {
puts map[Key]interface{}
puts map[Key][]byte
deletes map[Key]struct{}
target Datastore
......@@ -11,13 +11,13 @@ type basicBatch struct {
func NewBasicBatch(ds Datastore) Batch {
return &basicBatch{
puts: make(map[Key]interface{}),
puts: make(map[Key][]byte),
deletes: make(map[Key]struct{}),
target: ds,
}
}
func (bt *basicBatch) Put(key Key, val interface{}) error {
func (bt *basicBatch) Put(key Key, val []byte) error {
bt.puts[key] = val
return nil
}
......
......@@ -39,11 +39,11 @@ type Datastore interface {
// Ultimately, the lowest-level datastore will need to do some value checking
// or risk getting incorrect values. It may also be useful to expose a more
// type-safe interface to your application, and do the checking up-front.
Put(key Key, value interface{}) error
Put(key Key, value []byte) error
// Get retrieves the object `value` named by `key`.
// Get will return ErrNotFound if the key is not mapped to a value.
Get(key Key) (value interface{}, err error)
Get(key Key) (value []byte, err error)
// Has returns whether the `key` is mapped to a `value`.
// In some contexts, it may be much cheaper only to check for existence of
......@@ -160,7 +160,7 @@ func GetBackedHas(ds Datastore, key Key) (bool, error) {
}
type Batch interface {
Put(key Key, val interface{}) error
Put(key Key, val []byte) error
Delete(key Key) error
......
......@@ -18,12 +18,12 @@ type delayed struct {
delay delay.D
}
func (dds *delayed) Put(key ds.Key, value interface{}) (err error) {
func (dds *delayed) Put(key ds.Key, value []byte) (err error) {
dds.delay.Wait()
return dds.ds.Put(key, value)
}
func (dds *delayed) Get(key ds.Key) (value interface{}, err error) {
func (dds *delayed) Get(key ds.Key) (value []byte, err error) {
dds.delay.Wait()
return dds.ds.Get(key)
}
......
......@@ -12,7 +12,7 @@ func TestDelayed(t *testing.T) {
d := New(datastore.NewMapDatastore(), delay.Fixed(time.Second))
now := time.Now()
k := datastore.NewKey("test")
err := d.Put(k, "value")
err := d.Put(k, []byte("value"))
if err != nil {
t.Fatal(err)
}
......
......@@ -51,19 +51,7 @@ func (d *Datastore) KeyFilename(key ds.Key) string {
}
// Put stores the given value.
func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
// TODO: maybe use io.Readers/Writers?
// r, err := dsio.CastAsReader(value)
// if err != nil {
// return err
// }
val, ok := value.([]byte)
if !ok {
return ds.ErrInvalidType
}
func (d *Datastore) Put(key ds.Key, value []byte) (err error) {
fn := d.KeyFilename(key)
// mkdirall above.
......@@ -72,11 +60,11 @@ func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
return err
}
return ioutil.WriteFile(fn, val, 0666)
return ioutil.WriteFile(fn, value, 0666)
}
// Get returns the value for given key
func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
func (d *Datastore) Get(key ds.Key) (value []byte, err error) {
fn := d.KeyFilename(key)
if !isFile(fn) {
return nil, ds.ErrNotFound
......@@ -115,9 +103,13 @@ func (d *Datastore) Query(q query.Query) (query.Results, error) {
if strings.HasSuffix(path, ObjectKeySuffix) {
path = path[:len(path)-len(ObjectKeySuffix)]
}
var result query.Result
key := ds.NewKey(path)
entry := query.Entry{Key: key.String(), Value: query.NotFetched}
results <- query.Result{Entry: entry}
result.Entry.Key = key.String()
if !q.KeysOnly {
result.Entry.Value, result.Error = d.Get(key)
}
results <- result
}
return nil
}
......
......@@ -53,7 +53,7 @@ func (ks *DSSuite) TestBasic(c *C) {
for _, k := range keys {
v, err := ks.ds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true)
c.Check(bytes.Equal(v, []byte(k.String())), Equals, true)
}
r, err := ks.ds.Query(query.Query{Prefix: "/foo/bar/"})
......
......@@ -27,7 +27,7 @@ func NewFailstore(c ds.Datastore, efunc func(string) error) *Failstore {
}
// Put puts a key/value into the datastore.
func (d *Failstore) Put(k ds.Key, val interface{}) error {
func (d *Failstore) Put(k ds.Key, val []byte) error {
err := d.errfunc("put")
if err != nil {
return err
......@@ -37,7 +37,7 @@ func (d *Failstore) Put(k ds.Key, val interface{}) error {
}
// Get retrieves a value from the datastore.
func (d *Failstore) Get(k ds.Key) (interface{}, error) {
func (d *Failstore) Get(k ds.Key) ([]byte, error) {
err := d.errfunc("get")
if err != nil {
return nil, err
......@@ -108,7 +108,7 @@ func (d *Failstore) Batch() (ds.Batch, error) {
}
// Put does a batch put.
func (b *FailBatch) Put(k ds.Key, val interface{}) error {
func (b *FailBatch) Put(k ds.Key, val []byte) error {
if err := b.dstore.errfunc("batch-put"); err != nil {
return err
}
......
......@@ -33,12 +33,12 @@ func (d *ktds) Children() []ds.Datastore {
}
// Put stores the given value, transforming the key first.
func (d *ktds) Put(key ds.Key, value interface{}) (err error) {
func (d *ktds) Put(key ds.Key, value []byte) (err error) {
return d.child.Put(d.ConvertKey(key), value)
}
// Get returns the value for given key, transforming the key first.
func (d *ktds) Get(key ds.Key) (value interface{}, err error) {
func (d *ktds) Get(key ds.Key) (value []byte, err error) {
return d.child.Get(d.ConvertKey(key))
}
......@@ -111,7 +111,7 @@ type transformBatch struct {
f KeyMapping
}
func (t *transformBatch) Put(key ds.Key, val interface{}) error {
func (t *transformBatch) Put(key ds.Key, val []byte) error {
return t.dst.Put(t.f(key), val)
}
......
......@@ -56,11 +56,11 @@ func (ks *DSSuite) TestBasic(c *C) {
for _, k := range keys {
v1, err := ktds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
c.Check(bytes.Equal(v1, []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey("abc").Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
c.Check(bytes.Equal(v2, []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
......
......@@ -82,7 +82,7 @@ func (d *Datastore) lookupAll(key ds.Key) (dst []ds.Datastore, mountpoint, rest
return dst, mountpoint, rest
}
func (d *Datastore) Put(key ds.Key, value interface{}) error {
func (d *Datastore) Put(key ds.Key, value []byte) error {
cds, _, k := d.lookup(key)
if cds == nil {
return ErrNoMount
......@@ -90,7 +90,7 @@ func (d *Datastore) Put(key ds.Key, value interface{}) error {
return cds.Put(k, value)
}
func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
func (d *Datastore) Get(key ds.Key) (value []byte, err error) {
cds, _, k := d.lookup(key)
if cds == nil {
return nil, ds.ErrNotFound
......@@ -246,7 +246,7 @@ func (mt *mountBatch) lookupBatch(key ds.Key) (ds.Batch, ds.Key, error) {
return t, rest, nil
}
func (mt *mountBatch) Put(key ds.Key, val interface{}) error {
func (mt *mountBatch) Put(key ds.Key, val []byte) error {
t, rest, err := mt.lookupBatch(key)
if err != nil {
return err
......
......@@ -41,14 +41,10 @@ func TestPut(t *testing.T) {
t.Fatalf("Put error: %v", err)
}
val, err := mapds.Get(datastore.NewKey("/thud"))
buf, err := mapds.Get(datastore.NewKey("/thud"))
if err != nil {
t.Fatalf("Get error: %v", err)
}
buf, ok := val.([]byte)
if !ok {
t.Fatalf("Get value is not []byte: %T %v", val, val)
}
if g, e := string(buf), "foobar"; g != e {
t.Errorf("wrong value: %q != %q", g, e)
}
......@@ -97,15 +93,10 @@ func TestGet(t *testing.T) {
t.Fatalf("Get error: %v", err)
}
val, err := m.Get(datastore.NewKey("/quux/thud"))
buf, err := m.Get(datastore.NewKey("/quux/thud"))
if err != nil {
t.Fatalf("Put error: %v", err)
}
buf, ok := val.([]byte)
if !ok {
t.Fatalf("Get value is not []byte: %T %v", val, val)
}
if g, e := string(buf), "foobar"; g != e {
t.Errorf("wrong value: %q != %q", g, e)
}
......@@ -259,11 +250,11 @@ func TestQueryCross(t *testing.T) {
{Prefix: datastore.NewKey("/"), Datastore: mapds0},
})
m.Put(datastore.NewKey("/foo/lorem"), "123")
m.Put(datastore.NewKey("/bar/ipsum"), "234")
m.Put(datastore.NewKey("/bar/dolor"), "345")
m.Put(datastore.NewKey("/baz/sit"), "456")
m.Put(datastore.NewKey("/banana"), "567")
m.Put(datastore.NewKey("/foo/lorem"), []byte("123"))
m.Put(datastore.NewKey("/bar/ipsum"), []byte("234"))
m.Put(datastore.NewKey("/bar/dolor"), []byte("345"))
m.Put(datastore.NewKey("/baz/sit"), []byte("456"))
m.Put(datastore.NewKey("/banana"), []byte("567"))
res, err := m.Query(query.Query{Prefix: "/ba"})
if err != nil {
......@@ -288,7 +279,7 @@ func TestQueryCross(t *testing.T) {
t.Errorf("unexpected key %s", e.Key)
}
if v != e.Value {
if v != string(e.Value) {
t.Errorf("key value didn't match expected %s: '%s' - '%s'", e.Key, v, e.Value)
}
......@@ -315,8 +306,8 @@ func TestLookupPrio(t *testing.T) {
{Prefix: datastore.NewKey("/foo"), Datastore: mapds1},
})
m.Put(datastore.NewKey("/foo/bar"), "123")
m.Put(datastore.NewKey("/baz"), "234")
m.Put(datastore.NewKey("/foo/bar"), []byte("123"))
m.Put(datastore.NewKey("/baz"), []byte("234"))
found, err := mapds0.Has(datastore.NewKey("/baz"))
if err != nil {
......@@ -360,7 +351,7 @@ func TestErrQueryClose(t *testing.T) {
{Prefix: datastore.NewKey("/foo"), Datastore: eqds},
})
m.Put(datastore.NewKey("/baz"), "123")
m.Put(datastore.NewKey("/baz"), []byte("123"))
qr, err := m.Query(query.Query{})
if err != nil {
......
......@@ -14,7 +14,7 @@ func Example() {
k := ds.NewKey("/beep")
v := "boop"
ns.Put(k, v)
ns.Put(k, []byte(v))
fmt.Printf("ns.Put %s %s\n", k, v)
v2, _ := ns.Get(k)
......
......@@ -47,11 +47,11 @@ func (ks *DSSuite) testBasic(c *C, prefix string) {
for _, k := range keys {
v1, err := nsds.Get(k)
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
c.Check(bytes.Equal(v1, []byte(k.String())), Equals, true)
v2, err := mpds.Get(ds.NewKey(prefix).Child(k))
c.Check(err, Equals, nil)
c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
c.Check(bytes.Equal(v2, []byte(k.String())), Equals, true)
}
run := func(d ds.Datastore, q dsq.Query) []ds.Key {
......@@ -112,9 +112,7 @@ func (ks *DSSuite) TestQuery(c *C) {
for i, ent := range results {
c.Check(ent.Key, Equals, expect[i].Key)
entval, _ := ent.Value.([]byte)
expval, _ := expect[i].Value.([]byte)
c.Check(string(entval), Equals, string(expval))
c.Check(string(ent.Value), Equals, string(expect[i].Value))
}
err = qres.Close()
......@@ -134,9 +132,7 @@ func (ks *DSSuite) TestQuery(c *C) {
for i, ent := range results {
c.Check(ent.Key, Equals, expect[i].Key)
entval, _ := ent.Value.([]byte)
expval, _ := expect[i].Value.([]byte)
c.Check(string(entval), Equals, string(expval))
c.Check(string(ent.Value), Equals, string(expect[i].Value))
}
if err := nsds.Datastore.(ds.CheckedDatastore).Check(); err != dstest.TestError {
......
......@@ -64,16 +64,10 @@ type Query struct {
KeysOnly bool // return only keys.
}
// NotFetched is a special type that signals whether or not the value
// of an Entry has been fetched or not. This is needed because
// datastore implementations get to decide whether Query returns values
// or only keys. nil is not a good signal, as real values may be nil.
const NotFetched int = iota
// Entry is a query result entry.
type Entry struct {
Key string // cant be ds.Key because circular imports ...!!!
Value interface{}
Value []byte // Will be nil if KeysOnly has been passed.
}
// Result is a special entry that includes an error, so that the client
......
......@@ -118,7 +118,7 @@ func NaiveQueryApply(q Query, qr Results) Results {
return qr
}
func ResultEntriesFrom(keys []string, vals []interface{}) []Entry {
func ResultEntriesFrom(keys []string, vals [][]byte) []Entry {
re := make([]Entry, len(keys))
for i, k := range keys {
re[i] = Entry{Key: k, Value: vals[i]}
......
......@@ -55,8 +55,8 @@ func (d *Datastore) DiskUsage() (uint64, error) {
}
// Get retrieves a value given a key.
func (d *Datastore) Get(k ds.Key) (interface{}, error) {
var val interface{}
func (d *Datastore) Get(k ds.Key) ([]byte, error) {
var val []byte
err := d.runOp(func() error {
var err error
val, err = d.Batching.Get(k)
......@@ -67,7 +67,7 @@ func (d *Datastore) Get(k ds.Key) (interface{}, error) {
}
// Put stores a key/value.
func (d *Datastore) Put(k ds.Key, val interface{}) error {
func (d *Datastore) Put(k ds.Key, val []byte) error {
return d.runOp(func() error {
return d.Batching.Put(k, val)
})
......
......@@ -142,7 +142,7 @@ func TestSuccessAfterTemp(t *testing.T) {
t.Fatal(err)
}
if string(out.([]byte)) != string(val) {
if string(out) != string(val) {
t.Fatal("got wrong value")
}
}
......@@ -31,14 +31,14 @@ func (d *MutexDatastore) Children() []ds.Datastore {
func (d *MutexDatastore) IsThreadSafe() {}
// Put implements Datastore.Put
func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) {
func (d *MutexDatastore) Put(key ds.Key, value []byte) (err error) {
d.Lock()
defer d.Unlock()
return d.child.Put(key, value)
}
// Get implements Datastore.Get
func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) {
func (d *MutexDatastore) Get(key ds.Key) (value []byte, err error) {
d.RLock()
defer d.RUnlock()
return d.child.Get(key)
......@@ -104,7 +104,7 @@ type syncBatch struct {
mds *MutexDatastore
}
func (b *syncBatch) Put(key ds.Key, val interface{}) error {
func (b *syncBatch) Put(key ds.Key, val []byte) error {
b.mds.Lock()
defer b.mds.Unlock()
return b.batch.Put(key, val)
......
......@@ -34,13 +34,8 @@ func SubtestBasicPutGet(t *testing.T, ds dstore.Datastore) {
t.Fatal("error getting value after put: ", err)
}
outb, ok := out.([]byte)
if !ok {
t.Fatalf("output type wasnt []byte, it was %T", out)
}
if !bytes.Equal(outb, val) {
t.Fatal("value received on get wasnt what we expected:", outb)
if !bytes.Equal(out, val) {
t.Fatal("value received on get wasnt what we expected:", out)
}
have, err = ds.Has(k)
......@@ -118,12 +113,7 @@ func SubtestManyKeysAndQuery(t *testing.T, ds dstore.Datastore) {
t.Fatalf("error on get[%d]: %s", i, err)
}
valb, ok := val.([]byte)
if !ok {
t.Fatalf("expected []byte as output from get, got: %T", val)
}
if !bytes.Equal(valb, values[i]) {
if !bytes.Equal(val, values[i]) {
t.Fatal("input value didnt match the one returned from Get")
}
}
......
......@@ -56,7 +56,7 @@ func RunBatchTest(t *testing.T, ds dstore.Batching) {
t.Fatal(err)
}
if !bytes.Equal(blk.([]byte), blocks[i]) {
if !bytes.Equal(blk, blocks[i]) {
t.Fatal("blocks not correct!")
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment