tiered ds

tiered/tiered.go

+package tiered
+
+import (
+	"fmt"
+	"sync"
+
+	ds "github.com/jbenet/go-datastore"
+	dsq "github.com/jbenet/go-datastore/query"
+)
+
+type tiered []ds.Datastore
+
+// New returns a tiered datastore. Puts and Deletes will write-through to
+// all datastores, Has and Get will try each datastore sequentially, and
+// Query will always try the last one (most complete) first.
+func New(dses ...ds.Datastore) ds.Datastore {
+	return tiered(dses)
+}
+
+// Put stores the object `value` named by `key`.
+func (d tiered) Put(key ds.Key, value interface{}) (err error) {
+	errs := make(chan error, len(d))
+
+	var wg sync.WaitGroup
+	for _, cd := range d {
+		wg.Add(1)
+		go func(cd ds.Datastore) {
+			defer wg.Done()
+			if err := cd.Put(key, value); err != nil {
+				errs <- err
+			}
+		}(cd)
+	}
+	wg.Wait()
+
+	close(errs)
+	for err := range errs {
+		return err
+	}
+	return nil
+}
+
+// Get retrieves the object `value` named by `key`.
+func (d tiered) Get(key ds.Key) (value interface{}, err error) {
+	err = fmt.Errorf("no datastores")
+	for _, cd := range d {
+		value, err = cd.Get(key)
+		if err == nil {
+			break
+		}
+	}
+	return
+}
+
+// Has returns whether the `key` is mapped to a `value`.
+func (d tiered) Has(key ds.Key) (exists bool, err error) {
+	err = fmt.Errorf("no datastores")
+	for _, cd := range d {
+		exists, err = cd.Has(key)
+		if err == nil && exists {
+			break
+		}
+	}
+	return
+}
+
+// Delete removes the value for given `key`.
+func (d tiered) Delete(key ds.Key) (err error) {
+	errs := make(chan error, len(d))
+
+	var wg sync.WaitGroup
+	for _, cd := range d {
+		wg.Add(1)
+		go func(cd ds.Datastore) {
+			defer wg.Done()
+			if err := cd.Delete(key); err != nil {
+				errs <- err
+			}
+		}(cd)
+	}
+	wg.Wait()
+
+	close(errs)
+	for err := range errs {
+		return err
+	}
+	return nil
+}
+
+// Query returns a list of keys in the datastore
+func (d tiered) Query(q dsq.Query) (dsq.Results, error) {
+	// query always the last (most complete) one
+	return d[len(d)-1].Query(q)
+}

tiered/tiered_test.go

+package tiered
+
+import (
+	"testing"
+
+	ds "github.com/jbenet/go-datastore"
+	dscb "github.com/jbenet/go-datastore/callback"
+	dsq "github.com/jbenet/go-datastore/query"
+)
+
+func testHas(t *testing.T, dses []ds.Datastore, k ds.Key, v interface{}) {
+	// all under should have it
+	for _, d := range dses {
+		if v2, err := d.Get(k); err != nil {
+			t.Error(err)
+		} else if v2 != v {
+			t.Error("value incorrect", d, k, v, v2)
+		}
+
+		if has, err := d.Has(k); err != nil {
+			t.Error(err)
+		} else if !has {
+			t.Error("should have it", d, k, v)
+		}
+	}
+}
+
+func testNotHas(t *testing.T, dses []ds.Datastore, k ds.Key) {
+	// all under should not have it
+	for _, d := range dses {
+		if _, err := d.Get(k); err == nil {
+			t.Error("should not have it", d, k)
+		}
+
+		if has, err := d.Has(k); err != nil {
+			t.Error(err)
+		} else if has {
+			t.Error("should not have it", d, k)
+		}
+	}
+}
+
+func TestTiered(t *testing.T) {
+	d1 := ds.NewMapDatastore()
+	d2 := ds.NewMapDatastore()
+	d3 := ds.NewMapDatastore()
+	d4 := ds.NewMapDatastore()
+
+	td := New(d1, d2, d3, d4)
+	td.Put(ds.NewKey("foo"), "bar")
+	testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
+	testHas(t, td.(tiered), ds.NewKey("foo"), "bar") // all children
+
+	// remove it from, say, caches.
+	d1.Delete(ds.NewKey("foo"))
+	d2.Delete(ds.NewKey("foo"))
+	testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
+	testHas(t, td.(tiered)[2:], ds.NewKey("foo"), "bar")
+	testNotHas(t, td.(tiered)[:2], ds.NewKey("foo"))
+
+	// write it again.
+	td.Put(ds.NewKey("foo"), "bar2")
+	testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar2")
+	testHas(t, td.(tiered), ds.NewKey("foo"), "bar2")
+}
+
+func TestQueryCallsLast(t *testing.T) {
+	var d1n, d2n, d3n int
+	d1 := dscb.Wrap(ds.NewMapDatastore(), func() { d1n++ })
+	d2 := dscb.Wrap(ds.NewMapDatastore(), func() { d2n++ })
+	d3 := dscb.Wrap(ds.NewMapDatastore(), func() { d3n++ })
+
+	td := New(d1, d2, d3)
+
+	td.Query(dsq.Query{})
+	if d3n < 1 {
+		t.Error("should call last")
+	}
+}