Initial version of flatfs

flatfs.go

+// Package flatfs is a Datastore implementation that stores all
+// objects in a two-level directory structure in the local file
+// system, regardless of the hierarchy of the keys.
+package flatfs
+
+import (
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/jbenet/go-datastore"
+	"github.com/jbenet/go-datastore/query"
+)
+
+const (
+	extension    = ".data"
+	maxPrefixLen = 16
+)
+
+var (
+	ErrBadPrefixLen = errors.New("bad prefix length")
+)
+
+type Datastore struct {
+	path string
+	// length of the dir splay prefix, in bytes of hex digits
+	hexPrefixLen int
+}
+
+var _ datastore.Datastore = (*Datastore)(nil)
+
+func New(path string, prefixLen int) (*Datastore, error) {
+	if prefixLen <= 0 || prefixLen > maxPrefixLen {
+		return nil, ErrBadPrefixLen
+	}
+	fs := &Datastore{
+		path: path,
+		// convert from binary bytes to bytes of hex encoding
+		hexPrefixLen: prefixLen * hex.EncodedLen(1),
+	}
+	return fs, nil
+}
+
+var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))
+
+func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
+	safe := hex.EncodeToString(key.Bytes())
+	prefix := (safe + padding)[:fs.hexPrefixLen]
+	dir = path.Join(fs.path, prefix)
+	file = path.Join(dir, safe+extension)
+	return dir, file
+}
+
+func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
+	val, ok := value.([]byte)
+	if !ok {
+		return datastore.ErrInvalidType
+	}
+
+	dir, path := fs.encode(key)
+	if err := os.Mkdir(dir, 0777); err != nil {
+		// EEXIST is safe to ignore here, that just means the prefix
+		// directory already existed.
+		if !os.IsExist(err) {
+			return err
+		}
+	}
+
+	tmp, err := ioutil.TempFile(dir, "put-")
+	if err != nil {
+		return err
+	}
+	closed := false
+	removed := false
+	defer func() {
+		if !closed {
+			// silence errcheck
+			_ = tmp.Close()
+		}
+		if !removed {
+			// silence errcheck
+			_ = os.Remove(tmp.Name())
+		}
+	}()
+
+	if _, err := tmp.Write(val); err != nil {
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	closed = true
+
+	err = os.Rename(tmp.Name(), path)
+	if err != nil {
+		return err
+	}
+	removed = true
+
+	return nil
+}
+
+func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
+	_, path := fs.encode(key)
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, datastore.ErrNotFound
+		}
+		// no specific error to return, so just pass it through
+		return nil, err
+	}
+	return data, nil
+}
+
+func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
+	return false, errors.New("TODO")
+}
+
+func (fs *Datastore) Delete(key datastore.Key) error {
+	return errors.New("TODO")
+}
+
+func (fs *Datastore) Query(q query.Query) (query.Results, error) {
+	return nil, errors.New("TODO")
+}
+
+var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)
+
+func (*Datastore) IsThreadSafe() {}

flatfs_test.go

+package flatfs_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/jbenet/go-datastore"
+	"github.com/jbenet/go-datastore/flatfs"
+)
+
+func tempdir(t testing.TB) (path string, cleanup func()) {
+	path, err := ioutil.TempDir("", "test-datastore-flatfs-")
+	if err != nil {
+		t.Fatalf("cannot create temp directory: %v", err)
+	}
+
+	cleanup = func() {
+		if err := os.RemoveAll(path); err != nil {
+			t.Errorf("tempdir cleanup failed: %v", err)
+		}
+	}
+	return path, cleanup
+}
+
+func TestBadPrefixLen(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	for i := 0; i > -3; i-- {
+		_, err := flatfs.New(temp, 0)
+		if g, e := err, flatfs.ErrBadPrefixLen; g != e {
+			t.Errorf("expected ErrBadPrefixLen, got: %v", g)
+		}
+	}
+}
+
+func TestPutBadValueType(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	fs, err := flatfs.New(temp, 2)
+	if err != nil {
+		t.Fatalf("New fail: %v\n", err)
+	}
+
+	err = fs.Put(datastore.NewKey("quux"), 22)
+	if g, e := err, datastore.ErrInvalidType; g != e {
+		t.Fatalf("expected ErrInvalidType, got: %v\n", g)
+	}
+}
+
+func TestPut(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	fs, err := flatfs.New(temp, 2)
+	if err != nil {
+		t.Fatalf("New fail: %v\n", err)
+	}
+
+	err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
+	if err != nil {
+		t.Fatalf("Put fail: %v\n", err)
+	}
+}
+
+func TestGet(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	fs, err := flatfs.New(temp, 2)
+	if err != nil {
+		t.Fatalf("New fail: %v\n", err)
+	}
+
+	const input = "foobar"
+	err = fs.Put(datastore.NewKey("quux"), []byte(input))
+	if err != nil {
+		t.Fatalf("Put fail: %v\n", err)
+	}
+
+	data, err := fs.Get(datastore.NewKey("quux"))
+	if err != nil {
+		t.Fatalf("Get failed: %v", err)
+	}
+	buf, ok := data.([]byte)
+	if !ok {
+		t.Fatalf("expected []byte from Get, got %T: %v", data, data)
+	}
+	if g, e := string(buf), input; g != e {
+		t.Fatalf("Get gave wrong content: %q != %q", g, e)
+	}
+}
+
+func TestPutOverwrite(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	fs, err := flatfs.New(temp, 2)
+	if err != nil {
+		t.Fatalf("New fail: %v\n", err)
+	}
+
+	const (
+		loser  = "foobar"
+		winner = "xyzzy"
+	)
+	err = fs.Put(datastore.NewKey("quux"), []byte(loser))
+	if err != nil {
+		t.Fatalf("Put fail: %v\n", err)
+	}
+
+	err = fs.Put(datastore.NewKey("quux"), []byte(winner))
+	if err != nil {
+		t.Fatalf("Put fail: %v\n", err)
+	}
+
+	data, err := fs.Get(datastore.NewKey("quux"))
+	if err != nil {
+		t.Fatalf("Get failed: %v", err)
+	}
+	if g, e := string(data.([]byte)), winner; g != e {
+		t.Fatalf("Get gave wrong content: %q != %q", g, e)
+	}
+}
+
+func TestGetNotFoundError(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	fs, err := flatfs.New(temp, 2)
+	if err != nil {
+		t.Fatalf("New fail: %v\n", err)
+	}
+
+	_, err = fs.Get(datastore.NewKey("quux"))
+	if g, e := err, datastore.ErrNotFound; g != e {
+		t.Fatalf("expected ErrNotFound, got: %v\n", g)
+	}
+}
+
+func TestStorage(t *testing.T) {
+	temp, cleanup := tempdir(t)
+	defer cleanup()
+
+	const prefixLen = 2
+	const prefix = "2f71"
+	const target = prefix + "/2f71757578.data"
+	fs, err := flatfs.New(temp, prefixLen)
+	if err != nil {
+		t.Fatalf("New fail: %v\n", err)
+	}
+
+	err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
+	if err != nil {
+		t.Fatalf("Put fail: %v\n", err)
+	}
+
+	seen := false
+	walk := func(absPath string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		path, err := filepath.Rel(temp, absPath)
+		if err != nil {
+			return err
+		}
+		switch path {
+		case ".", "..":
+			// ignore
+		case prefix:
+			if !fi.IsDir() {
+				t.Errorf("prefix directory is not a file? %v", fi.Mode())
+			}
+			// we know it's there if we see the file, nothing more to
+			// do here
+		case target:
+			seen = true
+			if !fi.Mode().IsRegular() {
+				t.Errorf("expected a regular file, mode: %04o", fi.Mode())
+			}
+			if g, e := fi.Mode()&os.ModePerm&0007, os.FileMode(0000); g != e {
+				t.Errorf("file should not be world accessible: %04o", fi.Mode())
+			}
+		default:
+			t.Errorf("saw unexpected directory entry: %q %v", path, fi.Mode())
+		}
+		return nil
+	}
+	if err := filepath.Walk(temp, walk); err != nil {
+		t.Fatal("walk: %v", err)
+	}
+	if !seen {
+		t.Error("did not see the data file")
+	}
+}