Skip to content

GitLab

Commit ceebea7e authored by Kevin Atkinson's avatar Kevin Atkinson
Browse files
Options

Avoid using strings for shard func when creating the datastore.

parent 6db29c0a
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 82 deletions
flatfs.go
View file @ ceebea7e
......@@ -29,8 +29,8 @@ const (
type Datastore struct {
path string
shardFunc string
getDir ShardFunc
shardStr string
getDir ShardFunc
// sychronize all writes and directory changes for added safety
sync bool
......@@ -46,20 +46,16 @@ var (
ErrShardingFileMissing = fmt.Errorf("%s file not found in datastore", SHARDING_FN)
)
const IPFS_DEF_SHARD = "/repo/flatfs/shard/v1/next-to-last/2"
var IPFS_DEF_SHARD = NextToLast(2)
var IPFS_DEF_SHARD_STR = IPFS_DEF_SHARD.String()
func Create(path string, funStr string) error {
func Create(path string, fun *ShardIdV1) error {
err := os.Mkdir(path, 0777)
if err != nil && !os.IsExist(err) {
return err
}
fun, err := ParseShardFunc(funStr)
if err != nil {
return err
}
dsFun, err := ReadShardFunc(path)
switch err {
case ErrShardingFileMissing:
......@@ -102,16 +98,16 @@ func Open(path string, sync bool) (*Datastore, error) {
}
fs := &Datastore{
path: path,
shardFunc: shardId.String(),
getDir: shardId.Func(),
sync: sync,
path: path,
shardStr: shardId.String(),
getDir: shardId.Func(),
sync: sync,
}
return fs, nil
}
// convenience method
func CreateOrOpen(path string, fun string, sync bool) (*Datastore, error) {
func CreateOrOpen(path string, fun *ShardIdV1, sync bool) (*Datastore, error) {
err := Create(path, fun)
if err != nil && err != ErrDatastoreExists {
return nil, err
......@@ -119,8 +115,8 @@ func CreateOrOpen(path string, fun string, sync bool) (*Datastore, error) {
return Open(path, sync)
}
func (fs *Datastore) ShardFunc() string {
return fs.shardFunc
func (fs *Datastore) ShardStr() string {
return fs.shardStr
}
func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
......
flatfs_test.go
View file @ ceebea7e
......@@ -32,17 +32,17 @@ func tempdir(t testing.TB) (path string, cleanup func()) {
return path, cleanup
}
func tryAllShardFuncs(t *testing.T, testFunc func(string, *testing.T)) {
t.Run("prefix", func(t *testing.T) { testFunc("v1/prefix", t) })
t.Run("suffix", func(t *testing.T) { testFunc("v1/suffix", t) })
t.Run("next-to-last", func(t *testing.T) { testFunc("v1/next-to-last", t) })
func tryAllShardFuncs(t *testing.T, testFunc func(mkShardFunc, *testing.T)) {
t.Run("prefix", func(t *testing.T) { testFunc(flatfs.Prefix, t) })
t.Run("suffix", func(t *testing.T) { testFunc(flatfs.Suffix, t) })
t.Run("next-to-last", func(t *testing.T) { testFunc(flatfs.NextToLast, t) })
}
func TestPutBadValueType(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, "v1/prefix/2", false)
fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -53,11 +53,13 @@ func TestPutBadValueType(t *testing.T) {
}
}
func testPut(dirFunc string, t *testing.T) {
type mkShardFunc func(int) *flatfs.ShardIdV1
func testPut(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -70,11 +72,11 @@ func testPut(dirFunc string, t *testing.T) {
func TestPut(t *testing.T) { tryAllShardFuncs(t, testPut) }
func testGet(dirFunc string, t *testing.T) {
func testGet(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -100,11 +102,11 @@ func testGet(dirFunc string, t *testing.T) {
func TestGet(t *testing.T) { tryAllShardFuncs(t, testGet) }
func testPutOverwrite(dirFunc string, t *testing.T) {
func testPutOverwrite(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -134,11 +136,11 @@ func testPutOverwrite(dirFunc string, t *testing.T) {
func TestPutOverwrite(t *testing.T) { tryAllShardFuncs(t, testPutOverwrite) }
func testGetNotFoundError(dirFunc string, t *testing.T) {
func testGetNotFoundError(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -152,7 +154,7 @@ func testGetNotFoundError(dirFunc string, t *testing.T) {
func TestGetNotFoundError(t *testing.T) { tryAllShardFuncs(t, testGetNotFoundError) }
type params struct {
shard string
shard *flatfs.ShardIdV1
dir string
key string
}
......@@ -218,9 +220,9 @@ func testStorage(p *params, t *testing.T) {
if !seen {
t.Error("did not see the data file")
}
if fs.ShardFunc() == flatfs.IPFS_DEF_SHARD && !haveREADME {
if fs.ShardStr() == flatfs.IPFS_DEF_SHARD_STR && !haveREADME {
t.Error("expected _README file")
} else if fs.ShardFunc() != flatfs.IPFS_DEF_SHARD && haveREADME {
} else if fs.ShardStr() != flatfs.IPFS_DEF_SHARD_STR && haveREADME {
t.Error("did not expect _README file")
}
}
......@@ -228,32 +230,32 @@ func testStorage(p *params, t *testing.T) {
func TestStorage(t *testing.T) {
t.Run("prefix", func(t *testing.T) {
testStorage(&params{
shard: "v1/prefix/2",
shard: flatfs.Prefix(2),
dir: "qu",
key: "quux",
}, t)
})
t.Run("suffix", func(t *testing.T) {
testStorage(&params{
shard: "v1/suffix/2",
shard: flatfs.Suffix(2),
dir: "ux",
key: "quux",
}, t)
})
t.Run("next-to-last", func(t *testing.T) {
testStorage(&params{
shard: flatfs.IPFS_DEF_SHARD,
shard: flatfs.NextToLast(2),
dir: "uu",
key: "quux",
}, t)
})
}
func testHasNotFound(dirFunc string, t *testing.T) {
func testHasNotFound(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -269,11 +271,11 @@ func testHasNotFound(dirFunc string, t *testing.T) {
func TestHasNotFound(t *testing.T) { tryAllShardFuncs(t, testHasNotFound) }
func testHasFound(dirFunc string, t *testing.T) {
func testHasFound(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -293,11 +295,11 @@ func testHasFound(dirFunc string, t *testing.T) {
func TestHasFound(t *testing.T) { tryAllShardFuncs(t, testHasFound) }
func testDeleteNotFound(dirFunc string, t *testing.T) {
func testDeleteNotFound(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -310,11 +312,11 @@ func testDeleteNotFound(dirFunc string, t *testing.T) {
func TestDeleteNotFound(t *testing.T) { tryAllShardFuncs(t, testDeleteNotFound) }
func testDeleteFound(dirFunc string, t *testing.T) {
func testDeleteFound(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -337,11 +339,11 @@ func testDeleteFound(dirFunc string, t *testing.T) {
func TestDeleteFound(t *testing.T) { tryAllShardFuncs(t, testDeleteFound) }
func testQuerySimple(dirFunc string, t *testing.T) {
func testQuerySimple(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -375,11 +377,11 @@ func testQuerySimple(dirFunc string, t *testing.T) {
func TestQuerySimple(t *testing.T) { tryAllShardFuncs(t, testQuerySimple) }
func testBatchPut(dirFunc string, t *testing.T) {
func testBatchPut(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -389,11 +391,11 @@ func testBatchPut(dirFunc string, t *testing.T) {
func TestBatchPut(t *testing.T) { tryAllShardFuncs(t, testBatchPut) }
func testBatchDelete(dirFunc string, t *testing.T) {
func testBatchDelete(dirFunc mkShardFunc, t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, dirFunc+"/2", false)
fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -418,8 +420,8 @@ func TestSHARDINGFile(t *testing.T) {
if err != nil {
t.Fatalf("Open fail: %v\n", err)
}
if fs.ShardFunc() != flatfs.IPFS_DEF_SHARD {
t.Fatalf("Expected '%s' for shard function got '%s'", flatfs.IPFS_DEF_SHARD, fs.ShardFunc())
if fs.ShardStr() != flatfs.IPFS_DEF_SHARD_STR {
t.Fatalf("Expected '%s' for shard function got '%s'", flatfs.IPFS_DEF_SHARD_STR, fs.ShardStr())
}
fs.Close()
......@@ -429,19 +431,16 @@ func TestSHARDINGFile(t *testing.T) {
}
fs.Close()
fs, err = flatfs.CreateOrOpen(tempdir, "prefix/5", false)
fs, err = flatfs.CreateOrOpen(tempdir, flatfs.Prefix(5), false)
if err == nil {
t.Fatalf("Was able to open repo with incompatible sharding function")
}
}
func TestInvalidPrefix(t *testing.T) {
tempdir, cleanup := tempdir(t)
defer cleanup()
err := flatfs.Create(tempdir, "/bad/prefix/v1/next-to-last/2")
_, err := flatfs.ParseShardFunc("/bad/prefix/v1/next-to-last/2")
if err == nil {
t.Fatalf("Expected an error when creating a datastore with a bad prefix for the sharding function")
t.Fatalf("Expected an error while parsing a shard identifier with a bad prefix")
}
}
......@@ -451,7 +450,7 @@ func TestNonDatastoreDir(t *testing.T) {
ioutil.WriteFile(filepath.Join(tempdir, "afile"), []byte("Some Content"), 0644)
err := flatfs.Create(tempdir, "/bad/prefix/v1/next-to-last/2")
err := flatfs.Create(tempdir, flatfs.NextToLast(2))
if err == nil {
t.Fatalf("Expected an error when creating a datastore in a non-empty directory")
}
......@@ -461,7 +460,7 @@ func TestNoCluster(t *testing.T) {
tempdir, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(tempdir, "v1/next-to-last/1", false)
fs, err := flatfs.CreateOrOpen(tempdir, flatfs.NextToLast(1), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
......@@ -521,7 +520,7 @@ func BenchmarkConsecutivePut(b *testing.B) {
temp, cleanup := tempdir(b)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, "prefix/2", false)
fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
if err != nil {
b.Fatalf("New fail: %v\n", err)
}
......@@ -551,7 +550,7 @@ func BenchmarkBatchedPut(b *testing.B) {
temp, cleanup := tempdir(b)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, "prefix/2", false)
fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
if err != nil {
b.Fatalf("New fail: %v\n", err)
}
......
shard.go
View file @ ceebea7e
......@@ -51,26 +51,24 @@ func ParseShardFunc(str string) (*ShardIdV1, error) {
return nil, fmt.Errorf("expected 'v1' for version string got: %s\n", version)
}
id := &ShardIdV1{funName: parts[1]}
funName := parts[1]
param, err := strconv.Atoi(parts[2])
if err != nil {
return nil, fmt.Errorf("invalid parameter: %v", err)
}
id.param = param
switch id.funName {
switch funName {
case "prefix":
id.fun = Prefix(param)
return Prefix(param), nil
case "suffix":
id.fun = Suffix(param)
return Suffix(param), nil
case "next-to-last":
id.fun = NextToLast(param)
return NextToLast(param), nil
default:
return nil, fmt.Errorf("expected 'prefix', 'suffix' or 'next-to-last' got: %s", id.funName)
return nil, fmt.Errorf("expected 'prefix', 'suffix' or 'next-to-last' got: %s", funName)
}
return id, nil
}
func ReadShardFunc(dir string) (*ShardIdV1, error) {
......@@ -98,7 +96,7 @@ func WriteShardFunc(dir string, id *ShardIdV1) error {
}
func WriteReadme(dir string, id *ShardIdV1) error {
if id.String() == IPFS_DEF_SHARD {
if id.String() == IPFS_DEF_SHARD.String() {
err := ioutil.WriteFile(filepath.Join(dir, README_FN), []byte(README_IPFS_DEF_SHARD), 0444)
if err != nil {
return err
......@@ -107,26 +105,38 @@ func WriteReadme(dir string, id *ShardIdV1) error {
return nil
}
func Prefix(prefixLen int) ShardFunc {
func Prefix(prefixLen int) *ShardIdV1 {
padding := strings.Repeat("_", prefixLen)
return func(noslash string) string {
return (noslash + padding)[:prefixLen]
return &ShardIdV1{
funName: "prefix",
param: prefixLen,
fun: func(noslash string) string {
return (noslash + padding)[:prefixLen]
},
}
}
func Suffix(suffixLen int) ShardFunc {
func Suffix(suffixLen int) *ShardIdV1 {
padding := strings.Repeat("_", suffixLen)
return func(noslash string) string {
str := padding + noslash
return str[len(str)-suffixLen:]
return &ShardIdV1{
funName: "suffix",
param: suffixLen,
fun: func(noslash string) string {
str := padding + noslash
return str[len(str)-suffixLen:]
},
}
}
func NextToLast(suffixLen int) ShardFunc {
func NextToLast(suffixLen int) *ShardIdV1 {
padding := strings.Repeat("_", suffixLen+1)
return func(noslash string) string {
str := padding + noslash
offset := len(str) - suffixLen - 1
return str[offset : offset+suffixLen]
return &ShardIdV1{
funName: "next-to-last",
param: suffixLen,
fun: func(noslash string) string {
str := padding + noslash
offset := len(str) - suffixLen - 1
return str[offset : offset+suffixLen]
},
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment