Skip to content

GitLab

Unverified Commit a348e6ec authored by Steven Allen's avatar Steven Allen Committed by GitHub
Browse files
Options

Merge pull request #64 from ipfs/feat/improve-flatfs 

Make flatfs robust
parents fdacf94f 4ccc5e93
Hide whitespace changes
Inline Side-by-side
Showing with 250 additions and 85 deletions
.circleci/config.yml
View file @ a348e6ec
version: 2.1
orbs:
ci-go: ipfs/ci-go@0.1
ci-go: ipfs/ci-go@0.2.1
workflows:
version: 2
......
README.md
View file @ a348e6ec
......@@ -13,6 +13,9 @@
`go-ds-flatfs` is used by `go-ipfs` to store raw block contents on disk. It supports several sharding functions (prefix, suffix, next-to-last/*).
It is _not_ a general-purpose datastore and has several important restrictions.
See the restrictions section for details.
## Lead Maintainer
[Jakub Sztandera](https://github.com/kubuxu)
......@@ -33,13 +36,21 @@
import "github.com/ipfs/go-ds-flatfs"
```
`go-ds-flatfs` uses [`Gx`](https://github.com/whyrusleeping/gx) and [`Gx-go`](https://github.com/whyrusleeping/gx-go) to handle dependendencies. Run `make deps` to download and rewrite the imports to their fixed dependencies.
## Usage
Check the [GoDoc module documentation](https://godoc.org/github.com/ipfs/go-ds-flatfs) for an overview of this module's
functionality.
### Restrictions
FlatFS keys are severely restricted. Only keys that match `/[0-9A-Z+-_=]\+` are
allowed. That is, keys may only contain upper-case alpha-numeric characters,
'-', '+', '_', and '='. This is because values are written directly to the
filesystem without encoding.
Importantly, this means namespaced keys (e.g., /FOO/BAR), are _not_ allowed.
Attempts to write to such keys will result in an error.
### DiskUsage and Accuracy
This datastore implements the [`PersistentDatastore`](https://godoc.org/github.com/ipfs/go-datastore#PersistentDatastore) interface. It offers a `DiskUsage()` method which strives to find a balance between accuracy and performance. This implies:
......
convert.go
View file @ a348e6ec
......@@ -151,7 +151,7 @@ func Move(oldPath string, newPath string, out io.Writer) error {
}
} else {
// else we found something unexpected, so to be safe just move it
log.Warningf("found unexpected file in datastore directory: \"%s\", moving anyway\n", fn)
log.Warnw("found unexpected file in datastore directory, moving anyways", "file", fn)
newPath := filepath.Join(newDS.path, fn)
err := os.Rename(oldPath, newPath)
if err != nil {
......
convert_test.go
View file @ a348e6ec
......@@ -2,7 +2,7 @@ package flatfs_test
import (
"bytes"
"encoding/hex"
"encoding/base32"
"io/ioutil"
"math/rand"
"os"
......@@ -205,7 +205,7 @@ func populateDatastore(t *testing.T, dir string) ([]datastore.Key, [][]byte) {
r.Read(blk)
blocks = append(blocks, blk)
key := "x" + hex.EncodeToString(blk[:8])
key := "X" + base32.StdEncoding.EncodeToString(blk[:8])
keys = append(keys, datastore.NewKey(key))
err := ds.Put(keys[i], blocks[i])
if err != nil {
......
flatfs.go
View file @ a348e6ec
......@@ -95,6 +95,7 @@ var (
ErrDatastoreDoesNotExist = errors.New("datastore directory does not exist")
ErrShardingFileMissing = fmt.Errorf("%s file not found in datastore", SHARDING_FN)
ErrClosed = errors.New("datastore closed")
ErrInvalidKey = errors.New("key not supported by flatfs")
)
func init() {
......@@ -361,6 +362,10 @@ var putMaxRetries = 6
// concurrent Put and a Delete operation, we cannot guarantee which one
// will win.
func (fs *Datastore) Put(key datastore.Key, value []byte) error {
if !keyIsValid(key) {
return fmt.Errorf("when putting '%q': %w", key, ErrInvalidKey)
}
fs.shutdownLock.RLock()
defer fs.shutdownLock.RUnlock()
if fs.shutdown {
......@@ -580,6 +585,11 @@ func (fs *Datastore) putMany(data map[datastore.Key][]byte) error {
}
func (fs *Datastore) Get(key datastore.Key) (value []byte, err error) {
// Can't exist in datastore.
if !keyIsValid(key) {
return nil, datastore.ErrNotFound
}
_, path := fs.encode(key)
data, err := ioutil.ReadFile(path)
if err != nil {
......@@ -593,6 +603,11 @@ func (fs *Datastore) Get(key datastore.Key) (value []byte, err error) {
}
func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
// Can't exist in datastore.
if !keyIsValid(key) {
return false, nil
}
_, path := fs.encode(key)
switch _, err := os.Stat(path); {
case err == nil:
......@@ -605,6 +620,11 @@ func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
}
func (fs *Datastore) GetSize(key datastore.Key) (size int, err error) {
// Can't exist in datastore.
if !keyIsValid(key) {
return -1, datastore.ErrNotFound
}
_, path := fs.encode(key)
switch s, err := os.Stat(path); {
case err == nil:
......@@ -620,6 +640,11 @@ func (fs *Datastore) GetSize(key datastore.Key) (size int, err error) {
// the Put() explanation about the handling of concurrent write
// operations to the same key.
func (fs *Datastore) Delete(key datastore.Key) error {
// Can't exist in datastore.
if !keyIsValid(key) {
return nil
}
fs.shutdownLock.RLock()
defer fs.shutdownLock.RUnlock()
if fs.shutdown {
......@@ -654,17 +679,16 @@ func (fs *Datastore) doDelete(key datastore.Key) error {
func (fs *Datastore) Query(q query.Query) (query.Results, error) {
prefix := datastore.NewKey(q.Prefix).String()
if (prefix != "/") ||
len(q.Filters) > 0 ||
len(q.Orders) > 0 ||
q.Limit > 0 ||
q.Offset > 0 ||
!q.KeysOnly ||
q.ReturnExpirations ||
q.ReturnsSizes {
// TODO this is overly simplistic, but the only caller is
// `ipfs refs local` for now, and this gets us moving.
return nil, errors.New("flatfs only supports listing all keys in random order")
if prefix != "/" {
// This datastore can't include keys with multiple components.
// Therefore, it's always correct to return an empty result when
// the user requests a filter by prefix.
log.Warnw(
"flatfs was queried with a key prefix but flatfs only supports keys at the root",
"prefix", q.Prefix,
"query", q,
)
return query.ResultsWithEntries(q, nil), nil
}
// Replicates the logic in ResultsWithChan but actually respects calls
......@@ -682,7 +706,9 @@ func (fs *Datastore) Query(q query.Query) (query.Results, error) {
})
go b.Process.CloseAfterChildren() //nolint
return b.Results(), nil
// We don't apply _any_ of the query logic ourselves so we'll leave it
// all up to the naive query engine.
return query.NaiveQueryApply(q, b.Results()), nil
}
func (fs *Datastore) walkTopLevel(path string, result *query.ResultBuilder) error {
......@@ -893,7 +919,7 @@ func (fs *Datastore) checkpointLoop() {
if !more { // shutting down
fs.writeDiskUsageFile(du, true)
if fs.dirty {
log.Errorf("could not store final value of disk usage to file, future estimates may be inaccurate")
log.Error("could not store final value of disk usage to file, future estimates may be inaccurate")
}
return
}
......@@ -925,7 +951,7 @@ func (fs *Datastore) checkpointLoop() {
func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) {
tmp, err := ioutil.TempFile(fs.path, "du-")
if err != nil {
log.Warningf("cound not write disk usage: %v", err)
log.Warnw("could not write disk usage", "error", err)
return
}
......@@ -941,24 +967,24 @@ func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) {
toWrite.DiskUsage = du
encoder := json.NewEncoder(tmp)
if err := encoder.Encode(&toWrite); err != nil {
log.Warningf("cound not write disk usage: %v", err)
log.Warnw("cound not write disk usage", "error", err)
return
}
if doSync {
if err := tmp.Sync(); err != nil {
log.Warningf("cound not sync %s: %v", DiskUsageFile, err)
log.Warnw("cound not sync", "error", err, "file", DiskUsageFile)
return
}
}
if err := tmp.Close(); err != nil {
log.Warningf("cound not write disk usage: %v", err)
log.Warnw("cound not write disk usage", "error", err)
return
}
if err := os.Rename(tmp.Name(), filepath.Join(fs.path, DiskUsageFile)); err != nil {
log.Warningf("cound not write disk usage: %v", err)
log.Warnw("cound not write disk usage", "error", err)
return
}
removed = true
......@@ -1006,7 +1032,7 @@ func (fs *Datastore) Accuracy() string {
return string(fs.storedValue.Accuracy)
}
func (fs *Datastore) walk(path string, result *query.ResultBuilder) error {
func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error {
dir, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
......@@ -1038,17 +1064,35 @@ func (fs *Datastore) walk(path string, result *query.ResultBuilder) error {
key, ok := fs.decode(fn)
if !ok {
log.Warningf("failed to decode flatfs entry: %s", fn)
log.Warnw("failed to decode flatfs entry", "file", fn)
continue
}
var result query.Result
result.Key = key.String()
if !qrb.Query.KeysOnly {
value, err := ioutil.ReadFile(filepath.Join(path, fn))
if err != nil {
result.Error = err
} else {
// NOTE: Don't set the value/size on error. We
// don't want to return partial values.
result.Value = value
result.Size = len(value)
}
} else if qrb.Query.ReturnsSizes {
var stat os.FileInfo
stat, err := os.Stat(filepath.Join(path, fn))
if err != nil {
result.Error = err
} else {
result.Size = int(stat.Size())
}
}
select {
case result.Output <- query.Result{
Entry: query.Entry{
Key: key.String(),
},
}:
case <-result.Process.Closing():
case qrb.Output <- result:
case <-qrb.Process.Closing():
return nil
}
}
......@@ -1090,12 +1134,17 @@ func (fs *Datastore) Batch() (datastore.Batch, error) {
}
func (bt *flatfsBatch) Put(key datastore.Key, val []byte) error {
if !keyIsValid(key) {
return fmt.Errorf("when putting '%q': %w", key, ErrInvalidKey)
}
bt.puts[key] = val
return nil
}
func (bt *flatfsBatch) Delete(key datastore.Key) error {
bt.deletes[key] = struct{}{}
if keyIsValid(key) {
bt.deletes[key] = struct{}{}
} // otherwise, delete is a no-op anyways.
return nil
}
......
flatfs_test.go
View file @ a348e6ec
......@@ -16,6 +16,7 @@ import (
"time"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/mount"
"github.com/ipfs/go-datastore/query"
dstest "github.com/ipfs/go-datastore/test"
......@@ -54,10 +55,15 @@ func testPut(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("foo"), []byte("nonono"))
if err == nil {
t.Fatalf("did not expect to put a lowercase key")
}
}
func TestPut(t *testing.T) { tryAllShardFuncs(t, testPut) }
......@@ -73,18 +79,23 @@ func testGet(dirFunc mkShardFunc, t *testing.T) {
defer fs.Close()
const input = "foobar"
err = fs.Put(datastore.NewKey("quux"), []byte(input))
err = fs.Put(datastore.NewKey("QUUX"), []byte(input))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
buf, err := fs.Get(datastore.NewKey("quux"))
buf, err := fs.Get(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("Get failed: %v", err)
}
if g, e := string(buf), input; g != e {
t.Fatalf("Get gave wrong content: %q != %q", g, e)
}
_, err = fs.Get(datastore.NewKey("/FOO/BAR"))
if err != datastore.ErrNotFound {
t.Fatalf("expected ErrNotFound, got %s", err)
}
}
func TestGet(t *testing.T) { tryAllShardFuncs(t, testGet) }
......@@ -103,17 +114,17 @@ func testPutOverwrite(dirFunc mkShardFunc, t *testing.T) {
loser = "foobar"
winner = "xyzzy"
)
err = fs.Put(datastore.NewKey("quux"), []byte(loser))
err = fs.Put(datastore.NewKey("QUUX"), []byte(loser))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte(winner))
err = fs.Put(datastore.NewKey("QUUX"), []byte(winner))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
data, err := fs.Get(datastore.NewKey("quux"))
data, err := fs.Get(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("Get failed: %v", err)
}
......@@ -134,7 +145,7 @@ func testGetNotFoundError(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
_, err = fs.Get(datastore.NewKey("quux"))
_, err = fs.Get(datastore.NewKey("QUUX"))
if g, e := err, datastore.ErrNotFound; g != e {
t.Fatalf("expected ErrNotFound, got: %v\n", g)
}
......@@ -222,22 +233,22 @@ func TestStorage(t *testing.T) {
t.Run("prefix", func(t *testing.T) {
testStorage(&params{
shard: flatfs.Prefix(2),
dir: "qu",
key: "quux",
dir: "QU",
key: "QUUX",
}, t)
})
t.Run("suffix", func(t *testing.T) {
testStorage(&params{
shard: flatfs.Suffix(2),
dir: "ux",
key: "quux",
dir: "UX",
key: "QUUX",
}, t)
})
t.Run("next-to-last", func(t *testing.T) {
testStorage(&params{
shard: flatfs.NextToLast(2),
dir: "uu",
key: "quux",
dir: "UU",
key: "QUUX",
}, t)
})
}
......@@ -252,7 +263,7 @@ func testHasNotFound(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
found, err := fs.Has(datastore.NewKey("quux"))
found, err := fs.Has(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("Has fail: %v\n", err)
}
......@@ -273,12 +284,12 @@ func testHasFound(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
found, err := fs.Has(datastore.NewKey("quux"))
found, err := fs.Has(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("Has fail: %v\n", err)
}
......@@ -299,7 +310,7 @@ func testGetSizeFound(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
_, err = fs.GetSize(datastore.NewKey("quux"))
_, err = fs.GetSize(datastore.NewKey("QUUX"))
if err != datastore.ErrNotFound {
t.Fatalf("GetSize should have returned ErrNotFound, got: %v\n", err)
}
......@@ -317,12 +328,12 @@ func testGetSizeNotFound(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
size, err := fs.GetSize(datastore.NewKey("quux"))
size, err := fs.GetSize(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("GetSize failed with: %v\n", err)
}
......@@ -343,7 +354,7 @@ func testDeleteNotFound(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
err = fs.Delete(datastore.NewKey("quux"))
err = fs.Delete(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("expected nil, got: %v\n", err)
}
......@@ -361,18 +372,18 @@ func testDeleteFound(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
err = fs.Delete(datastore.NewKey("quux"))
err = fs.Delete(datastore.NewKey("QUUX"))
if err != nil {
t.Fatalf("Delete fail: %v\n", err)
}
// check that it's gone
_, err = fs.Get(datastore.NewKey("quux"))
_, err = fs.Get(datastore.NewKey("QUUX"))
if g, e := err, datastore.ErrNotFound; g != e {
t.Fatalf("expected Get after Delete to give ErrNotFound, got: %v\n", g)
}
......@@ -390,7 +401,7 @@ func testQuerySimple(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
const myKey = "quux"
const myKey = "QUUX"
err = fs.Put(datastore.NewKey(myKey), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
......@@ -439,7 +450,7 @@ func testDiskUsage(dirFunc mkShardFunc, t *testing.T) {
count := 200
for i := 0; i < count; i++ {
k := datastore.NewKey(fmt.Sprintf("test-%d", i))
k := datastore.NewKey(fmt.Sprintf("TEST-%d", i))
v := []byte("10bytes---")
err = fs.Put(k, v)
if err != nil {
......@@ -455,7 +466,7 @@ func testDiskUsage(dirFunc mkShardFunc, t *testing.T) {
t.Log("duPostPut:", duElems)
for i := 0; i < count; i++ {
k := datastore.NewKey(fmt.Sprintf("test-%d", i))
k := datastore.NewKey(fmt.Sprintf("TEST-%d", i))
err = fs.Delete(k)
if err != nil {
t.Fatalf("Delete fail: %v\n", err)
......@@ -553,7 +564,7 @@ func testDiskUsageDoubleCount(dirFunc mkShardFunc, t *testing.T) {
var count int
var wg sync.WaitGroup
testKey := datastore.NewKey("test")
testKey := datastore.NewKey("TEST")
put := func() {
defer wg.Done()
......@@ -633,7 +644,7 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
var wg sync.WaitGroup
testKeys := []datastore.Key{}
for i := 0; i < count; i++ {
k := datastore.NewKey(fmt.Sprintf("test%d", i))
k := datastore.NewKey(fmt.Sprintf("TEST%d", i))
testKeys = append(testKeys, k)
}
......@@ -726,7 +737,7 @@ func testDiskUsageEstimation(dirFunc mkShardFunc, t *testing.T) {
count := 50000
for i := 0; i < count; i++ {
k := datastore.NewKey(fmt.Sprintf("%d-test-%d", i, i))
k := datastore.NewKey(fmt.Sprintf("%d-TEST-%d", i, i))
v := make([]byte, 1000)
err = fs.Put(k, v)
if err != nil {
......@@ -837,14 +848,14 @@ func testClose(dirFunc mkShardFunc, t *testing.T) {
t.Fatalf("New fail: %v\n", err)
}
err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
if err != nil {
t.Fatalf("Put fail: %v\n", err)
}
fs.Close()
err = fs.Put(datastore.NewKey("qaax"), []byte("foobar"))
err = fs.Put(datastore.NewKey("QAAX"), []byte("foobar"))
if err == nil {
t.Fatal("expected put on closed datastore to fail")
}
......@@ -1065,3 +1076,29 @@ func TestQueryLeak(t *testing.T) {
t.Errorf("leaked %d goroutines", after-before)
}
}
func TestSuite(t *testing.T) {
temp, cleanup := tempdir(t)
defer cleanup()
fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
if err != nil {
t.Fatalf("New fail: %v\n", err)
}
ds := mount.New([]mount.Mount{{
Prefix: datastore.RawKey("/"),
Datastore: datastore.NewMapDatastore(),
}, {
Prefix: datastore.RawKey("/capital"),
Datastore: fs,
}})
defer func() {
err := ds.Close()
if err != nil {
t.Error(err)
}
}()
dstest.SubtestAll(t, ds)
}
go.sum
View file @ a348e6ec
......@@ -4,11 +4,13 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/ipfs/go-datastore v0.4.0 h1:Kiep/Ll245udr3DEnWBG0c5oofT6Dsin7fubWtFdmsc=
github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM=
github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0=
github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw=
github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
......@@ -20,24 +22,26 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
key.go 0 → 100644
View file @ a348e6ec
package flatfs
import (
"github.com/ipfs/go-datastore"
)
// keyIsValid returns true if the key is valid for flatfs.
// Allows keys that match [0-9A-Z+-_=].
func keyIsValid(key datastore.Key) bool {
ks := key.String()
if len(ks) < 2 || ks[0] != '/' {
return false
}
for _, b := range ks[1:] {
if '0' <= b && b <= '9' {
continue
}
if 'A' <= b && b <= 'Z' {
continue
}
switch b {
case '+', '-', '_', '=':
continue
}
return false
}
return true
}
key_test.go 0 → 100644
View file @ a348e6ec
package flatfs
import (
"testing"
"github.com/ipfs/go-datastore"
)
var (
validKeys = []string{
"/FOO",
"/1BAR1",
"/=EMACS-IS-KING=",
}
invalidKeys = []string{
"/foo/bar",
`/foo\bar`,
"/foo\000bar",
"/=Vim-IS-KING=",
}
)
func TestKeyIsValid(t *testing.T) {
for _, key := range validKeys {
k := datastore.NewKey(key)
if !keyIsValid(k) {
t.Errorf("expected key %s to be valid", k)
}
}
for _, key := range invalidKeys {
k := datastore.NewKey(key)
if keyIsValid(k) {
t.Errorf("expected key %s to be invalid", k)
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment