Skip to content

GitLab

Unverified Commit 3453260e authored by Steven Allen's avatar Steven Allen Committed by GitHub
Browse files
Options

Merge pull request #54 from ipfs/feat/idempotent-delete 

make delete idempotent
parents d5e3c1fa 70540a69
Hide whitespace changes
Inline Side-by-side
Showing with 57 additions and 100 deletions
.circleci/config.yml 0 → 100644
View file @ 3453260e
version: 2.1
orbs:
ci-go: ipfs/ci-go@0.1
workflows:
version: 2
test:
jobs:
- ci-go/build
- ci-go/lint
- ci-go/test
.travis.yml deleted 100644 → 0
View file @ d5e3c1fa
os:
- linux
language: go
go:
- 1.11.x
env:
global:
- GOTFLAGS="-race"
matrix:
- BUILD_DEPTYPE=gx
- BUILD_DEPTYPE=gomod
# disable travis install
install:
- true
script:
- bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
cache:
directories:
- $GOPATH/src/gx
- $GOPATH/pkg/mod
- $HOME/.cache/go-build
notifications:
email: false
Makefile deleted 100644 → 0
View file @ d5e3c1fa
export IPFS_API ?= v04x.ipfs.io
gx:
go get -u github.com/whyrusleeping/gx
go get -u github.com/whyrusleeping/gx-go
deps: gx
gx --verbose install --global
gx-go rewrite
flatfs.go
View file @ 3453260e
......@@ -490,7 +490,7 @@ func (fs *Datastore) putMany(data map[datastore.Key][]byte) error {
defer func() {
for fi := range files {
val, _ := ops[fi]
val := ops[fi]
switch val {
case 0:
_ = fi.Close()
......@@ -636,7 +636,7 @@ func (fs *Datastore) doDelete(key datastore.Key) error {
fs.checkpointDiskUsage()
return nil
case os.IsNotExist(err):
return datastore.ErrNotFound
return nil
default:
return err
}
......@@ -667,7 +667,7 @@ func (fs *Datastore) Query(q query.Query) (query.Results, error) {
case <-p.Closing():
}
})
go b.Process.CloseAfterChildren()
go b.Process.CloseAfterChildren() //nolint
return b.Results(), nil
}
......@@ -1045,20 +1045,20 @@ func (fs *Datastore) walk(path string, result *query.ResultBuilder) error {
// Deactivate closes background maintenance threads, most write
// operations will fail but readonly operations will continue to
// function
func (fs *Datastore) deactivate() error {
func (fs *Datastore) deactivate() {
fs.shutdownLock.Lock()
defer fs.shutdownLock.Unlock()
if fs.shutdown {
return nil
return
}
fs.shutdown = true
close(fs.checkpointCh)
<-fs.done
return nil
}
func (fs *Datastore) Close() error {
return fs.deactivate()
fs.deactivate()
return nil
}
type flatfsBatch struct {
......@@ -1091,7 +1091,7 @@ func (bt *flatfsBatch) Commit() error {
return err
}
for k, _ := range bt.deletes {
for k := range bt.deletes {
if err := bt.ds.Delete(k); err != nil {
return err
}
......
flatfs_test.go
View file @ 3453260e
......@@ -344,8 +344,8 @@ func testDeleteNotFound(dirFunc mkShardFunc, t *testing.T) {
defer fs.Close()
err = fs.Delete(datastore.NewKey("quux"))
if g, e := err, datastore.ErrNotFound; g != e {
t.Fatalf("expected ErrNotFound, got: %v\n", g)
if err != nil {
t.Fatalf("expected nil, got: %v\n", err)
}
}
......@@ -561,7 +561,7 @@ func testDiskUsageDoubleCount(dirFunc mkShardFunc, t *testing.T) {
v := []byte("10bytes---")
err := fs.Put(testKey, v)
if err != nil {
t.Fatalf("Put fail: %v\n", err)
t.Errorf("Put fail: %v\n", err)
}
}
}
......@@ -571,7 +571,7 @@ func testDiskUsageDoubleCount(dirFunc mkShardFunc, t *testing.T) {
for i := 0; i < count; i++ {
err := fs.Delete(testKey)
if err != nil && !strings.Contains(err.Error(), "key not found") {
t.Fatalf("Delete fail: %v\n", err)
t.Errorf("Delete fail: %v\n", err)
}
}
}
......@@ -624,7 +624,10 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
}
defer fs.Close()
fsBatch, _ := fs.Batch()
fsBatch, err := fs.Batch()
if err != nil {
t.Fatal(err)
}
count := 200
var wg sync.WaitGroup
......@@ -636,14 +639,17 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
put := func() {
for i := 0; i < count; i++ {
fsBatch.Put(testKeys[i], []byte("10bytes---"))
err := fsBatch.Put(testKeys[i], []byte("10bytes---"))
if err != nil {
t.Error(err)
}
}
}
commit := func() {
defer wg.Done()
err := fsBatch.Commit()
if err != nil {
t.Fatalf("Batch Put fail: %v\n", err)
t.Errorf("Batch Put fail: %v\n", err)
}
}
......@@ -652,7 +658,7 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
for _, k := range testKeys {
err := fs.Delete(k)
if err != nil && !strings.Contains(err.Error(), "key not found") {
t.Fatalf("Delete fail: %v\n", err)
t.Errorf("Delete fail: %v\n", err)
}
}
}
......@@ -662,9 +668,15 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
wg.Add(2)
put()
commit()
du, _ := fs.DiskUsage()
du, err := fs.DiskUsage()
if err != nil {
t.Fatal(err)
}
del()
du2, _ := fs.DiskUsage()
du2, err := fs.DiskUsage()
if err != nil {
t.Fatal(err)
}
if du-uint64(10*count) != du2 {
t.Errorf("should have deleted exactly %d bytes: %d %d", 10*count, du, du2)
}
......@@ -676,11 +688,17 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
go del()
wg.Wait()
du3, _ := fs.DiskUsage()
du3, err := fs.DiskUsage()
if err != nil {
t.Fatal(err)
}
// Now query how many keys we have
results, err := fs.Query(query.Query{
KeysOnly: true,
})
if err != nil {
t.Fatal(err)
}
rest, err := results.Rest()
if err != nil {
t.Fatal(err)
......@@ -862,6 +880,7 @@ func TestSHARDINGFile(t *testing.T) {
fs, err = flatfs.CreateOrOpen(tempdir, flatfs.Prefix(5), false)
if err == nil {
fs.Close()
t.Fatalf("Was able to open repo with incompatible sharding function")
}
}
......@@ -877,9 +896,12 @@ func TestNonDatastoreDir(t *testing.T) {
tempdir, cleanup := tempdir(t)
defer cleanup()
ioutil.WriteFile(filepath.Join(tempdir, "afile"), []byte("Some Content"), 0644)
err := ioutil.WriteFile(filepath.Join(tempdir, "afile"), []byte("Some Content"), 0644)
if err != nil {
t.Fatal(err)
}
err := flatfs.Create(tempdir, flatfs.NextToLast(2))
err = flatfs.Create(tempdir, flatfs.NextToLast(2))
if err == nil {
t.Fatalf("Expected an error when creating a datastore in a non-empty directory")
}
......
package.json deleted 100644 → 0
View file @ d5e3c1fa
{
"author": "whyrusleeping",
"bugs": {
"url": "https://github.com/ipfs/go-ds-flatfs"
},
"gx": {
"dvcsimport": "github.com/ipfs/go-ds-flatfs"
},
"gxDependencies": [
{
"hash": "QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF",
"name": "go-log",
"version": "1.5.9"
},
{
"author": "jbenet",
"hash": "QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9",
"name": "go-datastore",
"version": "3.6.1"
},
{
"author": "whyrusleeping",
"hash": "QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP",
"name": "goprocess",
"version": "1.0.0"
}
],
"gxVersion": "0.8.0",
"language": "go",
"license": "",
"name": "go-ds-flatfs",
"releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
"version": "1.3.7"
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment