Merge pull request #54 from ipfs/feat/idempotent-delete

make delete idempotent

.circleci/config.yml

+version: 2.1
+orbs:
+  ci-go: ipfs/ci-go@0.1
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - ci-go/build
+      - ci-go/lint
+      - ci-go/test

.travis.yml

-os:
-  - linux
-
-language: go
-
-go:
-  - 1.11.x
-
-env:
-  global:
-    - GOTFLAGS="-race"
-  matrix:
-    - BUILD_DEPTYPE=gx
-    - BUILD_DEPTYPE=gomod
-
-
-# disable travis install
-install:
-  - true
-
-script:
-  - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
-
-
-cache:
-  directories:
-    - $GOPATH/src/gx
-    - $GOPATH/pkg/mod
-    - $HOME/.cache/go-build
-
-notifications:
-  email: false

Makefile

-export IPFS_API ?= v04x.ipfs.io
-
-gx:
-	go get -u github.com/whyrusleeping/gx
-	go get -u github.com/whyrusleeping/gx-go
-
-deps: gx
-	gx --verbose install --global
-	gx-go rewrite

flatfs.go

@@ -490,7 +490,7 @@ func (fs *Datastore) putMany(data map[datastore.Key][]byte) error {
 
 	defer func() {
 		for fi := range files {
-			val, _ := ops[fi]
+			val := ops[fi]
 			switch val {
 			case 0:
 				_ = fi.Close()
@@ -636,7 +636,7 @@ func (fs *Datastore) doDelete(key datastore.Key) error {
 		fs.checkpointDiskUsage()
 		return nil
 	case os.IsNotExist(err):
-		return datastore.ErrNotFound
+		return nil
 	default:
 		return err
 	}
@@ -667,7 +667,7 @@ func (fs *Datastore) Query(q query.Query) (query.Results, error) {
 		case <-p.Closing():
 		}
 	})
-	go b.Process.CloseAfterChildren()
+	go b.Process.CloseAfterChildren() //nolint
 
 	return b.Results(), nil
 }
@@ -1045,20 +1045,20 @@ func (fs *Datastore) walk(path string, result *query.ResultBuilder) error {
 // Deactivate closes background maintenance threads, most write
 // operations will fail but readonly operations will continue to
 // function
-func (fs *Datastore) deactivate() error {
+func (fs *Datastore) deactivate() {
 	fs.shutdownLock.Lock()
 	defer fs.shutdownLock.Unlock()
 	if fs.shutdown {
-		return nil
+		return
 	}
 	fs.shutdown = true
 	close(fs.checkpointCh)
 	<-fs.done
-	return nil
 }
 
 func (fs *Datastore) Close() error {
-	return fs.deactivate()
+	fs.deactivate()
+	return nil
 }
 
 type flatfsBatch struct {
@@ -1091,7 +1091,7 @@ func (bt *flatfsBatch) Commit() error {
 		return err
 	}
 
-	for k, _ := range bt.deletes {
+	for k := range bt.deletes {
 		if err := bt.ds.Delete(k); err != nil {
 			return err
 		}

flatfs_test.go

@@ -344,8 +344,8 @@ func testDeleteNotFound(dirFunc mkShardFunc, t *testing.T) {
 	defer fs.Close()
 
 	err = fs.Delete(datastore.NewKey("quux"))
-	if g, e := err, datastore.ErrNotFound; g != e {
-		t.Fatalf("expected ErrNotFound, got: %v\n", g)
+	if err != nil {
+		t.Fatalf("expected nil, got: %v\n", err)
 	}
 }
 
@@ -561,7 +561,7 @@ func testDiskUsageDoubleCount(dirFunc mkShardFunc, t *testing.T) {
 			v := []byte("10bytes---")
 			err := fs.Put(testKey, v)
 			if err != nil {
-				t.Fatalf("Put fail: %v\n", err)
+				t.Errorf("Put fail: %v\n", err)
 			}
 		}
 	}
@@ -571,7 +571,7 @@ func testDiskUsageDoubleCount(dirFunc mkShardFunc, t *testing.T) {
 		for i := 0; i < count; i++ {
 			err := fs.Delete(testKey)
 			if err != nil && !strings.Contains(err.Error(), "key not found") {
-				t.Fatalf("Delete fail: %v\n", err)
+				t.Errorf("Delete fail: %v\n", err)
 			}
 		}
 	}
@@ -624,7 +624,10 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
 	}
 	defer fs.Close()
 
-	fsBatch, _ := fs.Batch()
+	fsBatch, err := fs.Batch()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	count := 200
 	var wg sync.WaitGroup
@@ -636,14 +639,17 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
 
 	put := func() {
 		for i := 0; i < count; i++ {
-			fsBatch.Put(testKeys[i], []byte("10bytes---"))
+			err := fsBatch.Put(testKeys[i], []byte("10bytes---"))
+			if err != nil {
+				t.Error(err)
+			}
 		}
 	}
 	commit := func() {
 		defer wg.Done()
 		err := fsBatch.Commit()
 		if err != nil {
-			t.Fatalf("Batch Put fail: %v\n", err)
+			t.Errorf("Batch Put fail: %v\n", err)
 		}
 	}
 
@@ -652,7 +658,7 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
 		for _, k := range testKeys {
 			err := fs.Delete(k)
 			if err != nil && !strings.Contains(err.Error(), "key not found") {
-				t.Fatalf("Delete fail: %v\n", err)
+				t.Errorf("Delete fail: %v\n", err)
 			}
 		}
 	}
@@ -662,9 +668,15 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
 	wg.Add(2)
 	put()
 	commit()
-	du, _ := fs.DiskUsage()
+	du, err := fs.DiskUsage()
+	if err != nil {
+		t.Fatal(err)
+	}
 	del()
-	du2, _ := fs.DiskUsage()
+	du2, err := fs.DiskUsage()
+	if err != nil {
+		t.Fatal(err)
+	}
 	if du-uint64(10*count) != du2 {
 		t.Errorf("should have deleted exactly %d bytes: %d %d", 10*count, du, du2)
 	}
@@ -676,11 +688,17 @@ func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
 	go del()
 	wg.Wait()
 
-	du3, _ := fs.DiskUsage()
+	du3, err := fs.DiskUsage()
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Now query how many keys we have
 	results, err := fs.Query(query.Query{
 		KeysOnly: true,
 	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	rest, err := results.Rest()
 	if err != nil {
 		t.Fatal(err)
@@ -862,6 +880,7 @@ func TestSHARDINGFile(t *testing.T) {
 
 	fs, err = flatfs.CreateOrOpen(tempdir, flatfs.Prefix(5), false)
 	if err == nil {
+		fs.Close()
 		t.Fatalf("Was able to open repo with incompatible sharding function")
 	}
 }
@@ -877,9 +896,12 @@ func TestNonDatastoreDir(t *testing.T) {
 	tempdir, cleanup := tempdir(t)
 	defer cleanup()
 
-	ioutil.WriteFile(filepath.Join(tempdir, "afile"), []byte("Some Content"), 0644)
+	err := ioutil.WriteFile(filepath.Join(tempdir, "afile"), []byte("Some Content"), 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	err := flatfs.Create(tempdir, flatfs.NextToLast(2))
+	err = flatfs.Create(tempdir, flatfs.NextToLast(2))
 	if err == nil {
 		t.Fatalf("Expected an error when creating a datastore in a non-empty directory")
 	}

go.mod

 module github.com/ipfs/go-ds-flatfs
 
 require (
-	github.com/ipfs/go-datastore v0.0.1
+	github.com/ipfs/go-datastore v0.1.0
 	github.com/ipfs/go-log v0.0.1
 	github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8
 )

go.sum

@@ -6,8 +6,8 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/ipfs/go-datastore v0.0.1 h1:AW/KZCScnBWlSb5JbnEnLKFWXL224LBEh/9KXXOrUms=
-github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI=
+github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=

package.json

-{
-  "author": "whyrusleeping",
-  "bugs": {
-    "url": "https://github.com/ipfs/go-ds-flatfs"
-  },
-  "gx": {
-    "dvcsimport": "github.com/ipfs/go-ds-flatfs"
-  },
-  "gxDependencies": [
-    {
-      "hash": "QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF",
-      "name": "go-log",
-      "version": "1.5.9"
-    },
-    {
-      "author": "jbenet",
-      "hash": "QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9",
-      "name": "go-datastore",
-      "version": "3.6.1"
-    },
-    {
-      "author": "whyrusleeping",
-      "hash": "QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP",
-      "name": "goprocess",
-      "version": "1.0.0"
-    }
-  ],
-  "gxVersion": "0.8.0",
-  "language": "go",
-  "license": "",
-  "name": "go-ds-flatfs",
-  "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
-  "version": "1.3.7"
-}
-