Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
dms3
go-datastore
Commits
5c508695
Commit
5c508695
authored
Oct 14, 2017
by
Łukasz Magiera
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Wire up Checked/Scrubbed/GC-DS to wrapper DSes
parent
b4b50d52
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
271 additions
and
2 deletions
+271
-2
basic_ds.go
basic_ds.go
+21
-0
keytransform/keytransform.go
keytransform/keytransform.go
+21
-0
keytransform/keytransform_test.go
keytransform/keytransform_test.go
+14
-1
mount/mount.go
mount/mount.go
+34
-0
mount/mount_test.go
mount/mount_test.go
+20
-0
namespace/namespace_test.go
namespace/namespace_test.go
+14
-1
sync/sync.go
sync/sync.go
+21
-0
syncmount/mount.go
syncmount/mount.go
+34
-0
syncmount/mount_test.go
syncmount/mount_test.go
+20
-0
test/test_util.go
test/test_util.go
+39
-0
tiered/tiered.go
tiered/tiered.go
+33
-0
No files found.
basic_ds.go
View file @
5c508695
...
@@ -226,3 +226,24 @@ func (d *LogDatastore) Close() error {
...
@@ -226,3 +226,24 @@ func (d *LogDatastore) Close() error {
}
}
return
nil
return
nil
}
}
func
(
d
*
LogDatastore
)
Check
()
error
{
if
c
,
ok
:=
d
.
child
.
(
CheckedDatastore
);
ok
{
return
c
.
Check
()
}
return
nil
}
func
(
d
*
LogDatastore
)
Scrub
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ScrubbedDatastore
);
ok
{
return
c
.
Scrub
()
}
return
nil
}
func
(
d
*
LogDatastore
)
CollectGarbage
()
error
{
if
c
,
ok
:=
d
.
child
.
(
GCDatastore
);
ok
{
return
c
.
CollectGarbage
()
}
return
nil
}
keytransform/keytransform.go
View file @
5c508695
...
@@ -117,3 +117,24 @@ func (t *transformBatch) Delete(key ds.Key) error {
...
@@ -117,3 +117,24 @@ func (t *transformBatch) Delete(key ds.Key) error {
func
(
t
*
transformBatch
)
Commit
()
error
{
func
(
t
*
transformBatch
)
Commit
()
error
{
return
t
.
dst
.
Commit
()
return
t
.
dst
.
Commit
()
}
}
func
(
d
*
ktds
)
Check
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ds
.
CheckedDatastore
);
ok
{
return
c
.
Check
()
}
return
nil
}
func
(
d
*
ktds
)
Scrub
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ds
.
ScrubbedDatastore
);
ok
{
return
c
.
Scrub
()
}
return
nil
}
func
(
d
*
ktds
)
CollectGarbage
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ds
.
GCDatastore
);
ok
{
return
c
.
CollectGarbage
()
}
return
nil
}
keytransform/keytransform_test.go
View file @
5c508695
...
@@ -10,6 +10,7 @@ import (
...
@@ -10,6 +10,7 @@ import (
ds
"github.com/ipfs/go-datastore"
ds
"github.com/ipfs/go-datastore"
kt
"github.com/ipfs/go-datastore/keytransform"
kt
"github.com/ipfs/go-datastore/keytransform"
dsq
"github.com/ipfs/go-datastore/query"
dsq
"github.com/ipfs/go-datastore/query"
dstest
"github.com/ipfs/go-datastore/test"
)
)
// Hook up gocheck into the "go test" runner.
// Hook up gocheck into the "go test" runner.
...
@@ -35,7 +36,7 @@ func (ks *DSSuite) TestBasic(c *C) {
...
@@ -35,7 +36,7 @@ func (ks *DSSuite) TestBasic(c *C) {
},
},
}
}
mpds
:=
ds
.
NewMap
Datastore
()
mpds
:=
ds
test
.
NewTest
Datastore
(
true
)
ktds
:=
kt
.
Wrap
(
mpds
,
pair
)
ktds
:=
kt
.
Wrap
(
mpds
,
pair
)
keys
:=
strsToKeys
([]
string
{
keys
:=
strsToKeys
([]
string
{
...
@@ -88,6 +89,18 @@ func (ks *DSSuite) TestBasic(c *C) {
...
@@ -88,6 +89,18 @@ func (ks *DSSuite) TestBasic(c *C) {
c
.
Log
(
"listA: "
,
listA
)
c
.
Log
(
"listA: "
,
listA
)
c
.
Log
(
"listB: "
,
listB
)
c
.
Log
(
"listB: "
,
listB
)
if
err
:=
ktds
.
Check
();
err
!=
dstest
.
TestError
{
c
.
Errorf
(
"Unexpected Check() error: %s"
,
err
)
}
if
err
:=
ktds
.
CollectGarbage
();
err
!=
dstest
.
TestError
{
c
.
Errorf
(
"Unexpected CollectGarbage() error: %s"
,
err
)
}
if
err
:=
ktds
.
Scrub
();
err
!=
dstest
.
TestError
{
c
.
Errorf
(
"Unexpected Scrub() error: %s"
,
err
)
}
}
}
func
strsToKeys
(
strs
[]
string
)
[]
ds
.
Key
{
func
strsToKeys
(
strs
[]
string
)
[]
ds
.
Key
{
...
...
mount/mount.go
View file @
5c508695
...
@@ -4,6 +4,7 @@ package mount
...
@@ -4,6 +4,7 @@ package mount
import
(
import
(
"errors"
"errors"
"fmt"
"io"
"io"
"sort"
"sort"
"strings"
"strings"
...
@@ -251,3 +252,36 @@ func (mt *mountBatch) Commit() error {
...
@@ -251,3 +252,36 @@ func (mt *mountBatch) Commit() error {
}
}
return
nil
return
nil
}
}
func
(
d
*
Datastore
)
Check
()
error
{
for
_
,
m
:=
range
d
.
mounts
{
if
c
,
ok
:=
m
.
Datastore
.
(
datastore
.
CheckedDatastore
);
ok
{
if
err
:=
c
.
Check
();
err
!=
nil
{
return
fmt
.
Errorf
(
"checking datastore at %s: %s"
,
m
.
Prefix
.
String
(),
err
.
Error
())
}
}
}
return
nil
}
func
(
d
*
Datastore
)
Scrub
()
error
{
for
_
,
m
:=
range
d
.
mounts
{
if
c
,
ok
:=
m
.
Datastore
.
(
datastore
.
ScrubbedDatastore
);
ok
{
if
err
:=
c
.
Scrub
();
err
!=
nil
{
return
fmt
.
Errorf
(
"scrubbing datastore at %s: %s"
,
m
.
Prefix
.
String
(),
err
.
Error
())
}
}
}
return
nil
}
func
(
d
*
Datastore
)
CollectGarbage
()
error
{
for
_
,
m
:=
range
d
.
mounts
{
if
c
,
ok
:=
m
.
Datastore
.
(
datastore
.
GCDatastore
);
ok
{
if
err
:=
c
.
CollectGarbage
();
err
!=
nil
{
return
fmt
.
Errorf
(
"gc on datastore at %s: %s"
,
m
.
Prefix
.
String
(),
err
.
Error
())
}
}
}
return
nil
}
mount/mount_test.go
View file @
5c508695
...
@@ -7,6 +7,7 @@ import (
...
@@ -7,6 +7,7 @@ import (
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/mount"
"github.com/ipfs/go-datastore/mount"
"github.com/ipfs/go-datastore/query"
"github.com/ipfs/go-datastore/query"
dstest
"github.com/ipfs/go-datastore/test"
)
)
func
TestPutBadNothing
(
t
*
testing
.
T
)
{
func
TestPutBadNothing
(
t
*
testing
.
T
)
{
...
@@ -371,3 +372,22 @@ func TestErrQueryClose(t *testing.T) {
...
@@ -371,3 +372,22 @@ func TestErrQueryClose(t *testing.T) {
t
.
Errorf
(
"Query was ok or q.Error was nil"
)
t
.
Errorf
(
"Query was ok or q.Error was nil"
)
}
}
}
}
func
TestMaintenanceFunctions
(
t
*
testing
.
T
)
{
mapds
:=
dstest
.
NewTestDatastore
(
true
)
m
:=
mount
.
New
([]
mount
.
Mount
{
{
Prefix
:
datastore
.
NewKey
(
"/"
),
Datastore
:
mapds
},
})
if
err
:=
m
.
Check
();
err
.
Error
()
!=
"checking datastore at /: test error"
{
t
.
Errorf
(
"Unexpected Check() error: %s"
,
err
)
}
if
err
:=
m
.
CollectGarbage
();
err
.
Error
()
!=
"gc on datastore at /: test error"
{
t
.
Errorf
(
"Unexpected CollectGarbage() error: %s"
,
err
)
}
if
err
:=
m
.
Scrub
();
err
.
Error
()
!=
"scrubbing datastore at /: test error"
{
t
.
Errorf
(
"Unexpected Scrub() error: %s"
,
err
)
}
}
namespace/namespace_test.go
View file @
5c508695
...
@@ -10,6 +10,7 @@ import (
...
@@ -10,6 +10,7 @@ import (
ds
"github.com/ipfs/go-datastore"
ds
"github.com/ipfs/go-datastore"
ns
"github.com/ipfs/go-datastore/namespace"
ns
"github.com/ipfs/go-datastore/namespace"
dsq
"github.com/ipfs/go-datastore/query"
dsq
"github.com/ipfs/go-datastore/query"
dstest
"github.com/ipfs/go-datastore/test"
)
)
// Hook up gocheck into the "go test" runner.
// Hook up gocheck into the "go test" runner.
...
@@ -79,7 +80,7 @@ func (ks *DSSuite) testBasic(c *C, prefix string) {
...
@@ -79,7 +80,7 @@ func (ks *DSSuite) testBasic(c *C, prefix string) {
}
}
func
(
ks
*
DSSuite
)
TestQuery
(
c
*
C
)
{
func
(
ks
*
DSSuite
)
TestQuery
(
c
*
C
)
{
mpds
:=
ds
.
NewMap
Datastore
()
mpds
:=
ds
test
.
NewTest
Datastore
(
true
)
nsds
:=
ns
.
Wrap
(
mpds
,
ds
.
NewKey
(
"/foo"
))
nsds
:=
ns
.
Wrap
(
mpds
,
ds
.
NewKey
(
"/foo"
))
keys
:=
strsToKeys
([]
string
{
keys
:=
strsToKeys
([]
string
{
...
@@ -137,6 +138,18 @@ func (ks *DSSuite) TestQuery(c *C) {
...
@@ -137,6 +138,18 @@ func (ks *DSSuite) TestQuery(c *C) {
expval
,
_
:=
expect
[
i
]
.
Value
.
([]
byte
)
expval
,
_
:=
expect
[
i
]
.
Value
.
([]
byte
)
c
.
Check
(
string
(
entval
),
Equals
,
string
(
expval
))
c
.
Check
(
string
(
entval
),
Equals
,
string
(
expval
))
}
}
if
err
:=
nsds
.
Datastore
.
(
ds
.
CheckedDatastore
)
.
Check
();
err
!=
dstest
.
TestError
{
c
.
Errorf
(
"Unexpected Check() error: %s"
,
err
)
}
if
err
:=
nsds
.
Datastore
.
(
ds
.
GCDatastore
)
.
CollectGarbage
();
err
!=
dstest
.
TestError
{
c
.
Errorf
(
"Unexpected CollectGarbage() error: %s"
,
err
)
}
if
err
:=
nsds
.
Datastore
.
(
ds
.
ScrubbedDatastore
)
.
Scrub
();
err
!=
dstest
.
TestError
{
c
.
Errorf
(
"Unexpected Scrub() error: %s"
,
err
)
}
}
}
func
strsToKeys
(
strs
[]
string
)
[]
ds
.
Key
{
func
strsToKeys
(
strs
[]
string
)
[]
ds
.
Key
{
...
...
sync/sync.go
View file @
5c508695
...
@@ -114,3 +114,24 @@ func (b *syncBatch) Commit() error {
...
@@ -114,3 +114,24 @@ func (b *syncBatch) Commit() error {
defer
b
.
mds
.
Unlock
()
defer
b
.
mds
.
Unlock
()
return
b
.
batch
.
Commit
()
return
b
.
batch
.
Commit
()
}
}
func
(
d
*
MutexDatastore
)
Check
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ds
.
CheckedDatastore
);
ok
{
return
c
.
Check
()
}
return
nil
}
func
(
d
*
MutexDatastore
)
Scrub
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ds
.
ScrubbedDatastore
);
ok
{
return
c
.
Scrub
()
}
return
nil
}
func
(
d
*
MutexDatastore
)
CollectGarbage
()
error
{
if
c
,
ok
:=
d
.
child
.
(
ds
.
GCDatastore
);
ok
{
return
c
.
CollectGarbage
()
}
return
nil
}
syncmount/mount.go
View file @
5c508695
...
@@ -4,6 +4,7 @@ package syncmount
...
@@ -4,6 +4,7 @@ package syncmount
import
(
import
(
"errors"
"errors"
"fmt"
"io"
"io"
"sort"
"sort"
"strings"
"strings"
...
@@ -264,3 +265,36 @@ func (mt *mountBatch) Commit() error {
...
@@ -264,3 +265,36 @@ func (mt *mountBatch) Commit() error {
}
}
return
nil
return
nil
}
}
func
(
d
*
Datastore
)
Check
()
error
{
for
_
,
m
:=
range
d
.
mounts
{
if
c
,
ok
:=
m
.
Datastore
.
(
ds
.
CheckedDatastore
);
ok
{
if
err
:=
c
.
Check
();
err
!=
nil
{
return
fmt
.
Errorf
(
"checking datastore at %s: %s"
,
m
.
Prefix
.
String
(),
err
.
Error
())
}
}
}
return
nil
}
func
(
d
*
Datastore
)
Scrub
()
error
{
for
_
,
m
:=
range
d
.
mounts
{
if
c
,
ok
:=
m
.
Datastore
.
(
ds
.
ScrubbedDatastore
);
ok
{
if
err
:=
c
.
Scrub
();
err
!=
nil
{
return
fmt
.
Errorf
(
"scrubbing datastore at %s: %s"
,
m
.
Prefix
.
String
(),
err
.
Error
())
}
}
}
return
nil
}
func
(
d
*
Datastore
)
CollectGarbage
()
error
{
for
_
,
m
:=
range
d
.
mounts
{
if
c
,
ok
:=
m
.
Datastore
.
(
ds
.
GCDatastore
);
ok
{
if
err
:=
c
.
CollectGarbage
();
err
!=
nil
{
return
fmt
.
Errorf
(
"gc on datastore at %s: %s"
,
m
.
Prefix
.
String
(),
err
.
Error
())
}
}
}
return
nil
}
syncmount/mount_test.go
View file @
5c508695
...
@@ -7,6 +7,7 @@ import (
...
@@ -7,6 +7,7 @@ import (
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
"github.com/ipfs/go-datastore/query"
mount
"github.com/ipfs/go-datastore/syncmount"
mount
"github.com/ipfs/go-datastore/syncmount"
dstest
"github.com/ipfs/go-datastore/test"
)
)
func
TestPutBadNothing
(
t
*
testing
.
T
)
{
func
TestPutBadNothing
(
t
*
testing
.
T
)
{
...
@@ -371,3 +372,22 @@ func TestErrQueryClose(t *testing.T) {
...
@@ -371,3 +372,22 @@ func TestErrQueryClose(t *testing.T) {
t
.
Errorf
(
"Query was ok or q.Error was nil"
)
t
.
Errorf
(
"Query was ok or q.Error was nil"
)
}
}
}
}
func
TestMaintenanceFunctions
(
t
*
testing
.
T
)
{
mapds
:=
dstest
.
NewTestDatastore
(
true
)
m
:=
mount
.
New
([]
mount
.
Mount
{
{
Prefix
:
datastore
.
NewKey
(
"/"
),
Datastore
:
mapds
},
})
if
err
:=
m
.
Check
();
err
.
Error
()
!=
"checking datastore at /: test error"
{
t
.
Errorf
(
"Unexpected Check() error: %s"
,
err
)
}
if
err
:=
m
.
CollectGarbage
();
err
.
Error
()
!=
"gc on datastore at /: test error"
{
t
.
Errorf
(
"Unexpected CollectGarbage() error: %s"
,
err
)
}
if
err
:=
m
.
Scrub
();
err
.
Error
()
!=
"scrubbing datastore at /: test error"
{
t
.
Errorf
(
"Unexpected Scrub() error: %s"
,
err
)
}
}
test/test_util.go
View file @
5c508695
...
@@ -3,12 +3,17 @@ package dstest
...
@@ -3,12 +3,17 @@ package dstest
import
(
import
(
"bytes"
"bytes"
"encoding/base32"
"encoding/base32"
"errors"
"math/rand"
"math/rand"
"testing"
"testing"
dstore
"github.com/ipfs/go-datastore"
dstore
"github.com/ipfs/go-datastore"
)
)
var
(
TestError
=
errors
.
New
(
"test error"
)
)
func
RunBatchTest
(
t
*
testing
.
T
,
ds
dstore
.
Batching
)
{
func
RunBatchTest
(
t
*
testing
.
T
,
ds
dstore
.
Batching
)
{
batch
,
err
:=
ds
.
Batch
()
batch
,
err
:=
ds
.
Batch
()
if
err
!=
nil
{
if
err
!=
nil
{
...
@@ -95,3 +100,37 @@ func RunBatchDeleteTest(t *testing.T, ds dstore.Batching) {
...
@@ -95,3 +100,37 @@ func RunBatchDeleteTest(t *testing.T, ds dstore.Batching) {
}
}
}
}
}
}
type
testDatastore
struct
{
testErrors
bool
*
dstore
.
MapDatastore
}
func
NewTestDatastore
(
testErrors
bool
)
*
testDatastore
{
return
&
testDatastore
{
testErrors
:
testErrors
,
MapDatastore
:
dstore
.
NewMapDatastore
(),
}
}
func
(
d
*
testDatastore
)
Check
()
error
{
if
d
.
testErrors
{
return
TestError
}
return
nil
}
func
(
d
*
testDatastore
)
Scrub
()
error
{
if
d
.
testErrors
{
return
TestError
}
return
nil
}
func
(
d
*
testDatastore
)
CollectGarbage
()
error
{
if
d
.
testErrors
{
return
TestError
}
return
nil
}
tiered/tiered.go
View file @
5c508695
...
@@ -92,3 +92,36 @@ func (d tiered) Query(q dsq.Query) (dsq.Results, error) {
...
@@ -92,3 +92,36 @@ func (d tiered) Query(q dsq.Query) (dsq.Results, error) {
// query always the last (most complete) one
// query always the last (most complete) one
return
d
[
len
(
d
)
-
1
]
.
Query
(
q
)
return
d
[
len
(
d
)
-
1
]
.
Query
(
q
)
}
}
func
(
d
tiered
)
Check
()
error
{
for
i
,
child
:=
range
d
{
if
c
,
ok
:=
child
.
(
ds
.
CheckedDatastore
);
ok
{
if
err
:=
c
.
Check
();
err
!=
nil
{
return
fmt
.
Errorf
(
"checking datastore %d: %s"
,
i
,
err
.
Error
())
}
}
}
return
nil
}
func
(
d
tiered
)
Scrub
()
error
{
for
i
,
child
:=
range
d
{
if
c
,
ok
:=
child
.
(
ds
.
ScrubbedDatastore
);
ok
{
if
err
:=
c
.
Scrub
();
err
!=
nil
{
return
fmt
.
Errorf
(
"scrubbing datastore at %d: %s"
,
i
,
err
.
Error
())
}
}
}
return
nil
}
func
(
d
tiered
)
CollectGarbage
()
error
{
for
i
,
child
:=
range
d
{
if
c
,
ok
:=
child
.
(
ds
.
GCDatastore
);
ok
{
if
err
:=
c
.
CollectGarbage
();
err
!=
nil
{
return
fmt
.
Errorf
(
"gc on datastore at %d: %s"
,
i
,
err
.
Error
())
}
}
}
return
nil
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment