Skip to content

GitLab

Unverified Commit 6bee6a41 authored by Steven Allen's avatar Steven Allen Committed by GitHub
Browse files
Options

Merge pull request #146 from ipfs/feat/try-harder 

feat(mount): don't give up on error
parents 9e9a3a83 ae47316e
Hide whitespace changes
Inline Side-by-side
Showing with 100 additions and 22 deletions
go.mod
View file @ 6bee6a41
......@@ -4,9 +4,9 @@ require (
github.com/google/uuid v1.1.1
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8
github.com/kr/pretty v0.1.0 // indirect
go.uber.org/multierr v1.4.0
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15
)
go 1.12
go 1.13
go.sum
View file @ 6bee6a41
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8 h1:NAviDvJ0WXgD+yiL2Rj35AmnfgI11+pHXbdciD917U0=
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
mount/mount.go
View file @ 6bee6a41
......@@ -5,13 +5,15 @@ package mount
import (
"container/heap"
"errors"
"fmt"
"sort"
"strings"
"sync"
ds "github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
xerrors "golang.org/x/xerrors"
"go.uber.org/multierr"
)
var (
......@@ -240,16 +242,22 @@ func (d *Datastore) Put(key ds.Key, value []byte) error {
// Sync implements Datastore.Sync
func (d *Datastore) Sync(prefix ds.Key) error {
var merr error
// Sync all mount points below the prefix
// Sync the mount point right at (or above) the prefix
dstores, _, rest := d.lookupAll(prefix)
dstores, prefixes, rest := d.lookupAll(prefix)
for i, suffix := range rest {
if err := dstores[i].Sync(suffix); err != nil {
return err
merr = multierr.Append(merr, fmt.Errorf(
"syncing datastore at %s: %w",
prefixes[i].String(),
err,
))
}
}
return nil
return merr
}
// Get returns the value associated with the key from the appropriate datastore.
......@@ -355,27 +363,39 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) {
// Close closes all mounted datastores.
func (d *Datastore) Close() error {
var merr error
for _, d := range d.mounts {
err := d.Datastore.Close()
if err != nil {
return err
merr = multierr.Append(merr, fmt.Errorf(
"closing datastore at %s: %w",
d.Prefix.String(),
err,
))
}
}
return nil
return merr
}
// DiskUsage returns the sum of DiskUsages for the mounted datastores.
// Non PersistentDatastores will not be accounted.
func (d *Datastore) DiskUsage() (uint64, error) {
var duTotal uint64 = 0
var (
merr error
duTotal uint64 = 0
)
for _, d := range d.mounts {
du, err := ds.DiskUsage(d.Datastore)
duTotal += du
if err != nil {
return duTotal, err
merr = multierr.Append(merr, fmt.Errorf(
"getting disk usage at %s: %w",
d.Prefix.String(),
err,
))
}
}
return duTotal, nil
return duTotal, merr
}
type mountBatch struct {
......@@ -436,44 +456,62 @@ func (mt *mountBatch) Commit() error {
mt.lk.Lock()
defer mt.lk.Unlock()
for _, t := range mt.mounts {
err := t.Commit()
if err != nil {
return err
var merr error
for p, t := range mt.mounts {
if err := t.Commit(); err != nil {
merr = multierr.Append(merr, fmt.Errorf(
"committing batch to datastore at %s: %w",
p, err,
))
}
}
return nil
return merr
}
func (d *Datastore) Check() error {
var merr error
for _, m := range d.mounts {
if c, ok := m.Datastore.(ds.CheckedDatastore); ok {
if err := c.Check(); err != nil {
return xerrors.Errorf("checking datastore at %s: %w", m.Prefix.String(), err)
merr = multierr.Append(merr, fmt.Errorf(
"checking datastore at %s: %w",
m.Prefix.String(),
err,
))
}
}
}
return nil
return merr
}
func (d *Datastore) Scrub() error {
var merr error
for _, m := range d.mounts {
if c, ok := m.Datastore.(ds.ScrubbedDatastore); ok {
if err := c.Scrub(); err != nil {
return xerrors.Errorf("scrubbing datastore at %s: %w", m.Prefix.String(), err)
merr = multierr.Append(merr, fmt.Errorf(
"scrubbing datastore at %s: %w",
m.Prefix.String(),
err,
))
}
}
}
return nil
return merr
}
func (d *Datastore) CollectGarbage() error {
var merr error
for _, m := range d.mounts {
if c, ok := m.Datastore.(ds.GCDatastore); ok {
if err := c.CollectGarbage(); err != nil {
return xerrors.Errorf("gc on datastore at %s: %w", m.Prefix.String(), err)
merr = multierr.Append(merr, fmt.Errorf(
"gc on datastore at %s: %w",
m.Prefix.String(),
err,
))
}
}
}
return nil
return merr
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment