Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
dms3
go-datastore
Commits
a77cfe36
Commit
a77cfe36
authored
Mar 11, 2015
by
Tommi Virtanen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Initial version of flatfs
parent
c4782ade
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
331 additions
and
0 deletions
+331
-0
flatfs/flatfs.go
flatfs/flatfs.go
+133
-0
flatfs/flatfs_test.go
flatfs/flatfs_test.go
+198
-0
No files found.
flatfs/flatfs.go
0 → 100644
View file @
a77cfe36
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package
flatfs
import
(
"encoding/hex"
"errors"
"io/ioutil"
"os"
"path"
"strings"
"github.com/jbenet/go-datastore"
"github.com/jbenet/go-datastore/query"
)
const
(
extension
=
".data"
maxPrefixLen
=
16
)
var
(
ErrBadPrefixLen
=
errors
.
New
(
"bad prefix length"
)
)
type
Datastore
struct
{
path
string
// length of the dir splay prefix, in bytes of hex digits
hexPrefixLen
int
}
var
_
datastore
.
Datastore
=
(
*
Datastore
)(
nil
)
func
New
(
path
string
,
prefixLen
int
)
(
*
Datastore
,
error
)
{
if
prefixLen
<=
0
||
prefixLen
>
maxPrefixLen
{
return
nil
,
ErrBadPrefixLen
}
fs
:=
&
Datastore
{
path
:
path
,
// convert from binary bytes to bytes of hex encoding
hexPrefixLen
:
prefixLen
*
hex
.
EncodedLen
(
1
),
}
return
fs
,
nil
}
var
padding
=
strings
.
Repeat
(
"_"
,
maxPrefixLen
*
hex
.
EncodedLen
(
1
))
func
(
fs
*
Datastore
)
encode
(
key
datastore
.
Key
)
(
dir
,
file
string
)
{
safe
:=
hex
.
EncodeToString
(
key
.
Bytes
())
prefix
:=
(
safe
+
padding
)[
:
fs
.
hexPrefixLen
]
dir
=
path
.
Join
(
fs
.
path
,
prefix
)
file
=
path
.
Join
(
dir
,
safe
+
extension
)
return
dir
,
file
}
func
(
fs
*
Datastore
)
Put
(
key
datastore
.
Key
,
value
interface
{})
error
{
val
,
ok
:=
value
.
([]
byte
)
if
!
ok
{
return
datastore
.
ErrInvalidType
}
dir
,
path
:=
fs
.
encode
(
key
)
if
err
:=
os
.
Mkdir
(
dir
,
0777
);
err
!=
nil
{
// EEXIST is safe to ignore here, that just means the prefix
// directory already existed.
if
!
os
.
IsExist
(
err
)
{
return
err
}
}
tmp
,
err
:=
ioutil
.
TempFile
(
dir
,
"put-"
)
if
err
!=
nil
{
return
err
}
closed
:=
false
removed
:=
false
defer
func
()
{
if
!
closed
{
// silence errcheck
_
=
tmp
.
Close
()
}
if
!
removed
{
// silence errcheck
_
=
os
.
Remove
(
tmp
.
Name
())
}
}()
if
_
,
err
:=
tmp
.
Write
(
val
);
err
!=
nil
{
return
err
}
if
err
:=
tmp
.
Close
();
err
!=
nil
{
return
err
}
closed
=
true
err
=
os
.
Rename
(
tmp
.
Name
(),
path
)
if
err
!=
nil
{
return
err
}
removed
=
true
return
nil
}
func
(
fs
*
Datastore
)
Get
(
key
datastore
.
Key
)
(
value
interface
{},
err
error
)
{
_
,
path
:=
fs
.
encode
(
key
)
data
,
err
:=
ioutil
.
ReadFile
(
path
)
if
err
!=
nil
{
if
os
.
IsNotExist
(
err
)
{
return
nil
,
datastore
.
ErrNotFound
}
// no specific error to return, so just pass it through
return
nil
,
err
}
return
data
,
nil
}
func
(
fs
*
Datastore
)
Has
(
key
datastore
.
Key
)
(
exists
bool
,
err
error
)
{
return
false
,
errors
.
New
(
"TODO"
)
}
func
(
fs
*
Datastore
)
Delete
(
key
datastore
.
Key
)
error
{
return
errors
.
New
(
"TODO"
)
}
func
(
fs
*
Datastore
)
Query
(
q
query
.
Query
)
(
query
.
Results
,
error
)
{
return
nil
,
errors
.
New
(
"TODO"
)
}
var
_
datastore
.
ThreadSafeDatastore
=
(
*
Datastore
)(
nil
)
func
(
*
Datastore
)
IsThreadSafe
()
{}
flatfs/flatfs_test.go
0 → 100644
View file @
a77cfe36
package
flatfs_test
import
(
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/jbenet/go-datastore"
"github.com/jbenet/go-datastore/flatfs"
)
func
tempdir
(
t
testing
.
TB
)
(
path
string
,
cleanup
func
())
{
path
,
err
:=
ioutil
.
TempDir
(
""
,
"test-datastore-flatfs-"
)
if
err
!=
nil
{
t
.
Fatalf
(
"cannot create temp directory: %v"
,
err
)
}
cleanup
=
func
()
{
if
err
:=
os
.
RemoveAll
(
path
);
err
!=
nil
{
t
.
Errorf
(
"tempdir cleanup failed: %v"
,
err
)
}
}
return
path
,
cleanup
}
func
TestBadPrefixLen
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
for
i
:=
0
;
i
>
-
3
;
i
--
{
_
,
err
:=
flatfs
.
New
(
temp
,
0
)
if
g
,
e
:=
err
,
flatfs
.
ErrBadPrefixLen
;
g
!=
e
{
t
.
Errorf
(
"expected ErrBadPrefixLen, got: %v"
,
g
)
}
}
}
func
TestPutBadValueType
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
fs
,
err
:=
flatfs
.
New
(
temp
,
2
)
if
err
!=
nil
{
t
.
Fatalf
(
"New fail: %v
\n
"
,
err
)
}
err
=
fs
.
Put
(
datastore
.
NewKey
(
"quux"
),
22
)
if
g
,
e
:=
err
,
datastore
.
ErrInvalidType
;
g
!=
e
{
t
.
Fatalf
(
"expected ErrInvalidType, got: %v
\n
"
,
g
)
}
}
func
TestPut
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
fs
,
err
:=
flatfs
.
New
(
temp
,
2
)
if
err
!=
nil
{
t
.
Fatalf
(
"New fail: %v
\n
"
,
err
)
}
err
=
fs
.
Put
(
datastore
.
NewKey
(
"quux"
),
[]
byte
(
"foobar"
))
if
err
!=
nil
{
t
.
Fatalf
(
"Put fail: %v
\n
"
,
err
)
}
}
func
TestGet
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
fs
,
err
:=
flatfs
.
New
(
temp
,
2
)
if
err
!=
nil
{
t
.
Fatalf
(
"New fail: %v
\n
"
,
err
)
}
const
input
=
"foobar"
err
=
fs
.
Put
(
datastore
.
NewKey
(
"quux"
),
[]
byte
(
input
))
if
err
!=
nil
{
t
.
Fatalf
(
"Put fail: %v
\n
"
,
err
)
}
data
,
err
:=
fs
.
Get
(
datastore
.
NewKey
(
"quux"
))
if
err
!=
nil
{
t
.
Fatalf
(
"Get failed: %v"
,
err
)
}
buf
,
ok
:=
data
.
([]
byte
)
if
!
ok
{
t
.
Fatalf
(
"expected []byte from Get, got %T: %v"
,
data
,
data
)
}
if
g
,
e
:=
string
(
buf
),
input
;
g
!=
e
{
t
.
Fatalf
(
"Get gave wrong content: %q != %q"
,
g
,
e
)
}
}
func
TestPutOverwrite
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
fs
,
err
:=
flatfs
.
New
(
temp
,
2
)
if
err
!=
nil
{
t
.
Fatalf
(
"New fail: %v
\n
"
,
err
)
}
const
(
loser
=
"foobar"
winner
=
"xyzzy"
)
err
=
fs
.
Put
(
datastore
.
NewKey
(
"quux"
),
[]
byte
(
loser
))
if
err
!=
nil
{
t
.
Fatalf
(
"Put fail: %v
\n
"
,
err
)
}
err
=
fs
.
Put
(
datastore
.
NewKey
(
"quux"
),
[]
byte
(
winner
))
if
err
!=
nil
{
t
.
Fatalf
(
"Put fail: %v
\n
"
,
err
)
}
data
,
err
:=
fs
.
Get
(
datastore
.
NewKey
(
"quux"
))
if
err
!=
nil
{
t
.
Fatalf
(
"Get failed: %v"
,
err
)
}
if
g
,
e
:=
string
(
data
.
([]
byte
)),
winner
;
g
!=
e
{
t
.
Fatalf
(
"Get gave wrong content: %q != %q"
,
g
,
e
)
}
}
func
TestGetNotFoundError
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
fs
,
err
:=
flatfs
.
New
(
temp
,
2
)
if
err
!=
nil
{
t
.
Fatalf
(
"New fail: %v
\n
"
,
err
)
}
_
,
err
=
fs
.
Get
(
datastore
.
NewKey
(
"quux"
))
if
g
,
e
:=
err
,
datastore
.
ErrNotFound
;
g
!=
e
{
t
.
Fatalf
(
"expected ErrNotFound, got: %v
\n
"
,
g
)
}
}
func
TestStorage
(
t
*
testing
.
T
)
{
temp
,
cleanup
:=
tempdir
(
t
)
defer
cleanup
()
const
prefixLen
=
2
const
prefix
=
"2f71"
const
target
=
prefix
+
"/2f71757578.data"
fs
,
err
:=
flatfs
.
New
(
temp
,
prefixLen
)
if
err
!=
nil
{
t
.
Fatalf
(
"New fail: %v
\n
"
,
err
)
}
err
=
fs
.
Put
(
datastore
.
NewKey
(
"quux"
),
[]
byte
(
"foobar"
))
if
err
!=
nil
{
t
.
Fatalf
(
"Put fail: %v
\n
"
,
err
)
}
seen
:=
false
walk
:=
func
(
absPath
string
,
fi
os
.
FileInfo
,
err
error
)
error
{
if
err
!=
nil
{
return
err
}
path
,
err
:=
filepath
.
Rel
(
temp
,
absPath
)
if
err
!=
nil
{
return
err
}
switch
path
{
case
"."
,
".."
:
// ignore
case
prefix
:
if
!
fi
.
IsDir
()
{
t
.
Errorf
(
"prefix directory is not a file? %v"
,
fi
.
Mode
())
}
// we know it's there if we see the file, nothing more to
// do here
case
target
:
seen
=
true
if
!
fi
.
Mode
()
.
IsRegular
()
{
t
.
Errorf
(
"expected a regular file, mode: %04o"
,
fi
.
Mode
())
}
if
g
,
e
:=
fi
.
Mode
()
&
os
.
ModePerm
&
0007
,
os
.
FileMode
(
0000
);
g
!=
e
{
t
.
Errorf
(
"file should not be world accessible: %04o"
,
fi
.
Mode
())
}
default
:
t
.
Errorf
(
"saw unexpected directory entry: %q %v"
,
path
,
fi
.
Mode
())
}
return
nil
}
if
err
:=
filepath
.
Walk
(
temp
,
walk
);
err
!=
nil
{
t
.
Fatal
(
"walk: %v"
,
err
)
}
if
!
seen
{
t
.
Error
(
"did not see the data file"
)
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment