Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
dms3
go-unixfs
Commits
f68092e8
Unverified
Commit
f68092e8
authored
6 years ago
by
Steven Allen
Committed by
GitHub
6 years ago
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #18 from overbool/fix/issue-#17
fix(fsnode): issue #17
parents
f2f968df
8bea61e6
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
48 additions
and
32 deletions
+48
-32
hamt/hamt.go
hamt/hamt.go
+7
-6
importer/trickle/trickledag.go
importer/trickle/trickledag.go
+6
-6
io/resolve.go
io/resolve.go
+2
-2
mod/dagmodifier.go
mod/dagmodifier.go
+9
-10
test/utils.go
test/utils.go
+2
-2
unixfs.go
unixfs.go
+16
-0
unixfs_test.go
unixfs_test.go
+6
-6
No files found.
hamt/hamt.go
View file @
f68092e8
...
...
@@ -102,28 +102,29 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
return
nil
,
dag
.
ErrNotProtobuf
}
pbd
,
err
:=
format
.
FromBytes
(
pbnd
.
Data
())
fsn
,
err
:=
format
.
FSNode
FromBytes
(
pbnd
.
Data
())
if
err
!=
nil
{
return
nil
,
err
}
if
pbd
.
GetType
()
!=
upb
.
Data_HAMTShard
{
if
fsn
.
Type
()
!=
upb
.
Data_HAMTShard
{
return
nil
,
fmt
.
Errorf
(
"node was not a dir shard"
)
}
if
pbd
.
Get
HashType
()
!=
HashMurmur3
{
if
fsn
.
HashType
()
!=
HashMurmur3
{
return
nil
,
fmt
.
Errorf
(
"only murmur3 supported as hash function"
)
}
ds
,
err
:=
makeShard
(
dserv
,
int
(
pbd
.
Get
Fanout
()))
ds
,
err
:=
makeShard
(
dserv
,
int
(
fsn
.
Fanout
()))
if
err
!=
nil
{
return
nil
,
err
}
ds
.
nd
=
pbnd
.
Copy
()
.
(
*
dag
.
ProtoNode
)
ds
.
children
=
make
([]
child
,
len
(
pbnd
.
Links
()))
ds
.
bitfield
.
SetBytes
(
pbd
.
Get
Data
())
ds
.
hashFunc
=
pbd
.
Get
HashType
()
ds
.
bitfield
.
SetBytes
(
fsn
.
Data
())
ds
.
hashFunc
=
fsn
.
HashType
()
ds
.
builder
=
ds
.
nd
.
CidBuilder
()
return
ds
,
nil
...
...
This diff is collapsed.
Click to expand it.
importer/trickle/trickledag.go
View file @
f68092e8
...
...
@@ -277,12 +277,12 @@ func verifyTDagRec(n ipld.Node, depth int, p VerifyParams) error {
// zero depth dag is raw data block
switch
nd
:=
n
.
(
type
)
{
case
*
dag
.
ProtoNode
:
pb
n
,
err
:=
ft
.
FromBytes
(
nd
.
Data
())
fs
n
,
err
:=
ft
.
FSNode
FromBytes
(
nd
.
Data
())
if
err
!=
nil
{
return
err
}
if
pbn
.
Get
Type
()
!=
ft
.
TRaw
{
if
fsn
.
Type
()
!=
ft
.
TRaw
{
return
errors
.
New
(
"expected raw block"
)
}
...
...
@@ -325,16 +325,16 @@ func verifyTDagRec(n ipld.Node, depth int, p VerifyParams) error {
}
// Verify this is a branch node
pb
n
,
err
:=
ft
.
FromBytes
(
nd
.
Data
())
fs
n
,
err
:=
ft
.
FSNode
FromBytes
(
nd
.
Data
())
if
err
!=
nil
{
return
err
}
if
pbn
.
Get
Type
()
!=
ft
.
TFile
{
return
fmt
.
Errorf
(
"expected file as branch node, got: %s"
,
pbn
.
Get
Type
())
if
fsn
.
Type
()
!=
ft
.
TFile
{
return
fmt
.
Errorf
(
"expected file as branch node, got: %s"
,
fsn
.
Type
())
}
if
len
(
pb
n
.
Data
)
>
0
{
if
len
(
fs
n
.
Data
()
)
>
0
{
return
errors
.
New
(
"branch node should not have data"
)
}
...
...
This diff is collapsed.
Click to expand it.
io/resolve.go
View file @
f68092e8
...
...
@@ -15,7 +15,7 @@ import (
func
ResolveUnixfsOnce
(
ctx
context
.
Context
,
ds
ipld
.
NodeGetter
,
nd
ipld
.
Node
,
names
[]
string
)
(
*
ipld
.
Link
,
[]
string
,
error
)
{
switch
nd
:=
nd
.
(
type
)
{
case
*
dag
.
ProtoNode
:
upb
,
err
:=
ft
.
FromBytes
(
nd
.
Data
())
fsn
,
err
:=
ft
.
FSNode
FromBytes
(
nd
.
Data
())
if
err
!=
nil
{
// Not a unixfs node, use standard object traversal code
lnk
,
err
:=
nd
.
GetNodeLink
(
names
[
0
])
...
...
@@ -26,7 +26,7 @@ func ResolveUnixfsOnce(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, na
return
lnk
,
names
[
1
:
],
nil
}
switch
upb
.
Get
Type
()
{
switch
fsn
.
Type
()
{
case
ft
.
THAMTShard
:
rods
:=
dag
.
NewReadOnlyDagService
(
ds
)
s
,
err
:=
hamt
.
NewHamtFromDag
(
rods
,
nd
)
...
...
This diff is collapsed.
Click to expand it.
mod/dagmodifier.go
View file @
f68092e8
...
...
@@ -13,7 +13,6 @@ import (
trickle
"github.com/ipfs/go-unixfs/importer/trickle"
uio
"github.com/ipfs/go-unixfs/io"
proto
"github.com/gogo/protobuf/proto"
cid
"github.com/ipfs/go-cid"
chunker
"github.com/ipfs/go-ipfs-chunker"
ipld
"github.com/ipfs/go-ipld-format"
...
...
@@ -173,11 +172,11 @@ func (dm *DagModifier) Size() (int64, error) {
func
fileSize
(
n
ipld
.
Node
)
(
uint64
,
error
)
{
switch
nd
:=
n
.
(
type
)
{
case
*
mdag
.
ProtoNode
:
f
,
err
:=
ft
.
FromBytes
(
nd
.
Data
())
f
sn
,
err
:=
ft
.
FSNode
FromBytes
(
nd
.
Data
())
if
err
!=
nil
{
return
0
,
err
}
return
f
.
Get
File
s
ize
(),
nil
return
f
sn
.
File
S
ize
(),
nil
case
*
mdag
.
RawNode
:
return
uint64
(
len
(
nd
.
RawData
())),
nil
default
:
...
...
@@ -238,18 +237,18 @@ func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) {
if
len
(
n
.
Links
())
==
0
{
switch
nd0
:=
n
.
(
type
)
{
case
*
mdag
.
ProtoNode
:
f
,
err
:=
ft
.
FromBytes
(
nd0
.
Data
())
f
sn
,
err
:=
ft
.
FSNode
FromBytes
(
nd0
.
Data
())
if
err
!=
nil
{
return
cid
.
Cid
{},
err
}
_
,
err
=
dm
.
wrBuf
.
Read
(
f
.
Data
[
offset
:
])
_
,
err
=
dm
.
wrBuf
.
Read
(
f
sn
.
Data
()
[
offset
:
])
if
err
!=
nil
&&
err
!=
io
.
EOF
{
return
cid
.
Cid
{},
err
}
// Update newly written node..
b
,
err
:=
proto
.
Marshal
(
f
)
b
,
err
:=
fsn
.
GetBytes
(
)
if
err
!=
nil
{
return
cid
.
Cid
{},
err
}
...
...
@@ -300,13 +299,13 @@ func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (cid.Cid, error) {
return
cid
.
Cid
{},
ErrNotUnixfs
}
f
,
err
:=
ft
.
FromBytes
(
node
.
Data
())
f
sn
,
err
:=
ft
.
FSNode
FromBytes
(
node
.
Data
())
if
err
!=
nil
{
return
cid
.
Cid
{},
err
}
var
cur
uint64
for
i
,
bs
:=
range
f
.
Get
Block
s
izes
()
{
for
i
,
bs
:=
range
f
sn
.
Block
S
izes
()
{
// We found the correct child to write into
if
cur
+
bs
>
offset
{
child
,
err
:=
node
.
Links
()[
i
]
.
GetNode
(
dm
.
ctx
,
dm
.
dagserv
)
...
...
@@ -510,11 +509,11 @@ func (dm *DagModifier) dagTruncate(ctx context.Context, n ipld.Node, size uint64
switch
nd
:=
n
.
(
type
)
{
case
*
mdag
.
ProtoNode
:
// TODO: this can likely be done without marshaling and remarshaling
pb
n
,
err
:=
ft
.
FromBytes
(
nd
.
Data
())
fs
n
,
err
:=
ft
.
FSNode
FromBytes
(
nd
.
Data
())
if
err
!=
nil
{
return
nil
,
err
}
nd
.
SetData
(
ft
.
WrapData
(
pb
n
.
Data
[
:
size
]))
nd
.
SetData
(
ft
.
WrapData
(
fs
n
.
Data
()
[
:
size
]))
return
nd
,
nil
case
*
mdag
.
RawNode
:
return
mdag
.
NewRawNodeWPrefix
(
nd
.
RawData
()[
:
size
],
nd
.
Cid
()
.
Prefix
())
...
...
This diff is collapsed.
Click to expand it.
test/utils.go
View file @
f68092e8
...
...
@@ -107,7 +107,7 @@ func ArrComp(a, b []byte) error {
// PrintDag pretty-prints the given dag to stdout.
func
PrintDag
(
nd
*
mdag
.
ProtoNode
,
ds
ipld
.
DAGService
,
indent
int
)
{
pbd
,
err
:=
ft
.
FromBytes
(
nd
.
Data
())
fsn
,
err
:=
ft
.
FSNode
FromBytes
(
nd
.
Data
())
if
err
!=
nil
{
panic
(
err
)
}
...
...
@@ -115,7 +115,7 @@ func PrintDag(nd *mdag.ProtoNode, ds ipld.DAGService, indent int) {
for
i
:=
0
;
i
<
indent
;
i
++
{
fmt
.
Print
(
" "
)
}
fmt
.
Printf
(
"{size = %d, type = %s, children = %d"
,
pbd
.
Get
File
s
ize
(),
pbd
.
Get
Type
()
.
String
(),
len
(
pbd
.
GetBlocksizes
()
))
fmt
.
Printf
(
"{size = %d, type = %s, children = %d"
,
fsn
.
File
S
ize
(),
fsn
.
Type
()
.
String
(),
fsn
.
NumChildren
(
))
if
len
(
nd
.
Links
())
>
0
{
fmt
.
Println
()
}
...
...
This diff is collapsed.
Click to expand it.
unixfs.go
View file @
f68092e8
...
...
@@ -29,6 +29,7 @@ var (
)
// FromBytes unmarshals a byte slice as protobuf Data.
// Deprecated: Use `FSNodeFromBytes` instead to avoid direct manipulation of `pb.Data`.
func
FromBytes
(
data
[]
byte
)
(
*
pb
.
Data
,
error
)
{
pbdata
:=
new
(
pb
.
Data
)
err
:=
proto
.
Unmarshal
(
data
,
pbdata
)
...
...
@@ -182,6 +183,16 @@ func NewFSNode(dataType pb.Data_DataType) *FSNode {
return
n
}
// HashType gets hash type of format
func
(
n
*
FSNode
)
HashType
()
uint64
{
return
n
.
format
.
GetHashType
()
}
// Fanout gets fanout of format
func
(
n
*
FSNode
)
Fanout
()
uint64
{
return
n
.
format
.
GetFanout
()
}
// AddBlockSize adds the size of the next child block of this node
func
(
n
*
FSNode
)
AddBlockSize
(
s
uint64
)
{
n
.
UpdateFilesize
(
int64
(
s
))
...
...
@@ -200,6 +211,11 @@ func (n *FSNode) BlockSize(i int) uint64 {
return
n
.
format
.
Blocksizes
[
i
]
}
// BlockSizes gets blocksizes of format
func
(
n
*
FSNode
)
BlockSizes
()
[]
uint64
{
return
n
.
format
.
GetBlocksizes
()
}
// RemoveAllBlockSizes removes all the child block sizes of this node.
func
(
n
*
FSNode
)
RemoveAllBlockSizes
()
{
n
.
format
.
Blocksizes
=
[]
uint64
{}
...
...
This diff is collapsed.
Click to expand it.
unixfs_test.go
View file @
f68092e8
...
...
@@ -76,12 +76,12 @@ func TestPBdataTools(t *testing.T) {
t
.
Fatal
(
"Unwrap failed to produce the correct wrapped data."
)
}
rawPBdata
,
err
:=
FromBytes
(
rawPB
)
rawPBdata
,
err
:=
FSNode
FromBytes
(
rawPB
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
isRaw
:=
rawPBdata
.
Get
Type
()
==
TRaw
isRaw
:=
rawPBdata
.
Type
()
==
TRaw
if
!
isRaw
{
t
.
Fatal
(
"WrapData does not create pb.Data_Raw!"
)
}
...
...
@@ -97,8 +97,8 @@ func TestPBdataTools(t *testing.T) {
}
dirPB
:=
FolderPBData
()
dir
,
err
:=
FromBytes
(
dirPB
)
isDir
:=
dir
.
Get
Type
()
==
TDirectory
dir
,
err
:=
FSNode
FromBytes
(
dirPB
)
isDir
:=
dir
.
Type
()
==
TDirectory
if
!
isDir
{
t
.
Fatal
(
"FolderPBData does not create a directory!"
)
}
...
...
@@ -115,8 +115,8 @@ func TestPBdataTools(t *testing.T) {
t
.
Fatal
(
err
)
}
catSymPB
,
err
:=
FromBytes
(
catSym
)
isSym
:=
catSymPB
.
Get
Type
()
==
TSymlink
catSymPB
,
err
:=
FSNode
FromBytes
(
catSym
)
isSym
:=
catSymPB
.
Type
()
==
TSymlink
if
!
isSym
{
t
.
Fatal
(
"Failed to make a Symlink."
)
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment