Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
dms3
go-unixfs
Commits
e2784eb5
Unverified
Commit
e2784eb5
authored
6 years ago
by
Hector Sanjuan
Committed by
GitHub
6 years ago
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #41 from ipfs/fix/27-remove-batching
Fix #27: Remove batching from importers
parents
0d29aa16
2d94c3b7
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
22 additions
and
59 deletions
+22
-59
.gx/lastpubver
.gx/lastpubver
+1
-1
importer/balanced/builder.go
importer/balanced/builder.go
+2
-2
importer/helpers/dagbuilder.go
importer/helpers/dagbuilder.go
+9
-29
importer/helpers/helpers.go
importer/helpers/helpers.go
+2
-2
importer/trickle/trickledag.go
importer/trickle/trickledag.go
+1
-18
package.json
package.json
+7
-7
No files found.
.gx/lastpubver
View file @
e2784eb5
1.1.1
5
: Qm
TJUySFxXjh54zEoFbzQEmGD3yj89XKS3A28y7Nqsn1TC
1.1.1
6
: Qm
ZF5iqaRD1hgh15tomihLJdrXjQcaY32iNH73aTCWvPKe
This diff is collapsed.
Click to expand it.
importer/balanced/builder.go
View file @
e2784eb5
...
...
@@ -130,7 +130,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
// This works without Filestore support (`ProcessFileStore`).
// TODO: Why? Is there a test case missing?
return
db
.
Add
NodeAndClose
(
root
)
return
root
,
db
.
Add
(
root
)
}
// The first `root` will be a single leaf node with data
...
...
@@ -160,7 +160,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
}
}
return
db
.
Add
NodeAndClose
(
root
)
return
root
,
db
.
Add
(
root
)
}
// fillNodeRec will "fill" the given internal (non-leaf) `node` with data by
...
...
This diff is collapsed.
Click to expand it.
importer/helpers/dagbuilder.go
View file @
e2784eb5
...
...
@@ -6,6 +6,7 @@ import (
"os"
dag
"github.com/ipfs/go-merkledag"
ft
"github.com/ipfs/go-unixfs"
pb
"github.com/ipfs/go-unixfs/pb"
...
...
@@ -25,7 +26,6 @@ type DagBuilderHelper struct {
rawLeaves
bool
nextData
[]
byte
// the next item to return.
maxlinks
int
batch
*
ipld
.
Batch
cidBuilder
cid
.
Builder
// Filestore support variables.
...
...
@@ -78,7 +78,6 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper {
rawLeaves
:
dbp
.
RawLeaves
,
cidBuilder
:
dbp
.
CidBuilder
,
maxlinks
:
dbp
.
Maxlinks
,
batch
:
ipld
.
NewBatch
(
context
.
TODO
(),
dbp
.
Dagserv
),
}
if
fi
,
ok
:=
spl
.
Reader
()
.
(
files
.
FileInfo
);
dbp
.
NoCopy
&&
ok
{
db
.
fullPath
=
fi
.
AbsPath
()
...
...
@@ -327,8 +326,8 @@ func (db *DagBuilderHelper) ProcessFileStore(node ipld.Node, dataSize uint64) ip
return
node
}
// Add sends a node to the DAGService, and returns it.
func
(
db
*
DagBuilderHelper
)
Add
(
node
*
UnixfsNode
)
(
ipld
.
Node
,
error
)
{
// Add
UnixfsNode
sends a node to the DAGService, and returns it
as ipld.Node
.
func
(
db
*
DagBuilderHelper
)
Add
UnixfsNode
(
node
*
UnixfsNode
)
(
ipld
.
Node
,
error
)
{
dn
,
err
:=
node
.
GetDagNode
()
if
err
!=
nil
{
return
nil
,
err
...
...
@@ -342,36 +341,17 @@ func (db *DagBuilderHelper) Add(node *UnixfsNode) (ipld.Node, error) {
return
dn
,
nil
}
// Add inserts the given node in the DAGService.
func
(
db
*
DagBuilderHelper
)
Add
(
node
ipld
.
Node
)
error
{
return
db
.
dserv
.
Add
(
context
.
TODO
(),
node
)
}
// Maxlinks returns the configured maximum number for links
// for nodes built with this helper.
func
(
db
*
DagBuilderHelper
)
Maxlinks
()
int
{
return
db
.
maxlinks
}
// Close has the DAGService perform a batch Commit operation.
// It should be called at the end of the building process to make
// sure all data is persisted.
func
(
db
*
DagBuilderHelper
)
Close
()
error
{
return
db
.
batch
.
Commit
()
}
// AddNodeAndClose adds the last `ipld.Node` from the DAG and
// closes the builder. It returns the same `node` passed as
// argument.
func
(
db
*
DagBuilderHelper
)
AddNodeAndClose
(
node
ipld
.
Node
)
(
ipld
.
Node
,
error
)
{
err
:=
db
.
batch
.
Add
(
node
)
if
err
!=
nil
{
return
nil
,
err
}
err
=
db
.
Close
()
if
err
!=
nil
{
return
nil
,
err
}
return
node
,
nil
}
// FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a
// `dag.ProtoNode`. Instead of just having a single `ipld.Node` that
// would need to be constantly (un)packed to access and modify its
...
...
@@ -421,7 +401,7 @@ func (n *FSNodeOverDag) AddChild(child ipld.Node, fileSize uint64, db *DagBuilde
n
.
file
.
AddBlockSize
(
fileSize
)
return
db
.
batch
.
Add
(
child
)
return
db
.
Add
(
child
)
}
// Commit unifies (resolves) the cache nodes into a single `ipld.Node`
...
...
This diff is collapsed.
Click to expand it.
importer/helpers/helpers.go
View file @
e2784eb5
...
...
@@ -6,6 +6,7 @@ import (
"os"
dag
"github.com/ipfs/go-merkledag"
ft
"github.com/ipfs/go-unixfs"
cid
"github.com/ipfs/go-cid"
...
...
@@ -103,8 +104,7 @@ func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
return
err
}
err
=
db
.
batch
.
Add
(
childnode
)
_
,
err
=
db
.
AddUnixfsNode
(
child
)
return
err
}
...
...
This diff is collapsed.
Click to expand it.
importer/trickle/trickledag.go
View file @
e2784eb5
...
...
@@ -42,16 +42,7 @@ func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
return
nil
,
err
}
out
,
err
:=
db
.
Add
(
root
)
if
err
!=
nil
{
return
nil
,
err
}
if
err
:=
db
.
Close
();
err
!=
nil
{
return
nil
,
err
}
return
out
,
nil
return
db
.
AddUnixfsNode
(
root
)
}
// fillTrickleRec creates a trickle (sub-)tree with an optional maximum specified depth
...
...
@@ -92,14 +83,6 @@ func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out i
return
nil
,
dag
.
ErrNotProtobuf
}
defer
func
()
{
if
errOut
==
nil
{
if
err
:=
db
.
Close
();
err
!=
nil
{
errOut
=
err
}
}
}()
// Convert to unixfs node for working with easily
ufsn
,
err
:=
h
.
NewUnixfsNodeFromDag
(
base
)
if
err
!=
nil
{
...
...
This diff is collapsed.
Click to expand it.
package.json
View file @
e2784eb5
...
...
@@ -27,15 +27,15 @@
},
{
"author"
:
"why"
,
"hash"
:
"Qm
Y8BMUSpCwNiTmFhACmC9Bt1qT63cHP35AoQAus4x14qH
"
,
"hash"
:
"Qm
dD9LnoZLYPu4LkkWBH2PkSKVneR38PRDd3b5LeRg2egv
"
,
"name"
:
"go-merkledag"
,
"version"
:
"1.1.1
3
"
"version"
:
"1.1.1
4
"
},
{
"author"
:
"hector"
,
"hash"
:
"QmP
G32VXR5jmpo9q8R9FNdR4Ae97Ky9CiZE6SctJLUB79H
"
,
"hash"
:
"QmP
TiYSsPRJTxkVqKb1DoK8zoPStqeBD5zgiXze7vrzagV
"
,
"name"
:
"go-ipfs-posinfo"
,
"version"
:
"0.1.
0
"
"version"
:
"0.1.
1
"
},
{
"author"
:
"Stebalien"
,
...
...
@@ -63,9 +63,9 @@
},
{
"author"
:
"whyrusleeping"
,
"hash"
:
"Qmd
DXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL
"
,
"hash"
:
"Qmd
VNBLt7RMYnZwqBQJeexmSbTEDzERjBQUfs5McuPfEtB
"
,
"name"
:
"go-ipld-format"
,
"version"
:
"0.
6
.0"
"version"
:
"0.
7
.0"
},
{
"author"
:
"magik6k"
,
...
...
@@ -79,6 +79,6 @@
"license"
:
""
,
"name"
:
"go-unixfs"
,
"releaseCmd"
:
"git commit -a -m
\"
gx publish $VERSION
\"
"
,
"version"
:
"1.1.1
5
"
"version"
:
"1.1.1
6
"
}
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment