Commit 0f9e9ed7 authored by Steven Allen's avatar Steven Allen

Make Batch a helper type/function

parent b3a1f4b9
package format
import (
cid "github.com/ipfs/go-cid"
)
func Batching(ds DAGService) *Batch {
return &Batch{
ds: ds,
MaxSize: 8 << 20,
// By default, only batch up to 128 nodes at a time.
// The current implementation of flatfs opens this many file
// descriptors at the same time for the optimized batch write.
MaxBlocks: 128,
}
}
type Batch struct {
ds DAGService
// TODO: try to re-use memory.
nodes []Node
size int
MaxSize int
MaxBlocks int
}
func (t *Batch) Add(nd Node) (*cid.Cid, error) {
t.nodes = append(t.nodes, nd)
t.size += len(nd.RawData())
if t.size > t.MaxSize || len(t.nodes) > t.MaxBlocks {
return nd.Cid(), t.Commit()
}
return nd.Cid(), nil
}
func (t *Batch) Commit() error {
_, err := t.ds.AddMany(t.nodes)
t.nodes = nil
t.size = 0
return err
}
......@@ -36,25 +36,11 @@ type DAGService interface {
// nodes of the passed in node.
GetMany(context.Context, []*cid.Cid) <-chan *NodeOption
Batch() Batch
AddMany([]Node) ([]*cid.Cid, error)
LinkService
}
// An interface for batch-adding nodes to a DAG.
// TODO: Is this really the *right* level to do this at?
// Why not just `DAGService.AddMany` + a concrete helper type?
//
// This will be a breaking change *regardless* of what we do as `Batch` *used*
// to be a plain struct (passed around by pointer). I had to change this to
// avoid requiring a `BlockService` (which would introduce the concept of
// exchanges and I really don't want to go down that rabbit hole).
type Batch interface {
Add(nd Node) (*cid.Cid, error)
Commit() error
}
// TODO: Replace this? I'm really not convinced this interface pulls its weight.
//
// Instead, we could add an `Offline()` function to `NodeGetter` that returns an
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment