Commit e23c2204 authored by Jeromy's avatar Jeromy

extract node interface

License: MIT
Signed-off-by: default avatarJeromy <why@ipfs.io>
parent 3e624eec
......@@ -7,6 +7,7 @@ import (
pb "github.com/ipfs/go-ipfs/merkledag/pb"
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
)
......@@ -22,9 +23,9 @@ func (n *ProtoNode) unmarshal(encoded []byte) error {
}
pbnl := pbn.GetLinks()
n.links = make([]*Link, len(pbnl))
n.links = make([]*node.Link, len(pbnl))
for i, l := range pbnl {
n.links[i] = &Link{Name: l.GetName(), Size: l.GetTsize()}
n.links[i] = &node.Link{Name: l.GetName(), Size: l.GetTsize()}
c, err := cid.Cast(l.GetHash())
if err != nil {
return fmt.Errorf("Link hash #%d is not valid multihash. %v", i, err)
......
......@@ -13,6 +13,7 @@ import (
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)
var log = logging.Logger("merkledag")
......@@ -20,9 +21,9 @@ var ErrNotFound = fmt.Errorf("merkledag: not found")
// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
Add(Node) (*cid.Cid, error)
Get(context.Context, *cid.Cid) (Node, error)
Remove(Node) error
Add(node.Node) (*cid.Cid, error)
Get(context.Context, *cid.Cid) (node.Node, error)
Remove(node.Node) error
// GetDAG returns, in order, all the single leve child
// nodes of the passed in node.
......@@ -36,7 +37,7 @@ type DAGService interface {
type LinkService interface {
// Return all links for a node, may be more effect than
// calling Get in DAGService
GetLinks(context.Context, *cid.Cid) ([]*Link, error)
GetLinks(context.Context, *cid.Cid) ([]*node.Link, error)
GetOfflineLinkService() LinkService
}
......@@ -45,19 +46,6 @@ func NewDAGService(bs bserv.BlockService) *dagService {
return &dagService{Blocks: bs}
}
type Node interface {
Resolve(path []string) (*Link, []string, error)
Links() []*Link
Tree() []string
Stat() (*NodeStat, error)
Size() (uint64, error)
Cid() *cid.Cid
Loggable() map[string]interface{}
RawData() []byte
String() string
}
// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
......@@ -68,7 +56,7 @@ type dagService struct {
}
// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd Node) (*cid.Cid, error) {
func (n *dagService) Add(nd node.Node) (*cid.Cid, error) {
if n == nil { // FIXME remove this assertion. protect with constructor invariant
return nil, fmt.Errorf("dagService is nil")
}
......@@ -81,7 +69,7 @@ func (n *dagService) Batch() *Batch {
}
// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, c *cid.Cid) (Node, error) {
func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) {
if n == nil {
return nil, fmt.Errorf("dagService is nil")
}
......@@ -97,7 +85,7 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (Node, error) {
return nil, fmt.Errorf("Failed to get block for %s: %v", c, err)
}
var res Node
var res node.Node
switch c.Type() {
case cid.Protobuf:
out, err := DecodeProtobuf(b.RawData())
......@@ -116,7 +104,7 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (Node, error) {
return res, nil
}
func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*Link, error) {
func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) {
node, err := n.Get(ctx, c)
if err != nil {
return nil, err
......@@ -133,7 +121,7 @@ func (n *dagService) GetOfflineLinkService() LinkService {
}
}
func (n *dagService) Remove(nd Node) error {
func (n *dagService) Remove(nd node.Node) error {
return n.Blocks.DeleteBlock(nd)
}
......@@ -155,7 +143,7 @@ func FindLinks(links []*cid.Cid, c *cid.Cid, start int) []int {
}
type NodeOption struct {
Node Node
Node node.Node
Err error
}
......@@ -178,7 +166,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *Node
c := b.Cid()
var nd Node
var nd node.Node
switch c.Type() {
case cid.Protobuf:
decnd, err := DecodeProtobuf(b.RawData())
......@@ -209,7 +197,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *Node
// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func GetDAG(ctx context.Context, ds DAGService, root Node) []NodeGetter {
func GetDAG(ctx context.Context, ds DAGService, root node.Node) []NodeGetter {
var cids []*cid.Cid
for _, lnk := range root.Links() {
cids = append(cids, lnk.Cid)
......@@ -281,16 +269,16 @@ func dedupeKeys(cids []*cid.Cid) []*cid.Cid {
func newNodePromise(ctx context.Context) NodeGetter {
return &nodePromise{
recv: make(chan Node, 1),
recv: make(chan node.Node, 1),
ctx: ctx,
err: make(chan error, 1),
}
}
type nodePromise struct {
cache Node
cache node.Node
clk sync.Mutex
recv chan Node
recv chan node.Node
ctx context.Context
err chan error
}
......@@ -300,9 +288,9 @@ type nodePromise struct {
// from its internal channels, subsequent calls will return the
// cached node.
type NodeGetter interface {
Get(context.Context) (Node, error)
Get(context.Context) (node.Node, error)
Fail(err error)
Send(Node)
Send(node.Node)
}
func (np *nodePromise) Fail(err error) {
......@@ -318,7 +306,7 @@ func (np *nodePromise) Fail(err error) {
np.err <- err
}
func (np *nodePromise) Send(nd Node) {
func (np *nodePromise) Send(nd node.Node) {
var already bool
np.clk.Lock()
if np.cache != nil {
......@@ -334,7 +322,7 @@ func (np *nodePromise) Send(nd Node) {
np.recv <- nd
}
func (np *nodePromise) Get(ctx context.Context) (Node, error) {
func (np *nodePromise) Get(ctx context.Context) (node.Node, error) {
np.clk.Lock()
c := np.cache
np.clk.Unlock()
......@@ -362,7 +350,7 @@ type Batch struct {
MaxSize int
}
func (t *Batch) Add(nd Node) (*cid.Cid, error) {
func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
t.blocks = append(t.blocks, nd)
t.size += len(nd.RawData())
if t.size > t.MaxSize {
......
......@@ -2,6 +2,7 @@ package merkledag_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
......@@ -19,10 +20,10 @@ import (
mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
dstest "github.com/ipfs/go-ipfs/merkledag/test"
uio "github.com/ipfs/go-ipfs/unixfs/io"
key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
"context"
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
)
......@@ -85,7 +86,7 @@ func SubtestNodeStat(t *testing.T, n *ProtoNode) {
k := n.Key()
expected := NodeStat{
expected := node.NodeStat{
NumLinks: len(n.Links()),
BlockSize: len(enc),
LinksSize: len(enc) - len(n.Data()), // includes framing.
......@@ -206,7 +207,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
}
}
func assertCanGet(t *testing.T, ds DAGService, n Node) {
func assertCanGet(t *testing.T, ds DAGService, n node.Node) {
if _, err := ds.Get(context.Background(), n.Cid()); err != nil {
t.Fatal(err)
}
......@@ -268,8 +269,8 @@ func TestEnumerateChildren(t *testing.T) {
t.Fatal(err)
}
var traverse func(n Node)
traverse = func(n Node) {
var traverse func(n node.Node)
traverse = func(n node.Node) {
// traverse dag and check
for _, lnk := range n.Links() {
c := lnk.Cid
......
package merkledag
import (
"fmt"
"context"
"fmt"
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
mh "gx/ipfs/QmYDds3421prZgqKbLpEK7T9Aa2eVdQ7o3YarX1LVLdP2J/go-multihash"
key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)
var ErrNotProtobuf = fmt.Errorf("expected protobuf dag node")
var ErrLinkNotFound = fmt.Errorf("no link by that name")
// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type ProtoNode struct {
links []*Link
links []*node.Link
data []byte
// cache encoded/marshaled value
......@@ -24,57 +25,12 @@ type ProtoNode struct {
cached *cid.Cid
}
// NodeStat is a statistics object for a Node. Mostly sizes.
type NodeStat struct {
Hash string
NumLinks int // number of links in link table
BlockSize int // size of the raw, encoded data
LinksSize int // size of the links segment
DataSize int // size of the data segment
CumulativeSize int // cumulative size of object and its references
}
func (ns NodeStat) String() string {
f := "NodeStat{NumLinks: %d, BlockSize: %d, LinksSize: %d, DataSize: %d, CumulativeSize: %d}"
return fmt.Sprintf(f, ns.NumLinks, ns.BlockSize, ns.LinksSize, ns.DataSize, ns.CumulativeSize)
}
// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
// utf string name. should be unique per object
Name string // utf8
// cumulative size of target object
Size uint64
// multihash of the target object
Cid *cid.Cid
}
type LinkSlice []*Link
type LinkSlice []*node.Link
func (ls LinkSlice) Len() int { return len(ls) }
func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] }
func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name }
// MakeLink creates a link to the given node
func MakeLink(n Node) (*Link, error) {
s, err := n.Size()
if err != nil {
return nil, err
}
return &Link{
Size: s,
Cid: n.Cid(),
}, nil
}
// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(ctx context.Context, serv DAGService) (Node, error) {
return serv.Get(ctx, l.Cid)
}
func NodeWithData(d []byte) *ProtoNode {
return &ProtoNode{data: d}
}
......@@ -83,13 +39,13 @@ func NodeWithData(d []byte) *ProtoNode {
func (n *ProtoNode) AddNodeLink(name string, that *ProtoNode) error {
n.encoded = nil
lnk, err := MakeLink(that)
lnk.Name = name
lnk, err := node.MakeLink(that)
if err != nil {
return err
}
lnk.Name = name
n.AddRawLink(name, lnk)
return nil
......@@ -97,9 +53,9 @@ func (n *ProtoNode) AddNodeLink(name string, that *ProtoNode) error {
// AddNodeLinkClean adds a link to another node. without keeping a reference to
// the child node
func (n *ProtoNode) AddNodeLinkClean(name string, that Node) error {
func (n *ProtoNode) AddNodeLinkClean(name string, that node.Node) error {
n.encoded = nil
lnk, err := MakeLink(that)
lnk, err := node.MakeLink(that)
if err != nil {
return err
}
......@@ -109,9 +65,9 @@ func (n *ProtoNode) AddNodeLinkClean(name string, that Node) error {
}
// AddRawLink adds a copy of a link to this node
func (n *ProtoNode) AddRawLink(name string, l *Link) error {
func (n *ProtoNode) AddRawLink(name string, l *node.Link) error {
n.encoded = nil
n.links = append(n.links, &Link{
n.links = append(n.links, &node.Link{
Name: name,
Size: l.Size,
Cid: l.Cid,
......@@ -123,7 +79,7 @@ func (n *ProtoNode) AddRawLink(name string, l *Link) error {
// Remove a link on this node by the given name
func (n *ProtoNode) RemoveNodeLink(name string) error {
n.encoded = nil
good := make([]*Link, 0, len(n.links))
good := make([]*node.Link, 0, len(n.links))
var found bool
for _, l := range n.links {
......@@ -143,10 +99,10 @@ func (n *ProtoNode) RemoveNodeLink(name string) error {
}
// Return a copy of the link with given name
func (n *ProtoNode) GetNodeLink(name string) (*Link, error) {
func (n *ProtoNode) GetNodeLink(name string) (*node.Link, error) {
for _, l := range n.links {
if l.Name == name {
return &Link{
return &node.Link{
Name: l.Name,
Size: l.Size,
Cid: l.Cid,
......@@ -156,8 +112,6 @@ func (n *ProtoNode) GetNodeLink(name string) (*Link, error) {
return nil, ErrLinkNotFound
}
var ErrNotProtobuf = fmt.Errorf("expected protobuf dag node")
func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds DAGService, name string) (*ProtoNode, error) {
nd, err := n.GetLinkedNode(ctx, ds, name)
if err != nil {
......@@ -172,7 +126,7 @@ func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds DAGService, name
return pbnd, nil
}
func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds DAGService, name string) (Node, error) {
func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds DAGService, name string) (node.Node, error) {
lnk, err := n.GetNodeLink(name)
if err != nil {
return nil, err
......@@ -191,7 +145,7 @@ func (n *ProtoNode) Copy() *ProtoNode {
}
if len(n.links) > 0 {
nnode.links = make([]*Link, len(n.links))
nnode.links = make([]*node.Link, len(n.links))
copy(nnode.links, n.links)
}
return nnode
......@@ -238,7 +192,7 @@ func (n *ProtoNode) Size() (uint64, error) {
}
// Stat returns statistics on the node.
func (n *ProtoNode) Stat() (*NodeStat, error) {
func (n *ProtoNode) Stat() (*node.NodeStat, error) {
enc, err := n.EncodeProtobuf(false)
if err != nil {
return nil, err
......@@ -249,7 +203,7 @@ func (n *ProtoNode) Stat() (*NodeStat, error) {
return nil, err
}
return &NodeStat{
return &node.NodeStat{
Hash: n.Key().B58String(),
NumLinks: len(n.links),
BlockSize: len(enc),
......@@ -291,15 +245,15 @@ func (n *ProtoNode) Multihash() mh.Multihash {
return n.cached.Hash()
}
func (n *ProtoNode) Links() []*Link {
func (n *ProtoNode) Links() []*node.Link {
return n.links
}
func (n *ProtoNode) SetLinks(links []*Link) {
func (n *ProtoNode) SetLinks(links []*node.Link) {
n.links = links
}
func (n *ProtoNode) Resolve(path []string) (*Link, []string, error) {
func (n *ProtoNode) Resolve(path []string) (*node.Link, []string, error) {
if len(path) == 0 {
return nil, nil, fmt.Errorf("end of path, no more links to resolve")
}
......
package merkledag_test
import (
"context"
"testing"
. "github.com/ipfs/go-ipfs/merkledag"
mdtest "github.com/ipfs/go-ipfs/merkledag/test"
"context"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)
func TestRemoveLink(t *testing.T) {
nd := &ProtoNode{}
nd.SetLinks([]*Link{
&Link{Name: "a"},
&Link{Name: "b"},
&Link{Name: "a"},
&Link{Name: "a"},
&Link{Name: "c"},
&Link{Name: "a"},
nd.SetLinks([]*node.Link{
{Name: "a"},
{Name: "b"},
{Name: "a"},
{Name: "a"},
{Name: "c"},
{Name: "a"},
})
err := nd.RemoveNodeLink("a")
......@@ -65,10 +66,10 @@ func TestFindLink(t *testing.T) {
}
nd := &ProtoNode{}
nd.SetLinks([]*Link{
&Link{Name: "a", Cid: k},
&Link{Name: "c", Cid: k},
&Link{Name: "b", Cid: k},
nd.SetLinks([]*node.Link{
{Name: "a", Cid: k},
{Name: "c", Cid: k},
{Name: "b", Cid: k},
})
_, err = ds.Add(nd)
......@@ -112,10 +113,10 @@ func TestFindLink(t *testing.T) {
func TestNodeCopy(t *testing.T) {
nd := &ProtoNode{}
nd.SetLinks([]*Link{
&Link{Name: "a"},
&Link{Name: "c"},
&Link{Name: "b"},
nd.SetLinks([]*node.Link{
{Name: "a"},
{Name: "c"},
{Name: "b"},
})
nd.SetData([]byte("testing"))
......
......@@ -2,11 +2,10 @@
package traverse
import (
"errors"
"context"
"errors"
mdag "github.com/ipfs/go-ipfs/merkledag"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)
// Order is an identifier for traversal algorithm orders
......@@ -20,7 +19,7 @@ const (
// Options specifies a series of traversal options
type Options struct {
DAG mdag.DAGService // the dagservice to fetch nodes
DAG node.NodeGetter // the dagservice to fetch nodes
Order Order // what order to traverse in
Func Func // the function to perform at each step
ErrFunc ErrFunc // see ErrFunc. Optional
......@@ -30,7 +29,7 @@ type Options struct {
// State is a current traversal state
type State struct {
Node mdag.Node
Node node.Node
Depth int
}
......@@ -39,7 +38,7 @@ type traversal struct {
seen map[string]struct{}
}
func (t *traversal) shouldSkip(n mdag.Node) (bool, error) {
func (t *traversal) shouldSkip(n node.Node) (bool, error) {
if t.opts.SkipDuplicates {
k := n.Cid()
if _, found := t.seen[k.KeyString()]; found {
......@@ -59,9 +58,9 @@ func (t *traversal) callFunc(next State) error {
// stop processing. if it returns a nil node, just skip it.
//
// the error handling is a little complicated.
func (t *traversal) getNode(link *mdag.Link) (mdag.Node, error) {
func (t *traversal) getNode(link *node.Link) (node.Node, error) {
getNode := func(l *mdag.Link) (mdag.Node, error) {
getNode := func(l *node.Link) (node.Node, error) {
next, err := l.GetNode(context.TODO(), t.opts.DAG)
if err != nil {
return nil, err
......@@ -99,7 +98,7 @@ type Func func(current State) error
//
type ErrFunc func(err error) error
func Traverse(root mdag.Node, o Options) error {
func Traverse(root node.Node, o Options) error {
t := traversal{
opts: o,
seen: map[string]struct{}{},
......
......@@ -7,6 +7,8 @@ import (
mdag "github.com/ipfs/go-ipfs/merkledag"
mdagtest "github.com/ipfs/go-ipfs/merkledag/test"
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
)
func TestDFSPreNoSkip(t *testing.T) {
......@@ -321,7 +323,7 @@ func TestBFSSkip(t *testing.T) {
`))
}
func testWalkOutputs(t *testing.T, root mdag.Node, opts Options, expect []byte) {
func testWalkOutputs(t *testing.T, root node.Node, opts Options, expect []byte) {
expect = bytes.TrimLeft(expect, "\n")
buf := new(bytes.Buffer)
......@@ -348,7 +350,7 @@ func testWalkOutputs(t *testing.T, root mdag.Node, opts Options, expect []byte)
}
}
func newFan(t *testing.T, ds mdag.DAGService) mdag.Node {
func newFan(t *testing.T, ds mdag.DAGService) node.Node {
a := mdag.NodeWithData([]byte("/a"))
addLink(t, ds, a, child(t, ds, a, "aa"))
addLink(t, ds, a, child(t, ds, a, "ab"))
......@@ -357,7 +359,7 @@ func newFan(t *testing.T, ds mdag.DAGService) mdag.Node {
return a
}
func newLinkedList(t *testing.T, ds mdag.DAGService) mdag.Node {
func newLinkedList(t *testing.T, ds mdag.DAGService) node.Node {
a := mdag.NodeWithData([]byte("/a"))
aa := child(t, ds, a, "aa")
aaa := child(t, ds, aa, "aaa")
......@@ -370,7 +372,7 @@ func newLinkedList(t *testing.T, ds mdag.DAGService) mdag.Node {
return a
}
func newBinaryTree(t *testing.T, ds mdag.DAGService) mdag.Node {
func newBinaryTree(t *testing.T, ds mdag.DAGService) node.Node {
a := mdag.NodeWithData([]byte("/a"))
aa := child(t, ds, a, "aa")
ab := child(t, ds, a, "ab")
......@@ -383,7 +385,7 @@ func newBinaryTree(t *testing.T, ds mdag.DAGService) mdag.Node {
return a
}
func newBinaryDAG(t *testing.T, ds mdag.DAGService) mdag.Node {
func newBinaryDAG(t *testing.T, ds mdag.DAGService) node.Node {
a := mdag.NodeWithData([]byte("/a"))
aa := child(t, ds, a, "aa")
aaa := child(t, ds, aa, "aaa")
......@@ -400,7 +402,7 @@ func newBinaryDAG(t *testing.T, ds mdag.DAGService) mdag.Node {
return a
}
func addLink(t *testing.T, ds mdag.DAGService, a, b mdag.Node) {
func addLink(t *testing.T, ds mdag.DAGService, a, b node.Node) {
to := string(a.(*mdag.ProtoNode).Data()) + "2" + string(b.(*mdag.ProtoNode).Data())
if _, err := ds.Add(b); err != nil {
t.Error(err)
......@@ -410,6 +412,6 @@ func addLink(t *testing.T, ds mdag.DAGService, a, b mdag.Node) {
}
}
func child(t *testing.T, ds mdag.DAGService, a mdag.Node, name string) mdag.Node {
func child(t *testing.T, ds mdag.DAGService, a node.Node, name string) node.Node {
return mdag.NodeWithData([]byte(string(a.(*mdag.ProtoNode).Data()) + "/" + name))
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment