cache encoded data when reading dag nodes from disk

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

coding.go

@@ -33,7 +33,8 @@ func (n *Node) unmarshal(encoded []byte) error {
 	}
 	sort.Stable(LinkSlice(n.Links)) // keep links sorted
 
-	n.Data = pbn.GetData()
+	n.data = pbn.GetData()
+	n.encoded = encoded
 	return nil
 }
 
@@ -62,8 +63,8 @@ func (n *Node) getPBNode() *pb.PBNode {
 		pbn.Links[i].Hash = []byte(l.Hash)
 	}
 
-	if len(n.Data) > 0 {
-		pbn.Data = n.Data
+	if len(n.data) > 0 {
+		pbn.Data = n.data
 	}
 	return pbn
 }
@@ -73,11 +74,15 @@ func (n *Node) getPBNode() *pb.PBNode {
 func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
 	sort.Stable(LinkSlice(n.Links)) // keep links sorted
 	if n.encoded == nil || force {
+		n.cached = nil
 		var err error
 		n.encoded, err = n.Marshal()
 		if err != nil {
 			return nil, err
 		}
+	}
+
+	if n.cached == nil {
 		n.cached = u.Hash(n.encoded)
 	}
 

merkledag.go

@@ -93,6 +93,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
 		}
 		return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
 	}
+
+	res.cached = k.ToMultihash()
+
 	return res, nil
 }
 
@@ -147,6 +150,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
 					out <- &NodeOption{Err: err}
 					return
 				}
+				nd.cached = b.Key().ToMultihash()
 
 				// buffered, no need to select
 				out <- &NodeOption{Node: nd}

merkledag_test.go

@@ -46,9 +46,9 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner {
 
 func TestNode(t *testing.T) {
 
-	n1 := &Node{Data: []byte("beep")}
-	n2 := &Node{Data: []byte("boop")}
-	n3 := &Node{Data: []byte("beep boop")}
+	n1 := NodeWithData([]byte("beep"))
+	n2 := NodeWithData([]byte("boop"))
+	n3 := NodeWithData([]byte("beep boop"))
 	if err := n3.AddNodeLink("beep-link", n1); err != nil {
 		t.Error(err)
 	}
@@ -58,7 +58,7 @@ func TestNode(t *testing.T) {
 
 	printn := func(name string, n *Node) {
 		fmt.Println(">", name)
-		fmt.Println("data:", string(n.Data))
+		fmt.Println("data:", string(n.Data()))
 
 		fmt.Println("links:")
 		for _, l := range n.Links {
@@ -118,8 +118,8 @@ func SubtestNodeStat(t *testing.T, n *Node) {
 	expected := NodeStat{
 		NumLinks:       len(n.Links),
 		BlockSize:      len(enc),
-		LinksSize:      len(enc) - len(n.Data), // includes framing.
-		DataSize:       len(n.Data),
+		LinksSize:      len(enc) - len(n.Data()), // includes framing.
+		DataSize:       len(n.Data()),
 		CumulativeSize: int(cumSize),
 		Hash:           k.B58String(),
 	}
@@ -255,7 +255,7 @@ func TestEmptyKey(t *testing.T) {
 
 func TestCantGet(t *testing.T) {
 	dsp := getDagservAndPinner(t)
-	a := &Node{Data: []byte("A")}
+	a := NodeWithData([]byte("A"))
 
 	k, err := a.Key()
 	if err != nil {
@@ -339,7 +339,7 @@ func TestFetchFailure(t *testing.T) {
 
 	top := new(Node)
 	for i := 0; i < 10; i++ {
-		nd := &Node{Data: []byte{byte('a' + i)}}
+		nd := NodeWithData([]byte{byte('a' + i)})
 		_, err := ds.Add(nd)
 		if err != nil {
 			t.Fatal(err)
@@ -352,7 +352,7 @@ func TestFetchFailure(t *testing.T) {
 	}
 
 	for i := 0; i < 10; i++ {
-		nd := &Node{Data: []byte{'f', 'a' + byte(i)}}
+		nd := NodeWithData([]byte{'f', 'a' + byte(i)})
 		_, err := ds_bad.Add(nd)
 		if err != nil {
 			t.Fatal(err)

node.go

@@ -15,7 +15,7 @@ var ErrLinkNotFound = fmt.Errorf("no link by that name")
 // nodes have opaque data and a set of navigable links.
 type Node struct {
 	Links []*Link
-	Data  []byte
+	data  []byte
 
 	// cache encoded/marshaled value
 	encoded []byte
@@ -78,6 +78,10 @@ func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
 	return serv.Get(ctx, key.Key(l.Hash))
 }
 
+func NodeWithData(d []byte) *Node {
+	return &Node{data: d}
+}
+
 // AddNodeLink adds a link to another node.
 func (n *Node) AddNodeLink(name string, that *Node) error {
 	n.encoded = nil
@@ -168,9 +172,9 @@ func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*
 // NOTE: Does not make copies of Node objects in the links.
 func (n *Node) Copy() *Node {
 	nnode := new(Node)
-	if len(n.Data) > 0 {
-		nnode.Data = make([]byte, len(n.Data))
-		copy(nnode.Data, n.Data)
+	if len(n.data) > 0 {
+		nnode.data = make([]byte, len(n.data))
+		copy(nnode.data, n.data)
 	}
 
 	if len(n.Links) > 0 {
@@ -180,6 +184,16 @@ func (n *Node) Copy() *Node {
 	return nnode
 }
 
+func (n *Node) Data() []byte {
+	return n.data
+}
+
+func (n *Node) SetData(d []byte) {
+	n.encoded = nil
+	n.cached = nil
+	n.data = d
+}
+
 // UpdateNodeLink return a copy of the node with the link name set to point to
 // that. If a link of the same name existed, it is removed.
 func (n *Node) UpdateNodeLink(name string, that *Node) (*Node, error) {
@@ -226,8 +240,8 @@ func (n *Node) Stat() (*NodeStat, error) {
 		Hash:           key.B58String(),
 		NumLinks:       len(n.Links),
 		BlockSize:      len(enc),
-		LinksSize:      len(enc) - len(n.Data), // includes framing.
-		DataSize:       len(n.Data),
+		LinksSize:      len(enc) - len(n.data), // includes framing.
+		DataSize:       len(n.data),
 		CumulativeSize: int(cumSize),
 	}, nil
 }

traverse/traverse_test.go

@@ -326,7 +326,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
 
 	buf := new(bytes.Buffer)
 	walk := func(current State) error {
-		s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data)
+		s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data())
 		t.Logf("walk: %s", s)
 		buf.Write([]byte(s))
 		return nil
@@ -349,7 +349,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
 }
 
 func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
-	a := &mdag.Node{Data: []byte("/a")}
+	a := mdag.NodeWithData([]byte("/a"))
 	addLink(t, ds, a, child(t, ds, a, "aa"))
 	addLink(t, ds, a, child(t, ds, a, "ab"))
 	addLink(t, ds, a, child(t, ds, a, "ac"))
@@ -358,7 +358,7 @@ func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
 }
 
 func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
-	a := &mdag.Node{Data: []byte("/a")}
+	a := mdag.NodeWithData([]byte("/a"))
 	aa := child(t, ds, a, "aa")
 	aaa := child(t, ds, aa, "aaa")
 	aaaa := child(t, ds, aaa, "aaaa")
@@ -371,7 +371,7 @@ func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
 }
 
 func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
-	a := &mdag.Node{Data: []byte("/a")}
+	a := mdag.NodeWithData([]byte("/a"))
 	aa := child(t, ds, a, "aa")
 	ab := child(t, ds, a, "ab")
 	addLink(t, ds, aa, child(t, ds, aa, "aaa"))
@@ -384,7 +384,7 @@ func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
 }
 
 func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
-	a := &mdag.Node{Data: []byte("/a")}
+	a := mdag.NodeWithData([]byte("/a"))
 	aa := child(t, ds, a, "aa")
 	aaa := child(t, ds, aa, "aaa")
 	aaaa := child(t, ds, aaa, "aaaa")
@@ -401,7 +401,7 @@ func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
 }
 
 func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
-	to := string(a.Data) + "2" + string(b.Data)
+	to := string(a.Data()) + "2" + string(b.Data())
 	if _, err := ds.Add(b); err != nil {
 		t.Error(err)
 	}
@@ -411,5 +411,5 @@ func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
 }
 
 func child(t *testing.T, ds mdag.DAGService, a *mdag.Node, name string) *mdag.Node {
-	return &mdag.Node{Data: []byte(string(a.Data) + "/" + name)}
+	return mdag.NodeWithData([]byte(string(a.Data()) + "/" + name))
 }

utils/utils_test.go

@@ -13,9 +13,7 @@ import (
 
 func TestAddLink(t *testing.T) {
 	ds := mdtest.Mock()
-	fishnode := &dag.Node{
-		Data: []byte("fishcakes!"),
-	}
+	fishnode := dag.NodeWithData([]byte("fishcakes!"))
 
 	fk, err := ds.Add(fishnode)
 	if err != nil {
@@ -90,7 +88,7 @@ func TestInsertNode(t *testing.T) {
 }
 
 func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
-	child := &dag.Node{Data: []byte(data)}
+	child := dag.NodeWithData([]byte(data))
 	ck, err := e.tmp.Add(child)
 	if err != nil {
 		t.Fatal(err)