unixfs: allow use of raw merkledag nodes for unixfs files

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

merkledag.go

@@ -85,23 +85,29 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) {
 		return nil, fmt.Errorf("Failed to get block for %s: %v", c, err)
 	}
 
-	var res node.Node
+	return decodeBlock(b)
+}
+
+func decodeBlock(b blocks.Block) (node.Node, error) {
+	c := b.Cid()
+
 	switch c.Type() {
 	case cid.Protobuf:
-		out, err := DecodeProtobuf(b.RawData())
+		decnd, err := DecodeProtobuf(b.RawData())
 		if err != nil {
 			if strings.Contains(err.Error(), "Unmarshal failed") {
 				return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c)
 			}
 			return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
 		}
-		out.cached = c
-		res = out
+
+		decnd.cached = b.Cid()
+		return decnd, nil
+	case cid.Raw:
+		return NewRawNode(b.RawData()), nil
 	default:
-		return nil, fmt.Errorf("unrecognized formatting type")
+		return nil, fmt.Errorf("unrecognized object type: %s", c.Type())
 	}
-
-	return res, nil
 }
 
 func (n *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) {
@@ -164,24 +170,12 @@ func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *Node
 					return
 				}
 
-				c := b.Cid()
-
-				var nd node.Node
-				switch c.Type() {
-				case cid.Protobuf:
-					decnd, err := DecodeProtobuf(b.RawData())
+				nd, err := decodeBlock(b)
 				if err != nil {
 					out <- &NodeOption{Err: err}
 					return
 				}
-					decnd.cached = b.Cid()
-					nd = decnd
-				default:
-					out <- &NodeOption{Err: fmt.Errorf("unrecognized object type: %s", c.Type())}
-					return
-				}
 
-				// buffered, no need to select
 				out <- &NodeOption{Node: nd}
 				count++
 

merkledag_test.go

@@ -373,3 +373,81 @@ func TestBasicAddGet(t *testing.T) {
 		t.Fatal("output didnt match input")
 	}
 }
+
+func TestGetRawNodes(t *testing.T) {
+	rn := NewRawNode([]byte("test"))
+
+	ds := dstest.Mock()
+
+	c, err := ds.Add(rn)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !c.Equals(rn.Cid()) {
+		t.Fatal("output cids didnt match")
+	}
+
+	out, err := ds.Get(context.TODO(), c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(out.RawData(), []byte("test")) {
+		t.Fatal("raw block should match input data")
+	}
+
+	if out.Links() != nil {
+		t.Fatal("raw blocks shouldnt have links")
+	}
+
+	if out.Tree() != nil {
+		t.Fatal("tree should return no paths in a raw block")
+	}
+
+	size, err := out.Size()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != 4 {
+		t.Fatal("expected size to be 4")
+	}
+
+	ns, err := out.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if ns.DataSize != 4 {
+		t.Fatal("expected size to be 4, got: ", ns.DataSize)
+	}
+
+	_, _, err = out.Resolve([]string{"foo"})
+	if err != ErrLinkNotFound {
+		t.Fatal("shouldnt find links under raw blocks")
+	}
+}
+
+func TestProtoNodeResolve(t *testing.T) {
+
+	nd := new(ProtoNode)
+	nd.SetLinks([]*node.Link{{Name: "foo"}})
+
+	lnk, left, err := nd.Resolve([]string{"foo", "bar"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(left) != 1 || left[0] != "bar" {
+		t.Fatal("expected the single path element 'bar' to remain")
+	}
+
+	if lnk.Name != "foo" {
+		t.Fatal("how did we get anything else?")
+	}
+
+	tvals := nd.Tree()
+	if len(tvals) != 1 || tvals[0] != "foo" {
+		t.Fatal("expected tree to return []{\"foo\"}")
+	}
+}

node.go

@@ -36,7 +36,7 @@ func NodeWithData(d []byte) *ProtoNode {
 }
 
 // AddNodeLink adds a link to another node.
-func (n *ProtoNode) AddNodeLink(name string, that *ProtoNode) error {
+func (n *ProtoNode) AddNodeLink(name string, that node.Node) error {
 	n.encoded = nil
 
 	lnk, err := node.MakeLink(that)

raw.go

+package merkledag
+
+import (
+	"github.com/ipfs/go-ipfs/blocks"
+
+	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
+	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
+)
+
+type RawNode struct {
+	blocks.Block
+}
+
+func NewRawNode(data []byte) *RawNode {
+	h := u.Hash(data)
+	c := cid.NewCidV1(cid.Raw, h)
+	blk, _ := blocks.NewBlockWithCid(data, c)
+
+	return &RawNode{blk}
+}
+
+func (rn *RawNode) Links() []*node.Link {
+	return nil
+}
+
+func (rn *RawNode) Resolve(path []string) (*node.Link, []string, error) {
+	return nil, nil, ErrLinkNotFound
+}
+
+func (rn *RawNode) Tree() []string {
+	return nil
+}
+
+func (rn *RawNode) Size() (uint64, error) {
+	return uint64(len(rn.RawData())), nil
+}
+
+func (rn *RawNode) Stat() (*node.NodeStat, error) {
+	return &node.NodeStat{
+		CumulativeSize: len(rn.RawData()),
+		DataSize:       len(rn.RawData()),
+	}, nil
+}
+
+var _ node.Node = (*RawNode)(nil)

utils/utils.go

 package dagutils
 
 import (
+	"context"
 	"errors"
 
-	context "context"
-	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
-	syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
-
 	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
 	bserv "github.com/ipfs/go-ipfs/blockservice"
 	offline "github.com/ipfs/go-ipfs/exchange/offline"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	path "github.com/ipfs/go-ipfs/path"
+
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
+	ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
+	syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
 )
 
 type Editor struct {
@@ -50,7 +51,7 @@ func (e *Editor) GetDagService() dag.DAGService {
 	return e.tmp
 }
 
-func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childname string, childnd *dag.ProtoNode) (*dag.ProtoNode, error) {
+func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childname string, childnd node.Node) (*dag.ProtoNode, error) {
 	if childname == "" {
 		return nil, errors.New("cannot create link with no name!")
 	}
@@ -76,7 +77,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.ProtoNode, childn
 	return root, nil
 }
 
-func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert *dag.ProtoNode, create func() *dag.ProtoNode) error {
+func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert node.Node, create func() *dag.ProtoNode) error {
 	splpath := path.SplitList(pth)
 	nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
 	if err != nil {
@@ -86,7 +87,7 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert *dag
 	return nil
 }
 
-func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert *dag.ProtoNode, create func() *dag.ProtoNode) (*dag.ProtoNode, error) {
+func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert node.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) {
 	if len(path) == 1 {
 		return addLink(ctx, e.tmp, root, path[0], toinsert)
 	}