unixfs: allow use of raw merkledag nodes for unixfs files

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

io/dagreader.go

@@ -2,17 +2,18 @@ package io
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 
-	"context"
-	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
-
 	mdag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	ftpb "github.com/ipfs/go-ipfs/unixfs/pb"
+
+	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 var ErrIsDir = errors.New("this dag node is a directory")
@@ -58,36 +59,45 @@ type ReadSeekCloser interface {
 
 // NewDagReader creates a new reader object that reads the data represented by
 // the given node, using the passed in DAGService for data retreival
-func NewDagReader(ctx context.Context, n *mdag.ProtoNode, serv mdag.DAGService) (*DagReader, error) {
-	pb := new(ftpb.Data)
-	if err := proto.Unmarshal(n.Data(), pb); err != nil {
-		return nil, err
-	}
-
-	switch pb.GetType() {
-	case ftpb.Data_Directory:
-		// Dont allow reading directories
-		return nil, ErrIsDir
-	case ftpb.Data_File, ftpb.Data_Raw:
-		return NewDataFileReader(ctx, n, pb, serv), nil
-	case ftpb.Data_Metadata:
-		if len(n.Links()) == 0 {
-			return nil, errors.New("incorrectly formatted metadata object")
-		}
-		child, err := n.Links()[0].GetNode(ctx, serv)
-		if err != nil {
+func NewDagReader(ctx context.Context, n node.Node, serv mdag.DAGService) (*DagReader, error) {
+	switch n := n.(type) {
+	case *mdag.RawNode:
+		return &DagReader{
+			buf: NewRSNCFromBytes(n.RawData()),
+		}, nil
+	case *mdag.ProtoNode:
+		pb := new(ftpb.Data)
+		if err := proto.Unmarshal(n.Data(), pb); err != nil {
 			return nil, err
 		}
 
-		childpb, ok := child.(*mdag.ProtoNode)
-		if !ok {
-			return nil, mdag.ErrNotProtobuf
+		switch pb.GetType() {
+		case ftpb.Data_Directory:
+			// Dont allow reading directories
+			return nil, ErrIsDir
+		case ftpb.Data_File, ftpb.Data_Raw:
+			return NewDataFileReader(ctx, n, pb, serv), nil
+		case ftpb.Data_Metadata:
+			if len(n.Links()) == 0 {
+				return nil, errors.New("incorrectly formatted metadata object")
+			}
+			child, err := n.Links()[0].GetNode(ctx, serv)
+			if err != nil {
+				return nil, err
+			}
+
+			childpb, ok := child.(*mdag.ProtoNode)
+			if !ok {
+				return nil, mdag.ErrNotProtobuf
+			}
+			return NewDagReader(ctx, childpb, serv)
+		case ftpb.Data_Symlink:
+			return nil, ErrCantReadSymlinks
+		default:
+			return nil, ft.ErrUnrecognizedType
 		}
-		return NewDagReader(ctx, childpb, serv)
-	case ftpb.Data_Symlink:
-		return nil, ErrCantReadSymlinks
 	default:
-		return nil, ft.ErrUnrecognizedType
+		return nil, fmt.Errorf("unrecognized node type")
 	}
 }
 

mod/dagmodifier.go

@@ -2,6 +2,7 @@ package mod
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"io"
 	"os"
@@ -13,10 +14,10 @@ import (
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
-	context "context"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
 	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
 	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 var ErrSeekFail = errors.New("failed to seek properly")
@@ -45,9 +46,14 @@ type DagModifier struct {
 	read *uio.DagReader
 }
 
-func NewDagModifier(ctx context.Context, from *mdag.ProtoNode, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) {
+func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) {
+	pbn, ok := from.(*mdag.ProtoNode)
+	if !ok {
+		return nil, mdag.ErrNotProtobuf
+	}
+
 	return &DagModifier{
-		curNode:  from.Copy(),
+		curNode:  pbn.Copy(),
 		dagserv:  serv,
 		splitter: spl,
 		ctx:      ctx,
@@ -109,7 +115,13 @@ func (dm *DagModifier) expandSparse(size int64) error {
 	if err != nil {
 		return err
 	}
-	dm.curNode = nnode
+
+	pbnnode, ok := nnode.(*mdag.ProtoNode)
+	if !ok {
+		return mdag.ErrNotProtobuf
+	}
+
+	dm.curNode = pbnnode
 	return nil
 }
 
@@ -197,7 +209,12 @@ func (dm *DagModifier) Sync() error {
 			return err
 		}
 
-		dm.curNode = nd
+		pbnode, ok := nd.(*mdag.ProtoNode)
+		if !ok {
+			return mdag.ErrNotProtobuf
+		}
+
+		dm.curNode = pbnode
 	}
 
 	dm.writeStart += uint64(buflen)
@@ -288,7 +305,7 @@ func (dm *DagModifier) modifyDag(node *mdag.ProtoNode, offset uint64, data io.Re
 }
 
 // appendData appends the blocks from the given chan to the end of this dag
-func (dm *DagModifier) appendData(node *mdag.ProtoNode, spl chunk.Splitter) (*mdag.ProtoNode, error) {
+func (dm *DagModifier) appendData(node *mdag.ProtoNode, spl chunk.Splitter) (node.Node, error) {
 	dbp := &help.DagBuilderParams{
 		Dagserv:  dm.dagserv,
 		Maxlinks: help.DefaultLinksPerBlock,

test/utils.go

@@ -2,6 +2,7 @@ package testu
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,7 +14,7 @@ import (
 	mdagmock "github.com/ipfs/go-ipfs/merkledag/test"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
-	context "context"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
 )
 
@@ -27,7 +28,7 @@ func GetDAGServ() mdag.DAGService {
 	return mdagmock.Mock()
 }
 
-func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) *mdag.ProtoNode {
+func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) node.Node {
 	in := bytes.NewReader(data)
 	node, err := imp.BuildTrickleDagFromReader(dserv, SizeSplitterGen(500)(in))
 	if err != nil {
@@ -37,11 +38,11 @@ func GetNode(t testing.TB, dserv mdag.DAGService, data []byte) *mdag.ProtoNode {
 	return node
 }
 
-func GetEmptyNode(t testing.TB, dserv mdag.DAGService) *mdag.ProtoNode {
+func GetEmptyNode(t testing.TB, dserv mdag.DAGService) node.Node {
 	return GetNode(t, dserv, []byte{})
 }
 
-func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, *mdag.ProtoNode) {
+func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64) ([]byte, node.Node) {
 	in := io.LimitReader(u.NewTimeSeededRand(), size)
 	buf, err := ioutil.ReadAll(in)
 	if err != nil {