raw dag: make raw nodes work in cat and get, add tests

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

importer/helpers/helpers.go

@@ -105,7 +105,7 @@ func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*U
 // the passed in DagBuilderHelper is used to store the child node an
 // pin it locally so it doesnt get lost
 func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
-	n.ufmt.AddBlockSize(child.ufmt.FileSize())
+	n.ufmt.AddBlockSize(child.DataSize())
 
 	childnode, err := child.GetDagNode()
 	if err != nil {
@@ -137,6 +137,13 @@ func (n *UnixfsNode) SetData(data []byte) {
 	n.ufmt.Data = data
 }
 
+func (n *UnixfsNode) DataSize() uint64 {
+	if n.raw {
+		return uint64(len(n.rawnode.RawData()))
+	}
+	return n.ufmt.FileSize()
+}
+
 // getDagNode fills out the proper formatting for the unixfs node
 // inside of a DAG node and returns the dag node
 func (n *UnixfsNode) GetDagNode() (node.Node, error) {

test/sharness/t0040-add-and-cat.sh

@@ -87,6 +87,9 @@ test_add_cat_file() {
 }
 
 test_add_cat_5MB() {
+	ADD_FLAGS="$1"
+	EXP_HASH="$2"
+
     test_expect_success "generate 5MB file using go-random" '
     	random 5242880 41 >mountdir/bigfile
     '
@@ -98,17 +101,16 @@ test_add_cat_5MB() {
     '
 
     test_expect_success "'ipfs add bigfile' succeeds" '
-    	ipfs add mountdir/bigfile >actual ||
+    	ipfs add $ADD_FLAGS mountdir/bigfile >actual ||
 		test_fsh cat daemon_err
     '
 
     test_expect_success "'ipfs add bigfile' output looks good" '
-    	HASH="QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" &&
-    	echo "added $HASH bigfile" >expected &&
+    	echo "added $EXP_HASH bigfile" >expected &&
     	test_cmp expected actual
     '
     test_expect_success "'ipfs cat' succeeds" '
-    	ipfs cat "$HASH" >actual
+    	ipfs cat "$EXP_HASH" >actual
     '
 
     test_expect_success "'ipfs cat' output looks good" '
@@ -116,7 +118,7 @@ test_add_cat_5MB() {
     '
 
     test_expect_success FUSE "cat ipfs/bigfile succeeds" '
-    	cat "ipfs/$HASH" >actual
+    	cat "ipfs/$EXP_HASH" >actual
     '
 
     test_expect_success FUSE "cat ipfs/bigfile looks good" '
@@ -380,7 +382,9 @@ test_expect_success "go-random is installed" '
     type random
 '
 
-test_add_cat_5MB
+test_add_cat_5MB "" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb"
+
+test_add_cat_5MB --raw-leaves "QmefsDaD3YVphd86mxjJfPLceKv8by98aB6J6sJxK13xS2"
 
 test_add_cat_expensive
 

unixfs/io/dagreader.go

@@ -129,33 +129,36 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error {
 	}
 	dr.linkPosition++
 
-	nxtpb, ok := nxt.(*mdag.ProtoNode)
-	if !ok {
-		return mdag.ErrNotProtobuf
-	}
-
-	pb := new(ftpb.Data)
-	err = proto.Unmarshal(nxtpb.Data(), pb)
-	if err != nil {
-		return fmt.Errorf("incorrectly formatted protobuf: %s", err)
-	}
+	switch nxt := nxt.(type) {
+	case *mdag.ProtoNode:
+		pb := new(ftpb.Data)
+		err = proto.Unmarshal(nxt.Data(), pb)
+		if err != nil {
+			return fmt.Errorf("incorrectly formatted protobuf: %s", err)
+		}
 
-	switch pb.GetType() {
-	case ftpb.Data_Directory:
-		// A directory should not exist within a file
-		return ft.ErrInvalidDirLocation
-	case ftpb.Data_File:
-		dr.buf = NewDataFileReader(dr.ctx, nxtpb, pb, dr.serv)
-		return nil
-	case ftpb.Data_Raw:
-		dr.buf = NewRSNCFromBytes(pb.GetData())
+		switch pb.GetType() {
+		case ftpb.Data_Directory:
+			// A directory should not exist within a file
+			return ft.ErrInvalidDirLocation
+		case ftpb.Data_File:
+			dr.buf = NewDataFileReader(dr.ctx, nxt, pb, dr.serv)
+			return nil
+		case ftpb.Data_Raw:
+			dr.buf = NewRSNCFromBytes(pb.GetData())
+			return nil
+		case ftpb.Data_Metadata:
+			return errors.New("shouldnt have had metadata object inside file")
+		case ftpb.Data_Symlink:
+			return errors.New("shouldnt have had symlink inside file")
+		default:
+			return ft.ErrUnrecognizedType
+		}
+	case *mdag.RawNode:
+		dr.buf = NewRSNCFromBytes(nxt.RawData())
 		return nil
-	case ftpb.Data_Metadata:
-		return errors.New("shouldnt have had metadata object inside file")
-	case ftpb.Data_Symlink:
-		return errors.New("shouldnt have had symlink inside file")
 	default:
-		return ft.ErrUnrecognizedType
+		return errors.New("unrecognized node type in DagReader")
 	}
 }