working on dag modification structures, factored out the data format into an...

working on dag modification structures, factored out the data format into an importer subpackage and added more ipns tests

Makefile

 
-all: node.pb.go data.pb.go
+all: node.pb.go
 
 node.pb.go: node.proto
 	protoc --gogo_out=. --proto_path=../../../../:/usr/local/opt/protobuf/include:. $<
 
-data.pb.go: data.proto
-	protoc --go_out=. data.proto
-
 clean:
 	rm node.pb.go

dagreader.go

@@ -6,6 +6,7 @@ import (
 	"io"
 
 	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
+	ft "github.com/jbenet/go-ipfs/importer/format"
 	u "github.com/jbenet/go-ipfs/util"
 )
 
@@ -20,21 +21,21 @@ type DagReader struct {
 }
 
 func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) {
-	pb := new(PBData)
+	pb := new(ft.PBData)
 	err := proto.Unmarshal(n.Data, pb)
 	if err != nil {
 		return nil, err
 	}
 	switch pb.GetType() {
-	case PBData_Directory:
+	case ft.PBData_Directory:
 		return nil, ErrIsDir
-	case PBData_File:
+	case ft.PBData_File:
 		return &DagReader{
 			node: n,
 			serv: serv,
 			buf:  bytes.NewBuffer(pb.GetData()),
 		}, nil
-	case PBData_Raw:
+	case ft.PBData_Raw:
 		return bytes.NewBuffer(pb.GetData()), nil
 	default:
 		panic("Unrecognized node type!")
@@ -54,7 +55,7 @@ func (dr *DagReader) precalcNextBuf() error {
 		}
 		nxt = nxtNode
 	}
-	pb := new(PBData)
+	pb := new(ft.PBData)
 	err := proto.Unmarshal(nxt.Data, pb)
 	if err != nil {
 		return err
@@ -62,13 +63,13 @@ func (dr *DagReader) precalcNextBuf() error {
 	dr.position++
 
 	switch pb.GetType() {
-	case PBData_Directory:
+	case ft.PBData_Directory:
 		panic("Why is there a directory under a file?")
-	case PBData_File:
+	case ft.PBData_File:
 		//TODO: this *should* work, needs testing first
 		//return NewDagReader(nxt, dr.serv)
 		panic("Not yet handling different layers of indirection!")
-	case PBData_Raw:
+	case ft.PBData_Raw:
 		dr.buf = bytes.NewBuffer(pb.GetData())
 		return nil
 	default:

data.pb.go

-// Code generated by protoc-gen-go.
-// source: data.proto
-// DO NOT EDIT!
-
-/*
-Package merkledag is a generated protocol buffer package.
-
-It is generated from these files:
-	data.proto
-
-It has these top-level messages:
-	PBData
-*/
-package merkledag
-
-import proto "code.google.com/p/goprotobuf/proto"
-import math "math"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = math.Inf
-
-type PBData_DataType int32
-
-const (
-	PBData_Raw       PBData_DataType = 0
-	PBData_Directory PBData_DataType = 1
-	PBData_File      PBData_DataType = 2
-)
-
-var PBData_DataType_name = map[int32]string{
-	0: "Raw",
-	1: "Directory",
-	2: "File",
-}
-var PBData_DataType_value = map[string]int32{
-	"Raw":       0,
-	"Directory": 1,
-	"File":      2,
-}
-
-func (x PBData_DataType) Enum() *PBData_DataType {
-	p := new(PBData_DataType)
-	*p = x
-	return p
-}
-func (x PBData_DataType) String() string {
-	return proto.EnumName(PBData_DataType_name, int32(x))
-}
-func (x *PBData_DataType) UnmarshalJSON(data []byte) error {
-	value, err := proto.UnmarshalJSONEnum(PBData_DataType_value, data, "PBData_DataType")
-	if err != nil {
-		return err
-	}
-	*x = PBData_DataType(value)
-	return nil
-}
-
-type PBData struct {
-	Type             *PBData_DataType `protobuf:"varint,1,req,enum=merkledag.PBData_DataType" json:"Type,omitempty"`
-	Data             []byte           `protobuf:"bytes,2,opt" json:"Data,omitempty"`
-	Filesize         *uint64          `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"`
-	XXX_unrecognized []byte           `json:"-"`
-}
-
-func (m *PBData) Reset()         { *m = PBData{} }
-func (m *PBData) String() string { return proto.CompactTextString(m) }
-func (*PBData) ProtoMessage()    {}
-
-func (m *PBData) GetType() PBData_DataType {
-	if m != nil && m.Type != nil {
-		return *m.Type
-	}
-	return PBData_Raw
-}
-
-func (m *PBData) GetData() []byte {
-	if m != nil {
-		return m.Data
-	}
-	return nil
-}
-
-func (m *PBData) GetFilesize() uint64 {
-	if m != nil && m.Filesize != nil {
-		return *m.Filesize
-	}
-	return 0
-}
-
-func init() {
-	proto.RegisterEnum("merkledag.PBData_DataType", PBData_DataType_name, PBData_DataType_value)
-}

data.proto

-package merkledag;
-
-message PBData {
-	enum DataType {
-		Raw = 0;
-		Directory = 1;
-		File = 2;
-	}
-
-	required DataType Type = 1;
-	optional bytes Data = 2;
-	optional uint64 filesize = 3;
-}

merkledag.go

 package merkledag
 
 import (
-	"errors"
 	"fmt"
 
-	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
-
 	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
 	blocks "github.com/jbenet/go-ipfs/blocks"
 	bserv "github.com/jbenet/go-ipfs/blockservice"
@@ -37,6 +34,9 @@ type Link struct {
 	// cumulative size of target object
 	Size uint64
 
+	// cumulative size of data stored in object
+	DataSize uint64
+
 	// multihash of the target object
 	Hash mh.Multihash
 
@@ -46,14 +46,28 @@ type Link struct {
 
 // AddNodeLink adds a link to another node.
 func (n *Node) AddNodeLink(name string, that *Node) error {
-	// DEBUG CODE
-	for _, l := range n.Links {
-		if l.Name == name {
-			panic("Trying to add child that already exists!")
+	s, err := that.Size()
+	if err != nil {
+		return err
 	}
+
+	h, err := that.Multihash()
+	if err != nil {
+		return err
 	}
-	//
 
+	n.Links = append(n.Links, &Link{
+		Name: name,
+		Size: s,
+		Hash: h,
+		Node: that,
+	})
+	return nil
+}
+
+// AddNodeLink adds a link to another node. without keeping a reference to
+// the child node
+func (n *Node) AddNodeLinkClean(name string, that *Node) error {
 	s, err := that.Size()
 	if err != nil {
 		return err
@@ -68,7 +82,6 @@ func (n *Node) AddNodeLink(name string, that *Node) error {
 		Name: name,
 		Size: s,
 		Hash: h,
-		Node: that,
 	})
 	return nil
 }
@@ -83,6 +96,8 @@ func (n *Node) RemoveNodeLink(name string) error {
 	return u.ErrNotFound
 }
 
+// Copy returns a copy of the node.
+// NOTE: does not make copies of Node objects in the links.
 func (n *Node) Copy() *Node {
 	nnode := new(Node)
 	nnode.Data = make([]byte, len(n.Data))
@@ -108,25 +123,6 @@ func (n *Node) Size() (uint64, error) {
 	return s, nil
 }
 
-func (n *Node) DataSize() (uint64, error) {
-	pbdata := new(PBData)
-	err := proto.Unmarshal(n.Data, pbdata)
-	if err != nil {
-		return 0, err
-	}
-
-	switch pbdata.GetType() {
-	case PBData_Directory:
-		return 0, errors.New("Cant get data size of directory!")
-	case PBData_File:
-		return pbdata.GetFilesize(), nil
-	case PBData_Raw:
-		return uint64(len(pbdata.GetData())), nil
-	default:
-		return 0, errors.New("Unrecognized node data type!")
-	}
-}
-
 // Multihash hashes the encoded data of this node.
 func (n *Node) Multihash() (mh.Multihash, error) {
 	b, err := n.Encoded(false)
@@ -230,46 +226,3 @@ func (n *DAGService) Get(k u.Key) (*Node, error) {
 
 	return Decoded(b.Data)
 }
-
-func FilePBData(data []byte, totalsize uint64) []byte {
-	pbfile := new(PBData)
-	typ := PBData_File
-	pbfile.Type = &typ
-	pbfile.Data = data
-	pbfile.Filesize = proto.Uint64(totalsize)
-
-	data, err := proto.Marshal(pbfile)
-	if err != nil {
-		//this really shouldnt happen, i promise
-		panic(err)
-	}
-	return data
-}
-
-func FolderPBData() []byte {
-	pbfile := new(PBData)
-	typ := PBData_Directory
-	pbfile.Type = &typ
-
-	data, err := proto.Marshal(pbfile)
-	if err != nil {
-		//this really shouldnt happen, i promise
-		panic(err)
-	}
-	return data
-}
-
-func WrapData(b []byte) []byte {
-	pbdata := new(PBData)
-	typ := PBData_Raw
-	pbdata.Data = b
-	pbdata.Type = &typ
-
-	out, err := proto.Marshal(pbdata)
-	if err != nil {
-		// This shouldnt happen. seriously.
-		panic(err)
-	}
-
-	return out
-}

merkledag_test.go

@@ -2,8 +2,9 @@ package merkledag
 
 import (
 	"fmt"
-	u "github.com/jbenet/go-ipfs/util"
 	"testing"
+
+	u "github.com/jbenet/go-ipfs/util"
 )
 
 func TestNode(t *testing.T) {