Merge branch 'encoding' into traversals

encoding/marshal.go

+package encoding
+
+import (
+	"fmt"
+
+	"github.com/polydawn/refmt/shared"
+	"github.com/polydawn/refmt/tok"
+
+	"github.com/ipld/go-ipld-prime"
+)
+
+// FUTURE there are very open questions on how to handle detection and special-track'ing for advLayout nodes when we get to that feature.
+
+func Marshal(n ipld.Node, sink shared.TokenSink) error {
+	var tk tok.Token
+	switch n.Kind() {
+	case ipld.ReprKind_Invalid:
+		return fmt.Errorf("cannot traverse a node that is undefined")
+	case ipld.ReprKind_Null:
+		tk.Type = tok.TNull
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Map:
+		// Emit start of map.
+		tk.Type = tok.TMapOpen
+		tk.Length = n.Length()
+		if _, err := sink.Step(&tk); err != nil {
+			return err
+		}
+		// Emit map contents (and recurse).
+		for itr := n.Keys(); itr.HasNext(); {
+			k, err := itr.Next()
+			if err != nil {
+				return err
+			}
+			tk.Type = tok.TString
+			tk.Str = k
+			if _, err := sink.Step(&tk); err != nil {
+				return err
+			}
+			v, err := n.TraverseField(k)
+			if err != nil {
+				return err
+			}
+			if err := Marshal(v, sink); err != nil {
+				return err
+			}
+		}
+		// Emit map close.
+		tk.Type = tok.TMapClose
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_List:
+		// Emit start of list.
+		tk.Type = tok.TArrOpen
+		l := n.Length()
+		tk.Length = l
+		if _, err := sink.Step(&tk); err != nil {
+			return err
+		}
+		// Emit list contents (and recurse).
+		for i := 0; i < l; i++ {
+			v, err := n.TraverseIndex(i)
+			if err != nil {
+				return err
+			}
+			if err := Marshal(v, sink); err != nil {
+				return err
+			}
+		}
+		// Emit list close.
+		tk.Type = tok.TArrClose
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Bool:
+		v, err := n.AsBool()
+		if err != nil {
+			return err
+		}
+		tk.Type = tok.TBool
+		tk.Bool = v
+		_, err = sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Int:
+		v, err := n.AsInt()
+		if err != nil {
+			return err
+		}
+		tk.Type = tok.TInt
+		tk.Int = int64(v)
+		_, err = sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Float:
+		v, err := n.AsFloat()
+		if err != nil {
+			return err
+		}
+		tk.Type = tok.TFloat64
+		tk.Float64 = v
+		_, err = sink.Step(&tk)
+		return err
+	case ipld.ReprKind_String:
+		v, err := n.AsString()
+		if err != nil {
+			return err
+		}
+		tk.Type = tok.TString
+		tk.Str = v
+		_, err = sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Bytes:
+		v, err := n.AsBytes()
+		if err != nil {
+			return err
+		}
+		tk.Type = tok.TBytes
+		tk.Bytes = v
+		_, err = sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Link:
+		panic("todo link emission")
+	default:
+		panic("unreachable")
+	}
+}

encoding/unmarshal.go

+package encoding
+
+import (
+	"fmt"
+
+	"github.com/polydawn/refmt/shared"
+	"github.com/polydawn/refmt/tok"
+
+	"github.com/ipld/go-ipld-prime"
+)
+
+// wishlist: if we could reconstruct the ipld.Path of an error while
+//  *unwinding* from that error... that'd be nice.
+//   (trying to build it proactively would waste tons of allocs on the happy path.)
+//  we can do this; it just requires well-typed errors and a bunch of work.
+
+// Tests for all this are in the ipld.Node impl tests!
+//  They're effectively doing double duty: testing the builders, too.
+//   (Is that sensible?  Should it be refactored?  Not sure; maybe!)
+
+func Unmarshal(nb ipld.NodeBuilder, tokSrc shared.TokenSource) (ipld.Node, error) {
+	var tk tok.Token
+	done, err := tokSrc.Step(&tk)
+	if done || err != nil {
+		return nil, err
+	}
+	return unmarshal(nb, tokSrc, &tk)
+}
+
+// starts with the first token already primed.  Necessary to get recursion
+//  to flow right without a peek+unpeek system.
+func unmarshal(nb ipld.NodeBuilder, tokSrc shared.TokenSource, tk *tok.Token) (ipld.Node, error) {
+	switch tk.Type {
+	case tok.TMapOpen:
+		mb, err := nb.CreateMap()
+		if err != nil {
+			return nil, err
+		}
+		for {
+			done, err := tokSrc.Step(tk)
+			if done {
+				return nil, fmt.Errorf("unexpected EOF")
+			}
+			if err != nil {
+				return nil, err
+			}
+			switch tk.Type {
+			case tok.TMapClose:
+				return mb.Build()
+			case tok.TString:
+				// continue
+			default:
+				return nil, fmt.Errorf("unexpected %s token while expecting map key", tk.Type)
+			}
+			k := tk.Str
+			// FUTURE: check for typed.NodeBuilder; need to specialize before recursing if so.
+			v, err := Unmarshal(nb, tokSrc)
+			if err != nil {
+				return nil, err
+			}
+			kn, err := nb.CreateString(k)
+			if err != nil {
+				panic(err) // TODO: I'm no longer sure Insert should take a Node instead of string, but not recursing into reviewing that choice now.
+			}
+			if err := mb.Insert(kn, v); err != nil {
+				return nil, err
+			}
+		}
+	case tok.TMapClose:
+		return nil, fmt.Errorf("unexpected mapClose token")
+	case tok.TArrOpen:
+		lb, err := nb.CreateList()
+		if err != nil {
+			return nil, err
+		}
+		for {
+			done, err := tokSrc.Step(tk)
+			if done {
+				return nil, fmt.Errorf("unexpected EOF")
+			}
+			if err != nil {
+				return nil, err
+			}
+			switch tk.Type {
+			case tok.TArrClose:
+				return lb.Build()
+			default:
+				// FUTURE: check for typed.NodeBuilder; need to specialize before recursing if so.
+				//  N.B. when considering optionals for tuple-represented structs, keep in mind how murky that will get here.
+				v, err := unmarshal(nb, tokSrc, tk)
+				if err != nil {
+					return nil, err
+				}
+				lb.Append(v)
+			}
+		}
+	case tok.TArrClose:
+		return nil, fmt.Errorf("unexpected arrClose token")
+	case tok.TNull:
+		return nb.CreateNull()
+	case tok.TString:
+		return nb.CreateString(tk.Str)
+	case tok.TBytes:
+		return nb.CreateBytes(tk.Bytes)
+		// TODO should also check tags to produce CIDs.
+		//  n.b. with schemas, we can comprehend links without tags;
+		//   but without schemas, tags are the only disambiguator.
+	case tok.TBool:
+		return nb.CreateBool(tk.Bool)
+	case tok.TInt:
+		return nb.CreateInt(int(tk.Int)) // FIXME overflow check
+	case tok.TUint:
+		return nb.CreateInt(int(tk.Uint)) // FIXME overflow check
+	case tok.TFloat64:
+		return nb.CreateFloat(tk.Float64)
+	default:
+		panic("unreachable")
+	}
+}

fluent/fluentNodeBuilder.go

@@ -6,10 +6,10 @@ import (
 )
 
 type NodeBuilder interface {
-	CreateMap() MapBuilder
-	AmendMap() MapBuilder
-	CreateList() ListBuilder
-	AmendList() ListBuilder
+	CreateMap(MapBuildingClosure) ipld.Node
+	AmendMap(MapBuildingClosure) ipld.Node
+	CreateList(ListBuildingClosure) ipld.Node
+	AmendList(ListBuildingClosure) ipld.Node
 	CreateNull() ipld.Node
 	CreateBool(bool) ipld.Node
 	CreateInt(int) ipld.Node
@@ -20,39 +20,110 @@ type NodeBuilder interface {
 }
 
 type MapBuilder interface {
-	InsertAll(map[ipld.Node]ipld.Node) MapBuilder
-	Insert(k, v ipld.Node) MapBuilder
-	Delete(k ipld.Node) MapBuilder
-	Build() ipld.Node
+	Insert(k, v ipld.Node)
+	Delete(k ipld.Node)
 }
 
 type ListBuilder interface {
-	AppendAll([]ipld.Node) ListBuilder
-	Append(v ipld.Node) ListBuilder
-	Set(idx int, v ipld.Node) ListBuilder
-	Build() ipld.Node
+	AppendAll([]ipld.Node)
+	Append(v ipld.Node)
+	Set(idx int, v ipld.Node)
 }
 
+// MapBuildingClosure is the signiture of a function which builds a Node of kind map.
+//
+// The MapBuilder parameter is used to accumulate the new Node for the
+// duration of the function; and when the function returns, that builder
+// will be invoked.  (In other words, there's no need to call `Build` within
+// the closure itself -- and correspondingly, note the lack of return value.)
+//
+// Additional NodeBuilder handles are provided for building keys and values.
+// These are used when handling typed Node implementations, since in that
+// case they may contain behavior related to the type contracts.
+// (For untyped nodes, this is degenerate: these builders are not
+// distinct from the parent builder driving this closure.)
+//
+// REVIEW : whether 'knb' is needed.  Not sure, and there are other pending
+// discussions on this.  (It's mostly a concern about having a place to do
+// validation on construction; but it's possible we can solve this without
+// additional Nodes and Builders by making that validation the responsibility
+// of the inside of the mb.Insert method; but will this compose well, and
+// will it convey the appropriate times to do the validations correctly?
+// Is 'knb' relevant even if that last question is 'no'?  If a concern is
+// to avoid double-validations, that argues for `mb.Insert(Node, Node)` over
+// `mb.Insert(string, Node)`, but if avoiding double-validations, that means
+// we already have a Node and don't need 'knb' to get one.  ... Design!)
+type MapBuildingClosure func(mb MapBuilder, knb NodeBuilder, vnb NodeBuilder)
+
+// ListBuildingClosure is the signiture of a function which builds a Node of kind list.
+//
+// The ListBuilder parameter is used to accumulate the new Node for the
+// duration of the function; and when the function returns, that builder
+// will be invoked.  (In other words, there's no need to call `Build` within
+// the closure itself -- and correspondingly, note the lack of return value.)
+//
+// Additional NodeBuilder handles are provided for building the values.
+// These are used when handling typed Node implementations, since in that
+// case they may contain behavior related to the type contracts.
+// (For untyped nodes, this is degenerate: the 'vnb' builder is not
+// distinct from the parent builder driving this closure.)
+type ListBuildingClosure func(lb ListBuilder, vnb NodeBuilder)
+
 func WrapNodeBuilder(nb ipld.NodeBuilder) NodeBuilder {
-	return &nodeBuilder{nb, nil}
+	return &nodeBuilder{nb}
 }
 
 type nodeBuilder struct {
-	nb  ipld.NodeBuilder
-	err error
+	nb ipld.NodeBuilder
 }
 
-func (nb *nodeBuilder) CreateMap() MapBuilder {
-	return mapBuilder{nb.nb.CreateMap()}
+func (nb *nodeBuilder) CreateMap(fn MapBuildingClosure) ipld.Node {
+	mb, err := nb.nb.CreateMap()
+	if err != nil {
+		panic(Error{err})
+	}
+	fn(mapBuilder{mb}, nb, nb) // FUTURE: check for typed.NodeBuilder; need to specialize latter params before calling down if so.
+	n, err := mb.Build()
+	if err != nil {
+		panic(Error{err})
+	}
+	return n
 }
-func (nb *nodeBuilder) AmendMap() MapBuilder {
-	return mapBuilder{nb.nb.AmendMap()}
+func (nb *nodeBuilder) AmendMap(fn MapBuildingClosure) ipld.Node {
+	mb, err := nb.nb.AmendMap()
+	if err != nil {
+		panic(Error{err})
+	}
+	fn(mapBuilder{mb}, nb, nb) // FUTURE: check for typed.NodeBuilder; need to specialize latter params before calling down if so.
+	n, err := mb.Build()
+	if err != nil {
+		panic(Error{err})
+	}
+	return n
 }
-func (nb *nodeBuilder) CreateList() ListBuilder {
-	return listBuilder{nb.nb.CreateList()}
+func (nb *nodeBuilder) CreateList(fn ListBuildingClosure) ipld.Node {
+	lb, err := nb.nb.CreateList()
+	if err != nil {
+		panic(Error{err})
+	}
+	fn(listBuilder{lb}, nb) // FUTURE: check for typed.NodeBuilder; need to specialize latter params before calling down if so.
+	n, err := lb.Build()
+	if err != nil {
+		panic(Error{err})
+	}
+	return n
 }
-func (nb *nodeBuilder) AmendList() ListBuilder {
-	return listBuilder{nb.nb.AmendList()}
+func (nb *nodeBuilder) AmendList(fn ListBuildingClosure) ipld.Node {
+	lb, err := nb.nb.AmendList()
+	if err != nil {
+		panic(Error{err})
+	}
+	fn(listBuilder{lb}, nb) // FUTURE: check for typed.NodeBuilder; need to specialize latter params before calling down if so.
+	n, err := lb.Build()
+	if err != nil {
+		panic(Error{err})
+	}
+	return n
 }
 func (nb *nodeBuilder) CreateNull() ipld.Node {
 	n, err := nb.nb.CreateNull()
@@ -108,46 +179,23 @@ type mapBuilder struct {
 	ipld.MapBuilder
 }
 
-func (mb mapBuilder) InsertAll(vs map[ipld.Node]ipld.Node) MapBuilder {
-	mb.MapBuilder.InsertAll(vs)
-	return mb
-}
-func (mb mapBuilder) Insert(k, v ipld.Node) MapBuilder {
+func (mb mapBuilder) Insert(k, v ipld.Node) {
 	mb.MapBuilder.Insert(k, v)
-	return mb
 }
-func (mb mapBuilder) Delete(k ipld.Node) MapBuilder {
+func (mb mapBuilder) Delete(k ipld.Node) {
 	mb.MapBuilder.Delete(k)
-	return mb
-}
-func (mb mapBuilder) Build() ipld.Node {
-	n, err := mb.MapBuilder.Build()
-	if err != nil {
-		panic(Error{err})
-	}
-	return n
 }
 
 type listBuilder struct {
 	ipld.ListBuilder
 }
 
-func (lb listBuilder) AppendAll(vs []ipld.Node) ListBuilder {
+func (lb listBuilder) AppendAll(vs []ipld.Node) {
 	lb.ListBuilder.AppendAll(vs)
-	return lb
 }
-func (lb listBuilder) Append(v ipld.Node) ListBuilder {
+func (lb listBuilder) Append(v ipld.Node) {
 	lb.ListBuilder.Append(v)
-	return lb
 }
-func (lb listBuilder) Set(idx int, v ipld.Node) ListBuilder {
+func (lb listBuilder) Set(idx int, v ipld.Node) {
 	lb.ListBuilder.Set(idx, v)
-	return lb
-}
-func (lb listBuilder) Build() ipld.Node {
-	n, err := lb.ListBuilder.Build()
-	if err != nil {
-		panic(Error{err})
-	}
-	return n
 }

impl/free/freeNodeBuilder.go

@@ -20,15 +20,15 @@ type nodeBuilder struct {
 	predecessor *Node // optional; only relevant for "Amend*" methods.
 }
 
-func (nb nodeBuilder) CreateMap() ipld.MapBuilder {
-	return &mapBuilder{n: &Node{kind: ipld.ReprKind_Map}}
+func (nb nodeBuilder) CreateMap() (ipld.MapBuilder, error) {
+	return &mapBuilder{n: &Node{kind: ipld.ReprKind_Map, _map: make(map[string]ipld.Node)}}, nil
 }
-func (nb nodeBuilder) AmendMap() ipld.MapBuilder {
+func (nb nodeBuilder) AmendMap() (ipld.MapBuilder, error) {
 	if nb.predecessor == nil {
 		return nb.CreateMap()
 	}
 	if nb.predecessor.kind != ipld.ReprKind_List {
-		return &mapBuilder{err: fmt.Errorf("AmendMap cannot be used when predecessor was not a map")}
+		return nil, fmt.Errorf("AmendMap cannot be used when predecessor was not a map")
 	}
 	newMap := make(map[string]ipld.Node, len(nb.predecessor._map))
 	for k, v := range nb.predecessor._map {
@@ -36,21 +36,21 @@ func (nb nodeBuilder) AmendMap() ipld.MapBuilder {
 	}
 	newArr := make([]string, len(nb.predecessor._mapOrd))
 	copy(newArr, nb.predecessor._mapOrd)
-	return &mapBuilder{n: &Node{kind: ipld.ReprKind_Map, _map: newMap, _mapOrd: newArr}}
+	return &mapBuilder{n: &Node{kind: ipld.ReprKind_Map, _map: newMap, _mapOrd: newArr}}, nil
 }
-func (nb nodeBuilder) CreateList() ipld.ListBuilder {
-	return &listBuilder{n: &Node{kind: ipld.ReprKind_List}}
+func (nb nodeBuilder) CreateList() (ipld.ListBuilder, error) {
+	return &listBuilder{n: &Node{kind: ipld.ReprKind_List}}, nil
 }
-func (nb nodeBuilder) AmendList() ipld.ListBuilder {
+func (nb nodeBuilder) AmendList() (ipld.ListBuilder, error) {
 	if nb.predecessor == nil {
 		return nb.CreateList()
 	}
 	if nb.predecessor.kind != ipld.ReprKind_List {
-		return &listBuilder{err: fmt.Errorf("AmendList cannot be used when predecessor was not a list")}
+		return nil, fmt.Errorf("AmendList cannot be used when predecessor was not a list")
 	}
 	newArr := make([]ipld.Node, len(nb.predecessor._arr))
 	copy(newArr, nb.predecessor._arr)
-	return &listBuilder{n: &Node{kind: ipld.ReprKind_List, _arr: newArr}}
+	return &listBuilder{n: &Node{kind: ipld.ReprKind_List, _arr: newArr}}, nil
 }
 func (nb nodeBuilder) CreateNull() (ipld.Node, error) {
 	return &Node{kind: ipld.ReprKind_Null}, nil
@@ -75,87 +75,57 @@ func (nb nodeBuilder) CreateLink(v cid.Cid) (ipld.Node, error) {
 }
 
 type mapBuilder struct {
-	n   *Node // a wip node; initialized at construction.
-	err error // gathered until the end (so we can have call chaining).
+	n *Node // a wip node; initialized at construction.
 	// whole builder object nil'd after terminal `Build()` call to prevent reuse.
 }
 
-func (mb *mapBuilder) InsertAll(map[ipld.Node]ipld.Node) ipld.MapBuilder {
-	// TODO needs to be sorted to be sane, i suspect.
-	// depends on the node implementation, but then, that's why there's a builder per implementation.
-	panic("NYI")
-}
-
 // Insert adds a k:v pair to the map.
 //
 // As is usual for maps, the key must have kind==ReprKind_String.
 //
 // Keys not already present in the map will be appened to the end of the
 // iteration order; keys already present retain their original order.
-func (mb *mapBuilder) Insert(k, v ipld.Node) ipld.MapBuilder {
-	if mb.err != nil {
-		return mb
-	}
+func (mb *mapBuilder) Insert(k, v ipld.Node) error {
 	ks, err := k.AsString()
 	if err != nil {
-		mb.err = fmt.Errorf("invalid node for map key: %s", err)
-		return mb
+		return fmt.Errorf("invalid node for map key: %s", err)
 	}
 	_, exists := mb.n._map[ks]
-	mb.n._map[ks] = v
 	if exists {
-		return mb
+		return fmt.Errorf("repeated map key: %s", ks)
 	}
+	mb.n._map[ks] = v
 	mb.n._mapOrd = append(mb.n._mapOrd, ks)
-	return mb
+	return nil
 }
-func (mb *mapBuilder) Delete(k ipld.Node) ipld.MapBuilder {
+func (mb *mapBuilder) Delete(k ipld.Node) error {
 	panic("NYI") // and see the "review: MapBuilder.Delete" comment in the interface defn file.
 }
 func (mb *mapBuilder) Build() (ipld.Node, error) {
-	if mb.err != nil {
-		return nil, mb.err
-	}
 	v := mb.n
 	mb = nil
 	return v, nil
 }
 
 type listBuilder struct {
-	n   *Node // a wip node; initialized at construction.
-	err error // gathered until the end (so we can have call chaining).
+	n *Node // a wip node; initialized at construction.
 	// whole builder object nil'd after terminal `Build()` call to prevent reuse.
 }
 
-func (lb *listBuilder) AppendAll(vs []ipld.Node) ipld.ListBuilder {
-	if lb.err != nil {
-		return lb
-	}
+func (lb *listBuilder) AppendAll(vs []ipld.Node) {
 	off := len(lb.n._arr)
 	new := off + len(vs)
 	growList(&lb.n._arr, new-1)
 	copy(lb.n._arr[off:new], vs)
-	return lb
 }
-func (lb *listBuilder) Append(v ipld.Node) ipld.ListBuilder {
-	if lb.err != nil {
-		return lb
-	}
+func (lb *listBuilder) Append(v ipld.Node) {
 	lb.n._arr = append(lb.n._arr, v)
-	return lb
 }
-func (lb *listBuilder) Set(idx int, v ipld.Node) ipld.ListBuilder {
-	if lb.err != nil {
-		return lb
-	}
+func (lb *listBuilder) Set(idx int, v ipld.Node) {
 	growList(&lb.n._arr, idx)
 	lb.n._arr[idx] = v
-	return lb
 }
 func (lb *listBuilder) Build() (ipld.Node, error) {
-	if lb.err != nil {
-		return nil, lb.err
-	}
 	v := lb.n
 	lb = nil
 	return v, nil

impl/free/freeNodeTokening.go

-package ipldfree
-
-import (
-	"fmt"
-
-	"github.com/polydawn/refmt/shared"
-	"github.com/polydawn/refmt/tok"
-
-	"github.com/ipld/go-ipld-prime"
-)
-
-// wishlist: if we could reconstruct the ipld.Path of an error while
-//  *unwinding* from that error... that'd be nice.
-//   (trying to build it proactively would waste tons of allocs on the happy path.)
-//  we can probably do this; it just requires well-typed errors.
-
-var (
-	_ ipld.NodeUnmarshaller = Unmarshal
-)
-
-func Unmarshal(tokSrc shared.TokenSource) (ipld.Node, error) {
-	var tk tok.Token
-	done, err := tokSrc.Step(&tk)
-	if done {
-		return &Node{}, nil // invalid node, but not exactly an error
-	}
-	if err != nil {
-		return nil, err
-	}
-	return unmarshal(tokSrc, &tk)
-}
-
-// starts with the first token already primed.  Necessary to get recursion
-//  to flow right without a peek+unpeek system.
-func unmarshal(tokSrc shared.TokenSource, tk *tok.Token) (ipld.Node, error) {
-	var n Node
-	switch tk.Type {
-	case tok.TMapOpen:
-		n.coerceType(ipld.ReprKind_Map)
-		for {
-			done, err := tokSrc.Step(tk)
-			if done {
-				return &n, fmt.Errorf("unexpected EOF")
-			}
-			if err != nil {
-				return &n, err
-			}
-			switch tk.Type {
-			case tok.TMapClose:
-				return &n, nil
-			case tok.TString:
-				// continue
-			default:
-				return &n, fmt.Errorf("unexpected %s token while expecting map key", tk.Type)
-			}
-			k := tk.Str
-			v, err := Unmarshal(tokSrc)
-			if err != nil {
-				return &n, err
-			}
-			if v.Kind() == ipld.ReprKind_Invalid {
-				return &n, fmt.Errorf("unexpected EOF")
-			}
-			if _, exists := n._map[k]; exists {
-				return &n, fmt.Errorf("repeated map key %q", tk)
-			}
-			n._mapOrd = append(n._mapOrd, k)
-			n._map[k] = v
-		}
-	case tok.TMapClose:
-		return nil, fmt.Errorf("unexpected mapClose token")
-	case tok.TArrOpen:
-		n.coerceType(ipld.ReprKind_List)
-		for {
-			done, err := tokSrc.Step(tk)
-			if done {
-				return &n, fmt.Errorf("unexpected EOF")
-			}
-			if err != nil {
-				return &n, err
-			}
-			switch tk.Type {
-			case tok.TArrClose:
-				return &n, nil
-			default:
-				v, err := unmarshal(tokSrc, tk)
-				if err != nil {
-					return &n, err
-				}
-				if v.Kind() == ipld.ReprKind_Invalid {
-					return &n, fmt.Errorf("unexpected EOF")
-				}
-				n._arr = append(n._arr, v)
-			}
-		}
-		return &n, nil
-	case tok.TArrClose:
-		return nil, fmt.Errorf("unexpected arrClose token")
-	case tok.TNull:
-		n.coerceType(ipld.ReprKind_Null)
-		return &n, nil
-	case tok.TString:
-		n.coerceType(ipld.ReprKind_String)
-		n._str = tk.Str
-		return &n, nil
-	case tok.TBytes:
-		n.coerceType(ipld.ReprKind_Bytes)
-		n._bytes = tk.Bytes
-		return &n, nil
-		// TODO should also check tags to produce CIDs.
-		//  n.b. with schemas, we can comprehend links without tags;
-		//   but without schemas, tags are the only disambiguator.
-	case tok.TBool:
-		n.coerceType(ipld.ReprKind_Bool)
-		n._bool = tk.Bool
-		return &n, nil
-	case tok.TInt:
-		n.coerceType(ipld.ReprKind_Int)
-		n._int = int(tk.Int) // FIXME overflow check
-		return &n, nil
-	case tok.TUint:
-		n.coerceType(ipld.ReprKind_Int)
-		n._int = int(tk.Uint) // FIXME overflow check
-		return &n, nil
-	case tok.TFloat64:
-		n.coerceType(ipld.ReprKind_Float)
-		n._float = tk.Float64
-		return &n, nil
-	default:
-		panic("unreachable")
-	}
-}
-
-func (n *Node) PushTokens(sink shared.TokenSink) error {
-	var tk tok.Token
-	switch n.kind {
-	case ipld.ReprKind_Invalid:
-		return fmt.Errorf("cannot traverse a node that is undefined")
-	case ipld.ReprKind_Null:
-		tk.Type = tok.TNull
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_Map:
-		// Emit start of map.
-		tk.Type = tok.TMapOpen
-		tk.Length = len(n._map)
-		if _, err := sink.Step(&tk); err != nil {
-			return err
-		}
-		// Emit map contents (and recurse).
-		for k, v := range n._map { // FIXME map order
-			tk.Type = tok.TString
-			tk.Str = k
-			if _, err := sink.Step(&tk); err != nil {
-				return err
-			}
-			switch v2 := v.(type) {
-			case ipld.TokenizableNode:
-				if err := v2.PushTokens(sink); err != nil {
-					return err
-				}
-			default:
-				panic("todo generic node tokenizer fallback")
-			}
-		}
-		// Emit map close.
-		tk.Type = tok.TMapClose
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_List:
-		// Emit start of list.
-		tk.Type = tok.TArrOpen
-		tk.Length = len(n._arr)
-		if _, err := sink.Step(&tk); err != nil {
-			return err
-		}
-		// Emit list contents (and recurse).
-		for _, v := range n._arr {
-			switch v2 := v.(type) {
-			case ipld.TokenizableNode:
-				if err := v2.PushTokens(sink); err != nil {
-					return err
-				}
-			default:
-				panic("todo generic node tokenizer fallback")
-			}
-		}
-		// Emit list close.
-		tk.Type = tok.TArrClose
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_Bool:
-		tk.Type = tok.TBool
-		tk.Bool = n._bool
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_Int:
-		tk.Type = tok.TInt
-		tk.Int = int64(n._int)
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_Float:
-		tk.Type = tok.TFloat64
-		tk.Float64 = n._float
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_String:
-		tk.Type = tok.TString
-		tk.Str = n._str
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_Bytes:
-		tk.Type = tok.TBytes
-		tk.Bytes = n._bytes
-		_, err := sink.Step(&tk)
-		return err
-	case ipld.ReprKind_Link:
-		panic("todo link emission")
-	default:
-		panic("unreachable")
-	}
-}

impl/free/freeNode_test.go

@@ -20,9 +20,8 @@ func TestMutableNode(t *testing.T) {
 }
 
 func TestTokening(t *testing.T) {
-	mutNodeFac := func() ipld.MutableNode { return &Node{} }
-	tests.TestScalarMarshal(t, mutNodeFac)
-	tests.TestRecursiveMarshal(t, mutNodeFac)
-	tests.TestScalarUnmarshal(t, Unmarshal)
-	tests.TestRecursiveUnmarshal(t, Unmarshal)
+	tests.TestScalarMarshal(t, NodeBuilder())
+	tests.TestRecursiveMarshal(t, NodeBuilder())
+	tests.TestScalarUnmarshal(t, NodeBuilder())
+	tests.TestRecursiveUnmarshal(t, NodeBuilder())
 }

nodeBuilder.go

@@ -47,10 +47,10 @@ import (
 // `n.GetBuilder().AmendMap().Insert(...)` is called with nodes of unmatching
 // type given to the insertion, the builder will error!)
 type NodeBuilder interface {
-	CreateMap() MapBuilder
-	AmendMap() MapBuilder
-	CreateList() ListBuilder
-	AmendList() ListBuilder
+	CreateMap() (MapBuilder, error)
+	AmendMap() (MapBuilder, error)
+	CreateList() (ListBuilder, error)
+	AmendList() (ListBuilder, error)
 	CreateNull() (Node, error)
 	CreateBool(bool) (Node, error)
 	CreateInt(int) (Node, error)
@@ -60,17 +60,29 @@ type NodeBuilder interface {
 	CreateLink(cid.Cid) (Node, error)
 }
 
+// MapBuilder is an interface for creating new Node instances of kind map.
+//
+// A MapBuilder is generally obtained by getting a NodeBuilder first,
+// and then using CreateMap or AmendMap to begin.
+//
+// Methods mutate the builder's internal state; when done, call Build to
+// produce a new immutable Node from the internal state.
+// (After calling Build, future mutations may be rejected.)
+//
+// Insertion methods error if the key already exists.
+//
+// You may be interested in the fluent package's fluent.MapBuilder equivalent
+// for common usage with less error-handling boilerplate requirements.
 type MapBuilder interface {
-	InsertAll(map[Node]Node) MapBuilder
-	Insert(k, v Node) MapBuilder
-	Delete(k Node) MapBuilder
+	Insert(k, v Node) error
+	Delete(k Node) error
 	Build() (Node, error)
 }
 
 type ListBuilder interface {
-	AppendAll([]Node) ListBuilder
-	Append(v Node) ListBuilder
-	Set(idx int, v Node) ListBuilder
+	AppendAll([]Node)
+	Append(v Node)
+	Set(idx int, v Node)
 	Build() (Node, error)
 }
 

nodeTokening.go

-package ipld
-
-import (
-	"github.com/polydawn/refmt/shared"
-)
-
-// future consideration: TokenizableNode.PushTokens is one of the easiest
-//  features in the whole area to code... but the least useful for composition.
-//  we might drop it outright at some point.
-
-// TokenizableNode is an optional interface which an ipld.Node may also
-// implement to indicate that it has an efficient method for translating
-// itself to a serial Token stream.
-//
-// Any ipld.Node is tokenizable via generic inspection, so providing
-// this interface is optional and purely for performance improvement.
-type TokenizableNode interface {
-	// PushTokens converts this node and its children into a stream of refmt Tokens
-	// and push them sequentially into the given TokenSink.
-	// This is useful for serializing, or hashing, or feeding to any other
-	// TokenSink (for example, converting to other ipld.Node implementations
-	// which can construct themselves from a token stream).
-	PushTokens(sink shared.TokenSink) error
-}
-
-type NodeUnmarshaller func(src shared.TokenSource) (Node, error)

repose/multicodecBuiltinCbor.go

+package repose
+
+import (
+	"io"
+
+	ipld "github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/encoding"
+	"github.com/polydawn/refmt/cbor"
+)
+
+var (
+	_ MulticodecDecoder = DecoderDagCbor
+	_ MulticodecEncoder = EncoderDagCbor
+)
+
+func DecoderDagCbor(nb ipld.NodeBuilder, r io.Reader) (ipld.Node, error) {
+	// Probe for a builtin fast path.  Shortcut to that if possible.
+	//  (ipldcbor.NodeBuilder supports this, for example.)
+	type detectFastPath interface {
+		DecodeCbor(io.Reader) (ipld.Node, error)
+	}
+	if nb2, ok := nb.(detectFastPath); ok {
+		return nb2.DecodeCbor(r)
+	}
+	// Okay, generic builder path.
+	return encoding.Unmarshal(nb, cbor.NewDecoder(r))
+}
+
+func EncoderDagCbor(n ipld.Node, w io.Writer) error {
+	// Probe for a builtin fast path.  Shortcut to that if possible.
+	//  (ipldcbor.Node supports this, for example.)
+	type detectFastPath interface {
+		EncodeCbor(io.Writer) error
+	}
+	if n2, ok := n.(detectFastPath); ok {
+		return n2.EncodeCbor(w)
+	}
+	// Okay, generic inspection path.
+	return encoding.Marshal(n, cbor.NewEncoder(w))
+}

repose/multicodecBuiltinJson.go

+package repose
+
+import (
+	"io"
+
+	ipld "github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/encoding"
+	"github.com/polydawn/refmt/json"
+)
+
+var (
+	_ MulticodecDecoder = DecoderDagJson
+	_ MulticodecEncoder = EncoderDagJson
+)
+
+func DecoderDagJson(nb ipld.NodeBuilder, r io.Reader) (ipld.Node, error) {
+	// Shell out directly to generic builder path.
+	//  (There's not really any fastpaths of note for json.)
+	return encoding.Unmarshal(nb, json.NewDecoder(r))
+}
+
+func EncoderDagJson(n ipld.Node, w io.Writer) error {
+	// Shell out directly to generic inspection path.
+	//  (There's not really any fastpaths of note for json.)
+	// Write another function if you need to tune encoding options about whitespace.
+	return encoding.Marshal(n, json.NewEncoder(w, json.EncodeOptions{
+		Line:   []byte{'\n'},
+		Indent: []byte{'\t'},
+	}))
+}

tests/building.go

@@ -50,19 +50,19 @@ func TestBuildingScalars(t *testing.T, nb ipld.NodeBuilder) {
 func TestBuildingRecursives(t *testing.T, nb ipld.NodeBuilder) {
 	t.Run("short list node", func(t *testing.T) {
 		nb := fluent.WrapNodeBuilder(nb)
-		n := nb.CreateList().
-			Append(nb.CreateString("asdf")).
-			Build()
+		n := nb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+			lb.Append(vnb.CreateString("asdf"))
+		})
 		Wish(t, fluent.WrapNode(n).TraverseIndex(0).AsString(), ShouldEqual, "asdf")
 	})
 	t.Run("nested list node", func(t *testing.T) {
 		nb := fluent.WrapNodeBuilder(nb)
-		n := nb.CreateList().
-			Append(nb.CreateList().
-				Append(nb.CreateString("asdf")).
-				Build()).
-			Append(nb.CreateString("quux")).
-			Build()
+		n := nb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+			lb.Append(vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+				lb.Append(vnb.CreateString("asdf"))
+			}))
+			lb.Append(vnb.CreateString("quux"))
+		})
 		nf := fluent.WrapNode(n)
 		Wish(t, nf.Kind(), ShouldEqual, ipld.ReprKind_List)
 		Wish(t, nf.TraverseIndex(0).TraverseIndex(0).AsString(), ShouldEqual, "asdf")
@@ -70,10 +70,10 @@ func TestBuildingRecursives(t *testing.T, nb ipld.NodeBuilder) {
 	})
 	t.Run("long list node", func(t *testing.T) {
 		nb := fluent.WrapNodeBuilder(nb)
-		n := nb.CreateList().
-			Set(9, nb.CreateString("quux")).
-			Set(19, nb.CreateString("quuux")).
-			Build()
+		n := nb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+			lb.Set(9, vnb.CreateString("quux"))
+			lb.Set(19, vnb.CreateString("quuux"))
+		})
 		nf := fluent.WrapNode(n)
 		Wish(t, nf.Kind(), ShouldEqual, ipld.ReprKind_List)
 		Wish(t, nf.Length(), ShouldEqual, 20)

tests/marshalling.go

@@ -7,6 +7,8 @@ import (
 	. "github.com/warpfork/go-wish"
 
 	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/encoding"
+	"github.com/ipld/go-ipld-prime/fluent"
 )
 
 // TokenBucket acts as a TokenSink; you can dump data into it and then
@@ -52,26 +54,26 @@ func (tb TokenBucket) Minimalize() TokenBucket {
 	return tb
 }
 
-func TestScalarMarshal(t *testing.T, newNode MutableNodeFactory) {
+func TestScalarMarshal(t *testing.T, nb ipld.NodeBuilder) {
 	t.Run("null node", func(t *testing.T) {
-		n0 := newNode()
-		n0.SetNull()
+		n, _ := nb.CreateNull()
 		var tb TokenBucket
-		(n0.(ipld.TokenizableNode)).PushTokens(&tb)
+		err := encoding.Marshal(n, &tb)
+		Wish(t, err, ShouldEqual, nil)
 		Wish(t, tb, ShouldEqual, TokenBucket{
 			{Type: tok.TNull},
 		})
 	})
 }
 
-func TestRecursiveMarshal(t *testing.T, newNode MutableNodeFactory) {
+func TestRecursiveMarshal(t *testing.T, nb ipld.NodeBuilder) {
 	t.Run("short list node", func(t *testing.T) {
-		n0 := newNode()
-		n00 := newNode()
-		n00.SetString("asdf")
-		n0.SetIndex(0, n00)
+		n := fluent.WrapNodeBuilder(nb).CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+			lb.Append(vnb.CreateString("asdf"))
+		})
 		var tb TokenBucket
-		(n0.(ipld.TokenizableNode)).PushTokens(&tb)
+		err := encoding.Marshal(n, &tb)
+		Wish(t, err, ShouldEqual, nil)
 		Wish(t, tb.Minimalize(), ShouldEqual, TokenBucket{
 			{Type: tok.TArrOpen, Length: 1},
 			{Type: tok.TString, Str: "asdf"},
@@ -79,17 +81,15 @@ func TestRecursiveMarshal(t *testing.T, newNode MutableNodeFactory) {
 		})
 	})
 	t.Run("nested list node", func(t *testing.T) {
-		n0 := newNode()
-		n00 := newNode()
-		n0.SetIndex(0, n00)
-		n000 := newNode()
-		n000.SetString("asdf")
-		n00.SetIndex(0, n000)
-		n01 := newNode()
-		n01.SetString("quux")
-		n0.SetIndex(1, n01)
+		n := fluent.WrapNodeBuilder(nb).CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+			lb.Append(vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+				lb.Append(vnb.CreateString("asdf"))
+			}))
+			lb.Append(vnb.CreateString("quux"))
+		})
 		var tb TokenBucket
-		(n0.(ipld.TokenizableNode)).PushTokens(&tb)
+		err := encoding.Marshal(n, &tb)
+		Wish(t, err, ShouldEqual, nil)
 		Wish(t, tb.Minimalize(), ShouldEqual, TokenBucket{
 			{Type: tok.TArrOpen, Length: 2},
 			{Type: tok.TArrOpen, Length: 1},

tests/unmarshalling.go

@@ -7,6 +7,7 @@ import (
 	. "github.com/warpfork/go-wish"
 
 	ipld "github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/encoding"
 	"github.com/ipld/go-ipld-prime/fluent"
 )
 
@@ -23,12 +24,12 @@ func (tb *TokenSourceBucket) Step(yield *tok.Token) (done bool, err error) {
 	return tb.read > len(tb.tokens), nil
 }
 
-func TestScalarUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
+func TestScalarUnmarshal(t *testing.T, nb ipld.NodeBuilder) {
 	t.Run("null node", func(t *testing.T) {
 		tb := &TokenSourceBucket{tokens: []tok.Token{
 			{Type: tok.TNull},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Wish(t, err, ShouldEqual, nil)
 		Wish(t, n.Kind(), ShouldEqual, ipld.ReprKind_Null)
 		Wish(t, tb.read, ShouldEqual, 1)
@@ -37,7 +38,7 @@ func TestScalarUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
 		tb := &TokenSourceBucket{tokens: []tok.Token{
 			{Type: tok.TInt, Int: 1400},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Wish(t, err, ShouldEqual, nil)
 		Wish(t, n.Kind(), ShouldEqual, ipld.ReprKind_Int)
 		Wish(t, fluent.WrapNode(n).AsInt(), ShouldEqual, 1400)
@@ -47,7 +48,7 @@ func TestScalarUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
 		tb := &TokenSourceBucket{tokens: []tok.Token{
 			{Type: tok.TString, Str: "zooooom"},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Wish(t, err, ShouldEqual, nil)
 		Wish(t, n.Kind(), ShouldEqual, ipld.ReprKind_String)
 		Wish(t, fluent.WrapNode(n).AsString(), ShouldEqual, "zooooom")
@@ -55,14 +56,14 @@ func TestScalarUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
 	})
 }
 
-func TestRecursiveUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
+func TestRecursiveUnmarshal(t *testing.T, nb ipld.NodeBuilder) {
 	t.Run("short list node", func(t *testing.T) {
 		tb := &TokenSourceBucket{tokens: []tok.Token{
 			{Type: tok.TArrOpen, Length: 1},
 			{Type: tok.TString, Str: "asdf"},
 			{Type: tok.TArrClose},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Require(t, err, ShouldEqual, nil)
 		Require(t, n.Kind(), ShouldEqual, ipld.ReprKind_List)
 		Require(t, n.Length(), ShouldEqual, 1)
@@ -79,7 +80,7 @@ func TestRecursiveUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
 			{Type: tok.TString, Str: "quux"},
 			{Type: tok.TArrClose},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Require(t, err, ShouldEqual, nil)
 		Require(t, n.Kind(), ShouldEqual, ipld.ReprKind_List)
 		Wish(t, n.Length(), ShouldEqual, 2)
@@ -96,7 +97,7 @@ func TestRecursiveUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
 			{Type: tok.TString, Str: "zomzom"},
 			{Type: tok.TMapClose},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Require(t, err, ShouldEqual, nil)
 		Require(t, n.Kind(), ShouldEqual, ipld.ReprKind_Map)
 		Require(t, n.Length(), ShouldEqual, 1)
@@ -117,7 +118,7 @@ func TestRecursiveUnmarshal(t *testing.T, unmarshalFn ipld.NodeUnmarshaller) {
 			{Type: tok.TInt, Int: 9},
 			{Type: tok.TMapClose},
 		}}
-		n, err := unmarshalFn(tb)
+		n, err := encoding.Unmarshal(nb, tb)
 		Require(t, err, ShouldEqual, nil)
 		Require(t, n.Kind(), ShouldEqual, ipld.ReprKind_Map)
 		Require(t, n.Length(), ShouldEqual, 2)