Hej, we've got tokenization.

You could now readily rig up the ipldfree.Node implementation to a
refmt cbor.TokenSink, for example, and go to town.  (At the moment,
doing so is left as an exercise to the reader.  We'll make that a
smooth built-in/one-word function at some point, but I'm not yet sure
exactly how that should look, so it's deferred for now.)

While working on this, a lot of other things are cooking on simmer:
I'm churning repeatedly over a lot of thoughts about A) API semantics
in general, B) how to cache CIDs of nodes, and C) how to memoize
serializations / reduce memcopies during partial tree updates...
And (unsurprisingly) I keep coming back to the conclusion that the API
for dang near everything should be immutable at heart in order to
keep things sane.  The problem is figuring out how to pursue this
A) efficiently, B) in tandem with reasonably low-friction nativeness
(i.e. I want autocompletion in a standard golang editor to be as useful
as possible!), and C) given an (as yet) lack of good builder or
mutation-applier patterns.  ipldbind was meant to be a solution to the
majority of the B and C issues there, but that rubs smack against the
grain of "let's be immutable" in golang >:/  So... a rock and a hard
place, in short.
Signed-off-by: Eric Myhre <hash@exultant.us>

impl/free/freeNodeTokening.go

+package ipldfree
+
+import (
+	"fmt"
+
+	"github.com/polydawn/refmt/shared"
+	"github.com/polydawn/refmt/tok"
+
+	"github.com/ipld/go-ipld-prime"
+)
+
+func (n *Node) PushTokens(sink shared.TokenSink) error {
+	var tk tok.Token
+	switch n.kind {
+	case ipld.ReprKind_Invalid:
+		return fmt.Errorf("cannot traverse a node that is undefined")
+	case ipld.ReprKind_Null:
+		tk.Type = tok.TNull
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Map:
+		// Emit start of map.
+		tk.Type = tok.TMapOpen
+		tk.Length = len(n._map)
+		if _, err := sink.Step(&tk); err != nil {
+			return err
+		}
+		// Emit map contents (and recurse).
+		for k, v := range n._map { // FIXME map order
+			tk.Type = tok.TString
+			tk.Str = k
+			if _, err := sink.Step(&tk); err != nil {
+				return err
+			}
+			if err := v.PushTokens(sink); err != nil {
+				return err
+			}
+		}
+		// Emit map close.
+		tk.Type = tok.TMapClose
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_List:
+		// Emit start of list.
+		tk.Type = tok.TArrOpen
+		tk.Length = len(n._arr)
+		if _, err := sink.Step(&tk); err != nil {
+			return err
+		}
+		// Emit list contents (and recurse).
+		for _, v := range n._arr {
+			if err := v.PushTokens(sink); err != nil {
+				return err
+			}
+		}
+		// Emit list close.
+		tk.Type = tok.TArrClose
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Bool:
+		tk.Type = tok.TBool
+		tk.Bool = n._bool
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Int:
+		tk.Type = tok.TInt
+		tk.Int = int64(n._int)
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Float:
+		tk.Type = tok.TFloat64
+		tk.Float64 = n._float
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_String:
+		tk.Type = tok.TString
+		tk.Str = n._str
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Bytes:
+		tk.Type = tok.TBytes
+		tk.Bytes = n._bytes
+		_, err := sink.Step(&tk)
+		return err
+	case ipld.ReprKind_Link:
+		panic("todo link emission")
+	default:
+		panic("unreachable")
+	}
+}

impl/free/freeNode_test.go

@@ -11,4 +11,6 @@ func Test(t *testing.T) {
 	mutNodeFac := func() ipld.MutableNode { return &Node{} }
 	tests.TestScalars(t, mutNodeFac)
 	tests.TestRecursives(t, mutNodeFac)
+	tests.TestScalarMarshal(t, mutNodeFac)
+	tests.TestRecursiveMarshal(t, mutNodeFac)
 }

node.go

 package ipld
 
-import "github.com/ipfs/go-cid"
+import (
+	"github.com/polydawn/refmt/shared"
+
+	"github.com/ipfs/go-cid"
+)
 
 type Node interface {
 	// Kind returns a value from the ReprKind enum describing what the
@@ -43,6 +47,13 @@ type Node interface {
 	AsString() (string, error)
 	AsBytes() ([]byte, error)
 	AsLink() (cid.Cid, error)
+
+	// PushTokens converts this node and its children into a stream of refmt Tokens
+	// and push them sequentially into the given TokenSink.
+	// This is useful for serializing, or hashing, or feeding to any other
+	// TokenSink (for example, converting to other ipld.Node implementations
+	// which can construct themselves from a token stream).
+	PushTokens(sink shared.TokenSink) error
 }
 
 type SerializableNode interface {

tests/marshalling.go

+package tests
+
+import (
+	"testing"
+
+	"github.com/polydawn/refmt/tok"
+	. "github.com/warpfork/go-wish"
+)
+
+// TokenBucket acts as a TokenSink; you can dump data into it and then
+// do test assertions on it with go-wish.
+type TokenBucket []tok.Token
+
+func (tb *TokenBucket) Step(consume *tok.Token) (done bool, err error) {
+	if tb == nil {
+		*tb = make(TokenBucket, 0, 10)
+	}
+	*tb = append(*tb, *consume)
+	return false, nil
+}
+
+// This should really be a utility func in refmt tok.  -.-
+func (tb TokenBucket) Minimalize() TokenBucket {
+	for i, v := range tb {
+		switch v.Type {
+		case tok.TMapOpen:
+			tb[i] = tok.Token{Type: v.Type, Length: v.Length, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TMapClose:
+			tb[i] = tok.Token{Type: v.Type}
+		case tok.TArrOpen:
+			tb[i] = tok.Token{Type: v.Type, Length: v.Length, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TArrClose:
+			tb[i] = tok.Token{Type: v.Type}
+		case tok.TNull:
+			tb[i] = tok.Token{Type: v.Type, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TString:
+			tb[i] = tok.Token{Type: v.Type, Str: v.Str, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TBytes:
+			tb[i] = tok.Token{Type: v.Type, Bytes: v.Bytes, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TBool:
+			tb[i] = tok.Token{Type: v.Type, Bool: v.Bool, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TInt:
+			tb[i] = tok.Token{Type: v.Type, Int: v.Int, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TUint:
+			tb[i] = tok.Token{Type: v.Type, Uint: v.Uint, Tagged: v.Tagged, Tag: v.Tag}
+		case tok.TFloat64:
+			tb[i] = tok.Token{Type: v.Type, Float64: v.Float64, Tagged: v.Tagged, Tag: v.Tag}
+		}
+	}
+	return tb
+}
+
+func TestScalarMarshal(t *testing.T, newNode MutableNodeFactory) {
+	t.Run("null node", func(t *testing.T) {
+		n0 := newNode()
+		n0.SetNull()
+		var tb TokenBucket
+		n0.PushTokens(&tb)
+		Wish(t, tb, ShouldEqual, TokenBucket{
+			{Type: tok.TNull},
+		})
+	})
+}
+
+func TestRecursiveMarshal(t *testing.T, newNode MutableNodeFactory) {
+	t.Run("short list node", func(t *testing.T) {
+		n0 := newNode()
+		n00 := newNode()
+		n00.SetString("asdf")
+		n0.SetIndex(0, n00)
+		var tb TokenBucket
+		n0.PushTokens(&tb)
+		Wish(t, tb.Minimalize(), ShouldEqual, TokenBucket{
+			{Type: tok.TArrOpen, Length: 1},
+			{Type: tok.TString, Str: "asdf"},
+			{Type: tok.TArrClose},
+		})
+	})
+	t.Run("nested list node", func(t *testing.T) {
+		n0 := newNode()
+		n00 := newNode()
+		n0.SetIndex(0, n00)
+		n000 := newNode()
+		n000.SetString("asdf")
+		n00.SetIndex(0, n000)
+		n01 := newNode()
+		n01.SetString("quux")
+		n0.SetIndex(1, n01)
+		var tb TokenBucket
+		n0.PushTokens(&tb)
+		Wish(t, tb.Minimalize(), ShouldEqual, TokenBucket{
+			{Type: tok.TArrOpen, Length: 2},
+			{Type: tok.TArrOpen, Length: 1},
+			{Type: tok.TString, Str: "asdf"},
+			{Type: tok.TArrClose},
+			{Type: tok.TString, Str: "quux"},
+			{Type: tok.TArrClose},
+		})
+	})
+}