Extract the interface for declaring a node to be self-tokenizing.

Signed-off-by: Eric Myhre <hash@exultant.us>

impl/free/freeNodeTokening.go

@@ -32,8 +32,13 @@ func (n *Node) PushTokens(sink shared.TokenSink) error {
 			if _, err := sink.Step(&tk); err != nil {
 				return err
 			}
-			if err := v.PushTokens(sink); err != nil {
-				return err
+			switch v2 := v.(type) {
+			case ipld.TokenizableNode:
+				if err := v2.PushTokens(sink); err != nil {
+					return err
+				}
+			default:
+				panic("todo generic node tokenizer fallback")
 			}
 		}
 		// Emit map close.
@@ -49,8 +54,13 @@ func (n *Node) PushTokens(sink shared.TokenSink) error {
 		}
 		// Emit list contents (and recurse).
 		for _, v := range n._arr {
-			if err := v.PushTokens(sink); err != nil {
-				return err
+			switch v2 := v.(type) {
+			case ipld.TokenizableNode:
+				if err := v2.PushTokens(sink); err != nil {
+					return err
+				}
+			default:
+				panic("todo generic node tokenizer fallback")
 			}
 		}
 		// Emit list close.

node.go

 package ipld
 
 import (
-	"github.com/polydawn/refmt/shared"
-
 	"github.com/ipfs/go-cid"
 )
 
@@ -57,13 +55,6 @@ type Node interface {
 	AsString() (string, error)
 	AsBytes() ([]byte, error)
 	AsLink() (cid.Cid, error)
-
-	// PushTokens converts this node and its children into a stream of refmt Tokens
-	// and push them sequentially into the given TokenSink.
-	// This is useful for serializing, or hashing, or feeding to any other
-	// TokenSink (for example, converting to other ipld.Node implementations
-	// which can construct themselves from a token stream).
-	PushTokens(sink shared.TokenSink) error
 }
 
 type SerializableNode interface {

nodeTokening.go

+package ipld
+
+import (
+	"github.com/polydawn/refmt/shared"
+)
+
+// TokenizableNode is an optional interface which an ipld.Node may also
+// implement to indicate that it has an efficient method for translating
+// itself to a serial Token stream.
+//
+// Any ipld.Node is tokenizable via generic inspection, so providing
+// this interface is optional and purely for performance improvement.
+type TokenizableNode interface {
+	// PushTokens converts this node and its children into a stream of refmt Tokens
+	// and push them sequentially into the given TokenSink.
+	// This is useful for serializing, or hashing, or feeding to any other
+	// TokenSink (for example, converting to other ipld.Node implementations
+	// which can construct themselves from a token stream).
+	PushTokens(sink shared.TokenSink) error
+}

tests/marshalling.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/polydawn/refmt/tok"
 	. "github.com/warpfork/go-wish"
+
+	"github.com/ipld/go-ipld-prime"
 )
 
 // TokenBucket acts as a TokenSink; you can dump data into it and then
@@ -55,7 +57,7 @@ func TestScalarMarshal(t *testing.T, newNode MutableNodeFactory) {
 		n0 := newNode()
 		n0.SetNull()
 		var tb TokenBucket
-		n0.PushTokens(&tb)
+		(n0.(ipld.TokenizableNode)).PushTokens(&tb)
 		Wish(t, tb, ShouldEqual, TokenBucket{
 			{Type: tok.TNull},
 		})
@@ -69,7 +71,7 @@ func TestRecursiveMarshal(t *testing.T, newNode MutableNodeFactory) {
 		n00.SetString("asdf")
 		n0.SetIndex(0, n00)
 		var tb TokenBucket
-		n0.PushTokens(&tb)
+		(n0.(ipld.TokenizableNode)).PushTokens(&tb)
 		Wish(t, tb.Minimalize(), ShouldEqual, TokenBucket{
 			{Type: tok.TArrOpen, Length: 1},
 			{Type: tok.TString, Str: "asdf"},
@@ -87,7 +89,7 @@ func TestRecursiveMarshal(t *testing.T, newNode MutableNodeFactory) {
 		n01.SetString("quux")
 		n0.SetIndex(1, n01)
 		var tb TokenBucket
-		n0.PushTokens(&tb)
+		(n0.(ipld.TokenizableNode)).PushTokens(&tb)
 		Wish(t, tb.Minimalize(), ShouldEqual, TokenBucket{
 			{Type: tok.TArrOpen, Length: 2},
 			{Type: tok.TArrOpen, Length: 1},