traversal.Focus implemented and tests: links go!

NodeBuilderChooser returned -- sorry.  There are some cases where it's
unavoidable.  (If we have schemas / typed nodes, we can do feature
detection on the node to see if it can recommend an appropriate
NodeBuilder for the far side of its link.  If we're fine using Node
implementations that have 'any' storage support, it's not an issue.
If we're using *bind* implementation Nodes -- which are constrained
by the Golang native type system, and yet not perfectly mapped onto
our own typed.Node constraint declarations -- then we need some way
to choose the NodeBuilder.  So.  NodeBuilderChooser.  Again: sorry.)

We can probably still expect the NodeBuilderChooser to be fixed
(probably returning one of the 'any' storage supporting Node impls)
in practice, since these traversal systems are generally for either
"low context" usage (e.g., dunno what the data is) or will be used
on typed/schema nodes, in which case we're again free of these issues.
And the default values will indeed do this, using our new helpful
default initialization for TraversalConfig.

TraversalConfig will now always be initialized to sensible default
values when using any of the traversal methods.  That means you can
always assume `tp.Ctx`, etc, are going to be set -- no nil checks
necssary -- even though they're optional to the caller.

Error messages from traversal.Focus are touched up a bit for both
consistency and clearer information.  There's more work to do here:
we want these to be full-on typed errors before we call it "done".
This work will probably begin soon, but be spread over quite a few
commits (we also need to go back up to the *main* ipld package and
make types for Node-level errors, and make sure tests exist to
standardize those errors across Node implementations as well).
Signed-off-by: Eric Myhre <hash@exultant.us>

traversal/common.go

+package traversal
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	ipld "github.com/ipld/go-ipld-prime"
+	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
+)
+
+// init sets all the values in TraveralConfig to reasonable defaults
+// if they're currently the zero value.
+func (tc *TraversalConfig) init() {
+	if tc.Ctx == nil {
+		tc.Ctx = context.Background()
+	}
+	if tc.LinkLoader == nil {
+		tc.LinkLoader = func(ipld.Link, ipld.LinkContext) (io.Reader, error) {
+			return nil, fmt.Errorf("no link loader configured")
+		}
+	}
+	if tc.LinkNodeBuilderChooser == nil {
+		tc.LinkNodeBuilderChooser = func(ipld.Link, ipld.LinkContext) ipld.NodeBuilder {
+			return ipldfree.NodeBuilder()
+		}
+	}
+	if tc.LinkStorer == nil {
+		tc.LinkStorer = func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
+			return nil, nil, fmt.Errorf("no link storer configured")
+		}
+	}
+}
+
+func (tp *TraversalProgress) init() {
+	if tp.TraversalConfig == nil {
+		tp.TraversalConfig = &TraversalConfig{}
+	}
+	tp.TraversalConfig.init()
+}

traversal/fns.go

@@ -6,6 +6,9 @@ import (
 	ipld "github.com/ipld/go-ipld-prime"
 )
 
+// This file defines interfaces for things users provide.
+//------------------------------------------------
+
 // VisitFn is a read-only visitor.
 type VisitFn func(TraversalProgress, ipld.Node) error
 
@@ -28,7 +31,19 @@ type TraversalProgress struct {
 }
 
 type TraversalConfig struct {
-	Ctx        context.Context // Context carried through a traversal.  Optional; use it if you need cancellation.
-	LinkLoader ipld.Loader     // Loader used for automatic link traversal.
-	LinkStorer ipld.Storer     // Storer used if any mutation features (e.g. traversal.Transform) are used.
+	Ctx                    context.Context    // Context carried through a traversal.  Optional; use it if you need cancellation.
+	LinkLoader             ipld.Loader        // Loader used for automatic link traversal.
+	LinkNodeBuilderChooser NodeBuilderChooser // Chooser for Node implementations to produce during automatic link traversal.
+	LinkStorer             ipld.Storer        // Storer used if any mutation features (e.g. traversal.Transform) are used.
 }
+
+// NodeBuilderChooser is a function that returns a NodeBuilder based on
+// the information in a Link its LinkContext.
+//
+// A NodeBuilderChooser can be used in a TraversalConfig to be clear about
+// what kind of Node implementation to use when loading a Link.
+// In a simple example, it could constantly return an `ipldfree.NodeBuilder`.
+// In a more complex example, a program using `bind` over native Go types
+// could decide what kind of native type is expected, and return a
+// `bind.NodeBuilder` for that specific concrete native type.
+type NodeBuilderChooser func(ipld.Link, ipld.LinkContext) ipld.NodeBuilder

traversal/focus.go

@@ -32,32 +32,53 @@ func FocusedTransform(n ipld.Node, p ipld.Path, fn TransformFn) (ipld.Node, erro
 // so far will continue to be extended, so continued nested uses of Focus
 // will see a fully contextualized Path.
 func (tp TraversalProgress) Focus(n ipld.Node, p ipld.Path, fn VisitFn) error {
+	tp.init()
 	segments := p.Segments()
+	var prev ipld.Node // for LinkContext
 	for i, seg := range segments {
+		// Traverse the segment.
 		switch n.ReprKind() {
 		case ipld.ReprKind_Invalid:
 			return fmt.Errorf("cannot traverse node at %q: it is undefined", p.Truncate(i))
 		case ipld.ReprKind_Map:
 			next, err := n.TraverseField(seg)
 			if err != nil {
-				return fmt.Errorf("error traversing node at %q: %s", p.Truncate(i), err)
+				return fmt.Errorf("error traversing segment %q on node at %q: %s", seg, p.Truncate(i), err)
 			}
-			n = next
+			prev, n = n, next
 		case ipld.ReprKind_List:
 			intSeg, err := strconv.Atoi(seg)
 			if err != nil {
-				return fmt.Errorf("cannot traverse node at %q: the next path segment (%q) cannot be parsed as a number and the node is a list", p.Truncate(i), seg)
+				return fmt.Errorf("error traversing segment %q on node at %q: the segment cannot be parsed as a number and the node is a list", seg, p.Truncate(i))
 			}
 			next, err := n.TraverseIndex(intSeg)
 			if err != nil {
-				return fmt.Errorf("error traversing node at %q: %s", p.Truncate(i), err)
+				return fmt.Errorf("error traversing segment %q on node at %q: %s", seg, p.Truncate(i), err)
 			}
-			n = next
-		case ipld.ReprKind_Link:
-			panic("NYI link loading") // TODO
-			// this would set a progress marker in `tp` as well
+			prev, n = n, next
 		default:
-			return fmt.Errorf("error traversing node at %q: %s", p.Truncate(i), fmt.Errorf("cannot traverse terminals"))
+			return fmt.Errorf("cannot traverse node at %q: %s", p.Truncate(i), fmt.Errorf("cannot traverse terminals"))
+		}
+		// Dereference any links.
+		for n.ReprKind() == ipld.ReprKind_Link {
+			lnk, _ := n.AsLink()
+			// Assemble the LinkContext in case the Loader or NBChooser want it.
+			lnkCtx := ipld.LinkContext{
+				LinkPath:   p.Truncate(i),
+				LinkNode:   n,
+				ParentNode: prev,
+			}
+			// Load link!
+			next, err := lnk.Load(
+				tp.Ctx,
+				lnkCtx,
+				tp.LinkNodeBuilderChooser(lnk, lnkCtx),
+				tp.LinkLoader,
+			)
+			if err != nil {
+				return fmt.Errorf("error traversing node at %q: could not load link %q: %s", p.Truncate(i+1), lnk, err)
+			}
+			prev, n = n, next
 		}
 	}
 	tp.Path = tp.Path.Join(p)

traversal/focus_test.go

+package traversal_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+	"unicode"
+
+	. "github.com/warpfork/go-wish"
+
+	cid "github.com/ipfs/go-cid"
+	ipld "github.com/ipld/go-ipld-prime"
+	_ "github.com/ipld/go-ipld-prime/encoding/dagjson"
+	"github.com/ipld/go-ipld-prime/fluent"
+	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/ipld/go-ipld-prime/traversal"
+)
+
+// Do some fixture fabrication.
+// We assume all the builders and serialization must Just Work here.
+
+var storage = make(map[ipld.Link][]byte)
+var fnb = fluent.WrapNodeBuilder(ipldfree.NodeBuilder()) // just for the other fixture building
+var (
+	leafAlpha, leafAlphaLnk         = encode(fnb.CreateString("alpha"))
+	leafBeta, leafBetaLnk           = encode(fnb.CreateString("beta"))
+	middleMapNode, middleMapNodeLnk = encode(fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+		mb.Insert(knb.CreateString("foo"), vnb.CreateBool(true))
+		mb.Insert(knb.CreateString("bar"), vnb.CreateBool(false))
+		mb.Insert(knb.CreateString("nested"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString("alink"), vnb.CreateLink(leafAlphaLnk))
+			mb.Insert(knb.CreateString("nonlink"), vnb.CreateString("zoo"))
+		}))
+	}))
+	middleListNode, middleListNodeLnk = encode(fnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+		lb.Append(vnb.CreateLink(leafAlphaLnk))
+		lb.Append(vnb.CreateLink(leafAlphaLnk))
+		lb.Append(vnb.CreateLink(leafBetaLnk))
+		lb.Append(vnb.CreateLink(leafAlphaLnk))
+	}))
+	rootNode, rootNodeLnk = encode(fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+		mb.Insert(knb.CreateString("plain"), vnb.CreateString("olde string"))
+		mb.Insert(knb.CreateString("linkedString"), vnb.CreateLink(leafAlphaLnk))
+		mb.Insert(knb.CreateString("linkedMap"), vnb.CreateLink(middleMapNodeLnk))
+		mb.Insert(knb.CreateString("linkedList"), vnb.CreateLink(middleListNodeLnk))
+	}))
+)
+
+// encode hardcodes some encoding choices for ease of use in fixture generation;
+// just gimme a link and stuff the bytes in a map.
+// (also return the node again for convenient assignment.)
+func encode(n ipld.Node) (ipld.Node, ipld.Link) {
+	lb := cidlink.LinkBuilder{cid.Prefix{
+		Version:  1,
+		Codec:    0x0129,
+		MhType:   0x17,
+		MhLength: 4,
+	}}
+	lnk, err := lb.Build(context.Background(), ipld.LinkContext{}, n,
+		func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
+			buf := bytes.Buffer{}
+			return &buf, func(lnk ipld.Link) error {
+				storage[lnk] = buf.Bytes()
+				return nil
+			}, nil
+		},
+	)
+	if err != nil {
+		panic(err)
+	}
+	return n, lnk
+}
+
+// Print a quick little table of our fixtures for sanity check purposes.
+func init() {
+	withoutWhitespace := func(s string) string {
+		return strings.Map(func(r rune) rune {
+			if !unicode.IsPrint(r) {
+				return -1
+			} else {
+				return r
+			}
+		}, s)
+	}
+	fmt.Printf("fixtures:\n"+strings.Repeat("\t%v\t%v\n", 5),
+		leafAlphaLnk, withoutWhitespace(string(storage[leafAlphaLnk])),
+		leafBetaLnk, withoutWhitespace(string(storage[leafBetaLnk])),
+		middleMapNodeLnk, withoutWhitespace(string(storage[middleMapNodeLnk])),
+		middleListNodeLnk, withoutWhitespace(string(storage[middleListNodeLnk])),
+		rootNodeLnk, withoutWhitespace(string(storage[rootNodeLnk])),
+	)
+}
+
+// covers Focus used on one already-loaded Node; no link-loading exercised.
+func TestFocusSingleTree(t *testing.T) {
+	t.Run("empty path on scalar node returns start node", func(t *testing.T) {
+		err := traversal.Focus(fnb.CreateString("x"), ipld.Path{}, func(tp traversal.TraversalProgress, n ipld.Node) error {
+			Wish(t, n, ShouldEqual, fnb.CreateString("x"))
+			Wish(t, tp.Path.String(), ShouldEqual, ipld.Path{}.String())
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+	})
+	t.Run("one step path on map node works", func(t *testing.T) {
+		err := traversal.Focus(middleMapNode, ipld.ParsePath("foo"), func(tp traversal.TraversalProgress, n ipld.Node) error {
+			Wish(t, n, ShouldEqual, fnb.CreateBool(true))
+			Wish(t, tp.Path, ShouldEqual, ipld.ParsePath("foo"))
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+	})
+	t.Run("two step path on map node works", func(t *testing.T) {
+		err := traversal.Focus(middleMapNode, ipld.ParsePath("nested/nonlink"), func(tp traversal.TraversalProgress, n ipld.Node) error {
+			Wish(t, n, ShouldEqual, fnb.CreateString("zoo"))
+			Wish(t, tp.Path, ShouldEqual, ipld.ParsePath("nested/nonlink"))
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+	})
+}
+
+func TestFocusWithLinkLoading(t *testing.T) {
+	t.Run("link traversal with no configured loader should fail", func(t *testing.T) {
+		t.Run("terminal link should fail", func(t *testing.T) {
+			err := traversal.Focus(middleMapNode, ipld.ParsePath("nested/alink"), func(tp traversal.TraversalProgress, n ipld.Node) error {
+				t.Errorf("should not be reached; no way to load this path")
+				return nil
+			})
+			Wish(t, err.Error(), ShouldEqual, `error traversing node at "nested/alink": could not load link "`+leafAlphaLnk.String()+`": no link loader configured`)
+		})
+		t.Run("mid-path link should fail", func(t *testing.T) {
+			err := traversal.Focus(rootNode, ipld.ParsePath("linkedMap/nested/nonlink"), func(tp traversal.TraversalProgress, n ipld.Node) error {
+				t.Errorf("should not be reached; no way to load this path")
+				return nil
+			})
+			Wish(t, err.Error(), ShouldEqual, `error traversing node at "linkedMap": could not load link "`+middleMapNodeLnk.String()+`": no link loader configured`)
+		})
+	})
+	t.Run("link traversal with loader should work", func(t *testing.T) {
+		err := traversal.TraversalProgress{
+			TraversalConfig: &traversal.TraversalConfig{
+				LinkLoader: func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
+					return bytes.NewBuffer(storage[lnk]), nil
+				},
+			},
+		}.Focus(rootNode, ipld.ParsePath("linkedMap/nested/nonlink"), func(tp traversal.TraversalProgress, n ipld.Node) error {
+			Wish(t, n, ShouldEqual, fnb.CreateString("zoo"))
+			Wish(t, tp.Path, ShouldEqual, ipld.ParsePath("linkedMap/nested/nonlink"))
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+	})
+}

traversal/traverse.go

@@ -18,6 +18,7 @@ func TraverseTransform(n ipld.Node, s selector.Selector, fn TransformFn) (ipld.N
 }
 
 func (tp TraversalProgress) Traverse(n ipld.Node, s selector.Selector, fn VisitFn) error {
+	tp.init()
 	return tp.TraverseInformatively(n, s, func(tp TraversalProgress, n ipld.Node, tr TraversalReason) error {
 		if tr != 1 {
 			return nil