feat(selectors): implement recursive selector

An initial implementation of recursive selectors that offers a way to traverse them. Not covered is
validating ExploreRecursiveEdge being under an ExploreRecursive selector

traversal/selector/exploreRecursive.go

+package selector
+
+import (
+	"fmt"
+
+	ipld "github.com/ipld/go-ipld-prime"
+)
+
+// ExploreRecursive traverses some structure recursively.
+// To guide this exploration, it uses a "sequence", which is another Selector
+// tree; some leaf node in this sequence should contain an ExploreRecursiveEdge
+// selector, which denotes the place recursion should occur.
+//
+// In implementation, whenever evaluation reaches an ExploreRecursiveEdge marker
+// in the recursion sequence's Selector tree, the implementation logically
+// produces another new Selector which is a copy of the original
+// ExploreRecursive selector, but with a decremented maxDepth parameter, and
+// continues evaluation thusly.
+//
+// It is not valid for an ExploreRecursive selector's sequence to contain
+// no instances of ExploreRecursiveEdge; it *is* valid for it to contain
+// more than one ExploreRecursiveEdge.
+//
+// ExploreRecursive can contain a nested ExploreRecursive!
+// This is comparable to a nested for-loop.
+// In these cases, any ExploreRecursiveEdge instance always refers to the
+// nearest parent ExploreRecursive (in other words, ExploreRecursiveEdge can
+// be thought of like the 'continue' statement, or end of a for-loop body;
+// it is *not* a 'goto' statement).
+//
+// Be careful when using ExploreRecursive with a large maxDepth parameter;
+// it can easily cause very large traversals (especially if used in combination
+// with selectors like ExploreAll inside the sequence).
+type ExploreRecursive struct {
+	sequence Selector // selector for element we're interested in
+	current  Selector // selector to apply to the current node
+	maxDepth int
+}
+
+// Interests for ExploreRecursive is empty (meaning traverse everything)
+func (s ExploreRecursive) Interests() []PathSegment {
+	return s.current.Interests()
+}
+
+// Explore returns the node's selector for all fields
+func (s ExploreRecursive) Explore(n ipld.Node, p PathSegment) Selector {
+	nextSelector := s.current.Explore(n, p)
+	if nextSelector == nil {
+		return nil
+	}
+	_, ok := nextSelector.(ExploreRecursiveEdge)
+	if !ok {
+		return ExploreRecursive{s.sequence, nextSelector, s.maxDepth}
+	}
+	if s.maxDepth < 2 {
+		return nil
+	}
+	return ExploreRecursive{s.sequence, s.sequence, s.maxDepth - 1}
+}
+
+// Decide always returns false because this is not a matcher
+func (s ExploreRecursive) Decide(n ipld.Node) bool {
+	return s.current.Decide(n)
+}
+
+// ParseExploreRecursive assembles a Selector from a ExploreRecursive selector node
+func ParseExploreRecursive(n ipld.Node) (Selector, error) {
+	if n.ReprKind() != ipld.ReprKind_Map {
+		return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map")
+	}
+
+	maxDepthNode, err := n.TraverseField(maxDepthKey)
+	if err != nil {
+		return nil, fmt.Errorf("selector spec parse rejected: maxDepth field must be present in ExploreRecursive selector")
+	}
+	maxDepthValue, err := maxDepthNode.AsInt()
+	if err != nil {
+		return nil, fmt.Errorf("selector spec parse rejected: maxDepth field must be a number in ExploreRecursive selector")
+	}
+	sequence, err := n.TraverseField(sequenceKey)
+	if err != nil {
+		return nil, fmt.Errorf("selector spec parse rejected: sequence field must be present in ExploreRecursive selector")
+	}
+	selector, err := ParseSelector(sequence)
+	if err != nil {
+		return nil, err
+	}
+	return ExploreRecursive{selector, selector, maxDepthValue}, nil
+}

traversal/selector/exploreRecursiveEdge.go

+package selector
+
+import (
+	"fmt"
+
+	ipld "github.com/ipld/go-ipld-prime"
+)
+
+// ExploreRecursiveEdge is a special sentinel value which is used to mark
+// the end of a sequence started by an ExploreRecursive selector: the recursion
+// goes back to the initial state of the earlier ExploreRecursive selector,
+// and proceeds again (with a decremented maxDepth value).
+//
+// An ExploreRecursive selector that doesn't contain an ExploreRecursiveEdge
+// is nonsensical.  Containing more than one ExploreRecursiveEdge is valid.
+// An ExploreRecursiveEdge without an enclosing ExploreRecursive is an error.
+type ExploreRecursiveEdge struct{}
+
+// Interests should ultimately never get called for an ExploreRecursiveEdge selector
+func (s ExploreRecursiveEdge) Interests() []PathSegment {
+	return []PathSegment{}
+}
+
+// Explore should ultimately never get called for an ExploreRecursiveEdge selector
+func (s ExploreRecursiveEdge) Explore(n ipld.Node, p PathSegment) Selector {
+	return nil
+}
+
+// Decide should ultimately never get called for an ExploreRecursiveEdge selector
+func (s ExploreRecursiveEdge) Decide(n ipld.Node) bool {
+	return false
+}
+
+// ParseExploreRecursiveEdge assembles a Selector
+// from a exploreRecursiveEdge selector node
+func ParseExploreRecursiveEdge(n ipld.Node) (Selector, error) {
+	if n.ReprKind() != ipld.ReprKind_Map {
+		return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map")
+	}
+	return ExploreRecursiveEdge{}, nil
+}

traversal/selector/exploreRecursive_test.go

+package selector
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/ipld/go-ipld-prime/fluent"
+	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
+	. "github.com/warpfork/go-wish"
+)
+
+func TestParseExploreRecursive(t *testing.T) {
+	fnb := fluent.WrapNodeBuilder(ipldfree.NodeBuilder()) // just for the other fixture building
+	t.Run("parsing non map node should error", func(t *testing.T) {
+		sn := fnb.CreateInt(0)
+		_, err := ParseExploreRecursive(sn)
+		Wish(t, err, ShouldEqual, fmt.Errorf("selector spec parse rejected: selector body must be a map"))
+	})
+	t.Run("parsing map node without sequence field should error", func(t *testing.T) {
+		sn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString(maxDepthKey), vnb.CreateInt(2))
+		})
+		_, err := ParseExploreRecursive(sn)
+		Wish(t, err, ShouldEqual, fmt.Errorf("selector spec parse rejected: sequence field must be present in ExploreRecursive selector"))
+	})
+	t.Run("parsing map node without maxDepth field should error", func(t *testing.T) {
+		sn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString(sequenceKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+				mb.Insert(knb.CreateString(matcherKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {}))
+			}))
+		})
+		_, err := ParseExploreRecursive(sn)
+		Wish(t, err, ShouldEqual, fmt.Errorf("selector spec parse rejected: maxDepth field must be present in ExploreRecursive selector"))
+	})
+	t.Run("parsing map node with maxDepth field that is not an int should error", func(t *testing.T) {
+		sn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString(maxDepthKey), vnb.CreateString("cheese"))
+			mb.Insert(knb.CreateString(sequenceKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+				mb.Insert(knb.CreateString(matcherKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {}))
+			}))
+		})
+		_, err := ParseExploreRecursive(sn)
+		Wish(t, err, ShouldEqual, fmt.Errorf("selector spec parse rejected: maxDepth field must be a number in ExploreRecursive selector"))
+	})
+	t.Run("parsing map node with sequence field with invalid selector node should return child's error", func(t *testing.T) {
+		sn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString(maxDepthKey), vnb.CreateInt(2))
+			mb.Insert(knb.CreateString(sequenceKey), vnb.CreateInt(0))
+		})
+		_, err := ParseExploreRecursive(sn)
+		Wish(t, err, ShouldEqual, fmt.Errorf("selector spec parse rejected: selector is a keyed union and thus must be a map"))
+	})
+	t.Run("parsing map node with sequence field with valid selector node should parse", func(t *testing.T) {
+		sn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString(maxDepthKey), vnb.CreateInt(2))
+			mb.Insert(knb.CreateString(sequenceKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+				mb.Insert(knb.CreateString(exploreAllKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+					mb.Insert(knb.CreateString(nextSelectorKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+						mb.Insert(knb.CreateString(exploreRecursiveEdgeKey), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {}))
+					}))
+				}))
+			}))
+		})
+		s, err := ParseExploreRecursive(sn)
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, s, ShouldEqual, ExploreRecursive{ExploreAll{ExploreRecursiveEdge{}}, ExploreAll{ExploreRecursiveEdge{}}, 2})
+	})
+}
+
+/*
+
+{
+	exploreRecursive: {
+		maxDepth: 3
+		sequence: {
+			exploreFields: {
+				fields: {
+					Parents: {
+						exploreAll: {
+							exploreRecursiveEdge: {}
+						}
+					}
+				}
+			}
+		}
+	}
+ }
+
+*/
+
+func TestExploreRecursiveExplore(t *testing.T) {
+	fnb := fluent.WrapNodeBuilder(ipldfree.NodeBuilder()) // just for the other fixture building
+	recursiveEdge := ExploreRecursiveEdge{}
+	maxDepth := 3
+	var err error
+	var rs Selector
+	t.Run("exploring should traverse until we get to maxDepth", func(t *testing.T) {
+		parentsSelector := ExploreAll{recursiveEdge}
+		subTree := ExploreFields{map[string]Selector{"Parents": parentsSelector}, []PathSegment{PathSegmentString{S: "Parents"}}}
+		rs = ExploreRecursive{subTree, subTree, maxDepth}
+		rn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+				lb.Append(vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+					mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+						lb.Append(vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+							mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+								lb.Append(vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+									mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {}))
+								}))
+							}))
+						}))
+					}))
+				}))
+			}))
+		})
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentInt{I: 0})
+		rn, err = rn.TraverseIndex(0)
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, subTree, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentInt{I: 0})
+		rn, err = rn.TraverseIndex(0)
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, subTree, maxDepth - 2})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth - 2})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentInt{I: 0})
+		rn, err = rn.TraverseIndex(0)
+		Wish(t, rs, ShouldEqual, nil)
+		Wish(t, err, ShouldEqual, nil)
+	})
+	t.Run("exploring should continue till we get to selector that returns nil on explore", func(t *testing.T) {
+		parentsSelector := ExploreIndex{recursiveEdge, [1]PathSegment{PathSegmentInt{I: 1}}}
+		subTree := ExploreFields{map[string]Selector{"Parents": parentsSelector}, []PathSegment{PathSegmentString{S: "Parents"}}}
+		rs = ExploreRecursive{subTree, subTree, maxDepth}
+		rn := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+				lb.Append(vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {}))
+			}))
+		})
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentInt{I: 0})
+		Wish(t, rs, ShouldEqual, nil)
+	})
+	t.Run("exploring should work when there is nested recursion", func(t *testing.T) {
+		parentsSelector := ExploreAll{recursiveEdge}
+		sideSelector := ExploreAll{recursiveEdge}
+		subTree := ExploreFields{map[string]Selector{
+			"Parents": parentsSelector,
+			"Side":    ExploreRecursive{sideSelector, sideSelector, maxDepth},
+		}, []PathSegment{
+			PathSegmentString{S: "Parents"},
+			PathSegmentString{S: "Side"},
+		},
+		}
+		s := ExploreRecursive{subTree, subTree, maxDepth}
+		n := fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+			mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {
+				lb.Append(vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+					mb.Insert(knb.CreateString("Parents"), vnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) {}))
+					mb.Insert(knb.CreateString("Side"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+						mb.Insert(knb.CreateString("cheese"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+							mb.Insert(knb.CreateString("whiz"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {}))
+						}))
+					}))
+				}))
+			}))
+			mb.Insert(knb.CreateString("Side"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+				mb.Insert(knb.CreateString("real"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+					mb.Insert(knb.CreateString("apple"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+						mb.Insert(knb.CreateString("sauce"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {}))
+					}))
+				}))
+			}))
+		})
+		rn := n
+		rs = s
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentInt{I: 0})
+		rn, err = rn.TraverseIndex(0)
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, subTree, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rn = n
+		rs = s
+		rs = rs.Explore(rn, PathSegmentString{S: "Side"})
+		rn, err = rn.TraverseField("Side")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, maxDepth}, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "real"})
+		rn, err = rn.TraverseField("real")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, maxDepth - 1}, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "apple"})
+		rn, err = rn.TraverseField("apple")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, maxDepth - 2}, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "sauce"})
+		rn, err = rn.TraverseField("sauce")
+		Wish(t, rs, ShouldEqual, nil)
+		Wish(t, err, ShouldEqual, nil)
+		rn = n
+		rs = s
+		rs = rs.Explore(rn, PathSegmentString{S: "Parents"})
+		rn, err = rn.TraverseField("Parents")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, parentsSelector, maxDepth})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentInt{I: 0})
+		rn, err = rn.TraverseIndex(0)
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, subTree, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "Side"})
+		rn, err = rn.TraverseField("Side")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, maxDepth}, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "cheese"})
+		rn, err = rn.TraverseField("cheese")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, maxDepth - 1}, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+		rs = rs.Explore(rn, PathSegmentString{S: "whiz"})
+		rn, err = rn.TraverseField("whiz")
+		Wish(t, rs, ShouldEqual, ExploreRecursive{subTree, ExploreRecursive{sideSelector, sideSelector, maxDepth - 2}, maxDepth - 1})
+		Wish(t, err, ShouldEqual, nil)
+	})
+}

traversal/selector/selector.go

@@ -38,6 +38,10 @@ func ParseSelector(n ipld.Node) (Selector, error) {
 		return ParseExploreRange(v)
 	case exploreUnionKey:
 		return ParseExploreUnion(v)
+	case exploreRecursiveKey:
+		return ParseExploreRecursive(v)
+	case exploreRecursiveEdgeKey:
+		return ParseExploreRecursiveEdge(v)
 	case matcherKey:
 		return ParseMatcher(v)
 	default: