Merge pull request #20 from ipld/feat/better-traversal

More complete selector traversal

encoding/dagcbor/roundtrip_test.go

@@ -43,3 +43,19 @@ func TestRoundtrip(t *testing.T) {
 		Wish(t, n2, ShouldEqual, n)
 	})
 }
+
+func TestRoundtripSimple(t *testing.T) {
+	simple := fnb.CreateString("applesauce")
+	t.Run("encoding", func(t *testing.T) {
+		var buf bytes.Buffer
+		err := Encoder(simple, &buf)
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, buf.String(), ShouldEqual, `japplesauce`)
+	})
+	t.Run("decoding", func(t *testing.T) {
+		buf := bytes.NewBufferString(`japplesauce`)
+		simple2, err := Decoder(ipldfree.NodeBuilder(), buf)
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, simple2, ShouldEqual, simple)
+	})
+}

encoding/dagcbor/unmarshal.go

@@ -24,7 +24,10 @@ var (
 func Unmarshal(nb ipld.NodeBuilder, tokSrc shared.TokenSource) (ipld.Node, error) {
 	var tk tok.Token
 	done, err := tokSrc.Step(&tk)
-	if done || err != nil {
+	if err != nil {
+		return nil, err
+	}
+	if done && !tk.Type.IsValue() {
 		return nil, err
 	}
 	return unmarshal(nb, tokSrc, &tk)

encoding/dagjson/multicodec.go

@@ -37,7 +37,7 @@ func Decoder(nb ipld.NodeBuilder, r io.Reader) (ipld.Node, error) {
 	for {
 		_, err := r.Read(buf[:])
 		switch buf[0] {
-		case ' ', '\t', '\r', '\n': // continue
+		case ' ', 0x0, '\t', '\r', '\n': // continue
 		default:
 			return n, fmt.Errorf("unexpected content after end of json object")
 		}

encoding/dagjson/roundtrip_test.go

@@ -59,3 +59,19 @@ func TestRoundtrip(t *testing.T) {
 		Wish(t, n2, ShouldEqual, n)
 	})
 }
+
+func TestRoundtripSimple(t *testing.T) {
+	simple := fnb.CreateString("applesauce")
+	t.Run("encoding", func(t *testing.T) {
+		var buf bytes.Buffer
+		err := Encoder(simple, &buf)
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, buf.String(), ShouldEqual, `"applesauce"`)
+	})
+	t.Run("decoding", func(t *testing.T) {
+		buf := bytes.NewBufferString(`"applesauce"`)
+		simple2, err := Decoder(ipldfree.NodeBuilder(), buf)
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, simple2, ShouldEqual, simple)
+	})
+}

encoding/dagjson/unmarshal.go

@@ -19,7 +19,10 @@ import (
 func Unmarshal(nb ipld.NodeBuilder, tokSrc shared.TokenSource) (ipld.Node, error) {
 	var tk tok.Token
 	done, err := tokSrc.Step(&tk)
-	if done || err != nil {
+	if err != nil {
+		return nil, err
+	}
+	if done && !tk.Type.IsValue() {
 		return nil, err
 	}
 	return unmarshal(nb, tokSrc, &tk)

encoding/unmarshal.go

@@ -30,7 +30,10 @@ import (
 func Unmarshal(nb ipld.NodeBuilder, tokSrc shared.TokenSource) (ipld.Node, error) {
 	var tk tok.Token
 	done, err := tokSrc.Step(&tk)
-	if done || err != nil {
+	if err != nil {
+		return nil, err
+	}
+	if done && !tk.Type.IsValue() {
 		return nil, err
 	}
 	return unmarshal(nb, tokSrc, &tk)

traversal/selector/selector.go

@@ -107,3 +107,48 @@ func (ps PathSegmentInt) String() string {
 func (ps PathSegmentInt) Index() (int, error) {
 	return ps.I, nil
 }
+
+// SegmentIterator iterates either a list or a map, generating PathSegments
+// instead of indexes or keys
+type SegmentIterator interface {
+	Next() (pathSegment PathSegment, value ipld.Node, err error)
+	Done() bool
+}
+
+// NewSegmentIterator generates a new iterator based on the node type
+func NewSegmentIterator(n ipld.Node) SegmentIterator {
+	if n.ReprKind() == ipld.ReprKind_List {
+		return listSegmentIterator{n.ListIterator()}
+	}
+	return mapSegmentIterator{n.MapIterator()}
+}
+
+type listSegmentIterator struct {
+	ipld.ListIterator
+}
+
+func (lsi listSegmentIterator) Next() (pathSegment PathSegment, value ipld.Node, err error) {
+	i, v, err := lsi.ListIterator.Next()
+	return PathSegmentInt{i}, v, err
+}
+
+func (lsi listSegmentIterator) Done() bool {
+	return lsi.ListIterator.Done()
+}
+
+type mapSegmentIterator struct {
+	ipld.MapIterator
+}
+
+func (msi mapSegmentIterator) Next() (pathSegment PathSegment, value ipld.Node, err error) {
+	k, v, err := msi.MapIterator.Next()
+	if err != nil {
+		return nil, v, err
+	}
+	kstr, _ := k.AsString()
+	return PathSegmentString{kstr}, v, err
+}
+
+func (msi mapSegmentIterator) Done() bool {
+	return msi.MapIterator.Done()
+}

traversal/traverse.go

 package traversal
 
 import (
+	"fmt"
+
 	ipld "github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/traversal/selector"
 )
@@ -48,22 +50,36 @@ func (tp TraversalProgress) traverseInformatively(n ipld.Node, s selector.Select
 	default:
 		return nil
 	}
-	// TODO: should only do this full loop if high-cardinality indicated.
-	//   attn := s.Interests()
-	//   if attn == nil {
-	// FIXME need another kind switch here, and list support!
-	for itr := n.MapIterator(); !itr.Done(); {
-		k, v, err := itr.Next()
+	attn := s.Interests()
+	if attn == nil {
+		return tp.traverseAll(n, s, fn)
+	}
+	return tp.traverseSelective(n, attn, s, fn)
+
+}
+
+func (tp TraversalProgress) traverseAll(n ipld.Node, s selector.Selector, fn AdvVisitFn) error {
+	for itr := selector.NewSegmentIterator(n); !itr.Done(); {
+		ps, v, err := itr.Next()
 		if err != nil {
 			return err
 		}
-		kstr, _ := k.AsString()
-		sNext := s.Explore(n, selector.PathSegmentString{kstr})
+		sNext := s.Explore(n, ps)
 		if sNext != nil {
-			// TODO when link load is implemented, it should go roughly here.
 			tpNext := tp
-			tpNext.Path = tp.Path.AppendSegment(kstr)
-			if err := tpNext.traverseInformatively(v, sNext, fn); err != nil {
+			tpNext.Path = tp.Path.AppendSegment(ps.String())
+			if v.ReprKind() == ipld.ReprKind_Link {
+				lnk, _ := v.AsLink()
+				tpNext.LastBlock.Path = tpNext.Path
+				tpNext.LastBlock.Link = lnk
+				v, err = tpNext.loadLink(v, n)
+				if err != nil {
+					return err
+				}
+			}
+
+			err = tpNext.traverseInformatively(v, sNext, fn)
+			if err != nil {
 				return err
 			}
 		}
@@ -71,6 +87,60 @@ func (tp TraversalProgress) traverseInformatively(n ipld.Node, s selector.Select
 	return nil
 }
 
+func (tp TraversalProgress) traverseSelective(n ipld.Node, attn []selector.PathSegment, s selector.Selector, fn AdvVisitFn) error {
+	for _, ps := range attn {
+		// TODO: Probably not the most efficient way to be doing this...
+		v, err := n.TraverseField(ps.String())
+		if err != nil {
+			continue
+		}
+		sNext := s.Explore(n, ps)
+		if sNext != nil {
+			tpNext := tp
+			tpNext.Path = tp.Path.AppendSegment(ps.String())
+			if v.ReprKind() == ipld.ReprKind_Link {
+				lnk, _ := v.AsLink()
+				tpNext.LastBlock.Path = tpNext.Path
+				tpNext.LastBlock.Link = lnk
+				v, err = tpNext.loadLink(v, n)
+				if err != nil {
+					return err
+				}
+			}
+
+			err = tpNext.traverseInformatively(v, sNext, fn)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (tp TraversalProgress) loadLink(v ipld.Node, parent ipld.Node) (ipld.Node, error) {
+	lnk, err := v.AsLink()
+	if err != nil {
+		return nil, err
+	}
+	// Assemble the LinkContext in case the Loader or NBChooser want it.
+	lnkCtx := ipld.LinkContext{
+		LinkPath:   tp.Path,
+		LinkNode:   v,
+		ParentNode: parent,
+	}
+	// Load link!
+	v, err = lnk.Load(
+		tp.Cfg.Ctx,
+		lnkCtx,
+		tp.Cfg.LinkNodeBuilderChooser(lnk, lnkCtx),
+		tp.Cfg.LinkLoader,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("error traversing node at %q: could not load link %q: %s", tp.Path, lnk, err)
+	}
+	return v, nil
+}
+
 func (tp TraversalProgress) TraverseTransform(n ipld.Node, s selector.Selector, fn TransformFn) (ipld.Node, error) {
 	panic("TODO")
 }

traversal/traverse_test.go

 package traversal_test
 
 import (
+	"bytes"
+	"io"
 	"testing"
 
 	. "github.com/warpfork/go-wish"
 
 	ipld "github.com/ipld/go-ipld-prime"
 	_ "github.com/ipld/go-ipld-prime/encoding/dagjson"
+	"github.com/ipld/go-ipld-prime/fluent"
 	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
 	"github.com/ipld/go-ipld-prime/traversal"
 	"github.com/ipld/go-ipld-prime/traversal/selector"
 	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
-
 )
 
 /* Remember, we've got the following fixtures in scope:
@@ -43,6 +45,7 @@ var (
 
 // covers traverse using a variety of selectors.
 // all cases here use one already-loaded Node; no link-loading exercised.
+
 func TestTraverse(t *testing.T) {
 	ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder())
 	t.Run("traverse selecting true should visit the root", func(t *testing.T) {
@@ -109,4 +112,146 @@ func TestTraverse(t *testing.T) {
 		Wish(t, err, ShouldEqual, nil)
 		Wish(t, order, ShouldEqual, 2)
 	})
+	t.Run("traversing across nodes should work", func(t *testing.T) {
+		ss := ssb.ExploreRecursive(3, ssb.ExploreUnion(
+			ssb.Matcher(),
+			ssb.ExploreAll(ssb.ExploreRecursiveEdge()),
+		))
+		s, err := ss.Selector()
+		var order int
+		err = traversal.TraversalProgress{
+			Cfg: &traversal.TraversalConfig{
+				LinkLoader: func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
+					return bytes.NewBuffer(storage[lnk]), nil
+				},
+			},
+		}.Traverse(middleMapNode, s, func(tp traversal.TraversalProgress, n ipld.Node) error {
+			switch order {
+			case 0:
+				Wish(t, n, ShouldEqual, middleMapNode)
+				Wish(t, tp.Path.String(), ShouldEqual, "")
+			case 1:
+				Wish(t, n, ShouldEqual, fnb.CreateBool(true))
+				Wish(t, tp.Path.String(), ShouldEqual, "foo")
+			case 2:
+				Wish(t, n, ShouldEqual, fnb.CreateBool(false))
+				Wish(t, tp.Path.String(), ShouldEqual, "bar")
+			case 3:
+				Wish(t, n, ShouldEqual, fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) {
+					mb.Insert(knb.CreateString("alink"), vnb.CreateLink(leafAlphaLnk))
+					mb.Insert(knb.CreateString("nonlink"), vnb.CreateString("zoo"))
+				}))
+				Wish(t, tp.Path.String(), ShouldEqual, "nested")
+			case 4:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "nested/alink")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "nested/alink")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+
+			case 5:
+				Wish(t, n, ShouldEqual, fnb.CreateString("zoo"))
+				Wish(t, tp.Path.String(), ShouldEqual, "nested/nonlink")
+			}
+			order++
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, order, ShouldEqual, 6)
+	})
+	t.Run("traversing lists should work", func(t *testing.T) {
+		ss := ssb.ExploreRange(0, 3, ssb.Matcher())
+		s, err := ss.Selector()
+		var order int
+		err = traversal.TraversalProgress{
+			Cfg: &traversal.TraversalConfig{
+				LinkLoader: func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
+					return bytes.NewBuffer(storage[lnk]), nil
+				},
+			},
+		}.Traverse(middleListNode, s, func(tp traversal.TraversalProgress, n ipld.Node) error {
+			switch order {
+			case 0:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "0")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "0")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+			case 1:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "1")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "1")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+			case 2:
+				Wish(t, n, ShouldEqual, fnb.CreateString("beta"))
+				Wish(t, tp.Path.String(), ShouldEqual, "2")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "2")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafBetaLnk.String())
+			}
+			order++
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, order, ShouldEqual, 3)
+	})
+	t.Run("multiple layers of link traversal should work", func(t *testing.T) {
+		ss := ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) {
+			efsb.Insert("linkedList", ssb.ExploreAll(ssb.Matcher()))
+			efsb.Insert("linkedMap", ssb.ExploreRecursive(3, ssb.ExploreFields(func(efsb builder.ExploreFieldsSpecBuilder) {
+				efsb.Insert("foo", ssb.Matcher())
+				efsb.Insert("nonlink", ssb.Matcher())
+				efsb.Insert("alink", ssb.Matcher())
+				efsb.Insert("nested", ssb.ExploreRecursiveEdge())
+			})))
+		})
+		s, err := ss.Selector()
+		var order int
+		err = traversal.TraversalProgress{
+			Cfg: &traversal.TraversalConfig{
+				LinkLoader: func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) {
+					return bytes.NewBuffer(storage[lnk]), nil
+				},
+			},
+		}.Traverse(rootNode, s, func(tp traversal.TraversalProgress, n ipld.Node) error {
+			switch order {
+			case 0:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedList/0")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedList/0")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+			case 1:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedList/1")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedList/1")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+			case 2:
+				Wish(t, n, ShouldEqual, fnb.CreateString("beta"))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedList/2")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedList/2")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafBetaLnk.String())
+			case 3:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedList/3")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedList/3")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+			case 4:
+				Wish(t, n, ShouldEqual, fnb.CreateBool(true))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedMap/foo")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedMap")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, middleMapNodeLnk.String())
+			case 5:
+				Wish(t, n, ShouldEqual, fnb.CreateString("zoo"))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedMap/nested/nonlink")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedMap")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, middleMapNodeLnk.String())
+			case 6:
+				Wish(t, n, ShouldEqual, fnb.CreateString("alpha"))
+				Wish(t, tp.Path.String(), ShouldEqual, "linkedMap/nested/alink")
+				Wish(t, tp.LastBlock.Path.String(), ShouldEqual, "linkedMap/nested/alink")
+				Wish(t, tp.LastBlock.Link.String(), ShouldEqual, leafAlphaLnk.String())
+			}
+			order++
+			return nil
+		})
+		Wish(t, err, ShouldEqual, nil)
+		Wish(t, order, ShouldEqual, 7)
+	})
 }