Merge pull request #152 from ipld/codec/json

add non-dag json codec

adl/rot13adl/example_test.go

@@ -20,7 +20,7 @@ func ExampleUnmarshallingToADL() {
 	nb := rot13adl.Prototype.SubstrateRoot.NewBuilder()
 
 	// Unmarshal -- using the substrate's nodebuilder just like you'd unmarshal with any other nodebuilder.
-	err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(`"n pbby fgevat"`)))
+	err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(`"n pbby fgevat"`)), true)
 	fmt.Printf("unmarshal error: %v\n", err)
 
 	// Use `Reify` to get the synthetic high-level view of the ADL data.
@@ -59,7 +59,7 @@ func ExampleCreatingViaADL() {
 
 	// To marshal the ADL, just use marshal methods on its substrate as normal:
 	var marshalBuffer bytes.Buffer
-	err := dagjson.Marshal(substrateNode, json.NewEncoder(&marshalBuffer, json.EncodeOptions{}))
+	err := dagjson.Marshal(substrateNode, json.NewEncoder(&marshalBuffer, json.EncodeOptions{}), true)
 	fmt.Printf("marshalled: %v\n", marshalBuffer.String())
 	fmt.Printf("marshal error: %v\n", err)
 

codec/dagjson/marshal.go

@@ -14,7 +14,7 @@ import (
 // except for the `case ipld.Kind_Link` block,
 // which is dag-json's special sauce for schemafree links.
 
-func Marshal(n ipld.Node, sink shared.TokenSink) error {
+func Marshal(n ipld.Node, sink shared.TokenSink, allowLinks bool) error {
 	var tk tok.Token
 	switch n.Kind() {
 	case ipld.Kind_Invalid:
@@ -44,7 +44,7 @@ func Marshal(n ipld.Node, sink shared.TokenSink) error {
 			if _, err := sink.Step(&tk); err != nil {
 				return err
 			}
-			if err := Marshal(v, sink); err != nil {
+			if err := Marshal(v, sink, allowLinks); err != nil {
 				return err
 			}
 		}
@@ -66,7 +66,7 @@ func Marshal(n ipld.Node, sink shared.TokenSink) error {
 			if err != nil {
 				return err
 			}
-			if err := Marshal(v, sink); err != nil {
+			if err := Marshal(v, sink, allowLinks); err != nil {
 				return err
 			}
 		}
@@ -120,6 +120,9 @@ func Marshal(n ipld.Node, sink shared.TokenSink) error {
 		_, err = sink.Step(&tk)
 		return err
 	case ipld.Kind_Link:
+		if !allowLinks {
+			return fmt.Errorf("cannot Marshal ipld links to JSON")
+		}
 		v, err := n.AsLink()
 		if err != nil {
 			return err

codec/dagjson/multicodec.go

@@ -21,9 +21,7 @@ func init() {
 }
 
 func Decode(na ipld.NodeAssembler, r io.Reader) error {
-	// Shell out directly to generic builder path.
-	//  (There's not really any fastpaths of note for json.)
-	err := Unmarshal(na, json.NewDecoder(r))
+	err := Unmarshal(na, json.NewDecoder(r), true)
 	if err != nil {
 		return err
 	}
@@ -49,7 +47,6 @@ func Decode(na ipld.NodeAssembler, r io.Reader) error {
 			return err
 		}
 	}
-	return err
 }
 
 func Encode(n ipld.Node, w io.Writer) error {
@@ -59,5 +56,5 @@ func Encode(n ipld.Node, w io.Writer) error {
 	return Marshal(n, json.NewEncoder(w, json.EncodeOptions{
 		Line:   []byte{'\n'},
 		Indent: []byte{'\t'},
-	}))
+	}), true)
 }

codec/dagjson/unmarshal.go

@@ -19,8 +19,9 @@ import (
 //       several steps of handling maps, because it necessitates peeking several
 //        tokens before deciding what kind of value to create).
 
-func Unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSource) error {
+func Unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSource, parseLinks bool) error {
 	var st unmarshalState
+	st.parseLinks = parseLinks
 	done, err := tokSrc.Step(&st.tk[0])
 	if err != nil {
 		return err
@@ -32,8 +33,9 @@ func Unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSource) error {
 }
 
 type unmarshalState struct {
-	tk    [4]tok.Token // mostly, only 0'th is used... but [1:4] are used during lookahead for links.
-	shift int          // how many times to slide something out of tk[1:4] instead of getting a new token.
+	tk         [4]tok.Token // mostly, only 0'th is used... but [1:4] are used during lookahead for links.
+	shift      int          // how many times to slide something out of tk[1:4] instead of getting a new token.
+	parseLinks bool
 }
 
 // step leaves a "new" token in tk[0],
@@ -120,7 +122,7 @@ func (st *unmarshalState) linkLookahead(na ipld.NodeAssembler, tokSrc shared.Tok
 	if err != nil {
 		return false, err
 	}
-	if err := na.AssignLink(cidlink.Link{elCid}); err != nil {
+	if err := na.AssignLink(cidlink.Link{Cid: elCid}); err != nil {
 		return false, err
 	}
 	return true, nil
@@ -135,12 +137,14 @@ func (st *unmarshalState) unmarshal(na ipld.NodeAssembler, tokSrc shared.TokenSo
 	case tok.TMapOpen:
 		// dag-json has special needs: we pump a few tokens ahead to look for dag-json's "link" pattern.
 		//  We can't actually call BeginMap until we're sure it's not gonna turn out to be a link.
-		gotLink, err := st.linkLookahead(na, tokSrc)
-		if err != nil { // return in error if any token peeks failed or if structure looked like a link but failed to parse as CID.
-			return err
-		}
-		if gotLink {
-			return nil
+		if st.parseLinks {
+			gotLink, err := st.linkLookahead(na, tokSrc)
+			if err != nil { // return in error if any token peeks failed or if structure looked like a link but failed to parse as CID.
+				return err
+			}
+			if gotLink {
+				return nil
+			}
 		}
 
 		// Okay, now back to regularly scheduled map logic.

codec/json/multicodec.go

+package json
+
+import (
+	"fmt"
+	"io"
+
+	rfmtjson "github.com/polydawn/refmt/json"
+
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/codec/dagjson"
+	"github.com/ipld/go-ipld-prime/multicodec"
+)
+
+var (
+	_ ipld.Decoder = Decode
+	_ ipld.Encoder = Encode
+)
+
+func init() {
+	multicodec.RegisterEncoder(0x0200, Encode)
+	multicodec.RegisterDecoder(0x0200, Decode)
+}
+
+func Decode(na ipld.NodeAssembler, r io.Reader) error {
+	// Shell out directly to generic builder path.
+	//  (There's not really any fastpaths of note for json.)
+	err := dagjson.Unmarshal(na, rfmtjson.NewDecoder(r), false)
+	if err != nil {
+		return err
+	}
+	// Slurp any remaining whitespace.
+	//  (This is relevant if our reader is tee'ing bytes to a hasher, and
+	//   the json contained any trailing whitespace.)
+	//  (We can't actually support multiple objects per reader from here;
+	//   we can't unpeek if we find a non-whitespace token, so our only
+	//    option is to error if this reader seems to contain more content.)
+	var buf [1]byte
+	for {
+		_, err := r.Read(buf[:])
+		switch buf[0] {
+		case ' ', 0x0, '\t', '\r', '\n': // continue
+		default:
+			return fmt.Errorf("unexpected content after end of json object")
+		}
+		if err == nil {
+			continue
+		} else if err == io.EOF {
+			return nil
+		} else {
+			return err
+		}
+	}
+}
+
+func Encode(n ipld.Node, w io.Writer) error {
+	// Shell out directly to generic inspection path.
+	//  (There's not really any fastpaths of note for json.)
+	// Write another function if you need to tune encoding options about whitespace.
+	return dagjson.Marshal(n, rfmtjson.NewEncoder(w, rfmtjson.EncodeOptions{
+		Line:   []byte{'\n'},
+		Indent: []byte{'\t'},
+	}), false)
+}

codec/jst/jst.go

@@ -168,8 +168,8 @@ func (tab *table) Finalize() {
 	var buf bytes.Buffer
 	for _, cn := range cols {
 		buf.Reset()
-		dagjson.Marshal(basicnode.NewString(string(cn)), json.NewEncoder(&buf, json.EncodeOptions{})) // FIXME this would be a lot less irritating if we had more plumbing access to the json encoding -- we want to encode exactly one string into a buffer, it literally can't error.
-		tab.keySize[cn] = buf.Len()                                                                   // FIXME this is ignoring charsets, renderable glyphs, etc at present.
+		dagjson.Marshal(basicnode.NewString(string(cn)), json.NewEncoder(&buf, json.EncodeOptions{}), false) // FIXME this would be a lot less irritating if we had more plumbing access to the json encoding -- we want to encode exactly one string into a buffer, it literally can't error.
+		tab.keySize[cn] = buf.Len()                                                                          // FIXME this is ignoring charsets, renderable glyphs, etc at present.
 	}
 }
 
@@ -299,7 +299,7 @@ func marshal(ctx *state, n ipld.Node, w io.Writer) error {
 func marshalPlain(ctx *state, n ipld.Node, w io.Writer) error {
 	err := dagjson.Marshal(n, json.NewEncoder(w, json.EncodeOptions{
 		// never indent here: these values will always end up being emitted mid-line.
-	}))
+	}), true)
 	if err != nil {
 		return recordErrorPosition(ctx, err)
 	}
@@ -470,7 +470,7 @@ func emitKey(ctx *state, k ipld.Node, w io.Writer) error {
 	if ctx.cfg.Color.Enabled {
 		w.Write(ctx.cfg.Color.KeyHighlight)
 	}
-	if err := dagjson.Marshal(k, json.NewEncoder(w, json.EncodeOptions{})); err != nil {
+	if err := dagjson.Marshal(k, json.NewEncoder(w, json.EncodeOptions{}), true); err != nil {
 		return recordErrorPosition(ctx, err)
 	}
 	if ctx.cfg.Color.Enabled {

schema/gen/go/testcase_test.go

@@ -190,7 +190,7 @@ func (tcase testcase) Test(t *testing.T, np, npr ipld.NodePrototype) {
 func testUnmarshal(t *testing.T, np ipld.NodePrototype, data string, expectFail error) ipld.Node {
 	t.Helper()
 	nb := np.NewBuilder()
-	err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(data)))
+	err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(data)), true)
 	switch {
 	case expectFail == nil && err != nil:
 		t.Fatalf("fixture parse failed: %s", err)
@@ -209,7 +209,7 @@ func testMarshal(t *testing.T, n ipld.Node, data string) {
 	// We'll marshal with "pretty" linebreaks and indents (and re-format the fixture to the same) for better diffing.
 	prettyprint := json.EncodeOptions{Line: []byte{'\n'}, Indent: []byte{'\t'}}
 	var buf bytes.Buffer
-	err := dagjson.Marshal(n, json.NewEncoder(&buf, prettyprint))
+	err := dagjson.Marshal(n, json.NewEncoder(&buf, prettyprint), true)
 	if err != nil {
 		t.Errorf("marshal failed: %s", err)
 	}

schema/schema2/typesystem_test.go

@@ -206,7 +206,7 @@ func testParse(t *testing.T, schemajson string, expectParseErr error, expectType
 
 func parseSchema(schemajson string) (schemadmt.Schema, error) {
 	nb := schemadmt.Type.Schema__Repr.NewBuilder()
-	if err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(schemajson))); err != nil {
+	if err := dagjson.Unmarshal(nb, json.NewDecoder(strings.NewReader(schemajson)), true); err != nil {
 		return nil, err
 	}
 	return nb.Build().(schemadmt.Schema), nil