From 7e69224426292b0adf1706d3ec4dc21ae9b8a67c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Fri, 5 Mar 2021 11:02:57 +0000 Subject: [PATCH] codec/raw: implement the raw codec It's small, it's simple, and it's already widely used as part of unixfs. So there's no reason it shouldn't be part of go-ipld-prime. The codec is tiny, but has three noteworthy parts: the Encode and Decode funcs, the cidlink multicodec registration, and the Bytes method shortcut. Each of these has its own dedicated regression test. I'm also using this commit to showcase the use of quicktest instead of go-wish. The result is extremely similar, but with less dot-import magic. For example, if I remove the Bytes shortcut in Decode: --- FAIL: TestDecodeBuffer (0.00s) codec_test.go:115: error: got non-nil error got: e"could not decode raw node: must not call Read" stack: /home/mvdan/src/ipld/codec/raw/codec_test.go:115 qt.Assert(t, err, qt.IsNil) --- codec/raw/codec.go | 63 ++++++++++++++++++++++ codec/raw/codec_test.go | 116 ++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 11 ++++ 4 files changed, 191 insertions(+) create mode 100644 codec/raw/codec.go create mode 100644 codec/raw/codec_test.go diff --git a/codec/raw/codec.go b/codec/raw/codec.go new file mode 100644 index 0000000..126a573 --- /dev/null +++ b/codec/raw/codec.go @@ -0,0 +1,63 @@ +// Package raw implements IPLD's raw codec, which simply writes and reads a Node +// which can be represented as bytes. +// +// The codec can be used with any node which supports AsBytes and AssignBytes. +// In general, it only makes sense to use this codec on a plain "bytes" node +// such as github.com/ipld/go-ipld-prime/node/basic.Prototype.Bytes. +package raw + +import ( + "fmt" + "io" + "io/ioutil" + + ipld "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +// TODO(mvdan): make go-ipld-prime use go-multicodec soon +const rawMulticodec = 0x55 + +func init() { + cidlink.RegisterMulticodecDecoder(rawMulticodec, Decode) + cidlink.RegisterMulticodecEncoder(rawMulticodec, Encode) +} + +// Decode implements decoding of a node with the raw codec. +// +// Note that if r has a Bytes method, such as is the case with *bytes.Buffer, we +// will use those bytes directly to save having to allocate and copy them. The +// Node interface is defined as immutable, so it is assumed that its bytes won't +// be modified in-place. Similarly, we assume that the incoming buffer's bytes +// won't get modified in-place later. +// +// To disable the shortcut above, hide the Bytes method by wrapping the buffer +// with an io.Reader: +// +// Decode([...], struct{io.Reader}{buf}) +func Decode(am ipld.NodeAssembler, r io.Reader) error { + var data []byte + if buf, ok := r.(interface{ Bytes() []byte }); ok { + data = buf.Bytes() + } else { + var err error + data, err = ioutil.ReadAll(r) + if err != nil { + return fmt.Errorf("could not decode raw node: %v", err) + } + } + return am.AssignBytes(data) +} + +// Encode implements encoding of a node with the raw codec. +// +// Note that Encode won't copy the node's bytes as returned by AsBytes, but the +// call to Write will typically have to copy the bytes anyway. +func Encode(node ipld.Node, w io.Writer) error { + data, err := node.AsBytes() + if err != nil { + return err + } + _, err = w.Write(data) + return err +} diff --git a/codec/raw/codec_test.go b/codec/raw/codec_test.go new file mode 100644 index 0000000..accc09f --- /dev/null +++ b/codec/raw/codec_test.go @@ -0,0 +1,116 @@ +package raw + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/ipfs/go-cid" + ipld "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" +) + +var tests = []struct { + name string + data []byte +}{ + {"Empty", nil}, + {"Plaintext", []byte("hello there")}, + {"JSON", []byte(`{"foo": "bar"}`)}, + {"NullBytes", []byte("\x00\x00")}, +} + +func TestRoundtrip(t *testing.T) { + t.Parallel() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + nb := basicnode.Prototype.Bytes.NewBuilder() + r := bytes.NewBuffer(test.data) + + err := Decode(nb, r) + qt.Assert(t, err, qt.IsNil) + node := nb.Build() + + buf := new(bytes.Buffer) + err = Encode(node, buf) + qt.Assert(t, err, qt.IsNil) + + qt.Assert(t, buf.Bytes(), qt.DeepEquals, test.data) + }) + } +} + +func TestRoundtripCidlink(t *testing.T) { + t.Parallel() + + lb := cidlink.LinkBuilder{Prefix: cid.Prefix{ + Version: 1, + Codec: rawMulticodec, + MhType: 0x17, + MhLength: 4, + }} + node := basicnode.NewBytes([]byte("hello there")) + + buf := bytes.Buffer{} + lnk, err := lb.Build(context.Background(), ipld.LinkContext{}, node, + func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) { + return &buf, func(lnk ipld.Link) error { return nil }, nil + }, + ) + qt.Assert(t, err, qt.IsNil) + + nb := basicnode.Prototype__Any{}.NewBuilder() + err = lnk.Load(context.Background(), ipld.LinkContext{}, nb, + func(lnk ipld.Link, _ ipld.LinkContext) (io.Reader, error) { + return bytes.NewReader(buf.Bytes()), nil + }, + ) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, nb.Build(), qt.DeepEquals, node) +} + +// mustOnlyUseRead only exposes Read, hiding Bytes. +type mustOnlyUseRead struct { + buf *bytes.Buffer +} + +func (r mustOnlyUseRead) Read(p []byte) (int, error) { + return r.buf.Read(p) +} + +// mustNotUseRead exposes Bytes and makes Read always error. +type mustNotUseRead struct { + buf *bytes.Buffer +} + +func (r mustNotUseRead) Read(p []byte) (int, error) { + return 0, fmt.Errorf("must not call Read") +} + +func (r mustNotUseRead) Bytes() []byte { + return r.buf.Bytes() +} + +func TestDecodeBuffer(t *testing.T) { + t.Parallel() + + var err error + buf := bytes.NewBuffer([]byte("hello there")) + + err = Decode( + basicnode.Prototype.Bytes.NewBuilder(), + mustOnlyUseRead{buf}, + ) + qt.Assert(t, err, qt.IsNil) + + err = Decode( + basicnode.Prototype.Bytes.NewBuilder(), + mustNotUseRead{buf}, + ) + qt.Assert(t, err, qt.IsNil) +} diff --git a/go.mod b/go.mod index 8d64288..b42b784 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ipld/go-ipld-prime go 1.14 require ( + github.com/frankban/quicktest v1.11.3 github.com/ipfs/go-cid v0.0.4 github.com/minio/sha256-simd v0.1.1 // indirect github.com/mr-tron/base58 v1.1.3 // indirect diff --git a/go.sum b/go.sum index bc17d60..25c92d9 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,18 @@ +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -41,3 +50,5 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgm golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -- GitLab