Merge branch 'port-2021-08-24'

LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2019 Jeromy Johnson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Makefile

+gentest:
+	rm -rf ./testing/cbor_gen.go ./testing/cbor_map_gen.go
+	go run ./testgen/main.go
+.PHONY: gentest
+
+test: gentest
+	go test ./...
+.PHONY: test

README.md

 # cbor-gen
 
-dms3 github.com/whyrusleeping/cbor-gen which has dependency on go-cid
\ No newline at end of file
+Some basic utilities to generate fast path cbor codecs for your types.
+
+## License
+MIT

cbor_cid.go

+package typegen
+
+import (
+	"io"
+
+	cid "gitlab.dms3.io/dms3/go-cid"
+)
+
+type CborCid cid.Cid
+
+func (c CborCid) MarshalCBOR(w io.Writer) error {
+	return WriteCid(w, cid.Cid(c))
+}
+
+func (c *CborCid) UnmarshalCBOR(r io.Reader) error {
+	oc, err := ReadCid(r)
+	if err != nil {
+		return err
+	}
+	*c = CborCid(oc)
+	return nil
+}

go.mod

+module gitlab.dms3.io/dms3/cbor-gen
+
+go 1.15
+
+require (
+	github.com/google/go-cmp v0.4.0
+	gitlab.dms3.io/dms3/go-cid v0.0.4
+	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+)

go.sum

+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
+github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM=
+github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
+github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
+github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+gitlab.dms3.io/dms3/go-cid v0.0.4 h1:exbA6/YstDUVBPDOtSW/ue+iIrb5nM9/7nfsc8rvcGA=
+gitlab.dms3.io/dms3/go-cid v0.0.4/go.mod h1:WLJV2nnU0k7PBj+758e8SGAZj/wklZQZfq1BUwwZbOA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

package.go

+package typegen
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+)
+
+var (
+	knownPackageNamesMu sync.Mutex
+	pkgNameToPkgPath    = make(map[string]string)
+	pkgPathToPkgName    = make(map[string]string)
+
+	defaultImports = []Import{
+		{Name: "cbg", PkgPath: "gitlab.dms3.io/dms3/cbor-gen"},
+		{Name: "xerrors", PkgPath: "golang.org/x/xerrors"},
+		{Name: "cid", PkgPath: "gitlab.dms3.io/dms3/go-cid"},
+	}
+)
+
+func init() {
+	for _, imp := range defaultImports {
+		if was, conflict := pkgNameToPkgPath[imp.Name]; conflict {
+			panic(fmt.Sprintf("reused pkg name %s for %s and %s", imp.Name, imp.PkgPath, was))
+		}
+		if _, conflict := pkgPathToPkgName[imp.Name]; conflict {
+			panic(fmt.Sprintf("duplicate default import %s", imp.PkgPath))
+		}
+		pkgNameToPkgPath[imp.Name] = imp.PkgPath
+		pkgPathToPkgName[imp.PkgPath] = imp.Name
+	}
+}
+
+func resolvePkgName(path, typeName string) string {
+	parts := strings.Split(typeName, ".")
+	if len(parts) != 2 {
+		panic(fmt.Sprintf("expected type to have a package name: %s", typeName))
+	}
+	defaultName := parts[0]
+
+	knownPackageNamesMu.Lock()
+	defer knownPackageNamesMu.Unlock()
+
+	// Check for a known name and use it.
+	if name, ok := pkgPathToPkgName[path]; ok {
+		return name
+	}
+
+	// Allocate a name.
+	for i := 0; ; i++ {
+		tryName := defaultName
+		if i > 0 {
+			tryName = fmt.Sprintf("%s%d", defaultName, i)
+		}
+		if _, taken := pkgNameToPkgPath[tryName]; !taken {
+			pkgNameToPkgPath[tryName] = path
+			pkgPathToPkgName[path] = tryName
+			return tryName
+		}
+	}
+
+}
+
+type Import struct {
+	Name, PkgPath string
+}
+
+func ImportsForType(currPkg string, t reflect.Type) []Import {
+	switch t.Kind() {
+	case reflect.Array, reflect.Slice, reflect.Ptr:
+		return ImportsForType(currPkg, t.Elem())
+	case reflect.Map:
+		return dedupImports(append(ImportsForType(currPkg, t.Key()), ImportsForType(currPkg, t.Elem())...))
+	default:
+		path := t.PkgPath()
+		if path == "" || path == currPkg {
+			// built-in or in current package.
+			return nil
+		}
+
+		return []Import{{PkgPath: path, Name: resolvePkgName(path, t.String())}}
+	}
+}
+
+func dedupImports(imps []Import) []Import {
+	impSet := make(map[string]string, len(imps))
+	for _, imp := range imps {
+		impSet[imp.PkgPath] = imp.Name
+	}
+	deduped := make([]Import, 0, len(imps))
+	for pkg, name := range impSet {
+		deduped = append(deduped, Import{Name: name, PkgPath: pkg})
+	}
+	sort.Slice(deduped, func(i, j int) bool {
+		return deduped[i].PkgPath < deduped[j].PkgPath
+	})
+	return deduped
+}

peeker.go

+package typegen
+
+import (
+	"bufio"
+	"io"
+)
+
+// BytePeeker combines the Reader and ByteScanner interfaces.
+type BytePeeker interface {
+	io.Reader
+	io.ByteScanner
+}
+
+func GetPeeker(r io.Reader) BytePeeker {
+	if r, ok := r.(BytePeeker); ok {
+		return r
+	}
+	return &peeker{reader: r}
+}
+
+// peeker is a non-buffering BytePeeker.
+type peeker struct {
+	reader    io.Reader
+	peekState int
+	lastByte  byte
+}
+
+const (
+	peekEmpty = iota
+	peekSet
+	peekUnread
+)
+
+func (p *peeker) Read(buf []byte) (n int, err error) {
+	// Read "nothing". I.e., read an error, maybe.
+	if len(buf) == 0 {
+		// There's something pending in the
+		if p.peekState == peekUnread {
+			return 0, nil
+		}
+		return p.reader.Read(nil)
+	}
+
+	if p.peekState == peekUnread {
+		buf[0] = p.lastByte
+		n, err = p.reader.Read(buf[1:])
+		n += 1
+	} else {
+		n, err = p.reader.Read(buf)
+	}
+	if n > 0 {
+		p.peekState = peekSet
+		p.lastByte = buf[n-1]
+	}
+	return n, err
+}
+
+func (p *peeker) ReadByte() (byte, error) {
+	if p.peekState == peekUnread {
+		p.peekState = peekSet
+		return p.lastByte, nil
+	}
+	var buf [1]byte
+	n, err := p.reader.Read(buf[:])
+	if n == 0 {
+		return 0, err
+	}
+	b := buf[0]
+	p.lastByte = b
+	p.peekState = peekSet
+	return b, err
+}
+
+func (p *peeker) UnreadByte() error {
+	if p.peekState != peekSet {
+		return bufio.ErrInvalidUnreadByte
+	}
+	p.peekState = peekUnread
+	return nil
+}

peeker_test.go

+package typegen
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"testing"
+)
+
+func TestPeeker(t *testing.T) {
+	buf := bytes.NewBuffer([]byte{0, 1, 2, 3})
+	p := peeker{reader: buf}
+	n, err := p.Read(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 0 {
+		t.Fatal(err)
+	}
+
+	err = p.UnreadByte()
+	if err != bufio.ErrInvalidUnreadByte {
+		t.Fatal(err)
+	}
+
+	// read 2 bytes
+	var out [2]byte
+	n, err = p.Read(out[:])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("expected 2 bytes, got %d", n)
+	}
+	if !bytes.Equal(out[:], []byte{0, 1}) {
+		t.Fatalf("unexpected output")
+	}
+
+	// unread that last byte and read it again.
+	err = p.UnreadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := p.ReadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if b != 1 {
+		t.Fatal("expected 1")
+	}
+
+	// unread that last byte then read 2
+	err = p.UnreadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = p.Read(out[:])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("expected 2 bytes, got %d", n)
+	}
+	if !bytes.Equal(out[:], []byte{1, 2}) {
+		t.Fatalf("unexpected output")
+	}
+
+	// read another byte
+	b, err = p.ReadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if b != 3 {
+		t.Fatal("expected 1")
+	}
+
+	// Should read eof at end.
+	n, err = p.Read(out[:])
+	if err != io.EOF {
+		t.Fatal(err)
+	}
+	if n != 0 {
+		t.Fatal("should have been at end")
+	}
+	// should unread eof
+	err = p.UnreadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = p.Read(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b, err = p.ReadByte()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if b != 3 {
+		t.Fatal("expected 1")
+	}
+}

testgen/main.go

+package main
+
+import (
+	cbg "gitlab.dms3.io/dms3/cbor-gen"
+	types "gitlab.dms3.io/dms3/cbor-gen/testing"
+)
+
+func main() {
+	if err := cbg.WriteTupleEncodersToFile("testing/cbor_gen.go", "testing",
+		types.SignedArray{},
+		types.SimpleTypeOne{},
+		types.SimpleTypeTwo{},
+		types.DeferredContainer{},
+		types.FixedArrays{},
+		types.ThingWithSomeTime{},
+	); err != nil {
+		panic(err)
+	}
+
+	if err := cbg.WriteMapEncodersToFile("testing/cbor_map_gen.go", "testing",
+		types.SimpleTypeTree{},
+		types.NeedScratchForMap{},
+		types.SimpleStructV1{},
+		types.SimpleStructV2{},
+	); err != nil {
+		panic(err)
+	}
+}

testing/bench_test.go

+package testing
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"reflect"
+	"testing"
+	"testing/quick"
+
+	cbg "gitlab.dms3.io/dms3/cbor-gen"
+	"gitlab.dms3.io/dms3/go-cid"
+)
+
+func BenchmarkMarshaling(b *testing.B) {
+	r := rand.New(rand.NewSource(56887))
+	val, ok := quick.Value(reflect.TypeOf(SimpleTypeTwo{}), r)
+	if !ok {
+		b.Fatal("failed to construct type")
+	}
+
+	tt := val.Interface().(SimpleTypeTwo)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if err := tt.MarshalCBOR(ioutil.Discard); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkUnmarshaling(b *testing.B) {
+	r := rand.New(rand.NewSource(123456))
+	val, ok := quick.Value(reflect.TypeOf(SimpleTypeTwo{}), r)
+	if !ok {
+		b.Fatal("failed to construct type")
+	}
+
+	tt := val.Interface().(SimpleTypeTwo)
+
+	buf := new(bytes.Buffer)
+	if err := tt.MarshalCBOR(buf); err != nil {
+		b.Fatal(err)
+	}
+
+	reader := bytes.NewReader(buf.Bytes())
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		reader.Seek(0, io.SeekStart)
+		var tt SimpleTypeTwo
+		if err := tt.UnmarshalCBOR(reader); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+}
+
+func BenchmarkLinkScan(b *testing.B) {
+	r := rand.New(rand.NewSource(123456))
+	val, ok := quick.Value(reflect.TypeOf(SimpleTypeTwo{}), r)
+	if !ok {
+		b.Fatal("failed to construct type")
+	}
+
+	tt := val.Interface().(SimpleTypeTwo)
+
+	buf := new(bytes.Buffer)
+	if err := tt.MarshalCBOR(buf); err != nil {
+		b.Fatal(err)
+	}
+
+	reader := bytes.NewReader(buf.Bytes())
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		reader.Seek(0, io.SeekStart)
+		if err := cbg.ScanForLinks(reader, func(cid.Cid) {}); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkDeferred(b *testing.B) {
+	r := rand.New(rand.NewSource(123456))
+	val, ok := quick.Value(reflect.TypeOf(SimpleTypeTwo{}), r)
+	if !ok {
+		b.Fatal("failed to construct type")
+	}
+
+	tt := val.Interface().(SimpleTypeTwo)
+
+	buf := new(bytes.Buffer)
+	if err := tt.MarshalCBOR(buf); err != nil {
+		b.Fatal(err)
+	}
+
+	var (
+		deferred cbg.Deferred
+		reader   = bytes.NewReader(buf.Bytes())
+	)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		reader.Seek(0, io.SeekStart)
+		if err := deferred.UnmarshalCBOR(reader); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

testing/roundtrip_test.go

+package testing
+
+import (
+	"bytes"
+	"encoding/json"
+	"math/rand"
+	"reflect"
+	"testing"
+	"testing/quick"
+	"time"
+
+	"gitlab.dms3.io/dms3/go-cid"
+
+	"github.com/google/go-cmp/cmp"
+	cbg "gitlab.dms3.io/dms3/cbor-gen"
+)
+
+var alwaysEqual = cmp.Comparer(func(_, _ interface{}) bool { return true })
+
+// This option handles slices and maps of any type.
+var alwaysEqualOpt = cmp.FilterValues(func(x, y interface{}) bool {
+	vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
+	return (vx.IsValid() && vy.IsValid() && vx.Type() == vy.Type()) &&
+		(vx.Kind() == reflect.Slice || vx.Kind() == reflect.Map) &&
+		(vx.Len() == 0 && vy.Len() == 0)
+}, alwaysEqual)
+
+func TestSimpleSigned(t *testing.T) {
+	testTypeRoundtrips(t, reflect.TypeOf(SignedArray{}))
+}
+
+func TestSimpleTypeOne(t *testing.T) {
+	testTypeRoundtrips(t, reflect.TypeOf(SimpleTypeOne{}))
+}
+
+func TestSimpleTypeTwo(t *testing.T) {
+	testTypeRoundtrips(t, reflect.TypeOf(SimpleTypeTwo{}))
+}
+
+func TestSimpleTypeTree(t *testing.T) {
+	testTypeRoundtrips(t, reflect.TypeOf(SimpleTypeTree{}))
+}
+
+func TestNeedScratchForMap(t *testing.T) {
+	testTypeRoundtrips(t, reflect.TypeOf(NeedScratchForMap{}))
+}
+
+func testValueRoundtrip(t *testing.T, obj cbg.CBORMarshaler, nobj cbg.CBORUnmarshaler) {
+
+	buf := new(bytes.Buffer)
+	if err := obj.MarshalCBOR(buf); err != nil {
+		t.Fatal("i guess its fine to fail marshaling")
+	}
+
+	enc := buf.Bytes()
+
+	if err := nobj.UnmarshalCBOR(bytes.NewReader(enc)); err != nil {
+		t.Logf("got bad bytes: %x", enc)
+		t.Fatal("failed to round trip object: ", err)
+	}
+
+	if !cmp.Equal(obj, nobj, alwaysEqualOpt) {
+		t.Logf("%#v != %#v", obj, nobj)
+		t.Log("not equal after round trip!")
+	}
+
+	nbuf := new(bytes.Buffer)
+	if err := nobj.(cbg.CBORMarshaler).MarshalCBOR(nbuf); err != nil {
+		t.Fatal("failed to remarshal object: ", err)
+	}
+
+	if !bytes.Equal(nbuf.Bytes(), enc) {
+		t.Fatalf("objects encodings different: %x != %x", nbuf.Bytes(), enc)
+	}
+
+}
+
+func testTypeRoundtrips(t *testing.T, typ reflect.Type) {
+	r := rand.New(rand.NewSource(56887))
+	for i := 0; i < 1000; i++ {
+		val, ok := quick.Value(typ, r)
+		if !ok {
+			t.Fatal("failed to generate test value")
+		}
+
+		obj := val.Addr().Interface().(cbg.CBORMarshaler)
+		nobj := reflect.New(typ).Interface().(cbg.CBORUnmarshaler)
+		testValueRoundtrip(t, obj, nobj)
+	}
+}
+
+func TestDeferredContainer(t *testing.T) {
+	zero := &DeferredContainer{}
+	recepticle := &DeferredContainer{}
+	testValueRoundtrip(t, zero, recepticle)
+}
+
+func TestNilValueDeferredUnmarshaling(t *testing.T) {
+	var zero DeferredContainer
+	zero.Deferred = &cbg.Deferred{Raw: []byte{0xf6}}
+
+	buf := new(bytes.Buffer)
+	if err := zero.MarshalCBOR(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	var n DeferredContainer
+	if err := n.UnmarshalCBOR(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	if n.Deferred == nil {
+		t.Fatal("shouldnt be nil!")
+	}
+}
+
+func TestFixedArrays(t *testing.T) {
+	zero := &FixedArrays{}
+	recepticle := &FixedArrays{}
+	testValueRoundtrip(t, zero, recepticle)
+}
+
+func TestTimeIsh(t *testing.T) {
+	val := &ThingWithSomeTime{
+		When:    cbg.CborTime(time.Now()),
+		Stuff:   1234,
+		CatName: "hank",
+	}
+
+	buf := new(bytes.Buffer)
+	if err := val.MarshalCBOR(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	out := ThingWithSomeTime{}
+	if err := out.UnmarshalCBOR(buf); err != nil {
+		t.Fatal(err)
+	}
+
+	if out.When.Time().UnixNano() != val.When.Time().UnixNano() {
+		t.Fatal("time didnt round trip properly", out.When.Time(), val.When.Time())
+	}
+
+	if out.Stuff != val.Stuff {
+		t.Fatal("no")
+	}
+
+	if out.CatName != val.CatName {
+		t.Fatal("no")
+	}
+
+	b, err := json.Marshal(val)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var out2 ThingWithSomeTime
+	if err := json.Unmarshal(b, &out2); err != nil {
+		t.Fatal(err)
+	}
+
+	if out2.When != out.When {
+		t.Fatal(err)
+	}
+
+}
+
+func TestLessToMoreFieldsRoundTrip(t *testing.T) {
+	dummyCid, _ := cid.Parse("bafkqaaa")
+	simpleTypeOne := SimpleTypeOne{
+		Foo:     "foo",
+		Value:   1,
+		Binary:  []byte("bin"),
+		Signed:  -1,
+		NString: "namedstr",
+	}
+	obj := &SimpleStructV1{
+		OldStr:    "hello",
+		OldBytes:  []byte("bytes"),
+		OldNum:    10,
+		OldPtr:    &dummyCid,
+		OldMap:    map[string]SimpleTypeOne{"first": simpleTypeOne},
+		OldArray:  []SimpleTypeOne{simpleTypeOne},
+		OldStruct: simpleTypeOne,
+	}
+
+	buf := new(bytes.Buffer)
+	if err := obj.MarshalCBOR(buf); err != nil {
+		t.Fatal("failed marshaling", err)
+	}
+
+	enc := buf.Bytes()
+
+	nobj := SimpleStructV2{}
+	if err := nobj.UnmarshalCBOR(bytes.NewReader(enc)); err != nil {
+		t.Logf("got bad bytes: %x", enc)
+		t.Fatal("failed to round trip object: ", err)
+	}
+
+	if obj.OldStr != nobj.OldStr {
+		t.Fatal("mismatch ", obj.OldStr, " != ", nobj.OldStr)
+	}
+	if nobj.NewStr != "" {
+		t.Fatal("expected field to be zero value")
+	}
+
+	if obj.OldNum != nobj.OldNum {
+		t.Fatal("mismatch ", obj.OldNum, " != ", nobj.OldNum)
+	}
+	if nobj.NewNum != 0 {
+		t.Fatal("expected field to be zero value")
+	}
+
+	if !bytes.Equal(obj.OldBytes, nobj.OldBytes) {
+		t.Fatal("mismatch ", obj.OldBytes, " != ", nobj.OldBytes)
+	}
+	if nobj.NewBytes != nil {
+		t.Fatal("expected field to be zero value")
+	}
+
+	if *obj.OldPtr != *nobj.OldPtr {
+		t.Fatal("mismatch ", obj.OldPtr, " != ", nobj.OldPtr)
+	}
+	if nobj.NewPtr != nil {
+		t.Fatal("expected field to be zero value")
+	}
+
+	if !cmp.Equal(obj.OldMap, nobj.OldMap) {
+		t.Fatal("mismatch map marshal / unmarshal")
+	}
+	if len(nobj.NewMap) != 0 {
+		t.Fatal("expected field to be zero value")
+	}
+
+	if !cmp.Equal(obj.OldArray, nobj.OldArray) {
+		t.Fatal("mismatch array marshal / unmarshal")
+	}
+	if len(nobj.NewArray) != 0 {
+		t.Fatal("expected field to be zero value")
+	}
+
+	if !cmp.Equal(obj.OldStruct, nobj.OldStruct) {
+		t.Fatal("mismatch struct marshal / unmarshal")
+	}
+	if !cmp.Equal(nobj.NewStruct, SimpleTypeOne{}) {
+		t.Fatal("expected field to be zero value")
+	}
+}
+
+func TestMoreToLessFieldsRoundTrip(t *testing.T) {
+	dummyCid1, _ := cid.Parse("bafkqaaa")
+	dummyCid2, _ := cid.Parse("bafkqaab")
+	simpleType1 := SimpleTypeOne{
+		Foo:     "foo",
+		Value:   1,
+		Binary:  []byte("bin"),
+		Signed:  -1,
+		NString: "namedstr",
+	}
+	simpleType2 := SimpleTypeOne{
+		Foo:     "bar",
+		Value:   2,
+		Binary:  []byte("bin2"),
+		Signed:  -2,
+		NString: "namedstr2",
+	}
+	obj := &SimpleStructV2{
+		OldStr:    "oldstr",
+		NewStr:    "newstr",
+		OldBytes:  []byte("oldbytes"),
+		NewBytes:  []byte("newbytes"),
+		OldNum:    10,
+		NewNum:    11,
+		OldPtr:    &dummyCid1,
+		NewPtr:    &dummyCid2,
+		OldMap:    map[string]SimpleTypeOne{"foo": simpleType1},
+		NewMap:    map[string]SimpleTypeOne{"bar": simpleType2},
+		OldArray:  []SimpleTypeOne{simpleType1},
+		NewArray:  []SimpleTypeOne{simpleType1, simpleType2},
+		OldStruct: simpleType1,
+		NewStruct: simpleType2,
+	}
+
+	buf := new(bytes.Buffer)
+	if err := obj.MarshalCBOR(buf); err != nil {
+		t.Fatal("failed marshaling", err)
+	}
+
+	enc := buf.Bytes()
+
+	nobj := SimpleStructV1{}
+	if err := nobj.UnmarshalCBOR(bytes.NewReader(enc)); err != nil {
+		t.Logf("got bad bytes: %x", enc)
+		t.Fatal("failed to round trip object: ", err)
+	}
+
+	if obj.OldStr != nobj.OldStr {
+		t.Fatal("mismatch", obj.OldStr, " != ", nobj.OldStr)
+	}
+	if obj.OldNum != nobj.OldNum {
+		t.Fatal("mismatch ", obj.OldNum, " != ", nobj.OldNum)
+	}
+	if !bytes.Equal(obj.OldBytes, nobj.OldBytes) {
+		t.Fatal("mismatch ", obj.OldBytes, " != ", nobj.OldBytes)
+	}
+	if *obj.OldPtr != *nobj.OldPtr {
+		t.Fatal("mismatch ", obj.OldPtr, " != ", nobj.OldPtr)
+	}
+	if !cmp.Equal(obj.OldMap, nobj.OldMap) {
+		t.Fatal("mismatch map marshal / unmarshal")
+	}
+	if !cmp.Equal(obj.OldArray, nobj.OldArray) {
+		t.Fatal("mismatch array marshal / unmarshal")
+	}
+	if !cmp.Equal(obj.OldStruct, nobj.OldStruct) {
+		t.Fatal("mismatch struct marshal / unmarshal")
+	}
+}

testing/types.go

+package testing
+
+import (
+	cbg "gitlab.dms3.io/dms3/cbor-gen"
+	"gitlab.dms3.io/dms3/go-cid"
+)
+
+const Thingc = 3
+
+type NamedNumber uint64
+type NamedString string
+
+type SignedArray struct {
+	Signed []uint64
+}
+
+type SimpleTypeOne struct {
+	Foo     string
+	Value   uint64
+	Binary  []byte
+	Signed  int64
+	NString NamedString
+}
+
+type SimpleTypeTwo struct {
+	Stuff        *SimpleTypeTwo
+	Others       []uint64
+	SignedOthers []int64
+	Test         [][]byte
+	Dog          string
+	Numbers      []NamedNumber
+	Pizza        *uint64
+	PointyPizza  *NamedNumber
+	Arrrrrghay   [Thingc]SimpleTypeOne
+}
+
+type SimpleTypeTree struct {
+	Stuff                            *SimpleTypeTree
+	Stufff                           *SimpleTypeTwo
+	Others                           []uint64
+	Test                             [][]byte
+	Dog                              string
+	SixtyThreeBitIntegerWithASignBit int64
+	NotPizza                         *uint64
+}
+
+type SimpleStructV1 struct {
+	OldStr    string
+	OldBytes  []byte
+	OldNum    uint64
+	OldPtr    *cid.Cid
+	OldMap    map[string]SimpleTypeOne
+	OldArray  []SimpleTypeOne
+	OldStruct SimpleTypeOne
+}
+
+type SimpleStructV2 struct {
+	OldStr string
+	NewStr string
+
+	OldBytes []byte
+	NewBytes []byte
+
+	OldNum uint64
+	NewNum uint64
+
+	OldPtr *cid.Cid
+	NewPtr *cid.Cid
+
+	OldMap map[string]SimpleTypeOne
+	NewMap map[string]SimpleTypeOne
+
+	OldArray []SimpleTypeOne
+	NewArray []SimpleTypeOne
+
+	OldStruct SimpleTypeOne
+	NewStruct SimpleTypeOne
+}
+
+type DeferredContainer struct {
+	Stuff    *SimpleTypeOne
+	Deferred *cbg.Deferred
+	Value    uint64
+}
+
+type FixedArrays struct {
+	Bytes  [20]byte
+	Uint8  [20]uint8
+	Uint64 [20]uint64
+}
+
+type ThingWithSomeTime struct {
+	When    cbg.CborTime
+	Stuff   int64
+	CatName string
+}
+
+// Do not add fields to this type.
+type NeedScratchForMap struct {
+	Thing bool
+}

utils_test.go

+package typegen
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	cid "gitlab.dms3.io/dms3/go-cid"
+)
+
+func TestLinkScan(t *testing.T) {
+	inp := "82442847c0498ba16131818242000484d82a57000155001266696c2f312f73746f72616765706f776572d82a5827000171a0e40220740d4196aaaee66d8e9b828bc6f9271662096e36782de248e3b3ed28443dbc810040a16131828242000584d82a5818000155001366696c2f312f73746f726167656d61726b6574d82a5827000171a0e402209e59ceb041921650967e8c77d36269c10049140d28d5015165cc8eb897a2555300408242006684d82a52000155000d66696c2f312f6163636f756e74d82a5827000171a0e40220f9556f0d5a735ff53cc327e85a46c7c094028b9da894de73caa88f162429c29d004b000a968163f0a57b400000a16131818242000084d82a51000155000c66696c2f312f73797374656dd82a5827000171a0e4022045b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c00040a16131818242006384d82a52000155000d66696c2f312f6163636f756e74d82a5827000171a0e4022045b0cfc220ceec5b7c1c62c4d4193d38e4eba48e8815729ce75f9c0ab0e4c1c00040a16131818242000184d82a4f000155000a66696c2f312f696e6974d82a5827000171a0e4022050f3c45d0e78f04688c6e8cbdc45f71a5cbcec731519ffdcdd92765fc5ba0da30040a16131818242000684d82a581b000155001666696c2f312f76657269666965647265676973747279d82a5827000171a0e40220fd3fee39acd88c8808110d9741149a79939f52c798b6539048e4835aa4d34fd50040a16131818242006584d82a52000155000d66696c2f312f6163636f756e74d82a5827000171a0e402200293716d8503737644624c102d9ba1514d599044a2e6f2038330124bc6f54361004c002116545850052128000000a16131818242000384d82a4f000155000a66696c2f312f63726f6ed82a5827000171a0e4022065d1dad76492ccd5d010197dc26bd5fb07c0cf85ccdcaff21084c0d47bfd17590040a16131818242005084d82a52000155000d66696c2f312f6163636f756e74d82a5827000171a0e402202e12f4eedaac06c2040df4923656f7f2d6c5991b1874a32e9f52b0e48e61d8410040a16131818242006484d82a52000155000d66696c2f312f6163636f756e74d82a5827000171a0e40220bde06af1782cb302e0973658dcb44b2cbd72891598b7a7b02381b563f1dc9c57004c002116545850052128000000a16131818242000284d82a51000155000c66696c2f312f726577617264d82a5827000171a0e4022083c127ddb0ba85f585b06365346eabe5ef98861d85770670dee402dc3810131e004d0004860d8812f0b38878000000"
+	inpb, err := hex.DecodeString(inp)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var cids []cid.Cid
+	if err := ScanForLinks(bytes.NewReader(inpb), func(c cid.Cid) {
+		cids = append(cids, c)
+	}); err != nil {
+		t.Fatal(err)
+	}
+	t.Log(cids)
+}
+
+func TestDeferredMaxLengthSingle(t *testing.T) {
+	var header bytes.Buffer
+	if err := WriteMajorTypeHeader(&header, MajByteString, ByteArrayMaxLen+1); err != nil {
+		t.Fatal("failed to write header")
+	}
+
+	var deferred Deferred
+	err := deferred.UnmarshalCBOR(&header)
+	if err != maxLengthError {
+		t.Fatal("deferred: allowed more than the maximum allocation supported")
+	}
+}

validate_test.go

+package typegen
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestValidateShort(t *testing.T) {
+	var buf bytes.Buffer
+	if err := WriteMajorTypeHeader(&buf, MajByteString, 100); err != nil {
+		t.Fatal("failed to write header")
+	}
+
+	if err := ValidateCBOR(buf.Bytes()); err == nil {
+		t.Fatal("expected an error checking truncated cbor")
+	}
+}
+
+func TestValidateDouble(t *testing.T) {
+	var buf bytes.Buffer
+	if err := WriteBool(&buf, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := WriteBool(&buf, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ValidateCBOR(buf.Bytes()); err == nil {
+		t.Fatal("expected an error checking cbor with two objects")
+	}
+}
+
+func TestValidate(t *testing.T) {
+	var buf bytes.Buffer
+	if err := WriteBool(&buf, false); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ValidateCBOR(buf.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+}