Skip to content

GitLab

Unverified Commit f75a516a authored by Rod Vagg's avatar Rod Vagg
Browse files
Options

remove old pb decoder/encoder

parent a8310bdf
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 1079 deletions
pb/basics_test.go deleted 100644 → 0
View file @ a8310bdf
package pb
import (
"encoding/hex"
"reflect"
"strings"
"testing"
cid "github.com/ipfs/go-cid"
)
func mkcid(t *testing.T, cidStr string) cid.Cid {
c, err := cid.Decode(cidStr)
if err != nil {
t.Fatal(err)
}
return c
}
func TestEmptyNode(t *testing.T) {
n := NewPBNode()
byts, err := MarshalPBNode(n)
if err != nil {
t.Fatal(err)
}
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
if rn.Data != nil {
t.Errorf("Expected nil PBNode#Data")
}
if rn.Links == nil || len(rn.Links) != 0 {
t.Errorf("Expected zero-length PBNode#Links")
}
}
func TestNodeWithData(t *testing.T) {
n := NewPBNode()
n.Data = []byte{0, 1, 2, 3, 4}
byts, err := MarshalPBNode(n)
if err != nil {
t.Fatal(err)
}
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
if rn.Data == nil {
t.Errorf("Expected non-nil PBNode#Data")
}
if !reflect.DeepEqual(rn.Data, []byte{0, 1, 2, 3, 4}) {
t.Errorf("Didn't get expected bytes")
}
if rn.Links == nil || len(rn.Links) != 0 {
t.Errorf("Expected zero-length PBNode#Links")
}
}
func TestNodeWithLink(t *testing.T) {
n := NewPBNode()
n.Links = append(n.Links, NewPBLinkFromCid(mkcid(t, "QmWDtUQj38YLW8v3q4A6LwPn4vYKEbuKWpgSm6bjKW6Xfe")))
byts, err := MarshalPBNode(n)
if err != nil {
t.Fatal(err)
}
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
if rn.Data != nil {
t.Errorf("Expected nil PBNode#Data")
}
if rn.Links == nil || len(rn.Links) != 1 {
t.Errorf("Expected one PBNode#Links element")
}
if rn.Links[0].Tsize != nil || rn.Links[0].Name != nil {
t.Errorf("Expected Tsize and Name to be nil")
}
if rn.Links[0].Hash.String() != "QmWDtUQj38YLW8v3q4A6LwPn4vYKEbuKWpgSm6bjKW6Xfe" {
t.Errorf("Got unexpected Hash")
}
}
func TestNodeWithTwoUnsortedLinks(t *testing.T) {
n := NewPBNodeFromData([]byte("some data"))
n.Links = append(n.Links, NewPBLink("some other link", mkcid(t, "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V"), 8))
n.Links = append(n.Links, NewPBLink("some link", mkcid(t, "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39U"), 100000000))
byts, err := MarshalPBNode(n)
if err == nil || !strings.Contains(err.Error(), "links must be sorted") || byts != nil {
t.Fatal("Expected to error from unsorted links")
}
n.SortLinks()
if byts, err = MarshalPBNode(n); err != nil {
t.Fatal(err)
}
expectedBytes := "12340a2212208ab7a6c5e74737878ac73863cb76739d15d4666de44e5756bf55a2f9e9ab5f431209736f6d65206c696e6b1880c2d72f12370a2212208ab7a6c5e74737878ac73863cb76739d15d4666de44e5756bf55a2f9e9ab5f44120f736f6d65206f74686572206c696e6b18080a09736f6d652064617461"
if hex.EncodeToString(byts) != expectedBytes {
t.Fatal("Did not get expected bytes")
}
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
if rn.Data == nil || !reflect.DeepEqual(rn.Data, []byte("some data")) {
t.Errorf("Did not match expected PBNode#Data")
}
if rn.Links == nil || len(rn.Links) != 2 {
t.Errorf("Expected two PBNode#Links elements")
}
if rn.Links[0].Tsize == nil || rn.Links[0].Name == nil || rn.Links[1].Tsize == nil || rn.Links[1].Name == nil {
t.Errorf("Expected Tsize and Name to not be nil")
}
if rn.Links[0].Hash.String() != "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39U" {
t.Errorf("Got unexpected Hash 0")
}
if rn.Links[1].Hash.String() != "QmXg9Pp2ytZ14xgmQjYEiHjVjMFXzCVVEcRTWJBmLgR39V" {
t.Errorf("Got unexpected Hash 1")
}
if *rn.Links[0].Name != "some link" {
t.Errorf("Got unexpected Name 0")
}
if *rn.Links[1].Name != "some other link" {
t.Errorf("Got unexpected Name 1")
}
if *rn.Links[0].Tsize != 100000000 {
t.Errorf("Got unexpected Tsize 0")
}
if *rn.Links[1].Tsize != 8 {
t.Errorf("Got unexpected Tsize 1")
}
}
func TestNodeWithStableSortedLinks(t *testing.T) {
n := NewPBNodeFromData([]byte("some data"))
cids := []string{
"QmUGhP2X8xo9dsj45vqx1H6i5WqPqLqmLQsHTTxd3ke8mp",
"QmP7SrR76KHK9A916RbHG1ufy2TzNABZgiE23PjZDMzZXy",
"QmQg1v4o9xdT3Q14wh4S7dxZkDjyZ9ssFzFzyep1YrVJBY",
"QmdP6fartWRrydZCUjHgrJ4XpxSE4SAoRsWJZ1zJ4MWiuf",
"QmNNjUStxtMC1WaSZYiDW6CmAUrvd5Q2e17qnxPgVdwrwW",
"QmWJwqZBJWerHsN1b7g4pRDYmzGNnaMYuD3KSbnpaxsB2h",
"QmRXPSdysBS3dbUXe6w8oXevZWHdPQWaR2d3fggNsjvieL",
"QmTUZAXfws6zrhEksnMqLxsbhXZBQs4FNiarjXSYQqVrjC",
"QmNNk7dTdh8UofwgqLNauq6N78DPc6LKK2yBs1MFdx7Mbg",
"QmW5mrJfyqh7B4ywSvraZgnWjS3q9CLiYURiJpCX3aro5i",
"QmTFHZL5CkgNz19MdPnSuyLAi6AVq9fFp81zmPpaL2amED",
}
// name is the same for all, they should remain unsorted
for _, c := range cids {
n.Links = append(n.Links, NewPBLink("", mkcid(t, c), 262158))
}
byts, err := MarshalPBNode(n)
if err != nil {
t.Fatal(err)
}
n.SortLinks()
byts2, err := MarshalPBNode(n)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(byts2, byts) {
t.Errorf("Sort was not stable for unnamed links")
}
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
for i, link := range rn.Links {
if link.Hash.String() != cids[i] {
t.Errorf("Link #%d does not match expected (%v)", i, cids[i])
}
}
}
func TestNodeWithUnnamedLinksFixture(t *testing.T) {
fixture := "122b0a2212203f29086b59b9e046b362b4b19c9371e834a9f5a80597af83be6d8b7d1a5ad33b120018aed4e015122b0a221220ae1a5afd7c770507dddf17f92bba7a326974af8ae5277c198cf13206373f7263120018aed4e015122b0a22122022ab2ebf9c3523077bd6a171d516ea0e1be1beb132d853778bcc62cd208e77f1120018aed4e015122b0a22122040a77fe7bc69bbef2491f7633b7c462d0bce968868f88e2cbcaae9d0996997e8120018aed4e015122b0a2212206ae1979b14dd43966b0241ebe80ac2a04ad48959078dc5affa12860648356ef6120018aed4e015122b0a221220a957d1f89eb9a861593bfcd19e0637b5c957699417e2b7f23c88653a240836c4120018aed4e015122b0a221220345f9c2137a2cd76d7b876af4bfecd01f80b7dd125f375cb0d56f8a2f96de2c31200189bfec10f0a2b080218cbc1819201208080e015208080e015208080e015208080e015208080e015208080e01520cbc1c10f"
expectedLinks := []*PBLink{
NewPBLink("", mkcid(t, "QmSbCgdsX12C4KDw3PDmpBN9iCzS87a5DjgSCoW9esqzXk"), 45623854),
NewPBLink("", mkcid(t, "Qma4GxWNhywSvWFzPKtEswPGqeZ9mLs2Kt76JuBq9g3fi2"), 45623854),
NewPBLink("", mkcid(t, "QmQfyxyys7a1e3mpz9XsntSsTGc8VgpjPj5BF1a1CGdGNc"), 45623854),
NewPBLink("", mkcid(t, "QmSh2wTTZT4N8fuSeCFw7wterzdqbE93j1XDhfN3vQHzDV"), 45623854),
NewPBLink("", mkcid(t, "QmVXsSVjwxMsCwKRCUxEkGb4f4B98gXVy3ih3v4otvcURK"), 45623854),
NewPBLink("", mkcid(t, "QmZjhH97MEYwQXzCqSQbdjGDhXWuwW4RyikR24pNqytWLj"), 45623854),
NewPBLink("", mkcid(t, "QmRs6U5YirCqC7taTynz3x2GNaHJZ3jDvMVAzaiXppwmNJ"), 32538395),
}
byts, err := hex.DecodeString(fixture)
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(rn.Links, expectedLinks) {
t.Errorf("Didn't get expected Links array")
}
}
func TestNodeWithNamedLinksFixture(t *testing.T) {
fixture := "12390a221220b4397c02da5513563d33eef894bf68f2ccdf1bdfc14a976956ab3d1c72f735a0120e617564696f5f6f6e6c792e6d346118cda88f0b12310a221220025c13fcd1a885df444f64a4a82a26aea867b1148c68cb671e83589f971149321208636861742e74787418e40712340a2212205d44a305b9b328ab80451d0daa72a12a7bf2763c5f8bbe327597a31ee40d1e48120c706c61796261636b2e6d3375187412360a2212202539ed6e85f2a6f9097db9d76cffd49bf3042eb2e3e8e9af4a3ce842d49dea22120a7a6f6f6d5f302e6d70341897fb8592010a020801"
expectedLinks := []*PBLink{
NewPBLink("audio_only.m4a", mkcid(t, "QmaUAwAQJNtvUdJB42qNbTTgDpzPYD1qdsKNtctM5i7DGB"), 23319629),
NewPBLink("chat.txt", mkcid(t, "QmNVrxbB25cKTRuKg2DuhUmBVEK9NmCwWEHtsHPV6YutHw"), 996),
NewPBLink("playback.m3u", mkcid(t, "QmUcjKzDLXBPmB6BKHeKSh6ZoFZjss4XDhMRdLYRVuvVfu"), 116),
NewPBLink("zoom_0.mp4", mkcid(t, "QmQqy2SiEkKgr2cw5UbQ93TtLKEMsD8TdcWggR8q9JabjX"), 306281879),
}
byts, err := hex.DecodeString(fixture)
rn, err := UnmarshalPBNode(byts)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(rn.Links, expectedLinks) {
t.Errorf("Didn't get expected Links array")
}
}
pb/compat_test.go deleted 100644 → 0
View file @ a8310bdf
package pb
// mirrored in JavaScript @ https://github.com/ipld/js-dag-pb/blob/master/test/test-compat.js
import (
"encoding/hex"
"encoding/json"
"strings"
"testing"
cid "github.com/ipfs/go-cid"
)
var dataZero []byte = make([]byte, 0)
var dataSome []byte = []byte{0, 1, 2, 3, 4}
var acid cid.Cid = _mkcid()
var zeroName string = ""
var someName string = "some name"
var zeroTsize uint64 = 0
var someTsize uint64 = 1010
var largeTsize uint64 = 9007199254740991 // JavaScript Number.MAX_SAFE_INTEGER
type testCase struct {
name string
node *PBNode
expectedBytes string
expectedForm string
encodeError string
decodeError string
}
var testCases = []testCase{
{
name: "empty",
node: &PBNode{},
expectedBytes: "",
expectedForm: `{
"Links": []
}`,
encodeError: "Links must be an array",
},
{
name: "Data zero",
node: &PBNode{Data: dataZero},
expectedBytes: "0a00",
expectedForm: `{
"Data": "",
"Links": []
}`,
encodeError: "Links must be an array",
},
{
name: "Data some",
node: &PBNode{Data: dataSome},
expectedBytes: "0a050001020304",
expectedForm: `{
"Data": "0001020304",
"Links": []
}`,
encodeError: "Links must be an array",
},
{
name: "Links zero",
node: &PBNode{Links: make([]*PBLink, 0)},
expectedBytes: "",
expectedForm: `{
"Links": []
}`,
},
{
name: "Data some Links zero",
node: &PBNode{Data: dataSome, Links: make([]*PBLink, 0)},
expectedBytes: "0a050001020304",
expectedForm: `{
"Data": "0001020304",
"Links": []
}`,
},
{
name: "Links empty",
node: &PBNode{Links: []*PBLink{{}}},
expectedBytes: "1200",
encodeError: "link must have a Hash",
decodeError: "expected CID",
},
{
name: "Data some Links empty",
node: &PBNode{Data: dataSome, Links: []*PBLink{{}}},
expectedBytes: "12000a050001020304",
encodeError: "link must have a Hash",
decodeError: "expected CID",
},
{
name: "Links Hash zero",
expectedBytes: "12020a00",
decodeError: "expected CID", // error should come up from go-cid too
},
{
name: "Links Hash some",
node: &PBNode{Links: []*PBLink{{Hash: &acid}}},
expectedBytes: "120b0a09015500050001020304",
expectedForm: `{
"Links": [
{
"Hash": "015500050001020304"
}
]
}`,
},
{
name: "Links Name zero",
node: &PBNode{Links: []*PBLink{{Name: &zeroName}}},
expectedBytes: "12021200",
encodeError: "link must have a Hash",
decodeError: "expected CID",
},
{
name: "Links Hash some Name zero",
node: &PBNode{Links: []*PBLink{{Hash: &acid, Name: &zeroName}}},
expectedBytes: "120d0a090155000500010203041200",
expectedForm: `{
"Links": [
{
"Hash": "015500050001020304",
"Name": ""
}
]
}`,
},
{
name: "Links Name some",
node: &PBNode{Links: []*PBLink{{Name: &someName}}},
expectedBytes: "120b1209736f6d65206e616d65",
encodeError: "link must have a Hash",
decodeError: "expected CID",
},
{
name: "Links Hash some Name some",
node: &PBNode{Links: []*PBLink{{Hash: &acid, Name: &someName}}},
expectedBytes: "12160a090155000500010203041209736f6d65206e616d65",
expectedForm: `{
"Links": [
{
"Hash": "015500050001020304",
"Name": "some name"
}
]
}`,
},
{
name: "Links Tsize zero",
node: &PBNode{Links: []*PBLink{{Tsize: &zeroTsize}}},
expectedBytes: "12021800",
encodeError: "link must have a Hash",
decodeError: "expected CID",
},
{
name: "Links Hash some Tsize zero",
node: &PBNode{Links: []*PBLink{{Hash: &acid, Tsize: &zeroTsize}}},
expectedBytes: "120d0a090155000500010203041800",
expectedForm: `{
"Links": [
{
"Hash": "015500050001020304",
"Tsize": 0
}
]
}`,
},
{
name: "Links Tsize some",
node: &PBNode{Links: []*PBLink{{Tsize: &someTsize}}},
expectedBytes: "120318f207",
encodeError: "link must have a Hash",
decodeError: "expected CID",
},
{
name: "Links Hash some Tsize some",
node: &PBNode{Links: []*PBLink{{Hash: &acid, Tsize: &largeTsize}}},
expectedBytes: "12140a0901550005000102030418ffffffffffffff0f",
expectedForm: `{
"Links": [
{
"Hash": "015500050001020304",
"Tsize": 9007199254740991
}
]
}`,
},
}
func TestCompat(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
verifyRoundTrip(t, tc)
})
}
}
func verifyRoundTrip(t *testing.T, tc testCase) {
var err error
var actualBytes string
var actualForm string
if tc.node != nil {
actualBytes, err = nodeToString(t, tc.node)
if tc.encodeError != "" {
if err != nil {
if !strings.Contains(err.Error(), tc.encodeError) {
t.Fatalf("got unexpeced encode error: [%v] (expected [%v])", err.Error(), tc.encodeError)
}
} else {
t.Fatalf("did not get expected encode error: %v", tc.encodeError)
}
} else {
if err != nil {
t.Fatal(err)
} else {
if actualBytes != tc.expectedBytes {
t.Logf(
"Expected bytes: [%v]\nGot: [%v]\n",
tc.expectedBytes,
actualBytes)
t.Error("Did not match")
}
}
}
}
actualForm, err = bytesToFormString(t, tc.expectedBytes)
if tc.decodeError != "" {
if err != nil {
if !strings.Contains(err.Error(), tc.decodeError) {
t.Fatalf("got unexpeced decode error: [%v] (expected [%v])", err.Error(), tc.decodeError)
}
} else {
t.Fatalf("did not get expected decode error: %v", tc.decodeError)
}
} else {
if err != nil {
t.Fatal(err)
}
if actualForm != tc.expectedForm {
t.Logf(
"Expected form: [%v]\nGot: [%v]\n",
tc.expectedForm,
actualForm)
t.Error("Did not match")
}
}
}
func nodeToString(t *testing.T, n *PBNode) (string, error) {
bytes, err := MarshalPBNode(n)
if err != nil {
return "", err
}
t.Logf("[%v]\n", hex.EncodeToString(bytes))
return hex.EncodeToString(bytes), nil
}
func bytesToFormString(t *testing.T, bytesHex string) (string, error) {
bytes, err := hex.DecodeString(bytesHex)
if err != nil {
return "", err
}
var rt *PBNode
if rt, err = UnmarshalPBNode(bytes); err != nil {
return "", err
}
str, err := json.MarshalIndent(cleanPBNode(t, rt), "", "\t")
if err != nil {
return "", err
}
return string(str), nil
}
// convert a PBLink into a map for clean JSON marshalling
func cleanPBLink(t *testing.T, link *PBLink) map[string]interface{} {
if link == nil {
return nil
}
nl := make(map[string]interface{})
if link.Hash != nil {
nl["Hash"] = hex.EncodeToString(link.Hash.Bytes())
}
if link.Name != nil {
nl["Name"] = link.Name
}
if link.Tsize != nil {
nl["Tsize"] = link.Tsize
}
return nl
}
// convert a PBNode into a map for clean JSON marshalling
func cleanPBNode(t *testing.T, node *PBNode) map[string]interface{} {
nn := make(map[string]interface{})
if node.Data != nil {
nn["Data"] = hex.EncodeToString(node.Data)
}
if node.Links != nil {
links := make([]map[string]interface{}, len(node.Links))
for i, l := range node.Links {
links[i] = cleanPBLink(t, l)
}
nn["Links"] = links
}
return nn
}
func _mkcid() cid.Cid {
_, c, _ := cid.CidFromBytes([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4})
return c
}
pb/dagpb.go deleted 100644 → 0
View file @ a8310bdf
package pb
import (
"sort"
cid "github.com/ipfs/go-cid"
)
// PBLink ...
type PBLink struct {
Hash *cid.Cid
Name *string
Tsize *uint64
}
// PBNode ...
type PBNode struct {
Links []*PBLink
Data []byte
}
func NewPBNode() *PBNode {
n := &PBNode{Links: make([]*PBLink, 0)}
return n
}
func NewPBNodeFromData(data []byte) *PBNode {
n := &PBNode{Data: data, Links: make([]*PBLink, 0)}
return n
}
type TokenType byte
const (
TypeData TokenType = 'd'
TypeLinks TokenType = '['
TypeLink TokenType = 'l'
TypeLinkEnd TokenType = 'e'
TypeHash TokenType = 'h'
TypeName TokenType = 'n'
TypeTSize TokenType = 's'
TypeEnd TokenType = 'x'
)
type Token struct {
Type TokenType
Bytes []byte
Cid *cid.Cid
Int uint64
}
func NewPBLinkFromCid(c cid.Cid) *PBLink {
l := &PBLink{Hash: &c}
return l
}
func NewPBLink(name string, c cid.Cid, tsize uint64) *PBLink {
l := &PBLink{Name: &name, Hash: &c, Tsize: &tsize}
return l
}
func (node *PBNode) SortLinks() {
SortLinks(node.Links)
}
func SortLinks(links []*PBLink) {
sort.Stable(pbLinkSlice(links))
}
type pbLinkSlice []*PBLink
func (ls pbLinkSlice) Len() int { return len(ls) }
func (ls pbLinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] }
func (ls pbLinkSlice) Less(a, b int) bool { return pbLinkLess(ls[a], ls[b]) }
func pbLinkLess(a *PBLink, b *PBLink) bool {
return *a.Name < *b.Name
}
pb/go.mod deleted 100644 → 0
View file @ a8310bdf
module github.com/rvagg/go-dagpb/pb
go 1.15
require (
github.com/ipfs/go-cid v0.0.7
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1
)
pb/go.sum deleted 100644 → 0
View file @ a8310bdf
github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI=
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
pb/marshal.go deleted 100644 → 0
View file @ a8310bdf
package pb
// based on the original pb codegen
import (
"fmt"
"io"
math_bits "math/bits"
)
func Marshal(out io.Writer, tokenSource func() (Token, error)) error {
writeLead := func(wire byte, size uint64) {
lead := make([]byte, SizeOfVarint(size)+1)
lead[0] = wire
EncodeVarint(lead, len(lead), size)
out.Write(lead)
}
var link PBLink
for {
tok, err := tokenSource()
if err != nil {
return err
}
if tok.Type == TypeEnd {
break
}
switch tok.Type {
case TypeData:
writeLead(0xa, uint64(len(tok.Bytes)))
out.Write(tok.Bytes)
case TypeLinkEnd:
l := link.EncodedSize()
writeLead(0x12, uint64(l))
chunk := make([]byte, l)
wrote, err := link.Marshal(chunk)
if err != nil {
return err
}
if wrote != l {
return fmt.Errorf("bad PBLink marshal, wrote wrong number of bytes")
}
out.Write(chunk)
link = PBLink{}
case TypeHash:
link.Hash = tok.Cid
case TypeName:
s := string(tok.Bytes)
link.Name = &s
case TypeTSize:
link.Tsize = &tok.Int
}
}
return nil
}
func MarshalPBNode(node *PBNode) ([]byte, error) {
if err := node.Validate(); err != nil {
return nil, err
}
size := node.EncodedSize()
data := make([]byte, size)
i := len(data)
if node.Data != nil {
i -= len(node.Data)
copy(data[i:], node.Data)
i = EncodeVarint(data, i, uint64(len(node.Data))) - 1
data[i] = 0xa
}
if len(node.Links) > 0 {
for index := len(node.Links) - 1; index >= 0; index-- {
size, err := node.Links[index].Marshal(data[:i])
if err != nil {
return nil, err
}
i -= size
i = EncodeVarint(data, i, uint64(size)) - 1
data[i] = 0x12
}
}
return data[:size], nil
}
func (link *PBLink) Marshal(data []byte) (int, error) {
i := len(data)
if link.Tsize != nil {
i = EncodeVarint(data, i, uint64(*link.Tsize)) - 1
data[i] = 0x18
}
if link.Name != nil {
i -= len(*link.Name)
copy(data[i:], *link.Name)
i = EncodeVarint(data, i, uint64(len(*link.Name))) - 1
data[i] = 0x12
}
if link.Hash != nil {
byts := link.Hash.Bytes()
i -= len(byts)
copy(data[i:], byts)
i = EncodeVarint(data, i, uint64(len(byts))) - 1
data[i] = 0xa
} else {
return 0, fmt.Errorf("invalid DAG-PB form (link must have a Hash)")
}
return len(data) - i, nil
}
func (node *PBNode) Validate() error {
if node == nil {
return fmt.Errorf("PBNode not defined")
}
if node.Links == nil {
return fmt.Errorf("invalid DAG-PB form (Links must be an array)")
}
for i, link := range node.Links {
if link.Hash == nil {
return fmt.Errorf("invalid DAG-PB form (link must have a Hash)")
}
if i > 0 && pbLinkLess(link, node.Links[i-1]) {
return fmt.Errorf("invalid DAG-PB form (links must be sorted by Name bytes)")
}
}
return nil
}
func (link *PBLink) EncodedSize() (n int) {
if link == nil {
return 0
}
var l int
if link.Hash != nil {
l = link.Hash.ByteLen()
n += 1 + l + SizeOfVarint(uint64(l))
}
if link.Name != nil {
l = len(*link.Name)
n += 1 + l + SizeOfVarint(uint64(l))
}
if link.Tsize != nil {
n += 1 + SizeOfVarint(uint64(*link.Tsize))
}
return n
}
func (node *PBNode) EncodedSize() (n int) {
if node == nil {
return 0
}
var l int
if node.Data != nil {
l = len(node.Data)
n += 1 + l + SizeOfVarint(uint64(l))
}
if len(node.Links) > 0 {
for _, e := range node.Links {
l = e.EncodedSize()
n += 1 + l + SizeOfVarint(uint64(l))
}
}
return n
}
func EncodeVarint(data []byte, offset int, v uint64) int {
offset -= SizeOfVarint(v)
base := offset
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return base
}
func SizeOfVarint(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
pb/unmarshal.go deleted 100644 → 0
View file @ a8310bdf
package pb
import (
"bytes"
"fmt"
"io"
cid "github.com/ipfs/go-cid"
"github.com/polydawn/refmt/shared"
)
type PBLinkBuilder interface {
SetHash(*cid.Cid) error
SetName(*string) error
SetTsize(uint64) error
Done() error
}
type PBNodeBuilder interface {
SetData([]byte) error
AddLink() (PBLinkBuilder, error)
Done() error
}
func UnmarshalPBNode(byts []byte) (*PBNode, error) {
node := NewPBNode()
tokenReceiver := func(tok Token) error {
switch tok.Type {
case TypeData:
node.Data = tok.Bytes
case TypeLink:
node.Links = append(node.Links, &PBLink{})
case TypeHash:
node.Links[len(node.Links)-1].Hash = tok.Cid
case TypeName:
s := string(tok.Bytes)
node.Links[len(node.Links)-1].Name = &s
case TypeTSize:
node.Links[len(node.Links)-1].Tsize = &tok.Int
}
return nil
}
if err := Unmarshal(bytes.NewReader(byts), tokenReceiver); err != nil {
return nil, err
}
return node, nil
}
func Unmarshal(in io.Reader, tokenReceiver func(tok Token) error) error {
haveData := false
reader := shared.NewReader(in)
for {
_, err := reader.Readn1()
if err == io.EOF {
break
}
reader.Unreadn1()
fieldNum, wireType, err := decodeKey(reader)
if err != nil {
return err
}
if wireType != 2 {
return fmt.Errorf("protobuf: (PBNode) invalid wireType, expected 2, got %d", wireType)
}
if fieldNum == 1 {
if haveData {
return fmt.Errorf("protobuf: (PBNode) duplicate Data section")
}
var chunk []byte
if chunk, err = decodeBytes(reader); err != nil {
return err
}
if err := tokenReceiver(Token{Type: TypeData, Bytes: chunk}); err != nil {
return err
}
haveData = true
} else if fieldNum == 2 {
if haveData {
return fmt.Errorf("protobuf: (PBNode) invalid order, found Data before Links content")
}
bytesLen, err := decodeVarint(reader)
if err != nil {
return err
}
if err = tokenReceiver(Token{Type: TypeLink}); err != nil {
return err
}
if err = unmarshalLink(reader, int(bytesLen), tokenReceiver); err != nil {
return err
}
if err = tokenReceiver(Token{Type: TypeLinkEnd}); err != nil {
return err
}
} else {
return fmt.Errorf("protobuf: (PBNode) invalid fieldNumber, expected 1 or 2, got %d", fieldNum)
}
}
return nil
}
func unmarshalLink(reader shared.SlickReader, length int, tokenReceiver func(tok Token) error) error {
haveHash := false
haveName := false
haveTsize := false
startOffset := reader.NumRead()
for {
readBytes := reader.NumRead() - startOffset
if readBytes == length {
break
} else if readBytes > length {
return fmt.Errorf("protobuf: (PBLink) bad length for link")
}
fieldNum, wireType, err := decodeKey(reader)
if err != nil {
return err
}
if fieldNum == 1 {
if haveHash {
return fmt.Errorf("protobuf: (PBLink) duplicate Hash section")
}
if haveName {
return fmt.Errorf("protobuf: (PBLink) invalid order, found Name before Hash")
}
if haveTsize {
return fmt.Errorf("protobuf: (PBLink) invalid order, found Tsize before Hash")
}
if wireType != 2 {
return fmt.Errorf("protobuf: (PBLink) wrong wireType (%d) for Hash", wireType)
}
var chunk []byte
if chunk, err = decodeBytes(reader); err != nil {
return err
}
var c cid.Cid
if _, c, err = cid.CidFromBytes(chunk); err != nil {
return fmt.Errorf("invalid Hash field found in link, expected CID (%v)", err)
}
if err := tokenReceiver(Token{Type: TypeHash, Cid: &c}); err != nil {
return err
}
haveHash = true
} else if fieldNum == 2 {
if haveName {
return fmt.Errorf("protobuf: (PBLink) duplicate Name section")
}
if haveTsize {
return fmt.Errorf("protobuf: (PBLink) invalid order, found Tsize before Name")
}
if wireType != 2 {
return fmt.Errorf("protobuf: (PBLink) wrong wireType (%d) for Name", wireType)
}
var chunk []byte
if chunk, err = decodeBytes(reader); err != nil {
return err
}
if err := tokenReceiver(Token{Type: TypeName, Bytes: chunk}); err != nil {
return err
}
haveName = true
} else if fieldNum == 3 {
if haveTsize {
return fmt.Errorf("protobuf: (PBLink) duplicate Tsize section")
}
if wireType != 0 {
return fmt.Errorf("protobuf: (PBLink) wrong wireType (%d) for Tsize", wireType)
}
var v uint64
if v, err = decodeVarint(reader); err != nil {
return err
}
if err := tokenReceiver(Token{Type: TypeTSize, Int: v}); err != nil {
return err
}
haveTsize = true
} else {
return fmt.Errorf("protobuf: (PBLink) invalid fieldNumber, expected 1, 2 or 3, got %d", fieldNum)
}
}
if !haveHash {
return fmt.Errorf("invalid Hash field found in link, expected CID")
}
return nil
}
func decodeKey(reader shared.SlickReader) (int, int, error) {
var wire uint64
var err error
if wire, err = decodeVarint(reader); err != nil {
return 0, 0, err
}
fieldNum := int(wire >> 3)
wireType := int(wire & 0x7)
return fieldNum, wireType, nil
}
func decodeBytes(reader shared.SlickReader) ([]byte, error) {
bytesLen, err := decodeVarint(reader)
if err != nil {
return nil, err
}
byts, err := reader.Readn(int(bytesLen))
if err != nil {
return nil, fmt.Errorf("protobuf: unexpected read error: %w", err)
}
return byts, nil
}
func decodeVarint(reader shared.SlickReader) (uint64, error) {
var v uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflow
}
b, err := reader.Readn1()
if err != nil {
return 0, fmt.Errorf("protobuf: unexpected read error: %w", err)
}
v |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
return v, nil
}
// ErrIntOverflow TODO
var ErrIntOverflow = fmt.Errorf("protobuf: varint overflow")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment