Commit fd4638f3 authored by Daniel Martí's avatar Daniel Martí

encode directly with a []byte

Like the previous commit, this helps reduce allocations as well as
improve performance thanks to the well-optimized protowire package.

And, as before, we get to remove unnecessary code.

	name         old time/op    new time/op    delta
	Roundtrip-8    4.81µs ± 0%    4.35µs ± 1%  -9.59%  (p=0.004 n=5+6)

	name         old alloc/op   new alloc/op   delta
	Roundtrip-8    7.14kB ± 0%    6.86kB ± 0%  -3.83%  (p=0.000 n=6+5)

	name         old allocs/op  new allocs/op  delta
	Roundtrip-8       119 ± 0%       112 ± 0%  -5.88%  (p=0.002 n=6+6)
parent b4150aed
...@@ -3,12 +3,12 @@ package dagpb ...@@ -3,12 +3,12 @@ package dagpb
import ( import (
"fmt" "fmt"
"io" "io"
math_bits "math/bits"
"sort" "sort"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
ipld "github.com/ipld/go-ipld-prime" ipld "github.com/ipld/go-ipld-prime"
cidlink "github.com/ipld/go-ipld-prime/linking/cid" cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"google.golang.org/protobuf/encoding/protowire"
) )
type pbLink struct { type pbLink struct {
...@@ -36,6 +36,7 @@ func Marshal(inNode ipld.Node, out io.Writer) error { ...@@ -36,6 +36,7 @@ func Marshal(inNode ipld.Node, out io.Writer) error {
if err != nil { if err != nil {
return err return err
} }
var enc []byte
if links.Length() > 0 { if links.Length() > 0 {
// collect links into a slice so we can properly sort for encoding // collect links into a slice so we can properly sort for encoding
pbLinks := make([]pbLink, links.Length()) pbLinks := make([]pbLink, links.Length())
...@@ -105,20 +106,35 @@ func Marshal(inNode ipld.Node, out io.Writer) error { ...@@ -105,20 +106,35 @@ func Marshal(inNode ipld.Node, out io.Writer) error {
// links must be strictly sorted by Name before encoding, leaving stable // links must be strictly sorted by Name before encoding, leaving stable
// ordering where the names are the same (or absent) // ordering where the names are the same (or absent)
sortLinks(pbLinks) sort.Stable(pbLinkSlice(pbLinks))
for _, link := range pbLinks { for _, link := range pbLinks {
size := link.encodedSize() hash := link.hash.Bytes()
chunk := make([]byte, size+sizeOfVarint(uint64(size))+1)
chunk[0] = 0x12 // field & wire type for Links size := 0
offset := encodeVarint(chunk, 1, uint64(size)) size += protowire.SizeTag(2)
wrote, err := link.marshal(chunk, offset) size += protowire.SizeBytes(len(hash))
if err != nil { if link.hasName {
return err size += protowire.SizeTag(2)
size += protowire.SizeBytes(len(link.name))
}
if link.hasTsize {
size += protowire.SizeTag(3)
size += protowire.SizeVarint(uint64(link.tsize))
}
enc = protowire.AppendTag(enc, 2, 2) // field & wire type for Links
enc = protowire.AppendVarint(enc, uint64(size))
enc = protowire.AppendTag(enc, 1, 2) // field & wire type for Hash
enc = protowire.AppendBytes(enc, hash)
if link.hasName {
enc = protowire.AppendTag(enc, 2, 2) // field & wire type for Name
enc = protowire.AppendString(enc, link.name)
} }
if wrote != size { if link.hasTsize {
return fmt.Errorf("bad PBLink marshal, wrote wrong number of bytes") enc = protowire.AppendTag(enc, 3, 0) // field & wire type for Tsize
enc = protowire.AppendVarint(enc, uint64(link.tsize))
} }
out.Write(chunk)
} }
} // if links } // if links
...@@ -132,71 +148,12 @@ func Marshal(inNode ipld.Node, out io.Writer) error { ...@@ -132,71 +148,12 @@ func Marshal(inNode ipld.Node, out io.Writer) error {
if err != nil { if err != nil {
return err return err
} }
size := uint64(len(byts)) enc = protowire.AppendTag(enc, 1, 2) // field & wire type for Data
lead := make([]byte, sizeOfVarint(size)+1) enc = protowire.AppendBytes(enc, byts)
lead[0] = 0xa // field and wireType for Data
encodeVarint(lead, 1, size)
out.Write(lead)
out.Write(byts)
}
return nil
}
// predict the byte size of the encoded Link
func (link pbLink) encodedSize() (n int) {
l := link.hash.ByteLen()
n += 1 + l + sizeOfVarint(uint64(l))
if link.hasName {
l = len(link.name)
n += 1 + l + sizeOfVarint(uint64(l))
}
if link.hasTsize {
n += 1 + sizeOfVarint(uint64(link.tsize))
} }
return n
}
// encode a Link to PB
func (link pbLink) marshal(data []byte, offset int) (int, error) {
base := offset
data[offset] = 0xa // field and wireType for Hash
byts := link.hash.Bytes()
offset = encodeVarint(data, offset+1, uint64(len(byts)))
copy(data[offset:], byts)
offset += len(byts)
if link.hasName {
data[offset] = 0x12 // field and wireType for Name
offset = encodeVarint(data, offset+1, uint64(len(link.name)))
copy(data[offset:], link.name)
offset += len(link.name)
}
if link.hasTsize {
data[offset] = 0x18 // field and wireType for Tsize
offset = encodeVarint(data, offset+1, uint64(link.tsize))
}
return offset - base, nil
}
// predict the size of a varint for PB before creating it
func sizeOfVarint(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
// encode a varint to a PB chunk
func encodeVarint(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
// stable sorting of Links using the strict sorting rules _, err = out.Write(enc)
func sortLinks(links []pbLink) { return err
sort.Stable(pbLinkSlice(links))
} }
type pbLinkSlice []pbLink type pbLinkSlice []pbLink
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment