Commit 47a43bf7 authored by Eric Myhre's avatar Eric Myhre

Introduce variable scale benchmark. Woot!

This is one of the key goals I've been trying to work toward with all
the fixture nomenclature efforts.  (Correspondingly, finally this is
a naming pattern I'm satisfied with, as well.)

Here's an example output, from the ipldfree package:

```
BenchmarkSpec_Walk_Map3StrInt-8                          1501033               802 ns/op             624 B/op          9 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=0-8               6288885               191 ns/op              96 B/op          3 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=1-8               1000000              1160 ns/op             896 B/op         13 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=2-8                520072              2326 ns/op            1696 B/op         23 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=4-8                239588              4293 ns/op            3296 B/op         43 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=8-8                138903              8461 ns/op            6496 B/op         83 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=16-8                70845             16951 ns/op           12896 B/op        163 allocs/op
BenchmarkSpec_Walk_MapNStrMap3StrInt/n=32-8                33601             33793 ns/op           25696 B/op        323 allocs/op
```

This roughly linear increases in time taken and in allocs required as
the size of the data we walk is increased.

(What I'm hoping to see when we land the NodeAssembler refactor and
other key improvements to reduce interface boxing costs that were
co-developed with that change is: we'll see the allocs become O(1),
and the linearity on the time will become a much less sharp incline!)

I suspect one could parse these names back apart to draw graphs for
the various sizes.  (I think such tools exist out there already, though
I don't have a working knowledge of them yet.  If something suitable
doesn't exist already, the basic pieces to assemble something do:
https://github.com/golang/perf/blob/36b577b0eb03b831f9f591c1338a115cafcb56a7/storage/benchfmt/benchfmt.go#L31
has a parser for the output format.)

We should also be able to reuse these benchmarks for other Node
implementations fairly effortlessly.  I'm particularly looking forward
to seeing how that shapes up when we apply it to codegen'd stuff.

In the future, I want to rack up even more data in name patterns like:

 baselines.BenchmarkSpec_Unmarshal_MapStrInt/codec=json/n=3
 baselines.BenchmarkSpec_Unmarshal_MapStrInt/codec=json/n=25
 basicnode.BenchmarkSpec_Unmarshal_MapStrInt/codec=json/n=3
 basicnode.BenchmarkSpec_Unmarshal_MapStrInt/codec=json/n=25
 basicnode.BenchmarkSpec_Unmarshal_MapStrInt/codec=cbor/n=3
 basicnode.BenchmarkSpec_Unmarshal_MapStrInt/codec=cbor/n=25
   gendemo.BenchmarkSpec_Unmarshal_MapStrInt/codec=json/n=3
   gendemo.BenchmarkSpec_Unmarshal_MapStrInt/codec=json/n=25
   gendemo.BenchmarkSpec_Unmarshal_MapStrInt/codec=cbor/n=3
   gendemo.BenchmarkSpec_Unmarshal_MapStrInt/codec=cbor/n=25

The general pattern being:

  {nodeImplementation}.BenchmarkSpec_{Application}_{FixtureCohort}/codec={codec}/size={size}

Some variations like "codec=" will only exist for some applications,
e.g. it does exist for marshal and unmarshal, but of course doesn't
apply for traversal.
parent 3ac88b30
......@@ -96,8 +96,11 @@ func BenchmarkMarshalMapStrInt_3n(b *testing.B) {
// benchmarks covering traversal -->
func BenchmarkWalkMapStrInt_3n(b *testing.B) {
tests.SpecBenchmarkWalkMapStrInt_3n(b, NodeBuilder())
func BenchmarkSpec_Walk_Map3StrInt(b *testing.B) {
tests.BenchmarkSpec_Walk_Map3StrInt(b, NodeBuilder())
}
func BenchmarkSpec_Walk_MapNStrMap3StrInt(b *testing.B) {
tests.BenchmarkSpec_Walk_MapNStrMap3StrInt(b, NodeBuilder())
}
// copy of helper functions from must package, because import cycles, sigh -->
......
......@@ -49,3 +49,10 @@ func MapNStrInt(n int) string {
return fmt.Sprintf(`"k%d":%d`, i, i)
}) + `}`
}
func MapNStrMap3StrInt(n int) string {
return `{` + ents(n, func(i int) string {
return fmt.Sprintf(`"k%d":`, i) +
fmt.Sprintf(`{"whee":%d,"woot":%d,"waga":%d}`, i*3+1, i*3+2, i*3+3)
}) + `}`
}
......@@ -16,4 +16,7 @@ func TestCorpusValidity(t *testing.T) {
must.True(json.Valid([]byte(MapNStrInt(0))))
must.True(json.Valid([]byte(MapNStrInt(1))))
must.True(json.Valid([]byte(MapNStrInt(2))))
must.True(json.Valid([]byte(MapNStrMap3StrInt(0))))
must.True(json.Valid([]byte(MapNStrMap3StrInt(1))))
must.True(json.Valid([]byte(MapNStrMap3StrInt(2))))
}
......@@ -2,6 +2,7 @@ package tests
import (
"bytes"
"fmt"
"testing"
refmtjson "github.com/polydawn/refmt/json"
......@@ -28,15 +29,15 @@ func mustSelectorFromJsonString(nb ipld.NodeBuilder, str string) selector.Select
return sel
}
func SpecBenchmarkWalkMapStrInt_3n(b *testing.B, nb ipld.NodeBuilder) {
n := mustNodeFromJsonString(nb, corpus.Map3StrInt())
func BenchmarkSpec_Walk_Map3StrInt(b *testing.B, nb ipld.NodeBuilder) {
node := mustNodeFromJsonString(nb, corpus.Map3StrInt())
sel := mustSelectorFromJsonString(nb, `{"a":{">":{".":{}}}}`)
b.ResetTimer()
var visitCountSanityCheck int
for i := 0; i < b.N; i++ {
visitCountSanityCheck = 0
traversal.WalkMatching(n, sel, func(tp traversal.Progress, n ipld.Node) error {
traversal.WalkMatching(node, sel, func(tp traversal.Progress, n ipld.Node) error {
visitCountSanityCheck++ // this sanity check is sufficiently cheap to be worth it
return nil // no need to do anything here; just care about exercising the walk internals.
})
......@@ -45,3 +46,26 @@ func SpecBenchmarkWalkMapStrInt_3n(b *testing.B, nb ipld.NodeBuilder) {
b.Fatalf("visitCountSanityCheck should be 3, got %d", visitCountSanityCheck)
}
}
func BenchmarkSpec_Walk_MapNStrMap3StrInt(b *testing.B, nb ipld.NodeBuilder) {
sel := mustSelectorFromJsonString(nb, `{"a":{">":{"a":{">":{".":{}}}}}}`)
for _, n := range []int{0, 1, 2, 4, 8, 16, 32} {
b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) {
node := mustNodeFromJsonString(nb, corpus.MapNStrMap3StrInt(n))
b.ResetTimer()
var visitCountSanityCheck int
for i := 0; i < b.N; i++ {
visitCountSanityCheck = 0
traversal.WalkMatching(node, sel, func(tp traversal.Progress, n ipld.Node) error {
visitCountSanityCheck++ // this sanity check is sufficiently cheap to be worth it
return nil // no need to do anything here; just care about exercising the walk internals.
})
}
if visitCountSanityCheck != 3*n {
b.Fatalf("visitCountSanityCheck should be %d, got %d", n*3, visitCountSanityCheck)
}
})
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment