Merge pull request #147 from ipfs/feat/benchmark-ci

ci(circleci): add benchmark comparisons

.travis.yml

@@ -9,17 +9,17 @@ go:
 env:
   global:
     - GOTFLAGS="-race"
-  matrix:
     - BUILD_DEPTYPE=gomod
-
+  matrix:
+    - TEST_PHASE=test
+    - TEST_PHASE=benchmark
 
 # disable travis install
 install:
   - true
 
 script:
-  - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
-
+  - ./bin/test-exec.sh
 
 cache:
   directories:

bin/benchmark-to-file.sh

+#!/bin/bash
+
+git checkout "$2"
+
+IPFS_LOGGING=critical go test -benchtime=3s -run=NONE -bench=. ./... | tee "$1"

bin/diff-benchmarks.sh

+#!/bin/bash
+
+parse() {
+    sed -n \
+        -e 's/ *\t */\t/g' \
+        -e '/^Benchmark/p' |
+        awk 'BEGIN{print "{\"results\": ["} {print "  {\"name\": \"",$1,"\", \"time\": ",$3," },"} END{print "]}"}' OFS="" ORS=" "|
+        sed -e 's/, ]/ ]/g' |
+        jq '.results[] | {name: .name, time: .time }'
+
+}
+
+benchcmp "$1" "$2"
+
+echo ""
+echo "Result:"
+
+{
+    parse < "$1"
+    parse < "$2"
+} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[0].time / .[1].time)} | select(.speedup < 0.75) | "\(.name)\t\(.speedup)x"'
+
+if [[ $? -ne 4 ]]; then
+    echo ""
+    echo "FAIL"
+    exit 1
+else
+    echo "PASS"
+fi

bin/run-benchmarks.sh

+#!/bin/bash
+set -eo pipefail
+
+display_and_run() {
+    echo "***" "$@"
+	eval "$(printf '%q ' "$@")"
+}
+
+# reset workdir to state from git (to remove possible rewritten dependencies)
+export GO111MODULE=on
+display_and_run go get golang.org/x/tools/cmd/benchcmp
+display_and_run git reset --hard
+git checkout -b after
+git fetch origin master:refs/remotes/origin/before
+git checkout remotes/origin/before
+git checkout -b before
+git checkout after
+display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-before.txt before
+git checkout after
+display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-after.txt after
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+display_and_run ./bin/diff-benchmarks.sh benchmark-before.txt benchmark-after.txt
+fi

bin/test-exec.sh

+#!/bin/bash
+if [[ "$TEST_PHASE" == "test" ]]; then
+  bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
+else
+  ./bin/run-benchmarks.sh
+fi
\ No newline at end of file

go.mod

@@ -36,5 +36,6 @@ require (
 	golang.org/x/net v0.0.0-20190611141213-3f473d35a33a // indirect
 	golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae // indirect
 	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect
 	gopkg.in/yaml.v2 v2.2.2 // indirect
 )

go.sum

@@ -300,11 +300,13 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -320,6 +322,10 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d h1:LQ06Vbju+Kwbcd94hb+6CgDsWoj/e7GOLPcYzHrG+iI=
+golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=