ci(circleci): add benchmark comparisons

Run benchmarks before and after, compare results

.circleci/config.yml

+version: 2.1
+
+aliases:
+  make_out_dirs: &make_out_dirs
+    run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_WORKSPACE $CIRCLE_TEST_REPORTS/{bitswap}
+  restore_gomod: &restore_gomod
+    restore_cache:
+      keys:
+        - go-bitswap-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-bitswap/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
+        - go-bitswap-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-bitswap/go.sum" }}-
+        - go-bitswap-dep-{{ .Branch }}-
+        - go-bitswap-dep--master-
+  store_gomod: &store_gomod
+      save_cache:
+        key: v4-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
+        paths:
+          - ~/go/pkg/mod
+          - ~/.cache/go-build/
+
+commands:
+  benchmark:
+    parameters:
+      output:
+        type: string
+      branch:
+        type: string
+        default: HEAD
+    steps:
+      - checkout
+      - run: git checkout << parameters.branch >>
+      - *make_out_dirs
+      - *restore_gomod
+      - run:
+          command: go test -run=NONE -bench=. ./... | tee /tmp/circleci-workspace/<< parameters.output >>
+          environment:
+            IPFS_LOGGING: critical
+            CGO_ENABLED: 0
+      - persist_to_workspace:
+          root: /tmp/circleci-workspace
+          paths:
+            - << parameters.output >>
+      - *store_gomod
+
+defaults: &defaults
+  working_directory: ~/ipfs/go-ipfs
+  environment:
+    GO111MODULE: "on"
+    TEST_NO_DOCKER: 1
+    TEST_NO_FUSE: 1
+    GOPATH: /home/circleci/go
+    CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+    CIRCLE: 1
+    SERVICE: circle-ci
+    CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    CIRCLE_WORKSPACE: /tmp/circleci-workspace
+    TEST_VERBOSE: 1
+    TRAVIS: 1
+
+jobs:
+  gobuild:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+    - checkout
+    - *make_out_dirs
+    - *restore_gomod
+    - run:
+        command: bin/build.sh
+
+    - *store_gomod
+  benchmark-release:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - benchmark:
+          output: benchmark-release.txt
+          branch: release
+  benchmark-before:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - benchmark:
+          output: benchmark-before.txt
+          branch: master
+  benchmark-after:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - benchmark:
+          output: benchmark-after.txt
+  benchmark-compare:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - checkout
+      - *make_out_dirs
+      - attach_workspace:
+          at: /tmp/circleci-workspace
+      - run: bin/diff-benchmarks.sh /tmp/circleci-workspace/benchmark-before.txt /tmp/circleci-workspace/benchmark-after.txt
+      - run: bin/diff-benchmarks.sh /tmp/circleci-workspace/benchmark-release.txt /tmp/circleci-workspace/benchmark-after.txt
+  
+workflows:
+  version: 2
+  test:
+    jobs:
+    - gobuild
+    - benchmark-before
+    - benchmark-after
+    - benchmark-release
+    - benchmark-compare:
+        requires:
+          - benchmark-after
+          - benchmark-before
+          - benchmark-release
\ No newline at end of file

bin/build.sh

+#!/bin/bash
+set -eo pipefail
+
+list_buildable() {
+    go list -f '{{if (len .GoFiles)}}{{.ImportPath}} {{if .Module}}{{.Module.Dir}}{{else}}{{.Dir}}{{end}}{{end}}' ./... | grep -v /vendor/
+}
+
+build_all() {
+    # Make sure everything can compile since some package may not have tests
+    # Note: that "go build ./..." will fail if some packages have only
+    #   tests (will get "no buildable Go source files" error) so we
+    #   have to do this the hard way.
+    list_buildable | while read -r pkg dir; do
+        echo '*** go build' "$pkg"
+        buildmode=archive
+        if [[ "$(go list -f '{{.Name}}')" == "main" ]]; then
+            # plugin works even when a "main" function is missing.
+            buildmode=plugin
+        fi
+        ( cd "$dir"; go build -buildmode=$buildmode -o /dev/null "$pkg")
+    done
+}
+
+build_all
\ No newline at end of file

bin/diff-benchmarks.sh

+#!/bin/bash
+
+before="$1"
+after="$2"
+
+parse() {
+    sed -n \
+        -e 's/ *\t */\t/g' \
+        -e '/^Benchmark/p' |
+        column -s'	' --json \
+               --table-columns name,count,time,rate \
+               --table-name "results" |
+        jq '.results[] | {name: .name, time: (.time | rtrimstr(" ns/op") | tonumber)}'
+}
+
+benchcmp "$1" "$2"
+
+echo ""
+echo "Result:"
+
+{
+    parse < "$1"
+    parse < "$2"
+} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[1].time / .[0].time)} | select(.speedup < 0.90) | "\(.name)\t\(.speedup)x"'
+
+if [[ $? -ne 4 ]]; then
+    echo ""
+    echo "FAIL"
+    exit 1
+else
+    echo "PASS"
+fi
\ No newline at end of file

go.mod

@@ -36,5 +36,6 @@ require (
 	golang.org/x/net v0.0.0-20190611141213-3f473d35a33a // indirect
 	golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae // indirect
 	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d // indirect
 	gopkg.in/yaml.v2 v2.2.2 // indirect
 )

go.sum

@@ -300,11 +300,13 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -320,6 +322,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d h1:LQ06Vbju+Kwbcd94hb+6CgDsWoj/e7GOLPcYzHrG+iI=
+golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=