ci(travis): redo benchmarks with travis

.circleci/config.yml

-version: 2.1
-
-aliases:
-  make_out_dirs: &make_out_dirs
-    run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_WORKSPACE $CIRCLE_TEST_REPORTS/{bitswap}
-  restore_gomod: &restore_gomod
-    restore_cache:
-      keys:
-        - go-bitswap-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-bitswap/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
-        - go-bitswap-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-bitswap/go.sum" }}-
-        - go-bitswap-dep-{{ .Branch }}-
-        - go-bitswap-dep--master-
-  store_gomod: &store_gomod
-      save_cache:
-        key: v4-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
-        paths:
-          - ~/go/pkg/mod
-          - ~/.cache/go-build/
-
-commands:
-  benchmark:
-    parameters:
-      output:
-        type: string
-      branch:
-        type: string
-        default: HEAD
-    steps:
-      - checkout
-      - run: git checkout << parameters.branch >>
-      - *make_out_dirs
-      - *restore_gomod
-      - run:
-          command: go test -run=NONE -bench=. ./... | tee /tmp/circleci-workspace/<< parameters.output >>
-          environment:
-            IPFS_LOGGING: critical
-            CGO_ENABLED: 0
-      - persist_to_workspace:
-          root: /tmp/circleci-workspace
-          paths:
-            - << parameters.output >>
-      - *store_gomod
-
-defaults: &defaults
-  working_directory: ~/ipfs/go-ipfs
-  environment:
-    GO111MODULE: "on"
-    TEST_NO_DOCKER: 1
-    TEST_NO_FUSE: 1
-    GOPATH: /home/circleci/go
-    CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
-    CIRCLE: 1
-    SERVICE: circle-ci
-    CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-    CIRCLE_WORKSPACE: /tmp/circleci-workspace
-    TEST_VERBOSE: 1
-    TRAVIS: 1
-
-jobs:
-  gobuild:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
-    steps:
-    - checkout
-    - *make_out_dirs
-    - *restore_gomod
-    - run:
-        command: bin/build.sh
-
-    - *store_gomod
-  benchmark-release:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
-    steps:
-      - benchmark:
-          output: benchmark-release.txt
-          branch: release
-  benchmark-before:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
-    steps:
-      - benchmark:
-          output: benchmark-before.txt
-          branch: master
-  benchmark-after:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
-    steps:
-      - benchmark:
-          output: benchmark-after.txt
-  benchmark-compare:
-    docker:
-      - image: circleci/golang:1.12
-    <<: *defaults
-    steps:
-      - checkout
-      - *make_out_dirs
-      - attach_workspace:
-          at: /tmp/circleci-workspace
-      - run: bin/diff-benchmarks.sh /tmp/circleci-workspace/benchmark-before.txt /tmp/circleci-workspace/benchmark-after.txt
-      - run: bin/diff-benchmarks.sh /tmp/circleci-workspace/benchmark-release.txt /tmp/circleci-workspace/benchmark-after.txt
-  
-workflows:
-  version: 2
-  test:
-    jobs:
-    - gobuild
-    - benchmark-before
-    - benchmark-after
-    - benchmark-release
-    - benchmark-compare:
-        requires:
-          - benchmark-after
-          - benchmark-before
-          - benchmark-release
\ No newline at end of file

.travis.yml

@@ -9,17 +9,17 @@ go:
 env:
   global:
     - GOTFLAGS="-race"
-  matrix:
     - BUILD_DEPTYPE=gomod
-
+  matrix:
+    - TEST_PHASE=test
+    - TEST_PHASE=benchmark
 
 # disable travis install
 install:
   - true
 
 script:
-  - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
-
+  - ./bin/test-exec.sh
 
 cache:
   directories:

bin/benchmark-to-file.sh

+#!/bin/bash
+output="$1"
+branch="$2"
+
+git checkout "$2"
+
+IPFS_LOGGING=critical go test -benchtime=3s -run=NONE -bench=. ./... | tee "$1"

bin/diff-benchmarks.sh

@@ -7,10 +7,10 @@ parse() {
     sed -n \
         -e 's/ *\t */\t/g' \
         -e '/^Benchmark/p' |
-        column -s'	' --json \
-               --table-columns name,count,time,rate \
-               --table-name "results" |
-        jq '.results[] | {name: .name, time: (.time | rtrimstr(" ns/op") | tonumber)}'
+        awk 'BEGIN{print "{\"results\": ["} {print "  {\"name\": \"",$1,"\", \"time\": ",$3," },"} END{print "]}"}' OFS="" ORS=" "|
+        sed -e 's/, ]/ ]/g' |
+        jq '.results[] | {name: .name, time: .time }'
+
 }
 
 benchcmp "$1" "$2"
@@ -21,7 +21,7 @@ echo "Result:"
 {
     parse < "$1"
     parse < "$2"
-} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[1].time / .[0].time)} | select(.speedup < 0.90) | "\(.name)\t\(.speedup)x"'
+} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[0].time / .[1].time)} | select(.speedup < 0.75) | "\(.name)\t\(.speedup)x"'
 
 if [[ $? -ne 4 ]]; then
     echo ""

bin/run-benchmarks.sh

+#!/bin/bash
+set -eo pipefail
+
+display_and_run() {
+    echo "***" "$@"
+	eval $(printf '%q ' "$@")
+}
+
+# reset workdir to state from git (to remove possible rewritten dependencies)
+display_and_run git reset --hard
+
+# Environment
+echo "*** Setting up test environment"
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    if [[ "$TRAVIS_SUDO" == true ]]; then
+        # Ensure that IPv6 is enabled.
+        # While this is unsupported by TravisCI, it still works for localhost.
+        sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0
+        sudo sysctl -w net.ipv6.conf.default.disable_ipv6=0
+        sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0
+    fi
+else
+    # OSX has a default file limit of 256, for some tests we need a
+    # maximum of 8192.
+    ulimit -Sn 8192
+fi
+
+list_buildable() {
+    go list -f '{{if (len .GoFiles)}}{{.ImportPath}} {{if .Module}}{{.Module.Dir}}{{else}}{{.Dir}}{{end}}{{end}}' ./... | grep -v /vendor/
+}
+
+build_all() {
+    # Make sure everything can compile since some package may not have tests
+    # Note: that "go build ./..." will fail if some packages have only
+    #   tests (will get "no buildable Go source files" error) so we
+    #   have to do this the hard way.
+    list_buildable | while read -r pkg dir; do
+        echo '*** go build' "$pkg"
+        buildmode=archive
+        if [[ "$(go list -f '{{.Name}}')" == "main" ]]; then
+            # plugin works even when a "main" function is missing.
+            buildmode=plugin
+        fi
+        ( cd "$dir"; go build -buildmode=$buildmode -o /dev/null "$pkg")
+    done
+}
+
+export GO111MODULE=on
+display_and_run go get golang.org/x/tools/cmd/benchcmp
+build_all
+display_and_run git reset --hard
+git checkout -b after
+git fetch origin master:refs/remotes/origin/before
+git checkout remotes/origin/before
+git checkout -b before
+git checkout after
+display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-before.txt before
+git checkout after
+display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-after.txt after
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+display_and_run ./bin/diff-benchmarks.sh benchmark-before.txt benchmark-after.txt
+fi
\ No newline at end of file

bin/test-exec.sh

+#!/bin/bash
+if [[ "$TEST_PHASE" == "test" ]]; then
+  bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
+else
+  ./bin/run-benchmarks.sh
+fi
\ No newline at end of file

go.mod

@@ -36,6 +36,6 @@ require (
 	golang.org/x/net v0.0.0-20190611141213-3f473d35a33a // indirect
 	golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae // indirect
 	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d // indirect
+	golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect
 	gopkg.in/yaml.v2 v2.2.2 // indirect
 )

go.sum

@@ -324,6 +324,8 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d h1:LQ06Vbju+Kwbcd94hb+6CgDsWoj/e7GOLPcYzHrG+iI=
 golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=