Commit 10f6b329 authored by Steven Allen's avatar Steven Allen

ci: remove travis and upstream benchmark logic

parent e86a535c
version: 2.1 version: 2.1
orbs: orbs:
ci-go: ipfs/ci-go@0.1 ci-go: ipfs/ci-go@0.1
commands:
benchcmp:
parameters:
before:
description: "File containing the old benchmarks."
type: string
after:
description: "File containing the new benchmarks."
type: string
tolerance:
description: "Tolerable slowdown (%)."
type: integer
default: 25
environment:
BEFORE: << parameters.before >>
AFTER: << parameters.after >>
steps:
- run:
name: "Comparing benchmarks."
command: |
parse() {
sed -n \
-e 's/ *\t */\t/g' \
-e '/^Benchmark/p' |
awk 'BEGIN{print "{\"results\": ["} {print " {\"name\": \"",$1,"\", \"time\": ",$3," },"} END{print "]}"}' OFS="" ORS=" "|
sed -e 's/, ]/ ]/g' |
jq '.results[] | {name: .name, time: .time }'
}
benchcmp "$BEFORE" "$AFTER"
echo ""
echo "Result:"
{
parse < "$BEFORE"
parse < "$AFTER"
} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[0].time / .[1].time)} | select(.speedup < (100 - << parameters.tolerance >>)/100) | "\(.name)\t\(.speedup)x"'
if [[ $? -ne 4 ]]; then
echo ""
echo "FAIL"
exit 1
else
echo "PASS"
fi
benchmark:
parameters:
output:
description: "File to write the results to."
type: string
branch:
description: "Branch to benchmark."
type: string
default: HEAD
command:
description: "Benchmark command."
type: string
default: go test -run=NONE -bench=. ./...
steps:
- run:
name: Check out << parameters.branch >>
command: |
mkdir /tmp/benchmark
git --work-tree=/tmp/benchmark checkout << parameters.branch >>
- run:
name: "Benchmarking"
command: |
{
<< parameters.command >>
} | tee << parameters.output >>
working_directory: /tmp/benchmark
environment:
IPFS_LOGGING: critical
- run:
when: always
name: Cleanup
command: rm -rf /tmp/benchmark
jobs:
benchmark:
parameters:
tolerance:
description: "Tolerable slowdown (%)."
type: integer
default: 25
baseline:
description: "Baseline branch to which to compare (if any)."
type: string
default: ""
command:
description: "Benchmark command."
type: string
default: go test -run=NONE -bench=. ./...
executor: ci-go/default
steps:
- checkout
- ci-go/restore-cache
- benchmark:
command: << parameters.command >>
branch: master
output: ~/benchmark-before.txt
- benchmark:
command: << parameters.command >>
output: ~/benchmark-after.txt
- benchcmp:
before: ~/benchmark-before.txt
after: ~/benchmark-after.txt
tolerance: << parameters.tolerance >>
- when:
condition: << parameters.baseline >>
steps:
- benchmark:
command: << parameters.command >>
branch: << parameters.baseline >>
output: ~/benchmark-baseline.txt
- benchcmp:
before: ~/benchmark-baseline.txt
after: ~/benchmark-after.txt
tolerance: << parameters.tolerance >>
- ci-go/save-cache
workflows: workflows:
version: 2 version: 2
test: test:
jobs: jobs:
- ci-go/build - ci-go/build
- ci-go/lint - ci-go/lint
- ci-go/test - ci-go/test
- benchmark: - ci-go/benchmark:
require: ci-go/test tolerance: 50
requires:
- ci-go/test
os:
- linux
language: go
go:
- 1.11.x
env:
global:
- GOTFLAGS="-race"
- BUILD_DEPTYPE=gomod
matrix:
- TEST_PHASE=test
- TEST_PHASE=benchmark
# disable travis install
install:
- true
script:
- ./bin/test-exec.sh
cache:
directories:
- $GOPATH/src/gx
- $GOPATH/pkg/mod
- $HOME/.cache/go-build
notifications:
email: false
#!/bin/bash
git checkout "$2"
IPFS_LOGGING=critical go test -benchtime=3s -run=NONE -bench=. ./... | tee "$1"
#!/bin/bash
parse() {
sed -n \
-e 's/ *\t */\t/g' \
-e '/^Benchmark/p' |
awk 'BEGIN{print "{\"results\": ["} {print " {\"name\": \"",$1,"\", \"time\": ",$3," },"} END{print "]}"}' OFS="" ORS=" "|
sed -e 's/, ]/ ]/g' |
jq '.results[] | {name: .name, time: .time }'
}
benchcmp "$1" "$2"
echo ""
echo "Result:"
{
parse < "$1"
parse < "$2"
} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[0].time / .[1].time)} | select(.speedup < 0.75) | "\(.name)\t\(.speedup)x"'
if [[ $? -ne 4 ]]; then
echo ""
echo "FAIL"
exit 1
else
echo "PASS"
fi
#!/bin/bash
set -eo pipefail
display_and_run() {
echo "***" "$@"
eval "$(printf '%q ' "$@")"
}
# reset workdir to state from git (to remove possible rewritten dependencies)
export GO111MODULE=on
display_and_run go get golang.org/x/tools/cmd/benchcmp
display_and_run git reset --hard
git checkout -b after
git fetch origin master:refs/remotes/origin/before
git checkout remotes/origin/before
git checkout -b before
git checkout after
display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-before.txt before
git checkout after
display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-after.txt after
if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
display_and_run ./bin/diff-benchmarks.sh benchmark-before.txt benchmark-after.txt
fi
#!/bin/bash
if [[ "$TEST_PHASE" == "test" ]]; then
bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
else
./bin/run-benchmarks.sh
fi
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment