ci: Coverage and Junit xmls in one run

License: MIT
Signed-off-by: Łukasz Magiera <magik6k@gmail.com>

.circleci/config.yml

@@ -51,8 +51,9 @@ jobs:
 
 
     - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && curl -s https://codecov.io/bash > codecov
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make -j 1 coverage/unit_tests.coverprofile
+    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make -j 1 test/unit/gotest.junit.xml
     - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && bash codecov -cF unittests -X search -f coverage/unit_tests.coverprofile
+    - run: mv "$HOME/.go_workspace/src/${IMPORT_PATH}/test/unit/gotest.junit.xml" /tmp/circleci-test-results
     - store_test_results:
         path: /tmp/circleci-test-results
     # Save artifacts
@@ -89,10 +90,13 @@ jobs:
         - ~/.go_workspace/src/gx/ipfs
         - ./node_modules
 
-
     - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && curl -s https://codecov.io/bash > codecov
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make -j 1 coverage/sharness_tests.coverprofile
+    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1
     - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && bash codecov -cF sharness -X search -f coverage/sharness_tests.coverprofile
+    - run: mv "$HOME/.go_workspace/src/${IMPORT_PATH}/test/sharness/test-results/sharness.xml" /tmp/circleci-test-results
+
+    # make sure we fail if there are test failures
+    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && ls test/sharness/test-results/t*-*.sh.*.counts | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0'
     - store_test_results:
         path: /tmp/circleci-test-results
     # Save artifacts

Rules.mk

@@ -41,7 +41,7 @@ include $(dir)/Rules.mk
 
 # include this file only if coverage target is executed
 # it is quite expensive
-ifneq ($(filter coverage% clean distclean,$(MAKECMDGOALS)),)
+ifneq ($(filter coverage% clean distclean test/unit/gotest.junit.xml,$(MAKECMDGOALS)),)
 	# has to be after cmd/ipfs due to PATH
 	dir := coverage
 	include $(dir)/Rules.mk

bin/circle.sh

-#!/usr/bin/env bash
-# the CircleCI build line got a bit out of hands
-# thus we have sparate file for it
-
-curl -s https://codecov.io/bash > codecov
-
-case $CIRCLE_NODE_INDEX in
-	0) make -j 1 coverage/unit_tests.coverprofile &&
-		bash codecov -cF unittests -X search -f coverage/unit_tests.coverprofile
-	;;
-	1) make -j 1 coverage/sharness_tests.coverprofile &&
-		bash codecov -cF sharness -X search -f coverage/sharness_tests.coverprofile
-	;;
-esac

coverage/Rules.mk

@@ -11,6 +11,7 @@ endif
 
 # unit tests coverage
 UTESTS_$(d) := $(shell go list -f '{{if (len .TestGoFiles)}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./...)
+UTESTS_$(d) += $(shell go list -f '{{if (len .XTestGoFiles)}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./... | grep -v go-ipfs/vendor | grep -v go-ipfs/Godeps)
 
 UCOVER_$(d) := $(addsuffix .coverprofile,$(addprefix $(d)/unitcover/, $(subst /,_,$(UTESTS_$(d)))))
 
@@ -18,7 +19,7 @@ $(UCOVER_$(d)): $(d)/coverage_deps ALWAYS
 	$(eval TMP_PKG := $(subst _,/,$(basename $(@F))))
 	$(eval TMP_DEPS := $(shell go list -f '{{range .Deps}}{{.}} {{end}}' $(go-flags-with-tags) $(TMP_PKG) | sed 's/ /\n/g' | grep ipfs/go-ipfs) $(TMP_PKG))
 	$(eval TMP_DEPS_LIST := $(call join-with,$(comma),$(TMP_DEPS)))
-	go test $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverpkg=$(TMP_DEPS_LIST) -coverprofile=$@ $(TMP_PKG)
+	go test $(go-flags-with-tags) $(GOTFLAGS) -v -covermode=atomic -json -coverpkg=$(TMP_DEPS_LIST) -coverprofile=$@ $(TMP_PKG) | tee -a test/unit/gotest.json
 
 
 $(d)/unit_tests.coverprofile: $(UCOVER_$(d))
@@ -26,6 +27,7 @@ $(d)/unit_tests.coverprofile: $(UCOVER_$(d))
 
 TGTS_$(d) := $(d)/unit_tests.coverprofile
 
+.PHONY: $(d)/unit_tests.coverprofile
 
 # sharness tests coverage
 $(d)/ipfs: GOTAGS += testrunmain

test/Rules.mk

@@ -6,4 +6,7 @@ include $(dir)/Rules.mk
 dir := $(d)/sharness
 include $(dir)/Rules.mk
 
+dir := $(d)/unit
+include $(dir)/Rules.mk
+
 include mk/footer.mk

test/bin/Rules.mk

@@ -18,6 +18,10 @@ $(d)/ma-pipe-unidir: test/dependencies/ma-pipe-unidir
 	$(go-build)
 TGTS_$(d) += $(d)/ma-pipe-unidir
 
+$(d)/json-to-junit: test/dependencies/json-to-junit
+	$(go-build)
+TGTS_$(d) += $(d)/json-to-junit
+
 TGTS_GX_$(d) := hang-fds iptb
 TGTS_GX_$(d) := $(addprefix $(d)/,$(TGTS_GX_$(d)))
 

test/dependencies/json-to-junit/main.go

+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"encoding/xml"
+	"log"
+	"os"
+)
+
+type testMsg struct {
+	Time    string
+	Action  string
+	Package string
+	Test    string
+	Output  string
+	Elapsed float64
+}
+
+type testsuites struct {
+	XMLName    xml.Name `xml:"testsuites"`
+	Name       string   `xml:"name,attr"`
+	Testsuites []testsuite
+}
+
+type testsuite struct {
+	XMLName   xml.Name `xml:"testsuite"`
+	Package   string   `xml:"package,attr"`
+	Errors    int      `xml:"errors,attr"`
+	Failures  int      `xml:"failures,attr"`
+	Tests     int      `xml:"tests,attr"`
+	Time      float64  `xml:"time,attr"`
+	Testcases []testcase
+}
+
+type testcase struct {
+	XMLName   xml.Name `xml:"testcase"`
+	Name      string   `xml:"name,attr"`
+	Classname string   `xml:"classname,attr"`
+	Time      float64  `xml:"time,attr"`
+	Sout      string   `xml:"system-out"`
+	Serr      string   `xml:"system-err"`
+	Skipped   string   `xml:"skipped,omitempty"`
+	Failure   string   `xml:"failure,omitempty"`
+}
+
+func main() {
+	scanner := bufio.NewScanner(os.Stdin)
+	tests := make(map[string]map[string]testcase)
+	var fails int
+	var packages []testsuite
+	for scanner.Scan() {
+		msg := testMsg{}
+		if err := json.Unmarshal(scanner.Bytes(), &msg); err != nil {
+			log.Fatal(err)
+		}
+
+		switch {
+		case msg.Action == "run":
+			if tests[msg.Package] == nil {
+				tests[msg.Package] = make(map[string]testcase)
+			}
+
+			tests[msg.Package][msg.Test] = testcase{
+				Name:      msg.Test,
+				Classname: msg.Package,
+			}
+		case msg.Action == "output":
+			if msg.Test == "" {
+				continue
+			}
+
+			test := tests[msg.Package][msg.Test]
+			test.Sout = test.Sout + msg.Output + "\n"
+			tests[msg.Package][msg.Test] = test
+		case msg.Action == "skip":
+			fallthrough
+		case msg.Action == "fail":
+			fallthrough
+		case msg.Action == "pass":
+			if msg.Test != "" {
+				test := tests[msg.Package][msg.Test]
+				test.Time = msg.Elapsed
+
+				if msg.Action == "skip" {
+					test.Skipped = "skipped"
+				}
+
+				if msg.Action == "fail" {
+					fails++
+					test.Failure = "failed"
+				}
+
+				tests[msg.Package][msg.Test] = test
+				continue
+			}
+
+			ts := testsuite{
+				Package: msg.Package,
+				Time:    msg.Elapsed,
+			}
+
+			for _, test := range tests[msg.Package] {
+				ts.Testcases = append(ts.Testcases, test)
+				ts.Tests++
+				if test.Failure != "" {
+					ts.Failures++
+				}
+			}
+			packages = append(packages, ts)
+		case msg.Action == "cont":
+		case msg.Action == "pause":
+			// ??
+		default:
+			log.Fatalf("unknown action %s", msg.Action)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	out := testsuites{
+		Name:       "go test",
+		Testsuites: packages,
+	}
+
+	output, err := xml.MarshalIndent(&out, "  ", "    ")
+	if err != nil {
+		log.Fatalf("error: %v\n", err)
+	}
+
+	os.Stdout.Write(output)
+	os.Exit(fails)
+}

test/unit/.gitignore

+gotest.json
+gotest.junit.xml

test/unit/Rules.mk

+include mk/header.mk
+
+CLEAN += $(d)/gotest.json $(d)/gotest.junit.xml
+
+$(d)/gotest.junit.xml: clean test/bin/json-to-junit coverage/unit_tests.coverprofile
+	cat $(@D)/gotest.json | json-to-junit > $(@D)/gotest.junit.xml
+
+include mk/footer.mk