Merge pull request #493 from jbenet/sharness-verbose

sharness: verbose output

.travis.yml

@@ -8,4 +8,4 @@ go:
 script:
   - make test
 
-env: TEST_NO_FUSE=1
+env: TEST_NO_FUSE=1 TEST_VERBOSE=1

Makefile

@@ -30,7 +30,7 @@ test_sharness:
 	cd test/ && make
 
 test_sharness_expensive:
-	cd test/ && make TEST_EXPENSIVE=1
+	cd test/ && TEST_EXPENSIVE=1 make
 
 test_all_commits:
 	@echo "testing all commits between origin/master..HEAD"

test/Makefile

@@ -4,6 +4,8 @@
 # MIT Licensed; see the LICENSE file in this repository.
 #
 
+# NOTE: run with TEST_VERBOSE=1 for verbose sharness tests.
+
 T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
 SHARNESS = lib/sharness/sharness.sh
 RANDOMSRC = Godeps/_workspace/src/github.com/jbenet/go-random/random

test/README.md

@@ -3,6 +3,19 @@
 ## Running all the tests
 
 Just use `make` in this directory to run all the tests.
+Run with `TEST_VERBOSE=1` to get helpful verbose output.
+
+```
+TEST_VERBOSE=1 make
+```
+
+The usual ipfs env flags also apply:
+
+```sh
+# the output will make your eyes bleed
+IPFS_LOGGING=debug TEST_VERBOSE=1 make
+```
+
 
 ## Running just one test
 
@@ -24,3 +37,66 @@ Please do not change anything in the "lib/sharness" directory.
 If you really need some changes in sharness, please fork it from
 [its cannonical repo](https://github.com/mlafeldt/sharness/) and
 send pull requests there.
+
+## Writing Tests
+
+
+### Diagnostics
+
+Make your test case output helpful for when running sharness verbosely.
+This means cating certain files, or running diagnostic commands.
+For example:
+
+```
+test_expect_success ".go-ipfs/ has been created" '
+  test -d ".go-ipfs" &&
+  test -f ".go-ipfs/config" &&
+  test -d ".go-ipfs/datastore" ||
+  fsh ls -al .go-ipfs
+'
+```
+
+The `|| ...` is a diagnostic run when the preceding command fails.
+`bin/fsh` is a trivial script that echoes the args, runs the cmd,
+and then also fails, making sure the test case fails. (wouldnt want
+the diagnostic accidentally returning true and making it _seem_ like
+the test case succeeded!).
+
+
+### Testing commands on daemon or mounted
+
+Use the provided functions in `lib/test-lib.sh` to run the daemon or mount:
+
+To init, run daemon, and mount in one go:
+
+```sh
+test_launch_ipfs_daemon_and_mount
+
+test_expect_success "'ipfs add --help' succeeds" '
+  ipfs add --help >actual
+'
+
+# other tests here...
+
+# dont forget to kill the daemon!!
+test_kill_ipfs_daemon
+```
+
+To init, run daemon, and then mount separately:
+
+```sh
+test_init_ipfs
+
+# tests inited but not running here
+
+test_launch_ipfs_daemon
+
+# tests running but not mounted here
+
+test_mount_ipfs
+
+# tests mounted here
+
+# dont forget to kill the daemon!!
+test_kill_ipfs_daemon
+```

test/bin/fsh

+#!/bin/sh
+# Author: Juan Batiz-Benet <juan@benet.ai>
+# MIT LICENSED
+
+# verbose eval, and exit with error, so we can avoid writing:
+# echo "cat version.txt" && cat version.txt && false
+
+# echo "# > $@"
+# eval $@ | sed -e 's/^/# /'
+echo "> $@"
+eval $@
+echo ""
+exit 1

test/lib/test-lib.sh

@@ -11,6 +11,11 @@
 # add current directory to path, for ipfs tool.
 PATH=$(pwd)/bin:${PATH}
 
+# set sharness verbosity. we set the env var directly as
+# it's too late to pass in --verbose, and --verbose is harder
+# to pass through in some cases.
+test "$TEST_VERBOSE" = 1 && verbose=t
+
 # assert the `ipfs` we're using is the right one.
 if test `which ipfs` != $(pwd)/bin/ipfs; then
 	echo >&2 "Cannot find the tests' local ipfs tool."
@@ -26,6 +31,13 @@ SHARNESS_LIB="lib/sharness/sharness.sh"
 	exit 1
 }
 
+# overriding testcmp to make it use fsh (to see it better in output)
+# have to do it twice so the first diff output doesnt show unless it's
+# broken.
+test_cmp() {
+	diff -q "$@" >/dev/null || fsh diff -u "$@"
+}
+
 # Please put go-ipfs specific shell functions below
 
 test "$TEST_NO_FUSE" != 1 && test_set_prereq FUSE
@@ -34,7 +46,7 @@ test "$TEST_EXPENSIVE" = 1 && test_set_prereq EXPENSIVE
 test_cmp_repeat_10_sec() {
 	for i in 1 2 3 4 5 6 7 8 9 10
 	do
-		test_cmp "$1" "$2" && return
+		test_cmp "$1" "$2" >/dev/null && return
 		sleep 1
 	done
 	test_cmp "$1" "$2"
@@ -52,24 +64,11 @@ test_wait_output_n_lines_60_sec() {
 	test_cmp "expected_waitn" "actual_waitn"
 }
 
-test_launch_ipfs_daemon() {
-
-	test_expect_success FUSE "'ipfs daemon' succeeds" '
-		ipfs daemon >actual &
-	'
-
-	test_expect_success FUSE "'ipfs daemon' output looks good" '
-		IPFS_PID=$! &&
-		echo "daemon listening on /ip4/127.0.0.1/tcp/5001" >expected &&
-		test_cmp_repeat_10_sec expected actual
-	'
-}
-
-test_launch_ipfs_daemon_and_mount() {
+test_init_ipfs() {
 
 	test_expect_success "ipfs init succeeds" '
 		export IPFS_DIR="$(pwd)/.go-ipfs" &&
-		ipfs init -b=1024
+		ipfs init -b=1024 > /dev/null
 	'
 
 	test_expect_success "prepare config" '
@@ -78,7 +77,23 @@ test_launch_ipfs_daemon_and_mount() {
 		ipfs config Mounts.IPNS "$(pwd)/ipns"
 	'
 
-	test_launch_ipfs_daemon
+}
+
+test_launch_ipfs_daemon() {
+
+	test_expect_success FUSE "'ipfs daemon' succeeds" '
+		ipfs daemon >actual 2>daemon_err &
+	'
+
+	test_expect_success FUSE "'ipfs daemon' output looks good" '
+		IPFS_PID=$! &&
+		echo "daemon listening on /ip4/127.0.0.1/tcp/5001" >expected &&
+		test_cmp_repeat_10_sec expected actual ||
+		fsh cat daemon_err
+	'
+}
+
+test_mount_ipfs() {
 
 	test_expect_success FUSE "'ipfs mount' succeeds" '
 		ipfs mount >actual
@@ -89,6 +104,15 @@ test_launch_ipfs_daemon_and_mount() {
 		echo "IPNS mounted at: $(pwd)/ipns" >>expected &&
 		test_cmp expected actual
 	'
+
+}
+
+test_launch_ipfs_daemon_and_mount() {
+
+	test_init_ipfs
+	test_launch_ipfs_daemon
+	test_mount_ipfs
+
 }
 
 test_kill_repeat_10_sec() {

test/t0010-basic-commands.sh

@@ -17,7 +17,8 @@ test_expect_success "ipfs version succeeds" '
 '
 
 test_expect_success "ipfs version output looks good" '
-	cat version.txt | egrep "^ipfs version [0-9]+\.[0-9]+\.[0-9]"
+	cat version.txt | egrep "^ipfs version [0-9]+\.[0-9]+\.[0-9]" >/dev/null ||
+	fsh cat version.txt
 '
 
 test_expect_success "ipfs help succeeds" '
@@ -25,9 +26,9 @@ test_expect_success "ipfs help succeeds" '
 '
 
 test_expect_success "ipfs help output looks good" '
-	cat help.txt | egrep -i "^Usage:" &&
-	cat help.txt | egrep "ipfs .* <command>"
+	cat help.txt | egrep -i "^Usage:" >/dev/null &&
+	cat help.txt | egrep "ipfs .* <command>" >/dev/null ||
+	fsh cat help.txt
 '
 
 test_done
-

test/t0020-init.sh

@@ -16,7 +16,8 @@ test_expect_success "ipfs init succeeds" '
 test_expect_success ".go-ipfs/ has been created" '
 	test -d ".go-ipfs" &&
 	test -f ".go-ipfs/config" &&
-	test -d ".go-ipfs/datastore"
+	test -d ".go-ipfs/datastore" ||
+	fsh ls -al .go-ipfs
 '
 
 test_expect_success "ipfs config succeeds" '

test/t0030-mount.sh

@@ -15,25 +15,32 @@ if ! test_have_prereq FUSE; then
 	test_done
 fi
 
-test_launch_ipfs_daemon_and_mount
-
-test_kill_ipfs_daemon
-
-test_expect_success "mount directories can be removed" '
-	rmdir ipfs ipns
-'
-
+test_init_ipfs
 test_launch_ipfs_daemon
 
+# run this mount failure before mounting properly.
+
 test_expect_failure "'ipfs mount' fails when no mount dir (issue #341)" '
-	test_must_fail ipfs mount >actual
+	test_must_fail ipfs mount -f=not_ipfs -n=not_ipns >actual
 '
 
 test_expect_failure "'ipfs mount' looks good when it fails (issue #341)" '
-	! grep "IPFS mounted at" actual &&
-	! grep "IPNS mounted at" actual
+	! grep "IPFS mounted at: $(pwd)/ipfs" actual >/dev/null &&
+	! grep "IPNS mounted at: $(pwd)/ipns" actual >/dev/null ||
+	fsh cat actual
+'
+
+# now mount properly, and keep going
+test_mount_ipfs
+
+test_expect_success "mount directories cannot be removed while active" '
+	test_must_fail rmdir ipfs ipns 2>/dev/null
 '
 
 test_kill_ipfs_daemon
 
+test_expect_success "mount directories can be removed after shutdown" '
+	rmdir ipfs ipns
+'
+
 test_done

test/t0040-add-and-cat.sh

@@ -15,7 +15,8 @@ test_expect_success "'ipfs add --help' succeeds" '
 '
 
 test_expect_success "'ipfs add --help' output looks good" '
-	egrep "ipfs add.*<path>" actual
+	egrep "ipfs add.*<path>" actual >/dev/null ||
+	fsh cat actual
 '
 
 test_expect_success "'ipfs cat --help' succeeds" '
@@ -23,7 +24,8 @@ test_expect_success "'ipfs cat --help' succeeds" '
 '
 
 test_expect_success "'ipfs cat --help' output looks good" '
-	egrep "ipfs cat.*<ipfs-path>" actual
+	egrep "ipfs cat.*<ipfs-path>" actual >/dev/null ||
+	fsh cat actual
 '
 
 test_expect_success "ipfs add succeeds" '

test/t0050-block.sh

@@ -8,10 +8,7 @@ test_description="Test block command"
 
 . lib/test-lib.sh
 
-test_expect_success "ipfs init succeeds" '
-	export IPFS_DIR="$(pwd)/.go-ipfs" &&
-	ipfs init
-'
+test_init_ipfs
 
 test_expect_success "'ipfs block put' succeeds" '
 	echo "Hello Mars!" >expected_in &&

test/t0060-daemon.sh

@@ -46,7 +46,8 @@ test_expect_success "ipfs daemon output looks good" '
 test_expect_success ".go-ipfs/ has been created" '
   test -d ".go-ipfs" &&
   test -f ".go-ipfs/config" &&
-  test -d ".go-ipfs/datastore"
+  test -d ".go-ipfs/datastore" ||
+  fsh ls .go-ipfs
 '
 
 test_expect_success "daemon is still running" '