Merge pull request #7140 from ipfs/chore/harden_impexp_test

Beef up and harden import/export tests

test/sharness/t0054-dag-car-import-export.sh

@@ -9,17 +9,19 @@ export -f ipfsi
 set -o pipefail
 
 tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz
+tab=$'\t'
 
 reset_blockstore() {
   node=$1
-  ipfsi $1 pin ls --quiet --type=recursive | ipfsi $1 pin rm &>/dev/null
-  ipfsi $1 repo gc &>/dev/null
+
+  ipfsi $node pin ls --quiet --type=recursive | ipfsi $node pin rm &>/dev/null
+  ipfsi $node repo gc &>/dev/null
 
   test_expect_success "pinlist empty" '
-    [ "$( ipfsi $1 pin ls )" = "" ]
+    [[ -z "$( ipfsi $node pin ls )" ]]
   '
   test_expect_success "nothing left to gc" '
-    [ "$( ipfsi $1 repo gc )" = "" ]
+    [[ -z "$( ipfsi $node repo gc )" ]]
   '
 }
 
@@ -27,14 +29,13 @@ reset_blockstore() {
 do_import() {
   node=$1; shift
 
-  bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc1_pid=$!
-  bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc2_pid=$!
+  touch spin.gc
+  timeout -s QUIT 15 bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc &>>gc_out; done" & gc1_pid=$!
+  timeout -s QUIT 15 bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc &>>gc_out; done" & gc2_pid=$!
 
-  ipfsi $node dag import "$@"
+  timeout -s QUIT 10 bash -c "ipfsi $node dag import $* 2>&1"
 
   rm -f spin.gc || true
-  sleep 3
-  kill $gc1_pid $gc2_pid || true
 }
 
 run_online_imp_exp_tests() {
@@ -42,11 +43,18 @@ run_online_imp_exp_tests() {
   reset_blockstore 0
   reset_blockstore 1
 
-  echo -e "Pinned root\tbafkqaaa\tsuccess" > basic_import_expected
-  echo -e "Pinned root\tbafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u\tsuccess" >> basic_import_expected
-  echo -e "Pinned root\tbafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy\tsuccess" >> basic_import_expected
+  cat > basic_import_expected <<EOE
+Pinned root${tab}bafkqaaa${tab}success
+Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
+Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
+EOE
+
+  cat >naked_root_import_json_expected <<EOE
+{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
+{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
+EOE
+
 
-  touch spin.gc
   test_expect_success "basic import" '
     do_import 0 \
       ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
@@ -57,12 +65,39 @@ run_online_imp_exp_tests() {
 
   # FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed
   test_expect_failure "concurrent GC did not manage to grab anything and remained silent" '
-    ! [[ -s gc_out ]]
+    test_cmp /dev/null gc_out
   '
   test_expect_success "basic import output as expected" '
     test_cmp basic_import_expected basic_import_actual
   '
 
+  test_expect_success "basic fetch+export 1" '
+    ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
+  '
+  test_expect_success "export of shuffled testnet export identical to canonical original" '
+    test_cmp reexported_testnet_128.car ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car
+  '
+
+  test_expect_success "basic fetch+export 2" '
+    ipfsi 1 dag export bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u > reexported_devnet_genesis.car
+  '
+  test_expect_success "export of shuffled devnet export identical to canonical original" '
+    test_cmp reexported_devnet_genesis.car ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car
+  '
+
+  test_expect_success "pinlist on node1 still empty" '
+    [[ -z "$( ipfsi 1 pin ls )" ]]
+  '
+
+  test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
+    ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
+      | sort > naked_import_result_json_actual
+  '
+
+  test_expect_success "naked import output as expected" '
+    test_cmp  naked_root_import_json_expected naked_import_result_json_actual
+  '
+
   reset_blockstore 0
   reset_blockstore 1
 
@@ -81,7 +116,6 @@ run_online_imp_exp_tests() {
     bash -c "sleep 60; kill $cat1_pid $cat2_pid 2>/dev/null" &
   ' &
 
-  touch spin.gc
   test_expect_success "fifo import" '
     do_import 0 \
       pipe_testnet \
@@ -91,7 +125,7 @@ run_online_imp_exp_tests() {
   '
   # FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed
   test_expect_failure "concurrent GC did not manage to grab anything and remained silent" '
-    ! [[ -s gc_out ]]
+    test_cmp /dev/null gc_out
   '
 
   test_expect_success "fifo-import output as expected" '
@@ -134,16 +168,16 @@ test_expect_success "correct error" '
 '
 
 
-cat >multiroot_import_expected <<EOE
+cat >multiroot_import_json_expected <<EOE
 {"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
 {"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
 EOE
 test_expect_success "multiroot import works" '
-  ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_actual
+  ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_json_actual
 '
 test_expect_success "multiroot import expected output" '
-  test_cmp multiroot_import_expected multiroot_import_actual
+  test_cmp multiroot_import_json_expected multiroot_import_json_actual
 '
 
 
@@ -154,21 +188,16 @@ test_expect_success "pin-less import works" '
     > no-pin_import_actual
 '
 test_expect_success "expected silence on --pin-roots=false" '
-  ! [[ -s no-pin_import_actual ]]
+  test_cmp /dev/null no-pin_import_actual
 '
 
 
-cat >naked_root_import_expected <<EOE
-{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
-{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
-EOE
 test_expect_success "naked root import works" '
   ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
-  | sort > naked_root_import_actual
+  | sort > naked_root_import_json_actual
 '
 test_expect_success "naked root import expected output" '
-   test_cmp naked_root_import_expected naked_root_import_actual
+   test_cmp naked_root_import_json_expected naked_root_import_json_actual
 '
 
-
 test_done