Uniquely keyed run artifacts

.gitignore

@@ -1,2 +1,3 @@
 bats/
-
+oats.log
+results/

README.md

@@ -92,25 +92,45 @@ cd OpenDroneMap/
 docker build -t opendronemap/odm:myversion .
 ```
 
-Then pass the `--tags` parameter to `run`:
+Then run the suite once per version with the `--tag` parameter:
 
 ```bash
-./run all --tags latest,myversion,0.3.1
+./run all --tag latest
+./run all --tag myversion
 ```
 
+Each run writes its results to a separate directory (see below), so you can compare them afterwards.
+
 ## Rerunning Tests
 
-By default OATS chooses the least destructive approach possible. Previous test results are never cleared between runs unless explicitely instructed by the user.
+Every run starts from a fresh directory and processes each dataset from scratch. Old runs pile up under `results/runs/` and can be deleted whenever you want.
+
+If you need to resume a pipeline from a previous run, invoke ODM directly against that run's output directory:
 
 ```bash
-./run all --clear
+docker run -v <old_output>:/datasets/code opendronemap/odm --project-path /datasets --rerun-from odm_meshing
 ```
 
 ## Examine Test Results
 
-All results are placed in `results/`. Each dataset directory will contain a `task_output.txt` file with the console output result. Most errors can be traced with this file.
+At the end of a run, `run` prints a PASS/FAIL summary for each dataset with its wall time, and lists the failures with the ODM exit status (and signal name if it was killed, e.g. SIGSEGV) and the path to the full log. An ODM exit status of 0 on a failed test means ODM finished but a post-run check in the `.oat` file failed.
+
+`run` exits with a non-zero status if any test failed, so it can be used as a CI gate.
+
+### Run artifacts
+
+Each run writes to its own directory (ignored by git):
+
+```
+results/runs/<odm revision>/<image key>/<timestamp>/
+```
+
+The ODM revision is read from the image's `org.opencontainers.image.revision` label and the image key from the image digest (or the image id for local builds); either falls back to `unknown`, and `--test` runs use `unknown/unknown`. Runs of the same ODM version end up next to each other, which makes them easy to compare. The directory contains:
 
-The output of `run` follows the [TAP Protocol](http://testanything.org/) so you can parse it with one of the many [TAP Consumers](http://testanything.org/consumers.html) available.
+- `run_manifest.json`: the run key, docker image name/id/digest, the ODM source revision, the OATS git hash, host info (kernel, total RAM, GPU model), the `run` argument line, and an entry per test with the ODM exit status, wall time (seconds) and output size.
+- `<tag>/<dataset>/<test>/`: the ODM output for each test, including a `task_output.txt` file with the console output of the OpenDroneMap run. Most errors can be traced with this file.
+- `reports/<dataset>_<tag>.xml`: a [JUnit XML](https://github.com/testmoapp/junitxml) report per dataset for CI.
+- `oats_manifest.tsv`: the per-test records in tab-separated form.
 
 If you want to aggregate all files into a single directory for ease of view, you can use `harvest`:
 

functions.bash

@@ -15,16 +15,11 @@ run_test(){
 
 	# Sync dataset images to test directory
 	# Publish output directory (for people to check files, do extra test logic)
-	export output_dir="results/$tag/$dataset/$BATS_TEST_NAME/"
-
-	if [ "$CLEAR" == "YES" ]; then
-		rm -fr $output_dir
-	fi
+	export output_dir="$OATS_RUN_DIR/$tag/$dataset/$BATS_TEST_NAME/"
+	mkdir -p $output_dir
 
 	if [ "$TESTRUN" == "NO" ]; then
 		check_download_dataset $dataset
-
-		mkdir -p $output_dir
 		rsync -a --delete datasets/$dataset/* $output_dir
 	fi
 
@@ -44,15 +39,22 @@ run_test(){
 			-v $(pwd)/$output_dir:/staging \
 			--entrypoint bash \
 			$DOCKER_IMAGE:$tag \
-			-c \"mkdir -p /datasets/code && cp -R /staging/* /datasets/code && ./run.sh --project-path /datasets $options $CMD_OPTIONS code; cp -R /datasets/code/* /staging\" "
+			-c \"mkdir -p /datasets/code && cp -R /staging/* /datasets/code && /code/scripts/docker-entrypoint.sh --project-path /datasets $options $CMD_OPTIONS code; cp -R /datasets/code/* /staging\" "
 	fi
 
+	wall_time_s=0
+	result_dir_bytes=0
+
 	if [ "$TESTRUN" == "YES" ]; then
 		log "About to run: $DOCKER_CMD"
 		run echo "$output_dir output"
 	else
 		log "About to run: $DOCKER_CMD"
+
+		local start
+		start=$SECONDS
 		run eval $DOCKER_CMD
+		wall_time_s=$((SECONDS - start))
 
 		sleep 1
 
@@ -62,11 +64,21 @@ run_test(){
 			--entrypoint /bin/chown \
 			$DOCKER_IMAGE:$tag \
 			-R $(id -u):$(id -u) /dataset
+
+		result_dir_bytes=$(du -sb "$output_dir" 2>/dev/null | cut -f1)
 	fi
 
 	# Save command output to log
 	echo "$output" > $output_dir/task_output.txt
-
+
+	# $status is the ODM exit code, used to tell an ODM failure from a
+	# failed post-run check
+	if [ -n "${OATS_MANIFEST:-}" ]; then
+		printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+			"$tag" "$dataset" "$BATS_TEST_DESCRIPTION" "$output_dir" "$status" \
+			"${wall_time_s:-0}" "${result_dir_bytes:-0}" >> "$OATS_MANIFEST"
+	fi
+
 	# Basic check
 	[ "$status" -eq 0 ]
 }
@@ -77,15 +89,15 @@ check_download_dataset(){
 	if [ ! -e ./datasets/$dataset/images ] && [ ! -z $DATASET_URL ]; then
 		if [ ! -e ./datasets/$dataset ]; then
 			mkdir ./datasets/$dataset
-		fi 
+		fi
 
 		wget $DATASET_URL -q -O ./datasets/$dataset/download.zip
 		cd ./datasets/$dataset/
-		unzip ./download.zip 2>/dev/null
+		unzip -q ./download.zip 2>/dev/null
 		rm ./download.zip
-		
+
 		# Remove top level directory if needed
-		for dir in $(ls -d */); do 
+		for dir in $(ls -d */); do
 			if [ "$dir" != "images/" ]; then
 				mv "$dir"/* .
 				rm -fr "$dir"

harvest

@@ -82,7 +82,7 @@ if [ -e "groups/$group.bash" ]; then
 			files=()
 			while IFS=  read -r -d $'\0'; do
 			    files+=("$REPLY")
-			done < <(cd results && find $tag/$dataset/ -name "$pattern" 2>/dev/null -print0)
+			done < <(cd results && find runs/ -path "*/$tag/$dataset/*" -name "$pattern" 2>/dev/null -print0)
 
 			if [ ! -z "$files" ]; then
 				if [ ! -e "$destination" ]; then