tests: Add a test command to get test status as an exit code

Add a "test" command to tests.py that queries a test pickle file and
returns different exit codes depending on the outcome of the tests in
the file. The following exit codes can currently be returned:

  * 0: All tests were successful or skipped.

  * 1: General fault in the script such as incorrect parameters or
    failing to parse a pickle file.

  * 2: At least one test failed to run. This is what the summary
    formatter usually shows as a 'FAILED'.

  * 3: All tests ran correctly, but at least one failed to verify
    its output. When displaying test output using the summary
    formatter, such a test would show up as 'CHANGED'.

The command can be invoked like this:

./tests/tests.py test `find build/ARM/tests/opt/ -name status.pickle`

Change-Id: I7e6bc661516f38ff08dfda7c4359a1e10bf97864
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Curtis Dunham <curtis.dunham@arm.com>

diff --git a/tests/tests.py b/tests/tests.py
index 05d68881e6e3d91b298cddc9144b6c121a965bda..85b194c56204200c2365f21a307092ed406856fe 100755 (executable)
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -241,10 +241,52 @@ def _show(args):
     suites = sum([ pickle.load(f) for f in args.result ], [])
     formatter.dump_suites(suites)
 
+def _test_args(subparsers):
+    parser = subparsers.add_parser(
+        "test",
+        formatter_class=ParagraphHelpFormatter,
+        help='Probe test results and set exit code',
+        epilog="""
+
+        Load one or more pickled test file and return an exit code
+        corresponding to the test outcome. The following exit codes
+        can be returned:
+
+        0: All tests were successful or skipped.
+
+        1: General fault in the script such as incorrect parameters or
+        failing to parse a pickle file.
+
+        2: At least one test failed to run. This is what the summary
+        formatter usually shows as a 'FAILED'.
+
+        3: All tests ran correctly, but at least one failed to
+        verify its output. When displaying test output using the
+        summary formatter, such a test would show up as 'CHANGED'.
+        """)
+
+    _add_format_args(parser)
+
+    parser.add_argument("result", type=argparse.FileType("rb"), nargs="*",
+                        help="Pickled test results")
+
+def _test(args):
+    suites = sum([ pickle.load(f) for f in args.result ], [])
+
+    if all(s for s in suites):
+        sys.exit(0)
+    elif any([ s.failed_run() for s in suites ]):
+        sys.exit(2)
+    elif any([ s.changed() for s in suites ]):
+        sys.exit(3)
+    else:
+        assert False, "Unexpected return status from test"
+
 _commands = {
     "list" : (_list_tests, _list_tests_args),
     "run" : (_run_tests, _run_tests_args),
     "show" : (_show, _show_args),
+    "test" : (_test, _test_args),
 }
 
 def main():