From dd0f54fed65de7962bc5621c4a601579c2f82184 Mon Sep 17 00:00:00 2001 From: Andreas Sandberg Date: Mon, 20 Jun 2016 14:50:43 +0100 Subject: [PATCH] tests: Add a test command to get test status as an exit code Add a "test" command to tests.py that queries a test pickle file and returns different exit codes depending on the outcome of the tests in the file. The following exit codes can currently be returned: * 0: All tests were successful or skipped. * 1: General fault in the script such as incorrect parameters or failing to parse a pickle file. * 2: At least one test failed to run. This is what the summary formatter usually shows as a 'FAILED'. * 3: All tests ran correctly, but at least one failed to verify its output. When displaying test output using the summary formatter, such a test would show up as 'CHANGED'. The command can be invoked like this: ./tests/tests.py test `find build/ARM/tests/opt/ -name status.pickle` Change-Id: I7e6bc661516f38ff08dfda7c4359a1e10bf97864 Signed-off-by: Andreas Sandberg Reviewed-by: Curtis Dunham --- tests/tests.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 05d68881e..85b194c56 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -241,10 +241,52 @@ def _show(args): suites = sum([ pickle.load(f) for f in args.result ], []) formatter.dump_suites(suites) +def _test_args(subparsers): + parser = subparsers.add_parser( + "test", + formatter_class=ParagraphHelpFormatter, + help='Probe test results and set exit code', + epilog=""" + + Load one or more pickled test file and return an exit code + corresponding to the test outcome. The following exit codes + can be returned: + + 0: All tests were successful or skipped. + + 1: General fault in the script such as incorrect parameters or + failing to parse a pickle file. + + 2: At least one test failed to run. This is what the summary + formatter usually shows as a 'FAILED'. + + 3: All tests ran correctly, but at least one failed to + verify its output. When displaying test output using the + summary formatter, such a test would show up as 'CHANGED'. + """) + + _add_format_args(parser) + + parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", + help="Pickled test results") + +def _test(args): + suites = sum([ pickle.load(f) for f in args.result ], []) + + if all(s for s in suites): + sys.exit(0) + elif any([ s.failed_run() for s in suites ]): + sys.exit(2) + elif any([ s.changed() for s in suites ]): + sys.exit(3) + else: + assert False, "Unexpected return status from test" + _commands = { "list" : (_list_tests, _list_tests_args), "run" : (_run_tests, _run_tests_args), "show" : (_show, _show_args), + "test" : (_test, _test_args), } def main(): -- 2.30.2