Minor testsuite improvements (#154)

* run_readelf_tests.py: add an option to continue tests after a failure

Also add a counter to display the number of failed tests over the number
of tests run.

* run_readelf_tests.py: run tests in alphabetical order

This gives deterministic runs, which is handy to run edit/test
development cycles.

diff --git a/test/run_readelf_tests.py b/test/run_readelf_tests.py
index f9991749799beb42434055937b47aa7ee24e0a83..7ccbb4d8cd43eb02de7feb12c3ae1ff9a45a3e58 100755 (executable)
--- a/test/run_readelf_tests.py
+++ b/test/run_readelf_tests.py
@@ -195,6 +195,9 @@ def main():
     optparser.add_option('-V', '--verbose',
         action='store_true', dest='verbose',
         help='Verbose output')
+    optparser.add_option('-k', '--keep-going',
+        action='store_true', dest='keep_going',
+        help="Run all tests, don't stop at the first failure")
     options, args = optparser.parse_args()
 
     if options.verbose:
@@ -208,18 +211,22 @@ def main():
     if len(args) > 0:
         filenames = args
     else:
-        filenames = list(discover_testfiles('test/testfiles_for_readelf'))
+        filenames = sorted(discover_testfiles('test/testfiles_for_readelf'))
 
-    success = True
+    failures = 0
     for filename in filenames:
-        if success:
-            success = success and run_test_on_file(
-                                    filename,
-                                    verbose=options.verbose)
+        if not run_test_on_file(filename, verbose=options.verbose):
+            failures += 1
+            if not options.keep_going:
+                break
 
-    if success:
+    if failures == 0:
         testlog.info('\nConclusion: SUCCESS')
         return 0
+    elif options.keep_going:
+        testlog.info('\nConclusion: FAIL ({}/{})'.format(
+            failures, len(filenames)))
+        return 1
     else:
         testlog.info('\nConclusion: FAIL')
         return 1