From c26453e6ee39f168fd7ce3a16bc2d96bf3e93bbb Mon Sep 17 00:00:00 2001
From: Pierre-Marie de Rodat <pmderodat@kawie.fr>
Date: Thu, 17 Aug 2017 21:33:44 -0400
Subject: [PATCH] Minor testsuite improvements (#154)

* run_readelf_tests.py: add an option to continue tests after a failure

Also add a counter to display the number of failed tests over the number
of tests run.

* run_readelf_tests.py: run tests in alphabetical order

This gives deterministic runs, which is handy to run edit/test
development cycles.
---
 test/run_readelf_tests.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/test/run_readelf_tests.py b/test/run_readelf_tests.py
index f999174..7ccbb4d 100755
--- a/test/run_readelf_tests.py
+++ b/test/run_readelf_tests.py
@@ -195,6 +195,9 @@ def main():
     optparser.add_option('-V', '--verbose',
         action='store_true', dest='verbose',
         help='Verbose output')
+    optparser.add_option('-k', '--keep-going',
+        action='store_true', dest='keep_going',
+        help="Run all tests, don't stop at the first failure")
     options, args = optparser.parse_args()
 
     if options.verbose:
@@ -208,18 +211,22 @@ def main():
     if len(args) > 0:
         filenames = args
     else:
-        filenames = list(discover_testfiles('test/testfiles_for_readelf'))
+        filenames = sorted(discover_testfiles('test/testfiles_for_readelf'))
 
-    success = True
+    failures = 0
     for filename in filenames:
-        if success:
-            success = success and run_test_on_file(
-                                    filename,
-                                    verbose=options.verbose)
+        if not run_test_on_file(filename, verbose=options.verbose):
+            failures += 1
+            if not options.keep_going:
+                break
 
-    if success:
+    if failures == 0:
         testlog.info('\nConclusion: SUCCESS')
         return 0
+    elif options.keep_going:
+        testlog.info('\nConclusion: FAIL ({}/{})'.format(
+            failures, len(filenames)))
+        return 1
     else:
         testlog.info('\nConclusion: FAIL')
         return 1
-- 
2.30.2