Use the newer automake test driver "parallel-tests". This driver:

* keeps test logs around
* provides parallel testing functionality (with make -jN).

I've also added new functionality in test/Makefile.am which deletes old test logs, ensures that ALL tests are tried (even if units fail), and provides a color-coded summary at the end of the test run, which shows how many units, regressions (per level), and system tests failed (or passed), and provides a link to the log file for further information.

Resolves bug 117.

diff --git a/configure.ac b/configure.ac
index e7d5a005aed2f774901bd3ef75d0ef7935827403..f805c445c2d4b7c0d018ca67bfa39b0a922dff8b 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -413,7 +413,7 @@ if test "$enable_profiling" = yes; then
   CVC4LDFLAGS="${CVC4LDFLAGS:+$CVC4LDFLAGS }-pg"
 fi
 
-AM_INIT_AUTOMAKE([1.11 no-define color-tests])
+AM_INIT_AUTOMAKE([1.11 no-define parallel-tests color-tests])
 AC_CONFIG_HEADERS([cvc4autoconfig.h])
 
 # Initialize libtool's configuration options.

diff --git a/test/Makefile.am b/test/Makefile.am
index 316b2a140a6a4d8399d37aed81bfffbe158cb912..31978b3c2e7c954bf3ea515b21d1df96abb5f7f4 100644 (file)
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -1,11 +1,46 @@
-SUBDIRS = unit system regress
+SUBDIRS = unit system regress .
+
+MAKEFLAGS = -k
 
 .PHONY: regress0 regress1 regress2 regress3
 regress0 regress1 regress2 regress3:
+       @$(MAKE) check-pre; \
        for dir in $(SUBDIRS); do \
-               (cd $$dir && $(MAKE) $(AM_MAKEFLAGS) $@) || exit 1; \
-       done
+               test $$dir = . || (cd $$dir && $(MAKE) $(AM_MAKEFLAGS) $@); \
+       done; \
+       $(MAKE) check-local
 
 # synonyms for "check"
 .PHONY: regress test
 regress test: check
+
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=; \
+test "X$(AM_COLOR_TESTS)" != Xno \
+&& test "X$$TERM" != Xdumb \
+&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
+&& { \
+  red='\e[0;31m'; \
+  grn='\e[0;32m'; \
+  lgn='\e[1;32m'; \
+  blu='\e[1;34m'; \
+  std='\e[m'; \
+}
+subdirs_to_check = unit system regress/regress0 regress/regress1 regress/regress2 regress/regress3
+check-recursive: check-pre
+.PHONY: check-pre
+check-pre:
+       @rm -f $(subdirs_to_check:=/test-suite.log)
+check-local:
+       @$(am__tty_colors); \
+       echo $${blu}===============================  TESTING SUMMARY  =============================$$std; \
+       for log in $(subdirs_to_check:=/test-suite.log); do \
+               if test -s "$$log"; then \
+                       status="`head -n 5 $$log | tail -1`"; \
+                       if echo "$$status" | grep -q failed; then col=$$red; \
+                       else col=$$grn; fi; \
+                       echo "$$col$$status"; \
+                       echo "    @abs_builddir@/$$log$$std"; \
+               fi; \
+       done; \
+       echo $${blu}===============================  TESTING SUMMARY  =============================$$std

diff --git a/test/regress/Makefile.am b/test/regress/Makefile.am
index 6ed661de9306b618674ad73376e1810a730a9abc..13ed5b5f6c6bb1377c8f4f6efe4776a21763abe8 100644 (file)
--- a/test/regress/Makefile.am
+++ b/test/regress/Makefile.am
@@ -6,7 +6,7 @@ regress1: regress0
 regress2: regress0 regress1
 regress3: regress0 regress1 regress2
 regress0 regress1 regress2 regress3:
-       cd $@ && $(MAKE) check
+       -cd $@ && $(MAKE) check
 
 # synonyms for "check"
 .PHONY: regress test

diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am
index fbca75676924d42a7680c0254f5a359bdec7e83b..ef60febf80a9fee20d66501ce3a90b1af03aacb2 100644 (file)
--- a/test/unit/Makefile.am
+++ b/test/unit/Makefile.am
@@ -29,7 +29,7 @@ UNIT_TESTS = \
        util/integer_black \
        util/integer_white \
        util/rational_black \
-    util/rational_white
+       util/rational_white
 
 # Things that aren't tests but that tests rely on and need to
 # go into the distribution
@@ -69,7 +69,7 @@ MOSTLYCLEANFILES = $(UNIT_TESTS) $(UNIT_TESTS:%=%.cpp)
 
 # We leave "TESTS" empty here; it's handled in Makefile.tests (see
 # that file for comment)
-# TESTS =
+TESTS = $(UNIT_TESTS)
 
 if STATIC_BINARY
 unit_LINK = $(CXXLINK) -all-static
@@ -83,39 +83,39 @@ $(UNIT_TESTS:%=@am__quote@./$(DEPDIR)/%.Plo@am__quote@): %.Plo:
        $(AM_V_at)$(MKDIR_P) `dirname "$@"`
        $(AM_V_GEN)test -e "$@" || touch "$@"
 
-$(UNIT_TESTS:%=%.cpp): %.cpp: %.h
+$(UNIT_TESTS:%=@abs_builddir@/%.cpp): @abs_builddir@/%.cpp: %.h
        $(AM_V_at)$(MKDIR_P) `dirname "$@"`
        $(AM_V_GEN)$(CXXTESTGEN) --have-eh --have-std --error-printer -o "$@" "$<"
 
-$(WHITE_TESTS:%=%.lo): %_white.lo: %_white.cpp
-@am__fastdepCXX_TRUE@  $(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ @abs_builddir@/$<
+$(WHITE_TESTS:%=%.lo): %_white.lo: @abs_builddir@/%_white.cpp
+@am__fastdepCXX_TRUE@  $(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ $<
 @am__fastdepCXX_TRUE@  $(AM_V_at)$(am__mv) $(DEPDIR)/$(@:%.lo=%).Tpo $(DEPDIR)/$(@:%.lo=%).Plo
 @am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -c -o $@ @abs_builddir@/$<
+@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) $(AM_CXXFLAGS_WHITE) -c -o $@ $<
 
 $(WHITE_TESTS): %_white: %_white.lo $(AM_LIBADD_WHITE)
        $(AM_V_CXXLD)$(unit_LINK) $(AM_LIBADD_WHITE) $(AM_LDFLAGS_WHITE) $<
 
-$(BLACK_TESTS:%=%.lo): %_black.lo: %_black.cpp
-@am__fastdepCXX_TRUE@  $(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ @abs_builddir@/$<
+$(BLACK_TESTS:%=%.lo): %_black.lo: @abs_builddir@/%_black.cpp
+@am__fastdepCXX_TRUE@  $(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ $<
 @am__fastdepCXX_TRUE@  $(AM_V_at)$(am__mv) $(DEPDIR)/$(@:%.lo=%).Tpo $(DEPDIR)/$(@:%.lo=%).Plo
 @am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -c -o $@ @abs_builddir@/$<
+@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) $(AM_CXXFLAGS_BLACK) -c -o $@ $<
 
 $(BLACK_TESTS): %_black: %_black.lo $(AM_LIBADD_BLACK)
        $(AM_V_CXXLD)$(unit_LINK) $(AM_LIBADD_BLACK) $(AM_LDFLAGS_BLACK) $<
 
-$(PUBLIC_TESTS:%=%.lo): %_public.lo: %_public.cpp
-@am__fastdepCXX_TRUE@  $(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ @abs_builddir@/$<
+$(PUBLIC_TESTS:%=%.lo): %_public.lo: @abs_builddir@/%_public.cpp
+@am__fastdepCXX_TRUE@  $(AM_V_CXX)$(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -MT $@ -MD -MP -MF $(DEPDIR)/$(@:%.lo=%).Tpo -c -o $@ $<
 @am__fastdepCXX_TRUE@  $(AM_V_at)$(am__mv) $(DEPDIR)/$(@:%.lo=%).Tpo $(DEPDIR)/$(@:%.lo=%).Plo
 @am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@     DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -c -o $@ @abs_builddir@/$<
+@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) $(AM_CXXFLAGS_PUBLIC) -c -o $@ $<
 
 $(PUBLIC_TESTS): %_public: %_public.lo $(AM_LIBADD_PUBLIC)
        $(AM_V_CXXLD)$(unit_LINK) $(AM_LIBADD_PUBLIC) $(AM_LDFLAGS_PUBLIC) $<