From df63d1b7f755a38d3f762899c91d48f893091088 Mon Sep 17 00:00:00 2001 From: Iain Buclaw Date: Fri, 12 Apr 2019 06:25:34 +0000 Subject: [PATCH] libphobos: Parallelize the libphobos testsuite The removal of the check programs in libphobos means that all tests will now be compiled one at a time, causing a notable slowdown in comparison with the previous method of testing. libphobos/ChangeLog: 2019-04-12 Iain Buclaw * testsuite/Makefile.am: * testsuite/Makefile.am (AUTOMAKE_OPTIONS): Remove dejagnu. (RUNTEST): Remove variable. (RUNTESTDEFAULTFLAGS, check_p_subno, check_p_numbers0, check_p_numbers1, check_p_numbers2, check_p_numbers3, check_p_numbers4, check_p_numbers5, check_p_numbers6, check_p_numbers, check_p_subdirs, check_DEJAGNU_libphobos_targets): New variables. (site.exp, %/site.exp, check-DEJAGNU, check-am, clean-local): New rules written so that all the *.exp files are ran parallelized. (CLEANFILES): Add *.exe, *.o, and site.exp. * testsuite/Makefile.in: Regenerate. From-SVN: r270303 --- libphobos/ChangeLog | 14 +++ libphobos/testsuite/Makefile.am | 108 ++++++++++++++++++- libphobos/testsuite/Makefile.in | 185 ++++++++++++++++++++------------ 3 files changed, 238 insertions(+), 69 deletions(-) diff --git a/libphobos/ChangeLog b/libphobos/ChangeLog index 1387496dcff..e974a64299f 100644 --- a/libphobos/ChangeLog +++ b/libphobos/ChangeLog @@ -1,3 +1,17 @@ +2019-04-12 Iain Buclaw + + * testsuite/Makefile.am: + * testsuite/Makefile.am (AUTOMAKE_OPTIONS): Remove dejagnu. + (RUNTEST): Remove variable. + (RUNTESTDEFAULTFLAGS, check_p_subno, check_p_numbers0, + check_p_numbers1, check_p_numbers2, check_p_numbers3, + check_p_numbers4, check_p_numbers5, check_p_numbers6, check_p_numbers, + check_p_subdirs, check_DEJAGNU_libphobos_targets): New variables. + (site.exp, %/site.exp, check-DEJAGNU, check-am, clean-local): New + rules written so that all the *.exp files are ran parallelized. + (CLEANFILES): Add *.exe, *.o, and site.exp. + * testsuite/Makefile.in: Regenerate. + 2019-04-12 Iain Buclaw * configure.ac (ENABLE_SHARED, ENABLE_STATIC): Remove conditionals. diff --git a/libphobos/testsuite/Makefile.am b/libphobos/testsuite/Makefile.am index def102941a7..55b2ba42640 100644 --- a/libphobos/testsuite/Makefile.am +++ b/libphobos/testsuite/Makefile.am @@ -17,7 +17,7 @@ # Process this file with automake to produce Makefile.in. -AUTOMAKE_OPTIONS = foreign dejagnu +AUTOMAKE_OPTIONS = foreign # Setup the testing framework, if you have one EXPECT = $(shell if test -f $(top_builddir)/../expect/expect; then \ @@ -25,8 +25,110 @@ EXPECT = $(shell if test -f $(top_builddir)/../expect/expect; then \ _RUNTEST = $(shell if test -f $(top_srcdir)/../dejagnu/runtest; then \ echo $(top_srcdir)/../dejagnu/runtest; else echo runtest; fi) -RUNTEST = $(_RUNTEST) $(AM_RUNTESTFLAGS) +RUNTESTDEFAULTFLAGS = --tool $$tool --srcdir $$srcdir + +check_p_subno=$(word 2,$(subst _, ,$*)) +check_p_numbers0:=1 2 3 4 5 6 7 8 9 +check_p_numbers1:=0 $(check_p_numbers0) +check_p_numbers2:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers1))) +check_p_numbers3:=$(addprefix 0,$(check_p_numbers1)) $(check_p_numbers2) +check_p_numbers4:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers3))) +check_p_numbers5:=$(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4) +check_p_numbers6:=$(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5))) +check_p_numbers:=$(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6) +check_p_subdirs=$(wordlist 1,$(if $(GCC_TEST_PARALLEL_SLOTS),$(GCC_TEST_PARALLEL_SLOTS),10),$(check_p_numbers)) +check_DEJAGNU_libphobos_targets = $(addprefix check-DEJAGNUlibphobos,$(check_p_subdirs)) +$(check_DEJAGNU_libphobos_targets): check-DEJAGNUlibphobos%: libphobos%/site.exp + +site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG) + @echo 'Making a new site.exp file ...' + @echo '## these variables are automatically generated by make ##' >site.tmp + @echo '# Do not edit here. If you wish to override these values' >>site.tmp + @echo '# edit the last section' >>site.tmp + @echo 'set srcdir "$(srcdir)"' >>site.tmp + @echo "set objdir `pwd`" >>site.tmp + @echo 'set build_alias "$(build_alias)"' >>site.tmp + @echo 'set build_triplet $(build_triplet)' >>site.tmp + @echo 'set host_alias "$(host_alias)"' >>site.tmp + @echo 'set host_triplet $(host_triplet)' >>site.tmp + @echo 'set target_alias "$(target_alias)"' >>site.tmp + @echo 'set target_triplet $(target_triplet)' >>site.tmp + @list='$(EXTRA_DEJAGNU_SITE_CONFIG)'; for f in $$list; do \ + echo "## Begin content included from file $$f. Do not modify. ##" \ + && cat `test -f "$$f" || echo '$(srcdir)/'`$$f \ + && echo "## End content included from file $$f. ##" \ + || exit 1; \ + done >> site.tmp + @echo "## End of auto-generated content; you can edit from here. ##" >> site.tmp + @if test -f site.exp; then \ + sed -e '1,/^## End of auto-generated content.*##/d' site.exp >> site.tmp; \ + fi + @-rm -f site.bak + @test ! -f site.exp || mv site.exp site.bak + @mv site.tmp site.exp + +%/site.exp: site.exp + -@test -d $* || mkdir $* + @srcdir=`cd $(srcdir); ${PWD_COMMAND}`; + @objdir=`${PWD_COMMAND}`/$*; \ + sed -e "s|^set srcdir .*$$|set srcdir $$srcdir|" \ + -e "s|^set objdir .*$$|set objdir $$objdir|" \ + site.exp > $*/site.exp.tmp + @-rm -f $*/site.bak + @test ! -f $*/site.exp || mv $*/site.exp $*/site.bak + @mv $*/site.exp.tmp $*/site.exp + +# Run the testsuite in normal mode. +check-DEJAGNU $(check_DEJAGNU_libphobos_targets): check-DEJAGNU%: site.exp + $(if $*,@)AR="$(AR)"; export AR; \ + RANLIB="$(RANLIB)"; export RANLIB; \ + if [ -z "$*" ] && [ -n "$(filter -j%, $(MFLAGS))" ]; then \ + rm -rf libphobos-parallel || true; \ + mkdir libphobos-parallel; \ + $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_libphobos_targets); \ + rm -rf libphobos-parallel || true; \ + for idx in $(check_p_subdirs); do \ + if [ -d libphobos$$idx ]; then \ + mv -f libphobos$$idx/libphobos.sum libphobos$$idx/libphobos.sum.sep; \ + mv -f libphobos$$idx/libphobos.log libphobos$$idx/libphobos.log.sep; \ + fi; \ + done; \ + $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh \ + libphobos[0-9]*/libphobos.sum.sep > libphobos.sum; \ + $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh -L \ + libphobos[0-9]*/libphobos.log.sep > libphobos.log; \ + exit 0; \ + fi; \ + srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \ + EXPECT=$(EXPECT); export EXPECT; \ + runtest=$(_RUNTEST); \ + if [ -z "$$runtest" ]; then runtest=runtest; fi; \ + tool=libphobos; \ + if [ -n "$*" ]; then \ + if [ -f libphobos-parallel/finished ]; then rm -rf "$*"; exit 0; fi; \ + GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/libphobos-parallel; \ + export GCC_RUNTEST_PARALLELIZE_DIR; \ + cd "$*"; \ + fi; \ + if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \ + $$runtest $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) \ + $(RUNTESTFLAGS); \ + if [ -n "$*" ]; then \ + touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished; \ + fi; \ + else \ + echo "WARNING: could not find \`runtest'" 1>&2; :;\ + fi + +check-am: + $(MAKE) $(AM_MAKEFLAGS) check-DEJAGNU + +.PHONY: check-DEJAGNU AM_MAKEFLAGS = "EXEEXT=$(EXEEXT)" -CLEANFILES = *.log *.sum +CLEANFILES = *.exe *.log *.o *.sum site.exp + +# To remove runtest-parallel directories. +clean-local: + rm -rf libphobos* diff --git a/libphobos/testsuite/Makefile.in b/libphobos/testsuite/Makefile.in index 78107d22680..b81209b9251 100644 --- a/libphobos/testsuite/Makefile.in +++ b/libphobos/testsuite/Makefile.in @@ -133,8 +133,6 @@ am__can_run_installinfo = \ *) (install-info --version) >/dev/null 2>&1;; \ esac am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) -DEJATOOL = $(PACKAGE) -RUNTESTDEFAULTFLAGS = --tool $$tool --srcdir $$srcdir ACLOCAL = @ACLOCAL@ AMTAR = @AMTAR@ AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ @@ -278,7 +276,7 @@ target_vendor = @target_vendor@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ -AUTOMAKE_OPTIONS = foreign dejagnu +AUTOMAKE_OPTIONS = foreign # Setup the testing framework, if you have one EXPECT = $(shell if test -f $(top_builddir)/../expect/expect; then \ @@ -287,9 +285,20 @@ EXPECT = $(shell if test -f $(top_builddir)/../expect/expect; then \ _RUNTEST = $(shell if test -f $(top_srcdir)/../dejagnu/runtest; then \ echo $(top_srcdir)/../dejagnu/runtest; else echo runtest; fi) -RUNTEST = $(_RUNTEST) $(AM_RUNTESTFLAGS) +RUNTESTDEFAULTFLAGS = --tool $$tool --srcdir $$srcdir +check_p_subno = $(word 2,$(subst _, ,$*)) +check_p_numbers0 := 1 2 3 4 5 6 7 8 9 +check_p_numbers1 := 0 $(check_p_numbers0) +check_p_numbers2 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers1))) +check_p_numbers3 := $(addprefix 0,$(check_p_numbers1)) $(check_p_numbers2) +check_p_numbers4 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers3))) +check_p_numbers5 := $(addprefix 0,$(check_p_numbers3)) $(check_p_numbers4) +check_p_numbers6 := $(foreach i,$(check_p_numbers0),$(addprefix $(i),$(check_p_numbers5))) +check_p_numbers := $(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6) +check_p_subdirs = $(wordlist 1,$(if $(GCC_TEST_PARALLEL_SLOTS),$(GCC_TEST_PARALLEL_SLOTS),10),$(check_p_numbers)) +check_DEJAGNU_libphobos_targets = $(addprefix check-DEJAGNUlibphobos,$(check_p_subdirs)) AM_MAKEFLAGS = "EXEEXT=$(EXEEXT)" -CLEANFILES = *.log *.sum +CLEANFILES = *.exe *.log *.o *.sum site.exp all: all-am .SUFFIXES: @@ -336,52 +345,7 @@ ctags CTAGS: cscope cscopelist: - -check-DEJAGNU: site.exp - srcdir='$(srcdir)'; export srcdir; \ - EXPECT=$(EXPECT); export EXPECT; \ - if $(SHELL) -c "$(RUNTEST) --version" > /dev/null 2>&1; then \ - exit_status=0; l='$(DEJATOOL)'; for tool in $$l; do \ - if $(RUNTEST) $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) $(RUNTESTFLAGS); \ - then :; else exit_status=1; fi; \ - done; \ - else echo "WARNING: could not find '$(RUNTEST)'" 1>&2; :;\ - fi; \ - exit $$exit_status -site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG) - @echo 'Making a new site.exp file ...' - @echo '## these variables are automatically generated by make ##' >site.tmp - @echo '# Do not edit here. If you wish to override these values' >>site.tmp - @echo '# edit the last section' >>site.tmp - @echo 'set srcdir "$(srcdir)"' >>site.tmp - @echo "set objdir `pwd`" >>site.tmp - @echo 'set build_alias "$(build_alias)"' >>site.tmp - @echo 'set build_triplet $(build_triplet)' >>site.tmp - @echo 'set host_alias "$(host_alias)"' >>site.tmp - @echo 'set host_triplet $(host_triplet)' >>site.tmp - @echo 'set target_alias "$(target_alias)"' >>site.tmp - @echo 'set target_triplet $(target_triplet)' >>site.tmp - @list='$(EXTRA_DEJAGNU_SITE_CONFIG)'; for f in $$list; do \ - echo "## Begin content included from file $$f. Do not modify. ##" \ - && cat `test -f "$$f" || echo '$(srcdir)/'`$$f \ - && echo "## End content included from file $$f. ##" \ - || exit 1; \ - done >> site.tmp - @echo "## End of auto-generated content; you can edit from here. ##" >> site.tmp - @if test -f site.exp; then \ - sed -e '1,/^## End of auto-generated content.*##/d' site.exp >> site.tmp; \ - fi - @-rm -f site.bak - @test ! -f site.exp || mv site.exp site.bak - @mv site.tmp site.exp - -distclean-DEJAGNU: - -rm -f site.exp site.bak - -l='$(DEJATOOL)'; for tool in $$l; do \ - rm -f $$tool.sum $$tool.log; \ - done check-am: all-am - $(MAKE) $(AM_MAKEFLAGS) check-DEJAGNU check: check-am all-am: Makefile installdirs: @@ -418,11 +382,11 @@ maintainer-clean-generic: @echo "it deletes files that may require special tools to rebuild." clean: clean-am -clean-am: clean-generic clean-libtool mostlyclean-am +clean-am: clean-generic clean-libtool clean-local mostlyclean-am distclean: distclean-am -rm -f Makefile -distclean-am: clean-am distclean-DEJAGNU distclean-generic +distclean-am: clean-am distclean-generic dvi: dvi-am @@ -482,23 +446,112 @@ ps-am: uninstall-am: -.MAKE: check-am install-am install-strip - -.PHONY: all all-am check check-DEJAGNU check-am clean clean-generic \ - clean-libtool cscopelist-am ctags-am distclean \ - distclean-DEJAGNU distclean-generic distclean-libtool dvi \ - dvi-am html html-am info info-am install install-am \ - install-data install-data-am install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-generic \ - mostlyclean-libtool pdf pdf-am ps ps-am tags-am uninstall \ - uninstall-am +.MAKE: install-am install-strip + +.PHONY: all all-am check check-am clean clean-generic clean-libtool \ + clean-local cscopelist-am ctags-am distclean distclean-generic \ + distclean-libtool dvi dvi-am html html-am info info-am install \ + install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags-am uninstall uninstall-am .PRECIOUS: Makefile +$(check_DEJAGNU_libphobos_targets): check-DEJAGNUlibphobos%: libphobos%/site.exp + +site.exp: Makefile $(EXTRA_DEJAGNU_SITE_CONFIG) + @echo 'Making a new site.exp file ...' + @echo '## these variables are automatically generated by make ##' >site.tmp + @echo '# Do not edit here. If you wish to override these values' >>site.tmp + @echo '# edit the last section' >>site.tmp + @echo 'set srcdir "$(srcdir)"' >>site.tmp + @echo "set objdir `pwd`" >>site.tmp + @echo 'set build_alias "$(build_alias)"' >>site.tmp + @echo 'set build_triplet $(build_triplet)' >>site.tmp + @echo 'set host_alias "$(host_alias)"' >>site.tmp + @echo 'set host_triplet $(host_triplet)' >>site.tmp + @echo 'set target_alias "$(target_alias)"' >>site.tmp + @echo 'set target_triplet $(target_triplet)' >>site.tmp + @list='$(EXTRA_DEJAGNU_SITE_CONFIG)'; for f in $$list; do \ + echo "## Begin content included from file $$f. Do not modify. ##" \ + && cat `test -f "$$f" || echo '$(srcdir)/'`$$f \ + && echo "## End content included from file $$f. ##" \ + || exit 1; \ + done >> site.tmp + @echo "## End of auto-generated content; you can edit from here. ##" >> site.tmp + @if test -f site.exp; then \ + sed -e '1,/^## End of auto-generated content.*##/d' site.exp >> site.tmp; \ + fi + @-rm -f site.bak + @test ! -f site.exp || mv site.exp site.bak + @mv site.tmp site.exp + +%/site.exp: site.exp + -@test -d $* || mkdir $* + @srcdir=`cd $(srcdir); ${PWD_COMMAND}`; + @objdir=`${PWD_COMMAND}`/$*; \ + sed -e "s|^set srcdir .*$$|set srcdir $$srcdir|" \ + -e "s|^set objdir .*$$|set objdir $$objdir|" \ + site.exp > $*/site.exp.tmp + @-rm -f $*/site.bak + @test ! -f $*/site.exp || mv $*/site.exp $*/site.bak + @mv $*/site.exp.tmp $*/site.exp + +# Run the testsuite in normal mode. +check-DEJAGNU $(check_DEJAGNU_libphobos_targets): check-DEJAGNU%: site.exp + $(if $*,@)AR="$(AR)"; export AR; \ + RANLIB="$(RANLIB)"; export RANLIB; \ + if [ -z "$*" ] && [ -n "$(filter -j%, $(MFLAGS))" ]; then \ + rm -rf libphobos-parallel || true; \ + mkdir libphobos-parallel; \ + $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_libphobos_targets); \ + rm -rf libphobos-parallel || true; \ + for idx in $(check_p_subdirs); do \ + if [ -d libphobos$$idx ]; then \ + mv -f libphobos$$idx/libphobos.sum libphobos$$idx/libphobos.sum.sep; \ + mv -f libphobos$$idx/libphobos.log libphobos$$idx/libphobos.log.sep; \ + fi; \ + done; \ + $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh \ + libphobos[0-9]*/libphobos.sum.sep > libphobos.sum; \ + $(SHELL) $(srcdir)/../../contrib/dg-extract-results.sh -L \ + libphobos[0-9]*/libphobos.log.sep > libphobos.log; \ + exit 0; \ + fi; \ + srcdir=`$(am__cd) $(srcdir) && pwd`; export srcdir; \ + EXPECT=$(EXPECT); export EXPECT; \ + runtest=$(_RUNTEST); \ + if [ -z "$$runtest" ]; then runtest=runtest; fi; \ + tool=libphobos; \ + if [ -n "$*" ]; then \ + if [ -f libphobos-parallel/finished ]; then rm -rf "$*"; exit 0; fi; \ + GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/libphobos-parallel; \ + export GCC_RUNTEST_PARALLELIZE_DIR; \ + cd "$*"; \ + fi; \ + if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \ + $$runtest $(AM_RUNTESTFLAGS) $(RUNTESTDEFAULTFLAGS) \ + $(RUNTESTFLAGS); \ + if [ -n "$*" ]; then \ + touch $$GCC_RUNTEST_PARALLELIZE_DIR/finished; \ + fi; \ + else \ + echo "WARNING: could not find \`runtest'" 1>&2; :;\ + fi + +check-am: + $(MAKE) $(AM_MAKEFLAGS) check-DEJAGNU + +.PHONY: check-DEJAGNU + +# To remove runtest-parallel directories. +clean-local: + rm -rf libphobos* # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. -- 2.30.2