From d7068b3d2acc18be3975a44362bc34ba3d9dc2e1 Mon Sep 17 00:00:00 2001 From: Mark Mitchell Date: Thu, 5 Jun 2003 00:17:16 +0000 Subject: [PATCH] Makefile.in (QMTESTRUNFLAGS): Set for DejaGNU emulation. * Makefile.in (QMTESTRUNFLAGS): Set for DejaGNU emulation. (QMTEST_GPP_TESTS): Use "g++" by default. (stamp-qmtest): Tweak database creation. (QMTEST_DIR/context): Update context file format. (qmtest-g++): Tweak command-line. * README.QMTEST: Update. From-SVN: r67474 --- gcc/ChangeLog | 8 ++ gcc/Makefile.in | 52 ++++-------- gcc/testsuite/ChangeLog | 4 + gcc/testsuite/README.QMTEST | 165 ++++++++++++++++-------------------- 4 files changed, 103 insertions(+), 126 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 9f6580e43a4..a3a252aa45f 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,11 @@ +2003-06-04 Mark Mitchell + + * Makefile.in (QMTESTRUNFLAGS): Set for DejaGNU emulation. + (QMTEST_GPP_TESTS): Use "g++" by default. + (stamp-qmtest): Tweak database creation. + (QMTEST_DIR/context): Update context file format. + (qmtest-g++): Tweak command-line. + 2003-06-04 Kaveh R. Ghazi * Makefile.in (varasm.o): Don't set -Wno-error. diff --git a/gcc/Makefile.in b/gcc/Makefile.in index 72a72d69336..76d2c8d30db 100644 --- a/gcc/Makefile.in +++ b/gcc/Makefile.in @@ -1,7 +1,7 @@ # Makefile for GNU Compiler Collection # Run 'configure' to generate Makefile from Makefile.in -# Copyright (C) 1987, 1988, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997 +# Copyright (C) 1987, 1988, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, # 1998, 1999, 2000, 2001, 2002, 2003 Free Software Foundation, Inc. #This file is part of GCC. @@ -3378,13 +3378,13 @@ QMTEST_PATH=qmtest QMTESTFLAGS= # The flags to pass to "qmtest run". -QMTESTRUNFLAGS= +QMTESTRUNFLAGS=-f none --result-stream dejagnu_stream.DejaGNUStream # The command to use to invoke qmtest. QMTEST=${QMTEST_PATH} ${QMTESTFLAGS} # The tests (or suites) to run. -QMTEST_GPP_TESTS=gpp +QMTEST_GPP_TESTS=g++ # The subdirectory of the OBJDIR that will be used to store the QMTest # test database configuration and that will be used for temporary @@ -3393,48 +3393,32 @@ QMTEST_DIR=qmtestsuite # Create the QMTest database configuration. ${QMTEST_DIR} stamp-qmtest: - debug_options=""; \ - ${STAMP} empty.C; \ - for option in \ - -gdwarf-2 -gstabs -gstabs+ -gxcoff -gxcoff+ -gcoff; do \ - (./cc1plus -q $${option} empty.C 2>&1 | \ - grep "unknown or unsupported -g option" > /dev/null) || \ - debug_options="$${debug_options}$${option} "; done; \ - ${QMTEST} -D ${QMTEST_DIR} create-tdb \ - -c gcc_database.GCCDatabase \ - -a GCCDatabase.testsuite_root=`cd ${srcdir}/testsuite && pwd` \ - -a GCCDatabase.debug_options="$${debug_options}" - rm -f empty.C empty.s - $(STAMP) stamp-qmtest + ${QMTEST} -D ${QMTEST_DIR} create-tdb \ + -c gcc_database.GCCDatabase \ + -a srcdir=`cd ${srcdir}/testsuite && pwd` && \ + $(STAMP) stamp-qmtest # Create the QMTest context file. ${QMTEST_DIR}/context: stamp-qmtest - echo "GCCTest.flags=-B${objdir}" >> $@ - echo "GCCTest.objdir=${objdir}/.." >> $@ - echo "GCCTest.host=${host_canonical}" >> $@ - echo "GCCTest.target=${target}" >> $@ - echo "GCCTest.gcov=${objdir}/gcov" >> $@ - echo "GPPTest.gpp=${objdir}/g++" >> $@ - echo "DGTest.demangler=${objdir}/c++filt" >> $@ + rm -f $@ + echo "CompilerTable.languages=c cplusplus" >> $@ + echo "CompilerTable.c_kind=GCC" >> $@ + echo "CompilerTable.c_path=${objdir}/xgcc" >> $@ + echo "CompilerTable.c_options=-B${objdir}/" >> $@ + echo "CompilerTable.cplusplus_kind=GCC" >> $@ + echo "CompilerTable.cplusplus_path=${objdir}/g++" >> $@ + echo "CompilerTable.cplusplus_options=-B${objdir}/" >> $@ + echo "DejaGNUTest.target=$(target)" >> $@ # Run the G++ testsuite using QMTest. -qmtest-g++: ${QMTEST_DIR}/context ${QMTEST_DIR}/gpp-expected.qmr +qmtest-g++: ${QMTEST_DIR}/context cd ${QMTEST_DIR} && ${QMTEST} run ${QMTESTRUNFLAGS} -C context \ - -o gpp.qmr -O gpp-expected.qmr \ - ${QMTEST_GPP_TESTS} + -o g++.qmr ${QMTEST_GPP_TESTS} # Use the QMTest GUI. qmtest-gui: ${QMTEST_DIR}/context cd ${QMTEST_DIR} && ${QMTEST} gui -C context -# Build the set of expected G++ failures. -${QMTEST_DIR}/gpp-expected.qmr: ${QMTEST_DIR}/context - echo "Determining expected results..." - cd ${QMTEST_DIR} && ${QMTEST} run ${QMTESTRUNFLAGS} -C context \ - -c "GCCTest.generate_xfails=1" -o gpp-expected.qmr \ - ${QMTEST_GPP_TESTS} \ - > /dev/null - .PHONY: qmtest-g++ # Run Paranoia on real.c. diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 1acdb65494c..9781d57008f 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,7 @@ +2003-06-04 Mark Mitchell + + * README.QMTEST: Update. + 2003-06-04 Richard Henderson * gcc.dg/cleanup-1.c: New. diff --git a/gcc/testsuite/README.QMTEST b/gcc/testsuite/README.QMTEST index 555b11aa5f4..19bd1a30497 100644 --- a/gcc/testsuite/README.QMTEST +++ b/gcc/testsuite/README.QMTEST @@ -10,51 +10,50 @@ officially supported testing procedure. Therefore, you must run the tests using DejaGNU (with "make check-g++") before committing changes that affect G++. -Differences from DejaGNU -======================== - -Although QMTest can be used to run the G++ test suite, it works -somewhat differently from DejaGNU. In particular: - -- In DejaGNU, a single source file contains many tests. Each - line where a diagnostic is expected is considered a separate - test. Testing for successful compilation and testing for - successful execution of the generated program are considered - separate tests. Thus, a test "test.C" could contain, say, - seven tests; some of which might pass and some of which might - fail. - - With QMTest, each source file is considered a single test. If any - of the seven sub-tests fail, the entire test is considered to fail. - However, QMTest does present information about *why* the test - failed, so the same information is effectively available. - - It is true that, therefore, causing an already failing test to "fail - more" is not immediately detectable through an additional unexpected - failure messages when using QMTest. On the other hand, most people - seem to think of each source file as "a test", not "twelve tests", - so the model QMTest uses may be more natural. - -- In DejaGNU, tests themselves keep track of expected and unexpected - failures. The QMTest philosophy is that expected failures should be - stored separately from the tests themselves; in particular, that - tonights results can be tomorrow's expectations. In order to - preserve compatibility with DejaGNU, the first time you use QMTest - to test G++, QMTest computes the set of expected failures indicated - by the tests, and then compares the actual results with these - results. Therefore, if you change the expected failure notations in - the DejaGNU tests, you must rebuild the set of expected failures. - - To do this, remove the file "qmtestsuite/gpp-expected.qmr". Then, - when you rerun the tests, the expected failures will be - automatically recalculated. +QMTest emulates DejaGNU behavior very closely when running the tests. + +QMTest has two output modes: a DejaGNU emulation mode and a native +QMTest mode. + +In the DejaGNU mode, you should receive output that is almost exactly +the same as the DejaGNU output; in particular, you should see the same +number of passes, failures, etc. When using the DejaGNU-style output, +QMTest uses the "xfail" indications in the test cases to determine +which tests are expected to pass and which are expected to fail, and +presents that information in the same way as DejaGNU. + +In the QMTest mode, the number of passes and failures will be +different from that obtained when using DejaGNU. The reason is that a +single source file may contain multiple DejaGNU tests. In DejaGNU, +each line where a diagnostic is expected is considered a separate +test. Testing for successful compilation and testing for successful +execution of the generated program are considered separate tests. So, +a single source file "test.C" could contain, say, seven tests; some of +which might pass and some of which might fail. + +In the QMTest mode, each source file is considered a single test. If +any of the seven sub-tests fail, the entire test is considered to +fail. However, QMTest does present information about *why* the test +failed, so the same information is effectively available. + +In the QMTest mode, whether or not a test is expected to fail is +determined not by an indication in the test, but rather by comparing +the new results to the results of a previous run. Testing for whether +a change caused a regression is very simple: run the tests before +making the change, run them again after making the change, and let +QMTest compare the results. + +The mode chosen only affects the output from QMTest, not how it runs +the tests or how it stores the data. Therefore, if you choose to run +in the QMTest mode and later want to get the DejaGNU style output, or +vice versa, you can do that as described below. Setting Up ========== You must download and install the following software: -- Python 2.1 (or greater) +- Python 2.2 (or greater) See http://www.python.org. @@ -63,50 +62,35 @@ You must download and install the following software: Installation instructions are available on the web-site. -- QMTest 1.1.4 (or greater) +- A current version of QMTest. No released version provides all of + the functionality required, so you must obtain QMTest from CVS. - See http://www.qmtest.com. + To do that, follow the instructions at: - QMTest is available at: + http://www.codesourcery.com/qm/qmtest_development - http://www.codesourcery.com/qm/qmtest_download - - or: - - ftp://ftp.codesourcery.com/pub/qmtest - - Installation instructions are available on the web-site. - -- QMTC 1.1 + Installation instructions are available in the file called README + after you check out QMTest. - This package is available from: +- The "qmtc" and "qmtest_gcc" QMTest support packages. These are + available from the same CVS repository as QMTest. For example, to + check out "qmtc", do: - ftp://ftp.codesourcery.com/pub/qmtest/qmtc/qmtc-.tar.gz - - See the file called INSTALL in the distribution. + cvs -d :pserver:anoncvs@cvs.codesourcery.com:/home/qm/Repository \ + co qmtc + You do not have to install these packages; you need only check them + out. + Running the Tests ================= -To run the tests, run "make qmtest-g++" in the gcc directory of your -build tree. The first time that you do this, QMTest will calculate -the set of tests that are expected to fail on your platform, so it -will take several minutes before you see any test results. After the -first time, QMTest will start running the tests much more quickly. - -If the test summary printed at the test run indicates no unexpected -failures, then G++ is behaving as expected on your target. (Some -unexpected passes are normal.) - -You can obtain detailed information about why tests failed in one -of two ways: +First, you must set QMTEST_CLASS_PATH so that it can find the qmtc and +qmtest_gcc support packages: + + export QMTEST_CLASS_PATH=/path/to/qmtc:/path/to/qmtest_gcc -1. By invoking QMTest with the "-f full" option. For example: - - make QMTESTRUNFLAGS="-f full" qmtest-g++ - -2. Examining the log file qmtestsuite/gpp.qmr after the tests have - run. +The, run "make qmtest-g++" in the gcc directory of your build tree. Here are some more advanced usage instructions: @@ -117,7 +101,7 @@ Here are some more advanced usage instructions: will run only the tests in the g++.dg subdirectory, and: - make QMTEST_GPP_TESTS="g++.dg/special/conpr1.C \ + make QMTEST_GPP_TESTS="g++.dg/special/conpr-1.C \ g++.old-deja/g++.other/access2.C" qmtest-g++ @@ -132,10 +116,16 @@ Here are some more advanced usage instructions: qmtest -v run -f full ... + (The "-f full" mode will provide detailed information about each + test as it runs.) + 3. To run the compiler with particular flags, use QMTESTRUNFLAGS to - set the QMTest context variable "GPPTest.flags", like this: + set the QMTest context variable "CompilerTable.cplusplus_options", + like this: - make QMTESTRUNFLAGS='-c GPPTest.flags="-funroll-loops"' qmtest-g++ + make \ + QMTESTRUNFLAGS='-c CompilerTable.cplusplus_options="-funroll-loops"' \ + qmtest-g++ The compiler will then use the "-funroll-loops" switch when compiling. @@ -149,7 +139,7 @@ Here are some more advanced usage instructions: make qmtest-gui - (Note that this will run the program called "netscape" in your path. + (Note that this will run the program called "mozilla" in your path. If you want to use another browser, you must configure qmtest as described in its manual.) @@ -168,20 +158,11 @@ Here are some more advanced usage instructions: across multiple machines; for more information see the QMTest manual.) -7. If you are testing a cross compiler, you must specify an interpreter - that is capable of running the generated program. It must be a - program "p" such that: - - p program arg1 arg2 arg3 ... - - behaves exactly like running: - - program arg1 arg2 arg3 ... - - would on the target machine. You specify this program via the - "CompilerTest.interpreter" context variable: - - make QMTESTRUNFLAGS='-c CompilerTest.interpreter=/path/to/interpreter' - qmtest-g++ +7. If a test (say "g++.dg/abi/bitfield1.C") fails, and you want to get + more detailed information, you can do: + cd qmtestsuite + qmtest summarize g++.qmr g++.dg/abi/bitfield1.C + to get more information about the commands that were run and the + output produced. -- 2.30.2