From 0a88346eb42c1871382b04c5e7d34bbc0180af29 Mon Sep 17 00:00:00 2001 From: Mathias Preiner Date: Mon, 8 Nov 2021 16:01:18 -0800 Subject: [PATCH] cmake: Use fastcov for generating coverage reports. (#7594) fastcov is a parallelized gcov wrapper. --- CMakeLists.txt | 27 +-- cmake/CodeCoverage.cmake | 421 ++++++--------------------------------- 2 files changed, 71 insertions(+), 377 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 11e12d89a..f7ff3bac3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -316,29 +316,14 @@ endif() if(ENABLE_COVERAGE) include(CodeCoverage) - APPEND_COVERAGE_COMPILER_FLAGS() + append_coverage_compiler_flags() add_definitions(-DCVC5_COVERAGE) - # Note: The ctest command returns a non-zero exit code if tests fail or run - # into a timeout. As a consequence, the coverage report is not generated. To - # prevent this we always return with exit code 0 after the ctest command has - # finished. - setup_target_for_coverage_gcovr_html( - NAME coverage-test - EXECUTABLE - ctest -j${CTEST_NTHREADS} -LE "example" - --output-on-failure $$ARGS || exit 0 - DEPENDS - build-tests) - - # Adds targets `coverage` and `coverage-reset` for manually generating - # coverage reports for specific executions. - # - # Target coverage-reset resets all the coverage counters to zero, while - # target coverage will generate a coverage report for all executions since - # the last coverage-reset. - setup_target_for_coverage_lcov_no_executable( + setup_code_coverage_fastcov( NAME coverage - DEPENDENCIES cvc5-bin) + PATH "${PROJECT_SOURCE_DIR}" + EXCLUDE "${CMAKE_BINARY_DIR}/deps/*" + DEPENDENCIES cvc5-bin + ) endif() if(ENABLE_DEBUG_CONTEXT_MM) diff --git a/cmake/CodeCoverage.cmake b/cmake/CodeCoverage.cmake index 035c2aa24..9027b21b3 100644 --- a/cmake/CodeCoverage.cmake +++ b/cmake/CodeCoverage.cmake @@ -1,360 +1,69 @@ -# Copyright (c) 2012 - 2017, Lars Bilke -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# CHANGES: -# -# 2012-01-31, Lars Bilke -# - Enable Code Coverage -# -# 2013-09-17, Joakim Söderberg -# - Added support for Clang. -# - Some additional usage instructions. -# -# 2016-02-03, Lars Bilke -# - Refactored functions to use named parameters -# -# 2017-06-02, Lars Bilke -# - Merged with modified version from github.com/ufz/ogs -# -# -# USAGE: -# -# 1. Copy this file into your cmake modules path. -# -# 2. Add the following line to your CMakeLists.txt: -# include(CodeCoverage) -# -# 3. Append necessary compiler flags: -# APPEND_COVERAGE_COMPILER_FLAGS() -# -# 4. If you need to exclude additional directories from the report, specify them -# using the COVERAGE_LCOV_EXCLUDES variable before calling SETUP_TARGET_FOR_COVERAGE_LCOV. -# Example: -# set(COVERAGE_LCOV_EXCLUDES 'dir1/*' 'dir2/*') -# -# 5. Use the functions described below to create a custom make target which -# runs your test executable and produces a code coverage report. -# -# 6. Build a Debug build: -# cmake -DCMAKE_BUILD_TYPE=Debug .. -# make -# make my_coverage_target -# - -include(CMakeParseArguments) - -# Check prereqs -find_program( GCOV_PATH gcov ) -find_program( LCOV_PATH NAMES lcov lcov.bat lcov.exe lcov.perl) -find_program( GENHTML_PATH NAMES genhtml genhtml.perl genhtml.bat ) -find_program( GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/scripts/test) -find_program( SIMPLE_PYTHON_EXECUTABLE python ) - -if(NOT GCOV_PATH) - message(FATAL_ERROR "gcov not found! Aborting...") -endif() # NOT GCOV_PATH - -if("${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang") - if("${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS 3) - message(FATAL_ERROR "Clang version must be 3.0.0 or greater! Aborting...") - endif() -elseif(NOT CMAKE_COMPILER_IS_GNUCXX) - message(FATAL_ERROR "Compiler is not GNU gcc! Aborting...") -endif() +find_program(LCOV_BINARY lcov REQUIRED) +find_program(GENHTML_BINARY NAMES genhtml REQUIRED) +find_program(FASTCOV_BINARY fastcov REQUIRED) set(COVERAGE_COMPILER_FLAGS "-g -O0 --coverage -fprofile-arcs -ftest-coverage" CACHE INTERNAL "") -set(CMAKE_CXX_FLAGS_COVERAGE - ${COVERAGE_COMPILER_FLAGS} - CACHE STRING "Flags used by the C++ compiler during coverage builds." - FORCE ) -set(CMAKE_C_FLAGS_COVERAGE - ${COVERAGE_COMPILER_FLAGS} - CACHE STRING "Flags used by the C compiler during coverage builds." - FORCE ) -set(CMAKE_EXE_LINKER_FLAGS_COVERAGE - "" - CACHE STRING "Flags used for linking binaries during coverage builds." - FORCE ) -set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE - "" - CACHE STRING "Flags used by the shared libraries linker during coverage builds." - FORCE ) -mark_as_advanced( - CMAKE_CXX_FLAGS_COVERAGE - CMAKE_C_FLAGS_COVERAGE - CMAKE_EXE_LINKER_FLAGS_COVERAGE - CMAKE_SHARED_LINKER_FLAGS_COVERAGE ) - -if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading") -endif() # NOT CMAKE_BUILD_TYPE STREQUAL "Debug" - -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - link_libraries(gcov) -else() - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") -endif() - -# Defines a target for running and collection code coverage information -# Builds dependencies, runs the given executable and outputs reports. -# NOTE! The executable should always have a ZERO as exit code otherwise -# the coverage generation will not complete. -# -# SETUP_TARGET_FOR_COVERAGE_LCOV( -# NAME testrunner_coverage # New target name -# EXECUTABLE testrunner -j ${PROCESSOR_COUNT} # Executable in PROJECT_BINARY_DIR -# DEPENDENCIES testrunner # Dependencies to build first -# ) -function(SETUP_TARGET_FOR_COVERAGE_LCOV) - - set(options NONE) - set(oneValueArgs NAME) - set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES) - cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if(NOT LCOV_PATH) - message(FATAL_ERROR "lcov not found! Aborting...") - endif() # NOT LCOV_PATH - - if(NOT GENHTML_PATH) - message(FATAL_ERROR "genhtml not found! Aborting...") - endif() # NOT GENHTML_PATH - - # Setup target - add_custom_target(${Coverage_NAME} - - # Cleanup lcov - COMMAND ${LCOV_PATH} --directory . --zerocounters - # Create baseline to make sure untouched files show up in the report - COMMAND ${LCOV_PATH} -c -i -d . -o ${Coverage_NAME}.base - - # Run tests - COMMAND ${Coverage_EXECUTABLE} - - # Capturing lcov counters and generating report - COMMAND ${LCOV_PATH} --directory . --capture --output-file ${Coverage_NAME}.info - # add baseline counters - COMMAND ${LCOV_PATH} -a ${Coverage_NAME}.base -a ${Coverage_NAME}.info --output-file ${Coverage_NAME}.total - COMMAND ${LCOV_PATH} --remove ${Coverage_NAME}.total ${COVERAGE_LCOV_EXCLUDES} --output-file ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned - COMMAND ${GENHTML_PATH} -o ${Coverage_NAME} ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned - COMMAND ${CMAKE_COMMAND} -E remove ${Coverage_NAME}.base ${Coverage_NAME}.total ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned - - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - DEPENDS ${Coverage_DEPENDENCIES} - COMMENT "Resetting code coverage counters to zero.\nProcessing code coverage counters and generating report." - ) - - # Show where to find the lcov info report - add_custom_command(TARGET ${Coverage_NAME} POST_BUILD - COMMAND ; - COMMENT "Lcov code coverage info report saved in ${Coverage_NAME}.info." - ) - - # Show info where to find the report - add_custom_command(TARGET ${Coverage_NAME} POST_BUILD - COMMAND ; - COMMENT "Open ./${Coverage_NAME}/index.html in your browser to view the coverage report." - ) - -endfunction() # SETUP_TARGET_FOR_COVERAGE_LCOV - -function(SETUP_TARGET_FOR_COVERAGE_LCOV_NO_EXECUTABLE) - - set(options NONE) - set(oneValueArgs NAME) - cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if(NOT LCOV_PATH) - message(FATAL_ERROR "lcov not found! Aborting...") - endif() # NOT LCOV_PATH - - if(NOT GENHTML_PATH) - message(FATAL_ERROR "genhtml not found! Aborting...") - endif() # NOT GENHTML_PATH - - set(DIRECTORIES -d .) - foreach(LPATH ${COVERAGE_LCOV_PATHS}) - list(APPEND DIRECTORIES "-d") - list(APPEND DIRECTORIES "${LPATH}") - endforeach() - - - add_custom_target(${Coverage_NAME}-reset - # Cleanup lcov - COMMAND ${LCOV_PATH} ${DIRECTORIES} --zerocounters - # Create baseline to make sure untouched files show up in the report - COMMAND ${LCOV_PATH} -c -i ${DIRECTORIES} -o ${Coverage_NAME}.base - COMMENT "Resetting code coverage counters to zero." - ) - - # Setup target - add_custom_target(${Coverage_NAME} - # Capturing lcov counters and generating report - COMMAND ${LCOV_PATH} ${DIRECTORIES} --capture --output-file ${Coverage_NAME}.info - # add baseline counters - COMMAND ${LCOV_PATH} -a ${Coverage_NAME}.base -a ${Coverage_NAME}.info --output-file ${Coverage_NAME}.total - COMMAND ${LCOV_PATH} --remove ${Coverage_NAME}.total ${COVERAGE_LCOV_EXCLUDES} --output-file ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned - COMMAND ${GENHTML_PATH} -o ${Coverage_NAME} ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned - - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - DEPENDS ${Coverage_DEPENDENCIES} - COMMENT "Processing code coverage counters and generating report." - ) - - # Show where to find the lcov info report - add_custom_command(TARGET ${Coverage_NAME} POST_BUILD - COMMAND ; - COMMENT "Lcov code coverage info report saved in ${Coverage_NAME}.info." - ) - - # Show info where to find the report - add_custom_command(TARGET ${Coverage_NAME} POST_BUILD - COMMAND ; - COMMENT "Open ./${Coverage_NAME}/index.html in your browser to view the coverage report." - ) - -endfunction() # SETUP_TARGET_FOR_COVERAGE_LCOV_NO_EXECUTABLE - -# Defines a target for running and collection code coverage information -# Builds dependencies, runs the given executable and outputs reports. -# NOTE! The executable should always have a ZERO as exit code otherwise -# the coverage generation will not complete. -# -# SETUP_TARGET_FOR_COVERAGE_GCOVR_XML( -# NAME ctest_coverage # New target name -# EXECUTABLE ctest -j ${PROCESSOR_COUNT} # Executable in PROJECT_BINARY_DIR -# DEPENDENCIES executable_target # Dependencies to build first -# ) -function(SETUP_TARGET_FOR_COVERAGE_GCOVR_XML) - - set(options NONE) - set(oneValueArgs NAME) - set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES) - cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if(NOT SIMPLE_PYTHON_EXECUTABLE) - message(FATAL_ERROR "python not found! Aborting...") - endif() # NOT SIMPLE_PYTHON_EXECUTABLE - - if(NOT GCOVR_PATH) - message(FATAL_ERROR "gcovr not found! Aborting...") - endif() # NOT GCOVR_PATH - - # Combine excludes to several -e arguments - set(GCOVR_EXCLUDES "") - foreach(EXCLUDE ${COVERAGE_GCOVR_EXCLUDES}) - list(APPEND GCOVR_EXCLUDES "-e") - list(APPEND GCOVR_EXCLUDES "${EXCLUDE}") - endforeach() - - add_custom_target(${Coverage_NAME} - # Run tests - ${Coverage_EXECUTABLE} - - # Running gcovr - COMMAND ${GCOVR_PATH} --xml - -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES} - --object-directory=${PROJECT_BINARY_DIR} - -o ${Coverage_NAME}.xml - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - DEPENDS ${Coverage_DEPENDENCIES} - COMMENT "Running gcovr to produce Cobertura code coverage report." - ) - - # Show info where to find the report - add_custom_command(TARGET ${Coverage_NAME} POST_BUILD - COMMAND ; - COMMENT "Cobertura code coverage report saved in ${Coverage_NAME}.xml." - ) - -endfunction() # SETUP_TARGET_FOR_COVERAGE_GCOVR_XML - -# Defines a target for running and collection code coverage information -# Builds dependencies, runs the given executable and outputs reports. -# NOTE! The executable should always have a ZERO as exit code otherwise -# the coverage generation will not complete. -# -# SETUP_TARGET_FOR_COVERAGE_GCOVR_HTML( -# NAME ctest_coverage # New target name -# EXECUTABLE ctest -j ${PROCESSOR_COUNT} # Executable in PROJECT_BINARY_DIR -# DEPENDENCIES executable_target # Dependencies to build first -# ) -function(SETUP_TARGET_FOR_COVERAGE_GCOVR_HTML) - - set(options NONE) - set(oneValueArgs NAME) - set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES) - cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if(NOT SIMPLE_PYTHON_EXECUTABLE) - message(FATAL_ERROR "python not found! Aborting...") - endif() # NOT SIMPLE_PYTHON_EXECUTABLE - - if(NOT GCOVR_PATH) - message(FATAL_ERROR "gcovr not found! Aborting...") - endif() # NOT GCOVR_PATH - - # Combine excludes to several -e arguments - set(GCOVR_EXCLUDES "") - foreach(EXCLUDE ${COVERAGE_GCOVR_EXCLUDES}) - list(APPEND GCOVR_EXCLUDES "-e") - list(APPEND GCOVR_EXCLUDES "${EXCLUDE}") - endforeach() - - add_custom_target(${Coverage_NAME} - # Run tests - ${Coverage_EXECUTABLE} - - # Create folder - COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/${Coverage_NAME} - - # Running gcovr - COMMAND ${GCOVR_PATH} --html --html-details - -r ${PROJECT_SOURCE_DIR} ${GCOVR_EXCLUDES} - --object-directory=${PROJECT_BINARY_DIR} - -o ${Coverage_NAME}/index.html - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - DEPENDS ${Coverage_DEPENDENCIES} - COMMENT "Running gcovr to produce HTML code coverage report." - ) - - # Show info where to find the report - add_custom_command(TARGET ${Coverage_NAME} POST_BUILD - COMMAND ; - COMMENT "Open ./${Coverage_NAME}/index.html in your browser to view the coverage report." - ) - -endfunction() # SETUP_TARGET_FOR_COVERAGE_GCOVR_HTML - -function(APPEND_COVERAGE_COMPILER_FLAGS) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE) - message(STATUS "Appending code coverage compiler flags: ${COVERAGE_COMPILER_FLAGS}") -endfunction() # APPEND_COVERAGE_COMPILER_FLAGS +## +# Add compilers flags for code coverage. +## +function(append_coverage_compiler_flags) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE) + message(STATUS "Appending code coverage compiler flags: ${COVERAGE_COMPILER_FLAGS}") +endfunction() + +## +# Sets up code coverage targets NAME and NAME-reset +# +# NAME-reset: Reset code coverage counters to zero. +# NAME: Generate code coverage report since the last reset. +# +# Options: +# NAME: name of the target +# DEPENDENCIES: list of dependencies +# PATHS: additional source code directories to include in the report +# +## +function(setup_code_coverage_fastcov) + set(options NONE) + set(oneValueArgs NAME PATH) + set(multiValueArgs DEPENDENCIES EXCLUDE) + cmake_parse_arguments( + COVERAGE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(EXCLUDES -e /usr/include) + foreach(DIR ${COVERAGE_EXCLUDE}) + list(APPEND EXCLUDES "${DIR}") + message(STATUS "Exclude ${DIR} in coverage reports") + endforeach() + + if(DEFINED ENV{FASTCOV_PARALLEL_JOBS}) + set(FASTCOV_PARALLEL_JOBS $ENV{FASTCOV_PARALLEL_JOBS}) + else() + include(ProcessorCount) + ProcessorCount(FASTCOV_PARALLEL_JOBS) + endif() + + add_custom_target(${COVERAGE_NAME}-reset + COMMAND + ${FASTCOV_BINARY} -d ${COVERAGE_PATH} ${EXCLUDES} --zerocounters + -j${FASTCOV_PARALLEL_JOBS} + COMMENT + "Resetting code coverage counters to zero." + ) + + add_custom_target(${COVERAGE_NAME} + COMMAND + ${FASTCOV_BINARY} + -d ${COVERAGE_PATH} --lcov ${EXCLUDES} -o coverage.info + -j${FASTCOV_PARALLEL_JOBS} + COMMAND + ${GENHTML_BINARY} --no-prefix -o coverage coverage.info + DEPENDS + ${COVERAGE_DEPENDENCIES} + COMMENT + "Generate code coverage report." + ) +endfunction() -- 2.30.2