scons: Add support for debug info compression.

If supported this will compress the debug information in object files,
libraries, and binaries. This decreases the size of the build/ARM
directory from 11GB to 7.2GB.

Because the benefit of this mechanism depends on the performance and
capacity of the build machine's storage, it can be disabled with the
--no-compress-debug scons flag. For instance if your storage is very
fast, writing out a larger binary may take less time than compressing
that same number of bytes, plus the time to write out the smaller
file.

This feature seems to make our presubmit test machine run out of memory
when trying to build gem5, and so is disabled in the testing scripts.

Change-Id: I71919062d23742b7658918b0fa9c4d91d0521fbf
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/40715
Reviewed-by: Gabe Black <gabe.black@gmail.com>
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Gabe Black <gabe.black@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

diff --git a/SConstruct b/SConstruct
index f744c77df65868c601db3973d7b0003449e3afcf..0e5d19fcfdad690568c27b0fa2c42f228562564c 100755 (executable)
--- a/SConstruct
+++ b/SConstruct
@@ -108,6 +108,8 @@ AddOption('--default',
 AddOption('--ignore-style', action='store_true',
           help='Disable style checking hooks')
 AddOption('--gold-linker', action='store_true', help='Use the gold linker')
+AddOption('--no-compress-debug', action='store_true',
+          help="Don't compress debug info in build files")
 AddOption('--no-lto', action='store_true',
           help='Disable Link-Time Optimization for fast')
 AddOption('--force-lto', action='store_true',
@@ -383,8 +385,8 @@ if main['GCC']:
         # https://gcc.gnu.org/ml/gcc-patches/2015-11/msg03161.html
         # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69866
         if not GetOption('force_lto'):
-            main.Append(PSHLINKFLAGS='-flinker-output=rel')
-            main.Append(PLINKFLAGS='-flinker-output=rel')
+            main.Append(PSHLINKFLAGS=['-flinker-output=rel'])
+            main.Append(PLINKFLAGS=['-flinker-output=rel'])
 
     disable_lto = GetOption('no_lto')
     if not disable_lto and main.get('BROKEN_INCREMENTAL_LTO', False) and \
@@ -553,6 +555,24 @@ timeout_version = timeout_lines[0].split() if timeout_lines else []
 main['TIMEOUT'] =  timeout_version and \
     compareVersions(timeout_version[-1], '8.13') >= 0
 
+def CheckCxxFlag(context, flag):
+    context.Message("Checking for compiler %s support... " % flag)
+    last_cxxflags = context.env['CXXFLAGS']
+    context.env.Append(CXXFLAGS=[flag])
+    ret = context.TryCompile('', '.cc')
+    context.env['CXXFLAGS'] = last_cxxflags
+    context.Result(ret)
+    return ret
+
+def CheckLinkFlag(context, flag):
+    context.Message("Checking for linker %s support... " % flag)
+    last_linkflags = context.env['LINKFLAGS']
+    context.env.Append(LINKFLAGS=[flag])
+    ret = context.TryLink('int main(int, char *[]) { return 0; }', '.cc')
+    context.env['LINKFLAGS'] = last_linkflags
+    context.Result(ret)
+    return ret
+
 # Add a custom Check function to test for structure members.
 def CheckMember(context, include, decl, member, include_quotes="<>"):
     context.Message("Checking for member %s in %s..." %
@@ -602,6 +622,8 @@ conf = Configure(main,
                  custom_tests = {
         'CheckMember' : CheckMember,
         'CheckPythonLib' : CheckPythonLib,
+        'CheckCxxFlag' : CheckCxxFlag,
+        'CheckLinkFlag' : CheckLinkFlag,
         })
 
 # Check if we should compile a 64 bit binary on Mac OS X/Darwin
@@ -641,6 +663,16 @@ if main['M5_BUILD_CACHE']:
     print('Using build cache located at', main['M5_BUILD_CACHE'])
     CacheDir(main['M5_BUILD_CACHE'])
 
+if not GetOption('no_compress_debug'):
+    if conf.CheckCxxFlag('-gz'):
+        main.Append(CXXFLAGS=['-gz'])
+    else:
+        warning("Can't enable object file debug section compression")
+    if conf.CheckLinkFlag('-gz'):
+        main.Append(LINKFLAGS=['-gz'], SHLINKFLAGS=['-gz'])
+    else:
+        warning("Can't enable executable debug section compression")
+
 main['USE_PYTHON'] = not GetOption('without_python')
 if main['USE_PYTHON']:
     # Find Python include and library directories for embedding the

diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py
index 5ffb2482adec10dbfe6e46173c574a314ae10473..a6b2881e8b1189314f2b48c3dec41e0c28e0fdbd 100644 (file)
--- a/tests/gem5/fixture.py
+++ b/tests/gem5/fixture.py
@@ -151,7 +151,8 @@ class SConsFixture(UniqueFixture):
         command = [
             'scons', '-C', self.directory,
             '-j', str(config.threads),
-            '--ignore-style'
+            '--ignore-style',
+            '--no-compress-debug'
         ]
 
         if not self.targets:

diff --git a/tests/jenkins/presubmit-stage2.sh b/tests/jenkins/presubmit-stage2.sh
index be90b2b1186b17c57a1bab15ea7260d209119eaf..aed60fd4a62577ddbaa2606968439ccaa671a6cb 100755 (executable)
--- a/tests/jenkins/presubmit-stage2.sh
+++ b/tests/jenkins/presubmit-stage2.sh
@@ -46,4 +46,5 @@ set -e
 # Look for tests in the gem5 subdirectory
 # Once complete, run the Google Tests
 cd tests
-./main.py run -j4 -t4 gem5 && scons -C .. build/NULL/unittests.opt
+./main.py run -j4 -t4 gem5 && scons -C .. --no-compress-debug \
+    build/NULL/unittests.opt

diff --git a/tests/jenkins/presubmit.sh b/tests/jenkins/presubmit.sh
index 8e76d981db2fa532caf53fd656615250e24494b3..f27c23c9caf70c2ea553431f13fdc1fd79812620 100755 (executable)
--- a/tests/jenkins/presubmit.sh
+++ b/tests/jenkins/presubmit.sh
@@ -62,4 +62,4 @@ docker run -u $UID:$GID --volume $(pwd):$(pwd) -w $(pwd) --rm \
 rm -rf build
 docker run -u $UID:$GID --volume $(pwd):$(pwd) -w $(pwd) --rm \
     "${DOCKER_IMAGE_CLANG_COMPILE}" /usr/bin/env python3 /usr/bin/scons \
-    build/X86/gem5.fast -j4
+    build/X86/gem5.fast -j4 --no-compress-debug