+2019-07-03 Martin Liska <mliska@suse.cz>
+
+ * Makefile.in: Define ZSTD_LIB.
+ * common.opt: Adjust compression level
+ to support also zstd levels.
+ * config.in: Regenerate.
+ * configure: Likewise.
+ * configure.ac: Add --with-zstd and --with-zstd-include options
+ and detect ZSTD.
+ * doc/install.texi: Mention zstd dependency.
+ * gcc.c: Print supported LTO compression algorithms.
+ * lto-compress.c (lto_normalized_zstd_level): Likewise.
+ (lto_compression_zstd): Likewise.
+ (lto_uncompression_zstd): Likewise.
+ (lto_end_compression): Dispatch in between zlib and zstd.
+ (lto_compression_zlib): Mark with ATTRIBUTE_UNUSED.
+ (lto_uncompression_zlib): Make it static.
+ * lto-compress.h (lto_end_uncompression): Fix GNU coding style.
+ * lto-section-in.c (lto_get_section_data): Pass info
+ about used compression.
+ * lto-streamer-out.c: By default use zstd when possible.
+ * timevar.def (TV_IPA_LTO_DECOMPRESS): Rename to decompression
+ (TV_IPA_LTO_COMPRESS): Likewise for compression.
+
2019-07-03 Martin Liska <mliska@suse.cz>
* lto-section-in.c (lto_get_section_data): Add "lto" section.
LIBS = @LIBS@ libcommon.a $(CPPLIB) $(LIBINTL) $(LIBICONV) $(LIBBACKTRACE) \
$(LIBIBERTY) $(LIBDECNUMBER) $(HOST_LIBS)
BACKENDLIBS = $(ISLLIBS) $(GMPLIBS) $(PLUGINLIBS) $(HOST_LIBS) \
- $(ZLIB)
+ $(ZLIB) $(ZSTD_LIB)
# Any system libraries needed just for GNAT.
SYSLIBS = @GNAT_LIBEXC@
# Libs needed (at present) just for jcf-dump.
LDEXP_LIB = @LDEXP_LIB@
+ZSTD_LIB = @ZSTD_LIB@
+
# Likewise, for use in the tools that must run on this machine
# even if we are cross-building GCC.
BUILD_LIBS = $(BUILD_LIBIBERTY)
; The initial value of -1 comes from Z_DEFAULT_COMPRESSION in zlib.h.
flto-compression-level=
-Common Joined RejectNegative UInteger Var(flag_lto_compression_level) Init(-1) IntegerRange(0, 9)
--flto-compression-level=<number> Use zlib compression level <number> for IL.
+Common Joined RejectNegative UInteger Var(flag_lto_compression_level) Init(-1) IntegerRange(0, 19)
+-flto-compression-level=<number> Use zlib/zstd compression level <number> for IL.
flto-odr-type-merging
Common Ignore
#endif
+/* Define if you have a working <zstd.h> header file. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_ZSTD_H
+#endif
+
+
/* Define if isl is in use. */
#ifndef USED_FOR_TARGET
#undef HAVE_isl
LIBICONV_DEP
LTLIBICONV
LIBICONV
+ZSTD_LIB
+ZSTD_INCLUDE
DL_LIB
LDEXP_LIB
EXTRA_GCC_LIBS
with_bugurl
enable_languages
with_multilib_list
+with_zstd
+with_zstd_include
+with_zstd_lib
enable_rpath
with_libiconv_prefix
enable_sjlj_exceptions
--with-pkgversion=PKG Use PKG in the version string in place of "GCC"
--with-bugurl=URL Direct users to URL to report a bug
--with-multilib-list select multilibs (AArch64, SH and x86-64 only)
+ --with-zstd=PATH specify prefix directory for installed zstd library.
+ Equivalent to --with-zstd-include=PATH/include plus
+ --with-zstd-lib=PATH/lib
+ --with-zstd-include=PATH
+ specify directory for installed zstd include files
+ --with-zstd-lib=PATH specify directory for the installed zstd library
--with-gnu-ld assume the C compiler uses GNU ld default=no
--with-libiconv-prefix[=DIR] search for libiconv in DIR/include and DIR/lib
--without-libiconv-prefix don't search for libiconv in includedir and libdir
fi
+# Look for the ZSTD package.
+ZSTD_INCLUDE=
+ZSTD_LIB=
+
+
+ZSTD_CPPFLAGS=
+ZSTD_LDFLAGS=
+
+# Check whether --with-zstd was given.
+if test "${with_zstd+set}" = set; then :
+ withval=$with_zstd;
+fi
+
+
+# Check whether --with-zstd-include was given.
+if test "${with_zstd_include+set}" = set; then :
+ withval=$with_zstd_include;
+fi
+
+
+# Check whether --with-zstd-lib was given.
+if test "${with_zstd_lib+set}" = set; then :
+ withval=$with_zstd_lib;
+fi
+
+case "x$with_zstd" in
+ x) ;;
+ xno)
+ ZSTD_INCLUDE=no
+ ZSTD_LIB=no
+ ;;
+ *) ZSTD_INCLUDE=$with_zstd/include
+ ZSTD_LIB=$with_zstd/lib
+ ;;
+esac
+if test "x$with_zstd_include" != x; then
+ ZSTD_INCLUDE=$with_zstd_include
+fi
+if test "x$with_zstd_lib" != x; then
+ ZSTD_LIB=$with_zstd_lib
+fi
+if test "x$ZSTD_INCLUDE" != x \
+ && test "x$ZSTD_INCLUDE" != xno; then
+ ZSTD_CPPFLAGS=-I$ZSTD_INCLUDE
+fi
+if test "x$ZSTD_LIB" != x \
+ && test "x$ZSTD_LIB" != xno; then
+ ZSTD_LDFLAGS=-L$ZSTD_LIB
+fi
+
+CXXFLAGS="$CXXFLAGS $ZSTD_CPPFLAGS"
+LDFLAGS="$LDFLAGS $ZSTD_LDFLAGS"
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for zstd.h" >&5
+$as_echo_n "checking for zstd.h... " >&6; }
+if ${gcc_cv_header_zstd_h+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <zstd.h>
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+ gcc_cv_header_zstd_h=yes
+else
+ gcc_cv_header_zstd_h=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_header_zstd_h" >&5
+$as_echo "$gcc_cv_header_zstd_h" >&6; }
+if test $gcc_cv_header_zstd_h = yes; then
+
+$as_echo "#define HAVE_ZSTD_H 1" >>confdefs.h
+
+fi
+
+# LTO can use zstd compression algorithm
+save_LIBS="$LIBS"
+LIBS=
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing ZSTD_compress" >&5
+$as_echo_n "checking for library containing ZSTD_compress... " >&6; }
+if ${ac_cv_search_ZSTD_compress+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ZSTD_compress ();
+int
+main ()
+{
+return ZSTD_compress ();
+ ;
+ return 0;
+}
+_ACEOF
+for ac_lib in '' zstd; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ fi
+ if ac_fn_cxx_try_link "$LINENO"; then :
+ ac_cv_search_ZSTD_compress=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext
+ if ${ac_cv_search_ZSTD_compress+:} false; then :
+ break
+fi
+done
+if ${ac_cv_search_ZSTD_compress+:} false; then :
+
+else
+ ac_cv_search_ZSTD_compress=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZSTD_compress" >&5
+$as_echo "$ac_cv_search_ZSTD_compress" >&6; }
+ac_res=$ac_cv_search_ZSTD_compress
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+fi
+
+ZSTD_LIB="$LIBS"
+LIBS="$save_LIBS"
+
+
for ac_func in times clock kill getrlimit setrlimit atoq \
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 18658 "configure"
+#line 18817 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 18764 "configure"
+#line 18923 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
[Define if you have a working <inttypes.h> header file.])
fi
+# Look for the ZSTD package.
+ZSTD_INCLUDE=
+ZSTD_LIB=
+AC_SUBST(ZSTD_INCLUDE)
+AC_SUBST(ZSTD_LIB)
+ZSTD_CPPFLAGS=
+ZSTD_LDFLAGS=
+AC_ARG_WITH(zstd,
+ [AS_HELP_STRING([--with-zstd=PATH],
+ [specify prefix directory for installed zstd library.
+ Equivalent to --with-zstd-include=PATH/include
+ plus --with-zstd-lib=PATH/lib])])
+AC_ARG_WITH(zstd-include,
+ [AS_HELP_STRING([--with-zstd-include=PATH],
+ [specify directory for installed zstd include files])])
+AC_ARG_WITH(zstd-lib,
+ [AS_HELP_STRING([--with-zstd-lib=PATH],
+ [specify directory for the installed zstd library])])
+case "x$with_zstd" in
+ x) ;;
+ xno)
+ ZSTD_INCLUDE=no
+ ZSTD_LIB=no
+ ;;
+ *) ZSTD_INCLUDE=$with_zstd/include
+ ZSTD_LIB=$with_zstd/lib
+ ;;
+esac
+if test "x$with_zstd_include" != x; then
+ ZSTD_INCLUDE=$with_zstd_include
+fi
+if test "x$with_zstd_lib" != x; then
+ ZSTD_LIB=$with_zstd_lib
+fi
+if test "x$ZSTD_INCLUDE" != x \
+ && test "x$ZSTD_INCLUDE" != xno; then
+ ZSTD_CPPFLAGS=-I$ZSTD_INCLUDE
+fi
+if test "x$ZSTD_LIB" != x \
+ && test "x$ZSTD_LIB" != xno; then
+ ZSTD_LDFLAGS=-L$ZSTD_LIB
+fi
+
+CXXFLAGS="$CXXFLAGS $ZSTD_CPPFLAGS"
+LDFLAGS="$LDFLAGS $ZSTD_LDFLAGS"
+
+AC_MSG_CHECKING(for zstd.h)
+AC_CACHE_VAL(gcc_cv_header_zstd_h,
+[AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+[[#include <zstd.h>]])],
+ [gcc_cv_header_zstd_h=yes],
+ [gcc_cv_header_zstd_h=no])])
+AC_MSG_RESULT($gcc_cv_header_zstd_h)
+if test $gcc_cv_header_zstd_h = yes; then
+ AC_DEFINE(HAVE_ZSTD_H, 1,
+ [Define if you have a working <zstd.h> header file.])
+fi
+
+# LTO can use zstd compression algorithm
+save_LIBS="$LIBS"
+LIBS=
+AC_SEARCH_LIBS(ZSTD_compress, zstd)
+ZSTD_LIB="$LIBS"
+LIBS="$save_LIBS"
+AC_SUBST(ZSTD_LIB)
+
dnl Disabled until we have a complete test for buggy enum bitfields.
dnl gcc_AC_C_ENUM_BF_UNSIGNED
option should be used if isl is not installed in your default library
search path.
+@item zstd Library.
+
+Necessary to build GCC with zstd compression used for LTO bytecode.
+The library is searched in your default library patch search.
+Alternatively, the @option{--with-zstd} configure option should be used.
+
@end table
@heading Tools/packages necessary for modifying GCC
#endif
fnotice (file, "Thread model: %s\n", thrmod);
+ fnotice (file, "Supported LTO compression algorithms: zlib");
+#ifdef HAVE_ZSTD_H
+ fnotice (file, " zstd");
+#endif
+ fnotice (file, "\n");
/* compiler_version is truncated at the first space when initialized
from version string, so truncate version_string at the first space
#include "lto-compress.h"
#include "timevar.h"
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
/* Compression stream structure, holds the flush callback and opaque token,
the buffered data, and a note of whether compressing or uncompressing. */
return level;
}
+/* Free the buffer and memory associated with STREAM. */
+
+static void
+lto_destroy_compression_stream (struct lto_compression_stream *stream)
+{
+ free (stream->buffer);
+ free (stream);
+}
+
+#ifdef HAVE_ZSTD_H
+/* Return a zstd compression level that zstd will not reject. Normalizes
+ the compression level from the command line flag, clamping non-default
+ values to the appropriate end of their valid range. */
+
+static int
+lto_normalized_zstd_level (void)
+{
+ int level = flag_lto_compression_level;
+
+ if (level != ZSTD_CLEVEL_DEFAULT)
+ {
+ if (level < 1)
+ level = 1;
+ else if (level > ZSTD_maxCLevel ())
+ level = ZSTD_maxCLevel ();
+ }
+
+ return level;
+}
+
+/* Compress STREAM using ZSTD algorithm. */
+
+static void
+lto_compression_zstd (struct lto_compression_stream *stream)
+{
+ unsigned char *cursor = (unsigned char *) stream->buffer;
+ size_t size = stream->bytes;
+
+ timevar_push (TV_IPA_LTO_COMPRESS);
+ size_t const outbuf_length = ZSTD_compressBound (size);
+ char *outbuf = (char *) xmalloc (outbuf_length);
+
+ size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
+ lto_normalized_zstd_level ());
+
+ if (ZSTD_isError (csize))
+ internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));
+
+ stream->callback (outbuf, csize, NULL);
+
+ lto_destroy_compression_stream (stream);
+ free (outbuf);
+ timevar_pop (TV_IPA_LTO_COMPRESS);
+}
+
+/* Uncompress STREAM using ZSTD algorithm. */
+
+static void
+lto_uncompression_zstd (struct lto_compression_stream *stream)
+{
+ unsigned char *cursor = (unsigned char *) stream->buffer;
+ size_t size = stream->bytes;
+
+ timevar_push (TV_IPA_LTO_DECOMPRESS);
+ unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
+ if (rsize == ZSTD_CONTENTSIZE_ERROR)
+ internal_error ("original not compressed with zstd");
+ else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
+ internal_error ("original size unknown");
+
+ char *outbuf = (char *) xmalloc (rsize);
+ size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);
+
+ if (ZSTD_isError (dsize))
+ internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));
+
+ stream->callback (outbuf, dsize, stream->opaque);
+
+ lto_destroy_compression_stream (stream);
+ free (outbuf);
+ timevar_pop (TV_IPA_LTO_DECOMPRESS);
+}
+
+#endif
+
/* Create a new compression stream, with CALLBACK flush function passed
OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */
stream->bytes += num_chars;
}
-/* Free the buffer and memory associated with STREAM. */
-
-static void
-lto_destroy_compression_stream (struct lto_compression_stream *stream)
-{
- free (stream->buffer);
- free (stream);
-}
-
/* Return a new compression stream, with CALLBACK flush function passed
OPAQUE token. */
lto_stats.num_output_il_bytes += num_chars;
}
-/* Finalize STREAM compression, and free stream allocations. */
-
-void
-lto_end_compression (struct lto_compression_stream *stream)
+static void ATTRIBUTE_UNUSED
+lto_compression_zlib (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t remaining = stream->bytes;
timevar_pop (TV_IPA_LTO_COMPRESS);
}
+void
+lto_end_compression (struct lto_compression_stream *stream)
+{
+#ifdef HAVE_ZSTD_H
+ lto_compression_zstd (stream);
+#else
+ lto_compression_zlib (stream);
+#endif
+}
+
/* Return a new uncompression stream, with CALLBACK flush function passed
OPAQUE token. */
lto_stats.num_input_il_bytes += num_chars;
}
-/* Finalize STREAM uncompression, and free stream allocations.
-
- Because of the way LTO IL streams are compressed, there may be several
- concatenated compressed segments in the accumulated data, so for this
- function we iterate decompressions until no data remains. */
-
-void
-lto_end_uncompression (struct lto_compression_stream *stream)
+static void
+lto_uncompression_zlib (struct lto_compression_stream *stream)
{
unsigned char *cursor = (unsigned char *) stream->buffer;
size_t remaining = stream->bytes;
free (outbuf);
timevar_pop (TV_IPA_LTO_DECOMPRESS);
}
+
+void
+lto_end_uncompression (struct lto_compression_stream *stream,
+ lto_compression compression)
+{
+#ifdef HAVE_ZSTD_H
+ if (compression == ZSTD)
+ {
+ lto_uncompression_zstd (stream);
+ return;
+ }
+#endif
+ if (compression == ZSTD)
+ internal_error ("compiler does not support ZSTD LTO compression");
+
+ lto_uncompression_zlib (stream);
+}
void *opaque);
extern void lto_uncompress_block (struct lto_compression_stream *stream,
const char *base, size_t num_chars);
-extern void lto_end_uncompression (struct lto_compression_stream *stream);
+extern void lto_end_uncompression (struct lto_compression_stream *stream,
+ lto_compression compression);
#endif /* GCC_LTO_COMPRESS_H */
stream = lto_start_uncompression (lto_append_data, &buffer);
lto_uncompress_block (stream, data, *len);
- lto_end_uncompression (stream);
+ lto_end_uncompression (stream, file_data->lto_section_header.compression);
*len = buffer.length - header_length;
data = buffer.data + header_length;
lto_begin_section (section_name, false);
free (section_name);
+#ifdef HAVE_ZSTD_H
+ lto_compression compression = ZSTD;
+#else
lto_compression compression = ZLIB;
+#endif
bool slim_object = flag_generate_lto && !flag_fat_lto_objects;
lto_section s
DEFTIMEVAR (TV_IPA_FNSPLIT , "ipa function splitting")
DEFTIMEVAR (TV_IPA_COMDATS , "ipa comdats")
DEFTIMEVAR (TV_IPA_OPT , "ipa various optimizations")
-DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream inflate")
-DEFTIMEVAR (TV_IPA_LTO_COMPRESS , "lto stream deflate")
+DEFTIMEVAR (TV_IPA_LTO_DECOMPRESS , "lto stream decompression")
+DEFTIMEVAR (TV_IPA_LTO_COMPRESS , "lto stream compression")
DEFTIMEVAR (TV_IPA_LTO_OUTPUT , "lto stream output")
DEFTIMEVAR (TV_IPA_LTO_GIMPLE_IN , "ipa lto gimple in")
DEFTIMEVAR (TV_IPA_LTO_GIMPLE_OUT , "ipa lto gimple out")
projects / gcc.git / commitdiff