rtl-optimization/98863 - fix PRE/CPROP memory usage check

This fixes overflow of the memory usage estimate in turn failing
to disable itself on WRF with LTO, causing a few GBs worth of
memory peak.

2021-01-29  Richard Biener  <rguenther@suse.de>

	PR rtl-optimization/98863
	* gcse.c (gcse_or_cprop_is_too_expensive): Use unsigned
	HOST_WIDE_INT for the memory estimate.

diff --git a/gcc/gcse.c b/gcc/gcse.c
index c4a6acb8aa141250b992b61c8cb4cc3442fb2732..29c9f900a8c3ed50a7f2a9a78b8124095dddd259 100644 (file)
--- a/gcc/gcse.c
+++ b/gcc/gcse.c
@@ -3982,9 +3982,9 @@ update_ld_motion_stores (struct gcse_expr * expr)
 bool
 gcse_or_cprop_is_too_expensive (const char *pass)
 {
-  int memory_request = (n_basic_blocks_for_fn (cfun)
-                       * SBITMAP_SET_SIZE (max_reg_num ())
-                       * sizeof (SBITMAP_ELT_TYPE));
+  unsigned HOST_WIDE_INT memory_request
+    = ((unsigned HOST_WIDE_INT)n_basic_blocks_for_fn (cfun)
+       * SBITMAP_SET_SIZE (max_reg_num ()) * sizeof (SBITMAP_ELT_TYPE));
   
   /* Trying to perform global optimizations on flow graphs which have
      a high connectivity will take a long time and is unlikely to be
@@ -4007,11 +4007,12 @@ gcse_or_cprop_is_too_expensive (const char *pass)
 
   /* If allocating memory for the dataflow bitmaps would take up too much
      storage it's better just to disable the optimization.  */
-  if (memory_request > param_max_gcse_memory)
+  if (memory_request > (unsigned HOST_WIDE_INT)param_max_gcse_memory)
     {
       warning (OPT_Wdisabled_optimization,
               "%s: %d basic blocks and %d registers; "
-              "increase %<--param max-gcse-memory%> above %d",
+              "increase %<--param max-gcse-memory%> above "
+              HOST_WIDE_INT_PRINT_UNSIGNED,
               pass, n_basic_blocks_for_fn (cfun), max_reg_num (),
               memory_request);