Reduce memory usage for storing LTO decl resolutions

With a LTO build of a large project (>11k subfiles incrementially linked)
storing the LTO resolutions took over 0.5GB memory:

lto/lto.c:1087 (lto_resolution_read)                      0: 0.0%  540398500           15903: 0.0%

The reason is that the declaration indexes are quite sparse, but every subfile
got a full continuous vector for them. Since there are so many of them the
many vectors add up.

This patch instead stores the resolutions initially in a compact (index, resolution)
format. This is only expanded into a sparse vector for fast lookup when
the subfile is actually read, but then immediately freed. This means only one
vector is allocated at a time.

This brings the overhead for this down to less than 3MB for the test case:

lto/lto.c:1087 (lto_resolution_read)                      0: 0.0%    2821456           42186: 0.0%

gcc/:

2012-09-06  Andi Kleen  <ak@linux.intel.com>

	* gcc/lto-streamer.h (res_pair): Add.
	(lto_file_decl_data): Replace resolutions with respairs.
	Add max_index.
	* gcc/lto/lto.c (lto_resolution_read): Remove max_index.  Add rp.
	Initialize respairs.
	(lto_file_finalize): Set up resolutions vector lazily from respairs.

From-SVN: r191051

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index c1639906f73a9b9b35ed7d6308f09354e0039f01..5aa6aa84a0890408d401b6660ba305a0a9fffbe9 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,12 @@
+2012-09-06  Andi Kleen  <ak@linux.intel.com>
+
+       * gcc/lto-streamer.h (res_pair): Add.
+       (lto_file_decl_data): Replace resolutions with respairs.
+       Add max_index.
+       * gcc/lto/lto.c (lto_resolution_read): Remove max_index.  Add rp.
+       Initialize respairs.
+       (lto_file_finalize): Set up resolutions vector lazily from respairs.
+
 2012-09-06  Lawrence Crowl  <crowl@google.com>
 
        * double-int.h (double_int::operator &=): New.

diff --git a/gcc/lto-streamer.h b/gcc/lto-streamer.h
index bed408aef0a92c4aa101a15d609f6c113f065b50..80fc19f861ad094b85bc09743ed380985f909649 100644 (file)
--- a/gcc/lto-streamer.h
+++ b/gcc/lto-streamer.h
@@ -513,6 +513,18 @@ typedef struct lto_out_decl_state *lto_out_decl_state_ptr;
 DEF_VEC_P(lto_out_decl_state_ptr);
 DEF_VEC_ALLOC_P(lto_out_decl_state_ptr, heap);
 
+/* Compact representation of a index <-> resolution pair. Unpacked to an 
+   vector later. */
+struct res_pair 
+{
+  ld_plugin_symbol_resolution_t res;
+  unsigned index;
+};
+typedef struct res_pair res_pair;
+
+DEF_VEC_O(res_pair);
+DEF_VEC_ALLOC_O(res_pair, heap);
+
 /* One of these is allocated for each object file that being compiled
    by lto.  This structure contains the tables that are needed by the
    serialized functions and ipa passes to connect themselves to the
@@ -548,7 +560,8 @@ struct GTY(()) lto_file_decl_data
   unsigned HOST_WIDE_INT id;
 
   /* Symbol resolutions for this file */
-  VEC(ld_plugin_symbol_resolution_t,heap) * GTY((skip)) resolutions;
+  VEC(res_pair, heap) * GTY((skip)) respairs;
+  unsigned max_index;
 
   struct gcov_ctr_summary GTY((skip)) profile_info;
 };

diff --git a/gcc/lto/lto.c b/gcc/lto/lto.c
index bd91c391fd142a12fecc41debdbce9c366a1d26b..5da54124f2db881f7bf51864942a22e799710397 100644 (file)
--- a/gcc/lto/lto.c
+++ b/gcc/lto/lto.c
@@ -1012,7 +1012,6 @@ lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
   unsigned int num_symbols;
   unsigned int i;
   struct lto_file_decl_data *file_data;
-  unsigned max_index = 0;
   splay_tree_node nd = NULL; 
 
   if (!resolution)
@@ -1054,13 +1053,12 @@ lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
       unsigned int j;
       unsigned int lto_resolution_str_len =
        sizeof (lto_resolution_str) / sizeof (char *);
+      res_pair rp;
 
       t = fscanf (resolution, "%u " HOST_WIDE_INT_PRINT_HEX_PURE " %26s %*[^\n]\n", 
                  &index, &id, r_str);
       if (t != 3)
         internal_error ("invalid line in the resolution file");
-      if (index > max_index)
-       max_index = index;
 
       for (j = 0; j < lto_resolution_str_len; j++)
        {
@@ -1082,11 +1080,13 @@ lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
        }
 
       file_data = (struct lto_file_decl_data *)nd->value;
-      VEC_safe_grow_cleared (ld_plugin_symbol_resolution_t, heap, 
-                            file_data->resolutions,
-                            max_index + 1);
-      VEC_replace (ld_plugin_symbol_resolution_t, 
-                  file_data->resolutions, index, r);
+      /* The indexes are very sparse. To save memory save them in a compact
+         format that is only unpacked later when the subfile is processed. */
+      rp.res = r;
+      rp.index = index;
+      VEC_safe_push (res_pair, heap, file_data->respairs, rp);
+      if (file_data->max_index < index)
+        file_data->max_index = index;
     }
 }
 
@@ -1166,6 +1166,18 @@ lto_file_finalize (struct lto_file_decl_data *file_data, lto_file *file)
 {
   const char *data;
   size_t len;
+  VEC(ld_plugin_symbol_resolution_t,heap) *resolutions = NULL;
+  int i;
+  res_pair *rp;
+
+  /* Create vector for fast access of resolution. We do this lazily
+     to save memory. */ 
+  VEC_safe_grow_cleared (ld_plugin_symbol_resolution_t, heap, 
+                            resolutions,
+                            file_data->max_index + 1);
+  for (i = 0; VEC_iterate (res_pair, file_data->respairs, i, rp); i++)
+    VEC_replace (ld_plugin_symbol_resolution_t, resolutions, rp->index, rp->res);
+  VEC_free (res_pair, heap, file_data->respairs);
 
   file_data->renaming_hash_table = lto_create_renaming_table ();
   file_data->file_name = file->filename;
@@ -1175,7 +1187,8 @@ lto_file_finalize (struct lto_file_decl_data *file_data, lto_file *file)
       internal_error ("cannot read LTO decls from %s", file_data->file_name);
       return;
     }
-  lto_read_decls (file_data, data, file_data->resolutions);
+  /* Frees resolutions */
+  lto_read_decls (file_data, data, resolutions);
   lto_free_section_data (file_data, LTO_section_decls, NULL, data, len);
 }