tree-vectorizer.h (_stmt_vec_info::v_reduc_type): Remove.
authorRichard Biener <rguenther@suse.de>
Tue, 8 Oct 2019 13:39:03 +0000 (13:39 +0000)
committerRichard Biener <rguenth@gcc.gnu.org>
Tue, 8 Oct 2019 13:39:03 +0000 (13:39 +0000)
2019-10-08  Richard Biener  <rguenther@suse.de>

* tree-vectorizer.h (_stmt_vec_info::v_reduc_type): Remove.
(_stmt_vec_info::is_reduc_info): Add.
(STMT_VINFO_VEC_REDUCTION_TYPE): Remove.
(vectorizable_condition): Remove.
(vectorizable_shift): Likewise.
(vectorizable_reduction): Adjust.
(info_for_reduction): New.
* tree-vect-loop.c (vect_force_simple_reduction): Fold into...
(vect_analyze_scalar_cycles_1): ... here.
(vect_analyze_loop_operations): Adjust.
(needs_fold_left_reduction_p): Simplify for single caller.
(vect_is_simple_reduction): Likewise.  Remove stmt restriction
for nested cycles not part of double reductions.
(vect_model_reduction_cost): Pass in the reduction type.
(info_for_reduction): New function.
(vect_create_epilog_for_reduction): Use it, access reduction
meta off the stmt info it returns.  Use STMT_VINFO_REDUC_TYPE
instead of STMT_VINFO_VEC_REDUCTION_TYPE.
(vectorize_fold_left_reduction): Remove pointless assert.
(vectorizable_reduction): Analyze the full reduction when
visiting the outermost PHI.  Simplify.  Use STMT_VINFO_REDUC_TYPE
instead of STMT_VINFO_VEC_REDUCTION_TYPE.  Direct reduction
stmt code-generation to vectorizable_* in most cases.  Verify
code-generation only for cases handled by
vect_transform_reductuon.
(vect_transform_reduction): Use info_for_reduction to get at
reduction meta.  Simplify.
(vect_transform_cycle_phi): Likewise.
(vectorizable_live_operation): Likewise.
* tree-vect-patterns.c (vect_reassociating_reduction_p): Look
at the PHI node for STMT_VINFO_REDUC_TYPE.
* tree-vect-slp.c (vect_schedule_slp_instance): Remove no
longer necessary code.
* tree-vect-stmts.c (vectorizable_shift): Make static again.
(vectorizable_condition): Likewise.  Get at reduction related
info via info_for_reduction.
(vect_analyze_stmt): Adjust.
(vect_transform_stmt): Likewise.
* tree-vectorizer.c (vec_info::new_stmt_vec_info): Initialize
STMT_VINFO_REDUC_TYPE instead of STMT_VINFO_VEC_REDUCTION_TYPE.

* gcc.dg/vect/pr65947-1.c: Adjust.
* gcc.dg/vect/pr65947-13.c: Likewise.
* gcc.dg/vect/pr65947-14.c: Likewise.
* gcc.dg/vect/pr65947-4.c: Likewise.
* gcc.dg/vect/pr80631-1.c: Likewise.
* gcc.dg/vect/pr80631-2.c: Likewise.

From-SVN: r276700

14 files changed:
gcc/ChangeLog
gcc/testsuite/ChangeLog
gcc/testsuite/gcc.dg/vect/pr65947-1.c
gcc/testsuite/gcc.dg/vect/pr65947-13.c
gcc/testsuite/gcc.dg/vect/pr65947-14.c
gcc/testsuite/gcc.dg/vect/pr65947-4.c
gcc/testsuite/gcc.dg/vect/pr80631-1.c
gcc/testsuite/gcc.dg/vect/pr80631-2.c
gcc/tree-vect-loop.c
gcc/tree-vect-patterns.c
gcc/tree-vect-slp.c
gcc/tree-vect-stmts.c
gcc/tree-vectorizer.c
gcc/tree-vectorizer.h

index 23e0a259e0c60524ce37e4153790c6b21b5e4c00..984a05d07565300ddaf610d67ac39a205c91d123 100644 (file)
@@ -1,3 +1,46 @@
+2019-10-08  Richard Biener  <rguenther@suse.de>
+
+       * tree-vectorizer.h (_stmt_vec_info::v_reduc_type): Remove.
+       (_stmt_vec_info::is_reduc_info): Add.
+       (STMT_VINFO_VEC_REDUCTION_TYPE): Remove.
+       (vectorizable_condition): Remove.
+       (vectorizable_shift): Likewise.
+       (vectorizable_reduction): Adjust.
+       (info_for_reduction): New.
+       * tree-vect-loop.c (vect_force_simple_reduction): Fold into...
+       (vect_analyze_scalar_cycles_1): ... here.
+       (vect_analyze_loop_operations): Adjust.
+       (needs_fold_left_reduction_p): Simplify for single caller.
+       (vect_is_simple_reduction): Likewise.  Remove stmt restriction
+       for nested cycles not part of double reductions.
+       (vect_model_reduction_cost): Pass in the reduction type.
+       (info_for_reduction): New function.
+       (vect_create_epilog_for_reduction): Use it, access reduction
+       meta off the stmt info it returns.  Use STMT_VINFO_REDUC_TYPE
+       instead of STMT_VINFO_VEC_REDUCTION_TYPE.
+       (vectorize_fold_left_reduction): Remove pointless assert.
+       (vectorizable_reduction): Analyze the full reduction when
+       visiting the outermost PHI.  Simplify.  Use STMT_VINFO_REDUC_TYPE
+       instead of STMT_VINFO_VEC_REDUCTION_TYPE.  Direct reduction
+       stmt code-generation to vectorizable_* in most cases.  Verify
+       code-generation only for cases handled by
+       vect_transform_reductuon.
+       (vect_transform_reduction): Use info_for_reduction to get at
+       reduction meta.  Simplify.
+       (vect_transform_cycle_phi): Likewise.
+       (vectorizable_live_operation): Likewise.
+       * tree-vect-patterns.c (vect_reassociating_reduction_p): Look
+       at the PHI node for STMT_VINFO_REDUC_TYPE.
+       * tree-vect-slp.c (vect_schedule_slp_instance): Remove no
+       longer necessary code.
+       * tree-vect-stmts.c (vectorizable_shift): Make static again.
+       (vectorizable_condition): Likewise.  Get at reduction related
+       info via info_for_reduction.
+       (vect_analyze_stmt): Adjust.
+       (vect_transform_stmt): Likewise.
+       * tree-vectorizer.c (vec_info::new_stmt_vec_info): Initialize
+       STMT_VINFO_REDUC_TYPE instead of STMT_VINFO_VEC_REDUCTION_TYPE.
+
 2019-10-08  Joseph Myers  <joseph@codesourcery.com>
 
        * doc/invoke.texi (-ffp-int-builtin-inexact): Document
index f4819f23d544576f18b6b81a14862b0101ca7be5..fb2264fa2ba220c1b179842532c0b537ac68a79b 100644 (file)
@@ -1,3 +1,12 @@
+2019-10-08  Richard Biener  <rguenther@suse.de>
+
+       * gcc.dg/vect/pr65947-1.c: Adjust.
+       * gcc.dg/vect/pr65947-13.c: Likewise.
+       * gcc.dg/vect/pr65947-14.c: Likewise.
+       * gcc.dg/vect/pr65947-4.c: Likewise.
+       * gcc.dg/vect/pr80631-1.c: Likewise.
+       * gcc.dg/vect/pr80631-2.c: Likewise.
+
 2019-10-08  Tobias Burnus  <tobias@codesourcery.com>
 
        * gfortran.dg/goacc/continuation-free-form.f95: Update dg-error.
index 879819d576ac738a6b18cbc2ebe3f5be8c1c2d29..b81baed914c8b606ffd60dfad863d79fb67eb8e7 100644 (file)
@@ -42,4 +42,4 @@ main (void)
 
 /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" } } */
 /* { dg-final { scan-tree-dump-times "optimizing condition reduction with FOLD_EXTRACT_LAST" 4 "vect" { target vect_fold_extract_last } } } */
-/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 4 "vect" { target { ! vect_fold_extract_last } } } } */
+/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 2 "vect" { target { ! vect_fold_extract_last } } } } */
index e1d3ff52f5c8b3d2230598e5c9d859bf246d6c0c..4ad5262019a6bb50608f5569b32af8af1602a9d8 100644 (file)
@@ -41,5 +41,5 @@ main (void)
 }
 
 /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" } } */
-/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 4 "vect" { xfail vect_fold_extract_last } } } */
+/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 2 "vect" { xfail vect_fold_extract_last } } } */
 /* { dg-final { scan-tree-dump-times "optimizing condition reduction with FOLD_EXTRACT_LAST" 4 "vect" { target vect_fold_extract_last } } } */
index 9f1e4e1eb6a6ffe731a12de08899f8d1230b7d18..d0194f237d4dc28079711366b5433e6cd975ad81 100644 (file)
@@ -42,4 +42,4 @@ main (void)
 
 /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" } } */
 /* { dg-final { scan-tree-dump-times "optimizing condition reduction with FOLD_EXTRACT_LAST" 4 "vect" { target vect_fold_extract_last } } } */
-/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 4 "vect" { target { ! vect_fold_extract_last } } } } */
+/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 2 "vect" { target { ! vect_fold_extract_last } } } } */
index 186e03a63465da695d25db76b17f80862b7768b3..405571047158143e8b01e320961602d1e088bf1a 100644 (file)
@@ -42,5 +42,5 @@ main (void)
 
 /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 2 "vect" } } */
 /* { dg-final { scan-tree-dump-times "optimizing condition reduction with FOLD_EXTRACT_LAST" 4 "vect" { target vect_fold_extract_last } } } */
-/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 4 "vect" { target { ! vect_fold_extract_last } } } } */
+/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 2 "vect" { target { ! vect_fold_extract_last } } } } */
 
index f2405198a10b2deef85e8d21cecc695f735142e3..b531fe6dbf9420baa973ea178a7cd540f2706e80 100644 (file)
@@ -73,4 +73,4 @@ main ()
 
 /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 5 "vect" { target vect_condition } } } */
 /* { dg-final { scan-tree-dump-times "optimizing condition reduction with FOLD_EXTRACT_LAST" 10 "vect" { target vect_fold_extract_last } } } */
-/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 10 "vect" { target { { ! vect_fold_extract_last } && vect_condition } } } } */
+/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 5 "vect" { target { { ! vect_fold_extract_last } && vect_condition } } } } */
index b334ca2345bdf20ab7ea1864e9c1d372f97e6944..07f1a721e477a15caac29d80257dde086f82d9aa 100644 (file)
@@ -72,5 +72,5 @@ main ()
 }
 
 /* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 5 "vect" { target vect_condition } } } */
-/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 10 "vect" { target vect_condition xfail vect_fold_extract_last } } } */
+/* { dg-final { scan-tree-dump-times "condition expression based on integer induction." 5 "vect" { target vect_condition xfail vect_fold_extract_last } } } */
 /* { dg-final { scan-tree-dump-times "optimizing condition reduction with FOLD_EXTRACT_LAST" 10 "vect" { target vect_fold_extract_last } } } */
index 3db4a5cdf78ae95ce711d164fcfcfa738b9a1bd7..f63bb8556181ddd518fb32a6ae32da53a67adfe3 100644 (file)
@@ -154,8 +154,8 @@ along with GCC; see the file COPYING3.  If not see
 */
 
 static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *);
-static stmt_vec_info vect_force_simple_reduction (loop_vec_info, stmt_vec_info,
-                                                 bool *, bool);
+static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
+                                              bool *);
 
 /* Subroutine of vect_determine_vf_for_stmt that handles only one
    statement.  VECTYPE_MAYBE_SET_P is true if STMT_VINFO_VECTYPE
@@ -561,19 +561,19 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop)
                  && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
 
       stmt_vec_info reduc_stmt_info
-       = vect_force_simple_reduction (loop_vinfo, stmt_vinfo,
-                                      &double_reduc, false);
+       = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc);
       if (reduc_stmt_info)
         {
-          if (double_reduc)
-            {
-              if (dump_enabled_p ())
-                dump_printf_loc (MSG_NOTE, vect_location,
+         STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info;
+         STMT_VINFO_REDUC_DEF (reduc_stmt_info) = stmt_vinfo;
+         if (double_reduc)
+           {
+             if (dump_enabled_p ())
+               dump_printf_loc (MSG_NOTE, vect_location,
                                 "Detected double reduction.\n");
 
               STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
-             STMT_VINFO_DEF_TYPE (reduc_stmt_info)
-               = vect_double_reduction_def;
+             STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_double_reduction_def;
             }
           else
             {
@@ -1558,8 +1558,7 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
                            == vect_double_reduction_def)
                        || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
                       && ! PURE_SLP_STMT (stmt_info))
-               ok = vectorizable_reduction (stmt_info, NULL, NULL, NULL, NULL,
-                                            &cost_vec);
+               ok = vectorizable_reduction (stmt_info, NULL, NULL, &cost_vec);
             }
 
          /* SLP PHIs are tested by vect_slp_analyze_node_operations.  */
@@ -2554,8 +2553,7 @@ vect_valid_reduction_input_p (stmt_vec_info def_stmt_info)
    overflow must wrap.  */
 
 static bool
-needs_fold_left_reduction_p (tree type, tree_code code,
-                            bool need_wrapping_integral_overflow)
+needs_fold_left_reduction_p (tree type, tree_code code)
 {
   /* CHECKME: check for !flag_finite_math_only too?  */
   if (SCALAR_FLOAT_TYPE_P (type))
@@ -2573,10 +2571,6 @@ needs_fold_left_reduction_p (tree type, tree_code code,
     {
       if (!operation_no_trapping_overflow (type, code))
        return true;
-      if (need_wrapping_integral_overflow
-         && !TYPE_OVERFLOW_WRAPS (type)
-         && operation_can_overflow (code))
-       return true;
       return false;
     }
 
@@ -2752,9 +2746,7 @@ check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi,
 
 static stmt_vec_info
 vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
-                         bool *double_reduc,
-                         bool need_wrapping_integral_overflow,
-                         enum vect_reduction_type *v_reduc_type)
+                         bool *double_reduc)
 {
   gphi *phi = as_a <gphi *> (phi_info->stmt);
   class loop *loop = (gimple_bb (phi))->loop_father;
@@ -2770,7 +2762,7 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
   bool phi_def;
 
   *double_reduc = false;
-  *v_reduc_type = TREE_CODE_REDUCTION;
+  STMT_VINFO_REDUC_TYPE (phi_info) = TREE_CODE_REDUCTION;
 
   tree phi_name = PHI_RESULT (phi);
   /* ???  If there are no uses of the PHI result the inner loop reduction
@@ -2928,22 +2920,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
 
   if (nested_in_vect_loop && !check_reduction)
     {
-      /* FIXME: Even for non-reductions code generation is funneled
-        through vectorizable_reduction for the stmt defining the
-        PHI latch value.  So we have to artificially restrict ourselves
-        for the supported operations.  */
-      switch (get_gimple_rhs_class (code))
-       {
-       case GIMPLE_BINARY_RHS:
-       case GIMPLE_TERNARY_RHS:
-         break;
-       default:
-         /* Not supported by vectorizable_reduction.  */
-         if (dump_enabled_p ())
-           report_vect_op (MSG_MISSED_OPTIMIZATION, def_stmt,
-                           "nested cycle: not handled operation: ");
-         return NULL;
-       }
       if (dump_enabled_p ())
        report_vect_op (MSG_NOTE, def_stmt, "detected nested cycle: ");
       return def_stmt_info;
@@ -2959,7 +2935,7 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
   if (code == COND_EXPR)
     {
       if (! nested_in_vect_loop)
-       *v_reduc_type = COND_REDUCTION;
+       STMT_VINFO_REDUC_TYPE (phi_info) = COND_REDUCTION;
 
       op3 = gimple_assign_rhs1 (def_stmt);
       if (COMPARISON_CLASS_P (op3))
@@ -3043,10 +3019,9 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
      and therefore vectorizing reductions in the inner-loop during
      outer-loop vectorization is safe.  */
   if (check_reduction
-      && *v_reduc_type == TREE_CODE_REDUCTION
-      && needs_fold_left_reduction_p (type, code,
-                                     need_wrapping_integral_overflow))
-    *v_reduc_type = FOLD_LEFT_REDUCTION;
+      && STMT_VINFO_REDUC_TYPE (phi_info) == TREE_CODE_REDUCTION
+      && needs_fold_left_reduction_p (type, code))
+    STMT_VINFO_REDUC_TYPE (phi_info) = FOLD_LEFT_REDUCTION;
 
   /* Reduction is safe. We're dealing with one of the following:
      1) integer arithmetic and no trapv
@@ -3142,30 +3117,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
   return NULL;
 }
 
-/* Wrapper around vect_is_simple_reduction, which will modify code
-   in-place if it enables detection of more reductions.  Arguments
-   as there.  */
-
-static stmt_vec_info
-vect_force_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
-                            bool *double_reduc,
-                            bool need_wrapping_integral_overflow)
-{
-  enum vect_reduction_type v_reduc_type;
-  stmt_vec_info def_info
-    = vect_is_simple_reduction (loop_info, phi_info, double_reduc,
-                               need_wrapping_integral_overflow,
-                               &v_reduc_type);
-  if (def_info)
-    {
-      STMT_VINFO_REDUC_TYPE (phi_info) = v_reduc_type;
-      STMT_VINFO_REDUC_DEF (phi_info) = def_info;
-      STMT_VINFO_REDUC_TYPE (def_info) = v_reduc_type;
-      STMT_VINFO_REDUC_DEF (def_info) = phi_info;
-    }
-  return def_info;
-}
-
 /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times.  */
 int
 vect_get_known_peeling_cost (loop_vec_info loop_vinfo, int peel_iters_prologue,
@@ -3754,6 +3705,7 @@ have_whole_vector_shift (machine_mode mode)
 
 static void
 vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn,
+                          vect_reduction_type reduction_type,
                           int ncopies, stmt_vector_for_cost *cost_vec)
 {
   int prologue_cost = 0, epilogue_cost = 0, inside_cost;
@@ -3768,8 +3720,6 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn,
     loop = LOOP_VINFO_LOOP (loop_vinfo);
 
   /* Condition reductions generate two reductions in the loop.  */
-  vect_reduction_type reduction_type
-    = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info);
   if (reduction_type == COND_REDUCTION)
     ncopies *= 2;
 
@@ -4208,6 +4158,32 @@ get_initial_defs_for_reduction (slp_tree slp_node,
     gsi_insert_seq_on_edge_immediate (pe, ctor_seq);
 }
 
+/* For a statement STMT_INFO taking part in a reduction operation return
+   the stmt_vec_info the meta information is stored on.  */
+
+stmt_vec_info
+info_for_reduction (stmt_vec_info stmt_info)
+{
+  stmt_info = vect_orig_stmt (stmt_info);
+  gcc_assert (STMT_VINFO_REDUC_DEF (stmt_info));
+  if (!is_a <gphi *> (stmt_info->stmt))
+    stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
+  gphi *phi = as_a <gphi *> (stmt_info->stmt);
+  if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
+    {
+      if (gimple_phi_num_args (phi) == 1)
+       stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
+    }
+  else if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
+    {
+      edge pe = loop_preheader_edge (gimple_bb (phi)->loop_father);
+      stmt_vec_info info
+         = stmt_info->vinfo->lookup_def (PHI_ARG_DEF_FROM_EDGE (phi, pe));
+      if (info && STMT_VINFO_DEF_TYPE (info) == vect_double_reduction_def)
+       stmt_info = info;
+    }
+  return stmt_info;
+}
 
 /* Function vect_create_epilog_for_reduction
 
@@ -4261,6 +4237,8 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
                                  slp_tree slp_node,
                                  slp_instance slp_node_instance)
 {
+  stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+  gcc_assert (reduc_info->is_reduc_info);
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
   /* For double reductions we need to get at the inner loop reduction
      stmt which has the meta info attached.  Our stmt_info is that of the
@@ -4278,8 +4256,8 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
     }
   gphi *reduc_def_stmt
     = as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))->stmt);
-  enum tree_code code = STMT_VINFO_REDUC_CODE (stmt_info);
-  internal_fn reduc_fn = STMT_VINFO_REDUC_FN (stmt_info);
+  enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
+  internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
   tree neutral_op = NULL_TREE;
   if (slp_node)
     neutral_op
@@ -4345,15 +4323,14 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
                                           loop_preheader_edge (loop));
       /* Optimize: for induction condition reduction, if we can't use zero
          for induc_val, use initial_def.  */
-      if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-         == INTEGER_INDUC_COND_REDUCTION)
-       induc_val = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info);
+      if (STMT_VINFO_REDUC_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
+       induc_val = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info);
       else if (double_reduc)
        ;
       else if (nested_in_vect_loop)
        ;
       else
-       adjustment_def = STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (stmt_info);
+       adjustment_def = STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info);
     }
 
   unsigned vec_num;
@@ -4383,7 +4360,7 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
      The first match will be a 1 to allow 0 to be used for non-matching
      indexes.  If there are no matches at all then the vector will be all
      zeroes.  */
-  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
+  if (STMT_VINFO_REDUC_TYPE (reduc_info) == COND_REDUCTION)
     {
       tree indx_before_incr, indx_after_incr;
       poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
@@ -4616,7 +4593,7 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
   else
     new_phi_result = PHI_RESULT (new_phis[0]);
 
-  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION
+  if (STMT_VINFO_REDUC_TYPE (reduc_info) == COND_REDUCTION
       && reduc_fn != IFN_LAST)
     {
       /* For condition reductions, we have a vector (NEW_PHI_RESULT) containing
@@ -4725,7 +4702,7 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
       gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT);
       scalar_results.safe_push (new_temp);
     }
-  else if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION
+  else if (STMT_VINFO_REDUC_TYPE (reduc_info) == COND_REDUCTION
           && reduc_fn == IFN_LAST)
     {
       /* Condition reduction without supported IFN_REDUC_MAX.  Generate
@@ -4838,8 +4815,7 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
       gimple_set_lhs (epilog_stmt, new_temp);
       gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
 
-      if ((STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-          == INTEGER_INDUC_COND_REDUCTION)
+      if ((STMT_VINFO_REDUC_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
          && induc_val)
        {
          /* Earlier we set the initial value to be a vector if induc_val
@@ -5201,8 +5177,7 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
             scalar_results.safe_push (new_temp);
         }
 
-      if ((STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-          == INTEGER_INDUC_COND_REDUCTION)
+      if ((STMT_VINFO_REDUC_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
          && induc_val)
        {
          /* Earlier we set the initial value to be a vector if induc_val
@@ -5486,8 +5461,6 @@ vectorize_fold_left_reduction (stmt_vec_info stmt_info,
   gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
   gcc_assert (ncopies == 1);
   gcc_assert (TREE_CODE_LENGTH (code) == binary_op);
-  gcc_assert (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-             == FOLD_LEFT_REDUCTION);
 
   if (slp_node)
     gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out),
@@ -5762,8 +5735,7 @@ build_vect_cond_expr (enum tree_code code, tree vop[3], tree mask,
    does *NOT* necessarily hold for reduction patterns.  */
 
 bool
-vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-                       stmt_vec_info *vec_stmt, slp_tree slp_node,
+vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
                        slp_instance slp_node_instance,
                        stmt_vector_for_cost *cost_vec)
 {
@@ -5774,9 +5746,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   enum tree_code code;
   internal_fn reduc_fn;
-  machine_mode vec_mode;
   int op_type;
-  optab optab;
   enum vect_def_type dt, cond_reduc_dt = vect_unknown_def_type;
   stmt_vec_info cond_stmt_vinfo = NULL;
   tree scalar_type;
@@ -5799,6 +5769,69 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
     return false;
 
+  /* The stmt we store reduction analysis meta on.  */
+  stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+  reduc_info->is_reduc_info = true;
+
+  if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
+    {
+      if (is_a <gphi *> (stmt_info->stmt))
+       {
+         /* Analysis for double-reduction is done on the outer
+            loop PHI, nested cycles have no further restrictions.  */
+         STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
+         /* For nested cycles we want to let regular vectorizable_*
+            routines handle code-generation.  */
+         if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_double_reduction_def)
+           {
+             stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
+             STMT_VINFO_DEF_TYPE (stmt_info) = vect_internal_def;
+             STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize (stmt_info))
+               = vect_internal_def;
+           }
+       }
+      else
+       STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+      return true;
+    }
+
+  stmt_vec_info orig_stmt_of_analysis = stmt_info;
+  if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
+      || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
+    {
+      if (!is_a <gphi *> (stmt_info->stmt))
+       {
+         STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+         return true;
+       }
+      if (slp_node)
+       {
+         slp_node_instance->reduc_phis = slp_node;
+         /* ???  We're leaving slp_node to point to the PHIs, we only
+            need it to get at the number of vector stmts which wasn't
+            yet initialized for the instance root.  */
+       }
+      if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+       stmt_info = vect_stmt_to_vectorize (STMT_VINFO_REDUC_DEF (stmt_info));
+      else /* STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def */
+       {
+         use_operand_p use_p;
+         gimple *use_stmt;
+         bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
+                                    &use_p, &use_stmt);
+         gcc_assert (res);
+         stmt_info = loop_vinfo->lookup_stmt (use_stmt);
+         stmt_info = vect_stmt_to_vectorize (STMT_VINFO_REDUC_DEF (stmt_info));
+       }
+      /* STMT_VINFO_REDUC_DEF doesn't point to the first but the last
+         element.  */
+      if (slp_node && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
+       {
+         gcc_assert (!REDUC_GROUP_NEXT_ELEMENT (stmt_info));
+         stmt_info = REDUC_GROUP_FIRST_ELEMENT (stmt_info);
+       }
+    }
+
   if (nested_in_vect_loop_p (loop, stmt_info))
     {
       loop = loop->inner;
@@ -5809,18 +5842,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     gcc_assert (slp_node
                && REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info);
 
-  if (is_a <gphi *> (stmt_info->stmt))
-    {
-      /* Analysis is fully done on the reduction stmt invocation.  */
-      gcc_assert (! vec_stmt);
-
-      if (slp_node)
-       slp_node_instance->reduc_phis = slp_node;
-
-      STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
-      return true;
-    }
-
   /* 1. Is vectorizable reduction?  */
   /* Not supportable if the reduction variable is used in the loop, unless
      it's a reduction chain.  */
@@ -5898,13 +5919,10 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
      The last use is the reduction variable.  In case of nested cycle this
      assumption is not true: we use reduc_index to record the index of the
      reduction variable.  */
-  stmt_vec_info reduc_def_info;
-  if (orig_stmt_info)
-    reduc_def_info = STMT_VINFO_REDUC_DEF (orig_stmt_info);
-  else
-    reduc_def_info = STMT_VINFO_REDUC_DEF (stmt_info);
-  gcc_assert (reduc_def_info);
-  gphi *reduc_def_phi = as_a <gphi *> (reduc_def_info->stmt);
+  stmt_vec_info phi_info = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info));
+  /* PHIs should not participate in patterns.  */
+  gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
+  gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
   tree reduc_def = PHI_RESULT (reduc_def_phi);
   int reduc_index = -1;
   for (i = 0; i < op_type; i++)
@@ -5973,7 +5991,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
      directy used in stmt.  */
   if (reduc_index == -1)
     {
-      if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION)
+      if (STMT_VINFO_REDUC_TYPE (phi_info) == FOLD_LEFT_REDUCTION)
        {
          if (dump_enabled_p ())
            dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -5998,17 +6016,10 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       return false;
     }
 
-  /* PHIs should not participate in patterns.  */
-  gcc_assert (!STMT_VINFO_RELATED_STMT (reduc_def_info));
-  enum vect_reduction_type v_reduc_type
-    = STMT_VINFO_REDUC_TYPE (reduc_def_info);
-  stmt_vec_info tmp = STMT_VINFO_REDUC_DEF (reduc_def_info);
-
-  if (!vec_stmt)
-    STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = v_reduc_type;
+  enum vect_reduction_type v_reduc_type = STMT_VINFO_REDUC_TYPE (phi_info);
+  STMT_VINFO_REDUC_TYPE (reduc_info) = v_reduc_type;
   /* If we have a condition reduction, see if we can simplify it further.  */
-  if (v_reduc_type == COND_REDUCTION
-      && !vec_stmt)
+  if (v_reduc_type == COND_REDUCTION)
     {
       /* TODO: We can't yet handle reduction chains, since we need to treat
         each COND_EXPR in the chain specially, not just the last one.
@@ -6037,7 +6048,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
            dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
                             "optimizing condition reduction with"
                             " FOLD_EXTRACT_LAST.\n");
-         STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = EXTRACT_LAST_REDUCTION;
+         STMT_VINFO_REDUC_TYPE (reduc_info) = EXTRACT_LAST_REDUCTION;
        }
       else if (cond_reduc_dt == vect_induction_def)
        {
@@ -6077,12 +6088,14 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
            }
          if (cond_reduc_val)
            {
-             STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info)
-               = cond_reduc_op_code;
-             STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info)
+             if (dump_enabled_p ())
+               dump_printf_loc (MSG_NOTE, vect_location,
+                                "condition expression based on "
+                                "integer induction.\n");
+             STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info) = cond_reduc_op_code;
+             STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info)
                = cond_reduc_val;
-             STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-               = INTEGER_INDUC_COND_REDUCTION;
+             STMT_VINFO_REDUC_TYPE (reduc_info) = INTEGER_INDUC_COND_REDUCTION;
            }
        }
       else if (cond_reduc_dt == vect_constant_def)
@@ -6107,30 +6120,21 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
                                     "condition expression based on "
                                     "compile time constant.\n");
                  /* Record reduction code at analysis stage.  */
-                 STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info)
+                 STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info)
                    = integer_onep (e) ? MAX_EXPR : MIN_EXPR;
-                 STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-                   = CONST_COND_REDUCTION;
+                 STMT_VINFO_REDUC_TYPE (reduc_info) = CONST_COND_REDUCTION;
                }
            }
        }
     }
-  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == INTEGER_INDUC_COND_REDUCTION
-      && dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-                    "condition expression based on "
-                    "integer induction.\n");
 
-  if (orig_stmt_info)
-    gcc_assert (tmp == orig_stmt_info
-               || REDUC_GROUP_FIRST_ELEMENT (tmp) == orig_stmt_info);
-  else
+  if (REDUC_GROUP_FIRST_ELEMENT (stmt_info))
     /* We changed STMT to be the first stmt in reduction chain, hence we
        check that in this case the first element in the chain is STMT.  */
-    gcc_assert (tmp == stmt_info
-               || REDUC_GROUP_FIRST_ELEMENT (tmp) == stmt_info);
+    gcc_assert (REDUC_GROUP_FIRST_ELEMENT (STMT_VINFO_REDUC_DEF (phi_info))
+               == vect_orig_stmt (stmt_info));
 
-  if (STMT_VINFO_LIVE_P (reduc_def_info))
+  if (STMT_VINFO_LIVE_P (phi_info))
     return false;
 
   if (slp_node)
@@ -6140,25 +6144,16 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 
   gcc_assert (ncopies >= 1);
 
-  vec_mode = TYPE_MODE (vectype_in);
   poly_uint64 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
 
   if (nested_cycle)
     {
-      basic_block def_bb = gimple_bb (reduc_def_phi);
-      class loop *def_stmt_loop = def_bb->loop_father;
-      tree def_arg = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi,
-                                           loop_preheader_edge (def_stmt_loop));
-      stmt_vec_info def_arg_stmt_info = loop_vinfo->lookup_def (def_arg);
-      if (def_arg_stmt_info
-         && (STMT_VINFO_DEF_TYPE (def_arg_stmt_info)
-             == vect_double_reduction_def))
-        double_reduc = true;
-      gcc_assert (!double_reduc || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer_by_reduction);
+      gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info)
+                 == vect_double_reduction_def);
+      double_reduc = true;
     }
 
-  vect_reduction_type reduction_type
-    = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info);
+  vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
   if ((double_reduc || reduction_type != TREE_CODE_REDUCTION)
       && ncopies > 1)
     {
@@ -6169,78 +6164,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       return false;
     }
 
-  if (code == COND_EXPR)
-    {
-      /* Only call during the analysis stage, otherwise we'll lose
-        STMT_VINFO_TYPE.  */
-      gcc_assert (nested_cycle || reduc_index > 0);
-      if (!vec_stmt && !vectorizable_condition (stmt_info, gsi, NULL,
-                                               true, reduc_index,
-                                               NULL, cost_vec))
-        {
-          if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "unsupported condition in reduction\n");
-         return false;
-        }
-    }
-  else if (code == LSHIFT_EXPR || code == RSHIFT_EXPR
-          || code == LROTATE_EXPR || code == RROTATE_EXPR)
-    {
-      /* Only call during the analysis stage, otherwise we'll lose
-        STMT_VINFO_TYPE.  We only support this for nested cycles
-        without double reductions at the moment.  */
-      if (!nested_cycle
-         || double_reduc
-         || (!vec_stmt && !vectorizable_shift (stmt_info, gsi, NULL,
-                                               NULL, cost_vec)))
-       {
-          if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "unsupported shift or rotation in reduction\n");
-         return false;
-       }
-    }
-  else
-    {
-      /* 4. Supportable by target?  */
-
-      /* 4.1. check support for the operation in the loop  */
-      optab = optab_for_tree_code (code, vectype_in, optab_default);
-      if (!optab)
-        {
-          if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "no optab.\n");
-
-          return false;
-        }
-
-      if (optab_handler (optab, vec_mode) == CODE_FOR_nothing)
-        {
-          if (dump_enabled_p ())
-            dump_printf (MSG_NOTE, "op not supported by target.\n");
-
-         if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
-             || !vect_worthwhile_without_simd_p (loop_vinfo, code))
-            return false;
-
-          if (dump_enabled_p ())
-           dump_printf (MSG_NOTE, "proceeding using word mode.\n");
-        }
-
-      /* Worthwhile without SIMD support?  */
-      if (!VECTOR_MODE_P (TYPE_MODE (vectype_in))
-         && !vect_worthwhile_without_simd_p (loop_vinfo, code))
-        {
-          if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "not worthwhile without SIMD support.\n");
-
-          return false;
-        }
-    }
-
   /* 4.2. Check support for the epilog operation.
 
           If STMT represents a reduction pattern, then the type of the
@@ -6285,7 +6208,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
          reduction variable, and get the tree-code from orig_stmt.  */
       orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt);
       gcc_assert (vectype_out);
-      vec_mode = TYPE_MODE (vectype_out);
     }
   else
     {
@@ -6301,11 +6223,11 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       if (reduction_type == CONST_COND_REDUCTION
          || reduction_type == INTEGER_INDUC_COND_REDUCTION)
        {
-         orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (stmt_info);
+         orig_code = STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info);
          gcc_assert (orig_code == MAX_EXPR || orig_code == MIN_EXPR);
        }
     }
-  STMT_VINFO_REDUC_CODE (stmt_info) = orig_code;
+  STMT_VINFO_REDUC_CODE (reduc_info) = orig_code;
 
   reduc_fn = IFN_LAST;
 
@@ -6353,7 +6275,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
                                          OPTIMIZE_FOR_SPEED))
        reduc_fn = IFN_REDUC_MAX;
     }
-  STMT_VINFO_REDUC_FN (stmt_info) = reduc_fn;
+  STMT_VINFO_REDUC_FN (reduc_info) = reduc_fn;
 
   if (reduction_type != EXTRACT_LAST_REDUCTION
       && (!nested_cycle || double_reduc)
@@ -6436,7 +6358,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
         which each SLP statement has its own initial value and in which
         that value needs to be repeated for every instance of the
         statement within the initial vector.  */
-      unsigned int group_size = SLP_TREE_SCALAR_STMTS (slp_node).length ();
+      unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
       scalar_mode elt_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype_out));
       if (!neutral_op
          && !can_duplicate_and_interleave_p (group_size, elt_mode))
@@ -6467,22 +6389,8 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
      constant fits the type in the pattern recognition pass.  */
   if (code == DOT_PROD_EXPR
       && !types_compatible_p (TREE_TYPE (ops[0]), TREE_TYPE (ops[1])))
-    {
-      gcc_unreachable ();
-      /* No testcase for this.  PR49478.  */
-      if (TREE_CODE (ops[0]) == INTEGER_CST)
-        ops[0] = fold_convert (TREE_TYPE (ops[1]), ops[0]);
-      else if (TREE_CODE (ops[1]) == INTEGER_CST)
-        ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]);
-      else
-        {
-          if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "invalid types in dot-prod\n");
-
-          return false;
-        }
-    }
+    /* No testcase for this.  PR49478.  */
+    gcc_unreachable ();
 
   if (reduction_type == COND_REDUCTION)
     {
@@ -6551,6 +6459,52 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       && vect_stmt_to_vectorize (use_stmt_info) == stmt_info)
     single_defuse_cycle = true;
 
+  if (single_defuse_cycle
+      || code == DOT_PROD_EXPR
+      || code == WIDEN_SUM_EXPR
+      || code == SAD_EXPR)
+    {
+      gcc_assert (code != COND_EXPR);
+
+      /* 4. Supportable by target?  */
+
+      /* 4.1. check support for the operation in the loop  */
+      optab optab = optab_for_tree_code (code, vectype_in, optab_default);
+      if (!optab)
+        {
+          if (dump_enabled_p ())
+           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+                            "no optab.\n");
+
+          return false;
+        }
+
+      machine_mode vec_mode = TYPE_MODE (vectype_in);
+      if (optab_handler (optab, vec_mode) == CODE_FOR_nothing)
+        {
+          if (dump_enabled_p ())
+            dump_printf (MSG_NOTE, "op not supported by target.\n");
+
+         if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
+             || !vect_worthwhile_without_simd_p (loop_vinfo, code))
+            return false;
+
+          if (dump_enabled_p ())
+           dump_printf (MSG_NOTE, "proceeding using word mode.\n");
+        }
+
+      /* Worthwhile without SIMD support?  */
+      if (!VECTOR_MODE_P (TYPE_MODE (vectype_in))
+         && !vect_worthwhile_without_simd_p (loop_vinfo, code))
+        {
+          if (dump_enabled_p ())
+           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+                            "not worthwhile without SIMD support.\n");
+
+          return false;
+        }
+    }
+
   /* If the reduction stmt is one of the patterns that have lane
      reduction embedded we cannot handle the case of ! single_defuse_cycle.  */
   if ((ncopies > 1
@@ -6575,10 +6529,8 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
   bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in);
 
-  /* transformation not required.  */
-  gcc_assert (!vec_stmt);
-
-  vect_model_reduction_cost (stmt_info, reduc_fn, ncopies, cost_vec);
+  vect_model_reduction_cost (stmt_info, reduc_fn, reduction_type, ncopies,
+                            cost_vec);
   if (loop_vinfo && LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo))
     {
       if (reduction_type != FOLD_LEFT_REDUCTION
@@ -6609,7 +6561,18 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       && reduction_type == FOLD_LEFT_REDUCTION)
     dump_printf_loc (MSG_NOTE, vect_location,
                     "using an in-order (fold-left) reduction.\n");
-  STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+  STMT_VINFO_TYPE (orig_stmt_of_analysis) = cycle_phi_info_type;
+  /* All but single defuse-cycle optimized, lane-reducing and fold-left
+     reductions go through their own vectorizable_* routines.  */
+  if (!single_defuse_cycle
+      && code != DOT_PROD_EXPR
+      && code != WIDEN_SUM_EXPR
+      && code != SAD_EXPR
+      && reduction_type != FOLD_LEFT_REDUCTION)
+    {
+      STMT_VINFO_DEF_TYPE (stmt_info) = vect_internal_def;
+      STMT_VINFO_DEF_TYPE (vect_orig_stmt (stmt_info)) = vect_internal_def;
+    }
   return true;
 }
 
@@ -6632,13 +6595,15 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   bool single_defuse_cycle = false;
   int j;
   tree ops[3];
-  bool nested_cycle = false;
   int vec_num;
 
+  stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+  gcc_assert (reduc_info->is_reduc_info);
+
   if (nested_in_vect_loop_p (loop, stmt_info))
     {
       loop = loop->inner;
-      nested_cycle = true;
+      gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info) == vect_double_reduction_def);
     }
 
   gassign *stmt = as_a <gassign *> (stmt_info->stmt);
@@ -6674,10 +6639,8 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
      The last use is the reduction variable.  In case of nested cycle this
      assumption is not true: we use reduc_index to record the index of the
      reduction variable.  */
-  stmt_vec_info reduc_def_info
-    = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info));
-  gcc_assert (reduc_def_info);
-  gphi *reduc_def_phi = as_a <gphi *> (reduc_def_info->stmt);
+  stmt_vec_info phi_info = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info));
+  gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
   tree reduc_def = PHI_RESULT (reduc_def_phi);
   int reduc_index = -1;
   for (i = 0; i < op_type; i++)
@@ -6718,19 +6681,12 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   if (!vectype_in)
     vectype_in = vectype_out;
 
-  if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == INTEGER_INDUC_COND_REDUCTION
-      && dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-                    "condition expression based on "
-                    "integer induction.\n");
-
   if (slp_node)
     ncopies = 1;
   else
     ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
 
-  vect_reduction_type reduction_type
-    = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info);
+  vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
 
   /* In case of widenning multiplication by a constant, we update the type
      of the constant to be the type of the other operand.  We check that the
@@ -6814,18 +6770,16 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 
   if (reduction_type == FOLD_LEFT_REDUCTION)
     {
-      internal_fn reduc_fn = STMT_VINFO_REDUC_FN (stmt_info);
+      internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info);
       return vectorize_fold_left_reduction
          (stmt_info, gsi, vec_stmt, slp_node, reduc_def_phi, code,
           reduc_fn, ops, vectype_in, reduc_index, masks);
     }
 
-  if (reduction_type == EXTRACT_LAST_REDUCTION)
-    {
-      gcc_assert (!slp_node && reduc_index > 0);
-      return vectorizable_condition (stmt_info, gsi, vec_stmt,
-                                    true, reduc_index, NULL, NULL);
-    }
+  gcc_assert (single_defuse_cycle
+             || code == DOT_PROD_EXPR
+             || code == WIDEN_SUM_EXPR
+             || code == SAD_EXPR);
 
   /* Create the destination vector  */
   tree scalar_dest = gimple_assign_lhs (stmt);
@@ -6842,20 +6796,6 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 
   for (j = 0; j < ncopies; j++)
     {
-      if (code == COND_EXPR)
-        {
-          gcc_assert (!slp_node && (nested_cycle || reduc_index > 0));
-         vectorizable_condition (stmt_info, gsi, vec_stmt, true,
-                                 reduc_index, NULL, NULL);
-          break;
-        }
-      if (code == LSHIFT_EXPR
-         || code == RSHIFT_EXPR)
-       {
-         vectorizable_shift (stmt_info, gsi, vec_stmt, slp_node, NULL);
-         break;
-       }
-
       /* Handle uses.  */
       if (j == 0)
         {
@@ -7015,17 +6955,13 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
   gphi *phi = as_a <gphi *> (stmt_info->stmt);
   tree phi_result = gimple_phi_result (phi);
 
-  if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION)
-    /* Leave the scalar phi in place.  Note that checking
-       STMT_VINFO_VEC_REDUCTION_TYPE (as below) only works
-       for reductions involving a single statement.  */
-    return true;
-
   stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info);
   reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info);
+  stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+  gcc_assert (reduc_info->is_reduc_info);
 
-  if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
-      == EXTRACT_LAST_REDUCTION)
+  if (STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION
+      || STMT_VINFO_REDUC_TYPE (reduc_info) == FOLD_LEFT_REDUCTION)
     /* Leave the scalar phi in place.  */
     return true;
 
@@ -7091,8 +7027,7 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
       stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info);
       tree neutral_op
        = neutral_op_for_slp_reduction (slp_node,
-                                       STMT_VINFO_REDUC_CODE
-                                         (first ? first : reduc_stmt_info),
+                                       STMT_VINFO_REDUC_CODE (reduc_info),
                                        first != NULL);
       get_initial_defs_for_reduction (slp_node_instance->reduc_phis,
                                      &vec_initial_defs, vec_num,
@@ -7107,27 +7042,20 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
       /* Optimize: if initial_def is for REDUC_MAX smaller than the base
         and we can't use zero for induc_val, use initial_def.  Similarly
         for REDUC_MIN and initial_def larger than the base.  */
-      if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
-         == INTEGER_INDUC_COND_REDUCTION)
+      if (STMT_VINFO_REDUC_TYPE (reduc_info) == INTEGER_INDUC_COND_REDUCTION)
        {
-         tree induc_val
-           = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info);
+         tree induc_val = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info);
          if (TREE_CODE (initial_def) == INTEGER_CST
-             && (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info)
-                 == INTEGER_INDUC_COND_REDUCTION)
              && !integer_zerop (induc_val)
-             && (((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info)
-                   == MAX_EXPR)
+             && (((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info) == MAX_EXPR)
                   && tree_int_cst_lt (initial_def, induc_val))
-                 || ((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info)
-                      == MIN_EXPR)
+                 || ((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_info) == MIN_EXPR)
                      && tree_int_cst_lt (induc_val, initial_def))))
            {
              induc_val = initial_def;
              /* Communicate we used the initial_def to epilouge
                 generation.  */
-             STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info)
-               = NULL_TREE;
+             STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info) = NULL_TREE;
            }
          vec_initial_def = build_vector_from_val (vectype_out, induc_val);
        }
@@ -7142,24 +7070,13 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
        {
          tree adjustment_def = NULL_TREE;
          tree *adjustment_defp = &adjustment_def;
-         enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_stmt_info);
-         /* ???  For the outer loop PHI we have to do a bit of searching
-            to find the stmt with the code.  reduc_stmt_info here is the
-            loop-closed PHI of the inner reduction which means we can look
-            at its single-arg def.  */
+         enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_info);
          if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
-           {
-             tree def = gimple_phi_arg_def
-                          (as_a <gphi *> (reduc_stmt_info->stmt), 0);
-             code = STMT_VINFO_REDUC_CODE
-                      (vect_stmt_to_vectorize (loop_vinfo->lookup_def (def)));
-             adjustment_defp = NULL;
-           }
+           adjustment_defp = NULL;
          vec_initial_def
            = get_initial_def_for_reduction (reduc_stmt_info, code,
                                             initial_def, adjustment_defp);
-         STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_stmt_info)
-           = adjustment_def;
+         STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info) = adjustment_def;
        }
       vec_initial_defs.create (1);
       vec_initial_defs.quick_push (vec_initial_def);
@@ -7894,30 +7811,26 @@ vectorizable_live_operation (stmt_vec_info stmt_info,
   /* The last stmt of a reduction is live and vectorized via
      vect_create_epilog_for_reduction.  vectorizable_reduction assessed
      validity so just trigger the transform here.  */
-  if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
-      || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
+  if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)))
     {
       if (!vec_stmt)
        return true;
-      if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+      if (slp_node)
        {
-         if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION
-             || (STMT_VINFO_REDUC_TYPE (stmt_info) == COND_REDUCTION
-                 && (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
-                     == EXTRACT_LAST_REDUCTION)))
+         /* For reduction chains the meta-info is attached to
+            the group leader.  */
+         if (REDUC_GROUP_FIRST_ELEMENT (stmt_info))
+           stmt_info = REDUC_GROUP_FIRST_ELEMENT (stmt_info);
+         /* For SLP reductions we vectorize the epilogue for
+            all involved stmts together.  */
+         else if (slp_index != 0)
            return true;
-         if (slp_node)
-           {
-             /* For reduction chains the meta-info is attached to
-                the group leader.  */
-             if (REDUC_GROUP_FIRST_ELEMENT (stmt_info))
-               stmt_info = REDUC_GROUP_FIRST_ELEMENT (stmt_info);
-             /* For SLP reductions we vectorize the epilogue for
-                all involved stmts together.  */
-             else if (slp_index != 0)
-               return true;
-           }
        }
+      stmt_vec_info reduc_info = info_for_reduction (stmt_info);
+      gcc_assert (reduc_info->is_reduc_info);
+      if (STMT_VINFO_REDUC_TYPE (reduc_info) == FOLD_LEFT_REDUCTION
+         || STMT_VINFO_REDUC_TYPE (reduc_info) == EXTRACT_LAST_REDUCTION)
+       return true;
       vect_create_epilog_for_reduction (stmt_info, slp_node,
                                        slp_node_instance);
       return true;
index 09db74bdd77f14f05ca54ade64c9de01ea4fcf34..f9c74087893586086817e3dc53d76c91d205acf8 100644 (file)
@@ -838,9 +838,11 @@ vect_convert_output (stmt_vec_info stmt_info, tree type, gimple *pattern_stmt,
 static bool
 vect_reassociating_reduction_p (stmt_vec_info stmt_vinfo)
 {
-  return (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
-         ? STMT_VINFO_REDUC_TYPE (stmt_vinfo) != FOLD_LEFT_REDUCTION
-         : REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL);
+  if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
+    return (STMT_VINFO_REDUC_TYPE (STMT_VINFO_REDUC_DEF (stmt_vinfo))
+           != FOLD_LEFT_REDUCTION);
+  else
+    return REDUC_GROUP_FIRST_ELEMENT (stmt_vinfo) != NULL;
 }
 
 /* As above, but also require it to have code CODE and to be a reduction
index 9b86b67734ad3e3506e9cee6a532b68decf24ae6..77b18ba36d02dc7b5fc92ab5a56ecc40fbab390a 100644 (file)
@@ -3957,17 +3957,6 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance,
   stmt_vec_info last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
   si = gsi_for_stmt (last_stmt_info->stmt);
 
-  /* Mark the first element of the reduction chain as reduction to properly
-     transform the node.  In the analysis phase only the last element of the
-     chain is marked as reduction.  */
-  if (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
-      && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
-      && REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info)
-    {
-      STMT_VINFO_DEF_TYPE (stmt_info) = vect_reduction_def;
-      STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
-    }
-
   /* Handle two-operation SLP nodes by vectorizing the group with
      both operations and then performing a merge.  */
   if (SLP_TREE_TWO_OPERATORS (node))
index cac7410387b90715494d91c41b966d263d2e3fb5..9ebd3e3fa369ad5023cb9fa1c375111844820cbb 100644 (file)
@@ -5501,7 +5501,7 @@ vect_supportable_shift (enum tree_code code, tree scalar_type)
    stmt to replace it, put it in VEC_STMT, and insert it at GSI.
    Return true if STMT_INFO is vectorizable in this way.  */
 
-bool
+static bool
 vectorizable_shift (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
                    stmt_vec_info *vec_stmt, slp_tree slp_node,
                    stmt_vector_for_cost *cost_vec)
@@ -9776,10 +9776,9 @@ vect_is_simple_cond (tree cond, vec_info *vinfo,
 
    Return true if STMT_INFO is vectorizable in this way.  */
 
-bool
+static bool
 vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-                       stmt_vec_info *vec_stmt, bool for_reduction,
-                       int reduc_index,
+                       stmt_vec_info *vec_stmt,
                        slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   vec_info *vinfo = stmt_info->vinfo;
@@ -9812,17 +9811,36 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
     return false;
 
-  if (for_reduction && STMT_SLP_TYPE (stmt_info))
+  /* Is vectorizable conditional operation?  */
+  gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
+  if (!stmt)
     return false;
 
-  vect_reduction_type reduction_type
-    = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info);
-  if (!for_reduction)
+  code = gimple_assign_rhs_code (stmt);
+  if (code != COND_EXPR)
+    return false;
+
+  stmt_vec_info reduc_info = NULL;
+  int reduc_index = -1;
+  vect_reduction_type reduction_type = TREE_CODE_REDUCTION;
+  bool for_reduction
+    = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)) != NULL;
+  if (for_reduction)
+    {
+      if (STMT_SLP_TYPE (stmt_info))
+       return false;
+      reduc_info = info_for_reduction (stmt_info);
+      reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
+      reduc_index = STMT_VINFO_REDUC_IDX (stmt_info);
+      gcc_assert (reduction_type != EXTRACT_LAST_REDUCTION
+                 || reduc_index != -1);
+    }
+  else
     {
       if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
        return false;
 
-      /* FORNOW: not yet supported.  */
+      /* FORNOW: only supported as part of a reduction.  */
       if (STMT_VINFO_LIVE_P (stmt_info))
        {
          if (dump_enabled_p ())
@@ -9832,16 +9850,6 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
        }
     }
 
-  /* Is vectorizable conditional operation?  */
-  gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt);
-  if (!stmt)
-    return false;
-
-  code = gimple_assign_rhs_code (stmt);
-
-  if (code != COND_EXPR)
-    return false;
-
   tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
 
@@ -10665,12 +10673,10 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
          || vectorizable_load (stmt_info, NULL, NULL, node, node_instance,
                                cost_vec)
          || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec)
-         || vectorizable_reduction (stmt_info, NULL, NULL, node,
-                                    node_instance, cost_vec)
+         || vectorizable_reduction (stmt_info, node, node_instance, cost_vec)
          || vectorizable_induction (stmt_info, NULL, NULL, node, cost_vec)
          || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec)
-         || vectorizable_condition (stmt_info, NULL, NULL, false, -1, node,
-                                    cost_vec)
+         || vectorizable_condition (stmt_info, NULL, NULL, node, cost_vec)
          || vectorizable_comparison (stmt_info, NULL, NULL, node,
                                      cost_vec)
          || vectorizable_lc_phi (stmt_info, NULL, node));
@@ -10689,8 +10695,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
              || vectorizable_load (stmt_info, NULL, NULL, node, node_instance,
                                    cost_vec)
              || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec)
-             || vectorizable_condition (stmt_info, NULL, NULL, false, -1, node,
-                                        cost_vec)
+             || vectorizable_condition (stmt_info, NULL, NULL, node, cost_vec)
              || vectorizable_comparison (stmt_info, NULL, NULL, node,
                                          cost_vec));
     }
@@ -10796,8 +10801,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       break;
 
     case condition_vec_info_type:
-      done = vectorizable_condition (stmt_info, gsi, &vec_stmt, false, -1,
-                                    slp_node, NULL);
+      done = vectorizable_condition (stmt_info, gsi, &vec_stmt, slp_node, NULL);
       gcc_assert (done);
       break;
 
@@ -10895,20 +10899,22 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   /* If this stmt defines a value used on a backedge, update the
      vectorized PHIs.  */
   stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
-  if (!slp_node && STMT_VINFO_REDUC_DEF (orig_stmt_info)
-      && STMT_VINFO_REDUC_TYPE (orig_stmt_info) != FOLD_LEFT_REDUCTION
-      && (STMT_VINFO_REDUC_TYPE (orig_stmt_info) != COND_REDUCTION
-         || (STMT_VINFO_VEC_REDUCTION_TYPE (orig_stmt_info)
-             != EXTRACT_LAST_REDUCTION))
-      && is_a <gphi *> (STMT_VINFO_REDUC_DEF (orig_stmt_info)->stmt))
-    {
-      gphi *phi = as_a <gphi *> (STMT_VINFO_REDUC_DEF (orig_stmt_info)->stmt);
-      if (dominated_by_p (CDI_DOMINATORS,
-                         gimple_bb (orig_stmt_info->stmt), gimple_bb (phi)))
+  stmt_vec_info reduc_info;
+  if (STMT_VINFO_REDUC_DEF (orig_stmt_info)
+      && (reduc_info = info_for_reduction (orig_stmt_info))
+      && STMT_VINFO_REDUC_TYPE (reduc_info) != FOLD_LEFT_REDUCTION
+      && STMT_VINFO_REDUC_TYPE (reduc_info) != EXTRACT_LAST_REDUCTION)
+    {
+      gphi *phi;
+      if (!slp_node
+         && (phi = dyn_cast <gphi *>
+                     (STMT_VINFO_REDUC_DEF (orig_stmt_info)->stmt))
+         && dominated_by_p (CDI_DOMINATORS,
+                            gimple_bb (orig_stmt_info->stmt), gimple_bb (phi)))
        {
          edge e = loop_latch_edge (gimple_bb (phi)->loop_father);
          stmt_vec_info phi_info
-             = STMT_VINFO_VEC_STMT (STMT_VINFO_REDUC_DEF (orig_stmt_info));
+           = STMT_VINFO_VEC_STMT (STMT_VINFO_REDUC_DEF (orig_stmt_info));
          stmt_vec_info vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
          do
            {
@@ -10921,24 +10927,19 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
          while (phi_info);
          gcc_assert (!vec_stmt);
        }
-    }
-  else if (slp_node && STMT_VINFO_REDUC_DEF (orig_stmt_info)
-          /* Going back and forth via STMT_VINFO_REDUC_DEF gets us to the
-             stmt with the reduction meta in case of reduction groups.  */
-          && (STMT_VINFO_REDUC_TYPE
-                (STMT_VINFO_REDUC_DEF (STMT_VINFO_REDUC_DEF (orig_stmt_info)))
-              != FOLD_LEFT_REDUCTION)
-          && slp_node != slp_node_instance->reduc_phis)
-    {
-      slp_tree phi_node = slp_node_instance->reduc_phis;
-      gphi *phi = as_a <gphi *> (SLP_TREE_SCALAR_STMTS (phi_node)[0]->stmt);
-      edge e = loop_latch_edge (gimple_bb (phi)->loop_father);
-      gcc_assert (SLP_TREE_VEC_STMTS (phi_node).length ()
-                 == SLP_TREE_VEC_STMTS (slp_node).length ());
-      for (unsigned i = 0; i < SLP_TREE_VEC_STMTS (phi_node).length (); ++i)
-       add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]->stmt),
-                    gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt),
-                    e, gimple_phi_arg_location (phi, e->dest_idx));
+      else if (slp_node
+              && slp_node != slp_node_instance->reduc_phis)
+       {
+         slp_tree phi_node = slp_node_instance->reduc_phis;
+         gphi *phi = as_a <gphi *> (SLP_TREE_SCALAR_STMTS (phi_node)[0]->stmt);
+         edge e = loop_latch_edge (gimple_bb (phi)->loop_father);
+         gcc_assert (SLP_TREE_VEC_STMTS (phi_node).length ()
+                     == SLP_TREE_VEC_STMTS (slp_node).length ());
+         for (unsigned i = 0; i < SLP_TREE_VEC_STMTS (phi_node).length (); ++i)
+           add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]->stmt),
+                        gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt),
+                        e, gimple_phi_arg_location (phi, e->dest_idx));
+       }
     }
 
   /* Handle stmts whose DEF is used outside the loop-nest that is
index 800c99fea266fc65f2941652d7bad562e4562c69..3e8637f070d5cd526d6626d2b7ba1c5f9243ce0a 100644 (file)
@@ -637,7 +637,7 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   STMT_VINFO_TYPE (res) = undef_vec_info_type;
   STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
   STMT_VINFO_VECTORIZABLE (res) = true;
-  STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
+  STMT_VINFO_REDUC_TYPE (res) = TREE_CODE_REDUCTION;
   STMT_VINFO_VEC_COND_REDUC_CODE (res) = ERROR_MARK;
   STMT_VINFO_REDUC_CODE (res) = ERROR_MARK;
   STMT_VINFO_REDUC_FN (res) = IFN_LAST;
index 837fb5ab52537cdf95a413557335b30704f9dc26..71ea6adde045ab9a8e22cb91cf9c3ffeee3d8f73 100644 (file)
@@ -933,9 +933,6 @@ public:
      for loop vectorization.  */
   vect_memory_access_type memory_access_type;
 
-  /* For reduction loops, this is the type of reduction.  */
-  enum vect_reduction_type v_reduc_type;
-
   /* For CONST_COND_REDUCTION and INTEGER_INDUC_COND_REDUCTION, the
      reduction code.  */
   enum tree_code cond_reduc_code;
@@ -947,7 +944,7 @@ public:
   tree reduc_epilogue_adjustment;
 
   /* On a reduction PHI the reduction type as detected by
-     vect_force_simple_reduction.  */
+     vect_is_simple_reduction and vectorizable_reduction.  */
   enum vect_reduction_type reduc_type;
 
   /* The original reduction code, to be used in the epilogue.  */
@@ -964,6 +961,9 @@ public:
      corresponding PHI.  */
   stmt_vec_info reduc_def;
 
+  /* Whether on this stmt reduction meta is recorded.  */
+  bool is_reduc_info;
+
   /* The number of scalar stmt references from active SLP instances.  */
   unsigned int num_slp_uses;
 
@@ -1046,7 +1046,6 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo)
 #define STMT_VINFO_STRIDED_P(S)                   (S)->strided_p
 #define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p
-#define STMT_VINFO_VEC_REDUCTION_TYPE(S)   (S)->v_reduc_type
 #define STMT_VINFO_VEC_COND_REDUC_CODE(S)  (S)->cond_reduc_code
 #define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
 #define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
@@ -1556,12 +1555,6 @@ extern bool vect_transform_stmt (stmt_vec_info, gimple_stmt_iterator *,
 extern void vect_remove_stores (stmt_vec_info);
 extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree,
                                     slp_instance, stmt_vector_for_cost *);
-extern bool vectorizable_condition (stmt_vec_info, gimple_stmt_iterator *,
-                                   stmt_vec_info *, bool, int, slp_tree,
-                                   stmt_vector_for_cost *);
-extern bool vectorizable_shift (stmt_vec_info, gimple_stmt_iterator *,
-                               stmt_vec_info *, slp_tree,
-                               stmt_vector_for_cost *);
 extern void vect_get_load_cost (stmt_vec_info, int, bool,
                                unsigned int *, unsigned int *,
                                stmt_vector_for_cost *,
@@ -1644,6 +1637,7 @@ extern void vect_record_loop_mask (loop_vec_info, vec_loop_masks *,
                                   unsigned int, tree);
 extern tree vect_get_loop_mask (gimple_stmt_iterator *, vec_loop_masks *,
                                unsigned int, tree, unsigned int);
+extern stmt_vec_info info_for_reduction (stmt_vec_info);
 
 /* Drive for loop transformation stage.  */
 extern class loop *vect_transform_loop (loop_vec_info);
@@ -1653,8 +1647,7 @@ extern bool vectorizable_live_operation (stmt_vec_info, gimple_stmt_iterator *,
                                         slp_tree, slp_instance, int,
                                         stmt_vec_info *,
                                         stmt_vector_for_cost *);
-extern bool vectorizable_reduction (stmt_vec_info, gimple_stmt_iterator *,
-                                   stmt_vec_info *, slp_tree, slp_instance,
+extern bool vectorizable_reduction (stmt_vec_info, slp_tree, slp_instance,
                                    stmt_vector_for_cost *);
 extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *,
                                    stmt_vec_info *, slp_tree,