Refactor vect_get_and_check_slp_defs some more

This refactors vect_get_and_check_slp_defs so that the ops and def_stmts
arrays are filled for all stmts and operands even when we signal failure.
This allows later changes for BB vectorization SLP discovery heuristics.

2020-10-16  Richard Biener  <rguenther@suse.de>

	* tree-vect-slp.c (vect_get_and_check_slp_defs): First analyze
	all operands and fill in the def_stmts and ops entries.
	(vect_def_types_match): New helper.

diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index 8037b27cddd556a78f19fd800b82414177012d91..cfb79e2651f20e067e3ed1a1d404e381aabbd5a0 100644 (file)
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -356,6 +356,16 @@ can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
     }
 }
 
+/* Return true if DTA and DTB match.  */
+
+static bool
+vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
+{
+  return (dta == dtb
+         || ((dta == vect_external_def || dta == vect_constant_def)
+             && (dtb == vect_external_def || dtb == vect_constant_def)));
+}
+
 /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
    they are of a valid type and that they match the defs of the first stmt of
    the SLP group (stored in OPRNDS_INFO).  This function tries to match stmts
@@ -421,9 +431,9 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
 
   bool swapped = (swap != 0);
   gcc_assert (!swapped || first_op_cond);
+  enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
   for (i = 0; i < number_of_oprnds; i++)
     {
-again:
       if (first_op_cond)
        {
          /* Map indicating how operands of cond_expr should be swapped.  */
@@ -444,7 +454,7 @@ again:
       oprnd_info = (*oprnds_info)[i];
 
       stmt_vec_info def_stmt_info;
-      if (!vect_is_simple_use (oprnd, vinfo, &dt, &def_stmt_info))
+      if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
        {
          if (dump_enabled_p ())
            dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -457,9 +467,13 @@ again:
       if (def_stmt_info && is_pattern_stmt_p (def_stmt_info))
        oprnd_info->any_pattern = true;
 
-      tree type = TREE_TYPE (oprnd);
+      oprnd_info->def_stmts.quick_push (def_stmt_info);
+      oprnd_info->ops.quick_push (oprnd);
+
       if (first)
        {
+         tree type = TREE_TYPE (oprnd);
+         dt = dts[i];
          if ((dt == vect_constant_def
               || dt == vect_external_def)
              && !GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
@@ -480,14 +494,48 @@ again:
              && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
              && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
              && def_stmt_info)
-           dt = vect_reduction_def;
+           dts[i] = dt = vect_reduction_def;
+
+         /* Check the types of the definition.  */
+         switch (dt)
+           {
+           case vect_external_def:
+           case vect_constant_def:
+           case vect_internal_def:
+           case vect_reduction_def:
+           case vect_induction_def:
+             break;
+
+           default:
+             /* FORNOW: Not supported.  */
+             if (dump_enabled_p ())
+               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+                                "Build SLP failed: illegal type of def %T\n",
+                                oprnd);
+             return -1;
+           }
+
          oprnd_info->first_dt = dt;
          oprnd_info->first_op_type = type;
        }
-      else
-       {
+    }
+  if (first)
+    return 0;
+
+  /* Now match the operand definition types to that of the first stmt.  */
+  for (i = 0; i < number_of_oprnds;)
+    {
+      oprnd_info = (*oprnds_info)[i];
+      dt = dts[i];
+      stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
+      oprnd = oprnd_info->ops[stmt_num];
+      tree type = TREE_TYPE (oprnd);
+
          if (!types_compatible_p (oprnd_info->first_op_type, type))
            {
+             gcc_assert ((i != commutative_op
+                          && (commutative_op == -1U
+                              || i != commutative_op + 1)));
              if (dump_enabled_p ())
                dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
                                 "Build SLP failed: different operand types\n");
@@ -499,18 +547,14 @@ again:
             types for reduction chains: the first stmt must be a
             vect_reduction_def (a phi node), and the rest
             end in the reduction chain.  */
-         if ((oprnd_info->first_dt != dt
+         if ((!vect_def_types_match (oprnd_info->first_dt, dt)
               && !(oprnd_info->first_dt == vect_reduction_def
                    && !STMT_VINFO_DATA_REF (stmt_info)
                    && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
                    && def_stmt_info
                    && !STMT_VINFO_DATA_REF (def_stmt_info)
                    && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
-                       == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
-              && !((oprnd_info->first_dt == vect_external_def
-                    || oprnd_info->first_dt == vect_constant_def)
-                   && (dt == vect_external_def
-                       || dt == vect_constant_def)))
+                       == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
              || (!STMT_VINFO_DATA_REF (stmt_info)
                  && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
                  && ((!def_stmt_info
@@ -519,69 +563,48 @@ again:
                           != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
                      != (oprnd_info->first_dt != vect_reduction_def))))
            {
-             /* Try swapping operands if we got a mismatch.  */
+             /* Try swapping operands if we got a mismatch.  For BB
+                vectorization only in case that will improve things.  */
              if (i == commutative_op && !swapped)
                {
                  if (dump_enabled_p ())
                    dump_printf_loc (MSG_NOTE, vect_location,
                                     "trying swapped operands\n");
+                 std::swap (dts[i], dts[i+1]);
+                 std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
+                            (*oprnds_info)[i+1]->def_stmts[stmt_num]);
+                 std::swap ((*oprnds_info)[i]->ops[stmt_num],
+                            (*oprnds_info)[i+1]->ops[stmt_num]);
                  swapped = true;
-                 goto again;
+                 continue;
                }
 
              if (dump_enabled_p ())
                dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
                                 "Build SLP failed: different types\n");
-
              return 1;
            }
-       }
 
-      /* Check the types of the definitions.  */
-      switch (dt)
+      /* Make sure to demote the overall operand to external.  */
+      if (dt == vect_external_def)
+       oprnd_info->first_dt = vect_external_def;
+      /* For a SLP reduction chain we want to duplicate the reduction to
+        each of the chain members.  That gets us a sane SLP graph (still
+        the stmts are not 100% correct wrt the initial values).  */
+      else if ((dt == vect_internal_def
+               || dt == vect_reduction_def)
+              && oprnd_info->first_dt == vect_reduction_def
+              && !STMT_VINFO_DATA_REF (stmt_info)
+              && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
+              && !STMT_VINFO_DATA_REF (def_stmt_info)
+              && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
+                  == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
        {
-       case vect_external_def:
-         /* Make sure to demote the overall operand to external.  */
-         oprnd_info->first_dt = vect_external_def;
-         /* Fallthru.  */
-       case vect_constant_def:
-         oprnd_info->def_stmts.quick_push (NULL);
-         oprnd_info->ops.quick_push (oprnd);
-         break;
-
-       case vect_internal_def:
-       case vect_reduction_def:
-         if (oprnd_info->first_dt == vect_reduction_def
-             && !STMT_VINFO_DATA_REF (stmt_info)
-             && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
-             && !STMT_VINFO_DATA_REF (def_stmt_info)
-             && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
-                 == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
-           {
-             /* For a SLP reduction chain we want to duplicate the
-                reduction to each of the chain members.  That gets
-                us a sane SLP graph (still the stmts are not 100%
-                correct wrt the initial values).  */
-             gcc_assert (!first);
-             oprnd_info->def_stmts.quick_push (oprnd_info->def_stmts[0]);
-             oprnd_info->ops.quick_push (oprnd_info->ops[0]);
-             break;
-           }
-         /* Fallthru.  */
-       case vect_induction_def:
-         oprnd_info->def_stmts.quick_push (def_stmt_info);
-         oprnd_info->ops.quick_push (oprnd);
-         break;
-
-       default:
-         /* FORNOW: Not supported.  */
-         if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "Build SLP failed: illegal type of def %T\n",
-                            oprnd);
-
-         return -1;
+         oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
+         oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
        }
+
+      ++i;
     }
 
   /* Swap operands.  */