Enforce correct COND_EXPR order for EXTRACT_LAST_REDUCTION

For conditional reductions, the "then" value needs to be the candidate
value calculated by this iteration while the "else" value needs to be
the result carried over from previous iterations.  If the COND_EXPR
is the other way around, we need to swap it.

2019-09-19  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (vectorizable_condition): Take an int
	reduction index instead of a boolean flag.
	* tree-vect-stmts.c (vectorizable_condition): Likewise.
	Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION
	reductions if the reduction accumulator is the "then" rather
	than the "else" value.
	(vect_analyze_stmt): Update call accordingly.
	(vect_transform_stmt): Likewise.
	* tree-vect-loop.c (vectorizable_reduction): Likewise,
	asserting that the index is > 0.

From-SVN: r275962

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index cbddb904bfff6c0818e48b65ecfe948a5d7387d2..506e93d757ca7bf218045eef572107c0bc9d4966 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,16 @@
+2019-09-19  Richard Sandiford  <richard.sandiford@arm.com>
+
+       * tree-vectorizer.h (vectorizable_condition): Take an int
+       reduction index instead of a boolean flag.
+       * tree-vect-stmts.c (vectorizable_condition): Likewise.
+       Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION
+       reductions if the reduction accumulator is the "then" rather
+       than the "else" value.
+       (vect_analyze_stmt): Update call accordingly.
+       (vect_transform_stmt): Likewise.
+       * tree-vect-loop.c (vectorizable_reduction): Likewise,
+       asserting that the index is > 0.
+
 2019-09-19  Martin Liska  <mliska@suse.cz>
 
        * ipa-icf.c (sort_sem_items_by_decl_uid): Simplify comparator.

diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 5ee4ee3063f07b82871abb22f0a7bb9b327326d9..aec44622e03c34701b912cefeec10b4b1abb42c8 100644 (file)
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -6659,8 +6659,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     {
       /* Only call during the analysis stage, otherwise we'll lose
         STMT_VINFO_TYPE.  */
+      gcc_assert (reduc_index > 0);
       if (!vec_stmt && !vectorizable_condition (stmt_info, gsi, NULL,
-                                               true, NULL, cost_vec))
+                                               reduc_index, NULL, cost_vec))
         {
           if (dump_enabled_p ())
            dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -7113,9 +7114,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 
   if (reduction_type == EXTRACT_LAST_REDUCTION)
     {
-      gcc_assert (!slp_node);
+      gcc_assert (!slp_node && reduc_index > 0);
       return vectorizable_condition (stmt_info, gsi, vec_stmt,
-                                    true, NULL, NULL);
+                                    reduc_index, NULL, NULL);
     }
 
   /* Create the destination vector  */
@@ -7145,9 +7146,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
     {
       if (code == COND_EXPR)
         {
-          gcc_assert (!slp_node);
+          gcc_assert (!slp_node && reduc_index > 0);
          vectorizable_condition (stmt_info, gsi, vec_stmt,
-                                 true, NULL, NULL);
+                                 reduc_index, NULL, NULL);
           break;
         }
       if (code == LSHIFT_EXPR

diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 7f5f1c3fafaffc3f62b255160a794bf94fc7f98e..0636e5c9ffa08b0e62cdf0f5d7d1bb775b716e37 100644 (file)
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -9778,7 +9778,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo,
 
 bool
 vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-                       stmt_vec_info *vec_stmt, bool for_reduction,
+                       stmt_vec_info *vec_stmt, int reduc_index,
                        slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   vec_info *vinfo = stmt_info->vinfo;
@@ -9807,6 +9807,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   vec<tree> vec_oprnds3 = vNULL;
   tree vec_cmp_type;
   bool masked = false;
+  bool for_reduction = (reduc_index > 0);
 
   if (for_reduction && STMT_SLP_TYPE (stmt_info))
     return false;
@@ -9888,6 +9889,29 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       cond_expr1 = TREE_OPERAND (cond_expr, 1);
     }
 
+  /* For conditional reductions, the "then" value needs to be the candidate
+     value calculated by this iteration while the "else" value needs to be
+     the result carried over from previous iterations.  If the COND_EXPR
+     is the other way around, we need to swap it.  */
+  bool must_invert_cmp_result = false;
+  if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1)
+    {
+      if (masked)
+       must_invert_cmp_result = true;
+      else
+       {
+         bool honor_nans = HONOR_NANS (TREE_TYPE (cond_expr0));
+         tree_code new_code = invert_tree_comparison (cond_code, honor_nans);
+         if (new_code == ERROR_MARK)
+           must_invert_cmp_result = true;
+         else
+           cond_code = new_code;
+       }
+      /* Make sure we don't accidentally use the old condition.  */
+      cond_expr = NULL_TREE;
+      std::swap (then_clause, else_clause);
+    }
+
   if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype))
     {
       /* Boolean values may have another representation in vectors
@@ -10102,6 +10126,15 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
                  vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
                  vec_compare = vec_compare_name;
                }
+             if (must_invert_cmp_result)
+               {
+                 tree vec_compare_name = make_ssa_name (vec_cmp_type);
+                 gassign *new_stmt = gimple_build_assign (vec_compare_name,
+                                                          BIT_NOT_EXPR,
+                                                          vec_compare);
+                 vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
+                 vec_compare = vec_compare_name;
+               }
              gcall *new_stmt = gimple_build_call_internal
                (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
                 vec_then_clause);
@@ -10635,7 +10668,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
                                     node_instance, cost_vec)
          || vectorizable_induction (stmt_info, NULL, NULL, node, cost_vec)
          || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec)
-         || vectorizable_condition (stmt_info, NULL, NULL, false, node,
+         || vectorizable_condition (stmt_info, NULL, NULL, 0, node,
                                     cost_vec)
          || vectorizable_comparison (stmt_info, NULL, NULL, node,
                                      cost_vec));
@@ -10654,7 +10687,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
              || vectorizable_load (stmt_info, NULL, NULL, node, node_instance,
                                    cost_vec)
              || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec)
-             || vectorizable_condition (stmt_info, NULL, NULL, false, node,
+             || vectorizable_condition (stmt_info, NULL, NULL, 0, node,
                                         cost_vec)
              || vectorizable_comparison (stmt_info, NULL, NULL, node,
                                          cost_vec));
@@ -10759,7 +10792,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       break;
 
     case condition_vec_info_type:
-      done = vectorizable_condition (stmt_info, gsi, &vec_stmt, false,
+      done = vectorizable_condition (stmt_info, gsi, &vec_stmt, 0,
                                     slp_node, NULL);
       gcc_assert (done);
       break;

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index ac6e89907a30eb7c04efd6eb82b6ab0ff743aaa1..aef0ff8663c7c19d20b0a6adfd7a5d19be2cfae9 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1534,7 +1534,7 @@ extern void vect_remove_stores (stmt_vec_info);
 extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree,
                                     slp_instance, stmt_vector_for_cost *);
 extern bool vectorizable_condition (stmt_vec_info, gimple_stmt_iterator *,
-                                   stmt_vec_info *, bool, slp_tree,
+                                   stmt_vec_info *, int, slp_tree,
                                    stmt_vector_for_cost *);
 extern bool vectorizable_shift (stmt_vec_info, gimple_stmt_iterator *,
                                stmt_vec_info *, slp_tree,