From: Richard Sandiford Date: Thu, 19 Sep 2019 13:33:55 +0000 (+0000) Subject: Enforce correct COND_EXPR order for EXTRACT_LAST_REDUCTION X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=c449d3ae28ff4e133114fb67dbf7dcc7a95ca5d5;p=gcc.git Enforce correct COND_EXPR order for EXTRACT_LAST_REDUCTION For conditional reductions, the "then" value needs to be the candidate value calculated by this iteration while the "else" value needs to be the result carried over from previous iterations. If the COND_EXPR is the other way around, we need to swap it. 2019-09-19 Richard Sandiford gcc/ * tree-vectorizer.h (vectorizable_condition): Take an int reduction index instead of a boolean flag. * tree-vect-stmts.c (vectorizable_condition): Likewise. Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION reductions if the reduction accumulator is the "then" rather than the "else" value. (vect_analyze_stmt): Update call accordingly. (vect_transform_stmt): Likewise. * tree-vect-loop.c (vectorizable_reduction): Likewise, asserting that the index is > 0. From-SVN: r275962 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index cbddb904bff..506e93d757c 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,16 @@ +2019-09-19 Richard Sandiford + + * tree-vectorizer.h (vectorizable_condition): Take an int + reduction index instead of a boolean flag. + * tree-vect-stmts.c (vectorizable_condition): Likewise. + Swap the "then" and "else" values for EXTRACT_LAST_REDUCTION + reductions if the reduction accumulator is the "then" rather + than the "else" value. + (vect_analyze_stmt): Update call accordingly. + (vect_transform_stmt): Likewise. + * tree-vect-loop.c (vectorizable_reduction): Likewise, + asserting that the index is > 0. + 2019-09-19 Martin Liska * ipa-icf.c (sort_sem_items_by_decl_uid): Simplify comparator. diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 5ee4ee3063f..aec44622e03 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -6659,8 +6659,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, { /* Only call during the analysis stage, otherwise we'll lose STMT_VINFO_TYPE. */ + gcc_assert (reduc_index > 0); if (!vec_stmt && !vectorizable_condition (stmt_info, gsi, NULL, - true, NULL, cost_vec)) + reduc_index, NULL, cost_vec)) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -7113,9 +7114,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, if (reduction_type == EXTRACT_LAST_REDUCTION) { - gcc_assert (!slp_node); + gcc_assert (!slp_node && reduc_index > 0); return vectorizable_condition (stmt_info, gsi, vec_stmt, - true, NULL, NULL); + reduc_index, NULL, NULL); } /* Create the destination vector */ @@ -7145,9 +7146,9 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, { if (code == COND_EXPR) { - gcc_assert (!slp_node); + gcc_assert (!slp_node && reduc_index > 0); vectorizable_condition (stmt_info, gsi, vec_stmt, - true, NULL, NULL); + reduc_index, NULL, NULL); break; } if (code == LSHIFT_EXPR diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 7f5f1c3fafa..0636e5c9ffa 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -9778,7 +9778,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, bool vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, bool for_reduction, + stmt_vec_info *vec_stmt, int reduc_index, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { vec_info *vinfo = stmt_info->vinfo; @@ -9807,6 +9807,7 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vec vec_oprnds3 = vNULL; tree vec_cmp_type; bool masked = false; + bool for_reduction = (reduc_index > 0); if (for_reduction && STMT_SLP_TYPE (stmt_info)) return false; @@ -9888,6 +9889,29 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, cond_expr1 = TREE_OPERAND (cond_expr, 1); } + /* For conditional reductions, the "then" value needs to be the candidate + value calculated by this iteration while the "else" value needs to be + the result carried over from previous iterations. If the COND_EXPR + is the other way around, we need to swap it. */ + bool must_invert_cmp_result = false; + if (reduction_type == EXTRACT_LAST_REDUCTION && reduc_index == 1) + { + if (masked) + must_invert_cmp_result = true; + else + { + bool honor_nans = HONOR_NANS (TREE_TYPE (cond_expr0)); + tree_code new_code = invert_tree_comparison (cond_code, honor_nans); + if (new_code == ERROR_MARK) + must_invert_cmp_result = true; + else + cond_code = new_code; + } + /* Make sure we don't accidentally use the old condition. */ + cond_expr = NULL_TREE; + std::swap (then_clause, else_clause); + } + if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype)) { /* Boolean values may have another representation in vectors @@ -10102,6 +10126,15 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vect_finish_stmt_generation (stmt_info, new_stmt, gsi); vec_compare = vec_compare_name; } + if (must_invert_cmp_result) + { + tree vec_compare_name = make_ssa_name (vec_cmp_type); + gassign *new_stmt = gimple_build_assign (vec_compare_name, + BIT_NOT_EXPR, + vec_compare); + vect_finish_stmt_generation (stmt_info, new_stmt, gsi); + vec_compare = vec_compare_name; + } gcall *new_stmt = gimple_build_call_internal (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, vec_then_clause); @@ -10635,7 +10668,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize, node_instance, cost_vec) || vectorizable_induction (stmt_info, NULL, NULL, node, cost_vec) || vectorizable_shift (stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (stmt_info, NULL, NULL, false, node, + || vectorizable_condition (stmt_info, NULL, NULL, 0, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, cost_vec)); @@ -10654,7 +10687,7 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize, || vectorizable_load (stmt_info, NULL, NULL, node, node_instance, cost_vec) || vectorizable_store (stmt_info, NULL, NULL, node, cost_vec) - || vectorizable_condition (stmt_info, NULL, NULL, false, node, + || vectorizable_condition (stmt_info, NULL, NULL, 0, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, cost_vec)); @@ -10759,7 +10792,7 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, break; case condition_vec_info_type: - done = vectorizable_condition (stmt_info, gsi, &vec_stmt, false, + done = vectorizable_condition (stmt_info, gsi, &vec_stmt, 0, slp_node, NULL); gcc_assert (done); break; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index ac6e89907a3..aef0ff8663c 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1534,7 +1534,7 @@ extern void vect_remove_stores (stmt_vec_info); extern opt_result vect_analyze_stmt (stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *); extern bool vectorizable_condition (stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, bool, slp_tree, + stmt_vec_info *, int, slp_tree, stmt_vector_for_cost *); extern bool vectorizable_shift (stmt_vec_info, gimple_stmt_iterator *, stmt_vec_info *, slp_tree,