From: Richard Biener Date: Wed, 1 Jun 2016 13:08:24 +0000 (+0000) Subject: re PR tree-optimization/71261 (Trunk GCC hangs on knl and broadwell targets) X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=fa2c9034583b7fe6300e547dc67792e4eb440f48;p=gcc.git re PR tree-optimization/71261 (Trunk GCC hangs on knl and broadwell targets) 2016-06-01 Richard Biener PR tree-optimization/71261 * tree-vect-patterns.c (check_bool_pattern): Gather a hash-set of stmts successfully put in the bool pattern. Remove single-use restriction. (adjust_bool_pattern_cast): Add cast at the use site via the pattern def sequence. (adjust_bool_pattern): Remove recursion, maintain a hash-map of patterned defs. Use the pattern def seqence instead of multiple independent patterns. (sort_after_uid): New qsort compare function. (adjust_bool_stmts): New function to process stmts in the bool pattern in IL order. (vect_recog_bool_pattern): Adjust. * tree-if-conv.c (ifcvt_split_def_stmt): Remove. (ifcvt_walk_pattern_tree): Likewise. (stmt_is_root_of_bool_pattern): Likewise. (ifcvt_repair_bool_pattern): Likewise. (tree_if_conversion): Do not call ifcvt_repair_bool_pattern. * gcc.dg/torture/vect-bool-1.c: New testcase. From-SVN: r236989 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index d7321aeff2f..8ab3fc62693 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,24 @@ +2016-06-01 Richard Biener + + PR tree-optimization/71261 + * tree-vect-patterns.c (check_bool_pattern): Gather a hash-set + of stmts successfully put in the bool pattern. Remove + single-use restriction. + (adjust_bool_pattern_cast): Add cast at the use site via the + pattern def sequence. + (adjust_bool_pattern): Remove recursion, maintain a hash-map + of patterned defs. Use the pattern def seqence instead of + multiple independent patterns. + (sort_after_uid): New qsort compare function. + (adjust_bool_stmts): New function to process stmts in the bool + pattern in IL order. + (vect_recog_bool_pattern): Adjust. + * tree-if-conv.c (ifcvt_split_def_stmt): Remove. + (ifcvt_walk_pattern_tree): Likewise. + (stmt_is_root_of_bool_pattern): Likewise. + (ifcvt_repair_bool_pattern): Likewise. + (tree_if_conversion): Do not call ifcvt_repair_bool_pattern. + 2016-06-01 Jan Hubicka * loop-unroll.c (decide_unroll_constant_iterations, diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 1efe0e1bb4a..ba28ae8d6f4 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,8 @@ +2016-06-01 Richard Biener + + PR tree-optimization/71261 + * gcc.dg/torture/vect-bool-1.c: New testcase. + 2016-06-01 Jakub Jelinek * gfortran.dg/gomp/order-1.f90: New test. diff --git a/gcc/testsuite/gcc.dg/torture/vect-bool-1.c b/gcc/testsuite/gcc.dg/torture/vect-bool-1.c new file mode 100644 index 00000000000..ca1456b4d8e --- /dev/null +++ b/gcc/testsuite/gcc.dg/torture/vect-bool-1.c @@ -0,0 +1,14 @@ +/* { dg-do compile } */ + +int a[64]; +long b[64]; + +void foo (void) +{ + for (int i = 0; i < 64; ++i) + { + _Bool x = a[i] < 10; + a[i] = x; + b[i] = x; + } +} diff --git a/gcc/tree-if-conv.c b/gcc/tree-if-conv.c index 5914a785ae8..e5a3372cd4e 100644 --- a/gcc/tree-if-conv.c +++ b/gcc/tree-if-conv.c @@ -2516,194 +2516,6 @@ ifcvt_split_critical_edges (struct loop *loop, bool aggressive_if_conv) return true; } -/* Assumes that lhs of DEF_STMT have multiple uses. - Delete one use by (1) creation of copy DEF_STMT with - unique lhs; (2) change original use of lhs in one - use statement with newly created lhs. */ - -static void -ifcvt_split_def_stmt (gimple *def_stmt, gimple *use_stmt) -{ - tree var; - tree lhs; - gimple *copy_stmt; - gimple_stmt_iterator gsi; - use_operand_p use_p; - ssa_op_iter iter; - - var = gimple_assign_lhs (def_stmt); - copy_stmt = gimple_copy (def_stmt); - lhs = make_temp_ssa_name (TREE_TYPE (var), NULL, "_ifc_"); - gimple_assign_set_lhs (copy_stmt, lhs); - SSA_NAME_DEF_STMT (lhs) = copy_stmt; - /* Insert copy of DEF_STMT. */ - gsi = gsi_for_stmt (def_stmt); - gsi_insert_after (&gsi, copy_stmt, GSI_SAME_STMT); - /* Change use of var to lhs in use_stmt. */ - if (dump_file && (dump_flags & TDF_DETAILS)) - { - fprintf (dump_file, "Change use of var "); - print_generic_expr (dump_file, var, TDF_SLIM); - fprintf (dump_file, " to "); - print_generic_expr (dump_file, lhs, TDF_SLIM); - fprintf (dump_file, "\n"); - } - FOR_EACH_SSA_USE_OPERAND (use_p, use_stmt, iter, SSA_OP_USE) - { - if (USE_FROM_PTR (use_p) != var) - continue; - SET_USE (use_p, lhs); - break; - } -} - -/* Traverse bool pattern recursively starting from VAR. - Save its def and use statements to defuse_list if VAR does - not have single use. */ - -static void -ifcvt_walk_pattern_tree (tree var, vec *defuse_list, - gimple *use_stmt) -{ - tree rhs1, rhs2; - enum tree_code code; - gimple *def_stmt; - - if (TREE_CODE (var) != SSA_NAME) - return; - - def_stmt = SSA_NAME_DEF_STMT (var); - if (gimple_code (def_stmt) != GIMPLE_ASSIGN) - return; - if (!has_single_use (var)) - { - /* Put def and use stmts into defuse_list. */ - defuse_list->safe_push (def_stmt); - defuse_list->safe_push (use_stmt); - if (dump_file && (dump_flags & TDF_DETAILS)) - { - fprintf (dump_file, "Multiple lhs uses in stmt\n"); - print_gimple_stmt (dump_file, def_stmt, 0, TDF_SLIM); - } - } - rhs1 = gimple_assign_rhs1 (def_stmt); - code = gimple_assign_rhs_code (def_stmt); - switch (code) - { - case SSA_NAME: - ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt); - break; - CASE_CONVERT: - if ((TYPE_PRECISION (TREE_TYPE (rhs1)) != 1 - || !TYPE_UNSIGNED (TREE_TYPE (rhs1))) - && TREE_CODE (TREE_TYPE (rhs1)) != BOOLEAN_TYPE) - break; - ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt); - break; - case BIT_NOT_EXPR: - ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt); - break; - case BIT_AND_EXPR: - case BIT_IOR_EXPR: - case BIT_XOR_EXPR: - ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt); - rhs2 = gimple_assign_rhs2 (def_stmt); - ifcvt_walk_pattern_tree (rhs2, defuse_list, def_stmt); - break; - default: - break; - } - return; -} - -/* Returns true if STMT can be a root of bool pattern applied - by vectorizer. */ - -static bool -stmt_is_root_of_bool_pattern (gimple *stmt) -{ - enum tree_code code; - tree lhs, rhs; - - code = gimple_assign_rhs_code (stmt); - if (CONVERT_EXPR_CODE_P (code)) - { - lhs = gimple_assign_lhs (stmt); - rhs = gimple_assign_rhs1 (stmt); - if (TREE_CODE (TREE_TYPE (rhs)) != BOOLEAN_TYPE) - return false; - if (TREE_CODE (TREE_TYPE (lhs)) == BOOLEAN_TYPE) - return false; - return true; - } - else if (code == COND_EXPR) - { - rhs = gimple_assign_rhs1 (stmt); - if (TREE_CODE (rhs) != SSA_NAME) - return false; - return true; - } - return false; -} - -/* Traverse all statements in BB which correspond to loop header to - find out all statements which can start bool pattern applied by - vectorizer and convert multiple uses in it to conform pattern - restrictions. Such case can occur if the same predicate is used both - for phi node conversion and load/store mask. */ - -static void -ifcvt_repair_bool_pattern (basic_block bb) -{ - tree rhs; - gimple *stmt; - gimple_stmt_iterator gsi; - auto_vec defuse_list; - auto_vec pattern_roots; - bool repeat = true; - int niter = 0; - unsigned int ix; - - /* Collect all root pattern statements. */ - for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) - { - stmt = gsi_stmt (gsi); - if (gimple_code (stmt) != GIMPLE_ASSIGN) - continue; - if (!stmt_is_root_of_bool_pattern (stmt)) - continue; - pattern_roots.safe_push (stmt); - } - - if (pattern_roots.is_empty ()) - return; - - /* Split all statements with multiple uses iteratively since splitting - may create new multiple uses. */ - while (repeat) - { - repeat = false; - niter++; - FOR_EACH_VEC_ELT (pattern_roots, ix, stmt) - { - rhs = gimple_assign_rhs1 (stmt); - ifcvt_walk_pattern_tree (rhs, &defuse_list, stmt); - while (defuse_list.length () > 0) - { - repeat = true; - gimple *def_stmt, *use_stmt; - use_stmt = defuse_list.pop (); - def_stmt = defuse_list.pop (); - ifcvt_split_def_stmt (def_stmt, use_stmt); - } - - } - } - if (dump_file && (dump_flags & TDF_DETAILS)) - fprintf (dump_file, "Repair bool pattern takes %d iterations. \n", - niter); -} - /* Delete redundant statements produced by predication which prevents loop vectorization. */ @@ -2850,10 +2662,8 @@ tree_if_conversion (struct loop *loop) on-the-fly. */ combine_blocks (loop); - /* Delete dead predicate computations and repair tree correspondent - to bool pattern to delete multiple uses of predicates. */ + /* Delete dead predicate computations. */ ifcvt_local_dce (loop->header); - ifcvt_repair_bool_pattern (loop->header); todo |= TODO_cleanup_cfg; mark_virtual_operands_for_renaming (cfun); diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index cc8c445863e..e1622a81b91 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -2888,10 +2888,11 @@ vect_recog_mixed_size_cond_pattern (vec *stmts, tree *type_in, /* Helper function of vect_recog_bool_pattern. Called recursively, return true if bool VAR can and should be optimized that way. Assume it shouldn't in case it's a result of a comparison which can be directly vectorized into - a vector comparison. */ + a vector comparison. Fills in STMTS with all stmts visited during the + walk. */ static bool -check_bool_pattern (tree var, vec_info *vinfo) +check_bool_pattern (tree var, vec_info *vinfo, hash_set &stmts) { gimple *def_stmt; enum vect_def_type dt; @@ -2907,37 +2908,44 @@ check_bool_pattern (tree var, vec_info *vinfo) if (!is_gimple_assign (def_stmt)) return false; - if (!has_single_use (var)) - return false; + if (stmts.contains (def_stmt)) + return true; rhs1 = gimple_assign_rhs1 (def_stmt); rhs_code = gimple_assign_rhs_code (def_stmt); switch (rhs_code) { case SSA_NAME: - return check_bool_pattern (rhs1, vinfo); + if (! check_bool_pattern (rhs1, vinfo, stmts)) + return false; + break; CASE_CONVERT: if ((TYPE_PRECISION (TREE_TYPE (rhs1)) != 1 || !TYPE_UNSIGNED (TREE_TYPE (rhs1))) && TREE_CODE (TREE_TYPE (rhs1)) != BOOLEAN_TYPE) return false; - return check_bool_pattern (rhs1, vinfo); + if (! check_bool_pattern (rhs1, vinfo, stmts)) + return false; + break; case BIT_NOT_EXPR: - return check_bool_pattern (rhs1, vinfo); + if (! check_bool_pattern (rhs1, vinfo, stmts)) + return false; + break; case BIT_AND_EXPR: case BIT_IOR_EXPR: case BIT_XOR_EXPR: - if (!check_bool_pattern (rhs1, vinfo)) + if (! check_bool_pattern (rhs1, vinfo, stmts) + || ! check_bool_pattern (gimple_assign_rhs2 (def_stmt), vinfo, stmts)) return false; - return check_bool_pattern (gimple_assign_rhs2 (def_stmt), vinfo); + break; default: if (TREE_CODE_CLASS (rhs_code) == tcc_comparison) { - tree vecitype, comp_vectype, mask_type; + tree vecitype, comp_vectype; /* If the comparison can throw, then is_gimple_condexpr will be false and we can't make a COND_EXPR/VEC_COND_EXPR out of it. */ @@ -2948,7 +2956,7 @@ check_bool_pattern (tree var, vec_info *vinfo) if (comp_vectype == NULL_TREE) return false; - mask_type = get_mask_type_for_scalar_type (TREE_TYPE (rhs1)); + tree mask_type = get_mask_type_for_scalar_type (TREE_TYPE (rhs1)); if (mask_type && expand_vec_cmp_expr_p (comp_vectype, mask_type)) return false; @@ -2964,50 +2972,54 @@ check_bool_pattern (tree var, vec_info *vinfo) } else vecitype = comp_vectype; - return expand_vec_cond_expr_p (vecitype, comp_vectype); + if (! expand_vec_cond_expr_p (vecitype, comp_vectype)) + return false; } - return false; + else + return false; + break; } + + bool res = stmts.add (def_stmt); + /* We can't end up recursing when just visiting SSA defs but not PHIs. */ + gcc_assert (!res); + + return true; } /* Helper function of adjust_bool_pattern. Add a cast to TYPE to a previous - stmt (SSA_NAME_DEF_STMT of VAR) by moving the COND_EXPR from RELATED_STMT - to PATTERN_DEF_SEQ and adding a cast as RELATED_STMT. */ + stmt (SSA_NAME_DEF_STMT of VAR) adding a cast to STMT_INFOs + pattern sequence. */ static tree -adjust_bool_pattern_cast (tree type, tree var) +adjust_bool_pattern_cast (tree type, tree var, stmt_vec_info stmt_info) { - stmt_vec_info stmt_vinfo = vinfo_for_stmt (SSA_NAME_DEF_STMT (var)); - gimple *cast_stmt, *pattern_stmt; - - gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo)); - pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo); - new_pattern_def_seq (stmt_vinfo, pattern_stmt); - cast_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type, NULL), - NOP_EXPR, gimple_assign_lhs (pattern_stmt)); - STMT_VINFO_RELATED_STMT (stmt_vinfo) = cast_stmt; + gimple *cast_stmt = gimple_build_assign (vect_recog_temp_ssa_var (type, NULL), + NOP_EXPR, var); + stmt_vec_info patt_vinfo = new_stmt_vec_info (cast_stmt, stmt_info->vinfo); + set_vinfo_for_stmt (cast_stmt, patt_vinfo); + STMT_VINFO_VECTYPE (patt_vinfo) = get_vectype_for_scalar_type (type); + append_pattern_def_seq (stmt_info, cast_stmt); return gimple_assign_lhs (cast_stmt); } +/* Helper function of vect_recog_bool_pattern. Do the actual transformations. + VAR is an SSA_NAME that should be transformed from bool to a wider integer + type, OUT_TYPE is the desired final integer type of the whole pattern. + STMT_INFO is the info of the pattern root and is where pattern stmts should + be associated with. DEFS is a map of pattern defs. */ -/* Helper function of vect_recog_bool_pattern. Do the actual transformations, - recursively. VAR is an SSA_NAME that should be transformed from bool - to a wider integer type, OUT_TYPE is the desired final integer type of - the whole pattern, TRUEVAL should be NULL unless optimizing - BIT_AND_EXPR into a COND_EXPR with one integer from one of the operands - in the then_clause, STMTS is where statements with added pattern stmts - should be pushed to. */ - -static tree -adjust_bool_pattern (tree var, tree out_type, tree trueval, - vec *stmts) +static void +adjust_bool_pattern (tree var, tree out_type, + stmt_vec_info stmt_info, hash_map &defs) { gimple *stmt = SSA_NAME_DEF_STMT (var); enum tree_code rhs_code, def_rhs_code; tree itype, cond_expr, rhs1, rhs2, irhs1, irhs2; location_t loc; gimple *pattern_stmt, *def_stmt; + tree trueval = NULL_TREE; rhs1 = gimple_assign_rhs1 (stmt); rhs2 = gimple_assign_rhs2 (stmt); @@ -3017,7 +3029,7 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval, { case SSA_NAME: CASE_CONVERT: - irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts); + irhs1 = *defs.get (rhs1); itype = TREE_TYPE (irhs1); pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL), @@ -3025,7 +3037,7 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval, break; case BIT_NOT_EXPR: - irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts); + irhs1 = *defs.get (rhs1); itype = TREE_TYPE (irhs1); pattern_stmt = gimple_build_assign (vect_recog_temp_ssa_var (itype, NULL), @@ -3070,57 +3082,45 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval, def_rhs_code = gimple_assign_rhs_code (def_stmt); if (TREE_CODE_CLASS (def_rhs_code) == tcc_comparison) { + irhs1 = *defs.get (rhs1); tree def_rhs1 = gimple_assign_rhs1 (def_stmt); - irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts); if (TYPE_PRECISION (TREE_TYPE (irhs1)) == GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (def_rhs1)))) { - gimple *tstmt; - stmt_vec_info stmt_def_vinfo = vinfo_for_stmt (def_stmt); - irhs2 = adjust_bool_pattern (rhs2, out_type, irhs1, stmts); - tstmt = stmts->pop (); - gcc_assert (tstmt == def_stmt); - stmts->quick_push (stmt); - STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)) - = STMT_VINFO_RELATED_STMT (stmt_def_vinfo); - gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo)); - STMT_VINFO_RELATED_STMT (stmt_def_vinfo) = NULL; - return irhs2; + rhs_code = def_rhs_code; + rhs1 = def_rhs1; + rhs2 = gimple_assign_rhs2 (def_stmt); + trueval = irhs1; + goto do_compare; } else - irhs2 = adjust_bool_pattern (rhs2, out_type, NULL_TREE, stmts); + irhs2 = *defs.get (rhs2); goto and_ior_xor; } def_stmt = SSA_NAME_DEF_STMT (rhs1); def_rhs_code = gimple_assign_rhs_code (def_stmt); if (TREE_CODE_CLASS (def_rhs_code) == tcc_comparison) { + irhs2 = *defs.get (rhs2); tree def_rhs1 = gimple_assign_rhs1 (def_stmt); - irhs2 = adjust_bool_pattern (rhs2, out_type, NULL_TREE, stmts); if (TYPE_PRECISION (TREE_TYPE (irhs2)) == GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (def_rhs1)))) { - gimple *tstmt; - stmt_vec_info stmt_def_vinfo = vinfo_for_stmt (def_stmt); - irhs1 = adjust_bool_pattern (rhs1, out_type, irhs2, stmts); - tstmt = stmts->pop (); - gcc_assert (tstmt == def_stmt); - stmts->quick_push (stmt); - STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)) - = STMT_VINFO_RELATED_STMT (stmt_def_vinfo); - gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo)); - STMT_VINFO_RELATED_STMT (stmt_def_vinfo) = NULL; - return irhs1; + rhs_code = def_rhs_code; + rhs1 = def_rhs1; + rhs2 = gimple_assign_rhs2 (def_stmt); + trueval = irhs2; + goto do_compare; } else - irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts); + irhs1 = *defs.get (rhs1); goto and_ior_xor; } /* FALLTHRU */ case BIT_IOR_EXPR: case BIT_XOR_EXPR: - irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts); - irhs2 = adjust_bool_pattern (rhs2, out_type, NULL_TREE, stmts); + irhs1 = *defs.get (rhs1); + irhs2 = *defs.get (rhs2); and_ior_xor: if (TYPE_PRECISION (TREE_TYPE (irhs1)) != TYPE_PRECISION (TREE_TYPE (irhs2))) @@ -3129,13 +3129,15 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval, int prec2 = TYPE_PRECISION (TREE_TYPE (irhs2)); int out_prec = TYPE_PRECISION (out_type); if (absu_hwi (out_prec - prec1) < absu_hwi (out_prec - prec2)) - irhs2 = adjust_bool_pattern_cast (TREE_TYPE (irhs1), rhs2); + irhs2 = adjust_bool_pattern_cast (TREE_TYPE (irhs1), irhs2, + stmt_info); else if (absu_hwi (out_prec - prec1) > absu_hwi (out_prec - prec2)) - irhs1 = adjust_bool_pattern_cast (TREE_TYPE (irhs2), rhs1); + irhs1 = adjust_bool_pattern_cast (TREE_TYPE (irhs2), irhs1, + stmt_info); else { - irhs1 = adjust_bool_pattern_cast (out_type, rhs1); - irhs2 = adjust_bool_pattern_cast (out_type, rhs2); + irhs1 = adjust_bool_pattern_cast (out_type, irhs1, stmt_info); + irhs2 = adjust_bool_pattern_cast (out_type, irhs2, stmt_info); } } itype = TREE_TYPE (irhs1); @@ -3145,6 +3147,7 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval, break; default: + do_compare: gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison); if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE || !TYPE_UNSIGNED (TREE_TYPE (rhs1)) @@ -3170,12 +3173,54 @@ adjust_bool_pattern (tree var, tree out_type, tree trueval, break; } - stmts->safe_push (stmt); gimple_set_location (pattern_stmt, loc); - STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)) = pattern_stmt; - return gimple_assign_lhs (pattern_stmt); + /* ??? Why does vect_mark_pattern_stmts set the vector type on all + pattern def seq stmts instead of just letting auto-detection do + its work? */ + stmt_vec_info patt_vinfo = new_stmt_vec_info (pattern_stmt, stmt_info->vinfo); + set_vinfo_for_stmt (pattern_stmt, patt_vinfo); + STMT_VINFO_VECTYPE (patt_vinfo) = get_vectype_for_scalar_type (itype); + append_pattern_def_seq (stmt_info, pattern_stmt); + defs.put (var, gimple_assign_lhs (pattern_stmt)); +} + +/* Comparison function to qsort a vector of gimple stmts after UID. */ + +static int +sort_after_uid (const void *p1, const void *p2) +{ + const gimple *stmt1 = *(const gimple * const *)p1; + const gimple *stmt2 = *(const gimple * const *)p2; + return gimple_uid (stmt1) - gimple_uid (stmt2); } +/* Create pattern stmts for all stmts participating in the bool pattern + specified by BOOL_STMT_SET and its root STMT with the desired type + OUT_TYPE. Return the def of the pattern root. */ + +static tree +adjust_bool_stmts (hash_set &bool_stmt_set, + tree out_type, gimple *stmt) +{ + /* Gather original stmts in the bool pattern in their order of appearance + in the IL. */ + auto_vec bool_stmts (bool_stmt_set.elements ()); + for (hash_set ::iterator i = bool_stmt_set.begin (); + i != bool_stmt_set.end (); ++i) + bool_stmts.quick_push (*i); + bool_stmts.qsort (sort_after_uid); + + /* Now process them in that order, producing pattern stmts. */ + hash_map defs; + for (unsigned i = 0; i < bool_stmts.length (); ++i) + adjust_bool_pattern (gimple_assign_lhs (bool_stmts[i]), + out_type, vinfo_for_stmt (stmt), defs); + + /* Pop the last pattern seq stmt and install it as pattern root for STMT. */ + gimple *pattern_stmt + = gimple_seq_last_stmt (STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (stmt))); + return gimple_assign_lhs (pattern_stmt); +} /* Return the proper type for converting bool VAR into an integer value or NULL_TREE if no such type exists. @@ -3338,6 +3383,8 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, && TREE_CODE (TREE_TYPE (var)) != BOOLEAN_TYPE) return NULL; + hash_set bool_stmts; + rhs_code = gimple_assign_rhs_code (last_stmt); if (CONVERT_EXPR_CODE_P (rhs_code)) { @@ -3348,9 +3395,9 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, if (vectype == NULL_TREE) return NULL; - if (check_bool_pattern (var, vinfo)) + if (check_bool_pattern (var, vinfo, bool_stmts)) { - rhs = adjust_bool_pattern (var, TREE_TYPE (lhs), NULL_TREE, stmts); + rhs = adjust_bool_stmts (bool_stmts, TREE_TYPE (lhs), last_stmt); lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); if (useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs))) pattern_stmt = gimple_build_assign (lhs, SSA_NAME, rhs); @@ -3420,10 +3467,10 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, if (get_vectype_for_scalar_type (type) == NULL_TREE) return NULL; - if (!check_bool_pattern (var, vinfo)) + if (!check_bool_pattern (var, vinfo, bool_stmts)) return NULL; - rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts); + rhs = adjust_bool_stmts (bool_stmts, type, last_stmt); lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL); pattern_stmt @@ -3450,9 +3497,8 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, if (!VECTOR_MODE_P (TYPE_MODE (vectype))) return NULL; - if (check_bool_pattern (var, vinfo)) - rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), - NULL_TREE, stmts); + if (check_bool_pattern (var, vinfo, bool_stmts)) + rhs = adjust_bool_stmts (bool_stmts, TREE_TYPE (vectype), last_stmt); else { tree type = search_type_for_mask (var, vinfo);