From b690cc0f331d0f63e2de4f0aa2e3883444cfead2 Mon Sep 17 00:00:00 2001 From: Richard Guenther Date: Fri, 9 Apr 2010 10:40:14 +0000 Subject: [PATCH] tree-vectorizer.h (struct _stmt_vec_info): Document that vectype is the type of the LHS. 2010-04-09 Richard Guenther * tree-vectorizer.h (struct _stmt_vec_info): Document that vectype is the type of the LHS. (supportable_widening_operation, supportable_narrowing_operation): Get both input and output vector types as arguments. (vect_is_simple_use_1): Declare. (get_same_sized_vectype): Likewise. * tree-vect-loop.c (vect_determine_vectorization_factor): Set STMT_VINFO_VECTYPE to the vector type of the def. (vectorizable_reduction): Adjust. * tree-vect-patterns.c (vect_recog_widen_mult_pattern): Adjust. Specify the output vector type. (vect_pattern_recog_1): Adjust. * tree-vect-stmts.c (get_same_sized_vectype): New function. (vectorizable_call): Adjust. (vectorizable_conversion): Likewise. (vectorizable_operation): Likewise. (vectorizable_type_demotion): Likewise. (vectorizable_type_promotion): Likewise. (vect_analyze_stmt): Set STMT_VINFO_VECTYPE to the vector type of the def. (vect_is_simple_use_1): New function. (supportable_widening_operation): Get both input and output vector types. (supportable_narrowing_operation): Likewise. * tree-vect-slp.c (vect_schedule_slp_instance): Adjust. From-SVN: r158157 --- gcc/ChangeLog | 28 ++++ gcc/tree-vect-loop.c | 105 ++++++++----- gcc/tree-vect-patterns.c | 32 ++-- gcc/tree-vect-slp.c | 2 +- gcc/tree-vect-stmts.c | 325 +++++++++++++++++++++++---------------- gcc/tree-vectorizer.h | 13 +- 6 files changed, 317 insertions(+), 188 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 37e496bc799..bdf0c5839ec 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,31 @@ +2010-04-09 Richard Guenther + + * tree-vectorizer.h (struct _stmt_vec_info): Document + that vectype is the type of the LHS. + (supportable_widening_operation, supportable_narrowing_operation): + Get both input and output vector types as arguments. + (vect_is_simple_use_1): Declare. + (get_same_sized_vectype): Likewise. + * tree-vect-loop.c (vect_determine_vectorization_factor): + Set STMT_VINFO_VECTYPE to the vector type of the def. + (vectorizable_reduction): Adjust. + * tree-vect-patterns.c (vect_recog_widen_mult_pattern): + Adjust. Specify the output vector type. + (vect_pattern_recog_1): Adjust. + * tree-vect-stmts.c (get_same_sized_vectype): New function. + (vectorizable_call): Adjust. + (vectorizable_conversion): Likewise. + (vectorizable_operation): Likewise. + (vectorizable_type_demotion): Likewise. + (vectorizable_type_promotion): Likewise. + (vect_analyze_stmt): Set STMT_VINFO_VECTYPE to the vector type of + the def. + (vect_is_simple_use_1): New function. + (supportable_widening_operation): Get both input and output + vector types. + (supportable_narrowing_operation): Likewise. + * tree-vect-slp.c (vect_schedule_slp_instance): Adjust. + 2010-04-09 Kai Tietz * config/i386/cygming.h (TARGET_OS_CPP_BUILTINS): Add diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index adce4ff18b4..c6547957341 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -240,6 +240,7 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { + tree vf_vectype; gimple stmt = gsi_stmt (si); stmt_info = vinfo_for_stmt (stmt); @@ -294,14 +295,12 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) gcc_assert (!STMT_VINFO_DATA_REF (stmt_info) && !is_pattern_stmt_p (stmt_info)); - scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, - &dummy); + scalar_type = TREE_TYPE (gimple_get_lhs (stmt)); if (vect_print_dump_info (REPORT_DETAILS)) { fprintf (vect_dump, "get vectype for scalar type: "); print_generic_expr (vect_dump, scalar_type, TDF_SLIM); } - vectype = get_vectype_for_scalar_type (scalar_type); if (!vectype) { @@ -313,23 +312,60 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) } return false; } + STMT_VINFO_VECTYPE (stmt_info) = vectype; } + /* The vectorization factor is according to the smallest + scalar type (or the largest vector size, but we only + support one vector size per loop). */ + scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, + &dummy); + if (vect_print_dump_info (REPORT_DETAILS)) + { + fprintf (vect_dump, "get vectype for scalar type: "); + print_generic_expr (vect_dump, scalar_type, TDF_SLIM); + } + vf_vectype = get_vectype_for_scalar_type (scalar_type); + if (!vf_vectype) + { + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) + { + fprintf (vect_dump, + "not vectorized: unsupported data-type "); + print_generic_expr (vect_dump, scalar_type, TDF_SLIM); + } + return false; + } + + if ((GET_MODE_SIZE (TYPE_MODE (vectype)) + != GET_MODE_SIZE (TYPE_MODE (vf_vectype)))) + { + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) + { + fprintf (vect_dump, + "not vectorized: different sized vector " + "types in statement, "); + print_generic_expr (vect_dump, vectype, TDF_SLIM); + fprintf (vect_dump, " and "); + print_generic_expr (vect_dump, vf_vectype, TDF_SLIM); + } + return false; + } + if (vect_print_dump_info (REPORT_DETAILS)) { fprintf (vect_dump, "vectype: "); - print_generic_expr (vect_dump, vectype, TDF_SLIM); + print_generic_expr (vect_dump, vf_vectype, TDF_SLIM); } - nunits = TYPE_VECTOR_SUBPARTS (vectype); + nunits = TYPE_VECTOR_SUBPARTS (vf_vectype); if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "nunits = %d", nunits); if (!vectorization_factor || (nunits > vectorization_factor)) vectorization_factor = nunits; - } } @@ -3446,7 +3482,8 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, tree scalar_dest; tree loop_vec_def0 = NULL_TREE, loop_vec_def1 = NULL_TREE; stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); + tree vectype_in = NULL_TREE; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); enum tree_code code, orig_code, epilog_reduc_code; @@ -3464,8 +3501,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, stmt_vec_info orig_stmt_info; tree expr = NULL_TREE; int i; - int nunits = TYPE_VECTOR_SUBPARTS (vectype); - int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits; + int ncopies; int epilog_copies; stmt_vec_info prev_stmt_info, prev_phi_info; gimple first_phi = NULL; @@ -3491,8 +3527,6 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, nested_cycle = true; } - gcc_assert (ncopies >= 1); - /* FORNOW: SLP not supported. */ if (STMT_SLP_TYPE (stmt_info)) return false; @@ -3579,12 +3613,16 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, reduction variable. */ for (i = 0; i < op_type-1; i++) { + tree tem; + /* The condition of COND_EXPR is checked in vectorizable_condition(). */ if (i == 0 && code == COND_EXPR) continue; - is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt, - &def, &dt); + is_simple_use = vect_is_simple_use_1 (ops[i], loop_vinfo, NULL, + &def_stmt, &def, &dt, &tem); + if (!vectype_in) + vectype_in = tem; gcc_assert (is_simple_use); if (dt != vect_internal_def && dt != vect_external_def @@ -3602,7 +3640,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, } is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt, - &def, &dt); + &def, &dt); gcc_assert (is_simple_use); gcc_assert (dt == vect_reduction_def || dt == vect_nested_cycle @@ -3625,7 +3663,12 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, if (STMT_VINFO_LIVE_P (vinfo_for_stmt (reduc_def_stmt))) return false; - vec_mode = TYPE_MODE (vectype); + + ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo) + / TYPE_VECTOR_SUBPARTS (vectype_in)); + gcc_assert (ncopies >= 1); + + vec_mode = TYPE_MODE (vectype_in); if (code == COND_EXPR) { @@ -3642,7 +3685,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, /* 4. Supportable by target? */ /* 4.1. check support for the operation in the loop */ - optab = optab_for_tree_code (code, vectype, optab_default); + optab = optab_for_tree_code (code, vectype_in, optab_default); if (!optab) { if (vect_print_dump_info (REPORT_DETAILS)) @@ -3666,7 +3709,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, } /* Worthwhile without SIMD support? */ - if (!VECTOR_MODE_P (TYPE_MODE (vectype)) + if (!VECTOR_MODE_P (TYPE_MODE (vectype_in)) && LOOP_VINFO_VECT_FACTOR (loop_vinfo) < vect_min_worthwhile_factor (code)) { @@ -3716,18 +3759,8 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, /* This is a reduction pattern: get the vectype from the type of the reduction variable, and get the tree-code from orig_stmt. */ orig_code = gimple_assign_rhs_code (orig_stmt); - vectype = get_vectype_for_scalar_type (TREE_TYPE (def)); - if (!vectype) - { - if (vect_print_dump_info (REPORT_DETAILS)) - { - fprintf (vect_dump, "unsupported data-type "); - print_generic_expr (vect_dump, TREE_TYPE (def), TDF_SLIM); - } - return false; - } - - vec_mode = TYPE_MODE (vectype); + gcc_assert (vectype_out); + vec_mode = TYPE_MODE (vectype_out); } else { @@ -3755,7 +3788,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, epilog_reduc_code = ERROR_MARK; if (reduction_code_for_scalar_code (orig_code, &epilog_reduc_code)) { - reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype, + reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype_out, optab_default); if (!reduc_optab) { @@ -3812,7 +3845,7 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, gcc_assert (ncopies == 1); /* Create the destination vector */ - vec_dest = vect_create_destination_var (scalar_dest, vectype); + vec_dest = vect_create_destination_var (scalar_dest, vectype_out); /* In case the vectorization factor (VF) is bigger than the number of elements that we can fit in a vectype (nunits), we have to generate @@ -3910,22 +3943,22 @@ vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi, if (op_type == binary_op) { if (reduc_index == 0) - expr = build2 (code, vectype, reduc_def, loop_vec_def0); + expr = build2 (code, vectype_out, reduc_def, loop_vec_def0); else - expr = build2 (code, vectype, loop_vec_def0, reduc_def); + expr = build2 (code, vectype_out, loop_vec_def0, reduc_def); } else { if (reduc_index == 0) - expr = build3 (code, vectype, reduc_def, loop_vec_def0, + expr = build3 (code, vectype_out, reduc_def, loop_vec_def0, loop_vec_def1); else { if (reduc_index == 1) - expr = build3 (code, vectype, loop_vec_def0, reduc_def, + expr = build3 (code, vectype_out, loop_vec_def0, reduc_def, loop_vec_def1); else - expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1, + expr = build3 (code, vectype_out, loop_vec_def0, loop_vec_def1, reduc_def); } } diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 37b06330f3a..55b9d50cca0 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -362,7 +362,7 @@ vect_recog_widen_mult_pattern (gimple last_stmt, tree oprnd0, oprnd1; tree type, half_type0, half_type1; gimple pattern_stmt; - tree vectype; + tree vectype, vectype_out; tree dummy; tree var; enum tree_code dummy_code; @@ -405,14 +405,16 @@ vect_recog_widen_mult_pattern (gimple last_stmt, /* Check target support */ vectype = get_vectype_for_scalar_type (half_type0); + vectype_out = get_vectype_for_scalar_type (type); if (!vectype - || !supportable_widening_operation (WIDEN_MULT_EXPR, last_stmt, vectype, + || !supportable_widening_operation (WIDEN_MULT_EXPR, last_stmt, + vectype_out, vectype, &dummy, &dummy, &dummy_code, &dummy_code, &dummy_int, &dummy_vec)) return NULL; *type_in = vectype; - *type_out = NULL_TREE; + *type_out = vectype_out; /* Pattern supported. Create a stmt to be used to replace the pattern: */ var = vect_recog_temp_ssa_var (type, NULL); @@ -677,7 +679,9 @@ vect_pattern_recog_1 ( { /* No need to check target support (already checked by the pattern recognition function). */ - pattern_vectype = type_in; + if (type_out) + gcc_assert (VECTOR_MODE_P (TYPE_MODE (type_out))); + pattern_vectype = type_out ? type_out : type_in; } else { @@ -686,9 +690,14 @@ vect_pattern_recog_1 ( optab optab; /* Check target support */ - pattern_vectype = get_vectype_for_scalar_type (type_in); - if (!pattern_vectype) - return; + type_in = get_vectype_for_scalar_type (type_in); + if (!type_in) + return; + if (type_out) + type_out = get_vectype_for_scalar_type (type_out); + else + type_out = type_in; + pattern_vectype = type_out; if (is_gimple_assign (pattern_stmt)) code = gimple_assign_rhs_code (pattern_stmt); @@ -698,15 +707,12 @@ vect_pattern_recog_1 ( code = CALL_EXPR; } - optab = optab_for_tree_code (code, pattern_vectype, optab_default); - vec_mode = TYPE_MODE (pattern_vectype); + optab = optab_for_tree_code (code, type_in, optab_default); + vec_mode = TYPE_MODE (type_in); if (!optab || (icode = optab_handler (optab, vec_mode)->insn_code) == CODE_FOR_nothing - || (type_out - && (!get_vectype_for_scalar_type (type_out) - || (insn_data[icode].operand[0].mode != - TYPE_MODE (get_vectype_for_scalar_type (type_out)))))) + || (insn_data[icode].operand[0].mode != TYPE_MODE (type_out))) return; } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 3f97d620910..0e4f6ce6151 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -1971,7 +1971,7 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance, stmt_info = vinfo_for_stmt (stmt); /* VECTYPE is the type of the destination. */ - vectype = get_vectype_for_scalar_type (TREE_TYPE (gimple_assign_lhs (stmt))); + vectype = STMT_VINFO_VECTYPE (stmt_info); nunits = (unsigned int) TYPE_VECTOR_SUBPARTS (vectype); group_size = SLP_INSTANCE_GROUP_SIZE (instance); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 48bf1bdd54a..26b9ca20520 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1223,7 +1223,7 @@ vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt) int nunits_in; int nunits_out; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); - tree fndecl, new_temp, def, rhs_type, lhs_type; + tree fndecl, new_temp, def, rhs_type; gimple def_stmt; enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type}; gimple new_stmt = NULL; @@ -1252,8 +1252,11 @@ vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt) if (TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME) return false; + vectype_out = STMT_VINFO_VECTYPE (stmt_info); + /* Process function arguments. */ rhs_type = NULL_TREE; + vectype_in = NULL_TREE; nargs = gimple_call_num_args (stmt); /* Bail out if the function has more than two arguments, we @@ -1264,6 +1267,8 @@ vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt) for (i = 0; i < nargs; i++) { + tree opvectype; + op = gimple_call_arg (stmt, i); /* We can only handle calls with arguments of the same type. */ @@ -1274,28 +1279,35 @@ vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt) fprintf (vect_dump, "argument types differ."); return false; } - rhs_type = TREE_TYPE (op); + if (!rhs_type) + rhs_type = TREE_TYPE (op); - if (!vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def, &dt[i])) + if (!vect_is_simple_use_1 (op, loop_vinfo, NULL, + &def_stmt, &def, &dt[i], &opvectype)) { if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "use not simple."); return false; } - } - vectype_in = get_vectype_for_scalar_type (rhs_type); + if (!vectype_in) + vectype_in = opvectype; + else if (opvectype + && opvectype != vectype_in) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "argument vector types differ."); + return false; + } + } + /* If all arguments are external or constant defs use a vector type with + the same size as the output vector type. */ if (!vectype_in) - return false; - nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); - - lhs_type = TREE_TYPE (gimple_call_lhs (stmt)); - vectype_out = get_vectype_for_scalar_type (lhs_type); - if (!vectype_out) - return false; - nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); + vectype_in = get_same_sized_vectype (rhs_type, vectype_out); /* FORNOW */ + nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); + nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); if (nunits_in == nunits_out / 2) modifier = NARROW; else if (nunits_out == nunits_in) @@ -1546,7 +1558,7 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, int nunits_out; tree vectype_out, vectype_in; int ncopies, j; - tree rhs_type, lhs_type; + tree rhs_type; tree builtin_decl; enum { NARROW, NONE, WIDEN } modifier; int i; @@ -1578,21 +1590,27 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, return false; /* Check types of lhs and rhs. */ + scalar_dest = gimple_assign_lhs (stmt); + vectype_out = STMT_VINFO_VECTYPE (stmt_info); + op0 = gimple_assign_rhs1 (stmt); rhs_type = TREE_TYPE (op0); - vectype_in = get_vectype_for_scalar_type (rhs_type); + /* Check the operands of the operation. */ + if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL, + &def_stmt, &def, &dt[0], &vectype_in)) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "use not simple."); + return false; + } + /* If op0 is an external or constant defs use a vector type of + the same size as the output vector type. */ if (!vectype_in) - return false; - nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); - - scalar_dest = gimple_assign_lhs (stmt); - lhs_type = TREE_TYPE (scalar_dest); - vectype_out = get_vectype_for_scalar_type (lhs_type); - if (!vectype_out) - return false; - nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); + vectype_in = get_same_sized_vectype (rhs_type, vectype_out); /* FORNOW */ + nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); + nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); if (nunits_in == nunits_out / 2) modifier = NARROW; else if (nunits_out == nunits_in) @@ -1602,14 +1620,6 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, else return false; - if (modifier == NONE) - gcc_assert (STMT_VINFO_VECTYPE (stmt_info) == vectype_out); - - /* Bail out if the types are both integral or non-integral. */ - if ((INTEGRAL_TYPE_P (rhs_type) && INTEGRAL_TYPE_P (lhs_type)) - || (!INTEGRAL_TYPE_P (rhs_type) && !INTEGRAL_TYPE_P (lhs_type))) - return false; - integral_type = INTEGRAL_TYPE_P (rhs_type) ? vectype_in : vectype_out; if (modifier == NARROW) @@ -1626,24 +1636,17 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, needs to be generated. */ gcc_assert (ncopies >= 1); - /* Check the operands of the operation. */ - if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0])) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "use not simple."); - return false; - } - /* Supportable by target? */ if ((modifier == NONE && !targetm.vectorize.builtin_conversion (code, integral_type)) || (modifier == WIDEN - && !supportable_widening_operation (code, stmt, vectype_in, + && !supportable_widening_operation (code, stmt, + vectype_out, vectype_in, &decl1, &decl2, &code1, &code2, &dummy_int, &dummy)) || (modifier == NARROW - && !supportable_narrowing_operation (code, stmt, vectype_in, + && !supportable_narrowing_operation (code, vectype_out, vectype_in, &code1, &dummy_int, &dummy))) { if (vect_print_dump_info (REPORT_DETAILS)) @@ -1653,7 +1656,6 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, if (modifier != NONE) { - STMT_VINFO_VECTYPE (stmt_info) = vectype_in; /* FORNOW: SLP not supported. */ if (STMT_SLP_TYPE (stmt_info)) return false; @@ -1719,8 +1721,6 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, else vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0); - STMT_VINFO_VECTYPE (stmt_info) = vectype_in; - /* Generate first half of the widened result: */ new_stmt = vect_gen_widened_results_half (code1, decl1, @@ -1918,7 +1918,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, tree op0, op1 = NULL; tree vec_oprnd1 = NULL_TREE; stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); enum tree_code code; enum machine_mode vec_mode; @@ -1932,7 +1932,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type}; gimple new_stmt = NULL; stmt_vec_info prev_stmt_info; - int nunits_in = TYPE_VECTOR_SUBPARTS (vectype); + int nunits_in; int nunits_out; tree vectype_out; int ncopies; @@ -1944,21 +1944,6 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); int vf; - if (loop_vinfo) - vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); - else - vf = 1; - - /* Multiple types in SLP are handled by creating the appropriate number of - vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in - case of SLP. */ - if (slp_node) - ncopies = 1; - else - ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in; - - gcc_assert (ncopies >= 1); - if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) return false; @@ -1972,14 +1957,6 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME) return false; - scalar_dest = gimple_assign_lhs (stmt); - vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest)); - if (!vectype_out) - return false; - nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); - if (nunits_out != nunits_in) - return false; - code = gimple_assign_rhs_code (stmt); /* For pointer addition, we should use the normal plus for @@ -1996,13 +1973,27 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, return false; } + scalar_dest = gimple_assign_lhs (stmt); + vectype_out = STMT_VINFO_VECTYPE (stmt_info); + op0 = gimple_assign_rhs1 (stmt); - if (!vect_is_simple_use (op0, loop_vinfo, bb_vinfo, &def_stmt, &def, &dt[0])) + if (!vect_is_simple_use_1 (op0, loop_vinfo, bb_vinfo, + &def_stmt, &def, &dt[0], &vectype)) { if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "use not simple."); return false; } + /* If op0 is an external or constant def use a vector type with + the same size as the output vector type. */ + if (!vectype) + vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out); + gcc_assert (vectype); + + nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); + nunits_in = TYPE_VECTOR_SUBPARTS (vectype); + if (nunits_out != nunits_in) + return false; if (op_type == binary_op) { @@ -2016,6 +2007,21 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, } } + if (loop_vinfo) + vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + else + vf = 1; + + /* Multiple types in SLP are handled by creating the appropriate number of + vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in + case of SLP. */ + if (slp_node) + ncopies = 1; + else + ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in; + + gcc_assert (ncopies >= 1); + /* If this is a shift/rotate, determine whether the shift amount is a vector, or scalar. If the shift/rotate amount is a vector, use the vector/vector shift optabs. */ @@ -2423,16 +2429,32 @@ vectorizable_type_demotion (gimple stmt, gimple_stmt_iterator *gsi, if (!CONVERT_EXPR_CODE_P (code)) return false; + scalar_dest = gimple_assign_lhs (stmt); + vectype_out = STMT_VINFO_VECTYPE (stmt_info); + + /* Check the operands of the operation. */ op0 = gimple_assign_rhs1 (stmt); - vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op0)); + if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest)) + && INTEGRAL_TYPE_P (TREE_TYPE (op0))) + || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest)) + && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0)) + && CONVERT_EXPR_CODE_P (code)))) + return false; + if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL, + &def_stmt, &def, &dt[0], &vectype_in)) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "use not simple."); + return false; + } + /* If op0 is an external def use a vector type with the + same size as the output vector type if possible. */ + if (!vectype_in) + vectype_in = get_same_sized_vectype (TREE_TYPE (op0), vectype_out); if (!vectype_in) return false; - nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); - scalar_dest = gimple_assign_lhs (stmt); - vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest)); - if (!vectype_out) - return false; + nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); if (nunits_in >= nunits_out) return false; @@ -2446,28 +2468,11 @@ vectorizable_type_demotion (gimple stmt, gimple_stmt_iterator *gsi, ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out; gcc_assert (ncopies >= 1); - if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest)) - && INTEGRAL_TYPE_P (TREE_TYPE (op0))) - || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest)) - && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0)) - && CONVERT_EXPR_CODE_P (code)))) - return false; - - /* Check the operands of the operation. */ - if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0])) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "use not simple."); - return false; - } - /* Supportable by target? */ - if (!supportable_narrowing_operation (code, stmt, vectype_in, &code1, - &multi_step_cvt, &interm_types)) + if (!supportable_narrowing_operation (code, vectype_out, vectype_in, + &code1, &multi_step_cvt, &interm_types)) return false; - STMT_VINFO_VECTYPE (stmt_info) = vectype_in; - if (!vec_stmt) /* transformation not required. */ { STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type; @@ -2693,16 +2698,32 @@ vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi, && code != WIDEN_MULT_EXPR) return false; + scalar_dest = gimple_assign_lhs (stmt); + vectype_out = STMT_VINFO_VECTYPE (stmt_info); + + /* Check the operands of the operation. */ op0 = gimple_assign_rhs1 (stmt); - vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op0)); + if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest)) + && INTEGRAL_TYPE_P (TREE_TYPE (op0))) + || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest)) + && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0)) + && CONVERT_EXPR_CODE_P (code)))) + return false; + if (!vect_is_simple_use_1 (op0, loop_vinfo, NULL, + &def_stmt, &def, &dt[0], &vectype_in)) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "use not simple."); + return false; + } + /* If op0 is an external or constant def use a vector type with + the same size as the output vector type. */ + if (!vectype_in) + vectype_in = get_same_sized_vectype (TREE_TYPE (op0), vectype_out); if (!vectype_in) return false; - nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); - scalar_dest = gimple_assign_lhs (stmt); - vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest)); - if (!vectype_out) - return false; + nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in); nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); if (nunits_in <= nunits_out) return false; @@ -2717,21 +2738,6 @@ vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi, gcc_assert (ncopies >= 1); - if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest)) - && INTEGRAL_TYPE_P (TREE_TYPE (op0))) - || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest)) - && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0)) - && CONVERT_EXPR_CODE_P (code)))) - return false; - - /* Check the operands of the operation. */ - if (!vect_is_simple_use (op0, loop_vinfo, NULL, &def_stmt, &def, &dt[0])) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "use not simple."); - return false; - } - op_type = TREE_CODE_LENGTH (code); if (op_type == binary_op) { @@ -2745,7 +2751,7 @@ vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi, } /* Supportable by target? */ - if (!supportable_widening_operation (code, stmt, vectype_in, + if (!supportable_widening_operation (code, stmt, vectype_out, vectype_in, &decl1, &decl2, &code1, &code2, &multi_step_cvt, &interm_types)) return false; @@ -2754,8 +2760,6 @@ vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi, architecture. */ gcc_assert (!(multi_step_cvt && op_type == binary_op)); - STMT_VINFO_VECTYPE (stmt_info) = vectype_in; - if (!vec_stmt) /* transformation not required. */ { STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type; @@ -3944,7 +3948,6 @@ vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node) bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); bool ok; - HOST_WIDE_INT dummy; tree scalar_type, vectype; if (vect_print_dump_info (REPORT_DETAILS)) @@ -4002,7 +4005,7 @@ vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node) { gcc_assert (PURE_SLP_STMT (stmt_info)); - scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy); + scalar_type = TREE_TYPE (gimple_get_lhs (stmt)); if (vect_print_dump_info (REPORT_DETAILS)) { fprintf (vect_dump, "get vectype for scalar type: "); @@ -4455,6 +4458,17 @@ get_vectype_for_scalar_type (tree scalar_type) return vectype; } +/* Function get_same_sized_vectype + + Returns a vector type corresponding to SCALAR_TYPE of size + VECTOR_TYPE if supported by the target. */ + +tree +get_same_sized_vectype (tree scalar_type, tree vector_type ATTRIBUTE_UNUSED) +{ + return get_vectype_for_scalar_type (scalar_type); +} + /* Function vect_is_simple_use. Input: @@ -4588,12 +4602,55 @@ vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, return true; } +/* Function vect_is_simple_use_1. + + Same as vect_is_simple_use_1 but also determines the vector operand + type of OPERAND and stores it to *VECTYPE. If the definition of + OPERAND is vect_uninitialized_def, vect_constant_def or + vect_external_def *VECTYPE will be set to NULL_TREE and the caller + is responsible to compute the best suited vector type for the + scalar operand. */ + +bool +vect_is_simple_use_1 (tree operand, loop_vec_info loop_vinfo, + bb_vec_info bb_vinfo, gimple *def_stmt, + tree *def, enum vect_def_type *dt, tree *vectype) +{ + if (!vect_is_simple_use (operand, loop_vinfo, bb_vinfo, def_stmt, def, dt)) + return false; + + /* Now get a vector type if the def is internal, otherwise supply + NULL_TREE and leave it up to the caller to figure out a proper + type for the use stmt. */ + if (*dt == vect_internal_def + || *dt == vect_induction_def + || *dt == vect_reduction_def + || *dt == vect_double_reduction_def + || *dt == vect_nested_cycle) + { + stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt); + if (STMT_VINFO_IN_PATTERN_P (stmt_info)) + stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); + *vectype = STMT_VINFO_VECTYPE (stmt_info); + gcc_assert (*vectype != NULL_TREE); + } + else if (*dt == vect_uninitialized_def + || *dt == vect_constant_def + || *dt == vect_external_def) + *vectype = NULL_TREE; + else + gcc_unreachable (); + + return true; +} + /* Function supportable_widening_operation Check whether an operation represented by the code CODE is a widening operation that is supported by the target platform in - vector form (i.e., when operating on arguments of type VECTYPE). + vector form (i.e., when operating on arguments of type VECTYPE_IN + producing a result of type VECTYPE_OUT). Widening operations we currently support are NOP (CONVERT), FLOAT and WIDEN_MULT. This function checks if these operations are supported @@ -4613,7 +4670,8 @@ vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, widening operation (short in the above example). */ bool -supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype, +supportable_widening_operation (enum tree_code code, gimple stmt, + tree vectype_out, tree vectype_in, tree *decl1, tree *decl2, enum tree_code *code1, enum tree_code *code2, int *multi_step_cvt, @@ -4626,8 +4684,8 @@ supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype, enum machine_mode vec_mode; enum insn_code icode1, icode2; optab optab1, optab2; - tree type = gimple_expr_type (stmt); - tree wide_vectype = get_vectype_for_scalar_type (type); + tree vectype = vectype_in; + tree wide_vectype = vectype_out; enum tree_code c1, c2; /* The result of a vectorized widening operation usually requires two vectors @@ -4726,8 +4784,8 @@ supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype, if (code == FIX_TRUNC_EXPR) { /* The signedness is determined from output operand. */ - optab1 = optab_for_tree_code (c1, type, optab_default); - optab2 = optab_for_tree_code (c2, type, optab_default); + optab1 = optab_for_tree_code (c1, vectype_out, optab_default); + optab2 = optab_for_tree_code (c2, vectype_out, optab_default); } else { @@ -4809,7 +4867,8 @@ supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype, Check whether an operation represented by the code CODE is a narrowing operation that is supported by the target platform in - vector form (i.e., when operating on arguments of type VECTYPE). + vector form (i.e., when operating on arguments of type VECTYPE_IN + and producing a result of type VECTYPE_OUT). Narrowing operations we currently support are NOP (CONVERT) and FIX_TRUNC. This function checks if these operations are supported by @@ -4826,15 +4885,15 @@ supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype, bool supportable_narrowing_operation (enum tree_code code, - const_gimple stmt, tree vectype, + tree vectype_out, tree vectype_in, enum tree_code *code1, int *multi_step_cvt, VEC (tree, heap) **interm_types) { enum machine_mode vec_mode; enum insn_code icode1; optab optab1, interm_optab; - tree type = gimple_expr_type (stmt); - tree narrow_vectype = get_vectype_for_scalar_type (type); + tree vectype = vectype_in; + tree narrow_vectype = vectype_out; enum tree_code c1; tree intermediate_type, prev_type; int i; @@ -4860,7 +4919,7 @@ supportable_narrowing_operation (enum tree_code code, if (code == FIX_TRUNC_EXPR) /* The signedness is determined from output operand. */ - optab1 = optab_for_tree_code (c1, type, optab_default); + optab1 = optab_for_tree_code (c1, vectype_out, optab_default); else optab1 = optab_for_tree_code (c1, vectype, optab_default); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 53a15e131cf..b46bc52c25b 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -409,7 +409,7 @@ typedef struct _stmt_vec_info { used outside the loop. */ bool live; - /* The vector type to be used. */ + /* The vector type to be used for the LHS of this statement. */ tree vectype; /* The vectorized version of the stmt. */ @@ -760,15 +760,18 @@ extern bool vect_can_advance_ivs_p (loop_vec_info); /* In tree-vect-stmts.c. */ extern tree get_vectype_for_scalar_type (tree); +extern tree get_same_sized_vectype (tree, tree); extern bool vect_is_simple_use (tree, loop_vec_info, bb_vec_info, gimple *, tree *, enum vect_def_type *); -extern bool supportable_widening_operation (enum tree_code, gimple, tree, +extern bool vect_is_simple_use_1 (tree, loop_vec_info, bb_vec_info, gimple *, + tree *, enum vect_def_type *, tree *); +extern bool supportable_widening_operation (enum tree_code, gimple, tree, tree, tree *, tree *, enum tree_code *, enum tree_code *, int *, VEC (tree, heap) **); -extern bool supportable_narrowing_operation (enum tree_code, const_gimple, - tree, enum tree_code *, int *, - VEC (tree, heap) **); +extern bool supportable_narrowing_operation (enum tree_code, tree, tree, + enum tree_code *, + int *, VEC (tree, heap) **); extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info, bb_vec_info); extern void free_stmt_vec_info (gimple stmt); -- 2.30.2