#include "tm.h"
#include "tree.h"
#include "stor-layout.h"
+#include "predict.h"
+#include "vec.h"
+#include "hashtab.h"
+#include "hash-set.h"
+#include "machmode.h"
+#include "hard-reg-set.h"
+#include "input.h"
+#include "function.h"
+#include "dominance.h"
+#include "cfg.h"
+#include "cfganal.h"
#include "basic-block.h"
#include "gimple-pretty-print.h"
#include "tree-ssa-alias.h"
#include "cfgloop.h"
#include "expr.h"
#include "recog.h"
+#include "insn-codes.h"
#include "optabs.h"
#include "params.h"
#include "diagnostic-core.h"
LOOP_VINFO_REDUCTIONS (loop_vinfo).release ();
LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo).release ();
- if (LOOP_VINFO_PEELING_HTAB (loop_vinfo).is_created ())
- LOOP_VINFO_PEELING_HTAB (loop_vinfo).dispose ();
+ delete LOOP_VINFO_PEELING_HTAB (loop_vinfo);
+ LOOP_VINFO_PEELING_HTAB (loop_vinfo) = NULL;
destroy_cost_data (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo));
Output:
REDUC_CODE - the corresponding tree-code to be used to reduce the
- vector of partial results into a single scalar result (which
- will also reside in a vector) or ERROR_MARK if the operation is
- a supported reduction operation, but does not have such tree-code.
+ vector of partial results into a single scalar result, or ERROR_MARK
+ if the operation is a supported reduction operation, but does not have
+ such a tree-code.
Return FALSE if CODE currently cannot be vectorized as reduction. */
}
def1 = SSA_NAME_DEF_STMT (op1);
- if (flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ if (gimple_bb (def1)
+ && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
&& loop->inner
&& flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
&& is_gimple_assign (def1))
*ret_min_profitable_estimate = min_profitable_estimate;
}
+/* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
+ vector elements (not bits) for a vector of mode MODE. */
+static void
+calc_vec_perm_mask_for_shift (enum machine_mode mode, unsigned int offset,
+ unsigned char *sel)
+{
+ unsigned int i, nelt = GET_MODE_NUNITS (mode);
+
+ for (i = 0; i < nelt; i++)
+ sel[i] = (i + offset) & (2*nelt - 1);
+}
+
+/* Checks whether the target supports whole-vector shifts for vectors of mode
+ MODE. This is the case if _either_ the platform handles vec_shr_optab, _or_
+ it supports vec_perm_const with masks for all necessary shift amounts. */
+static bool
+have_whole_vector_shift (enum machine_mode mode)
+{
+ if (optab_handler (vec_shr_optab, mode) != CODE_FOR_nothing)
+ return true;
+
+ if (direct_optab_handler (vec_perm_const_optab, mode) == CODE_FOR_nothing)
+ return false;
+
+ unsigned int i, nelt = GET_MODE_NUNITS (mode);
+ unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
+
+ for (i = nelt/2; i >= 1; i/=2)
+ {
+ calc_vec_perm_mask_for_shift (mode, i, sel);
+ if (!can_vec_perm_p (mode, false, sel))
+ return false;
+ }
+ return true;
+}
/* TODO: Close dependency between vect_model_*_cost and vectorizable_*
functions. Design better to avoid maintenance issues. */
tree vectype;
gimple stmt, orig_stmt;
tree reduction_op;
- enum machine_mode mode;
+ machine_mode mode;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
void *target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
/* We have a whole vector shift available. */
if (VECTOR_MODE_P (mode)
&& optab_handler (optab, mode) != CODE_FOR_nothing
- && optab_handler (vec_shr_optab, mode) != CODE_FOR_nothing)
+ && have_whole_vector_shift (mode))
{
/* Final reduction via vector shifts and the reduction operator.
Also requires scalar extract. */
{
case WIDEN_SUM_EXPR:
case DOT_PROD_EXPR:
+ case SAD_EXPR:
case PLUS_EXPR:
case MINUS_EXPR:
case BIT_IOR_EXPR:
return init_def;
}
-
/* Function vect_create_epilog_for_reduction
Create code at the loop-epilog to finalize the result of a reduction
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info prev_phi_info;
tree vectype;
- enum machine_mode mode;
+ machine_mode mode;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *outer_loop = NULL;
basic_block exit_bb;
gimple epilog_stmt = NULL;
enum tree_code code = gimple_assign_rhs_code (stmt);
gimple exit_phi;
- tree bitsize, bitpos;
+ tree bitsize;
tree adjustment_def = NULL;
tree vec_initial_def = NULL;
tree reduction_op, expr, def;
tree orig_name, scalar_result;
imm_use_iterator imm_iter, phi_imm_iter;
use_operand_p use_p, phi_use_p;
- bool extract_scalar_result = false;
gimple use_stmt, orig_stmt, reduction_phi = NULL;
bool nested_in_vect_loop = false;
auto_vec<gimple> new_phis;
/* Set phi nodes arguments. */
FOR_EACH_VEC_ELT (reduction_phis, i, phi)
{
- tree vec_init_def = vec_initial_defs[i];
- tree def = vect_defs[i];
+ tree vec_init_def, def;
+ gimple_seq stmts;
+ vec_init_def = force_gimple_operand (vec_initial_defs[i], &stmts,
+ true, NULL_TREE);
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ def = vect_defs[i];
for (j = 0; j < ncopies; j++)
{
/* Set the loop-entry arg of the reduction-phi. */
if (reduc_code != ERROR_MARK && !slp_reduc)
{
tree tmp;
+ tree vec_elem_type;
/*** Case 1: Create:
v_out2 = reduc_expr <v_out1> */
dump_printf_loc (MSG_NOTE, vect_location,
"Reduce using direct vector reduction.\n");
- vec_dest = vect_create_destination_var (scalar_dest, vectype);
- tmp = build1 (reduc_code, vectype, new_phi_result);
- epilog_stmt = gimple_build_assign (vec_dest, tmp);
- new_temp = make_ssa_name (vec_dest, epilog_stmt);
+ vec_elem_type = TREE_TYPE (TREE_TYPE (new_phi_result));
+ if (!useless_type_conversion_p (scalar_type, vec_elem_type))
+ {
+ tree tmp_dest =
+ vect_create_destination_var (scalar_dest, vec_elem_type);
+ tmp = build1 (reduc_code, vec_elem_type, new_phi_result);
+ epilog_stmt = gimple_build_assign (tmp_dest, tmp);
+ new_temp = make_ssa_name (tmp_dest, epilog_stmt);
+ gimple_assign_set_lhs (epilog_stmt, new_temp);
+ gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
+
+ tmp = build1 (NOP_EXPR, scalar_type, new_temp);
+ }
+ else
+ tmp = build1 (reduc_code, scalar_type, new_phi_result);
+ epilog_stmt = gimple_build_assign (new_scalar_dest, tmp);
+ new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
gimple_assign_set_lhs (epilog_stmt, new_temp);
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
-
- extract_scalar_result = true;
+ scalar_results.safe_push (new_temp);
}
else
{
- enum tree_code shift_code = ERROR_MARK;
- bool have_whole_vector_shift = true;
- int bit_offset;
+ bool reduce_with_shift = have_whole_vector_shift (mode);
int element_bitsize = tree_to_uhwi (bitsize);
int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
tree vec_temp;
- if (optab_handler (vec_shr_optab, mode) != CODE_FOR_nothing)
- shift_code = VEC_RSHIFT_EXPR;
- else
- have_whole_vector_shift = false;
-
/* Regardless of whether we have a whole vector shift, if we're
emulating the operation via tree-vect-generic, we don't want
to use it. Only the first round of the reduction is likely
/* ??? It might be better to emit a reduction tree code here, so that
tree-vect-generic can expand the first round via bit tricks. */
if (!VECTOR_MODE_P (mode))
- have_whole_vector_shift = false;
+ reduce_with_shift = false;
else
{
optab optab = optab_for_tree_code (code, vectype, optab_default);
if (optab_handler (optab, mode) == CODE_FOR_nothing)
- have_whole_vector_shift = false;
+ reduce_with_shift = false;
}
- if (have_whole_vector_shift && !slp_reduc)
+ if (reduce_with_shift && !slp_reduc)
{
+ int nelements = vec_size_in_bits / element_bitsize;
+ unsigned char *sel = XALLOCAVEC (unsigned char, nelements);
+
+ int elt_offset;
+
+ tree zero_vec = build_zero_cst (vectype);
/*** Case 2: Create:
- for (offset = VS/2; offset >= element_size; offset/=2)
+ for (offset = nelements/2; offset >= 1; offset/=2)
{
Create: va' = vec_shift <va, offset>
Create: va = vop <va, va'>
} */
+ tree rhs;
+
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"Reduce using vector shifts\n");
vec_dest = vect_create_destination_var (scalar_dest, vectype);
new_temp = new_phi_result;
- for (bit_offset = vec_size_in_bits/2;
- bit_offset >= element_bitsize;
- bit_offset /= 2)
+ for (elt_offset = nelements / 2;
+ elt_offset >= 1;
+ elt_offset /= 2)
{
- tree bitpos = size_int (bit_offset);
-
- epilog_stmt = gimple_build_assign_with_ops (shift_code,
- vec_dest, new_temp, bitpos);
+ calc_vec_perm_mask_for_shift (mode, elt_offset, sel);
+ tree mask = vect_gen_perm_mask_any (vectype, sel);
+ epilog_stmt = gimple_build_assign_with_ops (VEC_PERM_EXPR,
+ vec_dest, new_temp,
+ zero_vec, mask);
new_name = make_ssa_name (vec_dest, epilog_stmt);
gimple_assign_set_lhs (epilog_stmt, new_name);
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
}
- extract_scalar_result = true;
+ /* 2.4 Extract the final scalar result. Create:
+ s_out3 = extract_field <v_out2, bitpos> */
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "extract scalar result\n");
+
+ rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp,
+ bitsize, bitsize_zero_node);
+ epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
+ new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
+ gimple_assign_set_lhs (epilog_stmt, new_temp);
+ gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
+ scalar_results.safe_push (new_temp);
}
else
{
- tree rhs;
-
/*** Case 3: Create:
s = extract_field <v_out2, 0>
for (offset = element_size;
vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype));
FOR_EACH_VEC_ELT (new_phis, i, new_phi)
{
+ int bit_offset;
if (gimple_code (new_phi) == GIMPLE_PHI)
vec_temp = PHI_RESULT (new_phi);
else
vec_temp = gimple_assign_lhs (new_phi);
- rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
+ tree rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
bitsize_zero_node);
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
else
/* Not SLP - we have one scalar to keep in SCALAR_RESULTS. */
scalar_results.safe_push (new_temp);
-
- extract_scalar_result = false;
}
}
-
- /* 2.4 Extract the final scalar result. Create:
- s_out3 = extract_field <v_out2, bitpos> */
-
- if (extract_scalar_result)
- {
- tree rhs;
-
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "extract scalar result\n");
-
- if (BYTES_BIG_ENDIAN)
- bitpos = size_binop (MULT_EXPR,
- bitsize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1),
- TYPE_SIZE (scalar_type));
- else
- bitpos = bitsize_zero_node;
-
- rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp, bitsize, bitpos);
- epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
- new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- gimple_assign_set_lhs (epilog_stmt, new_temp);
- gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
- scalar_results.safe_push (new_temp);
- }
vect_finalize_reduction:
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum tree_code code, orig_code, epilog_reduc_code;
- enum machine_mode vec_mode;
+ machine_mode vec_mode;
int op_type;
optab optab, reduc_optab;
tree new_temp = NULL_TREE;
epilog_reduc_code = ERROR_MARK;
}
-
- if (reduc_optab
- && optab_handler (reduc_optab, vec_mode) == CODE_FOR_nothing)
+ else if (optab_handler (reduc_optab, vec_mode) == CODE_FOR_nothing)
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "reduc op not supported by target.\n");
+ optab = scalar_reduc_to_vector (reduc_optab, vectype_out);
+ if (optab_handler (optab, vec_mode) == CODE_FOR_nothing)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "reduc op not supported by target.\n");
- epilog_reduc_code = ERROR_MARK;
+ epilog_reduc_code = ERROR_MARK;
+ }
}
}
else
scale_loop_profile (loop, GCOV_COMPUTE_SCALE (1, vectorization_factor),
expected_iterations / vectorization_factor);
loop->nb_iterations_upper_bound
- = loop->nb_iterations_upper_bound.udiv (double_int::from_uhwi (vectorization_factor),
- FLOOR_DIV_EXPR);
+ = wi::udiv_floor (loop->nb_iterations_upper_bound, vectorization_factor);
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
- && loop->nb_iterations_upper_bound != double_int_zero)
- loop->nb_iterations_upper_bound = loop->nb_iterations_upper_bound - double_int_one;
+ && loop->nb_iterations_upper_bound != 0)
+ loop->nb_iterations_upper_bound = loop->nb_iterations_upper_bound - 1;
if (loop->any_estimate)
{
loop->nb_iterations_estimate
- = loop->nb_iterations_estimate.udiv (double_int::from_uhwi (vectorization_factor),
- FLOOR_DIV_EXPR);
+ = wi::udiv_floor (loop->nb_iterations_estimate, vectorization_factor);
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo)
- && loop->nb_iterations_estimate != double_int_zero)
- loop->nb_iterations_estimate = loop->nb_iterations_estimate - double_int_one;
+ && loop->nb_iterations_estimate != 0)
+ loop->nb_iterations_estimate = loop->nb_iterations_estimate - 1;
}
if (dump_enabled_p ())
projects / gcc.git / blobdiff