/* Statement Analysis and Transformation for Vectorization
- Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ Copyright (C) 2003-2016 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "dumpfile.h"
#include "backend.h"
+#include "target.h"
+#include "rtl.h"
#include "tree.h"
#include "gimple.h"
-#include "rtl.h"
#include "ssa.h"
+#include "optabs-tree.h"
+#include "insn-config.h"
+#include "recog.h" /* FIXME: for insn_data */
+#include "cgraph.h"
+#include "dumpfile.h"
#include "alias.h"
#include "fold-const.h"
#include "stor-layout.h"
-#include "target.h"
-#include "gimple-pretty-print.h"
-#include "internal-fn.h"
#include "tree-eh.h"
#include "gimplify.h"
#include "gimple-iterator.h"
#include "cfgloop.h"
#include "tree-ssa-loop.h"
#include "tree-scalar-evolution.h"
-#include "flags.h"
-#include "insn-config.h"
-#include "recog.h" /* FIXME: for insn_data */
-#include "insn-codes.h"
-#include "optabs-tree.h"
-#include "diagnostic-core.h"
#include "tree-vectorizer.h"
-#include "cgraph.h"
#include "builtins.h"
+#include "internal-fn.h"
/* For lang_hooks.types.type_for_mode. */
#include "langhooks.h"
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
stmt_vec_info dstmt_vinfo;
basic_block bb, def_bb;
- tree def;
gimple *def_stmt;
enum vect_def_type dt;
if (!force && !exist_non_indexing_operands_for_use_p (use, stmt))
return true;
- if (!vect_is_simple_use (use, stmt, loop_vinfo, &def_stmt, &def, &dt))
+ if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &dt))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
tree
vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi)
{
- tree new_var;
gimple *init_stmt;
- tree vec_oprnd;
tree new_temp;
if (TREE_CODE (type) == VECTOR_TYPE
{
if (!types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
{
- if (CONSTANT_CLASS_P (val))
- val = fold_unary (VIEW_CONVERT_EXPR, TREE_TYPE (type), val);
+ /* Scalar boolean value should be transformed into
+ all zeros or all ones value before building a vector. */
+ if (VECTOR_BOOLEAN_TYPE_P (type))
+ {
+ tree true_val = build_all_ones_cst (TREE_TYPE (type));
+ tree false_val = build_zero_cst (TREE_TYPE (type));
+
+ if (CONSTANT_CLASS_P (val))
+ val = integer_zerop (val) ? false_val : true_val;
+ else
+ {
+ new_temp = make_ssa_name (TREE_TYPE (type));
+ init_stmt = gimple_build_assign (new_temp, COND_EXPR,
+ val, true_val, false_val);
+ vect_init_vector_1 (stmt, init_stmt, gsi);
+ val = new_temp;
+ }
+ }
+ else if (CONSTANT_CLASS_P (val))
+ val = fold_convert (TREE_TYPE (type), val);
else
{
new_temp = make_ssa_name (TREE_TYPE (type));
val = build_vector_from_val (type, val);
}
- new_var = vect_get_new_vect_var (type, vect_simple_var, "cst_");
- init_stmt = gimple_build_assign (new_var, val);
- new_temp = make_ssa_name (new_var, init_stmt);
- gimple_assign_set_lhs (init_stmt, new_temp);
+ new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
+ init_stmt = gimple_build_assign (new_temp, val);
vect_init_vector_1 (stmt, init_stmt, gsi);
- vec_oprnd = gimple_assign_lhs (init_stmt);
- return vec_oprnd;
+ return new_temp;
}
STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
In case OP is an invariant or constant, a new stmt that creates a vector def
- needs to be introduced. */
+ needs to be introduced. VECTYPE may be used to specify a required type for
+ vector invariant. */
tree
-vect_get_vec_def_for_operand (tree op, gimple *stmt, tree *scalar_def)
+vect_get_vec_def_for_operand (tree op, gimple *stmt, tree vectype)
{
tree vec_oprnd;
gimple *vec_stmt;
gimple *def_stmt;
stmt_vec_info def_stmt_info = NULL;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
- unsigned int nunits;
+ tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- tree def;
enum vect_def_type dt;
bool is_simple_use;
tree vector_type;
dump_printf (MSG_NOTE, "\n");
}
- is_simple_use = vect_is_simple_use (op, stmt, loop_vinfo,
- &def_stmt, &def, &dt);
+ is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt);
gcc_assert (is_simple_use);
if (dump_enabled_p ())
{
int loc_printed = 0;
- if (def)
- {
- dump_printf_loc (MSG_NOTE, vect_location, "def = ");
- loc_printed = 1;
- dump_generic_expr (MSG_NOTE, TDF_SLIM, def);
- dump_printf (MSG_NOTE, "\n");
- }
if (def_stmt)
{
if (loc_printed)
switch (dt)
{
- /* Case 1: operand is a constant. */
+ /* operand is a constant or a loop invariant. */
case vect_constant_def:
- {
- vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
- gcc_assert (vector_type);
- nunits = TYPE_VECTOR_SUBPARTS (vector_type);
-
- if (scalar_def)
- *scalar_def = op;
-
- /* Create 'vect_cst_ = {cst,cst,...,cst}' */
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "Create vector_cst. nunits = %d\n", nunits);
-
- return vect_init_vector (stmt, op, vector_type, NULL);
- }
-
- /* Case 2: operand is defined outside the loop - loop invariant. */
case vect_external_def:
{
- vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
- gcc_assert (vector_type);
-
- if (scalar_def)
- *scalar_def = def;
-
- /* Create 'vec_inv = {inv,inv,..,inv}' */
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "Create vector_inv.\n");
+ if (vectype)
+ vector_type = vectype;
+ else if (TREE_CODE (TREE_TYPE (op)) == BOOLEAN_TYPE
+ && VECTOR_BOOLEAN_TYPE_P (stmt_vectype))
+ vector_type = build_same_sized_truth_vector_type (stmt_vectype);
+ else
+ vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
- return vect_init_vector (stmt, def, vector_type, NULL);
+ gcc_assert (vector_type);
+ return vect_init_vector (stmt, op, vector_type, NULL);
}
- /* Case 3: operand is defined inside the loop. */
+ /* operand is defined inside the loop. */
case vect_internal_def:
{
- if (scalar_def)
- *scalar_def = NULL/* FIXME tuples: def_stmt*/;
-
/* Get the def from the vectorized stmt. */
def_stmt_info = vinfo_for_stmt (def_stmt);
return vec_oprnd;
}
- /* Case 4: operand is defined by a loop header phi - reduction */
+ /* operand is defined by a loop header phi - reduction */
case vect_reduction_def:
case vect_double_reduction_def:
case vect_nested_cycle:
- {
- struct loop *loop;
-
- gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
- loop = (gimple_bb (def_stmt))->loop_father;
-
- /* Get the def before the loop */
- op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop));
- return get_initial_def_for_reduction (stmt, op, scalar_def);
- }
+ /* Code should use get_initial_def_for_reduction. */
+ gcc_unreachable ();
- /* Case 5: operand is defined by loop-header phi - induction. */
+ /* operand is defined by loop-header phi - induction. */
case vect_induction_def:
{
gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
gcc_assert (def_stmt_info);
vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
gcc_assert (vec_stmt_for_operand);
- vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
else
tree vec_oprnd;
vec_oprnds0->create (1);
- vec_oprnd = vect_get_vec_def_for_operand (op0, stmt, NULL);
+ vec_oprnd = vect_get_vec_def_for_operand (op0, stmt);
vec_oprnds0->quick_push (vec_oprnd);
if (op1)
{
vec_oprnds1->create (1);
- vec_oprnd = vect_get_vec_def_for_operand (op1, stmt, NULL);
+ vec_oprnd = vect_get_vec_def_for_operand (op1, stmt);
vec_oprnds1->quick_push (vec_oprnd);
}
}
add_stmt_to_eh_lp (vec_stmt, lp_nr);
}
-/* Checks if CALL can be vectorized in type VECTYPE. Returns
- a function declaration if the target has a vectorized version
- of the function, or NULL_TREE if the function cannot be vectorized. */
+/* We want to vectorize a call to combined function CFN with function
+ decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
+ as the types of all inputs. Check whether this is possible using
+ an internal function, returning its code if so or IFN_LAST if not. */
-tree
-vectorizable_function (gcall *call, tree vectype_out, tree vectype_in)
+static internal_fn
+vectorizable_internal_function (combined_fn cfn, tree fndecl,
+ tree vectype_out, tree vectype_in)
{
- tree fndecl = gimple_call_fndecl (call);
-
- /* We only handle functions that do not read or clobber memory -- i.e.
- const or novops ones. */
- if (!(gimple_call_flags (call) & (ECF_CONST | ECF_NOVOPS)))
- return NULL_TREE;
-
- if (!fndecl
- || TREE_CODE (fndecl) != FUNCTION_DECL
- || !DECL_BUILT_IN (fndecl))
- return NULL_TREE;
-
- return targetm.vectorize.builtin_vectorized_function (fndecl, vectype_out,
- vectype_in);
+ internal_fn ifn;
+ if (internal_fn_p (cfn))
+ ifn = as_internal_fn (cfn);
+ else
+ ifn = associated_internal_fn (fndecl);
+ if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
+ {
+ const direct_internal_fn_info &info = direct_internal_fn (ifn);
+ if (info.vectorizable)
+ {
+ tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
+ tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
+ if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
+ OPTIMIZE_FOR_SPEED))
+ return ifn;
+ }
+ }
+ return IFN_LAST;
}
bool nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ tree rhs_vectype = NULL_TREE;
+ tree mask_vectype;
tree elem_type;
gimple *new_stmt;
tree dummy;
bool is_store;
tree mask;
gimple *def_stmt;
- tree def;
enum vect_def_type dt;
if (slp_node != NULL)
is_store = gimple_call_internal_fn (stmt) == IFN_MASK_STORE;
mask = gimple_call_arg (stmt, 2);
- if (TYPE_PRECISION (TREE_TYPE (mask))
- != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype))))
+
+ if (TREE_CODE (TREE_TYPE (mask)) != BOOLEAN_TYPE)
return false;
/* FORNOW. This restriction should be relaxed. */
if (STMT_VINFO_STRIDED_P (stmt_info))
return false;
+ if (TREE_CODE (mask) != SSA_NAME)
+ return false;
+
+ if (!vect_is_simple_use (mask, loop_vinfo, &def_stmt, &dt, &mask_vectype))
+ return false;
+
+ if (!mask_vectype)
+ mask_vectype = get_mask_type_for_scalar_type (TREE_TYPE (vectype));
+
+ if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
+ return false;
+
+ if (is_store)
+ {
+ tree rhs = gimple_call_arg (stmt, 3);
+ if (!vect_is_simple_use (rhs, loop_vinfo, &def_stmt, &dt, &rhs_vectype))
+ return false;
+ }
+
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
gimple *def_stmt;
- tree def;
gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base,
&gather_off, &gather_scale);
gcc_assert (gather_decl);
- if (!vect_is_simple_use_1 (gather_off, NULL, loop_vinfo,
- &def_stmt, &def, &gather_dt,
- &gather_off_vectype))
+ if (!vect_is_simple_use (gather_off, loop_vinfo, &def_stmt, &gather_dt,
+ &gather_off_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
: DR_STEP (dr), size_zero_node) <= 0)
return false;
else if (!VECTOR_MODE_P (TYPE_MODE (vectype))
- || !can_vec_mask_load_store_p (TYPE_MODE (vectype), !is_store))
- return false;
-
- if (TREE_CODE (mask) != SSA_NAME)
+ || !can_vec_mask_load_store_p (TYPE_MODE (vectype),
+ TYPE_MODE (mask_vectype),
+ !is_store)
+ || (rhs_vectype
+ && !useless_type_conversion_p (vectype, rhs_vectype)))
return false;
- if (!vect_is_simple_use (mask, stmt, loop_vinfo,
- &def_stmt, &def, &dt))
- return false;
-
- if (is_store)
- {
- tree rhs = gimple_call_arg (stmt, 3);
- if (!vect_is_simple_use (rhs, stmt, loop_vinfo,
- &def_stmt, &def, &dt))
- return false;
- }
-
if (!vec_stmt) /* transformation not required. */
{
STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
perm_mask, stmt, gsi);
else if (j == 0)
op = vec_oprnd0
- = vect_get_vec_def_for_operand (gather_off, stmt, NULL);
+ = vect_get_vec_def_for_operand (gather_off, stmt);
else
op = vec_oprnd0
= vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
{
gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
== TYPE_VECTOR_SUBPARTS (idxtype));
- var = vect_get_new_vect_var (idxtype, vect_simple_var, NULL);
- var = make_ssa_name (var);
+ var = vect_get_new_ssa_name (idxtype, vect_simple_var);
op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
else
{
if (j == 0)
- vec_mask = vect_get_vec_def_for_operand (mask, stmt, NULL);
+ vec_mask = vect_get_vec_def_for_operand (mask, stmt);
else
{
- vect_is_simple_use (vec_mask, NULL, loop_vinfo,
- &def_stmt, &def, &dt);
+ vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt);
vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
}
{
gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op))
== TYPE_VECTOR_SUBPARTS (masktype));
- var = vect_get_new_vect_var (masktype, vect_simple_var,
- NULL);
- var = make_ssa_name (var);
+ var = vect_get_new_ssa_name (masktype, vect_simple_var);
mask_op = build1 (VIEW_CONVERT_EXPR, masktype, mask_op);
new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_op);
{
gcc_assert (TYPE_VECTOR_SUBPARTS (vectype)
== TYPE_VECTOR_SUBPARTS (rettype));
- var = vect_get_new_vect_var (rettype, vect_simple_var, NULL);
- op = make_ssa_name (var, new_stmt);
+ op = vect_get_new_ssa_name (rettype, vect_simple_var);
gimple_call_set_lhs (new_stmt, op);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
var = make_ssa_name (vec_dest);
/* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
from the IL. */
+ if (STMT_VINFO_RELATED_STMT (stmt_info))
+ {
+ stmt = STMT_VINFO_RELATED_STMT (stmt_info);
+ stmt_info = vinfo_for_stmt (stmt);
+ }
tree lhs = gimple_call_lhs (stmt);
new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
set_vinfo_for_stmt (new_stmt, stmt_info);
if (i == 0)
{
tree rhs = gimple_call_arg (stmt, 3);
- vec_rhs = vect_get_vec_def_for_operand (rhs, stmt, NULL);
- vec_mask = vect_get_vec_def_for_operand (mask, stmt, NULL);
+ vec_rhs = vect_get_vec_def_for_operand (rhs, stmt);
+ vec_mask = vect_get_vec_def_for_operand (mask, stmt);
/* We should have catched mismatched types earlier. */
gcc_assert (useless_type_conversion_p (vectype,
TREE_TYPE (vec_rhs)));
}
else
{
- vect_is_simple_use (vec_rhs, NULL, loop_vinfo, &def_stmt,
- &def, &dt);
+ vect_is_simple_use (vec_rhs, loop_vinfo, &def_stmt, &dt);
vec_rhs = vect_get_vec_def_for_stmt_copy (dt, vec_rhs);
- vect_is_simple_use (vec_mask, NULL, loop_vinfo, &def_stmt,
- &def, &dt);
+ vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt);
vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
TYPE_SIZE_UNIT (vectype));
misalign = DR_MISALIGNMENT (dr);
set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
misalign);
+ tree ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)),
+ misalign ? misalign & -misalign : align);
new_stmt
= gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
- gimple_call_arg (stmt, 1),
- vec_mask, vec_rhs);
+ ptr, vec_mask, vec_rhs);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
if (i == 0)
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
if (i == 0)
{
- vec_mask = vect_get_vec_def_for_operand (mask, stmt, NULL);
+ vec_mask = vect_get_vec_def_for_operand (mask, stmt);
dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL,
NULL_TREE, &dummy, gsi,
&ptr_incr, false, &inv_p);
}
else
{
- vect_is_simple_use (vec_mask, NULL, loop_vinfo, &def_stmt,
- &def, &dt);
+ vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt);
vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
TYPE_SIZE_UNIT (vectype));
misalign = DR_MISALIGNMENT (dr);
set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
misalign);
+ tree ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)),
+ misalign ? misalign & -misalign : align);
new_stmt
= gimple_build_call_internal (IFN_MASK_LOAD, 3, dataref_ptr,
- gimple_call_arg (stmt, 1),
- vec_mask);
+ ptr, vec_mask);
gimple_call_set_lhs (new_stmt, make_ssa_name (vec_dest));
vect_finish_stmt_generation (stmt, new_stmt, gsi);
if (i == 0)
{
/* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
from the IL. */
+ if (STMT_VINFO_RELATED_STMT (stmt_info))
+ {
+ stmt = STMT_VINFO_RELATED_STMT (stmt_info);
+ stmt_info = vinfo_for_stmt (stmt);
+ }
tree lhs = gimple_call_lhs (stmt);
new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
set_vinfo_for_stmt (new_stmt, stmt_info);
return true;
}
+/* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
+ integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
+ in a single step. On success, store the binary pack code in
+ *CONVERT_CODE. */
+
+static bool
+simple_integer_narrowing (tree vectype_out, tree vectype_in,
+ tree_code *convert_code)
+{
+ if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
+ || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
+ return false;
+
+ tree_code code;
+ int multi_step_cvt = 0;
+ auto_vec <tree, 8> interm_types;
+ if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
+ &code, &multi_step_cvt,
+ &interm_types)
+ || multi_step_cvt)
+ return false;
+
+ *convert_code = code;
+ return true;
+}
/* Function vectorizable_call.
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
vec_info *vinfo = stmt_info->vinfo;
- tree fndecl, new_temp, def, rhs_type;
+ tree fndecl, new_temp, rhs_type;
gimple *def_stmt;
enum vect_def_type dt[3]
= {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
if (!rhs_type)
rhs_type = TREE_TYPE (op);
- if (!vect_is_simple_use_1 (op, stmt, vinfo,
- &def_stmt, &def, &dt[i], &opvectype))
+ if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[i], &opvectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
else
return false;
+ /* We only handle functions that do not read or clobber memory. */
+ if (gimple_vuse (stmt))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "function reads from or writes to memory.\n");
+ return false;
+ }
+
/* For now, we only vectorize functions if a target specific builtin
is available. TODO -- in some cases, it might be profitable to
insert the calls for pieces of the vector, in order to be able
to vectorize other operations in the loop. */
- fndecl = vectorizable_function (stmt, vectype_out, vectype_in);
- if (fndecl == NULL_TREE)
+ fndecl = NULL_TREE;
+ internal_fn ifn = IFN_LAST;
+ combined_fn cfn = gimple_call_combined_fn (stmt);
+ tree callee = gimple_call_fndecl (stmt);
+
+ /* First try using an internal function. */
+ tree_code convert_code = ERROR_MARK;
+ if (cfn != CFN_LAST
+ && (modifier == NONE
+ || (modifier == NARROW
+ && simple_integer_narrowing (vectype_out, vectype_in,
+ &convert_code))))
+ ifn = vectorizable_internal_function (cfn, callee, vectype_out,
+ vectype_in);
+
+ /* If that fails, try asking for a target-specific built-in function. */
+ if (ifn == IFN_LAST)
+ {
+ if (cfn != CFN_LAST)
+ fndecl = targetm.vectorize.builtin_vectorized_function
+ (cfn, vectype_out, vectype_in);
+ else
+ fndecl = targetm.vectorize.builtin_md_vectorized_function
+ (callee, vectype_out, vectype_in);
+ }
+
+ if (ifn == IFN_LAST && !fndecl)
{
- if (gimple_call_internal_p (stmt)
- && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
+ if (cfn == CFN_GOMP_SIMD_LANE
&& !slp_node
&& loop_vinfo
&& LOOP_VINFO_LOOP (loop_vinfo)->simduid
}
}
- gcc_assert (!gimple_vuse (stmt));
-
if (slp_node || PURE_SLP_STMT (stmt_info))
ncopies = 1;
- else if (modifier == NARROW)
+ else if (modifier == NARROW && ifn == IFN_LAST)
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
else
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_call ==="
"\n");
vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
+ if (ifn != IFN_LAST && modifier == NARROW && !slp_node)
+ add_stmt_cost (stmt_info->vinfo->target_cost_data, ncopies / 2,
+ vec_promote_demote, stmt_info, 0, vect_body);
+
return true;
}
vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
prev_stmt_info = NULL;
- switch (modifier)
+ if (modifier == NONE || ifn != IFN_LAST)
{
- case NONE:
+ tree prev_res = NULL_TREE;
for (j = 0; j < ncopies; ++j)
{
/* Build argument list for the vectorized call. */
vec<tree> vec_oprndsk = vec_defs[k];
vargs[k] = vec_oprndsk[i];
}
- new_stmt = gimple_build_call_vec (fndecl, vargs);
- new_temp = make_ssa_name (vec_dest, new_stmt);
- gimple_call_set_lhs (new_stmt, new_temp);
+ if (modifier == NARROW)
+ {
+ tree half_res = make_ssa_name (vectype_in);
+ new_stmt = gimple_build_call_internal_vec (ifn, vargs);
+ gimple_call_set_lhs (new_stmt, half_res);
+ vect_finish_stmt_generation (stmt, new_stmt, gsi);
+ if ((i & 1) == 0)
+ {
+ prev_res = half_res;
+ continue;
+ }
+ new_temp = make_ssa_name (vec_dest);
+ new_stmt = gimple_build_assign (new_temp, convert_code,
+ prev_res, half_res);
+ }
+ else
+ {
+ if (ifn != IFN_LAST)
+ new_stmt = gimple_build_call_internal_vec (ifn, vargs);
+ else
+ new_stmt = gimple_build_call_vec (fndecl, vargs);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ gimple_call_set_lhs (new_stmt, new_temp);
+ }
vect_finish_stmt_generation (stmt, new_stmt, gsi);
SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
}
op = gimple_call_arg (stmt, i);
if (j == 0)
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt, NULL);
+ = vect_get_vec_def_for_operand (op, stmt);
else
{
vec_oprnd0 = gimple_call_arg (new_stmt, i);
v[k] = build_int_cst (unsigned_type_node, j * nunits_out + k);
tree cst = build_vector (vectype_out, v);
tree new_var
- = vect_get_new_vect_var (vectype_out, vect_simple_var, "cst_");
+ = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
gimple *init_stmt = gimple_build_assign (new_var, cst);
- new_temp = make_ssa_name (new_var, init_stmt);
- gimple_assign_set_lhs (init_stmt, new_temp);
vect_init_vector_1 (stmt, init_stmt, NULL);
new_temp = make_ssa_name (vec_dest);
- new_stmt = gimple_build_assign (new_temp,
- gimple_assign_lhs (init_stmt));
+ new_stmt = gimple_build_assign (new_temp, new_var);
+ }
+ else if (modifier == NARROW)
+ {
+ tree half_res = make_ssa_name (vectype_in);
+ new_stmt = gimple_build_call_internal_vec (ifn, vargs);
+ gimple_call_set_lhs (new_stmt, half_res);
+ vect_finish_stmt_generation (stmt, new_stmt, gsi);
+ if ((j & 1) == 0)
+ {
+ prev_res = half_res;
+ continue;
+ }
+ new_temp = make_ssa_name (vec_dest);
+ new_stmt = gimple_build_assign (new_temp, convert_code,
+ prev_res, half_res);
}
else
{
- new_stmt = gimple_build_call_vec (fndecl, vargs);
+ if (ifn != IFN_LAST)
+ new_stmt = gimple_build_call_internal_vec (ifn, vargs);
+ else
+ new_stmt = gimple_build_call_vec (fndecl, vargs);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_call_set_lhs (new_stmt, new_temp);
}
vect_finish_stmt_generation (stmt, new_stmt, gsi);
- if (j == 0)
+ if (j == (modifier == NARROW ? 1 : 0))
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
else
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
prev_stmt_info = vinfo_for_stmt (new_stmt);
}
-
- break;
-
- case NARROW:
+ }
+ else if (modifier == NARROW)
+ {
for (j = 0; j < ncopies; ++j)
{
/* Build argument list for the vectorized call. */
vargs.quick_push (vec_oprndsk[i]);
vargs.quick_push (vec_oprndsk[i + 1]);
}
- new_stmt = gimple_build_call_vec (fndecl, vargs);
+ if (ifn != IFN_LAST)
+ new_stmt = gimple_build_call_internal_vec (ifn, vargs);
+ else
+ new_stmt = gimple_build_call_vec (fndecl, vargs);
new_temp = make_ssa_name (vec_dest, new_stmt);
gimple_call_set_lhs (new_stmt, new_temp);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
if (j == 0)
{
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt, NULL);
+ = vect_get_vec_def_for_operand (op, stmt);
vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
}
}
*vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
-
- break;
-
- case WIDEN:
- /* No current target implements this case. */
- return false;
}
+ else
+ /* No current target implements this case. */
+ return false;
vargs.release ();
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
vec_info *vinfo = stmt_info->vinfo;
struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
- tree fndecl, new_temp, def;
+ tree fndecl, new_temp;
gimple *def_stmt;
gimple *new_stmt = NULL;
int ncopies, j;
thisarginfo.simd_lane_linear = false;
op = gimple_call_arg (stmt, i);
- if (!vect_is_simple_use_1 (op, stmt, vinfo,
- &def_stmt, &def, &thisarginfo.dt,
- &thisarginfo.vectype)
+ if (!vect_is_simple_use (op, vinfo, &def_stmt, &thisarginfo.dt,
+ &thisarginfo.vectype)
|| thisarginfo.dt == vect_uninitialized_def)
{
if (dump_enabled_p ())
case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
/* FORNOW */
i = -1;
break;
gcc_assert ((k & (k - 1)) == 0);
if (m == 0)
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt, NULL);
+ = vect_get_vec_def_for_operand (op, stmt);
else
{
vec_oprnd0 = arginfo[i].op;
{
if (m == 0 && l == 0)
vec_oprnd0
- = vect_get_vec_def_for_operand (op, stmt, NULL);
+ = vect_get_vec_def_for_operand (op, stmt);
else
vec_oprnd0
= vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
}
break;
case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
default:
gcc_unreachable ();
}
/* All the vector operands except the very first one (that is scalar oprnd)
are stmt copies. */
if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
- vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt, NULL);
+ vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt);
else
vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
tree decl1 = NULL_TREE, decl2 = NULL_TREE;
tree new_temp;
- tree def;
gimple *def_stmt;
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
gimple *new_stmt = NULL;
&& SCALAR_FLOAT_TYPE_P (rhs_type))))
return false;
- if ((INTEGRAL_TYPE_P (lhs_type)
- && (TYPE_PRECISION (lhs_type)
- != GET_MODE_PRECISION (TYPE_MODE (lhs_type))))
- || (INTEGRAL_TYPE_P (rhs_type)
- && (TYPE_PRECISION (rhs_type)
- != GET_MODE_PRECISION (TYPE_MODE (rhs_type)))))
+ if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
+ && ((INTEGRAL_TYPE_P (lhs_type)
+ && (TYPE_PRECISION (lhs_type)
+ != GET_MODE_PRECISION (TYPE_MODE (lhs_type))))
+ || (INTEGRAL_TYPE_P (rhs_type)
+ && (TYPE_PRECISION (rhs_type)
+ != GET_MODE_PRECISION (TYPE_MODE (rhs_type))))))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
}
/* Check the operands of the operation. */
- if (!vect_is_simple_use_1 (op0, stmt, vinfo,
- &def_stmt, &def, &dt[0], &vectype_in))
+ if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype_in))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
/* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
OP1. */
if (CONSTANT_CLASS_P (op0))
- ok = vect_is_simple_use_1 (op1, stmt, vinfo,
- &def_stmt, &def, &dt[1], &vectype_in);
+ ok = vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1], &vectype_in);
else
- ok = vect_is_simple_use (op1, stmt, vinfo, &def_stmt,
- &def, &dt[1]);
+ ok = vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1]);
if (!ok)
{
return false;
}
+ if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
+ && !VECTOR_BOOLEAN_TYPE_P (vectype_in))
+ {
+ if (dump_enabled_p ())
+ {
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't convert between boolean and non "
+ "boolean vectors");
+ dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
+ dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
+ }
+
+ return false;
+ }
+
nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
if (nunits_in < nunits_out)
}
else
{
- vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
+ vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt);
vec_oprnds0.quick_push (vec_oprnd0);
if (op_type == binary_op)
{
if (code == WIDEN_LSHIFT_EXPR)
vec_oprnd1 = op1;
else
- vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt,
- NULL);
+ vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt);
vec_oprnds1.quick_push (vec_oprnd1);
}
}
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree new_temp;
- tree def;
gimple *def_stmt;
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
int ncopies;
gcc_assert (ncopies >= 1);
- if (!vect_is_simple_use_1 (op, stmt, vinfo,
- &def_stmt, &def, &dt[0], &vectype_in))
+ if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[0], &vectype_in))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
/* But a conversion that does not change the bit-pattern is ok. */
&& !((TYPE_PRECISION (TREE_TYPE (scalar_dest))
> TYPE_PRECISION (TREE_TYPE (op)))
- && TYPE_UNSIGNED (TREE_TYPE (op))))
+ && TYPE_UNSIGNED (TREE_TYPE (op)))
+ /* Conversion between boolean types of different sizes is
+ a simple assignment in case their vectypes are same
+ boolean vectors. */
+ && (!VECTOR_BOOLEAN_TYPE_P (vectype)
+ || !VECTOR_BOOLEAN_TYPE_P (vectype_in)))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
optab optab;
int icode;
machine_mode optab_op2_mode;
- tree def;
gimple *def_stmt;
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
gimple *new_stmt = NULL;
}
op0 = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_use_1 (op0, stmt, vinfo,
- &def_stmt, &def, &dt[0], &vectype))
+ if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
return false;
op1 = gimple_assign_rhs2 (stmt);
- if (!vect_is_simple_use_1 (op1, stmt, vinfo, &def_stmt,
- &def, &dt[1], &op1_vectype))
+ if (!vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1], &op1_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
int op_type;
optab optab;
bool target_support_p;
- tree def;
gimple *def_stmt;
enum vect_def_type dt[3]
= {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
/* Most operations cannot handle bit-precision types without extra
truncations. */
- if ((TYPE_PRECISION (TREE_TYPE (scalar_dest))
- != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
+ if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
+ && (TYPE_PRECISION (TREE_TYPE (scalar_dest))
+ != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
/* Exception are bitwise binary operations. */
&& code != BIT_IOR_EXPR
&& code != BIT_XOR_EXPR
}
op0 = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_use_1 (op0, stmt, vinfo,
- &def_stmt, &def, &dt[0], &vectype))
+ if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
/* If op0 is an external or constant def use a vector type with
the same size as the output vector type. */
if (!vectype)
- vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
+ {
+ /* For boolean type we cannot determine vectype by
+ invariant value (don't know whether it is a vector
+ of booleans or vector of integers). We use output
+ vectype because operations on boolean don't change
+ type. */
+ if (TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE)
+ {
+ if (TREE_CODE (TREE_TYPE (scalar_dest)) != BOOLEAN_TYPE)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not supported operation on bool value.\n");
+ return false;
+ }
+ vectype = vectype_out;
+ }
+ else
+ vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
+ }
if (vec_stmt)
gcc_assert (vectype);
if (!vectype)
if (op_type == binary_op || op_type == ternary_op)
{
op1 = gimple_assign_rhs2 (stmt);
- if (!vect_is_simple_use (op1, stmt, vinfo, &def_stmt,
- &def, &dt[1]))
+ if (!vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1]))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
if (op_type == ternary_op)
{
op2 = gimple_assign_rhs3 (stmt);
- if (!vect_is_simple_use (op2, stmt, vinfo, &def_stmt,
- &def, &dt[2]))
+ if (!vect_is_simple_use (op2, vinfo, &def_stmt, &dt[2]))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
{
vec_oprnds2.create (1);
vec_oprnds2.quick_push (vect_get_vec_def_for_operand (op2,
- stmt,
- NULL));
+ stmt));
}
}
else
machine_mode vec_mode;
tree dummy;
enum dr_alignment_support alignment_support_scheme;
- tree def;
gimple *def_stmt;
enum vect_def_type dt;
stmt_vec_info prev_stmt_info = NULL;
}
op = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_use (op, stmt, vinfo, &def_stmt,
- &def, &dt))
+ if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
{
gcc_assert (gimple_assign_single_p (next_stmt));
op = gimple_assign_rhs1 (next_stmt);
- if (!vect_is_simple_use (op, next_stmt, vinfo,
- &def_stmt, &def, &dt))
+ if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
gimple *def_stmt;
- tree def;
scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo, &scatter_base,
&scatter_off, &scatter_scale);
gcc_assert (scatter_decl);
- if (!vect_is_simple_use_1 (scatter_off, NULL, vinfo,
- &def_stmt, &def, &scatter_idx_dt,
- &scatter_off_vectype))
+ if (!vect_is_simple_use (scatter_off, vinfo, &def_stmt, &scatter_idx_dt,
+ &scatter_off_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
if (j == 0)
{
src = vec_oprnd1
- = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt, NULL);
+ = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt);
op = vec_oprnd0
- = vect_get_vec_def_for_operand (scatter_off, stmt, NULL);
+ = vect_get_vec_def_for_operand (scatter_off, stmt);
}
else if (modifier != NONE && (j & 1))
{
{
gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src))
== TYPE_VECTOR_SUBPARTS (srctype));
- var = vect_get_new_vect_var (srctype, vect_simple_var, NULL);
- var = make_ssa_name (var);
+ var = vect_get_new_ssa_name (srctype, vect_simple_var);
src = build1 (VIEW_CONVERT_EXPR, srctype, src);
new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
{
gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
== TYPE_VECTOR_SUBPARTS (idxtype));
- var = vect_get_new_vect_var (idxtype, vect_simple_var, NULL);
- var = make_ssa_name (var);
+ var = vect_get_new_ssa_name (idxtype, vect_simple_var);
op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
group. */
vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
+ gcc_assert (GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_stmt)) == first_stmt);
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
op = gimple_assign_rhs1 (first_stmt);
}
{
gcc_assert (gimple_assign_single_p (next_stmt));
op = gimple_assign_rhs1 (next_stmt);
- vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt,
- NULL);
+ vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt);
}
}
else
vec_oprnd = vec_oprnds[j];
else
{
- vect_is_simple_use (vec_oprnd, NULL, vinfo,
- &def_stmt, &def, &dt);
+ vect_is_simple_use (vec_oprnd, vinfo, &def_stmt, &dt);
vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
}
}
&& gimple_assign_single_p (next_stmt));
op = gimple_assign_rhs1 (next_stmt);
- vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt,
- NULL);
+ vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt);
dr_chain.quick_push (vec_oprnd);
oprnds.quick_push (vec_oprnd);
next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
for (i = 0; i < group_size; i++)
{
op = oprnds[i];
- vect_is_simple_use (op, NULL, vinfo, &def_stmt,
- &def, &dt);
+ vect_is_simple_use (op, vinfo, &def_stmt, &dt);
vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
dr_chain[i] = vec_oprnd;
oprnds[i] = vec_oprnd;
bool grouped_load = false;
bool load_lanes_p = false;
gimple *first_stmt;
+ gimple *first_stmt_for_drptr = NULL;
bool inv_p;
bool negative = false;
bool compute_in_loop = false;
that leaves unused vector loads around punt - we at least create
very sub-optimal code in that case (and blow up memory,
see PR65518). */
+ bool force_peeling = false;
if (first_stmt == stmt
- && !GROUP_NEXT_ELEMENT (stmt_info)
- && GROUP_SIZE (stmt_info) > TYPE_VECTOR_SUBPARTS (vectype))
+ && !GROUP_NEXT_ELEMENT (stmt_info))
+ {
+ if (GROUP_SIZE (stmt_info) > TYPE_VECTOR_SUBPARTS (vectype))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "single-element interleaving not supported "
+ "for not adjacent vector loads\n");
+ return false;
+ }
+
+ /* Single-element interleaving requires peeling for gaps. */
+ force_peeling = true;
+ }
+
+ /* If there is a gap in the end of the group or the group size cannot
+ be made a multiple of the vector element count then we access excess
+ elements in the last iteration and thus need to peel that off. */
+ if (loop_vinfo
+ && ! STMT_VINFO_STRIDED_P (stmt_info)
+ && (force_peeling
+ || GROUP_GAP (vinfo_for_stmt (first_stmt)) != 0
+ || (!slp && vf % GROUP_SIZE (vinfo_for_stmt (first_stmt)) != 0)))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "single-element interleaving not supported "
- "for not adjacent vector loads\n");
- return false;
+ "Data access with gaps requires scalar "
+ "epilogue loop\n");
+ if (loop->inner)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Peeling for outer loop is not supported\n");
+ return false;
+ }
+
+ LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
}
if (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
gimple *def_stmt;
- tree def;
gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base,
&gather_off, &gather_scale);
gcc_assert (gather_decl);
- if (!vect_is_simple_use_1 (gather_off, NULL, vinfo,
- &def_stmt, &def, &gather_dt,
- &gather_off_vectype))
+ if (!vect_is_simple_use (gather_off, vinfo, &def_stmt, &gather_dt,
+ &gather_off_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
perm_mask, stmt, gsi);
else if (j == 0)
op = vec_oprnd0
- = vect_get_vec_def_for_operand (gather_off, stmt, NULL);
+ = vect_get_vec_def_for_operand (gather_off, stmt);
else
op = vec_oprnd0
= vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
{
gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
== TYPE_VECTOR_SUBPARTS (idxtype));
- var = vect_get_new_vect_var (idxtype, vect_simple_var, NULL);
- var = make_ssa_name (var);
+ var = vect_get_new_ssa_name (idxtype, vect_simple_var);
op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
new_stmt
= gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
{
gcc_assert (TYPE_VECTOR_SUBPARTS (vectype)
== TYPE_VECTOR_SUBPARTS (rettype));
- var = vect_get_new_vect_var (rettype, vect_simple_var, NULL);
- op = make_ssa_name (var, new_stmt);
+ op = vect_get_new_ssa_name (rettype, vect_simple_var);
gimple_call_set_lhs (new_stmt, op);
vect_finish_stmt_generation (stmt, new_stmt, gsi);
var = make_ssa_name (vec_dest);
if (grouped_load)
{
first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
- if (slp
- && !SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
- && first_stmt != SLP_TREE_SCALAR_STMTS (slp_node)[0])
- first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
+ /* For SLP vectorization we directly vectorize a subchain
+ without permutation. */
+ if (slp && ! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
+ first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
+ /* For BB vectorization always use the first stmt to base
+ the data ref pointer on. */
+ if (bb_vinfo)
+ first_stmt_for_drptr = SLP_TREE_SCALAR_STMTS (slp_node)[0];
/* Check if the chain of loads is already vectorized. */
if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt))
(DR_REF (first_dr)), 0);
inv_p = false;
}
+ else if (first_stmt_for_drptr
+ && first_stmt != first_stmt_for_drptr)
+ {
+ dataref_ptr
+ = vect_create_data_ref_ptr (first_stmt_for_drptr, aggr_type,
+ at_loop, offset, &dummy, gsi,
+ &ptr_incr, simd_lane_access_p,
+ &inv_p, byte_offset);
+ /* Adjust the pointer by the difference to first_stmt. */
+ data_reference_p ptrdr
+ = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr));
+ tree diff = fold_convert (sizetype,
+ size_binop (MINUS_EXPR,
+ DR_INIT (first_dr),
+ DR_INIT (ptrdr)));
+ dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
+ stmt, diff);
+ }
else
dataref_ptr
= vect_create_data_ref_ptr (first_stmt, aggr_type, at_loop,
unshare_expr
(gimple_assign_rhs1 (stmt))));
new_temp = vect_init_vector (stmt, tem, vectype, NULL);
+ new_stmt = SSA_NAME_DEF_STMT (new_temp);
+ set_vinfo_for_stmt (new_stmt,
+ new_stmt_vec_info (new_stmt, vinfo));
}
else
{
gsi_next (&gsi2);
new_temp = vect_init_vector (stmt, scalar_dest,
vectype, &gsi2);
+ new_stmt = SSA_NAME_DEF_STMT (new_temp);
}
- new_stmt = SSA_NAME_DEF_STMT (new_temp);
- set_vinfo_for_stmt (new_stmt,
- new_stmt_vec_info (new_stmt, vinfo));
}
if (negative)
condition operands are supportable using vec_is_simple_use. */
static bool
-vect_is_simple_cond (tree cond, gimple *stmt, vec_info *vinfo,
- tree *comp_vectype)
+vect_is_simple_cond (tree cond, vec_info *vinfo, tree *comp_vectype)
{
tree lhs, rhs;
- tree def;
enum vect_def_type dt;
tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
+ /* Mask case. */
+ if (TREE_CODE (cond) == SSA_NAME
+ && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE)
+ {
+ gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond);
+ if (!vect_is_simple_use (cond, vinfo, &lhs_def_stmt,
+ &dt, comp_vectype)
+ || !*comp_vectype
+ || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
+ return false;
+ return true;
+ }
+
if (!COMPARISON_CLASS_P (cond))
return false;
if (TREE_CODE (lhs) == SSA_NAME)
{
gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
- if (!vect_is_simple_use_1 (lhs, stmt, vinfo,
- &lhs_def_stmt, &def, &dt, &vectype1))
+ if (!vect_is_simple_use (lhs, vinfo, &lhs_def_stmt, &dt, &vectype1))
return false;
}
else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
if (TREE_CODE (rhs) == SSA_NAME)
{
gimple *rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
- if (!vect_is_simple_use_1 (rhs, stmt, vinfo,
- &rhs_def_stmt, &def, &dt, &vectype2))
+ if (!vect_is_simple_use (rhs, vinfo, &rhs_def_stmt, &dt, &vectype2))
return false;
}
else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
tree vec_compare, vec_cond_expr;
tree new_temp;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- tree def;
enum vect_def_type dt, dts[4];
int ncopies;
enum tree_code code;
vec<tree> vec_oprnds2 = vNULL;
vec<tree> vec_oprnds3 = vNULL;
tree vec_cmp_type;
+ bool masked = false;
if (reduc_index && STMT_SLP_TYPE (stmt_info))
return false;
- if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
- return false;
+ if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == TREE_CODE_REDUCTION)
+ {
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
+ return false;
- if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
- && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
- && reduc_def))
- return false;
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
+ && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
+ && reduc_def))
+ return false;
- /* FORNOW: not yet supported. */
- if (STMT_VINFO_LIVE_P (stmt_info))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "value used after loop.\n");
- return false;
+ /* FORNOW: not yet supported. */
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "value used after loop.\n");
+ return false;
+ }
}
/* Is vectorizable conditional operation? */
then_clause = gimple_assign_rhs2 (stmt);
else_clause = gimple_assign_rhs3 (stmt);
- if (!vect_is_simple_cond (cond_expr, stmt, stmt_info->vinfo, &comp_vectype)
+ if (!vect_is_simple_cond (cond_expr, stmt_info->vinfo, &comp_vectype)
|| !comp_vectype)
return false;
- if (TREE_CODE (then_clause) == SSA_NAME)
- {
- gimple *then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
- if (!vect_is_simple_use (then_clause, stmt, stmt_info->vinfo,
- &then_def_stmt, &def, &dt))
- return false;
- }
- else if (TREE_CODE (then_clause) != INTEGER_CST
- && TREE_CODE (then_clause) != REAL_CST
- && TREE_CODE (then_clause) != FIXED_CST)
+ gimple *def_stmt;
+ if (!vect_is_simple_use (then_clause, stmt_info->vinfo, &def_stmt, &dt))
+ return false;
+ if (!vect_is_simple_use (else_clause, stmt_info->vinfo, &def_stmt, &dt))
return false;
- if (TREE_CODE (else_clause) == SSA_NAME)
+ if (VECTOR_BOOLEAN_TYPE_P (comp_vectype))
{
- gimple *else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
- if (!vect_is_simple_use (else_clause, stmt, stmt_info->vinfo,
- &else_def_stmt, &def, &dt))
- return false;
+ vec_cmp_type = comp_vectype;
+ masked = true;
}
- else if (TREE_CODE (else_clause) != INTEGER_CST
- && TREE_CODE (else_clause) != REAL_CST
- && TREE_CODE (else_clause) != FIXED_CST)
- return false;
-
- unsigned int prec = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype)));
- /* The result of a vector comparison should be signed type. */
- tree cmp_type = build_nonstandard_integer_type (prec, 0);
- vec_cmp_type = get_same_sized_vectype (cmp_type, vectype);
+ else
+ vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
if (vec_cmp_type == NULL_TREE)
return false;
auto_vec<tree, 4> ops;
auto_vec<vec<tree>, 4> vec_defs;
- ops.safe_push (TREE_OPERAND (cond_expr, 0));
- ops.safe_push (TREE_OPERAND (cond_expr, 1));
+ if (masked)
+ ops.safe_push (cond_expr);
+ else
+ {
+ ops.safe_push (TREE_OPERAND (cond_expr, 0));
+ ops.safe_push (TREE_OPERAND (cond_expr, 1));
+ }
ops.safe_push (then_clause);
ops.safe_push (else_clause);
vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
vec_oprnds3 = vec_defs.pop ();
vec_oprnds2 = vec_defs.pop ();
- vec_oprnds1 = vec_defs.pop ();
+ if (!masked)
+ vec_oprnds1 = vec_defs.pop ();
vec_oprnds0 = vec_defs.pop ();
ops.release ();
else
{
gimple *gtemp;
- vec_cond_lhs =
- vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
- stmt, NULL);
- vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt,
- loop_vinfo, >emp, &def, &dts[0]);
-
- vec_cond_rhs =
- vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
- stmt, NULL);
- vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt,
- loop_vinfo, >emp, &def, &dts[1]);
+ if (masked)
+ {
+ vec_cond_lhs
+ = vect_get_vec_def_for_operand (cond_expr, stmt,
+ comp_vectype);
+ vect_is_simple_use (cond_expr, stmt_info->vinfo,
+ >emp, &dts[0]);
+ }
+ else
+ {
+ vec_cond_lhs =
+ vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
+ stmt, comp_vectype);
+ vect_is_simple_use (TREE_OPERAND (cond_expr, 0),
+ loop_vinfo, >emp, &dts[0]);
+
+ vec_cond_rhs =
+ vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
+ stmt, comp_vectype);
+ vect_is_simple_use (TREE_OPERAND (cond_expr, 1),
+ loop_vinfo, >emp, &dts[1]);
+ }
if (reduc_index == 1)
vec_then_clause = reduc_def;
else
{
vec_then_clause = vect_get_vec_def_for_operand (then_clause,
- stmt, NULL);
- vect_is_simple_use (then_clause, stmt, loop_vinfo,
- >emp, &def, &dts[2]);
+ stmt);
+ vect_is_simple_use (then_clause, loop_vinfo,
+ >emp, &dts[2]);
}
if (reduc_index == 2)
vec_else_clause = reduc_def;
else
{
vec_else_clause = vect_get_vec_def_for_operand (else_clause,
- stmt, NULL);
- vect_is_simple_use (else_clause, stmt, loop_vinfo,
- >emp, &def, &dts[3]);
+ stmt);
+ vect_is_simple_use (else_clause, loop_vinfo, >emp, &dts[3]);
}
}
}
else
{
- vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0],
- vec_oprnds0.pop ());
- vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1],
- vec_oprnds1.pop ());
+ vec_cond_lhs
+ = vect_get_vec_def_for_stmt_copy (dts[0],
+ vec_oprnds0.pop ());
+ if (!masked)
+ vec_cond_rhs
+ = vect_get_vec_def_for_stmt_copy (dts[1],
+ vec_oprnds1.pop ());
+
vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
vec_oprnds2.pop ());
vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
if (!slp_node)
{
vec_oprnds0.quick_push (vec_cond_lhs);
- vec_oprnds1.quick_push (vec_cond_rhs);
+ if (!masked)
+ vec_oprnds1.quick_push (vec_cond_rhs);
vec_oprnds2.quick_push (vec_then_clause);
vec_oprnds3.quick_push (vec_else_clause);
}
/* Arguments are ready. Create the new vector stmt. */
FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
{
- vec_cond_rhs = vec_oprnds1[i];
vec_then_clause = vec_oprnds2[i];
vec_else_clause = vec_oprnds3[i];
- vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
- vec_cond_lhs, vec_cond_rhs);
+ if (masked)
+ vec_compare = vec_cond_lhs;
+ else
+ {
+ vec_cond_rhs = vec_oprnds1[i];
+ vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
+ vec_cond_lhs, vec_cond_rhs);
+ }
vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
vec_compare, vec_then_clause, vec_else_clause);
return true;
}
+/* vectorizable_comparison.
+
+ Check if STMT is comparison expression that can be vectorized.
+ If VEC_STMT is also passed, vectorize the STMT: create a vectorized
+ comparison, put it in VEC_STMT, and insert it at GSI.
+
+ Return FALSE if not a vectorizable STMT, TRUE otherwise. */
+
+bool
+vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi,
+ gimple **vec_stmt, tree reduc_def,
+ slp_tree slp_node)
+{
+ tree lhs, rhs1, rhs2;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
+ tree new_temp;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
+ unsigned nunits;
+ int ncopies;
+ enum tree_code code;
+ stmt_vec_info prev_stmt_info = NULL;
+ int i, j;
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ vec<tree> vec_oprnds0 = vNULL;
+ vec<tree> vec_oprnds1 = vNULL;
+ gimple *def_stmt;
+ tree mask_type;
+ tree mask;
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
+ return false;
+
+ if (!VECTOR_BOOLEAN_TYPE_P (vectype))
+ return false;
+
+ mask_type = vectype;
+ nunits = TYPE_VECTOR_SUBPARTS (vectype);
+
+ if (slp_node || PURE_SLP_STMT (stmt_info))
+ ncopies = 1;
+ else
+ ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
+
+ gcc_assert (ncopies >= 1);
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
+ && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
+ && reduc_def))
+ return false;
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "value used after loop.\n");
+ return false;
+ }
+
+ if (!is_gimple_assign (stmt))
+ return false;
+
+ code = gimple_assign_rhs_code (stmt);
+
+ if (TREE_CODE_CLASS (code) != tcc_comparison)
+ return false;
+
+ rhs1 = gimple_assign_rhs1 (stmt);
+ rhs2 = gimple_assign_rhs2 (stmt);
+
+ if (!vect_is_simple_use (rhs1, stmt_info->vinfo, &def_stmt,
+ &dts[0], &vectype1))
+ return false;
+
+ if (!vect_is_simple_use (rhs2, stmt_info->vinfo, &def_stmt,
+ &dts[1], &vectype2))
+ return false;
+
+ if (vectype1 && vectype2
+ && TYPE_VECTOR_SUBPARTS (vectype1) != TYPE_VECTOR_SUBPARTS (vectype2))
+ return false;
+
+ vectype = vectype1 ? vectype1 : vectype2;
+
+ /* Invariant comparison. */
+ if (!vectype)
+ {
+ vectype = build_vector_type (TREE_TYPE (rhs1), nunits);
+ if (tree_to_shwi (TYPE_SIZE_UNIT (vectype)) != current_vector_size)
+ return false;
+ }
+ else if (nunits != TYPE_VECTOR_SUBPARTS (vectype))
+ return false;
+
+ if (!vec_stmt)
+ {
+ STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type;
+ vect_model_simple_cost (stmt_info, ncopies, dts, NULL, NULL);
+ return expand_vec_cmp_expr_p (vectype, mask_type);
+ }
+
+ /* Transform. */
+ if (!slp_node)
+ {
+ vec_oprnds0.create (1);
+ vec_oprnds1.create (1);
+ }
+
+ /* Handle def. */
+ lhs = gimple_assign_lhs (stmt);
+ mask = vect_create_destination_var (lhs, mask_type);
+
+ /* Handle cmp expr. */
+ for (j = 0; j < ncopies; j++)
+ {
+ gassign *new_stmt = NULL;
+ if (j == 0)
+ {
+ if (slp_node)
+ {
+ auto_vec<tree, 2> ops;
+ auto_vec<vec<tree>, 2> vec_defs;
+
+ ops.safe_push (rhs1);
+ ops.safe_push (rhs2);
+ vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
+ vec_oprnds1 = vec_defs.pop ();
+ vec_oprnds0 = vec_defs.pop ();
+ }
+ else
+ {
+ vec_rhs1 = vect_get_vec_def_for_operand (rhs1, stmt, vectype);
+ vec_rhs2 = vect_get_vec_def_for_operand (rhs2, stmt, vectype);
+ }
+ }
+ else
+ {
+ vec_rhs1 = vect_get_vec_def_for_stmt_copy (dts[0],
+ vec_oprnds0.pop ());
+ vec_rhs2 = vect_get_vec_def_for_stmt_copy (dts[1],
+ vec_oprnds1.pop ());
+ }
+
+ if (!slp_node)
+ {
+ vec_oprnds0.quick_push (vec_rhs1);
+ vec_oprnds1.quick_push (vec_rhs2);
+ }
+
+ /* Arguments are ready. Create the new vector stmt. */
+ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
+ {
+ vec_rhs2 = vec_oprnds1[i];
+
+ new_temp = make_ssa_name (mask);
+ new_stmt = gimple_build_assign (new_temp, code, vec_rhs1, vec_rhs2);
+ vect_finish_stmt_generation (stmt, new_stmt, gsi);
+ if (slp_node)
+ SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
+ }
+
+ if (slp_node)
+ continue;
+
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ else
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+
+ prev_stmt_info = vinfo_for_stmt (new_stmt);
+ }
+
+ vec_oprnds0.release ();
+ vec_oprnds1.release ();
+
+ return true;
+}
/* Make sure the statement is vectorizable. */
|| vectorizable_call (stmt, NULL, NULL, node)
|| vectorizable_store (stmt, NULL, NULL, node)
|| vectorizable_reduction (stmt, NULL, NULL, node)
- || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node));
+ || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
+ || vectorizable_comparison (stmt, NULL, NULL, NULL, node));
else
{
if (bb_vinfo)
|| vectorizable_load (stmt, NULL, NULL, node, NULL)
|| vectorizable_call (stmt, NULL, NULL, node)
|| vectorizable_store (stmt, NULL, NULL, node)
- || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node));
+ || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
+ || vectorizable_comparison (stmt, NULL, NULL, NULL, node));
}
if (!ok)
gcc_assert (done);
break;
+ case comparison_vec_info_type:
+ done = vectorizable_comparison (stmt, gsi, &vec_stmt, NULL, slp_node);
+ gcc_assert (done);
+ break;
+
case call_vec_info_type:
done = vectorizable_call (stmt, gsi, &vec_stmt, slp_node);
stmt = gsi_stmt (*gsi);
STMT_VINFO_RELATED_STMT (res) = NULL;
STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
STMT_VINFO_DATA_REF (res) = NULL;
+ STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
STMT_VINFO_DR_OFFSET (res) = NULL;
gimple *patt_stmt = STMT_VINFO_STMT (patt_info);
gimple_set_bb (patt_stmt, NULL);
tree lhs = gimple_get_lhs (patt_stmt);
- if (TREE_CODE (lhs) == SSA_NAME)
+ if (lhs && TREE_CODE (lhs) == SSA_NAME)
release_ssa_name (lhs);
if (seq)
{
gimple *seq_stmt = gsi_stmt (si);
gimple_set_bb (seq_stmt, NULL);
lhs = gimple_get_lhs (seq_stmt);
- if (TREE_CODE (lhs) == SSA_NAME)
+ if (lhs && TREE_CODE (lhs) == SSA_NAME)
release_ssa_name (lhs);
free_stmt_vec_info (seq_stmt);
}
return vectype;
}
+/* Function get_mask_type_for_scalar_type.
+
+ Returns the mask type corresponding to a result of comparison
+ of vectors of specified SCALAR_TYPE as supported by target. */
+
+tree
+get_mask_type_for_scalar_type (tree scalar_type)
+{
+ tree vectype = get_vectype_for_scalar_type (scalar_type);
+
+ if (!vectype)
+ return NULL;
+
+ return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype),
+ current_vector_size);
+}
+
/* Function get_same_sized_vectype
Returns a vector type corresponding to SCALAR_TYPE of size
tree
get_same_sized_vectype (tree scalar_type, tree vector_type)
{
+ if (TREE_CODE (scalar_type) == BOOLEAN_TYPE)
+ return build_same_sized_truth_vector_type (vector_type);
+
return get_vectype_for_scalar_type_and_size
(scalar_type, GET_MODE_SIZE (TYPE_MODE (vector_type)));
}
/* Function vect_is_simple_use.
Input:
- LOOP_VINFO - the vect info of the loop that is being vectorized.
- BB_VINFO - the vect info of the basic block that is being vectorized.
- OPERAND - operand of STMT in the loop or bb.
- DEF - the defining stmt in case OPERAND is an SSA_NAME.
+ VINFO - the vect info of the loop or basic block that is being vectorized.
+ OPERAND - operand in the loop or bb.
+ Output:
+ DEF_STMT - the defining stmt in case OPERAND is an SSA_NAME.
+ DT - the type of definition
Returns whether a stmt with OPERAND can be vectorized.
For loops, supportable operands are constants, loop invariants, and operands
For now, operands defined outside the basic block are not supported. */
bool
-vect_is_simple_use (tree operand, gimple *stmt, vec_info *vinfo,
- gimple **def_stmt, tree *def, enum vect_def_type *dt)
+vect_is_simple_use (tree operand, vec_info *vinfo,
+ gimple **def_stmt, enum vect_def_type *dt)
{
*def_stmt = NULL;
- *def = NULL_TREE;
*dt = vect_unknown_def_type;
if (dump_enabled_p ())
if (is_gimple_min_invariant (operand))
{
- *def = operand;
*dt = vect_external_def;
return true;
}
if (TREE_CODE (operand) != SSA_NAME)
{
if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not ssa-name.\n");
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not ssa-name.\n");
return false;
}
if (SSA_NAME_IS_DEFAULT_DEF (operand))
{
- *def = operand;
*dt = vect_external_def;
return true;
}
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, *def_stmt, 0);
}
- basic_block bb = gimple_bb (*def_stmt);
- if ((is_a <loop_vec_info> (vinfo)
- && !flow_bb_inside_loop_p (as_a <loop_vec_info> (vinfo)->loop, bb))
- || (is_a <bb_vec_info> (vinfo)
- && (bb != as_a <bb_vec_info> (vinfo)->bb
- || gimple_code (*def_stmt) == GIMPLE_PHI)))
+ if (! vect_stmt_in_region_p (vinfo, *def_stmt))
*dt = vect_external_def;
else
{
stmt_vec_info stmt_vinfo = vinfo_for_stmt (*def_stmt);
- if (is_a <bb_vec_info> (vinfo) && !STMT_VINFO_VECTORIZABLE (stmt_vinfo))
- *dt = vect_external_def;
- else
- *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
+ *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
}
if (dump_enabled_p ())
}
}
- if (*dt == vect_unknown_def_type
- || (stmt
- && *dt == vect_double_reduction_def
- && gimple_code (stmt) != GIMPLE_PHI))
+ if (*dt == vect_unknown_def_type)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
switch (gimple_code (*def_stmt))
{
case GIMPLE_PHI:
- *def = gimple_phi_result (*def_stmt);
- break;
-
case GIMPLE_ASSIGN:
- *def = gimple_assign_lhs (*def_stmt);
- break;
-
case GIMPLE_CALL:
- *def = gimple_call_lhs (*def_stmt);
- if (*def != NULL)
- break;
- /* FALLTHRU */
+ break;
default:
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
return true;
}
-/* Function vect_is_simple_use_1.
+/* Function vect_is_simple_use.
- Same as vect_is_simple_use_1 but also determines the vector operand
+ Same as vect_is_simple_use but also determines the vector operand
type of OPERAND and stores it to *VECTYPE. If the definition of
OPERAND is vect_uninitialized_def, vect_constant_def or
vect_external_def *VECTYPE will be set to NULL_TREE and the caller
scalar operand. */
bool
-vect_is_simple_use_1 (tree operand, gimple *stmt, vec_info *vinfo,
- gimple **def_stmt,
- tree *def, enum vect_def_type *dt, tree *vectype)
+vect_is_simple_use (tree operand, vec_info *vinfo,
+ gimple **def_stmt, enum vect_def_type *dt, tree *vectype)
{
- if (!vect_is_simple_use (operand, stmt, vinfo, def_stmt, def, dt))
+ if (!vect_is_simple_use (operand, vinfo, def_stmt, dt))
return false;
/* Now get a vector type if the def is internal, otherwise supply
c2 = VEC_WIDEN_MULT_HI_EXPR;
break;
+ case DOT_PROD_EXPR:
+ c1 = DOT_PROD_EXPR;
+ c2 = DOT_PROD_EXPR;
+ break;
+
+ case SAD_EXPR:
+ c1 = SAD_EXPR;
+ c2 = SAD_EXPR;
+ break;
+
case VEC_WIDEN_MULT_EVEN_EXPR:
/* Support the recursion induced just above. */
c1 = VEC_WIDEN_MULT_EVEN_EXPR;
projects / gcc.git / blobdiff