return false;
}
-/* Try to infer and assign a vector type to all the statements in STMTS.
- Used only for BB vectorization. */
-
-static bool
-vect_update_all_shared_vectypes (vec_info *vinfo, vec<stmt_vec_info> stmts)
-{
- tree vectype, nunits_vectype;
- if (!vect_get_vector_types_for_stmt (vinfo, stmts[0], &vectype,
- &nunits_vectype, stmts.length ()))
- return false;
-
- stmt_vec_info stmt_info;
- unsigned int i;
- FOR_EACH_VEC_ELT (stmts, i, stmt_info)
- if (!vect_update_shared_vectype (stmt_info, vectype))
- return false;
-
- return true;
-}
-
/* Return true if call statements CALL1 and CALL2 are similar enough
to be combined into the same SLP group. */
FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
{
slp_tree child;
- unsigned old_tree_size = this_tree_size;
unsigned int j;
if (oprnd_info->first_dt == vect_uninitialized_def)
matches, npermutes,
&this_tree_size, bst_map)) != NULL)
{
- /* If we have all children of a non-unary child built up from
- scalars then just throw that away and build it up this node
- from scalars. */
- if (is_a <bb_vec_info> (vinfo)
- && SLP_TREE_CHILDREN (child).length () > 1
- /* ??? Rejecting patterns this way doesn't work. We'd have to
- do extra work to cancel the pattern so the uses see the
- scalar version. */
- && !oprnd_info->any_pattern)
- {
- slp_tree grandchild;
-
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
- if (SLP_TREE_DEF_TYPE (grandchild) != vect_external_def)
- break;
- if (!grandchild
- && vect_update_all_shared_vectypes (vinfo,
- oprnd_info->def_stmts))
- {
- /* Roll back. */
- this_tree_size = old_tree_size;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
- vect_free_slp_tree (grandchild, false);
- SLP_TREE_CHILDREN (child).truncate (0);
-
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "Building parent vector operands from "
- "scalars instead\n");
- oprnd_info->def_stmts = vNULL;
- SLP_TREE_DEF_TYPE (child) = vect_external_def;
- SLP_TREE_SCALAR_OPS (child) = oprnd_info->ops;
- oprnd_info->ops = vNULL;
- ++this_tree_size;
- children.safe_push (child);
- continue;
- }
- }
-
oprnd_info->def_stmts = vNULL;
children.safe_push (child);
continue;
do extra work to cancel the pattern so the uses see the
scalar version. */
&& !is_pattern_stmt_p (stmt_info)
- && !oprnd_info->any_pattern
- && vect_update_all_shared_vectypes (vinfo, oprnd_info->def_stmts))
+ && !oprnd_info->any_pattern)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"Building vector operands from scalars\n");
this_tree_size++;
- child = vect_create_new_slp_node (oprnd_info->def_stmts, 0);
- SLP_TREE_DEF_TYPE (child) = vect_external_def;
- SLP_TREE_SCALAR_OPS (child) = oprnd_info->ops;
+ child = vect_create_new_slp_node (oprnd_info->ops);
children.safe_push (child);
oprnd_info->ops = vNULL;
oprnd_info->def_stmts = vNULL;
tem, npermutes,
&this_tree_size, bst_map)) != NULL)
{
- /* If we have all children of a non-unary child built up from
- scalars then just throw that away and build it up this node
- from scalars. */
- if (is_a <bb_vec_info> (vinfo)
- && SLP_TREE_CHILDREN (child).length () > 1
- /* ??? Rejecting patterns this way doesn't work. We'd have
- to do extra work to cancel the pattern so the uses see the
- scalar version. */
- && !oprnd_info->any_pattern)
- {
- unsigned int j;
- slp_tree grandchild;
-
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
- if (SLP_TREE_DEF_TYPE (grandchild) != vect_external_def)
- break;
- if (!grandchild
- && (vect_update_all_shared_vectypes
- (vinfo, oprnd_info->def_stmts)))
- {
- /* Roll back. */
- this_tree_size = old_tree_size;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
- vect_free_slp_tree (grandchild, false);
- SLP_TREE_CHILDREN (child).truncate (0);
-
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "Building parent vector operands from "
- "scalars instead\n");
- oprnd_info->def_stmts = vNULL;
- SLP_TREE_DEF_TYPE (child) = vect_external_def;
- SLP_TREE_SCALAR_OPS (child) = oprnd_info->ops;
- oprnd_info->ops = vNULL;
- ++this_tree_size;
- children.safe_push (child);
- continue;
- }
- }
-
oprnd_info->def_stmts = vNULL;
children.safe_push (child);
continue;
vect_free_oprnd_info (oprnds_info);
+ /* If we have all children of a non-unary child built up from
+ scalars then just throw that away, causing it built up
+ from scalars. */
+ if (nops > 1
+ && is_a <bb_vec_info> (vinfo)
+ /* ??? Rejecting patterns this way doesn't work. We'd have to
+ do extra work to cancel the pattern so the uses see the
+ scalar version. */
+ && !is_pattern_stmt_p (stmt_info))
+ {
+ slp_tree child;
+ unsigned j;
+ FOR_EACH_VEC_ELT (children, j, child)
+ if (SLP_TREE_DEF_TYPE (child) != vect_external_def)
+ break;
+ if (!child)
+ {
+ /* Roll back. */
+ FOR_EACH_VEC_ELT (children, j, child)
+ vect_free_slp_tree (child, false);
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Building parent vector operands from "
+ "scalars instead\n");
+ return NULL;
+ }
+ }
+
*tree_size += this_tree_size + 1;
*max_nunits = this_max_nunits;
visited, lvisited, cost_vec))
return false;
- /* ??? We have to catch the case late where two first scalar stmts appear
- in multiple SLP children with different def type and fail. Remember
- original def types first since SLP_TREE_DEF_TYPE doesn't necessarily
- match it when that is vect_internal_def. */
- auto_vec<vect_def_type, 4> dt;
- dt.safe_grow (SLP_TREE_CHILDREN (node).length ());
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
- if (SLP_TREE_SCALAR_STMTS (child).length () != 0)
- dt[j] = STMT_VINFO_DEF_TYPE (SLP_TREE_SCALAR_STMTS (child)[0]);
-
- /* Push SLP node def-type to stmt operands. */
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
- if (SLP_TREE_DEF_TYPE (child) != vect_internal_def
- && SLP_TREE_SCALAR_STMTS (child).length () != 0)
- STMT_VINFO_DEF_TYPE (SLP_TREE_SCALAR_STMTS (child)[0])
- = SLP_TREE_DEF_TYPE (child);
-
- /* Check everything worked out. */
- bool res = true;
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
- if (SLP_TREE_SCALAR_STMTS (child).length () != 0)
- {
- if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
- {
- if (STMT_VINFO_DEF_TYPE (SLP_TREE_SCALAR_STMTS (child)[0])
- != SLP_TREE_DEF_TYPE (child))
- res = false;
- }
- else if (STMT_VINFO_DEF_TYPE (SLP_TREE_SCALAR_STMTS (child)[0])
- != dt[j])
- res = false;
- }
- if (!res && dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: same operand with different "
- "def type in stmt.\n");
-
- if (res)
- res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
- cost_vec);
-
- /* Restore def-types. */
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
- if (SLP_TREE_SCALAR_STMTS (child).length () != 0)
- STMT_VINFO_DEF_TYPE (SLP_TREE_SCALAR_STMTS (child)[0]) = dt[j];
+ bool res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
+ cost_vec);
/* When the node can be vectorized cost invariant nodes it references.
This is not done in DFS order to allow the refering node
slp_tree node, slp_instance instance)
{
gimple_stmt_iterator si;
- int i, j;
+ int i;
slp_tree child;
/* See if we have already vectorized the node in the graph of the
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
vect_schedule_slp_instance (vinfo, child, instance);
- /* Push SLP node def-type to stmts. */
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
- if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
- {
- stmt_vec_info child_stmt_info;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (child), j, child_stmt_info)
- STMT_VINFO_DEF_TYPE (child_stmt_info) = SLP_TREE_DEF_TYPE (child);
- }
-
stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
/* VECTYPE is the type of the destination. */
}
if (!done_p)
vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
-
- /* Restore stmt def-types. */
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
- if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
- {
- stmt_vec_info child_stmt_info;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (child), j, child_stmt_info)
- STMT_VINFO_DEF_TYPE (child_stmt_info) = vect_internal_def;
- }
}
/* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
*RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
static bool
-vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info, tree rhs,
+vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
+ slp_tree slp_node, tree rhs,
vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
vec_load_store_type *vls_type_out)
{
enum vect_def_type rhs_dt;
tree rhs_vectype;
- if (!vect_is_simple_use (rhs, vinfo, &rhs_dt, &rhs_vectype))
+ slp_tree slp_op;
+ if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0,
+ &rhs, &slp_op, &rhs_dt, &rhs_vectype))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
return false;
}
- if (!vect_check_store_rhs (vinfo, stmt_info,
+ if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
op, &rhs_dt, &rhs_vectype, &vls_type))
return false;
condition operands are supportable using vec_is_simple_use. */
static bool
-vect_is_simple_cond (tree cond, vec_info *vinfo, slp_tree slp_node,
- tree *comp_vectype, enum vect_def_type *dts,
- tree vectype)
+vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
+ slp_tree slp_node, tree *comp_vectype,
+ enum vect_def_type *dts, tree vectype)
{
tree lhs, rhs;
tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
+ slp_tree slp_op;
/* Mask case. */
if (TREE_CODE (cond) == SSA_NAME
&& VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond)))
{
- if (!vect_is_simple_use (cond, vinfo, &dts[0], comp_vectype)
+ if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &cond,
+ &slp_op, &dts[0], comp_vectype)
|| !*comp_vectype
|| !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
return false;
if (TREE_CODE (lhs) == SSA_NAME)
{
- if (!vect_is_simple_use (lhs, vinfo, &dts[0], &vectype1))
+ if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0,
+ &lhs, &slp_op, &dts[0], &vectype1))
return false;
}
else if (TREE_CODE (lhs) == INTEGER_CST || TREE_CODE (lhs) == REAL_CST
if (TREE_CODE (rhs) == SSA_NAME)
{
- if (!vect_is_simple_use (rhs, vinfo, &dts[1], &vectype2))
+ if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1,
+ &rhs, &slp_op, &dts[1], &vectype2))
return false;
}
else if (TREE_CODE (rhs) == INTEGER_CST || TREE_CODE (rhs) == REAL_CST
cond_expr = gimple_assign_rhs1 (stmt);
- if (!vect_is_simple_cond (cond_expr, vinfo, slp_node,
+ if (!vect_is_simple_cond (cond_expr, vinfo, stmt_info, slp_node,
&comp_vectype, &dts[0], vectype)
|| !comp_vectype)
return false;
- unsigned slp_adjust = 0;
- if (slp_node && SLP_TREE_CHILDREN (slp_node).length () == 4)
- /* ??? Hack. Hope for COND_EXPR GIMPLE sanitizing or refactor
- things more... */
- slp_adjust = 1;
+ unsigned op_adjust = COMPARISON_CLASS_P (cond_expr) ? 1 : 0;
slp_tree then_slp_node, else_slp_node;
- if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1 + slp_adjust,
+ if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1 + op_adjust,
&then_clause, &then_slp_node, &dts[2], &vectype1))
return false;
- if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 2 + slp_adjust,
+ if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 2 + op_adjust,
&else_clause, &else_slp_node, &dts[3], &vectype2))
return false;
if (slp_node
&& (!vect_maybe_update_slp_op_vectype
(SLP_TREE_CHILDREN (slp_node)[0], comp_vectype)
- || (slp_adjust == 1
+ || (op_adjust == 1
&& !vect_maybe_update_slp_op_vectype
(SLP_TREE_CHILDREN (slp_node)[1], comp_vectype))
|| !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
*op = gimple_get_lhs (SLP_TREE_SCALAR_STMTS (child)[0]->stmt);
else
- *op = SLP_TREE_SCALAR_OPS (child)[0];
+ {
+ if (def_stmt_info_out)
+ *def_stmt_info_out = NULL;
+ *op = SLP_TREE_SCALAR_OPS (child)[0];
+ *dt = SLP_TREE_DEF_TYPE (child);
+ *vectype = SLP_TREE_VECTYPE (child);
+ return true;
+ }
}
else
{
if (gassign *ass = dyn_cast <gassign *> (stmt->stmt))
{
- *op = gimple_op (ass, operand + 1);
- /* ??? Ick. But it will vanish with SLP only. */
- if (TREE_CODE (*op) == VIEW_CONVERT_EXPR)
- *op = TREE_OPERAND (*op, 0);
+ if (gimple_assign_rhs_code (ass) == COND_EXPR
+ && COMPARISON_CLASS_P (gimple_assign_rhs1 (ass)))
+ {
+ if (operand < 2)
+ *op = TREE_OPERAND (gimple_assign_rhs1 (ass), operand);
+ else
+ *op = gimple_op (ass, operand);
+ }
+ else if (gimple_assign_rhs_code (ass) == VIEW_CONVERT_EXPR)
+ *op = TREE_OPERAND (gimple_assign_rhs1 (ass), 0);
+ else
+ *op = gimple_op (ass, operand + 1);
}
else if (gcall *call = dyn_cast <gcall *> (stmt->stmt))
- *op = gimple_call_arg (call, operand);
+ {
+ if (gimple_call_internal_p (call)
+ && internal_store_fn_p (gimple_call_internal_fn (call)))
+ operand = internal_fn_stored_value_index (gimple_call_internal_fn
+ (call));
+ *op = gimple_call_arg (call, operand);
+ }
else
gcc_unreachable ();
}
projects / gcc.git / commitdiff