+2020-05-13 Richard Biener <rguenther@suse.de>
+
+ * target.def (add_stmt_cost): Add new vectype parameter.
+ * targhooks.c (default_add_stmt_cost): Adjust.
+ * targhooks.h (default_add_stmt_cost): Likewise.
+ * config/aarch64/aarch64.c (aarch64_add_stmt_cost): Take new
+ vectype parameter.
+ * config/arm/arm.c (arm_add_stmt_cost): Likewise.
+ * config/i386/i386.c (ix86_add_stmt_cost): Likewise.
+ * config/rs6000/rs6000.c (rs6000_add_stmt_cost): Likewise.
+
+ * tree-vectorizer.h (stmt_info_for_cost::vectype): Add.
+ (dump_stmt_cost): Add new vectype parameter.
+ (add_stmt_cost): Likewise.
+ (record_stmt_cost): Likewise.
+ (record_stmt_cost): Add overload with old signature.
+ * tree-vect-loop.c (vect_compute_single_scalar_iteration_cost):
+ Adjust.
+ (vect_get_known_peeling_cost): Likewise.
+ (vect_estimate_min_profitable_iters): Likewise.
+ * tree-vectorizer.c (dump_stmt_cost): Add new vectype parameter.
+ * tree-vect-stmts.c (record_stmt_cost): Likewise.
+ (vect_prologue_cost_for_slp_op): Remove stmt_vec_info parameter
+ and pass down correct vectype and NULL stmt_info.
+ (vect_model_simple_cost): Adjust.
+ (vect_model_store_cost): Likewise.
+
2020-05-13 Richard Biener <rguenther@suse.de>
* tree-vectorizer.h (SLP_INSTANCE_GROUP_SIZE): Remove.
static unsigned
aarch64_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- struct _stmt_vec_info *stmt_info, int misalign,
- enum vect_cost_model_location where)
+ struct _stmt_vec_info *stmt_info, tree vectype,
+ int misalign, enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
if (flag_vect_cost_model)
{
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost =
aarch64_builtin_vectorization_cost (kind, vectype, misalign);
static unsigned arm_add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
struct _stmt_vec_info *stmt_info,
- int misalign,
+ tree vectype, int misalign,
enum vect_cost_model_location where);
static void arm_canonicalize_comparison (int *code, rtx *op0, rtx *op1,
static unsigned
arm_add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- struct _stmt_vec_info *stmt_info, int misalign,
- enum vect_cost_model_location where)
+ struct _stmt_vec_info *stmt_info, tree vectype,
+ int misalign, enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
if (flag_vect_cost_model)
{
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = arm_builtin_vectorization_cost (kind, vectype, misalign);
/* Statements in an inner loop relative to the loop being
static unsigned
ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- class _stmt_vec_info *stmt_info, int misalign,
+ class _stmt_vec_info *stmt_info, tree vectype,
+ int misalign,
enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
bool scalar_p
= (kind == scalar_stmt || kind == scalar_load || kind == scalar_store);
-
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = - 1;
bool fp = false;
static unsigned
rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- struct _stmt_vec_info *stmt_info, int misalign,
- enum vect_cost_model_location where)
+ struct _stmt_vec_info *stmt_info, tree vectype,
+ int misalign, enum vect_cost_model_location where)
{
rs6000_cost_data *cost_data = (rs6000_cost_data*) data;
unsigned retval = 0;
if (flag_vect_cost_model)
{
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = rs6000_builtin_vectorization_cost (kind, vectype,
misalign);
stmt_cost += adjust_vectorization_cost (kind, stmt_info);
This hook should initialize target-specific data structures in preparation for modeling the costs of vectorizing a loop or basic block. The default allocates three unsigned integers for accumulating costs for the prologue, body, and epilogue of the loop or basic block. If @var{loop_info} is non-NULL, it identifies the loop being vectorized; otherwise a single block is being vectorized.
@end deftypefn
-@deftypefn {Target Hook} unsigned TARGET_VECTORIZE_ADD_STMT_COST (class vec_info *@var{}, void *@var{data}, int @var{count}, enum vect_cost_for_stmt @var{kind}, class _stmt_vec_info *@var{stmt_info}, int @var{misalign}, enum vect_cost_model_location @var{where})
+@deftypefn {Target Hook} unsigned TARGET_VECTORIZE_ADD_STMT_COST (class vec_info *@var{}, void *@var{data}, int @var{count}, enum vect_cost_for_stmt @var{kind}, class _stmt_vec_info *@var{stmt_info}, tree @var{vectype}, int @var{misalign}, enum vect_cost_model_location @var{where})
This hook should update the target-specific @var{data} in response to adding @var{count} copies of the given @var{kind} of statement to a loop or basic block. The default adds the builtin vectorizer cost for the copies of the statement to the accumulator specified by @var{where}, (the prologue, body, or epilogue) and returns the amount added. The return value should be viewed as a tentative cost that may later be revised.
@end deftypefn
"revised.",
unsigned,
(class vec_info *, void *data, int count, enum vect_cost_for_stmt kind,
- class _stmt_vec_info *stmt_info, int misalign,
+ class _stmt_vec_info *stmt_info, tree vectype, int misalign,
enum vect_cost_model_location where),
default_add_stmt_cost)
unsigned
default_add_stmt_cost (class vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- class _stmt_vec_info *stmt_info, int misalign,
+ class _stmt_vec_info *stmt_info, tree vectype,
+ int misalign,
enum vect_cost_model_location where)
{
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
-
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
int stmt_cost = targetm.vectorize.builtin_vectorization_cost (kind, vectype,
misalign);
/* Statements in an inner loop relative to the loop being
extern void *default_init_cost (class loop *);
extern unsigned default_add_stmt_cost (class vec_info *, void *, int,
enum vect_cost_for_stmt,
- class _stmt_vec_info *, int,
+ class _stmt_vec_info *, tree, int,
enum vect_cost_model_location);
extern void default_finish_cost (void *, unsigned *, unsigned *, unsigned *);
extern void default_destroy_cost_data (void *);
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
(void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
- si->kind, si->stmt_info, si->misalign,
- vect_body);
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign, vect_body);
unsigned dummy, body_cost = 0;
finish_cost (target_cost_data, &dummy, &body_cost, &dummy);
destroy_cost_data (target_cost_data);
/* If peeled iterations are known but number of scalar loop
iterations are unknown, count a taken branch per peeled loop. */
retval = record_stmt_cost (prologue_cost_vec, 1, cond_branch_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
retval += record_stmt_cost (epilogue_cost_vec, 1, cond_branch_taken,
- NULL, 0, vect_epilogue);
+ NULL, NULL_TREE, 0, vect_epilogue);
}
else
{
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
(void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
/* FIXME: Make cost depend on complexity of individual check. */
unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
(void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
if (len)
/* Count LEN - 1 ANDs and LEN comparisons. */
(void) add_stmt_cost (loop_vinfo, target_cost_data, len * 2 - 1,
- scalar_stmt, NULL, 0, vect_prologue);
+ scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
if (len)
{
if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
nstmts += 1;
(void) add_stmt_cost (loop_vinfo, target_cost_data, nstmts,
- scalar_stmt, NULL, 0, vect_prologue);
+ scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
}
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
{
/* FIXME: Make cost depend on complexity of individual check. */
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
if (dump_enabled_p ())
dump_printf (MSG_NOTE,
"cost model: Adding cost of checks for loop "
if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
/* Count statements in scalar loop. Using this as scalar cost for a single
iteration for now.
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
j, si)
(void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
- si->kind, si->stmt_info, si->misalign,
- vect_epilogue);
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign, vect_epilogue);
}
/* Calculate how many masks we need to generate. */
probably better not to vectorize. */
(void) add_stmt_cost (loop_vinfo,
target_cost_data, num_masks, vector_stmt,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo,
target_cost_data, num_masks - 1, vector_stmt,
- NULL, 0, vect_body);
+ NULL, NULL_TREE, 0, vect_body);
}
else if (npeel < 0)
{
vector iterations are not known since peeled prologue iterations are
not known. Hence guards remain the same. */
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo,
target_cost_data, 1, cond_branch_not_taken,
- NULL, 0, vect_prologue);
+ NULL, NULL_TREE, 0, vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
- NULL, 0, vect_epilogue);
+ NULL, NULL_TREE, 0, vect_epilogue);
(void) add_stmt_cost (loop_vinfo,
target_cost_data, 1, cond_branch_not_taken,
- NULL, 0, vect_epilogue);
+ NULL, NULL_TREE, 0, vect_epilogue);
stmt_info_for_cost *si;
int j;
FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
{
(void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_prologue,
- si->kind, si->stmt_info, si->misalign,
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign,
vect_prologue);
(void) add_stmt_cost (loop_vinfo, target_cost_data,
si->count * peel_iters_epilogue,
- si->kind, si->stmt_info, si->misalign,
+ si->kind, si->stmt_info, si->vectype,
+ si->misalign,
vect_epilogue);
}
}
FOR_EACH_VEC_ELT (prologue_cost_vec, j, si)
(void) add_stmt_cost (loop_vinfo,
data, si->count, si->kind, si->stmt_info,
- si->misalign, vect_prologue);
+ si->vectype, si->misalign, vect_prologue);
FOR_EACH_VEC_ELT (epilogue_cost_vec, j, si)
(void) add_stmt_cost (loop_vinfo,
data, si->count, si->kind, si->stmt_info,
- si->misalign, vect_epilogue);
+ si->vectype, si->misalign, vect_epilogue);
prologue_cost_vec.release ();
epilogue_cost_vec.release ();
unsigned
record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
- int misalign, enum vect_cost_model_location where)
+ tree vectype, int misalign,
+ enum vect_cost_model_location where)
{
if ((kind == vector_load || kind == unaligned_load)
- && STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+ && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
kind = vector_gather_load;
if ((kind == vector_store || kind == unaligned_store)
- && STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+ && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
kind = vector_scatter_store;
- stmt_info_for_cost si = { count, kind, where, stmt_info, misalign };
+ stmt_info_for_cost si = { count, kind, where, stmt_info, vectype, misalign };
body_cost_vec->safe_push (si);
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
return (unsigned)
(builtin_vectorization_cost (kind, vectype, misalign) * count);
}
static unsigned
vect_prologue_cost_for_slp_op (vec_info *vinfo,
- slp_tree node, stmt_vec_info stmt_info,
+ slp_tree node,
unsigned opno, enum vect_def_type dt,
stmt_vector_for_cost *cost_vec)
{
nelt++;
if (nelt == nelt_limit)
{
- /* ??? We need to pass down stmt_info for a vector type
- even if it points to the wrong stmt. */
prologue_cost += record_stmt_cost
(cost_vec, 1,
dt == vect_external_def
- ? (elt ? scalar_to_vec : vec_construct)
- : vector_load,
- stmt_info, 0, vect_prologue);
+ ? (elt ? scalar_to_vec : vec_construct) : vector_load,
+ NULL, vectype, 0, vect_prologue);
nelt = 0;
}
}
if (vect_is_simple_use (op, vinfo, &dt)
&& (dt == vect_constant_def || dt == vect_external_def))
prologue_cost += vect_prologue_cost_for_slp_op (vinfo, node,
- stmt_info,
i, dt, cost_vec);
}
}
{
if (slp_node)
prologue_cost += vect_prologue_cost_for_slp_op (vinfo, slp_node,
- stmt_info,
1, dt, cost_vec);
else
prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
void
dump_stmt_cost (FILE *f, void *data, int count, enum vect_cost_for_stmt kind,
- stmt_vec_info stmt_info, int misalign, unsigned cost,
+ stmt_vec_info stmt_info, tree, int misalign, unsigned cost,
enum vect_cost_model_location where)
{
fprintf (f, "%p ", data);
enum vect_cost_for_stmt kind;
enum vect_cost_model_location where;
stmt_vec_info stmt_info;
+ tree vectype;
int misalign;
};
}
extern void dump_stmt_cost (FILE *, void *, int, enum vect_cost_for_stmt,
- stmt_vec_info, int, unsigned,
+ stmt_vec_info, tree, int, unsigned,
enum vect_cost_model_location);
/* Alias targetm.vectorize.add_stmt_cost. */
static inline unsigned
add_stmt_cost (vec_info *vinfo, void *data, int count,
enum vect_cost_for_stmt kind,
- stmt_vec_info stmt_info, int misalign,
+ stmt_vec_info stmt_info, tree vectype, int misalign,
enum vect_cost_model_location where)
{
unsigned cost = targetm.vectorize.add_stmt_cost (vinfo, data, count, kind,
- stmt_info, misalign, where);
+ stmt_info, vectype,
+ misalign, where);
if (dump_file && (dump_flags & TDF_DETAILS))
- dump_stmt_cost (dump_file, data, count, kind, stmt_info, misalign,
+ dump_stmt_cost (dump_file, data, count, kind, stmt_info, vectype, misalign,
cost, where);
return cost;
}
unsigned i;
FOR_EACH_VEC_ELT (*cost_vec, i, cost)
add_stmt_cost (vinfo, data, cost->count, cost->kind, cost->stmt_info,
- cost->misalign, cost->where);
+ cost->vectype, cost->misalign, cost->where);
}
/*-----------------------------------------------------------------*/
extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
enum tree_code *, int *,
vec<tree> *);
+
extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
enum vect_cost_for_stmt, stmt_vec_info,
- int, enum vect_cost_model_location);
+ tree, int, enum vect_cost_model_location);
+
+/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
+
+static inline unsigned
+record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
+ enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
+ int misalign, enum vect_cost_model_location where)
+{
+ return record_stmt_cost (body_cost_vec, count, kind, stmt_info,
+ STMT_VINFO_VECTYPE (stmt_info), misalign, where);
+}
+
extern stmt_vec_info vect_finish_replace_stmt (vec_info *,
stmt_vec_info, gimple *);
extern stmt_vec_info vect_finish_stmt_generation (vec_info *,
projects / gcc.git / commitdiff