From e31cd607e999ca6ab47b7e65a7045b1594e4fba4 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Wed, 27 May 2020 15:38:20 +0200 Subject: [PATCH] Code generate externals/invariants during the SLP graph walk This generates vector defs for externals and invariants during the SLP walk rather than as part of getting vectorized defs when vectorizing the users. This is a requirement to make sharing of external/invariant nodes be reflected in actual code generation. This temporarily adds a SLP_TREE_VEC_DEFS vector alongside the SLP_TREE_VEC_STMTS one. Eventually the latter can go away. 2020-05-27 Richard Biener * tree-vectorizer.h (_slp_tree::vec_defs): Add. (SLP_TREE_VEC_DEFS): Likewise. * tree-vect-slp.c (_slp_tree::_slp_tree): Adjust. (_slp_tree::~_slp_tree): Likewise. (vect_mask_constant_operand_p): Remove unused function. (vect_get_constant_vectors): Rename to... (vect_create_constant_vectors): ... this. Take the invariant node as argument and code generate it. Remove dead code, remove temporary asserts. Pass a NULL stmt_info to vect_init_vector. (vect_get_slp_defs): Simplify. (vect_schedule_slp_instance): Code-generate externals and invariants using vect_create_constant_vectors. --- gcc/tree-vect-slp.c | 158 +++++++++++------------------------------- gcc/tree-vectorizer.h | 2 + 2 files changed, 42 insertions(+), 118 deletions(-) diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index a6c5a9d9dc4..aa95c0a7f75 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -53,7 +53,8 @@ _slp_tree::_slp_tree () { SLP_TREE_SCALAR_STMTS (this) = vNULL; SLP_TREE_SCALAR_OPS (this) = vNULL; - SLP_TREE_VEC_STMTS (this).create (0); + SLP_TREE_VEC_STMTS (this) = vNULL; + SLP_TREE_VEC_DEFS (this) = vNULL; SLP_TREE_NUMBER_OF_VEC_STMTS (this) = 0; SLP_TREE_CHILDREN (this) = vNULL; SLP_TREE_LOAD_PERMUTATION (this) = vNULL; @@ -72,6 +73,7 @@ _slp_tree::~_slp_tree () SLP_TREE_SCALAR_STMTS (this).release (); SLP_TREE_SCALAR_OPS (this).release (); SLP_TREE_VEC_STMTS (this).release (); + SLP_TREE_VEC_DEFS (this).release (); SLP_TREE_LOAD_PERMUTATION (this).release (); } @@ -3480,56 +3482,6 @@ vect_slp_bb (basic_block bb) } -/* Return 1 if vector type STMT_VINFO is a boolean vector. */ - -static bool -vect_mask_constant_operand_p (vec_info *vinfo, - stmt_vec_info stmt_vinfo, unsigned op_num) -{ - enum tree_code code = gimple_expr_code (stmt_vinfo->stmt); - tree op, vectype; - enum vect_def_type dt; - - /* For comparison and COND_EXPR type is chosen depending - on the non-constant other comparison operand. */ - if (TREE_CODE_CLASS (code) == tcc_comparison) - { - gassign *stmt = as_a (stmt_vinfo->stmt); - op = gimple_assign_rhs1 (stmt); - - if (!vect_is_simple_use (op, vinfo, &dt, &vectype)) - gcc_unreachable (); - - return !vectype || VECTOR_BOOLEAN_TYPE_P (vectype); - } - - if (code == COND_EXPR) - { - gassign *stmt = as_a (stmt_vinfo->stmt); - tree cond = gimple_assign_rhs1 (stmt); - - if (TREE_CODE (cond) == SSA_NAME) - { - if (op_num > 0) - return VECTOR_BOOLEAN_TYPE_P (STMT_VINFO_VECTYPE (stmt_vinfo)); - op = cond; - } - else - { - if (op_num > 1) - return VECTOR_BOOLEAN_TYPE_P (STMT_VINFO_VECTYPE (stmt_vinfo)); - op = TREE_OPERAND (cond, 0); - } - - if (!vect_is_simple_use (op, vinfo, &dt, &vectype)) - gcc_unreachable (); - - return !vectype || VECTOR_BOOLEAN_TYPE_P (vectype); - } - - return VECTOR_BOOLEAN_TYPE_P (STMT_VINFO_VECTYPE (stmt_vinfo)); -} - /* Build a variable-length vector in which the elements in ELTS are repeated to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in RESULTS and add any new instructions to SEQ. @@ -3644,18 +3596,13 @@ duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type, } -/* For constant and loop invariant defs of SLP_NODE this function returns - (vector) defs (VEC_OPRNDS) that will be used in the vectorized stmts. - OP_NODE determines the node for the operand containing the scalar - operands. */ +/* For constant and loop invariant defs in OP_NODE this function creates + vector defs that will be used in the vectorized stmts and stores them + to SLP_TREE_VEC_DEFS of OP_NODE. */ static void -vect_get_constant_vectors (vec_info *vinfo, - slp_tree slp_node, unsigned op_num, - vec *vec_oprnds) +vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node) { - slp_tree op_node = SLP_TREE_CHILDREN (slp_node)[op_num]; - stmt_vec_info stmt_vinfo = SLP_TREE_SCALAR_STMTS (slp_node)[0]; unsigned HOST_WIDE_INT nunits; tree vec_cst; unsigned j, number_of_places_left_in_vector; @@ -3665,29 +3612,14 @@ vect_get_constant_vectors (vec_info *vinfo, unsigned int vec_num, i; unsigned number_of_copies = 1; bool constant_p; - tree neutral_op = NULL; gimple_seq ctor_seq = NULL; auto_vec permute_results; /* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */ vector_type = SLP_TREE_VECTYPE (op_node); - { - tree op = op_node->ops[0]; - tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo); - if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op)) - && vect_mask_constant_operand_p (vinfo, stmt_vinfo, op_num)) - gcc_assert (vector_type - && types_compatible_p (vector_type, - truth_type_for (stmt_vectype))); - else - gcc_assert (vector_type - && types_compatible_p (vector_type, - get_vectype_for_scalar_type - (vinfo, TREE_TYPE (op), op_node))); - } unsigned int number_of_vectors = SLP_TREE_NUMBER_OF_VEC_STMTS (op_node); - vec_oprnds->create (number_of_vectors); + SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors); auto_vec voprnds (number_of_vectors); /* NUMBER_OF_COPIES is the number of times we need to use the same values in @@ -3812,11 +3744,11 @@ vect_get_constant_vectors (vec_info *vinfo, gimple_stmt_iterator gsi = gsi_for_stmt (insert_after->stmt); /* vect_init_vector inserts before. */ gsi_next (&gsi); - init = vect_init_vector (vinfo, stmt_vinfo, vec_cst, + init = vect_init_vector (vinfo, NULL, vec_cst, vector_type, &gsi); } else - init = vect_init_vector (vinfo, stmt_vinfo, vec_cst, + init = vect_init_vector (vinfo, NULL, vec_cst, vector_type, NULL); if (ctor_seq != NULL) { @@ -3841,30 +3773,17 @@ vect_get_constant_vectors (vec_info *vinfo, for (j = vec_num; j != 0; j--) { vop = voprnds[j - 1]; - vec_oprnds->quick_push (vop); + SLP_TREE_VEC_DEFS (op_node).quick_push (vop); } /* In case that VF is greater than the unrolling factor needed for the SLP group of stmts, NUMBER_OF_VECTORS to be created is greater than NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have to replicate the vectors. */ - while (number_of_vectors > vec_oprnds->length ()) - { - tree neutral_vec = NULL; - - if (neutral_op) - { - if (!neutral_vec) - neutral_vec = build_vector_from_val (vector_type, neutral_op); - - vec_oprnds->quick_push (neutral_vec); - } - else - { - for (i = 0; vec_oprnds->iterate (i, &vop) && i < vec_num; i++) - vec_oprnds->quick_push (vop); - } - } + while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ()) + for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num; + i++) + SLP_TREE_VEC_DEFS (op_node).quick_push (vop); } @@ -3884,15 +3803,10 @@ vect_get_slp_vect_defs (slp_tree slp_node, vec *vec_oprnds) } -/* Get N vectorized definitions for SLP_NODE. - If the scalar definitions are loop invariants or constants, collect them and - call vect_get_constant_vectors() to create vector stmts. - Otherwise, the def-stmts must be already vectorized and the vectorized stmts - must be stored in the corresponding child of SLP_NODE, and we call - vect_get_slp_vect_defs () to retrieve them. */ +/* Get N vectorized definitions for SLP_NODE. */ void -vect_get_slp_defs (vec_info *vinfo, +vect_get_slp_defs (vec_info *, slp_tree slp_node, vec > *vec_oprnds, unsigned n) { if (n == -1U) @@ -3906,13 +3820,11 @@ vect_get_slp_defs (vec_info *vinfo, /* For each operand we check if it has vectorized definitions in a child node or we need to create them (for invariants and constants). */ + vec_defs.create (SLP_TREE_NUMBER_OF_VEC_STMTS (child)); if (SLP_TREE_DEF_TYPE (child) == vect_internal_def) - { - vec_defs.create (SLP_TREE_NUMBER_OF_VEC_STMTS (child)); - vect_get_slp_vect_defs (child, &vec_defs); - } + vect_get_slp_vect_defs (child, &vec_defs); else - vect_get_constant_vectors (vinfo, slp_node, i, &vec_defs); + vec_defs.splice (SLP_TREE_VEC_DEFS (child)); vec_oprnds->quick_push (vec_defs); } @@ -4137,20 +4049,30 @@ vect_schedule_slp_instance (vec_info *vinfo, slp_tree node, slp_instance instance) { gimple_stmt_iterator si; - stmt_vec_info stmt_info; - unsigned int group_size; - tree vectype; int i, j; slp_tree child; - if (SLP_TREE_DEF_TYPE (node) != vect_internal_def) - return; - /* See if we have already vectorized the node in the graph of the SLP instance. */ - if (SLP_TREE_VEC_STMTS (node).exists ()) + if ((SLP_TREE_DEF_TYPE (node) == vect_internal_def + && SLP_TREE_VEC_STMTS (node).exists ()) + || SLP_TREE_VEC_DEFS (node).exists ()) return; + /* Vectorize externals and constants. */ + if (SLP_TREE_DEF_TYPE (node) == vect_constant_def + || SLP_TREE_DEF_TYPE (node) == vect_external_def) + { + /* ??? vectorizable_shift can end up using a scalar operand which is + currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the + node in this case. */ + if (!SLP_TREE_VECTYPE (node)) + return; + + vect_create_constant_vectors (vinfo, node); + return; + } + FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child) vect_schedule_slp_instance (vinfo, child, instance); @@ -4163,12 +4085,12 @@ vect_schedule_slp_instance (vec_info *vinfo, STMT_VINFO_DEF_TYPE (child_stmt_info) = SLP_TREE_DEF_TYPE (child); } - stmt_info = SLP_TREE_SCALAR_STMTS (node)[0]; + stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0]; /* VECTYPE is the type of the destination. */ - vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = STMT_VINFO_VECTYPE (stmt_info); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); - group_size = SLP_TREE_SCALAR_STMTS (node).length (); + unsigned group_size = SLP_TREE_SCALAR_STMTS (node).length (); gcc_assert (SLP_TREE_NUMBER_OF_VEC_STMTS (node) != 0); SLP_TREE_VEC_STMTS (node).create (SLP_TREE_NUMBER_OF_VEC_STMTS (node)); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 2eb3ab5d280..2bde71760e5 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -136,6 +136,7 @@ struct _slp_tree { tree vectype; /* Vectorized stmt/s. */ vec vec_stmts; + vec vec_defs; /* Number of vector stmts that are created to replace the group of scalar stmts. It is calculated during the transformation phase as the number of scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF @@ -186,6 +187,7 @@ public: #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts #define SLP_TREE_SCALAR_OPS(S) (S)->ops #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts +#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size #define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation #define SLP_TREE_TWO_OPERATORS(S) (S)->two_operators -- 2.30.2