From f09552335030433018fd5f7f6b9848339b5ca2da Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Thu, 14 Nov 2019 14:36:26 +0000 Subject: [PATCH] Add a targetm.vectorize.related_mode hook This patch is the first of a series that tries to remove two assumptions: (1) that all vectors involved in vectorisation must be the same size (2) that there is only one vector mode for a given element mode and number of elements Relaxing (1) helps with targets that support multiple vector sizes or that require the number of elements to stay the same. E.g. if we're vectorising code that operates on narrow and wide elements, and the narrow elements use 64-bit vectors, then on AArch64 it would normally be better to use 128-bit vectors rather than pairs of 64-bit vectors for the wide elements. Relaxing (2) makes it possible for -msve-vector-bits=128 to produce fixed-length code for SVE. It also allows unpacked/half-size SVE vectors to work with -msve-vector-bits=256. The patch adds a new hook that targets can use to control how we move from one vector mode to another. The hook takes a starting vector mode, a new element mode, and (optionally) a new number of elements. The flexibility needed for (1) comes in when the number of elements isn't specified. All callers in this patch specify the number of elements, but a later vectoriser patch doesn't. 2019-11-14 Richard Sandiford gcc/ * target.def (related_mode): New hook. * doc/tm.texi.in (TARGET_VECTORIZE_RELATED_MODE): New hook. * doc/tm.texi: Regenerate. * targhooks.h (default_vectorize_related_mode): Declare. * targhooks.c (default_vectorize_related_mode): New function. * machmode.h (related_vector_mode): Declare. * stor-layout.c (related_vector_mode): New function. * expmed.c (extract_bit_field_1): Use it instead of mode_for_vector. * optabs-query.c (qimode_for_vec_perm): Likewise. * tree-vect-stmts.c (get_group_load_store_type): Likewise. (vectorizable_store, vectorizable_load): Likewise From-SVN: r278229 --- gcc/ChangeLog | 14 ++++++++++++++ gcc/doc/tm.texi | 24 ++++++++++++++++++++++++ gcc/doc/tm.texi.in | 2 ++ gcc/expmed.c | 8 +++----- gcc/machmode.h | 2 ++ gcc/optabs-query.c | 7 ++----- gcc/stor-layout.c | 20 ++++++++++++++++++++ gcc/target.def | 27 +++++++++++++++++++++++++++ gcc/targhooks.c | 19 +++++++++++++++++++ gcc/targhooks.h | 3 +++ gcc/tree-vect-stmts.c | 25 ++++++++++--------------- 11 files changed, 126 insertions(+), 25 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index b4427fa5cd2..54f244902a2 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,17 @@ +2019-11-14 Richard Sandiford + + * target.def (related_mode): New hook. + * doc/tm.texi.in (TARGET_VECTORIZE_RELATED_MODE): New hook. + * doc/tm.texi: Regenerate. + * targhooks.h (default_vectorize_related_mode): Declare. + * targhooks.c (default_vectorize_related_mode): New function. + * machmode.h (related_vector_mode): Declare. + * stor-layout.c (related_vector_mode): New function. + * expmed.c (extract_bit_field_1): Use it instead of mode_for_vector. + * optabs-query.c (qimode_for_vec_perm): Likewise. + * tree-vect-stmts.c (get_group_load_store_type): Likewise. + (vectorizable_store, vectorizable_load): Likewise + 2019-11-14 Richard Henderson * config/arm/aarch-common-protos.h (arm_md_asm_adjust): Declare. diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi index 11c236e1c65..5586e2edae2 100644 --- a/gcc/doc/tm.texi +++ b/gcc/doc/tm.texi @@ -6021,6 +6021,30 @@ The hook does not need to do anything if the vector returned by for autovectorization. The default implementation does nothing. @end deftypefn +@deftypefn {Target Hook} opt_machine_mode TARGET_VECTORIZE_RELATED_MODE (machine_mode @var{vector_mode}, scalar_mode @var{element_mode}, poly_uint64 @var{nunits}) +If a piece of code is using vector mode @var{vector_mode} and also wants +to operate on elements of mode @var{element_mode}, return the vector mode +it should use for those elements. If @var{nunits} is nonzero, ensure that +the mode has exactly @var{nunits} elements, otherwise pick whichever vector +size pairs the most naturally with @var{vector_mode}. Return an empty +@code{opt_machine_mode} if there is no supported vector mode with the +required properties. + +There is no prescribed way of handling the case in which @var{nunits} +is zero. One common choice is to pick a vector mode with the same size +as @var{vector_mode}; this is the natural choice if the target has a +fixed vector size. Another option is to choose a vector mode with the +same number of elements as @var{vector_mode}; this is the natural choice +if the target has a fixed number of elements. Alternatively, the hook +might choose a middle ground, such as trying to keep the number of +elements as similar as possible while applying maximum and minimum +vector sizes. + +The default implementation uses @code{mode_for_vector} to find the +requested mode, returning a mode with the same size as @var{vector_mode} +when @var{nunits} is zero. This is the correct behavior for most targets. +@end deftypefn + @deftypefn {Target Hook} opt_machine_mode TARGET_VECTORIZE_GET_MASK_MODE (poly_uint64 @var{nunits}, poly_uint64 @var{length}) A vector mask is a value that holds one boolean result for every element in a vector. This hook returns the machine mode that should be used to diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in index b8c41b5a7aa..727b88c70ce 100644 --- a/gcc/doc/tm.texi.in +++ b/gcc/doc/tm.texi.in @@ -4177,6 +4177,8 @@ address; but often a machine-dependent strategy can generate better code. @hook TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES +@hook TARGET_VECTORIZE_RELATED_MODE + @hook TARGET_VECTORIZE_GET_MASK_MODE @hook TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE diff --git a/gcc/expmed.c b/gcc/expmed.c index ff8554b1562..512944ef418 100644 --- a/gcc/expmed.c +++ b/gcc/expmed.c @@ -1643,12 +1643,10 @@ extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum, poly_uint64 nunits; if (!multiple_p (GET_MODE_BITSIZE (GET_MODE (op0)), GET_MODE_UNIT_BITSIZE (tmode), &nunits) - || !mode_for_vector (inner_mode, nunits).exists (&new_mode) - || !VECTOR_MODE_P (new_mode) + || !related_vector_mode (tmode, inner_mode, + nunits).exists (&new_mode) || maybe_ne (GET_MODE_SIZE (new_mode), - GET_MODE_SIZE (GET_MODE (op0))) - || GET_MODE_INNER (new_mode) != GET_MODE_INNER (tmode) - || !targetm.vector_mode_supported_p (new_mode)) + GET_MODE_SIZE (GET_MODE (op0)))) new_mode = VOIDmode; } poly_uint64 pos; diff --git a/gcc/machmode.h b/gcc/machmode.h index 64ac213fbac..906b2161cfa 100644 --- a/gcc/machmode.h +++ b/gcc/machmode.h @@ -880,6 +880,8 @@ extern opt_scalar_int_mode int_mode_for_mode (machine_mode); extern opt_machine_mode bitwise_mode_for_mode (machine_mode); extern opt_machine_mode mode_for_vector (scalar_mode, poly_uint64); extern opt_machine_mode mode_for_int_vector (unsigned int, poly_uint64); +extern opt_machine_mode related_vector_mode (machine_mode, scalar_mode, + poly_uint64 = 0); /* Return the integer vector equivalent of MODE, if one exists. In other words, return the mode for an integer vector that has the same number diff --git a/gcc/optabs-query.c b/gcc/optabs-query.c index 6465b5cf6c5..e610e969b78 100644 --- a/gcc/optabs-query.c +++ b/gcc/optabs-query.c @@ -354,11 +354,8 @@ can_conditionally_move_p (machine_mode mode) opt_machine_mode qimode_for_vec_perm (machine_mode mode) { - machine_mode qimode; - if (GET_MODE_INNER (mode) != QImode - && mode_for_vector (QImode, GET_MODE_SIZE (mode)).exists (&qimode) - && VECTOR_MODE_P (qimode)) - return qimode; + if (GET_MODE_INNER (mode) != QImode) + return related_vector_mode (mode, QImode, GET_MODE_SIZE (mode)); return opt_machine_mode (); } diff --git a/gcc/stor-layout.c b/gcc/stor-layout.c index 9aada97b038..c1b724f682c 100644 --- a/gcc/stor-layout.c +++ b/gcc/stor-layout.c @@ -530,6 +530,26 @@ mode_for_int_vector (unsigned int int_bits, poly_uint64 nunits) return opt_machine_mode (); } +/* If a piece of code is using vector mode VECTOR_MODE and also wants + to operate on elements of mode ELEMENT_MODE, return the vector mode + it should use for those elements. If NUNITS is nonzero, ensure that + the mode has exactly NUNITS elements, otherwise pick whichever vector + size pairs the most naturally with VECTOR_MODE; this may mean choosing + a mode with a different size and/or number of elements, depending on + what the target prefers. Return an empty opt_machine_mode if there + is no supported vector mode with the required properties. + + Unlike mode_for_vector. any returned mode is guaranteed to satisfy + both VECTOR_MODE_P and targetm.vector_mode_supported_p. */ + +opt_machine_mode +related_vector_mode (machine_mode vector_mode, scalar_mode element_mode, + poly_uint64 nunits) +{ + gcc_assert (VECTOR_MODE_P (vector_mode)); + return targetm.vectorize.related_mode (vector_mode, element_mode, nunits); +} + /* Return the alignment of MODE. This will be bounded by 1 and BIGGEST_ALIGNMENT. */ diff --git a/gcc/target.def b/gcc/target.def index 8e83c2c7a71..569aaa5da8e 100644 --- a/gcc/target.def +++ b/gcc/target.def @@ -1924,6 +1924,33 @@ for autovectorization. The default implementation does nothing.", (vector_sizes *sizes, bool all), default_autovectorize_vector_sizes) +DEFHOOK +(related_mode, + "If a piece of code is using vector mode @var{vector_mode} and also wants\n\ +to operate on elements of mode @var{element_mode}, return the vector mode\n\ +it should use for those elements. If @var{nunits} is nonzero, ensure that\n\ +the mode has exactly @var{nunits} elements, otherwise pick whichever vector\n\ +size pairs the most naturally with @var{vector_mode}. Return an empty\n\ +@code{opt_machine_mode} if there is no supported vector mode with the\n\ +required properties.\n\ +\n\ +There is no prescribed way of handling the case in which @var{nunits}\n\ +is zero. One common choice is to pick a vector mode with the same size\n\ +as @var{vector_mode}; this is the natural choice if the target has a\n\ +fixed vector size. Another option is to choose a vector mode with the\n\ +same number of elements as @var{vector_mode}; this is the natural choice\n\ +if the target has a fixed number of elements. Alternatively, the hook\n\ +might choose a middle ground, such as trying to keep the number of\n\ +elements as similar as possible while applying maximum and minimum\n\ +vector sizes.\n\ +\n\ +The default implementation uses @code{mode_for_vector} to find the\n\ +requested mode, returning a mode with the same size as @var{vector_mode}\n\ +when @var{nunits} is zero. This is the correct behavior for most targets.", + opt_machine_mode, + (machine_mode vector_mode, scalar_mode element_mode, poly_uint64 nunits), + default_vectorize_related_mode) + /* Function to get a target mode for a vector mask. */ DEFHOOK (get_mask_mode, diff --git a/gcc/targhooks.c b/gcc/targhooks.c index b6443d204a6..dcecd81e54d 100644 --- a/gcc/targhooks.c +++ b/gcc/targhooks.c @@ -1306,6 +1306,25 @@ default_autovectorize_vector_sizes (vector_sizes *, bool) { } +/* The default implementation of TARGET_VECTORIZE_RELATED_MODE. */ + +opt_machine_mode +default_vectorize_related_mode (machine_mode vector_mode, + scalar_mode element_mode, + poly_uint64 nunits) +{ + machine_mode result_mode; + if ((maybe_ne (nunits, 0U) + || multiple_p (GET_MODE_SIZE (vector_mode), + GET_MODE_SIZE (element_mode), &nunits)) + && mode_for_vector (element_mode, nunits).exists (&result_mode) + && VECTOR_MODE_P (result_mode) + && targetm.vector_mode_supported_p (result_mode)) + return result_mode; + + return opt_machine_mode (); +} + /* By default a vector of integers is used as a mask. */ opt_machine_mode diff --git a/gcc/targhooks.h b/gcc/targhooks.h index d4c3563e825..12e6ad49cf0 100644 --- a/gcc/targhooks.h +++ b/gcc/targhooks.h @@ -114,6 +114,9 @@ default_builtin_support_vector_misalignment (machine_mode mode, extern machine_mode default_preferred_simd_mode (scalar_mode mode); extern machine_mode default_split_reduction (machine_mode); extern void default_autovectorize_vector_sizes (vector_sizes *, bool); +extern opt_machine_mode default_vectorize_related_mode (machine_mode, + scalar_mode, + poly_uint64); extern opt_machine_mode default_get_mask_mode (poly_uint64, poly_uint64); extern bool default_empty_mask_is_expensive (unsigned); extern void *default_init_cost (class loop *); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index c8a43ada16f..73fe573ea3e 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -2309,9 +2309,8 @@ get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp, || alignment_support_scheme == dr_unaligned_supported) && known_eq (nunits, (group_size - gap) * 2) && known_eq (nunits, group_size) - && mode_for_vector (elmode, (group_size - gap)).exists (&vmode) - && VECTOR_MODE_P (vmode) - && targetm.vector_mode_supported_p (vmode) + && related_vector_mode (TYPE_MODE (vectype), elmode, + group_size - gap).exists (&vmode) && (convert_optab_handler (vec_init_optab, TYPE_MODE (vectype), vmode) != CODE_FOR_nothing)) @@ -7811,9 +7810,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, of vector elts directly. */ scalar_mode elmode = SCALAR_TYPE_MODE (elem_type); machine_mode vmode; - if (!mode_for_vector (elmode, group_size).exists (&vmode) - || !VECTOR_MODE_P (vmode) - || !targetm.vector_mode_supported_p (vmode) + if (!related_vector_mode (TYPE_MODE (vectype), elmode, + group_size).exists (&vmode) || (convert_optab_handler (vec_extract_optab, TYPE_MODE (vectype), vmode) == CODE_FOR_nothing)) @@ -7830,9 +7828,8 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, element extracts from the original vector type and element size stores. */ if (int_mode_for_size (lsize, 0).exists (&elmode) - && mode_for_vector (elmode, lnunits).exists (&vmode) - && VECTOR_MODE_P (vmode) - && targetm.vector_mode_supported_p (vmode) + && related_vector_mode (TYPE_MODE (vectype), elmode, + lnunits).exists (&vmode) && (convert_optab_handler (vec_extract_optab, vmode, elmode) != CODE_FOR_nothing)) @@ -8913,9 +8910,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, vector elts directly. */ scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vectype)); machine_mode vmode; - if (mode_for_vector (elmode, group_size).exists (&vmode) - && VECTOR_MODE_P (vmode) - && targetm.vector_mode_supported_p (vmode) + if (related_vector_mode (TYPE_MODE (vectype), elmode, + group_size).exists (&vmode) && (convert_optab_handler (vec_init_optab, TYPE_MODE (vectype), vmode) != CODE_FOR_nothing)) @@ -8939,9 +8935,8 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, /* If we can't construct such a vector fall back to element loads of the original vector type. */ if (int_mode_for_size (lsize, 0).exists (&elmode) - && mode_for_vector (elmode, lnunits).exists (&vmode) - && VECTOR_MODE_P (vmode) - && targetm.vector_mode_supported_p (vmode) + && related_vector_mode (TYPE_MODE (vectype), elmode, + lnunits).exists (&vmode) && (convert_optab_handler (vec_init_optab, vmode, elmode) != CODE_FOR_nothing)) { -- 2.30.2