From ee62a5a60299fdc12c884337dd36767fd2230ac0 Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Thu, 3 Dec 2015 14:31:55 +0000 Subject: [PATCH] Add an rsqrt_optab and IFN_RSQRT internal function All current uses of builtin_reciprocal convert 1.0/sqrt into rsqrt. This patch adds an rsqrt optab and associated internal function for that instead. We can then pick up the vector forms of rsqrt automatically, fixing an AArch64 regression from my internal_fn patches. With that change, builtin_reciprocal only needs to handle target-specific built-in functions. I've restricted the hook to those since, if we need a reciprocal of another standard function later, I think there should be a strong preference for adding a new optab and internal function for it, rather than hiding the code in a backend. Three targets implement builtin_reciprocal: aarch64, i386 and rs6000. i386 and rs6000 already used the obvious rsqrt2 pattern names for the instructions, so they pick up the new code automatically. aarch64 needs a slight rename. mn10300 is unusual in that its native operation is rsqrt, and sqrt is approximated as 1.0/rsqrt. The port also uses rsqrt2 for the rsqrt pattern, so after the patch we now pick it up as a native operation. Two other ports define rsqrt patterns: sh and v850. AFAICT these patterns aren't currently used, but I think the patch does what the authors of the patterns would have expected. There's obviously some risk of fallout though. Tested on x86_64-linux-gnu, aarch64-linux-gnu, arm-linux-gnueabihf (as a target without the hooks) and powerpc64-linux-gnu. gcc/ * internal-fn.def (RSQRT): New function. * optabs.def (rsqrt_optab): New optab. * doc/md.texi (rsqrtM2): Document. * target.def (builtin_reciprocal): Replace gcall argument with a function decl. Restrict hook to machine functions. * doc/tm.texi: Regenerate. * targhooks.h (default_builtin_reciprocal): Update prototype. * targhooks.c (default_builtin_reciprocal): Likewise. * tree-ssa-math-opts.c: Include internal-fn.h. (internal_fn_reciprocal): New function. (pass_cse_reciprocals::execute): Call it, and build a call to an internal function on success. Only call targetm.builtin_reciprocal for machine functions. * config/aarch64/aarch64-protos.h (aarch64_builtin_rsqrt): Remove second argument. * config/aarch64/aarch64-builtins.c (aarch64_expand_builtin_rsqrt): Rename aarch64_rsqrt_2 to rsqrt2. (aarch64_builtin_rsqrt): Remove md_fn argument and only handle machine functions. * config/aarch64/aarch64.c (use_rsqrt_p): New function. (aarch64_builtin_reciprocal): Replace gcall argument with a function decl. Use use_rsqrt_p. Remove optimize_size check. Only handle machine functions. Update call to aarch64_builtin_rsqrt. (aarch64_optab_supported_p): New function. (TARGET_OPTAB_SUPPORTED_P): Define. * config/aarch64/aarch64-simd.md (aarch64_rsqrt_2): Rename to... (rsqrt2): ...this. * config/i386/i386.c (use_rsqrt_p): New function. (ix86_builtin_reciprocal): Replace gcall argument with a function decl. Use use_rsqrt_p. Remove optimize_insn_for_size_p check. Only handle machine functions. (ix86_optab_supported_p): Handle rsqrt_optab. * config/rs6000/rs6000.c (TARGET_OPTAB_SUPPORTED_P): Define. (rs6000_builtin_reciprocal): Replace gcall argument with a function decl. Remove optimize_insn_for_size_p check. Only handle machine functions. (rs6000_optab_supported_p): New function. From-SVN: r231229 --- gcc/ChangeLog | 40 +++++++++++ gcc/config/aarch64/aarch64-builtins.c | 34 ++++------ gcc/config/aarch64/aarch64-protos.h | 2 +- gcc/config/aarch64/aarch64-simd.md | 2 +- gcc/config/aarch64/aarch64.c | 49 ++++++++++---- gcc/config/i386/i386.c | 79 ++++++++-------------- gcc/config/rs6000/rs6000.c | 95 +++++++++------------------ gcc/doc/md.texi | 12 ++++ gcc/doc/tm.texi | 6 +- gcc/internal-fn.def | 2 + gcc/optabs.def | 1 + gcc/target.def | 8 +-- gcc/targhooks.c | 2 +- gcc/targhooks.h | 2 +- gcc/tree-ssa-math-opts.c | 79 ++++++++++++++++------ 15 files changed, 228 insertions(+), 185 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 6e9c823d3aa..b2e43ff4bc7 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,43 @@ +2015-12-03 Richard Sandiford + + * internal-fn.def (RSQRT): New function. + * optabs.def (rsqrt_optab): New optab. + * doc/md.texi (rsqrtM2): Document. + * target.def (builtin_reciprocal): Replace gcall argument with + a function decl. Restrict hook to machine functions. + * doc/tm.texi: Regenerate. + * targhooks.h (default_builtin_reciprocal): Update prototype. + * targhooks.c (default_builtin_reciprocal): Likewise. + * tree-ssa-math-opts.c: Include internal-fn.h. + (internal_fn_reciprocal): New function. + (pass_cse_reciprocals::execute): Call it, and build a call to an + internal function on success. Only call targetm.builtin_reciprocal + for machine functions. + * config/aarch64/aarch64-protos.h (aarch64_builtin_rsqrt): Remove + second argument. + * config/aarch64/aarch64-builtins.c (aarch64_expand_builtin_rsqrt): + Rename aarch64_rsqrt_2 to rsqrt2. + (aarch64_builtin_rsqrt): Remove md_fn argument and only handle + machine functions. + * config/aarch64/aarch64.c (use_rsqrt_p): New function. + (aarch64_builtin_reciprocal): Replace gcall argument with a + function decl. Use use_rsqrt_p. Remove optimize_size check. + Only handle machine functions. Update call to aarch64_builtin_rsqrt. + (aarch64_optab_supported_p): New function. + (TARGET_OPTAB_SUPPORTED_P): Define. + * config/aarch64/aarch64-simd.md (aarch64_rsqrt_2): Rename to... + (rsqrt2): ...this. + * config/i386/i386.c (use_rsqrt_p): New function. + (ix86_builtin_reciprocal): Replace gcall argument with a + function decl. Use use_rsqrt_p. Remove optimize_insn_for_size_p + check. Only handle machine functions. + (ix86_optab_supported_p): Handle rsqrt_optab. + * config/rs6000/rs6000.c (TARGET_OPTAB_SUPPORTED_P): Define. + (rs6000_builtin_reciprocal): Replace gcall argument with a + function decl. Remove optimize_insn_for_size_p check. + Only handle machine functions. + (rs6000_optab_supported_p): New function. + 2015-12-03 Bernd Schmidt PR target/68471 diff --git a/gcc/config/aarch64/aarch64-builtins.c b/gcc/config/aarch64/aarch64-builtins.c index 45011f61d9b..b268a6aaf7c 100644 --- a/gcc/config/aarch64/aarch64-builtins.c +++ b/gcc/config/aarch64/aarch64-builtins.c @@ -1176,19 +1176,19 @@ aarch64_expand_builtin_rsqrt (int fcode, tree exp, rtx target) switch (fcode) { case AARCH64_BUILTIN_RSQRT_DF: - gen = gen_aarch64_rsqrt_df2; + gen = gen_rsqrtdf2; break; case AARCH64_BUILTIN_RSQRT_SF: - gen = gen_aarch64_rsqrt_sf2; + gen = gen_rsqrtsf2; break; case AARCH64_BUILTIN_RSQRT_V2DF: - gen = gen_aarch64_rsqrt_v2df2; + gen = gen_rsqrtv2df2; break; case AARCH64_BUILTIN_RSQRT_V2SF: - gen = gen_aarch64_rsqrt_v2sf2; + gen = gen_rsqrtv2sf2; break; case AARCH64_BUILTIN_RSQRT_V4SF: - gen = gen_aarch64_rsqrt_v4sf2; + gen = gen_rsqrtv4sf2; break; default: gcc_unreachable (); } @@ -1405,24 +1405,14 @@ aarch64_builtin_vectorized_function (unsigned int fn, tree type_out, /* Return builtin for reciprocal square root. */ tree -aarch64_builtin_rsqrt (unsigned int fn, bool md_fn) +aarch64_builtin_rsqrt (unsigned int fn) { - if (md_fn) - { - if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df) - return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF]; - if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf) - return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF]; - if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf) - return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF]; - } - else - { - if (fn == BUILT_IN_SQRT) - return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_DF]; - if (fn == BUILT_IN_SQRTF) - return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_SF]; - } + if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df) + return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF]; + if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf) + return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF]; + if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf) + return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF]; return NULL_TREE; } diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h index e0a050ce5bc..e6bfe06bc15 100644 --- a/gcc/config/aarch64/aarch64-protos.h +++ b/gcc/config/aarch64/aarch64-protos.h @@ -407,7 +407,7 @@ rtx aarch64_expand_builtin (tree exp, int ignore ATTRIBUTE_UNUSED); tree aarch64_builtin_decl (unsigned, bool ATTRIBUTE_UNUSED); -tree aarch64_builtin_rsqrt (unsigned int, bool); +tree aarch64_builtin_rsqrt (unsigned int); tree aarch64_builtin_vectorized_function (unsigned int, tree, tree); diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index ae1075c2895..030a1013caa 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -399,7 +399,7 @@ "frsqrts\\t%0, %1, %2" [(set_attr "type" "neon_fp_rsqrts_")]) -(define_expand "aarch64_rsqrt_2" +(define_expand "rsqrt2" [(set (match_operand:VALLF 0 "register_operand" "=w") (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")] UNSPEC_RSQRT))] diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 88dbe158c73..88f3ef8c8e8 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -7099,26 +7099,27 @@ aarch64_memory_move_cost (machine_mode mode ATTRIBUTE_UNUSED, return aarch64_tune_params.memmov_cost; } +/* Return true if it is safe and beneficial to use the rsqrt optabs to + optimize 1.0/sqrt. */ + +static bool +use_rsqrt_p (void) +{ + return (!flag_trapping_math + && flag_unsafe_math_optimizations + && (aarch64_tune_params.extra_tuning_flags + & AARCH64_EXTRA_TUNE_RECIP_SQRT)); +} + /* Function to decide when to use reciprocal square root builtins. */ static tree -aarch64_builtin_reciprocal (gcall *call) +aarch64_builtin_reciprocal (tree fndecl) { - if (flag_trapping_math - || !flag_unsafe_math_optimizations - || optimize_size - || ! (aarch64_tune_params.extra_tuning_flags - & AARCH64_EXTRA_TUNE_RECIP_SQRT)) + if (!use_rsqrt_p ()) return NULL_TREE; - - if (gimple_call_internal_p (call)) - return NULL_TREE; - - tree fndecl = gimple_call_fndecl (call); - enum built_in_function fn = DECL_FUNCTION_CODE (fndecl); - bool md_fn = DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD; - return aarch64_builtin_rsqrt (fn, md_fn); + return aarch64_builtin_rsqrt (DECL_FUNCTION_CODE (fndecl)); } typedef rtx (*rsqrte_type) (rtx, rtx); @@ -13546,6 +13547,23 @@ aarch64_promoted_type (const_tree t) return float_type_node; return NULL_TREE; } + +/* Implement the TARGET_OPTAB_SUPPORTED_P hook. */ + +static bool +aarch64_optab_supported_p (int op, machine_mode, machine_mode, + optimization_type opt_type) +{ + switch (op) + { + case rsqrt_optab: + return opt_type == OPTIMIZE_FOR_SPEED && use_rsqrt_p (); + + default: + return true; + } +} + #undef TARGET_ADDRESS_COST #define TARGET_ADDRESS_COST aarch64_address_cost @@ -13866,6 +13884,9 @@ aarch64_promoted_type (const_tree t) #undef TARGET_PRINT_OPERAND_ADDRESS #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address +#undef TARGET_OPTAB_SUPPORTED_P +#define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p + struct gcc_target targetm = TARGET_INITIALIZER; #include "gt-aarch64.h" diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c index 96d6c98d935..cd44375556e 100644 --- a/gcc/config/i386/i386.c +++ b/gcc/config/i386/i386.c @@ -42676,66 +42676,36 @@ ix86_vectorize_builtin_scatter (const_tree vectype, return ix86_builtins[code]; } +/* Return true if it is safe to use the rsqrt optabs to optimize + 1.0/sqrt. */ + +static bool +use_rsqrt_p () +{ + return (TARGET_SSE_MATH + && flag_finite_math_only + && !flag_trapping_math + && flag_unsafe_math_optimizations); +} + /* Returns a code for a target-specific builtin that implements reciprocal of the function, or NULL_TREE if not available. */ static tree -ix86_builtin_reciprocal (gcall *call) +ix86_builtin_reciprocal (tree fndecl) { - if (! (TARGET_SSE_MATH && !optimize_insn_for_size_p () - && flag_finite_math_only && !flag_trapping_math - && flag_unsafe_math_optimizations)) - return NULL_TREE; - - if (gimple_call_internal_p (call)) - switch (gimple_call_internal_fn (call)) - { - tree type; - case IFN_SQRT: - type = TREE_TYPE (gimple_call_lhs (call)); - switch (TYPE_MODE (type)) - { - /* Vectorized version of sqrt to rsqrt conversion. */ - case V4SFmode: - return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR); - - case V8SFmode: - return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR256); - - default: - return NULL_TREE; - } - - default: - return NULL_TREE; - } - - tree fndecl = gimple_call_fndecl (call); - if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD) - /* Machine dependent builtins. */ - switch (DECL_FUNCTION_CODE (fndecl)) - { - /* Vectorized version of sqrt to rsqrt conversion. */ - case IX86_BUILTIN_SQRTPS_NR: - return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR); + switch (DECL_FUNCTION_CODE (fndecl)) + { + /* Vectorized version of sqrt to rsqrt conversion. */ + case IX86_BUILTIN_SQRTPS_NR: + return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR); - case IX86_BUILTIN_SQRTPS_NR256: - return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR256); + case IX86_BUILTIN_SQRTPS_NR256: + return ix86_get_builtin (IX86_BUILTIN_RSQRTPS_NR256); - default: - return NULL_TREE; - } - else - /* Normal builtins. */ - switch (DECL_FUNCTION_CODE (fndecl)) - { - /* Sqrt to rsqrt conversion. */ - case BUILT_IN_SQRTF: - return ix86_get_builtin (IX86_BUILTIN_RSQRTF); - - default: - return NULL_TREE; - } + default: + return NULL_TREE; + } } /* Helper for avx_vpermilps256_operand et al. This is also used by @@ -54139,6 +54109,9 @@ ix86_optab_supported_p (int op, machine_mode mode1, machine_mode, return true; return opt_type == OPTIMIZE_FOR_SPEED; + case rsqrt_optab: + return opt_type == OPTIMIZE_FOR_SPEED && use_rsqrt_p (); + default: return true; } diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c index 22eb0e5b713..2b2c170d6e9 100644 --- a/gcc/config/rs6000/rs6000.c +++ b/gcc/config/rs6000/rs6000.c @@ -1722,6 +1722,9 @@ static const struct attribute_spec rs6000_attribute_table[] = #undef TARGET_INVALID_BINARY_OP #define TARGET_INVALID_BINARY_OP rs6000_invalid_binary_op + +#undef TARGET_OPTAB_SUPPORTED_P +#define TARGET_OPTAB_SUPPORTED_P rs6000_optab_supported_p /* Processor table. */ @@ -32713,77 +32716,25 @@ rs6000_memory_move_cost (machine_mode mode, reg_class_t rclass, reciprocal of the function, or NULL_TREE if not available. */ static tree -rs6000_builtin_reciprocal (gcall *call) +rs6000_builtin_reciprocal (tree fndecl) { - if (optimize_insn_for_size_p ()) - return NULL_TREE; - - if (gimple_call_internal_p (call)) - switch (gimple_call_internal_fn (call)) - { - tree type; - case IFN_SQRT: - type = TREE_TYPE (gimple_call_lhs (call)); - switch (TYPE_MODE (type)) - { - case V2DFmode: - if (!RS6000_RECIP_AUTO_RSQRTE_P (V2DFmode)) - return NULL_TREE; - - return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_2DF]; - - case V4SFmode: - if (!RS6000_RECIP_AUTO_RSQRTE_P (V4SFmode)) - return NULL_TREE; - - return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_4SF]; - - default: - return NULL_TREE; - } - - default: + switch (DECL_FUNCTION_CODE (fndecl)) + { + case VSX_BUILTIN_XVSQRTDP: + if (!RS6000_RECIP_AUTO_RSQRTE_P (V2DFmode)) return NULL_TREE; - } - - tree fndecl = gimple_call_fndecl (call); - if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD) - switch (DECL_FUNCTION_CODE (fndecl)) - { - case VSX_BUILTIN_XVSQRTDP: - if (!RS6000_RECIP_AUTO_RSQRTE_P (V2DFmode)) - return NULL_TREE; - - return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_2DF]; - case VSX_BUILTIN_XVSQRTSP: - if (!RS6000_RECIP_AUTO_RSQRTE_P (V4SFmode)) - return NULL_TREE; + return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_2DF]; - return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_4SF]; - - default: + case VSX_BUILTIN_XVSQRTSP: + if (!RS6000_RECIP_AUTO_RSQRTE_P (V4SFmode)) return NULL_TREE; - } - - else - switch (DECL_FUNCTION_CODE (fndecl)) - { - case BUILT_IN_SQRT: - if (!RS6000_RECIP_AUTO_RSQRTE_P (DFmode)) - return NULL_TREE; - - return rs6000_builtin_decls[RS6000_BUILTIN_RSQRT]; - - case BUILT_IN_SQRTF: - if (!RS6000_RECIP_AUTO_RSQRTE_P (SFmode)) - return NULL_TREE; - return rs6000_builtin_decls[RS6000_BUILTIN_RSQRTF]; + return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_4SF]; - default: - return NULL_TREE; - } + default: + return NULL_TREE; + } } /* Load up a constant. If the mode is a vector mode, splat the value across @@ -38007,6 +37958,22 @@ rs6000_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update) *update = build2 (COMPOUND_EXPR, void_type_node, update_mffs, update_mtfsf); } +/* Implement the TARGET_OPTAB_SUPPORTED_P hook. */ + +static bool +rs6000_optab_supported_p (int op, machine_mode mode1, machine_mode, + optimization_type opt_type) +{ + switch (op) + { + case rsqrt_optab: + return (opt_type == OPTIMIZE_FOR_SPEED + && RS6000_RECIP_AUTO_RSQRTE_P (mode1)); + + default: + return true; + } +} struct gcc_target targetm = TARGET_INITIALIZER; diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi index 8e3f8f58e5d..dcb3ee0c69b 100644 --- a/gcc/doc/md.texi +++ b/gcc/doc/md.texi @@ -5331,6 +5331,18 @@ corresponds to the C data type @code{double} and the @code{sqrtf} built-in function uses the mode which corresponds to the C data type @code{float}. +@cindex @code{rsqrt@var{m}2} instruction pattern +@item @samp{rsqrt@var{m}2} +Store the reciprocal of the square root of operand 1 into operand 0. +On most architectures this pattern is only approximate, so either +its C condition or the @code{TARGET_OPTAB_SUPPORTED_P} hook should +check for the appropriate math flags. (Using the C condition is +more direct, but using @code{TARGET_OPTAB_SUPPORTED_P} can be useful +if a target-specific built-in also uses the @samp{rsqrt@var{m}2} +pattern.) + +This pattern is not allowed to @code{FAIL}. + @cindex @code{fmod@var{m}3} instruction pattern @item @samp{fmod@var{m}3} Store the remainder of dividing operand 1 by operand 2 into diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi index a84ad5723b6..1ce7181c3d4 100644 --- a/gcc/doc/tm.texi +++ b/gcc/doc/tm.texi @@ -5608,9 +5608,9 @@ be placed in an @code{object_block} structure. The default version returns true for all decls. @end deftypefn -@deftypefn {Target Hook} tree TARGET_BUILTIN_RECIPROCAL (gcall *@var{call}) -This hook should return the DECL of a function that implements reciprocal of -the builtin or internal function call @var{call}, or +@deftypefn {Target Hook} tree TARGET_BUILTIN_RECIPROCAL (tree @var{fndecl}) +This hook should return the DECL of a function that implements the +reciprocal of the machine-specific builtin function @var{fndecl}, or @code{NULL_TREE} if such a function is not available. @end deftypefn diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index dee9332c3d6..a9118b30b5a 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -91,6 +91,8 @@ DEF_INTERNAL_OPTAB_FN (LOAD_LANES, ECF_CONST, vec_load_lanes, load_lanes) DEF_INTERNAL_OPTAB_FN (MASK_STORE, 0, maskstore, mask_store) DEF_INTERNAL_OPTAB_FN (STORE_LANES, ECF_CONST, vec_store_lanes, store_lanes) +DEF_INTERNAL_OPTAB_FN (RSQRT, ECF_CONST, rsqrt, unary) + /* Unary math functions. */ DEF_INTERNAL_FLT_FN (ACOS, ECF_CONST, acos, unary) DEF_INTERNAL_FLT_FN (ASIN, ECF_CONST, asin, unary) diff --git a/gcc/optabs.def b/gcc/optabs.def index 8feb3942ac5..a19466e3611 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -267,6 +267,7 @@ OPTAB_D (log_optab, "log$a2") OPTAB_D (logb_optab, "logb$a2") OPTAB_D (pow_optab, "pow$a3") OPTAB_D (remainder_optab, "remainder$a3") +OPTAB_D (rsqrt_optab, "rsqrt$a2") OPTAB_D (scalb_optab, "scalb$a3") OPTAB_D (signbit_optab, "signbit$F$a2") OPTAB_D (significand_optab, "significand$a2") diff --git a/gcc/target.def b/gcc/target.def index 197189282b1..d7543378fe7 100644 --- a/gcc/target.def +++ b/gcc/target.def @@ -2459,13 +2459,13 @@ identical versions.", tree, (void *decl), NULL) /* Returns a code for a target-specific builtin that implements - reciprocal of the function, or NULL_TREE if not available. */ + reciprocal of a target-specific function, or NULL_TREE if not available. */ DEFHOOK (builtin_reciprocal, - "This hook should return the DECL of a function that implements reciprocal of\n\ -the builtin or internal function call @var{call}, or\n\ + "This hook should return the DECL of a function that implements the\n\ +reciprocal of the machine-specific builtin function @var{fndecl}, or\n\ @code{NULL_TREE} if such a function is not available.", - tree, (gcall *call), + tree, (tree fndecl), default_builtin_reciprocal) /* For a vendor-specific TYPE, return a pointer to a statically-allocated diff --git a/gcc/targhooks.c b/gcc/targhooks.c index 70456096542..dcf08631c40 100644 --- a/gcc/targhooks.c +++ b/gcc/targhooks.c @@ -600,7 +600,7 @@ default_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost, /* Reciprocal. */ tree -default_builtin_reciprocal (gcall *) +default_builtin_reciprocal (tree) { return NULL_TREE; } diff --git a/gcc/targhooks.h b/gcc/targhooks.h index 281b5fed1fb..47b5cfc3b8b 100644 --- a/gcc/targhooks.h +++ b/gcc/targhooks.h @@ -90,7 +90,7 @@ extern tree default_builtin_vectorized_conversion (unsigned int, tree, tree); extern int default_builtin_vectorization_cost (enum vect_cost_for_stmt, tree, int); -extern tree default_builtin_reciprocal (gcall *); +extern tree default_builtin_reciprocal (tree); extern HOST_WIDE_INT default_vector_alignment (const_tree); diff --git a/gcc/tree-ssa-math-opts.c b/gcc/tree-ssa-math-opts.c index 66d750138a0..b00f046074b 100644 --- a/gcc/tree-ssa-math-opts.c +++ b/gcc/tree-ssa-math-opts.c @@ -110,6 +110,7 @@ along with GCC; see the file COPYING3. If not see #include "tree-ssa.h" #include "builtins.h" #include "params.h" +#include "internal-fn.h" #include "case-cfn-macros.h" /* This structure represents one basic block that either computes a @@ -497,6 +498,31 @@ execute_cse_reciprocals_1 (gimple_stmt_iterator *def_gsi, tree def) occ_head = NULL; } +/* Return an internal function that implements the reciprocal of CALL, + or IFN_LAST if there is no such function that the target supports. */ + +internal_fn +internal_fn_reciprocal (gcall *call) +{ + internal_fn ifn; + + switch (gimple_call_combined_fn (call)) + { + CASE_CFN_SQRT: + ifn = IFN_RSQRT; + break; + + default: + return IFN_LAST; + } + + tree_pair types = direct_internal_fn_types (ifn, call); + if (!direct_internal_fn_supported_p (ifn, types, OPTIMIZE_FOR_SPEED)) + return IFN_LAST; + + return ifn; +} + /* Go through all the floating-point SSA_NAMEs, and call execute_cse_reciprocals_1 on each of them. */ namespace { @@ -586,7 +612,6 @@ pass_cse_reciprocals::execute (function *fun) gsi_next (&gsi)) { gimple *stmt = gsi_stmt (gsi); - tree fndecl; if (is_gimple_assign (stmt) && gimple_assign_rhs_code (stmt) == RDIV_EXPR) @@ -600,20 +625,25 @@ pass_cse_reciprocals::execute (function *fun) stmt1 = SSA_NAME_DEF_STMT (arg1); if (is_gimple_call (stmt1) - && gimple_call_lhs (stmt1) - && (gimple_call_internal_p (stmt1) - || ((fndecl = gimple_call_fndecl (stmt1)) - && (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL - || (DECL_BUILT_IN_CLASS (fndecl) - == BUILT_IN_MD))))) + && gimple_call_lhs (stmt1)) { bool fail; imm_use_iterator ui; use_operand_p use_p; + tree fndecl = NULL_TREE; - fndecl = targetm.builtin_reciprocal (as_a (stmt1)); - if (!fndecl) - continue; + gcall *call = as_a (stmt1); + internal_fn ifn = internal_fn_reciprocal (call); + if (ifn == IFN_LAST) + { + fndecl = gimple_call_fndecl (call); + if (!fndecl + || DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_MD) + continue; + fndecl = targetm.builtin_reciprocal (fndecl); + if (!fndecl) + continue; + } /* Check that all uses of the SSA name are divisions, otherwise replacing the defining statement will do @@ -636,28 +666,35 @@ pass_cse_reciprocals::execute (function *fun) if (fail) continue; - gimple_replace_ssa_lhs (stmt1, arg1); - if (gimple_call_internal_p (stmt1)) + gimple_replace_ssa_lhs (call, arg1); + if (gimple_call_internal_p (call) != (ifn != IFN_LAST)) { auto_vec args; for (unsigned int i = 0; - i < gimple_call_num_args (stmt1); i++) - args.safe_push (gimple_call_arg (stmt1, i)); - gcall *stmt2 = gimple_build_call_vec (fndecl, args); + i < gimple_call_num_args (call); i++) + args.safe_push (gimple_call_arg (call, i)); + gcall *stmt2; + if (ifn == IFN_LAST) + stmt2 = gimple_build_call_vec (fndecl, args); + else + stmt2 = gimple_build_call_internal_vec (ifn, args); gimple_call_set_lhs (stmt2, arg1); - if (gimple_vdef (stmt1)) + if (gimple_vdef (call)) { - gimple_set_vdef (stmt2, gimple_vdef (stmt1)); + gimple_set_vdef (stmt2, gimple_vdef (call)); SSA_NAME_DEF_STMT (gimple_vdef (stmt2)) = stmt2; } - gimple_set_vuse (stmt2, gimple_vuse (stmt1)); - gimple_stmt_iterator gsi2 = gsi_for_stmt (stmt1); + gimple_set_vuse (stmt2, gimple_vuse (call)); + gimple_stmt_iterator gsi2 = gsi_for_stmt (call); gsi_replace (&gsi2, stmt2, true); } else { - gimple_call_set_fndecl (stmt1, fndecl); - update_stmt (stmt1); + if (ifn == IFN_LAST) + gimple_call_set_fndecl (call, fndecl); + else + gimple_call_set_internal_fn (call, ifn); + update_stmt (call); } reciprocal_stats.rfuncs_inserted++; -- 2.30.2