From b41d1f6ed753bf7ae7e68f745e50c26ee65b5711 Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Thu, 12 Jul 2018 13:01:33 +0000 Subject: [PATCH] Add IFN_COND_FMA functions This patch adds conditional equivalents of the IFN_FMA built-in functions. Most of it is just a mechanical extension of the binary stuff. 2018-07-12 Richard Sandiford gcc/ * doc/md.texi (cond_fma, cond_fms, cond_fnma, cond_fnms): Document. * optabs.def (cond_fma_optab, cond_fms_optab, cond_fnma_optab) (cond_fnms_optab): New optabs. * internal-fn.def (COND_FMA, COND_FMS, COND_FNMA, COND_FNMS): New internal functions. (FMA): Use DEF_INTERNAL_FLT_FN rather than DEF_INTERNAL_FLT_FLOATN_FN. * internal-fn.h (get_conditional_internal_fn): Declare. (get_unconditional_internal_fn): Likewise. * internal-fn.c (cond_ternary_direct): New macro. (expand_cond_ternary_optab_fn): Likewise. (direct_cond_ternary_optab_supported_p): Likewise. (FOR_EACH_COND_FN_PAIR): Likewise. (get_conditional_internal_fn): New function. (get_unconditional_internal_fn): Likewise. * gimple-match.h (gimple_match_op::MAX_NUM_OPS): Bump to 5. (gimple_match_op::gimple_match_op): Add a new overload for 5 operands. (gimple_match_op::set_op): Likewise. (gimple_resimplify5): Declare. * genmatch.c (decision_tree::gen): Generate simplifications for 5 operands. * gimple-match-head.c (gimple_simplify): Define an overload for 5 operands. Handle calls with 5 arguments in the top-level overload. (convert_conditional_op): Handle conversions from unconditional internal functions to conditional ones. (gimple_resimplify5): New function. (build_call_internal): Pass a fifth operand. (maybe_push_res_to_seq): Likewise. (try_conditional_simplification): Try converting conditional internal functions to unconditional internal functions. Handle 3-operand unconditional forms. * match.pd (UNCOND_TERNARY, COND_TERNARY): Operator lists. Define ternary equivalents of the current rules for binary conditional internal functions. * config/aarch64/aarch64.c (aarch64_preferred_else_value): Handle ternary operations. * config/aarch64/iterators.md (UNSPEC_COND_FMLA, UNSPEC_COND_FMLS) (UNSPEC_COND_FNMLA, UNSPEC_COND_FNMLS): New unspecs. (optab): Handle them. (SVE_COND_FP_TERNARY): New int iterator. (sve_fmla_op, sve_fmad_op): New int attributes. * config/aarch64/aarch64-sve.md (cond_) (*cond__2, *cond__any): New SVE_COND_FP_TERNARY patterns. gcc/testsuite/ * gcc.dg/vect/vect-cond-arith-3.c: New test. * gcc.target/aarch64/sve/vcond_13.c: Likewise. * gcc.target/aarch64/sve/vcond_13_run.c: Likewise. * gcc.target/aarch64/sve/vcond_14.c: Likewise. * gcc.target/aarch64/sve/vcond_14_run.c: Likewise. * gcc.target/aarch64/sve/vcond_15.c: Likewise. * gcc.target/aarch64/sve/vcond_15_run.c: Likewise. * gcc.target/aarch64/sve/vcond_16.c: Likewise. * gcc.target/aarch64/sve/vcond_16_run.c: Likewise. From-SVN: r262587 --- gcc/ChangeLog | 47 +++++++++ gcc/config/aarch64/aarch64-sve.md | 95 +++++++++++++++++++ gcc/config/aarch64/aarch64.c | 14 ++- gcc/config/aarch64/iterators.md | 25 ++++- gcc/doc/md.texi | 17 ++++ gcc/genmatch.c | 2 +- gcc/gimple-match-head.c | 66 +++++++++++-- gcc/gimple-match.h | 34 ++++++- gcc/internal-fn.c | 56 +++++++++++ gcc/internal-fn.def | 10 +- gcc/internal-fn.h | 2 + gcc/match.pd | 29 ++++++ gcc/optabs.def | 4 + gcc/testsuite/ChangeLog | 12 +++ gcc/testsuite/gcc.dg/vect/vect-cond-arith-3.c | 63 ++++++++++++ .../gcc.target/aarch64/sve/vcond_13.c | 58 +++++++++++ .../gcc.target/aarch64/sve/vcond_13_run.c | 37 ++++++++ .../gcc.target/aarch64/sve/vcond_14.c | 58 +++++++++++ .../gcc.target/aarch64/sve/vcond_14_run.c | 37 ++++++++ .../gcc.target/aarch64/sve/vcond_15.c | 58 +++++++++++ .../gcc.target/aarch64/sve/vcond_15_run.c | 37 ++++++++ .../gcc.target/aarch64/sve/vcond_16.c | 58 +++++++++++ .../gcc.target/aarch64/sve/vcond_16_run.c | 37 ++++++++ 23 files changed, 839 insertions(+), 17 deletions(-) create mode 100644 gcc/testsuite/gcc.dg/vect/vect-cond-arith-3.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_13.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_13_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_14.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_14_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_15.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_15_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_16.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/vcond_16_run.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index a8694e2c234..a12938054c7 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,50 @@ +2018-07-12 Richard Sandiford + + * doc/md.texi (cond_fma, cond_fms, cond_fnma, cond_fnms): Document. + * optabs.def (cond_fma_optab, cond_fms_optab, cond_fnma_optab) + (cond_fnms_optab): New optabs. + * internal-fn.def (COND_FMA, COND_FMS, COND_FNMA, COND_FNMS): New + internal functions. + (FMA): Use DEF_INTERNAL_FLT_FN rather than DEF_INTERNAL_FLT_FLOATN_FN. + * internal-fn.h (get_conditional_internal_fn): Declare. + (get_unconditional_internal_fn): Likewise. + * internal-fn.c (cond_ternary_direct): New macro. + (expand_cond_ternary_optab_fn): Likewise. + (direct_cond_ternary_optab_supported_p): Likewise. + (FOR_EACH_COND_FN_PAIR): Likewise. + (get_conditional_internal_fn): New function. + (get_unconditional_internal_fn): Likewise. + * gimple-match.h (gimple_match_op::MAX_NUM_OPS): Bump to 5. + (gimple_match_op::gimple_match_op): Add a new overload for 5 + operands. + (gimple_match_op::set_op): Likewise. + (gimple_resimplify5): Declare. + * genmatch.c (decision_tree::gen): Generate simplifications for + 5 operands. + * gimple-match-head.c (gimple_simplify): Define an overload for + 5 operands. Handle calls with 5 arguments in the top-level overload. + (convert_conditional_op): Handle conversions from unconditional + internal functions to conditional ones. + (gimple_resimplify5): New function. + (build_call_internal): Pass a fifth operand. + (maybe_push_res_to_seq): Likewise. + (try_conditional_simplification): Try converting conditional + internal functions to unconditional internal functions. + Handle 3-operand unconditional forms. + * match.pd (UNCOND_TERNARY, COND_TERNARY): Operator lists. + Define ternary equivalents of the current rules for binary conditional + internal functions. + * config/aarch64/aarch64.c (aarch64_preferred_else_value): Handle + ternary operations. + * config/aarch64/iterators.md (UNSPEC_COND_FMLA, UNSPEC_COND_FMLS) + (UNSPEC_COND_FNMLA, UNSPEC_COND_FNMLS): New unspecs. + (optab): Handle them. + (SVE_COND_FP_TERNARY): New int iterator. + (sve_fmla_op, sve_fmad_op): New int attributes. + * config/aarch64/aarch64-sve.md (cond_) + (*cond__2, *cond__any): New SVE_COND_FP_TERNARY patterns. + 2018-07-12 Richard Sandiford * target.def (preferred_else_value): New target hook. diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md index b16d0455159..5cd591b9433 100644 --- a/gcc/config/aarch64/aarch64-sve.md +++ b/gcc/config/aarch64/aarch64-sve.md @@ -2906,6 +2906,101 @@ UNSPEC_SEL))] ) +;; Predicated floating-point ternary operations with select. +(define_expand "cond_" + [(set (match_operand:SVE_F 0 "register_operand") + (unspec:SVE_F + [(match_operand: 1 "register_operand") + (unspec:SVE_F + [(match_operand:SVE_F 2 "register_operand") + (match_operand:SVE_F 3 "register_operand") + (match_operand:SVE_F 4 "register_operand")] + SVE_COND_FP_TERNARY) + (match_operand:SVE_F 5 "aarch64_simd_reg_or_zero")] + UNSPEC_SEL))] + "TARGET_SVE" +{ + /* Swap the multiplication operands if the fallback value is the + second of the two. */ + if (rtx_equal_p (operands[3], operands[5])) + std::swap (operands[2], operands[3]); +}) + +;; Predicated floating-point ternary operations using the FMAD-like form. +(define_insn "*cond__2" + [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") + (unspec:SVE_F + [(match_operand: 1 "register_operand" "Upl, Upl") + (unspec:SVE_F + [(match_operand:SVE_F 2 "register_operand" "0, w") + (match_operand:SVE_F 3 "register_operand" "w, w") + (match_operand:SVE_F 4 "register_operand" "w, w")] + SVE_COND_FP_TERNARY) + (match_dup 2)] + UNSPEC_SEL))] + "TARGET_SVE" + "@ + \t%0., %1/m, %3., %4. + movprfx\t%0, %2\;\t%0., %1/m, %3., %4." + [(set_attr "movprfx" "*,yes")] +) + +;; Predicated floating-point ternary operations using the FMLA-like form. +(define_insn "*cond__4" + [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") + (unspec:SVE_F + [(match_operand: 1 "register_operand" "Upl, Upl") + (unspec:SVE_F + [(match_operand:SVE_F 2 "register_operand" "w, w") + (match_operand:SVE_F 3 "register_operand" "w, w") + (match_operand:SVE_F 4 "register_operand" "0, w")] + SVE_COND_FP_TERNARY) + (match_dup 4)] + UNSPEC_SEL))] + "TARGET_SVE" + "@ + \t%0., %1/m, %2., %3. + movprfx\t%0, %4\;\t%0., %1/m, %2., %3." + [(set_attr "movprfx" "*,yes")] +) + +;; Predicated floating-point ternary operations in which the value for +;; inactive lanes is distinct from the other inputs. +(define_insn_and_split "*cond__any" + [(set (match_operand:SVE_F 0 "register_operand" "=&w, &w, ?&w") + (unspec:SVE_F + [(match_operand: 1 "register_operand" "Upl, Upl, Upl") + (unspec:SVE_F + [(match_operand:SVE_F 2 "register_operand" "w, w, w") + (match_operand:SVE_F 3 "register_operand" "w, w, w") + (match_operand:SVE_F 4 "register_operand" "w, w, w")] + SVE_COND_FP_TERNARY) + (match_operand:SVE_F 5 "aarch64_simd_reg_or_zero" "Dz, 0, w")] + UNSPEC_SEL))] + "TARGET_SVE + && !rtx_equal_p (operands[2], operands[5]) + && !rtx_equal_p (operands[3], operands[5]) + && !rtx_equal_p (operands[4], operands[5])" + "@ + movprfx\t%0., %1/z, %4.\;\t%0., %1/m, %2., %3. + movprfx\t%0., %1/m, %4.\;\t%0., %1/m, %2., %3. + #" + "&& reload_completed + && !CONSTANT_P (operands[5]) + && !rtx_equal_p (operands[0], operands[5])" + [(set (match_dup 0) + (unspec:SVE_F [(match_dup 1) (match_dup 4) (match_dup 5)] UNSPEC_SEL)) + (set (match_dup 0) + (unspec:SVE_F + [(match_dup 1) + (unspec:SVE_F [(match_dup 2) (match_dup 3) (match_dup 0)] + SVE_COND_FP_TERNARY) + (match_dup 0)] + UNSPEC_SEL))] + "" + [(set_attr "movprfx" "yes")] +) + ;; Shift an SVE vector left and insert a scalar into element 0. (define_insn "vec_shl_insert_" [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w") diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 6ef0cc75018..e15b4e83d20 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -1320,14 +1320,18 @@ aarch64_get_mask_mode (poly_uint64 nunits, poly_uint64 nbytes) return default_get_mask_mode (nunits, nbytes); } -/* Implement TARGET_PREFERRED_ELSE_VALUE. Prefer to use the first - arithmetic operand as the else value if the else value doesn't matter, - since that exactly matches the SVE destructive merging form. */ +/* Implement TARGET_PREFERRED_ELSE_VALUE. For binary operations, + prefer to use the first arithmetic operand as the else value if + the else value doesn't matter, since that exactly matches the SVE + destructive merging form. For ternary operations we could either + pick the first operand and use FMAD-like instructions or the last + operand and use FMLA-like instructions; the latter seems more + natural. */ static tree -aarch64_preferred_else_value (unsigned, tree, unsigned int, tree *ops) +aarch64_preferred_else_value (unsigned, tree, unsigned int nops, tree *ops) { - return ops[0]; + return nops == 3 ? ops[2] : ops[0]; } /* Implement TARGET_HARD_REGNO_NREGS. */ diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md index 949fd2aa09d..d846118d3b6 100644 --- a/gcc/config/aarch64/iterators.md +++ b/gcc/config/aarch64/iterators.md @@ -471,6 +471,10 @@ UNSPEC_COND_DIV ; Used in aarch64-sve.md. UNSPEC_COND_MAX ; Used in aarch64-sve.md. UNSPEC_COND_MIN ; Used in aarch64-sve.md. + UNSPEC_COND_FMLA ; Used in aarch64-sve.md. + UNSPEC_COND_FMLS ; Used in aarch64-sve.md. + UNSPEC_COND_FNMLA ; Used in aarch64-sve.md. + UNSPEC_COND_FNMLS ; Used in aarch64-sve.md. UNSPEC_COND_LT ; Used in aarch64-sve.md. UNSPEC_COND_LE ; Used in aarch64-sve.md. UNSPEC_COND_EQ ; Used in aarch64-sve.md. @@ -1567,6 +1571,11 @@ UNSPEC_COND_MUL UNSPEC_COND_DIV UNSPEC_COND_MAX UNSPEC_COND_MIN]) +(define_int_iterator SVE_COND_FP_TERNARY [UNSPEC_COND_FMLA + UNSPEC_COND_FMLS + UNSPEC_COND_FNMLA + UNSPEC_COND_FNMLS]) + (define_int_iterator SVE_COND_FP_CMP [UNSPEC_COND_LT UNSPEC_COND_LE UNSPEC_COND_EQ UNSPEC_COND_NE UNSPEC_COND_GE UNSPEC_COND_GT]) @@ -1599,7 +1608,11 @@ (UNSPEC_COND_MUL "mul") (UNSPEC_COND_DIV "div") (UNSPEC_COND_MAX "smax") - (UNSPEC_COND_MIN "smin")]) + (UNSPEC_COND_MIN "smin") + (UNSPEC_COND_FMLA "fma") + (UNSPEC_COND_FMLS "fnma") + (UNSPEC_COND_FNMLA "fnms") + (UNSPEC_COND_FNMLS "fms")]) (define_int_attr maxmin_uns [(UNSPEC_UMAXV "umax") (UNSPEC_UMINV "umin") @@ -1826,6 +1839,16 @@ (UNSPEC_COND_MAX "fmaxnm") (UNSPEC_COND_MIN "fminnm")]) +(define_int_attr sve_fmla_op [(UNSPEC_COND_FMLA "fmla") + (UNSPEC_COND_FMLS "fmls") + (UNSPEC_COND_FNMLA "fnmla") + (UNSPEC_COND_FNMLS "fnmls")]) + +(define_int_attr sve_fmad_op [(UNSPEC_COND_FMLA "fmad") + (UNSPEC_COND_FMLS "fmsb") + (UNSPEC_COND_FNMLA "fnmad") + (UNSPEC_COND_FNMLS "fnmsb")]) + (define_int_attr commutative [(UNSPEC_COND_ADD "true") (UNSPEC_COND_SUB "false") (UNSPEC_COND_MUL "true") diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi index c6bf77ef1a8..b4f4a9a9622 100644 --- a/gcc/doc/md.texi +++ b/gcc/doc/md.texi @@ -6438,6 +6438,23 @@ Operands 0, 2, 3 and 4 all have mode @var{m}. Operand 1 is a scalar integer if @var{m} is scalar, otherwise it has the mode returned by @code{TARGET_VECTORIZE_GET_MASK_MODE}. +@cindex @code{cond_fma@var{mode}} instruction pattern +@cindex @code{cond_fms@var{mode}} instruction pattern +@cindex @code{cond_fnma@var{mode}} instruction pattern +@cindex @code{cond_fnms@var{mode}} instruction pattern +@item @samp{cond_fma@var{mode}} +@itemx @samp{cond_fms@var{mode}} +@itemx @samp{cond_fnma@var{mode}} +@itemx @samp{cond_fnms@var{mode}} +Like @samp{cond_add@var{m}}, except that the conditional operation +takes 3 operands rather than two. For example, the vector form of +@samp{cond_fma@var{mode}} is equivalent to: + +@smallexample +for (i = 0; i < GET_MODE_NUNITS (@var{m}); i++) + op0[i] = op1[i] ? fma (op2[i], op3[i], op4[i]) : op5[i]; +@end smallexample + @cindex @code{neg@var{mode}cc} instruction pattern @item @samp{neg@var{mode}cc} Similar to @samp{mov@var{mode}cc} but for conditional negation. Conditionally diff --git a/gcc/genmatch.c b/gcc/genmatch.c index 0492e804fc3..5d6cf0b4d3a 100644 --- a/gcc/genmatch.c +++ b/gcc/genmatch.c @@ -3750,7 +3750,7 @@ decision_tree::gen (FILE *f, bool gimple) } fprintf (stderr, "removed %u duplicate tails\n", rcnt); - for (unsigned n = 1; n <= 4; ++n) + for (unsigned n = 1; n <= 5; ++n) { /* First generate split-out functions. */ for (unsigned i = 0; i < root->kids.length (); i++) diff --git a/gcc/gimple-match-head.c b/gcc/gimple-match-head.c index e165a77132b..414007ec1f9 100644 --- a/gcc/gimple-match-head.c +++ b/gcc/gimple-match-head.c @@ -54,6 +54,8 @@ static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree), code_helper, tree, tree, tree, tree); static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree), code_helper, tree, tree, tree, tree, tree); +static bool gimple_simplify (gimple_match_op *, gimple_seq *, tree (*)(tree), + code_helper, tree, tree, tree, tree, tree, tree); const unsigned int gimple_match_op::MAX_NUM_OPS; @@ -80,7 +82,12 @@ convert_conditional_op (gimple_match_op *orig_op, if (orig_op->code.is_tree_code ()) ifn = get_conditional_internal_fn ((tree_code) orig_op->code); else - return false; + { + combined_fn cfn = orig_op->code; + if (!internal_fn_p (cfn)) + return false; + ifn = get_conditional_internal_fn (as_internal_fn (cfn)); + } if (ifn == IFN_LAST) return false; unsigned int num_ops = orig_op->num_ops; @@ -403,6 +410,34 @@ gimple_resimplify4 (gimple_seq *seq, gimple_match_op *res_op, return false; } +/* Helper that matches and simplifies the toplevel result from + a gimple_simplify run (where we don't want to build + a stmt in case it's used in in-place folding). Replaces + RES_OP with a simplified and/or canonicalized result and + returns whether any change was made. */ + +bool +gimple_resimplify5 (gimple_seq *seq, gimple_match_op *res_op, + tree (*valueize)(tree)) +{ + /* No constant folding is defined for five-operand functions. */ + + gimple_match_op res_op2 (*res_op); + if (gimple_simplify (&res_op2, seq, valueize, + res_op->code, res_op->type, + res_op->ops[0], res_op->ops[1], res_op->ops[2], + res_op->ops[3], res_op->ops[4])) + { + *res_op = res_op2; + return true; + } + + if (maybe_resimplify_conditional_op (seq, res_op, valueize)) + return true; + + return false; +} + /* If in GIMPLE the operation described by RES_OP should be single-rhs, build a GENERIC tree for that expression and update RES_OP accordingly. */ @@ -444,7 +479,8 @@ build_call_internal (internal_fn fn, gimple_match_op *res_op) res_op->op_or_null (0), res_op->op_or_null (1), res_op->op_or_null (2), - res_op->op_or_null (3)); + res_op->op_or_null (3), + res_op->op_or_null (4)); } /* Push the exploded expression described by RES_OP as a statement to @@ -538,7 +574,8 @@ maybe_push_res_to_seq (gimple_match_op *res_op, gimple_seq *seq, tree res) res_op->op_or_null (0), res_op->op_or_null (1), res_op->op_or_null (2), - res_op->op_or_null (3)); + res_op->op_or_null (3), + res_op->op_or_null (4)); } if (!res) { @@ -745,14 +782,22 @@ static bool try_conditional_simplification (internal_fn ifn, gimple_match_op *res_op, gimple_seq *seq, tree (*valueize) (tree)) { + code_helper op; tree_code code = conditional_internal_fn_code (ifn); - if (code == ERROR_MARK) - return false; + if (code != ERROR_MARK) + op = code; + else + { + ifn = get_unconditional_internal_fn (ifn); + if (ifn == IFN_LAST) + return false; + op = as_combined_fn (ifn); + } unsigned int num_ops = res_op->num_ops; gimple_match_op cond_op (gimple_match_cond (res_op->ops[0], res_op->ops[num_ops - 1]), - code, res_op->type, num_ops - 2); + op, res_op->type, num_ops - 2); for (unsigned int i = 1; i < num_ops - 1; ++i) cond_op.ops[i - 1] = res_op->ops[i]; switch (num_ops - 2) @@ -761,6 +806,10 @@ try_conditional_simplification (internal_fn ifn, gimple_match_op *res_op, if (!gimple_resimplify2 (seq, &cond_op, valueize)) return false; break; + case 3: + if (!gimple_resimplify3 (seq, &cond_op, valueize)) + return false; + break; default: gcc_unreachable (); } @@ -893,7 +942,7 @@ gimple_simplify (gimple *stmt, gimple_match_op *res_op, gimple_seq *seq, /* ??? This way we can't simplify calls with side-effects. */ if (gimple_call_lhs (stmt) != NULL_TREE && gimple_call_num_args (stmt) >= 1 - && gimple_call_num_args (stmt) <= 4) + && gimple_call_num_args (stmt) <= 5) { bool valueized = false; combined_fn cfn; @@ -943,6 +992,9 @@ gimple_simplify (gimple *stmt, gimple_match_op *res_op, gimple_seq *seq, case 4: return (gimple_resimplify4 (seq, res_op, valueize) || valueized); + case 5: + return (gimple_resimplify5 (seq, res_op, valueize) + || valueized); default: gcc_unreachable (); } diff --git a/gcc/gimple-match.h b/gcc/gimple-match.h index 0fe394d169c..704fa76c0a4 100644 --- a/gcc/gimple-match.h +++ b/gcc/gimple-match.h @@ -91,18 +91,21 @@ struct gimple_match_op code_helper, tree, tree, tree, tree); gimple_match_op (const gimple_match_cond &, code_helper, tree, tree, tree, tree, tree); + gimple_match_op (const gimple_match_cond &, + code_helper, tree, tree, tree, tree, tree, tree); void set_op (code_helper, tree, unsigned int); void set_op (code_helper, tree, tree); void set_op (code_helper, tree, tree, tree); void set_op (code_helper, tree, tree, tree, tree); void set_op (code_helper, tree, tree, tree, tree, tree); + void set_op (code_helper, tree, tree, tree, tree, tree, tree); void set_value (tree); tree op_or_null (unsigned int) const; /* The maximum value of NUM_OPS. */ - static const unsigned int MAX_NUM_OPS = 4; + static const unsigned int MAX_NUM_OPS = 5; /* The conditions under which the operation is performed, and the value to use as a fallback. */ @@ -182,6 +185,20 @@ gimple_match_op::gimple_match_op (const gimple_match_cond &cond_in, ops[3] = op3; } +inline +gimple_match_op::gimple_match_op (const gimple_match_cond &cond_in, + code_helper code_in, tree type_in, + tree op0, tree op1, tree op2, tree op3, + tree op4) + : cond (cond_in), code (code_in), type (type_in), num_ops (5) +{ + ops[0] = op0; + ops[1] = op1; + ops[2] = op2; + ops[3] = op3; + ops[4] = op4; +} + /* Change the operation performed to CODE_IN, the type of the result to TYPE_IN, and the number of operands to NUM_OPS_IN. The caller needs to set the operands itself. */ @@ -242,6 +259,20 @@ gimple_match_op::set_op (code_helper code_in, tree type_in, ops[3] = op3; } +inline void +gimple_match_op::set_op (code_helper code_in, tree type_in, + tree op0, tree op1, tree op2, tree op3, tree op4) +{ + code = code_in; + type = type_in; + num_ops = 5; + ops[0] = op0; + ops[1] = op1; + ops[2] = op2; + ops[3] = op3; + ops[4] = op4; +} + /* Set the "operation" to be the single value VALUE, such as a constant or SSA_NAME. */ @@ -279,6 +310,7 @@ bool gimple_resimplify1 (gimple_seq *, gimple_match_op *, tree (*)(tree)); bool gimple_resimplify2 (gimple_seq *, gimple_match_op *, tree (*)(tree)); bool gimple_resimplify3 (gimple_seq *, gimple_match_op *, tree (*)(tree)); bool gimple_resimplify4 (gimple_seq *, gimple_match_op *, tree (*)(tree)); +bool gimple_resimplify5 (gimple_seq *, gimple_match_op *, tree (*)(tree)); tree maybe_push_res_to_seq (gimple_match_op *, gimple_seq *, tree res = NULL_TREE); void maybe_build_generic_op (gimple_match_op *); diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c index eb5493fcd81..474a16bc6fb 100644 --- a/gcc/internal-fn.c +++ b/gcc/internal-fn.c @@ -113,6 +113,7 @@ init_internal_fns () #define ternary_direct { 0, 0, true } #define cond_unary_direct { 1, 1, true } #define cond_binary_direct { 1, 1, true } +#define cond_ternary_direct { 1, 1, true } #define while_direct { 0, 2, false } #define fold_extract_direct { 2, 2, false } #define fold_left_direct { 1, 1, false } @@ -2993,6 +2994,9 @@ expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab) #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \ expand_direct_optab_fn (FN, STMT, OPTAB, 4) +#define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \ + expand_direct_optab_fn (FN, STMT, OPTAB, 5) + #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \ expand_direct_optab_fn (FN, STMT, OPTAB, 3) @@ -3075,6 +3079,7 @@ multi_vector_optab_supported_p (convert_optab optab, tree_pair types, #define direct_ternary_optab_supported_p direct_optab_supported_p #define direct_cond_unary_optab_supported_p direct_optab_supported_p #define direct_cond_binary_optab_supported_p direct_optab_supported_p +#define direct_cond_ternary_optab_supported_p direct_optab_supported_p #define direct_mask_load_optab_supported_p direct_optab_supported_p #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p @@ -3277,6 +3282,57 @@ conditional_internal_fn_code (internal_fn ifn) } } +/* Invoke T(IFN) for each internal function IFN that also has an + IFN_COND_* form. */ +#define FOR_EACH_COND_FN_PAIR(T) \ + T (FMA) \ + T (FMS) \ + T (FNMA) \ + T (FNMS) + +/* Return a function that only performs internal function FN when a + certain condition is met and that uses a given fallback value otherwise. + In other words, the returned function FN' is such that: + + LHS = FN' (COND, A1, ... An, ELSE) + + is equivalent to the C expression: + + LHS = COND ? FN (A1, ..., An) : ELSE; + + operating elementwise if the operands are vectors. + + Return IFN_LAST if no such function exists. */ + +internal_fn +get_conditional_internal_fn (internal_fn fn) +{ + switch (fn) + { +#define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME; + FOR_EACH_COND_FN_PAIR(CASE) +#undef CASE + default: + return IFN_LAST; + } +} + +/* If IFN implements the conditional form of an unconditional internal + function, return that unconditional function, otherwise return IFN_LAST. */ + +internal_fn +get_unconditional_internal_fn (internal_fn ifn) +{ + switch (ifn) + { +#define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME; + FOR_EACH_COND_FN_PAIR(CASE) +#undef CASE + default: + return IFN_LAST; + } +} + /* Return true if IFN is some form of load from memory. */ bool diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index 6293ab36dc9..cda314e1121 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -59,7 +59,8 @@ along with GCC; see the file COPYING3. If not see - binary: a normal binary optab, such as vec_interleave_lo_ - ternary: a normal ternary optab, such as fma4 - - cond_binary: a conditional binary optab, such as addcc + - cond_binary: a conditional binary optab, such as cond_add + - cond_ternary: a conditional ternary optab, such as cond_fma_rev - fold_left: for scalar = FN (scalar, vector), keyed off the vector mode @@ -167,6 +168,11 @@ DEF_INTERNAL_OPTAB_FN (COND_IOR, ECF_CONST | ECF_NOTHROW, DEF_INTERNAL_OPTAB_FN (COND_XOR, ECF_CONST | ECF_NOTHROW, cond_xor, cond_binary) +DEF_INTERNAL_OPTAB_FN (COND_FMA, ECF_CONST, cond_fma, cond_ternary) +DEF_INTERNAL_OPTAB_FN (COND_FMS, ECF_CONST, cond_fms, cond_ternary) +DEF_INTERNAL_OPTAB_FN (COND_FNMA, ECF_CONST, cond_fnma, cond_ternary) +DEF_INTERNAL_OPTAB_FN (COND_FNMS, ECF_CONST, cond_fnms, cond_ternary) + DEF_INTERNAL_OPTAB_FN (RSQRT, ECF_CONST, rsqrt, unary) DEF_INTERNAL_OPTAB_FN (REDUC_PLUS, ECF_CONST | ECF_NOTHROW, @@ -235,7 +241,7 @@ DEF_INTERNAL_OPTAB_FN (XORSIGN, ECF_CONST, xorsign, binary) DEF_INTERNAL_FLT_FN (LDEXP, ECF_CONST, ldexp, binary) /* Ternary math functions. */ -DEF_INTERNAL_FLT_FN (FMA, ECF_CONST, fma, ternary) +DEF_INTERNAL_FLT_FLOATN_FN (FMA, ECF_CONST, fma, ternary) /* Unary integer ops. */ DEF_INTERNAL_INT_FN (CLRSB, ECF_CONST | ECF_NOTHROW, clrsb, unary) diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h index a1c8b4cbef6..7105c3bbff8 100644 --- a/gcc/internal-fn.h +++ b/gcc/internal-fn.h @@ -193,7 +193,9 @@ direct_internal_fn_supported_p (internal_fn fn, tree type0, tree type1, extern bool set_edom_supported_p (void); extern internal_fn get_conditional_internal_fn (tree_code); +extern internal_fn get_conditional_internal_fn (internal_fn); extern tree_code conditional_internal_fn_code (internal_fn); +extern internal_fn get_unconditional_internal_fn (internal_fn); extern bool internal_load_fn_p (internal_fn); extern bool internal_store_fn_p (internal_fn); diff --git a/gcc/match.pd b/gcc/match.pd index 7e4020eb3ef..dea6cde49ab 100644 --- a/gcc/match.pd +++ b/gcc/match.pd @@ -86,6 +86,12 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) IFN_COND_MUL IFN_COND_DIV IFN_COND_MOD IFN_COND_RDIV IFN_COND_MIN IFN_COND_MAX IFN_COND_AND IFN_COND_IOR IFN_COND_XOR) + +/* Same for ternary operations. */ +(define_operator_list UNCOND_TERNARY + IFN_FMA IFN_FMS IFN_FNMA IFN_FNMS) +(define_operator_list COND_TERNARY + IFN_COND_FMA IFN_COND_FMS IFN_COND_FNMA IFN_COND_FNMS) /* As opposed to convert?, this still creates a single pattern, so it is not a suitable replacement for convert? in all cases. */ @@ -4885,6 +4891,21 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) (if (element_precision (type) == element_precision (op_type)) (view_convert (cond_op (bit_not @0) @2 @3 (view_convert:op_type @1))))))) +/* Same for ternary operations. */ +(for uncond_op (UNCOND_TERNARY) + cond_op (COND_TERNARY) + (simplify + (vec_cond @0 (view_convert? (uncond_op@5 @1 @2 @3)) @4) + (with { tree op_type = TREE_TYPE (@5); } + (if (element_precision (type) == element_precision (op_type)) + (view_convert (cond_op @0 @1 @2 @3 (view_convert:op_type @4)))))) + (simplify + (vec_cond @0 @1 (view_convert? (uncond_op@5 @2 @3 @4))) + (with { tree op_type = TREE_TYPE (@5); } + (if (element_precision (type) == element_precision (op_type)) + (view_convert (cond_op (bit_not @0) @2 @3 @4 + (view_convert:op_type @1))))))) + /* Detect cases in which a VEC_COND_EXPR effectively replaces the "else" value of an IFN_COND_*. */ (for cond_op (COND_BINARY) @@ -4893,3 +4914,11 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT) (with { tree op_type = TREE_TYPE (@3); } (if (element_precision (type) == element_precision (op_type)) (view_convert (cond_op @0 @1 @2 (view_convert:op_type @4))))))) + +/* Same for ternary operations. */ +(for cond_op (COND_TERNARY) + (simplify + (vec_cond @0 (view_convert? (cond_op @0 @1 @2 @3 @4)) @5) + (with { tree op_type = TREE_TYPE (@4); } + (if (element_precision (type) == element_precision (op_type)) + (view_convert (cond_op @0 @1 @2 @3 (view_convert:op_type @5))))))) diff --git a/gcc/optabs.def b/gcc/optabs.def index 707d9696b4c..5a67f5eed5e 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -234,6 +234,10 @@ OPTAB_D (cond_smin_optab, "cond_smin$a") OPTAB_D (cond_smax_optab, "cond_smax$a") OPTAB_D (cond_umin_optab, "cond_umin$a") OPTAB_D (cond_umax_optab, "cond_umax$a") +OPTAB_D (cond_fma_optab, "cond_fma$a") +OPTAB_D (cond_fms_optab, "cond_fms$a") +OPTAB_D (cond_fnma_optab, "cond_fnma$a") +OPTAB_D (cond_fnms_optab, "cond_fnms$a") OPTAB_D (cmov_optab, "cmov$a6") OPTAB_D (cstore_optab, "cstore$a4") OPTAB_D (ctrap_optab, "ctrap$a4") diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 8ce8a41a351..9e5287d4742 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,15 @@ +2018-07-12 Richard Sandiford + + * gcc.dg/vect/vect-cond-arith-3.c: New test. + * gcc.target/aarch64/sve/vcond_13.c: Likewise. + * gcc.target/aarch64/sve/vcond_13_run.c: Likewise. + * gcc.target/aarch64/sve/vcond_14.c: Likewise. + * gcc.target/aarch64/sve/vcond_14_run.c: Likewise. + * gcc.target/aarch64/sve/vcond_15.c: Likewise. + * gcc.target/aarch64/sve/vcond_15_run.c: Likewise. + * gcc.target/aarch64/sve/vcond_16.c: Likewise. + * gcc.target/aarch64/sve/vcond_16_run.c: Likewise. + 2018-07-12 Richard Sandiford * gcc.dg/vect/vect-cond-arith-2.c: New test. diff --git a/gcc/testsuite/gcc.dg/vect/vect-cond-arith-3.c b/gcc/testsuite/gcc.dg/vect/vect-cond-arith-3.c new file mode 100644 index 00000000000..f02b0dc5d3a --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-cond-arith-3.c @@ -0,0 +1,63 @@ +/* { dg-require-effective-target scalar_all_fma } */ +/* { dg-additional-options "-fdump-tree-optimized" } */ + +#include "tree-vect.h" + +#define N (VECTOR_BITS * 11 / 64 + 3) + +#define DEF(INV) \ + void __attribute__ ((noipa)) \ + f_##INV (double *restrict a, double *restrict b, \ + double *restrict c, double *restrict d) \ + { \ + for (int i = 0; i < N; ++i) \ + { \ + double mb = (INV & 1 ? -b[i] : b[i]); \ + double mc = c[i]; \ + double md = (INV & 2 ? -d[i] : d[i]); \ + double fma = __builtin_fma (mb, mc, md); \ + double truev = (INV & 4 ? -fma : fma); \ + a[i] = b[i] < 10 ? truev : 10.0; \ + } \ + } + +#define TEST(INV) \ + { \ + f_##INV (a, b, c, d); \ + for (int i = 0; i < N; ++i) \ + { \ + double mb = (INV & 1 ? -b[i] : b[i]); \ + double mc = c[i]; \ + double md = (INV & 2 ? -d[i] : d[i]); \ + double fma = __builtin_fma (mb, mc, md); \ + double truev = (INV & 4 ? -fma : fma); \ + if (a[i] != (i % 17 < 10 ? truev : 10.0)) \ + __builtin_abort (); \ + asm volatile ("" ::: "memory"); \ + } \ + } + +#define FOR_EACH_INV(T) \ + T (0) T (1) T (2) T (3) T (4) T (5) T (6) T (7) + +FOR_EACH_INV (DEF) + +int +main (void) +{ + double a[N], b[N], c[N], d[N]; + for (int i = 0; i < N; ++i) + { + b[i] = i % 17; + c[i] = i % 9 + 11; + d[i] = i % 13 + 14; + asm volatile ("" ::: "memory"); + } + FOR_EACH_INV (TEST) + return 0; +} + +/* { dg-final { scan-tree-dump-times { = \.COND_FMA } 2 "optimized" { target vect_double_cond_arith } } } */ +/* { dg-final { scan-tree-dump-times { = \.COND_FMS } 2 "optimized" { target vect_double_cond_arith } } } */ +/* { dg-final { scan-tree-dump-times { = \.COND_FNMA } 2 "optimized" { target vect_double_cond_arith } } } */ +/* { dg-final { scan-tree-dump-times { = \.COND_FNMS } 2 "optimized" { target vect_double_cond_arith } } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_13.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_13.c new file mode 100644 index 00000000000..3c3aa00f674 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_13.c @@ -0,0 +1,58 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define N 119 + +#define DEF_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + void __attribute__ ((noipa)) \ + f_##INV##_##SUFFIX (TYPE *restrict a, TYPE *restrict b, \ + TYPE *restrict c, TYPE *restrict d, \ + CMPTYPE *restrict cond) \ + { \ + for (int i = 0; i < N; ++i) \ + { \ + TYPE mb = (INV & 1 ? -b[i] : b[i]); \ + TYPE mc = c[i]; \ + TYPE md = (INV & 2 ? -d[i] : d[i]); \ + TYPE fma = __builtin_fma##SUFFIX (mb, mc, md); \ + TYPE truev = (INV & 4 ? -fma : fma); \ + a[i] = cond[i] < 10 ? truev : b[i]; \ + } \ + } + +#define FOR_EACH_TYPE(T, INV) \ + T (INV, _Float16, short, f16) \ + T (INV, float, float, f32) \ + T (INV, double, double, f64) + +#define FOR_EACH_INV(T) \ + FOR_EACH_TYPE (T, 0) \ + FOR_EACH_TYPE (T, 1) \ + FOR_EACH_TYPE (T, 2) \ + FOR_EACH_TYPE (T, 3) \ + FOR_EACH_TYPE (T, 4) \ + FOR_EACH_TYPE (T, 5) \ + FOR_EACH_TYPE (T, 6) \ + FOR_EACH_TYPE (T, 7) + +FOR_EACH_INV (DEF_LOOP) + +/* { dg-final { scan-assembler-not {\tsel\t} } } */ +/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */ +/* { dg-final { scan-assembler-not {\tmov\tz[0-9]+\.., z[0-9]+} } } */ + +/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.d,} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_13_run.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_13_run.c new file mode 100644 index 00000000000..f0f8385babc --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_13_run.c @@ -0,0 +1,37 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include "vcond_13.c" + +#define TEST_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + { \ + TYPE a[N], b[N], c[N], d[N]; \ + CMPTYPE cond[N]; \ + for (int i = 0; i < N; ++i) \ + { \ + b[i] = i % 15; \ + c[i] = i % 9 + 11; \ + d[i] = i % 13 + 14; \ + cond[i] = i % 17; \ + asm volatile ("" ::: "memory"); \ + } \ + f_##INV##_##SUFFIX (a, b, c, d, cond); \ + for (int i = 0; i < N; ++i) \ + { \ + double mb = (INV & 1 ? -b[i] : b[i]); \ + double mc = c[i]; \ + double md = (INV & 2 ? -d[i] : d[i]); \ + double fma = __builtin_fma (mb, mc, md); \ + double truev = (INV & 4 ? -fma : fma); \ + if (a[i] != (i % 17 < 10 ? truev : b[i])) \ + __builtin_abort (); \ + asm volatile ("" ::: "memory"); \ + } \ + } + +int +main (void) +{ + FOR_EACH_INV (TEST_LOOP) + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_14.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_14.c new file mode 100644 index 00000000000..1144461dec0 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_14.c @@ -0,0 +1,58 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define N 119 + +#define DEF_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + void __attribute__ ((noipa)) \ + f_##INV##_##SUFFIX (TYPE *restrict a, TYPE *restrict b, \ + TYPE *restrict c, TYPE *restrict d, \ + CMPTYPE *restrict cond) \ + { \ + for (int i = 0; i < N; ++i) \ + { \ + TYPE mb = (INV & 1 ? -b[i] : b[i]); \ + TYPE mc = c[i]; \ + TYPE md = (INV & 2 ? -d[i] : d[i]); \ + TYPE fma = __builtin_fma##SUFFIX (mb, mc, md); \ + TYPE truev = (INV & 4 ? -fma : fma); \ + a[i] = cond[i] < 10 ? truev : c[i]; \ + } \ + } + +#define FOR_EACH_TYPE(T, INV) \ + T (INV, _Float16, short, f16) \ + T (INV, float, float, f32) \ + T (INV, double, double, f64) + +#define FOR_EACH_INV(T) \ + FOR_EACH_TYPE (T, 0) \ + FOR_EACH_TYPE (T, 1) \ + FOR_EACH_TYPE (T, 2) \ + FOR_EACH_TYPE (T, 3) \ + FOR_EACH_TYPE (T, 4) \ + FOR_EACH_TYPE (T, 5) \ + FOR_EACH_TYPE (T, 6) \ + FOR_EACH_TYPE (T, 7) + +FOR_EACH_INV (DEF_LOOP) + +/* { dg-final { scan-assembler-not {\tsel\t} } } */ +/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */ +/* { dg-final { scan-assembler-not {\tmov\tz[0-9]+\.., z[0-9]+} } } */ + +/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.d,} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_14_run.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_14_run.c new file mode 100644 index 00000000000..090f35be315 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_14_run.c @@ -0,0 +1,37 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include "vcond_14.c" + +#define TEST_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + { \ + TYPE a[N], b[N], c[N], d[N]; \ + CMPTYPE cond[N]; \ + for (int i = 0; i < N; ++i) \ + { \ + b[i] = i % 15; \ + c[i] = i % 9 + 11; \ + d[i] = i % 13 + 14; \ + cond[i] = i % 17; \ + asm volatile ("" ::: "memory"); \ + } \ + f_##INV##_##SUFFIX (a, b, c, d, cond); \ + for (int i = 0; i < N; ++i) \ + { \ + double mb = (INV & 1 ? -b[i] : b[i]); \ + double mc = c[i]; \ + double md = (INV & 2 ? -d[i] : d[i]); \ + double fma = __builtin_fma (mb, mc, md); \ + double truev = (INV & 4 ? -fma : fma); \ + if (a[i] != (i % 17 < 10 ? truev : c[i])) \ + __builtin_abort (); \ + asm volatile ("" ::: "memory"); \ + } \ + } + +int +main (void) +{ + FOR_EACH_INV (TEST_LOOP) + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_15.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_15.c new file mode 100644 index 00000000000..4967074072c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_15.c @@ -0,0 +1,58 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define N 119 + +#define DEF_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + void __attribute__ ((noipa)) \ + f_##INV##_##SUFFIX (TYPE *restrict a, TYPE *restrict b, \ + TYPE *restrict c, TYPE *restrict d, \ + CMPTYPE *restrict cond) \ + { \ + for (int i = 0; i < N; ++i) \ + { \ + TYPE mb = (INV & 1 ? -b[i] : b[i]); \ + TYPE mc = c[i]; \ + TYPE md = (INV & 2 ? -d[i] : d[i]); \ + TYPE fma = __builtin_fma##SUFFIX (mb, mc, md); \ + TYPE truev = (INV & 4 ? -fma : fma); \ + a[i] = cond[i] < 10 ? truev : d[i]; \ + } \ + } + +#define FOR_EACH_TYPE(T, INV) \ + T (INV, _Float16, short, f16) \ + T (INV, float, float, f32) \ + T (INV, double, double, f64) + +#define FOR_EACH_INV(T) \ + FOR_EACH_TYPE (T, 0) \ + FOR_EACH_TYPE (T, 1) \ + FOR_EACH_TYPE (T, 2) \ + FOR_EACH_TYPE (T, 3) \ + FOR_EACH_TYPE (T, 4) \ + FOR_EACH_TYPE (T, 5) \ + FOR_EACH_TYPE (T, 6) \ + FOR_EACH_TYPE (T, 7) + +FOR_EACH_INV (DEF_LOOP) + +/* { dg-final { scan-assembler-not {\tsel\t} } } */ +/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */ +/* { dg-final { scan-assembler-not {\tmov\tz[0-9]+\.., z[0-9]+} } } */ + +/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.d,} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_15_run.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_15_run.c new file mode 100644 index 00000000000..b08d4d8ebfb --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_15_run.c @@ -0,0 +1,37 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include "vcond_15.c" + +#define TEST_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + { \ + TYPE a[N], b[N], c[N], d[N]; \ + CMPTYPE cond[N]; \ + for (int i = 0; i < N; ++i) \ + { \ + b[i] = i % 15; \ + c[i] = i % 9 + 11; \ + d[i] = i % 13 + 14; \ + cond[i] = i % 17; \ + asm volatile ("" ::: "memory"); \ + } \ + f_##INV##_##SUFFIX (a, b, c, d, cond); \ + for (int i = 0; i < N; ++i) \ + { \ + double mb = (INV & 1 ? -b[i] : b[i]); \ + double mc = c[i]; \ + double md = (INV & 2 ? -d[i] : d[i]); \ + double fma = __builtin_fma (mb, mc, md); \ + double truev = (INV & 4 ? -fma : fma); \ + if (a[i] != (i % 17 < 10 ? truev : d[i])) \ + __builtin_abort (); \ + asm volatile ("" ::: "memory"); \ + } \ + } + +int +main (void) +{ + FOR_EACH_INV (TEST_LOOP) + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_16.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_16.c new file mode 100644 index 00000000000..7fa98716830 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_16.c @@ -0,0 +1,58 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define N 119 + +#define DEF_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + void __attribute__ ((noipa)) \ + f_##INV##_##SUFFIX (TYPE *restrict a, TYPE *restrict b, \ + TYPE *restrict c, TYPE *restrict d, \ + CMPTYPE *restrict cond) \ + { \ + for (int i = 0; i < N; ++i) \ + { \ + TYPE mb = (INV & 1 ? -b[i] : b[i]); \ + TYPE mc = c[i]; \ + TYPE md = (INV & 2 ? -d[i] : d[i]); \ + TYPE fma = __builtin_fma##SUFFIX (mb, mc, md); \ + TYPE truev = (INV & 4 ? -fma : fma); \ + a[i] = cond[i] < 10 ? truev : 10; \ + } \ + } + +#define FOR_EACH_TYPE(T, INV) \ + T (INV, _Float16, short, f16) \ + T (INV, float, float, f32) \ + T (INV, double, double, f64) + +#define FOR_EACH_INV(T) \ + FOR_EACH_TYPE (T, 0) \ + FOR_EACH_TYPE (T, 1) \ + FOR_EACH_TYPE (T, 2) \ + FOR_EACH_TYPE (T, 3) \ + FOR_EACH_TYPE (T, 4) \ + FOR_EACH_TYPE (T, 5) \ + FOR_EACH_TYPE (T, 6) \ + FOR_EACH_TYPE (T, 7) + +FOR_EACH_INV (DEF_LOOP) + +/* { dg-final { scan-assembler-times {\tsel\t} 24 } } */ +/* { dg-final { scan-assembler-not {\tmovprfx\t} } } */ +/* { dg-final { scan-assembler-not {\tmov\tz[0-9]+\.., z[0-9]+} } } */ + +/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.d,} 2 } } */ + +/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s,} 2 } } */ +/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.d,} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vcond_16_run.c b/gcc/testsuite/gcc.target/aarch64/sve/vcond_16_run.c new file mode 100644 index 00000000000..d4e5afd4d57 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/vcond_16_run.c @@ -0,0 +1,37 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include "vcond_16.c" + +#define TEST_LOOP(INV, TYPE, CMPTYPE, SUFFIX) \ + { \ + TYPE a[N], b[N], c[N], d[N]; \ + CMPTYPE cond[N]; \ + for (int i = 0; i < N; ++i) \ + { \ + b[i] = i % 15; \ + c[i] = i % 9 + 11; \ + d[i] = i % 13 + 14; \ + cond[i] = i % 17; \ + asm volatile ("" ::: "memory"); \ + } \ + f_##INV##_##SUFFIX (a, b, c, d, cond); \ + for (int i = 0; i < N; ++i) \ + { \ + double mb = (INV & 1 ? -b[i] : b[i]); \ + double mc = c[i]; \ + double md = (INV & 2 ? -d[i] : d[i]); \ + double fma = __builtin_fma (mb, mc, md); \ + double truev = (INV & 4 ? -fma : fma); \ + if (a[i] != (i % 17 < 10 ? truev : 10)) \ + __builtin_abort (); \ + asm volatile ("" ::: "memory"); \ + } \ + } + +int +main (void) +{ + FOR_EACH_INV (TEST_LOOP) + return 0; +} -- 2.30.2