From aa652fb2a083c15678f82a5cb20b7f8cbc9c1437 Mon Sep 17 00:00:00 2001 From: Jonathan Wright Date: Wed, 3 Feb 2021 17:01:53 +0000 Subject: [PATCH] aarch64: Use RTL builtins for [su]mull_high_lane[q] intrinsics Rewrite [su]mull_high_lane[q] Neon intrinsics to use RTL builtins rather than inline assembly code, allowing for better scheduling and optimization. gcc/ChangeLog: 2021-02-03 Jonathan Wright * config/aarch64/aarch64-simd-builtins.def: Add [su]mull_hi_lane[q] builtin generator macros. * config/aarch64/aarch64-simd.md (aarch64_mull_hi_lane_insn): Define. (aarch64_mull_hi_lane): Define. (aarch64_mull_hi_laneq_insn): Define. (aarch64_mull_hi_laneq): Define. * config/aarch64/arm_neon.h (vmull_high_lane_s16): Use RTL builtin instead of inline asm. (vmull_high_lane_s32): Likewise. (vmull_high_lane_u16): Likewise. (vmull_high_lane_u32): Likewise. (vmull_high_laneq_s16): Likewise. (vmull_high_laneq_s32): Likewise. (vmull_high_laneq_u16): Likewise. (vmull_high_laneq_u32): Liekwise. --- gcc/config/aarch64/aarch64-simd-builtins.def | 5 + gcc/config/aarch64/aarch64-simd.md | 64 ++++++++ gcc/config/aarch64/arm_neon.h | 145 +++++++------------ 3 files changed, 118 insertions(+), 96 deletions(-) diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def b/gcc/config/aarch64/aarch64-simd-builtins.def index 4dac8e0d79b..29a7bbc24a7 100644 --- a/gcc/config/aarch64/aarch64-simd-builtins.def +++ b/gcc/config/aarch64/aarch64-simd-builtins.def @@ -303,6 +303,11 @@ BUILTIN_VQ_HSI (BINOP, smull_hi_n, 0, NONE) BUILTIN_VQ_HSI (BINOPU, umull_hi_n, 0, NONE) + BUILTIN_VQ_HSI (TERNOP_LANE, smull_hi_lane, 0, NONE) + BUILTIN_VQ_HSI (TERNOP_LANE, smull_hi_laneq, 0, NONE) + BUILTIN_VQ_HSI (TERNOPU_LANE, umull_hi_lane, 0, NONE) + BUILTIN_VQ_HSI (TERNOPU_LANE, umull_hi_laneq, 0, NONE) + BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_lane_, 0, NONE) BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_lane_, 0, NONE) BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_laneq_, 0, NONE) diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md index 1d1ba379c28..60eeddcc946 100644 --- a/gcc/config/aarch64/aarch64-simd.md +++ b/gcc/config/aarch64/aarch64-simd.md @@ -2253,6 +2253,70 @@ [(set_attr "type" "neon_mul__scalar_long")] ) +(define_insn "aarch64_mull_hi_lane_insn" + [(set (match_operand: 0 "register_operand" "=w") + (mult: + (ANY_EXTEND: (vec_select: + (match_operand:VQ_HSI 1 "register_operand" "w") + (match_operand:VQ_HSI 2 "vect_par_cnst_hi_half" ""))) + (ANY_EXTEND: (vec_duplicate: + (vec_select: + (match_operand: 3 "register_operand" "") + (parallel [(match_operand:SI 4 "immediate_operand" "i")]))))))] + "TARGET_SIMD" + { + operands[4] = aarch64_endian_lane_rtx (mode, INTVAL (operands[4])); + return "mull2\\t%0., %1., %3.[%4]"; + } + [(set_attr "type" "neon_mul__scalar_long")] +) + +(define_expand "aarch64_mull_hi_lane" + [(match_operand: 0 "register_operand") + (ANY_EXTEND:(match_operand:VQ_HSI 1 "register_operand")) + (match_operand: 2 "register_operand") + (match_operand:SI 3 "immediate_operand")] + "TARGET_SIMD" +{ + rtx p = aarch64_simd_vect_par_cnst_half (mode, , true); + emit_insn (gen_aarch64_mull_hi_lane_insn (operands[0], + operands[1], p, operands[2], operands[3])); + DONE; +} +) + +(define_insn "aarch64_mull_hi_laneq_insn" + [(set (match_operand: 0 "register_operand" "=w") + (mult: + (ANY_EXTEND: (vec_select: + (match_operand:VQ_HSI 1 "register_operand" "w") + (match_operand:VQ_HSI 2 "vect_par_cnst_hi_half" ""))) + (ANY_EXTEND: (vec_duplicate: + (vec_select: + (match_operand: 3 "register_operand" "") + (parallel [(match_operand:SI 4 "immediate_operand" "i")]))))))] + "TARGET_SIMD" + { + operands[4] = aarch64_endian_lane_rtx (mode, INTVAL (operands[4])); + return "mull2\\t%0., %1., %3.[%4]"; + } + [(set_attr "type" "neon_mul__scalar_long")] +) + +(define_expand "aarch64_mull_hi_laneq" + [(match_operand: 0 "register_operand") + (ANY_EXTEND:(match_operand:VQ_HSI 1 "register_operand")) + (match_operand: 2 "register_operand") + (match_operand:SI 3 "immediate_operand")] + "TARGET_SIMD" +{ + rtx p = aarch64_simd_vect_par_cnst_half (mode, , true); + emit_insn (gen_aarch64_mull_hi_laneq_insn (operands[0], + operands[1], p, operands[2], operands[3])); + DONE; +} +) + (define_insn "aarch64_mull_n" [(set (match_operand: 0 "register_operand" "=w") (mult: diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h index fffd7b7bd2f..2d776ef7ef4 100644 --- a/gcc/config/aarch64/arm_neon.h +++ b/gcc/config/aarch64/arm_neon.h @@ -8167,109 +8167,62 @@ vshrn_n_u64 (uint64x2_t __a, const int __b) { return (uint32x2_t)__builtin_aarch64_shrnv2di ((int64x2_t)__a, __b); } -#define vmull_high_lane_s16(a, b, c) \ - __extension__ \ - ({ \ - int16x4_t b_ = (b); \ - int16x8_t a_ = (a); \ - int32x4_t result; \ - __asm__ ("smull2 %0.4s, %1.8h, %2.h[%3]" \ - : "=w"(result) \ - : "w"(a_), "x"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) -#define vmull_high_lane_s32(a, b, c) \ - __extension__ \ - ({ \ - int32x2_t b_ = (b); \ - int32x4_t a_ = (a); \ - int64x2_t result; \ - __asm__ ("smull2 %0.2d, %1.4s, %2.s[%3]" \ - : "=w"(result) \ - : "w"(a_), "w"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline int32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_lane_s16 (int16x8_t __a, int16x4_t __v, const int __lane) +{ + return __builtin_aarch64_smull_hi_lanev8hi (__a, __v, __lane); +} -#define vmull_high_lane_u16(a, b, c) \ - __extension__ \ - ({ \ - uint16x4_t b_ = (b); \ - uint16x8_t a_ = (a); \ - uint32x4_t result; \ - __asm__ ("umull2 %0.4s, %1.8h, %2.h[%3]" \ - : "=w"(result) \ - : "w"(a_), "x"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline int64x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_lane_s32 (int32x4_t __a, int32x2_t __v, const int __lane) +{ + return __builtin_aarch64_smull_hi_lanev4si (__a, __v, __lane); +} -#define vmull_high_lane_u32(a, b, c) \ - __extension__ \ - ({ \ - uint32x2_t b_ = (b); \ - uint32x4_t a_ = (a); \ - uint64x2_t result; \ - __asm__ ("umull2 %0.2d, %1.4s, %2.s[%3]" \ - : "=w"(result) \ - : "w"(a_), "w"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline uint32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_lane_u16 (uint16x8_t __a, uint16x4_t __v, const int __lane) +{ + return __builtin_aarch64_umull_hi_lanev8hi_uuus (__a, __v, __lane); +} -#define vmull_high_laneq_s16(a, b, c) \ - __extension__ \ - ({ \ - int16x8_t b_ = (b); \ - int16x8_t a_ = (a); \ - int32x4_t result; \ - __asm__ ("smull2 %0.4s, %1.8h, %2.h[%3]" \ - : "=w"(result) \ - : "w"(a_), "x"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline uint64x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_lane_u32 (uint32x4_t __a, uint32x2_t __v, const int __lane) +{ + return __builtin_aarch64_umull_hi_lanev4si_uuus (__a, __v, __lane); +} -#define vmull_high_laneq_s32(a, b, c) \ - __extension__ \ - ({ \ - int32x4_t b_ = (b); \ - int32x4_t a_ = (a); \ - int64x2_t result; \ - __asm__ ("smull2 %0.2d, %1.4s, %2.s[%3]" \ - : "=w"(result) \ - : "w"(a_), "w"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline int32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_laneq_s16 (int16x8_t __a, int16x8_t __v, const int __lane) +{ + return __builtin_aarch64_smull_hi_laneqv8hi (__a, __v, __lane); +} -#define vmull_high_laneq_u16(a, b, c) \ - __extension__ \ - ({ \ - uint16x8_t b_ = (b); \ - uint16x8_t a_ = (a); \ - uint32x4_t result; \ - __asm__ ("umull2 %0.4s, %1.8h, %2.h[%3]" \ - : "=w"(result) \ - : "w"(a_), "x"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline int64x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_laneq_s32 (int32x4_t __a, int32x4_t __v, const int __lane) +{ + return __builtin_aarch64_smull_hi_laneqv4si (__a, __v, __lane); +} -#define vmull_high_laneq_u32(a, b, c) \ - __extension__ \ - ({ \ - uint32x4_t b_ = (b); \ - uint32x4_t a_ = (a); \ - uint64x2_t result; \ - __asm__ ("umull2 %0.2d, %1.4s, %2.s[%3]" \ - : "=w"(result) \ - : "w"(a_), "w"(b_), "i"(c) \ - : /* No clobbers */); \ - result; \ - }) +__extension__ extern __inline uint32x4_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_laneq_u16 (uint16x8_t __a, uint16x8_t __v, const int __lane) +{ + return __builtin_aarch64_umull_hi_laneqv8hi_uuus (__a, __v, __lane); +} + +__extension__ extern __inline uint64x2_t +__attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) +vmull_high_laneq_u32 (uint32x4_t __a, uint32x4_t __v, const int __lane) +{ + return __builtin_aarch64_umull_hi_laneqv4si_uuus (__a, __v, __lane); +} __extension__ extern __inline int32x4_t __attribute__ ((__always_inline__, __gnu_inline__, __artificial__)) -- 2.30.2