return result;
}
-#define vmul_lane_f32(a, b, c) \
- __extension__ \
- ({ \
- float32x2_t b_ = (b); \
- float32x2_t a_ = (a); \
- float32x2_t result; \
- __asm__ ("fmul %0.2s,%1.2s,%2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_lane_s16(a, b, c) \
- __extension__ \
- ({ \
- int16x4_t b_ = (b); \
- int16x4_t a_ = (a); \
- int16x4_t result; \
- __asm__ ("mul %0.4h,%1.4h,%2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_lane_s32(a, b, c) \
- __extension__ \
- ({ \
- int32x2_t b_ = (b); \
- int32x2_t a_ = (a); \
- int32x2_t result; \
- __asm__ ("mul %0.2s,%1.2s,%2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_lane_u16(a, b, c) \
- __extension__ \
- ({ \
- uint16x4_t b_ = (b); \
- uint16x4_t a_ = (a); \
- uint16x4_t result; \
- __asm__ ("mul %0.4h,%1.4h,%2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_lane_u32(a, b, c) \
- __extension__ \
- ({ \
- uint32x2_t b_ = (b); \
- uint32x2_t a_ = (a); \
- uint32x2_t result; \
- __asm__ ("mul %0.2s, %1.2s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_laneq_f32(a, b, c) \
- __extension__ \
- ({ \
- float32x4_t b_ = (b); \
- float32x2_t a_ = (a); \
- float32x2_t result; \
- __asm__ ("fmul %0.2s, %1.2s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_laneq_s16(a, b, c) \
- __extension__ \
- ({ \
- int16x8_t b_ = (b); \
- int16x4_t a_ = (a); \
- int16x4_t result; \
- __asm__ ("mul %0.4h, %1.4h, %2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_laneq_s32(a, b, c) \
- __extension__ \
- ({ \
- int32x4_t b_ = (b); \
- int32x2_t a_ = (a); \
- int32x2_t result; \
- __asm__ ("mul %0.2s, %1.2s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_laneq_u16(a, b, c) \
- __extension__ \
- ({ \
- uint16x8_t b_ = (b); \
- uint16x4_t a_ = (a); \
- uint16x4_t result; \
- __asm__ ("mul %0.4h, %1.4h, %2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmul_laneq_u32(a, b, c) \
- __extension__ \
- ({ \
- uint32x4_t b_ = (b); \
- uint32x2_t a_ = (a); \
- uint32x2_t result; \
- __asm__ ("mul %0.2s, %1.2s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
__extension__ static __inline float32x2_t __attribute__ ((__always_inline__))
vmul_n_f32 (float32x2_t a, float32_t b)
{
return result;
}
-#define vmulq_lane_f32(a, b, c) \
- __extension__ \
- ({ \
- float32x2_t b_ = (b); \
- float32x4_t a_ = (a); \
- float32x4_t result; \
- __asm__ ("fmul %0.4s, %1.4s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_lane_f64(a, b, c) \
- __extension__ \
- ({ \
- float64x1_t b_ = (b); \
- float64x2_t a_ = (a); \
- float64x2_t result; \
- __asm__ ("fmul %0.2d,%1.2d,%2.d[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_lane_s16(a, b, c) \
- __extension__ \
- ({ \
- int16x4_t b_ = (b); \
- int16x8_t a_ = (a); \
- int16x8_t result; \
- __asm__ ("mul %0.8h,%1.8h,%2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "x"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_lane_s32(a, b, c) \
- __extension__ \
- ({ \
- int32x2_t b_ = (b); \
- int32x4_t a_ = (a); \
- int32x4_t result; \
- __asm__ ("mul %0.4s,%1.4s,%2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_lane_u16(a, b, c) \
- __extension__ \
- ({ \
- uint16x4_t b_ = (b); \
- uint16x8_t a_ = (a); \
- uint16x8_t result; \
- __asm__ ("mul %0.8h,%1.8h,%2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "x"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_lane_u32(a, b, c) \
- __extension__ \
- ({ \
- uint32x2_t b_ = (b); \
- uint32x4_t a_ = (a); \
- uint32x4_t result; \
- __asm__ ("mul %0.4s, %1.4s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_laneq_f32(a, b, c) \
- __extension__ \
- ({ \
- float32x4_t b_ = (b); \
- float32x4_t a_ = (a); \
- float32x4_t result; \
- __asm__ ("fmul %0.4s, %1.4s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_laneq_f64(a, b, c) \
- __extension__ \
- ({ \
- float64x2_t b_ = (b); \
- float64x2_t a_ = (a); \
- float64x2_t result; \
- __asm__ ("fmul %0.2d,%1.2d,%2.d[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_laneq_s16(a, b, c) \
- __extension__ \
- ({ \
- int16x8_t b_ = (b); \
- int16x8_t a_ = (a); \
- int16x8_t result; \
- __asm__ ("mul %0.8h, %1.8h, %2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "x"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_laneq_s32(a, b, c) \
- __extension__ \
- ({ \
- int32x4_t b_ = (b); \
- int32x4_t a_ = (a); \
- int32x4_t result; \
- __asm__ ("mul %0.4s, %1.4s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_laneq_u16(a, b, c) \
- __extension__ \
- ({ \
- uint16x8_t b_ = (b); \
- uint16x8_t a_ = (a); \
- uint16x8_t result; \
- __asm__ ("mul %0.8h, %1.8h, %2.h[%3]" \
- : "=w"(result) \
- : "w"(a_), "x"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
-#define vmulq_laneq_u32(a, b, c) \
- __extension__ \
- ({ \
- uint32x4_t b_ = (b); \
- uint32x4_t a_ = (a); \
- uint32x4_t result; \
- __asm__ ("mul %0.4s, %1.4s, %2.s[%3]" \
- : "=w"(result) \
- : "w"(a_), "w"(b_), "i"(c) \
- : /* No clobbers */); \
- result; \
- })
-
__extension__ static __inline float32x4_t __attribute__ ((__always_inline__))
vmulq_n_f32 (float32x4_t a, float32_t b)
{
return a - b * c;
}
+/* vmul_lane */
+
+__extension__ static __inline float32x2_t __attribute__ ((__always_inline__))
+vmul_lane_f32 (float32x2_t __a, float32x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_f32 (__b, __lane);
+}
+
+__extension__ static __inline float64x1_t __attribute__ ((__always_inline__))
+vmul_lane_f64 (float64x1_t __a, float64x1_t __b, const int __lane)
+{
+ return __a * __b;
+}
+
+__extension__ static __inline int16x4_t __attribute__ ((__always_inline__))
+vmul_lane_s16 (int16x4_t __a, int16x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_s16 (__b, __lane);
+}
+
+__extension__ static __inline int32x2_t __attribute__ ((__always_inline__))
+vmul_lane_s32 (int32x2_t __a, int32x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_s32 (__b, __lane);
+}
+
+__extension__ static __inline uint16x4_t __attribute__ ((__always_inline__))
+vmul_lane_u16 (uint16x4_t __a, uint16x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_u16 (__b, __lane);
+}
+
+__extension__ static __inline uint32x2_t __attribute__ ((__always_inline__))
+vmul_lane_u32 (uint32x2_t __a, uint32x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_u32 (__b, __lane);
+}
+
+/* vmul_laneq */
+
+__extension__ static __inline float32x2_t __attribute__ ((__always_inline__))
+vmul_laneq_f32 (float32x2_t __a, float32x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_f32 (__b, __lane);
+}
+
+__extension__ static __inline float64x1_t __attribute__ ((__always_inline__))
+vmul_laneq_f64 (float64x1_t __a, float64x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_f64 (__b, __lane);
+}
+
+__extension__ static __inline int16x4_t __attribute__ ((__always_inline__))
+vmul_laneq_s16 (int16x4_t __a, int16x8_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_s16 (__b, __lane);
+}
+
+__extension__ static __inline int32x2_t __attribute__ ((__always_inline__))
+vmul_laneq_s32 (int32x2_t __a, int32x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_s32 (__b, __lane);
+}
+
+__extension__ static __inline uint16x4_t __attribute__ ((__always_inline__))
+vmul_laneq_u16 (uint16x4_t __a, uint16x8_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_u16 (__b, __lane);
+}
+
+__extension__ static __inline uint32x2_t __attribute__ ((__always_inline__))
+vmul_laneq_u32 (uint32x2_t __a, uint32x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_u32 (__b, __lane);
+}
+
+/* vmulq_lane */
+
+__extension__ static __inline float32x4_t __attribute__ ((__always_inline__))
+vmulq_lane_f32 (float32x4_t __a, float32x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_f32 (__b, __lane);
+}
+
+__extension__ static __inline float64x2_t __attribute__ ((__always_inline__))
+vmulq_lane_f64 (float64x2_t __a, float64x1_t __b, const int __lane)
+{
+ return __a * __b;
+}
+
+__extension__ static __inline int16x8_t __attribute__ ((__always_inline__))
+vmulq_lane_s16 (int16x8_t __a, int16x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_s16 (__b, __lane);
+}
+
+__extension__ static __inline int32x4_t __attribute__ ((__always_inline__))
+vmulq_lane_s32 (int32x4_t __a, int32x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_s32 (__b, __lane);
+}
+
+__extension__ static __inline uint16x8_t __attribute__ ((__always_inline__))
+vmulq_lane_u16 (uint16x8_t __a, uint16x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_u16 (__b, __lane);
+}
+
+__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__))
+vmulq_lane_u32 (uint32x4_t __a, uint32x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vget_lane_u32 (__b, __lane);
+}
+
+/* vmulq_laneq */
+
+__extension__ static __inline float32x4_t __attribute__ ((__always_inline__))
+vmulq_laneq_f32 (float32x4_t __a, float32x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_f32 (__b, __lane);
+}
+
+__extension__ static __inline float64x2_t __attribute__ ((__always_inline__))
+vmulq_laneq_f64 (float64x2_t __a, float64x2_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_f64 (__b, __lane);
+}
+
+__extension__ static __inline int16x8_t __attribute__ ((__always_inline__))
+vmulq_laneq_s16 (int16x8_t __a, int16x8_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_s16 (__b, __lane);
+}
+
+__extension__ static __inline int32x4_t __attribute__ ((__always_inline__))
+vmulq_laneq_s32 (int32x4_t __a, int32x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_s32 (__b, __lane);
+}
+
+__extension__ static __inline uint16x8_t __attribute__ ((__always_inline__))
+vmulq_laneq_u16 (uint16x8_t __a, uint16x8_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_u16 (__b, __lane);
+}
+
+__extension__ static __inline uint32x4_t __attribute__ ((__always_inline__))
+vmulq_laneq_u32 (uint32x4_t __a, uint32x4_t __b, const int __lane)
+{
+ return __a * __aarch64_vgetq_lane_u32 (__b, __lane);
+}
+
/* vqabs */
__extension__ static __inline int64x2_t __attribute__ ((__always_inline__))
--- /dev/null
+/* { dg-do run } */
+/* { dg-options "-O3 --save-temps" } */
+
+#include <arm_neon.h>
+
+#define DELTA 0.0001
+extern void abort (void);
+extern double fabs (double);
+
+#define TEST_VMUL(q1, q2, size, in1_lanes, in2_lanes) \
+static void \
+test_vmul##q1##_lane##q2##_f##size (float##size##_t * res, \
+ const float##size##_t *in1, \
+ const float##size##_t *in2) \
+{ \
+ float##size##x##in1_lanes##_t a = vld1##q1##_f##size (res); \
+ float##size##x##in1_lanes##_t b = vld1##q1##_f##size (in1); \
+ float##size##x##in2_lanes##_t c; \
+ if (in2_lanes > 1) \
+ { \
+ c = vld1##q2##_f##size (in2); \
+ a = vmul##q1##_lane##q2##_f##size (b, c, 1); \
+ } \
+ else \
+ { \
+ c = vld1##q2##_f##size (in2 + 1); \
+ a = vmul##q1##_lane##q2##_f##size (b, c, 0); \
+ } \
+ vst1##q1##_f##size (res, a); \
+}
+
+#define BUILD_VARS(width, n_lanes, n_half_lanes) \
+TEST_VMUL ( , , width, n_half_lanes, n_half_lanes) \
+TEST_VMUL (q, , width, n_lanes, n_half_lanes) \
+TEST_VMUL ( , q, width, n_half_lanes, n_lanes) \
+TEST_VMUL (q, q, width, n_lanes, n_lanes)
+
+BUILD_VARS (32, 4, 2)
+BUILD_VARS (64, 2, 1)
+
+#define POOL2 {0.0, 1.0}
+#define POOL4 {0.0, 1.0, 2.0, 3.0}
+#define EMPTY2 {0.0, 0.0}
+#define EMPTY4 {0.0, 0.0, 0.0, 0.0}
+
+#define BUILD_TEST(size, lanes) \
+static void \
+test_f##size (void) \
+{ \
+ int i; \
+ float##size##_t pool[lanes] = POOL##lanes; \
+ float##size##_t res[lanes] = EMPTY##lanes; \
+ float##size##_t res2[lanes] = EMPTY##lanes; \
+ float##size##_t res3[lanes] = EMPTY##lanes; \
+ float##size##_t res4[lanes] = EMPTY##lanes; \
+ \
+ /* Avoid constant folding the multiplication. */ \
+ asm volatile ("" : : : "memory"); \
+ test_vmul_lane_f##size (res, pool, pool); \
+ /* Avoid fusing multiplication and subtraction. */ \
+ asm volatile ("" : :"Q" (res) : "memory"); \
+ for (i = 0; i < lanes / 2; i++) \
+ if (fabs (res[i] - pool[i]) > DELTA) \
+ abort (); \
+ \
+ test_vmulq_lane_f##size (res2, pool, pool); \
+ /* Avoid fusing multiplication and subtraction. */ \
+ asm volatile ("" : :"Q" (res2) : "memory"); \
+ for (i = 0; i < lanes; i++) \
+ if (fabs (res2[i] - pool[i]) > DELTA) \
+ abort (); \
+ \
+ test_vmul_laneq_f##size (res3, pool, pool); \
+ /* Avoid fusing multiplication and subtraction. */ \
+ asm volatile ("" : :"Q" (res3) : "memory"); \
+ for (i = 0; i < lanes / 2; i++) \
+ if (fabs (res3[i] - pool[i]) > DELTA) \
+ abort (); \
+ \
+ test_vmulq_laneq_f##size (res4, pool, pool); \
+ /* Avoid fusing multiplication and subtraction. */ \
+ asm volatile ("" : :"Q" (res4) : "memory"); \
+ for (i = 0; i < lanes; i++) \
+ if (fabs (res4[i] - pool[i]) > DELTA) \
+ abort (); \
+}
+
+BUILD_TEST (32, 4)
+BUILD_TEST (64, 2)
+
+int
+main (int argc, char **argv)
+{
+ test_f32 ();
+ test_f64 ();
+ return 0;
+}
+
+/* vmul_laneq_f32.
+ vmul_lane_f32. */
+/* { dg-final { scan-assembler-times "fmul\\tv\[0-9\]+\.2s, v\[0-9\]+\.2s, v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+
+/* vmulq_lane_f32.
+ vmulq_laneq_f32. */
+/* { dg-final { scan-assembler-times "fmul\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s, v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 2 } } */
+
+/* vmul_lane_f64. */
+/* { dg-final { scan-assembler-times "fmul\\td\[0-9\]+, d\[0-9\]+, d\[0-9\]+" 1 } } */
+
+/* vmul_laneq_f64.
+ vmulq_lane_f64.
+ vmulq_laneq_f64. */
+/* { dg-final { scan-assembler-times "fmul\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 } } */
+
+/* { dg-final { cleanup-saved-temps } } */
+
--- /dev/null
+/* { dg-do run } */
+/* { dg-options "-O3 --save-temps" } */
+
+#include <arm_neon.h>
+
+extern void abort (void);
+
+#define MAPs(size, xx) int##size##xx##_t
+#define MAPu(size, xx) uint##size##xx##_t
+
+
+#define TEST_VMUL(q, su, size, in1_lanes, in2_lanes) \
+static void \
+test_vmulq_lane##q##_##su##size (MAP##su (size, ) * res, \
+ const MAP##su(size, ) *in1, \
+ const MAP##su(size, ) *in2) \
+{ \
+ MAP##su (size, x##in1_lanes) a = vld1q_##su##size (in1); \
+ MAP##su (size, x##in2_lanes) b = vld1##q##_##su##size (in2); \
+ a = vmulq_lane##q##_##su##size (a, b, 1); \
+ vst1q_##su##size (res, a); \
+}
+
+#define BUILD_VARS(width, n_lanes, n_half_lanes) \
+TEST_VMUL (, s, width, n_lanes, n_half_lanes) \
+TEST_VMUL (q, s, width, n_lanes, n_lanes) \
+TEST_VMUL (, u, width, n_lanes, n_half_lanes) \
+TEST_VMUL (q, u, width, n_lanes, n_lanes) \
+
+BUILD_VARS (32, 4, 2)
+BUILD_VARS (16, 8, 4)
+
+#define POOL4 {0, 1, 2, 3}
+#define POOL8 {0, 1, 2, 3, 4, 5, 6, 7}
+#define EMPTY4 {0, 0, 0, 0}
+#define EMPTY8 {0, 0, 0, 0, 0, 0, 0, 0}
+
+#define BUILD_TEST(su, size, lanes) \
+static void \
+test_##su##size (void) \
+{ \
+ int i; \
+ MAP##su (size,) pool[lanes] = POOL##lanes; \
+ MAP##su (size,) res[lanes] = EMPTY##lanes; \
+ MAP##su (size,) res2[lanes] = EMPTY##lanes; \
+ \
+ /* Forecfully avoid optimization. */ \
+ asm volatile ("" : : : "memory"); \
+ test_vmulq_lane_##su##size (res, pool, pool); \
+ for (i = 0; i < lanes; i++) \
+ if (res[i] != pool[i]) \
+ abort (); \
+ \
+ /* Forecfully avoid optimization. */ \
+ asm volatile ("" : : : "memory"); \
+ test_vmulq_laneq_##su##size (res2, pool, pool); \
+ for (i = 0; i < lanes; i++) \
+ if (res2[i] != pool[i]) \
+ abort (); \
+}
+
+#undef BUILD_VARS
+#define BUILD_VARS(size, lanes) \
+BUILD_TEST (s, size, lanes) \
+BUILD_TEST (u, size, lanes)
+
+BUILD_VARS (32, 4)
+BUILD_VARS (16, 8)
+
+int
+main (int argc, char **argv)
+{
+ test_s32 ();
+ test_u32 ();
+ test_s16 ();
+ test_u16 ();
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times "mul\\tv\[0-9\]+\.4s, v\[0-9\]+\.4s, v\[0-9\]+\.s\\\[\[0-9\]+\\\]" 4 } } */
+/* { dg-final { scan-assembler-times "mul\\tv\[0-9\]+\.8h, v\[0-9\]+\.8h, v\[0-9\]+\.h\\\[\[0-9\]+\\\]" 4 } } */
+/* { dg-final { cleanup-saved-temps } } */
+
projects / gcc.git / commitdiff