From: H.J. Lu Date: Fri, 21 Nov 2008 14:05:02 +0000 (+0000) Subject: config.gcc (extra_headers): For x86 and x86-64, remove gmmintrin.h, add immintrin... X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=04a412f9d4851d15fb904ed1052423ae54795413;p=gcc.git config.gcc (extra_headers): For x86 and x86-64, remove gmmintrin.h, add immintrin.h and avxintrin.h. gcc/ 2008-11-21 H.J. Lu Xuepeng Guo * config.gcc (extra_headers): For x86 and x86-64, remove gmmintrin.h, add immintrin.h and avxintrin.h. * config/i386/gmmintrin.h: Renamed to ... * config/i386/avxintrin.h: This. Don't include intrinsics if _IMMINTRIN_H_INCLUDED is undedined. * config/i386/immintrin.h: New. gcc/testsuite/ 2008-11-21 H.J. Lu * gcc.target/i386/avx-1.c: Include instead of . * gcc.target/i386/avx-2.c: Likewise. * gcc.target/i386/m256-check.h: Likewise. * g++.dg/other/i386-5.C: Likewise. * g++.dg/other/i386-6.C: Likewise. Co-Authored-By: Xuepeng Guo From-SVN: r142090 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 3967476f2d6..b73d83c9f3b 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,15 @@ +2008-11-21 H.J. Lu + Xuepeng Guo + + * config.gcc (extra_headers): For x86 and x86-64, remove + gmmintrin.h, add immintrin.h and avxintrin.h. + + * config/i386/gmmintrin.h: Renamed to ... + * config/i386/avxintrin.h: This. Don't include intrinsics if + _IMMINTRIN_H_INCLUDED is undedined. + + * config/i386/immintrin.h: New. + 2008-11-21 Jakub Jelinek PR target/38208 diff --git a/gcc/config.gcc b/gcc/config.gcc index a78dcc04e57..b0e9340bc78 100644 --- a/gcc/config.gcc +++ b/gcc/config.gcc @@ -298,7 +298,7 @@ i[34567]86-*-*) extra_headers="cpuid.h mmintrin.h mm3dnow.h xmmintrin.h emmintrin.h pmmintrin.h tmmintrin.h ammintrin.h smmintrin.h nmmintrin.h bmmintrin.h mmintrin-common.h - wmmintrin.h gmmintrin.h cross-stdarg.h" + wmmintrin.h immintrin.h avxintrin.h cross-stdarg.h" ;; x86_64-*-*) cpu_type=i386 @@ -307,7 +307,7 @@ x86_64-*-*) extra_headers="cpuid.h mmintrin.h mm3dnow.h xmmintrin.h emmintrin.h pmmintrin.h tmmintrin.h ammintrin.h smmintrin.h nmmintrin.h bmmintrin.h mmintrin-common.h - wmmintrin.h gmmintrin.h cross-stdarg.h" + wmmintrin.h immintrin.h avxintrin.h cross-stdarg.h" need_64bit_hwint=yes ;; ia64-*-*) diff --git a/gcc/config/i386/avxintrin.h b/gcc/config/i386/avxintrin.h new file mode 100644 index 00000000000..7aea85d355d --- /dev/null +++ b/gcc/config/i386/avxintrin.h @@ -0,0 +1,1471 @@ +/* Copyright (C) 2008 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GCC; see the file COPYING. If not, write to + the Free Software Foundation, 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. */ + +/* As a special exception, if you include this header file into source + files compiled by GCC, this header file does not by itself cause + the resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 11.0. */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +/* Internal data types for implementing the intrinsics. */ +typedef double __v4df __attribute__ ((__vector_size__ (32))); +typedef float __v8sf __attribute__ ((__vector_size__ (32))); +typedef long long __v4di __attribute__ ((__vector_size__ (32))); +typedef int __v8si __attribute__ ((__vector_size__ (32))); +typedef short __v16hi __attribute__ ((__vector_size__ (32))); +typedef char __v32qi __attribute__ ((__vector_size__ (32))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m256 __attribute__ ((__vector_size__ (32), + __may_alias__)); +typedef long long __m256i __attribute__ ((__vector_size__ (32), + __may_alias__)); +typedef double __m256d __attribute__ ((__vector_size__ (32), + __may_alias__)); + +/* Compare predicates for scalar and packed compare intrinsics. */ + +/* Equal (ordered, non-signaling) */ +#define _CMP_EQ_OQ 0x00 +/* Less-than (ordered, signaling) */ +#define _CMP_LT_OS 0x01 +/* Less-than-or-equal (ordered, signaling) */ +#define _CMP_LE_OS 0x02 +/* Unordered (non-signaling) */ +#define _CMP_UNORD_Q 0x03 +/* Not-equal (unordered, non-signaling) */ +#define _CMP_NEQ_UQ 0x04 +/* Not-less-than (unordered, signaling) */ +#define _CMP_NLT_US 0x05 +/* Not-less-than-or-equal (unordered, signaling) */ +#define _CMP_NLE_US 0x06 +/* Ordered (nonsignaling) */ +#define _CMP_ORD_Q 0x07 +/* Equal (unordered, non-signaling) */ +#define _CMP_EQ_UQ 0x08 +/* Not-greater-than-or-equal (unordered, signaling) */ +#define _CMP_NGE_US 0x09 +/* Not-greater-than (unordered, signaling) */ +#define _CMP_NGT_US 0x0a +/* False (ordered, non-signaling) */ +#define _CMP_FALSE_OQ 0x0b +/* Not-equal (ordered, non-signaling) */ +#define _CMP_NEQ_OQ 0x0c +/* Greater-than-or-equal (ordered, signaling) */ +#define _CMP_GE_OS 0x0d +/* Greater-than (ordered, signaling) */ +#define _CMP_GT_OS 0x0e +/* True (unordered, non-signaling) */ +#define _CMP_TRUE_UQ 0x0f +/* Equal (ordered, signaling) */ +#define _CMP_EQ_OS 0x10 +/* Less-than (ordered, non-signaling) */ +#define _CMP_LT_OQ 0x11 +/* Less-than-or-equal (ordered, non-signaling) */ +#define _CMP_LE_OQ 0x12 +/* Unordered (signaling) */ +#define _CMP_UNORD_S 0x13 +/* Not-equal (unordered, signaling) */ +#define _CMP_NEQ_US 0x14 +/* Not-less-than (unordered, non-signaling) */ +#define _CMP_NLT_UQ 0x15 +/* Not-less-than-or-equal (unordered, non-signaling) */ +#define _CMP_NLE_UQ 0x16 +/* Ordered (signaling) */ +#define _CMP_ORD_S 0x17 +/* Equal (unordered, signaling) */ +#define _CMP_EQ_US 0x18 +/* Not-greater-than-or-equal (unordered, non-signaling) */ +#define _CMP_NGE_UQ 0x19 +/* Not-greater-than (unordered, non-signaling) */ +#define _CMP_NGT_UQ 0x1a +/* False (ordered, signaling) */ +#define _CMP_FALSE_OS 0x1b +/* Not-equal (ordered, signaling) */ +#define _CMP_NEQ_OS 0x1c +/* Greater-than-or-equal (ordered, non-signaling) */ +#define _CMP_GE_OQ 0x1d +/* Greater-than (ordered, non-signaling) */ +#define _CMP_GT_OQ 0x1e +/* True (unordered, signaling) */ +#define _CMP_TRUE_US 0x1f + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_addpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_addsub_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_addsubpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_addsub_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addsubps256 ((__v8sf)__A, (__v8sf)__B); +} + + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256 ((__v8sf)__A, (__v8sf)__B); +} + +/* Double/single precision floating point blend instructions - select + data from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_pd (__m256d __X, __m256d __Y, const int __M) +{ + return (__m256d) __builtin_ia32_blendpd256 ((__v4df)__X, + (__v4df)__Y, + __M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_ps (__m256 __X, __m256 __Y, const int __M) +{ + return (__m256) __builtin_ia32_blendps256 ((__v8sf)__X, + (__v8sf)__Y, + __M); +} +#else +#define _mm256_blend_pd(X, Y, M) \ + ((__m256d) __builtin_ia32_blendpd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(M))) + +#define _mm256_blend_ps(X, Y, M) \ + ((__m256) __builtin_ia32_blendps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(M))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_pd (__m256d __X, __m256d __Y, __m256d __M) +{ + return (__m256d) __builtin_ia32_blendvpd256 ((__v4df)__X, + (__v4df)__Y, + (__v4df)__M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_ps (__m256 __X, __m256 __Y, __m256 __M) +{ + return (__m256) __builtin_ia32_blendvps256 ((__v8sf)__X, + (__v8sf)__Y, + (__v8sf)__M); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_div_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_divpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_div_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_divps256 ((__v8sf)__A, (__v8sf)__B); +} + +/* Dot product instructions with mask-defined summing and zeroing parts + of result. */ + +#ifdef __OPTIMIZE__ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_dp_ps (__m256 __X, __m256 __Y, const int __M) +{ + return (__m256) __builtin_ia32_dpps256 ((__v8sf)__X, + (__v8sf)__Y, + __M); +} +#else +#define _mm256_dp_ps(X, Y, M) \ + ((__m256) __builtin_ia32_dpps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(M))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_pd (__m256d __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_haddpd256 ((__v4df)__X, (__v4df)__Y); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_ps (__m256 __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_haddps256 ((__v8sf)__X, (__v8sf)__Y); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_pd (__m256d __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_hsubpd256 ((__v4df)__X, (__v4df)__Y); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_ps (__m256 __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_hsubps256 ((__v8sf)__X, (__v8sf)__Y); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_mulpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_mulps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256 ((__v8sf)__A, (__v8sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_pd (__m256d __A, __m256d __B, const int __mask) +{ + return (__m256d) __builtin_ia32_shufpd256 ((__v4df)__A, (__v4df)__B, + __mask); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_ps (__m256 __A, __m256 __B, const int __mask) +{ + return (__m256) __builtin_ia32_shufps256 ((__v8sf)__A, (__v8sf)__B, + __mask); +} +#else +#define _mm256_shuffle_pd(A, B, N) \ + ((__m256d)__builtin_ia32_shufpd256 ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(N))) + +#define _mm256_shuffle_ps(A, B, N) \ + ((__m256) __builtin_ia32_shufps256 ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(N))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_subpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_subps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256 ((__v8sf)__A, (__v8sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_pd (__m128d __X, __m128d __Y, const int __P) +{ + return (__m128d) __builtin_ia32_cmppd ((__v2df)__X, (__v2df)__Y, __P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ps (__m128 __X, __m128 __Y, const int __P) +{ + return (__m128) __builtin_ia32_cmpps ((__v4sf)__X, (__v4sf)__Y, __P); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_pd (__m256d __X, __m256d __Y, const int __P) +{ + return (__m256d) __builtin_ia32_cmppd256 ((__v4df)__X, (__v4df)__Y, + __P); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_ps (__m256 __X, __m256 __Y, const int __P) +{ + return (__m256) __builtin_ia32_cmpps256 ((__v8sf)__X, (__v8sf)__Y, + __P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_sd (__m128d __X, __m128d __Y, const int __P) +{ + return (__m128d) __builtin_ia32_cmpsd ((__v2df)__X, (__v2df)__Y, __P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ss (__m128 __X, __m128 __Y, const int __P) +{ + return (__m128) __builtin_ia32_cmpss ((__v4sf)__X, (__v4sf)__Y, __P); +} +#else +#define _mm_cmp_pd(X, Y, P) \ + ((__m128d) __builtin_ia32_cmppd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P))) + +#define _mm_cmp_ps(X, Y, P) \ + ((__m128) __builtin_ia32_cmpps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P))) + +#define _mm256_cmp_pd(X, Y, P) \ + ((__m256d) __builtin_ia32_cmppd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P))) + +#define _mm256_cmp_ps(X, Y, P) \ + ((__m256) __builtin_ia32_cmpps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P))) + +#define _mm_cmp_sd(X, Y, P) \ + ((__m128d) __builtin_ia32_cmpsd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P))) + +#define _mm_cmp_ss(X, Y, P) \ + ((__m128) __builtin_ia32_cmpss ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_pd (__m128i __A) +{ + return (__m256d)__builtin_ia32_cvtdq2pd256 ((__v4si) __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_ps (__m256i __A) +{ + return (__m256)__builtin_ia32_cvtdq2ps256 ((__v8si) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_ps (__m256d __A) +{ + return (__m128)__builtin_ia32_cvtpd2ps256 ((__v4df) __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epi32 (__m256 __A) +{ + return (__m256i)__builtin_ia32_cvtps2dq256 ((__v8sf) __A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_pd (__m128 __A) +{ + return (__m256d)__builtin_ia32_cvtps2pd256 ((__v4sf) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epi32 (__m256d __A) +{ + return (__m128i)__builtin_ia32_cvttpd2dq256 ((__v4df) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epi32 (__m256d __A) +{ + return (__m128i)__builtin_ia32_cvtpd2dq256 ((__v4df) __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epi32 (__m256 __A) +{ + return (__m256i)__builtin_ia32_cvttps2dq256 ((__v8sf) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_pd (__m256d __X, const int __N) +{ + return (__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)__X, __N); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_ps (__m256 __X, const int __N) +{ + return (__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)__X, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_si256 (__m256i __X, const int __N) +{ + return (__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)__X, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi32 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); + return _mm_extract_epi32 (__Y, __N % 4); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi16 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); + return _mm_extract_epi16 (__Y, __N % 8); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi8 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); + return _mm_extract_epi8 (__Y, __N % 16); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi64 (__m256i __X, const int __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); + return _mm_extract_epi64 (__Y, __N % 2); +} +#endif +#else +#define _mm256_extractf128_pd(X, N) \ + ((__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)(__m256d)(X), \ + (int)(N))) + +#define _mm256_extractf128_ps(X, N) \ + ((__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)(__m256)(X), \ + (int)(N))) + +#define _mm256_extractf128_si256(X, N) \ + ((__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)(__m256i)(X), \ + (int)(N))) + +#define _mm256_extract_epi32(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ + _mm_extract_epi32 (__Y, (N) % 4); \ + })) + +#define _mm256_extract_epi16(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ + _mm_extract_epi16 (__Y, (N) % 8); \ + })) + +#define _mm256_extract_epi8(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ + _mm_extract_epi8 (__Y, (N) % 16); \ + })) + +#ifdef __x86_64__ +#define _mm256_extract_epi64(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ + _mm_extract_epi64 (__Y, (N) % 2); \ + })) +#endif +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_zeroall (void) +{ + __builtin_ia32_vzeroall (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_zeroupper (void) +{ + __builtin_ia32_vzeroupper (); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutevar_pd (__m128d __A, __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd ((__v2df)__A, + (__v2di)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar_pd (__m256d __A, __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256 ((__v4df)__A, + (__v4di)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutevar_ps (__m128 __A, __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps ((__v4sf)__A, + (__v4si)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar_ps (__m256 __A, __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256 ((__v8sf)__A, + (__v8si)__C); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute_pd (__m128d __X, const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd ((__v2df)__X, __C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute_pd (__m256d __X, const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256 ((__v4df)__X, __C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute_ps (__m128 __X, const int __C) +{ + return (__m128) __builtin_ia32_vpermilps ((__v4sf)__X, __C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute_ps (__m256 __X, const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256 ((__v8sf)__X, __C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute2_pd (__m128d __X, __m128d __Y, __m128i __C, const int __I) +{ + return (__m128d) __builtin_ia32_vpermil2pd ((__v2df)__X, + (__v2df)__Y, + (__v2di)__C, + __I); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2_pd (__m256d __X, __m256d __Y, __m256i __C, const int __I) +{ + return (__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)__X, + (__v4df)__Y, + (__v4di)__C, + __I); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute2_ps (__m128 __X, __m128 __Y, __m128i __C, const int __I) +{ + return (__m128) __builtin_ia32_vpermil2ps ((__v4sf)__X, + (__v4sf)__Y, + (__v4si)__C, + __I); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2_ps (__m256 __X, __m256 __Y, __m256i __C, const int __I) +{ + return (__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)__X, + (__v8sf)__Y, + (__v8si)__C, + __I); +} +#else +#define _mm_permute_pd(X, C) \ + ((__m128d) __builtin_ia32_vpermilpd ((__v2df)(__m128d)(X), (int)(C))) + +#define _mm256_permute_pd(X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256 ((__v4df)(__m256d)(X), (int)(C))) + +#define _mm_permute_ps(X, C) \ + ((__m128) __builtin_ia32_vpermilps ((__v4sf)(__m128)(X), (int)(C))) + +#define _mm256_permute_ps(X, C) \ + ((__m256) __builtin_ia32_vpermilps256 ((__v8sf)(__m256)(X), (int)(C))) + +#define _mm_permute2_pd(X, Y, C, I) \ + ((__m128d) __builtin_ia32_vpermil2pd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128d)(C), \ + (int)(I))) + +#define _mm256_permute2_pd(X, Y, C, I) \ + ((__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256d)(C), \ + (int)(I))) + +#define _mm_permute2_ps(X, Y, C, I) \ + ((__m128) __builtin_ia32_vpermil2ps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128)(C), \ + (int)(I))) + +#define _mm256_permute2_ps(X, Y, C, I) \ + ((__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256)(C), \ + (int)(I))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_pd (__m256d __X, __m256d __Y, const int __C) +{ + return (__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)__X, + (__v4df)__Y, + __C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_ps (__m256 __X, __m256 __Y, const int __C) +{ + return (__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)__X, + (__v8sf)__Y, + __C); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_si256 (__m256i __X, __m256i __Y, const int __C) +{ + return (__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)__X, + (__v8si)__Y, + __C); +} +#else +#define _mm256_permute2f128_pd(X, Y, C) \ + ((__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (int)(C))) + +#define _mm256_permute2f128_ps(X, Y, C) \ + ((__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (int)(C))) + +#define _mm256_permute2f128_si256(X, Y, C) \ + ((__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), \ + (int)(C))) +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcast_ss (float const *__X) +{ + return (__m128) __builtin_ia32_vbroadcastss (__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_sd (double const *__X) +{ + return (__m256d) __builtin_ia32_vbroadcastsd256 (__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_ss (float const *__X) +{ + return (__m256) __builtin_ia32_vbroadcastss256 (__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_pd (__m128d const *__X) +{ + return (__m256d) __builtin_ia32_vbroadcastf128_pd256 (__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_ps (__m128 const *__X) +{ + return (__m256) __builtin_ia32_vbroadcastf128_ps256 (__X); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_pd (__m256d __X, __m128d __Y, const int __O) +{ + return (__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)__X, + (__v2df)__Y, + __O); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_ps (__m256 __X, __m128 __Y, const int __O) +{ + return (__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)__X, + (__v4sf)__Y, + __O); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_si256 (__m256i __X, __m128i __Y, const int __O) +{ + return (__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)__X, + (__v4si)__Y, + __O); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi32 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); + __Y = _mm_insert_epi16 (__Y, __D, __N % 4); + return _mm256_insertf128_si256 (__X, __Y, __N >> 2); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi16 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); + __Y = _mm_insert_epi16 (__Y, __D, __N % 8); + return _mm256_insertf128_si256 (__X, __Y, __N >> 3); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi8 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); + __Y = _mm_insert_epi8 (__Y, __D, __N % 16); + return _mm256_insertf128_si256 (__X, __Y, __N >> 4); +} + +#ifdef __x86_64__ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi64 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); + __Y = _mm_insert_epi16 (__Y, __D, __N % 2); + return _mm256_insertf128_si256 (__X, __Y, __N >> 1); +} +#endif +#else +#define _mm256_insertf128_pd(X, Y, O) \ + ((__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)(__m256d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(O))) + +#define _mm256_insertf128_ps(X, Y, O) \ + ((__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)(__m256)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(O))) + +#define _mm256_insertf128_si256(X, Y, O) \ + ((__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)(__m256i)(X), \ + (__v4si)(__m128i)(Y), \ + (int)(O))) + +#define _mm256_insert_epi32(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ + __Y = _mm_insert_epi32 (__Y, (D), (N) % 4); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 2); \ + })) + +#define _mm256_insert_epi16(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ + __Y = _mm_insert_epi16 (__Y, (D), (N) % 8); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 3); \ + })) + +#define _mm256_insert_epi8(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ + __Y = _mm_insert_epi8 (__Y, (D), (N) % 16); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 4); \ + })) + +#ifdef __x86_64__ +#define _mm256_insert_epi64(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ + __Y = _mm_insert_epi64 (__Y, (D), (N) % 2); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 1); \ + })) +#endif +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_pd (double const *__P) +{ + return *(__m256d *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_pd (double *__P, __m256d __A) +{ + *(__m256d *)__P = __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_ps (float const *__P) +{ + return *(__m256 *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_ps (float *__P, __m256 __A) +{ + *(__m256 *)__P = __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_pd (double const *__P) +{ + return (__m256d) __builtin_ia32_loadupd256 (__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_pd (double *__P, __m256d __A) +{ + __builtin_ia32_storeupd256 (__P, (__v4df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_ps (float const *__P) +{ + return (__m256) __builtin_ia32_loadups256 (__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_ps (float *__P, __m256 __A) +{ + __builtin_ia32_storeups256 (__P, (__v8sf)__A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_si256 (__m256i const *__P) +{ + return *__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_si256 (__m256i *__P, __m256i __A) +{ + *__P = __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_si256 (__m256i const *__P) +{ + return (__m256i) __builtin_ia32_loaddqu256 ((char const *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_si256 (__m256i *__P, __m256i __A) +{ + __builtin_ia32_storedqu256 ((char *)__P, (__v32qi)__A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_pd (double const *__P, __m128d __M) +{ + return (__m128d) __builtin_ia32_maskloadpd ((const __v2df *)__P, + (__v2df)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_pd (double *__P, __m128d __M, __m128d __A) +{ + __builtin_ia32_maskstorepd ((__v2df *)__P, (__v2df)__M, (__v2df)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_pd (double const *__P, __m256d __M) +{ + return (__m256d) __builtin_ia32_maskloadpd256 ((const __v4df *)__P, + (__v4df)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_pd (double *__P, __m256d __M, __m256d __A) +{ + __builtin_ia32_maskstorepd256 ((__v4df *)__P, (__v4df)__M, (__v4df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_ps (float const *__P, __m128 __M) +{ + return (__m128) __builtin_ia32_maskloadps ((const __v4sf *)__P, + (__v4sf)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_ps (float *__P, __m128 __M, __m128 __A) +{ + __builtin_ia32_maskstoreps ((__v4sf *)__P, (__v4sf)__M, (__v4sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_ps (float const *__P, __m256 __M) +{ + return (__m256) __builtin_ia32_maskloadps256 ((const __v8sf *)__P, + (__v8sf)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_ps (float *__P, __m256 __M, __m256 __A) +{ + __builtin_ia32_maskstoreps256 ((__v8sf *)__P, (__v8sf)__M, (__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movehdup_ps (__m256 __X) +{ + return (__m256) __builtin_ia32_movshdup256 ((__v8sf)__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_moveldup_ps (__m256 __X) +{ + return (__m256) __builtin_ia32_movsldup256 ((__v8sf)__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movedup_pd (__m256d __X) +{ + return (__m256d) __builtin_ia32_movddup256 ((__v4df)__X); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lddqu_si256 (__m256i const *__P) +{ + return (__m256i) __builtin_ia32_lddqu256 ((char const *)__P); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rcpps256 ((__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rsqrtps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sqrt_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256 ((__v4df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sqrt_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256 ((__v8sf)__A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_round_pd (__m256d __V, const int __M) +{ + return (__m256d) __builtin_ia32_roundpd256 ((__v4df)__V, __M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_round_ps (__m256 __V, const int __M) +{ + return (__m256) __builtin_ia32_roundps256 ((__v8sf)__V, __M); +} +#else +#define _mm256_round_pd(V, M) \ + ((__m256d) __builtin_ia32_roundpd256 ((__v4df)(__m256d)(V), (int)(M))) + +#define _mm256_round_ps(V, M) \ + ((__m256) __builtin_ia32_roundps256 ((__v8sf)(__m256)(V), (int)(M))) +#endif + +#define _mm256_ceil_pd(V) _mm256_round_pd ((V), _MM_FROUND_CEIL) +#define _mm256_floor_pd(V) _mm256_round_pd ((V), _MM_FROUND_FLOOR) +#define _mm256_ceil_ps(V) _mm256_round_ps ((V), _MM_FROUND_CEIL) +#define _mm256_floor_ps(V) _mm256_round_ps ((V), _MM_FROUND_FLOOR) + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestzpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestcpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestnzcpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestzps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestcps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestnzcps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestzpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestcpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestnzcpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestzps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestcps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestnzcps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestz256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestc256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestnzc256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_pd (__m256d __A) +{ + return __builtin_ia32_movmskpd256 ((__v4df)__A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_ps (__m256 __A) +{ + return __builtin_ia32_movmskps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_pd (void) +{ + return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_ps (void) +{ + return __extension__ (__m256){ 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_si256 (void) +{ + return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 }; +} + +/* Create the vector [A B C D]. */ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_pd (double __A, double __B, double __C, double __D) +{ + return __extension__ (__m256d){ __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H]. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H) +{ + return __extension__ (__m256){ __H, __G, __F, __E, + __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H]. */ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H) +{ + return __extension__ (__m256i)(__v8si){ __H, __G, __F, __E, + __D, __C, __B, __A }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi16 (short __q15, short __q14, short __q13, short __q12, + short __q11, short __q10, short __q09, short __q08, + short __q07, short __q06, short __q05, short __q04, + short __q03, short __q02, short __q01, short __q00) +{ + return __extension__ (__m256i)(__v16hi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15 + }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi8 (char __q31, char __q30, char __q29, char __q28, + char __q27, char __q26, char __q25, char __q24, + char __q23, char __q22, char __q21, char __q20, + char __q19, char __q18, char __q17, char __q16, + char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return __extension__ (__m256i)(__v32qi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15, + __q16, __q17, __q18, __q19, __q20, __q21, __q22, __q23, + __q24, __q25, __q26, __q27, __q28, __q29, __q30, __q31 + }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi64x (long long __A, long long __B, long long __C, + long long __D) +{ + return __extension__ (__m256i)(__v4di){ __D, __C, __B, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_pd (double __A) +{ + return __extension__ (__m256d){ __A, __A, __A, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_ps (float __A) +{ + return __extension__ (__m256){ __A, __A, __A, __A, + __A, __A, __A, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi32 (int __A) +{ + return __extension__ (__m256i)(__v8si){ __A, __A, __A, __A, + __A, __A, __A, __A }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi16 (short __A) +{ + return _mm256_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi8 (char __A) +{ + return _mm256_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi64x (long long __A) +{ + return __extension__ (__m256i)(__v4di){ __A, __A, __A, __A }; +} + +/* Create vectors of elements in the reversed order from the + _mm256_set_XXX functions. */ + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_pd (double __A, double __B, double __C, double __D) +{ + return _mm256_set_pd (__D, __C, __B, __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H) +{ + return _mm256_set_ps (__H, __G, __F, __E, __D, __C, __B, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H) +{ + return _mm256_set_epi32 (__H, __G, __F, __E, __D, __C, __B, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi16 (short __q15, short __q14, short __q13, short __q12, + short __q11, short __q10, short __q09, short __q08, + short __q07, short __q06, short __q05, short __q04, + short __q03, short __q02, short __q01, short __q00) +{ + return _mm256_set_epi16 (__q00, __q01, __q02, __q03, + __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, + __q12, __q13, __q14, __q15); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi8 (char __q31, char __q30, char __q29, char __q28, + char __q27, char __q26, char __q25, char __q24, + char __q23, char __q22, char __q21, char __q20, + char __q19, char __q18, char __q17, char __q16, + char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return _mm256_set_epi8 (__q00, __q01, __q02, __q03, + __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, + __q12, __q13, __q14, __q15, + __q16, __q17, __q18, __q19, + __q20, __q21, __q22, __q23, + __q24, __q25, __q26, __q27, + __q28, __q29, __q30, __q31); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi64x (long long __A, long long __B, long long __C, + long long __D) +{ + return _mm256_set_epi64x (__D, __C, __B, __A); +} + +/* Casts between various SP, DP, INT vector types. Note that these do no + conversion of values, they just change the type. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd_ps (__m256d __A) +{ + return (__m256) __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd_si256 (__m256d __A) +{ + return (__m256i) __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps_pd (__m256 __A) +{ + return (__m256d) __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps_si256(__m256 __A) +{ + return (__m256i) __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_ps (__m256i __A) +{ + return (__m256) __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_pd (__m256i __A) +{ + return (__m256d) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd256_pd128 (__m256d __A) +{ + return (__m128d) __builtin_ia32_pd_pd256 ((__v4df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps256_ps128 (__m256 __A) +{ + return (__m128) __builtin_ia32_ps_ps256 ((__v8sf)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_si128 (__m256i __A) +{ + return (__m128i) __builtin_ia32_si_si256 ((__v8si)__A); +} + +/* When cast is done from a 128 to 256-bit type, the low 128 bits of + the 256-bit result contain source parameter value and the upper 128 + bits of the result are undefined. Those intrinsics shouldn't + generate any extra moves. */ + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd128_pd256 (__m128d __A) +{ + return (__m256d) __builtin_ia32_pd256_pd ((__v2df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps128_ps256 (__m128 __A) +{ + return (__m256) __builtin_ia32_ps256_ps ((__v4sf)__A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi128_si256 (__m128i __A) +{ + return (__m256i) __builtin_ia32_si256_si ((__v4si)__A); +} diff --git a/gcc/config/i386/gmmintrin.h b/gcc/config/i386/gmmintrin.h deleted file mode 100644 index 1c6bb18be83..00000000000 --- a/gcc/config/i386/gmmintrin.h +++ /dev/null @@ -1,1482 +0,0 @@ -/* Copyright (C) 2008 Free Software Foundation, Inc. - - This file is part of GCC. - - GCC is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2, or (at your option) - any later version. - - GCC is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GCC; see the file COPYING. If not, write to - the Free Software Foundation, 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. */ - -/* As a special exception, if you include this header file into source - files compiled by GCC, this header file does not by itself cause - the resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. */ - -/* Implemented from the specification included in the Intel C++ Compiler - User Guide and Reference, version 11.0. */ - -#ifndef _GMMINTRIN_H_INCLUDED -#define _GMMINTRIN_H_INCLUDED - -#ifndef __AVX__ -# error "AVX instruction set not enabled" -#else - -/* We need definitions from the SSE4, SSSE3, SSE3, SSE2 and SSE header - files. */ -#include - -/* Internal data types for implementing the intrinsics. */ -typedef double __v4df __attribute__ ((__vector_size__ (32))); -typedef float __v8sf __attribute__ ((__vector_size__ (32))); -typedef long long __v4di __attribute__ ((__vector_size__ (32))); -typedef int __v8si __attribute__ ((__vector_size__ (32))); -typedef short __v16hi __attribute__ ((__vector_size__ (32))); -typedef char __v32qi __attribute__ ((__vector_size__ (32))); - -/* The Intel API is flexible enough that we must allow aliasing with other - vector types, and their scalar components. */ -typedef float __m256 __attribute__ ((__vector_size__ (32), - __may_alias__)); -typedef long long __m256i __attribute__ ((__vector_size__ (32), - __may_alias__)); -typedef double __m256d __attribute__ ((__vector_size__ (32), - __may_alias__)); - -/* Compare predicates for scalar and packed compare intrinsics. */ - -/* Equal (ordered, non-signaling) */ -#define _CMP_EQ_OQ 0x00 -/* Less-than (ordered, signaling) */ -#define _CMP_LT_OS 0x01 -/* Less-than-or-equal (ordered, signaling) */ -#define _CMP_LE_OS 0x02 -/* Unordered (non-signaling) */ -#define _CMP_UNORD_Q 0x03 -/* Not-equal (unordered, non-signaling) */ -#define _CMP_NEQ_UQ 0x04 -/* Not-less-than (unordered, signaling) */ -#define _CMP_NLT_US 0x05 -/* Not-less-than-or-equal (unordered, signaling) */ -#define _CMP_NLE_US 0x06 -/* Ordered (nonsignaling) */ -#define _CMP_ORD_Q 0x07 -/* Equal (unordered, non-signaling) */ -#define _CMP_EQ_UQ 0x08 -/* Not-greater-than-or-equal (unordered, signaling) */ -#define _CMP_NGE_US 0x09 -/* Not-greater-than (unordered, signaling) */ -#define _CMP_NGT_US 0x0a -/* False (ordered, non-signaling) */ -#define _CMP_FALSE_OQ 0x0b -/* Not-equal (ordered, non-signaling) */ -#define _CMP_NEQ_OQ 0x0c -/* Greater-than-or-equal (ordered, signaling) */ -#define _CMP_GE_OS 0x0d -/* Greater-than (ordered, signaling) */ -#define _CMP_GT_OS 0x0e -/* True (unordered, non-signaling) */ -#define _CMP_TRUE_UQ 0x0f -/* Equal (ordered, signaling) */ -#define _CMP_EQ_OS 0x10 -/* Less-than (ordered, non-signaling) */ -#define _CMP_LT_OQ 0x11 -/* Less-than-or-equal (ordered, non-signaling) */ -#define _CMP_LE_OQ 0x12 -/* Unordered (signaling) */ -#define _CMP_UNORD_S 0x13 -/* Not-equal (unordered, signaling) */ -#define _CMP_NEQ_US 0x14 -/* Not-less-than (unordered, non-signaling) */ -#define _CMP_NLT_UQ 0x15 -/* Not-less-than-or-equal (unordered, non-signaling) */ -#define _CMP_NLE_UQ 0x16 -/* Ordered (signaling) */ -#define _CMP_ORD_S 0x17 -/* Equal (unordered, signaling) */ -#define _CMP_EQ_US 0x18 -/* Not-greater-than-or-equal (unordered, non-signaling) */ -#define _CMP_NGE_UQ 0x19 -/* Not-greater-than (unordered, non-signaling) */ -#define _CMP_NGT_UQ 0x1a -/* False (ordered, signaling) */ -#define _CMP_FALSE_OS 0x1b -/* Not-equal (ordered, signaling) */ -#define _CMP_NEQ_OS 0x1c -/* Greater-than-or-equal (ordered, non-signaling) */ -#define _CMP_GE_OQ 0x1d -/* Greater-than (ordered, non-signaling) */ -#define _CMP_GT_OQ 0x1e -/* True (unordered, signaling) */ -#define _CMP_TRUE_US 0x1f - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_add_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_addpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_add_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_addps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_addsub_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_addsubpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_addsub_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_addsubps256 ((__v8sf)__A, (__v8sf)__B); -} - - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_and_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_andpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_and_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_andps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_andnot_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_andnpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_andnot_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_andnps256 ((__v8sf)__A, (__v8sf)__B); -} - -/* Double/single precision floating point blend instructions - select - data from 2 sources using constant/variable mask. */ - -#ifdef __OPTIMIZE__ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_blend_pd (__m256d __X, __m256d __Y, const int __M) -{ - return (__m256d) __builtin_ia32_blendpd256 ((__v4df)__X, - (__v4df)__Y, - __M); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_blend_ps (__m256 __X, __m256 __Y, const int __M) -{ - return (__m256) __builtin_ia32_blendps256 ((__v8sf)__X, - (__v8sf)__Y, - __M); -} -#else -#define _mm256_blend_pd(X, Y, M) \ - ((__m256d) __builtin_ia32_blendpd256 ((__v4df)(__m256d)(X), \ - (__v4df)(__m256d)(Y), (int)(M))) - -#define _mm256_blend_ps(X, Y, M) \ - ((__m256) __builtin_ia32_blendps256 ((__v8sf)(__m256)(X), \ - (__v8sf)(__m256)(Y), (int)(M))) -#endif - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_blendv_pd (__m256d __X, __m256d __Y, __m256d __M) -{ - return (__m256d) __builtin_ia32_blendvpd256 ((__v4df)__X, - (__v4df)__Y, - (__v4df)__M); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_blendv_ps (__m256 __X, __m256 __Y, __m256 __M) -{ - return (__m256) __builtin_ia32_blendvps256 ((__v8sf)__X, - (__v8sf)__Y, - (__v8sf)__M); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_div_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_divpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_div_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_divps256 ((__v8sf)__A, (__v8sf)__B); -} - -/* Dot product instructions with mask-defined summing and zeroing parts - of result. */ - -#ifdef __OPTIMIZE__ -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_dp_ps (__m256 __X, __m256 __Y, const int __M) -{ - return (__m256) __builtin_ia32_dpps256 ((__v8sf)__X, - (__v8sf)__Y, - __M); -} -#else -#define _mm256_dp_ps(X, Y, M) \ - ((__m256) __builtin_ia32_dpps256 ((__v8sf)(__m256)(X), \ - (__v8sf)(__m256)(Y), (int)(M))) -#endif - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_hadd_pd (__m256d __X, __m256d __Y) -{ - return (__m256d) __builtin_ia32_haddpd256 ((__v4df)__X, (__v4df)__Y); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_hadd_ps (__m256 __X, __m256 __Y) -{ - return (__m256) __builtin_ia32_haddps256 ((__v8sf)__X, (__v8sf)__Y); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_hsub_pd (__m256d __X, __m256d __Y) -{ - return (__m256d) __builtin_ia32_hsubpd256 ((__v4df)__X, (__v4df)__Y); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_hsub_ps (__m256 __X, __m256 __Y) -{ - return (__m256) __builtin_ia32_hsubps256 ((__v8sf)__X, (__v8sf)__Y); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_max_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_maxpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_max_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_maxps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_min_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_minpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_min_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_minps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_mul_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_mulpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_mul_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_mulps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_or_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_orpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_or_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_orps256 ((__v8sf)__A, (__v8sf)__B); -} - -#ifdef __OPTIMIZE__ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_shuffle_pd (__m256d __A, __m256d __B, const int __mask) -{ - return (__m256d) __builtin_ia32_shufpd256 ((__v4df)__A, (__v4df)__B, - __mask); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_shuffle_ps (__m256 __A, __m256 __B, const int __mask) -{ - return (__m256) __builtin_ia32_shufps256 ((__v8sf)__A, (__v8sf)__B, - __mask); -} -#else -#define _mm256_shuffle_pd(A, B, N) \ - ((__m256d)__builtin_ia32_shufpd256 ((__v4df)(__m256d)(A), \ - (__v4df)(__m256d)(B), (int)(N))) - -#define _mm256_shuffle_ps(A, B, N) \ - ((__m256) __builtin_ia32_shufps256 ((__v8sf)(__m256)(A), \ - (__v8sf)(__m256)(B), (int)(N))) -#endif - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_sub_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_subpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_sub_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_subps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_xor_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_xorpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_xor_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_xorps256 ((__v8sf)__A, (__v8sf)__B); -} - -#ifdef __OPTIMIZE__ -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_cmp_pd (__m128d __X, __m128d __Y, const int __P) -{ - return (__m128d) __builtin_ia32_cmppd ((__v2df)__X, (__v2df)__Y, __P); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_cmp_ps (__m128 __X, __m128 __Y, const int __P) -{ - return (__m128) __builtin_ia32_cmpps ((__v4sf)__X, (__v4sf)__Y, __P); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cmp_pd (__m256d __X, __m256d __Y, const int __P) -{ - return (__m256d) __builtin_ia32_cmppd256 ((__v4df)__X, (__v4df)__Y, - __P); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cmp_ps (__m256 __X, __m256 __Y, const int __P) -{ - return (__m256) __builtin_ia32_cmpps256 ((__v8sf)__X, (__v8sf)__Y, - __P); -} - -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_cmp_sd (__m128d __X, __m128d __Y, const int __P) -{ - return (__m128d) __builtin_ia32_cmpsd ((__v2df)__X, (__v2df)__Y, __P); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_cmp_ss (__m128 __X, __m128 __Y, const int __P) -{ - return (__m128) __builtin_ia32_cmpss ((__v4sf)__X, (__v4sf)__Y, __P); -} -#else -#define _mm_cmp_pd(X, Y, P) \ - ((__m128d) __builtin_ia32_cmppd ((__v2df)(__m128d)(X), \ - (__v2df)(__m128d)(Y), (int)(P))) - -#define _mm_cmp_ps(X, Y, P) \ - ((__m128) __builtin_ia32_cmpps ((__v4sf)(__m128)(X), \ - (__v4sf)(__m128)(Y), (int)(P))) - -#define _mm256_cmp_pd(X, Y, P) \ - ((__m256d) __builtin_ia32_cmppd256 ((__v4df)(__m256d)(X), \ - (__v4df)(__m256d)(Y), (int)(P))) - -#define _mm256_cmp_ps(X, Y, P) \ - ((__m256) __builtin_ia32_cmpps256 ((__v8sf)(__m256)(X), \ - (__v8sf)(__m256)(Y), (int)(P))) - -#define _mm_cmp_sd(X, Y, P) \ - ((__m128d) __builtin_ia32_cmpsd ((__v2df)(__m128d)(X), \ - (__v2df)(__m128d)(Y), (int)(P))) - -#define _mm_cmp_ss(X, Y, P) \ - ((__m128) __builtin_ia32_cmpss ((__v4sf)(__m128)(X), \ - (__v4sf)(__m128)(Y), (int)(P))) -#endif - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvtepi32_pd (__m128i __A) -{ - return (__m256d)__builtin_ia32_cvtdq2pd256 ((__v4si) __A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvtepi32_ps (__m256i __A) -{ - return (__m256)__builtin_ia32_cvtdq2ps256 ((__v8si) __A); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvtpd_ps (__m256d __A) -{ - return (__m128)__builtin_ia32_cvtpd2ps256 ((__v4df) __A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvtps_epi32 (__m256 __A) -{ - return (__m256i)__builtin_ia32_cvtps2dq256 ((__v8sf) __A); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvtps_pd (__m128 __A) -{ - return (__m256d)__builtin_ia32_cvtps2pd256 ((__v4sf) __A); -} - -extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvttpd_epi32 (__m256d __A) -{ - return (__m128i)__builtin_ia32_cvttpd2dq256 ((__v4df) __A); -} - -extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvtpd_epi32 (__m256d __A) -{ - return (__m128i)__builtin_ia32_cvtpd2dq256 ((__v4df) __A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_cvttps_epi32 (__m256 __A) -{ - return (__m256i)__builtin_ia32_cvttps2dq256 ((__v8sf) __A); -} - -#ifdef __OPTIMIZE__ -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extractf128_pd (__m256d __X, const int __N) -{ - return (__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)__X, __N); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extractf128_ps (__m256 __X, const int __N) -{ - return (__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)__X, __N); -} - -extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extractf128_si256 (__m256i __X, const int __N) -{ - return (__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)__X, __N); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extract_epi32 (__m256i __X, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); - return _mm_extract_epi32 (__Y, __N % 4); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extract_epi16 (__m256i __X, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); - return _mm_extract_epi16 (__Y, __N % 8); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extract_epi8 (__m256i __X, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); - return _mm_extract_epi8 (__Y, __N % 16); -} - -#ifdef __x86_64__ -extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_extract_epi64 (__m256i __X, const int __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); - return _mm_extract_epi64 (__Y, __N % 2); -} -#endif -#else -#define _mm256_extractf128_pd(X, N) \ - ((__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)(__m256d)(X), \ - (int)(N))) - -#define _mm256_extractf128_ps(X, N) \ - ((__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)(__m256)(X), \ - (int)(N))) - -#define _mm256_extractf128_si256(X, N) \ - ((__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)(__m256i)(X), \ - (int)(N))) - -#define _mm256_extract_epi32(X, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ - _mm_extract_epi32 (__Y, (N) % 4); \ - })) - -#define _mm256_extract_epi16(X, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ - _mm_extract_epi16 (__Y, (N) % 8); \ - })) - -#define _mm256_extract_epi8(X, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ - _mm_extract_epi8 (__Y, (N) % 16); \ - })) - -#ifdef __x86_64__ -#define _mm256_extract_epi64(X, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ - _mm_extract_epi64 (__Y, (N) % 2); \ - })) -#endif -#endif - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_zeroall (void) -{ - __builtin_ia32_vzeroall (); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_zeroupper (void) -{ - __builtin_ia32_vzeroupper (); -} - -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_permutevar_pd (__m128d __A, __m128i __C) -{ - return (__m128d) __builtin_ia32_vpermilvarpd ((__v2df)__A, - (__v2di)__C); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permutevar_pd (__m256d __A, __m256i __C) -{ - return (__m256d) __builtin_ia32_vpermilvarpd256 ((__v4df)__A, - (__v4di)__C); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_permutevar_ps (__m128 __A, __m128i __C) -{ - return (__m128) __builtin_ia32_vpermilvarps ((__v4sf)__A, - (__v4si)__C); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permutevar_ps (__m256 __A, __m256i __C) -{ - return (__m256) __builtin_ia32_vpermilvarps256 ((__v8sf)__A, - (__v8si)__C); -} - -#ifdef __OPTIMIZE__ -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_permute_pd (__m128d __X, const int __C) -{ - return (__m128d) __builtin_ia32_vpermilpd ((__v2df)__X, __C); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute_pd (__m256d __X, const int __C) -{ - return (__m256d) __builtin_ia32_vpermilpd256 ((__v4df)__X, __C); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_permute_ps (__m128 __X, const int __C) -{ - return (__m128) __builtin_ia32_vpermilps ((__v4sf)__X, __C); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute_ps (__m256 __X, const int __C) -{ - return (__m256) __builtin_ia32_vpermilps256 ((__v8sf)__X, __C); -} - -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_permute2_pd (__m128d __X, __m128d __Y, __m128i __C, const int __I) -{ - return (__m128d) __builtin_ia32_vpermil2pd ((__v2df)__X, - (__v2df)__Y, - (__v2di)__C, - __I); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute2_pd (__m256d __X, __m256d __Y, __m256i __C, const int __I) -{ - return (__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)__X, - (__v4df)__Y, - (__v4di)__C, - __I); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_permute2_ps (__m128 __X, __m128 __Y, __m128i __C, const int __I) -{ - return (__m128) __builtin_ia32_vpermil2ps ((__v4sf)__X, - (__v4sf)__Y, - (__v4si)__C, - __I); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute2_ps (__m256 __X, __m256 __Y, __m256i __C, const int __I) -{ - return (__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)__X, - (__v8sf)__Y, - (__v8si)__C, - __I); -} -#else -#define _mm_permute_pd(X, C) \ - ((__m128d) __builtin_ia32_vpermilpd ((__v2df)(__m128d)(X), (int)(C))) - -#define _mm256_permute_pd(X, C) \ - ((__m256d) __builtin_ia32_vpermilpd256 ((__v4df)(__m256d)(X), (int)(C))) - -#define _mm_permute_ps(X, C) \ - ((__m128) __builtin_ia32_vpermilps ((__v4sf)(__m128)(X), (int)(C))) - -#define _mm256_permute_ps(X, C) \ - ((__m256) __builtin_ia32_vpermilps256 ((__v8sf)(__m256)(X), (int)(C))) - -#define _mm_permute2_pd(X, Y, C, I) \ - ((__m128d) __builtin_ia32_vpermil2pd ((__v2df)(__m128d)(X), \ - (__v2df)(__m128d)(Y), \ - (__v2di)(__m128d)(C), \ - (int)(I))) - -#define _mm256_permute2_pd(X, Y, C, I) \ - ((__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)(__m256d)(X), \ - (__v4df)(__m256d)(Y), \ - (__v4di)(__m256d)(C), \ - (int)(I))) - -#define _mm_permute2_ps(X, Y, C, I) \ - ((__m128) __builtin_ia32_vpermil2ps ((__v4sf)(__m128)(X), \ - (__v4sf)(__m128)(Y), \ - (__v4si)(__m128)(C), \ - (int)(I))) - -#define _mm256_permute2_ps(X, Y, C, I) \ - ((__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)(__m256)(X), \ - (__v8sf)(__m256)(Y), \ - (__v8si)(__m256)(C), \ - (int)(I))) -#endif - -#ifdef __OPTIMIZE__ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute2f128_pd (__m256d __X, __m256d __Y, const int __C) -{ - return (__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)__X, - (__v4df)__Y, - __C); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute2f128_ps (__m256 __X, __m256 __Y, const int __C) -{ - return (__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)__X, - (__v8sf)__Y, - __C); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_permute2f128_si256 (__m256i __X, __m256i __Y, const int __C) -{ - return (__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)__X, - (__v8si)__Y, - __C); -} -#else -#define _mm256_permute2f128_pd(X, Y, C) \ - ((__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)(__m256d)(X), \ - (__v4df)(__m256d)(Y), \ - (int)(C))) - -#define _mm256_permute2f128_ps(X, Y, C) \ - ((__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)(__m256)(X), \ - (__v8sf)(__m256)(Y), \ - (int)(C))) - -#define _mm256_permute2f128_si256(X, Y, C) \ - ((__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)(__m256i)(X), \ - (__v8si)(__m256i)(Y), \ - (int)(C))) -#endif - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_broadcast_ss (float const *__X) -{ - return (__m128) __builtin_ia32_vbroadcastss (__X); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_broadcast_sd (double const *__X) -{ - return (__m256d) __builtin_ia32_vbroadcastsd256 (__X); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_broadcast_ss (float const *__X) -{ - return (__m256) __builtin_ia32_vbroadcastss256 (__X); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_broadcast_pd (__m128d const *__X) -{ - return (__m256d) __builtin_ia32_vbroadcastf128_pd256 (__X); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_broadcast_ps (__m128 const *__X) -{ - return (__m256) __builtin_ia32_vbroadcastf128_ps256 (__X); -} - -#ifdef __OPTIMIZE__ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insertf128_pd (__m256d __X, __m128d __Y, const int __O) -{ - return (__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)__X, - (__v2df)__Y, - __O); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insertf128_ps (__m256 __X, __m128 __Y, const int __O) -{ - return (__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)__X, - (__v4sf)__Y, - __O); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insertf128_si256 (__m256i __X, __m128i __Y, const int __O) -{ - return (__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)__X, - (__v4si)__Y, - __O); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insert_epi32 (__m256i __X, int __D, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); - __Y = _mm_insert_epi16 (__Y, __D, __N % 4); - return _mm256_insertf128_si256 (__X, __Y, __N >> 2); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insert_epi16 (__m256i __X, int __D, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); - __Y = _mm_insert_epi16 (__Y, __D, __N % 8); - return _mm256_insertf128_si256 (__X, __Y, __N >> 3); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insert_epi8 (__m256i __X, int __D, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); - __Y = _mm_insert_epi8 (__Y, __D, __N % 16); - return _mm256_insertf128_si256 (__X, __Y, __N >> 4); -} - -#ifdef __x86_64__ -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_insert_epi64 (__m256i __X, int __D, int const __N) -{ - __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); - __Y = _mm_insert_epi16 (__Y, __D, __N % 2); - return _mm256_insertf128_si256 (__X, __Y, __N >> 1); -} -#endif -#else -#define _mm256_insertf128_pd(X, Y, O) \ - ((__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)(__m256d)(X), \ - (__v2df)(__m128d)(Y), \ - (int)(O))) - -#define _mm256_insertf128_ps(X, Y, O) \ - ((__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)(__m256)(X), \ - (__v4sf)(__m128)(Y), \ - (int)(O))) - -#define _mm256_insertf128_si256(X, Y, O) \ - ((__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)(__m256i)(X), \ - (__v4si)(__m128i)(Y), \ - (int)(O))) - -#define _mm256_insert_epi32(X, D, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ - __Y = _mm_insert_epi32 (__Y, (D), (N) % 4); \ - _mm256_insertf128_si256 ((X), __Y, (N) >> 2); \ - })) - -#define _mm256_insert_epi16(X, D, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ - __Y = _mm_insert_epi16 (__Y, (D), (N) % 8); \ - _mm256_insertf128_si256 ((X), __Y, (N) >> 3); \ - })) - -#define _mm256_insert_epi8(X, D, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ - __Y = _mm_insert_epi8 (__Y, (D), (N) % 16); \ - _mm256_insertf128_si256 ((X), __Y, (N) >> 4); \ - })) - -#ifdef __x86_64__ -#define _mm256_insert_epi64(X, D, N) \ - (__extension__ \ - ({ \ - __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ - __Y = _mm_insert_epi64 (__Y, (D), (N) % 2); \ - _mm256_insertf128_si256 ((X), __Y, (N) >> 1); \ - })) -#endif -#endif - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_load_pd (double const *__P) -{ - return *(__m256d *)__P; -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_store_pd (double *__P, __m256d __A) -{ - *(__m256d *)__P = __A; -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_load_ps (float const *__P) -{ - return *(__m256 *)__P; -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_store_ps (float *__P, __m256 __A) -{ - *(__m256 *)__P = __A; -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_loadu_pd (double const *__P) -{ - return (__m256d) __builtin_ia32_loadupd256 (__P); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_storeu_pd (double *__P, __m256d __A) -{ - __builtin_ia32_storeupd256 (__P, (__v4df)__A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_loadu_ps (float const *__P) -{ - return (__m256) __builtin_ia32_loadups256 (__P); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_storeu_ps (float *__P, __m256 __A) -{ - __builtin_ia32_storeups256 (__P, (__v8sf)__A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_load_si256 (__m256i const *__P) -{ - return *__P; -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_store_si256 (__m256i *__P, __m256i __A) -{ - *__P = __A; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_loadu_si256 (__m256i const *__P) -{ - return (__m256i) __builtin_ia32_loaddqu256 ((char const *)__P); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_storeu_si256 (__m256i *__P, __m256i __A) -{ - __builtin_ia32_storedqu256 ((char *)__P, (__v32qi)__A); -} - -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_maskload_pd (double const *__P, __m128d __M) -{ - return (__m128d) __builtin_ia32_maskloadpd ((const __v2df *)__P, - (__v2df)__M); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_maskstore_pd (double *__P, __m128d __M, __m128d __A) -{ - __builtin_ia32_maskstorepd ((__v2df *)__P, (__v2df)__M, (__v2df)__A); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_maskload_pd (double const *__P, __m256d __M) -{ - return (__m256d) __builtin_ia32_maskloadpd256 ((const __v4df *)__P, - (__v4df)__M); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_maskstore_pd (double *__P, __m256d __M, __m256d __A) -{ - __builtin_ia32_maskstorepd256 ((__v4df *)__P, (__v4df)__M, (__v4df)__A); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_maskload_ps (float const *__P, __m128 __M) -{ - return (__m128) __builtin_ia32_maskloadps ((const __v4sf *)__P, - (__v4sf)__M); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_maskstore_ps (float *__P, __m128 __M, __m128 __A) -{ - __builtin_ia32_maskstoreps ((__v4sf *)__P, (__v4sf)__M, (__v4sf)__A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_maskload_ps (float const *__P, __m256 __M) -{ - return (__m256) __builtin_ia32_maskloadps256 ((const __v8sf *)__P, - (__v8sf)__M); -} - -extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_maskstore_ps (float *__P, __m256 __M, __m256 __A) -{ - __builtin_ia32_maskstoreps256 ((__v8sf *)__P, (__v8sf)__M, (__v8sf)__A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_movehdup_ps (__m256 __X) -{ - return (__m256) __builtin_ia32_movshdup256 ((__v8sf)__X); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_moveldup_ps (__m256 __X) -{ - return (__m256) __builtin_ia32_movsldup256 ((__v8sf)__X); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_movedup_pd (__m256d __X) -{ - return (__m256d) __builtin_ia32_movddup256 ((__v4df)__X); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_lddqu_si256 (__m256i const *__P) -{ - return (__m256i) __builtin_ia32_lddqu256 ((char const *)__P); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_rcp_ps (__m256 __A) -{ - return (__m256) __builtin_ia32_rcpps256 ((__v8sf)__A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_rsqrt_ps (__m256 __A) -{ - return (__m256) __builtin_ia32_rsqrtps256 ((__v8sf)__A); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_sqrt_pd (__m256d __A) -{ - return (__m256d) __builtin_ia32_sqrtpd256 ((__v4df)__A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_sqrt_ps (__m256 __A) -{ - return (__m256) __builtin_ia32_sqrtps256 ((__v8sf)__A); -} - -#ifdef __OPTIMIZE__ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_round_pd (__m256d __V, const int __M) -{ - return (__m256d) __builtin_ia32_roundpd256 ((__v4df)__V, __M); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_round_ps (__m256 __V, const int __M) -{ - return (__m256) __builtin_ia32_roundps256 ((__v8sf)__V, __M); -} -#else -#define _mm256_round_pd(V, M) \ - ((__m256d) __builtin_ia32_roundpd256 ((__v4df)(__m256d)(V), (int)(M))) - -#define _mm256_round_ps(V, M) \ - ((__m256) __builtin_ia32_roundps256 ((__v8sf)(__m256)(V), (int)(M))) -#endif - -#define _mm256_ceil_pd(V) _mm256_round_pd ((V), _MM_FROUND_CEIL) -#define _mm256_floor_pd(V) _mm256_round_pd ((V), _MM_FROUND_FLOOR) -#define _mm256_ceil_ps(V) _mm256_round_ps ((V), _MM_FROUND_CEIL) -#define _mm256_floor_ps(V) _mm256_round_ps ((V), _MM_FROUND_FLOOR) - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_unpackhi_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_unpckhpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_unpacklo_pd (__m256d __A, __m256d __B) -{ - return (__m256d) __builtin_ia32_unpcklpd256 ((__v4df)__A, (__v4df)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_unpackhi_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_unpckhps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_unpacklo_ps (__m256 __A, __m256 __B) -{ - return (__m256) __builtin_ia32_unpcklps256 ((__v8sf)__A, (__v8sf)__B); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_testz_pd (__m128d __M, __m128d __V) -{ - return __builtin_ia32_vtestzpd ((__v2df)__M, (__v2df)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_testc_pd (__m128d __M, __m128d __V) -{ - return __builtin_ia32_vtestcpd ((__v2df)__M, (__v2df)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_testnzc_pd (__m128d __M, __m128d __V) -{ - return __builtin_ia32_vtestnzcpd ((__v2df)__M, (__v2df)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_testz_ps (__m128 __M, __m128 __V) -{ - return __builtin_ia32_vtestzps ((__v4sf)__M, (__v4sf)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_testc_ps (__m128 __M, __m128 __V) -{ - return __builtin_ia32_vtestcps ((__v4sf)__M, (__v4sf)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm_testnzc_ps (__m128 __M, __m128 __V) -{ - return __builtin_ia32_vtestnzcps ((__v4sf)__M, (__v4sf)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testz_pd (__m256d __M, __m256d __V) -{ - return __builtin_ia32_vtestzpd256 ((__v4df)__M, (__v4df)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testc_pd (__m256d __M, __m256d __V) -{ - return __builtin_ia32_vtestcpd256 ((__v4df)__M, (__v4df)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testnzc_pd (__m256d __M, __m256d __V) -{ - return __builtin_ia32_vtestnzcpd256 ((__v4df)__M, (__v4df)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testz_ps (__m256 __M, __m256 __V) -{ - return __builtin_ia32_vtestzps256 ((__v8sf)__M, (__v8sf)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testc_ps (__m256 __M, __m256 __V) -{ - return __builtin_ia32_vtestcps256 ((__v8sf)__M, (__v8sf)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testnzc_ps (__m256 __M, __m256 __V) -{ - return __builtin_ia32_vtestnzcps256 ((__v8sf)__M, (__v8sf)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testz_si256 (__m256i __M, __m256i __V) -{ - return __builtin_ia32_ptestz256 ((__v4di)__M, (__v4di)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testc_si256 (__m256i __M, __m256i __V) -{ - return __builtin_ia32_ptestc256 ((__v4di)__M, (__v4di)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_testnzc_si256 (__m256i __M, __m256i __V) -{ - return __builtin_ia32_ptestnzc256 ((__v4di)__M, (__v4di)__V); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_movemask_pd (__m256d __A) -{ - return __builtin_ia32_movmskpd256 ((__v4df)__A); -} - -extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_movemask_ps (__m256 __A) -{ - return __builtin_ia32_movmskps256 ((__v8sf)__A); -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setzero_pd (void) -{ - return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 }; -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setzero_ps (void) -{ - return __extension__ (__m256){ 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0 }; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setzero_si256 (void) -{ - return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 }; -} - -/* Create the vector [A B C D]. */ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set_pd (double __A, double __B, double __C, double __D) -{ - return __extension__ (__m256d){ __D, __C, __B, __A }; -} - -/* Create the vector [A B C D E F G H]. */ -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set_ps (float __A, float __B, float __C, float __D, - float __E, float __F, float __G, float __H) -{ - return __extension__ (__m256){ __H, __G, __F, __E, - __D, __C, __B, __A }; -} - -/* Create the vector [A B C D E F G H]. */ -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set_epi32 (int __A, int __B, int __C, int __D, - int __E, int __F, int __G, int __H) -{ - return __extension__ (__m256i)(__v8si){ __H, __G, __F, __E, - __D, __C, __B, __A }; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set_epi16 (short __q15, short __q14, short __q13, short __q12, - short __q11, short __q10, short __q09, short __q08, - short __q07, short __q06, short __q05, short __q04, - short __q03, short __q02, short __q01, short __q00) -{ - return __extension__ (__m256i)(__v16hi){ - __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, - __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15 - }; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set_epi8 (char __q31, char __q30, char __q29, char __q28, - char __q27, char __q26, char __q25, char __q24, - char __q23, char __q22, char __q21, char __q20, - char __q19, char __q18, char __q17, char __q16, - char __q15, char __q14, char __q13, char __q12, - char __q11, char __q10, char __q09, char __q08, - char __q07, char __q06, char __q05, char __q04, - char __q03, char __q02, char __q01, char __q00) -{ - return __extension__ (__m256i)(__v32qi){ - __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, - __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15, - __q16, __q17, __q18, __q19, __q20, __q21, __q22, __q23, - __q24, __q25, __q26, __q27, __q28, __q29, __q30, __q31 - }; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set_epi64x (long long __A, long long __B, long long __C, - long long __D) -{ - return __extension__ (__m256i)(__v4di){ __D, __C, __B, __A }; -} - -/* Create a vector with all elements equal to A. */ -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set1_pd (double __A) -{ - return __extension__ (__m256d){ __A, __A, __A, __A }; -} - -/* Create a vector with all elements equal to A. */ -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set1_ps (float __A) -{ - return __extension__ (__m256){ __A, __A, __A, __A, - __A, __A, __A, __A }; -} - -/* Create a vector with all elements equal to A. */ -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set1_epi32 (int __A) -{ - return __extension__ (__m256i)(__v8si){ __A, __A, __A, __A, - __A, __A, __A, __A }; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set1_epi16 (short __A) -{ - return _mm256_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A, - __A, __A, __A, __A, __A, __A, __A, __A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set1_epi8 (char __A) -{ - return _mm256_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A, - __A, __A, __A, __A, __A, __A, __A, __A, - __A, __A, __A, __A, __A, __A, __A, __A, - __A, __A, __A, __A, __A, __A, __A, __A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_set1_epi64x (long long __A) -{ - return __extension__ (__m256i)(__v4di){ __A, __A, __A, __A }; -} - -/* Create vectors of elements in the reversed order from the - _mm256_set_XXX functions. */ - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setr_pd (double __A, double __B, double __C, double __D) -{ - return _mm256_set_pd (__D, __C, __B, __A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setr_ps (float __A, float __B, float __C, float __D, - float __E, float __F, float __G, float __H) -{ - return _mm256_set_ps (__H, __G, __F, __E, __D, __C, __B, __A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setr_epi32 (int __A, int __B, int __C, int __D, - int __E, int __F, int __G, int __H) -{ - return _mm256_set_epi32 (__H, __G, __F, __E, __D, __C, __B, __A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setr_epi16 (short __q15, short __q14, short __q13, short __q12, - short __q11, short __q10, short __q09, short __q08, - short __q07, short __q06, short __q05, short __q04, - short __q03, short __q02, short __q01, short __q00) -{ - return _mm256_set_epi16 (__q00, __q01, __q02, __q03, - __q04, __q05, __q06, __q07, - __q08, __q09, __q10, __q11, - __q12, __q13, __q14, __q15); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setr_epi8 (char __q31, char __q30, char __q29, char __q28, - char __q27, char __q26, char __q25, char __q24, - char __q23, char __q22, char __q21, char __q20, - char __q19, char __q18, char __q17, char __q16, - char __q15, char __q14, char __q13, char __q12, - char __q11, char __q10, char __q09, char __q08, - char __q07, char __q06, char __q05, char __q04, - char __q03, char __q02, char __q01, char __q00) -{ - return _mm256_set_epi8 (__q00, __q01, __q02, __q03, - __q04, __q05, __q06, __q07, - __q08, __q09, __q10, __q11, - __q12, __q13, __q14, __q15, - __q16, __q17, __q18, __q19, - __q20, __q21, __q22, __q23, - __q24, __q25, __q26, __q27, - __q28, __q29, __q30, __q31); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_setr_epi64x (long long __A, long long __B, long long __C, - long long __D) -{ - return _mm256_set_epi64x (__D, __C, __B, __A); -} - -/* Casts between various SP, DP, INT vector types. Note that these do no - conversion of values, they just change the type. */ -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castpd_ps (__m256d __A) -{ - return (__m256) __A; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castpd_si256 (__m256d __A) -{ - return (__m256i) __A; -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castps_pd (__m256 __A) -{ - return (__m256d) __A; -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castps_si256(__m256 __A) -{ - return (__m256i) __A; -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castsi256_ps (__m256i __A) -{ - return (__m256) __A; -} - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castsi256_pd (__m256i __A) -{ - return (__m256d) __A; -} - -extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castpd256_pd128 (__m256d __A) -{ - return (__m128d) __builtin_ia32_pd_pd256 ((__v4df)__A); -} - -extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castps256_ps128 (__m256 __A) -{ - return (__m128) __builtin_ia32_ps_ps256 ((__v8sf)__A); -} - -extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castsi256_si128 (__m256i __A) -{ - return (__m128i) __builtin_ia32_si_si256 ((__v8si)__A); -} - -/* When cast is done from a 128 to 256-bit type, the low 128 bits of - the 256-bit result contain source parameter value and the upper 128 - bits of the result are undefined. Those intrinsics shouldn't - generate any extra moves. */ - -extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castpd128_pd256 (__m128d __A) -{ - return (__m256d) __builtin_ia32_pd256_pd ((__v2df)__A); -} - -extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castps128_ps256 (__m128 __A) -{ - return (__m256) __builtin_ia32_ps256_ps ((__v4sf)__A); -} - -extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) -_mm256_castsi128_si256 (__m128i __A) -{ - return (__m256i) __builtin_ia32_si256_si ((__v4si)__A); -} - -#endif /* __AVX__ */ - -#endif /* _GMMINTRIN_H_INCLUDED */ diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 38c36f20a2c..3f99b5aa3d7 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,12 @@ +2008-11-21 H.J. Lu + + * gcc.target/i386/avx-1.c: Include instead of + . + * gcc.target/i386/avx-2.c: Likewise. + * gcc.target/i386/m256-check.h: Likewise. + * g++.dg/other/i386-5.C: Likewise. + * g++.dg/other/i386-6.C: Likewise. + 2008-11-20 Jerry DeLisle PR libfortran/37472 diff --git a/gcc/testsuite/g++.dg/other/i386-5.C b/gcc/testsuite/g++.dg/other/i386-5.C index 2556e7e021b..d9b5460a8dc 100644 --- a/gcc/testsuite/g++.dg/other/i386-5.C +++ b/gcc/testsuite/g++.dg/other/i386-5.C @@ -5,5 +5,5 @@ #include #include -#include +#include #include diff --git a/gcc/testsuite/g++.dg/other/i386-6.C b/gcc/testsuite/g++.dg/other/i386-6.C index 905ac97b011..15f1d5258df 100644 --- a/gcc/testsuite/g++.dg/other/i386-6.C +++ b/gcc/testsuite/g++.dg/other/i386-6.C @@ -5,7 +5,7 @@ #include #include -#include +#include #include int dummy; diff --git a/gcc/testsuite/gcc.target/i386/avx-1.c b/gcc/testsuite/gcc.target/i386/avx-1.c index 3cda080a3df..d093d6c1cc3 100644 --- a/gcc/testsuite/gcc.target/i386/avx-1.c +++ b/gcc/testsuite/gcc.target/i386/avx-1.c @@ -18,7 +18,7 @@ #define __builtin_ia32_extrqi(X, I, L) __builtin_ia32_extrqi(X, 1, 1) #define __builtin_ia32_insertqi(X, Y, I, L) __builtin_ia32_insertqi(X, Y, 1, 1) -/* gmmintrin.h */ +/* immintrin.h */ #define __builtin_ia32_blendpd256(X, Y, M) __builtin_ia32_blendpd256(X, Y, 1) #define __builtin_ia32_blendps256(X, Y, M) __builtin_ia32_blendps256(X, Y, 1) #define __builtin_ia32_dpps256(X, Y, M) __builtin_ia32_dpps256(X, Y, 1) @@ -135,5 +135,5 @@ #include #include -#include +#include #include diff --git a/gcc/testsuite/gcc.target/i386/avx-2.c b/gcc/testsuite/gcc.target/i386/avx-2.c index 6699558d8f9..62f6538bbe6 100644 --- a/gcc/testsuite/gcc.target/i386/avx-2.c +++ b/gcc/testsuite/gcc.target/i386/avx-2.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #define _CONCAT(x,y) x ## y @@ -53,7 +53,7 @@ test_1x (_mm_extracti_si64, __m128i, __m128i, 1, 1) test_2x (_mm_inserti_si64, __m128i, __m128i, __m128i, 1, 1) -/* gmmintrin.h */ +/* immintrin.h */ test_2 (_mm256_blend_pd, __m256d, __m256d, __m256d, 1) test_2 (_mm256_blend_ps, __m256, __m256, __m256, 1) test_2 (_mm256_dp_ps, __m256, __m256, __m256, 1) diff --git a/gcc/testsuite/gcc.target/i386/m256-check.h b/gcc/testsuite/gcc.target/i386/m256-check.h index 324af75ba64..e1843550ecd 100644 --- a/gcc/testsuite/gcc.target/i386/m256-check.h +++ b/gcc/testsuite/gcc.target/i386/m256-check.h @@ -1,4 +1,4 @@ -#include +#include #include "m128-check.h" #ifndef max