gcc/config/aarch64/aarch64-simd-builtins.def

   1 /* Machine description for AArch64 architecture.
   2    Copyright (C) 2012-2021 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 /* In the list below, the BUILTIN_<ITERATOR> macros expand to create
  22    builtins for each of the modes described by <ITERATOR>.  When adding
  23    new builtins to this list, a helpful idiom to follow is to add
  24    a line for each pattern in the md file.  Thus, ADDP, which has one
  25    pattern defined for the VD_BHSI iterator, and one for DImode, has two
  26    entries below.
  27
  28    Parameter 1 is the 'type' of the intrinsic.  This is used to
  29    describe the type modifiers (for example; unsigned) applied to
  30    each of the parameters to the intrinsic function.
  31
  32    Parameter 2 is the name of the intrinsic.  This is appended
  33    to `__builtin_aarch64_<name><mode>` to give the intrinsic name
  34    as exported to the front-ends.
  35
  36    Parameter 3 describes how to map from the name to the CODE_FOR_
  37    macro holding the RTL pattern for the intrinsic.  This mapping is:
  38    0 - CODE_FOR_aarch64_<name><mode>
  39    1-9 - CODE_FOR_<name><mode><1-9>
  40    10 - CODE_FOR_<name><mode>.
  41
  42    Parameter 4 is the 'flag' of the intrinsic.  This is used to
  43    help describe the attributes (for example, pure) for the intrinsic
  44    function.  */
  45
  46   BUILTIN_VDC (COMBINE, combine, 0, ALL)
  47   VAR1 (COMBINEP, combine, 0, ALL, di)
  48   BUILTIN_VB (BINOP, pmul, 0, NONE)
  49   BUILTIN_VHSDF_HSDF (BINOP, fmulx, 0, FP)
  50   BUILTIN_VHSDF_DF (UNOP, sqrt, 2, FP)
  51   BUILTIN_VD_BHSI (BINOP, addp, 0, NONE)
  52   VAR1 (UNOP, addp, 0, NONE, di)
  53   BUILTIN_VDQ_BHSI (UNOP, clrsb, 2, ALL)
  54   BUILTIN_VDQ_BHSI (UNOP, clz, 2, ALL)
  55   BUILTIN_VS (UNOP, ctz, 2, ALL)
  56   BUILTIN_VB (UNOP, popcount, 2, ALL)
  57
  58   /* Implemented by aarch64_<sur>q<r>shl<mode>.  */
  59   BUILTIN_VSDQ_I (BINOP, sqshl, 0, NONE)
  60   BUILTIN_VSDQ_I (BINOP_UUS, uqshl, 0, NONE)
  61   BUILTIN_VSDQ_I (BINOP, sqrshl, 0, NONE)
  62   BUILTIN_VSDQ_I (BINOP_UUS, uqrshl, 0, NONE)
  63   /* Implemented by aarch64_<su_optab><optab><mode>.  */
  64   BUILTIN_VSDQ_I (BINOP, sqadd, 0, NONE)
  65   BUILTIN_VSDQ_I (BINOPU, uqadd, 0, NONE)
  66   BUILTIN_VSDQ_I (BINOP, sqsub, 0, NONE)
  67   BUILTIN_VSDQ_I (BINOPU, uqsub, 0, NONE)
  68   /* Implemented by aarch64_<sur>qadd<mode>.  */
  69   BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0, NONE)
  70   BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0, NONE)
  71
  72   /* Implemented by aarch64_get_dreg<VSTRUCT:mode><VDC:mode>.  */
  73   BUILTIN_VDC (GETREG, get_dregoi, 0, AUTO_FP)
  74   BUILTIN_VDC (GETREG, get_dregci, 0, AUTO_FP)
  75   BUILTIN_VDC (GETREG, get_dregxi, 0, AUTO_FP)
  76   VAR1 (GETREGP, get_dregoi, 0, AUTO_FP, di)
  77   VAR1 (GETREGP, get_dregci, 0, AUTO_FP, di)
  78   VAR1 (GETREGP, get_dregxi, 0, AUTO_FP, di)
  79   /* Implemented by aarch64_get_qreg<VSTRUCT:mode><VQ:mode>.  */
  80   BUILTIN_VQ (GETREG, get_qregoi, 0, AUTO_FP)
  81   BUILTIN_VQ (GETREG, get_qregci, 0, AUTO_FP)
  82   BUILTIN_VQ (GETREG, get_qregxi, 0, AUTO_FP)
  83   VAR1 (GETREGP, get_qregoi, 0, AUTO_FP, v2di)
  84   VAR1 (GETREGP, get_qregci, 0, AUTO_FP, v2di)
  85   VAR1 (GETREGP, get_qregxi, 0, AUTO_FP, v2di)
  86   /* Implemented by aarch64_set_qreg<VSTRUCT:mode><VQ:mode>.  */
  87   BUILTIN_VQ (SETREG, set_qregoi, 0, AUTO_FP)
  88   BUILTIN_VQ (SETREG, set_qregci, 0, AUTO_FP)
  89   BUILTIN_VQ (SETREG, set_qregxi, 0, AUTO_FP)
  90   VAR1 (SETREGP, set_qregoi, 0, AUTO_FP, v2di)
  91   VAR1 (SETREGP, set_qregci, 0, AUTO_FP, v2di)
  92   VAR1 (SETREGP, set_qregxi, 0, AUTO_FP, v2di)
  93   /* Implemented by aarch64_ld1x2<VQ:mode>. */
  94   BUILTIN_VQ (LOADSTRUCT, ld1x2, 0, ALL)
  95   /* Implemented by aarch64_ld1x2<VDC:mode>. */
  96   BUILTIN_VDC (LOADSTRUCT, ld1x2, 0, ALL)
  97   /* Implemented by aarch64_ld<VSTRUCT:nregs><VDC:mode>.  */
  98   BUILTIN_VDC (LOADSTRUCT, ld2, 0, ALL)
  99   BUILTIN_VDC (LOADSTRUCT, ld3, 0, ALL)
 100   BUILTIN_VDC (LOADSTRUCT, ld4, 0, ALL)
 101   /* Implemented by aarch64_ld<VSTRUCT:nregs><VQ:mode>.  */
 102   BUILTIN_VQ (LOADSTRUCT, ld2, 0, ALL)
 103   BUILTIN_VQ (LOADSTRUCT, ld3, 0, ALL)
 104   BUILTIN_VQ (LOADSTRUCT, ld4, 0, ALL)
 105   /* Implemented by aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>.  */
 106   BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0, ALL)
 107   BUILTIN_VALLDIF (LOADSTRUCT, ld3r, 0, ALL)
 108   BUILTIN_VALLDIF (LOADSTRUCT, ld4r, 0, ALL)
 109   /* Implemented by aarch64_ld<VSTRUCT:nregs>_lane<VQ:mode>.  */
 110   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld2_lane, 0, ALL)
 111   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld3_lane, 0, ALL)
 112   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld4_lane, 0, ALL)
 113   /* Implemented by aarch64_st<VSTRUCT:nregs><VDC:mode>.  */
 114   BUILTIN_VDC (STORESTRUCT, st2, 0, STORE)
 115   BUILTIN_VDC (STORESTRUCT, st3, 0, STORE)
 116   BUILTIN_VDC (STORESTRUCT, st4, 0, STORE)
 117   /* Implemented by aarch64_st<VSTRUCT:nregs><VQ:mode>.  */
 118   BUILTIN_VQ (STORESTRUCT, st2, 0, STORE)
 119   BUILTIN_VQ (STORESTRUCT, st3, 0, STORE)
 120   BUILTIN_VQ (STORESTRUCT, st4, 0, STORE)
 121
 122   BUILTIN_VALLDIF (STORESTRUCT_LANE, st2_lane, 0, ALL)
 123   BUILTIN_VALLDIF (STORESTRUCT_LANE, st3_lane, 0, ALL)
 124   BUILTIN_VALLDIF (STORESTRUCT_LANE, st4_lane, 0, ALL)
 125
 126   BUILTIN_VQW (BINOP, saddl2, 0, NONE)
 127   BUILTIN_VQW (BINOP, uaddl2, 0, NONE)
 128   BUILTIN_VQW (BINOP, ssubl2, 0, NONE)
 129   BUILTIN_VQW (BINOP, usubl2, 0, NONE)
 130   BUILTIN_VQW (BINOP, saddw2, 0, NONE)
 131   BUILTIN_VQW (BINOP, uaddw2, 0, NONE)
 132   BUILTIN_VQW (BINOP, ssubw2, 0, NONE)
 133   BUILTIN_VQW (BINOP, usubw2, 0, NONE)
 134   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
 135   BUILTIN_VD_BHSI (BINOP, saddl, 0, NONE)
 136   BUILTIN_VD_BHSI (BINOP, uaddl, 0, NONE)
 137   BUILTIN_VD_BHSI (BINOP, ssubl, 0, NONE)
 138   BUILTIN_VD_BHSI (BINOP, usubl, 0, NONE)
 139   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
 140   BUILTIN_VD_BHSI (BINOP, saddw, 0, NONE)
 141   BUILTIN_VD_BHSI (BINOP, uaddw, 0, NONE)
 142   BUILTIN_VD_BHSI (BINOP, ssubw, 0, NONE)
 143   BUILTIN_VD_BHSI (BINOP, usubw, 0, NONE)
 144   /* Implemented by aarch64_<sur>h<addsub><mode>.  */
 145   BUILTIN_VDQ_BHSI (BINOP, shadd, 0, NONE)
 146   BUILTIN_VDQ_BHSI (BINOP, shsub, 0, NONE)
 147   BUILTIN_VDQ_BHSI (BINOP, uhadd, 0, NONE)
 148   BUILTIN_VDQ_BHSI (BINOP, uhsub, 0, NONE)
 149   BUILTIN_VDQ_BHSI (BINOP, srhadd, 0, NONE)
 150   BUILTIN_VDQ_BHSI (BINOP, urhadd, 0, NONE)
 151
 152   /* Implemented by aarch64_<su>abd<mode>.  */
 153   BUILTIN_VDQ_BHSI (BINOP, sabd, 0, NONE)
 154   BUILTIN_VDQ_BHSI (BINOPU, uabd, 0, NONE)
 155
 156   /* Implemented by aarch64_<su>aba<mode>.  */
 157   BUILTIN_VDQ_BHSI (TERNOP, saba, 0, NONE)
 158   BUILTIN_VDQ_BHSI (TERNOPU, uaba, 0, NONE)
 159
 160   BUILTIN_VDQV_S (BINOP, sadalp, 0, NONE)
 161   BUILTIN_VDQV_S (BINOPU, uadalp, 0, NONE)
 162
 163   /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
 164   BUILTIN_VQN (BINOP, addhn, 0, NONE)
 165   BUILTIN_VQN (BINOP, subhn, 0, NONE)
 166   BUILTIN_VQN (BINOP, raddhn, 0, NONE)
 167   BUILTIN_VQN (BINOP, rsubhn, 0, NONE)
 168   /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
 169   BUILTIN_VQN (TERNOP, addhn2, 0, NONE)
 170   BUILTIN_VQN (TERNOP, subhn2, 0, NONE)
 171   BUILTIN_VQN (TERNOP, raddhn2, 0, NONE)
 172   BUILTIN_VQN (TERNOP, rsubhn2, 0, NONE)
 173
 174   /* Implemented by aarch64_<us>xtl<mode>.  */
 175   BUILTIN_VQN (UNOP, sxtl, 0, NONE)
 176   BUILTIN_VQN (UNOPU, uxtl, 0, NONE)
 177
 178   /* Implemented by aarch64_xtn<mode>.  */
 179   BUILTIN_VQN (UNOP, xtn, 0, NONE)
 180
 181   /* Implemented by aarch64_mla<mode>.  */
 182   BUILTIN_VDQ_BHSI (TERNOP, mla, 0, NONE)
 183
 184   /* Implemented by aarch64_<su>mlsl<mode>.  */
 185   BUILTIN_VD_BHSI (TERNOP, smlsl, 0, NONE)
 186   BUILTIN_VD_BHSI (TERNOPU, umlsl, 0, NONE)
 187
 188   /* Implemented by aarch64_<su>mlsl_hi<mode>.  */
 189   BUILTIN_VQW (TERNOP, smlsl_hi, 0, NONE)
 190   BUILTIN_VQW (TERNOPU, umlsl_hi, 0, NONE)
 191
 192   BUILTIN_VSQN_HSDI (UNOPUS, sqmovun, 0, NONE)
 193   /* Implemented by aarch64_<sur>qmovn<mode>.  */
 194   BUILTIN_VSQN_HSDI (UNOP, sqmovn, 0, NONE)
 195   BUILTIN_VSQN_HSDI (UNOP, uqmovn, 0, NONE)
 196
 197   /* Implemented by aarch64_<su>qxtn2<mode>.  */
 198   BUILTIN_VQN (BINOP, sqxtn2, 0, NONE)
 199   BUILTIN_VQN (BINOPU, uqxtn2, 0, NONE)
 200
 201   /* Implemented by aarch64_s<optab><mode>.  */
 202   BUILTIN_VSDQ_I (UNOP, sqabs, 0, NONE)
 203   BUILTIN_VSDQ_I (UNOP, sqneg, 0, NONE)
 204
 205   /* Implemented by aarch64_sqdml<SBINQOPS:as>l<mode>.  */
 206   BUILTIN_VSD_HSI (TERNOP, sqdmlal, 0, NONE)
 207   BUILTIN_VSD_HSI (TERNOP, sqdmlsl, 0, NONE)
 208   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_lane<mode>.  */
 209   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_lane, 0, NONE)
 210   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_lane, 0, NONE)
 211   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_laneq<mode>.  */
 212   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_laneq, 0, NONE)
 213   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_laneq, 0, NONE)
 214   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_n<mode>.  */
 215   BUILTIN_VD_HSI (TERNOP, sqdmlal_n, 0, NONE)
 216   BUILTIN_VD_HSI (TERNOP, sqdmlsl_n, 0, NONE)
 217
 218   BUILTIN_VQ_HSI (TERNOP, sqdmlal2, 0, NONE)
 219   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2, 0, NONE)
 220   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_lane, 0, NONE)
 221   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_lane, 0, NONE)
 222   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_laneq, 0, NONE)
 223   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_laneq, 0, NONE)
 224   BUILTIN_VQ_HSI (TERNOP, sqdmlal2_n, 0, NONE)
 225   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2_n, 0, NONE)
 226
 227   BUILTIN_VD_BHSI (BINOP, intrinsic_vec_smult_lo_, 0, NONE)
 228   BUILTIN_VD_BHSI (BINOPU, intrinsic_vec_umult_lo_, 0, NONE)
 229
 230   BUILTIN_VQW (BINOP, vec_widen_smult_hi_, 10, NONE)
 231   BUILTIN_VQW (BINOPU, vec_widen_umult_hi_, 10, NONE)
 232
 233   BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_lane_, 0, ALL)
 234   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_lane_, 0, ALL)
 235   BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_laneq_, 0, ALL)
 236   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_laneq_, 0, ALL)
 237   BUILTIN_VD_HSI (TERNOPU_LANE, vec_umult_lane_, 0, ALL)
 238   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlal_lane_, 0, ALL)
 239   BUILTIN_VD_HSI (TERNOPU_LANE, vec_umult_laneq_, 0, ALL)
 240   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlal_laneq_, 0, ALL)
 241
 242   BUILTIN_VSD_HSI (BINOP, sqdmull, 0, NONE)
 243   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_lane, 0, NONE)
 244   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_laneq, 0, NONE)
 245   BUILTIN_VD_HSI (BINOP, sqdmull_n, 0, NONE)
 246   BUILTIN_VQ_HSI (BINOP, sqdmull2, 0, NONE)
 247   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_lane, 0, NONE)
 248   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_laneq, 0, NONE)
 249   BUILTIN_VQ_HSI (BINOP, sqdmull2_n, 0, NONE)
 250   /* Implemented by aarch64_sq<r>dmulh<mode>.  */
 251   BUILTIN_VSDQ_HSI (BINOP, sqdmulh, 0, NONE)
 252   BUILTIN_VSDQ_HSI (BINOP, sqrdmulh, 0, NONE)
 253   /* Implemented by aarch64_sq<r>dmulh_lane<q><mode>.  */
 254   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_lane, 0, NONE)
 255   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_laneq, 0, NONE)
 256   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_lane, 0, NONE)
 257   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_laneq, 0, NONE)
 258
 259   BUILTIN_VSDQ_I_DI (BINOP, ashl, 3, ALL)
 260   /* Implemented by aarch64_<sur>shl<mode>.  */
 261   BUILTIN_VSDQ_I_DI (BINOP, sshl, 0, ALL)
 262   BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0, ALL)
 263   BUILTIN_VSDQ_I_DI (BINOP, srshl, 0, ALL)
 264   BUILTIN_VSDQ_I_DI (BINOP_UUS, urshl, 0, ALL)
 265
 266   /* Implemented by aarch64_<sur><dotprod>{_lane}{q}<dot_mode>.  */
 267   BUILTIN_VB (TERNOP, sdot, 0, NONE)
 268   BUILTIN_VB (TERNOPU, udot, 0, NONE)
 269   BUILTIN_VB (TERNOP_SSUS, usdot, 0, NONE)
 270   BUILTIN_VB (QUADOP_LANE, sdot_lane, 0, ALL)
 271   BUILTIN_VB (QUADOPU_LANE, udot_lane, 0, ALL)
 272   BUILTIN_VB (QUADOP_LANE, sdot_laneq, 0, ALL)
 273   BUILTIN_VB (QUADOPU_LANE, udot_laneq, 0, ALL)
 274   BUILTIN_VB (QUADOPSSUS_LANE_QUADTUP, usdot_lane, 0, ALL)
 275   BUILTIN_VB (QUADOPSSUS_LANE_QUADTUP, usdot_laneq, 0, ALL)
 276   BUILTIN_VB (QUADOPSSSU_LANE_QUADTUP, sudot_lane, 0, ALL)
 277   BUILTIN_VB (QUADOPSSSU_LANE_QUADTUP, sudot_laneq, 0, ALL)
 278
 279   /* Implemented by aarch64_fcadd<rot><mode>.   */
 280   BUILTIN_VHSDF (BINOP, fcadd90, 0, FP)
 281   BUILTIN_VHSDF (BINOP, fcadd270, 0, FP)
 282
 283   /* Implemented by aarch64_fcmla{_lane}{q}<rot><mode>.   */
 284   BUILTIN_VHSDF (TERNOP, fcmla0, 0, FP)
 285   BUILTIN_VHSDF (TERNOP, fcmla90, 0, FP)
 286   BUILTIN_VHSDF (TERNOP, fcmla180, 0, FP)
 287   BUILTIN_VHSDF (TERNOP, fcmla270, 0, FP)
 288   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane0, 0, ALL)
 289   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane90, 0, ALL)
 290   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane180, 0, ALL)
 291   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane270, 0, ALL)
 292
 293   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane0, 0, ALL)
 294   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane90, 0, ALL)
 295   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane180, 0, ALL)
 296   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane270, 0, ALL)
 297
 298   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3, ALL)
 299   VAR1 (SHIFTIMM, ashr_simd, 0, ALL, di)
 300   BUILTIN_VDQ_I (SHIFTIMM, lshr, 3, ALL)
 301   VAR1 (USHIFTIMM, lshr_simd, 0, ALL, di)
 302   /* Implemented by aarch64_<sur>shr_n<mode>.  */
 303   BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n, 0, ALL)
 304   BUILTIN_VSDQ_I_DI (USHIFTIMM, urshr_n, 0, ALL)
 305   /* Implemented by aarch64_<sur>sra_n<mode>.  */
 306   BUILTIN_VSDQ_I_DI (SHIFTACC, ssra_n, 0, ALL)
 307   BUILTIN_VSDQ_I_DI (USHIFTACC, usra_n, 0, ALL)
 308   BUILTIN_VSDQ_I_DI (SHIFTACC, srsra_n, 0, ALL)
 309   BUILTIN_VSDQ_I_DI (USHIFTACC, ursra_n, 0, ALL)
 310   /* Implemented by aarch64_<sur>shll_n<mode>.  */
 311   BUILTIN_VD_BHSI (SHIFTIMM, sshll_n, 0, ALL)
 312   BUILTIN_VD_BHSI (USHIFTIMM, ushll_n, 0, ALL)
 313   /* Implemented by aarch64_<sur>shll2_n<mode>.  */
 314   BUILTIN_VQW (SHIFTIMM, sshll2_n, 0, ALL)
 315   BUILTIN_VQW (SHIFTIMM, ushll2_n, 0, ALL)
 316   /* Implemented by aarch64_<sur>q<r>shr<u>n_n<mode>.  */
 317   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrun_n, 0, NONE)
 318   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrun_n, 0, NONE)
 319   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrn_n, 0, NONE)
 320   BUILTIN_VSQN_HSDI (USHIFTIMM, uqshrn_n, 0, NONE)
 321   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrn_n, 0, NONE)
 322   BUILTIN_VSQN_HSDI (USHIFTIMM, uqrshrn_n, 0, NONE)
 323   /* Implemented by aarch64_<sur>q<r>shr<u>n2_n<mode>.  */
 324   BUILTIN_VQN (SHIFT2IMM_UUSS, sqshrun2_n, 0, NONE)
 325   BUILTIN_VQN (SHIFT2IMM_UUSS, sqrshrun2_n, 0, NONE)
 326   BUILTIN_VQN (SHIFT2IMM, sqshrn2_n, 0, NONE)
 327   BUILTIN_VQN (USHIFT2IMM, uqshrn2_n, 0, NONE)
 328   BUILTIN_VQN (SHIFT2IMM, sqrshrn2_n, 0, NONE)
 329   BUILTIN_VQN (USHIFT2IMM, uqrshrn2_n, 0, NONE)
 330   /* Implemented by aarch64_<sur>s<lr>i_n<mode>.  */
 331   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssri_n, 0, ALL)
 332   BUILTIN_VSDQ_I_DI (USHIFTACC, usri_n, 0, ALL)
 333   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssli_n, 0, ALL)
 334   VAR2 (SHIFTINSERTP, ssli_n, 0, ALL, di, v2di)
 335   BUILTIN_VSDQ_I_DI (USHIFTACC, usli_n, 0, ALL)
 336   /* Implemented by aarch64_<sur>qshl<u>_n<mode>.  */
 337   BUILTIN_VSDQ_I (SHIFTIMM_USS, sqshlu_n, 0, NONE)
 338   BUILTIN_VSDQ_I (SHIFTIMM, sqshl_n, 0, NONE)
 339   BUILTIN_VSDQ_I (USHIFTIMM, uqshl_n, 0, NONE)
 340
 341   /* Implemented by aarch64_xtn2<mode>.  */
 342   BUILTIN_VQN (UNOP, xtn2, 0, NONE)
 343
 344   /* Implemented by aarch64_reduc_plus_<mode>.  */
 345   BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, NONE)
 346
 347   /* Implemented by reduc_<maxmin_uns>_scal_<mode> (producing scalar).  */
 348   BUILTIN_VDQIF_F16 (UNOP, reduc_smax_scal_, 10, NONE)
 349   BUILTIN_VDQIF_F16 (UNOP, reduc_smin_scal_, 10, NONE)
 350   BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10, NONE)
 351   BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10, NONE)
 352   BUILTIN_VHSDF (UNOP, reduc_smax_nan_scal_, 10, NONE)
 353   BUILTIN_VHSDF (UNOP, reduc_smin_nan_scal_, 10, NONE)
 354
 355   /* Implemented by <maxmin_uns><mode>3.
 356      smax variants map to fmaxnm,
 357      smax_nan variants map to fmax.  */
 358   BUILTIN_VDQ_BHSI (BINOP, smax, 3, NONE)
 359   BUILTIN_VDQ_BHSI (BINOP, smin, 3, NONE)
 360   BUILTIN_VDQ_BHSI (BINOP, umax, 3, NONE)
 361   BUILTIN_VDQ_BHSI (BINOP, umin, 3, NONE)
 362   BUILTIN_VHSDF_DF (BINOP, smax_nan, 3, NONE)
 363   BUILTIN_VHSDF_DF (BINOP, smin_nan, 3, NONE)
 364
 365   /* Implemented by <maxmin_uns><mode>3.  */
 366   BUILTIN_VHSDF_HSDF (BINOP, fmax, 3, FP)
 367   BUILTIN_VHSDF_HSDF (BINOP, fmin, 3, FP)
 368
 369   /* Implemented by aarch64_<maxmin_uns>p<mode>.  */
 370   BUILTIN_VDQ_BHSI (BINOP, smaxp, 0, NONE)
 371   BUILTIN_VDQ_BHSI (BINOP, sminp, 0, NONE)
 372   BUILTIN_VDQ_BHSI (BINOP, umaxp, 0, NONE)
 373   BUILTIN_VDQ_BHSI (BINOP, uminp, 0, NONE)
 374   BUILTIN_VHSDF (BINOP, smaxp, 0, NONE)
 375   BUILTIN_VHSDF (BINOP, sminp, 0, NONE)
 376   BUILTIN_VHSDF (BINOP, smax_nanp, 0, NONE)
 377   BUILTIN_VHSDF (BINOP, smin_nanp, 0, NONE)
 378
 379   /* Implemented by <frint_pattern><mode>2.  */
 380   BUILTIN_VHSDF (UNOP, btrunc, 2, FP)
 381   BUILTIN_VHSDF (UNOP, ceil, 2, FP)
 382   BUILTIN_VHSDF (UNOP, floor, 2, FP)
 383   BUILTIN_VHSDF (UNOP, nearbyint, 2, FP)
 384   BUILTIN_VHSDF (UNOP, rint, 2, FP)
 385   BUILTIN_VHSDF (UNOP, round, 2, FP)
 386   BUILTIN_VHSDF_HSDF (UNOP, frintn, 2, FP)
 387
 388   VAR1 (UNOP, btrunc, 2, FP, hf)
 389   VAR1 (UNOP, ceil, 2, FP, hf)
 390   VAR1 (UNOP, floor, 2, FP, hf)
 391   VAR1 (UNOP, nearbyint, 2, FP, hf)
 392   VAR1 (UNOP, rint, 2, FP, hf)
 393   VAR1 (UNOP, round, 2, FP, hf)
 394
 395   /* Implemented by l<fcvt_pattern><su_optab><VQDF:mode><vcvt_target>2.  */
 396   VAR1 (UNOP, lbtruncv4hf, 2, FP, v4hi)
 397   VAR1 (UNOP, lbtruncv8hf, 2, FP, v8hi)
 398   VAR1 (UNOP, lbtruncv2sf, 2, FP, v2si)
 399   VAR1 (UNOP, lbtruncv4sf, 2, FP, v4si)
 400   VAR1 (UNOP, lbtruncv2df, 2, FP, v2di)
 401
 402   VAR1 (UNOPUS, lbtruncuv4hf, 2, FP, v4hi)
 403   VAR1 (UNOPUS, lbtruncuv8hf, 2, FP, v8hi)
 404   VAR1 (UNOPUS, lbtruncuv2sf, 2, FP, v2si)
 405   VAR1 (UNOPUS, lbtruncuv4sf, 2, FP, v4si)
 406   VAR1 (UNOPUS, lbtruncuv2df, 2, FP, v2di)
 407
 408   VAR1 (UNOP, lroundv4hf, 2, FP, v4hi)
 409   VAR1 (UNOP, lroundv8hf, 2, FP, v8hi)
 410   VAR1 (UNOP, lroundv2sf, 2, FP, v2si)
 411   VAR1 (UNOP, lroundv4sf, 2, FP, v4si)
 412   VAR1 (UNOP, lroundv2df, 2, FP, v2di)
 413   /* Implemented by l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2.  */
 414   BUILTIN_GPI_I16 (UNOP, lroundhf, 2, FP)
 415   VAR1 (UNOP, lroundsf, 2, FP, si)
 416   VAR1 (UNOP, lrounddf, 2, FP, di)
 417
 418   VAR1 (UNOPUS, lrounduv4hf, 2, FP, v4hi)
 419   VAR1 (UNOPUS, lrounduv8hf, 2, FP, v8hi)
 420   VAR1 (UNOPUS, lrounduv2sf, 2, FP, v2si)
 421   VAR1 (UNOPUS, lrounduv4sf, 2, FP, v4si)
 422   VAR1 (UNOPUS, lrounduv2df, 2, FP, v2di)
 423   BUILTIN_GPI_I16 (UNOPUS, lrounduhf, 2, FP)
 424   VAR1 (UNOPUS, lroundusf, 2, FP, si)
 425   VAR1 (UNOPUS, lroundudf, 2, FP, di)
 426
 427   VAR1 (UNOP, lceilv4hf, 2, FP, v4hi)
 428   VAR1 (UNOP, lceilv8hf, 2, FP, v8hi)
 429   VAR1 (UNOP, lceilv2sf, 2, FP, v2si)
 430   VAR1 (UNOP, lceilv4sf, 2, FP, v4si)
 431   VAR1 (UNOP, lceilv2df, 2, FP, v2di)
 432   BUILTIN_GPI_I16 (UNOP, lceilhf, 2, FP)
 433
 434   VAR1 (UNOPUS, lceiluv4hf, 2, FP, v4hi)
 435   VAR1 (UNOPUS, lceiluv8hf, 2, FP, v8hi)
 436   VAR1 (UNOPUS, lceiluv2sf, 2, FP, v2si)
 437   VAR1 (UNOPUS, lceiluv4sf, 2, FP, v4si)
 438   VAR1 (UNOPUS, lceiluv2df, 2, FP, v2di)
 439   BUILTIN_GPI_I16 (UNOPUS, lceiluhf, 2, FP)
 440   VAR1 (UNOPUS, lceilusf, 2, FP, si)
 441   VAR1 (UNOPUS, lceiludf, 2, FP, di)
 442
 443   VAR1 (UNOP, lfloorv4hf, 2, FP, v4hi)
 444   VAR1 (UNOP, lfloorv8hf, 2, FP, v8hi)
 445   VAR1 (UNOP, lfloorv2sf, 2, FP, v2si)
 446   VAR1 (UNOP, lfloorv4sf, 2, FP, v4si)
 447   VAR1 (UNOP, lfloorv2df, 2, FP, v2di)
 448   BUILTIN_GPI_I16 (UNOP, lfloorhf, 2, FP)
 449
 450   VAR1 (UNOPUS, lflooruv4hf, 2, FP, v4hi)
 451   VAR1 (UNOPUS, lflooruv8hf, 2, FP, v8hi)
 452   VAR1 (UNOPUS, lflooruv2sf, 2, FP, v2si)
 453   VAR1 (UNOPUS, lflooruv4sf, 2, FP, v4si)
 454   VAR1 (UNOPUS, lflooruv2df, 2, FP, v2di)
 455   BUILTIN_GPI_I16 (UNOPUS, lflooruhf, 2, FP)
 456   VAR1 (UNOPUS, lfloorusf, 2, FP, si)
 457   VAR1 (UNOPUS, lfloorudf, 2, FP, di)
 458
 459   VAR1 (UNOP, lfrintnv4hf, 2, FP, v4hi)
 460   VAR1 (UNOP, lfrintnv8hf, 2, FP, v8hi)
 461   VAR1 (UNOP, lfrintnv2sf, 2, FP, v2si)
 462   VAR1 (UNOP, lfrintnv4sf, 2, FP, v4si)
 463   VAR1 (UNOP, lfrintnv2df, 2, FP, v2di)
 464   BUILTIN_GPI_I16 (UNOP, lfrintnhf, 2, FP)
 465   VAR1 (UNOP, lfrintnsf, 2, FP, si)
 466   VAR1 (UNOP, lfrintndf, 2, FP, di)
 467
 468   VAR1 (UNOPUS, lfrintnuv4hf, 2, FP, v4hi)
 469   VAR1 (UNOPUS, lfrintnuv8hf, 2, FP, v8hi)
 470   VAR1 (UNOPUS, lfrintnuv2sf, 2, FP, v2si)
 471   VAR1 (UNOPUS, lfrintnuv4sf, 2, FP, v4si)
 472   VAR1 (UNOPUS, lfrintnuv2df, 2, FP, v2di)
 473   BUILTIN_GPI_I16 (UNOPUS, lfrintnuhf, 2, FP)
 474   VAR1 (UNOPUS, lfrintnusf, 2, FP, si)
 475   VAR1 (UNOPUS, lfrintnudf, 2, FP, di)
 476
 477   /* Implemented by <optab><fcvt_target><VDQF:mode>2.  */
 478   VAR1 (UNOP, floatv4hi, 2, FP, v4hf)
 479   VAR1 (UNOP, floatv8hi, 2, FP, v8hf)
 480   VAR1 (UNOP, floatv2si, 2, FP, v2sf)
 481   VAR1 (UNOP, floatv4si, 2, FP, v4sf)
 482   VAR1 (UNOP, floatv2di, 2, FP, v2df)
 483
 484   VAR1 (UNOP, floatunsv4hi, 2, FP, v4hf)
 485   VAR1 (UNOP, floatunsv8hi, 2, FP, v8hf)
 486   VAR1 (UNOP, floatunsv2si, 2, FP, v2sf)
 487   VAR1 (UNOP, floatunsv4si, 2, FP, v4sf)
 488   VAR1 (UNOP, floatunsv2di, 2, FP, v2df)
 489
 490   VAR5 (UNOPU, bswap, 2, ALL, v4hi, v8hi, v2si, v4si, v2di)
 491
 492   BUILTIN_VB (UNOP, rbit, 0, ALL)
 493
 494   /* Implemented by
 495      aarch64_<PERMUTE:perm_insn><mode>.  */
 496   BUILTIN_VALL (BINOP, zip1, 0, ALL)
 497   BUILTIN_VALL (BINOP, zip2, 0, ALL)
 498   BUILTIN_VALL (BINOP, uzp1, 0, ALL)
 499   BUILTIN_VALL (BINOP, uzp2, 0, ALL)
 500   BUILTIN_VALL (BINOP, trn1, 0, ALL)
 501   BUILTIN_VALL (BINOP, trn2, 0, ALL)
 502
 503   BUILTIN_GPF_F16 (UNOP, frecpe, 0, FP)
 504   BUILTIN_GPF_F16 (UNOP, frecpx, 0, FP)
 505
 506   BUILTIN_VDQ_SI (UNOP, urecpe, 0, NONE)
 507
 508   BUILTIN_VHSDF (UNOP, frecpe, 0, FP)
 509   BUILTIN_VHSDF_HSDF (BINOP, frecps, 0, FP)
 510
 511   /* Implemented by a mixture of abs2 patterns.  Note the DImode builtin is
 512      only ever used for the int64x1_t intrinsic, there is no scalar version.  */
 513   BUILTIN_VSDQ_I_DI (UNOP, abs, 0, AUTO_FP)
 514   BUILTIN_VHSDF (UNOP, abs, 2, AUTO_FP)
 515   VAR1 (UNOP, abs, 2, AUTO_FP, hf)
 516
 517   BUILTIN_VQ_HSF (UNOP, vec_unpacks_hi_, 10, FP)
 518   VAR1 (BINOP, float_truncate_hi_, 0, FP, v4sf)
 519   VAR1 (BINOP, float_truncate_hi_, 0, FP, v8hf)
 520
 521   VAR1 (UNOP, float_extend_lo_, 0, FP, v2df)
 522   VAR1 (UNOP, float_extend_lo_,  0, FP, v4sf)
 523   BUILTIN_VDF (UNOP, float_truncate_lo_, 0, FP)
 524
 525   /* Implemented by aarch64_ld1<VALL_F16:mode>.  */
 526   BUILTIN_VALL_F16 (LOAD1, ld1, 0, ALL)
 527   VAR1(STORE1P, ld1, 0, ALL, v2di)
 528
 529   /* Implemented by aarch64_st1<VALL_F16:mode>.  */
 530   BUILTIN_VALL_F16 (STORE1, st1, 0, STORE)
 531   VAR1 (STORE1P, st1, 0, STORE, v2di)
 532
 533   /* Implemented by aarch64_ld1x3<VALLDIF:mode>.  */
 534   BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0, ALL)
 535
 536   /* Implemented by aarch64_ld1x4<VALLDIF:mode>.  */
 537   BUILTIN_VALLDIF (LOADSTRUCT, ld1x4, 0, ALL)
 538
 539   /* Implemented by aarch64_st1x2<VALLDIF:mode>.  */
 540   BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0, STORE)
 541
 542   /* Implemented by aarch64_st1x3<VALLDIF:mode>.  */
 543   BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0, STORE)
 544
 545   /* Implemented by aarch64_st1x4<VALLDIF:mode>.  */
 546   BUILTIN_VALLDIF (STORESTRUCT, st1x4, 0, STORE)
 547
 548   /* Implemented by fma<mode>4.  */
 549   BUILTIN_VHSDF (TERNOP, fma, 4, FP)
 550   VAR1 (TERNOP, fma, 4, FP, hf)
 551   /* Implemented by fnma<mode>4.  */
 552   BUILTIN_VHSDF (TERNOP, fnma, 4, FP)
 553   VAR1 (TERNOP, fnma, 4, FP, hf)
 554
 555   /* Implemented by aarch64_simd_bsl<mode>.  */
 556   BUILTIN_VDQQH (BSL_P, simd_bsl, 0, ALL)
 557   VAR2 (BSL_P, simd_bsl,0, ALL, di, v2di)
 558   BUILTIN_VSDQ_I_DI (BSL_U, simd_bsl, 0, ALL)
 559   BUILTIN_VALLDIF (BSL_S, simd_bsl, 0, ALL)
 560
 561   /* Implemented by aarch64_crypto_aes<op><mode>.  */
 562   VAR1 (BINOPU, crypto_aese, 0, NONE, v16qi)
 563   VAR1 (BINOPU, crypto_aesd, 0, NONE, v16qi)
 564   VAR1 (UNOPU, crypto_aesmc, 0, NONE, v16qi)
 565   VAR1 (UNOPU, crypto_aesimc, 0, NONE, v16qi)
 566
 567   /* Implemented by aarch64_crypto_sha1<op><mode>.  */
 568   VAR1 (UNOPU, crypto_sha1h, 0, NONE, si)
 569   VAR1 (BINOPU, crypto_sha1su1, 0, NONE, v4si)
 570   VAR1 (TERNOPU, crypto_sha1c, 0, NONE, v4si)
 571   VAR1 (TERNOPU, crypto_sha1m, 0, NONE, v4si)
 572   VAR1 (TERNOPU, crypto_sha1p, 0, NONE, v4si)
 573   VAR1 (TERNOPU, crypto_sha1su0, 0, NONE, v4si)
 574
 575   /* Implemented by aarch64_crypto_sha256<op><mode>.  */
 576   VAR1 (TERNOPU, crypto_sha256h, 0, NONE, v4si)
 577   VAR1 (TERNOPU, crypto_sha256h2, 0, NONE, v4si)
 578   VAR1 (BINOPU, crypto_sha256su0, 0, NONE, v4si)
 579   VAR1 (TERNOPU, crypto_sha256su1, 0, NONE, v4si)
 580
 581   /* Implemented by aarch64_crypto_pmull<mode>.  */
 582   VAR1 (BINOPP, crypto_pmull, 0, NONE, di)
 583   VAR1 (BINOPP, crypto_pmull, 0, NONE, v2di)
 584
 585   /* Implemented by aarch64_tbl3<mode>.  */
 586   VAR1 (BINOP, tbl3, 0, NONE, v8qi)
 587   VAR1 (BINOP, tbl3, 0, NONE, v16qi)
 588
 589   /* Implemented by aarch64_qtbl3<mode>.  */
 590   VAR1 (BINOP, qtbl3, 0, NONE, v8qi)
 591   VAR1 (BINOP, qtbl3, 0, NONE, v16qi)
 592
 593   /* Implemented by aarch64_qtbl4<mode>.  */
 594   VAR1 (BINOP, qtbl4, 0, NONE, v8qi)
 595   VAR1 (BINOP, qtbl4, 0, NONE, v16qi)
 596
 597   /* Implemented by aarch64_tbx4<mode>.  */
 598   VAR1 (TERNOP, tbx4, 0, NONE, v8qi)
 599   VAR1 (TERNOP, tbx4, 0, NONE, v16qi)
 600
 601   /* Implemented by aarch64_qtbx3<mode>.  */
 602   VAR1 (TERNOP, qtbx3, 0, NONE, v8qi)
 603   VAR1 (TERNOP, qtbx3, 0, NONE, v16qi)
 604
 605   /* Implemented by aarch64_qtbx4<mode>.  */
 606   VAR1 (TERNOP, qtbx4, 0, NONE, v8qi)
 607   VAR1 (TERNOP, qtbx4, 0, NONE, v16qi)
 608
 609   /* Builtins for ARMv8.1-A Adv.SIMD instructions.  */
 610
 611   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>.  */
 612   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlah, 0, NONE)
 613   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlsh, 0, NONE)
 614
 615   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>.  */
 616   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_lane, 0, NONE)
 617   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_lane, 0, NONE)
 618
 619   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>.  */
 620   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_laneq, 0, NONE)
 621   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_laneq, 0, NONE)
 622
 623   /* Implemented by <FCVT_F2FIXED/FIXED2F:fcvt_fixed_insn><*><*>3.  */
 624   BUILTIN_VSDQ_HSDI (SHIFTIMM, scvtf, 3, ALL)
 625   BUILTIN_VSDQ_HSDI (FCVTIMM_SUS, ucvtf, 3, ALL)
 626   BUILTIN_VHSDF_HSDF (SHIFTIMM, fcvtzs, 3, ALL)
 627   BUILTIN_VHSDF_HSDF (SHIFTIMM_USS, fcvtzu, 3, ALL)
 628   VAR1 (SHIFTIMM, scvtfsi, 3, ALL, hf)
 629   VAR1 (SHIFTIMM, scvtfdi, 3, ALL, hf)
 630   VAR1 (FCVTIMM_SUS, ucvtfsi, 3, ALL, hf)
 631   VAR1 (FCVTIMM_SUS, ucvtfdi, 3, ALL, hf)
 632   BUILTIN_GPI (SHIFTIMM, fcvtzshf, 3, ALL)
 633   BUILTIN_GPI (SHIFTIMM_USS, fcvtzuhf, 3, ALL)
 634
 635   /* Implemented by aarch64_rsqrte<mode>.  */
 636   BUILTIN_VHSDF_HSDF (UNOP, rsqrte, 0, FP)
 637
 638   /* Implemented by aarch64_rsqrts<mode>.  */
 639   BUILTIN_VHSDF_HSDF (BINOP, rsqrts, 0, FP)
 640
 641   /* Implemented by fabd<mode>3.  */
 642   BUILTIN_VHSDF_HSDF (BINOP, fabd, 3, FP)
 643
 644   /* Implemented by aarch64_faddp<mode>.  */
 645   BUILTIN_VHSDF (BINOP, faddp, 0, FP)
 646
 647   /* Implemented by aarch64_cm<optab><mode>.  */
 648   BUILTIN_VHSDF_HSDF (BINOP_USS, cmeq, 0, FP)
 649   BUILTIN_VHSDF_HSDF (BINOP_USS, cmge, 0, FP)
 650   BUILTIN_VHSDF_HSDF (BINOP_USS, cmgt, 0, FP)
 651   BUILTIN_VHSDF_HSDF (BINOP_USS, cmle, 0, FP)
 652   BUILTIN_VHSDF_HSDF (BINOP_USS, cmlt, 0, FP)
 653
 654   /* Implemented by neg<mode>2.  */
 655   BUILTIN_VHSDF_HSDF (UNOP, neg, 2, ALL)
 656
 657   /* Implemented by aarch64_fac<optab><mode>.  */
 658   BUILTIN_VHSDF_HSDF (BINOP_USS, faclt, 0, FP)
 659   BUILTIN_VHSDF_HSDF (BINOP_USS, facle, 0, FP)
 660   BUILTIN_VHSDF_HSDF (BINOP_USS, facgt, 0, FP)
 661   BUILTIN_VHSDF_HSDF (BINOP_USS, facge, 0, FP)
 662
 663   /* Implemented by sqrt<mode>2.  */
 664   VAR1 (UNOP, sqrt, 2, FP, hf)
 665
 666   /* Implemented by <optab><mode>hf2.  */
 667   VAR1 (UNOP, floatdi, 2, FP, hf)
 668   VAR1 (UNOP, floatsi, 2, FP, hf)
 669   VAR1 (UNOP, floathi, 2, FP, hf)
 670   VAR1 (UNOPUS, floatunsdi, 2, FP, hf)
 671   VAR1 (UNOPUS, floatunssi, 2, FP, hf)
 672   VAR1 (UNOPUS, floatunshi, 2, FP, hf)
 673   BUILTIN_GPI_I16 (UNOP, fix_trunchf, 2, FP)
 674   BUILTIN_GPI (UNOP, fix_truncsf, 2, FP)
 675   BUILTIN_GPI (UNOP, fix_truncdf, 2, FP)
 676   BUILTIN_GPI_I16 (UNOPUS, fixuns_trunchf, 2, FP)
 677   BUILTIN_GPI (UNOPUS, fixuns_truncsf, 2, FP)
 678   BUILTIN_GPI (UNOPUS, fixuns_truncdf, 2, FP)
 679
 680   /* Implemented by aarch64_sm3ss1qv4si.  */
 681   VAR1 (TERNOPU, sm3ss1q, 0, NONE, v4si)
 682   /* Implemented by aarch64_sm3tt<sm3tt_op>qv4si.  */
 683   VAR1 (QUADOPUI, sm3tt1aq, 0, NONE, v4si)
 684   VAR1 (QUADOPUI, sm3tt1bq, 0, NONE, v4si)
 685   VAR1 (QUADOPUI, sm3tt2aq, 0, NONE, v4si)
 686   VAR1 (QUADOPUI, sm3tt2bq, 0, NONE, v4si)
 687   /* Implemented by aarch64_sm3partw<sm3part_op>qv4si.  */
 688   VAR1 (TERNOPU, sm3partw1q, 0, NONE, v4si)
 689   VAR1 (TERNOPU, sm3partw2q, 0, NONE, v4si)
 690   /* Implemented by aarch64_sm4eqv4si.  */
 691   VAR1 (BINOPU, sm4eq, 0, NONE, v4si)
 692   /* Implemented by aarch64_sm4ekeyqv4si.  */
 693   VAR1 (BINOPU, sm4ekeyq, 0, NONE, v4si)
 694   /* Implemented by aarch64_crypto_sha512hqv2di.  */
 695   VAR1 (TERNOPU, crypto_sha512hq, 0, NONE, v2di)
 696   /* Implemented by aarch64_sha512h2qv2di.  */
 697   VAR1 (TERNOPU, crypto_sha512h2q, 0, NONE, v2di)
 698   /* Implemented by aarch64_crypto_sha512su0qv2di.  */
 699   VAR1 (BINOPU, crypto_sha512su0q, 0, NONE, v2di)
 700   /* Implemented by aarch64_crypto_sha512su1qv2di.  */
 701   VAR1 (TERNOPU, crypto_sha512su1q, 0, NONE, v2di)
 702   /* Implemented by eor3q<mode>4.  */
 703   BUILTIN_VQ_I (TERNOPU, eor3q, 4, ALL)
 704   BUILTIN_VQ_I (TERNOP, eor3q, 4, ALL)
 705   /* Implemented by aarch64_rax1qv2di.  */
 706   VAR1 (BINOPU, rax1q, 0, ALL, v2di)
 707   /* Implemented by aarch64_xarqv2di.  */
 708   VAR1 (TERNOPUI, xarq, 0, ALL, v2di)
 709   /* Implemented by bcaxq<mode>4.  */
 710   BUILTIN_VQ_I (TERNOPU, bcaxq, 4, ALL)
 711   BUILTIN_VQ_I (TERNOP, bcaxq, 4, ALL)
 712
 713   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_low<mode>.  */
 714   VAR1 (TERNOP, fmlal_low, 0, FP, v2sf)
 715   VAR1 (TERNOP, fmlsl_low, 0, FP, v2sf)
 716   VAR1 (TERNOP, fmlalq_low, 0, FP, v4sf)
 717   VAR1 (TERNOP, fmlslq_low, 0, FP, v4sf)
 718   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_high<mode>.  */
 719   VAR1 (TERNOP, fmlal_high, 0, FP, v2sf)
 720   VAR1 (TERNOP, fmlsl_high, 0, FP, v2sf)
 721   VAR1 (TERNOP, fmlalq_high, 0, FP, v4sf)
 722   VAR1 (TERNOP, fmlslq_high, 0, FP, v4sf)
 723   /* Implemented by aarch64_fml<f16mac1>l_lane_lowv2sf.  */
 724   VAR1 (QUADOP_LANE, fmlal_lane_low, 0, ALL, v2sf)
 725   VAR1 (QUADOP_LANE, fmlsl_lane_low, 0, ALL, v2sf)
 726   /* Implemented by aarch64_fml<f16mac1>l_laneq_lowv2sf.  */
 727   VAR1 (QUADOP_LANE, fmlal_laneq_low, 0, ALL, v2sf)
 728   VAR1 (QUADOP_LANE, fmlsl_laneq_low, 0, ALL, v2sf)
 729   /* Implemented by aarch64_fml<f16mac1>lq_lane_lowv4sf.  */
 730   VAR1 (QUADOP_LANE, fmlalq_lane_low, 0, ALL, v4sf)
 731   VAR1 (QUADOP_LANE, fmlslq_lane_low, 0, ALL, v4sf)
 732   /* Implemented by aarch64_fml<f16mac1>lq_laneq_lowv4sf.  */
 733   VAR1 (QUADOP_LANE, fmlalq_laneq_low, 0, ALL, v4sf)
 734   VAR1 (QUADOP_LANE, fmlslq_laneq_low, 0, ALL, v4sf)
 735   /* Implemented by aarch64_fml<f16mac1>l_lane_highv2sf.  */
 736   VAR1 (QUADOP_LANE, fmlal_lane_high, 0, ALL, v2sf)
 737   VAR1 (QUADOP_LANE, fmlsl_lane_high, 0, ALL, v2sf)
 738   /* Implemented by aarch64_fml<f16mac1>l_laneq_highv2sf.  */
 739   VAR1 (QUADOP_LANE, fmlal_laneq_high, 0, ALL, v2sf)
 740   VAR1 (QUADOP_LANE, fmlsl_laneq_high, 0, ALL, v2sf)
 741   /* Implemented by aarch64_fml<f16mac1>lq_lane_highv4sf.  */
 742   VAR1 (QUADOP_LANE, fmlalq_lane_high, 0, ALL, v4sf)
 743   VAR1 (QUADOP_LANE, fmlslq_lane_high, 0, ALL, v4sf)
 744   /* Implemented by aarch64_fml<f16mac1>lq_laneq_highv4sf.  */
 745   VAR1 (QUADOP_LANE, fmlalq_laneq_high, 0, ALL, v4sf)
 746   VAR1 (QUADOP_LANE, fmlslq_laneq_high, 0, ALL, v4sf)
 747
 748   /* Implemented by aarch64_<frintnzs_op><mode>.  */
 749   BUILTIN_VSFDF (UNOP, frint32z, 0, FP)
 750   BUILTIN_VSFDF (UNOP, frint32x, 0, FP)
 751   BUILTIN_VSFDF (UNOP, frint64z, 0, FP)
 752   BUILTIN_VSFDF (UNOP, frint64x, 0, FP)
 753
 754   /* Implemented by aarch64_bfdot{_lane}{q}<mode>.  */
 755   VAR2 (TERNOP, bfdot, 0, AUTO_FP, v2sf, v4sf)
 756   VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, ALL, v2sf, v4sf)
 757   VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, ALL, v2sf, v4sf)
 758
 759   /* Implemented by aarch64_bfmmlaqv4sf  */
 760   VAR1 (TERNOP, bfmmlaq, 0, AUTO_FP, v4sf)
 761
 762   /* Implemented by aarch64_bfmlal<bt>{_lane{q}}v4sf  */
 763   VAR1 (TERNOP, bfmlalb, 0, FP, v4sf)
 764   VAR1 (TERNOP, bfmlalt, 0, FP, v4sf)
 765   VAR1 (QUADOP_LANE, bfmlalb_lane, 0, ALL, v4sf)
 766   VAR1 (QUADOP_LANE, bfmlalt_lane, 0, ALL, v4sf)
 767   VAR1 (QUADOP_LANE, bfmlalb_lane_q, 0, ALL, v4sf)
 768   VAR1 (QUADOP_LANE, bfmlalt_lane_q, 0, ALL, v4sf)
 769
 770   /* Implemented by aarch64_vget_lo/hi_halfv8bf.  */
 771   VAR1 (UNOP, vget_lo_half, 0, AUTO_FP, v8bf)
 772   VAR1 (UNOP, vget_hi_half, 0, AUTO_FP, v8bf)
 773
 774   /* Implemented by aarch64_simd_<sur>mmlav16qi.  */
 775   VAR1 (TERNOP, simd_smmla, 0, NONE, v16qi)
 776   VAR1 (TERNOPU, simd_ummla, 0, NONE, v16qi)
 777   VAR1 (TERNOP_SSUS, simd_usmmla, 0, NONE, v16qi)
 778
 779   /* Implemented by aarch64_bfcvtn{q}{2}<mode>  */
 780   VAR1 (UNOP, bfcvtn, 0, FP, v4bf)
 781   VAR1 (UNOP, bfcvtn_q, 0, FP, v8bf)
 782   VAR1 (BINOP, bfcvtn2, 0, FP, v8bf)
 783   VAR1 (UNOP, bfcvt, 0, FP, bf)
 784
 785   /* Implemented by aarch64_{v}bfcvt{_high}<mode>.  */
 786   VAR2 (UNOP, vbfcvt, 0, AUTO_FP, v4bf, v8bf)
 787   VAR1 (UNOP, vbfcvt_high, 0, AUTO_FP, v8bf)
 788   VAR1 (UNOP, bfcvt, 0, AUTO_FP, sf)