gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012
   3 ;;   Free Software Foundation, Inc.
   4 ;; Written by CodeSourcery.
   5 ;;
   6 ;; This file is part of GCC.
   7 ;;
   8 ;; GCC is free software; you can redistribute it and/or modify it
   9 ;; under the terms of the GNU General Public License as published by
  10 ;; the Free Software Foundation; either version 3, or (at your option)
  11 ;; any later version.
  12 ;;
  13 ;; GCC is distributed in the hope that it will be useful, but
  14 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 ;; General Public License for more details.
  17 ;;
  18 ;; You should have received a copy of the GNU General Public License
  19 ;; along with GCC; see the file COPYING3.  If not see
  20 ;; <http://www.gnu.org/licenses/>.
  21
  22 ;; Enumerators for unspecs.
  23 (define_c_enum "unspec" [
  24   UNSPEC_ASHIFT_SIGNED
  25   UNSPEC_ASHIFT_UNSIGNED
  26   UNSPEC_LOAD_COUNT
  27   UNSPEC_VABD
  28   UNSPEC_VABDL
  29   UNSPEC_VADD
  30   UNSPEC_VADDHN
  31   UNSPEC_VADDL
  32   UNSPEC_VADDW
  33   UNSPEC_VBSL
  34   UNSPEC_VCAGE
  35   UNSPEC_VCAGT
  36   UNSPEC_VCEQ
  37   UNSPEC_VCGE
  38   UNSPEC_VCGEU
  39   UNSPEC_VCGT
  40   UNSPEC_VCGTU
  41   UNSPEC_VCLS
  42   UNSPEC_VCONCAT
  43   UNSPEC_VCVT
  44   UNSPEC_VCVT_N
  45   UNSPEC_VEXT
  46   UNSPEC_VHADD
  47   UNSPEC_VHSUB
  48   UNSPEC_VLD1
  49   UNSPEC_VLD1_LANE
  50   UNSPEC_VLD2
  51   UNSPEC_VLD2_DUP
  52   UNSPEC_VLD2_LANE
  53   UNSPEC_VLD3
  54   UNSPEC_VLD3A
  55   UNSPEC_VLD3B
  56   UNSPEC_VLD3_DUP
  57   UNSPEC_VLD3_LANE
  58   UNSPEC_VLD4
  59   UNSPEC_VLD4A
  60   UNSPEC_VLD4B
  61   UNSPEC_VLD4_DUP
  62   UNSPEC_VLD4_LANE
  63   UNSPEC_VMAX
  64   UNSPEC_VMIN
  65   UNSPEC_VMLA
  66   UNSPEC_VMLAL
  67   UNSPEC_VMLA_LANE
  68   UNSPEC_VMLAL_LANE
  69   UNSPEC_VMLS
  70   UNSPEC_VMLSL
  71   UNSPEC_VMLS_LANE
  72   UNSPEC_VMLSL_LANE
  73   UNSPEC_VMOVL
  74   UNSPEC_VMOVN
  75   UNSPEC_VMUL
  76   UNSPEC_VMULL
  77   UNSPEC_VMUL_LANE
  78   UNSPEC_VMULL_LANE
  79   UNSPEC_VPADAL
  80   UNSPEC_VPADD
  81   UNSPEC_VPADDL
  82   UNSPEC_VPMAX
  83   UNSPEC_VPMIN
  84   UNSPEC_VPSMAX
  85   UNSPEC_VPSMIN
  86   UNSPEC_VPUMAX
  87   UNSPEC_VPUMIN
  88   UNSPEC_VQABS
  89   UNSPEC_VQADD
  90   UNSPEC_VQDMLAL
  91   UNSPEC_VQDMLAL_LANE
  92   UNSPEC_VQDMLSL
  93   UNSPEC_VQDMLSL_LANE
  94   UNSPEC_VQDMULH
  95   UNSPEC_VQDMULH_LANE
  96   UNSPEC_VQDMULL
  97   UNSPEC_VQDMULL_LANE
  98   UNSPEC_VQMOVN
  99   UNSPEC_VQMOVUN
 100   UNSPEC_VQNEG
 101   UNSPEC_VQSHL
 102   UNSPEC_VQSHL_N
 103   UNSPEC_VQSHLU_N
 104   UNSPEC_VQSHRN_N
 105   UNSPEC_VQSHRUN_N
 106   UNSPEC_VQSUB
 107   UNSPEC_VRECPE
 108   UNSPEC_VRECPS
 109   UNSPEC_VREV16
 110   UNSPEC_VREV32
 111   UNSPEC_VREV64
 112   UNSPEC_VRSQRTE
 113   UNSPEC_VRSQRTS
 114   UNSPEC_VSHL
 115   UNSPEC_VSHLL_N
 116   UNSPEC_VSHL_N
 117   UNSPEC_VSHR_N
 118   UNSPEC_VSHRN_N
 119   UNSPEC_VSLI
 120   UNSPEC_VSRA_N
 121   UNSPEC_VSRI
 122   UNSPEC_VST1
 123   UNSPEC_VST1_LANE
 124   UNSPEC_VST2
 125   UNSPEC_VST2_LANE
 126   UNSPEC_VST3
 127   UNSPEC_VST3A
 128   UNSPEC_VST3B
 129   UNSPEC_VST3_LANE
 130   UNSPEC_VST4
 131   UNSPEC_VST4A
 132   UNSPEC_VST4B
 133   UNSPEC_VST4_LANE
 134   UNSPEC_VSTRUCTDUMMY
 135   UNSPEC_VSUB
 136   UNSPEC_VSUBHN
 137   UNSPEC_VSUBL
 138   UNSPEC_VSUBW
 139   UNSPEC_VTBL
 140   UNSPEC_VTBX
 141   UNSPEC_VTRN1
 142   UNSPEC_VTRN2
 143   UNSPEC_VTST
 144   UNSPEC_VUZP1
 145   UNSPEC_VUZP2
 146   UNSPEC_VZIP1
 147   UNSPEC_VZIP2
 148   UNSPEC_MISALIGNED_ACCESS
 149   UNSPEC_VCLE
 150   UNSPEC_VCLT
 151 ])
 152
 153
 154 ;; Attribute used to permit string comparisons against <VQH_mnem> in
 155 ;; neon_type attribute definitions.
 156 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
 157
 158 (define_insn "*neon_mov<mode>"
 159   [(set (match_operand:VDX 0 "nonimmediate_operand"
 160           "=w,Un,w, w,  ?r,?w,?r,?r, ?Us")
 161         (match_operand:VDX 1 "general_operand"
 162           " w,w, Dn,Uni, w, r, r, Usi,r"))]
 163   "TARGET_NEON
 164    && (register_operand (operands[0], <MODE>mode)
 165        || register_operand (operands[1], <MODE>mode))"
 166 {
 167   if (which_alternative == 2)
 168     {
 169       int width, is_valid;
 170       static char templ[40];
 171
 172       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 173         &operands[1], &width);
 174
 175       gcc_assert (is_valid != 0);
 176
 177       if (width == 0)
 178         return "vmov.f32\t%P0, %1  @ <mode>";
 179       else
 180         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
 181
 182       return templ;
 183     }
 184
 185   switch (which_alternative)
 186     {
 187     case 0: return "vmov\t%P0, %P1  @ <mode>";
 188     case 1: case 3: return output_move_neon (operands);
 189     case 2: gcc_unreachable ();
 190     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
 191     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
 192     default: return output_move_double (operands, true, NULL);
 193     }
 194 }
 195  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 196   (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
 197   (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 198   (set_attr "length" "4,4,4,4,4,4,8,8,8")
 199   (set_attr "pool_range"     "*,*,*,1020,*,*,*,1020,*")
 200   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
 201
 202 (define_insn "*neon_mov<mode>"
 203   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
 204           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
 205         (match_operand:VQXMOV 1 "general_operand"
 206           " w,w, Dn,Uni, w, r, r, Usi, r"))]
 207   "TARGET_NEON
 208    && (register_operand (operands[0], <MODE>mode)
 209        || register_operand (operands[1], <MODE>mode))"
 210 {
 211   if (which_alternative == 2)
 212     {
 213       int width, is_valid;
 214       static char templ[40];
 215
 216       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 217         &operands[1], &width);
 218
 219       gcc_assert (is_valid != 0);
 220
 221       if (width == 0)
 222         return "vmov.f32\t%q0, %1  @ <mode>";
 223       else
 224         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
 225
 226       return templ;
 227     }
 228
 229   switch (which_alternative)
 230     {
 231     case 0: return "vmov\t%q0, %q1  @ <mode>";
 232     case 1: case 3: return output_move_neon (operands);
 233     case 2: gcc_unreachable ();
 234     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 235     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 236     default: return output_move_quad (operands);
 237     }
 238 }
 239   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 240                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 241    (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
 242    (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 243    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 244    (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
 245    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 246
 247 (define_expand "movti"
 248   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 249         (match_operand:TI 1 "general_operand" ""))]
 250   "TARGET_NEON"
 251 {
 252   if (can_create_pseudo_p ())
 253     {
 254       if (!REG_P (operands[0]))
 255         operands[1] = force_reg (TImode, operands[1]);
 256     }
 257 })
 258
 259 (define_expand "mov<mode>"
 260   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 261         (match_operand:VSTRUCT 1 "general_operand" ""))]
 262   "TARGET_NEON"
 263 {
 264   if (can_create_pseudo_p ())
 265     {
 266       if (!REG_P (operands[0]))
 267         operands[1] = force_reg (<MODE>mode, operands[1]);
 268     }
 269 })
 270
 271 (define_insn "*neon_mov<mode>"
 272   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 273         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 274   "TARGET_NEON
 275    && (register_operand (operands[0], <MODE>mode)
 276        || register_operand (operands[1], <MODE>mode))"
 277 {
 278   switch (which_alternative)
 279     {
 280     case 0: return "#";
 281     case 1: case 2: return output_move_neon (operands);
 282     default: gcc_unreachable ();
 283     }
 284 }
 285   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 286    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 287
 288 (define_split
 289   [(set (match_operand:EI 0 "s_register_operand" "")
 290         (match_operand:EI 1 "s_register_operand" ""))]
 291   "TARGET_NEON && reload_completed"
 292   [(set (match_dup 0) (match_dup 1))
 293    (set (match_dup 2) (match_dup 3))]
 294 {
 295   int rdest = REGNO (operands[0]);
 296   int rsrc = REGNO (operands[1]);
 297   rtx dest[2], src[2];
 298
 299   dest[0] = gen_rtx_REG (TImode, rdest);
 300   src[0] = gen_rtx_REG (TImode, rsrc);
 301   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 302   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 303
 304   neon_disambiguate_copy (operands, dest, src, 2);
 305 })
 306
 307 (define_split
 308   [(set (match_operand:OI 0 "s_register_operand" "")
 309         (match_operand:OI 1 "s_register_operand" ""))]
 310   "TARGET_NEON && reload_completed"
 311   [(set (match_dup 0) (match_dup 1))
 312    (set (match_dup 2) (match_dup 3))]
 313 {
 314   int rdest = REGNO (operands[0]);
 315   int rsrc = REGNO (operands[1]);
 316   rtx dest[2], src[2];
 317
 318   dest[0] = gen_rtx_REG (TImode, rdest);
 319   src[0] = gen_rtx_REG (TImode, rsrc);
 320   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 321   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 322
 323   neon_disambiguate_copy (operands, dest, src, 2);
 324 })
 325
 326 (define_split
 327   [(set (match_operand:CI 0 "s_register_operand" "")
 328         (match_operand:CI 1 "s_register_operand" ""))]
 329   "TARGET_NEON && reload_completed"
 330   [(set (match_dup 0) (match_dup 1))
 331    (set (match_dup 2) (match_dup 3))
 332    (set (match_dup 4) (match_dup 5))]
 333 {
 334   int rdest = REGNO (operands[0]);
 335   int rsrc = REGNO (operands[1]);
 336   rtx dest[3], src[3];
 337
 338   dest[0] = gen_rtx_REG (TImode, rdest);
 339   src[0] = gen_rtx_REG (TImode, rsrc);
 340   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 341   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 342   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 343   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 344
 345   neon_disambiguate_copy (operands, dest, src, 3);
 346 })
 347
 348 (define_split
 349   [(set (match_operand:XI 0 "s_register_operand" "")
 350         (match_operand:XI 1 "s_register_operand" ""))]
 351   "TARGET_NEON && reload_completed"
 352   [(set (match_dup 0) (match_dup 1))
 353    (set (match_dup 2) (match_dup 3))
 354    (set (match_dup 4) (match_dup 5))
 355    (set (match_dup 6) (match_dup 7))]
 356 {
 357   int rdest = REGNO (operands[0]);
 358   int rsrc = REGNO (operands[1]);
 359   rtx dest[4], src[4];
 360
 361   dest[0] = gen_rtx_REG (TImode, rdest);
 362   src[0] = gen_rtx_REG (TImode, rsrc);
 363   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 364   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 365   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 366   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 367   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 368   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 369
 370   neon_disambiguate_copy (operands, dest, src, 4);
 371 })
 372
 373 (define_expand "movmisalign<mode>"
 374   [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
 375         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
 376                      UNSPEC_MISALIGNED_ACCESS))]
 377   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 378 {
 379   /* This pattern is not permitted to fail during expansion: if both arguments
 380      are non-registers (e.g. memory := constant, which can be created by the
 381      auto-vectorizer), force operand 1 into a register.  */
 382   if (!s_register_operand (operands[0], <MODE>mode)
 383       && !s_register_operand (operands[1], <MODE>mode))
 384     operands[1] = force_reg (<MODE>mode, operands[1]);
 385 })
 386
 387 (define_insn "*movmisalign<mode>_neon_store"
 388   [(set (match_operand:VDX 0 "neon_struct_operand"             "=Um")
 389         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 390                     UNSPEC_MISALIGNED_ACCESS))]
 391   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 392   "vst1.<V_sz_elem>\t{%P1}, %A0"
 393   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 394
 395 (define_insn "*movmisalign<mode>_neon_load"
 396   [(set (match_operand:VDX 0 "s_register_operand"               "=w")
 397         (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
 398                     UNSPEC_MISALIGNED_ACCESS))]
 399   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 400   "vld1.<V_sz_elem>\t{%P0}, %A1"
 401   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 402
 403 (define_insn "*movmisalign<mode>_neon_store"
 404   [(set (match_operand:VQX 0 "neon_struct_operand"             "=Um")
 405         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 406                     UNSPEC_MISALIGNED_ACCESS))]
 407   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 408   "vst1.<V_sz_elem>\t{%q1}, %A0"
 409   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 410
 411 (define_insn "*movmisalign<mode>_neon_load"
 412   [(set (match_operand:VQX 0 "s_register_operand"                "=w")
 413         (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
 414                     UNSPEC_MISALIGNED_ACCESS))]
 415   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 416   "vld1.<V_sz_elem>\t{%q0}, %A1"
 417   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 418
 419 (define_insn "vec_set<mode>_internal"
 420   [(set (match_operand:VD 0 "s_register_operand" "=w,w")
 421         (vec_merge:VD
 422           (vec_duplicate:VD
 423             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 424           (match_operand:VD 3 "s_register_operand" "0,0")
 425           (match_operand:SI 2 "immediate_operand" "i,i")))]
 426   "TARGET_NEON"
 427 {
 428   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 429   if (BYTES_BIG_ENDIAN)
 430     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 431   operands[2] = GEN_INT (elt);
 432
 433   if (which_alternative == 0)
 434     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 435   else
 436     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 437 }
 438   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")])
 439
 440 (define_insn "vec_set<mode>_internal"
 441   [(set (match_operand:VQ 0 "s_register_operand" "=w,w")
 442         (vec_merge:VQ
 443           (vec_duplicate:VQ
 444             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 445           (match_operand:VQ 3 "s_register_operand" "0,0")
 446           (match_operand:SI 2 "immediate_operand" "i,i")))]
 447   "TARGET_NEON"
 448 {
 449   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 450   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 451   int elt = elem % half_elts;
 452   int hi = (elem / half_elts) * 2;
 453   int regno = REGNO (operands[0]);
 454
 455   if (BYTES_BIG_ENDIAN)
 456     elt = half_elts - 1 - elt;
 457
 458   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 459   operands[2] = GEN_INT (elt);
 460
 461   if (which_alternative == 0)
 462     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 463   else
 464     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 465 }
 466   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
 467 )
 468
 469 (define_insn "vec_setv2di_internal"
 470   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
 471         (vec_merge:V2DI
 472           (vec_duplicate:V2DI
 473             (match_operand:DI 1 "nonimmediate_operand" "Um,r"))
 474           (match_operand:V2DI 3 "s_register_operand" "0,0")
 475           (match_operand:SI 2 "immediate_operand" "i,i")))]
 476   "TARGET_NEON"
 477 {
 478   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 479   int regno = REGNO (operands[0]) + 2 * elem;
 480
 481   operands[0] = gen_rtx_REG (DImode, regno);
 482
 483   if (which_alternative == 0)
 484     return "vld1.64\t%P0, %A1";
 485   else
 486     return "vmov\t%P0, %Q1, %R1";
 487 }
 488   [(set_attr "neon_type" "neon_vld1_1_2_regs,neon_mcr_2_mcrr")]
 489 )
 490
 491 (define_expand "vec_set<mode>"
 492   [(match_operand:VDQ 0 "s_register_operand" "")
 493    (match_operand:<V_elem> 1 "s_register_operand" "")
 494    (match_operand:SI 2 "immediate_operand" "")]
 495   "TARGET_NEON"
 496 {
 497   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 498   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 499                                          GEN_INT (elem), operands[0]));
 500   DONE;
 501 })
 502
 503 (define_insn "vec_extract<mode>"
 504   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 505         (vec_select:<V_elem>
 506           (match_operand:VD 1 "s_register_operand" "w,w")
 507           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 508   "TARGET_NEON"
 509 {
 510   if (BYTES_BIG_ENDIAN)
 511     {
 512       int elt = INTVAL (operands[2]);
 513       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 514       operands[2] = GEN_INT (elt);
 515     }
 516
 517   if (which_alternative == 0)
 518     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 519   else
 520     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 521 }
 522   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 523 )
 524
 525 (define_insn "vec_extract<mode>"
 526   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 527         (vec_select:<V_elem>
 528           (match_operand:VQ 1 "s_register_operand" "w,w")
 529           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 530   "TARGET_NEON"
 531 {
 532   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 533   int elt = INTVAL (operands[2]) % half_elts;
 534   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 535   int regno = REGNO (operands[1]);
 536
 537   if (BYTES_BIG_ENDIAN)
 538     elt = half_elts - 1 - elt;
 539
 540   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 541   operands[2] = GEN_INT (elt);
 542
 543   if (which_alternative == 0)
 544     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 545   else
 546     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 547 }
 548   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 549 )
 550
 551 (define_insn "vec_extractv2di"
 552   [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r")
 553         (vec_select:DI
 554           (match_operand:V2DI 1 "s_register_operand" "w,w")
 555           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 556   "TARGET_NEON"
 557 {
 558   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 559
 560   operands[1] = gen_rtx_REG (DImode, regno);
 561
 562   if (which_alternative == 0)
 563     return "vst1.64\t{%P1}, %A0  @ v2di";
 564   else
 565     return "vmov\t%Q0, %R0, %P1  @ v2di";
 566 }
 567   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_int_1")]
 568 )
 569
 570 (define_expand "vec_init<mode>"
 571   [(match_operand:VDQ 0 "s_register_operand" "")
 572    (match_operand 1 "" "")]
 573   "TARGET_NEON"
 574 {
 575   neon_expand_vector_init (operands[0], operands[1]);
 576   DONE;
 577 })
 578
 579 ;; Doubleword and quadword arithmetic.
 580
 581 ;; NOTE: some other instructions also support 64-bit integer
 582 ;; element size, which we could potentially use for "long long" operations.
 583
 584 (define_insn "*add<mode>3_neon"
 585   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 586         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 587                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 588   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 589   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 590   [(set (attr "neon_type")
 591       (if_then_else (match_test "<Is_float_mode>")
 592                     (if_then_else (match_test "<Is_d_reg>")
 593                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 594                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 595                     (const_string "neon_int_1")))]
 596 )
 597
 598 (define_insn "adddi3_neon"
 599   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 600         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 601                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 602    (clobber (reg:CC CC_REGNUM))]
 603   "TARGET_NEON"
 604 {
 605   switch (which_alternative)
 606     {
 607     case 0: /* fall through */
 608     case 3: return "vadd.i64\t%P0, %P1, %P2";
 609     case 1: return "#";
 610     case 2: return "#";
 611     case 4: return "#";
 612     case 5: return "#";
 613     case 6: return "#";
 614     default: gcc_unreachable ();
 615     }
 616 }
 617   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 618    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 619    (set_attr "length" "*,8,8,*,8,8,8")
 620    (set_attr "arch" "nota8,*,*,onlya8,*,*,*")]
 621 )
 622
 623 (define_insn "*sub<mode>3_neon"
 624   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 625         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 626                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 627   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 628   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 629   [(set (attr "neon_type")
 630       (if_then_else (match_test "<Is_float_mode>")
 631                     (if_then_else (match_test "<Is_d_reg>")
 632                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 633                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 634                     (const_string "neon_int_2")))]
 635 )
 636
 637 (define_insn "subdi3_neon"
 638   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 639         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 640                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 641    (clobber (reg:CC CC_REGNUM))]
 642   "TARGET_NEON"
 643 {
 644   switch (which_alternative)
 645     {
 646     case 0: /* fall through */
 647     case 4: return "vsub.i64\t%P0, %P1, %P2";
 648     case 1: /* fall through */
 649     case 2: /* fall through */
 650     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 651     default: gcc_unreachable ();
 652     }
 653 }
 654   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 655    (set_attr "conds" "*,clob,clob,clob,*")
 656    (set_attr "length" "*,8,8,8,*")
 657    (set_attr "arch" "nota8,*,*,*,onlya8")]
 658 )
 659
 660 (define_insn "*mul<mode>3_neon"
 661   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 662         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 663                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 664   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 665   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 666   [(set (attr "neon_type")
 667       (if_then_else (match_test "<Is_float_mode>")
 668                     (if_then_else (match_test "<Is_d_reg>")
 669                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 670                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 671                     (if_then_else (match_test "<Is_d_reg>")
 672                                   (if_then_else
 673                                     (match_test "<Scalar_mul_8_16>")
 674                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 675                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 676                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 677                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 678                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 679 )
 680
 681 (define_insn "mul<mode>3add<mode>_neon"
 682   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 683         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 684                             (match_operand:VDQ 3 "s_register_operand" "w"))
 685                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 686   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 687   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 688   [(set (attr "neon_type")
 689       (if_then_else (match_test "<Is_float_mode>")
 690                     (if_then_else (match_test "<Is_d_reg>")
 691                                   (const_string "neon_fp_vmla_ddd")
 692                                   (const_string "neon_fp_vmla_qqq"))
 693                     (if_then_else (match_test "<Is_d_reg>")
 694                                   (if_then_else
 695                                     (match_test "<Scalar_mul_8_16>")
 696                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 697                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 698                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 699                                     (const_string "neon_mla_qqq_8_16")
 700                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 701 )
 702
 703 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 704   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 705         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 706                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 707                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 708   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 709   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 710   [(set (attr "neon_type")
 711       (if_then_else (match_test "<Is_float_mode>")
 712                     (if_then_else (match_test "<Is_d_reg>")
 713                                   (const_string "neon_fp_vmla_ddd")
 714                                   (const_string "neon_fp_vmla_qqq"))
 715                     (if_then_else (match_test "<Is_d_reg>")
 716                                   (if_then_else
 717                                     (match_test "<Scalar_mul_8_16>")
 718                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 719                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 720                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 721                                     (const_string "neon_mla_qqq_8_16")
 722                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 723 )
 724
 725 ;; Fused multiply-accumulate
 726 ;; We define each insn twice here:
 727 ;;    1: with flag_unsafe_math_optimizations for the widening multiply phase
 728 ;;       to be able to use when converting to FMA.
 729 ;;    2: without flag_unsafe_math_optimizations for the intrinsics to use.
 730 (define_insn "fma<VCVTF:mode>4"
 731   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 732         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 733                  (match_operand:VCVTF 2 "register_operand" "w")
 734                  (match_operand:VCVTF 3 "register_operand" "0")))]
 735   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 736   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 737   [(set (attr "neon_type")
 738         (if_then_else (match_test "<Is_d_reg>")
 739                       (const_string "neon_fp_vmla_ddd")
 740                       (const_string "neon_fp_vmla_qqq")))]
 741 )
 742
 743 (define_insn "fma<VCVTF:mode>4_intrinsic"
 744   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 745         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 746                  (match_operand:VCVTF 2 "register_operand" "w")
 747                  (match_operand:VCVTF 3 "register_operand" "0")))]
 748   "TARGET_NEON && TARGET_FMA"
 749   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 750   [(set (attr "neon_type")
 751         (if_then_else (match_test "<Is_d_reg>")
 752                       (const_string "neon_fp_vmla_ddd")
 753                       (const_string "neon_fp_vmla_qqq")))]
 754 )
 755
 756 (define_insn "*fmsub<VCVTF:mode>4"
 757   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 758         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 759                    (match_operand:VCVTF 2 "register_operand" "w")
 760                    (match_operand:VCVTF 3 "register_operand" "0")))]
 761   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 762   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 763   [(set (attr "neon_type")
 764         (if_then_else (match_test "<Is_d_reg>")
 765                       (const_string "neon_fp_vmla_ddd")
 766                       (const_string "neon_fp_vmla_qqq")))]
 767 )
 768
 769 (define_insn "fmsub<VCVTF:mode>4_intrinsic"
 770   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 771         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 772                    (match_operand:VCVTF 2 "register_operand" "w")
 773                    (match_operand:VCVTF 3 "register_operand" "0")))]
 774   "TARGET_NEON && TARGET_FMA"
 775   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 776   [(set (attr "neon_type")
 777         (if_then_else (match_test "<Is_d_reg>")
 778                       (const_string "neon_fp_vmla_ddd")
 779                       (const_string "neon_fp_vmla_qqq")))]
 780 )
 781
 782 (define_insn "ior<mode>3"
 783   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 784         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 785                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 786   "TARGET_NEON"
 787 {
 788   switch (which_alternative)
 789     {
 790     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 791     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 792                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 793     default: gcc_unreachable ();
 794     }
 795 }
 796   [(set_attr "neon_type" "neon_int_1")]
 797 )
 798
 799 (define_insn "iordi3_neon"
 800   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 801         (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 802                 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
 803   "TARGET_NEON"
 804 {
 805   switch (which_alternative)
 806     {
 807     case 0: /* fall through */
 808     case 4: return "vorr\t%P0, %P1, %P2";
 809     case 1: /* fall through */
 810     case 5: return neon_output_logic_immediate ("vorr", &operands[2],
 811                      DImode, 0, VALID_NEON_QREG_MODE (DImode));
 812     case 2: return "#";
 813     case 3: return "#";
 814     default: gcc_unreachable ();
 815     }
 816 }
 817   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 818    (set_attr "length" "*,*,8,8,*,*")
 819    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 820 )
 821
 822 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 823 ;; vorr. We support the pseudo-instruction vand instead, because that
 824 ;; corresponds to the canonical form the middle-end expects to use for
 825 ;; immediate bitwise-ANDs.
 826
 827 (define_insn "and<mode>3"
 828   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 829         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 830                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 831   "TARGET_NEON"
 832 {
 833   switch (which_alternative)
 834     {
 835     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 836     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 837                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 838     default: gcc_unreachable ();
 839     }
 840 }
 841   [(set_attr "neon_type" "neon_int_1")]
 842 )
 843
 844 (define_insn "anddi3_neon"
 845   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 846         (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 847                 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
 848   "TARGET_NEON"
 849 {
 850   switch (which_alternative)
 851     {
 852     case 0: /* fall through */
 853     case 4: return "vand\t%P0, %P1, %P2";
 854     case 1: /* fall through */
 855     case 5: return neon_output_logic_immediate ("vand", &operands[2],
 856                      DImode, 1, VALID_NEON_QREG_MODE (DImode));
 857     case 2: return "#";
 858     case 3: return "#";
 859     default: gcc_unreachable ();
 860     }
 861 }
 862   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 863    (set_attr "length" "*,*,8,8,*,*")
 864    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 865 )
 866
 867 (define_insn "orn<mode>3_neon"
 868   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 869         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 870                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 871   "TARGET_NEON"
 872   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 873   [(set_attr "neon_type" "neon_int_1")]
 874 )
 875
 876 ;; TODO: investigate whether we should disable
 877 ;; this and bicdi3_neon for the A8 in line with the other
 878 ;; changes above.
 879 (define_insn_and_split "orndi3_neon"
 880   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 881         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 882                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 883   "TARGET_NEON"
 884   "@
 885    vorn\t%P0, %P1, %P2
 886    #
 887    #
 888    #"
 889   "reload_completed &&
 890    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 891   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 892    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 893   "
 894   {
 895     if (TARGET_THUMB2)
 896       {
 897         operands[3] = gen_highpart (SImode, operands[0]);
 898         operands[0] = gen_lowpart (SImode, operands[0]);
 899         operands[4] = gen_highpart (SImode, operands[2]);
 900         operands[2] = gen_lowpart (SImode, operands[2]);
 901         operands[5] = gen_highpart (SImode, operands[1]);
 902         operands[1] = gen_lowpart (SImode, operands[1]);
 903       }
 904     else
 905       {
 906         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 907         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 908         DONE;
 909       }
 910   }"
 911   [(set_attr "neon_type" "neon_int_1,*,*,*")
 912    (set_attr "length" "*,16,8,8")
 913    (set_attr "arch" "any,a,t2,t2")]
 914 )
 915
 916 (define_insn "bic<mode>3_neon"
 917   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 918         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 919                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 920   "TARGET_NEON"
 921   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 922   [(set_attr "neon_type" "neon_int_1")]
 923 )
 924
 925 ;; Compare to *anddi_notdi_di.
 926 (define_insn "bicdi3_neon"
 927   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 928         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 929                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 930   "TARGET_NEON"
 931   "@
 932    vbic\t%P0, %P1, %P2
 933    #
 934    #"
 935   [(set_attr "neon_type" "neon_int_1,*,*")
 936    (set_attr "length" "*,8,8")]
 937 )
 938
 939 (define_insn "xor<mode>3"
 940   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 941         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 942                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 943   "TARGET_NEON"
 944   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 945   [(set_attr "neon_type" "neon_int_1")]
 946 )
 947
 948 (define_insn "xordi3_neon"
 949   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
 950         (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
 951                 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
 952   "TARGET_NEON"
 953   "@
 954    veor\t%P0, %P1, %P2
 955    #
 956    #
 957    veor\t%P0, %P1, %P2"
 958   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
 959    (set_attr "length" "*,8,8,*")
 960    (set_attr "arch" "nota8,*,*,onlya8")]
 961 )
 962
 963 (define_insn "one_cmpl<mode>2"
 964   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 965         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 966   "TARGET_NEON"
 967   "vmvn\t%<V_reg>0, %<V_reg>1"
 968   [(set_attr "neon_type" "neon_int_1")]
 969 )
 970
 971 (define_insn "abs<mode>2"
 972   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 973         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 974   "TARGET_NEON"
 975   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 976   [(set (attr "neon_type")
 977       (if_then_else (match_test "<Is_float_mode>")
 978                     (if_then_else (match_test "<Is_d_reg>")
 979                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 980                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 981                     (const_string "neon_int_3")))]
 982 )
 983
 984 (define_insn "neg<mode>2"
 985   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 986         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 987   "TARGET_NEON"
 988   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 989   [(set (attr "neon_type")
 990       (if_then_else (match_test "<Is_float_mode>")
 991                     (if_then_else (match_test "<Is_d_reg>")
 992                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 993                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 994                     (const_string "neon_int_3")))]
 995 )
 996
 997 (define_insn "negdi2_neon"
 998   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 999         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
1000    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
1001    (clobber (reg:CC CC_REGNUM))]
1002   "TARGET_NEON"
1003   "#"
1004   [(set_attr "length" "8")]
1005 )
1006
1007 ; Split negdi2_neon for vfp registers
1008 (define_split
1009   [(set (match_operand:DI 0 "s_register_operand" "")
1010         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
1011    (clobber (match_scratch:DI 2 ""))
1012    (clobber (reg:CC CC_REGNUM))]
1013   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
1014   [(set (match_dup 2) (const_int 0))
1015    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
1016               (clobber (reg:CC CC_REGNUM))])]
1017   {
1018     if (!REG_P (operands[2]))
1019       operands[2] = operands[0];
1020   }
1021 )
1022
1023 ; Split negdi2_neon for core registers
1024 (define_split
1025   [(set (match_operand:DI 0 "s_register_operand" "")
1026         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
1027    (clobber (match_scratch:DI 2 ""))
1028    (clobber (reg:CC CC_REGNUM))]
1029   "TARGET_32BIT && reload_completed
1030    && arm_general_register_operand (operands[0], DImode)"
1031   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
1032               (clobber (reg:CC CC_REGNUM))])]
1033   ""
1034 )
1035
1036 (define_insn "*umin<mode>3_neon"
1037   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1038         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1039                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
1040   "TARGET_NEON"
1041   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1042   [(set_attr "neon_type" "neon_int_5")]
1043 )
1044
1045 (define_insn "*umax<mode>3_neon"
1046   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1047         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1048                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
1049   "TARGET_NEON"
1050   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1051   [(set_attr "neon_type" "neon_int_5")]
1052 )
1053
1054 (define_insn "*smin<mode>3_neon"
1055   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1056         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
1057                    (match_operand:VDQW 2 "s_register_operand" "w")))]
1058   "TARGET_NEON"
1059   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1060   [(set (attr "neon_type")
1061       (if_then_else (match_test "<Is_float_mode>")
1062                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1063                     (const_string "neon_int_5")))]
1064 )
1065
1066 (define_insn "*smax<mode>3_neon"
1067   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1068         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
1069                    (match_operand:VDQW 2 "s_register_operand" "w")))]
1070   "TARGET_NEON"
1071   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1072   [(set (attr "neon_type")
1073       (if_then_else (match_test "<Is_float_mode>")
1074                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1075                     (const_string "neon_int_5")))]
1076 )
1077
1078 ; TODO: V2DI shifts are current disabled because there are bugs in the
1079 ; generic vectorizer code.  It ends up creating a V2DI constructor with
1080 ; SImode elements.
1081
1082 (define_insn "vashl<mode>3"
1083   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
1084         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
1085                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
1086   "TARGET_NEON"
1087   {
1088     switch (which_alternative)
1089       {
1090         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
1091         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
1092                                                     <MODE>mode,
1093                                                     VALID_NEON_QREG_MODE (<MODE>mode),
1094                                                     true);
1095         default: gcc_unreachable ();
1096       }
1097   }
1098   [(set (attr "neon_type")
1099       (if_then_else (match_test "<Is_d_reg>")
1100                     (const_string "neon_vshl_ddd")
1101                     (const_string "neon_shift_3")))]
1102 )
1103
1104 (define_insn "vashr<mode>3_imm"
1105   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1106         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1107                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1108   "TARGET_NEON"
1109   {
1110     return neon_output_shift_immediate ("vshr", 's', &operands[2],
1111                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1112                                         false);
1113   }
1114   [(set (attr "neon_type")
1115       (if_then_else (match_test "<Is_d_reg>")
1116                     (const_string "neon_vshl_ddd")
1117                     (const_string "neon_shift_3")))]
1118 )
1119
1120 (define_insn "vlshr<mode>3_imm"
1121   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1122         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1123                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1124   "TARGET_NEON"
1125   {
1126     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1127                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1128                                         false);
1129   }
1130   [(set (attr "neon_type")
1131         (if_then_else (match_test "<Is_d_reg>")
1132                       (const_string "neon_vshl_ddd")
1133                       (const_string "neon_shift_3")))]
1134 )
1135
1136 ; Used for implementing logical shift-right, which is a left-shift by a negative
1137 ; amount, with signed operands. This is essentially the same as ashl<mode>3
1138 ; above, but using an unspec in case GCC tries anything tricky with negative
1139 ; shift amounts.
1140
1141 (define_insn "ashl<mode>3_signed"
1142   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1143         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1144                       (match_operand:VDQI 2 "s_register_operand" "w")]
1145                      UNSPEC_ASHIFT_SIGNED))]
1146   "TARGET_NEON"
1147   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1148   [(set (attr "neon_type")
1149       (if_then_else (match_test "<Is_d_reg>")
1150                     (const_string "neon_vshl_ddd")
1151                     (const_string "neon_shift_3")))]
1152 )
1153
1154 ; Used for implementing logical shift-right, which is a left-shift by a negative
1155 ; amount, with unsigned operands.
1156
1157 (define_insn "ashl<mode>3_unsigned"
1158   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1159         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1160                       (match_operand:VDQI 2 "s_register_operand" "w")]
1161                      UNSPEC_ASHIFT_UNSIGNED))]
1162   "TARGET_NEON"
1163   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1164   [(set (attr "neon_type")
1165       (if_then_else (match_test "<Is_d_reg>")
1166                     (const_string "neon_vshl_ddd")
1167                     (const_string "neon_shift_3")))]
1168 )
1169
1170 (define_expand "vashr<mode>3"
1171   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1172         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1173                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1174   "TARGET_NEON"
1175 {
1176   if (s_register_operand (operands[2], <MODE>mode))
1177     {
1178       rtx neg = gen_reg_rtx (<MODE>mode);
1179       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1180       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1181     }
1182   else
1183     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1184   DONE;
1185 })
1186
1187 (define_expand "vlshr<mode>3"
1188   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1189         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1190                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1191   "TARGET_NEON"
1192 {
1193   if (s_register_operand (operands[2], <MODE>mode))
1194     {
1195       rtx neg = gen_reg_rtx (<MODE>mode);
1196       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1197       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1198     }
1199   else
1200     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1201   DONE;
1202 })
1203
1204 ;; 64-bit shifts
1205
1206 ;; This pattern loads a 32-bit shift count into a 64-bit NEON register,
1207 ;; leaving the upper half uninitalized.  This is OK since the shift
1208 ;; instruction only looks at the low 8 bits anyway.  To avoid confusing
1209 ;; data flow analysis however, we pretend the full register is set
1210 ;; using an unspec.
1211 (define_insn "neon_load_count"
1212   [(set (match_operand:DI 0 "s_register_operand" "=w,w")
1213         (unspec:DI [(match_operand:SI 1 "nonimmediate_operand" "Um,r")]
1214                    UNSPEC_LOAD_COUNT))]
1215   "TARGET_NEON"
1216   "@
1217    vld1.32\t{%P0[0]}, %A1
1218    vmov.32\t%P0[0], %1"
1219   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
1220 )
1221
1222 (define_insn "ashldi3_neon_noclobber"
1223   [(set (match_operand:DI 0 "s_register_operand"            "=w,w")
1224         (ashift:DI (match_operand:DI 1 "s_register_operand" " w,w")
1225                    (match_operand:DI 2 "reg_or_int_operand" " i,w")))]
1226   "TARGET_NEON && reload_completed
1227    && (!CONST_INT_P (operands[2])
1228        || (INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) < 64))"
1229   "@
1230    vshl.u64\t%P0, %P1, %2
1231    vshl.u64\t%P0, %P1, %P2"
1232   [(set_attr "neon_type" "neon_vshl_ddd,neon_vshl_ddd")]
1233 )
1234
1235 (define_insn_and_split "ashldi3_neon"
1236   [(set (match_operand:DI 0 "s_register_operand"            "= w, w,?&r,?r, ?w,w")
1237         (ashift:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r, 0w,w")
1238                    (match_operand:SI 2 "general_operand"    "rUm, i,  r, i,rUm,i")))
1239    (clobber (match_scratch:SI 3                             "= X, X,?&r, X,  X,X"))
1240    (clobber (match_scratch:SI 4                             "= X, X,?&r, X,  X,X"))
1241    (clobber (match_scratch:DI 5                             "=&w, X,  X, X, &w,X"))
1242    (clobber (reg:CC_C CC_REGNUM))]
1243   "TARGET_NEON"
1244   "#"
1245   "TARGET_NEON && reload_completed"
1246   [(const_int 0)]
1247   "
1248   {
1249     if (IS_VFP_REGNUM (REGNO (operands[0])))
1250       {
1251         if (CONST_INT_P (operands[2]))
1252           {
1253             if (INTVAL (operands[2]) < 1)
1254               {
1255                 emit_insn (gen_movdi (operands[0], operands[1]));
1256                 DONE;
1257               }
1258             else if (INTVAL (operands[2]) > 63)
1259               operands[2] = gen_rtx_CONST_INT (VOIDmode, 63);
1260           }
1261         else
1262           {
1263             emit_insn (gen_neon_load_count (operands[5], operands[2]));
1264             operands[2] = operands[5];
1265           }
1266
1267         /* Ditch the unnecessary clobbers.  */
1268         emit_insn (gen_ashldi3_neon_noclobber (operands[0], operands[1],
1269                                                operands[2]));
1270       }
1271     else
1272       {
1273         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1274           /* This clobbers CC.  */
1275           emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
1276         else
1277           arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
1278                                          operands[2], operands[3], operands[4]);
1279       }
1280     DONE;
1281   }"
1282   [(set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")
1283    (set_attr "opt" "*,*,speed,speed,*,*")]
1284 )
1285
1286 ; The shift amount needs to be negated for right-shifts
1287 (define_insn "signed_shift_di3_neon"
1288   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1289         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1290                     (match_operand:DI 2 "s_register_operand" " w")]
1291                    UNSPEC_ASHIFT_SIGNED))]
1292   "TARGET_NEON && reload_completed"
1293   "vshl.s64\t%P0, %P1, %P2"
1294   [(set_attr "neon_type" "neon_vshl_ddd")]
1295 )
1296
1297 ; The shift amount needs to be negated for right-shifts
1298 (define_insn "unsigned_shift_di3_neon"
1299   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1300         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1301                     (match_operand:DI 2 "s_register_operand" " w")]
1302                    UNSPEC_ASHIFT_UNSIGNED))]
1303   "TARGET_NEON && reload_completed"
1304   "vshl.u64\t%P0, %P1, %P2"
1305   [(set_attr "neon_type" "neon_vshl_ddd")]
1306 )
1307
1308 (define_insn "ashrdi3_neon_imm_noclobber"
1309   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1310         (ashiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1311                      (match_operand:DI 2 "const_int_operand"  " i")))]
1312   "TARGET_NEON && reload_completed
1313    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1314   "vshr.s64\t%P0, %P1, %2"
1315   [(set_attr "neon_type" "neon_vshl_ddd")]
1316 )
1317
1318 (define_insn "lshrdi3_neon_imm_noclobber"
1319   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1320         (lshiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1321                      (match_operand:DI 2 "const_int_operand"  " i")))]
1322   "TARGET_NEON && reload_completed
1323    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1324   "vshr.u64\t%P0, %P1, %2"
1325   [(set_attr "neon_type" "neon_vshl_ddd")]
1326 )
1327
1328 ;; ashrdi3_neon
1329 ;; lshrdi3_neon
1330 (define_insn_and_split "<shift>di3_neon"
1331   [(set (match_operand:DI 0 "s_register_operand"             "= w, w,?&r,?r,?w,?w")
1332         (rshifts:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r,0w, w")
1333                     (match_operand:SI 2 "reg_or_int_operand" "  r, i,  r, i, r, i")))
1334    (clobber (match_scratch:SI 3                              "=2r, X, &r, X,2r, X"))
1335    (clobber (match_scratch:SI 4                              "= X, X, &r, X, X, X"))
1336    (clobber (match_scratch:DI 5                              "=&w, X,  X, X,&w, X"))
1337    (clobber (reg:CC CC_REGNUM))]
1338   "TARGET_NEON"
1339   "#"
1340   "TARGET_NEON && reload_completed"
1341   [(const_int 0)]
1342   "
1343   {
1344     if (IS_VFP_REGNUM (REGNO (operands[0])))
1345       {
1346         if (CONST_INT_P (operands[2]))
1347           {
1348             if (INTVAL (operands[2]) < 1)
1349               {
1350                 emit_insn (gen_movdi (operands[0], operands[1]));
1351                 DONE;
1352               }
1353             else if (INTVAL (operands[2]) > 64)
1354               operands[2] = gen_rtx_CONST_INT (VOIDmode, 64);
1355
1356             /* Ditch the unnecessary clobbers.  */
1357             emit_insn (gen_<shift>di3_neon_imm_noclobber (operands[0],
1358                                                           operands[1],
1359                                                           operands[2]));
1360           }
1361         else
1362           {
1363             /* We must use a negative left-shift.  */
1364             emit_insn (gen_negsi2 (operands[3], operands[2]));
1365             emit_insn (gen_neon_load_count (operands[5], operands[3]));
1366             emit_insn (gen_<shifttype>_shift_di3_neon (operands[0], operands[1],
1367                                                        operands[5]));
1368           }
1369       }
1370     else
1371       {
1372         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1373           /* This clobbers CC.  */
1374           emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1]));
1375         else
1376           /* This clobbers CC (ASHIFTRT by register only).  */
1377           arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
1378                                          operands[2], operands[3], operands[4]);
1379       }
1380
1381     DONE;
1382   }"
1383   [(set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")
1384    (set_attr "opt" "*,*,speed,speed,*,*")]
1385 )
1386
1387 ;; Widening operations
1388
1389 (define_insn "widen_ssum<mode>3"
1390   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1391         (plus:<V_widen> (sign_extend:<V_widen>
1392                           (match_operand:VW 1 "s_register_operand" "%w"))
1393                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1394   "TARGET_NEON"
1395   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1396   [(set_attr "neon_type" "neon_int_3")]
1397 )
1398
1399 (define_insn "widen_usum<mode>3"
1400   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1401         (plus:<V_widen> (zero_extend:<V_widen>
1402                           (match_operand:VW 1 "s_register_operand" "%w"))
1403                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1404   "TARGET_NEON"
1405   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1406   [(set_attr "neon_type" "neon_int_3")]
1407 )
1408
1409 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1410 ;; shift-count granularity. That's good enough for the middle-end's current
1411 ;; needs.
1412
1413 ;; Note that it's not safe to perform such an operation in big-endian mode,
1414 ;; due to element-ordering issues.
1415
1416 (define_expand "vec_shr_<mode>"
1417   [(match_operand:VDQ 0 "s_register_operand" "")
1418    (match_operand:VDQ 1 "s_register_operand" "")
1419    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1420   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1421 {
1422   rtx zero_reg;
1423   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1424   const int width = GET_MODE_BITSIZE (<MODE>mode);
1425   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1426   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1427     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1428
1429   if (num_bits == width)
1430     {
1431       emit_move_insn (operands[0], operands[1]);
1432       DONE;
1433     }
1434
1435   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1436   operands[0] = gen_lowpart (bvecmode, operands[0]);
1437   operands[1] = gen_lowpart (bvecmode, operands[1]);
1438
1439   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1440                       GEN_INT (num_bits / BITS_PER_UNIT)));
1441   DONE;
1442 })
1443
1444 (define_expand "vec_shl_<mode>"
1445   [(match_operand:VDQ 0 "s_register_operand" "")
1446    (match_operand:VDQ 1 "s_register_operand" "")
1447    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1448   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1449 {
1450   rtx zero_reg;
1451   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1452   const int width = GET_MODE_BITSIZE (<MODE>mode);
1453   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1454   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1455     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1456
1457   if (num_bits == 0)
1458     {
1459       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1460       DONE;
1461     }
1462
1463   num_bits = width - num_bits;
1464
1465   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1466   operands[0] = gen_lowpart (bvecmode, operands[0]);
1467   operands[1] = gen_lowpart (bvecmode, operands[1]);
1468
1469   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1470                       GEN_INT (num_bits / BITS_PER_UNIT)));
1471   DONE;
1472 })
1473
1474 ;; Helpers for quad-word reduction operations
1475
1476 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1477 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1478 ; N/2-element vector.
1479
1480 (define_insn "quad_halves_<code>v4si"
1481   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1482         (vqh_ops:V2SI
1483           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1484                            (parallel [(const_int 0) (const_int 1)]))
1485           (vec_select:V2SI (match_dup 1)
1486                            (parallel [(const_int 2) (const_int 3)]))))]
1487   "TARGET_NEON"
1488   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1489   [(set_attr "vqh_mnem" "<VQH_mnem>")
1490    (set (attr "neon_type")
1491       (if_then_else (eq_attr "vqh_mnem" "vadd")
1492                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1493 )
1494
1495 (define_insn "quad_halves_<code>v4sf"
1496   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1497         (vqhs_ops:V2SF
1498           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1499                            (parallel [(const_int 0) (const_int 1)]))
1500           (vec_select:V2SF (match_dup 1)
1501                            (parallel [(const_int 2) (const_int 3)]))))]
1502   "TARGET_NEON && flag_unsafe_math_optimizations"
1503   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1504   [(set_attr "vqh_mnem" "<VQH_mnem>")
1505    (set (attr "neon_type")
1506       (if_then_else (eq_attr "vqh_mnem" "vadd")
1507                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1508 )
1509
1510 (define_insn "quad_halves_<code>v8hi"
1511   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1512         (vqh_ops:V4HI
1513           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1514                            (parallel [(const_int 0) (const_int 1)
1515                                       (const_int 2) (const_int 3)]))
1516           (vec_select:V4HI (match_dup 1)
1517                            (parallel [(const_int 4) (const_int 5)
1518                                       (const_int 6) (const_int 7)]))))]
1519   "TARGET_NEON"
1520   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1521   [(set_attr "vqh_mnem" "<VQH_mnem>")
1522    (set (attr "neon_type")
1523       (if_then_else (eq_attr "vqh_mnem" "vadd")
1524                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1525 )
1526
1527 (define_insn "quad_halves_<code>v16qi"
1528   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1529         (vqh_ops:V8QI
1530           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1531                            (parallel [(const_int 0) (const_int 1)
1532                                       (const_int 2) (const_int 3)
1533                                       (const_int 4) (const_int 5)
1534                                       (const_int 6) (const_int 7)]))
1535           (vec_select:V8QI (match_dup 1)
1536                            (parallel [(const_int 8) (const_int 9)
1537                                       (const_int 10) (const_int 11)
1538                                       (const_int 12) (const_int 13)
1539                                       (const_int 14) (const_int 15)]))))]
1540   "TARGET_NEON"
1541   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1542   [(set_attr "vqh_mnem" "<VQH_mnem>")
1543    (set (attr "neon_type")
1544       (if_then_else (eq_attr "vqh_mnem" "vadd")
1545                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1546 )
1547
1548 (define_expand "move_hi_quad_<mode>"
1549  [(match_operand:ANY128 0 "s_register_operand" "")
1550   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1551  "TARGET_NEON"
1552 {
1553   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1554                                        GET_MODE_SIZE (<V_HALF>mode)),
1555                   operands[1]);
1556   DONE;
1557 })
1558
1559 (define_expand "move_lo_quad_<mode>"
1560  [(match_operand:ANY128 0 "s_register_operand" "")
1561   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1562  "TARGET_NEON"
1563 {
1564   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1565                                        <MODE>mode, 0),
1566                   operands[1]);
1567   DONE;
1568 })
1569
1570 ;; Reduction operations
1571
1572 (define_expand "reduc_splus_<mode>"
1573   [(match_operand:VD 0 "s_register_operand" "")
1574    (match_operand:VD 1 "s_register_operand" "")]
1575   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1576 {
1577   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1578                         &gen_neon_vpadd_internal<mode>);
1579   DONE;
1580 })
1581
1582 (define_expand "reduc_splus_<mode>"
1583   [(match_operand:VQ 0 "s_register_operand" "")
1584    (match_operand:VQ 1 "s_register_operand" "")]
1585   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1586    && !BYTES_BIG_ENDIAN"
1587 {
1588   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1589   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1590
1591   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1592   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1593   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1594
1595   DONE;
1596 })
1597
1598 (define_insn "reduc_splus_v2di"
1599   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1600         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1601                      UNSPEC_VPADD))]
1602   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1603   "vadd.i64\t%e0, %e1, %f1"
1604   [(set_attr "neon_type" "neon_int_1")]
1605 )
1606
1607 ;; NEON does not distinguish between signed and unsigned addition except on
1608 ;; widening operations.
1609 (define_expand "reduc_uplus_<mode>"
1610   [(match_operand:VDQI 0 "s_register_operand" "")
1611    (match_operand:VDQI 1 "s_register_operand" "")]
1612   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1613 {
1614   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1615   DONE;
1616 })
1617
1618 (define_expand "reduc_smin_<mode>"
1619   [(match_operand:VD 0 "s_register_operand" "")
1620    (match_operand:VD 1 "s_register_operand" "")]
1621   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1622 {
1623   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1624                         &gen_neon_vpsmin<mode>);
1625   DONE;
1626 })
1627
1628 (define_expand "reduc_smin_<mode>"
1629   [(match_operand:VQ 0 "s_register_operand" "")
1630    (match_operand:VQ 1 "s_register_operand" "")]
1631   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1632    && !BYTES_BIG_ENDIAN"
1633 {
1634   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1635   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1636
1637   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1638   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1639   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1640
1641   DONE;
1642 })
1643
1644 (define_expand "reduc_smax_<mode>"
1645   [(match_operand:VD 0 "s_register_operand" "")
1646    (match_operand:VD 1 "s_register_operand" "")]
1647   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1648 {
1649   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1650                         &gen_neon_vpsmax<mode>);
1651   DONE;
1652 })
1653
1654 (define_expand "reduc_smax_<mode>"
1655   [(match_operand:VQ 0 "s_register_operand" "")
1656    (match_operand:VQ 1 "s_register_operand" "")]
1657   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1658    && !BYTES_BIG_ENDIAN"
1659 {
1660   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1661   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1662
1663   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1664   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1665   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1666
1667   DONE;
1668 })
1669
1670 (define_expand "reduc_umin_<mode>"
1671   [(match_operand:VDI 0 "s_register_operand" "")
1672    (match_operand:VDI 1 "s_register_operand" "")]
1673   "TARGET_NEON"
1674 {
1675   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1676                         &gen_neon_vpumin<mode>);
1677   DONE;
1678 })
1679
1680 (define_expand "reduc_umin_<mode>"
1681   [(match_operand:VQI 0 "s_register_operand" "")
1682    (match_operand:VQI 1 "s_register_operand" "")]
1683   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1684 {
1685   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1686   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1687
1688   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1689   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1690   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1691
1692   DONE;
1693 })
1694
1695 (define_expand "reduc_umax_<mode>"
1696   [(match_operand:VDI 0 "s_register_operand" "")
1697    (match_operand:VDI 1 "s_register_operand" "")]
1698   "TARGET_NEON"
1699 {
1700   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1701                         &gen_neon_vpumax<mode>);
1702   DONE;
1703 })
1704
1705 (define_expand "reduc_umax_<mode>"
1706   [(match_operand:VQI 0 "s_register_operand" "")
1707    (match_operand:VQI 1 "s_register_operand" "")]
1708   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1709 {
1710   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1711   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1712
1713   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1714   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1715   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1716
1717   DONE;
1718 })
1719
1720 (define_insn "neon_vpadd_internal<mode>"
1721   [(set (match_operand:VD 0 "s_register_operand" "=w")
1722         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1723                     (match_operand:VD 2 "s_register_operand" "w")]
1724                    UNSPEC_VPADD))]
1725   "TARGET_NEON"
1726   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1727   ;; Assume this schedules like vadd.
1728   [(set (attr "neon_type")
1729       (if_then_else (match_test "<Is_float_mode>")
1730                     (if_then_else (match_test "<Is_d_reg>")
1731                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1732                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1733                     (const_string "neon_int_1")))]
1734 )
1735
1736 (define_insn "neon_vpsmin<mode>"
1737   [(set (match_operand:VD 0 "s_register_operand" "=w")
1738         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1739                     (match_operand:VD 2 "s_register_operand" "w")]
1740                    UNSPEC_VPSMIN))]
1741   "TARGET_NEON"
1742   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1743   ;; Assume this schedules like vmin.
1744   [(set (attr "neon_type")
1745       (if_then_else (match_test "<Is_float_mode>")
1746                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1747                     (const_string "neon_int_5")))]
1748 )
1749
1750 (define_insn "neon_vpsmax<mode>"
1751   [(set (match_operand:VD 0 "s_register_operand" "=w")
1752         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1753                     (match_operand:VD 2 "s_register_operand" "w")]
1754                    UNSPEC_VPSMAX))]
1755   "TARGET_NEON"
1756   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1757   ;; Assume this schedules like vmax.
1758   [(set (attr "neon_type")
1759       (if_then_else (match_test "<Is_float_mode>")
1760                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1761                     (const_string "neon_int_5")))]
1762 )
1763
1764 (define_insn "neon_vpumin<mode>"
1765   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1766         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1767                      (match_operand:VDI 2 "s_register_operand" "w")]
1768                    UNSPEC_VPUMIN))]
1769   "TARGET_NEON"
1770   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1771   ;; Assume this schedules like umin.
1772   [(set_attr "neon_type" "neon_int_5")]
1773 )
1774
1775 (define_insn "neon_vpumax<mode>"
1776   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1777         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1778                      (match_operand:VDI 2 "s_register_operand" "w")]
1779                    UNSPEC_VPUMAX))]
1780   "TARGET_NEON"
1781   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1782   ;; Assume this schedules like umax.
1783   [(set_attr "neon_type" "neon_int_5")]
1784 )
1785
1786 ;; Saturating arithmetic
1787
1788 ; NOTE: Neon supports many more saturating variants of instructions than the
1789 ; following, but these are all GCC currently understands.
1790 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1791 ; yet either, although these patterns may be used by intrinsics when they're
1792 ; added.
1793
1794 (define_insn "*ss_add<mode>_neon"
1795   [(set (match_operand:VD 0 "s_register_operand" "=w")
1796        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1797                    (match_operand:VD 2 "s_register_operand" "w")))]
1798   "TARGET_NEON"
1799   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1800   [(set_attr "neon_type" "neon_int_4")]
1801 )
1802
1803 (define_insn "*us_add<mode>_neon"
1804   [(set (match_operand:VD 0 "s_register_operand" "=w")
1805        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1806                    (match_operand:VD 2 "s_register_operand" "w")))]
1807   "TARGET_NEON"
1808   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1809   [(set_attr "neon_type" "neon_int_4")]
1810 )
1811
1812 (define_insn "*ss_sub<mode>_neon"
1813   [(set (match_operand:VD 0 "s_register_operand" "=w")
1814        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1815                     (match_operand:VD 2 "s_register_operand" "w")))]
1816   "TARGET_NEON"
1817   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1818   [(set_attr "neon_type" "neon_int_5")]
1819 )
1820
1821 (define_insn "*us_sub<mode>_neon"
1822   [(set (match_operand:VD 0 "s_register_operand" "=w")
1823        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1824                     (match_operand:VD 2 "s_register_operand" "w")))]
1825   "TARGET_NEON"
1826   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1827   [(set_attr "neon_type" "neon_int_5")]
1828 )
1829
1830 ;; Conditional instructions.  These are comparisons with conditional moves for
1831 ;; vectors.  They perform the assignment:
1832 ;;
1833 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1834 ;;
1835 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1836 ;; element-wise.
1837
1838 (define_expand "vcond<mode><mode>"
1839   [(set (match_operand:VDQW 0 "s_register_operand" "")
1840         (if_then_else:VDQW
1841           (match_operator 3 "arm_comparison_operator"
1842             [(match_operand:VDQW 4 "s_register_operand" "")
1843              (match_operand:VDQW 5 "nonmemory_operand" "")])
1844           (match_operand:VDQW 1 "s_register_operand" "")
1845           (match_operand:VDQW 2 "s_register_operand" "")))]
1846   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1847 {
1848   rtx mask;
1849   int inverse = 0, immediate_zero = 0;
1850   /* See the description of "magic" bits in the 'T' case of
1851      arm_print_operand.  */
1852   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1853                              ? 3 : 1;
1854   rtx magic_rtx = GEN_INT (magic_word);
1855
1856   mask = gen_reg_rtx (<V_cmp_result>mode);
1857
1858   if (operands[5] == CONST0_RTX (<MODE>mode))
1859     immediate_zero = 1;
1860   else if (!REG_P (operands[5]))
1861     operands[5] = force_reg (<MODE>mode, operands[5]);
1862
1863   switch (GET_CODE (operands[3]))
1864     {
1865     case GE:
1866       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1867                                       magic_rtx));
1868       break;
1869
1870     case GT:
1871       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1872                                       magic_rtx));
1873       break;
1874
1875     case EQ:
1876       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1877                                       magic_rtx));
1878       break;
1879
1880     case LE:
1881       if (immediate_zero)
1882         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1883                                         magic_rtx));
1884       else
1885         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1886                                         magic_rtx));
1887       break;
1888
1889     case LT:
1890       if (immediate_zero)
1891         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1892                                         magic_rtx));
1893       else
1894         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1895                                         magic_rtx));
1896       break;
1897
1898     case NE:
1899       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1900                                       magic_rtx));
1901       inverse = 1;
1902       break;
1903
1904     default:
1905       gcc_unreachable ();
1906     }
1907
1908   if (inverse)
1909     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1910                                     operands[1]));
1911   else
1912     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1913                                     operands[2]));
1914
1915   DONE;
1916 })
1917
1918 (define_expand "vcondu<mode><mode>"
1919   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1920         (if_then_else:VDQIW
1921           (match_operator 3 "arm_comparison_operator"
1922             [(match_operand:VDQIW 4 "s_register_operand" "")
1923              (match_operand:VDQIW 5 "s_register_operand" "")])
1924           (match_operand:VDQIW 1 "s_register_operand" "")
1925           (match_operand:VDQIW 2 "s_register_operand" "")))]
1926   "TARGET_NEON"
1927 {
1928   rtx mask;
1929   int inverse = 0, immediate_zero = 0;
1930
1931   mask = gen_reg_rtx (<V_cmp_result>mode);
1932
1933   if (operands[5] == CONST0_RTX (<MODE>mode))
1934     immediate_zero = 1;
1935   else if (!REG_P (operands[5]))
1936     operands[5] = force_reg (<MODE>mode, operands[5]);
1937
1938   switch (GET_CODE (operands[3]))
1939     {
1940     case GEU:
1941       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1942                                       const0_rtx));
1943       break;
1944
1945     case GTU:
1946       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1947                                       const0_rtx));
1948       break;
1949
1950     case EQ:
1951       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1952                                       const0_rtx));
1953       break;
1954
1955     case LEU:
1956       if (immediate_zero)
1957         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1958                                         const0_rtx));
1959       else
1960         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1961                                         const0_rtx));
1962       break;
1963
1964     case LTU:
1965       if (immediate_zero)
1966         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1967                                         const0_rtx));
1968       else
1969         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1970                                         const0_rtx));
1971       break;
1972
1973     case NE:
1974       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1975                                       const0_rtx));
1976       inverse = 1;
1977       break;
1978
1979     default:
1980       gcc_unreachable ();
1981     }
1982
1983   if (inverse)
1984     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1985                                     operands[1]));
1986   else
1987     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1988                                     operands[2]));
1989
1990   DONE;
1991 })
1992
1993 ;; Patterns for builtins.
1994
1995 ; good for plain vadd, vaddq.
1996
1997 (define_expand "neon_vadd<mode>"
1998   [(match_operand:VDQX 0 "s_register_operand" "=w")
1999    (match_operand:VDQX 1 "s_register_operand" "w")
2000    (match_operand:VDQX 2 "s_register_operand" "w")
2001    (match_operand:SI 3 "immediate_operand" "i")]
2002   "TARGET_NEON"
2003 {
2004   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2005     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
2006   else
2007     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
2008                                            operands[2]));
2009   DONE;
2010 })
2011
2012 ; Note that NEON operations don't support the full IEEE 754 standard: in
2013 ; particular, denormal values are flushed to zero.  This means that GCC cannot
2014 ; use those instructions for autovectorization, etc. unless
2015 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
2016 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
2017 ; header) must work in either case: if -funsafe-math-optimizations is given,
2018 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
2019 ; expand to unspecs (which may potentially limit the extent to which they might
2020 ; be optimized by generic code).
2021
2022 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2023
2024 (define_insn "neon_vadd<mode>_unspec"
2025   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2026         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2027                       (match_operand:VDQX 2 "s_register_operand" "w")]
2028                      UNSPEC_VADD))]
2029   "TARGET_NEON"
2030   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2031   [(set (attr "neon_type")
2032       (if_then_else (match_test "<Is_float_mode>")
2033                     (if_then_else (match_test "<Is_d_reg>")
2034                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2035                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2036                     (const_string "neon_int_1")))]
2037 )
2038
2039 ; operand 3 represents in bits:
2040 ;  bit 0: signed (vs unsigned).
2041 ;  bit 1: rounding (vs none).
2042
2043 (define_insn "neon_vaddl<mode>"
2044   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2045         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2046                            (match_operand:VDI 2 "s_register_operand" "w")
2047                            (match_operand:SI 3 "immediate_operand" "i")]
2048                           UNSPEC_VADDL))]
2049   "TARGET_NEON"
2050   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2051   [(set_attr "neon_type" "neon_int_3")]
2052 )
2053
2054 (define_insn "neon_vaddw<mode>"
2055   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2056         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2057                            (match_operand:VDI 2 "s_register_operand" "w")
2058                            (match_operand:SI 3 "immediate_operand" "i")]
2059                           UNSPEC_VADDW))]
2060   "TARGET_NEON"
2061   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2062   [(set_attr "neon_type" "neon_int_2")]
2063 )
2064
2065 ; vhadd and vrhadd.
2066
2067 (define_insn "neon_vhadd<mode>"
2068   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2069         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2070                        (match_operand:VDQIW 2 "s_register_operand" "w")
2071                        (match_operand:SI 3 "immediate_operand" "i")]
2072                       UNSPEC_VHADD))]
2073   "TARGET_NEON"
2074   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2075   [(set_attr "neon_type" "neon_int_4")]
2076 )
2077
2078 (define_insn "neon_vqadd<mode>"
2079   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2080         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2081                        (match_operand:VDQIX 2 "s_register_operand" "w")
2082                        (match_operand:SI 3 "immediate_operand" "i")]
2083                      UNSPEC_VQADD))]
2084   "TARGET_NEON"
2085   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2086   [(set_attr "neon_type" "neon_int_4")]
2087 )
2088
2089 (define_insn "neon_vaddhn<mode>"
2090   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2091         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2092                             (match_operand:VN 2 "s_register_operand" "w")
2093                             (match_operand:SI 3 "immediate_operand" "i")]
2094                            UNSPEC_VADDHN))]
2095   "TARGET_NEON"
2096   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
2097   [(set_attr "neon_type" "neon_int_4")]
2098 )
2099
2100 ;; We cannot replace this unspec with mul<mode>3 because of the odd
2101 ;; polynomial multiplication case that can specified by operand 3.
2102 (define_insn "neon_vmul<mode>"
2103   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2104         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2105                       (match_operand:VDQW 2 "s_register_operand" "w")
2106                       (match_operand:SI 3 "immediate_operand" "i")]
2107                      UNSPEC_VMUL))]
2108   "TARGET_NEON"
2109   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2110   [(set (attr "neon_type")
2111       (if_then_else (match_test "<Is_float_mode>")
2112                     (if_then_else (match_test "<Is_d_reg>")
2113                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2114                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2115                     (if_then_else (match_test "<Is_d_reg>")
2116                                   (if_then_else
2117                                     (match_test "<Scalar_mul_8_16>")
2118                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2119                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2120                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2121                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
2122                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
2123 )
2124
2125 (define_expand "neon_vmla<mode>"
2126   [(match_operand:VDQW 0 "s_register_operand" "=w")
2127    (match_operand:VDQW 1 "s_register_operand" "0")
2128    (match_operand:VDQW 2 "s_register_operand" "w")
2129    (match_operand:VDQW 3 "s_register_operand" "w")
2130    (match_operand:SI 4 "immediate_operand" "i")]
2131   "TARGET_NEON"
2132 {
2133   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2134     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
2135                                              operands[2], operands[3]));
2136   else
2137     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
2138                                            operands[2], operands[3]));
2139   DONE;
2140 })
2141
2142 (define_expand "neon_vfma<VCVTF:mode>"
2143   [(match_operand:VCVTF 0 "s_register_operand")
2144    (match_operand:VCVTF 1 "s_register_operand")
2145    (match_operand:VCVTF 2 "s_register_operand")
2146    (match_operand:VCVTF 3 "s_register_operand")
2147    (match_operand:SI 4 "immediate_operand")]
2148   "TARGET_NEON && TARGET_FMA"
2149 {
2150   emit_insn (gen_fma<mode>4_intrinsic (operands[0], operands[2], operands[3],
2151                                        operands[1]));
2152   DONE;
2153 })
2154
2155 (define_expand "neon_vfms<VCVTF:mode>"
2156   [(match_operand:VCVTF 0 "s_register_operand")
2157    (match_operand:VCVTF 1 "s_register_operand")
2158    (match_operand:VCVTF 2 "s_register_operand")
2159    (match_operand:VCVTF 3 "s_register_operand")
2160    (match_operand:SI 4 "immediate_operand")]
2161   "TARGET_NEON && TARGET_FMA"
2162 {
2163   emit_insn (gen_fmsub<mode>4_intrinsic (operands[0], operands[2], operands[3],
2164                                          operands[1]));
2165   DONE;
2166 })
2167
2168 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2169
2170 (define_insn "neon_vmla<mode>_unspec"
2171   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2172         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2173                      (match_operand:VDQ 2 "s_register_operand" "w")
2174                      (match_operand:VDQ 3 "s_register_operand" "w")]
2175                     UNSPEC_VMLA))]
2176   "TARGET_NEON"
2177   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2178   [(set (attr "neon_type")
2179       (if_then_else (match_test "<Is_float_mode>")
2180                     (if_then_else (match_test "<Is_d_reg>")
2181                                   (const_string "neon_fp_vmla_ddd")
2182                                   (const_string "neon_fp_vmla_qqq"))
2183                     (if_then_else (match_test "<Is_d_reg>")
2184                                   (if_then_else
2185                                     (match_test "<Scalar_mul_8_16>")
2186                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2187                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2188                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2189                                     (const_string "neon_mla_qqq_8_16")
2190                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2191 )
2192
2193 (define_insn "neon_vmlal<mode>"
2194   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2195         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2196                            (match_operand:VW 2 "s_register_operand" "w")
2197                            (match_operand:VW 3 "s_register_operand" "w")
2198                            (match_operand:SI 4 "immediate_operand" "i")]
2199                           UNSPEC_VMLAL))]
2200   "TARGET_NEON"
2201   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2202   [(set (attr "neon_type")
2203      (if_then_else (match_test "<Scalar_mul_8_16>")
2204                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2205                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2206 )
2207
2208 (define_expand "neon_vmls<mode>"
2209   [(match_operand:VDQW 0 "s_register_operand" "=w")
2210    (match_operand:VDQW 1 "s_register_operand" "0")
2211    (match_operand:VDQW 2 "s_register_operand" "w")
2212    (match_operand:VDQW 3 "s_register_operand" "w")
2213    (match_operand:SI 4 "immediate_operand" "i")]
2214   "TARGET_NEON"
2215 {
2216   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2217     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
2218                  operands[1], operands[2], operands[3]));
2219   else
2220     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
2221                                            operands[2], operands[3]));
2222   DONE;
2223 })
2224
2225 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2226
2227 (define_insn "neon_vmls<mode>_unspec"
2228   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2229         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2230                      (match_operand:VDQ 2 "s_register_operand" "w")
2231                      (match_operand:VDQ 3 "s_register_operand" "w")]
2232                     UNSPEC_VMLS))]
2233   "TARGET_NEON"
2234   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2235   [(set (attr "neon_type")
2236       (if_then_else (match_test "<Is_float_mode>")
2237                     (if_then_else (match_test "<Is_d_reg>")
2238                                   (const_string "neon_fp_vmla_ddd")
2239                                   (const_string "neon_fp_vmla_qqq"))
2240                     (if_then_else (match_test "<Is_d_reg>")
2241                                   (if_then_else
2242                                     (match_test "<Scalar_mul_8_16>")
2243                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2244                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2245                                   (if_then_else
2246                                     (match_test "<Scalar_mul_8_16>")
2247                                     (const_string "neon_mla_qqq_8_16")
2248                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2249 )
2250
2251 (define_insn "neon_vmlsl<mode>"
2252   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2253         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2254                            (match_operand:VW 2 "s_register_operand" "w")
2255                            (match_operand:VW 3 "s_register_operand" "w")
2256                            (match_operand:SI 4 "immediate_operand" "i")]
2257                           UNSPEC_VMLSL))]
2258   "TARGET_NEON"
2259   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2260   [(set (attr "neon_type")
2261      (if_then_else (match_test "<Scalar_mul_8_16>")
2262                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2263                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2264 )
2265
2266 (define_insn "neon_vqdmulh<mode>"
2267   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
2268         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
2269                        (match_operand:VMDQI 2 "s_register_operand" "w")
2270                        (match_operand:SI 3 "immediate_operand" "i")]
2271                       UNSPEC_VQDMULH))]
2272   "TARGET_NEON"
2273   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2274   [(set (attr "neon_type")
2275      (if_then_else (match_test "<Is_d_reg>")
2276         (if_then_else (match_test "<Scalar_mul_8_16>")
2277                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2278                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2279         (if_then_else (match_test "<Scalar_mul_8_16>")
2280                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2281                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2282 )
2283
2284 (define_insn "neon_vqdmlal<mode>"
2285   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2286         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2287                            (match_operand:VMDI 2 "s_register_operand" "w")
2288                            (match_operand:VMDI 3 "s_register_operand" "w")
2289                            (match_operand:SI 4 "immediate_operand" "i")]
2290                           UNSPEC_VQDMLAL))]
2291   "TARGET_NEON"
2292   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2293   [(set (attr "neon_type")
2294      (if_then_else (match_test "<Scalar_mul_8_16>")
2295                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2296                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2297 )
2298
2299 (define_insn "neon_vqdmlsl<mode>"
2300   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2301         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2302                            (match_operand:VMDI 2 "s_register_operand" "w")
2303                            (match_operand:VMDI 3 "s_register_operand" "w")
2304                            (match_operand:SI 4 "immediate_operand" "i")]
2305                           UNSPEC_VQDMLSL))]
2306   "TARGET_NEON"
2307   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2308   [(set (attr "neon_type")
2309      (if_then_else (match_test "<Scalar_mul_8_16>")
2310                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2311                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2312 )
2313
2314 (define_insn "neon_vmull<mode>"
2315   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2316         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2317                            (match_operand:VW 2 "s_register_operand" "w")
2318                            (match_operand:SI 3 "immediate_operand" "i")]
2319                           UNSPEC_VMULL))]
2320   "TARGET_NEON"
2321   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2322   [(set (attr "neon_type")
2323      (if_then_else (match_test "<Scalar_mul_8_16>")
2324                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2325                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2326 )
2327
2328 (define_insn "neon_vqdmull<mode>"
2329   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2330         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2331                            (match_operand:VMDI 2 "s_register_operand" "w")
2332                            (match_operand:SI 3 "immediate_operand" "i")]
2333                           UNSPEC_VQDMULL))]
2334   "TARGET_NEON"
2335   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2336   [(set (attr "neon_type")
2337      (if_then_else (match_test "<Scalar_mul_8_16>")
2338                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2339                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2340 )
2341
2342 (define_expand "neon_vsub<mode>"
2343   [(match_operand:VDQX 0 "s_register_operand" "=w")
2344    (match_operand:VDQX 1 "s_register_operand" "w")
2345    (match_operand:VDQX 2 "s_register_operand" "w")
2346    (match_operand:SI 3 "immediate_operand" "i")]
2347   "TARGET_NEON"
2348 {
2349   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2350     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2351   else
2352     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2353                                            operands[2]));
2354   DONE;
2355 })
2356
2357 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2358
2359 (define_insn "neon_vsub<mode>_unspec"
2360   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2361         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2362                       (match_operand:VDQX 2 "s_register_operand" "w")]
2363                      UNSPEC_VSUB))]
2364   "TARGET_NEON"
2365   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2366   [(set (attr "neon_type")
2367       (if_then_else (match_test "<Is_float_mode>")
2368                     (if_then_else (match_test "<Is_d_reg>")
2369                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2370                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2371                     (const_string "neon_int_2")))]
2372 )
2373
2374 (define_insn "neon_vsubl<mode>"
2375   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2376         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2377                            (match_operand:VDI 2 "s_register_operand" "w")
2378                            (match_operand:SI 3 "immediate_operand" "i")]
2379                           UNSPEC_VSUBL))]
2380   "TARGET_NEON"
2381   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2382   [(set_attr "neon_type" "neon_int_2")]
2383 )
2384
2385 (define_insn "neon_vsubw<mode>"
2386   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2387         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2388                            (match_operand:VDI 2 "s_register_operand" "w")
2389                            (match_operand:SI 3 "immediate_operand" "i")]
2390                           UNSPEC_VSUBW))]
2391   "TARGET_NEON"
2392   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2393   [(set_attr "neon_type" "neon_int_2")]
2394 )
2395
2396 (define_insn "neon_vqsub<mode>"
2397   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2398         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2399                        (match_operand:VDQIX 2 "s_register_operand" "w")
2400                        (match_operand:SI 3 "immediate_operand" "i")]
2401                       UNSPEC_VQSUB))]
2402   "TARGET_NEON"
2403   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2404   [(set_attr "neon_type" "neon_int_5")]
2405 )
2406
2407 (define_insn "neon_vhsub<mode>"
2408   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2409         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2410                        (match_operand:VDQIW 2 "s_register_operand" "w")
2411                        (match_operand:SI 3 "immediate_operand" "i")]
2412                       UNSPEC_VHSUB))]
2413   "TARGET_NEON"
2414   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2415   [(set_attr "neon_type" "neon_int_5")]
2416 )
2417
2418 (define_insn "neon_vsubhn<mode>"
2419   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2420         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2421                             (match_operand:VN 2 "s_register_operand" "w")
2422                             (match_operand:SI 3 "immediate_operand" "i")]
2423                            UNSPEC_VSUBHN))]
2424   "TARGET_NEON"
2425   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2426   [(set_attr "neon_type" "neon_int_4")]
2427 )
2428
2429 (define_insn "neon_vceq<mode>"
2430   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2431         (unspec:<V_cmp_result>
2432           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2433            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2434            (match_operand:SI 3 "immediate_operand" "i,i")]
2435           UNSPEC_VCEQ))]
2436   "TARGET_NEON"
2437   "@
2438   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2439   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2440   [(set (attr "neon_type")
2441       (if_then_else (match_test "<Is_float_mode>")
2442                     (if_then_else (match_test "<Is_d_reg>")
2443                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2444                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2445                     (const_string "neon_int_5")))]
2446 )
2447
2448 (define_insn "neon_vcge<mode>"
2449   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2450         (unspec:<V_cmp_result>
2451           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2452            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2453            (match_operand:SI 3 "immediate_operand" "i,i")]
2454           UNSPEC_VCGE))]
2455   "TARGET_NEON"
2456   "@
2457   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2458   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2459   [(set (attr "neon_type")
2460      (if_then_else (match_test "<Is_float_mode>")
2461                    (if_then_else (match_test "<Is_d_reg>")
2462                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2463                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2464                    (const_string "neon_int_5")))]
2465 )
2466
2467 (define_insn "neon_vcgeu<mode>"
2468   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2469         (unspec:<V_cmp_result>
2470           [(match_operand:VDQIW 1 "s_register_operand" "w")
2471            (match_operand:VDQIW 2 "s_register_operand" "w")
2472            (match_operand:SI 3 "immediate_operand" "i")]
2473           UNSPEC_VCGEU))]
2474   "TARGET_NEON"
2475   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2476   [(set_attr "neon_type" "neon_int_5")]
2477 )
2478
2479 (define_insn "neon_vcgt<mode>"
2480   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2481         (unspec:<V_cmp_result>
2482           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2483            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2484            (match_operand:SI 3 "immediate_operand" "i,i")]
2485           UNSPEC_VCGT))]
2486   "TARGET_NEON"
2487   "@
2488   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2489   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2490   [(set (attr "neon_type")
2491      (if_then_else (match_test "<Is_float_mode>")
2492                    (if_then_else (match_test "<Is_d_reg>")
2493                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2494                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2495                    (const_string "neon_int_5")))]
2496 )
2497
2498 (define_insn "neon_vcgtu<mode>"
2499   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2500         (unspec:<V_cmp_result>
2501           [(match_operand:VDQIW 1 "s_register_operand" "w")
2502            (match_operand:VDQIW 2 "s_register_operand" "w")
2503            (match_operand:SI 3 "immediate_operand" "i")]
2504           UNSPEC_VCGTU))]
2505   "TARGET_NEON"
2506   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2507   [(set_attr "neon_type" "neon_int_5")]
2508 )
2509
2510 ;; VCLE and VCLT only support comparisons with immediate zero (register
2511 ;; variants are VCGE and VCGT with operands reversed).
2512
2513 (define_insn "neon_vcle<mode>"
2514   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2515         (unspec:<V_cmp_result>
2516           [(match_operand:VDQW 1 "s_register_operand" "w")
2517            (match_operand:VDQW 2 "zero_operand" "Dz")
2518            (match_operand:SI 3 "immediate_operand" "i")]
2519           UNSPEC_VCLE))]
2520   "TARGET_NEON"
2521   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2522   [(set (attr "neon_type")
2523       (if_then_else (match_test "<Is_float_mode>")
2524                     (if_then_else (match_test "<Is_d_reg>")
2525                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2526                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2527                     (const_string "neon_int_5")))]
2528 )
2529
2530 (define_insn "neon_vclt<mode>"
2531   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2532         (unspec:<V_cmp_result>
2533           [(match_operand:VDQW 1 "s_register_operand" "w")
2534            (match_operand:VDQW 2 "zero_operand" "Dz")
2535            (match_operand:SI 3 "immediate_operand" "i")]
2536           UNSPEC_VCLT))]
2537   "TARGET_NEON"
2538   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2539   [(set (attr "neon_type")
2540       (if_then_else (match_test "<Is_float_mode>")
2541                     (if_then_else (match_test "<Is_d_reg>")
2542                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2543                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2544                     (const_string "neon_int_5")))]
2545 )
2546
2547 (define_insn "neon_vcage<mode>"
2548   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2549         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2550                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2551                                 (match_operand:SI 3 "immediate_operand" "i")]
2552                                UNSPEC_VCAGE))]
2553   "TARGET_NEON"
2554   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2555   [(set (attr "neon_type")
2556      (if_then_else (match_test "<Is_d_reg>")
2557                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2558                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2559 )
2560
2561 (define_insn "neon_vcagt<mode>"
2562   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2563         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2564                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2565                                 (match_operand:SI 3 "immediate_operand" "i")]
2566                                UNSPEC_VCAGT))]
2567   "TARGET_NEON"
2568   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2569   [(set (attr "neon_type")
2570      (if_then_else (match_test "<Is_d_reg>")
2571                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2572                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2573 )
2574
2575 (define_insn "neon_vtst<mode>"
2576   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2577         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2578                        (match_operand:VDQIW 2 "s_register_operand" "w")
2579                        (match_operand:SI 3 "immediate_operand" "i")]
2580                       UNSPEC_VTST))]
2581   "TARGET_NEON"
2582   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2583   [(set_attr "neon_type" "neon_int_4")]
2584 )
2585
2586 (define_insn "neon_vabd<mode>"
2587   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2588         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2589                       (match_operand:VDQW 2 "s_register_operand" "w")
2590                       (match_operand:SI 3 "immediate_operand" "i")]
2591                      UNSPEC_VABD))]
2592   "TARGET_NEON"
2593   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2594   [(set (attr "neon_type")
2595      (if_then_else (match_test "<Is_float_mode>")
2596                    (if_then_else (match_test "<Is_d_reg>")
2597                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2598                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2599                    (const_string "neon_int_5")))]
2600 )
2601
2602 (define_insn "neon_vabdl<mode>"
2603   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2604         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2605                            (match_operand:VW 2 "s_register_operand" "w")
2606                            (match_operand:SI 3 "immediate_operand" "i")]
2607                           UNSPEC_VABDL))]
2608   "TARGET_NEON"
2609   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2610   [(set_attr "neon_type" "neon_int_5")]
2611 )
2612
2613 (define_insn "neon_vaba<mode>"
2614   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2615         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2616                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2617                                    (match_operand:SI 4 "immediate_operand" "i")]
2618                                   UNSPEC_VABD)
2619                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2620   "TARGET_NEON"
2621   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2622   [(set (attr "neon_type")
2623      (if_then_else (match_test "<Is_d_reg>")
2624                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2625 )
2626
2627 (define_insn "neon_vabal<mode>"
2628   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2629         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2630                                            (match_operand:VW 3 "s_register_operand" "w")
2631                                            (match_operand:SI 4 "immediate_operand" "i")]
2632                                            UNSPEC_VABDL)
2633                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2634   "TARGET_NEON"
2635   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2636   [(set_attr "neon_type" "neon_vaba")]
2637 )
2638
2639 (define_insn "neon_vmax<mode>"
2640   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2641         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2642                       (match_operand:VDQW 2 "s_register_operand" "w")
2643                       (match_operand:SI 3 "immediate_operand" "i")]
2644                      UNSPEC_VMAX))]
2645   "TARGET_NEON"
2646   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2647   [(set (attr "neon_type")
2648     (if_then_else (match_test "<Is_float_mode>")
2649                   (if_then_else (match_test "<Is_d_reg>")
2650                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2651                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2652                   (const_string "neon_int_5")))]
2653 )
2654
2655 (define_insn "neon_vmin<mode>"
2656   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2657         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2658                       (match_operand:VDQW 2 "s_register_operand" "w")
2659                       (match_operand:SI 3 "immediate_operand" "i")]
2660                      UNSPEC_VMIN))]
2661   "TARGET_NEON"
2662   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2663   [(set (attr "neon_type")
2664     (if_then_else (match_test "<Is_float_mode>")
2665                   (if_then_else (match_test "<Is_d_reg>")
2666                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2667                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2668                   (const_string "neon_int_5")))]
2669 )
2670
2671 (define_expand "neon_vpadd<mode>"
2672   [(match_operand:VD 0 "s_register_operand" "=w")
2673    (match_operand:VD 1 "s_register_operand" "w")
2674    (match_operand:VD 2 "s_register_operand" "w")
2675    (match_operand:SI 3 "immediate_operand" "i")]
2676   "TARGET_NEON"
2677 {
2678   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2679                                             operands[2]));
2680   DONE;
2681 })
2682
2683 (define_insn "neon_vpaddl<mode>"
2684   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2685         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2686                                   (match_operand:SI 2 "immediate_operand" "i")]
2687                                  UNSPEC_VPADDL))]
2688   "TARGET_NEON"
2689   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2690   ;; Assume this schedules like vaddl.
2691   [(set_attr "neon_type" "neon_int_3")]
2692 )
2693
2694 (define_insn "neon_vpadal<mode>"
2695   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2696         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2697                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2698                                   (match_operand:SI 3 "immediate_operand" "i")]
2699                                  UNSPEC_VPADAL))]
2700   "TARGET_NEON"
2701   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2702   ;; Assume this schedules like vpadd.
2703   [(set_attr "neon_type" "neon_int_1")]
2704 )
2705
2706 (define_insn "neon_vpmax<mode>"
2707   [(set (match_operand:VD 0 "s_register_operand" "=w")
2708         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2709                     (match_operand:VD 2 "s_register_operand" "w")
2710                     (match_operand:SI 3 "immediate_operand" "i")]
2711                    UNSPEC_VPMAX))]
2712   "TARGET_NEON"
2713   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2714   ;; Assume this schedules like vmax.
2715   [(set (attr "neon_type")
2716     (if_then_else (match_test "<Is_float_mode>")
2717                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2718                   (const_string "neon_int_5")))]
2719 )
2720
2721 (define_insn "neon_vpmin<mode>"
2722   [(set (match_operand:VD 0 "s_register_operand" "=w")
2723         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2724                     (match_operand:VD 2 "s_register_operand" "w")
2725                     (match_operand:SI 3 "immediate_operand" "i")]
2726                    UNSPEC_VPMIN))]
2727   "TARGET_NEON"
2728   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2729   ;; Assume this schedules like vmin.
2730   [(set (attr "neon_type")
2731     (if_then_else (match_test "<Is_float_mode>")
2732                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2733                   (const_string "neon_int_5")))]
2734 )
2735
2736 (define_insn "neon_vrecps<mode>"
2737   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2738         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2739                        (match_operand:VCVTF 2 "s_register_operand" "w")
2740                        (match_operand:SI 3 "immediate_operand" "i")]
2741                       UNSPEC_VRECPS))]
2742   "TARGET_NEON"
2743   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2744   [(set (attr "neon_type")
2745       (if_then_else (match_test "<Is_d_reg>")
2746                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2747                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2748 )
2749
2750 (define_insn "neon_vrsqrts<mode>"
2751   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2752         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2753                        (match_operand:VCVTF 2 "s_register_operand" "w")
2754                        (match_operand:SI 3 "immediate_operand" "i")]
2755                       UNSPEC_VRSQRTS))]
2756   "TARGET_NEON"
2757   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2758   [(set (attr "neon_type")
2759       (if_then_else (match_test "<Is_d_reg>")
2760                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2761                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2762 )
2763
2764 (define_expand "neon_vabs<mode>"
2765   [(match_operand:VDQW 0 "s_register_operand" "")
2766    (match_operand:VDQW 1 "s_register_operand" "")
2767    (match_operand:SI 2 "immediate_operand" "")]
2768   "TARGET_NEON"
2769 {
2770   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2771   DONE;
2772 })
2773
2774 (define_insn "neon_vqabs<mode>"
2775   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2776         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2777                        (match_operand:SI 2 "immediate_operand" "i")]
2778                       UNSPEC_VQABS))]
2779   "TARGET_NEON"
2780   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2781   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2782 )
2783
2784 (define_expand "neon_vneg<mode>"
2785   [(match_operand:VDQW 0 "s_register_operand" "")
2786    (match_operand:VDQW 1 "s_register_operand" "")
2787    (match_operand:SI 2 "immediate_operand" "")]
2788   "TARGET_NEON"
2789 {
2790   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2791   DONE;
2792 })
2793
2794 (define_insn "neon_vqneg<mode>"
2795   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2796         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2797                        (match_operand:SI 2 "immediate_operand" "i")]
2798                       UNSPEC_VQNEG))]
2799   "TARGET_NEON"
2800   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2801   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2802 )
2803
2804 (define_insn "neon_vcls<mode>"
2805   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2806         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2807                        (match_operand:SI 2 "immediate_operand" "i")]
2808                       UNSPEC_VCLS))]
2809   "TARGET_NEON"
2810   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2811   [(set_attr "neon_type" "neon_int_1")]
2812 )
2813
2814 (define_insn "clz<mode>2"
2815   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2816         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2817   "TARGET_NEON"
2818   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2819   [(set_attr "neon_type" "neon_int_1")]
2820 )
2821
2822 (define_expand "neon_vclz<mode>"
2823   [(match_operand:VDQIW 0 "s_register_operand" "")
2824    (match_operand:VDQIW 1 "s_register_operand" "")
2825    (match_operand:SI 2 "immediate_operand" "")]
2826   "TARGET_NEON"
2827 {
2828   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2829   DONE;
2830 })
2831
2832 (define_insn "popcount<mode>2"
2833   [(set (match_operand:VE 0 "s_register_operand" "=w")
2834         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2835   "TARGET_NEON"
2836   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2837   [(set_attr "neon_type" "neon_int_1")]
2838 )
2839
2840 (define_expand "neon_vcnt<mode>"
2841   [(match_operand:VE 0 "s_register_operand" "=w")
2842    (match_operand:VE 1 "s_register_operand" "w")
2843    (match_operand:SI 2 "immediate_operand" "i")]
2844   "TARGET_NEON"
2845 {
2846   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2847   DONE;
2848 })
2849
2850 (define_insn "neon_vrecpe<mode>"
2851   [(set (match_operand:V32 0 "s_register_operand" "=w")
2852         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2853                      (match_operand:SI 2 "immediate_operand" "i")]
2854                     UNSPEC_VRECPE))]
2855   "TARGET_NEON"
2856   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2857   [(set (attr "neon_type")
2858       (if_then_else (match_test "<Is_d_reg>")
2859                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2860                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2861 )
2862
2863 (define_insn "neon_vrsqrte<mode>"
2864   [(set (match_operand:V32 0 "s_register_operand" "=w")
2865         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2866                      (match_operand:SI 2 "immediate_operand" "i")]
2867                     UNSPEC_VRSQRTE))]
2868   "TARGET_NEON"
2869   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2870   [(set (attr "neon_type")
2871       (if_then_else (match_test "<Is_d_reg>")
2872                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2873                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2874 )
2875
2876 (define_expand "neon_vmvn<mode>"
2877   [(match_operand:VDQIW 0 "s_register_operand" "")
2878    (match_operand:VDQIW 1 "s_register_operand" "")
2879    (match_operand:SI 2 "immediate_operand" "")]
2880   "TARGET_NEON"
2881 {
2882   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2883   DONE;
2884 })
2885
2886 (define_insn "neon_vget_lane<mode>_sext_internal"
2887   [(set (match_operand:SI 0 "s_register_operand" "=r")
2888         (sign_extend:SI
2889           (vec_select:<V_elem>
2890             (match_operand:VD 1 "s_register_operand" "w")
2891             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2892   "TARGET_NEON"
2893 {
2894   if (BYTES_BIG_ENDIAN)
2895     {
2896       int elt = INTVAL (operands[2]);
2897       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2898       operands[2] = GEN_INT (elt);
2899     }
2900   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2901 }
2902   [(set_attr "neon_type" "neon_bp_simple")]
2903 )
2904
2905 (define_insn "neon_vget_lane<mode>_zext_internal"
2906   [(set (match_operand:SI 0 "s_register_operand" "=r")
2907         (zero_extend:SI
2908           (vec_select:<V_elem>
2909             (match_operand:VD 1 "s_register_operand" "w")
2910             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2911   "TARGET_NEON"
2912 {
2913   if (BYTES_BIG_ENDIAN)
2914     {
2915       int elt = INTVAL (operands[2]);
2916       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2917       operands[2] = GEN_INT (elt);
2918     }
2919   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2920 }
2921   [(set_attr "neon_type" "neon_bp_simple")]
2922 )
2923
2924 (define_insn "neon_vget_lane<mode>_sext_internal"
2925   [(set (match_operand:SI 0 "s_register_operand" "=r")
2926         (sign_extend:SI
2927           (vec_select:<V_elem>
2928             (match_operand:VQ 1 "s_register_operand" "w")
2929             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2930   "TARGET_NEON"
2931 {
2932   rtx ops[3];
2933   int regno = REGNO (operands[1]);
2934   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2935   unsigned int elt = INTVAL (operands[2]);
2936   unsigned int elt_adj = elt % halfelts;
2937
2938   if (BYTES_BIG_ENDIAN)
2939     elt_adj = halfelts - 1 - elt_adj;
2940
2941   ops[0] = operands[0];
2942   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2943   ops[2] = GEN_INT (elt_adj);
2944   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2945
2946   return "";
2947 }
2948   [(set_attr "neon_type" "neon_bp_simple")]
2949 )
2950
2951 (define_insn "neon_vget_lane<mode>_zext_internal"
2952   [(set (match_operand:SI 0 "s_register_operand" "=r")
2953         (zero_extend:SI
2954           (vec_select:<V_elem>
2955             (match_operand:VQ 1 "s_register_operand" "w")
2956             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2957   "TARGET_NEON"
2958 {
2959   rtx ops[3];
2960   int regno = REGNO (operands[1]);
2961   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2962   unsigned int elt = INTVAL (operands[2]);
2963   unsigned int elt_adj = elt % halfelts;
2964
2965   if (BYTES_BIG_ENDIAN)
2966     elt_adj = halfelts - 1 - elt_adj;
2967
2968   ops[0] = operands[0];
2969   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2970   ops[2] = GEN_INT (elt_adj);
2971   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2972
2973   return "";
2974 }
2975   [(set_attr "neon_type" "neon_bp_simple")]
2976 )
2977
2978 (define_expand "neon_vget_lane<mode>"
2979   [(match_operand:<V_ext> 0 "s_register_operand" "")
2980    (match_operand:VDQW 1 "s_register_operand" "")
2981    (match_operand:SI 2 "immediate_operand" "")
2982    (match_operand:SI 3 "immediate_operand" "")]
2983   "TARGET_NEON"
2984 {
2985   HOST_WIDE_INT magic = INTVAL (operands[3]);
2986   rtx insn;
2987
2988   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2989
2990   if (BYTES_BIG_ENDIAN)
2991     {
2992       /* The intrinsics are defined in terms of a model where the
2993          element ordering in memory is vldm order, whereas the generic
2994          RTL is defined in terms of a model where the element ordering
2995          in memory is array order.  Convert the lane number to conform
2996          to this model.  */
2997       unsigned int elt = INTVAL (operands[2]);
2998       unsigned int reg_nelts
2999         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
3000       elt ^= reg_nelts - 1;
3001       operands[2] = GEN_INT (elt);
3002     }
3003
3004   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
3005     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
3006   else
3007     {
3008       if ((magic & 1) != 0)
3009         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
3010                                                        operands[2]);
3011       else
3012         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
3013                                                        operands[2]);
3014     }
3015   emit_insn (insn);
3016   DONE;
3017 })
3018
3019 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
3020 ; elements.
3021
3022 (define_expand "neon_vget_lanedi"
3023   [(match_operand:DI 0 "s_register_operand" "=r")
3024    (match_operand:DI 1 "s_register_operand" "w")
3025    (match_operand:SI 2 "immediate_operand" "i")
3026    (match_operand:SI 3 "immediate_operand" "i")]
3027   "TARGET_NEON"
3028 {
3029   neon_lane_bounds (operands[2], 0, 1);
3030   emit_move_insn (operands[0], operands[1]);
3031   DONE;
3032 })
3033
3034 (define_expand "neon_vget_lanev2di"
3035   [(match_operand:DI 0 "s_register_operand" "")
3036    (match_operand:V2DI 1 "s_register_operand" "")
3037    (match_operand:SI 2 "immediate_operand" "")
3038    (match_operand:SI 3 "immediate_operand" "")]
3039   "TARGET_NEON"
3040 {
3041   switch (INTVAL (operands[2]))
3042     {
3043     case 0:
3044       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
3045       break;
3046     case 1:
3047       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
3048       break;
3049     default:
3050       neon_lane_bounds (operands[2], 0, 1);
3051       FAIL;
3052     }
3053   DONE;
3054 })
3055
3056 (define_expand "neon_vset_lane<mode>"
3057   [(match_operand:VDQ 0 "s_register_operand" "=w")
3058    (match_operand:<V_elem> 1 "s_register_operand" "r")
3059    (match_operand:VDQ 2 "s_register_operand" "0")
3060    (match_operand:SI 3 "immediate_operand" "i")]
3061   "TARGET_NEON"
3062 {
3063   unsigned int elt = INTVAL (operands[3]);
3064   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3065
3066   if (BYTES_BIG_ENDIAN)
3067     {
3068       unsigned int reg_nelts
3069         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
3070       elt ^= reg_nelts - 1;
3071     }
3072
3073   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
3074                                          GEN_INT (1 << elt), operands[2]));
3075   DONE;
3076 })
3077
3078 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
3079
3080 (define_expand "neon_vset_lanedi"
3081   [(match_operand:DI 0 "s_register_operand" "=w")
3082    (match_operand:DI 1 "s_register_operand" "r")
3083    (match_operand:DI 2 "s_register_operand" "0")
3084    (match_operand:SI 3 "immediate_operand" "i")]
3085   "TARGET_NEON"
3086 {
3087   neon_lane_bounds (operands[3], 0, 1);
3088   emit_move_insn (operands[0], operands[1]);
3089   DONE;
3090 })
3091
3092 (define_expand "neon_vcreate<mode>"
3093   [(match_operand:VDX 0 "s_register_operand" "")
3094    (match_operand:DI 1 "general_operand" "")]
3095   "TARGET_NEON"
3096 {
3097   rtx src = gen_lowpart (<MODE>mode, operands[1]);
3098   emit_move_insn (operands[0], src);
3099   DONE;
3100 })
3101
3102 (define_insn "neon_vdup_n<mode>"
3103   [(set (match_operand:VX 0 "s_register_operand" "=w")
3104         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
3105   "TARGET_NEON"
3106   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
3107   ;; Assume this schedules like vmov.
3108   [(set_attr "neon_type" "neon_bp_simple")]
3109 )
3110
3111 (define_insn "neon_vdup_n<mode>"
3112   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
3113         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
3114   "TARGET_NEON"
3115   "@
3116   vdup.<V_sz_elem>\t%<V_reg>0, %1
3117   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
3118   ;; Assume this schedules like vmov.
3119   [(set_attr "neon_type" "neon_bp_simple")]
3120 )
3121
3122 (define_expand "neon_vdup_ndi"
3123   [(match_operand:DI 0 "s_register_operand" "=w")
3124    (match_operand:DI 1 "s_register_operand" "r")]
3125   "TARGET_NEON"
3126 {
3127   emit_move_insn (operands[0], operands[1]);
3128   DONE;
3129 }
3130 )
3131
3132 (define_insn "neon_vdup_nv2di"
3133   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
3134         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
3135   "TARGET_NEON"
3136   "@
3137   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
3138   vmov\t%e0, %P1\;vmov\t%f0, %P1"
3139   [(set_attr "length" "8")
3140    (set_attr "neon_type" "neon_bp_simple")]
3141 )
3142
3143 (define_insn "neon_vdup_lane<mode>_internal"
3144   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
3145         (vec_duplicate:VDQW
3146           (vec_select:<V_elem>
3147             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3148             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
3149   "TARGET_NEON"
3150 {
3151   if (BYTES_BIG_ENDIAN)
3152     {
3153       int elt = INTVAL (operands[2]);
3154       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
3155       operands[2] = GEN_INT (elt);
3156     }
3157   if (<Is_d_reg>)
3158     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
3159   else
3160     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
3161 }
3162   ;; Assume this schedules like vmov.
3163   [(set_attr "neon_type" "neon_bp_simple")]
3164 )
3165
3166 (define_expand "neon_vdup_lane<mode>"
3167   [(match_operand:VDQW 0 "s_register_operand" "=w")
3168    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3169    (match_operand:SI 2 "immediate_operand" "i")]
3170   "TARGET_NEON"
3171 {
3172   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
3173   if (BYTES_BIG_ENDIAN)
3174     {
3175       unsigned int elt = INTVAL (operands[2]);
3176       unsigned int reg_nelts
3177         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
3178       elt ^= reg_nelts - 1;
3179       operands[2] = GEN_INT (elt);
3180     }
3181     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
3182                                                   operands[2]));
3183     DONE;
3184 })
3185
3186 ; Scalar index is ignored, since only zero is valid here.
3187 (define_expand "neon_vdup_lanedi"
3188   [(match_operand:DI 0 "s_register_operand" "=w")
3189    (match_operand:DI 1 "s_register_operand" "w")
3190    (match_operand:SI 2 "immediate_operand" "i")]
3191   "TARGET_NEON"
3192 {
3193   neon_lane_bounds (operands[2], 0, 1);
3194   emit_move_insn (operands[0], operands[1]);
3195   DONE;
3196 })
3197
3198 ; Likewise for v2di, as the DImode second operand has only a single element.
3199 (define_expand "neon_vdup_lanev2di"
3200   [(match_operand:V2DI 0 "s_register_operand" "=w")
3201    (match_operand:DI 1 "s_register_operand" "w")
3202    (match_operand:SI 2 "immediate_operand" "i")]
3203   "TARGET_NEON"
3204 {
3205   neon_lane_bounds (operands[2], 0, 1);
3206   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
3207   DONE;
3208 })
3209
3210 ; Disabled before reload because we don't want combine doing something silly,
3211 ; but used by the post-reload expansion of neon_vcombine.
3212 (define_insn "*neon_vswp<mode>"
3213   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
3214         (match_operand:VDQX 1 "s_register_operand" "+w"))
3215    (set (match_dup 1) (match_dup 0))]
3216   "TARGET_NEON && reload_completed"
3217   "vswp\t%<V_reg>0, %<V_reg>1"
3218   [(set (attr "neon_type")
3219         (if_then_else (match_test "<Is_d_reg>")
3220                       (const_string "neon_bp_simple")
3221                       (const_string "neon_bp_2cycle")))]
3222 )
3223
3224 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
3225 ;; dest vector.
3226 ;; FIXME: A different implementation of this builtin could make it much
3227 ;; more likely that we wouldn't actually need to output anything (we could make
3228 ;; it so that the reg allocator puts things in the right places magically
3229 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
3230
3231 (define_insn_and_split "neon_vcombine<mode>"
3232   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
3233         (vec_concat:<V_DOUBLE>
3234           (match_operand:VDX 1 "s_register_operand" "w")
3235           (match_operand:VDX 2 "s_register_operand" "w")))]
3236   "TARGET_NEON"
3237   "#"
3238   "&& reload_completed"
3239   [(const_int 0)]
3240 {
3241   neon_split_vcombine (operands);
3242   DONE;
3243 })
3244
3245 (define_expand "neon_vget_high<mode>"
3246   [(match_operand:<V_HALF> 0 "s_register_operand")
3247    (match_operand:VQX 1 "s_register_operand")]
3248   "TARGET_NEON"
3249 {
3250   emit_move_insn (operands[0],
3251                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
3252                                        GET_MODE_SIZE (<V_HALF>mode)));
3253   DONE;
3254 })
3255
3256 (define_expand "neon_vget_low<mode>"
3257   [(match_operand:<V_HALF> 0 "s_register_operand")
3258    (match_operand:VQX 1 "s_register_operand")]
3259   "TARGET_NEON"
3260 {
3261   emit_move_insn (operands[0],
3262                   simplify_gen_subreg (<V_HALF>mode, operands[1],
3263                                        <MODE>mode, 0));
3264   DONE;
3265 })
3266
3267 (define_insn "float<mode><V_cvtto>2"
3268   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3269         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3270   "TARGET_NEON && !flag_rounding_math"
3271   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3272   [(set (attr "neon_type")
3273      (if_then_else (match_test "<Is_d_reg>")
3274                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3275                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3276 )
3277
3278 (define_insn "floatuns<mode><V_cvtto>2"
3279   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3280         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3281   "TARGET_NEON && !flag_rounding_math"
3282   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3283   [(set (attr "neon_type")
3284      (if_then_else (match_test "<Is_d_reg>")
3285                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3286                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3287 )
3288
3289 (define_insn "fix_trunc<mode><V_cvtto>2"
3290   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3291         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3292   "TARGET_NEON"
3293   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3294   [(set (attr "neon_type")
3295      (if_then_else (match_test "<Is_d_reg>")
3296                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3297                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3298 )
3299
3300 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3301   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3302         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3303   "TARGET_NEON"
3304   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3305   [(set (attr "neon_type")
3306      (if_then_else (match_test "<Is_d_reg>")
3307                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3308                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3309 )
3310
3311 (define_insn "neon_vcvt<mode>"
3312   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3313         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3314                            (match_operand:SI 2 "immediate_operand" "i")]
3315                           UNSPEC_VCVT))]
3316   "TARGET_NEON"
3317   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3318   [(set (attr "neon_type")
3319      (if_then_else (match_test "<Is_d_reg>")
3320                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3321                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3322 )
3323
3324 (define_insn "neon_vcvt<mode>"
3325   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3326         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3327                            (match_operand:SI 2 "immediate_operand" "i")]
3328                           UNSPEC_VCVT))]
3329   "TARGET_NEON"
3330   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3331   [(set (attr "neon_type")
3332      (if_then_else (match_test "<Is_d_reg>")
3333                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3334                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3335 )
3336
3337 (define_insn "neon_vcvt_n<mode>"
3338   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3339         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3340                            (match_operand:SI 2 "immediate_operand" "i")
3341                            (match_operand:SI 3 "immediate_operand" "i")]
3342                           UNSPEC_VCVT_N))]
3343   "TARGET_NEON"
3344 {
3345   neon_const_bounds (operands[2], 1, 33);
3346   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3347 }
3348   [(set (attr "neon_type")
3349      (if_then_else (match_test "<Is_d_reg>")
3350                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3351                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3352 )
3353
3354 (define_insn "neon_vcvt_n<mode>"
3355   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3356         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3357                            (match_operand:SI 2 "immediate_operand" "i")
3358                            (match_operand:SI 3 "immediate_operand" "i")]
3359                           UNSPEC_VCVT_N))]
3360   "TARGET_NEON"
3361 {
3362   neon_const_bounds (operands[2], 1, 33);
3363   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3364 }
3365   [(set (attr "neon_type")
3366      (if_then_else (match_test "<Is_d_reg>")
3367                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3368                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3369 )
3370
3371 (define_insn "neon_vmovn<mode>"
3372   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3373         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3374                             (match_operand:SI 2 "immediate_operand" "i")]
3375                            UNSPEC_VMOVN))]
3376   "TARGET_NEON"
3377   "vmovn.<V_if_elem>\t%P0, %q1"
3378   [(set_attr "neon_type" "neon_bp_simple")]
3379 )
3380
3381 (define_insn "neon_vqmovn<mode>"
3382   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3383         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3384                             (match_operand:SI 2 "immediate_operand" "i")]
3385                            UNSPEC_VQMOVN))]
3386   "TARGET_NEON"
3387   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3388   [(set_attr "neon_type" "neon_shift_2")]
3389 )
3390
3391 (define_insn "neon_vqmovun<mode>"
3392   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3393         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3394                             (match_operand:SI 2 "immediate_operand" "i")]
3395                            UNSPEC_VQMOVUN))]
3396   "TARGET_NEON"
3397   "vqmovun.<V_s_elem>\t%P0, %q1"
3398   [(set_attr "neon_type" "neon_shift_2")]
3399 )
3400
3401 (define_insn "neon_vmovl<mode>"
3402   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3403         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3404                            (match_operand:SI 2 "immediate_operand" "i")]
3405                           UNSPEC_VMOVL))]
3406   "TARGET_NEON"
3407   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3408   [(set_attr "neon_type" "neon_shift_1")]
3409 )
3410
3411 (define_insn "neon_vmul_lane<mode>"
3412   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3413         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3414                      (match_operand:VMD 2 "s_register_operand"
3415                                         "<scalar_mul_constraint>")
3416                      (match_operand:SI 3 "immediate_operand" "i")
3417                      (match_operand:SI 4 "immediate_operand" "i")]
3418                     UNSPEC_VMUL_LANE))]
3419   "TARGET_NEON"
3420 {
3421   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3422   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3423 }
3424   [(set (attr "neon_type")
3425      (if_then_else (match_test "<Is_float_mode>")
3426                    (const_string "neon_fp_vmul_ddd")
3427                    (if_then_else (match_test "<Scalar_mul_8_16>")
3428                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3429                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3430 )
3431
3432 (define_insn "neon_vmul_lane<mode>"
3433   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3434         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3435                      (match_operand:<V_HALF> 2 "s_register_operand"
3436                                              "<scalar_mul_constraint>")
3437                      (match_operand:SI 3 "immediate_operand" "i")
3438                      (match_operand:SI 4 "immediate_operand" "i")]
3439                     UNSPEC_VMUL_LANE))]
3440   "TARGET_NEON"
3441 {
3442   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3443   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3444 }
3445   [(set (attr "neon_type")
3446      (if_then_else (match_test "<Is_float_mode>")
3447                    (const_string "neon_fp_vmul_qqd")
3448                    (if_then_else (match_test "<Scalar_mul_8_16>")
3449                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3450                                  (const_string "neon_mul_qqd_32_scalar"))))]
3451 )
3452
3453 (define_insn "neon_vmull_lane<mode>"
3454   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3455         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3456                            (match_operand:VMDI 2 "s_register_operand"
3457                                                "<scalar_mul_constraint>")
3458                            (match_operand:SI 3 "immediate_operand" "i")
3459                            (match_operand:SI 4 "immediate_operand" "i")]
3460                           UNSPEC_VMULL_LANE))]
3461   "TARGET_NEON"
3462 {
3463   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3464   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3465 }
3466   [(set (attr "neon_type")
3467      (if_then_else (match_test "<Scalar_mul_8_16>")
3468                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3469                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3470 )
3471
3472 (define_insn "neon_vqdmull_lane<mode>"
3473   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3474         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3475                            (match_operand:VMDI 2 "s_register_operand"
3476                                                "<scalar_mul_constraint>")
3477                            (match_operand:SI 3 "immediate_operand" "i")
3478                            (match_operand:SI 4 "immediate_operand" "i")]
3479                           UNSPEC_VQDMULL_LANE))]
3480   "TARGET_NEON"
3481 {
3482   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3483   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3484 }
3485   [(set (attr "neon_type")
3486      (if_then_else (match_test "<Scalar_mul_8_16>")
3487                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3488                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3489 )
3490
3491 (define_insn "neon_vqdmulh_lane<mode>"
3492   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3493         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3494                       (match_operand:<V_HALF> 2 "s_register_operand"
3495                                               "<scalar_mul_constraint>")
3496                       (match_operand:SI 3 "immediate_operand" "i")
3497                       (match_operand:SI 4 "immediate_operand" "i")]
3498                       UNSPEC_VQDMULH_LANE))]
3499   "TARGET_NEON"
3500 {
3501   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3502   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3503 }
3504   [(set (attr "neon_type")
3505      (if_then_else (match_test "<Scalar_mul_8_16>")
3506                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3507                    (const_string "neon_mul_qqd_32_scalar")))]
3508 )
3509
3510 (define_insn "neon_vqdmulh_lane<mode>"
3511   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3512         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3513                       (match_operand:VMDI 2 "s_register_operand"
3514                                           "<scalar_mul_constraint>")
3515                       (match_operand:SI 3 "immediate_operand" "i")
3516                       (match_operand:SI 4 "immediate_operand" "i")]
3517                       UNSPEC_VQDMULH_LANE))]
3518   "TARGET_NEON"
3519 {
3520   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3521   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3522 }
3523   [(set (attr "neon_type")
3524      (if_then_else (match_test "<Scalar_mul_8_16>")
3525                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3526                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3527 )
3528
3529 (define_insn "neon_vmla_lane<mode>"
3530   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3531         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3532                      (match_operand:VMD 2 "s_register_operand" "w")
3533                      (match_operand:VMD 3 "s_register_operand"
3534                                         "<scalar_mul_constraint>")
3535                      (match_operand:SI 4 "immediate_operand" "i")
3536                      (match_operand:SI 5 "immediate_operand" "i")]
3537                      UNSPEC_VMLA_LANE))]
3538   "TARGET_NEON"
3539 {
3540   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3541   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3542 }
3543   [(set (attr "neon_type")
3544      (if_then_else (match_test "<Is_float_mode>")
3545                    (const_string "neon_fp_vmla_ddd_scalar")
3546                    (if_then_else (match_test "<Scalar_mul_8_16>")
3547                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3548                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3549 )
3550
3551 (define_insn "neon_vmla_lane<mode>"
3552   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3553         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3554                      (match_operand:VMQ 2 "s_register_operand" "w")
3555                      (match_operand:<V_HALF> 3 "s_register_operand"
3556                                              "<scalar_mul_constraint>")
3557                      (match_operand:SI 4 "immediate_operand" "i")
3558                      (match_operand:SI 5 "immediate_operand" "i")]
3559                      UNSPEC_VMLA_LANE))]
3560   "TARGET_NEON"
3561 {
3562   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3563   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3564 }
3565   [(set (attr "neon_type")
3566      (if_then_else (match_test "<Is_float_mode>")
3567                    (const_string "neon_fp_vmla_qqq_scalar")
3568                    (if_then_else (match_test "<Scalar_mul_8_16>")
3569                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3570                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3571 )
3572
3573 (define_insn "neon_vmlal_lane<mode>"
3574   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3575         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3576                            (match_operand:VMDI 2 "s_register_operand" "w")
3577                            (match_operand:VMDI 3 "s_register_operand"
3578                                                "<scalar_mul_constraint>")
3579                            (match_operand:SI 4 "immediate_operand" "i")
3580                            (match_operand:SI 5 "immediate_operand" "i")]
3581                           UNSPEC_VMLAL_LANE))]
3582   "TARGET_NEON"
3583 {
3584   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3585   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3586 }
3587   [(set (attr "neon_type")
3588      (if_then_else (match_test "<Scalar_mul_8_16>")
3589                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3590                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3591 )
3592
3593 (define_insn "neon_vqdmlal_lane<mode>"
3594   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3595         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3596                            (match_operand:VMDI 2 "s_register_operand" "w")
3597                            (match_operand:VMDI 3 "s_register_operand"
3598                                                "<scalar_mul_constraint>")
3599                            (match_operand:SI 4 "immediate_operand" "i")
3600                            (match_operand:SI 5 "immediate_operand" "i")]
3601                           UNSPEC_VQDMLAL_LANE))]
3602   "TARGET_NEON"
3603 {
3604   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3605   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3606 }
3607   [(set (attr "neon_type")
3608      (if_then_else (match_test "<Scalar_mul_8_16>")
3609                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3610                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3611 )
3612
3613 (define_insn "neon_vmls_lane<mode>"
3614   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3615         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3616                      (match_operand:VMD 2 "s_register_operand" "w")
3617                      (match_operand:VMD 3 "s_register_operand"
3618                                         "<scalar_mul_constraint>")
3619                      (match_operand:SI 4 "immediate_operand" "i")
3620                      (match_operand:SI 5 "immediate_operand" "i")]
3621                     UNSPEC_VMLS_LANE))]
3622   "TARGET_NEON"
3623 {
3624   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3625   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3626 }
3627   [(set (attr "neon_type")
3628      (if_then_else (match_test "<Is_float_mode>")
3629                    (const_string "neon_fp_vmla_ddd_scalar")
3630                    (if_then_else (match_test "<Scalar_mul_8_16>")
3631                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3632                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3633 )
3634
3635 (define_insn "neon_vmls_lane<mode>"
3636   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3637         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3638                      (match_operand:VMQ 2 "s_register_operand" "w")
3639                      (match_operand:<V_HALF> 3 "s_register_operand"
3640                                              "<scalar_mul_constraint>")
3641                      (match_operand:SI 4 "immediate_operand" "i")
3642                      (match_operand:SI 5 "immediate_operand" "i")]
3643                     UNSPEC_VMLS_LANE))]
3644   "TARGET_NEON"
3645 {
3646   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3647   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3648 }
3649   [(set (attr "neon_type")
3650      (if_then_else (match_test "<Is_float_mode>")
3651                    (const_string "neon_fp_vmla_qqq_scalar")
3652                    (if_then_else (match_test "<Scalar_mul_8_16>")
3653                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3654                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3655 )
3656
3657 (define_insn "neon_vmlsl_lane<mode>"
3658   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3659         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3660                            (match_operand:VMDI 2 "s_register_operand" "w")
3661                            (match_operand:VMDI 3 "s_register_operand"
3662                                                "<scalar_mul_constraint>")
3663                            (match_operand:SI 4 "immediate_operand" "i")
3664                            (match_operand:SI 5 "immediate_operand" "i")]
3665                           UNSPEC_VMLSL_LANE))]
3666   "TARGET_NEON"
3667 {
3668   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3669   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3670 }
3671   [(set (attr "neon_type")
3672      (if_then_else (match_test "<Scalar_mul_8_16>")
3673                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3674                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3675 )
3676
3677 (define_insn "neon_vqdmlsl_lane<mode>"
3678   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3679         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3680                            (match_operand:VMDI 2 "s_register_operand" "w")
3681                            (match_operand:VMDI 3 "s_register_operand"
3682                                                "<scalar_mul_constraint>")
3683                            (match_operand:SI 4 "immediate_operand" "i")
3684                            (match_operand:SI 5 "immediate_operand" "i")]
3685                           UNSPEC_VQDMLSL_LANE))]
3686   "TARGET_NEON"
3687 {
3688   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3689   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3690 }
3691   [(set (attr "neon_type")
3692      (if_then_else (match_test "<Scalar_mul_8_16>")
3693                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3694                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3695 )
3696
3697 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3698 ; core register into a temp register, then use a scalar taken from that. This
3699 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3700 ; or extracted from another vector. The latter case it's currently better to
3701 ; use the "_lane" variant, and the former case can probably be implemented
3702 ; using vld1_lane, but that hasn't been done yet.
3703
3704 (define_expand "neon_vmul_n<mode>"
3705   [(match_operand:VMD 0 "s_register_operand" "")
3706    (match_operand:VMD 1 "s_register_operand" "")
3707    (match_operand:<V_elem> 2 "s_register_operand" "")
3708    (match_operand:SI 3 "immediate_operand" "")]
3709   "TARGET_NEON"
3710 {
3711   rtx tmp = gen_reg_rtx (<MODE>mode);
3712   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3713   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3714                                        const0_rtx, const0_rtx));
3715   DONE;
3716 })
3717
3718 (define_expand "neon_vmul_n<mode>"
3719   [(match_operand:VMQ 0 "s_register_operand" "")
3720    (match_operand:VMQ 1 "s_register_operand" "")
3721    (match_operand:<V_elem> 2 "s_register_operand" "")
3722    (match_operand:SI 3 "immediate_operand" "")]
3723   "TARGET_NEON"
3724 {
3725   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3726   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3727   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3728                                        const0_rtx, const0_rtx));
3729   DONE;
3730 })
3731
3732 (define_expand "neon_vmull_n<mode>"
3733   [(match_operand:<V_widen> 0 "s_register_operand" "")
3734    (match_operand:VMDI 1 "s_register_operand" "")
3735    (match_operand:<V_elem> 2 "s_register_operand" "")
3736    (match_operand:SI 3 "immediate_operand" "")]
3737   "TARGET_NEON"
3738 {
3739   rtx tmp = gen_reg_rtx (<MODE>mode);
3740   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3741   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3742                                         const0_rtx, operands[3]));
3743   DONE;
3744 })
3745
3746 (define_expand "neon_vqdmull_n<mode>"
3747   [(match_operand:<V_widen> 0 "s_register_operand" "")
3748    (match_operand:VMDI 1 "s_register_operand" "")
3749    (match_operand:<V_elem> 2 "s_register_operand" "")
3750    (match_operand:SI 3 "immediate_operand" "")]
3751   "TARGET_NEON"
3752 {
3753   rtx tmp = gen_reg_rtx (<MODE>mode);
3754   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3755   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3756                                           const0_rtx, const0_rtx));
3757   DONE;
3758 })
3759
3760 (define_expand "neon_vqdmulh_n<mode>"
3761   [(match_operand:VMDI 0 "s_register_operand" "")
3762    (match_operand:VMDI 1 "s_register_operand" "")
3763    (match_operand:<V_elem> 2 "s_register_operand" "")
3764    (match_operand:SI 3 "immediate_operand" "")]
3765   "TARGET_NEON"
3766 {
3767   rtx tmp = gen_reg_rtx (<MODE>mode);
3768   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3769   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3770                                           const0_rtx, operands[3]));
3771   DONE;
3772 })
3773
3774 (define_expand "neon_vqdmulh_n<mode>"
3775   [(match_operand:VMQI 0 "s_register_operand" "")
3776    (match_operand:VMQI 1 "s_register_operand" "")
3777    (match_operand:<V_elem> 2 "s_register_operand" "")
3778    (match_operand:SI 3 "immediate_operand" "")]
3779   "TARGET_NEON"
3780 {
3781   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3782   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3783   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3784                                           const0_rtx, operands[3]));
3785   DONE;
3786 })
3787
3788 (define_expand "neon_vmla_n<mode>"
3789   [(match_operand:VMD 0 "s_register_operand" "")
3790    (match_operand:VMD 1 "s_register_operand" "")
3791    (match_operand:VMD 2 "s_register_operand" "")
3792    (match_operand:<V_elem> 3 "s_register_operand" "")
3793    (match_operand:SI 4 "immediate_operand" "")]
3794   "TARGET_NEON"
3795 {
3796   rtx tmp = gen_reg_rtx (<MODE>mode);
3797   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3798   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3799                                        tmp, const0_rtx, operands[4]));
3800   DONE;
3801 })
3802
3803 (define_expand "neon_vmla_n<mode>"
3804   [(match_operand:VMQ 0 "s_register_operand" "")
3805    (match_operand:VMQ 1 "s_register_operand" "")
3806    (match_operand:VMQ 2 "s_register_operand" "")
3807    (match_operand:<V_elem> 3 "s_register_operand" "")
3808    (match_operand:SI 4 "immediate_operand" "")]
3809   "TARGET_NEON"
3810 {
3811   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3812   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3813   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3814                                        tmp, const0_rtx, operands[4]));
3815   DONE;
3816 })
3817
3818 (define_expand "neon_vmlal_n<mode>"
3819   [(match_operand:<V_widen> 0 "s_register_operand" "")
3820    (match_operand:<V_widen> 1 "s_register_operand" "")
3821    (match_operand:VMDI 2 "s_register_operand" "")
3822    (match_operand:<V_elem> 3 "s_register_operand" "")
3823    (match_operand:SI 4 "immediate_operand" "")]
3824   "TARGET_NEON"
3825 {
3826   rtx tmp = gen_reg_rtx (<MODE>mode);
3827   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3828   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3829                                         tmp, const0_rtx, operands[4]));
3830   DONE;
3831 })
3832
3833 (define_expand "neon_vqdmlal_n<mode>"
3834   [(match_operand:<V_widen> 0 "s_register_operand" "")
3835    (match_operand:<V_widen> 1 "s_register_operand" "")
3836    (match_operand:VMDI 2 "s_register_operand" "")
3837    (match_operand:<V_elem> 3 "s_register_operand" "")
3838    (match_operand:SI 4 "immediate_operand" "")]
3839   "TARGET_NEON"
3840 {
3841   rtx tmp = gen_reg_rtx (<MODE>mode);
3842   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3843   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3844                                           tmp, const0_rtx, operands[4]));
3845   DONE;
3846 })
3847
3848 (define_expand "neon_vmls_n<mode>"
3849   [(match_operand:VMD 0 "s_register_operand" "")
3850    (match_operand:VMD 1 "s_register_operand" "")
3851    (match_operand:VMD 2 "s_register_operand" "")
3852    (match_operand:<V_elem> 3 "s_register_operand" "")
3853    (match_operand:SI 4 "immediate_operand" "")]
3854   "TARGET_NEON"
3855 {
3856   rtx tmp = gen_reg_rtx (<MODE>mode);
3857   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3858   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3859                                        tmp, const0_rtx, operands[4]));
3860   DONE;
3861 })
3862
3863 (define_expand "neon_vmls_n<mode>"
3864   [(match_operand:VMQ 0 "s_register_operand" "")
3865    (match_operand:VMQ 1 "s_register_operand" "")
3866    (match_operand:VMQ 2 "s_register_operand" "")
3867    (match_operand:<V_elem> 3 "s_register_operand" "")
3868    (match_operand:SI 4 "immediate_operand" "")]
3869   "TARGET_NEON"
3870 {
3871   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3872   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3873   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3874                                        tmp, const0_rtx, operands[4]));
3875   DONE;
3876 })
3877
3878 (define_expand "neon_vmlsl_n<mode>"
3879   [(match_operand:<V_widen> 0 "s_register_operand" "")
3880    (match_operand:<V_widen> 1 "s_register_operand" "")
3881    (match_operand:VMDI 2 "s_register_operand" "")
3882    (match_operand:<V_elem> 3 "s_register_operand" "")
3883    (match_operand:SI 4 "immediate_operand" "")]
3884   "TARGET_NEON"
3885 {
3886   rtx tmp = gen_reg_rtx (<MODE>mode);
3887   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3888   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3889                                         tmp, const0_rtx, operands[4]));
3890   DONE;
3891 })
3892
3893 (define_expand "neon_vqdmlsl_n<mode>"
3894   [(match_operand:<V_widen> 0 "s_register_operand" "")
3895    (match_operand:<V_widen> 1 "s_register_operand" "")
3896    (match_operand:VMDI 2 "s_register_operand" "")
3897    (match_operand:<V_elem> 3 "s_register_operand" "")
3898    (match_operand:SI 4 "immediate_operand" "")]
3899   "TARGET_NEON"
3900 {
3901   rtx tmp = gen_reg_rtx (<MODE>mode);
3902   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3903   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3904                                           tmp, const0_rtx, operands[4]));
3905   DONE;
3906 })
3907
3908 (define_insn "neon_vext<mode>"
3909   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3910         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3911                       (match_operand:VDQX 2 "s_register_operand" "w")
3912                       (match_operand:SI 3 "immediate_operand" "i")]
3913                      UNSPEC_VEXT))]
3914   "TARGET_NEON"
3915 {
3916   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3917   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3918 }
3919   [(set (attr "neon_type")
3920       (if_then_else (match_test "<Is_d_reg>")
3921                     (const_string "neon_bp_simple")
3922                     (const_string "neon_bp_2cycle")))]
3923 )
3924
3925 (define_insn "neon_vrev64<mode>"
3926   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3927         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3928                      (match_operand:SI 2 "immediate_operand" "i")]
3929                     UNSPEC_VREV64))]
3930   "TARGET_NEON"
3931   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3932   [(set_attr "neon_type" "neon_bp_simple")]
3933 )
3934
3935 (define_insn "neon_vrev32<mode>"
3936   [(set (match_operand:VX 0 "s_register_operand" "=w")
3937         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3938                     (match_operand:SI 2 "immediate_operand" "i")]
3939                    UNSPEC_VREV32))]
3940   "TARGET_NEON"
3941   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3942   [(set_attr "neon_type" "neon_bp_simple")]
3943 )
3944
3945 (define_insn "neon_vrev16<mode>"
3946   [(set (match_operand:VE 0 "s_register_operand" "=w")
3947         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3948                     (match_operand:SI 2 "immediate_operand" "i")]
3949                    UNSPEC_VREV16))]
3950   "TARGET_NEON"
3951   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3952   [(set_attr "neon_type" "neon_bp_simple")]
3953 )
3954
3955 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3956 ; allocation. For an intrinsic of form:
3957 ;   rD = vbsl_* (rS, rN, rM)
3958 ; We can use any of:
3959 ;   vbsl rS, rN, rM  (if D = S)
3960 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3961 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3962
3963 (define_insn "neon_vbsl<mode>_internal"
3964   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3965         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3966                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3967                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3968                      UNSPEC_VBSL))]
3969   "TARGET_NEON"
3970   "@
3971   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3972   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3973   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3974   [(set_attr "neon_type" "neon_int_1")]
3975 )
3976
3977 (define_expand "neon_vbsl<mode>"
3978   [(set (match_operand:VDQX 0 "s_register_operand" "")
3979         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3980                       (match_operand:VDQX 2 "s_register_operand" "")
3981                       (match_operand:VDQX 3 "s_register_operand" "")]
3982                      UNSPEC_VBSL))]
3983   "TARGET_NEON"
3984 {
3985   /* We can't alias operands together if they have different modes.  */
3986   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3987 })
3988
3989 (define_insn "neon_vshl<mode>"
3990   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3991         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3992                        (match_operand:VDQIX 2 "s_register_operand" "w")
3993                        (match_operand:SI 3 "immediate_operand" "i")]
3994                       UNSPEC_VSHL))]
3995   "TARGET_NEON"
3996   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3997   [(set (attr "neon_type")
3998       (if_then_else (match_test "<Is_d_reg>")
3999                     (const_string "neon_vshl_ddd")
4000                     (const_string "neon_shift_3")))]
4001 )
4002
4003 (define_insn "neon_vqshl<mode>"
4004   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4005         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4006                        (match_operand:VDQIX 2 "s_register_operand" "w")
4007                        (match_operand:SI 3 "immediate_operand" "i")]
4008                       UNSPEC_VQSHL))]
4009   "TARGET_NEON"
4010   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
4011   [(set (attr "neon_type")
4012       (if_then_else (match_test "<Is_d_reg>")
4013                     (const_string "neon_shift_2")
4014                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
4015 )
4016
4017 (define_insn "neon_vshr_n<mode>"
4018   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4019         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4020                        (match_operand:SI 2 "immediate_operand" "i")
4021                        (match_operand:SI 3 "immediate_operand" "i")]
4022                       UNSPEC_VSHR_N))]
4023   "TARGET_NEON"
4024 {
4025   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
4026   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4027 }
4028   [(set_attr "neon_type" "neon_shift_1")]
4029 )
4030
4031 (define_insn "neon_vshrn_n<mode>"
4032   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
4033         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
4034                             (match_operand:SI 2 "immediate_operand" "i")
4035                             (match_operand:SI 3 "immediate_operand" "i")]
4036                            UNSPEC_VSHRN_N))]
4037   "TARGET_NEON"
4038 {
4039   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
4040   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
4041 }
4042   [(set_attr "neon_type" "neon_shift_1")]
4043 )
4044
4045 (define_insn "neon_vqshrn_n<mode>"
4046   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
4047         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
4048                             (match_operand:SI 2 "immediate_operand" "i")
4049                             (match_operand:SI 3 "immediate_operand" "i")]
4050                            UNSPEC_VQSHRN_N))]
4051   "TARGET_NEON"
4052 {
4053   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
4054   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
4055 }
4056   [(set_attr "neon_type" "neon_shift_2")]
4057 )
4058
4059 (define_insn "neon_vqshrun_n<mode>"
4060   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
4061         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
4062                             (match_operand:SI 2 "immediate_operand" "i")
4063                             (match_operand:SI 3 "immediate_operand" "i")]
4064                            UNSPEC_VQSHRUN_N))]
4065   "TARGET_NEON"
4066 {
4067   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
4068   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
4069 }
4070   [(set_attr "neon_type" "neon_shift_2")]
4071 )
4072
4073 (define_insn "neon_vshl_n<mode>"
4074   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4075         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4076                        (match_operand:SI 2 "immediate_operand" "i")
4077                        (match_operand:SI 3 "immediate_operand" "i")]
4078                       UNSPEC_VSHL_N))]
4079   "TARGET_NEON"
4080 {
4081   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4082   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
4083 }
4084   [(set_attr "neon_type" "neon_shift_1")]
4085 )
4086
4087 (define_insn "neon_vqshl_n<mode>"
4088   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4089         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4090                        (match_operand:SI 2 "immediate_operand" "i")
4091                        (match_operand:SI 3 "immediate_operand" "i")]
4092                       UNSPEC_VQSHL_N))]
4093   "TARGET_NEON"
4094 {
4095   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4096   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4097 }
4098   [(set_attr "neon_type" "neon_shift_2")]
4099 )
4100
4101 (define_insn "neon_vqshlu_n<mode>"
4102   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4103         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4104                        (match_operand:SI 2 "immediate_operand" "i")
4105                        (match_operand:SI 3 "immediate_operand" "i")]
4106                       UNSPEC_VQSHLU_N))]
4107   "TARGET_NEON"
4108 {
4109   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4110   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4111 }
4112   [(set_attr "neon_type" "neon_shift_2")]
4113 )
4114
4115 (define_insn "neon_vshll_n<mode>"
4116   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
4117         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
4118                            (match_operand:SI 2 "immediate_operand" "i")
4119                            (match_operand:SI 3 "immediate_operand" "i")]
4120                           UNSPEC_VSHLL_N))]
4121   "TARGET_NEON"
4122 {
4123   /* The boundaries are: 0 < imm <= size.  */
4124   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
4125   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
4126 }
4127   [(set_attr "neon_type" "neon_shift_1")]
4128 )
4129
4130 (define_insn "neon_vsra_n<mode>"
4131   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4132         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4133                        (match_operand:VDQIX 2 "s_register_operand" "w")
4134                        (match_operand:SI 3 "immediate_operand" "i")
4135                        (match_operand:SI 4 "immediate_operand" "i")]
4136                       UNSPEC_VSRA_N))]
4137   "TARGET_NEON"
4138 {
4139   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4140   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4141 }
4142   [(set_attr "neon_type" "neon_vsra_vrsra")]
4143 )
4144
4145 (define_insn "neon_vsri_n<mode>"
4146   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4147         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4148                        (match_operand:VDQIX 2 "s_register_operand" "w")
4149                        (match_operand:SI 3 "immediate_operand" "i")]
4150                       UNSPEC_VSRI))]
4151   "TARGET_NEON"
4152 {
4153   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4154   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4155 }
4156   [(set (attr "neon_type")
4157       (if_then_else (match_test "<Is_d_reg>")
4158                     (const_string "neon_shift_1")
4159                     (const_string "neon_shift_3")))]
4160 )
4161
4162 (define_insn "neon_vsli_n<mode>"
4163   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4164         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4165                        (match_operand:VDQIX 2 "s_register_operand" "w")
4166                        (match_operand:SI 3 "immediate_operand" "i")]
4167                       UNSPEC_VSLI))]
4168   "TARGET_NEON"
4169 {
4170   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
4171   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4172 }
4173   [(set (attr "neon_type")
4174       (if_then_else (match_test "<Is_d_reg>")
4175                     (const_string "neon_shift_1")
4176                     (const_string "neon_shift_3")))]
4177 )
4178
4179 (define_insn "neon_vtbl1v8qi"
4180   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4181         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
4182                       (match_operand:V8QI 2 "s_register_operand" "w")]
4183                      UNSPEC_VTBL))]
4184   "TARGET_NEON"
4185   "vtbl.8\t%P0, {%P1}, %P2"
4186   [(set_attr "neon_type" "neon_bp_2cycle")]
4187 )
4188
4189 (define_insn "neon_vtbl2v8qi"
4190   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4191         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
4192                       (match_operand:V8QI 2 "s_register_operand" "w")]
4193                      UNSPEC_VTBL))]
4194   "TARGET_NEON"
4195 {
4196   rtx ops[4];
4197   int tabbase = REGNO (operands[1]);
4198
4199   ops[0] = operands[0];
4200   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4201   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4202   ops[3] = operands[2];
4203   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
4204
4205   return "";
4206 }
4207   [(set_attr "neon_type" "neon_bp_2cycle")]
4208 )
4209
4210 (define_insn "neon_vtbl3v8qi"
4211   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4212         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
4213                       (match_operand:V8QI 2 "s_register_operand" "w")]
4214                      UNSPEC_VTBL))]
4215   "TARGET_NEON"
4216 {
4217   rtx ops[5];
4218   int tabbase = REGNO (operands[1]);
4219
4220   ops[0] = operands[0];
4221   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4222   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4223   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4224   ops[4] = operands[2];
4225   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4226
4227   return "";
4228 }
4229   [(set_attr "neon_type" "neon_bp_3cycle")]
4230 )
4231
4232 (define_insn "neon_vtbl4v8qi"
4233   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4234         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
4235                       (match_operand:V8QI 2 "s_register_operand" "w")]
4236                      UNSPEC_VTBL))]
4237   "TARGET_NEON"
4238 {
4239   rtx ops[6];
4240   int tabbase = REGNO (operands[1]);
4241
4242   ops[0] = operands[0];
4243   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4244   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4245   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4246   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4247   ops[5] = operands[2];
4248   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4249
4250   return "";
4251 }
4252   [(set_attr "neon_type" "neon_bp_3cycle")]
4253 )
4254
4255 ;; These three are used by the vec_perm infrastructure for V16QImode.
4256 (define_insn_and_split "neon_vtbl1v16qi"
4257   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4258         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
4259                        (match_operand:V16QI 2 "s_register_operand" "w")]
4260                       UNSPEC_VTBL))]
4261   "TARGET_NEON"
4262   "#"
4263   "&& reload_completed"
4264   [(const_int 0)]
4265 {
4266   rtx op0, op1, op2, part0, part2;
4267   unsigned ofs;
4268
4269   op0 = operands[0];
4270   op1 = gen_lowpart (TImode, operands[1]);
4271   op2 = operands[2];
4272
4273   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4274   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4275   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4276   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4277
4278   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4279   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4280   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4281   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4282   DONE;
4283 })
4284
4285 (define_insn_and_split "neon_vtbl2v16qi"
4286   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4287         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4288                        (match_operand:V16QI 2 "s_register_operand" "w")]
4289                       UNSPEC_VTBL))]
4290   "TARGET_NEON"
4291   "#"
4292   "&& reload_completed"
4293   [(const_int 0)]
4294 {
4295   rtx op0, op1, op2, part0, part2;
4296   unsigned ofs;
4297
4298   op0 = operands[0];
4299   op1 = operands[1];
4300   op2 = operands[2];
4301
4302   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4303   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4304   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4305   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4306
4307   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4308   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4309   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4310   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4311   DONE;
4312 })
4313
4314 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4315 ;; handle quad-word input modes, producing octa-word output modes.  But
4316 ;; that requires us to add support for octa-word vector modes in moves.
4317 ;; That seems overkill for this one use in vec_perm.
4318 (define_insn_and_split "neon_vcombinev16qi"
4319   [(set (match_operand:OI 0 "s_register_operand" "=w")
4320         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4321                     (match_operand:V16QI 2 "s_register_operand" "w")]
4322                    UNSPEC_VCONCAT))]
4323   "TARGET_NEON"
4324   "#"
4325   "&& reload_completed"
4326   [(const_int 0)]
4327 {
4328   neon_split_vcombine (operands);
4329   DONE;
4330 })
4331
4332 (define_insn "neon_vtbx1v8qi"
4333   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4334         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4335                       (match_operand:V8QI 2 "s_register_operand" "w")
4336                       (match_operand:V8QI 3 "s_register_operand" "w")]
4337                      UNSPEC_VTBX))]
4338   "TARGET_NEON"
4339   "vtbx.8\t%P0, {%P2}, %P3"
4340   [(set_attr "neon_type" "neon_bp_2cycle")]
4341 )
4342
4343 (define_insn "neon_vtbx2v8qi"
4344   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4345         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4346                       (match_operand:TI 2 "s_register_operand" "w")
4347                       (match_operand:V8QI 3 "s_register_operand" "w")]
4348                      UNSPEC_VTBX))]
4349   "TARGET_NEON"
4350 {
4351   rtx ops[4];
4352   int tabbase = REGNO (operands[2]);
4353
4354   ops[0] = operands[0];
4355   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4356   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4357   ops[3] = operands[3];
4358   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4359
4360   return "";
4361 }
4362   [(set_attr "neon_type" "neon_bp_2cycle")]
4363 )
4364
4365 (define_insn "neon_vtbx3v8qi"
4366   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4367         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4368                       (match_operand:EI 2 "s_register_operand" "w")
4369                       (match_operand:V8QI 3 "s_register_operand" "w")]
4370                      UNSPEC_VTBX))]
4371   "TARGET_NEON"
4372 {
4373   rtx ops[5];
4374   int tabbase = REGNO (operands[2]);
4375
4376   ops[0] = operands[0];
4377   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4378   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4379   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4380   ops[4] = operands[3];
4381   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4382
4383   return "";
4384 }
4385   [(set_attr "neon_type" "neon_bp_3cycle")]
4386 )
4387
4388 (define_insn "neon_vtbx4v8qi"
4389   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4390         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4391                       (match_operand:OI 2 "s_register_operand" "w")
4392                       (match_operand:V8QI 3 "s_register_operand" "w")]
4393                      UNSPEC_VTBX))]
4394   "TARGET_NEON"
4395 {
4396   rtx ops[6];
4397   int tabbase = REGNO (operands[2]);
4398
4399   ops[0] = operands[0];
4400   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4401   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4402   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4403   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4404   ops[5] = operands[3];
4405   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4406
4407   return "";
4408 }
4409   [(set_attr "neon_type" "neon_bp_3cycle")]
4410 )
4411
4412 (define_insn "neon_vtrn<mode>_internal"
4413   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4414         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4415                       (match_operand:VDQW 2 "s_register_operand" "w")]
4416                      UNSPEC_VTRN1))
4417    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4418          (unspec:VDQW [(match_dup 1) (match_dup 2)]
4419                      UNSPEC_VTRN2))]
4420   "TARGET_NEON"
4421   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4422   [(set (attr "neon_type")
4423       (if_then_else (match_test "<Is_d_reg>")
4424                     (const_string "neon_bp_simple")
4425                     (const_string "neon_bp_3cycle")))]
4426 )
4427
4428 (define_expand "neon_vtrn<mode>"
4429   [(match_operand:SI 0 "s_register_operand" "r")
4430    (match_operand:VDQW 1 "s_register_operand" "w")
4431    (match_operand:VDQW 2 "s_register_operand" "w")]
4432   "TARGET_NEON"
4433 {
4434   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4435                               operands[0], operands[1], operands[2]);
4436   DONE;
4437 })
4438
4439 (define_insn "neon_vzip<mode>_internal"
4440   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4441         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4442                       (match_operand:VDQW 2 "s_register_operand" "w")]
4443                      UNSPEC_VZIP1))
4444    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4445         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4446                      UNSPEC_VZIP2))]
4447   "TARGET_NEON"
4448   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4449   [(set (attr "neon_type")
4450       (if_then_else (match_test "<Is_d_reg>")
4451                     (const_string "neon_bp_simple")
4452                     (const_string "neon_bp_3cycle")))]
4453 )
4454
4455 (define_expand "neon_vzip<mode>"
4456   [(match_operand:SI 0 "s_register_operand" "r")
4457    (match_operand:VDQW 1 "s_register_operand" "w")
4458    (match_operand:VDQW 2 "s_register_operand" "w")]
4459   "TARGET_NEON"
4460 {
4461   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4462                               operands[0], operands[1], operands[2]);
4463   DONE;
4464 })
4465
4466 (define_insn "neon_vuzp<mode>_internal"
4467   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4468         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4469                       (match_operand:VDQW 2 "s_register_operand" "w")]
4470                      UNSPEC_VUZP1))
4471    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4472         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4473                      UNSPEC_VUZP2))]
4474   "TARGET_NEON"
4475   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4476   [(set (attr "neon_type")
4477       (if_then_else (match_test "<Is_d_reg>")
4478                     (const_string "neon_bp_simple")
4479                     (const_string "neon_bp_3cycle")))]
4480 )
4481
4482 (define_expand "neon_vuzp<mode>"
4483   [(match_operand:SI 0 "s_register_operand" "r")
4484    (match_operand:VDQW 1 "s_register_operand" "w")
4485    (match_operand:VDQW 2 "s_register_operand" "w")]
4486   "TARGET_NEON"
4487 {
4488   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4489                               operands[0], operands[1], operands[2]);
4490   DONE;
4491 })
4492
4493 (define_expand "neon_vreinterpretv8qi<mode>"
4494   [(match_operand:V8QI 0 "s_register_operand" "")
4495    (match_operand:VDX 1 "s_register_operand" "")]
4496   "TARGET_NEON"
4497 {
4498   neon_reinterpret (operands[0], operands[1]);
4499   DONE;
4500 })
4501
4502 (define_expand "neon_vreinterpretv4hi<mode>"
4503   [(match_operand:V4HI 0 "s_register_operand" "")
4504    (match_operand:VDX 1 "s_register_operand" "")]
4505   "TARGET_NEON"
4506 {
4507   neon_reinterpret (operands[0], operands[1]);
4508   DONE;
4509 })
4510
4511 (define_expand "neon_vreinterpretv2si<mode>"
4512   [(match_operand:V2SI 0 "s_register_operand" "")
4513    (match_operand:VDX 1 "s_register_operand" "")]
4514   "TARGET_NEON"
4515 {
4516   neon_reinterpret (operands[0], operands[1]);
4517   DONE;
4518 })
4519
4520 (define_expand "neon_vreinterpretv2sf<mode>"
4521   [(match_operand:V2SF 0 "s_register_operand" "")
4522    (match_operand:VDX 1 "s_register_operand" "")]
4523   "TARGET_NEON"
4524 {
4525   neon_reinterpret (operands[0], operands[1]);
4526   DONE;
4527 })
4528
4529 (define_expand "neon_vreinterpretdi<mode>"
4530   [(match_operand:DI 0 "s_register_operand" "")
4531    (match_operand:VDX 1 "s_register_operand" "")]
4532   "TARGET_NEON"
4533 {
4534   neon_reinterpret (operands[0], operands[1]);
4535   DONE;
4536 })
4537
4538 (define_expand "neon_vreinterpretv16qi<mode>"
4539   [(match_operand:V16QI 0 "s_register_operand" "")
4540    (match_operand:VQX 1 "s_register_operand" "")]
4541   "TARGET_NEON"
4542 {
4543   neon_reinterpret (operands[0], operands[1]);
4544   DONE;
4545 })
4546
4547 (define_expand "neon_vreinterpretv8hi<mode>"
4548   [(match_operand:V8HI 0 "s_register_operand" "")
4549    (match_operand:VQX 1 "s_register_operand" "")]
4550   "TARGET_NEON"
4551 {
4552   neon_reinterpret (operands[0], operands[1]);
4553   DONE;
4554 })
4555
4556 (define_expand "neon_vreinterpretv4si<mode>"
4557   [(match_operand:V4SI 0 "s_register_operand" "")
4558    (match_operand:VQX 1 "s_register_operand" "")]
4559   "TARGET_NEON"
4560 {
4561   neon_reinterpret (operands[0], operands[1]);
4562   DONE;
4563 })
4564
4565 (define_expand "neon_vreinterpretv4sf<mode>"
4566   [(match_operand:V4SF 0 "s_register_operand" "")
4567    (match_operand:VQX 1 "s_register_operand" "")]
4568   "TARGET_NEON"
4569 {
4570   neon_reinterpret (operands[0], operands[1]);
4571   DONE;
4572 })
4573
4574 (define_expand "neon_vreinterpretv2di<mode>"
4575   [(match_operand:V2DI 0 "s_register_operand" "")
4576    (match_operand:VQX 1 "s_register_operand" "")]
4577   "TARGET_NEON"
4578 {
4579   neon_reinterpret (operands[0], operands[1]);
4580   DONE;
4581 })
4582
4583 (define_expand "vec_load_lanes<mode><mode>"
4584   [(set (match_operand:VDQX 0 "s_register_operand")
4585         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4586                      UNSPEC_VLD1))]
4587   "TARGET_NEON")
4588
4589 (define_insn "neon_vld1<mode>"
4590   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4591         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4592                     UNSPEC_VLD1))]
4593   "TARGET_NEON"
4594   "vld1.<V_sz_elem>\t%h0, %A1"
4595   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4596 )
4597
4598 (define_insn "neon_vld1_lane<mode>"
4599   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4600         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4601                      (match_operand:VDX 2 "s_register_operand" "0")
4602                      (match_operand:SI 3 "immediate_operand" "i")]
4603                     UNSPEC_VLD1_LANE))]
4604   "TARGET_NEON"
4605 {
4606   HOST_WIDE_INT lane = INTVAL (operands[3]);
4607   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4608   if (lane < 0 || lane >= max)
4609     error ("lane out of range");
4610   if (max == 1)
4611     return "vld1.<V_sz_elem>\t%P0, %A1";
4612   else
4613     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4614 }
4615   [(set (attr "neon_type")
4616       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4617                     (const_string "neon_vld1_1_2_regs")
4618                     (const_string "neon_vld1_vld2_lane")))]
4619 )
4620
4621 (define_insn "neon_vld1_lane<mode>"
4622   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4623         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4624                      (match_operand:VQX 2 "s_register_operand" "0")
4625                      (match_operand:SI 3 "immediate_operand" "i")]
4626                     UNSPEC_VLD1_LANE))]
4627   "TARGET_NEON"
4628 {
4629   HOST_WIDE_INT lane = INTVAL (operands[3]);
4630   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4631   int regno = REGNO (operands[0]);
4632   if (lane < 0 || lane >= max)
4633     error ("lane out of range");
4634   else if (lane >= max / 2)
4635     {
4636       lane -= max / 2;
4637       regno += 2;
4638       operands[3] = GEN_INT (lane);
4639     }
4640   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4641   if (max == 2)
4642     return "vld1.<V_sz_elem>\t%P0, %A1";
4643   else
4644     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4645 }
4646   [(set (attr "neon_type")
4647       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4648                     (const_string "neon_vld1_1_2_regs")
4649                     (const_string "neon_vld1_vld2_lane")))]
4650 )
4651
4652 (define_insn "neon_vld1_dup<mode>"
4653   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4654         (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4655   "TARGET_NEON"
4656 {
4657   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4658     return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4659   else
4660     return "vld1.<V_sz_elem>\t%h0, %A1";
4661 }
4662   [(set (attr "neon_type")
4663       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4664                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4665                     (const_string "neon_vld1_1_2_regs")))]
4666 )
4667
4668 (define_insn "neon_vld1_dup<mode>"
4669   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4670         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4671   "TARGET_NEON"
4672 {
4673   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4674 }
4675   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4676 )
4677
4678 (define_insn_and_split "neon_vld1_dupv2di"
4679    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4680     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4681    "TARGET_NEON"
4682    "#"
4683    "&& reload_completed"
4684    [(const_int 0)]
4685    {
4686     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4687     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4688     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4689     DONE;
4690     }
4691   [(set_attr "length" "8")
4692    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4693 )
4694
4695 (define_expand "vec_store_lanes<mode><mode>"
4696   [(set (match_operand:VDQX 0 "neon_struct_operand")
4697         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4698                      UNSPEC_VST1))]
4699   "TARGET_NEON")
4700
4701 (define_insn "neon_vst1<mode>"
4702   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4703         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4704                      UNSPEC_VST1))]
4705   "TARGET_NEON"
4706   "vst1.<V_sz_elem>\t%h1, %A0"
4707   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4708
4709 (define_insn "neon_vst1_lane<mode>"
4710   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4711         (unspec:<V_elem>
4712           [(match_operand:VDX 1 "s_register_operand" "w")
4713            (match_operand:SI 2 "immediate_operand" "i")]
4714           UNSPEC_VST1_LANE))]
4715   "TARGET_NEON"
4716 {
4717   HOST_WIDE_INT lane = INTVAL (operands[2]);
4718   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4719   if (lane < 0 || lane >= max)
4720     error ("lane out of range");
4721   if (max == 1)
4722     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4723   else
4724     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4725 }
4726   [(set (attr "neon_type")
4727       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4728                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4729                     (const_string "neon_vst1_vst2_lane")))])
4730
4731 (define_insn "neon_vst1_lane<mode>"
4732   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4733         (unspec:<V_elem>
4734           [(match_operand:VQX 1 "s_register_operand" "w")
4735            (match_operand:SI 2 "immediate_operand" "i")]
4736           UNSPEC_VST1_LANE))]
4737   "TARGET_NEON"
4738 {
4739   HOST_WIDE_INT lane = INTVAL (operands[2]);
4740   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4741   int regno = REGNO (operands[1]);
4742   if (lane < 0 || lane >= max)
4743     error ("lane out of range");
4744   else if (lane >= max / 2)
4745     {
4746       lane -= max / 2;
4747       regno += 2;
4748       operands[2] = GEN_INT (lane);
4749     }
4750   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4751   if (max == 2)
4752     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4753   else
4754     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4755 }
4756   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4757 )
4758
4759 (define_expand "vec_load_lanesti<mode>"
4760   [(set (match_operand:TI 0 "s_register_operand")
4761         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4762                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4763                    UNSPEC_VLD2))]
4764   "TARGET_NEON")
4765
4766 (define_insn "neon_vld2<mode>"
4767   [(set (match_operand:TI 0 "s_register_operand" "=w")
4768         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4769                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4770                    UNSPEC_VLD2))]
4771   "TARGET_NEON"
4772 {
4773   if (<V_sz_elem> == 64)
4774     return "vld1.64\t%h0, %A1";
4775   else
4776     return "vld2.<V_sz_elem>\t%h0, %A1";
4777 }
4778   [(set (attr "neon_type")
4779       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4780                     (const_string "neon_vld1_1_2_regs")
4781                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4782 )
4783
4784 (define_expand "vec_load_lanesoi<mode>"
4785   [(set (match_operand:OI 0 "s_register_operand")
4786         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4787                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4788                    UNSPEC_VLD2))]
4789   "TARGET_NEON")
4790
4791 (define_insn "neon_vld2<mode>"
4792   [(set (match_operand:OI 0 "s_register_operand" "=w")
4793         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4794                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795                    UNSPEC_VLD2))]
4796   "TARGET_NEON"
4797   "vld2.<V_sz_elem>\t%h0, %A1"
4798   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4799
4800 (define_insn "neon_vld2_lane<mode>"
4801   [(set (match_operand:TI 0 "s_register_operand" "=w")
4802         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4803                     (match_operand:TI 2 "s_register_operand" "0")
4804                     (match_operand:SI 3 "immediate_operand" "i")
4805                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4806                    UNSPEC_VLD2_LANE))]
4807   "TARGET_NEON"
4808 {
4809   HOST_WIDE_INT lane = INTVAL (operands[3]);
4810   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4811   int regno = REGNO (operands[0]);
4812   rtx ops[4];
4813   if (lane < 0 || lane >= max)
4814     error ("lane out of range");
4815   ops[0] = gen_rtx_REG (DImode, regno);
4816   ops[1] = gen_rtx_REG (DImode, regno + 2);
4817   ops[2] = operands[1];
4818   ops[3] = operands[3];
4819   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4820   return "";
4821 }
4822   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4823 )
4824
4825 (define_insn "neon_vld2_lane<mode>"
4826   [(set (match_operand:OI 0 "s_register_operand" "=w")
4827         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4828                     (match_operand:OI 2 "s_register_operand" "0")
4829                     (match_operand:SI 3 "immediate_operand" "i")
4830                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4831                    UNSPEC_VLD2_LANE))]
4832   "TARGET_NEON"
4833 {
4834   HOST_WIDE_INT lane = INTVAL (operands[3]);
4835   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4836   int regno = REGNO (operands[0]);
4837   rtx ops[4];
4838   if (lane < 0 || lane >= max)
4839     error ("lane out of range");
4840   else if (lane >= max / 2)
4841     {
4842       lane -= max / 2;
4843       regno += 2;
4844     }
4845   ops[0] = gen_rtx_REG (DImode, regno);
4846   ops[1] = gen_rtx_REG (DImode, regno + 4);
4847   ops[2] = operands[1];
4848   ops[3] = GEN_INT (lane);
4849   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4850   return "";
4851 }
4852   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4853 )
4854
4855 (define_insn "neon_vld2_dup<mode>"
4856   [(set (match_operand:TI 0 "s_register_operand" "=w")
4857         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4858                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4859                    UNSPEC_VLD2_DUP))]
4860   "TARGET_NEON"
4861 {
4862   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4863     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4864   else
4865     return "vld1.<V_sz_elem>\t%h0, %A1";
4866 }
4867   [(set (attr "neon_type")
4868       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4869                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4870                     (const_string "neon_vld1_1_2_regs")))]
4871 )
4872
4873 (define_expand "vec_store_lanesti<mode>"
4874   [(set (match_operand:TI 0 "neon_struct_operand")
4875         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4876                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4877                    UNSPEC_VST2))]
4878   "TARGET_NEON")
4879
4880 (define_insn "neon_vst2<mode>"
4881   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4882         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4883                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4884                    UNSPEC_VST2))]
4885   "TARGET_NEON"
4886 {
4887   if (<V_sz_elem> == 64)
4888     return "vst1.64\t%h1, %A0";
4889   else
4890     return "vst2.<V_sz_elem>\t%h1, %A0";
4891 }
4892   [(set (attr "neon_type")
4893       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4894                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4895                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4896 )
4897
4898 (define_expand "vec_store_lanesoi<mode>"
4899   [(set (match_operand:OI 0 "neon_struct_operand")
4900         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4901                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4902                    UNSPEC_VST2))]
4903   "TARGET_NEON")
4904
4905 (define_insn "neon_vst2<mode>"
4906   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4907         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4908                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4909                    UNSPEC_VST2))]
4910   "TARGET_NEON"
4911   "vst2.<V_sz_elem>\t%h1, %A0"
4912   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4913 )
4914
4915 (define_insn "neon_vst2_lane<mode>"
4916   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4917         (unspec:<V_two_elem>
4918           [(match_operand:TI 1 "s_register_operand" "w")
4919            (match_operand:SI 2 "immediate_operand" "i")
4920            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4921           UNSPEC_VST2_LANE))]
4922   "TARGET_NEON"
4923 {
4924   HOST_WIDE_INT lane = INTVAL (operands[2]);
4925   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4926   int regno = REGNO (operands[1]);
4927   rtx ops[4];
4928   if (lane < 0 || lane >= max)
4929     error ("lane out of range");
4930   ops[0] = operands[0];
4931   ops[1] = gen_rtx_REG (DImode, regno);
4932   ops[2] = gen_rtx_REG (DImode, regno + 2);
4933   ops[3] = operands[2];
4934   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4935   return "";
4936 }
4937   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4938 )
4939
4940 (define_insn "neon_vst2_lane<mode>"
4941   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4942         (unspec:<V_two_elem>
4943            [(match_operand:OI 1 "s_register_operand" "w")
4944             (match_operand:SI 2 "immediate_operand" "i")
4945             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4946            UNSPEC_VST2_LANE))]
4947   "TARGET_NEON"
4948 {
4949   HOST_WIDE_INT lane = INTVAL (operands[2]);
4950   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4951   int regno = REGNO (operands[1]);
4952   rtx ops[4];
4953   if (lane < 0 || lane >= max)
4954     error ("lane out of range");
4955   else if (lane >= max / 2)
4956     {
4957       lane -= max / 2;
4958       regno += 2;
4959     }
4960   ops[0] = operands[0];
4961   ops[1] = gen_rtx_REG (DImode, regno);
4962   ops[2] = gen_rtx_REG (DImode, regno + 4);
4963   ops[3] = GEN_INT (lane);
4964   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4965   return "";
4966 }
4967   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4968 )
4969
4970 (define_expand "vec_load_lanesei<mode>"
4971   [(set (match_operand:EI 0 "s_register_operand")
4972         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4973                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4974                    UNSPEC_VLD3))]
4975   "TARGET_NEON")
4976
4977 (define_insn "neon_vld3<mode>"
4978   [(set (match_operand:EI 0 "s_register_operand" "=w")
4979         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4980                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4981                    UNSPEC_VLD3))]
4982   "TARGET_NEON"
4983 {
4984   if (<V_sz_elem> == 64)
4985     return "vld1.64\t%h0, %A1";
4986   else
4987     return "vld3.<V_sz_elem>\t%h0, %A1";
4988 }
4989   [(set (attr "neon_type")
4990       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4991                     (const_string "neon_vld1_1_2_regs")
4992                     (const_string "neon_vld3_vld4")))]
4993 )
4994
4995 (define_expand "vec_load_lanesci<mode>"
4996   [(match_operand:CI 0 "s_register_operand")
4997    (match_operand:CI 1 "neon_struct_operand")
4998    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4999   "TARGET_NEON"
5000 {
5001   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
5002   DONE;
5003 })
5004
5005 (define_expand "neon_vld3<mode>"
5006   [(match_operand:CI 0 "s_register_operand")
5007    (match_operand:CI 1 "neon_struct_operand")
5008    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5009   "TARGET_NEON"
5010 {
5011   rtx mem;
5012
5013   mem = adjust_address (operands[1], EImode, 0);
5014   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
5015   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
5016   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
5017   DONE;
5018 })
5019
5020 (define_insn "neon_vld3qa<mode>"
5021   [(set (match_operand:CI 0 "s_register_operand" "=w")
5022         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
5023                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5024                    UNSPEC_VLD3A))]
5025   "TARGET_NEON"
5026 {
5027   int regno = REGNO (operands[0]);
5028   rtx ops[4];
5029   ops[0] = gen_rtx_REG (DImode, regno);
5030   ops[1] = gen_rtx_REG (DImode, regno + 4);
5031   ops[2] = gen_rtx_REG (DImode, regno + 8);
5032   ops[3] = operands[1];
5033   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
5034   return "";
5035 }
5036   [(set_attr "neon_type" "neon_vld3_vld4")]
5037 )
5038
5039 (define_insn "neon_vld3qb<mode>"
5040   [(set (match_operand:CI 0 "s_register_operand" "=w")
5041         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
5042                     (match_operand:CI 2 "s_register_operand" "0")
5043                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5044                    UNSPEC_VLD3B))]
5045   "TARGET_NEON"
5046 {
5047   int regno = REGNO (operands[0]);
5048   rtx ops[4];
5049   ops[0] = gen_rtx_REG (DImode, regno + 2);
5050   ops[1] = gen_rtx_REG (DImode, regno + 6);
5051   ops[2] = gen_rtx_REG (DImode, regno + 10);
5052   ops[3] = operands[1];
5053   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
5054   return "";
5055 }
5056   [(set_attr "neon_type" "neon_vld3_vld4")]
5057 )
5058
5059 (define_insn "neon_vld3_lane<mode>"
5060   [(set (match_operand:EI 0 "s_register_operand" "=w")
5061         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5062                     (match_operand:EI 2 "s_register_operand" "0")
5063                     (match_operand:SI 3 "immediate_operand" "i")
5064                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5065                    UNSPEC_VLD3_LANE))]
5066   "TARGET_NEON"
5067 {
5068   HOST_WIDE_INT lane = INTVAL (operands[3]);
5069   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5070   int regno = REGNO (operands[0]);
5071   rtx ops[5];
5072   if (lane < 0 || lane >= max)
5073     error ("lane out of range");
5074   ops[0] = gen_rtx_REG (DImode, regno);
5075   ops[1] = gen_rtx_REG (DImode, regno + 2);
5076   ops[2] = gen_rtx_REG (DImode, regno + 4);
5077   ops[3] = operands[1];
5078   ops[4] = operands[3];
5079   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5080                    ops);
5081   return "";
5082 }
5083   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5084 )
5085
5086 (define_insn "neon_vld3_lane<mode>"
5087   [(set (match_operand:CI 0 "s_register_operand" "=w")
5088         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5089                     (match_operand:CI 2 "s_register_operand" "0")
5090                     (match_operand:SI 3 "immediate_operand" "i")
5091                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5092                    UNSPEC_VLD3_LANE))]
5093   "TARGET_NEON"
5094 {
5095   HOST_WIDE_INT lane = INTVAL (operands[3]);
5096   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5097   int regno = REGNO (operands[0]);
5098   rtx ops[5];
5099   if (lane < 0 || lane >= max)
5100     error ("lane out of range");
5101   else if (lane >= max / 2)
5102     {
5103       lane -= max / 2;
5104       regno += 2;
5105     }
5106   ops[0] = gen_rtx_REG (DImode, regno);
5107   ops[1] = gen_rtx_REG (DImode, regno + 4);
5108   ops[2] = gen_rtx_REG (DImode, regno + 8);
5109   ops[3] = operands[1];
5110   ops[4] = GEN_INT (lane);
5111   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5112                    ops);
5113   return "";
5114 }
5115   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5116 )
5117
5118 (define_insn "neon_vld3_dup<mode>"
5119   [(set (match_operand:EI 0 "s_register_operand" "=w")
5120         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5121                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5122                    UNSPEC_VLD3_DUP))]
5123   "TARGET_NEON"
5124 {
5125   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5126     {
5127       int regno = REGNO (operands[0]);
5128       rtx ops[4];
5129       ops[0] = gen_rtx_REG (DImode, regno);
5130       ops[1] = gen_rtx_REG (DImode, regno + 2);
5131       ops[2] = gen_rtx_REG (DImode, regno + 4);
5132       ops[3] = operands[1];
5133       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
5134       return "";
5135     }
5136   else
5137     return "vld1.<V_sz_elem>\t%h0, %A1";
5138 }
5139   [(set (attr "neon_type")
5140       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5141                     (const_string "neon_vld3_vld4_all_lanes")
5142                     (const_string "neon_vld1_1_2_regs")))])
5143
5144 (define_expand "vec_store_lanesei<mode>"
5145   [(set (match_operand:EI 0 "neon_struct_operand")
5146         (unspec:EI [(match_operand:EI 1 "s_register_operand")
5147                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5148                    UNSPEC_VST3))]
5149   "TARGET_NEON")
5150
5151 (define_insn "neon_vst3<mode>"
5152   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5153         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
5154                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5155                    UNSPEC_VST3))]
5156   "TARGET_NEON"
5157 {
5158   if (<V_sz_elem> == 64)
5159     return "vst1.64\t%h1, %A0";
5160   else
5161     return "vst3.<V_sz_elem>\t%h1, %A0";
5162 }
5163   [(set (attr "neon_type")
5164       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5165                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5166                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
5167
5168 (define_expand "vec_store_lanesci<mode>"
5169   [(match_operand:CI 0 "neon_struct_operand")
5170    (match_operand:CI 1 "s_register_operand")
5171    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5172   "TARGET_NEON"
5173 {
5174   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
5175   DONE;
5176 })
5177
5178 (define_expand "neon_vst3<mode>"
5179   [(match_operand:CI 0 "neon_struct_operand")
5180    (match_operand:CI 1 "s_register_operand")
5181    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5182   "TARGET_NEON"
5183 {
5184   rtx mem;
5185
5186   mem = adjust_address (operands[0], EImode, 0);
5187   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
5188   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
5189   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
5190   DONE;
5191 })
5192
5193 (define_insn "neon_vst3qa<mode>"
5194   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5195         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5196                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5197                    UNSPEC_VST3A))]
5198   "TARGET_NEON"
5199 {
5200   int regno = REGNO (operands[1]);
5201   rtx ops[4];
5202   ops[0] = operands[0];
5203   ops[1] = gen_rtx_REG (DImode, regno);
5204   ops[2] = gen_rtx_REG (DImode, regno + 4);
5205   ops[3] = gen_rtx_REG (DImode, regno + 8);
5206   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5207   return "";
5208 }
5209   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5210 )
5211
5212 (define_insn "neon_vst3qb<mode>"
5213   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5214         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5215                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5216                    UNSPEC_VST3B))]
5217   "TARGET_NEON"
5218 {
5219   int regno = REGNO (operands[1]);
5220   rtx ops[4];
5221   ops[0] = operands[0];
5222   ops[1] = gen_rtx_REG (DImode, regno + 2);
5223   ops[2] = gen_rtx_REG (DImode, regno + 6);
5224   ops[3] = gen_rtx_REG (DImode, regno + 10);
5225   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5226   return "";
5227 }
5228   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5229 )
5230
5231 (define_insn "neon_vst3_lane<mode>"
5232   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5233         (unspec:<V_three_elem>
5234            [(match_operand:EI 1 "s_register_operand" "w")
5235             (match_operand:SI 2 "immediate_operand" "i")
5236             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5237            UNSPEC_VST3_LANE))]
5238   "TARGET_NEON"
5239 {
5240   HOST_WIDE_INT lane = INTVAL (operands[2]);
5241   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5242   int regno = REGNO (operands[1]);
5243   rtx ops[5];
5244   if (lane < 0 || lane >= max)
5245     error ("lane out of range");
5246   ops[0] = operands[0];
5247   ops[1] = gen_rtx_REG (DImode, regno);
5248   ops[2] = gen_rtx_REG (DImode, regno + 2);
5249   ops[3] = gen_rtx_REG (DImode, regno + 4);
5250   ops[4] = operands[2];
5251   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5252                    ops);
5253   return "";
5254 }
5255   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5256 )
5257
5258 (define_insn "neon_vst3_lane<mode>"
5259   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5260         (unspec:<V_three_elem>
5261            [(match_operand:CI 1 "s_register_operand" "w")
5262             (match_operand:SI 2 "immediate_operand" "i")
5263             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5264            UNSPEC_VST3_LANE))]
5265   "TARGET_NEON"
5266 {
5267   HOST_WIDE_INT lane = INTVAL (operands[2]);
5268   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5269   int regno = REGNO (operands[1]);
5270   rtx ops[5];
5271   if (lane < 0 || lane >= max)
5272     error ("lane out of range");
5273   else if (lane >= max / 2)
5274     {
5275       lane -= max / 2;
5276       regno += 2;
5277     }
5278   ops[0] = operands[0];
5279   ops[1] = gen_rtx_REG (DImode, regno);
5280   ops[2] = gen_rtx_REG (DImode, regno + 4);
5281   ops[3] = gen_rtx_REG (DImode, regno + 8);
5282   ops[4] = GEN_INT (lane);
5283   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5284                    ops);
5285   return "";
5286 }
5287 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5288
5289 (define_expand "vec_load_lanesoi<mode>"
5290   [(set (match_operand:OI 0 "s_register_operand")
5291         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5292                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5293                    UNSPEC_VLD4))]
5294   "TARGET_NEON")
5295
5296 (define_insn "neon_vld4<mode>"
5297   [(set (match_operand:OI 0 "s_register_operand" "=w")
5298         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5299                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5300                    UNSPEC_VLD4))]
5301   "TARGET_NEON"
5302 {
5303   if (<V_sz_elem> == 64)
5304     return "vld1.64\t%h0, %A1";
5305   else
5306     return "vld4.<V_sz_elem>\t%h0, %A1";
5307 }
5308   [(set (attr "neon_type")
5309       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5310                     (const_string "neon_vld1_1_2_regs")
5311                     (const_string "neon_vld3_vld4")))]
5312 )
5313
5314 (define_expand "vec_load_lanesxi<mode>"
5315   [(match_operand:XI 0 "s_register_operand")
5316    (match_operand:XI 1 "neon_struct_operand")
5317    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5318   "TARGET_NEON"
5319 {
5320   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5321   DONE;
5322 })
5323
5324 (define_expand "neon_vld4<mode>"
5325   [(match_operand:XI 0 "s_register_operand")
5326    (match_operand:XI 1 "neon_struct_operand")
5327    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5328   "TARGET_NEON"
5329 {
5330   rtx mem;
5331
5332   mem = adjust_address (operands[1], OImode, 0);
5333   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5334   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5335   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5336   DONE;
5337 })
5338
5339 (define_insn "neon_vld4qa<mode>"
5340   [(set (match_operand:XI 0 "s_register_operand" "=w")
5341         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5342                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5343                    UNSPEC_VLD4A))]
5344   "TARGET_NEON"
5345 {
5346   int regno = REGNO (operands[0]);
5347   rtx ops[5];
5348   ops[0] = gen_rtx_REG (DImode, regno);
5349   ops[1] = gen_rtx_REG (DImode, regno + 4);
5350   ops[2] = gen_rtx_REG (DImode, regno + 8);
5351   ops[3] = gen_rtx_REG (DImode, regno + 12);
5352   ops[4] = operands[1];
5353   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5354   return "";
5355 }
5356   [(set_attr "neon_type" "neon_vld3_vld4")]
5357 )
5358
5359 (define_insn "neon_vld4qb<mode>"
5360   [(set (match_operand:XI 0 "s_register_operand" "=w")
5361         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5362                     (match_operand:XI 2 "s_register_operand" "0")
5363                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5364                    UNSPEC_VLD4B))]
5365   "TARGET_NEON"
5366 {
5367   int regno = REGNO (operands[0]);
5368   rtx ops[5];
5369   ops[0] = gen_rtx_REG (DImode, regno + 2);
5370   ops[1] = gen_rtx_REG (DImode, regno + 6);
5371   ops[2] = gen_rtx_REG (DImode, regno + 10);
5372   ops[3] = gen_rtx_REG (DImode, regno + 14);
5373   ops[4] = operands[1];
5374   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5375   return "";
5376 }
5377   [(set_attr "neon_type" "neon_vld3_vld4")]
5378 )
5379
5380 (define_insn "neon_vld4_lane<mode>"
5381   [(set (match_operand:OI 0 "s_register_operand" "=w")
5382         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5383                     (match_operand:OI 2 "s_register_operand" "0")
5384                     (match_operand:SI 3 "immediate_operand" "i")
5385                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5386                    UNSPEC_VLD4_LANE))]
5387   "TARGET_NEON"
5388 {
5389   HOST_WIDE_INT lane = INTVAL (operands[3]);
5390   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5391   int regno = REGNO (operands[0]);
5392   rtx ops[6];
5393   if (lane < 0 || lane >= max)
5394     error ("lane out of range");
5395   ops[0] = gen_rtx_REG (DImode, regno);
5396   ops[1] = gen_rtx_REG (DImode, regno + 2);
5397   ops[2] = gen_rtx_REG (DImode, regno + 4);
5398   ops[3] = gen_rtx_REG (DImode, regno + 6);
5399   ops[4] = operands[1];
5400   ops[5] = operands[3];
5401   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5402                    ops);
5403   return "";
5404 }
5405   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5406 )
5407
5408 (define_insn "neon_vld4_lane<mode>"
5409   [(set (match_operand:XI 0 "s_register_operand" "=w")
5410         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5411                     (match_operand:XI 2 "s_register_operand" "0")
5412                     (match_operand:SI 3 "immediate_operand" "i")
5413                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5414                    UNSPEC_VLD4_LANE))]
5415   "TARGET_NEON"
5416 {
5417   HOST_WIDE_INT lane = INTVAL (operands[3]);
5418   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5419   int regno = REGNO (operands[0]);
5420   rtx ops[6];
5421   if (lane < 0 || lane >= max)
5422     error ("lane out of range");
5423   else if (lane >= max / 2)
5424     {
5425       lane -= max / 2;
5426       regno += 2;
5427     }
5428   ops[0] = gen_rtx_REG (DImode, regno);
5429   ops[1] = gen_rtx_REG (DImode, regno + 4);
5430   ops[2] = gen_rtx_REG (DImode, regno + 8);
5431   ops[3] = gen_rtx_REG (DImode, regno + 12);
5432   ops[4] = operands[1];
5433   ops[5] = GEN_INT (lane);
5434   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5435                    ops);
5436   return "";
5437 }
5438   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5439 )
5440
5441 (define_insn "neon_vld4_dup<mode>"
5442   [(set (match_operand:OI 0 "s_register_operand" "=w")
5443         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5444                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5445                    UNSPEC_VLD4_DUP))]
5446   "TARGET_NEON"
5447 {
5448   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5449     {
5450       int regno = REGNO (operands[0]);
5451       rtx ops[5];
5452       ops[0] = gen_rtx_REG (DImode, regno);
5453       ops[1] = gen_rtx_REG (DImode, regno + 2);
5454       ops[2] = gen_rtx_REG (DImode, regno + 4);
5455       ops[3] = gen_rtx_REG (DImode, regno + 6);
5456       ops[4] = operands[1];
5457       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5458                        ops);
5459       return "";
5460     }
5461   else
5462     return "vld1.<V_sz_elem>\t%h0, %A1";
5463 }
5464   [(set (attr "neon_type")
5465       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5466                     (const_string "neon_vld3_vld4_all_lanes")
5467                     (const_string "neon_vld1_1_2_regs")))]
5468 )
5469
5470 (define_expand "vec_store_lanesoi<mode>"
5471   [(set (match_operand:OI 0 "neon_struct_operand")
5472         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5473                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5474                    UNSPEC_VST4))]
5475   "TARGET_NEON")
5476
5477 (define_insn "neon_vst4<mode>"
5478   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5479         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5480                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5481                    UNSPEC_VST4))]
5482   "TARGET_NEON"
5483 {
5484   if (<V_sz_elem> == 64)
5485     return "vst1.64\t%h1, %A0";
5486   else
5487     return "vst4.<V_sz_elem>\t%h1, %A0";
5488 }
5489   [(set (attr "neon_type")
5490       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5491                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5492                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5493 )
5494
5495 (define_expand "vec_store_lanesxi<mode>"
5496   [(match_operand:XI 0 "neon_struct_operand")
5497    (match_operand:XI 1 "s_register_operand")
5498    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5499   "TARGET_NEON"
5500 {
5501   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5502   DONE;
5503 })
5504
5505 (define_expand "neon_vst4<mode>"
5506   [(match_operand:XI 0 "neon_struct_operand")
5507    (match_operand:XI 1 "s_register_operand")
5508    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5509   "TARGET_NEON"
5510 {
5511   rtx mem;
5512
5513   mem = adjust_address (operands[0], OImode, 0);
5514   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5515   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5516   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5517   DONE;
5518 })
5519
5520 (define_insn "neon_vst4qa<mode>"
5521   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5522         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5523                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5524                    UNSPEC_VST4A))]
5525   "TARGET_NEON"
5526 {
5527   int regno = REGNO (operands[1]);
5528   rtx ops[5];
5529   ops[0] = operands[0];
5530   ops[1] = gen_rtx_REG (DImode, regno);
5531   ops[2] = gen_rtx_REG (DImode, regno + 4);
5532   ops[3] = gen_rtx_REG (DImode, regno + 8);
5533   ops[4] = gen_rtx_REG (DImode, regno + 12);
5534   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5535   return "";
5536 }
5537   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5538 )
5539
5540 (define_insn "neon_vst4qb<mode>"
5541   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5542         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5543                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5544                    UNSPEC_VST4B))]
5545   "TARGET_NEON"
5546 {
5547   int regno = REGNO (operands[1]);
5548   rtx ops[5];
5549   ops[0] = operands[0];
5550   ops[1] = gen_rtx_REG (DImode, regno + 2);
5551   ops[2] = gen_rtx_REG (DImode, regno + 6);
5552   ops[3] = gen_rtx_REG (DImode, regno + 10);
5553   ops[4] = gen_rtx_REG (DImode, regno + 14);
5554   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5555   return "";
5556 }
5557   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5558 )
5559
5560 (define_insn "neon_vst4_lane<mode>"
5561   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5562         (unspec:<V_four_elem>
5563            [(match_operand:OI 1 "s_register_operand" "w")
5564             (match_operand:SI 2 "immediate_operand" "i")
5565             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5566            UNSPEC_VST4_LANE))]
5567   "TARGET_NEON"
5568 {
5569   HOST_WIDE_INT lane = INTVAL (operands[2]);
5570   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5571   int regno = REGNO (operands[1]);
5572   rtx ops[6];
5573   if (lane < 0 || lane >= max)
5574     error ("lane out of range");
5575   ops[0] = operands[0];
5576   ops[1] = gen_rtx_REG (DImode, regno);
5577   ops[2] = gen_rtx_REG (DImode, regno + 2);
5578   ops[3] = gen_rtx_REG (DImode, regno + 4);
5579   ops[4] = gen_rtx_REG (DImode, regno + 6);
5580   ops[5] = operands[2];
5581   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5582                    ops);
5583   return "";
5584 }
5585   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5586 )
5587
5588 (define_insn "neon_vst4_lane<mode>"
5589   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5590         (unspec:<V_four_elem>
5591            [(match_operand:XI 1 "s_register_operand" "w")
5592             (match_operand:SI 2 "immediate_operand" "i")
5593             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5594            UNSPEC_VST4_LANE))]
5595   "TARGET_NEON"
5596 {
5597   HOST_WIDE_INT lane = INTVAL (operands[2]);
5598   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5599   int regno = REGNO (operands[1]);
5600   rtx ops[6];
5601   if (lane < 0 || lane >= max)
5602     error ("lane out of range");
5603   else if (lane >= max / 2)
5604     {
5605       lane -= max / 2;
5606       regno += 2;
5607     }
5608   ops[0] = operands[0];
5609   ops[1] = gen_rtx_REG (DImode, regno);
5610   ops[2] = gen_rtx_REG (DImode, regno + 4);
5611   ops[3] = gen_rtx_REG (DImode, regno + 8);
5612   ops[4] = gen_rtx_REG (DImode, regno + 12);
5613   ops[5] = GEN_INT (lane);
5614   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5615                    ops);
5616   return "";
5617 }
5618   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5619 )
5620
5621 (define_expand "neon_vand<mode>"
5622   [(match_operand:VDQX 0 "s_register_operand" "")
5623    (match_operand:VDQX 1 "s_register_operand" "")
5624    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5625    (match_operand:SI 3 "immediate_operand" "")]
5626   "TARGET_NEON"
5627 {
5628   emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5629   DONE;
5630 })
5631
5632 (define_expand "neon_vorr<mode>"
5633   [(match_operand:VDQX 0 "s_register_operand" "")
5634    (match_operand:VDQX 1 "s_register_operand" "")
5635    (match_operand:VDQX 2 "neon_logic_op2" "")
5636    (match_operand:SI 3 "immediate_operand" "")]
5637   "TARGET_NEON"
5638 {
5639   emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5640   DONE;
5641 })
5642
5643 (define_expand "neon_veor<mode>"
5644   [(match_operand:VDQX 0 "s_register_operand" "")
5645    (match_operand:VDQX 1 "s_register_operand" "")
5646    (match_operand:VDQX 2 "s_register_operand" "")
5647    (match_operand:SI 3 "immediate_operand" "")]
5648   "TARGET_NEON"
5649 {
5650   emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5651   DONE;
5652 })
5653
5654 (define_expand "neon_vbic<mode>"
5655   [(match_operand:VDQX 0 "s_register_operand" "")
5656    (match_operand:VDQX 1 "s_register_operand" "")
5657    (match_operand:VDQX 2 "neon_logic_op2" "")
5658    (match_operand:SI 3 "immediate_operand" "")]
5659   "TARGET_NEON"
5660 {
5661   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5662   DONE;
5663 })
5664
5665 (define_expand "neon_vorn<mode>"
5666   [(match_operand:VDQX 0 "s_register_operand" "")
5667    (match_operand:VDQX 1 "s_register_operand" "")
5668    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5669    (match_operand:SI 3 "immediate_operand" "")]
5670   "TARGET_NEON"
5671 {
5672   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5673   DONE;
5674 })
5675
5676 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5677   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5678         (SE:<V_unpack> (vec_select:<V_HALF>
5679                           (match_operand:VU 1 "register_operand" "w")
5680                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5681   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5682   "vmovl.<US><V_sz_elem> %q0, %e1"
5683   [(set_attr "neon_type" "neon_shift_1")]
5684 )
5685
5686 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5687   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5688         (SE:<V_unpack> (vec_select:<V_HALF>
5689                           (match_operand:VU 1 "register_operand" "w")
5690                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5691   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5692   "vmovl.<US><V_sz_elem> %q0, %f1"
5693   [(set_attr "neon_type" "neon_shift_1")]
5694 )
5695
5696 (define_expand "vec_unpack<US>_hi_<mode>"
5697   [(match_operand:<V_unpack> 0 "register_operand" "")
5698    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5699  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5700   {
5701    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5702    rtx t1;
5703    int i;
5704    for (i = 0; i < (<V_mode_nunits>/2); i++)
5705      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5706
5707    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5708    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5709                                                  operands[1],
5710                                                  t1));
5711    DONE;
5712   }
5713 )
5714
5715 (define_expand "vec_unpack<US>_lo_<mode>"
5716   [(match_operand:<V_unpack> 0 "register_operand" "")
5717    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5718  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5719   {
5720    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5721    rtx t1;
5722    int i;
5723    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5724      RTVEC_ELT (v, i) = GEN_INT (i);
5725    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5726    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5727                                                  operands[1],
5728                                                  t1));
5729    DONE;
5730   }
5731 )
5732
5733 (define_insn "neon_vec_<US>mult_lo_<mode>"
5734  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5735        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5736                            (match_operand:VU 1 "register_operand" "w")
5737                            (match_operand:VU 2 "vect_par_constant_low" "")))
5738                         (SE:<V_unpack> (vec_select:<V_HALF>
5739                            (match_operand:VU 3 "register_operand" "w")
5740                            (match_dup 2)))))]
5741   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5742   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5743   [(set_attr "neon_type" "neon_shift_1")]
5744 )
5745
5746 (define_expand "vec_widen_<US>mult_lo_<mode>"
5747   [(match_operand:<V_unpack> 0 "register_operand" "")
5748    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5749    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5750  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5751  {
5752    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5753    rtx t1;
5754    int i;
5755    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5756      RTVEC_ELT (v, i) = GEN_INT (i);
5757    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5758
5759    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5760                                                operands[1],
5761                                                t1,
5762                                                operands[2]));
5763    DONE;
5764  }
5765 )
5766
5767 (define_insn "neon_vec_<US>mult_hi_<mode>"
5768  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5769       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5770                             (match_operand:VU 1 "register_operand" "w")
5771                             (match_operand:VU 2 "vect_par_constant_high" "")))
5772                        (SE:<V_unpack> (vec_select:<V_HALF>
5773                             (match_operand:VU 3 "register_operand" "w")
5774                             (match_dup 2)))))]
5775   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5776   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5777   [(set_attr "neon_type" "neon_shift_1")]
5778 )
5779
5780 (define_expand "vec_widen_<US>mult_hi_<mode>"
5781   [(match_operand:<V_unpack> 0 "register_operand" "")
5782    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5783    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5784  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5785  {
5786    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5787    rtx t1;
5788    int i;
5789    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5790      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5791    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5792
5793    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5794                                                operands[1],
5795                                                t1,
5796                                                operands[2]));
5797    DONE;
5798
5799  }
5800 )
5801
5802 (define_insn "neon_vec_<US>shiftl_<mode>"
5803  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5804        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5805        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5806   "TARGET_NEON"
5807 {
5808   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5809 }
5810   [(set_attr "neon_type" "neon_shift_1")]
5811 )
5812
5813 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5814   [(match_operand:<V_unpack> 0 "register_operand" "")
5815    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5816    (match_operand:SI 2 "immediate_operand" "i")]
5817  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5818  {
5819   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5820                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5821                 operands[2]));
5822    DONE;
5823  }
5824 )
5825
5826 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5827   [(match_operand:<V_unpack> 0 "register_operand" "")
5828    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5829    (match_operand:SI 2 "immediate_operand" "i")]
5830  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5831  {
5832   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5833                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5834                                      GET_MODE_SIZE (<V_HALF>mode)),
5835                 operands[2]));
5836    DONE;
5837  }
5838 )
5839
5840 ;; Vectorize for non-neon-quad case
5841 (define_insn "neon_unpack<US>_<mode>"
5842  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5843        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5844  "TARGET_NEON"
5845  "vmovl.<US><V_sz_elem> %q0, %P1"
5846   [(set_attr "neon_type" "neon_shift_1")]
5847 )
5848
5849 (define_expand "vec_unpack<US>_lo_<mode>"
5850  [(match_operand:<V_double_width> 0 "register_operand" "")
5851   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5852  "TARGET_NEON"
5853 {
5854   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5855   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5856   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5857
5858   DONE;
5859 }
5860 )
5861
5862 (define_expand "vec_unpack<US>_hi_<mode>"
5863  [(match_operand:<V_double_width> 0 "register_operand" "")
5864   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5865  "TARGET_NEON"
5866 {
5867   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5868   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5869   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5870
5871   DONE;
5872 }
5873 )
5874
5875 (define_insn "neon_vec_<US>mult_<mode>"
5876  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5877        (mult:<V_widen> (SE:<V_widen>
5878                            (match_operand:VDI 1 "register_operand" "w"))
5879                        (SE:<V_widen>
5880                            (match_operand:VDI 2 "register_operand" "w"))))]
5881   "TARGET_NEON"
5882   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5883   [(set_attr "neon_type" "neon_shift_1")]
5884 )
5885
5886 (define_expand "vec_widen_<US>mult_hi_<mode>"
5887   [(match_operand:<V_double_width> 0 "register_operand" "")
5888    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5889    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5890  "TARGET_NEON"
5891  {
5892    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5893    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5894    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5895
5896    DONE;
5897
5898  }
5899 )
5900
5901 (define_expand "vec_widen_<US>mult_lo_<mode>"
5902   [(match_operand:<V_double_width> 0 "register_operand" "")
5903    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5904    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5905  "TARGET_NEON"
5906  {
5907    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5908    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5909    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5910
5911    DONE;
5912
5913  }
5914 )
5915
5916 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5917  [(match_operand:<V_double_width> 0 "register_operand" "")
5918    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5919    (match_operand:SI 2 "immediate_operand" "i")]
5920  "TARGET_NEON"
5921  {
5922    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5923    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5924    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5925
5926    DONE;
5927  }
5928 )
5929
5930 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5931   [(match_operand:<V_double_width> 0 "register_operand" "")
5932    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5933    (match_operand:SI 2 "immediate_operand" "i")]
5934  "TARGET_NEON"
5935  {
5936    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5937    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5938    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5939
5940    DONE;
5941  }
5942 )
5943
5944 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5945 ; because the ordering of vector elements in Q registers is different from what
5946 ; the semantics of the instructions require.
5947
5948 (define_insn "vec_pack_trunc_<mode>"
5949  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5950        (vec_concat:<V_narrow_pack>
5951                 (truncate:<V_narrow>
5952                         (match_operand:VN 1 "register_operand" "w"))
5953                 (truncate:<V_narrow>
5954                         (match_operand:VN 2 "register_operand" "w"))))]
5955  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5956  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5957  [(set_attr "neon_type" "neon_shift_1")
5958   (set_attr "length" "8")]
5959 )
5960
5961 ;; For the non-quad case.
5962 (define_insn "neon_vec_pack_trunc_<mode>"
5963  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5964        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5965  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5966  "vmovn.i<V_sz_elem>\t%P0, %q1"
5967  [(set_attr "neon_type" "neon_shift_1")]
5968 )
5969
5970 (define_expand "vec_pack_trunc_<mode>"
5971  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5972   (match_operand:VSHFT 1 "register_operand" "")
5973   (match_operand:VSHFT 2 "register_operand")]
5974  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5975 {
5976   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5977
5978   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5979   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5980   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5981   DONE;
5982 })
5983
5984 (define_insn "neon_vabd<mode>_2"
5985  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5986        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5987                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5988  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5989  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5990  [(set (attr "neon_type")
5991        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5992                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5993                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5994                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5995                      (const_string "neon_int_5")))]
5996 )
5997
5998 (define_insn "neon_vabd<mode>_3"
5999  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
6000        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
6001                              (match_operand:VDQ 2 "s_register_operand" "w")]
6002                  UNSPEC_VSUB)))]
6003  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
6004  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
6005  [(set (attr "neon_type")
6006        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
6007                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
6008                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
6009                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
6010                      (const_string "neon_int_5")))]
6011 )