gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012
   3 ;;   Free Software Foundation, Inc.
   4 ;; Written by CodeSourcery.
   5 ;;
   6 ;; This file is part of GCC.
   7 ;;
   8 ;; GCC is free software; you can redistribute it and/or modify it
   9 ;; under the terms of the GNU General Public License as published by
  10 ;; the Free Software Foundation; either version 3, or (at your option)
  11 ;; any later version.
  12 ;;
  13 ;; GCC is distributed in the hope that it will be useful, but
  14 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 ;; General Public License for more details.
  17 ;;
  18 ;; You should have received a copy of the GNU General Public License
  19 ;; along with GCC; see the file COPYING3.  If not see
  20 ;; <http://www.gnu.org/licenses/>.
  21
  22 ;; Enumerators for unspecs.
  23 (define_c_enum "unspec" [
  24   UNSPEC_ASHIFT_SIGNED
  25   UNSPEC_ASHIFT_UNSIGNED
  26   UNSPEC_VABD
  27   UNSPEC_VABDL
  28   UNSPEC_VADD
  29   UNSPEC_VADDHN
  30   UNSPEC_VADDL
  31   UNSPEC_VADDW
  32   UNSPEC_VBSL
  33   UNSPEC_VCAGE
  34   UNSPEC_VCAGT
  35   UNSPEC_VCEQ
  36   UNSPEC_VCGE
  37   UNSPEC_VCGEU
  38   UNSPEC_VCGT
  39   UNSPEC_VCGTU
  40   UNSPEC_VCLS
  41   UNSPEC_VCONCAT
  42   UNSPEC_VCVT
  43   UNSPEC_VCVT_N
  44   UNSPEC_VEXT
  45   UNSPEC_VHADD
  46   UNSPEC_VHSUB
  47   UNSPEC_VLD1
  48   UNSPEC_VLD1_LANE
  49   UNSPEC_VLD2
  50   UNSPEC_VLD2_DUP
  51   UNSPEC_VLD2_LANE
  52   UNSPEC_VLD3
  53   UNSPEC_VLD3A
  54   UNSPEC_VLD3B
  55   UNSPEC_VLD3_DUP
  56   UNSPEC_VLD3_LANE
  57   UNSPEC_VLD4
  58   UNSPEC_VLD4A
  59   UNSPEC_VLD4B
  60   UNSPEC_VLD4_DUP
  61   UNSPEC_VLD4_LANE
  62   UNSPEC_VMAX
  63   UNSPEC_VMIN
  64   UNSPEC_VMLA
  65   UNSPEC_VMLAL
  66   UNSPEC_VMLA_LANE
  67   UNSPEC_VMLAL_LANE
  68   UNSPEC_VMLS
  69   UNSPEC_VMLSL
  70   UNSPEC_VMLS_LANE
  71   UNSPEC_VMLSL_LANE
  72   UNSPEC_VMOVL
  73   UNSPEC_VMOVN
  74   UNSPEC_VMUL
  75   UNSPEC_VMULL
  76   UNSPEC_VMUL_LANE
  77   UNSPEC_VMULL_LANE
  78   UNSPEC_VPADAL
  79   UNSPEC_VPADD
  80   UNSPEC_VPADDL
  81   UNSPEC_VPMAX
  82   UNSPEC_VPMIN
  83   UNSPEC_VPSMAX
  84   UNSPEC_VPSMIN
  85   UNSPEC_VPUMAX
  86   UNSPEC_VPUMIN
  87   UNSPEC_VQABS
  88   UNSPEC_VQADD
  89   UNSPEC_VQDMLAL
  90   UNSPEC_VQDMLAL_LANE
  91   UNSPEC_VQDMLSL
  92   UNSPEC_VQDMLSL_LANE
  93   UNSPEC_VQDMULH
  94   UNSPEC_VQDMULH_LANE
  95   UNSPEC_VQDMULL
  96   UNSPEC_VQDMULL_LANE
  97   UNSPEC_VQMOVN
  98   UNSPEC_VQMOVUN
  99   UNSPEC_VQNEG
 100   UNSPEC_VQSHL
 101   UNSPEC_VQSHL_N
 102   UNSPEC_VQSHLU_N
 103   UNSPEC_VQSHRN_N
 104   UNSPEC_VQSHRUN_N
 105   UNSPEC_VQSUB
 106   UNSPEC_VRECPE
 107   UNSPEC_VRECPS
 108   UNSPEC_VREV16
 109   UNSPEC_VREV32
 110   UNSPEC_VREV64
 111   UNSPEC_VRSQRTE
 112   UNSPEC_VRSQRTS
 113   UNSPEC_VSHL
 114   UNSPEC_VSHLL_N
 115   UNSPEC_VSHL_N
 116   UNSPEC_VSHR_N
 117   UNSPEC_VSHRN_N
 118   UNSPEC_VSLI
 119   UNSPEC_VSRA_N
 120   UNSPEC_VSRI
 121   UNSPEC_VST1
 122   UNSPEC_VST1_LANE
 123   UNSPEC_VST2
 124   UNSPEC_VST2_LANE
 125   UNSPEC_VST3
 126   UNSPEC_VST3A
 127   UNSPEC_VST3B
 128   UNSPEC_VST3_LANE
 129   UNSPEC_VST4
 130   UNSPEC_VST4A
 131   UNSPEC_VST4B
 132   UNSPEC_VST4_LANE
 133   UNSPEC_VSTRUCTDUMMY
 134   UNSPEC_VSUB
 135   UNSPEC_VSUBHN
 136   UNSPEC_VSUBL
 137   UNSPEC_VSUBW
 138   UNSPEC_VTBL
 139   UNSPEC_VTBX
 140   UNSPEC_VTRN1
 141   UNSPEC_VTRN2
 142   UNSPEC_VTST
 143   UNSPEC_VUZP1
 144   UNSPEC_VUZP2
 145   UNSPEC_VZIP1
 146   UNSPEC_VZIP2
 147   UNSPEC_MISALIGNED_ACCESS
 148   UNSPEC_VCLE
 149   UNSPEC_VCLT
 150 ])
 151
 152
 153 ;; Attribute used to permit string comparisons against <VQH_mnem> in
 154 ;; neon_type attribute definitions.
 155 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
 156
 157 (define_insn "*neon_mov<mode>"
 158   [(set (match_operand:VDX 0 "nonimmediate_operand"
 159           "=w,Uv,w, w,  ?r,?w,?r,?r, ?Us")
 160         (match_operand:VDX 1 "general_operand"
 161           " w,w, Dn,Uvi, w, r, r, Usi,r"))]
 162   "TARGET_NEON
 163    && (register_operand (operands[0], <MODE>mode)
 164        || register_operand (operands[1], <MODE>mode))"
 165 {
 166   if (which_alternative == 2)
 167     {
 168       int width, is_valid;
 169       static char templ[40];
 170
 171       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 172         &operands[1], &width);
 173
 174       gcc_assert (is_valid != 0);
 175
 176       if (width == 0)
 177         return "vmov.f32\t%P0, %1  @ <mode>";
 178       else
 179         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
 180
 181       return templ;
 182     }
 183
 184   /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp
 185      below must be changed to output_move_neon (which will use the
 186      element/structure loads/stores), and the constraint changed to 'Um' instead
 187      of 'Uv'.  */
 188
 189   switch (which_alternative)
 190     {
 191     case 0: return "vmov\t%P0, %P1  @ <mode>";
 192     case 1: case 3: return output_move_vfp (operands);
 193     case 2: gcc_unreachable ();
 194     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
 195     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
 196     default: return output_move_double (operands, true, NULL);
 197     }
 198 }
 199  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 200   (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
 201   (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 202   (set_attr "length" "4,4,4,4,4,4,8,8,8")
 203   (set_attr "pool_range"     "*,*,*,1020,*,*,*,1020,*")
 204   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
 205
 206 (define_insn "*neon_mov<mode>"
 207   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
 208           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
 209         (match_operand:VQXMOV 1 "general_operand"
 210           " w,w, Dn,Uni, w, r, r, Usi, r"))]
 211   "TARGET_NEON
 212    && (register_operand (operands[0], <MODE>mode)
 213        || register_operand (operands[1], <MODE>mode))"
 214 {
 215   if (which_alternative == 2)
 216     {
 217       int width, is_valid;
 218       static char templ[40];
 219
 220       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
 221         &operands[1], &width);
 222
 223       gcc_assert (is_valid != 0);
 224
 225       if (width == 0)
 226         return "vmov.f32\t%q0, %1  @ <mode>";
 227       else
 228         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
 229
 230       return templ;
 231     }
 232
 233   switch (which_alternative)
 234     {
 235     case 0: return "vmov\t%q0, %q1  @ <mode>";
 236     case 1: case 3: return output_move_neon (operands);
 237     case 2: gcc_unreachable ();
 238     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 239     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 240     default: return output_move_quad (operands);
 241     }
 242 }
 243   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 244                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 245    (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
 246    (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
 247    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 248    (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
 249    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 250
 251 (define_expand "movti"
 252   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 253         (match_operand:TI 1 "general_operand" ""))]
 254   "TARGET_NEON"
 255 {
 256   if (can_create_pseudo_p ())
 257     {
 258       if (GET_CODE (operands[0]) != REG)
 259         operands[1] = force_reg (TImode, operands[1]);
 260     }
 261 })
 262
 263 (define_expand "mov<mode>"
 264   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 265         (match_operand:VSTRUCT 1 "general_operand" ""))]
 266   "TARGET_NEON"
 267 {
 268   if (can_create_pseudo_p ())
 269     {
 270       if (GET_CODE (operands[0]) != REG)
 271         operands[1] = force_reg (<MODE>mode, operands[1]);
 272     }
 273 })
 274
 275 (define_insn "*neon_mov<mode>"
 276   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 277         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 278   "TARGET_NEON
 279    && (register_operand (operands[0], <MODE>mode)
 280        || register_operand (operands[1], <MODE>mode))"
 281 {
 282   switch (which_alternative)
 283     {
 284     case 0: return "#";
 285     case 1: case 2: return output_move_neon (operands);
 286     default: gcc_unreachable ();
 287     }
 288 }
 289   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 290    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 291
 292 (define_split
 293   [(set (match_operand:EI 0 "s_register_operand" "")
 294         (match_operand:EI 1 "s_register_operand" ""))]
 295   "TARGET_NEON && reload_completed"
 296   [(set (match_dup 0) (match_dup 1))
 297    (set (match_dup 2) (match_dup 3))]
 298 {
 299   int rdest = REGNO (operands[0]);
 300   int rsrc = REGNO (operands[1]);
 301   rtx dest[2], src[2];
 302
 303   dest[0] = gen_rtx_REG (TImode, rdest);
 304   src[0] = gen_rtx_REG (TImode, rsrc);
 305   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 306   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 307
 308   neon_disambiguate_copy (operands, dest, src, 2);
 309 })
 310
 311 (define_split
 312   [(set (match_operand:OI 0 "s_register_operand" "")
 313         (match_operand:OI 1 "s_register_operand" ""))]
 314   "TARGET_NEON && reload_completed"
 315   [(set (match_dup 0) (match_dup 1))
 316    (set (match_dup 2) (match_dup 3))]
 317 {
 318   int rdest = REGNO (operands[0]);
 319   int rsrc = REGNO (operands[1]);
 320   rtx dest[2], src[2];
 321
 322   dest[0] = gen_rtx_REG (TImode, rdest);
 323   src[0] = gen_rtx_REG (TImode, rsrc);
 324   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 325   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 326
 327   neon_disambiguate_copy (operands, dest, src, 2);
 328 })
 329
 330 (define_split
 331   [(set (match_operand:CI 0 "s_register_operand" "")
 332         (match_operand:CI 1 "s_register_operand" ""))]
 333   "TARGET_NEON && reload_completed"
 334   [(set (match_dup 0) (match_dup 1))
 335    (set (match_dup 2) (match_dup 3))
 336    (set (match_dup 4) (match_dup 5))]
 337 {
 338   int rdest = REGNO (operands[0]);
 339   int rsrc = REGNO (operands[1]);
 340   rtx dest[3], src[3];
 341
 342   dest[0] = gen_rtx_REG (TImode, rdest);
 343   src[0] = gen_rtx_REG (TImode, rsrc);
 344   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 345   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 346   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 347   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 348
 349   neon_disambiguate_copy (operands, dest, src, 3);
 350 })
 351
 352 (define_split
 353   [(set (match_operand:XI 0 "s_register_operand" "")
 354         (match_operand:XI 1 "s_register_operand" ""))]
 355   "TARGET_NEON && reload_completed"
 356   [(set (match_dup 0) (match_dup 1))
 357    (set (match_dup 2) (match_dup 3))
 358    (set (match_dup 4) (match_dup 5))
 359    (set (match_dup 6) (match_dup 7))]
 360 {
 361   int rdest = REGNO (operands[0]);
 362   int rsrc = REGNO (operands[1]);
 363   rtx dest[4], src[4];
 364
 365   dest[0] = gen_rtx_REG (TImode, rdest);
 366   src[0] = gen_rtx_REG (TImode, rsrc);
 367   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 368   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 369   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 370   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 371   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 372   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 373
 374   neon_disambiguate_copy (operands, dest, src, 4);
 375 })
 376
 377 (define_expand "movmisalign<mode>"
 378   [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
 379         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
 380                      UNSPEC_MISALIGNED_ACCESS))]
 381   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 382 {
 383   /* This pattern is not permitted to fail during expansion: if both arguments
 384      are non-registers (e.g. memory := constant, which can be created by the
 385      auto-vectorizer), force operand 1 into a register.  */
 386   if (!s_register_operand (operands[0], <MODE>mode)
 387       && !s_register_operand (operands[1], <MODE>mode))
 388     operands[1] = force_reg (<MODE>mode, operands[1]);
 389 })
 390
 391 (define_insn "*movmisalign<mode>_neon_store"
 392   [(set (match_operand:VDX 0 "neon_struct_operand"             "=Um")
 393         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 394                     UNSPEC_MISALIGNED_ACCESS))]
 395   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 396   "vst1.<V_sz_elem>\t{%P1}, %A0"
 397   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 398
 399 (define_insn "*movmisalign<mode>_neon_load"
 400   [(set (match_operand:VDX 0 "s_register_operand"               "=w")
 401         (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
 402                     UNSPEC_MISALIGNED_ACCESS))]
 403   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 404   "vld1.<V_sz_elem>\t{%P0}, %A1"
 405   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 406
 407 (define_insn "*movmisalign<mode>_neon_store"
 408   [(set (match_operand:VQX 0 "neon_struct_operand"             "=Um")
 409         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 410                     UNSPEC_MISALIGNED_ACCESS))]
 411   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 412   "vst1.<V_sz_elem>\t{%q1}, %A0"
 413   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 414
 415 (define_insn "*movmisalign<mode>_neon_load"
 416   [(set (match_operand:VQX 0 "s_register_operand"                "=w")
 417         (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
 418                     UNSPEC_MISALIGNED_ACCESS))]
 419   "TARGET_NEON && !BYTES_BIG_ENDIAN"
 420   "vld1.<V_sz_elem>\t{%q0}, %A1"
 421   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 422
 423 (define_insn "vec_set<mode>_internal"
 424   [(set (match_operand:VD 0 "s_register_operand" "=w")
 425         (vec_merge:VD
 426           (vec_duplicate:VD
 427             (match_operand:<V_elem> 1 "s_register_operand" "r"))
 428           (match_operand:VD 3 "s_register_operand" "0")
 429           (match_operand:SI 2 "immediate_operand" "i")))]
 430   "TARGET_NEON"
 431 {
 432   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 433   if (BYTES_BIG_ENDIAN)
 434     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 435   operands[2] = GEN_INT (elt);
 436
 437   return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
 438 }
 439   [(set_attr "predicable" "yes")
 440    (set_attr "neon_type" "neon_mcr")])
 441
 442 (define_insn "vec_set<mode>_internal"
 443   [(set (match_operand:VQ 0 "s_register_operand" "=w")
 444         (vec_merge:VQ
 445           (vec_duplicate:VQ
 446             (match_operand:<V_elem> 1 "s_register_operand" "r"))
 447           (match_operand:VQ 3 "s_register_operand" "0")
 448           (match_operand:SI 2 "immediate_operand" "i")))]
 449   "TARGET_NEON"
 450 {
 451   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 452   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 453   int elt = elem % half_elts;
 454   int hi = (elem / half_elts) * 2;
 455   int regno = REGNO (operands[0]);
 456
 457   if (BYTES_BIG_ENDIAN)
 458     elt = half_elts - 1 - elt;
 459
 460   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 461   operands[2] = GEN_INT (elt);
 462
 463   return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
 464 }
 465   [(set_attr "predicable" "yes")
 466    (set_attr "neon_type" "neon_mcr")]
 467 )
 468
 469 (define_insn "vec_setv2di_internal"
 470   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
 471         (vec_merge:V2DI
 472           (vec_duplicate:V2DI
 473             (match_operand:DI 1 "s_register_operand" "r"))
 474           (match_operand:V2DI 3 "s_register_operand" "0")
 475           (match_operand:SI 2 "immediate_operand" "i")))]
 476   "TARGET_NEON"
 477 {
 478   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 479   int regno = REGNO (operands[0]) + 2 * elem;
 480
 481   operands[0] = gen_rtx_REG (DImode, regno);
 482
 483   return "vmov%?\t%P0, %Q1, %R1";
 484 }
 485   [(set_attr "predicable" "yes")
 486    (set_attr "neon_type" "neon_mcr_2_mcrr")]
 487 )
 488
 489 (define_expand "vec_set<mode>"
 490   [(match_operand:VDQ 0 "s_register_operand" "")
 491    (match_operand:<V_elem> 1 "s_register_operand" "")
 492    (match_operand:SI 2 "immediate_operand" "")]
 493   "TARGET_NEON"
 494 {
 495   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 496   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 497                                          GEN_INT (elem), operands[0]));
 498   DONE;
 499 })
 500
 501 (define_insn "vec_extract<mode>"
 502   [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
 503         (vec_select:<V_elem>
 504           (match_operand:VD 1 "s_register_operand" "w")
 505           (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
 506   "TARGET_NEON"
 507 {
 508   if (BYTES_BIG_ENDIAN)
 509     {
 510       int elt = INTVAL (operands[2]);
 511       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 512       operands[2] = GEN_INT (elt);
 513     }
 514   return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
 515 }
 516   [(set_attr "predicable" "yes")
 517    (set_attr "neon_type" "neon_bp_simple")]
 518 )
 519
 520 (define_insn "vec_extract<mode>"
 521   [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
 522         (vec_select:<V_elem>
 523           (match_operand:VQ 1 "s_register_operand" "w")
 524           (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
 525   "TARGET_NEON"
 526 {
 527   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 528   int elt = INTVAL (operands[2]) % half_elts;
 529   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 530   int regno = REGNO (operands[1]);
 531
 532   if (BYTES_BIG_ENDIAN)
 533     elt = half_elts - 1 - elt;
 534
 535   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 536   operands[2] = GEN_INT (elt);
 537
 538   return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
 539 }
 540   [(set_attr "predicable" "yes")
 541    (set_attr "neon_type" "neon_bp_simple")]
 542 )
 543
 544 (define_insn "vec_extractv2di"
 545   [(set (match_operand:DI 0 "s_register_operand" "=r")
 546         (vec_select:DI
 547           (match_operand:V2DI 1 "s_register_operand" "w")
 548           (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
 549   "TARGET_NEON"
 550 {
 551   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 552
 553   operands[1] = gen_rtx_REG (DImode, regno);
 554
 555   return "vmov%?\t%Q0, %R0, %P1  @ v2di";
 556 }
 557   [(set_attr "predicable" "yes")
 558    (set_attr "neon_type" "neon_int_1")]
 559 )
 560
 561 (define_expand "vec_init<mode>"
 562   [(match_operand:VDQ 0 "s_register_operand" "")
 563    (match_operand 1 "" "")]
 564   "TARGET_NEON"
 565 {
 566   neon_expand_vector_init (operands[0], operands[1]);
 567   DONE;
 568 })
 569
 570 ;; Doubleword and quadword arithmetic.
 571
 572 ;; NOTE: some other instructions also support 64-bit integer
 573 ;; element size, which we could potentially use for "long long" operations.
 574
 575 (define_insn "*add<mode>3_neon"
 576   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 577         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 578                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 579   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 580   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 581   [(set (attr "neon_type")
 582       (if_then_else (match_test "<Is_float_mode>")
 583                     (if_then_else (match_test "<Is_d_reg>")
 584                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 585                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 586                     (const_string "neon_int_1")))]
 587 )
 588
 589 (define_insn "adddi3_neon"
 590   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 591         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 592                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 593    (clobber (reg:CC CC_REGNUM))]
 594   "TARGET_NEON"
 595 {
 596   switch (which_alternative)
 597     {
 598     case 0: /* fall through */
 599     case 3: return "vadd.i64\t%P0, %P1, %P2";
 600     case 1: return "#";
 601     case 2: return "#";
 602     case 4: return "#";
 603     case 5: return "#";
 604     case 6: return "#";
 605     default: gcc_unreachable ();
 606     }
 607 }
 608   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 609    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 610    (set_attr "length" "*,8,8,*,8,8,8")
 611    (set_attr "arch" "nota8,*,*,onlya8,*,*,*")]
 612 )
 613
 614 (define_insn "*sub<mode>3_neon"
 615   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 616         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 617                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 618   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 619   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 620   [(set (attr "neon_type")
 621       (if_then_else (match_test "<Is_float_mode>")
 622                     (if_then_else (match_test "<Is_d_reg>")
 623                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 624                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 625                     (const_string "neon_int_2")))]
 626 )
 627
 628 (define_insn "subdi3_neon"
 629   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 630         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 631                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 632    (clobber (reg:CC CC_REGNUM))]
 633   "TARGET_NEON"
 634 {
 635   switch (which_alternative)
 636     {
 637     case 0: /* fall through */
 638     case 4: return "vsub.i64\t%P0, %P1, %P2";
 639     case 1: /* fall through */
 640     case 2: /* fall through */
 641     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 642     default: gcc_unreachable ();
 643     }
 644 }
 645   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 646    (set_attr "conds" "*,clob,clob,clob,*")
 647    (set_attr "length" "*,8,8,8,*")
 648    (set_attr "arch" "nota8,*,*,*,onlya8")]
 649 )
 650
 651 (define_insn "*mul<mode>3_neon"
 652   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 653         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 654                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 655   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 656   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 657   [(set (attr "neon_type")
 658       (if_then_else (match_test "<Is_float_mode>")
 659                     (if_then_else (match_test "<Is_d_reg>")
 660                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 661                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 662                     (if_then_else (match_test "<Is_d_reg>")
 663                                   (if_then_else
 664                                     (match_test "<Scalar_mul_8_16>")
 665                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 666                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 667                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 668                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 669                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 670 )
 671
 672 (define_insn "mul<mode>3add<mode>_neon"
 673   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 674         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 675                             (match_operand:VDQ 3 "s_register_operand" "w"))
 676                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 677   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 678   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 679   [(set (attr "neon_type")
 680       (if_then_else (match_test "<Is_float_mode>")
 681                     (if_then_else (match_test "<Is_d_reg>")
 682                                   (const_string "neon_fp_vmla_ddd")
 683                                   (const_string "neon_fp_vmla_qqq"))
 684                     (if_then_else (match_test "<Is_d_reg>")
 685                                   (if_then_else
 686                                     (match_test "<Scalar_mul_8_16>")
 687                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 688                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 689                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 690                                     (const_string "neon_mla_qqq_8_16")
 691                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 692 )
 693
 694 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 695   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 696         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 697                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 698                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 699   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 700   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 701   [(set (attr "neon_type")
 702       (if_then_else (match_test "<Is_float_mode>")
 703                     (if_then_else (match_test "<Is_d_reg>")
 704                                   (const_string "neon_fp_vmla_ddd")
 705                                   (const_string "neon_fp_vmla_qqq"))
 706                     (if_then_else (match_test "<Is_d_reg>")
 707                                   (if_then_else
 708                                     (match_test "<Scalar_mul_8_16>")
 709                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 710                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 711                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 712                                     (const_string "neon_mla_qqq_8_16")
 713                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 714 )
 715
 716 (define_insn "ior<mode>3"
 717   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 718         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 719                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 720   "TARGET_NEON"
 721 {
 722   switch (which_alternative)
 723     {
 724     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 725     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 726                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 727     default: gcc_unreachable ();
 728     }
 729 }
 730   [(set_attr "neon_type" "neon_int_1")]
 731 )
 732
 733 (define_insn "iordi3_neon"
 734   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 735         (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 736                 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
 737   "TARGET_NEON"
 738 {
 739   switch (which_alternative)
 740     {
 741     case 0: /* fall through */
 742     case 4: return "vorr\t%P0, %P1, %P2";
 743     case 1: /* fall through */
 744     case 5: return neon_output_logic_immediate ("vorr", &operands[2],
 745                      DImode, 0, VALID_NEON_QREG_MODE (DImode));
 746     case 2: return "#";
 747     case 3: return "#";
 748     default: gcc_unreachable ();
 749     }
 750 }
 751   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 752    (set_attr "length" "*,*,8,8,*,*")
 753    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 754 )
 755
 756 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 757 ;; vorr. We support the pseudo-instruction vand instead, because that
 758 ;; corresponds to the canonical form the middle-end expects to use for
 759 ;; immediate bitwise-ANDs.
 760
 761 (define_insn "and<mode>3"
 762   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 763         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 764                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 765   "TARGET_NEON"
 766 {
 767   switch (which_alternative)
 768     {
 769     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 770     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 771                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 772     default: gcc_unreachable ();
 773     }
 774 }
 775   [(set_attr "neon_type" "neon_int_1")]
 776 )
 777
 778 (define_insn "anddi3_neon"
 779   [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
 780         (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
 781                 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
 782   "TARGET_NEON"
 783 {
 784   switch (which_alternative)
 785     {
 786     case 0: /* fall through */
 787     case 4: return "vand\t%P0, %P1, %P2";
 788     case 1: /* fall through */
 789     case 5: return neon_output_logic_immediate ("vand", &operands[2],
 790                      DImode, 1, VALID_NEON_QREG_MODE (DImode));
 791     case 2: return "#";
 792     case 3: return "#";
 793     default: gcc_unreachable ();
 794     }
 795 }
 796   [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
 797    (set_attr "length" "*,*,8,8,*,*")
 798    (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
 799 )
 800
 801 (define_insn "orn<mode>3_neon"
 802   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 803         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 804                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 805   "TARGET_NEON"
 806   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 807   [(set_attr "neon_type" "neon_int_1")]
 808 )
 809
 810 ;; TODO: investigate whether we should disable
 811 ;; this and bicdi3_neon for the A8 in line with the other
 812 ;; changes above.
 813 (define_insn_and_split "orndi3_neon"
 814   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 815         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 816                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 817   "TARGET_NEON"
 818   "@
 819    vorn\t%P0, %P1, %P2
 820    #
 821    #
 822    #"
 823   "reload_completed &&
 824    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 825   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 826    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 827   "
 828   {
 829     if (TARGET_THUMB2)
 830       {
 831         operands[3] = gen_highpart (SImode, operands[0]);
 832         operands[0] = gen_lowpart (SImode, operands[0]);
 833         operands[4] = gen_highpart (SImode, operands[2]);
 834         operands[2] = gen_lowpart (SImode, operands[2]);
 835         operands[5] = gen_highpart (SImode, operands[1]);
 836         operands[1] = gen_lowpart (SImode, operands[1]);
 837       }
 838     else
 839       {
 840         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 841         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 842         DONE;
 843       }
 844   }"
 845   [(set_attr "neon_type" "neon_int_1,*,*,*")
 846    (set_attr "length" "*,16,8,8")
 847    (set_attr "arch" "any,a,t2,t2")]
 848 )
 849
 850 (define_insn "bic<mode>3_neon"
 851   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 852         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 853                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 854   "TARGET_NEON"
 855   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 856   [(set_attr "neon_type" "neon_int_1")]
 857 )
 858
 859 ;; Compare to *anddi_notdi_di.
 860 (define_insn "bicdi3_neon"
 861   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 862         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 863                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 864   "TARGET_NEON"
 865   "@
 866    vbic\t%P0, %P1, %P2
 867    #
 868    #"
 869   [(set_attr "neon_type" "neon_int_1,*,*")
 870    (set_attr "length" "*,8,8")]
 871 )
 872
 873 (define_insn "xor<mode>3"
 874   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 875         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 876                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 877   "TARGET_NEON"
 878   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 879   [(set_attr "neon_type" "neon_int_1")]
 880 )
 881
 882 (define_insn "xordi3_neon"
 883   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
 884         (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
 885                 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
 886   "TARGET_NEON"
 887   "@
 888    veor\t%P0, %P1, %P2
 889    #
 890    #
 891    veor\t%P0, %P1, %P2"
 892   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
 893    (set_attr "length" "*,8,8,*")
 894    (set_attr "arch" "nota8,*,*,onlya8")]
 895 )
 896
 897 (define_insn "one_cmpl<mode>2"
 898   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 899         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 900   "TARGET_NEON"
 901   "vmvn\t%<V_reg>0, %<V_reg>1"
 902   [(set_attr "neon_type" "neon_int_1")]
 903 )
 904
 905 (define_insn "abs<mode>2"
 906   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 907         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 908   "TARGET_NEON"
 909   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 910   [(set (attr "neon_type")
 911       (if_then_else (match_test "<Is_float_mode>")
 912                     (if_then_else (match_test "<Is_d_reg>")
 913                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 914                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 915                     (const_string "neon_int_3")))]
 916 )
 917
 918 (define_insn "neg<mode>2"
 919   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 920         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 921   "TARGET_NEON"
 922   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 923   [(set (attr "neon_type")
 924       (if_then_else (match_test "<Is_float_mode>")
 925                     (if_then_else (match_test "<Is_d_reg>")
 926                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 927                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 928                     (const_string "neon_int_3")))]
 929 )
 930
 931 (define_insn "negdi2_neon"
 932   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 933         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 934    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 935    (clobber (reg:CC CC_REGNUM))]
 936   "TARGET_NEON"
 937   "#"
 938   [(set_attr "length" "8")]
 939 )
 940
 941 ; Split negdi2_neon for vfp registers
 942 (define_split
 943   [(set (match_operand:DI 0 "s_register_operand" "")
 944         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 945    (clobber (match_scratch:DI 2 ""))
 946    (clobber (reg:CC CC_REGNUM))]
 947   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 948   [(set (match_dup 2) (const_int 0))
 949    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 950               (clobber (reg:CC CC_REGNUM))])]
 951   {
 952     if (!REG_P (operands[2]))
 953       operands[2] = operands[0];
 954   }
 955 )
 956
 957 ; Split negdi2_neon for core registers
 958 (define_split
 959   [(set (match_operand:DI 0 "s_register_operand" "")
 960         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 961    (clobber (match_scratch:DI 2 ""))
 962    (clobber (reg:CC CC_REGNUM))]
 963   "TARGET_32BIT && reload_completed
 964    && arm_general_register_operand (operands[0], DImode)"
 965   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
 966               (clobber (reg:CC CC_REGNUM))])]
 967   ""
 968 )
 969
 970 (define_insn "*umin<mode>3_neon"
 971   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 972         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 973                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 974   "TARGET_NEON"
 975   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 976   [(set_attr "neon_type" "neon_int_5")]
 977 )
 978
 979 (define_insn "*umax<mode>3_neon"
 980   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 981         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 982                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 983   "TARGET_NEON"
 984   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 985   [(set_attr "neon_type" "neon_int_5")]
 986 )
 987
 988 (define_insn "*smin<mode>3_neon"
 989   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 990         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 991                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 992   "TARGET_NEON"
 993   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 994   [(set (attr "neon_type")
 995       (if_then_else (match_test "<Is_float_mode>")
 996                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 997                     (const_string "neon_int_5")))]
 998 )
 999
1000 (define_insn "*smax<mode>3_neon"
1001   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1002         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
1003                    (match_operand:VDQW 2 "s_register_operand" "w")))]
1004   "TARGET_NEON"
1005   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1006   [(set (attr "neon_type")
1007       (if_then_else (match_test "<Is_float_mode>")
1008                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1009                     (const_string "neon_int_5")))]
1010 )
1011
1012 ; TODO: V2DI shifts are current disabled because there are bugs in the
1013 ; generic vectorizer code.  It ends up creating a V2DI constructor with
1014 ; SImode elements.
1015
1016 (define_insn "vashl<mode>3"
1017   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
1018         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
1019                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
1020   "TARGET_NEON"
1021   {
1022     switch (which_alternative)
1023       {
1024         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
1025         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
1026                                                     <MODE>mode,
1027                                                     VALID_NEON_QREG_MODE (<MODE>mode),
1028                                                     true);
1029         default: gcc_unreachable ();
1030       }
1031   }
1032   [(set (attr "neon_type")
1033       (if_then_else (match_test "<Is_d_reg>")
1034                     (const_string "neon_vshl_ddd")
1035                     (const_string "neon_shift_3")))]
1036 )
1037
1038 (define_insn "vashr<mode>3_imm"
1039   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1040         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1041                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1042   "TARGET_NEON"
1043   {
1044     return neon_output_shift_immediate ("vshr", 's', &operands[2],
1045                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1046                                         false);
1047   }
1048   [(set (attr "neon_type")
1049       (if_then_else (match_test "<Is_d_reg>")
1050                     (const_string "neon_vshl_ddd")
1051                     (const_string "neon_shift_3")))]
1052 )
1053
1054 (define_insn "vlshr<mode>3_imm"
1055   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1056         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1057                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1058   "TARGET_NEON"
1059   {
1060     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1061                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1062                                         false);
1063   }
1064   [(set (attr "neon_type")
1065         (if_then_else (match_test "<Is_d_reg>")
1066                       (const_string "neon_vshl_ddd")
1067                       (const_string "neon_shift_3")))]
1068 )
1069
1070 ; Used for implementing logical shift-right, which is a left-shift by a negative
1071 ; amount, with signed operands. This is essentially the same as ashl<mode>3
1072 ; above, but using an unspec in case GCC tries anything tricky with negative
1073 ; shift amounts.
1074
1075 (define_insn "ashl<mode>3_signed"
1076   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1077         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1078                       (match_operand:VDQI 2 "s_register_operand" "w")]
1079                      UNSPEC_ASHIFT_SIGNED))]
1080   "TARGET_NEON"
1081   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1082   [(set (attr "neon_type")
1083       (if_then_else (match_test "<Is_d_reg>")
1084                     (const_string "neon_vshl_ddd")
1085                     (const_string "neon_shift_3")))]
1086 )
1087
1088 ; Used for implementing logical shift-right, which is a left-shift by a negative
1089 ; amount, with unsigned operands.
1090
1091 (define_insn "ashl<mode>3_unsigned"
1092   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1093         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1094                       (match_operand:VDQI 2 "s_register_operand" "w")]
1095                      UNSPEC_ASHIFT_UNSIGNED))]
1096   "TARGET_NEON"
1097   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1098   [(set (attr "neon_type")
1099       (if_then_else (match_test "<Is_d_reg>")
1100                     (const_string "neon_vshl_ddd")
1101                     (const_string "neon_shift_3")))]
1102 )
1103
1104 (define_expand "vashr<mode>3"
1105   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1106         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1107                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1108   "TARGET_NEON"
1109 {
1110   if (s_register_operand (operands[2], <MODE>mode))
1111     {
1112       rtx neg = gen_reg_rtx (<MODE>mode);
1113       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1114       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1115     }
1116   else
1117     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1118   DONE;
1119 })
1120
1121 (define_expand "vlshr<mode>3"
1122   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1123         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1124                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1125   "TARGET_NEON"
1126 {
1127   if (s_register_operand (operands[2], <MODE>mode))
1128     {
1129       rtx neg = gen_reg_rtx (<MODE>mode);
1130       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1131       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1132     }
1133   else
1134     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1135   DONE;
1136 })
1137
1138 ;; Widening operations
1139
1140 (define_insn "widen_ssum<mode>3"
1141   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1142         (plus:<V_widen> (sign_extend:<V_widen>
1143                           (match_operand:VW 1 "s_register_operand" "%w"))
1144                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1145   "TARGET_NEON"
1146   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1147   [(set_attr "neon_type" "neon_int_3")]
1148 )
1149
1150 (define_insn "widen_usum<mode>3"
1151   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1152         (plus:<V_widen> (zero_extend:<V_widen>
1153                           (match_operand:VW 1 "s_register_operand" "%w"))
1154                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1155   "TARGET_NEON"
1156   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1157   [(set_attr "neon_type" "neon_int_3")]
1158 )
1159
1160 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1161 ;; shift-count granularity. That's good enough for the middle-end's current
1162 ;; needs.
1163
1164 ;; Note that it's not safe to perform such an operation in big-endian mode,
1165 ;; due to element-ordering issues.
1166
1167 (define_expand "vec_shr_<mode>"
1168   [(match_operand:VDQ 0 "s_register_operand" "")
1169    (match_operand:VDQ 1 "s_register_operand" "")
1170    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1171   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1172 {
1173   rtx zero_reg;
1174   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1175   const int width = GET_MODE_BITSIZE (<MODE>mode);
1176   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1177   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1178     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1179
1180   if (num_bits == width)
1181     {
1182       emit_move_insn (operands[0], operands[1]);
1183       DONE;
1184     }
1185
1186   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1187   operands[0] = gen_lowpart (bvecmode, operands[0]);
1188   operands[1] = gen_lowpart (bvecmode, operands[1]);
1189
1190   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1191                       GEN_INT (num_bits / BITS_PER_UNIT)));
1192   DONE;
1193 })
1194
1195 (define_expand "vec_shl_<mode>"
1196   [(match_operand:VDQ 0 "s_register_operand" "")
1197    (match_operand:VDQ 1 "s_register_operand" "")
1198    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1199   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1200 {
1201   rtx zero_reg;
1202   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1203   const int width = GET_MODE_BITSIZE (<MODE>mode);
1204   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1205   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1206     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1207
1208   if (num_bits == 0)
1209     {
1210       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1211       DONE;
1212     }
1213
1214   num_bits = width - num_bits;
1215
1216   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1217   operands[0] = gen_lowpart (bvecmode, operands[0]);
1218   operands[1] = gen_lowpart (bvecmode, operands[1]);
1219
1220   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1221                       GEN_INT (num_bits / BITS_PER_UNIT)));
1222   DONE;
1223 })
1224
1225 ;; Helpers for quad-word reduction operations
1226
1227 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1228 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1229 ; N/2-element vector.
1230
1231 (define_insn "quad_halves_<code>v4si"
1232   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1233         (vqh_ops:V2SI
1234           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1235                            (parallel [(const_int 0) (const_int 1)]))
1236           (vec_select:V2SI (match_dup 1)
1237                            (parallel [(const_int 2) (const_int 3)]))))]
1238   "TARGET_NEON"
1239   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1240   [(set_attr "vqh_mnem" "<VQH_mnem>")
1241    (set (attr "neon_type")
1242       (if_then_else (eq_attr "vqh_mnem" "vadd")
1243                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1244 )
1245
1246 (define_insn "quad_halves_<code>v4sf"
1247   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1248         (vqhs_ops:V2SF
1249           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1250                            (parallel [(const_int 0) (const_int 1)]))
1251           (vec_select:V2SF (match_dup 1)
1252                            (parallel [(const_int 2) (const_int 3)]))))]
1253   "TARGET_NEON && flag_unsafe_math_optimizations"
1254   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1255   [(set_attr "vqh_mnem" "<VQH_mnem>")
1256    (set (attr "neon_type")
1257       (if_then_else (eq_attr "vqh_mnem" "vadd")
1258                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1259 )
1260
1261 (define_insn "quad_halves_<code>v8hi"
1262   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1263         (vqh_ops:V4HI
1264           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1265                            (parallel [(const_int 0) (const_int 1)
1266                                       (const_int 2) (const_int 3)]))
1267           (vec_select:V4HI (match_dup 1)
1268                            (parallel [(const_int 4) (const_int 5)
1269                                       (const_int 6) (const_int 7)]))))]
1270   "TARGET_NEON"
1271   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1272   [(set_attr "vqh_mnem" "<VQH_mnem>")
1273    (set (attr "neon_type")
1274       (if_then_else (eq_attr "vqh_mnem" "vadd")
1275                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1276 )
1277
1278 (define_insn "quad_halves_<code>v16qi"
1279   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1280         (vqh_ops:V8QI
1281           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1282                            (parallel [(const_int 0) (const_int 1)
1283                                       (const_int 2) (const_int 3)
1284                                       (const_int 4) (const_int 5)
1285                                       (const_int 6) (const_int 7)]))
1286           (vec_select:V8QI (match_dup 1)
1287                            (parallel [(const_int 8) (const_int 9)
1288                                       (const_int 10) (const_int 11)
1289                                       (const_int 12) (const_int 13)
1290                                       (const_int 14) (const_int 15)]))))]
1291   "TARGET_NEON"
1292   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1293   [(set_attr "vqh_mnem" "<VQH_mnem>")
1294    (set (attr "neon_type")
1295       (if_then_else (eq_attr "vqh_mnem" "vadd")
1296                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1297 )
1298
1299 (define_expand "move_hi_quad_<mode>"
1300  [(match_operand:ANY128 0 "s_register_operand" "")
1301   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1302  "TARGET_NEON"
1303 {
1304   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1305                                        GET_MODE_SIZE (<V_HALF>mode)),
1306                   operands[1]);
1307   DONE;
1308 })
1309
1310 (define_expand "move_lo_quad_<mode>"
1311  [(match_operand:ANY128 0 "s_register_operand" "")
1312   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1313  "TARGET_NEON"
1314 {
1315   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1316                                        <MODE>mode, 0),
1317                   operands[1]);
1318   DONE;
1319 })
1320
1321 ;; Reduction operations
1322
1323 (define_expand "reduc_splus_<mode>"
1324   [(match_operand:VD 0 "s_register_operand" "")
1325    (match_operand:VD 1 "s_register_operand" "")]
1326   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1327 {
1328   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1329                         &gen_neon_vpadd_internal<mode>);
1330   DONE;
1331 })
1332
1333 (define_expand "reduc_splus_<mode>"
1334   [(match_operand:VQ 0 "s_register_operand" "")
1335    (match_operand:VQ 1 "s_register_operand" "")]
1336   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1337    && !BYTES_BIG_ENDIAN"
1338 {
1339   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1340   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1341
1342   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1343   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1344   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1345
1346   DONE;
1347 })
1348
1349 (define_insn "reduc_splus_v2di"
1350   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1351         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1352                      UNSPEC_VPADD))]
1353   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1354   "vadd.i64\t%e0, %e1, %f1"
1355   [(set_attr "neon_type" "neon_int_1")]
1356 )
1357
1358 ;; NEON does not distinguish between signed and unsigned addition except on
1359 ;; widening operations.
1360 (define_expand "reduc_uplus_<mode>"
1361   [(match_operand:VDQI 0 "s_register_operand" "")
1362    (match_operand:VDQI 1 "s_register_operand" "")]
1363   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1364 {
1365   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1366   DONE;
1367 })
1368
1369 (define_expand "reduc_smin_<mode>"
1370   [(match_operand:VD 0 "s_register_operand" "")
1371    (match_operand:VD 1 "s_register_operand" "")]
1372   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1373 {
1374   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1375                         &gen_neon_vpsmin<mode>);
1376   DONE;
1377 })
1378
1379 (define_expand "reduc_smin_<mode>"
1380   [(match_operand:VQ 0 "s_register_operand" "")
1381    (match_operand:VQ 1 "s_register_operand" "")]
1382   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1383    && !BYTES_BIG_ENDIAN"
1384 {
1385   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1386   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1387
1388   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1389   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1390   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1391
1392   DONE;
1393 })
1394
1395 (define_expand "reduc_smax_<mode>"
1396   [(match_operand:VD 0 "s_register_operand" "")
1397    (match_operand:VD 1 "s_register_operand" "")]
1398   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1399 {
1400   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1401                         &gen_neon_vpsmax<mode>);
1402   DONE;
1403 })
1404
1405 (define_expand "reduc_smax_<mode>"
1406   [(match_operand:VQ 0 "s_register_operand" "")
1407    (match_operand:VQ 1 "s_register_operand" "")]
1408   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1409    && !BYTES_BIG_ENDIAN"
1410 {
1411   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1412   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1413
1414   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1415   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1416   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1417
1418   DONE;
1419 })
1420
1421 (define_expand "reduc_umin_<mode>"
1422   [(match_operand:VDI 0 "s_register_operand" "")
1423    (match_operand:VDI 1 "s_register_operand" "")]
1424   "TARGET_NEON"
1425 {
1426   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1427                         &gen_neon_vpumin<mode>);
1428   DONE;
1429 })
1430
1431 (define_expand "reduc_umin_<mode>"
1432   [(match_operand:VQI 0 "s_register_operand" "")
1433    (match_operand:VQI 1 "s_register_operand" "")]
1434   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1435 {
1436   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1437   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1438
1439   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1440   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1441   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1442
1443   DONE;
1444 })
1445
1446 (define_expand "reduc_umax_<mode>"
1447   [(match_operand:VDI 0 "s_register_operand" "")
1448    (match_operand:VDI 1 "s_register_operand" "")]
1449   "TARGET_NEON"
1450 {
1451   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1452                         &gen_neon_vpumax<mode>);
1453   DONE;
1454 })
1455
1456 (define_expand "reduc_umax_<mode>"
1457   [(match_operand:VQI 0 "s_register_operand" "")
1458    (match_operand:VQI 1 "s_register_operand" "")]
1459   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1460 {
1461   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1462   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1463
1464   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1465   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1466   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1467
1468   DONE;
1469 })
1470
1471 (define_insn "neon_vpadd_internal<mode>"
1472   [(set (match_operand:VD 0 "s_register_operand" "=w")
1473         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1474                     (match_operand:VD 2 "s_register_operand" "w")]
1475                    UNSPEC_VPADD))]
1476   "TARGET_NEON"
1477   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1478   ;; Assume this schedules like vadd.
1479   [(set (attr "neon_type")
1480       (if_then_else (match_test "<Is_float_mode>")
1481                     (if_then_else (match_test "<Is_d_reg>")
1482                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1483                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1484                     (const_string "neon_int_1")))]
1485 )
1486
1487 (define_insn "neon_vpsmin<mode>"
1488   [(set (match_operand:VD 0 "s_register_operand" "=w")
1489         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1490                     (match_operand:VD 2 "s_register_operand" "w")]
1491                    UNSPEC_VPSMIN))]
1492   "TARGET_NEON"
1493   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1494   ;; Assume this schedules like vmin.
1495   [(set (attr "neon_type")
1496       (if_then_else (match_test "<Is_float_mode>")
1497                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1498                     (const_string "neon_int_5")))]
1499 )
1500
1501 (define_insn "neon_vpsmax<mode>"
1502   [(set (match_operand:VD 0 "s_register_operand" "=w")
1503         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1504                     (match_operand:VD 2 "s_register_operand" "w")]
1505                    UNSPEC_VPSMAX))]
1506   "TARGET_NEON"
1507   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1508   ;; Assume this schedules like vmax.
1509   [(set (attr "neon_type")
1510       (if_then_else (match_test "<Is_float_mode>")
1511                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1512                     (const_string "neon_int_5")))]
1513 )
1514
1515 (define_insn "neon_vpumin<mode>"
1516   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1517         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1518                      (match_operand:VDI 2 "s_register_operand" "w")]
1519                    UNSPEC_VPUMIN))]
1520   "TARGET_NEON"
1521   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1522   ;; Assume this schedules like umin.
1523   [(set_attr "neon_type" "neon_int_5")]
1524 )
1525
1526 (define_insn "neon_vpumax<mode>"
1527   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1528         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1529                      (match_operand:VDI 2 "s_register_operand" "w")]
1530                    UNSPEC_VPUMAX))]
1531   "TARGET_NEON"
1532   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1533   ;; Assume this schedules like umax.
1534   [(set_attr "neon_type" "neon_int_5")]
1535 )
1536
1537 ;; Saturating arithmetic
1538
1539 ; NOTE: Neon supports many more saturating variants of instructions than the
1540 ; following, but these are all GCC currently understands.
1541 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1542 ; yet either, although these patterns may be used by intrinsics when they're
1543 ; added.
1544
1545 (define_insn "*ss_add<mode>_neon"
1546   [(set (match_operand:VD 0 "s_register_operand" "=w")
1547        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1548                    (match_operand:VD 2 "s_register_operand" "w")))]
1549   "TARGET_NEON"
1550   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1551   [(set_attr "neon_type" "neon_int_4")]
1552 )
1553
1554 (define_insn "*us_add<mode>_neon"
1555   [(set (match_operand:VD 0 "s_register_operand" "=w")
1556        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1557                    (match_operand:VD 2 "s_register_operand" "w")))]
1558   "TARGET_NEON"
1559   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1560   [(set_attr "neon_type" "neon_int_4")]
1561 )
1562
1563 (define_insn "*ss_sub<mode>_neon"
1564   [(set (match_operand:VD 0 "s_register_operand" "=w")
1565        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1566                     (match_operand:VD 2 "s_register_operand" "w")))]
1567   "TARGET_NEON"
1568   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1569   [(set_attr "neon_type" "neon_int_5")]
1570 )
1571
1572 (define_insn "*us_sub<mode>_neon"
1573   [(set (match_operand:VD 0 "s_register_operand" "=w")
1574        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1575                     (match_operand:VD 2 "s_register_operand" "w")))]
1576   "TARGET_NEON"
1577   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1578   [(set_attr "neon_type" "neon_int_5")]
1579 )
1580
1581 ;; Conditional instructions.  These are comparisons with conditional moves for
1582 ;; vectors.  They perform the assignment:
1583 ;;
1584 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1585 ;;
1586 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1587 ;; element-wise.
1588
1589 (define_expand "vcond<mode><mode>"
1590   [(set (match_operand:VDQW 0 "s_register_operand" "")
1591         (if_then_else:VDQW
1592           (match_operator 3 "arm_comparison_operator"
1593             [(match_operand:VDQW 4 "s_register_operand" "")
1594              (match_operand:VDQW 5 "nonmemory_operand" "")])
1595           (match_operand:VDQW 1 "s_register_operand" "")
1596           (match_operand:VDQW 2 "s_register_operand" "")))]
1597   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1598 {
1599   rtx mask;
1600   int inverse = 0, immediate_zero = 0;
1601   /* See the description of "magic" bits in the 'T' case of
1602      arm_print_operand.  */
1603   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1604                              ? 3 : 1;
1605   rtx magic_rtx = GEN_INT (magic_word);
1606
1607   mask = gen_reg_rtx (<V_cmp_result>mode);
1608
1609   if (operands[5] == CONST0_RTX (<MODE>mode))
1610     immediate_zero = 1;
1611   else if (!REG_P (operands[5]))
1612     operands[5] = force_reg (<MODE>mode, operands[5]);
1613
1614   switch (GET_CODE (operands[3]))
1615     {
1616     case GE:
1617       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1618                                       magic_rtx));
1619       break;
1620
1621     case GT:
1622       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1623                                       magic_rtx));
1624       break;
1625
1626     case EQ:
1627       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1628                                       magic_rtx));
1629       break;
1630
1631     case LE:
1632       if (immediate_zero)
1633         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1634                                         magic_rtx));
1635       else
1636         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1637                                         magic_rtx));
1638       break;
1639
1640     case LT:
1641       if (immediate_zero)
1642         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1643                                         magic_rtx));
1644       else
1645         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1646                                         magic_rtx));
1647       break;
1648
1649     case NE:
1650       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1651                                       magic_rtx));
1652       inverse = 1;
1653       break;
1654
1655     default:
1656       gcc_unreachable ();
1657     }
1658
1659   if (inverse)
1660     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1661                                     operands[1]));
1662   else
1663     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1664                                     operands[2]));
1665
1666   DONE;
1667 })
1668
1669 (define_expand "vcondu<mode><mode>"
1670   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1671         (if_then_else:VDQIW
1672           (match_operator 3 "arm_comparison_operator"
1673             [(match_operand:VDQIW 4 "s_register_operand" "")
1674              (match_operand:VDQIW 5 "s_register_operand" "")])
1675           (match_operand:VDQIW 1 "s_register_operand" "")
1676           (match_operand:VDQIW 2 "s_register_operand" "")))]
1677   "TARGET_NEON"
1678 {
1679   rtx mask;
1680   int inverse = 0, immediate_zero = 0;
1681
1682   mask = gen_reg_rtx (<V_cmp_result>mode);
1683
1684   if (operands[5] == CONST0_RTX (<MODE>mode))
1685     immediate_zero = 1;
1686   else if (!REG_P (operands[5]))
1687     operands[5] = force_reg (<MODE>mode, operands[5]);
1688
1689   switch (GET_CODE (operands[3]))
1690     {
1691     case GEU:
1692       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1693                                       const0_rtx));
1694       break;
1695
1696     case GTU:
1697       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1698                                       const0_rtx));
1699       break;
1700
1701     case EQ:
1702       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1703                                       const0_rtx));
1704       break;
1705
1706     case LEU:
1707       if (immediate_zero)
1708         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1709                                         const0_rtx));
1710       else
1711         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1712                                         const0_rtx));
1713       break;
1714
1715     case LTU:
1716       if (immediate_zero)
1717         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1718                                         const0_rtx));
1719       else
1720         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1721                                         const0_rtx));
1722       break;
1723
1724     case NE:
1725       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1726                                       const0_rtx));
1727       inverse = 1;
1728       break;
1729
1730     default:
1731       gcc_unreachable ();
1732     }
1733
1734   if (inverse)
1735     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1736                                     operands[1]));
1737   else
1738     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1739                                     operands[2]));
1740
1741   DONE;
1742 })
1743
1744 ;; Patterns for builtins.
1745
1746 ; good for plain vadd, vaddq.
1747
1748 (define_expand "neon_vadd<mode>"
1749   [(match_operand:VDQX 0 "s_register_operand" "=w")
1750    (match_operand:VDQX 1 "s_register_operand" "w")
1751    (match_operand:VDQX 2 "s_register_operand" "w")
1752    (match_operand:SI 3 "immediate_operand" "i")]
1753   "TARGET_NEON"
1754 {
1755   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1756     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1757   else
1758     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1759                                            operands[2]));
1760   DONE;
1761 })
1762
1763 ; Note that NEON operations don't support the full IEEE 754 standard: in
1764 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1765 ; use those instructions for autovectorization, etc. unless
1766 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1767 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1768 ; header) must work in either case: if -funsafe-math-optimizations is given,
1769 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1770 ; expand to unspecs (which may potentially limit the extent to which they might
1771 ; be optimized by generic code).
1772
1773 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1774
1775 (define_insn "neon_vadd<mode>_unspec"
1776   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1777         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1778                       (match_operand:VDQX 2 "s_register_operand" "w")]
1779                      UNSPEC_VADD))]
1780   "TARGET_NEON"
1781   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1782   [(set (attr "neon_type")
1783       (if_then_else (match_test "<Is_float_mode>")
1784                     (if_then_else (match_test "<Is_d_reg>")
1785                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1786                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1787                     (const_string "neon_int_1")))]
1788 )
1789
1790 ; operand 3 represents in bits:
1791 ;  bit 0: signed (vs unsigned).
1792 ;  bit 1: rounding (vs none).
1793
1794 (define_insn "neon_vaddl<mode>"
1795   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1796         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1797                            (match_operand:VDI 2 "s_register_operand" "w")
1798                            (match_operand:SI 3 "immediate_operand" "i")]
1799                           UNSPEC_VADDL))]
1800   "TARGET_NEON"
1801   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1802   [(set_attr "neon_type" "neon_int_3")]
1803 )
1804
1805 (define_insn "neon_vaddw<mode>"
1806   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1807         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1808                            (match_operand:VDI 2 "s_register_operand" "w")
1809                            (match_operand:SI 3 "immediate_operand" "i")]
1810                           UNSPEC_VADDW))]
1811   "TARGET_NEON"
1812   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1813   [(set_attr "neon_type" "neon_int_2")]
1814 )
1815
1816 ; vhadd and vrhadd.
1817
1818 (define_insn "neon_vhadd<mode>"
1819   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1820         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1821                        (match_operand:VDQIW 2 "s_register_operand" "w")
1822                        (match_operand:SI 3 "immediate_operand" "i")]
1823                       UNSPEC_VHADD))]
1824   "TARGET_NEON"
1825   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1826   [(set_attr "neon_type" "neon_int_4")]
1827 )
1828
1829 (define_insn "neon_vqadd<mode>"
1830   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1831         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1832                        (match_operand:VDQIX 2 "s_register_operand" "w")
1833                        (match_operand:SI 3 "immediate_operand" "i")]
1834                      UNSPEC_VQADD))]
1835   "TARGET_NEON"
1836   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1837   [(set_attr "neon_type" "neon_int_4")]
1838 )
1839
1840 (define_insn "neon_vaddhn<mode>"
1841   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1842         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1843                             (match_operand:VN 2 "s_register_operand" "w")
1844                             (match_operand:SI 3 "immediate_operand" "i")]
1845                            UNSPEC_VADDHN))]
1846   "TARGET_NEON"
1847   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1848   [(set_attr "neon_type" "neon_int_4")]
1849 )
1850
1851 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1852 ;; polynomial multiplication case that can specified by operand 3.
1853 (define_insn "neon_vmul<mode>"
1854   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1855         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1856                       (match_operand:VDQW 2 "s_register_operand" "w")
1857                       (match_operand:SI 3 "immediate_operand" "i")]
1858                      UNSPEC_VMUL))]
1859   "TARGET_NEON"
1860   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1861   [(set (attr "neon_type")
1862       (if_then_else (match_test "<Is_float_mode>")
1863                     (if_then_else (match_test "<Is_d_reg>")
1864                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1865                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1866                     (if_then_else (match_test "<Is_d_reg>")
1867                                   (if_then_else
1868                                     (match_test "<Scalar_mul_8_16>")
1869                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1870                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1871                                   (if_then_else (match_test "<Scalar_mul_8_16>")
1872                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
1873                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1874 )
1875
1876 (define_expand "neon_vmla<mode>"
1877   [(match_operand:VDQW 0 "s_register_operand" "=w")
1878    (match_operand:VDQW 1 "s_register_operand" "0")
1879    (match_operand:VDQW 2 "s_register_operand" "w")
1880    (match_operand:VDQW 3 "s_register_operand" "w")
1881    (match_operand:SI 4 "immediate_operand" "i")]
1882   "TARGET_NEON"
1883 {
1884   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1885     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1886                                              operands[2], operands[3]));
1887   else
1888     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1889                                            operands[2], operands[3]));
1890   DONE;
1891 })
1892
1893 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1894
1895 (define_insn "neon_vmla<mode>_unspec"
1896   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1897         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1898                      (match_operand:VDQ 2 "s_register_operand" "w")
1899                      (match_operand:VDQ 3 "s_register_operand" "w")]
1900                     UNSPEC_VMLA))]
1901   "TARGET_NEON"
1902   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1903   [(set (attr "neon_type")
1904       (if_then_else (match_test "<Is_float_mode>")
1905                     (if_then_else (match_test "<Is_d_reg>")
1906                                   (const_string "neon_fp_vmla_ddd")
1907                                   (const_string "neon_fp_vmla_qqq"))
1908                     (if_then_else (match_test "<Is_d_reg>")
1909                                   (if_then_else
1910                                     (match_test "<Scalar_mul_8_16>")
1911                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1912                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1913                                   (if_then_else (match_test "<Scalar_mul_8_16>")
1914                                     (const_string "neon_mla_qqq_8_16")
1915                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1916 )
1917
1918 (define_insn "neon_vmlal<mode>"
1919   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1920         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1921                            (match_operand:VW 2 "s_register_operand" "w")
1922                            (match_operand:VW 3 "s_register_operand" "w")
1923                            (match_operand:SI 4 "immediate_operand" "i")]
1924                           UNSPEC_VMLAL))]
1925   "TARGET_NEON"
1926   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1927   [(set (attr "neon_type")
1928      (if_then_else (match_test "<Scalar_mul_8_16>")
1929                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1930                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1931 )
1932
1933 (define_expand "neon_vmls<mode>"
1934   [(match_operand:VDQW 0 "s_register_operand" "=w")
1935    (match_operand:VDQW 1 "s_register_operand" "0")
1936    (match_operand:VDQW 2 "s_register_operand" "w")
1937    (match_operand:VDQW 3 "s_register_operand" "w")
1938    (match_operand:SI 4 "immediate_operand" "i")]
1939   "TARGET_NEON"
1940 {
1941   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1942     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1943                  operands[1], operands[2], operands[3]));
1944   else
1945     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1946                                            operands[2], operands[3]));
1947   DONE;
1948 })
1949
1950 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1951
1952 (define_insn "neon_vmls<mode>_unspec"
1953   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1954         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1955                      (match_operand:VDQ 2 "s_register_operand" "w")
1956                      (match_operand:VDQ 3 "s_register_operand" "w")]
1957                     UNSPEC_VMLS))]
1958   "TARGET_NEON"
1959   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1960   [(set (attr "neon_type")
1961       (if_then_else (match_test "<Is_float_mode>")
1962                     (if_then_else (match_test "<Is_d_reg>")
1963                                   (const_string "neon_fp_vmla_ddd")
1964                                   (const_string "neon_fp_vmla_qqq"))
1965                     (if_then_else (match_test "<Is_d_reg>")
1966                                   (if_then_else
1967                                     (match_test "<Scalar_mul_8_16>")
1968                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1969                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1970                                   (if_then_else
1971                                     (match_test "<Scalar_mul_8_16>")
1972                                     (const_string "neon_mla_qqq_8_16")
1973                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1974 )
1975
1976 (define_insn "neon_vmlsl<mode>"
1977   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1978         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1979                            (match_operand:VW 2 "s_register_operand" "w")
1980                            (match_operand:VW 3 "s_register_operand" "w")
1981                            (match_operand:SI 4 "immediate_operand" "i")]
1982                           UNSPEC_VMLSL))]
1983   "TARGET_NEON"
1984   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1985   [(set (attr "neon_type")
1986      (if_then_else (match_test "<Scalar_mul_8_16>")
1987                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1988                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1989 )
1990
1991 (define_insn "neon_vqdmulh<mode>"
1992   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
1993         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
1994                        (match_operand:VMDQI 2 "s_register_operand" "w")
1995                        (match_operand:SI 3 "immediate_operand" "i")]
1996                       UNSPEC_VQDMULH))]
1997   "TARGET_NEON"
1998   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1999   [(set (attr "neon_type")
2000      (if_then_else (match_test "<Is_d_reg>")
2001         (if_then_else (match_test "<Scalar_mul_8_16>")
2002                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2003                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2004         (if_then_else (match_test "<Scalar_mul_8_16>")
2005                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2006                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2007 )
2008
2009 (define_insn "neon_vqdmlal<mode>"
2010   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2011         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2012                            (match_operand:VMDI 2 "s_register_operand" "w")
2013                            (match_operand:VMDI 3 "s_register_operand" "w")
2014                            (match_operand:SI 4 "immediate_operand" "i")]
2015                           UNSPEC_VQDMLAL))]
2016   "TARGET_NEON"
2017   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2018   [(set (attr "neon_type")
2019      (if_then_else (match_test "<Scalar_mul_8_16>")
2020                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2021                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2022 )
2023
2024 (define_insn "neon_vqdmlsl<mode>"
2025   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2026         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2027                            (match_operand:VMDI 2 "s_register_operand" "w")
2028                            (match_operand:VMDI 3 "s_register_operand" "w")
2029                            (match_operand:SI 4 "immediate_operand" "i")]
2030                           UNSPEC_VQDMLSL))]
2031   "TARGET_NEON"
2032   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2033   [(set (attr "neon_type")
2034      (if_then_else (match_test "<Scalar_mul_8_16>")
2035                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2036                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2037 )
2038
2039 (define_insn "neon_vmull<mode>"
2040   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2041         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2042                            (match_operand:VW 2 "s_register_operand" "w")
2043                            (match_operand:SI 3 "immediate_operand" "i")]
2044                           UNSPEC_VMULL))]
2045   "TARGET_NEON"
2046   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2047   [(set (attr "neon_type")
2048      (if_then_else (match_test "<Scalar_mul_8_16>")
2049                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2050                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2051 )
2052
2053 (define_insn "neon_vqdmull<mode>"
2054   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2055         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2056                            (match_operand:VMDI 2 "s_register_operand" "w")
2057                            (match_operand:SI 3 "immediate_operand" "i")]
2058                           UNSPEC_VQDMULL))]
2059   "TARGET_NEON"
2060   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2061   [(set (attr "neon_type")
2062      (if_then_else (match_test "<Scalar_mul_8_16>")
2063                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2064                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2065 )
2066
2067 (define_expand "neon_vsub<mode>"
2068   [(match_operand:VDQX 0 "s_register_operand" "=w")
2069    (match_operand:VDQX 1 "s_register_operand" "w")
2070    (match_operand:VDQX 2 "s_register_operand" "w")
2071    (match_operand:SI 3 "immediate_operand" "i")]
2072   "TARGET_NEON"
2073 {
2074   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2075     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2076   else
2077     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2078                                            operands[2]));
2079   DONE;
2080 })
2081
2082 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2083
2084 (define_insn "neon_vsub<mode>_unspec"
2085   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2086         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2087                       (match_operand:VDQX 2 "s_register_operand" "w")]
2088                      UNSPEC_VSUB))]
2089   "TARGET_NEON"
2090   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2091   [(set (attr "neon_type")
2092       (if_then_else (match_test "<Is_float_mode>")
2093                     (if_then_else (match_test "<Is_d_reg>")
2094                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2095                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2096                     (const_string "neon_int_2")))]
2097 )
2098
2099 (define_insn "neon_vsubl<mode>"
2100   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2101         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2102                            (match_operand:VDI 2 "s_register_operand" "w")
2103                            (match_operand:SI 3 "immediate_operand" "i")]
2104                           UNSPEC_VSUBL))]
2105   "TARGET_NEON"
2106   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2107   [(set_attr "neon_type" "neon_int_2")]
2108 )
2109
2110 (define_insn "neon_vsubw<mode>"
2111   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2112         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2113                            (match_operand:VDI 2 "s_register_operand" "w")
2114                            (match_operand:SI 3 "immediate_operand" "i")]
2115                           UNSPEC_VSUBW))]
2116   "TARGET_NEON"
2117   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2118   [(set_attr "neon_type" "neon_int_2")]
2119 )
2120
2121 (define_insn "neon_vqsub<mode>"
2122   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2123         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2124                        (match_operand:VDQIX 2 "s_register_operand" "w")
2125                        (match_operand:SI 3 "immediate_operand" "i")]
2126                       UNSPEC_VQSUB))]
2127   "TARGET_NEON"
2128   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2129   [(set_attr "neon_type" "neon_int_5")]
2130 )
2131
2132 (define_insn "neon_vhsub<mode>"
2133   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2134         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2135                        (match_operand:VDQIW 2 "s_register_operand" "w")
2136                        (match_operand:SI 3 "immediate_operand" "i")]
2137                       UNSPEC_VHSUB))]
2138   "TARGET_NEON"
2139   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2140   [(set_attr "neon_type" "neon_int_5")]
2141 )
2142
2143 (define_insn "neon_vsubhn<mode>"
2144   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2145         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2146                             (match_operand:VN 2 "s_register_operand" "w")
2147                             (match_operand:SI 3 "immediate_operand" "i")]
2148                            UNSPEC_VSUBHN))]
2149   "TARGET_NEON"
2150   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2151   [(set_attr "neon_type" "neon_int_4")]
2152 )
2153
2154 (define_insn "neon_vceq<mode>"
2155   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2156         (unspec:<V_cmp_result>
2157           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2158            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2159            (match_operand:SI 3 "immediate_operand" "i,i")]
2160           UNSPEC_VCEQ))]
2161   "TARGET_NEON"
2162   "@
2163   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2164   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2165   [(set (attr "neon_type")
2166       (if_then_else (match_test "<Is_float_mode>")
2167                     (if_then_else (match_test "<Is_d_reg>")
2168                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2169                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2170                     (const_string "neon_int_5")))]
2171 )
2172
2173 (define_insn "neon_vcge<mode>"
2174   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2175         (unspec:<V_cmp_result>
2176           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2177            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2178            (match_operand:SI 3 "immediate_operand" "i,i")]
2179           UNSPEC_VCGE))]
2180   "TARGET_NEON"
2181   "@
2182   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2183   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2184   [(set (attr "neon_type")
2185      (if_then_else (match_test "<Is_float_mode>")
2186                    (if_then_else (match_test "<Is_d_reg>")
2187                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2188                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2189                    (const_string "neon_int_5")))]
2190 )
2191
2192 (define_insn "neon_vcgeu<mode>"
2193   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2194         (unspec:<V_cmp_result>
2195           [(match_operand:VDQIW 1 "s_register_operand" "w")
2196            (match_operand:VDQIW 2 "s_register_operand" "w")
2197            (match_operand:SI 3 "immediate_operand" "i")]
2198           UNSPEC_VCGEU))]
2199   "TARGET_NEON"
2200   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2201   [(set_attr "neon_type" "neon_int_5")]
2202 )
2203
2204 (define_insn "neon_vcgt<mode>"
2205   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2206         (unspec:<V_cmp_result>
2207           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2208            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2209            (match_operand:SI 3 "immediate_operand" "i,i")]
2210           UNSPEC_VCGT))]
2211   "TARGET_NEON"
2212   "@
2213   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2214   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2215   [(set (attr "neon_type")
2216      (if_then_else (match_test "<Is_float_mode>")
2217                    (if_then_else (match_test "<Is_d_reg>")
2218                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2219                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2220                    (const_string "neon_int_5")))]
2221 )
2222
2223 (define_insn "neon_vcgtu<mode>"
2224   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2225         (unspec:<V_cmp_result>
2226           [(match_operand:VDQIW 1 "s_register_operand" "w")
2227            (match_operand:VDQIW 2 "s_register_operand" "w")
2228            (match_operand:SI 3 "immediate_operand" "i")]
2229           UNSPEC_VCGTU))]
2230   "TARGET_NEON"
2231   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2232   [(set_attr "neon_type" "neon_int_5")]
2233 )
2234
2235 ;; VCLE and VCLT only support comparisons with immediate zero (register
2236 ;; variants are VCGE and VCGT with operands reversed).
2237
2238 (define_insn "neon_vcle<mode>"
2239   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2240         (unspec:<V_cmp_result>
2241           [(match_operand:VDQW 1 "s_register_operand" "w")
2242            (match_operand:VDQW 2 "zero_operand" "Dz")
2243            (match_operand:SI 3 "immediate_operand" "i")]
2244           UNSPEC_VCLE))]
2245   "TARGET_NEON"
2246   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2247   [(set (attr "neon_type")
2248       (if_then_else (match_test "<Is_float_mode>")
2249                     (if_then_else (match_test "<Is_d_reg>")
2250                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2251                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2252                     (const_string "neon_int_5")))]
2253 )
2254
2255 (define_insn "neon_vclt<mode>"
2256   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2257         (unspec:<V_cmp_result>
2258           [(match_operand:VDQW 1 "s_register_operand" "w")
2259            (match_operand:VDQW 2 "zero_operand" "Dz")
2260            (match_operand:SI 3 "immediate_operand" "i")]
2261           UNSPEC_VCLT))]
2262   "TARGET_NEON"
2263   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2264   [(set (attr "neon_type")
2265       (if_then_else (match_test "<Is_float_mode>")
2266                     (if_then_else (match_test "<Is_d_reg>")
2267                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2268                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2269                     (const_string "neon_int_5")))]
2270 )
2271
2272 (define_insn "neon_vcage<mode>"
2273   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2274         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2275                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2276                                 (match_operand:SI 3 "immediate_operand" "i")]
2277                                UNSPEC_VCAGE))]
2278   "TARGET_NEON"
2279   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2280   [(set (attr "neon_type")
2281      (if_then_else (match_test "<Is_d_reg>")
2282                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2283                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2284 )
2285
2286 (define_insn "neon_vcagt<mode>"
2287   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2288         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2289                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2290                                 (match_operand:SI 3 "immediate_operand" "i")]
2291                                UNSPEC_VCAGT))]
2292   "TARGET_NEON"
2293   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2294   [(set (attr "neon_type")
2295      (if_then_else (match_test "<Is_d_reg>")
2296                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2297                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2298 )
2299
2300 (define_insn "neon_vtst<mode>"
2301   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2302         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2303                        (match_operand:VDQIW 2 "s_register_operand" "w")
2304                        (match_operand:SI 3 "immediate_operand" "i")]
2305                       UNSPEC_VTST))]
2306   "TARGET_NEON"
2307   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2308   [(set_attr "neon_type" "neon_int_4")]
2309 )
2310
2311 (define_insn "neon_vabd<mode>"
2312   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2313         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2314                       (match_operand:VDQW 2 "s_register_operand" "w")
2315                       (match_operand:SI 3 "immediate_operand" "i")]
2316                      UNSPEC_VABD))]
2317   "TARGET_NEON"
2318   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2319   [(set (attr "neon_type")
2320      (if_then_else (match_test "<Is_float_mode>")
2321                    (if_then_else (match_test "<Is_d_reg>")
2322                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2323                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2324                    (const_string "neon_int_5")))]
2325 )
2326
2327 (define_insn "neon_vabdl<mode>"
2328   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2329         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2330                            (match_operand:VW 2 "s_register_operand" "w")
2331                            (match_operand:SI 3 "immediate_operand" "i")]
2332                           UNSPEC_VABDL))]
2333   "TARGET_NEON"
2334   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2335   [(set_attr "neon_type" "neon_int_5")]
2336 )
2337
2338 (define_insn "neon_vaba<mode>"
2339   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2340         (plus:VDQIW (match_operand:VDQIW 1 "s_register_operand" "0")
2341                     (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2342                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2343                                    (match_operand:SI 4 "immediate_operand" "i")]
2344                                   UNSPEC_VABD)))]
2345   "TARGET_NEON"
2346   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2347   [(set (attr "neon_type")
2348      (if_then_else (match_test "<Is_d_reg>")
2349                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2350 )
2351
2352 (define_insn "neon_vabal<mode>"
2353   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2354         (plus:<V_widen> (match_operand:<V_widen> 1 "s_register_operand" "0")
2355                         (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2356                                            (match_operand:VW 3 "s_register_operand" "w")
2357                                            (match_operand:SI 4 "immediate_operand" "i")]
2358                           UNSPEC_VABDL)))]
2359   "TARGET_NEON"
2360   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2361   [(set_attr "neon_type" "neon_vaba")]
2362 )
2363
2364 (define_insn "neon_vmax<mode>"
2365   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2366         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2367                       (match_operand:VDQW 2 "s_register_operand" "w")
2368                       (match_operand:SI 3 "immediate_operand" "i")]
2369                      UNSPEC_VMAX))]
2370   "TARGET_NEON"
2371   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2372   [(set (attr "neon_type")
2373     (if_then_else (match_test "<Is_float_mode>")
2374                   (if_then_else (match_test "<Is_d_reg>")
2375                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2376                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2377                   (const_string "neon_int_5")))]
2378 )
2379
2380 (define_insn "neon_vmin<mode>"
2381   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2382         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2383                       (match_operand:VDQW 2 "s_register_operand" "w")
2384                       (match_operand:SI 3 "immediate_operand" "i")]
2385                      UNSPEC_VMIN))]
2386   "TARGET_NEON"
2387   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2388   [(set (attr "neon_type")
2389     (if_then_else (match_test "<Is_float_mode>")
2390                   (if_then_else (match_test "<Is_d_reg>")
2391                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2392                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2393                   (const_string "neon_int_5")))]
2394 )
2395
2396 (define_expand "neon_vpadd<mode>"
2397   [(match_operand:VD 0 "s_register_operand" "=w")
2398    (match_operand:VD 1 "s_register_operand" "w")
2399    (match_operand:VD 2 "s_register_operand" "w")
2400    (match_operand:SI 3 "immediate_operand" "i")]
2401   "TARGET_NEON"
2402 {
2403   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2404                                             operands[2]));
2405   DONE;
2406 })
2407
2408 (define_insn "neon_vpaddl<mode>"
2409   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2410         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2411                                   (match_operand:SI 2 "immediate_operand" "i")]
2412                                  UNSPEC_VPADDL))]
2413   "TARGET_NEON"
2414   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2415   ;; Assume this schedules like vaddl.
2416   [(set_attr "neon_type" "neon_int_3")]
2417 )
2418
2419 (define_insn "neon_vpadal<mode>"
2420   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2421         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2422                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2423                                   (match_operand:SI 3 "immediate_operand" "i")]
2424                                  UNSPEC_VPADAL))]
2425   "TARGET_NEON"
2426   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2427   ;; Assume this schedules like vpadd.
2428   [(set_attr "neon_type" "neon_int_1")]
2429 )
2430
2431 (define_insn "neon_vpmax<mode>"
2432   [(set (match_operand:VD 0 "s_register_operand" "=w")
2433         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2434                     (match_operand:VD 2 "s_register_operand" "w")
2435                     (match_operand:SI 3 "immediate_operand" "i")]
2436                    UNSPEC_VPMAX))]
2437   "TARGET_NEON"
2438   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2439   ;; Assume this schedules like vmax.
2440   [(set (attr "neon_type")
2441     (if_then_else (match_test "<Is_float_mode>")
2442                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2443                   (const_string "neon_int_5")))]
2444 )
2445
2446 (define_insn "neon_vpmin<mode>"
2447   [(set (match_operand:VD 0 "s_register_operand" "=w")
2448         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2449                     (match_operand:VD 2 "s_register_operand" "w")
2450                     (match_operand:SI 3 "immediate_operand" "i")]
2451                    UNSPEC_VPMIN))]
2452   "TARGET_NEON"
2453   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2454   ;; Assume this schedules like vmin.
2455   [(set (attr "neon_type")
2456     (if_then_else (match_test "<Is_float_mode>")
2457                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2458                   (const_string "neon_int_5")))]
2459 )
2460
2461 (define_insn "neon_vrecps<mode>"
2462   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2463         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2464                        (match_operand:VCVTF 2 "s_register_operand" "w")
2465                        (match_operand:SI 3 "immediate_operand" "i")]
2466                       UNSPEC_VRECPS))]
2467   "TARGET_NEON"
2468   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2469   [(set (attr "neon_type")
2470       (if_then_else (match_test "<Is_d_reg>")
2471                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2472                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2473 )
2474
2475 (define_insn "neon_vrsqrts<mode>"
2476   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2477         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2478                        (match_operand:VCVTF 2 "s_register_operand" "w")
2479                        (match_operand:SI 3 "immediate_operand" "i")]
2480                       UNSPEC_VRSQRTS))]
2481   "TARGET_NEON"
2482   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2483   [(set (attr "neon_type")
2484       (if_then_else (match_test "<Is_d_reg>")
2485                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2486                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2487 )
2488
2489 (define_expand "neon_vabs<mode>"
2490   [(match_operand:VDQW 0 "s_register_operand" "")
2491    (match_operand:VDQW 1 "s_register_operand" "")
2492    (match_operand:SI 2 "immediate_operand" "")]
2493   "TARGET_NEON"
2494 {
2495   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2496   DONE;
2497 })
2498
2499 (define_insn "neon_vqabs<mode>"
2500   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2501         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2502                        (match_operand:SI 2 "immediate_operand" "i")]
2503                       UNSPEC_VQABS))]
2504   "TARGET_NEON"
2505   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2506   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2507 )
2508
2509 (define_expand "neon_vneg<mode>"
2510   [(match_operand:VDQW 0 "s_register_operand" "")
2511    (match_operand:VDQW 1 "s_register_operand" "")
2512    (match_operand:SI 2 "immediate_operand" "")]
2513   "TARGET_NEON"
2514 {
2515   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2516   DONE;
2517 })
2518
2519 (define_insn "neon_vqneg<mode>"
2520   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2521         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2522                        (match_operand:SI 2 "immediate_operand" "i")]
2523                       UNSPEC_VQNEG))]
2524   "TARGET_NEON"
2525   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2526   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2527 )
2528
2529 (define_insn "neon_vcls<mode>"
2530   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2531         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2532                        (match_operand:SI 2 "immediate_operand" "i")]
2533                       UNSPEC_VCLS))]
2534   "TARGET_NEON"
2535   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2536   [(set_attr "neon_type" "neon_int_1")]
2537 )
2538
2539 (define_insn "clz<mode>2"
2540   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2541         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2542   "TARGET_NEON"
2543   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2544   [(set_attr "neon_type" "neon_int_1")]
2545 )
2546
2547 (define_expand "neon_vclz<mode>"
2548   [(match_operand:VDQIW 0 "s_register_operand" "")
2549    (match_operand:VDQIW 1 "s_register_operand" "")
2550    (match_operand:SI 2 "immediate_operand" "")]
2551   "TARGET_NEON"
2552 {
2553   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2554   DONE;
2555 })
2556
2557 (define_insn "popcount<mode>2"
2558   [(set (match_operand:VE 0 "s_register_operand" "=w")
2559         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2560   "TARGET_NEON"
2561   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2562   [(set_attr "neon_type" "neon_int_1")]
2563 )
2564
2565 (define_expand "neon_vcnt<mode>"
2566   [(match_operand:VE 0 "s_register_operand" "=w")
2567    (match_operand:VE 1 "s_register_operand" "w")
2568    (match_operand:SI 2 "immediate_operand" "i")]
2569   "TARGET_NEON"
2570 {
2571   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2572   DONE;
2573 })
2574
2575 (define_insn "neon_vrecpe<mode>"
2576   [(set (match_operand:V32 0 "s_register_operand" "=w")
2577         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2578                      (match_operand:SI 2 "immediate_operand" "i")]
2579                     UNSPEC_VRECPE))]
2580   "TARGET_NEON"
2581   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2582   [(set (attr "neon_type")
2583       (if_then_else (match_test "<Is_d_reg>")
2584                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2585                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2586 )
2587
2588 (define_insn "neon_vrsqrte<mode>"
2589   [(set (match_operand:V32 0 "s_register_operand" "=w")
2590         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2591                      (match_operand:SI 2 "immediate_operand" "i")]
2592                     UNSPEC_VRSQRTE))]
2593   "TARGET_NEON"
2594   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2595   [(set (attr "neon_type")
2596       (if_then_else (match_test "<Is_d_reg>")
2597                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2598                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2599 )
2600
2601 (define_expand "neon_vmvn<mode>"
2602   [(match_operand:VDQIW 0 "s_register_operand" "")
2603    (match_operand:VDQIW 1 "s_register_operand" "")
2604    (match_operand:SI 2 "immediate_operand" "")]
2605   "TARGET_NEON"
2606 {
2607   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2608   DONE;
2609 })
2610
2611 (define_insn "neon_vget_lane<mode>_sext_internal"
2612   [(set (match_operand:SI 0 "s_register_operand" "=r")
2613         (sign_extend:SI
2614           (vec_select:<V_elem>
2615             (match_operand:VD 1 "s_register_operand" "w")
2616             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2617   "TARGET_NEON"
2618 {
2619   if (BYTES_BIG_ENDIAN)
2620     {
2621       int elt = INTVAL (operands[2]);
2622       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2623       operands[2] = GEN_INT (elt);
2624     }
2625   return "vmov%?.s<V_sz_elem>\t%0, %P1[%c2]";
2626 }
2627   [(set_attr "predicable" "yes")
2628    (set_attr "neon_type" "neon_bp_simple")]
2629 )
2630
2631 (define_insn "neon_vget_lane<mode>_zext_internal"
2632   [(set (match_operand:SI 0 "s_register_operand" "=r")
2633         (zero_extend:SI
2634           (vec_select:<V_elem>
2635             (match_operand:VD 1 "s_register_operand" "w")
2636             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2637   "TARGET_NEON"
2638 {
2639   if (BYTES_BIG_ENDIAN)
2640     {
2641       int elt = INTVAL (operands[2]);
2642       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2643       operands[2] = GEN_INT (elt);
2644     }
2645   return "vmov%?.u<V_sz_elem>\t%0, %P1[%c2]";
2646 }
2647   [(set_attr "predicable" "yes")
2648    (set_attr "neon_type" "neon_bp_simple")]
2649 )
2650
2651 (define_insn "neon_vget_lane<mode>_sext_internal"
2652   [(set (match_operand:SI 0 "s_register_operand" "=r")
2653         (sign_extend:SI
2654           (vec_select:<V_elem>
2655             (match_operand:VQ 1 "s_register_operand" "w")
2656             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2657   "TARGET_NEON"
2658 {
2659   rtx ops[3];
2660   int regno = REGNO (operands[1]);
2661   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2662   unsigned int elt = INTVAL (operands[2]);
2663   unsigned int elt_adj = elt % halfelts;
2664
2665   if (BYTES_BIG_ENDIAN)
2666     elt_adj = halfelts - 1 - elt_adj;
2667
2668   ops[0] = operands[0];
2669   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2670   ops[2] = GEN_INT (elt_adj);
2671   output_asm_insn ("vmov%?.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2672
2673   return "";
2674 }
2675   [(set_attr "predicable" "yes")
2676    (set_attr "neon_type" "neon_bp_simple")]
2677 )
2678
2679 (define_insn "neon_vget_lane<mode>_zext_internal"
2680   [(set (match_operand:SI 0 "s_register_operand" "=r")
2681         (zero_extend:SI
2682           (vec_select:<V_elem>
2683             (match_operand:VQ 1 "s_register_operand" "w")
2684             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2685   "TARGET_NEON"
2686 {
2687   rtx ops[3];
2688   int regno = REGNO (operands[1]);
2689   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2690   unsigned int elt = INTVAL (operands[2]);
2691   unsigned int elt_adj = elt % halfelts;
2692
2693   if (BYTES_BIG_ENDIAN)
2694     elt_adj = halfelts - 1 - elt_adj;
2695
2696   ops[0] = operands[0];
2697   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2698   ops[2] = GEN_INT (elt_adj);
2699   output_asm_insn ("vmov%?.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2700
2701   return "";
2702 }
2703   [(set_attr "predicable" "yes")
2704    (set_attr "neon_type" "neon_bp_simple")]
2705 )
2706
2707 (define_expand "neon_vget_lane<mode>"
2708   [(match_operand:<V_ext> 0 "s_register_operand" "")
2709    (match_operand:VDQW 1 "s_register_operand" "")
2710    (match_operand:SI 2 "immediate_operand" "")
2711    (match_operand:SI 3 "immediate_operand" "")]
2712   "TARGET_NEON"
2713 {
2714   HOST_WIDE_INT magic = INTVAL (operands[3]);
2715   rtx insn;
2716
2717   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2718
2719   if (BYTES_BIG_ENDIAN)
2720     {
2721       /* The intrinsics are defined in terms of a model where the
2722          element ordering in memory is vldm order, whereas the generic
2723          RTL is defined in terms of a model where the element ordering
2724          in memory is array order.  Convert the lane number to conform
2725          to this model.  */
2726       unsigned int elt = INTVAL (operands[2]);
2727       unsigned int reg_nelts
2728         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2729       elt ^= reg_nelts - 1;
2730       operands[2] = GEN_INT (elt);
2731     }
2732
2733   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2734     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2735   else
2736     {
2737       if ((magic & 1) != 0)
2738         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2739                                                        operands[2]);
2740       else
2741         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2742                                                        operands[2]);
2743     }
2744   emit_insn (insn);
2745   DONE;
2746 })
2747
2748 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2749 ; elements.
2750
2751 (define_expand "neon_vget_lanedi"
2752   [(match_operand:DI 0 "s_register_operand" "=r")
2753    (match_operand:DI 1 "s_register_operand" "w")
2754    (match_operand:SI 2 "immediate_operand" "i")
2755    (match_operand:SI 3 "immediate_operand" "i")]
2756   "TARGET_NEON"
2757 {
2758   neon_lane_bounds (operands[2], 0, 1);
2759   emit_move_insn (operands[0], operands[1]);
2760   DONE;
2761 })
2762
2763 (define_expand "neon_vget_lanev2di"
2764   [(match_operand:DI 0 "s_register_operand" "")
2765    (match_operand:V2DI 1 "s_register_operand" "")
2766    (match_operand:SI 2 "immediate_operand" "")
2767    (match_operand:SI 3 "immediate_operand" "")]
2768   "TARGET_NEON"
2769 {
2770   switch (INTVAL (operands[2]))
2771     {
2772     case 0:
2773       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2774       break;
2775     case 1:
2776       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2777       break;
2778     default:
2779       neon_lane_bounds (operands[2], 0, 1);
2780       FAIL;
2781     }
2782   DONE;
2783 })
2784
2785 (define_expand "neon_vset_lane<mode>"
2786   [(match_operand:VDQ 0 "s_register_operand" "=w")
2787    (match_operand:<V_elem> 1 "s_register_operand" "r")
2788    (match_operand:VDQ 2 "s_register_operand" "0")
2789    (match_operand:SI 3 "immediate_operand" "i")]
2790   "TARGET_NEON"
2791 {
2792   unsigned int elt = INTVAL (operands[3]);
2793   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2794
2795   if (BYTES_BIG_ENDIAN)
2796     {
2797       unsigned int reg_nelts
2798         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2799       elt ^= reg_nelts - 1;
2800     }
2801
2802   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2803                                          GEN_INT (1 << elt), operands[2]));
2804   DONE;
2805 })
2806
2807 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2808
2809 (define_expand "neon_vset_lanedi"
2810   [(match_operand:DI 0 "s_register_operand" "=w")
2811    (match_operand:DI 1 "s_register_operand" "r")
2812    (match_operand:DI 2 "s_register_operand" "0")
2813    (match_operand:SI 3 "immediate_operand" "i")]
2814   "TARGET_NEON"
2815 {
2816   neon_lane_bounds (operands[3], 0, 1);
2817   emit_move_insn (operands[0], operands[1]);
2818   DONE;
2819 })
2820
2821 (define_expand "neon_vcreate<mode>"
2822   [(match_operand:VDX 0 "s_register_operand" "")
2823    (match_operand:DI 1 "general_operand" "")]
2824   "TARGET_NEON"
2825 {
2826   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2827   emit_move_insn (operands[0], src);
2828   DONE;
2829 })
2830
2831 (define_insn "neon_vdup_n<mode>"
2832   [(set (match_operand:VX 0 "s_register_operand" "=w")
2833         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2834   "TARGET_NEON"
2835   "vdup%?.<V_sz_elem>\t%<V_reg>0, %1"
2836   ;; Assume this schedules like vmov.
2837   [(set_attr "predicable" "yes")
2838    (set_attr "neon_type" "neon_bp_simple")]
2839 )
2840
2841 (define_insn "neon_vdup_n<mode>"
2842   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2843         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2844   "TARGET_NEON"
2845   "@
2846   vdup%?.<V_sz_elem>\t%<V_reg>0, %1
2847   vdup%?.<V_sz_elem>\t%<V_reg>0, %y1"
2848   ;; Assume this schedules like vmov.
2849   [(set_attr "predicable" "yes")
2850    (set_attr "neon_type" "neon_bp_simple")]
2851 )
2852
2853 (define_expand "neon_vdup_ndi"
2854   [(match_operand:DI 0 "s_register_operand" "=w")
2855    (match_operand:DI 1 "s_register_operand" "r")]
2856   "TARGET_NEON"
2857 {
2858   emit_move_insn (operands[0], operands[1]);
2859   DONE;
2860 }
2861 )
2862
2863 (define_insn "neon_vdup_nv2di"
2864   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2865         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2866   "TARGET_NEON"
2867   "@
2868   vmov%?\t%e0, %Q1, %R1\;vmov%?\t%f0, %Q1, %R1
2869   vmov%?\t%e0, %P1\;vmov%?\t%f0, %P1"
2870   [(set_attr "predicable" "yes")
2871    (set_attr "length" "8")
2872    (set_attr "neon_type" "neon_bp_simple")]
2873 )
2874
2875 (define_insn "neon_vdup_lane<mode>_internal"
2876   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2877         (vec_duplicate:VDQW
2878           (vec_select:<V_elem>
2879             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2880             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2881   "TARGET_NEON"
2882 {
2883   if (BYTES_BIG_ENDIAN)
2884     {
2885       int elt = INTVAL (operands[2]);
2886       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2887       operands[2] = GEN_INT (elt);
2888     }
2889   if (<Is_d_reg>)
2890     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2891   else
2892     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2893 }
2894   ;; Assume this schedules like vmov.
2895   [(set_attr "neon_type" "neon_bp_simple")]
2896 )
2897
2898 (define_expand "neon_vdup_lane<mode>"
2899   [(match_operand:VDQW 0 "s_register_operand" "=w")
2900    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2901    (match_operand:SI 2 "immediate_operand" "i")]
2902   "TARGET_NEON"
2903 {
2904   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2905   if (BYTES_BIG_ENDIAN)
2906     {
2907       unsigned int elt = INTVAL (operands[2]);
2908       unsigned int reg_nelts
2909         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2910       elt ^= reg_nelts - 1;
2911       operands[2] = GEN_INT (elt);
2912     }
2913     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2914                                                   operands[2]));
2915     DONE;
2916 })
2917
2918 ; Scalar index is ignored, since only zero is valid here.
2919 (define_expand "neon_vdup_lanedi"
2920   [(match_operand:DI 0 "s_register_operand" "=w")
2921    (match_operand:DI 1 "s_register_operand" "w")
2922    (match_operand:SI 2 "immediate_operand" "i")]
2923   "TARGET_NEON"
2924 {
2925   neon_lane_bounds (operands[2], 0, 1);
2926   emit_move_insn (operands[0], operands[1]);
2927   DONE;
2928 })
2929
2930 ; Likewise for v2di, as the DImode second operand has only a single element.
2931 (define_expand "neon_vdup_lanev2di"
2932   [(match_operand:V2DI 0 "s_register_operand" "=w")
2933    (match_operand:DI 1 "s_register_operand" "w")
2934    (match_operand:SI 2 "immediate_operand" "i")]
2935   "TARGET_NEON"
2936 {
2937   neon_lane_bounds (operands[2], 0, 1);
2938   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2939   DONE;
2940 })
2941
2942 ; Disabled before reload because we don't want combine doing something silly,
2943 ; but used by the post-reload expansion of neon_vcombine.
2944 (define_insn "*neon_vswp<mode>"
2945   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
2946         (match_operand:VDQX 1 "s_register_operand" "+w"))
2947    (set (match_dup 1) (match_dup 0))]
2948   "TARGET_NEON && reload_completed"
2949   "vswp\t%<V_reg>0, %<V_reg>1"
2950   [(set (attr "neon_type")
2951         (if_then_else (match_test "<Is_d_reg>")
2952                       (const_string "neon_bp_simple")
2953                       (const_string "neon_bp_2cycle")))]
2954 )
2955
2956 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2957 ;; dest vector.
2958 ;; FIXME: A different implementation of this builtin could make it much
2959 ;; more likely that we wouldn't actually need to output anything (we could make
2960 ;; it so that the reg allocator puts things in the right places magically
2961 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2962
2963 (define_insn_and_split "neon_vcombine<mode>"
2964   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2965         (vec_concat:<V_DOUBLE>
2966           (match_operand:VDX 1 "s_register_operand" "w")
2967           (match_operand:VDX 2 "s_register_operand" "w")))]
2968   "TARGET_NEON"
2969   "#"
2970   "&& reload_completed"
2971   [(const_int 0)]
2972 {
2973   neon_split_vcombine (operands);
2974   DONE;
2975 })
2976
2977 (define_expand "neon_vget_high<mode>"
2978   [(match_operand:<V_HALF> 0 "s_register_operand")
2979    (match_operand:VQX 1 "s_register_operand")]
2980   "TARGET_NEON"
2981 {
2982   emit_move_insn (operands[0],
2983                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
2984                                        GET_MODE_SIZE (<V_HALF>mode)));
2985   DONE;
2986 })
2987
2988 (define_expand "neon_vget_low<mode>"
2989   [(match_operand:<V_HALF> 0 "s_register_operand")
2990    (match_operand:VQX 1 "s_register_operand")]
2991   "TARGET_NEON"
2992 {
2993   emit_move_insn (operands[0],
2994                   simplify_gen_subreg (<V_HALF>mode, operands[1],
2995                                        <MODE>mode, 0));
2996   DONE;
2997 })
2998
2999 (define_insn "float<mode><V_cvtto>2"
3000   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3001         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3002   "TARGET_NEON && !flag_rounding_math"
3003   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3004   [(set (attr "neon_type")
3005      (if_then_else (match_test "<Is_d_reg>")
3006                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3007                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3008 )
3009
3010 (define_insn "floatuns<mode><V_cvtto>2"
3011   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3012         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3013   "TARGET_NEON && !flag_rounding_math"
3014   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3015   [(set (attr "neon_type")
3016      (if_then_else (match_test "<Is_d_reg>")
3017                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3018                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3019 )
3020
3021 (define_insn "fix_trunc<mode><V_cvtto>2"
3022   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3023         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3024   "TARGET_NEON"
3025   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3026   [(set (attr "neon_type")
3027      (if_then_else (match_test "<Is_d_reg>")
3028                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3029                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3030 )
3031
3032 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3033   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3034         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3035   "TARGET_NEON"
3036   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3037   [(set (attr "neon_type")
3038      (if_then_else (match_test "<Is_d_reg>")
3039                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3040                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3041 )
3042
3043 (define_insn "neon_vcvt<mode>"
3044   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3045         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3046                            (match_operand:SI 2 "immediate_operand" "i")]
3047                           UNSPEC_VCVT))]
3048   "TARGET_NEON"
3049   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3050   [(set (attr "neon_type")
3051      (if_then_else (match_test "<Is_d_reg>")
3052                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3053                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3054 )
3055
3056 (define_insn "neon_vcvt<mode>"
3057   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3058         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3059                            (match_operand:SI 2 "immediate_operand" "i")]
3060                           UNSPEC_VCVT))]
3061   "TARGET_NEON"
3062   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3063   [(set (attr "neon_type")
3064      (if_then_else (match_test "<Is_d_reg>")
3065                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3066                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3067 )
3068
3069 (define_insn "neon_vcvt_n<mode>"
3070   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3071         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3072                            (match_operand:SI 2 "immediate_operand" "i")
3073                            (match_operand:SI 3 "immediate_operand" "i")]
3074                           UNSPEC_VCVT_N))]
3075   "TARGET_NEON"
3076 {
3077   neon_const_bounds (operands[2], 1, 33);
3078   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3079 }
3080   [(set (attr "neon_type")
3081      (if_then_else (match_test "<Is_d_reg>")
3082                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3083                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3084 )
3085
3086 (define_insn "neon_vcvt_n<mode>"
3087   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3088         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3089                            (match_operand:SI 2 "immediate_operand" "i")
3090                            (match_operand:SI 3 "immediate_operand" "i")]
3091                           UNSPEC_VCVT_N))]
3092   "TARGET_NEON"
3093 {
3094   neon_const_bounds (operands[2], 1, 33);
3095   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3096 }
3097   [(set (attr "neon_type")
3098      (if_then_else (match_test "<Is_d_reg>")
3099                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3100                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3101 )
3102
3103 (define_insn "neon_vmovn<mode>"
3104   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3105         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3106                             (match_operand:SI 2 "immediate_operand" "i")]
3107                            UNSPEC_VMOVN))]
3108   "TARGET_NEON"
3109   "vmovn.<V_if_elem>\t%P0, %q1"
3110   [(set_attr "neon_type" "neon_bp_simple")]
3111 )
3112
3113 (define_insn "neon_vqmovn<mode>"
3114   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3115         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3116                             (match_operand:SI 2 "immediate_operand" "i")]
3117                            UNSPEC_VQMOVN))]
3118   "TARGET_NEON"
3119   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3120   [(set_attr "neon_type" "neon_shift_2")]
3121 )
3122
3123 (define_insn "neon_vqmovun<mode>"
3124   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3125         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3126                             (match_operand:SI 2 "immediate_operand" "i")]
3127                            UNSPEC_VQMOVUN))]
3128   "TARGET_NEON"
3129   "vqmovun.<V_s_elem>\t%P0, %q1"
3130   [(set_attr "neon_type" "neon_shift_2")]
3131 )
3132
3133 (define_insn "neon_vmovl<mode>"
3134   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3135         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3136                            (match_operand:SI 2 "immediate_operand" "i")]
3137                           UNSPEC_VMOVL))]
3138   "TARGET_NEON"
3139   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3140   [(set_attr "neon_type" "neon_shift_1")]
3141 )
3142
3143 (define_insn "neon_vmul_lane<mode>"
3144   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3145         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3146                      (match_operand:VMD 2 "s_register_operand"
3147                                         "<scalar_mul_constraint>")
3148                      (match_operand:SI 3 "immediate_operand" "i")
3149                      (match_operand:SI 4 "immediate_operand" "i")]
3150                     UNSPEC_VMUL_LANE))]
3151   "TARGET_NEON"
3152 {
3153   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3154   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3155 }
3156   [(set (attr "neon_type")
3157      (if_then_else (match_test "<Is_float_mode>")
3158                    (const_string "neon_fp_vmul_ddd")
3159                    (if_then_else (match_test "<Scalar_mul_8_16>")
3160                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3161                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3162 )
3163
3164 (define_insn "neon_vmul_lane<mode>"
3165   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3166         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3167                      (match_operand:<V_HALF> 2 "s_register_operand"
3168                                              "<scalar_mul_constraint>")
3169                      (match_operand:SI 3 "immediate_operand" "i")
3170                      (match_operand:SI 4 "immediate_operand" "i")]
3171                     UNSPEC_VMUL_LANE))]
3172   "TARGET_NEON"
3173 {
3174   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3175   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3176 }
3177   [(set (attr "neon_type")
3178      (if_then_else (match_test "<Is_float_mode>")
3179                    (const_string "neon_fp_vmul_qqd")
3180                    (if_then_else (match_test "<Scalar_mul_8_16>")
3181                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3182                                  (const_string "neon_mul_qqd_32_scalar"))))]
3183 )
3184
3185 (define_insn "neon_vmull_lane<mode>"
3186   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3187         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3188                            (match_operand:VMDI 2 "s_register_operand"
3189                                                "<scalar_mul_constraint>")
3190                            (match_operand:SI 3 "immediate_operand" "i")
3191                            (match_operand:SI 4 "immediate_operand" "i")]
3192                           UNSPEC_VMULL_LANE))]
3193   "TARGET_NEON"
3194 {
3195   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3196   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3197 }
3198   [(set (attr "neon_type")
3199      (if_then_else (match_test "<Scalar_mul_8_16>")
3200                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3201                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3202 )
3203
3204 (define_insn "neon_vqdmull_lane<mode>"
3205   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3206         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3207                            (match_operand:VMDI 2 "s_register_operand"
3208                                                "<scalar_mul_constraint>")
3209                            (match_operand:SI 3 "immediate_operand" "i")
3210                            (match_operand:SI 4 "immediate_operand" "i")]
3211                           UNSPEC_VQDMULL_LANE))]
3212   "TARGET_NEON"
3213 {
3214   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3215   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3216 }
3217   [(set (attr "neon_type")
3218      (if_then_else (match_test "<Scalar_mul_8_16>")
3219                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3220                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3221 )
3222
3223 (define_insn "neon_vqdmulh_lane<mode>"
3224   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3225         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3226                       (match_operand:<V_HALF> 2 "s_register_operand"
3227                                               "<scalar_mul_constraint>")
3228                       (match_operand:SI 3 "immediate_operand" "i")
3229                       (match_operand:SI 4 "immediate_operand" "i")]
3230                       UNSPEC_VQDMULH_LANE))]
3231   "TARGET_NEON"
3232 {
3233   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3234   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3235 }
3236   [(set (attr "neon_type")
3237      (if_then_else (match_test "<Scalar_mul_8_16>")
3238                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3239                    (const_string "neon_mul_qqd_32_scalar")))]
3240 )
3241
3242 (define_insn "neon_vqdmulh_lane<mode>"
3243   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3244         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3245                       (match_operand:VMDI 2 "s_register_operand"
3246                                           "<scalar_mul_constraint>")
3247                       (match_operand:SI 3 "immediate_operand" "i")
3248                       (match_operand:SI 4 "immediate_operand" "i")]
3249                       UNSPEC_VQDMULH_LANE))]
3250   "TARGET_NEON"
3251 {
3252   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3253   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3254 }
3255   [(set (attr "neon_type")
3256      (if_then_else (match_test "<Scalar_mul_8_16>")
3257                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3258                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3259 )
3260
3261 (define_insn "neon_vmla_lane<mode>"
3262   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3263         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3264                      (match_operand:VMD 2 "s_register_operand" "w")
3265                      (match_operand:VMD 3 "s_register_operand"
3266                                         "<scalar_mul_constraint>")
3267                      (match_operand:SI 4 "immediate_operand" "i")
3268                      (match_operand:SI 5 "immediate_operand" "i")]
3269                      UNSPEC_VMLA_LANE))]
3270   "TARGET_NEON"
3271 {
3272   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3273   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3274 }
3275   [(set (attr "neon_type")
3276      (if_then_else (match_test "<Is_float_mode>")
3277                    (const_string "neon_fp_vmla_ddd_scalar")
3278                    (if_then_else (match_test "<Scalar_mul_8_16>")
3279                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3280                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3281 )
3282
3283 (define_insn "neon_vmla_lane<mode>"
3284   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3285         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3286                      (match_operand:VMQ 2 "s_register_operand" "w")
3287                      (match_operand:<V_HALF> 3 "s_register_operand"
3288                                              "<scalar_mul_constraint>")
3289                      (match_operand:SI 4 "immediate_operand" "i")
3290                      (match_operand:SI 5 "immediate_operand" "i")]
3291                      UNSPEC_VMLA_LANE))]
3292   "TARGET_NEON"
3293 {
3294   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3295   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3296 }
3297   [(set (attr "neon_type")
3298      (if_then_else (match_test "<Is_float_mode>")
3299                    (const_string "neon_fp_vmla_qqq_scalar")
3300                    (if_then_else (match_test "<Scalar_mul_8_16>")
3301                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3302                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3303 )
3304
3305 (define_insn "neon_vmlal_lane<mode>"
3306   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3307         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3308                            (match_operand:VMDI 2 "s_register_operand" "w")
3309                            (match_operand:VMDI 3 "s_register_operand"
3310                                                "<scalar_mul_constraint>")
3311                            (match_operand:SI 4 "immediate_operand" "i")
3312                            (match_operand:SI 5 "immediate_operand" "i")]
3313                           UNSPEC_VMLAL_LANE))]
3314   "TARGET_NEON"
3315 {
3316   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3317   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3318 }
3319   [(set (attr "neon_type")
3320      (if_then_else (match_test "<Scalar_mul_8_16>")
3321                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3322                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3323 )
3324
3325 (define_insn "neon_vqdmlal_lane<mode>"
3326   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3327         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3328                            (match_operand:VMDI 2 "s_register_operand" "w")
3329                            (match_operand:VMDI 3 "s_register_operand"
3330                                                "<scalar_mul_constraint>")
3331                            (match_operand:SI 4 "immediate_operand" "i")
3332                            (match_operand:SI 5 "immediate_operand" "i")]
3333                           UNSPEC_VQDMLAL_LANE))]
3334   "TARGET_NEON"
3335 {
3336   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3337   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3338 }
3339   [(set (attr "neon_type")
3340      (if_then_else (match_test "<Scalar_mul_8_16>")
3341                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3342                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3343 )
3344
3345 (define_insn "neon_vmls_lane<mode>"
3346   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3347         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3348                      (match_operand:VMD 2 "s_register_operand" "w")
3349                      (match_operand:VMD 3 "s_register_operand"
3350                                         "<scalar_mul_constraint>")
3351                      (match_operand:SI 4 "immediate_operand" "i")
3352                      (match_operand:SI 5 "immediate_operand" "i")]
3353                     UNSPEC_VMLS_LANE))]
3354   "TARGET_NEON"
3355 {
3356   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3357   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3358 }
3359   [(set (attr "neon_type")
3360      (if_then_else (match_test "<Is_float_mode>")
3361                    (const_string "neon_fp_vmla_ddd_scalar")
3362                    (if_then_else (match_test "<Scalar_mul_8_16>")
3363                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3364                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3365 )
3366
3367 (define_insn "neon_vmls_lane<mode>"
3368   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3369         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3370                      (match_operand:VMQ 2 "s_register_operand" "w")
3371                      (match_operand:<V_HALF> 3 "s_register_operand"
3372                                              "<scalar_mul_constraint>")
3373                      (match_operand:SI 4 "immediate_operand" "i")
3374                      (match_operand:SI 5 "immediate_operand" "i")]
3375                     UNSPEC_VMLS_LANE))]
3376   "TARGET_NEON"
3377 {
3378   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3379   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3380 }
3381   [(set (attr "neon_type")
3382      (if_then_else (match_test "<Is_float_mode>")
3383                    (const_string "neon_fp_vmla_qqq_scalar")
3384                    (if_then_else (match_test "<Scalar_mul_8_16>")
3385                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3386                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3387 )
3388
3389 (define_insn "neon_vmlsl_lane<mode>"
3390   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3391         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3392                            (match_operand:VMDI 2 "s_register_operand" "w")
3393                            (match_operand:VMDI 3 "s_register_operand"
3394                                                "<scalar_mul_constraint>")
3395                            (match_operand:SI 4 "immediate_operand" "i")
3396                            (match_operand:SI 5 "immediate_operand" "i")]
3397                           UNSPEC_VMLSL_LANE))]
3398   "TARGET_NEON"
3399 {
3400   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3401   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3402 }
3403   [(set (attr "neon_type")
3404      (if_then_else (match_test "<Scalar_mul_8_16>")
3405                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3406                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3407 )
3408
3409 (define_insn "neon_vqdmlsl_lane<mode>"
3410   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3411         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3412                            (match_operand:VMDI 2 "s_register_operand" "w")
3413                            (match_operand:VMDI 3 "s_register_operand"
3414                                                "<scalar_mul_constraint>")
3415                            (match_operand:SI 4 "immediate_operand" "i")
3416                            (match_operand:SI 5 "immediate_operand" "i")]
3417                           UNSPEC_VQDMLSL_LANE))]
3418   "TARGET_NEON"
3419 {
3420   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3421   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3422 }
3423   [(set (attr "neon_type")
3424      (if_then_else (match_test "<Scalar_mul_8_16>")
3425                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3426                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3427 )
3428
3429 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3430 ; core register into a temp register, then use a scalar taken from that. This
3431 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3432 ; or extracted from another vector. The latter case it's currently better to
3433 ; use the "_lane" variant, and the former case can probably be implemented
3434 ; using vld1_lane, but that hasn't been done yet.
3435
3436 (define_expand "neon_vmul_n<mode>"
3437   [(match_operand:VMD 0 "s_register_operand" "")
3438    (match_operand:VMD 1 "s_register_operand" "")
3439    (match_operand:<V_elem> 2 "s_register_operand" "")
3440    (match_operand:SI 3 "immediate_operand" "")]
3441   "TARGET_NEON"
3442 {
3443   rtx tmp = gen_reg_rtx (<MODE>mode);
3444   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3445   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3446                                        const0_rtx, const0_rtx));
3447   DONE;
3448 })
3449
3450 (define_expand "neon_vmul_n<mode>"
3451   [(match_operand:VMQ 0 "s_register_operand" "")
3452    (match_operand:VMQ 1 "s_register_operand" "")
3453    (match_operand:<V_elem> 2 "s_register_operand" "")
3454    (match_operand:SI 3 "immediate_operand" "")]
3455   "TARGET_NEON"
3456 {
3457   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3458   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3459   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3460                                        const0_rtx, const0_rtx));
3461   DONE;
3462 })
3463
3464 (define_expand "neon_vmull_n<mode>"
3465   [(match_operand:<V_widen> 0 "s_register_operand" "")
3466    (match_operand:VMDI 1 "s_register_operand" "")
3467    (match_operand:<V_elem> 2 "s_register_operand" "")
3468    (match_operand:SI 3 "immediate_operand" "")]
3469   "TARGET_NEON"
3470 {
3471   rtx tmp = gen_reg_rtx (<MODE>mode);
3472   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3473   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3474                                         const0_rtx, operands[3]));
3475   DONE;
3476 })
3477
3478 (define_expand "neon_vqdmull_n<mode>"
3479   [(match_operand:<V_widen> 0 "s_register_operand" "")
3480    (match_operand:VMDI 1 "s_register_operand" "")
3481    (match_operand:<V_elem> 2 "s_register_operand" "")
3482    (match_operand:SI 3 "immediate_operand" "")]
3483   "TARGET_NEON"
3484 {
3485   rtx tmp = gen_reg_rtx (<MODE>mode);
3486   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3487   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3488                                           const0_rtx, const0_rtx));
3489   DONE;
3490 })
3491
3492 (define_expand "neon_vqdmulh_n<mode>"
3493   [(match_operand:VMDI 0 "s_register_operand" "")
3494    (match_operand:VMDI 1 "s_register_operand" "")
3495    (match_operand:<V_elem> 2 "s_register_operand" "")
3496    (match_operand:SI 3 "immediate_operand" "")]
3497   "TARGET_NEON"
3498 {
3499   rtx tmp = gen_reg_rtx (<MODE>mode);
3500   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3501   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3502                                           const0_rtx, operands[3]));
3503   DONE;
3504 })
3505
3506 (define_expand "neon_vqdmulh_n<mode>"
3507   [(match_operand:VMQI 0 "s_register_operand" "")
3508    (match_operand:VMQI 1 "s_register_operand" "")
3509    (match_operand:<V_elem> 2 "s_register_operand" "")
3510    (match_operand:SI 3 "immediate_operand" "")]
3511   "TARGET_NEON"
3512 {
3513   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3514   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3515   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3516                                           const0_rtx, operands[3]));
3517   DONE;
3518 })
3519
3520 (define_expand "neon_vmla_n<mode>"
3521   [(match_operand:VMD 0 "s_register_operand" "")
3522    (match_operand:VMD 1 "s_register_operand" "")
3523    (match_operand:VMD 2 "s_register_operand" "")
3524    (match_operand:<V_elem> 3 "s_register_operand" "")
3525    (match_operand:SI 4 "immediate_operand" "")]
3526   "TARGET_NEON"
3527 {
3528   rtx tmp = gen_reg_rtx (<MODE>mode);
3529   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3530   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3531                                        tmp, const0_rtx, operands[4]));
3532   DONE;
3533 })
3534
3535 (define_expand "neon_vmla_n<mode>"
3536   [(match_operand:VMQ 0 "s_register_operand" "")
3537    (match_operand:VMQ 1 "s_register_operand" "")
3538    (match_operand:VMQ 2 "s_register_operand" "")
3539    (match_operand:<V_elem> 3 "s_register_operand" "")
3540    (match_operand:SI 4 "immediate_operand" "")]
3541   "TARGET_NEON"
3542 {
3543   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3544   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3545   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3546                                        tmp, const0_rtx, operands[4]));
3547   DONE;
3548 })
3549
3550 (define_expand "neon_vmlal_n<mode>"
3551   [(match_operand:<V_widen> 0 "s_register_operand" "")
3552    (match_operand:<V_widen> 1 "s_register_operand" "")
3553    (match_operand:VMDI 2 "s_register_operand" "")
3554    (match_operand:<V_elem> 3 "s_register_operand" "")
3555    (match_operand:SI 4 "immediate_operand" "")]
3556   "TARGET_NEON"
3557 {
3558   rtx tmp = gen_reg_rtx (<MODE>mode);
3559   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3560   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3561                                         tmp, const0_rtx, operands[4]));
3562   DONE;
3563 })
3564
3565 (define_expand "neon_vqdmlal_n<mode>"
3566   [(match_operand:<V_widen> 0 "s_register_operand" "")
3567    (match_operand:<V_widen> 1 "s_register_operand" "")
3568    (match_operand:VMDI 2 "s_register_operand" "")
3569    (match_operand:<V_elem> 3 "s_register_operand" "")
3570    (match_operand:SI 4 "immediate_operand" "")]
3571   "TARGET_NEON"
3572 {
3573   rtx tmp = gen_reg_rtx (<MODE>mode);
3574   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3575   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3576                                           tmp, const0_rtx, operands[4]));
3577   DONE;
3578 })
3579
3580 (define_expand "neon_vmls_n<mode>"
3581   [(match_operand:VMD 0 "s_register_operand" "")
3582    (match_operand:VMD 1 "s_register_operand" "")
3583    (match_operand:VMD 2 "s_register_operand" "")
3584    (match_operand:<V_elem> 3 "s_register_operand" "")
3585    (match_operand:SI 4 "immediate_operand" "")]
3586   "TARGET_NEON"
3587 {
3588   rtx tmp = gen_reg_rtx (<MODE>mode);
3589   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3590   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3591                                        tmp, const0_rtx, operands[4]));
3592   DONE;
3593 })
3594
3595 (define_expand "neon_vmls_n<mode>"
3596   [(match_operand:VMQ 0 "s_register_operand" "")
3597    (match_operand:VMQ 1 "s_register_operand" "")
3598    (match_operand:VMQ 2 "s_register_operand" "")
3599    (match_operand:<V_elem> 3 "s_register_operand" "")
3600    (match_operand:SI 4 "immediate_operand" "")]
3601   "TARGET_NEON"
3602 {
3603   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3604   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3605   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3606                                        tmp, const0_rtx, operands[4]));
3607   DONE;
3608 })
3609
3610 (define_expand "neon_vmlsl_n<mode>"
3611   [(match_operand:<V_widen> 0 "s_register_operand" "")
3612    (match_operand:<V_widen> 1 "s_register_operand" "")
3613    (match_operand:VMDI 2 "s_register_operand" "")
3614    (match_operand:<V_elem> 3 "s_register_operand" "")
3615    (match_operand:SI 4 "immediate_operand" "")]
3616   "TARGET_NEON"
3617 {
3618   rtx tmp = gen_reg_rtx (<MODE>mode);
3619   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3620   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3621                                         tmp, const0_rtx, operands[4]));
3622   DONE;
3623 })
3624
3625 (define_expand "neon_vqdmlsl_n<mode>"
3626   [(match_operand:<V_widen> 0 "s_register_operand" "")
3627    (match_operand:<V_widen> 1 "s_register_operand" "")
3628    (match_operand:VMDI 2 "s_register_operand" "")
3629    (match_operand:<V_elem> 3 "s_register_operand" "")
3630    (match_operand:SI 4 "immediate_operand" "")]
3631   "TARGET_NEON"
3632 {
3633   rtx tmp = gen_reg_rtx (<MODE>mode);
3634   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3635   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3636                                           tmp, const0_rtx, operands[4]));
3637   DONE;
3638 })
3639
3640 (define_insn "neon_vext<mode>"
3641   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3642         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3643                       (match_operand:VDQX 2 "s_register_operand" "w")
3644                       (match_operand:SI 3 "immediate_operand" "i")]
3645                      UNSPEC_VEXT))]
3646   "TARGET_NEON"
3647 {
3648   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3649   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3650 }
3651   [(set (attr "neon_type")
3652       (if_then_else (match_test "<Is_d_reg>")
3653                     (const_string "neon_bp_simple")
3654                     (const_string "neon_bp_2cycle")))]
3655 )
3656
3657 (define_insn "neon_vrev64<mode>"
3658   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3659         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3660                      (match_operand:SI 2 "immediate_operand" "i")]
3661                     UNSPEC_VREV64))]
3662   "TARGET_NEON"
3663   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3664   [(set_attr "neon_type" "neon_bp_simple")]
3665 )
3666
3667 (define_insn "neon_vrev32<mode>"
3668   [(set (match_operand:VX 0 "s_register_operand" "=w")
3669         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3670                     (match_operand:SI 2 "immediate_operand" "i")]
3671                    UNSPEC_VREV32))]
3672   "TARGET_NEON"
3673   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3674   [(set_attr "neon_type" "neon_bp_simple")]
3675 )
3676
3677 (define_insn "neon_vrev16<mode>"
3678   [(set (match_operand:VE 0 "s_register_operand" "=w")
3679         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3680                     (match_operand:SI 2 "immediate_operand" "i")]
3681                    UNSPEC_VREV16))]
3682   "TARGET_NEON"
3683   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3684   [(set_attr "neon_type" "neon_bp_simple")]
3685 )
3686
3687 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3688 ; allocation. For an intrinsic of form:
3689 ;   rD = vbsl_* (rS, rN, rM)
3690 ; We can use any of:
3691 ;   vbsl rS, rN, rM  (if D = S)
3692 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3693 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3694
3695 (define_insn "neon_vbsl<mode>_internal"
3696   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3697         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3698                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3699                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3700                      UNSPEC_VBSL))]
3701   "TARGET_NEON"
3702   "@
3703   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3704   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3705   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3706   [(set_attr "neon_type" "neon_int_1")]
3707 )
3708
3709 (define_expand "neon_vbsl<mode>"
3710   [(set (match_operand:VDQX 0 "s_register_operand" "")
3711         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3712                       (match_operand:VDQX 2 "s_register_operand" "")
3713                       (match_operand:VDQX 3 "s_register_operand" "")]
3714                      UNSPEC_VBSL))]
3715   "TARGET_NEON"
3716 {
3717   /* We can't alias operands together if they have different modes.  */
3718   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3719 })
3720
3721 (define_insn "neon_vshl<mode>"
3722   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3723         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3724                        (match_operand:VDQIX 2 "s_register_operand" "w")
3725                        (match_operand:SI 3 "immediate_operand" "i")]
3726                       UNSPEC_VSHL))]
3727   "TARGET_NEON"
3728   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3729   [(set (attr "neon_type")
3730       (if_then_else (match_test "<Is_d_reg>")
3731                     (const_string "neon_vshl_ddd")
3732                     (const_string "neon_shift_3")))]
3733 )
3734
3735 (define_insn "neon_vqshl<mode>"
3736   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3737         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3738                        (match_operand:VDQIX 2 "s_register_operand" "w")
3739                        (match_operand:SI 3 "immediate_operand" "i")]
3740                       UNSPEC_VQSHL))]
3741   "TARGET_NEON"
3742   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3743   [(set (attr "neon_type")
3744       (if_then_else (match_test "<Is_d_reg>")
3745                     (const_string "neon_shift_2")
3746                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3747 )
3748
3749 (define_insn "neon_vshr_n<mode>"
3750   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3751         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3752                        (match_operand:SI 2 "immediate_operand" "i")
3753                        (match_operand:SI 3 "immediate_operand" "i")]
3754                       UNSPEC_VSHR_N))]
3755   "TARGET_NEON"
3756 {
3757   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3758   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3759 }
3760   [(set_attr "neon_type" "neon_shift_1")]
3761 )
3762
3763 (define_insn "neon_vshrn_n<mode>"
3764   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3765         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3766                             (match_operand:SI 2 "immediate_operand" "i")
3767                             (match_operand:SI 3 "immediate_operand" "i")]
3768                            UNSPEC_VSHRN_N))]
3769   "TARGET_NEON"
3770 {
3771   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3772   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3773 }
3774   [(set_attr "neon_type" "neon_shift_1")]
3775 )
3776
3777 (define_insn "neon_vqshrn_n<mode>"
3778   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3779         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3780                             (match_operand:SI 2 "immediate_operand" "i")
3781                             (match_operand:SI 3 "immediate_operand" "i")]
3782                            UNSPEC_VQSHRN_N))]
3783   "TARGET_NEON"
3784 {
3785   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3786   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3787 }
3788   [(set_attr "neon_type" "neon_shift_2")]
3789 )
3790
3791 (define_insn "neon_vqshrun_n<mode>"
3792   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3793         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3794                             (match_operand:SI 2 "immediate_operand" "i")
3795                             (match_operand:SI 3 "immediate_operand" "i")]
3796                            UNSPEC_VQSHRUN_N))]
3797   "TARGET_NEON"
3798 {
3799   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3800   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3801 }
3802   [(set_attr "neon_type" "neon_shift_2")]
3803 )
3804
3805 (define_insn "neon_vshl_n<mode>"
3806   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3807         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3808                        (match_operand:SI 2 "immediate_operand" "i")
3809                        (match_operand:SI 3 "immediate_operand" "i")]
3810                       UNSPEC_VSHL_N))]
3811   "TARGET_NEON"
3812 {
3813   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3814   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3815 }
3816   [(set_attr "neon_type" "neon_shift_1")]
3817 )
3818
3819 (define_insn "neon_vqshl_n<mode>"
3820   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3821         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3822                        (match_operand:SI 2 "immediate_operand" "i")
3823                        (match_operand:SI 3 "immediate_operand" "i")]
3824                       UNSPEC_VQSHL_N))]
3825   "TARGET_NEON"
3826 {
3827   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3828   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3829 }
3830   [(set_attr "neon_type" "neon_shift_2")]
3831 )
3832
3833 (define_insn "neon_vqshlu_n<mode>"
3834   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3835         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3836                        (match_operand:SI 2 "immediate_operand" "i")
3837                        (match_operand:SI 3 "immediate_operand" "i")]
3838                       UNSPEC_VQSHLU_N))]
3839   "TARGET_NEON"
3840 {
3841   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3842   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3843 }
3844   [(set_attr "neon_type" "neon_shift_2")]
3845 )
3846
3847 (define_insn "neon_vshll_n<mode>"
3848   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3849         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3850                            (match_operand:SI 2 "immediate_operand" "i")
3851                            (match_operand:SI 3 "immediate_operand" "i")]
3852                           UNSPEC_VSHLL_N))]
3853   "TARGET_NEON"
3854 {
3855   /* The boundaries are: 0 < imm <= size.  */
3856   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3857   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3858 }
3859   [(set_attr "neon_type" "neon_shift_1")]
3860 )
3861
3862 (define_insn "neon_vsra_n<mode>"
3863   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3864         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3865                        (match_operand:VDQIX 2 "s_register_operand" "w")
3866                        (match_operand:SI 3 "immediate_operand" "i")
3867                        (match_operand:SI 4 "immediate_operand" "i")]
3868                       UNSPEC_VSRA_N))]
3869   "TARGET_NEON"
3870 {
3871   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3872   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3873 }
3874   [(set_attr "neon_type" "neon_vsra_vrsra")]
3875 )
3876
3877 (define_insn "neon_vsri_n<mode>"
3878   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3879         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3880                        (match_operand:VDQIX 2 "s_register_operand" "w")
3881                        (match_operand:SI 3 "immediate_operand" "i")]
3882                       UNSPEC_VSRI))]
3883   "TARGET_NEON"
3884 {
3885   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3886   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3887 }
3888   [(set (attr "neon_type")
3889       (if_then_else (match_test "<Is_d_reg>")
3890                     (const_string "neon_shift_1")
3891                     (const_string "neon_shift_3")))]
3892 )
3893
3894 (define_insn "neon_vsli_n<mode>"
3895   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3896         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3897                        (match_operand:VDQIX 2 "s_register_operand" "w")
3898                        (match_operand:SI 3 "immediate_operand" "i")]
3899                       UNSPEC_VSLI))]
3900   "TARGET_NEON"
3901 {
3902   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3903   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3904 }
3905   [(set (attr "neon_type")
3906       (if_then_else (match_test "<Is_d_reg>")
3907                     (const_string "neon_shift_1")
3908                     (const_string "neon_shift_3")))]
3909 )
3910
3911 (define_insn "neon_vtbl1v8qi"
3912   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3913         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3914                       (match_operand:V8QI 2 "s_register_operand" "w")]
3915                      UNSPEC_VTBL))]
3916   "TARGET_NEON"
3917   "vtbl.8\t%P0, {%P1}, %P2"
3918   [(set_attr "neon_type" "neon_bp_2cycle")]
3919 )
3920
3921 (define_insn "neon_vtbl2v8qi"
3922   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3923         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3924                       (match_operand:V8QI 2 "s_register_operand" "w")]
3925                      UNSPEC_VTBL))]
3926   "TARGET_NEON"
3927 {
3928   rtx ops[4];
3929   int tabbase = REGNO (operands[1]);
3930
3931   ops[0] = operands[0];
3932   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3933   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3934   ops[3] = operands[2];
3935   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3936
3937   return "";
3938 }
3939   [(set_attr "neon_type" "neon_bp_2cycle")]
3940 )
3941
3942 (define_insn "neon_vtbl3v8qi"
3943   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3944         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3945                       (match_operand:V8QI 2 "s_register_operand" "w")]
3946                      UNSPEC_VTBL))]
3947   "TARGET_NEON"
3948 {
3949   rtx ops[5];
3950   int tabbase = REGNO (operands[1]);
3951
3952   ops[0] = operands[0];
3953   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3954   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3955   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3956   ops[4] = operands[2];
3957   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3958
3959   return "";
3960 }
3961   [(set_attr "neon_type" "neon_bp_3cycle")]
3962 )
3963
3964 (define_insn "neon_vtbl4v8qi"
3965   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3966         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3967                       (match_operand:V8QI 2 "s_register_operand" "w")]
3968                      UNSPEC_VTBL))]
3969   "TARGET_NEON"
3970 {
3971   rtx ops[6];
3972   int tabbase = REGNO (operands[1]);
3973
3974   ops[0] = operands[0];
3975   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3976   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3977   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3978   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
3979   ops[5] = operands[2];
3980   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
3981
3982   return "";
3983 }
3984   [(set_attr "neon_type" "neon_bp_3cycle")]
3985 )
3986
3987 ;; These three are used by the vec_perm infrastructure for V16QImode.
3988 (define_insn_and_split "neon_vtbl1v16qi"
3989   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3990         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
3991                        (match_operand:V16QI 2 "s_register_operand" "w")]
3992                       UNSPEC_VTBL))]
3993   "TARGET_NEON"
3994   "#"
3995   "&& reload_completed"
3996   [(const_int 0)]
3997 {
3998   rtx op0, op1, op2, part0, part2;
3999   unsigned ofs;
4000
4001   op0 = operands[0];
4002   op1 = gen_lowpart (TImode, operands[1]);
4003   op2 = operands[2];
4004
4005   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4006   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4007   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4008   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4009
4010   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4011   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4012   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4013   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4014   DONE;
4015 })
4016
4017 (define_insn_and_split "neon_vtbl2v16qi"
4018   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4019         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4020                        (match_operand:V16QI 2 "s_register_operand" "w")]
4021                       UNSPEC_VTBL))]
4022   "TARGET_NEON"
4023   "#"
4024   "&& reload_completed"
4025   [(const_int 0)]
4026 {
4027   rtx op0, op1, op2, part0, part2;
4028   unsigned ofs;
4029
4030   op0 = operands[0];
4031   op1 = operands[1];
4032   op2 = operands[2];
4033
4034   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4035   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4036   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4037   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4038
4039   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4040   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4041   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4042   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4043   DONE;
4044 })
4045
4046 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4047 ;; handle quad-word input modes, producing octa-word output modes.  But
4048 ;; that requires us to add support for octa-word vector modes in moves.
4049 ;; That seems overkill for this one use in vec_perm.
4050 (define_insn_and_split "neon_vcombinev16qi"
4051   [(set (match_operand:OI 0 "s_register_operand" "=w")
4052         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4053                     (match_operand:V16QI 2 "s_register_operand" "w")]
4054                    UNSPEC_VCONCAT))]
4055   "TARGET_NEON"
4056   "#"
4057   "&& reload_completed"
4058   [(const_int 0)]
4059 {
4060   neon_split_vcombine (operands);
4061   DONE;
4062 })
4063
4064 (define_insn "neon_vtbx1v8qi"
4065   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4066         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4067                       (match_operand:V8QI 2 "s_register_operand" "w")
4068                       (match_operand:V8QI 3 "s_register_operand" "w")]
4069                      UNSPEC_VTBX))]
4070   "TARGET_NEON"
4071   "vtbx.8\t%P0, {%P2}, %P3"
4072   [(set_attr "neon_type" "neon_bp_2cycle")]
4073 )
4074
4075 (define_insn "neon_vtbx2v8qi"
4076   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4077         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4078                       (match_operand:TI 2 "s_register_operand" "w")
4079                       (match_operand:V8QI 3 "s_register_operand" "w")]
4080                      UNSPEC_VTBX))]
4081   "TARGET_NEON"
4082 {
4083   rtx ops[4];
4084   int tabbase = REGNO (operands[2]);
4085
4086   ops[0] = operands[0];
4087   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4088   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4089   ops[3] = operands[3];
4090   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4091
4092   return "";
4093 }
4094   [(set_attr "neon_type" "neon_bp_2cycle")]
4095 )
4096
4097 (define_insn "neon_vtbx3v8qi"
4098   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4099         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4100                       (match_operand:EI 2 "s_register_operand" "w")
4101                       (match_operand:V8QI 3 "s_register_operand" "w")]
4102                      UNSPEC_VTBX))]
4103   "TARGET_NEON"
4104 {
4105   rtx ops[5];
4106   int tabbase = REGNO (operands[2]);
4107
4108   ops[0] = operands[0];
4109   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4110   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4111   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4112   ops[4] = operands[3];
4113   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4114
4115   return "";
4116 }
4117   [(set_attr "neon_type" "neon_bp_3cycle")]
4118 )
4119
4120 (define_insn "neon_vtbx4v8qi"
4121   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4122         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4123                       (match_operand:OI 2 "s_register_operand" "w")
4124                       (match_operand:V8QI 3 "s_register_operand" "w")]
4125                      UNSPEC_VTBX))]
4126   "TARGET_NEON"
4127 {
4128   rtx ops[6];
4129   int tabbase = REGNO (operands[2]);
4130
4131   ops[0] = operands[0];
4132   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4133   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4134   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4135   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4136   ops[5] = operands[3];
4137   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4138
4139   return "";
4140 }
4141   [(set_attr "neon_type" "neon_bp_3cycle")]
4142 )
4143
4144 (define_insn "neon_vtrn<mode>_internal"
4145   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4146         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4147                       (match_operand:VDQW 2 "s_register_operand" "w")]
4148                      UNSPEC_VTRN1))
4149    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4150          (unspec:VDQW [(match_dup 1) (match_dup 2)]
4151                      UNSPEC_VTRN2))]
4152   "TARGET_NEON"
4153   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4154   [(set (attr "neon_type")
4155       (if_then_else (match_test "<Is_d_reg>")
4156                     (const_string "neon_bp_simple")
4157                     (const_string "neon_bp_3cycle")))]
4158 )
4159
4160 (define_expand "neon_vtrn<mode>"
4161   [(match_operand:SI 0 "s_register_operand" "r")
4162    (match_operand:VDQW 1 "s_register_operand" "w")
4163    (match_operand:VDQW 2 "s_register_operand" "w")]
4164   "TARGET_NEON"
4165 {
4166   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4167                               operands[0], operands[1], operands[2]);
4168   DONE;
4169 })
4170
4171 (define_insn "neon_vzip<mode>_internal"
4172   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4173         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4174                       (match_operand:VDQW 2 "s_register_operand" "w")]
4175                      UNSPEC_VZIP1))
4176    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4177         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4178                      UNSPEC_VZIP2))]
4179   "TARGET_NEON"
4180   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4181   [(set (attr "neon_type")
4182       (if_then_else (match_test "<Is_d_reg>")
4183                     (const_string "neon_bp_simple")
4184                     (const_string "neon_bp_3cycle")))]
4185 )
4186
4187 (define_expand "neon_vzip<mode>"
4188   [(match_operand:SI 0 "s_register_operand" "r")
4189    (match_operand:VDQW 1 "s_register_operand" "w")
4190    (match_operand:VDQW 2 "s_register_operand" "w")]
4191   "TARGET_NEON"
4192 {
4193   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4194                               operands[0], operands[1], operands[2]);
4195   DONE;
4196 })
4197
4198 (define_insn "neon_vuzp<mode>_internal"
4199   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4200         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4201                       (match_operand:VDQW 2 "s_register_operand" "w")]
4202                      UNSPEC_VUZP1))
4203    (set (match_operand:VDQW 3 "s_register_operand" "=2")
4204         (unspec:VDQW [(match_dup 1) (match_dup 2)]
4205                      UNSPEC_VUZP2))]
4206   "TARGET_NEON"
4207   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4208   [(set (attr "neon_type")
4209       (if_then_else (match_test "<Is_d_reg>")
4210                     (const_string "neon_bp_simple")
4211                     (const_string "neon_bp_3cycle")))]
4212 )
4213
4214 (define_expand "neon_vuzp<mode>"
4215   [(match_operand:SI 0 "s_register_operand" "r")
4216    (match_operand:VDQW 1 "s_register_operand" "w")
4217    (match_operand:VDQW 2 "s_register_operand" "w")]
4218   "TARGET_NEON"
4219 {
4220   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4221                               operands[0], operands[1], operands[2]);
4222   DONE;
4223 })
4224
4225 (define_expand "neon_vreinterpretv8qi<mode>"
4226   [(match_operand:V8QI 0 "s_register_operand" "")
4227    (match_operand:VDX 1 "s_register_operand" "")]
4228   "TARGET_NEON"
4229 {
4230   neon_reinterpret (operands[0], operands[1]);
4231   DONE;
4232 })
4233
4234 (define_expand "neon_vreinterpretv4hi<mode>"
4235   [(match_operand:V4HI 0 "s_register_operand" "")
4236    (match_operand:VDX 1 "s_register_operand" "")]
4237   "TARGET_NEON"
4238 {
4239   neon_reinterpret (operands[0], operands[1]);
4240   DONE;
4241 })
4242
4243 (define_expand "neon_vreinterpretv2si<mode>"
4244   [(match_operand:V2SI 0 "s_register_operand" "")
4245    (match_operand:VDX 1 "s_register_operand" "")]
4246   "TARGET_NEON"
4247 {
4248   neon_reinterpret (operands[0], operands[1]);
4249   DONE;
4250 })
4251
4252 (define_expand "neon_vreinterpretv2sf<mode>"
4253   [(match_operand:V2SF 0 "s_register_operand" "")
4254    (match_operand:VDX 1 "s_register_operand" "")]
4255   "TARGET_NEON"
4256 {
4257   neon_reinterpret (operands[0], operands[1]);
4258   DONE;
4259 })
4260
4261 (define_expand "neon_vreinterpretdi<mode>"
4262   [(match_operand:DI 0 "s_register_operand" "")
4263    (match_operand:VDX 1 "s_register_operand" "")]
4264   "TARGET_NEON"
4265 {
4266   neon_reinterpret (operands[0], operands[1]);
4267   DONE;
4268 })
4269
4270 (define_expand "neon_vreinterpretv16qi<mode>"
4271   [(match_operand:V16QI 0 "s_register_operand" "")
4272    (match_operand:VQX 1 "s_register_operand" "")]
4273   "TARGET_NEON"
4274 {
4275   neon_reinterpret (operands[0], operands[1]);
4276   DONE;
4277 })
4278
4279 (define_expand "neon_vreinterpretv8hi<mode>"
4280   [(match_operand:V8HI 0 "s_register_operand" "")
4281    (match_operand:VQX 1 "s_register_operand" "")]
4282   "TARGET_NEON"
4283 {
4284   neon_reinterpret (operands[0], operands[1]);
4285   DONE;
4286 })
4287
4288 (define_expand "neon_vreinterpretv4si<mode>"
4289   [(match_operand:V4SI 0 "s_register_operand" "")
4290    (match_operand:VQX 1 "s_register_operand" "")]
4291   "TARGET_NEON"
4292 {
4293   neon_reinterpret (operands[0], operands[1]);
4294   DONE;
4295 })
4296
4297 (define_expand "neon_vreinterpretv4sf<mode>"
4298   [(match_operand:V4SF 0 "s_register_operand" "")
4299    (match_operand:VQX 1 "s_register_operand" "")]
4300   "TARGET_NEON"
4301 {
4302   neon_reinterpret (operands[0], operands[1]);
4303   DONE;
4304 })
4305
4306 (define_expand "neon_vreinterpretv2di<mode>"
4307   [(match_operand:V2DI 0 "s_register_operand" "")
4308    (match_operand:VQX 1 "s_register_operand" "")]
4309   "TARGET_NEON"
4310 {
4311   neon_reinterpret (operands[0], operands[1]);
4312   DONE;
4313 })
4314
4315 (define_expand "vec_load_lanes<mode><mode>"
4316   [(set (match_operand:VDQX 0 "s_register_operand")
4317         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4318                      UNSPEC_VLD1))]
4319   "TARGET_NEON")
4320
4321 (define_insn "neon_vld1<mode>"
4322   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4323         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4324                     UNSPEC_VLD1))]
4325   "TARGET_NEON"
4326   "vld1.<V_sz_elem>\t%h0, %A1"
4327   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4328 )
4329
4330 (define_insn "neon_vld1_lane<mode>"
4331   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4332         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4333                      (match_operand:VDX 2 "s_register_operand" "0")
4334                      (match_operand:SI 3 "immediate_operand" "i")]
4335                     UNSPEC_VLD1_LANE))]
4336   "TARGET_NEON"
4337 {
4338   HOST_WIDE_INT lane = INTVAL (operands[3]);
4339   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4340   if (lane < 0 || lane >= max)
4341     error ("lane out of range");
4342   if (max == 1)
4343     return "vld1.<V_sz_elem>\t%P0, %A1";
4344   else
4345     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4346 }
4347   [(set (attr "neon_type")
4348       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4349                     (const_string "neon_vld1_1_2_regs")
4350                     (const_string "neon_vld1_vld2_lane")))]
4351 )
4352
4353 (define_insn "neon_vld1_lane<mode>"
4354   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4355         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4356                      (match_operand:VQX 2 "s_register_operand" "0")
4357                      (match_operand:SI 3 "immediate_operand" "i")]
4358                     UNSPEC_VLD1_LANE))]
4359   "TARGET_NEON"
4360 {
4361   HOST_WIDE_INT lane = INTVAL (operands[3]);
4362   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4363   int regno = REGNO (operands[0]);
4364   if (lane < 0 || lane >= max)
4365     error ("lane out of range");
4366   else if (lane >= max / 2)
4367     {
4368       lane -= max / 2;
4369       regno += 2;
4370       operands[3] = GEN_INT (lane);
4371     }
4372   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4373   if (max == 2)
4374     return "vld1.<V_sz_elem>\t%P0, %A1";
4375   else
4376     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4377 }
4378   [(set (attr "neon_type")
4379       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4380                     (const_string "neon_vld1_1_2_regs")
4381                     (const_string "neon_vld1_vld2_lane")))]
4382 )
4383
4384 (define_insn "neon_vld1_dup<mode>"
4385   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4386         (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4387   "TARGET_NEON"
4388 {
4389   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4390     return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4391   else
4392     return "vld1.<V_sz_elem>\t%h0, %A1";
4393 }
4394   [(set (attr "neon_type")
4395       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4396                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4397                     (const_string "neon_vld1_1_2_regs")))]
4398 )
4399
4400 (define_insn "neon_vld1_dup<mode>"
4401   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4402         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4403   "TARGET_NEON"
4404 {
4405   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4406 }
4407   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4408 )
4409
4410 (define_insn_and_split "neon_vld1_dupv2di"
4411    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4412     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4413    "TARGET_NEON"
4414    "#"
4415    "&& reload_completed"
4416    [(const_int 0)]
4417    {
4418     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4419     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4420     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4421     DONE;
4422     }
4423   [(set_attr "length" "8")
4424    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4425 )
4426
4427 (define_expand "vec_store_lanes<mode><mode>"
4428   [(set (match_operand:VDQX 0 "neon_struct_operand")
4429         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4430                      UNSPEC_VST1))]
4431   "TARGET_NEON")
4432
4433 (define_insn "neon_vst1<mode>"
4434   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4435         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4436                      UNSPEC_VST1))]
4437   "TARGET_NEON"
4438   "vst1.<V_sz_elem>\t%h1, %A0"
4439   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4440
4441 (define_insn "neon_vst1_lane<mode>"
4442   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4443         (vec_select:<V_elem>
4444           (match_operand:VDX 1 "s_register_operand" "w")
4445           (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4446   "TARGET_NEON"
4447 {
4448   HOST_WIDE_INT lane = INTVAL (operands[2]);
4449   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4450   if (lane < 0 || lane >= max)
4451     error ("lane out of range");
4452   if (max == 1)
4453     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4454   else
4455     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4456 }
4457   [(set (attr "neon_type")
4458       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4459                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4460                     (const_string "neon_vst1_vst2_lane")))])
4461
4462 (define_insn "neon_vst1_lane<mode>"
4463   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4464         (vec_select:<V_elem>
4465            (match_operand:VQX 1 "s_register_operand" "w")
4466            (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4467   "TARGET_NEON"
4468 {
4469   HOST_WIDE_INT lane = INTVAL (operands[2]);
4470   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4471   int regno = REGNO (operands[1]);
4472   if (lane < 0 || lane >= max)
4473     error ("lane out of range");
4474   else if (lane >= max / 2)
4475     {
4476       lane -= max / 2;
4477       regno += 2;
4478       operands[2] = GEN_INT (lane);
4479     }
4480   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4481   if (max == 2)
4482     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4483   else
4484     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4485 }
4486   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4487 )
4488
4489 (define_expand "vec_load_lanesti<mode>"
4490   [(set (match_operand:TI 0 "s_register_operand")
4491         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4492                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4493                    UNSPEC_VLD2))]
4494   "TARGET_NEON")
4495
4496 (define_insn "neon_vld2<mode>"
4497   [(set (match_operand:TI 0 "s_register_operand" "=w")
4498         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4499                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4500                    UNSPEC_VLD2))]
4501   "TARGET_NEON"
4502 {
4503   if (<V_sz_elem> == 64)
4504     return "vld1.64\t%h0, %A1";
4505   else
4506     return "vld2.<V_sz_elem>\t%h0, %A1";
4507 }
4508   [(set (attr "neon_type")
4509       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4510                     (const_string "neon_vld1_1_2_regs")
4511                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4512 )
4513
4514 (define_expand "vec_load_lanesoi<mode>"
4515   [(set (match_operand:OI 0 "s_register_operand")
4516         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4517                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4518                    UNSPEC_VLD2))]
4519   "TARGET_NEON")
4520
4521 (define_insn "neon_vld2<mode>"
4522   [(set (match_operand:OI 0 "s_register_operand" "=w")
4523         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4524                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4525                    UNSPEC_VLD2))]
4526   "TARGET_NEON"
4527   "vld2.<V_sz_elem>\t%h0, %A1"
4528   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4529
4530 (define_insn "neon_vld2_lane<mode>"
4531   [(set (match_operand:TI 0 "s_register_operand" "=w")
4532         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4533                     (match_operand:TI 2 "s_register_operand" "0")
4534                     (match_operand:SI 3 "immediate_operand" "i")
4535                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4536                    UNSPEC_VLD2_LANE))]
4537   "TARGET_NEON"
4538 {
4539   HOST_WIDE_INT lane = INTVAL (operands[3]);
4540   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4541   int regno = REGNO (operands[0]);
4542   rtx ops[4];
4543   if (lane < 0 || lane >= max)
4544     error ("lane out of range");
4545   ops[0] = gen_rtx_REG (DImode, regno);
4546   ops[1] = gen_rtx_REG (DImode, regno + 2);
4547   ops[2] = operands[1];
4548   ops[3] = operands[3];
4549   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4550   return "";
4551 }
4552   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4553 )
4554
4555 (define_insn "neon_vld2_lane<mode>"
4556   [(set (match_operand:OI 0 "s_register_operand" "=w")
4557         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4558                     (match_operand:OI 2 "s_register_operand" "0")
4559                     (match_operand:SI 3 "immediate_operand" "i")
4560                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4561                    UNSPEC_VLD2_LANE))]
4562   "TARGET_NEON"
4563 {
4564   HOST_WIDE_INT lane = INTVAL (operands[3]);
4565   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4566   int regno = REGNO (operands[0]);
4567   rtx ops[4];
4568   if (lane < 0 || lane >= max)
4569     error ("lane out of range");
4570   else if (lane >= max / 2)
4571     {
4572       lane -= max / 2;
4573       regno += 2;
4574     }
4575   ops[0] = gen_rtx_REG (DImode, regno);
4576   ops[1] = gen_rtx_REG (DImode, regno + 4);
4577   ops[2] = operands[1];
4578   ops[3] = GEN_INT (lane);
4579   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4580   return "";
4581 }
4582   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4583 )
4584
4585 (define_insn "neon_vld2_dup<mode>"
4586   [(set (match_operand:TI 0 "s_register_operand" "=w")
4587         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4588                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4589                    UNSPEC_VLD2_DUP))]
4590   "TARGET_NEON"
4591 {
4592   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4593     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4594   else
4595     return "vld1.<V_sz_elem>\t%h0, %A1";
4596 }
4597   [(set (attr "neon_type")
4598       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4599                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4600                     (const_string "neon_vld1_1_2_regs")))]
4601 )
4602
4603 (define_expand "vec_store_lanesti<mode>"
4604   [(set (match_operand:TI 0 "neon_struct_operand")
4605         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4606                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4607                    UNSPEC_VST2))]
4608   "TARGET_NEON")
4609
4610 (define_insn "neon_vst2<mode>"
4611   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4612         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4613                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4614                    UNSPEC_VST2))]
4615   "TARGET_NEON"
4616 {
4617   if (<V_sz_elem> == 64)
4618     return "vst1.64\t%h1, %A0";
4619   else
4620     return "vst2.<V_sz_elem>\t%h1, %A0";
4621 }
4622   [(set (attr "neon_type")
4623       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4624                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4625                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4626 )
4627
4628 (define_expand "vec_store_lanesoi<mode>"
4629   [(set (match_operand:OI 0 "neon_struct_operand")
4630         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4631                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4632                    UNSPEC_VST2))]
4633   "TARGET_NEON")
4634
4635 (define_insn "neon_vst2<mode>"
4636   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4637         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4638                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4639                    UNSPEC_VST2))]
4640   "TARGET_NEON"
4641   "vst2.<V_sz_elem>\t%h1, %A0"
4642   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4643 )
4644
4645 (define_insn "neon_vst2_lane<mode>"
4646   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4647         (unspec:<V_two_elem>
4648           [(match_operand:TI 1 "s_register_operand" "w")
4649            (match_operand:SI 2 "immediate_operand" "i")
4650            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4651           UNSPEC_VST2_LANE))]
4652   "TARGET_NEON"
4653 {
4654   HOST_WIDE_INT lane = INTVAL (operands[2]);
4655   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4656   int regno = REGNO (operands[1]);
4657   rtx ops[4];
4658   if (lane < 0 || lane >= max)
4659     error ("lane out of range");
4660   ops[0] = operands[0];
4661   ops[1] = gen_rtx_REG (DImode, regno);
4662   ops[2] = gen_rtx_REG (DImode, regno + 2);
4663   ops[3] = operands[2];
4664   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4665   return "";
4666 }
4667   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4668 )
4669
4670 (define_insn "neon_vst2_lane<mode>"
4671   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4672         (unspec:<V_two_elem>
4673            [(match_operand:OI 1 "s_register_operand" "w")
4674             (match_operand:SI 2 "immediate_operand" "i")
4675             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4676            UNSPEC_VST2_LANE))]
4677   "TARGET_NEON"
4678 {
4679   HOST_WIDE_INT lane = INTVAL (operands[2]);
4680   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4681   int regno = REGNO (operands[1]);
4682   rtx ops[4];
4683   if (lane < 0 || lane >= max)
4684     error ("lane out of range");
4685   else if (lane >= max / 2)
4686     {
4687       lane -= max / 2;
4688       regno += 2;
4689     }
4690   ops[0] = operands[0];
4691   ops[1] = gen_rtx_REG (DImode, regno);
4692   ops[2] = gen_rtx_REG (DImode, regno + 4);
4693   ops[3] = GEN_INT (lane);
4694   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4695   return "";
4696 }
4697   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4698 )
4699
4700 (define_expand "vec_load_lanesei<mode>"
4701   [(set (match_operand:EI 0 "s_register_operand")
4702         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4703                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4704                    UNSPEC_VLD3))]
4705   "TARGET_NEON")
4706
4707 (define_insn "neon_vld3<mode>"
4708   [(set (match_operand:EI 0 "s_register_operand" "=w")
4709         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4710                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4711                    UNSPEC_VLD3))]
4712   "TARGET_NEON"
4713 {
4714   if (<V_sz_elem> == 64)
4715     return "vld1.64\t%h0, %A1";
4716   else
4717     return "vld3.<V_sz_elem>\t%h0, %A1";
4718 }
4719   [(set (attr "neon_type")
4720       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4721                     (const_string "neon_vld1_1_2_regs")
4722                     (const_string "neon_vld3_vld4")))]
4723 )
4724
4725 (define_expand "vec_load_lanesci<mode>"
4726   [(match_operand:CI 0 "s_register_operand")
4727    (match_operand:CI 1 "neon_struct_operand")
4728    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4729   "TARGET_NEON"
4730 {
4731   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4732   DONE;
4733 })
4734
4735 (define_expand "neon_vld3<mode>"
4736   [(match_operand:CI 0 "s_register_operand")
4737    (match_operand:CI 1 "neon_struct_operand")
4738    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4739   "TARGET_NEON"
4740 {
4741   rtx mem;
4742
4743   mem = adjust_address (operands[1], EImode, 0);
4744   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4745   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4746   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4747   DONE;
4748 })
4749
4750 (define_insn "neon_vld3qa<mode>"
4751   [(set (match_operand:CI 0 "s_register_operand" "=w")
4752         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4753                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4754                    UNSPEC_VLD3A))]
4755   "TARGET_NEON"
4756 {
4757   int regno = REGNO (operands[0]);
4758   rtx ops[4];
4759   ops[0] = gen_rtx_REG (DImode, regno);
4760   ops[1] = gen_rtx_REG (DImode, regno + 4);
4761   ops[2] = gen_rtx_REG (DImode, regno + 8);
4762   ops[3] = operands[1];
4763   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4764   return "";
4765 }
4766   [(set_attr "neon_type" "neon_vld3_vld4")]
4767 )
4768
4769 (define_insn "neon_vld3qb<mode>"
4770   [(set (match_operand:CI 0 "s_register_operand" "=w")
4771         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4772                     (match_operand:CI 2 "s_register_operand" "0")
4773                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4774                    UNSPEC_VLD3B))]
4775   "TARGET_NEON"
4776 {
4777   int regno = REGNO (operands[0]);
4778   rtx ops[4];
4779   ops[0] = gen_rtx_REG (DImode, regno + 2);
4780   ops[1] = gen_rtx_REG (DImode, regno + 6);
4781   ops[2] = gen_rtx_REG (DImode, regno + 10);
4782   ops[3] = operands[1];
4783   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4784   return "";
4785 }
4786   [(set_attr "neon_type" "neon_vld3_vld4")]
4787 )
4788
4789 (define_insn "neon_vld3_lane<mode>"
4790   [(set (match_operand:EI 0 "s_register_operand" "=w")
4791         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4792                     (match_operand:EI 2 "s_register_operand" "0")
4793                     (match_operand:SI 3 "immediate_operand" "i")
4794                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795                    UNSPEC_VLD3_LANE))]
4796   "TARGET_NEON"
4797 {
4798   HOST_WIDE_INT lane = INTVAL (operands[3]);
4799   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4800   int regno = REGNO (operands[0]);
4801   rtx ops[5];
4802   if (lane < 0 || lane >= max)
4803     error ("lane out of range");
4804   ops[0] = gen_rtx_REG (DImode, regno);
4805   ops[1] = gen_rtx_REG (DImode, regno + 2);
4806   ops[2] = gen_rtx_REG (DImode, regno + 4);
4807   ops[3] = operands[1];
4808   ops[4] = operands[3];
4809   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4810                    ops);
4811   return "";
4812 }
4813   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4814 )
4815
4816 (define_insn "neon_vld3_lane<mode>"
4817   [(set (match_operand:CI 0 "s_register_operand" "=w")
4818         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4819                     (match_operand:CI 2 "s_register_operand" "0")
4820                     (match_operand:SI 3 "immediate_operand" "i")
4821                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4822                    UNSPEC_VLD3_LANE))]
4823   "TARGET_NEON"
4824 {
4825   HOST_WIDE_INT lane = INTVAL (operands[3]);
4826   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4827   int regno = REGNO (operands[0]);
4828   rtx ops[5];
4829   if (lane < 0 || lane >= max)
4830     error ("lane out of range");
4831   else if (lane >= max / 2)
4832     {
4833       lane -= max / 2;
4834       regno += 2;
4835     }
4836   ops[0] = gen_rtx_REG (DImode, regno);
4837   ops[1] = gen_rtx_REG (DImode, regno + 4);
4838   ops[2] = gen_rtx_REG (DImode, regno + 8);
4839   ops[3] = operands[1];
4840   ops[4] = GEN_INT (lane);
4841   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4842                    ops);
4843   return "";
4844 }
4845   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4846 )
4847
4848 (define_insn "neon_vld3_dup<mode>"
4849   [(set (match_operand:EI 0 "s_register_operand" "=w")
4850         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4851                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4852                    UNSPEC_VLD3_DUP))]
4853   "TARGET_NEON"
4854 {
4855   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4856     {
4857       int regno = REGNO (operands[0]);
4858       rtx ops[4];
4859       ops[0] = gen_rtx_REG (DImode, regno);
4860       ops[1] = gen_rtx_REG (DImode, regno + 2);
4861       ops[2] = gen_rtx_REG (DImode, regno + 4);
4862       ops[3] = operands[1];
4863       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
4864       return "";
4865     }
4866   else
4867     return "vld1.<V_sz_elem>\t%h0, %A1";
4868 }
4869   [(set (attr "neon_type")
4870       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4871                     (const_string "neon_vld3_vld4_all_lanes")
4872                     (const_string "neon_vld1_1_2_regs")))])
4873
4874 (define_expand "vec_store_lanesei<mode>"
4875   [(set (match_operand:EI 0 "neon_struct_operand")
4876         (unspec:EI [(match_operand:EI 1 "s_register_operand")
4877                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4878                    UNSPEC_VST3))]
4879   "TARGET_NEON")
4880
4881 (define_insn "neon_vst3<mode>"
4882   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4883         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4884                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4885                    UNSPEC_VST3))]
4886   "TARGET_NEON"
4887 {
4888   if (<V_sz_elem> == 64)
4889     return "vst1.64\t%h1, %A0";
4890   else
4891     return "vst3.<V_sz_elem>\t%h1, %A0";
4892 }
4893   [(set (attr "neon_type")
4894       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4895                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4896                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
4897
4898 (define_expand "vec_store_lanesci<mode>"
4899   [(match_operand:CI 0 "neon_struct_operand")
4900    (match_operand:CI 1 "s_register_operand")
4901    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4902   "TARGET_NEON"
4903 {
4904   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
4905   DONE;
4906 })
4907
4908 (define_expand "neon_vst3<mode>"
4909   [(match_operand:CI 0 "neon_struct_operand")
4910    (match_operand:CI 1 "s_register_operand")
4911    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4912   "TARGET_NEON"
4913 {
4914   rtx mem;
4915
4916   mem = adjust_address (operands[0], EImode, 0);
4917   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
4918   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4919   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
4920   DONE;
4921 })
4922
4923 (define_insn "neon_vst3qa<mode>"
4924   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4925         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4926                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4927                    UNSPEC_VST3A))]
4928   "TARGET_NEON"
4929 {
4930   int regno = REGNO (operands[1]);
4931   rtx ops[4];
4932   ops[0] = operands[0];
4933   ops[1] = gen_rtx_REG (DImode, regno);
4934   ops[2] = gen_rtx_REG (DImode, regno + 4);
4935   ops[3] = gen_rtx_REG (DImode, regno + 8);
4936   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4937   return "";
4938 }
4939   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4940 )
4941
4942 (define_insn "neon_vst3qb<mode>"
4943   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4944         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4945                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4946                    UNSPEC_VST3B))]
4947   "TARGET_NEON"
4948 {
4949   int regno = REGNO (operands[1]);
4950   rtx ops[4];
4951   ops[0] = operands[0];
4952   ops[1] = gen_rtx_REG (DImode, regno + 2);
4953   ops[2] = gen_rtx_REG (DImode, regno + 6);
4954   ops[3] = gen_rtx_REG (DImode, regno + 10);
4955   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4956   return "";
4957 }
4958   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4959 )
4960
4961 (define_insn "neon_vst3_lane<mode>"
4962   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4963         (unspec:<V_three_elem>
4964            [(match_operand:EI 1 "s_register_operand" "w")
4965             (match_operand:SI 2 "immediate_operand" "i")
4966             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4967            UNSPEC_VST3_LANE))]
4968   "TARGET_NEON"
4969 {
4970   HOST_WIDE_INT lane = INTVAL (operands[2]);
4971   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4972   int regno = REGNO (operands[1]);
4973   rtx ops[5];
4974   if (lane < 0 || lane >= max)
4975     error ("lane out of range");
4976   ops[0] = operands[0];
4977   ops[1] = gen_rtx_REG (DImode, regno);
4978   ops[2] = gen_rtx_REG (DImode, regno + 2);
4979   ops[3] = gen_rtx_REG (DImode, regno + 4);
4980   ops[4] = operands[2];
4981   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
4982                    ops);
4983   return "";
4984 }
4985   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
4986 )
4987
4988 (define_insn "neon_vst3_lane<mode>"
4989   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4990         (unspec:<V_three_elem>
4991            [(match_operand:CI 1 "s_register_operand" "w")
4992             (match_operand:SI 2 "immediate_operand" "i")
4993             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4994            UNSPEC_VST3_LANE))]
4995   "TARGET_NEON"
4996 {
4997   HOST_WIDE_INT lane = INTVAL (operands[2]);
4998   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4999   int regno = REGNO (operands[1]);
5000   rtx ops[5];
5001   if (lane < 0 || lane >= max)
5002     error ("lane out of range");
5003   else if (lane >= max / 2)
5004     {
5005       lane -= max / 2;
5006       regno += 2;
5007     }
5008   ops[0] = operands[0];
5009   ops[1] = gen_rtx_REG (DImode, regno);
5010   ops[2] = gen_rtx_REG (DImode, regno + 4);
5011   ops[3] = gen_rtx_REG (DImode, regno + 8);
5012   ops[4] = GEN_INT (lane);
5013   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5014                    ops);
5015   return "";
5016 }
5017 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5018
5019 (define_expand "vec_load_lanesoi<mode>"
5020   [(set (match_operand:OI 0 "s_register_operand")
5021         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5022                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5023                    UNSPEC_VLD4))]
5024   "TARGET_NEON")
5025
5026 (define_insn "neon_vld4<mode>"
5027   [(set (match_operand:OI 0 "s_register_operand" "=w")
5028         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5029                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5030                    UNSPEC_VLD4))]
5031   "TARGET_NEON"
5032 {
5033   if (<V_sz_elem> == 64)
5034     return "vld1.64\t%h0, %A1";
5035   else
5036     return "vld4.<V_sz_elem>\t%h0, %A1";
5037 }
5038   [(set (attr "neon_type")
5039       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5040                     (const_string "neon_vld1_1_2_regs")
5041                     (const_string "neon_vld3_vld4")))]
5042 )
5043
5044 (define_expand "vec_load_lanesxi<mode>"
5045   [(match_operand:XI 0 "s_register_operand")
5046    (match_operand:XI 1 "neon_struct_operand")
5047    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5048   "TARGET_NEON"
5049 {
5050   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5051   DONE;
5052 })
5053
5054 (define_expand "neon_vld4<mode>"
5055   [(match_operand:XI 0 "s_register_operand")
5056    (match_operand:XI 1 "neon_struct_operand")
5057    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5058   "TARGET_NEON"
5059 {
5060   rtx mem;
5061
5062   mem = adjust_address (operands[1], OImode, 0);
5063   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5064   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5065   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5066   DONE;
5067 })
5068
5069 (define_insn "neon_vld4qa<mode>"
5070   [(set (match_operand:XI 0 "s_register_operand" "=w")
5071         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5072                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5073                    UNSPEC_VLD4A))]
5074   "TARGET_NEON"
5075 {
5076   int regno = REGNO (operands[0]);
5077   rtx ops[5];
5078   ops[0] = gen_rtx_REG (DImode, regno);
5079   ops[1] = gen_rtx_REG (DImode, regno + 4);
5080   ops[2] = gen_rtx_REG (DImode, regno + 8);
5081   ops[3] = gen_rtx_REG (DImode, regno + 12);
5082   ops[4] = operands[1];
5083   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5084   return "";
5085 }
5086   [(set_attr "neon_type" "neon_vld3_vld4")]
5087 )
5088
5089 (define_insn "neon_vld4qb<mode>"
5090   [(set (match_operand:XI 0 "s_register_operand" "=w")
5091         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5092                     (match_operand:XI 2 "s_register_operand" "0")
5093                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5094                    UNSPEC_VLD4B))]
5095   "TARGET_NEON"
5096 {
5097   int regno = REGNO (operands[0]);
5098   rtx ops[5];
5099   ops[0] = gen_rtx_REG (DImode, regno + 2);
5100   ops[1] = gen_rtx_REG (DImode, regno + 6);
5101   ops[2] = gen_rtx_REG (DImode, regno + 10);
5102   ops[3] = gen_rtx_REG (DImode, regno + 14);
5103   ops[4] = operands[1];
5104   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5105   return "";
5106 }
5107   [(set_attr "neon_type" "neon_vld3_vld4")]
5108 )
5109
5110 (define_insn "neon_vld4_lane<mode>"
5111   [(set (match_operand:OI 0 "s_register_operand" "=w")
5112         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5113                     (match_operand:OI 2 "s_register_operand" "0")
5114                     (match_operand:SI 3 "immediate_operand" "i")
5115                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5116                    UNSPEC_VLD4_LANE))]
5117   "TARGET_NEON"
5118 {
5119   HOST_WIDE_INT lane = INTVAL (operands[3]);
5120   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5121   int regno = REGNO (operands[0]);
5122   rtx ops[6];
5123   if (lane < 0 || lane >= max)
5124     error ("lane out of range");
5125   ops[0] = gen_rtx_REG (DImode, regno);
5126   ops[1] = gen_rtx_REG (DImode, regno + 2);
5127   ops[2] = gen_rtx_REG (DImode, regno + 4);
5128   ops[3] = gen_rtx_REG (DImode, regno + 6);
5129   ops[4] = operands[1];
5130   ops[5] = operands[3];
5131   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5132                    ops);
5133   return "";
5134 }
5135   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5136 )
5137
5138 (define_insn "neon_vld4_lane<mode>"
5139   [(set (match_operand:XI 0 "s_register_operand" "=w")
5140         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5141                     (match_operand:XI 2 "s_register_operand" "0")
5142                     (match_operand:SI 3 "immediate_operand" "i")
5143                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5144                    UNSPEC_VLD4_LANE))]
5145   "TARGET_NEON"
5146 {
5147   HOST_WIDE_INT lane = INTVAL (operands[3]);
5148   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5149   int regno = REGNO (operands[0]);
5150   rtx ops[6];
5151   if (lane < 0 || lane >= max)
5152     error ("lane out of range");
5153   else if (lane >= max / 2)
5154     {
5155       lane -= max / 2;
5156       regno += 2;
5157     }
5158   ops[0] = gen_rtx_REG (DImode, regno);
5159   ops[1] = gen_rtx_REG (DImode, regno + 4);
5160   ops[2] = gen_rtx_REG (DImode, regno + 8);
5161   ops[3] = gen_rtx_REG (DImode, regno + 12);
5162   ops[4] = operands[1];
5163   ops[5] = GEN_INT (lane);
5164   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5165                    ops);
5166   return "";
5167 }
5168   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5169 )
5170
5171 (define_insn "neon_vld4_dup<mode>"
5172   [(set (match_operand:OI 0 "s_register_operand" "=w")
5173         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5174                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5175                    UNSPEC_VLD4_DUP))]
5176   "TARGET_NEON"
5177 {
5178   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5179     {
5180       int regno = REGNO (operands[0]);
5181       rtx ops[5];
5182       ops[0] = gen_rtx_REG (DImode, regno);
5183       ops[1] = gen_rtx_REG (DImode, regno + 2);
5184       ops[2] = gen_rtx_REG (DImode, regno + 4);
5185       ops[3] = gen_rtx_REG (DImode, regno + 6);
5186       ops[4] = operands[1];
5187       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5188                        ops);
5189       return "";
5190     }
5191   else
5192     return "vld1.<V_sz_elem>\t%h0, %A1";
5193 }
5194   [(set (attr "neon_type")
5195       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5196                     (const_string "neon_vld3_vld4_all_lanes")
5197                     (const_string "neon_vld1_1_2_regs")))]
5198 )
5199
5200 (define_expand "vec_store_lanesoi<mode>"
5201   [(set (match_operand:OI 0 "neon_struct_operand")
5202         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5203                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5204                    UNSPEC_VST4))]
5205   "TARGET_NEON")
5206
5207 (define_insn "neon_vst4<mode>"
5208   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5209         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5210                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5211                    UNSPEC_VST4))]
5212   "TARGET_NEON"
5213 {
5214   if (<V_sz_elem> == 64)
5215     return "vst1.64\t%h1, %A0";
5216   else
5217     return "vst4.<V_sz_elem>\t%h1, %A0";
5218 }
5219   [(set (attr "neon_type")
5220       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5221                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5222                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5223 )
5224
5225 (define_expand "vec_store_lanesxi<mode>"
5226   [(match_operand:XI 0 "neon_struct_operand")
5227    (match_operand:XI 1 "s_register_operand")
5228    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5229   "TARGET_NEON"
5230 {
5231   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5232   DONE;
5233 })
5234
5235 (define_expand "neon_vst4<mode>"
5236   [(match_operand:XI 0 "neon_struct_operand")
5237    (match_operand:XI 1 "s_register_operand")
5238    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5239   "TARGET_NEON"
5240 {
5241   rtx mem;
5242
5243   mem = adjust_address (operands[0], OImode, 0);
5244   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5245   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5246   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5247   DONE;
5248 })
5249
5250 (define_insn "neon_vst4qa<mode>"
5251   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5252         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5253                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5254                    UNSPEC_VST4A))]
5255   "TARGET_NEON"
5256 {
5257   int regno = REGNO (operands[1]);
5258   rtx ops[5];
5259   ops[0] = operands[0];
5260   ops[1] = gen_rtx_REG (DImode, regno);
5261   ops[2] = gen_rtx_REG (DImode, regno + 4);
5262   ops[3] = gen_rtx_REG (DImode, regno + 8);
5263   ops[4] = gen_rtx_REG (DImode, regno + 12);
5264   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5265   return "";
5266 }
5267   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5268 )
5269
5270 (define_insn "neon_vst4qb<mode>"
5271   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5272         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5273                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5274                    UNSPEC_VST4B))]
5275   "TARGET_NEON"
5276 {
5277   int regno = REGNO (operands[1]);
5278   rtx ops[5];
5279   ops[0] = operands[0];
5280   ops[1] = gen_rtx_REG (DImode, regno + 2);
5281   ops[2] = gen_rtx_REG (DImode, regno + 6);
5282   ops[3] = gen_rtx_REG (DImode, regno + 10);
5283   ops[4] = gen_rtx_REG (DImode, regno + 14);
5284   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5285   return "";
5286 }
5287   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5288 )
5289
5290 (define_insn "neon_vst4_lane<mode>"
5291   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5292         (unspec:<V_four_elem>
5293            [(match_operand:OI 1 "s_register_operand" "w")
5294             (match_operand:SI 2 "immediate_operand" "i")
5295             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5296            UNSPEC_VST4_LANE))]
5297   "TARGET_NEON"
5298 {
5299   HOST_WIDE_INT lane = INTVAL (operands[2]);
5300   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5301   int regno = REGNO (operands[1]);
5302   rtx ops[6];
5303   if (lane < 0 || lane >= max)
5304     error ("lane out of range");
5305   ops[0] = operands[0];
5306   ops[1] = gen_rtx_REG (DImode, regno);
5307   ops[2] = gen_rtx_REG (DImode, regno + 2);
5308   ops[3] = gen_rtx_REG (DImode, regno + 4);
5309   ops[4] = gen_rtx_REG (DImode, regno + 6);
5310   ops[5] = operands[2];
5311   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5312                    ops);
5313   return "";
5314 }
5315   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5316 )
5317
5318 (define_insn "neon_vst4_lane<mode>"
5319   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5320         (unspec:<V_four_elem>
5321            [(match_operand:XI 1 "s_register_operand" "w")
5322             (match_operand:SI 2 "immediate_operand" "i")
5323             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5324            UNSPEC_VST4_LANE))]
5325   "TARGET_NEON"
5326 {
5327   HOST_WIDE_INT lane = INTVAL (operands[2]);
5328   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5329   int regno = REGNO (operands[1]);
5330   rtx ops[6];
5331   if (lane < 0 || lane >= max)
5332     error ("lane out of range");
5333   else if (lane >= max / 2)
5334     {
5335       lane -= max / 2;
5336       regno += 2;
5337     }
5338   ops[0] = operands[0];
5339   ops[1] = gen_rtx_REG (DImode, regno);
5340   ops[2] = gen_rtx_REG (DImode, regno + 4);
5341   ops[3] = gen_rtx_REG (DImode, regno + 8);
5342   ops[4] = gen_rtx_REG (DImode, regno + 12);
5343   ops[5] = GEN_INT (lane);
5344   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5345                    ops);
5346   return "";
5347 }
5348   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5349 )
5350
5351 (define_expand "neon_vand<mode>"
5352   [(match_operand:VDQX 0 "s_register_operand" "")
5353    (match_operand:VDQX 1 "s_register_operand" "")
5354    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5355    (match_operand:SI 3 "immediate_operand" "")]
5356   "TARGET_NEON"
5357 {
5358   emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5359   DONE;
5360 })
5361
5362 (define_expand "neon_vorr<mode>"
5363   [(match_operand:VDQX 0 "s_register_operand" "")
5364    (match_operand:VDQX 1 "s_register_operand" "")
5365    (match_operand:VDQX 2 "neon_logic_op2" "")
5366    (match_operand:SI 3 "immediate_operand" "")]
5367   "TARGET_NEON"
5368 {
5369   emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5370   DONE;
5371 })
5372
5373 (define_expand "neon_veor<mode>"
5374   [(match_operand:VDQX 0 "s_register_operand" "")
5375    (match_operand:VDQX 1 "s_register_operand" "")
5376    (match_operand:VDQX 2 "s_register_operand" "")
5377    (match_operand:SI 3 "immediate_operand" "")]
5378   "TARGET_NEON"
5379 {
5380   emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5381   DONE;
5382 })
5383
5384 (define_expand "neon_vbic<mode>"
5385   [(match_operand:VDQX 0 "s_register_operand" "")
5386    (match_operand:VDQX 1 "s_register_operand" "")
5387    (match_operand:VDQX 2 "neon_logic_op2" "")
5388    (match_operand:SI 3 "immediate_operand" "")]
5389   "TARGET_NEON"
5390 {
5391   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5392   DONE;
5393 })
5394
5395 (define_expand "neon_vorn<mode>"
5396   [(match_operand:VDQX 0 "s_register_operand" "")
5397    (match_operand:VDQX 1 "s_register_operand" "")
5398    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5399    (match_operand:SI 3 "immediate_operand" "")]
5400   "TARGET_NEON"
5401 {
5402   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5403   DONE;
5404 })
5405
5406 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5407   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5408         (SE:<V_unpack> (vec_select:<V_HALF>
5409                           (match_operand:VU 1 "register_operand" "w")
5410                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5411   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5412   "vmovl.<US><V_sz_elem> %q0, %e1"
5413   [(set_attr "neon_type" "neon_shift_1")]
5414 )
5415
5416 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5417   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5418         (SE:<V_unpack> (vec_select:<V_HALF>
5419                           (match_operand:VU 1 "register_operand" "w")
5420                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5421   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5422   "vmovl.<US><V_sz_elem> %q0, %f1"
5423   [(set_attr "neon_type" "neon_shift_1")]
5424 )
5425
5426 (define_expand "vec_unpack<US>_hi_<mode>"
5427   [(match_operand:<V_unpack> 0 "register_operand" "")
5428    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5429  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5430   {
5431    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5432    rtx t1;
5433    int i;
5434    for (i = 0; i < (<V_mode_nunits>/2); i++)
5435      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5436
5437    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5438    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5439                                                  operands[1],
5440                                                  t1));
5441    DONE;
5442   }
5443 )
5444
5445 (define_expand "vec_unpack<US>_lo_<mode>"
5446   [(match_operand:<V_unpack> 0 "register_operand" "")
5447    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5448  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5449   {
5450    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5451    rtx t1;
5452    int i;
5453    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5454      RTVEC_ELT (v, i) = GEN_INT (i);
5455    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5456    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5457                                                  operands[1],
5458                                                  t1));
5459    DONE;
5460   }
5461 )
5462
5463 (define_insn "neon_vec_<US>mult_lo_<mode>"
5464  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5465        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5466                            (match_operand:VU 1 "register_operand" "w")
5467                            (match_operand:VU 2 "vect_par_constant_low" "")))
5468                         (SE:<V_unpack> (vec_select:<V_HALF>
5469                            (match_operand:VU 3 "register_operand" "w")
5470                            (match_dup 2)))))]
5471   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5472   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5473   [(set_attr "neon_type" "neon_shift_1")]
5474 )
5475
5476 (define_expand "vec_widen_<US>mult_lo_<mode>"
5477   [(match_operand:<V_unpack> 0 "register_operand" "")
5478    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5479    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5480  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5481  {
5482    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5483    rtx t1;
5484    int i;
5485    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5486      RTVEC_ELT (v, i) = GEN_INT (i);
5487    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5488
5489    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5490                                                operands[1],
5491                                                t1,
5492                                                operands[2]));
5493    DONE;
5494  }
5495 )
5496
5497 (define_insn "neon_vec_<US>mult_hi_<mode>"
5498  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5499       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5500                             (match_operand:VU 1 "register_operand" "w")
5501                             (match_operand:VU 2 "vect_par_constant_high" "")))
5502                        (SE:<V_unpack> (vec_select:<V_HALF>
5503                             (match_operand:VU 3 "register_operand" "w")
5504                             (match_dup 2)))))]
5505   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5506   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5507   [(set_attr "neon_type" "neon_shift_1")]
5508 )
5509
5510 (define_expand "vec_widen_<US>mult_hi_<mode>"
5511   [(match_operand:<V_unpack> 0 "register_operand" "")
5512    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5513    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5514  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5515  {
5516    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5517    rtx t1;
5518    int i;
5519    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5520      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5521    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5522
5523    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5524                                                operands[1],
5525                                                t1,
5526                                                operands[2]));
5527    DONE;
5528
5529  }
5530 )
5531
5532 (define_insn "neon_vec_<US>shiftl_<mode>"
5533  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5534        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5535        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5536   "TARGET_NEON"
5537 {
5538   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5539 }
5540   [(set_attr "neon_type" "neon_shift_1")]
5541 )
5542
5543 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5544   [(match_operand:<V_unpack> 0 "register_operand" "")
5545    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5546    (match_operand:SI 2 "immediate_operand" "i")]
5547  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5548  {
5549   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5550                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5551                 operands[2]));
5552    DONE;
5553  }
5554 )
5555
5556 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5557   [(match_operand:<V_unpack> 0 "register_operand" "")
5558    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5559    (match_operand:SI 2 "immediate_operand" "i")]
5560  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5561  {
5562   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5563                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5564                                      GET_MODE_SIZE (<V_HALF>mode)),
5565                 operands[2]));
5566    DONE;
5567  }
5568 )
5569
5570 ;; Vectorize for non-neon-quad case
5571 (define_insn "neon_unpack<US>_<mode>"
5572  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5573        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5574  "TARGET_NEON"
5575  "vmovl.<US><V_sz_elem> %q0, %P1"
5576   [(set_attr "neon_type" "neon_shift_1")]
5577 )
5578
5579 (define_expand "vec_unpack<US>_lo_<mode>"
5580  [(match_operand:<V_double_width> 0 "register_operand" "")
5581   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5582  "TARGET_NEON"
5583 {
5584   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5585   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5586   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5587
5588   DONE;
5589 }
5590 )
5591
5592 (define_expand "vec_unpack<US>_hi_<mode>"
5593  [(match_operand:<V_double_width> 0 "register_operand" "")
5594   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5595  "TARGET_NEON"
5596 {
5597   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5598   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5599   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5600
5601   DONE;
5602 }
5603 )
5604
5605 (define_insn "neon_vec_<US>mult_<mode>"
5606  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5607        (mult:<V_widen> (SE:<V_widen>
5608                            (match_operand:VDI 1 "register_operand" "w"))
5609                        (SE:<V_widen>
5610                            (match_operand:VDI 2 "register_operand" "w"))))]
5611   "TARGET_NEON"
5612   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5613   [(set_attr "neon_type" "neon_shift_1")]
5614 )
5615
5616 (define_expand "vec_widen_<US>mult_hi_<mode>"
5617   [(match_operand:<V_double_width> 0 "register_operand" "")
5618    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5619    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5620  "TARGET_NEON"
5621  {
5622    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5623    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5624    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5625
5626    DONE;
5627
5628  }
5629 )
5630
5631 (define_expand "vec_widen_<US>mult_lo_<mode>"
5632   [(match_operand:<V_double_width> 0 "register_operand" "")
5633    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5634    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5635  "TARGET_NEON"
5636  {
5637    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5638    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5639    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5640
5641    DONE;
5642
5643  }
5644 )
5645
5646 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5647  [(match_operand:<V_double_width> 0 "register_operand" "")
5648    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5649    (match_operand:SI 2 "immediate_operand" "i")]
5650  "TARGET_NEON"
5651  {
5652    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5653    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5654    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5655
5656    DONE;
5657  }
5658 )
5659
5660 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5661   [(match_operand:<V_double_width> 0 "register_operand" "")
5662    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5663    (match_operand:SI 2 "immediate_operand" "i")]
5664  "TARGET_NEON"
5665  {
5666    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5667    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5668    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5669
5670    DONE;
5671  }
5672 )
5673
5674 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5675 ; because the ordering of vector elements in Q registers is different from what
5676 ; the semantics of the instructions require.
5677
5678 (define_insn "vec_pack_trunc_<mode>"
5679  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5680        (vec_concat:<V_narrow_pack>
5681                 (truncate:<V_narrow>
5682                         (match_operand:VN 1 "register_operand" "w"))
5683                 (truncate:<V_narrow>
5684                         (match_operand:VN 2 "register_operand" "w"))))]
5685  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5686  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5687  [(set_attr "neon_type" "neon_shift_1")
5688   (set_attr "length" "8")]
5689 )
5690
5691 ;; For the non-quad case.
5692 (define_insn "neon_vec_pack_trunc_<mode>"
5693  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5694        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5695  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5696  "vmovn.i<V_sz_elem>\t%P0, %q1"
5697  [(set_attr "neon_type" "neon_shift_1")]
5698 )
5699
5700 (define_expand "vec_pack_trunc_<mode>"
5701  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5702   (match_operand:VSHFT 1 "register_operand" "")
5703   (match_operand:VSHFT 2 "register_operand")]
5704  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5705 {
5706   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5707
5708   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5709   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5710   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5711   DONE;
5712 })
5713
5714 (define_insn "neon_vabd<mode>_2"
5715  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5716        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5717                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5718  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5719  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5720  [(set (attr "neon_type")
5721        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5722                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5723                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5724                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5725                      (const_string "neon_int_5")))]
5726 )
5727
5728 (define_insn "neon_vabd<mode>_3"
5729  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5730        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5731                              (match_operand:VDQ 2 "s_register_operand" "w")]
5732                  UNSPEC_VSUB)))]
5733  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5734  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5735  [(set (attr "neon_type")
5736        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5737                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5738                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5739                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5740                      (const_string "neon_int_5")))]
5741 )