gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006-2013 Free Software Foundation, Inc.
   3 ;; Written by CodeSourcery.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 ;; Attribute used to permit string comparisons against <VQH_mnem> in
  23 ;; neon_type attribute definitions.
  24 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
  25
  26 (define_insn "*neon_mov<mode>"
  27   [(set (match_operand:VDX 0 "nonimmediate_operand"
  28           "=w,Un,w, w,  ?r,?w,?r,?r, ?Us")
  29         (match_operand:VDX 1 "general_operand"
  30           " w,w, Dn,Uni, w, r, r, Usi,r"))]
  31   "TARGET_NEON
  32    && (register_operand (operands[0], <MODE>mode)
  33        || register_operand (operands[1], <MODE>mode))"
  34 {
  35   if (which_alternative == 2)
  36     {
  37       int width, is_valid;
  38       static char templ[40];
  39
  40       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  41         &operands[1], &width);
  42
  43       gcc_assert (is_valid != 0);
  44
  45       if (width == 0)
  46         return "vmov.f32\t%P0, %1  @ <mode>";
  47       else
  48         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
  49
  50       return templ;
  51     }
  52
  53   switch (which_alternative)
  54     {
  55     case 0: return "vmov\t%P0, %P1  @ <mode>";
  56     case 1: case 3: return output_move_neon (operands);
  57     case 2: gcc_unreachable ();
  58     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
  59     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
  60     default: return output_move_double (operands, true, NULL);
  61     }
  62 }
  63  [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
  64   (set_attr "type" "*,f_stored,*,f_loadd,*,*,mov_reg,load2,store2")
  65   (set_attr "length" "4,4,4,4,4,4,8,8,8")
  66   (set_attr "arm_pool_range"     "*,*,*,1020,*,*,*,1020,*")
  67   (set_attr "thumb2_pool_range"     "*,*,*,1018,*,*,*,1018,*")
  68   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
  69
  70 (define_insn "*neon_mov<mode>"
  71   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
  72           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
  73         (match_operand:VQXMOV 1 "general_operand"
  74           " w,w, Dn,Uni, w, r, r, Usi, r"))]
  75   "TARGET_NEON
  76    && (register_operand (operands[0], <MODE>mode)
  77        || register_operand (operands[1], <MODE>mode))"
  78 {
  79   if (which_alternative == 2)
  80     {
  81       int width, is_valid;
  82       static char templ[40];
  83
  84       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  85         &operands[1], &width);
  86
  87       gcc_assert (is_valid != 0);
  88
  89       if (width == 0)
  90         return "vmov.f32\t%q0, %1  @ <mode>";
  91       else
  92         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
  93
  94       return templ;
  95     }
  96
  97   switch (which_alternative)
  98     {
  99     case 0: return "vmov\t%q0, %q1  @ <mode>";
 100     case 1: case 3: return output_move_neon (operands);
 101     case 2: gcc_unreachable ();
 102     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 103     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 104     default: return output_move_quad (operands);
 105     }
 106 }
 107   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
 108                           neon_mrrc,neon_mcr_2_mcrr,*,*,*")
 109    (set_attr "type" "*,*,*,*,*,*,mov_reg,load4,store4")
 110    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 111    (set_attr "arm_pool_range" "*,*,*,1020,*,*,*,1020,*")
 112    (set_attr "thumb2_pool_range" "*,*,*,1018,*,*,*,1018,*")
 113    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 114
 115 (define_expand "movti"
 116   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 117         (match_operand:TI 1 "general_operand" ""))]
 118   "TARGET_NEON"
 119 {
 120   if (can_create_pseudo_p ())
 121     {
 122       if (!REG_P (operands[0]))
 123         operands[1] = force_reg (TImode, operands[1]);
 124     }
 125 })
 126
 127 (define_expand "mov<mode>"
 128   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 129         (match_operand:VSTRUCT 1 "general_operand" ""))]
 130   "TARGET_NEON"
 131 {
 132   if (can_create_pseudo_p ())
 133     {
 134       if (!REG_P (operands[0]))
 135         operands[1] = force_reg (<MODE>mode, operands[1]);
 136     }
 137 })
 138
 139 (define_insn "*neon_mov<mode>"
 140   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 141         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 142   "TARGET_NEON
 143    && (register_operand (operands[0], <MODE>mode)
 144        || register_operand (operands[1], <MODE>mode))"
 145 {
 146   switch (which_alternative)
 147     {
 148     case 0: return "#";
 149     case 1: case 2: return output_move_neon (operands);
 150     default: gcc_unreachable ();
 151     }
 152 }
 153   [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
 154    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 155
 156 (define_split
 157   [(set (match_operand:EI 0 "s_register_operand" "")
 158         (match_operand:EI 1 "s_register_operand" ""))]
 159   "TARGET_NEON && reload_completed"
 160   [(set (match_dup 0) (match_dup 1))
 161    (set (match_dup 2) (match_dup 3))]
 162 {
 163   int rdest = REGNO (operands[0]);
 164   int rsrc = REGNO (operands[1]);
 165   rtx dest[2], src[2];
 166
 167   dest[0] = gen_rtx_REG (TImode, rdest);
 168   src[0] = gen_rtx_REG (TImode, rsrc);
 169   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 170   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 171
 172   neon_disambiguate_copy (operands, dest, src, 2);
 173 })
 174
 175 (define_split
 176   [(set (match_operand:OI 0 "s_register_operand" "")
 177         (match_operand:OI 1 "s_register_operand" ""))]
 178   "TARGET_NEON && reload_completed"
 179   [(set (match_dup 0) (match_dup 1))
 180    (set (match_dup 2) (match_dup 3))]
 181 {
 182   int rdest = REGNO (operands[0]);
 183   int rsrc = REGNO (operands[1]);
 184   rtx dest[2], src[2];
 185
 186   dest[0] = gen_rtx_REG (TImode, rdest);
 187   src[0] = gen_rtx_REG (TImode, rsrc);
 188   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 189   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 190
 191   neon_disambiguate_copy (operands, dest, src, 2);
 192 })
 193
 194 (define_split
 195   [(set (match_operand:CI 0 "s_register_operand" "")
 196         (match_operand:CI 1 "s_register_operand" ""))]
 197   "TARGET_NEON && reload_completed"
 198   [(set (match_dup 0) (match_dup 1))
 199    (set (match_dup 2) (match_dup 3))
 200    (set (match_dup 4) (match_dup 5))]
 201 {
 202   int rdest = REGNO (operands[0]);
 203   int rsrc = REGNO (operands[1]);
 204   rtx dest[3], src[3];
 205
 206   dest[0] = gen_rtx_REG (TImode, rdest);
 207   src[0] = gen_rtx_REG (TImode, rsrc);
 208   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 209   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 210   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 211   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 212
 213   neon_disambiguate_copy (operands, dest, src, 3);
 214 })
 215
 216 (define_split
 217   [(set (match_operand:XI 0 "s_register_operand" "")
 218         (match_operand:XI 1 "s_register_operand" ""))]
 219   "TARGET_NEON && reload_completed"
 220   [(set (match_dup 0) (match_dup 1))
 221    (set (match_dup 2) (match_dup 3))
 222    (set (match_dup 4) (match_dup 5))
 223    (set (match_dup 6) (match_dup 7))]
 224 {
 225   int rdest = REGNO (operands[0]);
 226   int rsrc = REGNO (operands[1]);
 227   rtx dest[4], src[4];
 228
 229   dest[0] = gen_rtx_REG (TImode, rdest);
 230   src[0] = gen_rtx_REG (TImode, rsrc);
 231   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 232   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 233   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 234   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 235   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 236   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 237
 238   neon_disambiguate_copy (operands, dest, src, 4);
 239 })
 240
 241 (define_expand "movmisalign<mode>"
 242   [(set (match_operand:VDQX 0 "neon_perm_struct_or_reg_operand")
 243         (unspec:VDQX [(match_operand:VDQX 1 "neon_perm_struct_or_reg_operand")]
 244                      UNSPEC_MISALIGNED_ACCESS))]
 245   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 246 {
 247   /* This pattern is not permitted to fail during expansion: if both arguments
 248      are non-registers (e.g. memory := constant, which can be created by the
 249      auto-vectorizer), force operand 1 into a register.  */
 250   if (!s_register_operand (operands[0], <MODE>mode)
 251       && !s_register_operand (operands[1], <MODE>mode))
 252     operands[1] = force_reg (<MODE>mode, operands[1]);
 253 })
 254
 255 (define_insn "*movmisalign<mode>_neon_store"
 256   [(set (match_operand:VDX 0 "neon_permissive_struct_operand"   "=Um")
 257         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 258                     UNSPEC_MISALIGNED_ACCESS))]
 259   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 260   "vst1.<V_sz_elem>\t{%P1}, %A0"
 261   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 262
 263 (define_insn "*movmisalign<mode>_neon_load"
 264   [(set (match_operand:VDX 0 "s_register_operand"                       "=w")
 265         (unspec:VDX [(match_operand:VDX 1 "neon_permissive_struct_operand"
 266                                                                         " Um")]
 267                     UNSPEC_MISALIGNED_ACCESS))]
 268   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 269   "vld1.<V_sz_elem>\t{%P0}, %A1"
 270   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 271
 272 (define_insn "*movmisalign<mode>_neon_store"
 273   [(set (match_operand:VQX 0 "neon_permissive_struct_operand"  "=Um")
 274         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 275                     UNSPEC_MISALIGNED_ACCESS))]
 276   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 277   "vst1.<V_sz_elem>\t{%q1}, %A0"
 278   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
 279
 280 (define_insn "*movmisalign<mode>_neon_load"
 281   [(set (match_operand:VQX 0 "s_register_operand"                       "=w")
 282         (unspec:VQX [(match_operand:VQX 1 "neon_permissive_struct_operand"
 283                                                                         " Um")]
 284                     UNSPEC_MISALIGNED_ACCESS))]
 285   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 286   "vld1.<V_sz_elem>\t{%q0}, %A1"
 287   [(set_attr "neon_type" "neon_vld1_1_2_regs")])
 288
 289 (define_insn "vec_set<mode>_internal"
 290   [(set (match_operand:VD 0 "s_register_operand" "=w,w")
 291         (vec_merge:VD
 292           (vec_duplicate:VD
 293             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 294           (match_operand:VD 3 "s_register_operand" "0,0")
 295           (match_operand:SI 2 "immediate_operand" "i,i")))]
 296   "TARGET_NEON"
 297 {
 298   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 299   if (BYTES_BIG_ENDIAN)
 300     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 301   operands[2] = GEN_INT (elt);
 302
 303   if (which_alternative == 0)
 304     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 305   else
 306     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 307 }
 308   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")])
 309
 310 (define_insn "vec_set<mode>_internal"
 311   [(set (match_operand:VQ 0 "s_register_operand" "=w,w")
 312         (vec_merge:VQ
 313           (vec_duplicate:VQ
 314             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 315           (match_operand:VQ 3 "s_register_operand" "0,0")
 316           (match_operand:SI 2 "immediate_operand" "i,i")))]
 317   "TARGET_NEON"
 318 {
 319   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 320   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 321   int elt = elem % half_elts;
 322   int hi = (elem / half_elts) * 2;
 323   int regno = REGNO (operands[0]);
 324
 325   if (BYTES_BIG_ENDIAN)
 326     elt = half_elts - 1 - elt;
 327
 328   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 329   operands[2] = GEN_INT (elt);
 330
 331   if (which_alternative == 0)
 332     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 333   else
 334     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 335 }
 336   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
 337 )
 338
 339 (define_insn "vec_setv2di_internal"
 340   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
 341         (vec_merge:V2DI
 342           (vec_duplicate:V2DI
 343             (match_operand:DI 1 "nonimmediate_operand" "Um,r"))
 344           (match_operand:V2DI 3 "s_register_operand" "0,0")
 345           (match_operand:SI 2 "immediate_operand" "i,i")))]
 346   "TARGET_NEON"
 347 {
 348   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 349   int regno = REGNO (operands[0]) + 2 * elem;
 350
 351   operands[0] = gen_rtx_REG (DImode, regno);
 352
 353   if (which_alternative == 0)
 354     return "vld1.64\t%P0, %A1";
 355   else
 356     return "vmov\t%P0, %Q1, %R1";
 357 }
 358   [(set_attr "neon_type" "neon_vld1_1_2_regs,neon_mcr_2_mcrr")]
 359 )
 360
 361 (define_expand "vec_set<mode>"
 362   [(match_operand:VDQ 0 "s_register_operand" "")
 363    (match_operand:<V_elem> 1 "s_register_operand" "")
 364    (match_operand:SI 2 "immediate_operand" "")]
 365   "TARGET_NEON"
 366 {
 367   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 368   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 369                                          GEN_INT (elem), operands[0]));
 370   DONE;
 371 })
 372
 373 (define_insn "vec_extract<mode>"
 374   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 375         (vec_select:<V_elem>
 376           (match_operand:VD 1 "s_register_operand" "w,w")
 377           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 378   "TARGET_NEON"
 379 {
 380   if (BYTES_BIG_ENDIAN)
 381     {
 382       int elt = INTVAL (operands[2]);
 383       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 384       operands[2] = GEN_INT (elt);
 385     }
 386
 387   if (which_alternative == 0)
 388     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 389   else
 390     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 391 }
 392   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 393 )
 394
 395 (define_insn "vec_extract<mode>"
 396   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 397         (vec_select:<V_elem>
 398           (match_operand:VQ 1 "s_register_operand" "w,w")
 399           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 400   "TARGET_NEON"
 401 {
 402   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 403   int elt = INTVAL (operands[2]) % half_elts;
 404   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 405   int regno = REGNO (operands[1]);
 406
 407   if (BYTES_BIG_ENDIAN)
 408     elt = half_elts - 1 - elt;
 409
 410   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 411   operands[2] = GEN_INT (elt);
 412
 413   if (which_alternative == 0)
 414     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 415   else
 416     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 417 }
 418   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")]
 419 )
 420
 421 (define_insn "vec_extractv2di"
 422   [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r")
 423         (vec_select:DI
 424           (match_operand:V2DI 1 "s_register_operand" "w,w")
 425           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 426   "TARGET_NEON"
 427 {
 428   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 429
 430   operands[1] = gen_rtx_REG (DImode, regno);
 431
 432   if (which_alternative == 0)
 433     return "vst1.64\t{%P1}, %A0  @ v2di";
 434   else
 435     return "vmov\t%Q0, %R0, %P1  @ v2di";
 436 }
 437   [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_int_1")]
 438 )
 439
 440 (define_expand "vec_init<mode>"
 441   [(match_operand:VDQ 0 "s_register_operand" "")
 442    (match_operand 1 "" "")]
 443   "TARGET_NEON"
 444 {
 445   neon_expand_vector_init (operands[0], operands[1]);
 446   DONE;
 447 })
 448
 449 ;; Doubleword and quadword arithmetic.
 450
 451 ;; NOTE: some other instructions also support 64-bit integer
 452 ;; element size, which we could potentially use for "long long" operations.
 453
 454 (define_insn "*add<mode>3_neon"
 455   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 456         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 457                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 458   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 459   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 460   [(set (attr "neon_type")
 461       (if_then_else (match_test "<Is_float_mode>")
 462                     (if_then_else (match_test "<Is_d_reg>")
 463                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 464                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 465                     (const_string "neon_int_1")))]
 466 )
 467
 468 (define_insn "adddi3_neon"
 469   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 470         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 471                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 472    (clobber (reg:CC CC_REGNUM))]
 473   "TARGET_NEON"
 474 {
 475   switch (which_alternative)
 476     {
 477     case 0: /* fall through */
 478     case 3: return "vadd.i64\t%P0, %P1, %P2";
 479     case 1: return "#";
 480     case 2: return "#";
 481     case 4: return "#";
 482     case 5: return "#";
 483     case 6: return "#";
 484     default: gcc_unreachable ();
 485     }
 486 }
 487   [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*")
 488    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 489    (set_attr "length" "*,8,8,*,8,8,8")
 490    (set_attr "arch" "neon_for_64bits,*,*,avoid_neon_for_64bits,*,*,*")]
 491 )
 492
 493 (define_insn "*sub<mode>3_neon"
 494   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 495         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 496                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 497   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 498   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 499   [(set (attr "neon_type")
 500       (if_then_else (match_test "<Is_float_mode>")
 501                     (if_then_else (match_test "<Is_d_reg>")
 502                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 503                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 504                     (const_string "neon_int_2")))]
 505 )
 506
 507 (define_insn "subdi3_neon"
 508   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 509         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 510                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 511    (clobber (reg:CC CC_REGNUM))]
 512   "TARGET_NEON"
 513 {
 514   switch (which_alternative)
 515     {
 516     case 0: /* fall through */
 517     case 4: return "vsub.i64\t%P0, %P1, %P2";
 518     case 1: /* fall through */
 519     case 2: /* fall through */
 520     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 521     default: gcc_unreachable ();
 522     }
 523 }
 524   [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
 525    (set_attr "conds" "*,clob,clob,clob,*")
 526    (set_attr "length" "*,8,8,8,*")
 527    (set_attr "arch" "neon_for_64bits,*,*,*,avoid_neon_for_64bits")]
 528 )
 529
 530 (define_insn "*mul<mode>3_neon"
 531   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 532         (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 533                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 534   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 535   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 536   [(set (attr "neon_type")
 537       (if_then_else (match_test "<Is_float_mode>")
 538                     (if_then_else (match_test "<Is_d_reg>")
 539                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 540                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 541                     (if_then_else (match_test "<Is_d_reg>")
 542                                   (if_then_else
 543                                     (match_test "<Scalar_mul_8_16>")
 544                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 545                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
 546                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 547                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
 548                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
 549 )
 550
 551 (define_insn "mul<mode>3add<mode>_neon"
 552   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 553         (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 554                             (match_operand:VDQ 3 "s_register_operand" "w"))
 555                   (match_operand:VDQ 1 "s_register_operand" "0")))]
 556   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 557   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 558   [(set (attr "neon_type")
 559       (if_then_else (match_test "<Is_float_mode>")
 560                     (if_then_else (match_test "<Is_d_reg>")
 561                                   (const_string "neon_fp_vmla_ddd")
 562                                   (const_string "neon_fp_vmla_qqq"))
 563                     (if_then_else (match_test "<Is_d_reg>")
 564                                   (if_then_else
 565                                     (match_test "<Scalar_mul_8_16>")
 566                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 567                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 568                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 569                                     (const_string "neon_mla_qqq_8_16")
 570                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 571 )
 572
 573 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 574   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 575         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
 576                    (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
 577                              (match_operand:VDQ 3 "s_register_operand" "w"))))]
 578   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 579   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 580   [(set (attr "neon_type")
 581       (if_then_else (match_test "<Is_float_mode>")
 582                     (if_then_else (match_test "<Is_d_reg>")
 583                                   (const_string "neon_fp_vmla_ddd")
 584                                   (const_string "neon_fp_vmla_qqq"))
 585                     (if_then_else (match_test "<Is_d_reg>")
 586                                   (if_then_else
 587                                     (match_test "<Scalar_mul_8_16>")
 588                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 589                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
 590                                   (if_then_else (match_test "<Scalar_mul_8_16>")
 591                                     (const_string "neon_mla_qqq_8_16")
 592                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
 593 )
 594
 595 ;; Fused multiply-accumulate
 596 ;; We define each insn twice here:
 597 ;;    1: with flag_unsafe_math_optimizations for the widening multiply phase
 598 ;;       to be able to use when converting to FMA.
 599 ;;    2: without flag_unsafe_math_optimizations for the intrinsics to use.
 600 (define_insn "fma<VCVTF:mode>4"
 601   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 602         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 603                  (match_operand:VCVTF 2 "register_operand" "w")
 604                  (match_operand:VCVTF 3 "register_operand" "0")))]
 605   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 606   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 607   [(set (attr "neon_type")
 608         (if_then_else (match_test "<Is_d_reg>")
 609                       (const_string "neon_fp_vmla_ddd")
 610                       (const_string "neon_fp_vmla_qqq")))]
 611 )
 612
 613 (define_insn "fma<VCVTF:mode>4_intrinsic"
 614   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 615         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 616                  (match_operand:VCVTF 2 "register_operand" "w")
 617                  (match_operand:VCVTF 3 "register_operand" "0")))]
 618   "TARGET_NEON && TARGET_FMA"
 619   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 620   [(set (attr "neon_type")
 621         (if_then_else (match_test "<Is_d_reg>")
 622                       (const_string "neon_fp_vmla_ddd")
 623                       (const_string "neon_fp_vmla_qqq")))]
 624 )
 625
 626 (define_insn "*fmsub<VCVTF:mode>4"
 627   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 628         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 629                    (match_operand:VCVTF 2 "register_operand" "w")
 630                    (match_operand:VCVTF 3 "register_operand" "0")))]
 631   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 632   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 633   [(set (attr "neon_type")
 634         (if_then_else (match_test "<Is_d_reg>")
 635                       (const_string "neon_fp_vmla_ddd")
 636                       (const_string "neon_fp_vmla_qqq")))]
 637 )
 638
 639 (define_insn "fmsub<VCVTF:mode>4_intrinsic"
 640   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 641         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 642                    (match_operand:VCVTF 2 "register_operand" "w")
 643                    (match_operand:VCVTF 3 "register_operand" "0")))]
 644   "TARGET_NEON && TARGET_FMA"
 645   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 646   [(set (attr "neon_type")
 647         (if_then_else (match_test "<Is_d_reg>")
 648                       (const_string "neon_fp_vmla_ddd")
 649                       (const_string "neon_fp_vmla_qqq")))]
 650 )
 651
 652 (define_insn "neon_vrint<NEON_VRINT:nvrint_variant><VCVTF:mode>"
 653   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
 654         (unspec:VCVTF [(match_operand:VCVTF 1
 655                          "s_register_operand" "w")]
 656                 NEON_VRINT))]
 657   "TARGET_NEON && TARGET_FPU_ARMV8"
 658   "vrint<nvrint_variant>%?.f32\\t%<V_reg>0, %<V_reg>1"
 659   [(set (attr "neon_type")
 660         (if_then_else (match_test "<Is_d_reg>")
 661                 (const_string "neon_fp_vadd_ddd_vabs_dd")
 662                 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
 663 )
 664
 665 (define_insn "ior<mode>3"
 666   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 667         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 668                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 669   "TARGET_NEON"
 670 {
 671   switch (which_alternative)
 672     {
 673     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 674     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 675                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 676     default: gcc_unreachable ();
 677     }
 678 }
 679   [(set_attr "neon_type" "neon_int_1")]
 680 )
 681
 682 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 683 ;; vorr. We support the pseudo-instruction vand instead, because that
 684 ;; corresponds to the canonical form the middle-end expects to use for
 685 ;; immediate bitwise-ANDs.
 686
 687 (define_insn "and<mode>3"
 688   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 689         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 690                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 691   "TARGET_NEON"
 692 {
 693   switch (which_alternative)
 694     {
 695     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 696     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 697                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 698     default: gcc_unreachable ();
 699     }
 700 }
 701   [(set_attr "neon_type" "neon_int_1")]
 702 )
 703
 704 (define_insn "orn<mode>3_neon"
 705   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 706         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 707                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 708   "TARGET_NEON"
 709   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 710   [(set_attr "neon_type" "neon_int_1")]
 711 )
 712
 713 ;; TODO: investigate whether we should disable
 714 ;; this and bicdi3_neon for the A8 in line with the other
 715 ;; changes above.
 716 (define_insn_and_split "orndi3_neon"
 717   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 718         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 719                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 720   "TARGET_NEON"
 721   "@
 722    vorn\t%P0, %P1, %P2
 723    #
 724    #
 725    #"
 726   "reload_completed &&
 727    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 728   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 729    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 730   "
 731   {
 732     if (TARGET_THUMB2)
 733       {
 734         operands[3] = gen_highpart (SImode, operands[0]);
 735         operands[0] = gen_lowpart (SImode, operands[0]);
 736         operands[4] = gen_highpart (SImode, operands[2]);
 737         operands[2] = gen_lowpart (SImode, operands[2]);
 738         operands[5] = gen_highpart (SImode, operands[1]);
 739         operands[1] = gen_lowpart (SImode, operands[1]);
 740       }
 741     else
 742       {
 743         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 744         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 745         DONE;
 746       }
 747   }"
 748   [(set_attr "neon_type" "neon_int_1,*,*,*")
 749    (set_attr "length" "*,16,8,8")
 750    (set_attr "arch" "any,a,t2,t2")]
 751 )
 752
 753 (define_insn "bic<mode>3_neon"
 754   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 755         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 756                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 757   "TARGET_NEON"
 758   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 759   [(set_attr "neon_type" "neon_int_1")]
 760 )
 761
 762 ;; Compare to *anddi_notdi_di.
 763 (define_insn "bicdi3_neon"
 764   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 765         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 766                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 767   "TARGET_NEON"
 768   "@
 769    vbic\t%P0, %P1, %P2
 770    #
 771    #"
 772   [(set_attr "neon_type" "neon_int_1,*,*")
 773    (set_attr "length" "*,8,8")]
 774 )
 775
 776 (define_insn "xor<mode>3"
 777   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 778         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 779                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 780   "TARGET_NEON"
 781   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 782   [(set_attr "neon_type" "neon_int_1")]
 783 )
 784
 785 (define_insn "one_cmpl<mode>2"
 786   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 787         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 788   "TARGET_NEON"
 789   "vmvn\t%<V_reg>0, %<V_reg>1"
 790   [(set_attr "neon_type" "neon_int_1")]
 791 )
 792
 793 (define_insn "abs<mode>2"
 794   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 795         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 796   "TARGET_NEON"
 797   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 798   [(set (attr "neon_type")
 799       (if_then_else (match_test "<Is_float_mode>")
 800                     (if_then_else (match_test "<Is_d_reg>")
 801                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 802                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 803                     (const_string "neon_int_3")))]
 804 )
 805
 806 (define_insn "neg<mode>2"
 807   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 808         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 809   "TARGET_NEON"
 810   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 811   [(set (attr "neon_type")
 812       (if_then_else (match_test "<Is_float_mode>")
 813                     (if_then_else (match_test "<Is_d_reg>")
 814                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
 815                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
 816                     (const_string "neon_int_3")))]
 817 )
 818
 819 (define_insn "negdi2_neon"
 820   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 821         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 822    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 823    (clobber (reg:CC CC_REGNUM))]
 824   "TARGET_NEON"
 825   "#"
 826   [(set_attr "length" "8")]
 827 )
 828
 829 ; Split negdi2_neon for vfp registers
 830 (define_split
 831   [(set (match_operand:DI 0 "s_register_operand" "")
 832         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 833    (clobber (match_scratch:DI 2 ""))
 834    (clobber (reg:CC CC_REGNUM))]
 835   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 836   [(set (match_dup 2) (const_int 0))
 837    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 838               (clobber (reg:CC CC_REGNUM))])]
 839   {
 840     if (!REG_P (operands[2]))
 841       operands[2] = operands[0];
 842   }
 843 )
 844
 845 ; Split negdi2_neon for core registers
 846 (define_split
 847   [(set (match_operand:DI 0 "s_register_operand" "")
 848         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 849    (clobber (match_scratch:DI 2 ""))
 850    (clobber (reg:CC CC_REGNUM))]
 851   "TARGET_32BIT && reload_completed
 852    && arm_general_register_operand (operands[0], DImode)"
 853   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
 854               (clobber (reg:CC CC_REGNUM))])]
 855   ""
 856 )
 857
 858 (define_insn "*umin<mode>3_neon"
 859   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 860         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 861                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 862   "TARGET_NEON"
 863   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 864   [(set_attr "neon_type" "neon_int_5")]
 865 )
 866
 867 (define_insn "*umax<mode>3_neon"
 868   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 869         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 870                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 871   "TARGET_NEON"
 872   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 873   [(set_attr "neon_type" "neon_int_5")]
 874 )
 875
 876 (define_insn "*smin<mode>3_neon"
 877   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 878         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 879                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 880   "TARGET_NEON"
 881   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 882   [(set (attr "neon_type")
 883       (if_then_else (match_test "<Is_float_mode>")
 884                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 885                     (const_string "neon_int_5")))]
 886 )
 887
 888 (define_insn "*smax<mode>3_neon"
 889   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 890         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 891                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 892   "TARGET_NEON"
 893   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 894   [(set (attr "neon_type")
 895       (if_then_else (match_test "<Is_float_mode>")
 896                     (const_string "neon_fp_vadd_ddd_vabs_dd")
 897                     (const_string "neon_int_5")))]
 898 )
 899
 900 ; TODO: V2DI shifts are current disabled because there are bugs in the
 901 ; generic vectorizer code.  It ends up creating a V2DI constructor with
 902 ; SImode elements.
 903
 904 (define_insn "vashl<mode>3"
 905   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
 906         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
 907                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
 908   "TARGET_NEON"
 909   {
 910     switch (which_alternative)
 911       {
 912         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 913         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
 914                                                     <MODE>mode,
 915                                                     VALID_NEON_QREG_MODE (<MODE>mode),
 916                                                     true);
 917         default: gcc_unreachable ();
 918       }
 919   }
 920   [(set (attr "neon_type")
 921       (if_then_else (match_test "<Is_d_reg>")
 922                     (const_string "neon_vshl_ddd")
 923                     (const_string "neon_shift_3")))]
 924 )
 925
 926 (define_insn "vashr<mode>3_imm"
 927   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 928         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 929                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 930   "TARGET_NEON"
 931   {
 932     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 933                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 934                                         false);
 935   }
 936   [(set (attr "neon_type")
 937       (if_then_else (match_test "<Is_d_reg>")
 938                     (const_string "neon_vshl_ddd")
 939                     (const_string "neon_shift_3")))]
 940 )
 941
 942 (define_insn "vlshr<mode>3_imm"
 943   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 944         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 945                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 946   "TARGET_NEON"
 947   {
 948     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 949                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 950                                         false);
 951   }
 952   [(set (attr "neon_type")
 953         (if_then_else (match_test "<Is_d_reg>")
 954                       (const_string "neon_vshl_ddd")
 955                       (const_string "neon_shift_3")))]
 956 )
 957
 958 ; Used for implementing logical shift-right, which is a left-shift by a negative
 959 ; amount, with signed operands. This is essentially the same as ashl<mode>3
 960 ; above, but using an unspec in case GCC tries anything tricky with negative
 961 ; shift amounts.
 962
 963 (define_insn "ashl<mode>3_signed"
 964   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
 965         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
 966                       (match_operand:VDQI 2 "s_register_operand" "w")]
 967                      UNSPEC_ASHIFT_SIGNED))]
 968   "TARGET_NEON"
 969   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 970   [(set (attr "neon_type")
 971       (if_then_else (match_test "<Is_d_reg>")
 972                     (const_string "neon_vshl_ddd")
 973                     (const_string "neon_shift_3")))]
 974 )
 975
 976 ; Used for implementing logical shift-right, which is a left-shift by a negative
 977 ; amount, with unsigned operands.
 978
 979 (define_insn "ashl<mode>3_unsigned"
 980   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
 981         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
 982                       (match_operand:VDQI 2 "s_register_operand" "w")]
 983                      UNSPEC_ASHIFT_UNSIGNED))]
 984   "TARGET_NEON"
 985   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 986   [(set (attr "neon_type")
 987       (if_then_else (match_test "<Is_d_reg>")
 988                     (const_string "neon_vshl_ddd")
 989                     (const_string "neon_shift_3")))]
 990 )
 991
 992 (define_expand "vashr<mode>3"
 993   [(set (match_operand:VDQIW 0 "s_register_operand" "")
 994         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
 995                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
 996   "TARGET_NEON"
 997 {
 998   if (s_register_operand (operands[2], <MODE>mode))
 999     {
1000       rtx neg = gen_reg_rtx (<MODE>mode);
1001       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1002       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1003     }
1004   else
1005     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1006   DONE;
1007 })
1008
1009 (define_expand "vlshr<mode>3"
1010   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1011         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1012                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1013   "TARGET_NEON"
1014 {
1015   if (s_register_operand (operands[2], <MODE>mode))
1016     {
1017       rtx neg = gen_reg_rtx (<MODE>mode);
1018       emit_insn (gen_neg<mode>2 (neg, operands[2]));
1019       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1020     }
1021   else
1022     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1023   DONE;
1024 })
1025
1026 ;; 64-bit shifts
1027
1028 ;; This pattern loads a 32-bit shift count into a 64-bit NEON register,
1029 ;; leaving the upper half uninitalized.  This is OK since the shift
1030 ;; instruction only looks at the low 8 bits anyway.  To avoid confusing
1031 ;; data flow analysis however, we pretend the full register is set
1032 ;; using an unspec.
1033 (define_insn "neon_load_count"
1034   [(set (match_operand:DI 0 "s_register_operand" "=w,w")
1035         (unspec:DI [(match_operand:SI 1 "nonimmediate_operand" "Um,r")]
1036                    UNSPEC_LOAD_COUNT))]
1037   "TARGET_NEON"
1038   "@
1039    vld1.32\t{%P0[0]}, %A1
1040    vmov.32\t%P0[0], %1"
1041   [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]
1042 )
1043
1044 (define_insn "ashldi3_neon_noclobber"
1045   [(set (match_operand:DI 0 "s_register_operand"            "=w,w")
1046         (ashift:DI (match_operand:DI 1 "s_register_operand" " w,w")
1047                    (match_operand:DI 2 "reg_or_int_operand" " i,w")))]
1048   "TARGET_NEON && reload_completed
1049    && (!CONST_INT_P (operands[2])
1050        || (INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) < 64))"
1051   "@
1052    vshl.u64\t%P0, %P1, %2
1053    vshl.u64\t%P0, %P1, %P2"
1054   [(set_attr "neon_type" "neon_vshl_ddd,neon_vshl_ddd")]
1055 )
1056
1057 (define_insn_and_split "ashldi3_neon"
1058   [(set (match_operand:DI 0 "s_register_operand"            "= w, w,?&r,?r, ?w,w")
1059         (ashift:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r, 0w,w")
1060                    (match_operand:SI 2 "general_operand"    "rUm, i,  r, i,rUm,i")))
1061    (clobber (match_scratch:SI 3                             "= X, X,?&r, X,  X,X"))
1062    (clobber (match_scratch:SI 4                             "= X, X,?&r, X,  X,X"))
1063    (clobber (match_scratch:DI 5                             "=&w, X,  X, X, &w,X"))
1064    (clobber (reg:CC_C CC_REGNUM))]
1065   "TARGET_NEON"
1066   "#"
1067   "TARGET_NEON && reload_completed"
1068   [(const_int 0)]
1069   "
1070   {
1071     if (IS_VFP_REGNUM (REGNO (operands[0])))
1072       {
1073         if (CONST_INT_P (operands[2]))
1074           {
1075             if (INTVAL (operands[2]) < 1)
1076               {
1077                 emit_insn (gen_movdi (operands[0], operands[1]));
1078                 DONE;
1079               }
1080             else if (INTVAL (operands[2]) > 63)
1081               operands[2] = gen_rtx_CONST_INT (VOIDmode, 63);
1082           }
1083         else
1084           {
1085             emit_insn (gen_neon_load_count (operands[5], operands[2]));
1086             operands[2] = operands[5];
1087           }
1088
1089         /* Ditch the unnecessary clobbers.  */
1090         emit_insn (gen_ashldi3_neon_noclobber (operands[0], operands[1],
1091                                                operands[2]));
1092       }
1093     else
1094       {
1095         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1096           /* This clobbers CC.  */
1097           emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
1098         else
1099           arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
1100                                          operands[2], operands[3], operands[4]);
1101       }
1102     DONE;
1103   }"
1104   [(set_attr "arch" "neon_for_64bits,neon_for_64bits,*,*,avoid_neon_for_64bits,avoid_neon_for_64bits")
1105    (set_attr "opt" "*,*,speed,speed,*,*")]
1106 )
1107
1108 ; The shift amount needs to be negated for right-shifts
1109 (define_insn "signed_shift_di3_neon"
1110   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1111         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1112                     (match_operand:DI 2 "s_register_operand" " w")]
1113                    UNSPEC_ASHIFT_SIGNED))]
1114   "TARGET_NEON && reload_completed"
1115   "vshl.s64\t%P0, %P1, %P2"
1116   [(set_attr "neon_type" "neon_vshl_ddd")]
1117 )
1118
1119 ; The shift amount needs to be negated for right-shifts
1120 (define_insn "unsigned_shift_di3_neon"
1121   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1122         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1123                     (match_operand:DI 2 "s_register_operand" " w")]
1124                    UNSPEC_ASHIFT_UNSIGNED))]
1125   "TARGET_NEON && reload_completed"
1126   "vshl.u64\t%P0, %P1, %P2"
1127   [(set_attr "neon_type" "neon_vshl_ddd")]
1128 )
1129
1130 (define_insn "ashrdi3_neon_imm_noclobber"
1131   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1132         (ashiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1133                      (match_operand:DI 2 "const_int_operand"  " i")))]
1134   "TARGET_NEON && reload_completed
1135    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1136   "vshr.s64\t%P0, %P1, %2"
1137   [(set_attr "neon_type" "neon_vshl_ddd")]
1138 )
1139
1140 (define_insn "lshrdi3_neon_imm_noclobber"
1141   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1142         (lshiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1143                      (match_operand:DI 2 "const_int_operand"  " i")))]
1144   "TARGET_NEON && reload_completed
1145    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1146   "vshr.u64\t%P0, %P1, %2"
1147   [(set_attr "neon_type" "neon_vshl_ddd")]
1148 )
1149
1150 ;; ashrdi3_neon
1151 ;; lshrdi3_neon
1152 (define_insn_and_split "<shift>di3_neon"
1153   [(set (match_operand:DI 0 "s_register_operand"             "= w, w,?&r,?r,?w,?w")
1154         (rshifts:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r,0w, w")
1155                     (match_operand:SI 2 "reg_or_int_operand" "  r, i,  r, i, r, i")))
1156    (clobber (match_scratch:SI 3                              "=2r, X, &r, X,2r, X"))
1157    (clobber (match_scratch:SI 4                              "= X, X, &r, X, X, X"))
1158    (clobber (match_scratch:DI 5                              "=&w, X,  X, X,&w, X"))
1159    (clobber (reg:CC CC_REGNUM))]
1160   "TARGET_NEON"
1161   "#"
1162   "TARGET_NEON && reload_completed"
1163   [(const_int 0)]
1164   "
1165   {
1166     if (IS_VFP_REGNUM (REGNO (operands[0])))
1167       {
1168         if (CONST_INT_P (operands[2]))
1169           {
1170             if (INTVAL (operands[2]) < 1)
1171               {
1172                 emit_insn (gen_movdi (operands[0], operands[1]));
1173                 DONE;
1174               }
1175             else if (INTVAL (operands[2]) > 64)
1176               operands[2] = gen_rtx_CONST_INT (VOIDmode, 64);
1177
1178             /* Ditch the unnecessary clobbers.  */
1179             emit_insn (gen_<shift>di3_neon_imm_noclobber (operands[0],
1180                                                           operands[1],
1181                                                           operands[2]));
1182           }
1183         else
1184           {
1185             /* We must use a negative left-shift.  */
1186             emit_insn (gen_negsi2 (operands[3], operands[2]));
1187             emit_insn (gen_neon_load_count (operands[5], operands[3]));
1188             emit_insn (gen_<shifttype>_shift_di3_neon (operands[0], operands[1],
1189                                                        operands[5]));
1190           }
1191       }
1192     else
1193       {
1194         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1195           /* This clobbers CC.  */
1196           emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1]));
1197         else
1198           /* This clobbers CC (ASHIFTRT by register only).  */
1199           arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
1200                                          operands[2], operands[3], operands[4]);
1201       }
1202
1203     DONE;
1204   }"
1205   [(set_attr "arch" "neon_for_64bits,neon_for_64bits,*,*,avoid_neon_for_64bits,avoid_neon_for_64bits")
1206    (set_attr "opt" "*,*,speed,speed,*,*")]
1207 )
1208
1209 ;; Widening operations
1210
1211 (define_insn "widen_ssum<mode>3"
1212   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1213         (plus:<V_widen> (sign_extend:<V_widen>
1214                           (match_operand:VW 1 "s_register_operand" "%w"))
1215                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1216   "TARGET_NEON"
1217   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1218   [(set_attr "neon_type" "neon_int_3")]
1219 )
1220
1221 (define_insn "widen_usum<mode>3"
1222   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1223         (plus:<V_widen> (zero_extend:<V_widen>
1224                           (match_operand:VW 1 "s_register_operand" "%w"))
1225                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1226   "TARGET_NEON"
1227   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1228   [(set_attr "neon_type" "neon_int_3")]
1229 )
1230
1231 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1232 ;; shift-count granularity. That's good enough for the middle-end's current
1233 ;; needs.
1234
1235 ;; Note that it's not safe to perform such an operation in big-endian mode,
1236 ;; due to element-ordering issues.
1237
1238 (define_expand "vec_shr_<mode>"
1239   [(match_operand:VDQ 0 "s_register_operand" "")
1240    (match_operand:VDQ 1 "s_register_operand" "")
1241    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1242   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1243 {
1244   rtx zero_reg;
1245   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1246   const int width = GET_MODE_BITSIZE (<MODE>mode);
1247   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1248   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1249     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1250
1251   if (num_bits == width)
1252     {
1253       emit_move_insn (operands[0], operands[1]);
1254       DONE;
1255     }
1256
1257   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1258   operands[0] = gen_lowpart (bvecmode, operands[0]);
1259   operands[1] = gen_lowpart (bvecmode, operands[1]);
1260
1261   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1262                       GEN_INT (num_bits / BITS_PER_UNIT)));
1263   DONE;
1264 })
1265
1266 (define_expand "vec_shl_<mode>"
1267   [(match_operand:VDQ 0 "s_register_operand" "")
1268    (match_operand:VDQ 1 "s_register_operand" "")
1269    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1270   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1271 {
1272   rtx zero_reg;
1273   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1274   const int width = GET_MODE_BITSIZE (<MODE>mode);
1275   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1276   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1277     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1278
1279   if (num_bits == 0)
1280     {
1281       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1282       DONE;
1283     }
1284
1285   num_bits = width - num_bits;
1286
1287   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1288   operands[0] = gen_lowpart (bvecmode, operands[0]);
1289   operands[1] = gen_lowpart (bvecmode, operands[1]);
1290
1291   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1292                       GEN_INT (num_bits / BITS_PER_UNIT)));
1293   DONE;
1294 })
1295
1296 ;; Helpers for quad-word reduction operations
1297
1298 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1299 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1300 ; N/2-element vector.
1301
1302 (define_insn "quad_halves_<code>v4si"
1303   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1304         (vqh_ops:V2SI
1305           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1306                            (parallel [(const_int 0) (const_int 1)]))
1307           (vec_select:V2SI (match_dup 1)
1308                            (parallel [(const_int 2) (const_int 3)]))))]
1309   "TARGET_NEON"
1310   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1311   [(set_attr "vqh_mnem" "<VQH_mnem>")
1312    (set (attr "neon_type")
1313       (if_then_else (eq_attr "vqh_mnem" "vadd")
1314                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1315 )
1316
1317 (define_insn "quad_halves_<code>v4sf"
1318   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1319         (vqhs_ops:V2SF
1320           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1321                            (parallel [(const_int 0) (const_int 1)]))
1322           (vec_select:V2SF (match_dup 1)
1323                            (parallel [(const_int 2) (const_int 3)]))))]
1324   "TARGET_NEON && flag_unsafe_math_optimizations"
1325   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1326   [(set_attr "vqh_mnem" "<VQH_mnem>")
1327    (set (attr "neon_type")
1328       (if_then_else (eq_attr "vqh_mnem" "vadd")
1329                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1330 )
1331
1332 (define_insn "quad_halves_<code>v8hi"
1333   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1334         (vqh_ops:V4HI
1335           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1336                            (parallel [(const_int 0) (const_int 1)
1337                                       (const_int 2) (const_int 3)]))
1338           (vec_select:V4HI (match_dup 1)
1339                            (parallel [(const_int 4) (const_int 5)
1340                                       (const_int 6) (const_int 7)]))))]
1341   "TARGET_NEON"
1342   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1343   [(set_attr "vqh_mnem" "<VQH_mnem>")
1344    (set (attr "neon_type")
1345       (if_then_else (eq_attr "vqh_mnem" "vadd")
1346                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1347 )
1348
1349 (define_insn "quad_halves_<code>v16qi"
1350   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1351         (vqh_ops:V8QI
1352           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1353                            (parallel [(const_int 0) (const_int 1)
1354                                       (const_int 2) (const_int 3)
1355                                       (const_int 4) (const_int 5)
1356                                       (const_int 6) (const_int 7)]))
1357           (vec_select:V8QI (match_dup 1)
1358                            (parallel [(const_int 8) (const_int 9)
1359                                       (const_int 10) (const_int 11)
1360                                       (const_int 12) (const_int 13)
1361                                       (const_int 14) (const_int 15)]))))]
1362   "TARGET_NEON"
1363   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1364   [(set_attr "vqh_mnem" "<VQH_mnem>")
1365    (set (attr "neon_type")
1366       (if_then_else (eq_attr "vqh_mnem" "vadd")
1367                     (const_string "neon_int_1") (const_string "neon_int_5")))]
1368 )
1369
1370 (define_expand "move_hi_quad_<mode>"
1371  [(match_operand:ANY128 0 "s_register_operand" "")
1372   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1373  "TARGET_NEON"
1374 {
1375   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1376                                        GET_MODE_SIZE (<V_HALF>mode)),
1377                   operands[1]);
1378   DONE;
1379 })
1380
1381 (define_expand "move_lo_quad_<mode>"
1382  [(match_operand:ANY128 0 "s_register_operand" "")
1383   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1384  "TARGET_NEON"
1385 {
1386   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1387                                        <MODE>mode, 0),
1388                   operands[1]);
1389   DONE;
1390 })
1391
1392 ;; Reduction operations
1393
1394 (define_expand "reduc_splus_<mode>"
1395   [(match_operand:VD 0 "s_register_operand" "")
1396    (match_operand:VD 1 "s_register_operand" "")]
1397   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1398 {
1399   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1400                         &gen_neon_vpadd_internal<mode>);
1401   DONE;
1402 })
1403
1404 (define_expand "reduc_splus_<mode>"
1405   [(match_operand:VQ 0 "s_register_operand" "")
1406    (match_operand:VQ 1 "s_register_operand" "")]
1407   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1408    && !BYTES_BIG_ENDIAN"
1409 {
1410   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1411   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1412
1413   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1414   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1415   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1416
1417   DONE;
1418 })
1419
1420 (define_insn "reduc_splus_v2di"
1421   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1422         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1423                      UNSPEC_VPADD))]
1424   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1425   "vadd.i64\t%e0, %e1, %f1"
1426   [(set_attr "neon_type" "neon_int_1")]
1427 )
1428
1429 ;; NEON does not distinguish between signed and unsigned addition except on
1430 ;; widening operations.
1431 (define_expand "reduc_uplus_<mode>"
1432   [(match_operand:VDQI 0 "s_register_operand" "")
1433    (match_operand:VDQI 1 "s_register_operand" "")]
1434   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1435 {
1436   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1437   DONE;
1438 })
1439
1440 (define_expand "reduc_smin_<mode>"
1441   [(match_operand:VD 0 "s_register_operand" "")
1442    (match_operand:VD 1 "s_register_operand" "")]
1443   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1444 {
1445   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1446                         &gen_neon_vpsmin<mode>);
1447   DONE;
1448 })
1449
1450 (define_expand "reduc_smin_<mode>"
1451   [(match_operand:VQ 0 "s_register_operand" "")
1452    (match_operand:VQ 1 "s_register_operand" "")]
1453   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1454    && !BYTES_BIG_ENDIAN"
1455 {
1456   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1457   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1458
1459   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1460   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1461   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1462
1463   DONE;
1464 })
1465
1466 (define_expand "reduc_smax_<mode>"
1467   [(match_operand:VD 0 "s_register_operand" "")
1468    (match_operand:VD 1 "s_register_operand" "")]
1469   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1470 {
1471   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1472                         &gen_neon_vpsmax<mode>);
1473   DONE;
1474 })
1475
1476 (define_expand "reduc_smax_<mode>"
1477   [(match_operand:VQ 0 "s_register_operand" "")
1478    (match_operand:VQ 1 "s_register_operand" "")]
1479   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1480    && !BYTES_BIG_ENDIAN"
1481 {
1482   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1483   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1484
1485   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1486   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1487   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1488
1489   DONE;
1490 })
1491
1492 (define_expand "reduc_umin_<mode>"
1493   [(match_operand:VDI 0 "s_register_operand" "")
1494    (match_operand:VDI 1 "s_register_operand" "")]
1495   "TARGET_NEON"
1496 {
1497   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1498                         &gen_neon_vpumin<mode>);
1499   DONE;
1500 })
1501
1502 (define_expand "reduc_umin_<mode>"
1503   [(match_operand:VQI 0 "s_register_operand" "")
1504    (match_operand:VQI 1 "s_register_operand" "")]
1505   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1506 {
1507   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1508   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1509
1510   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1511   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1512   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1513
1514   DONE;
1515 })
1516
1517 (define_expand "reduc_umax_<mode>"
1518   [(match_operand:VDI 0 "s_register_operand" "")
1519    (match_operand:VDI 1 "s_register_operand" "")]
1520   "TARGET_NEON"
1521 {
1522   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1523                         &gen_neon_vpumax<mode>);
1524   DONE;
1525 })
1526
1527 (define_expand "reduc_umax_<mode>"
1528   [(match_operand:VQI 0 "s_register_operand" "")
1529    (match_operand:VQI 1 "s_register_operand" "")]
1530   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1531 {
1532   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1533   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1534
1535   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1536   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1537   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1538
1539   DONE;
1540 })
1541
1542 (define_insn "neon_vpadd_internal<mode>"
1543   [(set (match_operand:VD 0 "s_register_operand" "=w")
1544         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1545                     (match_operand:VD 2 "s_register_operand" "w")]
1546                    UNSPEC_VPADD))]
1547   "TARGET_NEON"
1548   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1549   ;; Assume this schedules like vadd.
1550   [(set (attr "neon_type")
1551       (if_then_else (match_test "<Is_float_mode>")
1552                     (if_then_else (match_test "<Is_d_reg>")
1553                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1554                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1555                     (const_string "neon_int_1")))]
1556 )
1557
1558 (define_insn "neon_vpsmin<mode>"
1559   [(set (match_operand:VD 0 "s_register_operand" "=w")
1560         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1561                     (match_operand:VD 2 "s_register_operand" "w")]
1562                    UNSPEC_VPSMIN))]
1563   "TARGET_NEON"
1564   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1565   ;; Assume this schedules like vmin.
1566   [(set (attr "neon_type")
1567       (if_then_else (match_test "<Is_float_mode>")
1568                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1569                     (const_string "neon_int_5")))]
1570 )
1571
1572 (define_insn "neon_vpsmax<mode>"
1573   [(set (match_operand:VD 0 "s_register_operand" "=w")
1574         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1575                     (match_operand:VD 2 "s_register_operand" "w")]
1576                    UNSPEC_VPSMAX))]
1577   "TARGET_NEON"
1578   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1579   ;; Assume this schedules like vmax.
1580   [(set (attr "neon_type")
1581       (if_then_else (match_test "<Is_float_mode>")
1582                     (const_string "neon_fp_vadd_ddd_vabs_dd")
1583                     (const_string "neon_int_5")))]
1584 )
1585
1586 (define_insn "neon_vpumin<mode>"
1587   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1588         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1589                      (match_operand:VDI 2 "s_register_operand" "w")]
1590                    UNSPEC_VPUMIN))]
1591   "TARGET_NEON"
1592   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1593   ;; Assume this schedules like umin.
1594   [(set_attr "neon_type" "neon_int_5")]
1595 )
1596
1597 (define_insn "neon_vpumax<mode>"
1598   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1599         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1600                      (match_operand:VDI 2 "s_register_operand" "w")]
1601                    UNSPEC_VPUMAX))]
1602   "TARGET_NEON"
1603   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1604   ;; Assume this schedules like umax.
1605   [(set_attr "neon_type" "neon_int_5")]
1606 )
1607
1608 ;; Saturating arithmetic
1609
1610 ; NOTE: Neon supports many more saturating variants of instructions than the
1611 ; following, but these are all GCC currently understands.
1612 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1613 ; yet either, although these patterns may be used by intrinsics when they're
1614 ; added.
1615
1616 (define_insn "*ss_add<mode>_neon"
1617   [(set (match_operand:VD 0 "s_register_operand" "=w")
1618        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1619                    (match_operand:VD 2 "s_register_operand" "w")))]
1620   "TARGET_NEON"
1621   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1622   [(set_attr "neon_type" "neon_int_4")]
1623 )
1624
1625 (define_insn "*us_add<mode>_neon"
1626   [(set (match_operand:VD 0 "s_register_operand" "=w")
1627        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1628                    (match_operand:VD 2 "s_register_operand" "w")))]
1629   "TARGET_NEON"
1630   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1631   [(set_attr "neon_type" "neon_int_4")]
1632 )
1633
1634 (define_insn "*ss_sub<mode>_neon"
1635   [(set (match_operand:VD 0 "s_register_operand" "=w")
1636        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1637                     (match_operand:VD 2 "s_register_operand" "w")))]
1638   "TARGET_NEON"
1639   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1640   [(set_attr "neon_type" "neon_int_5")]
1641 )
1642
1643 (define_insn "*us_sub<mode>_neon"
1644   [(set (match_operand:VD 0 "s_register_operand" "=w")
1645        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1646                     (match_operand:VD 2 "s_register_operand" "w")))]
1647   "TARGET_NEON"
1648   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1649   [(set_attr "neon_type" "neon_int_5")]
1650 )
1651
1652 ;; Conditional instructions.  These are comparisons with conditional moves for
1653 ;; vectors.  They perform the assignment:
1654 ;;
1655 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1656 ;;
1657 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1658 ;; element-wise.
1659
1660 (define_expand "vcond<mode><mode>"
1661   [(set (match_operand:VDQW 0 "s_register_operand" "")
1662         (if_then_else:VDQW
1663           (match_operator 3 "comparison_operator"
1664             [(match_operand:VDQW 4 "s_register_operand" "")
1665              (match_operand:VDQW 5 "nonmemory_operand" "")])
1666           (match_operand:VDQW 1 "s_register_operand" "")
1667           (match_operand:VDQW 2 "s_register_operand" "")))]
1668   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1669 {
1670   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1671                              ? 3 : 1;
1672   rtx magic_rtx = GEN_INT (magic_word);
1673   int inverse = 0;
1674   int swap_bsl_operands = 0;
1675   rtx mask = gen_reg_rtx (<V_cmp_result>mode);
1676   rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
1677
1678   rtx (*base_comparison) (rtx, rtx, rtx, rtx);
1679   rtx (*complimentary_comparison) (rtx, rtx, rtx, rtx);
1680
1681   switch (GET_CODE (operands[3]))
1682     {
1683     case GE:
1684     case LE:
1685     case EQ:
1686       if (!REG_P (operands[5])
1687           && (operands[5] != CONST0_RTX (<MODE>mode)))
1688         operands[5] = force_reg (<MODE>mode, operands[5]);
1689       break;
1690     default:
1691       if (!REG_P (operands[5]))
1692         operands[5] = force_reg (<MODE>mode, operands[5]);
1693     }
1694
1695   switch (GET_CODE (operands[3]))
1696     {
1697     case LT:
1698     case UNLT:
1699       inverse = 1;
1700       /* Fall through.  */
1701     case GE:
1702     case UNGE:
1703     case ORDERED:
1704     case UNORDERED:
1705       base_comparison = gen_neon_vcge<mode>;
1706       complimentary_comparison = gen_neon_vcgt<mode>;
1707       break;
1708     case LE:
1709     case UNLE:
1710       inverse = 1;
1711       /* Fall through.  */
1712     case GT:
1713     case UNGT:
1714       base_comparison = gen_neon_vcgt<mode>;
1715       complimentary_comparison = gen_neon_vcge<mode>;
1716       break;
1717     case EQ:
1718     case NE:
1719     case UNEQ:
1720       base_comparison = gen_neon_vceq<mode>;
1721       complimentary_comparison = gen_neon_vceq<mode>;
1722       break;
1723     default:
1724       gcc_unreachable ();
1725     }
1726
1727   switch (GET_CODE (operands[3]))
1728     {
1729     case LT:
1730     case LE:
1731     case GT:
1732     case GE:
1733     case EQ:
1734       /* The easy case.  Here we emit one of vcge, vcgt or vceq.
1735          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
1736          a GE b -> a GE b
1737          a GT b -> a GT b
1738          a LE b -> b GE a
1739          a LT b -> b GT a
1740          a EQ b -> a EQ b  */
1741
1742       if (!inverse)
1743         emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx));
1744       else
1745         emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx));
1746       break;
1747     case UNLT:
1748     case UNLE:
1749     case UNGT:
1750     case UNGE:
1751     case NE:
1752       /* Vector compare returns false for lanes which are unordered, so if we use
1753          the inverse of the comparison we actually want to emit, then
1754          swap the operands to BSL, we will end up with the correct result.
1755          Note that a NE NaN and NaN NE b are true for all a, b.
1756
1757          Our transformations are:
1758          a GE b -> !(b GT a)
1759          a GT b -> !(b GE a)
1760          a LE b -> !(a GT b)
1761          a LT b -> !(a GE b)
1762          a NE b -> !(a EQ b)  */
1763
1764       if (inverse)
1765         emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx));
1766       else
1767         emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx));
1768
1769       swap_bsl_operands = 1;
1770       break;
1771     case UNEQ:
1772       /* We check (a > b ||  b > a).  combining these comparisons give us
1773          true iff !(a != b && a ORDERED b), swapping the operands to BSL
1774          will then give us (a == b ||  a UNORDERED b) as intended.  */
1775
1776       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], magic_rtx));
1777       emit_insn (gen_neon_vcgt<mode> (tmp, operands[5], operands[4], magic_rtx));
1778       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1779       swap_bsl_operands = 1;
1780       break;
1781     case UNORDERED:
1782        /* Operands are ORDERED iff (a > b || b >= a).
1783          Swapping the operands to BSL will give the UNORDERED case.  */
1784      swap_bsl_operands = 1;
1785      /* Fall through.  */
1786     case ORDERED:
1787       emit_insn (gen_neon_vcgt<mode> (tmp, operands[4], operands[5], magic_rtx));
1788       emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], magic_rtx));
1789       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1790       break;
1791     default:
1792       gcc_unreachable ();
1793     }
1794
1795   if (swap_bsl_operands)
1796     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1797                                     operands[1]));
1798   else
1799     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1800                                     operands[2]));
1801   DONE;
1802 })
1803
1804 (define_expand "vcondu<mode><mode>"
1805   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1806         (if_then_else:VDQIW
1807           (match_operator 3 "arm_comparison_operator"
1808             [(match_operand:VDQIW 4 "s_register_operand" "")
1809              (match_operand:VDQIW 5 "s_register_operand" "")])
1810           (match_operand:VDQIW 1 "s_register_operand" "")
1811           (match_operand:VDQIW 2 "s_register_operand" "")))]
1812   "TARGET_NEON"
1813 {
1814   rtx mask;
1815   int inverse = 0, immediate_zero = 0;
1816
1817   mask = gen_reg_rtx (<V_cmp_result>mode);
1818
1819   if (operands[5] == CONST0_RTX (<MODE>mode))
1820     immediate_zero = 1;
1821   else if (!REG_P (operands[5]))
1822     operands[5] = force_reg (<MODE>mode, operands[5]);
1823
1824   switch (GET_CODE (operands[3]))
1825     {
1826     case GEU:
1827       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1828                                       const0_rtx));
1829       break;
1830
1831     case GTU:
1832       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1833                                       const0_rtx));
1834       break;
1835
1836     case EQ:
1837       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1838                                       const0_rtx));
1839       break;
1840
1841     case LEU:
1842       if (immediate_zero)
1843         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1844                                         const0_rtx));
1845       else
1846         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1847                                         const0_rtx));
1848       break;
1849
1850     case LTU:
1851       if (immediate_zero)
1852         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1853                                         const0_rtx));
1854       else
1855         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1856                                         const0_rtx));
1857       break;
1858
1859     case NE:
1860       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1861                                       const0_rtx));
1862       inverse = 1;
1863       break;
1864
1865     default:
1866       gcc_unreachable ();
1867     }
1868
1869   if (inverse)
1870     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1871                                     operands[1]));
1872   else
1873     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1874                                     operands[2]));
1875
1876   DONE;
1877 })
1878
1879 ;; Patterns for builtins.
1880
1881 ; good for plain vadd, vaddq.
1882
1883 (define_expand "neon_vadd<mode>"
1884   [(match_operand:VDQX 0 "s_register_operand" "=w")
1885    (match_operand:VDQX 1 "s_register_operand" "w")
1886    (match_operand:VDQX 2 "s_register_operand" "w")
1887    (match_operand:SI 3 "immediate_operand" "i")]
1888   "TARGET_NEON"
1889 {
1890   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1891     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1892   else
1893     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1894                                            operands[2]));
1895   DONE;
1896 })
1897
1898 ; Note that NEON operations don't support the full IEEE 754 standard: in
1899 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1900 ; use those instructions for autovectorization, etc. unless
1901 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1902 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1903 ; header) must work in either case: if -funsafe-math-optimizations is given,
1904 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1905 ; expand to unspecs (which may potentially limit the extent to which they might
1906 ; be optimized by generic code).
1907
1908 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1909
1910 (define_insn "neon_vadd<mode>_unspec"
1911   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1912         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1913                       (match_operand:VDQX 2 "s_register_operand" "w")]
1914                      UNSPEC_VADD))]
1915   "TARGET_NEON"
1916   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1917   [(set (attr "neon_type")
1918       (if_then_else (match_test "<Is_float_mode>")
1919                     (if_then_else (match_test "<Is_d_reg>")
1920                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
1921                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
1922                     (const_string "neon_int_1")))]
1923 )
1924
1925 ; operand 3 represents in bits:
1926 ;  bit 0: signed (vs unsigned).
1927 ;  bit 1: rounding (vs none).
1928
1929 (define_insn "neon_vaddl<mode>"
1930   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1931         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1932                            (match_operand:VDI 2 "s_register_operand" "w")
1933                            (match_operand:SI 3 "immediate_operand" "i")]
1934                           UNSPEC_VADDL))]
1935   "TARGET_NEON"
1936   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1937   [(set_attr "neon_type" "neon_int_3")]
1938 )
1939
1940 (define_insn "neon_vaddw<mode>"
1941   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1942         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1943                            (match_operand:VDI 2 "s_register_operand" "w")
1944                            (match_operand:SI 3 "immediate_operand" "i")]
1945                           UNSPEC_VADDW))]
1946   "TARGET_NEON"
1947   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1948   [(set_attr "neon_type" "neon_int_2")]
1949 )
1950
1951 ; vhadd and vrhadd.
1952
1953 (define_insn "neon_vhadd<mode>"
1954   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1955         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1956                        (match_operand:VDQIW 2 "s_register_operand" "w")
1957                        (match_operand:SI 3 "immediate_operand" "i")]
1958                       UNSPEC_VHADD))]
1959   "TARGET_NEON"
1960   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1961   [(set_attr "neon_type" "neon_int_4")]
1962 )
1963
1964 (define_insn "neon_vqadd<mode>"
1965   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1966         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1967                        (match_operand:VDQIX 2 "s_register_operand" "w")
1968                        (match_operand:SI 3 "immediate_operand" "i")]
1969                      UNSPEC_VQADD))]
1970   "TARGET_NEON"
1971   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1972   [(set_attr "neon_type" "neon_int_4")]
1973 )
1974
1975 (define_insn "neon_vaddhn<mode>"
1976   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1977         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1978                             (match_operand:VN 2 "s_register_operand" "w")
1979                             (match_operand:SI 3 "immediate_operand" "i")]
1980                            UNSPEC_VADDHN))]
1981   "TARGET_NEON"
1982   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1983   [(set_attr "neon_type" "neon_int_4")]
1984 )
1985
1986 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1987 ;; polynomial multiplication case that can specified by operand 3.
1988 (define_insn "neon_vmul<mode>"
1989   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1990         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1991                       (match_operand:VDQW 2 "s_register_operand" "w")
1992                       (match_operand:SI 3 "immediate_operand" "i")]
1993                      UNSPEC_VMUL))]
1994   "TARGET_NEON"
1995   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1996   [(set (attr "neon_type")
1997       (if_then_else (match_test "<Is_float_mode>")
1998                     (if_then_else (match_test "<Is_d_reg>")
1999                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2000                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2001                     (if_then_else (match_test "<Is_d_reg>")
2002                                   (if_then_else
2003                                     (match_test "<Scalar_mul_8_16>")
2004                                     (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2005                                     (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2006                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2007                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")
2008                                     (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
2009 )
2010
2011 (define_expand "neon_vmla<mode>"
2012   [(match_operand:VDQW 0 "s_register_operand" "=w")
2013    (match_operand:VDQW 1 "s_register_operand" "0")
2014    (match_operand:VDQW 2 "s_register_operand" "w")
2015    (match_operand:VDQW 3 "s_register_operand" "w")
2016    (match_operand:SI 4 "immediate_operand" "i")]
2017   "TARGET_NEON"
2018 {
2019   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2020     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
2021                                              operands[2], operands[3]));
2022   else
2023     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
2024                                            operands[2], operands[3]));
2025   DONE;
2026 })
2027
2028 (define_expand "neon_vfma<VCVTF:mode>"
2029   [(match_operand:VCVTF 0 "s_register_operand")
2030    (match_operand:VCVTF 1 "s_register_operand")
2031    (match_operand:VCVTF 2 "s_register_operand")
2032    (match_operand:VCVTF 3 "s_register_operand")
2033    (match_operand:SI 4 "immediate_operand")]
2034   "TARGET_NEON && TARGET_FMA"
2035 {
2036   emit_insn (gen_fma<mode>4_intrinsic (operands[0], operands[2], operands[3],
2037                                        operands[1]));
2038   DONE;
2039 })
2040
2041 (define_expand "neon_vfms<VCVTF:mode>"
2042   [(match_operand:VCVTF 0 "s_register_operand")
2043    (match_operand:VCVTF 1 "s_register_operand")
2044    (match_operand:VCVTF 2 "s_register_operand")
2045    (match_operand:VCVTF 3 "s_register_operand")
2046    (match_operand:SI 4 "immediate_operand")]
2047   "TARGET_NEON && TARGET_FMA"
2048 {
2049   emit_insn (gen_fmsub<mode>4_intrinsic (operands[0], operands[2], operands[3],
2050                                          operands[1]));
2051   DONE;
2052 })
2053
2054 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2055
2056 (define_insn "neon_vmla<mode>_unspec"
2057   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2058         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2059                      (match_operand:VDQ 2 "s_register_operand" "w")
2060                      (match_operand:VDQ 3 "s_register_operand" "w")]
2061                     UNSPEC_VMLA))]
2062   "TARGET_NEON"
2063   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2064   [(set (attr "neon_type")
2065       (if_then_else (match_test "<Is_float_mode>")
2066                     (if_then_else (match_test "<Is_d_reg>")
2067                                   (const_string "neon_fp_vmla_ddd")
2068                                   (const_string "neon_fp_vmla_qqq"))
2069                     (if_then_else (match_test "<Is_d_reg>")
2070                                   (if_then_else
2071                                     (match_test "<Scalar_mul_8_16>")
2072                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2073                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2074                                   (if_then_else (match_test "<Scalar_mul_8_16>")
2075                                     (const_string "neon_mla_qqq_8_16")
2076                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2077 )
2078
2079 (define_insn "neon_vmlal<mode>"
2080   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2081         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2082                            (match_operand:VW 2 "s_register_operand" "w")
2083                            (match_operand:VW 3 "s_register_operand" "w")
2084                            (match_operand:SI 4 "immediate_operand" "i")]
2085                           UNSPEC_VMLAL))]
2086   "TARGET_NEON"
2087   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2088   [(set (attr "neon_type")
2089      (if_then_else (match_test "<Scalar_mul_8_16>")
2090                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2091                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2092 )
2093
2094 (define_expand "neon_vmls<mode>"
2095   [(match_operand:VDQW 0 "s_register_operand" "=w")
2096    (match_operand:VDQW 1 "s_register_operand" "0")
2097    (match_operand:VDQW 2 "s_register_operand" "w")
2098    (match_operand:VDQW 3 "s_register_operand" "w")
2099    (match_operand:SI 4 "immediate_operand" "i")]
2100   "TARGET_NEON"
2101 {
2102   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2103     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
2104                  operands[1], operands[2], operands[3]));
2105   else
2106     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
2107                                            operands[2], operands[3]));
2108   DONE;
2109 })
2110
2111 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2112
2113 (define_insn "neon_vmls<mode>_unspec"
2114   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
2115         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
2116                      (match_operand:VDQ 2 "s_register_operand" "w")
2117                      (match_operand:VDQ 3 "s_register_operand" "w")]
2118                     UNSPEC_VMLS))]
2119   "TARGET_NEON"
2120   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2121   [(set (attr "neon_type")
2122       (if_then_else (match_test "<Is_float_mode>")
2123                     (if_then_else (match_test "<Is_d_reg>")
2124                                   (const_string "neon_fp_vmla_ddd")
2125                                   (const_string "neon_fp_vmla_qqq"))
2126                     (if_then_else (match_test "<Is_d_reg>")
2127                                   (if_then_else
2128                                     (match_test "<Scalar_mul_8_16>")
2129                                     (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2130                                     (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
2131                                   (if_then_else
2132                                     (match_test "<Scalar_mul_8_16>")
2133                                     (const_string "neon_mla_qqq_8_16")
2134                                     (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
2135 )
2136
2137 (define_insn "neon_vmlsl<mode>"
2138   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2139         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2140                            (match_operand:VW 2 "s_register_operand" "w")
2141                            (match_operand:VW 3 "s_register_operand" "w")
2142                            (match_operand:SI 4 "immediate_operand" "i")]
2143                           UNSPEC_VMLSL))]
2144   "TARGET_NEON"
2145   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2146   [(set (attr "neon_type")
2147      (if_then_else (match_test "<Scalar_mul_8_16>")
2148                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2149                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2150 )
2151
2152 (define_insn "neon_vqdmulh<mode>"
2153   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
2154         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
2155                        (match_operand:VMDQI 2 "s_register_operand" "w")
2156                        (match_operand:SI 3 "immediate_operand" "i")]
2157                       UNSPEC_VQDMULH))]
2158   "TARGET_NEON"
2159   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2160   [(set (attr "neon_type")
2161      (if_then_else (match_test "<Is_d_reg>")
2162         (if_then_else (match_test "<Scalar_mul_8_16>")
2163                       (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2164                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))
2165         (if_then_else (match_test "<Scalar_mul_8_16>")
2166                       (const_string "neon_mul_qqq_8_16_32_ddd_32")
2167                       (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
2168 )
2169
2170 (define_insn "neon_vqdmlal<mode>"
2171   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2172         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2173                            (match_operand:VMDI 2 "s_register_operand" "w")
2174                            (match_operand:VMDI 3 "s_register_operand" "w")
2175                            (match_operand:SI 4 "immediate_operand" "i")]
2176                           UNSPEC_VQDMLAL))]
2177   "TARGET_NEON"
2178   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2179   [(set (attr "neon_type")
2180      (if_then_else (match_test "<Scalar_mul_8_16>")
2181                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2182                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2183 )
2184
2185 (define_insn "neon_vqdmlsl<mode>"
2186   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2187         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2188                            (match_operand:VMDI 2 "s_register_operand" "w")
2189                            (match_operand:VMDI 3 "s_register_operand" "w")
2190                            (match_operand:SI 4 "immediate_operand" "i")]
2191                           UNSPEC_VQDMLSL))]
2192   "TARGET_NEON"
2193   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2194   [(set (attr "neon_type")
2195      (if_then_else (match_test "<Scalar_mul_8_16>")
2196                    (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
2197                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
2198 )
2199
2200 (define_insn "neon_vmull<mode>"
2201   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2202         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2203                            (match_operand:VW 2 "s_register_operand" "w")
2204                            (match_operand:SI 3 "immediate_operand" "i")]
2205                           UNSPEC_VMULL))]
2206   "TARGET_NEON"
2207   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2208   [(set (attr "neon_type")
2209      (if_then_else (match_test "<Scalar_mul_8_16>")
2210                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2211                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2212 )
2213
2214 (define_insn "neon_vqdmull<mode>"
2215   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2216         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2217                            (match_operand:VMDI 2 "s_register_operand" "w")
2218                            (match_operand:SI 3 "immediate_operand" "i")]
2219                           UNSPEC_VQDMULL))]
2220   "TARGET_NEON"
2221   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2222   [(set (attr "neon_type")
2223      (if_then_else (match_test "<Scalar_mul_8_16>")
2224                    (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2225                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2226 )
2227
2228 (define_expand "neon_vsub<mode>"
2229   [(match_operand:VDQX 0 "s_register_operand" "=w")
2230    (match_operand:VDQX 1 "s_register_operand" "w")
2231    (match_operand:VDQX 2 "s_register_operand" "w")
2232    (match_operand:SI 3 "immediate_operand" "i")]
2233   "TARGET_NEON"
2234 {
2235   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2236     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2237   else
2238     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2239                                            operands[2]));
2240   DONE;
2241 })
2242
2243 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2244
2245 (define_insn "neon_vsub<mode>_unspec"
2246   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2247         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2248                       (match_operand:VDQX 2 "s_register_operand" "w")]
2249                      UNSPEC_VSUB))]
2250   "TARGET_NEON"
2251   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2252   [(set (attr "neon_type")
2253       (if_then_else (match_test "<Is_float_mode>")
2254                     (if_then_else (match_test "<Is_d_reg>")
2255                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2256                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2257                     (const_string "neon_int_2")))]
2258 )
2259
2260 (define_insn "neon_vsubl<mode>"
2261   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2262         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2263                            (match_operand:VDI 2 "s_register_operand" "w")
2264                            (match_operand:SI 3 "immediate_operand" "i")]
2265                           UNSPEC_VSUBL))]
2266   "TARGET_NEON"
2267   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2268   [(set_attr "neon_type" "neon_int_2")]
2269 )
2270
2271 (define_insn "neon_vsubw<mode>"
2272   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2273         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2274                            (match_operand:VDI 2 "s_register_operand" "w")
2275                            (match_operand:SI 3 "immediate_operand" "i")]
2276                           UNSPEC_VSUBW))]
2277   "TARGET_NEON"
2278   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2279   [(set_attr "neon_type" "neon_int_2")]
2280 )
2281
2282 (define_insn "neon_vqsub<mode>"
2283   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2284         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2285                        (match_operand:VDQIX 2 "s_register_operand" "w")
2286                        (match_operand:SI 3 "immediate_operand" "i")]
2287                       UNSPEC_VQSUB))]
2288   "TARGET_NEON"
2289   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2290   [(set_attr "neon_type" "neon_int_5")]
2291 )
2292
2293 (define_insn "neon_vhsub<mode>"
2294   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2295         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2296                        (match_operand:VDQIW 2 "s_register_operand" "w")
2297                        (match_operand:SI 3 "immediate_operand" "i")]
2298                       UNSPEC_VHSUB))]
2299   "TARGET_NEON"
2300   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2301   [(set_attr "neon_type" "neon_int_5")]
2302 )
2303
2304 (define_insn "neon_vsubhn<mode>"
2305   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2306         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2307                             (match_operand:VN 2 "s_register_operand" "w")
2308                             (match_operand:SI 3 "immediate_operand" "i")]
2309                            UNSPEC_VSUBHN))]
2310   "TARGET_NEON"
2311   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2312   [(set_attr "neon_type" "neon_int_4")]
2313 )
2314
2315 (define_insn "neon_vceq<mode>"
2316   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2317         (unspec:<V_cmp_result>
2318           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2319            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2320            (match_operand:SI 3 "immediate_operand" "i,i")]
2321           UNSPEC_VCEQ))]
2322   "TARGET_NEON"
2323   "@
2324   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2325   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2326   [(set (attr "neon_type")
2327       (if_then_else (match_test "<Is_float_mode>")
2328                     (if_then_else (match_test "<Is_d_reg>")
2329                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2330                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2331                     (const_string "neon_int_5")))]
2332 )
2333
2334 (define_insn "neon_vcge<mode>"
2335   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2336         (unspec:<V_cmp_result>
2337           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2338            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2339            (match_operand:SI 3 "immediate_operand" "i,i")]
2340           UNSPEC_VCGE))]
2341   "TARGET_NEON"
2342   "@
2343   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2344   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2345   [(set (attr "neon_type")
2346      (if_then_else (match_test "<Is_float_mode>")
2347                    (if_then_else (match_test "<Is_d_reg>")
2348                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2349                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2350                    (const_string "neon_int_5")))]
2351 )
2352
2353 (define_insn "neon_vcgeu<mode>"
2354   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2355         (unspec:<V_cmp_result>
2356           [(match_operand:VDQIW 1 "s_register_operand" "w")
2357            (match_operand:VDQIW 2 "s_register_operand" "w")
2358            (match_operand:SI 3 "immediate_operand" "i")]
2359           UNSPEC_VCGEU))]
2360   "TARGET_NEON"
2361   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2362   [(set_attr "neon_type" "neon_int_5")]
2363 )
2364
2365 (define_insn "neon_vcgt<mode>"
2366   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2367         (unspec:<V_cmp_result>
2368           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2369            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2370            (match_operand:SI 3 "immediate_operand" "i,i")]
2371           UNSPEC_VCGT))]
2372   "TARGET_NEON"
2373   "@
2374   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2375   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2376   [(set (attr "neon_type")
2377      (if_then_else (match_test "<Is_float_mode>")
2378                    (if_then_else (match_test "<Is_d_reg>")
2379                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2380                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2381                    (const_string "neon_int_5")))]
2382 )
2383
2384 (define_insn "neon_vcgtu<mode>"
2385   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2386         (unspec:<V_cmp_result>
2387           [(match_operand:VDQIW 1 "s_register_operand" "w")
2388            (match_operand:VDQIW 2 "s_register_operand" "w")
2389            (match_operand:SI 3 "immediate_operand" "i")]
2390           UNSPEC_VCGTU))]
2391   "TARGET_NEON"
2392   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2393   [(set_attr "neon_type" "neon_int_5")]
2394 )
2395
2396 ;; VCLE and VCLT only support comparisons with immediate zero (register
2397 ;; variants are VCGE and VCGT with operands reversed).
2398
2399 (define_insn "neon_vcle<mode>"
2400   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2401         (unspec:<V_cmp_result>
2402           [(match_operand:VDQW 1 "s_register_operand" "w")
2403            (match_operand:VDQW 2 "zero_operand" "Dz")
2404            (match_operand:SI 3 "immediate_operand" "i")]
2405           UNSPEC_VCLE))]
2406   "TARGET_NEON"
2407   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2408   [(set (attr "neon_type")
2409       (if_then_else (match_test "<Is_float_mode>")
2410                     (if_then_else (match_test "<Is_d_reg>")
2411                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2412                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2413                     (const_string "neon_int_5")))]
2414 )
2415
2416 (define_insn "neon_vclt<mode>"
2417   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2418         (unspec:<V_cmp_result>
2419           [(match_operand:VDQW 1 "s_register_operand" "w")
2420            (match_operand:VDQW 2 "zero_operand" "Dz")
2421            (match_operand:SI 3 "immediate_operand" "i")]
2422           UNSPEC_VCLT))]
2423   "TARGET_NEON"
2424   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2425   [(set (attr "neon_type")
2426       (if_then_else (match_test "<Is_float_mode>")
2427                     (if_then_else (match_test "<Is_d_reg>")
2428                                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2429                                   (const_string "neon_fp_vadd_qqq_vabs_qq"))
2430                     (const_string "neon_int_5")))]
2431 )
2432
2433 (define_insn "neon_vcage<mode>"
2434   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2435         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2436                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2437                                 (match_operand:SI 3 "immediate_operand" "i")]
2438                                UNSPEC_VCAGE))]
2439   "TARGET_NEON"
2440   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2441   [(set (attr "neon_type")
2442      (if_then_else (match_test "<Is_d_reg>")
2443                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2444                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2445 )
2446
2447 (define_insn "neon_vcagt<mode>"
2448   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2449         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2450                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2451                                 (match_operand:SI 3 "immediate_operand" "i")]
2452                                UNSPEC_VCAGT))]
2453   "TARGET_NEON"
2454   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2455   [(set (attr "neon_type")
2456      (if_then_else (match_test "<Is_d_reg>")
2457                    (const_string "neon_fp_vadd_ddd_vabs_dd")
2458                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2459 )
2460
2461 (define_insn "neon_vtst<mode>"
2462   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2463         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2464                        (match_operand:VDQIW 2 "s_register_operand" "w")
2465                        (match_operand:SI 3 "immediate_operand" "i")]
2466                       UNSPEC_VTST))]
2467   "TARGET_NEON"
2468   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2469   [(set_attr "neon_type" "neon_int_4")]
2470 )
2471
2472 (define_insn "neon_vabd<mode>"
2473   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2474         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2475                       (match_operand:VDQW 2 "s_register_operand" "w")
2476                       (match_operand:SI 3 "immediate_operand" "i")]
2477                      UNSPEC_VABD))]
2478   "TARGET_NEON"
2479   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2480   [(set (attr "neon_type")
2481      (if_then_else (match_test "<Is_float_mode>")
2482                    (if_then_else (match_test "<Is_d_reg>")
2483                                  (const_string "neon_fp_vadd_ddd_vabs_dd")
2484                                  (const_string "neon_fp_vadd_qqq_vabs_qq"))
2485                    (const_string "neon_int_5")))]
2486 )
2487
2488 (define_insn "neon_vabdl<mode>"
2489   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2490         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2491                            (match_operand:VW 2 "s_register_operand" "w")
2492                            (match_operand:SI 3 "immediate_operand" "i")]
2493                           UNSPEC_VABDL))]
2494   "TARGET_NEON"
2495   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2496   [(set_attr "neon_type" "neon_int_5")]
2497 )
2498
2499 (define_insn "neon_vaba<mode>"
2500   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2501         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2502                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2503                                    (match_operand:SI 4 "immediate_operand" "i")]
2504                                   UNSPEC_VABD)
2505                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2506   "TARGET_NEON"
2507   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2508   [(set (attr "neon_type")
2509      (if_then_else (match_test "<Is_d_reg>")
2510                    (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2511 )
2512
2513 (define_insn "neon_vabal<mode>"
2514   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2515         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2516                                            (match_operand:VW 3 "s_register_operand" "w")
2517                                            (match_operand:SI 4 "immediate_operand" "i")]
2518                                            UNSPEC_VABDL)
2519                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2520   "TARGET_NEON"
2521   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2522   [(set_attr "neon_type" "neon_vaba")]
2523 )
2524
2525 (define_insn "neon_vmax<mode>"
2526   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2527         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2528                       (match_operand:VDQW 2 "s_register_operand" "w")
2529                       (match_operand:SI 3 "immediate_operand" "i")]
2530                      UNSPEC_VMAX))]
2531   "TARGET_NEON"
2532   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2533   [(set (attr "neon_type")
2534     (if_then_else (match_test "<Is_float_mode>")
2535                   (if_then_else (match_test "<Is_d_reg>")
2536                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2537                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2538                   (const_string "neon_int_5")))]
2539 )
2540
2541 (define_insn "neon_vmin<mode>"
2542   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2543         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2544                       (match_operand:VDQW 2 "s_register_operand" "w")
2545                       (match_operand:SI 3 "immediate_operand" "i")]
2546                      UNSPEC_VMIN))]
2547   "TARGET_NEON"
2548   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2549   [(set (attr "neon_type")
2550     (if_then_else (match_test "<Is_float_mode>")
2551                   (if_then_else (match_test "<Is_d_reg>")
2552                                 (const_string "neon_fp_vadd_ddd_vabs_dd")
2553                                 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2554                   (const_string "neon_int_5")))]
2555 )
2556
2557 (define_expand "neon_vpadd<mode>"
2558   [(match_operand:VD 0 "s_register_operand" "=w")
2559    (match_operand:VD 1 "s_register_operand" "w")
2560    (match_operand:VD 2 "s_register_operand" "w")
2561    (match_operand:SI 3 "immediate_operand" "i")]
2562   "TARGET_NEON"
2563 {
2564   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2565                                             operands[2]));
2566   DONE;
2567 })
2568
2569 (define_insn "neon_vpaddl<mode>"
2570   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2571         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2572                                   (match_operand:SI 2 "immediate_operand" "i")]
2573                                  UNSPEC_VPADDL))]
2574   "TARGET_NEON"
2575   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2576   ;; Assume this schedules like vaddl.
2577   [(set_attr "neon_type" "neon_int_3")]
2578 )
2579
2580 (define_insn "neon_vpadal<mode>"
2581   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2582         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2583                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2584                                   (match_operand:SI 3 "immediate_operand" "i")]
2585                                  UNSPEC_VPADAL))]
2586   "TARGET_NEON"
2587   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2588   ;; Assume this schedules like vpadd.
2589   [(set_attr "neon_type" "neon_int_1")]
2590 )
2591
2592 (define_insn "neon_vpmax<mode>"
2593   [(set (match_operand:VD 0 "s_register_operand" "=w")
2594         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2595                     (match_operand:VD 2 "s_register_operand" "w")
2596                     (match_operand:SI 3 "immediate_operand" "i")]
2597                    UNSPEC_VPMAX))]
2598   "TARGET_NEON"
2599   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2600   ;; Assume this schedules like vmax.
2601   [(set (attr "neon_type")
2602     (if_then_else (match_test "<Is_float_mode>")
2603                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2604                   (const_string "neon_int_5")))]
2605 )
2606
2607 (define_insn "neon_vpmin<mode>"
2608   [(set (match_operand:VD 0 "s_register_operand" "=w")
2609         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2610                     (match_operand:VD 2 "s_register_operand" "w")
2611                     (match_operand:SI 3 "immediate_operand" "i")]
2612                    UNSPEC_VPMIN))]
2613   "TARGET_NEON"
2614   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2615   ;; Assume this schedules like vmin.
2616   [(set (attr "neon_type")
2617     (if_then_else (match_test "<Is_float_mode>")
2618                   (const_string "neon_fp_vadd_ddd_vabs_dd")
2619                   (const_string "neon_int_5")))]
2620 )
2621
2622 (define_insn "neon_vrecps<mode>"
2623   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2624         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2625                        (match_operand:VCVTF 2 "s_register_operand" "w")
2626                        (match_operand:SI 3 "immediate_operand" "i")]
2627                       UNSPEC_VRECPS))]
2628   "TARGET_NEON"
2629   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2630   [(set (attr "neon_type")
2631       (if_then_else (match_test "<Is_d_reg>")
2632                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2633                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2634 )
2635
2636 (define_insn "neon_vrsqrts<mode>"
2637   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2638         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2639                        (match_operand:VCVTF 2 "s_register_operand" "w")
2640                        (match_operand:SI 3 "immediate_operand" "i")]
2641                       UNSPEC_VRSQRTS))]
2642   "TARGET_NEON"
2643   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2644   [(set (attr "neon_type")
2645       (if_then_else (match_test "<Is_d_reg>")
2646                     (const_string "neon_fp_vrecps_vrsqrts_ddd")
2647                     (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2648 )
2649
2650 (define_expand "neon_vabs<mode>"
2651   [(match_operand:VDQW 0 "s_register_operand" "")
2652    (match_operand:VDQW 1 "s_register_operand" "")
2653    (match_operand:SI 2 "immediate_operand" "")]
2654   "TARGET_NEON"
2655 {
2656   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2657   DONE;
2658 })
2659
2660 (define_insn "neon_vqabs<mode>"
2661   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2662         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2663                        (match_operand:SI 2 "immediate_operand" "i")]
2664                       UNSPEC_VQABS))]
2665   "TARGET_NEON"
2666   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2667   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2668 )
2669
2670 (define_expand "neon_vneg<mode>"
2671   [(match_operand:VDQW 0 "s_register_operand" "")
2672    (match_operand:VDQW 1 "s_register_operand" "")
2673    (match_operand:SI 2 "immediate_operand" "")]
2674   "TARGET_NEON"
2675 {
2676   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2677   DONE;
2678 })
2679
2680 (define_insn "neon_vqneg<mode>"
2681   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2682         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2683                        (match_operand:SI 2 "immediate_operand" "i")]
2684                       UNSPEC_VQNEG))]
2685   "TARGET_NEON"
2686   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2687   [(set_attr "neon_type" "neon_vqneg_vqabs")]
2688 )
2689
2690 (define_insn "neon_vcls<mode>"
2691   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2692         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2693                        (match_operand:SI 2 "immediate_operand" "i")]
2694                       UNSPEC_VCLS))]
2695   "TARGET_NEON"
2696   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2697   [(set_attr "neon_type" "neon_int_1")]
2698 )
2699
2700 (define_insn "clz<mode>2"
2701   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2702         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2703   "TARGET_NEON"
2704   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2705   [(set_attr "neon_type" "neon_int_1")]
2706 )
2707
2708 (define_expand "neon_vclz<mode>"
2709   [(match_operand:VDQIW 0 "s_register_operand" "")
2710    (match_operand:VDQIW 1 "s_register_operand" "")
2711    (match_operand:SI 2 "immediate_operand" "")]
2712   "TARGET_NEON"
2713 {
2714   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2715   DONE;
2716 })
2717
2718 (define_insn "popcount<mode>2"
2719   [(set (match_operand:VE 0 "s_register_operand" "=w")
2720         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2721   "TARGET_NEON"
2722   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2723   [(set_attr "neon_type" "neon_int_1")]
2724 )
2725
2726 (define_expand "neon_vcnt<mode>"
2727   [(match_operand:VE 0 "s_register_operand" "=w")
2728    (match_operand:VE 1 "s_register_operand" "w")
2729    (match_operand:SI 2 "immediate_operand" "i")]
2730   "TARGET_NEON"
2731 {
2732   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2733   DONE;
2734 })
2735
2736 (define_insn "neon_vrecpe<mode>"
2737   [(set (match_operand:V32 0 "s_register_operand" "=w")
2738         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2739                      (match_operand:SI 2 "immediate_operand" "i")]
2740                     UNSPEC_VRECPE))]
2741   "TARGET_NEON"
2742   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2743   [(set (attr "neon_type")
2744       (if_then_else (match_test "<Is_d_reg>")
2745                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2746                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2747 )
2748
2749 (define_insn "neon_vrsqrte<mode>"
2750   [(set (match_operand:V32 0 "s_register_operand" "=w")
2751         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2752                      (match_operand:SI 2 "immediate_operand" "i")]
2753                     UNSPEC_VRSQRTE))]
2754   "TARGET_NEON"
2755   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2756   [(set (attr "neon_type")
2757       (if_then_else (match_test "<Is_d_reg>")
2758                     (const_string "neon_fp_vadd_ddd_vabs_dd")
2759                     (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2760 )
2761
2762 (define_expand "neon_vmvn<mode>"
2763   [(match_operand:VDQIW 0 "s_register_operand" "")
2764    (match_operand:VDQIW 1 "s_register_operand" "")
2765    (match_operand:SI 2 "immediate_operand" "")]
2766   "TARGET_NEON"
2767 {
2768   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2769   DONE;
2770 })
2771
2772 (define_insn "neon_vget_lane<mode>_sext_internal"
2773   [(set (match_operand:SI 0 "s_register_operand" "=r")
2774         (sign_extend:SI
2775           (vec_select:<V_elem>
2776             (match_operand:VD 1 "s_register_operand" "w")
2777             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2778   "TARGET_NEON"
2779 {
2780   if (BYTES_BIG_ENDIAN)
2781     {
2782       int elt = INTVAL (operands[2]);
2783       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2784       operands[2] = GEN_INT (elt);
2785     }
2786   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2787 }
2788   [(set_attr "neon_type" "neon_bp_simple")]
2789 )
2790
2791 (define_insn "neon_vget_lane<mode>_zext_internal"
2792   [(set (match_operand:SI 0 "s_register_operand" "=r")
2793         (zero_extend:SI
2794           (vec_select:<V_elem>
2795             (match_operand:VD 1 "s_register_operand" "w")
2796             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2797   "TARGET_NEON"
2798 {
2799   if (BYTES_BIG_ENDIAN)
2800     {
2801       int elt = INTVAL (operands[2]);
2802       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2803       operands[2] = GEN_INT (elt);
2804     }
2805   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2806 }
2807   [(set_attr "neon_type" "neon_bp_simple")]
2808 )
2809
2810 (define_insn "neon_vget_lane<mode>_sext_internal"
2811   [(set (match_operand:SI 0 "s_register_operand" "=r")
2812         (sign_extend:SI
2813           (vec_select:<V_elem>
2814             (match_operand:VQ 1 "s_register_operand" "w")
2815             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2816   "TARGET_NEON"
2817 {
2818   rtx ops[3];
2819   int regno = REGNO (operands[1]);
2820   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2821   unsigned int elt = INTVAL (operands[2]);
2822   unsigned int elt_adj = elt % halfelts;
2823
2824   if (BYTES_BIG_ENDIAN)
2825     elt_adj = halfelts - 1 - elt_adj;
2826
2827   ops[0] = operands[0];
2828   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2829   ops[2] = GEN_INT (elt_adj);
2830   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2831
2832   return "";
2833 }
2834   [(set_attr "neon_type" "neon_bp_simple")]
2835 )
2836
2837 (define_insn "neon_vget_lane<mode>_zext_internal"
2838   [(set (match_operand:SI 0 "s_register_operand" "=r")
2839         (zero_extend:SI
2840           (vec_select:<V_elem>
2841             (match_operand:VQ 1 "s_register_operand" "w")
2842             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2843   "TARGET_NEON"
2844 {
2845   rtx ops[3];
2846   int regno = REGNO (operands[1]);
2847   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2848   unsigned int elt = INTVAL (operands[2]);
2849   unsigned int elt_adj = elt % halfelts;
2850
2851   if (BYTES_BIG_ENDIAN)
2852     elt_adj = halfelts - 1 - elt_adj;
2853
2854   ops[0] = operands[0];
2855   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2856   ops[2] = GEN_INT (elt_adj);
2857   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2858
2859   return "";
2860 }
2861   [(set_attr "neon_type" "neon_bp_simple")]
2862 )
2863
2864 (define_expand "neon_vget_lane<mode>"
2865   [(match_operand:<V_ext> 0 "s_register_operand" "")
2866    (match_operand:VDQW 1 "s_register_operand" "")
2867    (match_operand:SI 2 "immediate_operand" "")
2868    (match_operand:SI 3 "immediate_operand" "")]
2869   "TARGET_NEON"
2870 {
2871   HOST_WIDE_INT magic = INTVAL (operands[3]);
2872   rtx insn;
2873
2874   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2875
2876   if (BYTES_BIG_ENDIAN)
2877     {
2878       /* The intrinsics are defined in terms of a model where the
2879          element ordering in memory is vldm order, whereas the generic
2880          RTL is defined in terms of a model where the element ordering
2881          in memory is array order.  Convert the lane number to conform
2882          to this model.  */
2883       unsigned int elt = INTVAL (operands[2]);
2884       unsigned int reg_nelts
2885         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2886       elt ^= reg_nelts - 1;
2887       operands[2] = GEN_INT (elt);
2888     }
2889
2890   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2891     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2892   else
2893     {
2894       if ((magic & 1) != 0)
2895         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2896                                                        operands[2]);
2897       else
2898         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2899                                                        operands[2]);
2900     }
2901   emit_insn (insn);
2902   DONE;
2903 })
2904
2905 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2906 ; elements.
2907
2908 (define_expand "neon_vget_lanedi"
2909   [(match_operand:DI 0 "s_register_operand" "=r")
2910    (match_operand:DI 1 "s_register_operand" "w")
2911    (match_operand:SI 2 "immediate_operand" "i")
2912    (match_operand:SI 3 "immediate_operand" "i")]
2913   "TARGET_NEON"
2914 {
2915   neon_lane_bounds (operands[2], 0, 1);
2916   emit_move_insn (operands[0], operands[1]);
2917   DONE;
2918 })
2919
2920 (define_expand "neon_vget_lanev2di"
2921   [(match_operand:DI 0 "s_register_operand" "")
2922    (match_operand:V2DI 1 "s_register_operand" "")
2923    (match_operand:SI 2 "immediate_operand" "")
2924    (match_operand:SI 3 "immediate_operand" "")]
2925   "TARGET_NEON"
2926 {
2927   switch (INTVAL (operands[2]))
2928     {
2929     case 0:
2930       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2931       break;
2932     case 1:
2933       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2934       break;
2935     default:
2936       neon_lane_bounds (operands[2], 0, 1);
2937       FAIL;
2938     }
2939   DONE;
2940 })
2941
2942 (define_expand "neon_vset_lane<mode>"
2943   [(match_operand:VDQ 0 "s_register_operand" "=w")
2944    (match_operand:<V_elem> 1 "s_register_operand" "r")
2945    (match_operand:VDQ 2 "s_register_operand" "0")
2946    (match_operand:SI 3 "immediate_operand" "i")]
2947   "TARGET_NEON"
2948 {
2949   unsigned int elt = INTVAL (operands[3]);
2950   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2951
2952   if (BYTES_BIG_ENDIAN)
2953     {
2954       unsigned int reg_nelts
2955         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2956       elt ^= reg_nelts - 1;
2957     }
2958
2959   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2960                                          GEN_INT (1 << elt), operands[2]));
2961   DONE;
2962 })
2963
2964 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2965
2966 (define_expand "neon_vset_lanedi"
2967   [(match_operand:DI 0 "s_register_operand" "=w")
2968    (match_operand:DI 1 "s_register_operand" "r")
2969    (match_operand:DI 2 "s_register_operand" "0")
2970    (match_operand:SI 3 "immediate_operand" "i")]
2971   "TARGET_NEON"
2972 {
2973   neon_lane_bounds (operands[3], 0, 1);
2974   emit_move_insn (operands[0], operands[1]);
2975   DONE;
2976 })
2977
2978 (define_expand "neon_vcreate<mode>"
2979   [(match_operand:VDX 0 "s_register_operand" "")
2980    (match_operand:DI 1 "general_operand" "")]
2981   "TARGET_NEON"
2982 {
2983   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2984   emit_move_insn (operands[0], src);
2985   DONE;
2986 })
2987
2988 (define_insn "neon_vdup_n<mode>"
2989   [(set (match_operand:VX 0 "s_register_operand" "=w")
2990         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2991   "TARGET_NEON"
2992   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
2993   ;; Assume this schedules like vmov.
2994   [(set_attr "neon_type" "neon_bp_simple")]
2995 )
2996
2997 (define_insn "neon_vdup_n<mode>"
2998   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2999         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
3000   "TARGET_NEON"
3001   "@
3002   vdup.<V_sz_elem>\t%<V_reg>0, %1
3003   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
3004   ;; Assume this schedules like vmov.
3005   [(set_attr "neon_type" "neon_bp_simple")]
3006 )
3007
3008 (define_expand "neon_vdup_ndi"
3009   [(match_operand:DI 0 "s_register_operand" "=w")
3010    (match_operand:DI 1 "s_register_operand" "r")]
3011   "TARGET_NEON"
3012 {
3013   emit_move_insn (operands[0], operands[1]);
3014   DONE;
3015 }
3016 )
3017
3018 (define_insn "neon_vdup_nv2di"
3019   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
3020         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
3021   "TARGET_NEON"
3022   "@
3023   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
3024   vmov\t%e0, %P1\;vmov\t%f0, %P1"
3025   [(set_attr "length" "8")
3026    (set_attr "neon_type" "neon_bp_simple")]
3027 )
3028
3029 (define_insn "neon_vdup_lane<mode>_internal"
3030   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
3031         (vec_duplicate:VDQW
3032           (vec_select:<V_elem>
3033             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3034             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
3035   "TARGET_NEON"
3036 {
3037   if (BYTES_BIG_ENDIAN)
3038     {
3039       int elt = INTVAL (operands[2]);
3040       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
3041       operands[2] = GEN_INT (elt);
3042     }
3043   if (<Is_d_reg>)
3044     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
3045   else
3046     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
3047 }
3048   ;; Assume this schedules like vmov.
3049   [(set_attr "neon_type" "neon_bp_simple")]
3050 )
3051
3052 (define_expand "neon_vdup_lane<mode>"
3053   [(match_operand:VDQW 0 "s_register_operand" "=w")
3054    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
3055    (match_operand:SI 2 "immediate_operand" "i")]
3056   "TARGET_NEON"
3057 {
3058   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
3059   if (BYTES_BIG_ENDIAN)
3060     {
3061       unsigned int elt = INTVAL (operands[2]);
3062       unsigned int reg_nelts
3063         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
3064       elt ^= reg_nelts - 1;
3065       operands[2] = GEN_INT (elt);
3066     }
3067     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
3068                                                   operands[2]));
3069     DONE;
3070 })
3071
3072 ; Scalar index is ignored, since only zero is valid here.
3073 (define_expand "neon_vdup_lanedi"
3074   [(match_operand:DI 0 "s_register_operand" "=w")
3075    (match_operand:DI 1 "s_register_operand" "w")
3076    (match_operand:SI 2 "immediate_operand" "i")]
3077   "TARGET_NEON"
3078 {
3079   neon_lane_bounds (operands[2], 0, 1);
3080   emit_move_insn (operands[0], operands[1]);
3081   DONE;
3082 })
3083
3084 ; Likewise for v2di, as the DImode second operand has only a single element.
3085 (define_expand "neon_vdup_lanev2di"
3086   [(match_operand:V2DI 0 "s_register_operand" "=w")
3087    (match_operand:DI 1 "s_register_operand" "w")
3088    (match_operand:SI 2 "immediate_operand" "i")]
3089   "TARGET_NEON"
3090 {
3091   neon_lane_bounds (operands[2], 0, 1);
3092   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
3093   DONE;
3094 })
3095
3096 ; Disabled before reload because we don't want combine doing something silly,
3097 ; but used by the post-reload expansion of neon_vcombine.
3098 (define_insn "*neon_vswp<mode>"
3099   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
3100         (match_operand:VDQX 1 "s_register_operand" "+w"))
3101    (set (match_dup 1) (match_dup 0))]
3102   "TARGET_NEON && reload_completed"
3103   "vswp\t%<V_reg>0, %<V_reg>1"
3104   [(set (attr "neon_type")
3105         (if_then_else (match_test "<Is_d_reg>")
3106                       (const_string "neon_bp_simple")
3107                       (const_string "neon_bp_2cycle")))]
3108 )
3109
3110 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
3111 ;; dest vector.
3112 ;; FIXME: A different implementation of this builtin could make it much
3113 ;; more likely that we wouldn't actually need to output anything (we could make
3114 ;; it so that the reg allocator puts things in the right places magically
3115 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
3116
3117 (define_insn_and_split "neon_vcombine<mode>"
3118   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
3119         (vec_concat:<V_DOUBLE>
3120           (match_operand:VDX 1 "s_register_operand" "w")
3121           (match_operand:VDX 2 "s_register_operand" "w")))]
3122   "TARGET_NEON"
3123   "#"
3124   "&& reload_completed"
3125   [(const_int 0)]
3126 {
3127   neon_split_vcombine (operands);
3128   DONE;
3129 })
3130
3131 (define_expand "neon_vget_high<mode>"
3132   [(match_operand:<V_HALF> 0 "s_register_operand")
3133    (match_operand:VQX 1 "s_register_operand")]
3134   "TARGET_NEON"
3135 {
3136   emit_move_insn (operands[0],
3137                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
3138                                        GET_MODE_SIZE (<V_HALF>mode)));
3139   DONE;
3140 })
3141
3142 (define_expand "neon_vget_low<mode>"
3143   [(match_operand:<V_HALF> 0 "s_register_operand")
3144    (match_operand:VQX 1 "s_register_operand")]
3145   "TARGET_NEON"
3146 {
3147   emit_move_insn (operands[0],
3148                   simplify_gen_subreg (<V_HALF>mode, operands[1],
3149                                        <MODE>mode, 0));
3150   DONE;
3151 })
3152
3153 (define_insn "float<mode><V_cvtto>2"
3154   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3155         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3156   "TARGET_NEON && !flag_rounding_math"
3157   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3158   [(set (attr "neon_type")
3159      (if_then_else (match_test "<Is_d_reg>")
3160                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3161                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3162 )
3163
3164 (define_insn "floatuns<mode><V_cvtto>2"
3165   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3166         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3167   "TARGET_NEON && !flag_rounding_math"
3168   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3169   [(set (attr "neon_type")
3170      (if_then_else (match_test "<Is_d_reg>")
3171                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3172                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3173 )
3174
3175 (define_insn "fix_trunc<mode><V_cvtto>2"
3176   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3177         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3178   "TARGET_NEON"
3179   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3180   [(set (attr "neon_type")
3181      (if_then_else (match_test "<Is_d_reg>")
3182                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3183                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3184 )
3185
3186 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3187   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3188         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3189   "TARGET_NEON"
3190   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3191   [(set (attr "neon_type")
3192      (if_then_else (match_test "<Is_d_reg>")
3193                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3194                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3195 )
3196
3197 (define_insn "neon_vcvt<mode>"
3198   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3199         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3200                            (match_operand:SI 2 "immediate_operand" "i")]
3201                           UNSPEC_VCVT))]
3202   "TARGET_NEON"
3203   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3204   [(set (attr "neon_type")
3205      (if_then_else (match_test "<Is_d_reg>")
3206                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3207                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3208 )
3209
3210 (define_insn "neon_vcvt<mode>"
3211   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3212         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3213                            (match_operand:SI 2 "immediate_operand" "i")]
3214                           UNSPEC_VCVT))]
3215   "TARGET_NEON"
3216   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3217   [(set (attr "neon_type")
3218      (if_then_else (match_test "<Is_d_reg>")
3219                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3220                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3221 )
3222
3223 (define_insn "neon_vcvtv4sfv4hf"
3224   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
3225         (unspec:V4SF [(match_operand:V4HF 1 "s_register_operand" "w")]
3226                           UNSPEC_VCVT))]
3227   "TARGET_NEON && TARGET_FP16"
3228   "vcvt.f32.f16\t%q0, %P1"
3229   [(set_attr "neon_type" "neon_fp_vadd_ddd_vabs_dd")]
3230 )
3231
3232 (define_insn "neon_vcvtv4hfv4sf"
3233   [(set (match_operand:V4HF 0 "s_register_operand" "=w")
3234         (unspec:V4HF [(match_operand:V4SF 1 "s_register_operand" "w")]
3235                           UNSPEC_VCVT))]
3236   "TARGET_NEON && TARGET_FP16"
3237   "vcvt.f16.f32\t%P0, %q1"
3238   [(set_attr "neon_type" "neon_fp_vadd_ddd_vabs_dd")]
3239 )
3240
3241 (define_insn "neon_vcvt_n<mode>"
3242   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3243         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3244                            (match_operand:SI 2 "immediate_operand" "i")
3245                            (match_operand:SI 3 "immediate_operand" "i")]
3246                           UNSPEC_VCVT_N))]
3247   "TARGET_NEON"
3248 {
3249   neon_const_bounds (operands[2], 1, 33);
3250   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3251 }
3252   [(set (attr "neon_type")
3253      (if_then_else (match_test "<Is_d_reg>")
3254                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3255                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3256 )
3257
3258 (define_insn "neon_vcvt_n<mode>"
3259   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3260         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3261                            (match_operand:SI 2 "immediate_operand" "i")
3262                            (match_operand:SI 3 "immediate_operand" "i")]
3263                           UNSPEC_VCVT_N))]
3264   "TARGET_NEON"
3265 {
3266   neon_const_bounds (operands[2], 1, 33);
3267   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3268 }
3269   [(set (attr "neon_type")
3270      (if_then_else (match_test "<Is_d_reg>")
3271                    (const_string "neon_fp_vadd_ddd_vabs_dd")
3272                    (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3273 )
3274
3275 (define_insn "neon_vmovn<mode>"
3276   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3277         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3278                             (match_operand:SI 2 "immediate_operand" "i")]
3279                            UNSPEC_VMOVN))]
3280   "TARGET_NEON"
3281   "vmovn.<V_if_elem>\t%P0, %q1"
3282   [(set_attr "neon_type" "neon_bp_simple")]
3283 )
3284
3285 (define_insn "neon_vqmovn<mode>"
3286   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3287         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3288                             (match_operand:SI 2 "immediate_operand" "i")]
3289                            UNSPEC_VQMOVN))]
3290   "TARGET_NEON"
3291   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3292   [(set_attr "neon_type" "neon_shift_2")]
3293 )
3294
3295 (define_insn "neon_vqmovun<mode>"
3296   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3297         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3298                             (match_operand:SI 2 "immediate_operand" "i")]
3299                            UNSPEC_VQMOVUN))]
3300   "TARGET_NEON"
3301   "vqmovun.<V_s_elem>\t%P0, %q1"
3302   [(set_attr "neon_type" "neon_shift_2")]
3303 )
3304
3305 (define_insn "neon_vmovl<mode>"
3306   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3307         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3308                            (match_operand:SI 2 "immediate_operand" "i")]
3309                           UNSPEC_VMOVL))]
3310   "TARGET_NEON"
3311   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3312   [(set_attr "neon_type" "neon_shift_1")]
3313 )
3314
3315 (define_insn "neon_vmul_lane<mode>"
3316   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3317         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3318                      (match_operand:VMD 2 "s_register_operand"
3319                                         "<scalar_mul_constraint>")
3320                      (match_operand:SI 3 "immediate_operand" "i")
3321                      (match_operand:SI 4 "immediate_operand" "i")]
3322                     UNSPEC_VMUL_LANE))]
3323   "TARGET_NEON"
3324 {
3325   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3326   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3327 }
3328   [(set (attr "neon_type")
3329      (if_then_else (match_test "<Is_float_mode>")
3330                    (const_string "neon_fp_vmul_ddd")
3331                    (if_then_else (match_test "<Scalar_mul_8_16>")
3332                                  (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3333                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3334 )
3335
3336 (define_insn "neon_vmul_lane<mode>"
3337   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3338         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3339                      (match_operand:<V_HALF> 2 "s_register_operand"
3340                                              "<scalar_mul_constraint>")
3341                      (match_operand:SI 3 "immediate_operand" "i")
3342                      (match_operand:SI 4 "immediate_operand" "i")]
3343                     UNSPEC_VMUL_LANE))]
3344   "TARGET_NEON"
3345 {
3346   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3347   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3348 }
3349   [(set (attr "neon_type")
3350      (if_then_else (match_test "<Is_float_mode>")
3351                    (const_string "neon_fp_vmul_qqd")
3352                    (if_then_else (match_test "<Scalar_mul_8_16>")
3353                                  (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3354                                  (const_string "neon_mul_qqd_32_scalar"))))]
3355 )
3356
3357 (define_insn "neon_vmull_lane<mode>"
3358   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3359         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3360                            (match_operand:VMDI 2 "s_register_operand"
3361                                                "<scalar_mul_constraint>")
3362                            (match_operand:SI 3 "immediate_operand" "i")
3363                            (match_operand:SI 4 "immediate_operand" "i")]
3364                           UNSPEC_VMULL_LANE))]
3365   "TARGET_NEON"
3366 {
3367   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3368   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3369 }
3370   [(set (attr "neon_type")
3371      (if_then_else (match_test "<Scalar_mul_8_16>")
3372                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3373                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3374 )
3375
3376 (define_insn "neon_vqdmull_lane<mode>"
3377   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3378         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3379                            (match_operand:VMDI 2 "s_register_operand"
3380                                                "<scalar_mul_constraint>")
3381                            (match_operand:SI 3 "immediate_operand" "i")
3382                            (match_operand:SI 4 "immediate_operand" "i")]
3383                           UNSPEC_VQDMULL_LANE))]
3384   "TARGET_NEON"
3385 {
3386   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3387   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3388 }
3389   [(set (attr "neon_type")
3390      (if_then_else (match_test "<Scalar_mul_8_16>")
3391                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3392                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3393 )
3394
3395 (define_insn "neon_vqdmulh_lane<mode>"
3396   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3397         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3398                       (match_operand:<V_HALF> 2 "s_register_operand"
3399                                               "<scalar_mul_constraint>")
3400                       (match_operand:SI 3 "immediate_operand" "i")
3401                       (match_operand:SI 4 "immediate_operand" "i")]
3402                       UNSPEC_VQDMULH_LANE))]
3403   "TARGET_NEON"
3404 {
3405   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3406   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3407 }
3408   [(set (attr "neon_type")
3409      (if_then_else (match_test "<Scalar_mul_8_16>")
3410                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3411                    (const_string "neon_mul_qqd_32_scalar")))]
3412 )
3413
3414 (define_insn "neon_vqdmulh_lane<mode>"
3415   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3416         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3417                       (match_operand:VMDI 2 "s_register_operand"
3418                                           "<scalar_mul_constraint>")
3419                       (match_operand:SI 3 "immediate_operand" "i")
3420                       (match_operand:SI 4 "immediate_operand" "i")]
3421                       UNSPEC_VQDMULH_LANE))]
3422   "TARGET_NEON"
3423 {
3424   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3425   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3426 }
3427   [(set (attr "neon_type")
3428      (if_then_else (match_test "<Scalar_mul_8_16>")
3429                    (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3430                    (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3431 )
3432
3433 (define_insn "neon_vmla_lane<mode>"
3434   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3435         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3436                      (match_operand:VMD 2 "s_register_operand" "w")
3437                      (match_operand:VMD 3 "s_register_operand"
3438                                         "<scalar_mul_constraint>")
3439                      (match_operand:SI 4 "immediate_operand" "i")
3440                      (match_operand:SI 5 "immediate_operand" "i")]
3441                      UNSPEC_VMLA_LANE))]
3442   "TARGET_NEON"
3443 {
3444   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3445   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3446 }
3447   [(set (attr "neon_type")
3448      (if_then_else (match_test "<Is_float_mode>")
3449                    (const_string "neon_fp_vmla_ddd_scalar")
3450                    (if_then_else (match_test "<Scalar_mul_8_16>")
3451                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3452                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3453 )
3454
3455 (define_insn "neon_vmla_lane<mode>"
3456   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3457         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3458                      (match_operand:VMQ 2 "s_register_operand" "w")
3459                      (match_operand:<V_HALF> 3 "s_register_operand"
3460                                              "<scalar_mul_constraint>")
3461                      (match_operand:SI 4 "immediate_operand" "i")
3462                      (match_operand:SI 5 "immediate_operand" "i")]
3463                      UNSPEC_VMLA_LANE))]
3464   "TARGET_NEON"
3465 {
3466   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3467   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3468 }
3469   [(set (attr "neon_type")
3470      (if_then_else (match_test "<Is_float_mode>")
3471                    (const_string "neon_fp_vmla_qqq_scalar")
3472                    (if_then_else (match_test "<Scalar_mul_8_16>")
3473                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3474                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3475 )
3476
3477 (define_insn "neon_vmlal_lane<mode>"
3478   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3479         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3480                            (match_operand:VMDI 2 "s_register_operand" "w")
3481                            (match_operand:VMDI 3 "s_register_operand"
3482                                                "<scalar_mul_constraint>")
3483                            (match_operand:SI 4 "immediate_operand" "i")
3484                            (match_operand:SI 5 "immediate_operand" "i")]
3485                           UNSPEC_VMLAL_LANE))]
3486   "TARGET_NEON"
3487 {
3488   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3489   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3490 }
3491   [(set (attr "neon_type")
3492      (if_then_else (match_test "<Scalar_mul_8_16>")
3493                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3494                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3495 )
3496
3497 (define_insn "neon_vqdmlal_lane<mode>"
3498   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3499         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3500                            (match_operand:VMDI 2 "s_register_operand" "w")
3501                            (match_operand:VMDI 3 "s_register_operand"
3502                                                "<scalar_mul_constraint>")
3503                            (match_operand:SI 4 "immediate_operand" "i")
3504                            (match_operand:SI 5 "immediate_operand" "i")]
3505                           UNSPEC_VQDMLAL_LANE))]
3506   "TARGET_NEON"
3507 {
3508   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3509   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3510 }
3511   [(set (attr "neon_type")
3512      (if_then_else (match_test "<Scalar_mul_8_16>")
3513                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3514                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3515 )
3516
3517 (define_insn "neon_vmls_lane<mode>"
3518   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3519         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3520                      (match_operand:VMD 2 "s_register_operand" "w")
3521                      (match_operand:VMD 3 "s_register_operand"
3522                                         "<scalar_mul_constraint>")
3523                      (match_operand:SI 4 "immediate_operand" "i")
3524                      (match_operand:SI 5 "immediate_operand" "i")]
3525                     UNSPEC_VMLS_LANE))]
3526   "TARGET_NEON"
3527 {
3528   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3529   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3530 }
3531   [(set (attr "neon_type")
3532      (if_then_else (match_test "<Is_float_mode>")
3533                    (const_string "neon_fp_vmla_ddd_scalar")
3534                    (if_then_else (match_test "<Scalar_mul_8_16>")
3535                                  (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3536                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3537 )
3538
3539 (define_insn "neon_vmls_lane<mode>"
3540   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3541         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3542                      (match_operand:VMQ 2 "s_register_operand" "w")
3543                      (match_operand:<V_HALF> 3 "s_register_operand"
3544                                              "<scalar_mul_constraint>")
3545                      (match_operand:SI 4 "immediate_operand" "i")
3546                      (match_operand:SI 5 "immediate_operand" "i")]
3547                     UNSPEC_VMLS_LANE))]
3548   "TARGET_NEON"
3549 {
3550   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3551   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3552 }
3553   [(set (attr "neon_type")
3554      (if_then_else (match_test "<Is_float_mode>")
3555                    (const_string "neon_fp_vmla_qqq_scalar")
3556                    (if_then_else (match_test "<Scalar_mul_8_16>")
3557                                  (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3558                                  (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3559 )
3560
3561 (define_insn "neon_vmlsl_lane<mode>"
3562   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3563         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3564                            (match_operand:VMDI 2 "s_register_operand" "w")
3565                            (match_operand:VMDI 3 "s_register_operand"
3566                                                "<scalar_mul_constraint>")
3567                            (match_operand:SI 4 "immediate_operand" "i")
3568                            (match_operand:SI 5 "immediate_operand" "i")]
3569                           UNSPEC_VMLSL_LANE))]
3570   "TARGET_NEON"
3571 {
3572   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3573   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3574 }
3575   [(set (attr "neon_type")
3576      (if_then_else (match_test "<Scalar_mul_8_16>")
3577                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3578                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3579 )
3580
3581 (define_insn "neon_vqdmlsl_lane<mode>"
3582   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3583         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3584                            (match_operand:VMDI 2 "s_register_operand" "w")
3585                            (match_operand:VMDI 3 "s_register_operand"
3586                                                "<scalar_mul_constraint>")
3587                            (match_operand:SI 4 "immediate_operand" "i")
3588                            (match_operand:SI 5 "immediate_operand" "i")]
3589                           UNSPEC_VQDMLSL_LANE))]
3590   "TARGET_NEON"
3591 {
3592   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3593   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3594 }
3595   [(set (attr "neon_type")
3596      (if_then_else (match_test "<Scalar_mul_8_16>")
3597                    (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3598                    (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3599 )
3600
3601 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3602 ; core register into a temp register, then use a scalar taken from that. This
3603 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3604 ; or extracted from another vector. The latter case it's currently better to
3605 ; use the "_lane" variant, and the former case can probably be implemented
3606 ; using vld1_lane, but that hasn't been done yet.
3607
3608 (define_expand "neon_vmul_n<mode>"
3609   [(match_operand:VMD 0 "s_register_operand" "")
3610    (match_operand:VMD 1 "s_register_operand" "")
3611    (match_operand:<V_elem> 2 "s_register_operand" "")
3612    (match_operand:SI 3 "immediate_operand" "")]
3613   "TARGET_NEON"
3614 {
3615   rtx tmp = gen_reg_rtx (<MODE>mode);
3616   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3617   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3618                                        const0_rtx, const0_rtx));
3619   DONE;
3620 })
3621
3622 (define_expand "neon_vmul_n<mode>"
3623   [(match_operand:VMQ 0 "s_register_operand" "")
3624    (match_operand:VMQ 1 "s_register_operand" "")
3625    (match_operand:<V_elem> 2 "s_register_operand" "")
3626    (match_operand:SI 3 "immediate_operand" "")]
3627   "TARGET_NEON"
3628 {
3629   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3630   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3631   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3632                                        const0_rtx, const0_rtx));
3633   DONE;
3634 })
3635
3636 (define_expand "neon_vmull_n<mode>"
3637   [(match_operand:<V_widen> 0 "s_register_operand" "")
3638    (match_operand:VMDI 1 "s_register_operand" "")
3639    (match_operand:<V_elem> 2 "s_register_operand" "")
3640    (match_operand:SI 3 "immediate_operand" "")]
3641   "TARGET_NEON"
3642 {
3643   rtx tmp = gen_reg_rtx (<MODE>mode);
3644   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3645   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3646                                         const0_rtx, operands[3]));
3647   DONE;
3648 })
3649
3650 (define_expand "neon_vqdmull_n<mode>"
3651   [(match_operand:<V_widen> 0 "s_register_operand" "")
3652    (match_operand:VMDI 1 "s_register_operand" "")
3653    (match_operand:<V_elem> 2 "s_register_operand" "")
3654    (match_operand:SI 3 "immediate_operand" "")]
3655   "TARGET_NEON"
3656 {
3657   rtx tmp = gen_reg_rtx (<MODE>mode);
3658   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3659   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3660                                           const0_rtx, const0_rtx));
3661   DONE;
3662 })
3663
3664 (define_expand "neon_vqdmulh_n<mode>"
3665   [(match_operand:VMDI 0 "s_register_operand" "")
3666    (match_operand:VMDI 1 "s_register_operand" "")
3667    (match_operand:<V_elem> 2 "s_register_operand" "")
3668    (match_operand:SI 3 "immediate_operand" "")]
3669   "TARGET_NEON"
3670 {
3671   rtx tmp = gen_reg_rtx (<MODE>mode);
3672   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3673   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3674                                           const0_rtx, operands[3]));
3675   DONE;
3676 })
3677
3678 (define_expand "neon_vqdmulh_n<mode>"
3679   [(match_operand:VMQI 0 "s_register_operand" "")
3680    (match_operand:VMQI 1 "s_register_operand" "")
3681    (match_operand:<V_elem> 2 "s_register_operand" "")
3682    (match_operand:SI 3 "immediate_operand" "")]
3683   "TARGET_NEON"
3684 {
3685   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3686   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3687   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3688                                           const0_rtx, operands[3]));
3689   DONE;
3690 })
3691
3692 (define_expand "neon_vmla_n<mode>"
3693   [(match_operand:VMD 0 "s_register_operand" "")
3694    (match_operand:VMD 1 "s_register_operand" "")
3695    (match_operand:VMD 2 "s_register_operand" "")
3696    (match_operand:<V_elem> 3 "s_register_operand" "")
3697    (match_operand:SI 4 "immediate_operand" "")]
3698   "TARGET_NEON"
3699 {
3700   rtx tmp = gen_reg_rtx (<MODE>mode);
3701   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3702   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3703                                        tmp, const0_rtx, operands[4]));
3704   DONE;
3705 })
3706
3707 (define_expand "neon_vmla_n<mode>"
3708   [(match_operand:VMQ 0 "s_register_operand" "")
3709    (match_operand:VMQ 1 "s_register_operand" "")
3710    (match_operand:VMQ 2 "s_register_operand" "")
3711    (match_operand:<V_elem> 3 "s_register_operand" "")
3712    (match_operand:SI 4 "immediate_operand" "")]
3713   "TARGET_NEON"
3714 {
3715   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3716   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3717   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3718                                        tmp, const0_rtx, operands[4]));
3719   DONE;
3720 })
3721
3722 (define_expand "neon_vmlal_n<mode>"
3723   [(match_operand:<V_widen> 0 "s_register_operand" "")
3724    (match_operand:<V_widen> 1 "s_register_operand" "")
3725    (match_operand:VMDI 2 "s_register_operand" "")
3726    (match_operand:<V_elem> 3 "s_register_operand" "")
3727    (match_operand:SI 4 "immediate_operand" "")]
3728   "TARGET_NEON"
3729 {
3730   rtx tmp = gen_reg_rtx (<MODE>mode);
3731   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3732   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3733                                         tmp, const0_rtx, operands[4]));
3734   DONE;
3735 })
3736
3737 (define_expand "neon_vqdmlal_n<mode>"
3738   [(match_operand:<V_widen> 0 "s_register_operand" "")
3739    (match_operand:<V_widen> 1 "s_register_operand" "")
3740    (match_operand:VMDI 2 "s_register_operand" "")
3741    (match_operand:<V_elem> 3 "s_register_operand" "")
3742    (match_operand:SI 4 "immediate_operand" "")]
3743   "TARGET_NEON"
3744 {
3745   rtx tmp = gen_reg_rtx (<MODE>mode);
3746   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3747   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3748                                           tmp, const0_rtx, operands[4]));
3749   DONE;
3750 })
3751
3752 (define_expand "neon_vmls_n<mode>"
3753   [(match_operand:VMD 0 "s_register_operand" "")
3754    (match_operand:VMD 1 "s_register_operand" "")
3755    (match_operand:VMD 2 "s_register_operand" "")
3756    (match_operand:<V_elem> 3 "s_register_operand" "")
3757    (match_operand:SI 4 "immediate_operand" "")]
3758   "TARGET_NEON"
3759 {
3760   rtx tmp = gen_reg_rtx (<MODE>mode);
3761   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3762   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3763                                        tmp, const0_rtx, operands[4]));
3764   DONE;
3765 })
3766
3767 (define_expand "neon_vmls_n<mode>"
3768   [(match_operand:VMQ 0 "s_register_operand" "")
3769    (match_operand:VMQ 1 "s_register_operand" "")
3770    (match_operand:VMQ 2 "s_register_operand" "")
3771    (match_operand:<V_elem> 3 "s_register_operand" "")
3772    (match_operand:SI 4 "immediate_operand" "")]
3773   "TARGET_NEON"
3774 {
3775   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3776   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3777   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3778                                        tmp, const0_rtx, operands[4]));
3779   DONE;
3780 })
3781
3782 (define_expand "neon_vmlsl_n<mode>"
3783   [(match_operand:<V_widen> 0 "s_register_operand" "")
3784    (match_operand:<V_widen> 1 "s_register_operand" "")
3785    (match_operand:VMDI 2 "s_register_operand" "")
3786    (match_operand:<V_elem> 3 "s_register_operand" "")
3787    (match_operand:SI 4 "immediate_operand" "")]
3788   "TARGET_NEON"
3789 {
3790   rtx tmp = gen_reg_rtx (<MODE>mode);
3791   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3792   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3793                                         tmp, const0_rtx, operands[4]));
3794   DONE;
3795 })
3796
3797 (define_expand "neon_vqdmlsl_n<mode>"
3798   [(match_operand:<V_widen> 0 "s_register_operand" "")
3799    (match_operand:<V_widen> 1 "s_register_operand" "")
3800    (match_operand:VMDI 2 "s_register_operand" "")
3801    (match_operand:<V_elem> 3 "s_register_operand" "")
3802    (match_operand:SI 4 "immediate_operand" "")]
3803   "TARGET_NEON"
3804 {
3805   rtx tmp = gen_reg_rtx (<MODE>mode);
3806   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3807   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3808                                           tmp, const0_rtx, operands[4]));
3809   DONE;
3810 })
3811
3812 (define_insn "neon_vext<mode>"
3813   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3814         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3815                       (match_operand:VDQX 2 "s_register_operand" "w")
3816                       (match_operand:SI 3 "immediate_operand" "i")]
3817                      UNSPEC_VEXT))]
3818   "TARGET_NEON"
3819 {
3820   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3821   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3822 }
3823   [(set (attr "neon_type")
3824       (if_then_else (match_test "<Is_d_reg>")
3825                     (const_string "neon_bp_simple")
3826                     (const_string "neon_bp_2cycle")))]
3827 )
3828
3829 (define_insn "neon_vrev64<mode>"
3830   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3831         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3832                      (match_operand:SI 2 "immediate_operand" "i")]
3833                     UNSPEC_VREV64))]
3834   "TARGET_NEON"
3835   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3836   [(set_attr "neon_type" "neon_bp_simple")]
3837 )
3838
3839 (define_insn "neon_vrev32<mode>"
3840   [(set (match_operand:VX 0 "s_register_operand" "=w")
3841         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3842                     (match_operand:SI 2 "immediate_operand" "i")]
3843                    UNSPEC_VREV32))]
3844   "TARGET_NEON"
3845   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3846   [(set_attr "neon_type" "neon_bp_simple")]
3847 )
3848
3849 (define_insn "neon_vrev16<mode>"
3850   [(set (match_operand:VE 0 "s_register_operand" "=w")
3851         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3852                     (match_operand:SI 2 "immediate_operand" "i")]
3853                    UNSPEC_VREV16))]
3854   "TARGET_NEON"
3855   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3856   [(set_attr "neon_type" "neon_bp_simple")]
3857 )
3858
3859 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3860 ; allocation. For an intrinsic of form:
3861 ;   rD = vbsl_* (rS, rN, rM)
3862 ; We can use any of:
3863 ;   vbsl rS, rN, rM  (if D = S)
3864 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3865 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3866
3867 (define_insn "neon_vbsl<mode>_internal"
3868   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3869         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3870                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3871                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3872                      UNSPEC_VBSL))]
3873   "TARGET_NEON"
3874   "@
3875   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3876   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3877   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3878   [(set_attr "neon_type" "neon_int_1")]
3879 )
3880
3881 (define_expand "neon_vbsl<mode>"
3882   [(set (match_operand:VDQX 0 "s_register_operand" "")
3883         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3884                       (match_operand:VDQX 2 "s_register_operand" "")
3885                       (match_operand:VDQX 3 "s_register_operand" "")]
3886                      UNSPEC_VBSL))]
3887   "TARGET_NEON"
3888 {
3889   /* We can't alias operands together if they have different modes.  */
3890   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3891 })
3892
3893 (define_insn "neon_vshl<mode>"
3894   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3895         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3896                        (match_operand:VDQIX 2 "s_register_operand" "w")
3897                        (match_operand:SI 3 "immediate_operand" "i")]
3898                       UNSPEC_VSHL))]
3899   "TARGET_NEON"
3900   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3901   [(set (attr "neon_type")
3902       (if_then_else (match_test "<Is_d_reg>")
3903                     (const_string "neon_vshl_ddd")
3904                     (const_string "neon_shift_3")))]
3905 )
3906
3907 (define_insn "neon_vqshl<mode>"
3908   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3909         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3910                        (match_operand:VDQIX 2 "s_register_operand" "w")
3911                        (match_operand:SI 3 "immediate_operand" "i")]
3912                       UNSPEC_VQSHL))]
3913   "TARGET_NEON"
3914   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3915   [(set (attr "neon_type")
3916       (if_then_else (match_test "<Is_d_reg>")
3917                     (const_string "neon_shift_2")
3918                     (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3919 )
3920
3921 (define_insn "neon_vshr_n<mode>"
3922   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3923         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3924                        (match_operand:SI 2 "immediate_operand" "i")
3925                        (match_operand:SI 3 "immediate_operand" "i")]
3926                       UNSPEC_VSHR_N))]
3927   "TARGET_NEON"
3928 {
3929   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3930   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3931 }
3932   [(set_attr "neon_type" "neon_shift_1")]
3933 )
3934
3935 (define_insn "neon_vshrn_n<mode>"
3936   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3937         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3938                             (match_operand:SI 2 "immediate_operand" "i")
3939                             (match_operand:SI 3 "immediate_operand" "i")]
3940                            UNSPEC_VSHRN_N))]
3941   "TARGET_NEON"
3942 {
3943   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3944   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3945 }
3946   [(set_attr "neon_type" "neon_shift_1")]
3947 )
3948
3949 (define_insn "neon_vqshrn_n<mode>"
3950   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3951         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3952                             (match_operand:SI 2 "immediate_operand" "i")
3953                             (match_operand:SI 3 "immediate_operand" "i")]
3954                            UNSPEC_VQSHRN_N))]
3955   "TARGET_NEON"
3956 {
3957   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3958   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3959 }
3960   [(set_attr "neon_type" "neon_shift_2")]
3961 )
3962
3963 (define_insn "neon_vqshrun_n<mode>"
3964   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3965         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3966                             (match_operand:SI 2 "immediate_operand" "i")
3967                             (match_operand:SI 3 "immediate_operand" "i")]
3968                            UNSPEC_VQSHRUN_N))]
3969   "TARGET_NEON"
3970 {
3971   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3972   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3973 }
3974   [(set_attr "neon_type" "neon_shift_2")]
3975 )
3976
3977 (define_insn "neon_vshl_n<mode>"
3978   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3979         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3980                        (match_operand:SI 2 "immediate_operand" "i")
3981                        (match_operand:SI 3 "immediate_operand" "i")]
3982                       UNSPEC_VSHL_N))]
3983   "TARGET_NEON"
3984 {
3985   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3986   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3987 }
3988   [(set_attr "neon_type" "neon_shift_1")]
3989 )
3990
3991 (define_insn "neon_vqshl_n<mode>"
3992   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3993         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3994                        (match_operand:SI 2 "immediate_operand" "i")
3995                        (match_operand:SI 3 "immediate_operand" "i")]
3996                       UNSPEC_VQSHL_N))]
3997   "TARGET_NEON"
3998 {
3999   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4000   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4001 }
4002   [(set_attr "neon_type" "neon_shift_2")]
4003 )
4004
4005 (define_insn "neon_vqshlu_n<mode>"
4006   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4007         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
4008                        (match_operand:SI 2 "immediate_operand" "i")
4009                        (match_operand:SI 3 "immediate_operand" "i")]
4010                       UNSPEC_VQSHLU_N))]
4011   "TARGET_NEON"
4012 {
4013   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
4014   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
4015 }
4016   [(set_attr "neon_type" "neon_shift_2")]
4017 )
4018
4019 (define_insn "neon_vshll_n<mode>"
4020   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
4021         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
4022                            (match_operand:SI 2 "immediate_operand" "i")
4023                            (match_operand:SI 3 "immediate_operand" "i")]
4024                           UNSPEC_VSHLL_N))]
4025   "TARGET_NEON"
4026 {
4027   /* The boundaries are: 0 < imm <= size.  */
4028   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
4029   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
4030 }
4031   [(set_attr "neon_type" "neon_shift_1")]
4032 )
4033
4034 (define_insn "neon_vsra_n<mode>"
4035   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4036         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4037                        (match_operand:VDQIX 2 "s_register_operand" "w")
4038                        (match_operand:SI 3 "immediate_operand" "i")
4039                        (match_operand:SI 4 "immediate_operand" "i")]
4040                       UNSPEC_VSRA_N))]
4041   "TARGET_NEON"
4042 {
4043   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4044   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4045 }
4046   [(set_attr "neon_type" "neon_vsra_vrsra")]
4047 )
4048
4049 (define_insn "neon_vsri_n<mode>"
4050   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4051         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4052                        (match_operand:VDQIX 2 "s_register_operand" "w")
4053                        (match_operand:SI 3 "immediate_operand" "i")]
4054                       UNSPEC_VSRI))]
4055   "TARGET_NEON"
4056 {
4057   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
4058   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4059 }
4060   [(set (attr "neon_type")
4061       (if_then_else (match_test "<Is_d_reg>")
4062                     (const_string "neon_shift_1")
4063                     (const_string "neon_shift_3")))]
4064 )
4065
4066 (define_insn "neon_vsli_n<mode>"
4067   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
4068         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
4069                        (match_operand:VDQIX 2 "s_register_operand" "w")
4070                        (match_operand:SI 3 "immediate_operand" "i")]
4071                       UNSPEC_VSLI))]
4072   "TARGET_NEON"
4073 {
4074   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
4075   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
4076 }
4077   [(set (attr "neon_type")
4078       (if_then_else (match_test "<Is_d_reg>")
4079                     (const_string "neon_shift_1")
4080                     (const_string "neon_shift_3")))]
4081 )
4082
4083 (define_insn "neon_vtbl1v8qi"
4084   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4085         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
4086                       (match_operand:V8QI 2 "s_register_operand" "w")]
4087                      UNSPEC_VTBL))]
4088   "TARGET_NEON"
4089   "vtbl.8\t%P0, {%P1}, %P2"
4090   [(set_attr "neon_type" "neon_bp_2cycle")]
4091 )
4092
4093 (define_insn "neon_vtbl2v8qi"
4094   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4095         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
4096                       (match_operand:V8QI 2 "s_register_operand" "w")]
4097                      UNSPEC_VTBL))]
4098   "TARGET_NEON"
4099 {
4100   rtx ops[4];
4101   int tabbase = REGNO (operands[1]);
4102
4103   ops[0] = operands[0];
4104   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4105   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4106   ops[3] = operands[2];
4107   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
4108
4109   return "";
4110 }
4111   [(set_attr "neon_type" "neon_bp_2cycle")]
4112 )
4113
4114 (define_insn "neon_vtbl3v8qi"
4115   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4116         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
4117                       (match_operand:V8QI 2 "s_register_operand" "w")]
4118                      UNSPEC_VTBL))]
4119   "TARGET_NEON"
4120 {
4121   rtx ops[5];
4122   int tabbase = REGNO (operands[1]);
4123
4124   ops[0] = operands[0];
4125   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4126   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4127   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4128   ops[4] = operands[2];
4129   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4130
4131   return "";
4132 }
4133   [(set_attr "neon_type" "neon_bp_3cycle")]
4134 )
4135
4136 (define_insn "neon_vtbl4v8qi"
4137   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4138         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
4139                       (match_operand:V8QI 2 "s_register_operand" "w")]
4140                      UNSPEC_VTBL))]
4141   "TARGET_NEON"
4142 {
4143   rtx ops[6];
4144   int tabbase = REGNO (operands[1]);
4145
4146   ops[0] = operands[0];
4147   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4148   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4149   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4150   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4151   ops[5] = operands[2];
4152   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4153
4154   return "";
4155 }
4156   [(set_attr "neon_type" "neon_bp_3cycle")]
4157 )
4158
4159 ;; These three are used by the vec_perm infrastructure for V16QImode.
4160 (define_insn_and_split "neon_vtbl1v16qi"
4161   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4162         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
4163                        (match_operand:V16QI 2 "s_register_operand" "w")]
4164                       UNSPEC_VTBL))]
4165   "TARGET_NEON"
4166   "#"
4167   "&& reload_completed"
4168   [(const_int 0)]
4169 {
4170   rtx op0, op1, op2, part0, part2;
4171   unsigned ofs;
4172
4173   op0 = operands[0];
4174   op1 = gen_lowpart (TImode, operands[1]);
4175   op2 = operands[2];
4176
4177   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4178   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4179   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4180   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4181
4182   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4183   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4184   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4185   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4186   DONE;
4187 })
4188
4189 (define_insn_and_split "neon_vtbl2v16qi"
4190   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
4191         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
4192                        (match_operand:V16QI 2 "s_register_operand" "w")]
4193                       UNSPEC_VTBL))]
4194   "TARGET_NEON"
4195   "#"
4196   "&& reload_completed"
4197   [(const_int 0)]
4198 {
4199   rtx op0, op1, op2, part0, part2;
4200   unsigned ofs;
4201
4202   op0 = operands[0];
4203   op1 = operands[1];
4204   op2 = operands[2];
4205
4206   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
4207   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4208   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4209   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4210
4211   ofs = subreg_highpart_offset (V8QImode, V16QImode);
4212   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
4213   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4214   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4215   DONE;
4216 })
4217
4218 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4219 ;; handle quad-word input modes, producing octa-word output modes.  But
4220 ;; that requires us to add support for octa-word vector modes in moves.
4221 ;; That seems overkill for this one use in vec_perm.
4222 (define_insn_and_split "neon_vcombinev16qi"
4223   [(set (match_operand:OI 0 "s_register_operand" "=w")
4224         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4225                     (match_operand:V16QI 2 "s_register_operand" "w")]
4226                    UNSPEC_VCONCAT))]
4227   "TARGET_NEON"
4228   "#"
4229   "&& reload_completed"
4230   [(const_int 0)]
4231 {
4232   neon_split_vcombine (operands);
4233   DONE;
4234 })
4235
4236 (define_insn "neon_vtbx1v8qi"
4237   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4238         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4239                       (match_operand:V8QI 2 "s_register_operand" "w")
4240                       (match_operand:V8QI 3 "s_register_operand" "w")]
4241                      UNSPEC_VTBX))]
4242   "TARGET_NEON"
4243   "vtbx.8\t%P0, {%P2}, %P3"
4244   [(set_attr "neon_type" "neon_bp_2cycle")]
4245 )
4246
4247 (define_insn "neon_vtbx2v8qi"
4248   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4249         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4250                       (match_operand:TI 2 "s_register_operand" "w")
4251                       (match_operand:V8QI 3 "s_register_operand" "w")]
4252                      UNSPEC_VTBX))]
4253   "TARGET_NEON"
4254 {
4255   rtx ops[4];
4256   int tabbase = REGNO (operands[2]);
4257
4258   ops[0] = operands[0];
4259   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4260   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4261   ops[3] = operands[3];
4262   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4263
4264   return "";
4265 }
4266   [(set_attr "neon_type" "neon_bp_2cycle")]
4267 )
4268
4269 (define_insn "neon_vtbx3v8qi"
4270   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4271         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4272                       (match_operand:EI 2 "s_register_operand" "w")
4273                       (match_operand:V8QI 3 "s_register_operand" "w")]
4274                      UNSPEC_VTBX))]
4275   "TARGET_NEON"
4276 {
4277   rtx ops[5];
4278   int tabbase = REGNO (operands[2]);
4279
4280   ops[0] = operands[0];
4281   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4282   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4283   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4284   ops[4] = operands[3];
4285   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4286
4287   return "";
4288 }
4289   [(set_attr "neon_type" "neon_bp_3cycle")]
4290 )
4291
4292 (define_insn "neon_vtbx4v8qi"
4293   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4294         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4295                       (match_operand:OI 2 "s_register_operand" "w")
4296                       (match_operand:V8QI 3 "s_register_operand" "w")]
4297                      UNSPEC_VTBX))]
4298   "TARGET_NEON"
4299 {
4300   rtx ops[6];
4301   int tabbase = REGNO (operands[2]);
4302
4303   ops[0] = operands[0];
4304   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4305   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4306   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4307   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4308   ops[5] = operands[3];
4309   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4310
4311   return "";
4312 }
4313   [(set_attr "neon_type" "neon_bp_3cycle")]
4314 )
4315
4316 (define_expand "neon_vtrn<mode>_internal"
4317   [(parallel
4318     [(set (match_operand:VDQW 0 "s_register_operand" "")
4319           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4320                         (match_operand:VDQW 2 "s_register_operand" "")]
4321            UNSPEC_VTRN1))
4322      (set (match_operand:VDQW 3 "s_register_operand" "")
4323           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VTRN2))])]
4324   "TARGET_NEON"
4325   ""
4326 )
4327
4328 ;; Note: Different operand numbering to handle tied registers correctly.
4329 (define_insn "*neon_vtrn<mode>_insn"
4330   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4331         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4332                       (match_operand:VDQW 3 "s_register_operand" "2")]
4333                      UNSPEC_VTRN1))
4334    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4335          (unspec:VDQW [(match_dup 1) (match_dup 3)]
4336                      UNSPEC_VTRN2))]
4337   "TARGET_NEON"
4338   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4339   [(set (attr "neon_type")
4340       (if_then_else (match_test "<Is_d_reg>")
4341                     (const_string "neon_bp_simple")
4342                     (const_string "neon_bp_3cycle")))]
4343 )
4344
4345 (define_expand "neon_vtrn<mode>"
4346   [(match_operand:SI 0 "s_register_operand" "r")
4347    (match_operand:VDQW 1 "s_register_operand" "w")
4348    (match_operand:VDQW 2 "s_register_operand" "w")]
4349   "TARGET_NEON"
4350 {
4351   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4352                               operands[0], operands[1], operands[2]);
4353   DONE;
4354 })
4355
4356 (define_expand "neon_vzip<mode>_internal"
4357   [(parallel
4358     [(set (match_operand:VDQW 0 "s_register_operand" "")
4359           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4360                         (match_operand:VDQW 2 "s_register_operand" "")]
4361                        UNSPEC_VZIP1))
4362     (set (match_operand:VDQW 3 "s_register_operand" "")
4363          (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VZIP2))])]
4364   "TARGET_NEON"
4365   ""
4366 )
4367
4368 ;; Note: Different operand numbering to handle tied registers correctly.
4369 (define_insn "*neon_vzip<mode>_insn"
4370   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4371         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4372                       (match_operand:VDQW 3 "s_register_operand" "2")]
4373                      UNSPEC_VZIP1))
4374    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4375         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4376                      UNSPEC_VZIP2))]
4377   "TARGET_NEON"
4378   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4379   [(set (attr "neon_type")
4380       (if_then_else (match_test "<Is_d_reg>")
4381                     (const_string "neon_bp_simple")
4382                     (const_string "neon_bp_3cycle")))]
4383 )
4384
4385 (define_expand "neon_vzip<mode>"
4386   [(match_operand:SI 0 "s_register_operand" "r")
4387    (match_operand:VDQW 1 "s_register_operand" "w")
4388    (match_operand:VDQW 2 "s_register_operand" "w")]
4389   "TARGET_NEON"
4390 {
4391   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4392                               operands[0], operands[1], operands[2]);
4393   DONE;
4394 })
4395
4396 (define_expand "neon_vuzp<mode>_internal"
4397   [(parallel
4398     [(set (match_operand:VDQW 0 "s_register_operand" "")
4399           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4400                         (match_operand:VDQW 2 "s_register_operand" "")]
4401            UNSPEC_VUZP1))
4402      (set (match_operand:VDQW 3 "s_register_operand" "")
4403           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VUZP2))])]
4404   "TARGET_NEON"
4405   ""
4406 )
4407
4408 ;; Note: Different operand numbering to handle tied registers correctly.
4409 (define_insn "*neon_vuzp<mode>_insn"
4410   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4411         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4412                       (match_operand:VDQW 3 "s_register_operand" "2")]
4413                      UNSPEC_VUZP1))
4414    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4415         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4416                      UNSPEC_VUZP2))]
4417   "TARGET_NEON"
4418   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4419   [(set (attr "neon_type")
4420       (if_then_else (match_test "<Is_d_reg>")
4421                     (const_string "neon_bp_simple")
4422                     (const_string "neon_bp_3cycle")))]
4423 )
4424
4425 (define_expand "neon_vuzp<mode>"
4426   [(match_operand:SI 0 "s_register_operand" "r")
4427    (match_operand:VDQW 1 "s_register_operand" "w")
4428    (match_operand:VDQW 2 "s_register_operand" "w")]
4429   "TARGET_NEON"
4430 {
4431   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4432                               operands[0], operands[1], operands[2]);
4433   DONE;
4434 })
4435
4436 (define_expand "neon_vreinterpretv8qi<mode>"
4437   [(match_operand:V8QI 0 "s_register_operand" "")
4438    (match_operand:VDX 1 "s_register_operand" "")]
4439   "TARGET_NEON"
4440 {
4441   neon_reinterpret (operands[0], operands[1]);
4442   DONE;
4443 })
4444
4445 (define_expand "neon_vreinterpretv4hi<mode>"
4446   [(match_operand:V4HI 0 "s_register_operand" "")
4447    (match_operand:VDX 1 "s_register_operand" "")]
4448   "TARGET_NEON"
4449 {
4450   neon_reinterpret (operands[0], operands[1]);
4451   DONE;
4452 })
4453
4454 (define_expand "neon_vreinterpretv2si<mode>"
4455   [(match_operand:V2SI 0 "s_register_operand" "")
4456    (match_operand:VDX 1 "s_register_operand" "")]
4457   "TARGET_NEON"
4458 {
4459   neon_reinterpret (operands[0], operands[1]);
4460   DONE;
4461 })
4462
4463 (define_expand "neon_vreinterpretv2sf<mode>"
4464   [(match_operand:V2SF 0 "s_register_operand" "")
4465    (match_operand:VDX 1 "s_register_operand" "")]
4466   "TARGET_NEON"
4467 {
4468   neon_reinterpret (operands[0], operands[1]);
4469   DONE;
4470 })
4471
4472 (define_expand "neon_vreinterpretdi<mode>"
4473   [(match_operand:DI 0 "s_register_operand" "")
4474    (match_operand:VDX 1 "s_register_operand" "")]
4475   "TARGET_NEON"
4476 {
4477   neon_reinterpret (operands[0], operands[1]);
4478   DONE;
4479 })
4480
4481 (define_expand "neon_vreinterpretv16qi<mode>"
4482   [(match_operand:V16QI 0 "s_register_operand" "")
4483    (match_operand:VQX 1 "s_register_operand" "")]
4484   "TARGET_NEON"
4485 {
4486   neon_reinterpret (operands[0], operands[1]);
4487   DONE;
4488 })
4489
4490 (define_expand "neon_vreinterpretv8hi<mode>"
4491   [(match_operand:V8HI 0 "s_register_operand" "")
4492    (match_operand:VQX 1 "s_register_operand" "")]
4493   "TARGET_NEON"
4494 {
4495   neon_reinterpret (operands[0], operands[1]);
4496   DONE;
4497 })
4498
4499 (define_expand "neon_vreinterpretv4si<mode>"
4500   [(match_operand:V4SI 0 "s_register_operand" "")
4501    (match_operand:VQX 1 "s_register_operand" "")]
4502   "TARGET_NEON"
4503 {
4504   neon_reinterpret (operands[0], operands[1]);
4505   DONE;
4506 })
4507
4508 (define_expand "neon_vreinterpretv4sf<mode>"
4509   [(match_operand:V4SF 0 "s_register_operand" "")
4510    (match_operand:VQX 1 "s_register_operand" "")]
4511   "TARGET_NEON"
4512 {
4513   neon_reinterpret (operands[0], operands[1]);
4514   DONE;
4515 })
4516
4517 (define_expand "neon_vreinterpretv2di<mode>"
4518   [(match_operand:V2DI 0 "s_register_operand" "")
4519    (match_operand:VQX 1 "s_register_operand" "")]
4520   "TARGET_NEON"
4521 {
4522   neon_reinterpret (operands[0], operands[1]);
4523   DONE;
4524 })
4525
4526 (define_expand "vec_load_lanes<mode><mode>"
4527   [(set (match_operand:VDQX 0 "s_register_operand")
4528         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4529                      UNSPEC_VLD1))]
4530   "TARGET_NEON")
4531
4532 (define_insn "neon_vld1<mode>"
4533   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4534         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4535                     UNSPEC_VLD1))]
4536   "TARGET_NEON"
4537   "vld1.<V_sz_elem>\t%h0, %A1"
4538   [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4539 )
4540
4541 (define_insn "neon_vld1_lane<mode>"
4542   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4543         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4544                      (match_operand:VDX 2 "s_register_operand" "0")
4545                      (match_operand:SI 3 "immediate_operand" "i")]
4546                     UNSPEC_VLD1_LANE))]
4547   "TARGET_NEON"
4548 {
4549   HOST_WIDE_INT lane = INTVAL (operands[3]);
4550   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4551   if (lane < 0 || lane >= max)
4552     error ("lane out of range");
4553   if (max == 1)
4554     return "vld1.<V_sz_elem>\t%P0, %A1";
4555   else
4556     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4557 }
4558   [(set (attr "neon_type")
4559       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4560                     (const_string "neon_vld1_1_2_regs")
4561                     (const_string "neon_vld1_vld2_lane")))]
4562 )
4563
4564 (define_insn "neon_vld1_lane<mode>"
4565   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4566         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4567                      (match_operand:VQX 2 "s_register_operand" "0")
4568                      (match_operand:SI 3 "immediate_operand" "i")]
4569                     UNSPEC_VLD1_LANE))]
4570   "TARGET_NEON"
4571 {
4572   HOST_WIDE_INT lane = INTVAL (operands[3]);
4573   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4574   int regno = REGNO (operands[0]);
4575   if (lane < 0 || lane >= max)
4576     error ("lane out of range");
4577   else if (lane >= max / 2)
4578     {
4579       lane -= max / 2;
4580       regno += 2;
4581       operands[3] = GEN_INT (lane);
4582     }
4583   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4584   if (max == 2)
4585     return "vld1.<V_sz_elem>\t%P0, %A1";
4586   else
4587     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4588 }
4589   [(set (attr "neon_type")
4590       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4591                     (const_string "neon_vld1_1_2_regs")
4592                     (const_string "neon_vld1_vld2_lane")))]
4593 )
4594
4595 (define_insn "neon_vld1_dup<mode>"
4596   [(set (match_operand:VD 0 "s_register_operand" "=w")
4597         (vec_duplicate:VD (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4598   "TARGET_NEON"
4599   "vld1.<V_sz_elem>\t{%P0[]}, %A1"
4600   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4601 )
4602
4603 ;; Special case for DImode.  Treat it exactly like a simple load.
4604 (define_expand "neon_vld1_dupdi"
4605   [(set (match_operand:DI 0 "s_register_operand" "")
4606         (unspec:DI [(match_operand:DI 1 "neon_struct_operand" "")]
4607                    UNSPEC_VLD1))]
4608   "TARGET_NEON"
4609   ""
4610 )
4611
4612 (define_insn "neon_vld1_dup<mode>"
4613   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4614         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4615   "TARGET_NEON"
4616 {
4617   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4618 }
4619   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4620 )
4621
4622 (define_insn_and_split "neon_vld1_dupv2di"
4623    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4624     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4625    "TARGET_NEON"
4626    "#"
4627    "&& reload_completed"
4628    [(const_int 0)]
4629    {
4630     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4631     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4632     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4633     DONE;
4634     }
4635   [(set_attr "length" "8")
4636    (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]
4637 )
4638
4639 (define_expand "vec_store_lanes<mode><mode>"
4640   [(set (match_operand:VDQX 0 "neon_struct_operand")
4641         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4642                      UNSPEC_VST1))]
4643   "TARGET_NEON")
4644
4645 (define_insn "neon_vst1<mode>"
4646   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4647         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4648                      UNSPEC_VST1))]
4649   "TARGET_NEON"
4650   "vst1.<V_sz_elem>\t%h1, %A0"
4651   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4652
4653 (define_insn "neon_vst1_lane<mode>"
4654   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4655         (unspec:<V_elem>
4656           [(match_operand:VDX 1 "s_register_operand" "w")
4657            (match_operand:SI 2 "immediate_operand" "i")]
4658           UNSPEC_VST1_LANE))]
4659   "TARGET_NEON"
4660 {
4661   HOST_WIDE_INT lane = INTVAL (operands[2]);
4662   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4663   if (lane < 0 || lane >= max)
4664     error ("lane out of range");
4665   if (max == 1)
4666     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4667   else
4668     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4669 }
4670   [(set (attr "neon_type")
4671       (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4672                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4673                     (const_string "neon_vst1_vst2_lane")))])
4674
4675 (define_insn "neon_vst1_lane<mode>"
4676   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4677         (unspec:<V_elem>
4678           [(match_operand:VQX 1 "s_register_operand" "w")
4679            (match_operand:SI 2 "immediate_operand" "i")]
4680           UNSPEC_VST1_LANE))]
4681   "TARGET_NEON"
4682 {
4683   HOST_WIDE_INT lane = INTVAL (operands[2]);
4684   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4685   int regno = REGNO (operands[1]);
4686   if (lane < 0 || lane >= max)
4687     error ("lane out of range");
4688   else if (lane >= max / 2)
4689     {
4690       lane -= max / 2;
4691       regno += 2;
4692       operands[2] = GEN_INT (lane);
4693     }
4694   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4695   if (max == 2)
4696     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4697   else
4698     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4699 }
4700   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4701 )
4702
4703 (define_expand "vec_load_lanesti<mode>"
4704   [(set (match_operand:TI 0 "s_register_operand")
4705         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4706                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4707                    UNSPEC_VLD2))]
4708   "TARGET_NEON")
4709
4710 (define_insn "neon_vld2<mode>"
4711   [(set (match_operand:TI 0 "s_register_operand" "=w")
4712         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4713                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4714                    UNSPEC_VLD2))]
4715   "TARGET_NEON"
4716 {
4717   if (<V_sz_elem> == 64)
4718     return "vld1.64\t%h0, %A1";
4719   else
4720     return "vld2.<V_sz_elem>\t%h0, %A1";
4721 }
4722   [(set (attr "neon_type")
4723       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4724                     (const_string "neon_vld1_1_2_regs")
4725                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4726 )
4727
4728 (define_expand "vec_load_lanesoi<mode>"
4729   [(set (match_operand:OI 0 "s_register_operand")
4730         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4731                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4732                    UNSPEC_VLD2))]
4733   "TARGET_NEON")
4734
4735 (define_insn "neon_vld2<mode>"
4736   [(set (match_operand:OI 0 "s_register_operand" "=w")
4737         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4738                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4739                    UNSPEC_VLD2))]
4740   "TARGET_NEON"
4741   "vld2.<V_sz_elem>\t%h0, %A1"
4742   [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4743
4744 (define_insn "neon_vld2_lane<mode>"
4745   [(set (match_operand:TI 0 "s_register_operand" "=w")
4746         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4747                     (match_operand:TI 2 "s_register_operand" "0")
4748                     (match_operand:SI 3 "immediate_operand" "i")
4749                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4750                    UNSPEC_VLD2_LANE))]
4751   "TARGET_NEON"
4752 {
4753   HOST_WIDE_INT lane = INTVAL (operands[3]);
4754   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4755   int regno = REGNO (operands[0]);
4756   rtx ops[4];
4757   if (lane < 0 || lane >= max)
4758     error ("lane out of range");
4759   ops[0] = gen_rtx_REG (DImode, regno);
4760   ops[1] = gen_rtx_REG (DImode, regno + 2);
4761   ops[2] = operands[1];
4762   ops[3] = operands[3];
4763   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4764   return "";
4765 }
4766   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4767 )
4768
4769 (define_insn "neon_vld2_lane<mode>"
4770   [(set (match_operand:OI 0 "s_register_operand" "=w")
4771         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4772                     (match_operand:OI 2 "s_register_operand" "0")
4773                     (match_operand:SI 3 "immediate_operand" "i")
4774                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4775                    UNSPEC_VLD2_LANE))]
4776   "TARGET_NEON"
4777 {
4778   HOST_WIDE_INT lane = INTVAL (operands[3]);
4779   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4780   int regno = REGNO (operands[0]);
4781   rtx ops[4];
4782   if (lane < 0 || lane >= max)
4783     error ("lane out of range");
4784   else if (lane >= max / 2)
4785     {
4786       lane -= max / 2;
4787       regno += 2;
4788     }
4789   ops[0] = gen_rtx_REG (DImode, regno);
4790   ops[1] = gen_rtx_REG (DImode, regno + 4);
4791   ops[2] = operands[1];
4792   ops[3] = GEN_INT (lane);
4793   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4794   return "";
4795 }
4796   [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4797 )
4798
4799 (define_insn "neon_vld2_dup<mode>"
4800   [(set (match_operand:TI 0 "s_register_operand" "=w")
4801         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4802                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4803                    UNSPEC_VLD2_DUP))]
4804   "TARGET_NEON"
4805 {
4806   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4807     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4808   else
4809     return "vld1.<V_sz_elem>\t%h0, %A1";
4810 }
4811   [(set (attr "neon_type")
4812       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4813                     (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4814                     (const_string "neon_vld1_1_2_regs")))]
4815 )
4816
4817 (define_expand "vec_store_lanesti<mode>"
4818   [(set (match_operand:TI 0 "neon_struct_operand")
4819         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4820                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4821                    UNSPEC_VST2))]
4822   "TARGET_NEON")
4823
4824 (define_insn "neon_vst2<mode>"
4825   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4826         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4827                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4828                    UNSPEC_VST2))]
4829   "TARGET_NEON"
4830 {
4831   if (<V_sz_elem> == 64)
4832     return "vst1.64\t%h1, %A0";
4833   else
4834     return "vst2.<V_sz_elem>\t%h1, %A0";
4835 }
4836   [(set (attr "neon_type")
4837       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4838                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4839                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4840 )
4841
4842 (define_expand "vec_store_lanesoi<mode>"
4843   [(set (match_operand:OI 0 "neon_struct_operand")
4844         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4845                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4846                    UNSPEC_VST2))]
4847   "TARGET_NEON")
4848
4849 (define_insn "neon_vst2<mode>"
4850   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4851         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4852                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4853                    UNSPEC_VST2))]
4854   "TARGET_NEON"
4855   "vst2.<V_sz_elem>\t%h1, %A0"
4856   [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4857 )
4858
4859 (define_insn "neon_vst2_lane<mode>"
4860   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4861         (unspec:<V_two_elem>
4862           [(match_operand:TI 1 "s_register_operand" "w")
4863            (match_operand:SI 2 "immediate_operand" "i")
4864            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4865           UNSPEC_VST2_LANE))]
4866   "TARGET_NEON"
4867 {
4868   HOST_WIDE_INT lane = INTVAL (operands[2]);
4869   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4870   int regno = REGNO (operands[1]);
4871   rtx ops[4];
4872   if (lane < 0 || lane >= max)
4873     error ("lane out of range");
4874   ops[0] = operands[0];
4875   ops[1] = gen_rtx_REG (DImode, regno);
4876   ops[2] = gen_rtx_REG (DImode, regno + 2);
4877   ops[3] = operands[2];
4878   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4879   return "";
4880 }
4881   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4882 )
4883
4884 (define_insn "neon_vst2_lane<mode>"
4885   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4886         (unspec:<V_two_elem>
4887            [(match_operand:OI 1 "s_register_operand" "w")
4888             (match_operand:SI 2 "immediate_operand" "i")
4889             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4890            UNSPEC_VST2_LANE))]
4891   "TARGET_NEON"
4892 {
4893   HOST_WIDE_INT lane = INTVAL (operands[2]);
4894   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4895   int regno = REGNO (operands[1]);
4896   rtx ops[4];
4897   if (lane < 0 || lane >= max)
4898     error ("lane out of range");
4899   else if (lane >= max / 2)
4900     {
4901       lane -= max / 2;
4902       regno += 2;
4903     }
4904   ops[0] = operands[0];
4905   ops[1] = gen_rtx_REG (DImode, regno);
4906   ops[2] = gen_rtx_REG (DImode, regno + 4);
4907   ops[3] = GEN_INT (lane);
4908   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4909   return "";
4910 }
4911   [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4912 )
4913
4914 (define_expand "vec_load_lanesei<mode>"
4915   [(set (match_operand:EI 0 "s_register_operand")
4916         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4917                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4918                    UNSPEC_VLD3))]
4919   "TARGET_NEON")
4920
4921 (define_insn "neon_vld3<mode>"
4922   [(set (match_operand:EI 0 "s_register_operand" "=w")
4923         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4924                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4925                    UNSPEC_VLD3))]
4926   "TARGET_NEON"
4927 {
4928   if (<V_sz_elem> == 64)
4929     return "vld1.64\t%h0, %A1";
4930   else
4931     return "vld3.<V_sz_elem>\t%h0, %A1";
4932 }
4933   [(set (attr "neon_type")
4934       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4935                     (const_string "neon_vld1_1_2_regs")
4936                     (const_string "neon_vld3_vld4")))]
4937 )
4938
4939 (define_expand "vec_load_lanesci<mode>"
4940   [(match_operand:CI 0 "s_register_operand")
4941    (match_operand:CI 1 "neon_struct_operand")
4942    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4943   "TARGET_NEON"
4944 {
4945   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4946   DONE;
4947 })
4948
4949 (define_expand "neon_vld3<mode>"
4950   [(match_operand:CI 0 "s_register_operand")
4951    (match_operand:CI 1 "neon_struct_operand")
4952    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4953   "TARGET_NEON"
4954 {
4955   rtx mem;
4956
4957   mem = adjust_address (operands[1], EImode, 0);
4958   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4959   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4960   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4961   DONE;
4962 })
4963
4964 (define_insn "neon_vld3qa<mode>"
4965   [(set (match_operand:CI 0 "s_register_operand" "=w")
4966         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4967                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4968                    UNSPEC_VLD3A))]
4969   "TARGET_NEON"
4970 {
4971   int regno = REGNO (operands[0]);
4972   rtx ops[4];
4973   ops[0] = gen_rtx_REG (DImode, regno);
4974   ops[1] = gen_rtx_REG (DImode, regno + 4);
4975   ops[2] = gen_rtx_REG (DImode, regno + 8);
4976   ops[3] = operands[1];
4977   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4978   return "";
4979 }
4980   [(set_attr "neon_type" "neon_vld3_vld4")]
4981 )
4982
4983 (define_insn "neon_vld3qb<mode>"
4984   [(set (match_operand:CI 0 "s_register_operand" "=w")
4985         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4986                     (match_operand:CI 2 "s_register_operand" "0")
4987                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4988                    UNSPEC_VLD3B))]
4989   "TARGET_NEON"
4990 {
4991   int regno = REGNO (operands[0]);
4992   rtx ops[4];
4993   ops[0] = gen_rtx_REG (DImode, regno + 2);
4994   ops[1] = gen_rtx_REG (DImode, regno + 6);
4995   ops[2] = gen_rtx_REG (DImode, regno + 10);
4996   ops[3] = operands[1];
4997   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4998   return "";
4999 }
5000   [(set_attr "neon_type" "neon_vld3_vld4")]
5001 )
5002
5003 (define_insn "neon_vld3_lane<mode>"
5004   [(set (match_operand:EI 0 "s_register_operand" "=w")
5005         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5006                     (match_operand:EI 2 "s_register_operand" "0")
5007                     (match_operand:SI 3 "immediate_operand" "i")
5008                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5009                    UNSPEC_VLD3_LANE))]
5010   "TARGET_NEON"
5011 {
5012   HOST_WIDE_INT lane = INTVAL (operands[3]);
5013   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5014   int regno = REGNO (operands[0]);
5015   rtx ops[5];
5016   if (lane < 0 || lane >= max)
5017     error ("lane out of range");
5018   ops[0] = gen_rtx_REG (DImode, regno);
5019   ops[1] = gen_rtx_REG (DImode, regno + 2);
5020   ops[2] = gen_rtx_REG (DImode, regno + 4);
5021   ops[3] = operands[1];
5022   ops[4] = operands[3];
5023   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5024                    ops);
5025   return "";
5026 }
5027   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5028 )
5029
5030 (define_insn "neon_vld3_lane<mode>"
5031   [(set (match_operand:CI 0 "s_register_operand" "=w")
5032         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5033                     (match_operand:CI 2 "s_register_operand" "0")
5034                     (match_operand:SI 3 "immediate_operand" "i")
5035                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5036                    UNSPEC_VLD3_LANE))]
5037   "TARGET_NEON"
5038 {
5039   HOST_WIDE_INT lane = INTVAL (operands[3]);
5040   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5041   int regno = REGNO (operands[0]);
5042   rtx ops[5];
5043   if (lane < 0 || lane >= max)
5044     error ("lane out of range");
5045   else if (lane >= max / 2)
5046     {
5047       lane -= max / 2;
5048       regno += 2;
5049     }
5050   ops[0] = gen_rtx_REG (DImode, regno);
5051   ops[1] = gen_rtx_REG (DImode, regno + 4);
5052   ops[2] = gen_rtx_REG (DImode, regno + 8);
5053   ops[3] = operands[1];
5054   ops[4] = GEN_INT (lane);
5055   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
5056                    ops);
5057   return "";
5058 }
5059   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5060 )
5061
5062 (define_insn "neon_vld3_dup<mode>"
5063   [(set (match_operand:EI 0 "s_register_operand" "=w")
5064         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
5065                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5066                    UNSPEC_VLD3_DUP))]
5067   "TARGET_NEON"
5068 {
5069   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5070     {
5071       int regno = REGNO (operands[0]);
5072       rtx ops[4];
5073       ops[0] = gen_rtx_REG (DImode, regno);
5074       ops[1] = gen_rtx_REG (DImode, regno + 2);
5075       ops[2] = gen_rtx_REG (DImode, regno + 4);
5076       ops[3] = operands[1];
5077       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
5078       return "";
5079     }
5080   else
5081     return "vld1.<V_sz_elem>\t%h0, %A1";
5082 }
5083   [(set (attr "neon_type")
5084       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5085                     (const_string "neon_vld3_vld4_all_lanes")
5086                     (const_string "neon_vld1_1_2_regs")))])
5087
5088 (define_expand "vec_store_lanesei<mode>"
5089   [(set (match_operand:EI 0 "neon_struct_operand")
5090         (unspec:EI [(match_operand:EI 1 "s_register_operand")
5091                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5092                    UNSPEC_VST3))]
5093   "TARGET_NEON")
5094
5095 (define_insn "neon_vst3<mode>"
5096   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5097         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
5098                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5099                    UNSPEC_VST3))]
5100   "TARGET_NEON"
5101 {
5102   if (<V_sz_elem> == 64)
5103     return "vst1.64\t%h1, %A0";
5104   else
5105     return "vst3.<V_sz_elem>\t%h1, %A0";
5106 }
5107   [(set (attr "neon_type")
5108       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5109                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5110                     (const_string "neon_vst2_4_regs_vst3_vst4")))])
5111
5112 (define_expand "vec_store_lanesci<mode>"
5113   [(match_operand:CI 0 "neon_struct_operand")
5114    (match_operand:CI 1 "s_register_operand")
5115    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5116   "TARGET_NEON"
5117 {
5118   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
5119   DONE;
5120 })
5121
5122 (define_expand "neon_vst3<mode>"
5123   [(match_operand:CI 0 "neon_struct_operand")
5124    (match_operand:CI 1 "s_register_operand")
5125    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5126   "TARGET_NEON"
5127 {
5128   rtx mem;
5129
5130   mem = adjust_address (operands[0], EImode, 0);
5131   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
5132   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
5133   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
5134   DONE;
5135 })
5136
5137 (define_insn "neon_vst3qa<mode>"
5138   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5139         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5140                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5141                    UNSPEC_VST3A))]
5142   "TARGET_NEON"
5143 {
5144   int regno = REGNO (operands[1]);
5145   rtx ops[4];
5146   ops[0] = operands[0];
5147   ops[1] = gen_rtx_REG (DImode, regno);
5148   ops[2] = gen_rtx_REG (DImode, regno + 4);
5149   ops[3] = gen_rtx_REG (DImode, regno + 8);
5150   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5151   return "";
5152 }
5153   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5154 )
5155
5156 (define_insn "neon_vst3qb<mode>"
5157   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
5158         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
5159                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5160                    UNSPEC_VST3B))]
5161   "TARGET_NEON"
5162 {
5163   int regno = REGNO (operands[1]);
5164   rtx ops[4];
5165   ops[0] = operands[0];
5166   ops[1] = gen_rtx_REG (DImode, regno + 2);
5167   ops[2] = gen_rtx_REG (DImode, regno + 6);
5168   ops[3] = gen_rtx_REG (DImode, regno + 10);
5169   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
5170   return "";
5171 }
5172   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5173 )
5174
5175 (define_insn "neon_vst3_lane<mode>"
5176   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5177         (unspec:<V_three_elem>
5178            [(match_operand:EI 1 "s_register_operand" "w")
5179             (match_operand:SI 2 "immediate_operand" "i")
5180             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5181            UNSPEC_VST3_LANE))]
5182   "TARGET_NEON"
5183 {
5184   HOST_WIDE_INT lane = INTVAL (operands[2]);
5185   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5186   int regno = REGNO (operands[1]);
5187   rtx ops[5];
5188   if (lane < 0 || lane >= max)
5189     error ("lane out of range");
5190   ops[0] = operands[0];
5191   ops[1] = gen_rtx_REG (DImode, regno);
5192   ops[2] = gen_rtx_REG (DImode, regno + 2);
5193   ops[3] = gen_rtx_REG (DImode, regno + 4);
5194   ops[4] = operands[2];
5195   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5196                    ops);
5197   return "";
5198 }
5199   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5200 )
5201
5202 (define_insn "neon_vst3_lane<mode>"
5203   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
5204         (unspec:<V_three_elem>
5205            [(match_operand:CI 1 "s_register_operand" "w")
5206             (match_operand:SI 2 "immediate_operand" "i")
5207             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5208            UNSPEC_VST3_LANE))]
5209   "TARGET_NEON"
5210 {
5211   HOST_WIDE_INT lane = INTVAL (operands[2]);
5212   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5213   int regno = REGNO (operands[1]);
5214   rtx ops[5];
5215   if (lane < 0 || lane >= max)
5216     error ("lane out of range");
5217   else if (lane >= max / 2)
5218     {
5219       lane -= max / 2;
5220       regno += 2;
5221     }
5222   ops[0] = operands[0];
5223   ops[1] = gen_rtx_REG (DImode, regno);
5224   ops[2] = gen_rtx_REG (DImode, regno + 4);
5225   ops[3] = gen_rtx_REG (DImode, regno + 8);
5226   ops[4] = GEN_INT (lane);
5227   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5228                    ops);
5229   return "";
5230 }
5231 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
5232
5233 (define_expand "vec_load_lanesoi<mode>"
5234   [(set (match_operand:OI 0 "s_register_operand")
5235         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5236                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5237                    UNSPEC_VLD4))]
5238   "TARGET_NEON")
5239
5240 (define_insn "neon_vld4<mode>"
5241   [(set (match_operand:OI 0 "s_register_operand" "=w")
5242         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5243                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5244                    UNSPEC_VLD4))]
5245   "TARGET_NEON"
5246 {
5247   if (<V_sz_elem> == 64)
5248     return "vld1.64\t%h0, %A1";
5249   else
5250     return "vld4.<V_sz_elem>\t%h0, %A1";
5251 }
5252   [(set (attr "neon_type")
5253       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5254                     (const_string "neon_vld1_1_2_regs")
5255                     (const_string "neon_vld3_vld4")))]
5256 )
5257
5258 (define_expand "vec_load_lanesxi<mode>"
5259   [(match_operand:XI 0 "s_register_operand")
5260    (match_operand:XI 1 "neon_struct_operand")
5261    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5262   "TARGET_NEON"
5263 {
5264   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5265   DONE;
5266 })
5267
5268 (define_expand "neon_vld4<mode>"
5269   [(match_operand:XI 0 "s_register_operand")
5270    (match_operand:XI 1 "neon_struct_operand")
5271    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5272   "TARGET_NEON"
5273 {
5274   rtx mem;
5275
5276   mem = adjust_address (operands[1], OImode, 0);
5277   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5278   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5279   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5280   DONE;
5281 })
5282
5283 (define_insn "neon_vld4qa<mode>"
5284   [(set (match_operand:XI 0 "s_register_operand" "=w")
5285         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5286                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5287                    UNSPEC_VLD4A))]
5288   "TARGET_NEON"
5289 {
5290   int regno = REGNO (operands[0]);
5291   rtx ops[5];
5292   ops[0] = gen_rtx_REG (DImode, regno);
5293   ops[1] = gen_rtx_REG (DImode, regno + 4);
5294   ops[2] = gen_rtx_REG (DImode, regno + 8);
5295   ops[3] = gen_rtx_REG (DImode, regno + 12);
5296   ops[4] = operands[1];
5297   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5298   return "";
5299 }
5300   [(set_attr "neon_type" "neon_vld3_vld4")]
5301 )
5302
5303 (define_insn "neon_vld4qb<mode>"
5304   [(set (match_operand:XI 0 "s_register_operand" "=w")
5305         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5306                     (match_operand:XI 2 "s_register_operand" "0")
5307                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5308                    UNSPEC_VLD4B))]
5309   "TARGET_NEON"
5310 {
5311   int regno = REGNO (operands[0]);
5312   rtx ops[5];
5313   ops[0] = gen_rtx_REG (DImode, regno + 2);
5314   ops[1] = gen_rtx_REG (DImode, regno + 6);
5315   ops[2] = gen_rtx_REG (DImode, regno + 10);
5316   ops[3] = gen_rtx_REG (DImode, regno + 14);
5317   ops[4] = operands[1];
5318   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5319   return "";
5320 }
5321   [(set_attr "neon_type" "neon_vld3_vld4")]
5322 )
5323
5324 (define_insn "neon_vld4_lane<mode>"
5325   [(set (match_operand:OI 0 "s_register_operand" "=w")
5326         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5327                     (match_operand:OI 2 "s_register_operand" "0")
5328                     (match_operand:SI 3 "immediate_operand" "i")
5329                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5330                    UNSPEC_VLD4_LANE))]
5331   "TARGET_NEON"
5332 {
5333   HOST_WIDE_INT lane = INTVAL (operands[3]);
5334   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5335   int regno = REGNO (operands[0]);
5336   rtx ops[6];
5337   if (lane < 0 || lane >= max)
5338     error ("lane out of range");
5339   ops[0] = gen_rtx_REG (DImode, regno);
5340   ops[1] = gen_rtx_REG (DImode, regno + 2);
5341   ops[2] = gen_rtx_REG (DImode, regno + 4);
5342   ops[3] = gen_rtx_REG (DImode, regno + 6);
5343   ops[4] = operands[1];
5344   ops[5] = operands[3];
5345   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5346                    ops);
5347   return "";
5348 }
5349   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5350 )
5351
5352 (define_insn "neon_vld4_lane<mode>"
5353   [(set (match_operand:XI 0 "s_register_operand" "=w")
5354         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5355                     (match_operand:XI 2 "s_register_operand" "0")
5356                     (match_operand:SI 3 "immediate_operand" "i")
5357                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5358                    UNSPEC_VLD4_LANE))]
5359   "TARGET_NEON"
5360 {
5361   HOST_WIDE_INT lane = INTVAL (operands[3]);
5362   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5363   int regno = REGNO (operands[0]);
5364   rtx ops[6];
5365   if (lane < 0 || lane >= max)
5366     error ("lane out of range");
5367   else if (lane >= max / 2)
5368     {
5369       lane -= max / 2;
5370       regno += 2;
5371     }
5372   ops[0] = gen_rtx_REG (DImode, regno);
5373   ops[1] = gen_rtx_REG (DImode, regno + 4);
5374   ops[2] = gen_rtx_REG (DImode, regno + 8);
5375   ops[3] = gen_rtx_REG (DImode, regno + 12);
5376   ops[4] = operands[1];
5377   ops[5] = GEN_INT (lane);
5378   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5379                    ops);
5380   return "";
5381 }
5382   [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5383 )
5384
5385 (define_insn "neon_vld4_dup<mode>"
5386   [(set (match_operand:OI 0 "s_register_operand" "=w")
5387         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5388                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5389                    UNSPEC_VLD4_DUP))]
5390   "TARGET_NEON"
5391 {
5392   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5393     {
5394       int regno = REGNO (operands[0]);
5395       rtx ops[5];
5396       ops[0] = gen_rtx_REG (DImode, regno);
5397       ops[1] = gen_rtx_REG (DImode, regno + 2);
5398       ops[2] = gen_rtx_REG (DImode, regno + 4);
5399       ops[3] = gen_rtx_REG (DImode, regno + 6);
5400       ops[4] = operands[1];
5401       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5402                        ops);
5403       return "";
5404     }
5405   else
5406     return "vld1.<V_sz_elem>\t%h0, %A1";
5407 }
5408   [(set (attr "neon_type")
5409       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5410                     (const_string "neon_vld3_vld4_all_lanes")
5411                     (const_string "neon_vld1_1_2_regs")))]
5412 )
5413
5414 (define_expand "vec_store_lanesoi<mode>"
5415   [(set (match_operand:OI 0 "neon_struct_operand")
5416         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5417                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5418                    UNSPEC_VST4))]
5419   "TARGET_NEON")
5420
5421 (define_insn "neon_vst4<mode>"
5422   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5423         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5424                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5425                    UNSPEC_VST4))]
5426   "TARGET_NEON"
5427 {
5428   if (<V_sz_elem> == 64)
5429     return "vst1.64\t%h1, %A0";
5430   else
5431     return "vst4.<V_sz_elem>\t%h1, %A0";
5432 }
5433   [(set (attr "neon_type")
5434       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5435                     (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5436                     (const_string "neon_vst2_4_regs_vst3_vst4")))]
5437 )
5438
5439 (define_expand "vec_store_lanesxi<mode>"
5440   [(match_operand:XI 0 "neon_struct_operand")
5441    (match_operand:XI 1 "s_register_operand")
5442    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5443   "TARGET_NEON"
5444 {
5445   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5446   DONE;
5447 })
5448
5449 (define_expand "neon_vst4<mode>"
5450   [(match_operand:XI 0 "neon_struct_operand")
5451    (match_operand:XI 1 "s_register_operand")
5452    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5453   "TARGET_NEON"
5454 {
5455   rtx mem;
5456
5457   mem = adjust_address (operands[0], OImode, 0);
5458   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5459   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5460   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5461   DONE;
5462 })
5463
5464 (define_insn "neon_vst4qa<mode>"
5465   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5466         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5467                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5468                    UNSPEC_VST4A))]
5469   "TARGET_NEON"
5470 {
5471   int regno = REGNO (operands[1]);
5472   rtx ops[5];
5473   ops[0] = operands[0];
5474   ops[1] = gen_rtx_REG (DImode, regno);
5475   ops[2] = gen_rtx_REG (DImode, regno + 4);
5476   ops[3] = gen_rtx_REG (DImode, regno + 8);
5477   ops[4] = gen_rtx_REG (DImode, regno + 12);
5478   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5479   return "";
5480 }
5481   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5482 )
5483
5484 (define_insn "neon_vst4qb<mode>"
5485   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5486         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5487                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5488                    UNSPEC_VST4B))]
5489   "TARGET_NEON"
5490 {
5491   int regno = REGNO (operands[1]);
5492   rtx ops[5];
5493   ops[0] = operands[0];
5494   ops[1] = gen_rtx_REG (DImode, regno + 2);
5495   ops[2] = gen_rtx_REG (DImode, regno + 6);
5496   ops[3] = gen_rtx_REG (DImode, regno + 10);
5497   ops[4] = gen_rtx_REG (DImode, regno + 14);
5498   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5499   return "";
5500 }
5501   [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5502 )
5503
5504 (define_insn "neon_vst4_lane<mode>"
5505   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5506         (unspec:<V_four_elem>
5507            [(match_operand:OI 1 "s_register_operand" "w")
5508             (match_operand:SI 2 "immediate_operand" "i")
5509             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5510            UNSPEC_VST4_LANE))]
5511   "TARGET_NEON"
5512 {
5513   HOST_WIDE_INT lane = INTVAL (operands[2]);
5514   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5515   int regno = REGNO (operands[1]);
5516   rtx ops[6];
5517   if (lane < 0 || lane >= max)
5518     error ("lane out of range");
5519   ops[0] = operands[0];
5520   ops[1] = gen_rtx_REG (DImode, regno);
5521   ops[2] = gen_rtx_REG (DImode, regno + 2);
5522   ops[3] = gen_rtx_REG (DImode, regno + 4);
5523   ops[4] = gen_rtx_REG (DImode, regno + 6);
5524   ops[5] = operands[2];
5525   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5526                    ops);
5527   return "";
5528 }
5529   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5530 )
5531
5532 (define_insn "neon_vst4_lane<mode>"
5533   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5534         (unspec:<V_four_elem>
5535            [(match_operand:XI 1 "s_register_operand" "w")
5536             (match_operand:SI 2 "immediate_operand" "i")
5537             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5538            UNSPEC_VST4_LANE))]
5539   "TARGET_NEON"
5540 {
5541   HOST_WIDE_INT lane = INTVAL (operands[2]);
5542   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5543   int regno = REGNO (operands[1]);
5544   rtx ops[6];
5545   if (lane < 0 || lane >= max)
5546     error ("lane out of range");
5547   else if (lane >= max / 2)
5548     {
5549       lane -= max / 2;
5550       regno += 2;
5551     }
5552   ops[0] = operands[0];
5553   ops[1] = gen_rtx_REG (DImode, regno);
5554   ops[2] = gen_rtx_REG (DImode, regno + 4);
5555   ops[3] = gen_rtx_REG (DImode, regno + 8);
5556   ops[4] = gen_rtx_REG (DImode, regno + 12);
5557   ops[5] = GEN_INT (lane);
5558   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5559                    ops);
5560   return "";
5561 }
5562   [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5563 )
5564
5565 (define_expand "neon_vand<mode>"
5566   [(match_operand:VDQX 0 "s_register_operand" "")
5567    (match_operand:VDQX 1 "s_register_operand" "")
5568    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5569    (match_operand:SI 3 "immediate_operand" "")]
5570   "TARGET_NEON"
5571 {
5572   emit_insn (gen_and<mode>3 (operands[0], operands[1], operands[2]));
5573   DONE;
5574 })
5575
5576 (define_expand "neon_vorr<mode>"
5577   [(match_operand:VDQX 0 "s_register_operand" "")
5578    (match_operand:VDQX 1 "s_register_operand" "")
5579    (match_operand:VDQX 2 "neon_logic_op2" "")
5580    (match_operand:SI 3 "immediate_operand" "")]
5581   "TARGET_NEON"
5582 {
5583   emit_insn (gen_ior<mode>3 (operands[0], operands[1], operands[2]));
5584   DONE;
5585 })
5586
5587 (define_expand "neon_veor<mode>"
5588   [(match_operand:VDQX 0 "s_register_operand" "")
5589    (match_operand:VDQX 1 "s_register_operand" "")
5590    (match_operand:VDQX 2 "s_register_operand" "")
5591    (match_operand:SI 3 "immediate_operand" "")]
5592   "TARGET_NEON"
5593 {
5594   emit_insn (gen_xor<mode>3 (operands[0], operands[1], operands[2]));
5595   DONE;
5596 })
5597
5598 (define_expand "neon_vbic<mode>"
5599   [(match_operand:VDQX 0 "s_register_operand" "")
5600    (match_operand:VDQX 1 "s_register_operand" "")
5601    (match_operand:VDQX 2 "neon_logic_op2" "")
5602    (match_operand:SI 3 "immediate_operand" "")]
5603   "TARGET_NEON"
5604 {
5605   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5606   DONE;
5607 })
5608
5609 (define_expand "neon_vorn<mode>"
5610   [(match_operand:VDQX 0 "s_register_operand" "")
5611    (match_operand:VDQX 1 "s_register_operand" "")
5612    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5613    (match_operand:SI 3 "immediate_operand" "")]
5614   "TARGET_NEON"
5615 {
5616   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5617   DONE;
5618 })
5619
5620 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5621   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5622         (SE:<V_unpack> (vec_select:<V_HALF>
5623                           (match_operand:VU 1 "register_operand" "w")
5624                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5625   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5626   "vmovl.<US><V_sz_elem> %q0, %e1"
5627   [(set_attr "neon_type" "neon_shift_1")]
5628 )
5629
5630 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5631   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5632         (SE:<V_unpack> (vec_select:<V_HALF>
5633                           (match_operand:VU 1 "register_operand" "w")
5634                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5635   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5636   "vmovl.<US><V_sz_elem> %q0, %f1"
5637   [(set_attr "neon_type" "neon_shift_1")]
5638 )
5639
5640 (define_expand "vec_unpack<US>_hi_<mode>"
5641   [(match_operand:<V_unpack> 0 "register_operand" "")
5642    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5643  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5644   {
5645    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5646    rtx t1;
5647    int i;
5648    for (i = 0; i < (<V_mode_nunits>/2); i++)
5649      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5650
5651    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5652    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5653                                                  operands[1],
5654                                                  t1));
5655    DONE;
5656   }
5657 )
5658
5659 (define_expand "vec_unpack<US>_lo_<mode>"
5660   [(match_operand:<V_unpack> 0 "register_operand" "")
5661    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5662  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5663   {
5664    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5665    rtx t1;
5666    int i;
5667    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5668      RTVEC_ELT (v, i) = GEN_INT (i);
5669    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5670    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5671                                                  operands[1],
5672                                                  t1));
5673    DONE;
5674   }
5675 )
5676
5677 (define_insn "neon_vec_<US>mult_lo_<mode>"
5678  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5679        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5680                            (match_operand:VU 1 "register_operand" "w")
5681                            (match_operand:VU 2 "vect_par_constant_low" "")))
5682                         (SE:<V_unpack> (vec_select:<V_HALF>
5683                            (match_operand:VU 3 "register_operand" "w")
5684                            (match_dup 2)))))]
5685   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5686   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5687   [(set_attr "neon_type" "neon_shift_1")]
5688 )
5689
5690 (define_expand "vec_widen_<US>mult_lo_<mode>"
5691   [(match_operand:<V_unpack> 0 "register_operand" "")
5692    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5693    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5694  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5695  {
5696    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5697    rtx t1;
5698    int i;
5699    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5700      RTVEC_ELT (v, i) = GEN_INT (i);
5701    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5702
5703    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5704                                                operands[1],
5705                                                t1,
5706                                                operands[2]));
5707    DONE;
5708  }
5709 )
5710
5711 (define_insn "neon_vec_<US>mult_hi_<mode>"
5712  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5713       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5714                             (match_operand:VU 1 "register_operand" "w")
5715                             (match_operand:VU 2 "vect_par_constant_high" "")))
5716                        (SE:<V_unpack> (vec_select:<V_HALF>
5717                             (match_operand:VU 3 "register_operand" "w")
5718                             (match_dup 2)))))]
5719   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5720   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5721   [(set_attr "neon_type" "neon_shift_1")]
5722 )
5723
5724 (define_expand "vec_widen_<US>mult_hi_<mode>"
5725   [(match_operand:<V_unpack> 0 "register_operand" "")
5726    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5727    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5728  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5729  {
5730    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5731    rtx t1;
5732    int i;
5733    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5734      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5735    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5736
5737    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5738                                                operands[1],
5739                                                t1,
5740                                                operands[2]));
5741    DONE;
5742
5743  }
5744 )
5745
5746 (define_insn "neon_vec_<US>shiftl_<mode>"
5747  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5748        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5749        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5750   "TARGET_NEON"
5751 {
5752   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5753 }
5754   [(set_attr "neon_type" "neon_shift_1")]
5755 )
5756
5757 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5758   [(match_operand:<V_unpack> 0 "register_operand" "")
5759    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5760    (match_operand:SI 2 "immediate_operand" "i")]
5761  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5762  {
5763   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5764                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5765                 operands[2]));
5766    DONE;
5767  }
5768 )
5769
5770 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5771   [(match_operand:<V_unpack> 0 "register_operand" "")
5772    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5773    (match_operand:SI 2 "immediate_operand" "i")]
5774  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5775  {
5776   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5777                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5778                                      GET_MODE_SIZE (<V_HALF>mode)),
5779                 operands[2]));
5780    DONE;
5781  }
5782 )
5783
5784 ;; Vectorize for non-neon-quad case
5785 (define_insn "neon_unpack<US>_<mode>"
5786  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5787        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5788  "TARGET_NEON"
5789  "vmovl.<US><V_sz_elem> %q0, %P1"
5790   [(set_attr "neon_type" "neon_shift_1")]
5791 )
5792
5793 (define_expand "vec_unpack<US>_lo_<mode>"
5794  [(match_operand:<V_double_width> 0 "register_operand" "")
5795   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5796  "TARGET_NEON"
5797 {
5798   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5799   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5800   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5801
5802   DONE;
5803 }
5804 )
5805
5806 (define_expand "vec_unpack<US>_hi_<mode>"
5807  [(match_operand:<V_double_width> 0 "register_operand" "")
5808   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5809  "TARGET_NEON"
5810 {
5811   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5812   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5813   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5814
5815   DONE;
5816 }
5817 )
5818
5819 (define_insn "neon_vec_<US>mult_<mode>"
5820  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5821        (mult:<V_widen> (SE:<V_widen>
5822                            (match_operand:VDI 1 "register_operand" "w"))
5823                        (SE:<V_widen>
5824                            (match_operand:VDI 2 "register_operand" "w"))))]
5825   "TARGET_NEON"
5826   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5827   [(set_attr "neon_type" "neon_shift_1")]
5828 )
5829
5830 (define_expand "vec_widen_<US>mult_hi_<mode>"
5831   [(match_operand:<V_double_width> 0 "register_operand" "")
5832    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5833    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5834  "TARGET_NEON"
5835  {
5836    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5837    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5838    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5839
5840    DONE;
5841
5842  }
5843 )
5844
5845 (define_expand "vec_widen_<US>mult_lo_<mode>"
5846   [(match_operand:<V_double_width> 0 "register_operand" "")
5847    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5848    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5849  "TARGET_NEON"
5850  {
5851    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5852    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5853    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5854
5855    DONE;
5856
5857  }
5858 )
5859
5860 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5861  [(match_operand:<V_double_width> 0 "register_operand" "")
5862    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5863    (match_operand:SI 2 "immediate_operand" "i")]
5864  "TARGET_NEON"
5865  {
5866    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5867    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5868    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5869
5870    DONE;
5871  }
5872 )
5873
5874 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5875   [(match_operand:<V_double_width> 0 "register_operand" "")
5876    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5877    (match_operand:SI 2 "immediate_operand" "i")]
5878  "TARGET_NEON"
5879  {
5880    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5881    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5882    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5883
5884    DONE;
5885  }
5886 )
5887
5888 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5889 ; because the ordering of vector elements in Q registers is different from what
5890 ; the semantics of the instructions require.
5891
5892 (define_insn "vec_pack_trunc_<mode>"
5893  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5894        (vec_concat:<V_narrow_pack>
5895                 (truncate:<V_narrow>
5896                         (match_operand:VN 1 "register_operand" "w"))
5897                 (truncate:<V_narrow>
5898                         (match_operand:VN 2 "register_operand" "w"))))]
5899  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5900  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5901  [(set_attr "neon_type" "neon_shift_1")
5902   (set_attr "length" "8")]
5903 )
5904
5905 ;; For the non-quad case.
5906 (define_insn "neon_vec_pack_trunc_<mode>"
5907  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5908        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5909  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5910  "vmovn.i<V_sz_elem>\t%P0, %q1"
5911  [(set_attr "neon_type" "neon_shift_1")]
5912 )
5913
5914 (define_expand "vec_pack_trunc_<mode>"
5915  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5916   (match_operand:VSHFT 1 "register_operand" "")
5917   (match_operand:VSHFT 2 "register_operand")]
5918  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5919 {
5920   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5921
5922   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5923   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5924   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5925   DONE;
5926 })
5927
5928 (define_insn "neon_vabd<mode>_2"
5929  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5930        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5931                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5932  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5933  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5934  [(set (attr "neon_type")
5935        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5936                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5937                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5938                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5939                      (const_string "neon_int_5")))]
5940 )
5941
5942 (define_insn "neon_vabd<mode>_3"
5943  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5944        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5945                              (match_operand:VDQ 2 "s_register_operand" "w")]
5946                  UNSPEC_VSUB)))]
5947  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5948  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5949  [(set (attr "neon_type")
5950        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5951                      (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5952                                    (const_string "neon_fp_vadd_ddd_vabs_dd")
5953                                    (const_string "neon_fp_vadd_qqq_vabs_qq"))
5954                      (const_string "neon_int_5")))]
5955 )
5956
5957 ;; Copy from core-to-neon regs, then extend, not vice-versa
5958
5959 (define_split
5960   [(set (match_operand:DI 0 "s_register_operand" "")
5961         (sign_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5962   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5963   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5964    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 32)))]
5965   {
5966     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5967   })
5968
5969 (define_split
5970   [(set (match_operand:DI 0 "s_register_operand" "")
5971         (sign_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
5972   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5973   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
5974    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 48)))]
5975   {
5976     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
5977   })
5978
5979 (define_split
5980   [(set (match_operand:DI 0 "s_register_operand" "")
5981         (sign_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
5982   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5983   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
5984    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))]
5985   {
5986     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
5987   })
5988
5989 (define_split
5990   [(set (match_operand:DI 0 "s_register_operand" "")
5991         (zero_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5992   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5993   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5994    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 32)))]
5995   {
5996     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5997   })
5998
5999 (define_split
6000   [(set (match_operand:DI 0 "s_register_operand" "")
6001         (zero_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
6002   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
6003   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
6004    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 48)))]
6005   {
6006     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
6007   })
6008
6009 (define_split
6010   [(set (match_operand:DI 0 "s_register_operand" "")
6011         (zero_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
6012   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
6013   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
6014    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))]
6015   {
6016     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
6017   })