1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2014 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
23 (match_operand:VALL 1 "general_operand" ""))]
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
47 (match_operand:<VEL> 1 "register_operand" "r, w")))]
50 dup\\t%0.<Vtype>, %<vw>1
51 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
52 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
55 (define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF 0 "register_operand" "=w")
57 (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
59 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
60 [(set_attr "type" "neon_dup<q>")]
63 (define_insn "aarch64_dup_lane<mode>"
64 [(set (match_operand:VALL 0 "register_operand" "=w")
67 (match_operand:VALL 1 "register_operand" "w")
68 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
72 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
73 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
75 [(set_attr "type" "neon_dup<q>")]
78 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
79 [(set (match_operand:VALL 0 "register_operand" "=w")
82 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
83 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
87 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
88 INTVAL (operands[2])));
89 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
91 [(set_attr "type" "neon_dup<q>")]
94 (define_insn "*aarch64_simd_mov<mode>"
95 [(set (match_operand:VD 0 "nonimmediate_operand"
96 "=w, m, w, ?r, ?w, ?r, w")
97 (match_operand:VD 1 "general_operand"
98 "m, w, w, w, r, r, Dn"))]
100 && (register_operand (operands[0], <MODE>mode)
101 || register_operand (operands[1], <MODE>mode))"
103 switch (which_alternative)
105 case 0: return "ldr\\t%d0, %1";
106 case 1: return "str\\t%d1, %0";
107 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
108 case 3: return "umov\t%0, %1.d[0]";
109 case 4: return "ins\t%0.d[0], %1";
110 case 5: return "mov\t%0, %1";
112 return aarch64_output_simd_mov_immediate (operands[1],
114 default: gcc_unreachable ();
117 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
118 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
119 mov_reg, neon_move<q>")]
122 (define_insn "*aarch64_simd_mov<mode>"
123 [(set (match_operand:VQ 0 "nonimmediate_operand"
124 "=w, m, w, ?r, ?w, ?r, w")
125 (match_operand:VQ 1 "general_operand"
126 "m, w, w, w, r, r, Dn"))]
128 && (register_operand (operands[0], <MODE>mode)
129 || register_operand (operands[1], <MODE>mode))"
131 switch (which_alternative)
134 return "ldr\\t%q0, %1";
136 return "str\\t%q1, %0";
138 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
144 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
149 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
150 neon_logic<q>, multiple, multiple, multiple,\
152 (set_attr "length" "4,4,4,8,8,8,4")]
156 [(set (match_operand:VQ 0 "register_operand" "")
157 (match_operand:VQ 1 "register_operand" ""))]
158 "TARGET_SIMD && reload_completed
159 && GP_REGNUM_P (REGNO (operands[0]))
160 && GP_REGNUM_P (REGNO (operands[1]))"
161 [(set (match_dup 0) (match_dup 1))
162 (set (match_dup 2) (match_dup 3))]
164 int rdest = REGNO (operands[0]);
165 int rsrc = REGNO (operands[1]);
168 dest[0] = gen_rtx_REG (DImode, rdest);
169 src[0] = gen_rtx_REG (DImode, rsrc);
170 dest[1] = gen_rtx_REG (DImode, rdest + 1);
171 src[1] = gen_rtx_REG (DImode, rsrc + 1);
173 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
177 [(set (match_operand:VQ 0 "register_operand" "")
178 (match_operand:VQ 1 "register_operand" ""))]
179 "TARGET_SIMD && reload_completed
180 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
181 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
184 aarch64_split_simd_move (operands[0], operands[1]);
188 (define_expand "aarch64_split_simd_mov<mode>"
189 [(set (match_operand:VQ 0)
190 (match_operand:VQ 1))]
193 rtx dst = operands[0];
194 rtx src = operands[1];
196 if (GP_REGNUM_P (REGNO (src)))
198 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
199 rtx src_high_part = gen_highpart (<VHALF>mode, src);
202 (gen_move_lo_quad_<mode> (dst, src_low_part));
204 (gen_move_hi_quad_<mode> (dst, src_high_part));
209 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
210 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
211 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
212 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
215 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
217 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
223 (define_insn "aarch64_simd_mov_from_<mode>low"
224 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
226 (match_operand:VQ 1 "register_operand" "w")
227 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
228 "TARGET_SIMD && reload_completed"
230 [(set_attr "type" "neon_to_gp<q>")
231 (set_attr "length" "4")
234 (define_insn "aarch64_simd_mov_from_<mode>high"
235 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
237 (match_operand:VQ 1 "register_operand" "w")
238 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
239 "TARGET_SIMD && reload_completed"
241 [(set_attr "type" "neon_to_gp<q>")
242 (set_attr "length" "4")
245 (define_insn "orn<mode>3"
246 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
247 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
248 (match_operand:VDQ_I 2 "register_operand" "w")))]
250 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
251 [(set_attr "type" "neon_logic<q>")]
254 (define_insn "bic<mode>3"
255 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
256 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
257 (match_operand:VDQ_I 2 "register_operand" "w")))]
259 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
260 [(set_attr "type" "neon_logic<q>")]
263 (define_insn "add<mode>3"
264 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
265 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
266 (match_operand:VDQ_I 2 "register_operand" "w")))]
268 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
269 [(set_attr "type" "neon_add<q>")]
272 (define_insn "sub<mode>3"
273 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
274 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
275 (match_operand:VDQ_I 2 "register_operand" "w")))]
277 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
278 [(set_attr "type" "neon_sub<q>")]
281 (define_insn "mul<mode>3"
282 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
283 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
284 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
286 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
287 [(set_attr "type" "neon_mul_<Vetype><q>")]
290 (define_insn "bswap<mode>2"
291 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
292 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
294 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
295 [(set_attr "type" "neon_rev<q>")]
298 (define_insn "aarch64_rbit<mode>"
299 [(set (match_operand:VB 0 "register_operand" "=w")
300 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
303 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
304 [(set_attr "type" "neon_rbit")]
307 (define_expand "ctz<mode>2"
308 [(set (match_operand:VS 0 "register_operand")
309 (ctz:VS (match_operand:VS 1 "register_operand")))]
312 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
313 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
315 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
316 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
321 (define_insn "*aarch64_mul3_elt<mode>"
322 [(set (match_operand:VMUL 0 "register_operand" "=w")
326 (match_operand:VMUL 1 "register_operand" "<h_con>")
327 (parallel [(match_operand:SI 2 "immediate_operand")])))
328 (match_operand:VMUL 3 "register_operand" "w")))]
331 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
332 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
334 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
337 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
338 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
339 (mult:VMUL_CHANGE_NLANES
340 (vec_duplicate:VMUL_CHANGE_NLANES
342 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
343 (parallel [(match_operand:SI 2 "immediate_operand")])))
344 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
347 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
348 INTVAL (operands[2])));
349 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
351 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
354 (define_insn "*aarch64_mul3_elt_to_128df"
355 [(set (match_operand:V2DF 0 "register_operand" "=w")
358 (match_operand:DF 2 "register_operand" "w"))
359 (match_operand:V2DF 1 "register_operand" "w")))]
361 "fmul\\t%0.2d, %1.2d, %2.d[0]"
362 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
365 (define_insn "*aarch64_mul3_elt_to_64v2df"
366 [(set (match_operand:DF 0 "register_operand" "=w")
369 (match_operand:V2DF 1 "register_operand" "w")
370 (parallel [(match_operand:SI 2 "immediate_operand")]))
371 (match_operand:DF 3 "register_operand" "w")))]
374 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
375 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
377 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
380 (define_insn "neg<mode>2"
381 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
382 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
384 "neg\t%0.<Vtype>, %1.<Vtype>"
385 [(set_attr "type" "neon_neg<q>")]
388 (define_insn "abs<mode>2"
389 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
390 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
392 "abs\t%0.<Vtype>, %1.<Vtype>"
393 [(set_attr "type" "neon_abs<q>")]
396 (define_insn "abd<mode>_3"
397 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
398 (abs:VDQ_BHSI (minus:VDQ_BHSI
399 (match_operand:VDQ_BHSI 1 "register_operand" "w")
400 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
402 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
403 [(set_attr "type" "neon_abd<q>")]
406 (define_insn "aba<mode>_3"
407 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
408 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
409 (match_operand:VDQ_BHSI 1 "register_operand" "w")
410 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
411 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
413 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
414 [(set_attr "type" "neon_arith_acc<q>")]
417 (define_insn "fabd<mode>_3"
418 [(set (match_operand:VDQF 0 "register_operand" "=w")
419 (abs:VDQF (minus:VDQF
420 (match_operand:VDQF 1 "register_operand" "w")
421 (match_operand:VDQF 2 "register_operand" "w"))))]
423 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
424 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
427 (define_insn "*fabd_scalar<mode>3"
428 [(set (match_operand:GPF 0 "register_operand" "=w")
430 (match_operand:GPF 1 "register_operand" "w")
431 (match_operand:GPF 2 "register_operand" "w"))))]
433 "fabd\t%<s>0, %<s>1, %<s>2"
434 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
437 (define_insn "and<mode>3"
438 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
439 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
440 (match_operand:VDQ_I 2 "register_operand" "w")))]
442 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
443 [(set_attr "type" "neon_logic<q>")]
446 (define_insn "ior<mode>3"
447 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
448 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
449 (match_operand:VDQ_I 2 "register_operand" "w")))]
451 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
452 [(set_attr "type" "neon_logic<q>")]
455 (define_insn "xor<mode>3"
456 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
457 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
458 (match_operand:VDQ_I 2 "register_operand" "w")))]
460 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
461 [(set_attr "type" "neon_logic<q>")]
464 (define_insn "one_cmpl<mode>2"
465 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
466 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
468 "not\t%0.<Vbtype>, %1.<Vbtype>"
469 [(set_attr "type" "neon_logic<q>")]
472 (define_insn "aarch64_simd_vec_set<mode>"
473 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
475 (vec_duplicate:VDQ_BHSI
476 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
477 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
478 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
481 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
482 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
483 switch (which_alternative)
486 return "ins\\t%0.<Vetype>[%p2], %w1";
488 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
490 return "ld1\\t{%0.<Vetype>}[%p2], %1";
495 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_1reg<q>")]
498 (define_insn "aarch64_simd_lshr<mode>"
499 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
500 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
501 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
503 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
504 [(set_attr "type" "neon_shift_imm<q>")]
507 (define_insn "aarch64_simd_ashr<mode>"
508 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
509 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
510 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
512 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
513 [(set_attr "type" "neon_shift_imm<q>")]
516 (define_insn "aarch64_simd_imm_shl<mode>"
517 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
518 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
519 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
521 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
522 [(set_attr "type" "neon_shift_imm<q>")]
525 (define_insn "aarch64_simd_reg_sshl<mode>"
526 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
527 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
528 (match_operand:VDQ_I 2 "register_operand" "w")))]
530 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
531 [(set_attr "type" "neon_shift_reg<q>")]
534 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
535 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
536 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
537 (match_operand:VDQ_I 2 "register_operand" "w")]
538 UNSPEC_ASHIFT_UNSIGNED))]
540 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
541 [(set_attr "type" "neon_shift_reg<q>")]
544 (define_insn "aarch64_simd_reg_shl<mode>_signed"
545 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
546 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
547 (match_operand:VDQ_I 2 "register_operand" "w")]
548 UNSPEC_ASHIFT_SIGNED))]
550 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
551 [(set_attr "type" "neon_shift_reg<q>")]
554 (define_expand "ashl<mode>3"
555 [(match_operand:VDQ_I 0 "register_operand" "")
556 (match_operand:VDQ_I 1 "register_operand" "")
557 (match_operand:SI 2 "general_operand" "")]
560 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
563 if (CONST_INT_P (operands[2]))
565 shift_amount = INTVAL (operands[2]);
566 if (shift_amount >= 0 && shift_amount < bit_width)
568 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
570 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
577 operands[2] = force_reg (SImode, operands[2]);
580 else if (MEM_P (operands[2]))
582 operands[2] = force_reg (SImode, operands[2]);
585 if (REG_P (operands[2]))
587 rtx tmp = gen_reg_rtx (<MODE>mode);
588 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
589 convert_to_mode (<VEL>mode,
592 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
601 (define_expand "lshr<mode>3"
602 [(match_operand:VDQ_I 0 "register_operand" "")
603 (match_operand:VDQ_I 1 "register_operand" "")
604 (match_operand:SI 2 "general_operand" "")]
607 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
610 if (CONST_INT_P (operands[2]))
612 shift_amount = INTVAL (operands[2]);
613 if (shift_amount > 0 && shift_amount <= bit_width)
615 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
617 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
623 operands[2] = force_reg (SImode, operands[2]);
625 else if (MEM_P (operands[2]))
627 operands[2] = force_reg (SImode, operands[2]);
630 if (REG_P (operands[2]))
632 rtx tmp = gen_reg_rtx (SImode);
633 rtx tmp1 = gen_reg_rtx (<MODE>mode);
634 emit_insn (gen_negsi2 (tmp, operands[2]));
635 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
636 convert_to_mode (<VEL>mode,
638 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
648 (define_expand "ashr<mode>3"
649 [(match_operand:VDQ_I 0 "register_operand" "")
650 (match_operand:VDQ_I 1 "register_operand" "")
651 (match_operand:SI 2 "general_operand" "")]
654 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
657 if (CONST_INT_P (operands[2]))
659 shift_amount = INTVAL (operands[2]);
660 if (shift_amount > 0 && shift_amount <= bit_width)
662 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
664 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
670 operands[2] = force_reg (SImode, operands[2]);
672 else if (MEM_P (operands[2]))
674 operands[2] = force_reg (SImode, operands[2]);
677 if (REG_P (operands[2]))
679 rtx tmp = gen_reg_rtx (SImode);
680 rtx tmp1 = gen_reg_rtx (<MODE>mode);
681 emit_insn (gen_negsi2 (tmp, operands[2]));
682 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
683 convert_to_mode (<VEL>mode,
685 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
695 (define_expand "vashl<mode>3"
696 [(match_operand:VDQ_I 0 "register_operand" "")
697 (match_operand:VDQ_I 1 "register_operand" "")
698 (match_operand:VDQ_I 2 "register_operand" "")]
701 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
706 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
707 ;; Negating individual lanes most certainly offsets the
708 ;; gain from vectorization.
709 (define_expand "vashr<mode>3"
710 [(match_operand:VDQ_BHSI 0 "register_operand" "")
711 (match_operand:VDQ_BHSI 1 "register_operand" "")
712 (match_operand:VDQ_BHSI 2 "register_operand" "")]
715 rtx neg = gen_reg_rtx (<MODE>mode);
716 emit (gen_neg<mode>2 (neg, operands[2]));
717 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
723 (define_expand "aarch64_ashr_simddi"
724 [(match_operand:DI 0 "register_operand" "=w")
725 (match_operand:DI 1 "register_operand" "w")
726 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
729 /* An arithmetic shift right by 64 fills the result with copies of the sign
730 bit, just like asr by 63 - however the standard pattern does not handle
732 if (INTVAL (operands[2]) == 64)
733 operands[2] = GEN_INT (63);
734 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
739 (define_expand "vlshr<mode>3"
740 [(match_operand:VDQ_BHSI 0 "register_operand" "")
741 (match_operand:VDQ_BHSI 1 "register_operand" "")
742 (match_operand:VDQ_BHSI 2 "register_operand" "")]
745 rtx neg = gen_reg_rtx (<MODE>mode);
746 emit (gen_neg<mode>2 (neg, operands[2]));
747 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
752 (define_expand "aarch64_lshr_simddi"
753 [(match_operand:DI 0 "register_operand" "=w")
754 (match_operand:DI 1 "register_operand" "w")
755 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
758 if (INTVAL (operands[2]) == 64)
759 emit_insn (gen_aarch64_ushr_simddi (operands[0], operands[1]));
761 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
766 ;; SIMD shift by 64. This pattern is a special case as standard pattern does
767 ;; not handle NEON shifts by 64.
768 (define_insn "aarch64_ushr_simddi"
769 [(set (match_operand:DI 0 "register_operand" "=w")
771 [(match_operand:DI 1 "register_operand" "w")] UNSPEC_USHR64))]
774 [(set_attr "type" "neon_shift_imm")]
777 (define_expand "vec_set<mode>"
778 [(match_operand:VDQ_BHSI 0 "register_operand")
779 (match_operand:<VEL> 1 "register_operand")
780 (match_operand:SI 2 "immediate_operand")]
783 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
784 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
785 GEN_INT (elem), operands[0]));
790 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
791 (define_insn "vec_shr_<mode>"
792 [(set (match_operand:VD 0 "register_operand" "=w")
793 (lshiftrt:VD (match_operand:VD 1 "register_operand" "w")
794 (match_operand:SI 2 "immediate_operand" "i")))]
797 if (BYTES_BIG_ENDIAN)
798 return "ushl %d0, %d1, %2";
800 return "ushr %d0, %d1, %2";
802 [(set_attr "type" "neon_shift_imm")]
805 (define_insn "aarch64_simd_vec_setv2di"
806 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
809 (match_operand:DI 1 "register_operand" "r,w"))
810 (match_operand:V2DI 3 "register_operand" "0,0")
811 (match_operand:SI 2 "immediate_operand" "i,i")))]
814 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
815 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
816 switch (which_alternative)
819 return "ins\\t%0.d[%p2], %1";
821 return "ins\\t%0.d[%p2], %1.d[0]";
826 [(set_attr "type" "neon_from_gp, neon_ins_q")]
829 (define_expand "vec_setv2di"
830 [(match_operand:V2DI 0 "register_operand")
831 (match_operand:DI 1 "register_operand")
832 (match_operand:SI 2 "immediate_operand")]
835 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
836 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
837 GEN_INT (elem), operands[0]));
842 (define_insn "aarch64_simd_vec_set<mode>"
843 [(set (match_operand:VDQF 0 "register_operand" "=w")
846 (match_operand:<VEL> 1 "register_operand" "w"))
847 (match_operand:VDQF 3 "register_operand" "0")
848 (match_operand:SI 2 "immediate_operand" "i")))]
851 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
853 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
854 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
856 [(set_attr "type" "neon_ins<q>")]
859 (define_expand "vec_set<mode>"
860 [(match_operand:VDQF 0 "register_operand" "+w")
861 (match_operand:<VEL> 1 "register_operand" "w")
862 (match_operand:SI 2 "immediate_operand" "")]
865 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
866 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
867 GEN_INT (elem), operands[0]));
873 (define_insn "aarch64_mla<mode>"
874 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
875 (plus:VDQ_BHSI (mult:VDQ_BHSI
876 (match_operand:VDQ_BHSI 2 "register_operand" "w")
877 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
878 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
880 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
881 [(set_attr "type" "neon_mla_<Vetype><q>")]
884 (define_insn "*aarch64_mla_elt<mode>"
885 [(set (match_operand:VDQHS 0 "register_operand" "=w")
890 (match_operand:VDQHS 1 "register_operand" "<h_con>")
891 (parallel [(match_operand:SI 2 "immediate_operand")])))
892 (match_operand:VDQHS 3 "register_operand" "w"))
893 (match_operand:VDQHS 4 "register_operand" "0")))]
896 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
897 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
899 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
902 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
903 [(set (match_operand:VDQHS 0 "register_operand" "=w")
908 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
909 (parallel [(match_operand:SI 2 "immediate_operand")])))
910 (match_operand:VDQHS 3 "register_operand" "w"))
911 (match_operand:VDQHS 4 "register_operand" "0")))]
914 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
915 INTVAL (operands[2])));
916 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
918 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
921 (define_insn "aarch64_mls<mode>"
922 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
923 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
924 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
925 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
927 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
928 [(set_attr "type" "neon_mla_<Vetype><q>")]
931 (define_insn "*aarch64_mls_elt<mode>"
932 [(set (match_operand:VDQHS 0 "register_operand" "=w")
934 (match_operand:VDQHS 4 "register_operand" "0")
938 (match_operand:VDQHS 1 "register_operand" "<h_con>")
939 (parallel [(match_operand:SI 2 "immediate_operand")])))
940 (match_operand:VDQHS 3 "register_operand" "w"))))]
943 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
944 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
946 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
949 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
950 [(set (match_operand:VDQHS 0 "register_operand" "=w")
952 (match_operand:VDQHS 4 "register_operand" "0")
956 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
957 (parallel [(match_operand:SI 2 "immediate_operand")])))
958 (match_operand:VDQHS 3 "register_operand" "w"))))]
961 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
962 INTVAL (operands[2])));
963 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
965 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
968 ;; Max/Min operations.
969 (define_insn "<su><maxmin><mode>3"
970 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
971 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
972 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
974 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
975 [(set_attr "type" "neon_minmax<q>")]
978 (define_expand "<su><maxmin>v2di3"
979 [(set (match_operand:V2DI 0 "register_operand" "")
980 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
981 (match_operand:V2DI 2 "register_operand" "")))]
984 enum rtx_code cmp_operator;
1005 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1006 emit_insn (gen_aarch64_vcond_internalv2div2di (operands[0], operands[1],
1007 operands[2], cmp_fmt, operands[1], operands[2]));
1011 ;; vec_concat gives a new vector with the low elements from operand 1, and
1012 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1013 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1014 ;; What that means, is that the RTL descriptions of the below patterns
1015 ;; need to change depending on endianness.
1017 ;; Move to the low architectural bits of the register.
1018 ;; On little-endian this is { operand, zeroes }
1019 ;; On big-endian this is { zeroes, operand }
1021 (define_insn "move_lo_quad_internal_<mode>"
1022 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1024 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1025 (vec_duplicate:<VHALF> (const_int 0))))]
1026 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1031 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1032 (set_attr "simd" "yes,*,yes")
1033 (set_attr "fp" "*,yes,*")
1034 (set_attr "length" "4")]
1037 (define_insn "move_lo_quad_internal_<mode>"
1038 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1040 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1042 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1047 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1048 (set_attr "simd" "yes,*,yes")
1049 (set_attr "fp" "*,yes,*")
1050 (set_attr "length" "4")]
1053 (define_insn "move_lo_quad_internal_be_<mode>"
1054 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1056 (vec_duplicate:<VHALF> (const_int 0))
1057 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1058 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1063 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1064 (set_attr "simd" "yes,*,yes")
1065 (set_attr "fp" "*,yes,*")
1066 (set_attr "length" "4")]
1069 (define_insn "move_lo_quad_internal_be_<mode>"
1070 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1073 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1074 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1079 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1080 (set_attr "simd" "yes,*,yes")
1081 (set_attr "fp" "*,yes,*")
1082 (set_attr "length" "4")]
1085 (define_expand "move_lo_quad_<mode>"
1086 [(match_operand:VQ 0 "register_operand")
1087 (match_operand:VQ 1 "register_operand")]
1090 if (BYTES_BIG_ENDIAN)
1091 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1093 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1098 ;; Move operand1 to the high architectural bits of the register, keeping
1099 ;; the low architectural bits of operand2.
1100 ;; For little-endian this is { operand2, operand1 }
1101 ;; For big-endian this is { operand1, operand2 }
1103 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1104 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1108 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1109 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1110 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1112 ins\\t%0.d[1], %1.d[0]
1114 [(set_attr "type" "neon_ins")]
1117 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1118 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1120 (match_operand:<VHALF> 1 "register_operand" "w,r")
1123 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1124 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1126 ins\\t%0.d[1], %1.d[0]
1128 [(set_attr "type" "neon_ins")]
1131 (define_expand "move_hi_quad_<mode>"
1132 [(match_operand:VQ 0 "register_operand" "")
1133 (match_operand:<VHALF> 1 "register_operand" "")]
1136 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1137 if (BYTES_BIG_ENDIAN)
1138 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1141 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1146 ;; Narrowing operations.
1149 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1150 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1151 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1153 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1154 [(set_attr "type" "neon_shift_imm_narrow_q")]
1157 (define_expand "vec_pack_trunc_<mode>"
1158 [(match_operand:<VNARROWD> 0 "register_operand" "")
1159 (match_operand:VDN 1 "register_operand" "")
1160 (match_operand:VDN 2 "register_operand" "")]
1163 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1164 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1165 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1167 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1168 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1169 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1175 (define_insn "vec_pack_trunc_<mode>"
1176 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1177 (vec_concat:<VNARROWQ2>
1178 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1179 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1182 if (BYTES_BIG_ENDIAN)
1183 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1185 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1187 [(set_attr "type" "multiple")
1188 (set_attr "length" "8")]
1191 ;; Widening operations.
1193 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1194 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1195 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1196 (match_operand:VQW 1 "register_operand" "w")
1197 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1200 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1201 [(set_attr "type" "neon_shift_imm_long")]
1204 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1205 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1206 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1207 (match_operand:VQW 1 "register_operand" "w")
1208 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1211 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1212 [(set_attr "type" "neon_shift_imm_long")]
1215 (define_expand "vec_unpack<su>_hi_<mode>"
1216 [(match_operand:<VWIDE> 0 "register_operand" "")
1217 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1220 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1221 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1227 (define_expand "vec_unpack<su>_lo_<mode>"
1228 [(match_operand:<VWIDE> 0 "register_operand" "")
1229 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1232 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1233 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1239 ;; Widening arithmetic.
1241 (define_insn "*aarch64_<su>mlal_lo<mode>"
1242 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1245 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1246 (match_operand:VQW 2 "register_operand" "w")
1247 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1248 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1249 (match_operand:VQW 4 "register_operand" "w")
1251 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1253 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1254 [(set_attr "type" "neon_mla_<Vetype>_long")]
1257 (define_insn "*aarch64_<su>mlal_hi<mode>"
1258 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1261 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1262 (match_operand:VQW 2 "register_operand" "w")
1263 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1264 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1265 (match_operand:VQW 4 "register_operand" "w")
1267 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1269 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1270 [(set_attr "type" "neon_mla_<Vetype>_long")]
1273 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1274 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1276 (match_operand:<VWIDE> 1 "register_operand" "0")
1278 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1279 (match_operand:VQW 2 "register_operand" "w")
1280 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1281 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1282 (match_operand:VQW 4 "register_operand" "w")
1285 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1286 [(set_attr "type" "neon_mla_<Vetype>_long")]
1289 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1290 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1292 (match_operand:<VWIDE> 1 "register_operand" "0")
1294 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1295 (match_operand:VQW 2 "register_operand" "w")
1296 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1297 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1298 (match_operand:VQW 4 "register_operand" "w")
1301 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1302 [(set_attr "type" "neon_mla_<Vetype>_long")]
1305 (define_insn "*aarch64_<su>mlal<mode>"
1306 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1310 (match_operand:VD_BHSI 1 "register_operand" "w"))
1312 (match_operand:VD_BHSI 2 "register_operand" "w")))
1313 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1315 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1316 [(set_attr "type" "neon_mla_<Vetype>_long")]
1319 (define_insn "*aarch64_<su>mlsl<mode>"
1320 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1322 (match_operand:<VWIDE> 1 "register_operand" "0")
1325 (match_operand:VD_BHSI 2 "register_operand" "w"))
1327 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1329 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1330 [(set_attr "type" "neon_mla_<Vetype>_long")]
1333 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1334 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1335 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1336 (match_operand:VQW 1 "register_operand" "w")
1337 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1338 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1339 (match_operand:VQW 2 "register_operand" "w")
1342 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1343 [(set_attr "type" "neon_mul_<Vetype>_long")]
1346 (define_expand "vec_widen_<su>mult_lo_<mode>"
1347 [(match_operand:<VWIDE> 0 "register_operand" "")
1348 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1349 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1352 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1353 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1360 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1361 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1362 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1363 (match_operand:VQW 1 "register_operand" "w")
1364 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1365 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1366 (match_operand:VQW 2 "register_operand" "w")
1369 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1370 [(set_attr "type" "neon_mul_<Vetype>_long")]
1373 (define_expand "vec_widen_<su>mult_hi_<mode>"
1374 [(match_operand:<VWIDE> 0 "register_operand" "")
1375 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1376 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1379 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1380 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1388 ;; FP vector operations.
1389 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1390 ;; double-precision (64-bit) floating-point data types and arithmetic as
1391 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1392 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1394 ;; Floating-point operations can raise an exception. Vectorizing such
1395 ;; operations are safe because of reasons explained below.
1397 ;; ARMv8 permits an extension to enable trapped floating-point
1398 ;; exception handling, however this is an optional feature. In the
1399 ;; event of a floating-point exception being raised by vectorised
1401 ;; 1. If trapped floating-point exceptions are available, then a trap
1402 ;; will be taken when any lane raises an enabled exception. A trap
1403 ;; handler may determine which lane raised the exception.
1404 ;; 2. Alternatively a sticky exception flag is set in the
1405 ;; floating-point status register (FPSR). Software may explicitly
1406 ;; test the exception flags, in which case the tests will either
1407 ;; prevent vectorisation, allowing precise identification of the
1408 ;; failing operation, or if tested outside of vectorisable regions
1409 ;; then the specific operation and lane are not of interest.
1411 ;; FP arithmetic operations.
1413 (define_insn "add<mode>3"
1414 [(set (match_operand:VDQF 0 "register_operand" "=w")
1415 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1416 (match_operand:VDQF 2 "register_operand" "w")))]
1418 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1419 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1422 (define_insn "sub<mode>3"
1423 [(set (match_operand:VDQF 0 "register_operand" "=w")
1424 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1425 (match_operand:VDQF 2 "register_operand" "w")))]
1427 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1428 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1431 (define_insn "mul<mode>3"
1432 [(set (match_operand:VDQF 0 "register_operand" "=w")
1433 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1434 (match_operand:VDQF 2 "register_operand" "w")))]
1436 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1437 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1440 (define_insn "div<mode>3"
1441 [(set (match_operand:VDQF 0 "register_operand" "=w")
1442 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1443 (match_operand:VDQF 2 "register_operand" "w")))]
1445 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1446 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1449 (define_insn "neg<mode>2"
1450 [(set (match_operand:VDQF 0 "register_operand" "=w")
1451 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1453 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1454 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1457 (define_insn "abs<mode>2"
1458 [(set (match_operand:VDQF 0 "register_operand" "=w")
1459 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1461 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1462 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1465 (define_insn "fma<mode>4"
1466 [(set (match_operand:VDQF 0 "register_operand" "=w")
1467 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1468 (match_operand:VDQF 2 "register_operand" "w")
1469 (match_operand:VDQF 3 "register_operand" "0")))]
1471 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1472 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1475 (define_insn "*aarch64_fma4_elt<mode>"
1476 [(set (match_operand:VDQF 0 "register_operand" "=w")
1480 (match_operand:VDQF 1 "register_operand" "<h_con>")
1481 (parallel [(match_operand:SI 2 "immediate_operand")])))
1482 (match_operand:VDQF 3 "register_operand" "w")
1483 (match_operand:VDQF 4 "register_operand" "0")))]
1486 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1487 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1489 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1492 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1493 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1495 (vec_duplicate:VDQSF
1497 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1498 (parallel [(match_operand:SI 2 "immediate_operand")])))
1499 (match_operand:VDQSF 3 "register_operand" "w")
1500 (match_operand:VDQSF 4 "register_operand" "0")))]
1503 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1504 INTVAL (operands[2])));
1505 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1507 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1510 (define_insn "*aarch64_fma4_elt_to_128df"
1511 [(set (match_operand:V2DF 0 "register_operand" "=w")
1514 (match_operand:DF 1 "register_operand" "w"))
1515 (match_operand:V2DF 2 "register_operand" "w")
1516 (match_operand:V2DF 3 "register_operand" "0")))]
1518 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1519 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1522 (define_insn "*aarch64_fma4_elt_to_64v2df"
1523 [(set (match_operand:DF 0 "register_operand" "=w")
1526 (match_operand:V2DF 1 "register_operand" "w")
1527 (parallel [(match_operand:SI 2 "immediate_operand")]))
1528 (match_operand:DF 3 "register_operand" "w")
1529 (match_operand:DF 4 "register_operand" "0")))]
1532 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1533 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1535 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1538 (define_insn "fnma<mode>4"
1539 [(set (match_operand:VDQF 0 "register_operand" "=w")
1541 (match_operand:VDQF 1 "register_operand" "w")
1543 (match_operand:VDQF 2 "register_operand" "w"))
1544 (match_operand:VDQF 3 "register_operand" "0")))]
1546 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1547 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1550 (define_insn "*aarch64_fnma4_elt<mode>"
1551 [(set (match_operand:VDQF 0 "register_operand" "=w")
1554 (match_operand:VDQF 3 "register_operand" "w"))
1557 (match_operand:VDQF 1 "register_operand" "<h_con>")
1558 (parallel [(match_operand:SI 2 "immediate_operand")])))
1559 (match_operand:VDQF 4 "register_operand" "0")))]
1562 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1563 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1565 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1568 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1569 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1572 (match_operand:VDQSF 3 "register_operand" "w"))
1573 (vec_duplicate:VDQSF
1575 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1576 (parallel [(match_operand:SI 2 "immediate_operand")])))
1577 (match_operand:VDQSF 4 "register_operand" "0")))]
1580 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1581 INTVAL (operands[2])));
1582 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1584 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1587 (define_insn "*aarch64_fnma4_elt_to_128df"
1588 [(set (match_operand:V2DF 0 "register_operand" "=w")
1591 (match_operand:V2DF 2 "register_operand" "w"))
1593 (match_operand:DF 1 "register_operand" "w"))
1594 (match_operand:V2DF 3 "register_operand" "0")))]
1596 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1597 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1600 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1601 [(set (match_operand:DF 0 "register_operand" "=w")
1604 (match_operand:V2DF 1 "register_operand" "w")
1605 (parallel [(match_operand:SI 2 "immediate_operand")]))
1607 (match_operand:DF 3 "register_operand" "w"))
1608 (match_operand:DF 4 "register_operand" "0")))]
1611 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1612 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1614 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1617 ;; Vector versions of the floating-point frint patterns.
1618 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1619 (define_insn "<frint_pattern><mode>2"
1620 [(set (match_operand:VDQF 0 "register_operand" "=w")
1621 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1624 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1625 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1628 ;; Vector versions of the fcvt standard patterns.
1629 ;; Expands to lbtrunc, lround, lceil, lfloor
1630 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1631 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1632 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1633 [(match_operand:VDQF 1 "register_operand" "w")]
1636 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1637 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1640 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1641 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1642 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1643 [(match_operand:VDQF 1 "register_operand")]
1648 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1649 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1650 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1651 [(match_operand:VDQF 1 "register_operand")]
1656 (define_expand "ftrunc<VDQF:mode>2"
1657 [(set (match_operand:VDQF 0 "register_operand")
1658 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1663 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1664 [(set (match_operand:VDQF 0 "register_operand" "=w")
1666 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1668 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1669 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1672 ;; Conversions between vectors of floats and doubles.
1673 ;; Contains a mix of patterns to match standard pattern names
1674 ;; and those for intrinsics.
1676 ;; Float widening operations.
1678 (define_insn "vec_unpacks_lo_v4sf"
1679 [(set (match_operand:V2DF 0 "register_operand" "=w")
1682 (match_operand:V4SF 1 "register_operand" "w")
1683 (parallel [(const_int 0) (const_int 1)])
1686 "fcvtl\\t%0.2d, %1.2s"
1687 [(set_attr "type" "neon_fp_cvt_widen_s")]
1690 (define_insn "aarch64_float_extend_lo_v2df"
1691 [(set (match_operand:V2DF 0 "register_operand" "=w")
1693 (match_operand:V2SF 1 "register_operand" "w")))]
1695 "fcvtl\\t%0.2d, %1.2s"
1696 [(set_attr "type" "neon_fp_cvt_widen_s")]
1699 (define_insn "vec_unpacks_hi_v4sf"
1700 [(set (match_operand:V2DF 0 "register_operand" "=w")
1703 (match_operand:V4SF 1 "register_operand" "w")
1704 (parallel [(const_int 2) (const_int 3)])
1707 "fcvtl2\\t%0.2d, %1.4s"
1708 [(set_attr "type" "neon_fp_cvt_widen_s")]
1711 ;; Float narrowing operations.
1713 (define_insn "aarch64_float_truncate_lo_v2sf"
1714 [(set (match_operand:V2SF 0 "register_operand" "=w")
1715 (float_truncate:V2SF
1716 (match_operand:V2DF 1 "register_operand" "w")))]
1718 "fcvtn\\t%0.2s, %1.2d"
1719 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1722 (define_insn "aarch64_float_truncate_hi_v4sf"
1723 [(set (match_operand:V4SF 0 "register_operand" "=w")
1725 (match_operand:V2SF 1 "register_operand" "0")
1726 (float_truncate:V2SF
1727 (match_operand:V2DF 2 "register_operand" "w"))))]
1729 "fcvtn2\\t%0.4s, %2.2d"
1730 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1733 (define_expand "vec_pack_trunc_v2df"
1734 [(set (match_operand:V4SF 0 "register_operand")
1736 (float_truncate:V2SF
1737 (match_operand:V2DF 1 "register_operand"))
1738 (float_truncate:V2SF
1739 (match_operand:V2DF 2 "register_operand"))
1743 rtx tmp = gen_reg_rtx (V2SFmode);
1744 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1745 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1747 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1748 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1749 tmp, operands[hi]));
1754 (define_expand "vec_pack_trunc_df"
1755 [(set (match_operand:V2SF 0 "register_operand")
1758 (match_operand:DF 1 "register_operand"))
1760 (match_operand:DF 2 "register_operand"))
1764 rtx tmp = gen_reg_rtx (V2SFmode);
1765 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1766 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1768 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1769 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1770 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1775 (define_insn "aarch64_vmls<mode>"
1776 [(set (match_operand:VDQF 0 "register_operand" "=w")
1777 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1778 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1779 (match_operand:VDQF 3 "register_operand" "w"))))]
1781 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1782 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1786 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1788 ;; a = (b < c) ? b : c;
1789 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1790 ;; either explicitly or indirectly via -ffast-math.
1792 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1793 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1794 ;; operand will be returned when both operands are zero (i.e. they may not
1795 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1796 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1799 (define_insn "<su><maxmin><mode>3"
1800 [(set (match_operand:VDQF 0 "register_operand" "=w")
1801 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1802 (match_operand:VDQF 2 "register_operand" "w")))]
1804 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1805 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1808 (define_insn "<maxmin_uns><mode>3"
1809 [(set (match_operand:VDQF 0 "register_operand" "=w")
1810 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1811 (match_operand:VDQF 2 "register_operand" "w")]
1814 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1815 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1818 ;; 'across lanes' add.
1820 (define_expand "reduc_plus_scal_<mode>"
1821 [(match_operand:<VEL> 0 "register_operand" "=w")
1822 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
1826 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1827 rtx scratch = gen_reg_rtx (<MODE>mode);
1828 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1829 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1834 (define_expand "reduc_plus_scal_<mode>"
1835 [(match_operand:<VEL> 0 "register_operand" "=w")
1836 (match_operand:V2F 1 "register_operand" "w")]
1839 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1840 rtx scratch = gen_reg_rtx (<MODE>mode);
1841 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1842 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1847 (define_insn "aarch64_reduc_plus_internal<mode>"
1848 [(set (match_operand:VDQV 0 "register_operand" "=w")
1849 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1852 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
1853 [(set_attr "type" "neon_reduc_add<q>")]
1856 (define_insn "aarch64_reduc_plus_internalv2si"
1857 [(set (match_operand:V2SI 0 "register_operand" "=w")
1858 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1861 "addp\\t%0.2s, %1.2s, %1.2s"
1862 [(set_attr "type" "neon_reduc_add")]
1865 (define_insn "aarch64_reduc_plus_internal<mode>"
1866 [(set (match_operand:V2F 0 "register_operand" "=w")
1867 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1870 "faddp\\t%<Vetype>0, %1.<Vtype>"
1871 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
1874 (define_insn "aarch64_addpv4sf"
1875 [(set (match_operand:V4SF 0 "register_operand" "=w")
1876 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1879 "faddp\\t%0.4s, %1.4s, %1.4s"
1880 [(set_attr "type" "neon_fp_reduc_add_s_q")]
1883 (define_expand "reduc_plus_scal_v4sf"
1884 [(set (match_operand:SF 0 "register_operand")
1885 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1889 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
1890 rtx scratch = gen_reg_rtx (V4SFmode);
1891 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
1892 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
1893 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
1897 (define_insn "clrsb<mode>2"
1898 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1899 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1901 "cls\\t%0.<Vtype>, %1.<Vtype>"
1902 [(set_attr "type" "neon_cls<q>")]
1905 (define_insn "clz<mode>2"
1906 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1907 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1909 "clz\\t%0.<Vtype>, %1.<Vtype>"
1910 [(set_attr "type" "neon_cls<q>")]
1913 (define_insn "popcount<mode>2"
1914 [(set (match_operand:VB 0 "register_operand" "=w")
1915 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
1917 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
1918 [(set_attr "type" "neon_cnt<q>")]
1921 ;; 'across lanes' max and min ops.
1923 ;; Template for outputting a scalar, so we can create __builtins which can be
1924 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
1925 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1926 [(match_operand:<VEL> 0 "register_operand")
1927 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1931 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1932 rtx scratch = gen_reg_rtx (<MODE>mode);
1933 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1935 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1940 ;; Likewise for integer cases, signed and unsigned.
1941 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1942 [(match_operand:<VEL> 0 "register_operand")
1943 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
1947 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1948 rtx scratch = gen_reg_rtx (<MODE>mode);
1949 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1951 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1956 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1957 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
1958 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
1961 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1962 [(set_attr "type" "neon_reduc_minmax<q>")]
1965 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
1966 [(set (match_operand:V2SI 0 "register_operand" "=w")
1967 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1970 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1971 [(set_attr "type" "neon_reduc_minmax")]
1974 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1975 [(set (match_operand:VDQF 0 "register_operand" "=w")
1976 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1979 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
1980 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
1983 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1985 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1988 ;; Thus our BSL is of the form:
1989 ;; op0 = bsl (mask, op2, op3)
1990 ;; We can use any of:
1993 ;; bsl mask, op1, op2
1994 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1995 ;; bit op0, op2, mask
1996 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1997 ;; bif op0, op1, mask
1999 (define_insn "aarch64_simd_bsl<mode>_internal"
2000 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2004 (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w"))
2005 (match_operand:VSDQ_I_DI 3 "register_operand" " w,0,w"))
2007 (match_dup:<V_cmp_result> 1)
2008 (match_operand:VSDQ_I_DI 2 "register_operand" " w,w,0"))
2012 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2013 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2014 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2015 [(set_attr "type" "neon_bsl<q>")]
2018 (define_expand "aarch64_simd_bsl<mode>"
2019 [(match_operand:VALLDIF 0 "register_operand")
2020 (match_operand:<V_cmp_result> 1 "register_operand")
2021 (match_operand:VALLDIF 2 "register_operand")
2022 (match_operand:VALLDIF 3 "register_operand")]
2025 /* We can't alias operands together if they have different modes. */
2026 rtx tmp = operands[0];
2027 if (FLOAT_MODE_P (<MODE>mode))
2029 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2030 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2031 tmp = gen_reg_rtx (<V_cmp_result>mode);
2033 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
2034 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2038 if (tmp != operands[0])
2039 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2044 (define_expand "aarch64_vcond_internal<mode><mode>"
2045 [(set (match_operand:VDQ_I 0 "register_operand")
2047 (match_operator 3 "comparison_operator"
2048 [(match_operand:VDQ_I 4 "register_operand")
2049 (match_operand:VDQ_I 5 "nonmemory_operand")])
2050 (match_operand:VDQ_I 1 "nonmemory_operand")
2051 (match_operand:VDQ_I 2 "nonmemory_operand")))]
2054 rtx op1 = operands[1];
2055 rtx op2 = operands[2];
2056 rtx mask = gen_reg_rtx (<MODE>mode);
2057 enum rtx_code code = GET_CODE (operands[3]);
2059 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
2060 and desirable for other comparisons if it results in FOO ? -1 : 0
2061 (this allows direct use of the comparison result without a bsl). */
2064 && op1 == CONST0_RTX (<V_cmp_result>mode)
2065 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
2071 case LE: code = GT; break;
2072 case LT: code = GE; break;
2073 case GE: code = LT; break;
2074 case GT: code = LE; break;
2076 case NE: code = EQ; break;
2077 case LTU: code = GEU; break;
2078 case LEU: code = GTU; break;
2079 case GTU: code = LEU; break;
2080 case GEU: code = LTU; break;
2081 default: gcc_unreachable ();
2085 /* Make sure we can handle the last operand. */
2089 /* Normalized to EQ above. */
2097 /* These instructions have a form taking an immediate zero. */
2098 if (operands[5] == CONST0_RTX (<MODE>mode))
2100 /* Fall through, as may need to load into register. */
2102 if (!REG_P (operands[5]))
2103 operands[5] = force_reg (<MODE>mode, operands[5]);
2110 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2114 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2118 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2122 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2126 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2130 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2134 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2138 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2141 /* NE has been normalized to EQ above. */
2143 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2150 /* If we have (a = (b CMP c) ? -1 : 0);
2151 Then we can simply move the generated mask. */
2153 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2154 && op2 == CONST0_RTX (<V_cmp_result>mode))
2155 emit_move_insn (operands[0], mask);
2159 op1 = force_reg (<MODE>mode, op1);
2161 op2 = force_reg (<MODE>mode, op2);
2162 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2169 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2170 [(set (match_operand:VDQF_COND 0 "register_operand")
2172 (match_operator 3 "comparison_operator"
2173 [(match_operand:VDQF 4 "register_operand")
2174 (match_operand:VDQF 5 "nonmemory_operand")])
2175 (match_operand:VDQF_COND 1 "nonmemory_operand")
2176 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2180 int use_zero_form = 0;
2181 int swap_bsl_operands = 0;
2182 rtx op1 = operands[1];
2183 rtx op2 = operands[2];
2184 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2185 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2187 rtx (*base_comparison) (rtx, rtx, rtx);
2188 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2190 switch (GET_CODE (operands[3]))
2197 if (operands[5] == CONST0_RTX (<MODE>mode))
2204 if (!REG_P (operands[5]))
2205 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2208 switch (GET_CODE (operands[3]))
2218 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2219 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2227 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2228 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2233 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2234 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2240 switch (GET_CODE (operands[3]))
2247 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2248 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2254 Note that there also exist direct comparison against 0 forms,
2255 so catch those as a special case. */
2259 switch (GET_CODE (operands[3]))
2262 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2265 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2268 /* Do nothing, other zero form cases already have the correct
2275 emit_insn (base_comparison (mask, operands[4], operands[5]));
2277 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2284 /* FCM returns false for lanes which are unordered, so if we use
2285 the inverse of the comparison we actually want to emit, then
2286 swap the operands to BSL, we will end up with the correct result.
2287 Note that a NE NaN and NaN NE b are true for all a, b.
2289 Our transformations are:
2294 a NE b -> !(a EQ b) */
2297 emit_insn (base_comparison (mask, operands[4], operands[5]));
2299 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2301 swap_bsl_operands = 1;
2304 /* We check (a > b || b > a). combining these comparisons give us
2305 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2306 will then give us (a == b || a UNORDERED b) as intended. */
2308 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2309 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2310 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2311 swap_bsl_operands = 1;
2314 /* Operands are ORDERED iff (a > b || b >= a).
2315 Swapping the operands to BSL will give the UNORDERED case. */
2316 swap_bsl_operands = 1;
2319 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2320 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2321 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2327 if (swap_bsl_operands)
2333 /* If we have (a = (b CMP c) ? -1 : 0);
2334 Then we can simply move the generated mask. */
2336 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2337 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2338 emit_move_insn (operands[0], mask);
2342 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2344 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2345 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2352 (define_expand "vcond<mode><mode>"
2353 [(set (match_operand:VALL 0 "register_operand")
2355 (match_operator 3 "comparison_operator"
2356 [(match_operand:VALL 4 "register_operand")
2357 (match_operand:VALL 5 "nonmemory_operand")])
2358 (match_operand:VALL 1 "nonmemory_operand")
2359 (match_operand:VALL 2 "nonmemory_operand")))]
2362 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2363 operands[2], operands[3],
2364 operands[4], operands[5]));
2368 (define_expand "vcond<v_cmp_result><mode>"
2369 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2370 (if_then_else:<V_cmp_result>
2371 (match_operator 3 "comparison_operator"
2372 [(match_operand:VDQF 4 "register_operand")
2373 (match_operand:VDQF 5 "nonmemory_operand")])
2374 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2375 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2378 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2379 operands[0], operands[1],
2380 operands[2], operands[3],
2381 operands[4], operands[5]));
2385 (define_expand "vcondu<mode><mode>"
2386 [(set (match_operand:VDQ_I 0 "register_operand")
2388 (match_operator 3 "comparison_operator"
2389 [(match_operand:VDQ_I 4 "register_operand")
2390 (match_operand:VDQ_I 5 "nonmemory_operand")])
2391 (match_operand:VDQ_I 1 "nonmemory_operand")
2392 (match_operand:VDQ_I 2 "nonmemory_operand")))]
2395 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2396 operands[2], operands[3],
2397 operands[4], operands[5]));
2401 ;; Patterns for AArch64 SIMD Intrinsics.
2403 ;; Lane extraction with sign extension to general purpose register.
2404 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2405 [(set (match_operand:GPI 0 "register_operand" "=r")
2408 (match_operand:VDQQH 1 "register_operand" "w")
2409 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2412 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2413 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2415 [(set_attr "type" "neon_to_gp<q>")]
2418 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2419 [(set (match_operand:SI 0 "register_operand" "=r")
2422 (match_operand:VDQQH 1 "register_operand" "w")
2423 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2426 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2427 return "umov\\t%w0, %1.<Vetype>[%2]";
2429 [(set_attr "type" "neon_to_gp<q>")]
2432 ;; Lane extraction of a value, neither sign nor zero extension
2433 ;; is guaranteed so upper bits should be considered undefined.
2434 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2435 (define_insn "aarch64_get_lane<mode>"
2436 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2438 (match_operand:VALL 1 "register_operand" "w, w, w")
2439 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2442 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2443 switch (which_alternative)
2446 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2448 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2450 return "st1\\t{%1.<Vetype>}[%2], %0";
2455 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2458 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2461 (define_insn "*aarch64_combinez<mode>"
2462 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2464 (match_operand:VD_BHSI 1 "register_operand" "w")
2465 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz")))]
2466 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2467 "mov\\t%0.8b, %1.8b"
2468 [(set_attr "type" "neon_move<q>")]
2471 (define_insn "*aarch64_combinez_be<mode>"
2472 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2474 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz")
2475 (match_operand:VD_BHSI 1 "register_operand" "w")))]
2476 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2477 "mov\\t%0.8b, %1.8b"
2478 [(set_attr "type" "neon_move<q>")]
2481 (define_expand "aarch64_combine<mode>"
2482 [(match_operand:<VDBL> 0 "register_operand")
2483 (match_operand:VDC 1 "register_operand")
2484 (match_operand:VDC 2 "register_operand")]
2488 if (BYTES_BIG_ENDIAN)
2498 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2503 (define_insn_and_split "aarch64_combine_internal<mode>"
2504 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2505 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2506 (match_operand:VDC 2 "register_operand" "w")))]
2509 "&& reload_completed"
2512 if (BYTES_BIG_ENDIAN)
2513 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2515 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2518 [(set_attr "type" "multiple")]
2521 (define_expand "aarch64_simd_combine<mode>"
2522 [(match_operand:<VDBL> 0 "register_operand")
2523 (match_operand:VDC 1 "register_operand")
2524 (match_operand:VDC 2 "register_operand")]
2527 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2528 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2531 [(set_attr "type" "multiple")]
2534 ;; <su><addsub>l<q>.
2536 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2537 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2538 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2539 (match_operand:VQW 1 "register_operand" "w")
2540 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2541 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2542 (match_operand:VQW 2 "register_operand" "w")
2545 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2546 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2549 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2550 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2551 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2552 (match_operand:VQW 1 "register_operand" "w")
2553 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2554 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2555 (match_operand:VQW 2 "register_operand" "w")
2558 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2559 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2563 (define_expand "aarch64_saddl2<mode>"
2564 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2565 (match_operand:VQW 1 "register_operand" "w")
2566 (match_operand:VQW 2 "register_operand" "w")]
2569 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2570 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2575 (define_expand "aarch64_uaddl2<mode>"
2576 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2577 (match_operand:VQW 1 "register_operand" "w")
2578 (match_operand:VQW 2 "register_operand" "w")]
2581 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2582 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2587 (define_expand "aarch64_ssubl2<mode>"
2588 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2589 (match_operand:VQW 1 "register_operand" "w")
2590 (match_operand:VQW 2 "register_operand" "w")]
2593 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2594 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2599 (define_expand "aarch64_usubl2<mode>"
2600 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2601 (match_operand:VQW 1 "register_operand" "w")
2602 (match_operand:VQW 2 "register_operand" "w")]
2605 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2606 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2611 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2612 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2613 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2614 (match_operand:VD_BHSI 1 "register_operand" "w"))
2616 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2618 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2619 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2622 ;; <su><addsub>w<q>.
2624 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2625 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2626 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2628 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2630 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2631 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2634 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2635 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2636 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2639 (match_operand:VQW 2 "register_operand" "w")
2640 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2642 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2643 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2646 (define_expand "aarch64_saddw2<mode>"
2647 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2648 (match_operand:<VWIDE> 1 "register_operand" "w")
2649 (match_operand:VQW 2 "register_operand" "w")]
2652 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2653 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2658 (define_expand "aarch64_uaddw2<mode>"
2659 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2660 (match_operand:<VWIDE> 1 "register_operand" "w")
2661 (match_operand:VQW 2 "register_operand" "w")]
2664 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2665 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2671 (define_expand "aarch64_ssubw2<mode>"
2672 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2673 (match_operand:<VWIDE> 1 "register_operand" "w")
2674 (match_operand:VQW 2 "register_operand" "w")]
2677 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2678 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2683 (define_expand "aarch64_usubw2<mode>"
2684 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2685 (match_operand:<VWIDE> 1 "register_operand" "w")
2686 (match_operand:VQW 2 "register_operand" "w")]
2689 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2690 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2695 ;; <su><r>h<addsub>.
2697 (define_insn "aarch64_<sur>h<addsub><mode>"
2698 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2699 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
2700 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
2703 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2704 [(set_attr "type" "neon_<addsub>_halve<q>")]
2707 ;; <r><addsub>hn<q>.
2709 (define_insn "aarch64_<sur><addsub>hn<mode>"
2710 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2711 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2712 (match_operand:VQN 2 "register_operand" "w")]
2715 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2716 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2719 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2720 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2721 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2722 (match_operand:VQN 2 "register_operand" "w")
2723 (match_operand:VQN 3 "register_operand" "w")]
2726 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2727 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2732 (define_insn "aarch64_pmul<mode>"
2733 [(set (match_operand:VB 0 "register_operand" "=w")
2734 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2735 (match_operand:VB 2 "register_operand" "w")]
2738 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2739 [(set_attr "type" "neon_mul_<Vetype><q>")]
2744 (define_insn "aarch64_<su_optab><optab><mode>"
2745 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2746 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2747 (match_operand:VSDQ_I 2 "register_operand" "w")))]
2749 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2750 [(set_attr "type" "neon_<optab><q>")]
2753 ;; suqadd and usqadd
2755 (define_insn "aarch64_<sur>qadd<mode>"
2756 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2757 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2758 (match_operand:VSDQ_I 2 "register_operand" "w")]
2761 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2762 [(set_attr "type" "neon_qadd<q>")]
2767 (define_insn "aarch64_sqmovun<mode>"
2768 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2769 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2772 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2773 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2776 ;; sqmovn and uqmovn
2778 (define_insn "aarch64_<sur>qmovn<mode>"
2779 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2780 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2783 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2784 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2789 (define_insn "aarch64_s<optab><mode>"
2790 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2792 (match_operand:VSDQ_I 1 "register_operand" "w")))]
2794 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2795 [(set_attr "type" "neon_<optab><q>")]
2800 (define_insn "aarch64_sq<r>dmulh<mode>"
2801 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2803 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2804 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2807 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2808 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
2813 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2814 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2816 [(match_operand:VDQHS 1 "register_operand" "w")
2818 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2819 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2823 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2824 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2825 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2828 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2829 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2831 [(match_operand:VDQHS 1 "register_operand" "w")
2833 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2834 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2838 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2839 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2840 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2843 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2844 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2846 [(match_operand:SD_HSI 1 "register_operand" "w")
2848 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2849 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2853 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2854 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2855 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2858 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2859 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2861 [(match_operand:SD_HSI 1 "register_operand" "w")
2863 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2864 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2868 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2869 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2870 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2875 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2876 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2878 (match_operand:<VWIDE> 1 "register_operand" "0")
2881 (sign_extend:<VWIDE>
2882 (match_operand:VSD_HSI 2 "register_operand" "w"))
2883 (sign_extend:<VWIDE>
2884 (match_operand:VSD_HSI 3 "register_operand" "w")))
2887 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2888 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
2893 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
2894 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2896 (match_operand:<VWIDE> 1 "register_operand" "0")
2899 (sign_extend:<VWIDE>
2900 (match_operand:VD_HSI 2 "register_operand" "w"))
2901 (sign_extend:<VWIDE>
2902 (vec_duplicate:VD_HSI
2904 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2905 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2910 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2912 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2914 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2917 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
2918 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2920 (match_operand:<VWIDE> 1 "register_operand" "0")
2923 (sign_extend:<VWIDE>
2924 (match_operand:VD_HSI 2 "register_operand" "w"))
2925 (sign_extend:<VWIDE>
2926 (vec_duplicate:VD_HSI
2928 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
2929 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2934 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2936 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2938 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2941 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
2942 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2944 (match_operand:<VWIDE> 1 "register_operand" "0")
2947 (sign_extend:<VWIDE>
2948 (match_operand:SD_HSI 2 "register_operand" "w"))
2949 (sign_extend:<VWIDE>
2951 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2952 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2957 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2959 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2961 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2964 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
2965 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2967 (match_operand:<VWIDE> 1 "register_operand" "0")
2970 (sign_extend:<VWIDE>
2971 (match_operand:SD_HSI 2 "register_operand" "w"))
2972 (sign_extend:<VWIDE>
2974 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
2975 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2980 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2982 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2984 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2989 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2990 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2992 (match_operand:<VWIDE> 1 "register_operand" "0")
2995 (sign_extend:<VWIDE>
2996 (match_operand:VD_HSI 2 "register_operand" "w"))
2997 (sign_extend:<VWIDE>
2998 (vec_duplicate:VD_HSI
2999 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3002 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3003 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3008 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3009 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3011 (match_operand:<VWIDE> 1 "register_operand" "0")
3014 (sign_extend:<VWIDE>
3016 (match_operand:VQ_HSI 2 "register_operand" "w")
3017 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3018 (sign_extend:<VWIDE>
3020 (match_operand:VQ_HSI 3 "register_operand" "w")
3024 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3025 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3028 (define_expand "aarch64_sqdmlal2<mode>"
3029 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3030 (match_operand:<VWIDE> 1 "register_operand" "w")
3031 (match_operand:VQ_HSI 2 "register_operand" "w")
3032 (match_operand:VQ_HSI 3 "register_operand" "w")]
3035 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3036 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3037 operands[2], operands[3], p));
3041 (define_expand "aarch64_sqdmlsl2<mode>"
3042 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3043 (match_operand:<VWIDE> 1 "register_operand" "w")
3044 (match_operand:VQ_HSI 2 "register_operand" "w")
3045 (match_operand:VQ_HSI 3 "register_operand" "w")]
3048 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3049 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3050 operands[2], operands[3], p));
3056 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3057 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3059 (match_operand:<VWIDE> 1 "register_operand" "0")
3062 (sign_extend:<VWIDE>
3064 (match_operand:VQ_HSI 2 "register_operand" "w")
3065 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3066 (sign_extend:<VWIDE>
3067 (vec_duplicate:<VHALF>
3069 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3070 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3075 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3077 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3079 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3082 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3083 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3085 (match_operand:<VWIDE> 1 "register_operand" "0")
3088 (sign_extend:<VWIDE>
3090 (match_operand:VQ_HSI 2 "register_operand" "w")
3091 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3092 (sign_extend:<VWIDE>
3093 (vec_duplicate:<VHALF>
3095 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3096 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3101 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3103 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3105 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3108 (define_expand "aarch64_sqdmlal2_lane<mode>"
3109 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3110 (match_operand:<VWIDE> 1 "register_operand" "w")
3111 (match_operand:VQ_HSI 2 "register_operand" "w")
3112 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3113 (match_operand:SI 4 "immediate_operand" "i")]
3116 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3117 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3118 operands[2], operands[3],
3123 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3124 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3125 (match_operand:<VWIDE> 1 "register_operand" "w")
3126 (match_operand:VQ_HSI 2 "register_operand" "w")
3127 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3128 (match_operand:SI 4 "immediate_operand" "i")]
3131 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3132 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3133 operands[2], operands[3],
3138 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3139 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3140 (match_operand:<VWIDE> 1 "register_operand" "w")
3141 (match_operand:VQ_HSI 2 "register_operand" "w")
3142 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3143 (match_operand:SI 4 "immediate_operand" "i")]
3146 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3147 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3148 operands[2], operands[3],
3153 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3154 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3155 (match_operand:<VWIDE> 1 "register_operand" "w")
3156 (match_operand:VQ_HSI 2 "register_operand" "w")
3157 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3158 (match_operand:SI 4 "immediate_operand" "i")]
3161 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3162 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3163 operands[2], operands[3],
3168 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3169 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3171 (match_operand:<VWIDE> 1 "register_operand" "0")
3174 (sign_extend:<VWIDE>
3176 (match_operand:VQ_HSI 2 "register_operand" "w")
3177 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3178 (sign_extend:<VWIDE>
3179 (vec_duplicate:<VHALF>
3180 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3183 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3184 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3187 (define_expand "aarch64_sqdmlal2_n<mode>"
3188 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3189 (match_operand:<VWIDE> 1 "register_operand" "w")
3190 (match_operand:VQ_HSI 2 "register_operand" "w")
3191 (match_operand:<VEL> 3 "register_operand" "w")]
3194 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3195 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3196 operands[2], operands[3],
3201 (define_expand "aarch64_sqdmlsl2_n<mode>"
3202 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3203 (match_operand:<VWIDE> 1 "register_operand" "w")
3204 (match_operand:VQ_HSI 2 "register_operand" "w")
3205 (match_operand:<VEL> 3 "register_operand" "w")]
3208 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3209 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3210 operands[2], operands[3],
3217 (define_insn "aarch64_sqdmull<mode>"
3218 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3221 (sign_extend:<VWIDE>
3222 (match_operand:VSD_HSI 1 "register_operand" "w"))
3223 (sign_extend:<VWIDE>
3224 (match_operand:VSD_HSI 2 "register_operand" "w")))
3227 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3228 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3233 (define_insn "aarch64_sqdmull_lane<mode>"
3234 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3237 (sign_extend:<VWIDE>
3238 (match_operand:VD_HSI 1 "register_operand" "w"))
3239 (sign_extend:<VWIDE>
3240 (vec_duplicate:VD_HSI
3242 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3243 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3248 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3249 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3251 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3254 (define_insn "aarch64_sqdmull_laneq<mode>"
3255 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3258 (sign_extend:<VWIDE>
3259 (match_operand:VD_HSI 1 "register_operand" "w"))
3260 (sign_extend:<VWIDE>
3261 (vec_duplicate:VD_HSI
3263 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3264 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3269 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3270 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3272 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3275 (define_insn "aarch64_sqdmull_lane<mode>"
3276 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3279 (sign_extend:<VWIDE>
3280 (match_operand:SD_HSI 1 "register_operand" "w"))
3281 (sign_extend:<VWIDE>
3283 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3284 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3289 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3290 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3292 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3295 (define_insn "aarch64_sqdmull_laneq<mode>"
3296 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3299 (sign_extend:<VWIDE>
3300 (match_operand:SD_HSI 1 "register_operand" "w"))
3301 (sign_extend:<VWIDE>
3303 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3304 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3309 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3310 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3312 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3317 (define_insn "aarch64_sqdmull_n<mode>"
3318 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3321 (sign_extend:<VWIDE>
3322 (match_operand:VD_HSI 1 "register_operand" "w"))
3323 (sign_extend:<VWIDE>
3324 (vec_duplicate:VD_HSI
3325 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3329 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3330 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3337 (define_insn "aarch64_sqdmull2<mode>_internal"
3338 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3341 (sign_extend:<VWIDE>
3343 (match_operand:VQ_HSI 1 "register_operand" "w")
3344 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3345 (sign_extend:<VWIDE>
3347 (match_operand:VQ_HSI 2 "register_operand" "w")
3352 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3353 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3356 (define_expand "aarch64_sqdmull2<mode>"
3357 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3358 (match_operand:VQ_HSI 1 "register_operand" "w")
3359 (match_operand:VQ_HSI 2 "register_operand" "w")]
3362 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3363 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3370 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3371 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3374 (sign_extend:<VWIDE>
3376 (match_operand:VQ_HSI 1 "register_operand" "w")
3377 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3378 (sign_extend:<VWIDE>
3379 (vec_duplicate:<VHALF>
3381 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3382 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3387 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3388 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3390 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3393 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3394 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3397 (sign_extend:<VWIDE>
3399 (match_operand:VQ_HSI 1 "register_operand" "w")
3400 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3401 (sign_extend:<VWIDE>
3402 (vec_duplicate:<VHALF>
3404 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3405 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3410 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3411 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3413 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3416 (define_expand "aarch64_sqdmull2_lane<mode>"
3417 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3418 (match_operand:VQ_HSI 1 "register_operand" "w")
3419 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3420 (match_operand:SI 3 "immediate_operand" "i")]
3423 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3424 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3425 operands[2], operands[3],
3430 (define_expand "aarch64_sqdmull2_laneq<mode>"
3431 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3432 (match_operand:VQ_HSI 1 "register_operand" "w")
3433 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3434 (match_operand:SI 3 "immediate_operand" "i")]
3437 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3438 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3439 operands[2], operands[3],
3446 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3447 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3450 (sign_extend:<VWIDE>
3452 (match_operand:VQ_HSI 1 "register_operand" "w")
3453 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3454 (sign_extend:<VWIDE>
3455 (vec_duplicate:<VHALF>
3456 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3460 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3461 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3464 (define_expand "aarch64_sqdmull2_n<mode>"
3465 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3466 (match_operand:VQ_HSI 1 "register_operand" "w")
3467 (match_operand:<VEL> 2 "register_operand" "w")]
3470 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3471 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3478 (define_insn "aarch64_<sur>shl<mode>"
3479 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3481 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3482 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3485 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3486 [(set_attr "type" "neon_shift_reg<q>")]
3492 (define_insn "aarch64_<sur>q<r>shl<mode>"
3493 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3495 [(match_operand:VSDQ_I 1 "register_operand" "w")
3496 (match_operand:VSDQ_I 2 "register_operand" "w")]
3499 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3500 [(set_attr "type" "neon_sat_shift_reg<q>")]
3505 (define_insn "aarch64_<sur>shll_n<mode>"
3506 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3507 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
3509 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
3513 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3514 if (INTVAL (operands[2]) == bit_width)
3516 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3519 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3521 [(set_attr "type" "neon_shift_imm_long")]
3526 (define_insn "aarch64_<sur>shll2_n<mode>"
3527 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3528 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3529 (match_operand:SI 2 "immediate_operand" "i")]
3533 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3534 if (INTVAL (operands[2]) == bit_width)
3536 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3539 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3541 [(set_attr "type" "neon_shift_imm_long")]
3546 (define_insn "aarch64_<sur>shr_n<mode>"
3547 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3548 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3550 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3553 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3554 [(set_attr "type" "neon_sat_shift_imm<q>")]
3559 (define_insn "aarch64_<sur>sra_n<mode>"
3560 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3561 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3562 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3564 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3567 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3568 [(set_attr "type" "neon_shift_acc<q>")]
3573 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3574 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3575 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3576 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3578 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
3581 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3582 [(set_attr "type" "neon_shift_imm<q>")]
3587 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3588 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3589 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3591 "aarch64_simd_shift_imm_<ve_mode>" "i")]
3594 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3595 [(set_attr "type" "neon_sat_shift_imm<q>")]
3601 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3602 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3603 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3605 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3608 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
3609 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3613 ;; cm(eq|ge|gt|lt|le)
3614 ;; Note, we have constraints for Dz and Z as different expanders
3615 ;; have different ideas of what should be passed to this pattern.
3617 (define_insn "aarch64_cm<optab><mode>"
3618 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3620 (COMPARISONS:<V_cmp_result>
3621 (match_operand:VDQ_I 1 "register_operand" "w,w")
3622 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3626 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3627 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3628 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
3631 (define_insn_and_split "aarch64_cm<optab>di"
3632 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3635 (match_operand:DI 1 "register_operand" "w,w,r")
3636 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3638 (clobber (reg:CC CC_REGNUM))]
3642 [(set (match_operand:DI 0 "register_operand")
3645 (match_operand:DI 1 "register_operand")
3646 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3649 /* If we are in the general purpose register file,
3650 we split to a sequence of comparison and store. */
3651 if (GP_REGNUM_P (REGNO (operands[0]))
3652 && GP_REGNUM_P (REGNO (operands[1])))
3654 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3655 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3656 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3657 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3660 /* Otherwise, we expand to a similar pattern which does not
3661 clobber CC_REGNUM. */
3663 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
3666 (define_insn "*aarch64_cm<optab>di"
3667 [(set (match_operand:DI 0 "register_operand" "=w,w")
3670 (match_operand:DI 1 "register_operand" "w,w")
3671 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3673 "TARGET_SIMD && reload_completed"
3675 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3676 cm<optab>\t%d0, %d1, #0"
3677 [(set_attr "type" "neon_compare, neon_compare_zero")]
3682 (define_insn "aarch64_cm<optab><mode>"
3683 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3685 (UCOMPARISONS:<V_cmp_result>
3686 (match_operand:VDQ_I 1 "register_operand" "w")
3687 (match_operand:VDQ_I 2 "register_operand" "w")
3690 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3691 [(set_attr "type" "neon_compare<q>")]
3694 (define_insn_and_split "aarch64_cm<optab>di"
3695 [(set (match_operand:DI 0 "register_operand" "=w,r")
3698 (match_operand:DI 1 "register_operand" "w,r")
3699 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3701 (clobber (reg:CC CC_REGNUM))]
3705 [(set (match_operand:DI 0 "register_operand")
3708 (match_operand:DI 1 "register_operand")
3709 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3712 /* If we are in the general purpose register file,
3713 we split to a sequence of comparison and store. */
3714 if (GP_REGNUM_P (REGNO (operands[0]))
3715 && GP_REGNUM_P (REGNO (operands[1])))
3717 machine_mode mode = CCmode;
3718 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3719 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3720 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3723 /* Otherwise, we expand to a similar pattern which does not
3724 clobber CC_REGNUM. */
3726 [(set_attr "type" "neon_compare,multiple")]
3729 (define_insn "*aarch64_cm<optab>di"
3730 [(set (match_operand:DI 0 "register_operand" "=w")
3733 (match_operand:DI 1 "register_operand" "w")
3734 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
3736 "TARGET_SIMD && reload_completed"
3737 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
3738 [(set_attr "type" "neon_compare")]
3743 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
3744 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
3745 ;; not (neg (eq (and x y) 0))
3746 ;; which is rewritten by simplify_rtx as
3747 ;; plus (eq (and x y) 0) -1.
3749 (define_insn "aarch64_cmtst<mode>"
3750 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3751 (plus:<V_cmp_result>
3754 (match_operand:VDQ_I 1 "register_operand" "w")
3755 (match_operand:VDQ_I 2 "register_operand" "w"))
3756 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
3757 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
3760 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3761 [(set_attr "type" "neon_tst<q>")]
3764 (define_insn_and_split "aarch64_cmtstdi"
3765 [(set (match_operand:DI 0 "register_operand" "=w,r")
3769 (match_operand:DI 1 "register_operand" "w,r")
3770 (match_operand:DI 2 "register_operand" "w,r"))
3772 (clobber (reg:CC CC_REGNUM))]
3776 [(set (match_operand:DI 0 "register_operand")
3780 (match_operand:DI 1 "register_operand")
3781 (match_operand:DI 2 "register_operand"))
3784 /* If we are in the general purpose register file,
3785 we split to a sequence of comparison and store. */
3786 if (GP_REGNUM_P (REGNO (operands[0]))
3787 && GP_REGNUM_P (REGNO (operands[1])))
3789 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3790 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3791 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3792 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3793 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3796 /* Otherwise, we expand to a similar pattern which does not
3797 clobber CC_REGNUM. */
3799 [(set_attr "type" "neon_tst,multiple")]
3802 (define_insn "*aarch64_cmtstdi"
3803 [(set (match_operand:DI 0 "register_operand" "=w")
3807 (match_operand:DI 1 "register_operand" "w")
3808 (match_operand:DI 2 "register_operand" "w"))
3811 "cmtst\t%d0, %d1, %d2"
3812 [(set_attr "type" "neon_tst")]
3815 ;; fcm(eq|ge|gt|le|lt)
3817 (define_insn "aarch64_cm<optab><mode>"
3818 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3820 (COMPARISONS:<V_cmp_result>
3821 (match_operand:VALLF 1 "register_operand" "w,w")
3822 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3826 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3827 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3828 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
3832 ;; Note we can also handle what would be fac(le|lt) by
3833 ;; generating fac(ge|gt).
3835 (define_insn "*aarch64_fac<optab><mode>"
3836 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3838 (FAC_COMPARISONS:<V_cmp_result>
3839 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3840 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3843 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3844 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
3849 (define_insn "aarch64_addp<mode>"
3850 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3852 [(match_operand:VD_BHSI 1 "register_operand" "w")
3853 (match_operand:VD_BHSI 2 "register_operand" "w")]
3856 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3857 [(set_attr "type" "neon_reduc_add<q>")]
3860 (define_insn "aarch64_addpdi"
3861 [(set (match_operand:DI 0 "register_operand" "=w")
3863 [(match_operand:V2DI 1 "register_operand" "w")]
3867 [(set_attr "type" "neon_reduc_add")]
3872 (define_insn "sqrt<mode>2"
3873 [(set (match_operand:VDQF 0 "register_operand" "=w")
3874 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3876 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
3877 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
3880 ;; Patterns for vector struct loads and stores.
3882 (define_insn "vec_load_lanesoi<mode>"
3883 [(set (match_operand:OI 0 "register_operand" "=w")
3884 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3885 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3888 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3889 [(set_attr "type" "neon_load2_2reg<q>")]
3892 (define_insn "aarch64_simd_ld2r<mode>"
3893 [(set (match_operand:OI 0 "register_operand" "=w")
3894 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3895 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3898 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3899 [(set_attr "type" "neon_load2_all_lanes<q>")]
3902 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
3903 [(set (match_operand:OI 0 "register_operand" "=w")
3904 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3905 (match_operand:OI 2 "register_operand" "0")
3906 (match_operand:SI 3 "immediate_operand" "i")
3907 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3910 "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1"
3911 [(set_attr "type" "neon_load2_one_lane")]
3914 (define_insn "vec_store_lanesoi<mode>"
3915 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3916 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3917 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3920 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3921 [(set_attr "type" "neon_store2_2reg<q>")]
3924 (define_insn "vec_store_lanesoi_lane<mode>"
3925 [(set (match_operand:<V_TWO_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
3926 (unspec:<V_TWO_ELEM> [(match_operand:OI 1 "register_operand" "w")
3927 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
3928 (match_operand:SI 2 "immediate_operand" "i")]
3931 "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0"
3932 [(set_attr "type" "neon_store3_one_lane<q>")]
3935 (define_insn "vec_load_lanesci<mode>"
3936 [(set (match_operand:CI 0 "register_operand" "=w")
3937 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3938 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3941 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3942 [(set_attr "type" "neon_load3_3reg<q>")]
3945 (define_insn "aarch64_simd_ld3r<mode>"
3946 [(set (match_operand:CI 0 "register_operand" "=w")
3947 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3948 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
3951 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3952 [(set_attr "type" "neon_load3_all_lanes<q>")]
3955 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
3956 [(set (match_operand:CI 0 "register_operand" "=w")
3957 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
3958 (match_operand:CI 2 "register_operand" "0")
3959 (match_operand:SI 3 "immediate_operand" "i")
3960 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3963 "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1"
3964 [(set_attr "type" "neon_load3_one_lane")]
3967 (define_insn "vec_store_lanesci<mode>"
3968 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
3969 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
3970 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3973 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3974 [(set_attr "type" "neon_store3_3reg<q>")]
3977 (define_insn "vec_store_lanesci_lane<mode>"
3978 [(set (match_operand:<V_THREE_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
3979 (unspec:<V_THREE_ELEM> [(match_operand:CI 1 "register_operand" "w")
3980 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
3981 (match_operand:SI 2 "immediate_operand" "i")]
3984 "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0"
3985 [(set_attr "type" "neon_store3_one_lane<q>")]
3988 (define_insn "vec_load_lanesxi<mode>"
3989 [(set (match_operand:XI 0 "register_operand" "=w")
3990 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
3991 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3994 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3995 [(set_attr "type" "neon_load4_4reg<q>")]
3998 (define_insn "aarch64_simd_ld4r<mode>"
3999 [(set (match_operand:XI 0 "register_operand" "=w")
4000 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4001 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4004 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4005 [(set_attr "type" "neon_load4_all_lanes<q>")]
4008 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4009 [(set (match_operand:XI 0 "register_operand" "=w")
4010 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4011 (match_operand:XI 2 "register_operand" "0")
4012 (match_operand:SI 3 "immediate_operand" "i")
4013 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4016 "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1"
4017 [(set_attr "type" "neon_load4_one_lane")]
4020 (define_insn "vec_store_lanesxi<mode>"
4021 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4022 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4023 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4026 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4027 [(set_attr "type" "neon_store4_4reg<q>")]
4030 (define_insn "vec_store_lanesxi_lane<mode>"
4031 [(set (match_operand:<V_FOUR_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4032 (unspec:<V_FOUR_ELEM> [(match_operand:XI 1 "register_operand" "w")
4033 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4034 (match_operand:SI 2 "immediate_operand" "i")]
4037 "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0"
4038 [(set_attr "type" "neon_store4_one_lane<q>")]
4041 ;; Reload patterns for AdvSIMD register list operands.
4043 (define_expand "mov<mode>"
4044 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
4045 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
4048 if (can_create_pseudo_p ())
4050 if (GET_CODE (operands[0]) != REG)
4051 operands[1] = force_reg (<MODE>mode, operands[1]);
4055 (define_insn "*aarch64_mov<mode>"
4056 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4057 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4059 && (register_operand (operands[0], <MODE>mode)
4060 || register_operand (operands[1], <MODE>mode))"
4063 switch (which_alternative)
4066 case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
4067 case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
4068 default: gcc_unreachable ();
4071 [(set_attr "type" "neon_move,neon_store<nregs>_<nregs>reg_q,\
4072 neon_load<nregs>_<nregs>reg_q")
4073 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4076 (define_insn "aarch64_be_ld1<mode>"
4077 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4078 (unspec:VALLDI [(match_operand:VALLDI 1 "aarch64_simd_struct_operand" "Utv")]
4081 "ld1\\t{%0<Vmtype>}, %1"
4082 [(set_attr "type" "neon_load1_1reg<q>")]
4085 (define_insn "aarch64_be_st1<mode>"
4086 [(set (match_operand:VALLDI 0 "aarch64_simd_struct_operand" "=Utv")
4087 (unspec:VALLDI [(match_operand:VALLDI 1 "register_operand" "w")]
4090 "st1\\t{%1<Vmtype>}, %0"
4091 [(set_attr "type" "neon_store1_1reg<q>")]
4095 [(set (match_operand:OI 0 "register_operand" "")
4096 (match_operand:OI 1 "register_operand" ""))]
4097 "TARGET_SIMD && reload_completed"
4098 [(set (match_dup 0) (match_dup 1))
4099 (set (match_dup 2) (match_dup 3))]
4101 int rdest = REGNO (operands[0]);
4102 int rsrc = REGNO (operands[1]);
4103 rtx dest[2], src[2];
4105 dest[0] = gen_rtx_REG (TFmode, rdest);
4106 src[0] = gen_rtx_REG (TFmode, rsrc);
4107 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4108 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4110 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
4114 [(set (match_operand:CI 0 "register_operand" "")
4115 (match_operand:CI 1 "register_operand" ""))]
4116 "TARGET_SIMD && reload_completed"
4117 [(set (match_dup 0) (match_dup 1))
4118 (set (match_dup 2) (match_dup 3))
4119 (set (match_dup 4) (match_dup 5))]
4121 int rdest = REGNO (operands[0]);
4122 int rsrc = REGNO (operands[1]);
4123 rtx dest[3], src[3];
4125 dest[0] = gen_rtx_REG (TFmode, rdest);
4126 src[0] = gen_rtx_REG (TFmode, rsrc);
4127 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4128 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4129 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4130 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4132 aarch64_simd_disambiguate_copy (operands, dest, src, 3);
4136 [(set (match_operand:XI 0 "register_operand" "")
4137 (match_operand:XI 1 "register_operand" ""))]
4138 "TARGET_SIMD && reload_completed"
4139 [(set (match_dup 0) (match_dup 1))
4140 (set (match_dup 2) (match_dup 3))
4141 (set (match_dup 4) (match_dup 5))
4142 (set (match_dup 6) (match_dup 7))]
4144 int rdest = REGNO (operands[0]);
4145 int rsrc = REGNO (operands[1]);
4146 rtx dest[4], src[4];
4148 dest[0] = gen_rtx_REG (TFmode, rdest);
4149 src[0] = gen_rtx_REG (TFmode, rsrc);
4150 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4151 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4152 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4153 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4154 dest[3] = gen_rtx_REG (TFmode, rdest + 3);
4155 src[3] = gen_rtx_REG (TFmode, rsrc + 3);
4157 aarch64_simd_disambiguate_copy (operands, dest, src, 4);
4160 (define_expand "aarch64_ld2r<mode>"
4161 [(match_operand:OI 0 "register_operand" "=w")
4162 (match_operand:DI 1 "register_operand" "w")
4163 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4166 machine_mode mode = <V_TWO_ELEM>mode;
4167 rtx mem = gen_rtx_MEM (mode, operands[1]);
4169 emit_insn (gen_aarch64_simd_ld2r<mode> (operands[0], mem));
4173 (define_expand "aarch64_ld3r<mode>"
4174 [(match_operand:CI 0 "register_operand" "=w")
4175 (match_operand:DI 1 "register_operand" "w")
4176 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4179 machine_mode mode = <V_THREE_ELEM>mode;
4180 rtx mem = gen_rtx_MEM (mode, operands[1]);
4182 emit_insn (gen_aarch64_simd_ld3r<mode> (operands[0], mem));
4186 (define_expand "aarch64_ld4r<mode>"
4187 [(match_operand:XI 0 "register_operand" "=w")
4188 (match_operand:DI 1 "register_operand" "w")
4189 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4192 machine_mode mode = <V_FOUR_ELEM>mode;
4193 rtx mem = gen_rtx_MEM (mode, operands[1]);
4195 emit_insn (gen_aarch64_simd_ld4r<mode> (operands[0],mem));
4199 (define_insn "aarch64_ld2<mode>_dreg"
4200 [(set (match_operand:OI 0 "register_operand" "=w")
4204 (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4206 (vec_duplicate:VD (const_int 0)))
4208 (unspec:VD [(match_dup 1)]
4210 (vec_duplicate:VD (const_int 0)))) 0))]
4212 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4213 [(set_attr "type" "neon_load2_2reg<q>")]
4216 (define_insn "aarch64_ld2<mode>_dreg"
4217 [(set (match_operand:OI 0 "register_operand" "=w")
4221 (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4225 (unspec:DX [(match_dup 1)]
4227 (const_int 0))) 0))]
4229 "ld1\\t{%S0.1d - %T0.1d}, %1"
4230 [(set_attr "type" "neon_load1_2reg<q>")]
4233 (define_insn "aarch64_ld3<mode>_dreg"
4234 [(set (match_operand:CI 0 "register_operand" "=w")
4239 (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4241 (vec_duplicate:VD (const_int 0)))
4243 (unspec:VD [(match_dup 1)]
4245 (vec_duplicate:VD (const_int 0))))
4247 (unspec:VD [(match_dup 1)]
4249 (vec_duplicate:VD (const_int 0)))) 0))]
4251 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4252 [(set_attr "type" "neon_load3_3reg<q>")]
4255 (define_insn "aarch64_ld3<mode>_dreg"
4256 [(set (match_operand:CI 0 "register_operand" "=w")
4261 (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4265 (unspec:DX [(match_dup 1)]
4269 (unspec:DX [(match_dup 1)]
4271 (const_int 0))) 0))]
4273 "ld1\\t{%S0.1d - %U0.1d}, %1"
4274 [(set_attr "type" "neon_load1_3reg<q>")]
4277 (define_insn "aarch64_ld4<mode>_dreg"
4278 [(set (match_operand:XI 0 "register_operand" "=w")
4283 (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4285 (vec_duplicate:VD (const_int 0)))
4287 (unspec:VD [(match_dup 1)]
4289 (vec_duplicate:VD (const_int 0))))
4292 (unspec:VD [(match_dup 1)]
4294 (vec_duplicate:VD (const_int 0)))
4296 (unspec:VD [(match_dup 1)]
4298 (vec_duplicate:VD (const_int 0))))) 0))]
4300 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4301 [(set_attr "type" "neon_load4_4reg<q>")]
4304 (define_insn "aarch64_ld4<mode>_dreg"
4305 [(set (match_operand:XI 0 "register_operand" "=w")
4310 (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4314 (unspec:DX [(match_dup 1)]
4319 (unspec:DX [(match_dup 1)]
4323 (unspec:DX [(match_dup 1)]
4325 (const_int 0)))) 0))]
4327 "ld1\\t{%S0.1d - %V0.1d}, %1"
4328 [(set_attr "type" "neon_load1_4reg<q>")]
4331 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4332 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4333 (match_operand:DI 1 "register_operand" "r")
4334 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4337 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4338 rtx mem = gen_rtx_MEM (mode, operands[1]);
4340 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4344 (define_expand "aarch64_ld1<VALL:mode>"
4345 [(match_operand:VALL 0 "register_operand")
4346 (match_operand:DI 1 "register_operand")]
4349 machine_mode mode = <VALL:MODE>mode;
4350 rtx mem = gen_rtx_MEM (mode, operands[1]);
4352 if (BYTES_BIG_ENDIAN)
4353 emit_insn (gen_aarch64_be_ld1<VALL:mode> (operands[0], mem));
4355 emit_move_insn (operands[0], mem);
4359 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4360 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4361 (match_operand:DI 1 "register_operand" "r")
4362 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4365 machine_mode mode = <VSTRUCT:MODE>mode;
4366 rtx mem = gen_rtx_MEM (mode, operands[1]);
4368 emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
4372 (define_expand "aarch64_ld2_lane<mode>"
4373 [(match_operand:OI 0 "register_operand" "=w")
4374 (match_operand:DI 1 "register_operand" "w")
4375 (match_operand:OI 2 "register_operand" "0")
4376 (match_operand:SI 3 "immediate_operand" "i")
4377 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4380 machine_mode mode = <V_TWO_ELEM>mode;
4381 rtx mem = gen_rtx_MEM (mode, operands[1]);
4383 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4385 emit_insn (gen_aarch64_vec_load_lanesoi_lane<mode> (operands[0],
4392 (define_expand "aarch64_ld3_lane<mode>"
4393 [(match_operand:CI 0 "register_operand" "=w")
4394 (match_operand:DI 1 "register_operand" "w")
4395 (match_operand:CI 2 "register_operand" "0")
4396 (match_operand:SI 3 "immediate_operand" "i")
4397 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4400 machine_mode mode = <V_THREE_ELEM>mode;
4401 rtx mem = gen_rtx_MEM (mode, operands[1]);
4403 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4405 emit_insn (gen_aarch64_vec_load_lanesci_lane<mode> (operands[0],
4412 (define_expand "aarch64_ld4_lane<mode>"
4413 [(match_operand:XI 0 "register_operand" "=w")
4414 (match_operand:DI 1 "register_operand" "w")
4415 (match_operand:XI 2 "register_operand" "0")
4416 (match_operand:SI 3 "immediate_operand" "i")
4417 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4420 machine_mode mode = <V_FOUR_ELEM>mode;
4421 rtx mem = gen_rtx_MEM (mode, operands[1]);
4423 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode),
4425 emit_insn (gen_aarch64_vec_load_lanesxi_lane<mode> (operands[0],
4434 ;; Expanders for builtins to extract vector registers from large
4435 ;; opaque integer modes.
4439 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4440 [(match_operand:VDC 0 "register_operand" "=w")
4441 (match_operand:VSTRUCT 1 "register_operand" "w")
4442 (match_operand:SI 2 "immediate_operand" "i")]
4445 int part = INTVAL (operands[2]);
4446 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4447 int offset = part * 16;
4449 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4450 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4456 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4457 [(match_operand:VQ 0 "register_operand" "=w")
4458 (match_operand:VSTRUCT 1 "register_operand" "w")
4459 (match_operand:SI 2 "immediate_operand" "i")]
4462 int part = INTVAL (operands[2]);
4463 int offset = part * 16;
4465 emit_move_insn (operands[0],
4466 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4470 ;; Permuted-store expanders for neon intrinsics.
4472 ;; Permute instructions
4476 (define_expand "vec_perm_const<mode>"
4477 [(match_operand:VALL 0 "register_operand")
4478 (match_operand:VALL 1 "register_operand")
4479 (match_operand:VALL 2 "register_operand")
4480 (match_operand:<V_cmp_result> 3)]
4483 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4484 operands[2], operands[3]))
4490 (define_expand "vec_perm<mode>"
4491 [(match_operand:VB 0 "register_operand")
4492 (match_operand:VB 1 "register_operand")
4493 (match_operand:VB 2 "register_operand")
4494 (match_operand:VB 3 "register_operand")]
4497 aarch64_expand_vec_perm (operands[0], operands[1],
4498 operands[2], operands[3]);
4502 (define_insn "aarch64_tbl1<mode>"
4503 [(set (match_operand:VB 0 "register_operand" "=w")
4504 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4505 (match_operand:VB 2 "register_operand" "w")]
4508 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
4509 [(set_attr "type" "neon_tbl1<q>")]
4512 ;; Two source registers.
4514 (define_insn "aarch64_tbl2v16qi"
4515 [(set (match_operand:V16QI 0 "register_operand" "=w")
4516 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4517 (match_operand:V16QI 2 "register_operand" "w")]
4520 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
4521 [(set_attr "type" "neon_tbl2_q")]
4524 (define_insn_and_split "aarch64_combinev16qi"
4525 [(set (match_operand:OI 0 "register_operand" "=w")
4526 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4527 (match_operand:V16QI 2 "register_operand" "w")]
4531 "&& reload_completed"
4534 aarch64_split_combinev16qi (operands);
4537 [(set_attr "type" "multiple")]
4540 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4541 [(set (match_operand:VALL 0 "register_operand" "=w")
4542 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4543 (match_operand:VALL 2 "register_operand" "w")]
4546 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4547 [(set_attr "type" "neon_permute<q>")]
4550 ;; Note immediate (third) operand is lane index not byte index.
4551 (define_insn "aarch64_ext<mode>"
4552 [(set (match_operand:VALL 0 "register_operand" "=w")
4553 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4554 (match_operand:VALL 2 "register_operand" "w")
4555 (match_operand:SI 3 "immediate_operand" "i")]
4559 operands[3] = GEN_INT (INTVAL (operands[3])
4560 * GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
4561 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
4563 [(set_attr "type" "neon_ext<q>")]
4566 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
4567 [(set (match_operand:VALL 0 "register_operand" "=w")
4568 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
4571 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
4572 [(set_attr "type" "neon_rev<q>")]
4575 (define_insn "aarch64_st2<mode>_dreg"
4576 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4577 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4578 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4581 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4582 [(set_attr "type" "neon_store2_2reg")]
4585 (define_insn "aarch64_st2<mode>_dreg"
4586 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4587 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4588 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4591 "st1\\t{%S1.1d - %T1.1d}, %0"
4592 [(set_attr "type" "neon_store1_2reg")]
4595 (define_insn "aarch64_st3<mode>_dreg"
4596 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4597 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4598 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4601 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4602 [(set_attr "type" "neon_store3_3reg")]
4605 (define_insn "aarch64_st3<mode>_dreg"
4606 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4607 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4608 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4611 "st1\\t{%S1.1d - %U1.1d}, %0"
4612 [(set_attr "type" "neon_store1_3reg")]
4615 (define_insn "aarch64_st4<mode>_dreg"
4616 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4617 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4618 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4621 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4622 [(set_attr "type" "neon_store4_4reg")]
4625 (define_insn "aarch64_st4<mode>_dreg"
4626 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4627 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4628 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4631 "st1\\t{%S1.1d - %V1.1d}, %0"
4632 [(set_attr "type" "neon_store1_4reg")]
4635 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4636 [(match_operand:DI 0 "register_operand" "r")
4637 (match_operand:VSTRUCT 1 "register_operand" "w")
4638 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4641 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4642 rtx mem = gen_rtx_MEM (mode, operands[0]);
4644 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4648 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4649 [(match_operand:DI 0 "register_operand" "r")
4650 (match_operand:VSTRUCT 1 "register_operand" "w")
4651 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4654 machine_mode mode = <VSTRUCT:MODE>mode;
4655 rtx mem = gen_rtx_MEM (mode, operands[0]);
4657 emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4661 (define_expand "aarch64_st2_lane<VQ:mode>"
4662 [(match_operand:DI 0 "register_operand" "r")
4663 (match_operand:OI 1 "register_operand" "w")
4664 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4665 (match_operand:SI 2 "immediate_operand")]
4668 machine_mode mode = <V_TWO_ELEM>mode;
4669 rtx mem = gen_rtx_MEM (mode, operands[0]);
4670 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4672 emit_insn (gen_vec_store_lanesoi_lane<VQ:mode> (mem,
4678 (define_expand "aarch64_st3_lane<VQ:mode>"
4679 [(match_operand:DI 0 "register_operand" "r")
4680 (match_operand:CI 1 "register_operand" "w")
4681 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4682 (match_operand:SI 2 "immediate_operand")]
4685 machine_mode mode = <V_THREE_ELEM>mode;
4686 rtx mem = gen_rtx_MEM (mode, operands[0]);
4687 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4689 emit_insn (gen_vec_store_lanesci_lane<VQ:mode> (mem,
4695 (define_expand "aarch64_st4_lane<VQ:mode>"
4696 [(match_operand:DI 0 "register_operand" "r")
4697 (match_operand:XI 1 "register_operand" "w")
4698 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4699 (match_operand:SI 2 "immediate_operand")]
4702 machine_mode mode = <V_FOUR_ELEM>mode;
4703 rtx mem = gen_rtx_MEM (mode, operands[0]);
4704 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4706 emit_insn (gen_vec_store_lanesxi_lane<VQ:mode> (mem,
4712 (define_expand "aarch64_st1<VALL:mode>"
4713 [(match_operand:DI 0 "register_operand")
4714 (match_operand:VALL 1 "register_operand")]
4717 machine_mode mode = <VALL:MODE>mode;
4718 rtx mem = gen_rtx_MEM (mode, operands[0]);
4720 if (BYTES_BIG_ENDIAN)
4721 emit_insn (gen_aarch64_be_st1<VALL:mode> (mem, operands[1]));
4723 emit_move_insn (mem, operands[1]);
4727 ;; Expander for builtins to insert vector registers into large
4728 ;; opaque integer modes.
4730 ;; Q-register list. We don't need a D-reg inserter as we zero
4731 ;; extend them in arm_neon.h and insert the resulting Q-regs.
4733 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4734 [(match_operand:VSTRUCT 0 "register_operand" "+w")
4735 (match_operand:VSTRUCT 1 "register_operand" "0")
4736 (match_operand:VQ 2 "register_operand" "w")
4737 (match_operand:SI 3 "immediate_operand" "i")]
4740 int part = INTVAL (operands[3]);
4741 int offset = part * 16;
4743 emit_move_insn (operands[0], operands[1]);
4744 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4749 ;; Standard pattern name vec_init<mode>.
4751 (define_expand "vec_init<mode>"
4752 [(match_operand:VALL 0 "register_operand" "")
4753 (match_operand 1 "" "")]
4756 aarch64_expand_vector_init (operands[0], operands[1]);
4760 (define_insn "*aarch64_simd_ld1r<mode>"
4761 [(set (match_operand:VALL 0 "register_operand" "=w")
4763 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4765 "ld1r\\t{%0.<Vtype>}, %1"
4766 [(set_attr "type" "neon_load1_all_lanes")]
4769 (define_insn "aarch64_frecpe<mode>"
4770 [(set (match_operand:VDQF 0 "register_operand" "=w")
4771 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4774 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4775 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
4778 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4779 [(set (match_operand:GPF 0 "register_operand" "=w")
4780 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4783 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
4784 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
4787 (define_insn "aarch64_frecps<mode>"
4788 [(set (match_operand:VALLF 0 "register_operand" "=w")
4789 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4790 (match_operand:VALLF 2 "register_operand" "w")]
4793 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4794 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
4797 (define_insn "aarch64_urecpe<mode>"
4798 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
4799 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
4802 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
4803 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
4805 ;; Standard pattern name vec_extract<mode>.
4807 (define_expand "vec_extract<mode>"
4808 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
4809 (match_operand:VALL 1 "register_operand" "")
4810 (match_operand:SI 2 "immediate_operand" "")]
4814 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
4820 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
4821 [(set (match_operand:V16QI 0 "register_operand" "=w")
4822 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
4823 (match_operand:V16QI 2 "register_operand" "w")]
4825 "TARGET_SIMD && TARGET_CRYPTO"
4826 "aes<aes_op>\\t%0.16b, %2.16b"
4827 [(set_attr "type" "crypto_aese")]
4830 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
4831 [(set (match_operand:V16QI 0 "register_operand" "=w")
4832 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
4834 "TARGET_SIMD && TARGET_CRYPTO"
4835 "aes<aesmc_op>\\t%0.16b, %1.16b"
4836 [(set_attr "type" "crypto_aesmc")]
4841 (define_insn "aarch64_crypto_sha1hsi"
4842 [(set (match_operand:SI 0 "register_operand" "=w")
4843 (unspec:SI [(match_operand:SI 1
4844 "register_operand" "w")]
4846 "TARGET_SIMD && TARGET_CRYPTO"
4848 [(set_attr "type" "crypto_sha1_fast")]
4851 (define_insn "aarch64_crypto_sha1su1v4si"
4852 [(set (match_operand:V4SI 0 "register_operand" "=w")
4853 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4854 (match_operand:V4SI 2 "register_operand" "w")]
4856 "TARGET_SIMD && TARGET_CRYPTO"
4857 "sha1su1\\t%0.4s, %2.4s"
4858 [(set_attr "type" "crypto_sha1_fast")]
4861 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
4862 [(set (match_operand:V4SI 0 "register_operand" "=w")
4863 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4864 (match_operand:SI 2 "register_operand" "w")
4865 (match_operand:V4SI 3 "register_operand" "w")]
4867 "TARGET_SIMD && TARGET_CRYPTO"
4868 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
4869 [(set_attr "type" "crypto_sha1_slow")]
4872 (define_insn "aarch64_crypto_sha1su0v4si"
4873 [(set (match_operand:V4SI 0 "register_operand" "=w")
4874 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4875 (match_operand:V4SI 2 "register_operand" "w")
4876 (match_operand:V4SI 3 "register_operand" "w")]
4878 "TARGET_SIMD && TARGET_CRYPTO"
4879 "sha1su0\\t%0.4s, %2.4s, %3.4s"
4880 [(set_attr "type" "crypto_sha1_xor")]
4885 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
4886 [(set (match_operand:V4SI 0 "register_operand" "=w")
4887 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4888 (match_operand:V4SI 2 "register_operand" "w")
4889 (match_operand:V4SI 3 "register_operand" "w")]
4891 "TARGET_SIMD && TARGET_CRYPTO"
4892 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
4893 [(set_attr "type" "crypto_sha256_slow")]
4896 (define_insn "aarch64_crypto_sha256su0v4si"
4897 [(set (match_operand:V4SI 0 "register_operand" "=w")
4898 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4899 (match_operand:V4SI 2 "register_operand" "w")]
4901 "TARGET_SIMD &&TARGET_CRYPTO"
4902 "sha256su0\\t%0.4s, %2.4s"
4903 [(set_attr "type" "crypto_sha256_fast")]
4906 (define_insn "aarch64_crypto_sha256su1v4si"
4907 [(set (match_operand:V4SI 0 "register_operand" "=w")
4908 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
4909 (match_operand:V4SI 2 "register_operand" "w")
4910 (match_operand:V4SI 3 "register_operand" "w")]
4912 "TARGET_SIMD &&TARGET_CRYPTO"
4913 "sha256su1\\t%0.4s, %2.4s, %3.4s"
4914 [(set_attr "type" "crypto_sha256_slow")]
4919 (define_insn "aarch64_crypto_pmulldi"
4920 [(set (match_operand:TI 0 "register_operand" "=w")
4921 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
4922 (match_operand:DI 2 "register_operand" "w")]
4924 "TARGET_SIMD && TARGET_CRYPTO"
4925 "pmull\\t%0.1q, %1.1d, %2.1d"
4926 [(set_attr "type" "neon_mul_d_long")]
4929 (define_insn "aarch64_crypto_pmullv2di"
4930 [(set (match_operand:TI 0 "register_operand" "=w")
4931 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
4932 (match_operand:V2DI 2 "register_operand" "w")]
4934 "TARGET_SIMD && TARGET_CRYPTO"
4935 "pmull2\\t%0.1q, %1.2d, %2.2d"
4936 [(set_attr "type" "neon_mul_d_long")]