1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
88 (LAST_SAVED_REGNUM 63)
92 ;; Defined only to make the DWARF description simpler.
110 ;; Scratch register used by stack clash protection to calculate
111 ;; SVE CFA offsets during probing.
112 (STACK_CLASH_SVE_CFA_REGNUM 11)
113 ;; Scratch registers for prologue/epilogue use.
116 ;; A couple of call-clobbered registers that we need to reserve when
117 ;; tracking speculation this is not ABI, so is subject to change.
118 (SPECULATION_SCRATCH_REGNUM 14)
119 (SPECULATION_TRACKER_REGNUM 15)
120 ;; Scratch registers used in frame layout.
128 (define_c_enum "unspec" [
162 UNSPEC_GOTSMALLPIC28K
244 UNSPEC_SPECULATION_TRACKER
246 UNSPEC_TTEST ; Represent transaction test.
249 (define_c_enum "unspecv" [
250 UNSPECV_EH_RETURN ; Represent EH_RETURN
251 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
252 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
253 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
254 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
255 UNSPECV_BLOCKAGE ; Represent a blockage
256 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
257 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
258 UNSPECV_BTI_NOARG ; Represent BTI.
259 UNSPECV_BTI_C ; Represent BTI c.
260 UNSPECV_BTI_J ; Represent BTI j.
261 UNSPECV_BTI_JC ; Represent BTI jc.
262 UNSPECV_TSTART ; Represent transaction start.
263 UNSPECV_TCOMMIT ; Represent transaction commit.
264 UNSPECV_TCANCEL ; Represent transaction cancel.
268 ;; These constants are used as a const_int in various SVE unspecs
269 ;; to indicate whether the governing predicate is known to be a PTRUE.
271 [; Indicates that the predicate might not be a PTRUE.
272 (SVE_MAYBE_NOT_PTRUE 0)
274 ; Indicates that the predicate is known to be a PTRUE.
275 (SVE_KNOWN_PTRUE 1)])
277 ;; These constants are used as a const_int in predicated SVE FP arithmetic
278 ;; to indicate whether the operation is allowed to make additional lanes
279 ;; active without worrying about the effect on faulting behavior.
281 [; Indicates either that all lanes are active or that the instruction may
282 ; operate on inactive inputs even if doing so could induce a fault.
285 ; Indicates that some lanes might be inactive and that the instruction
286 ; must not operate on inactive inputs if doing so could induce a fault.
289 ;; If further include files are added the defintion of MD_INCLUDES
292 (include "constraints.md")
293 (include "predicates.md")
294 (include "iterators.md")
296 ;; -------------------------------------------------------------------
297 ;; Instruction types and attributes
298 ;; -------------------------------------------------------------------
300 ; The "type" attribute is included here from AArch32 backend to be able
301 ; to share pipeline descriptions.
302 (include "../arm/types.md")
304 ;; It is important to set the fp or simd attributes to yes when a pattern
305 ;; alternative uses the FP or SIMD register files, usually signified by use of
306 ;; the 'w' constraint. This will ensure that the alternative will be
307 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
308 ;; architecture extensions. If all the alternatives in a pattern use the
309 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
312 ;; Attributes of the architecture required to support the instruction (or
313 ;; alternative). This attribute is used to compute attribute "enabled", use type
314 ;; "any" to enable an alternative in all cases.
316 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
318 (define_enum_attr "arch" "arches" (const_string "any"))
320 ;; [For compatibility with Arm in pipeline models]
321 ;; Attribute that specifies whether or not the instruction touches fp
323 ;; Note that this attribute is not used anywhere in either the arm or aarch64
324 ;; backends except in the scheduling description for xgene1. In that
325 ;; scheduling description this attribute is used to subclass the load_4 and
327 (define_attr "fp" "no,yes"
329 (eq_attr "arch" "fp")
331 (const_string "no")))
333 (define_attr "arch_enabled" "no,yes"
336 (eq_attr "arch" "any")
338 (and (eq_attr "arch" "rcpc8_4")
339 (match_test "AARCH64_ISA_RCPC8_4"))
341 (and (eq_attr "arch" "fp")
342 (match_test "TARGET_FLOAT"))
344 (and (eq_attr "arch" "simd")
345 (match_test "TARGET_SIMD"))
347 (and (eq_attr "arch" "fp16")
348 (match_test "TARGET_FP_F16INST"))
350 (and (eq_attr "arch" "sve")
351 (match_test "TARGET_SVE")))
353 (const_string "no")))
355 ;; Attribute that controls whether an alternative is enabled or not.
356 ;; Currently it is only used to disable alternatives which touch fp or simd
357 ;; registers when -mgeneral-regs-only is specified or to require a special
358 ;; architecture support.
359 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
361 ;; Attribute that specifies whether we are dealing with a branch to a
362 ;; label that is far away, i.e. further away than the maximum/minimum
363 ;; representable in a signed 21-bits number.
366 (define_attr "far_branch" "" (const_int 0))
368 ;; Attribute that specifies whether the alternative uses MOVPRFX.
369 (define_attr "movprfx" "no,yes" (const_string "no"))
371 (define_attr "length" ""
372 (cond [(eq_attr "movprfx" "yes")
376 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
377 ;; no predicated insns.
378 (define_attr "predicated" "yes,no" (const_string "no"))
380 ;; Set to true on an insn that requires the speculation tracking state to be
381 ;; in the tracking register before the insn issues. Otherwise the compiler
382 ;; may chose to hold the tracking state encoded in SP.
383 (define_attr "speculation_barrier" "true,false" (const_string "false"))
385 ;; -------------------------------------------------------------------
386 ;; Pipeline descriptions and scheduling
387 ;; -------------------------------------------------------------------
390 (include "aarch64-tune.md")
393 (include "../arm/cortex-a53.md")
394 (include "../arm/cortex-a57.md")
395 (include "../arm/exynos-m1.md")
396 (include "falkor.md")
397 (include "saphira.md")
398 (include "thunderx.md")
399 (include "../arm/xgene1.md")
400 (include "thunderx2t99.md")
401 (include "tsv110.md")
403 ;; -------------------------------------------------------------------
404 ;; Jumps and other miscellaneous insns
405 ;; -------------------------------------------------------------------
407 (define_insn "indirect_jump"
408 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
411 [(set_attr "type" "branch")]
415 [(set (pc) (label_ref (match_operand 0 "" "")))]
418 [(set_attr "type" "branch")]
421 (define_expand "cbranch<mode>4"
422 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
423 [(match_operand:GPI 1 "register_operand")
424 (match_operand:GPI 2 "aarch64_plus_operand")])
425 (label_ref (match_operand 3 "" ""))
429 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
431 operands[2] = const0_rtx;
435 (define_expand "cbranch<mode>4"
436 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
437 [(match_operand:GPF 1 "register_operand")
438 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
439 (label_ref (match_operand 3 "" ""))
443 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
445 operands[2] = const0_rtx;
449 (define_expand "cbranchcc4"
450 [(set (pc) (if_then_else
451 (match_operator 0 "aarch64_comparison_operator"
452 [(match_operand 1 "cc_register")
453 (match_operand 2 "const0_operand")])
454 (label_ref (match_operand 3 "" ""))
459 (define_insn "ccmp<mode>"
460 [(set (match_operand:CC 1 "cc_register" "")
462 (match_operator 4 "aarch64_comparison_operator"
463 [(match_operand 0 "cc_register" "")
466 (match_operand:GPI 2 "register_operand" "r,r,r")
467 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
468 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
471 ccmp\\t%<w>2, %<w>3, %k5, %m4
472 ccmp\\t%<w>2, %3, %k5, %m4
473 ccmn\\t%<w>2, #%n3, %k5, %m4"
474 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
477 (define_insn "fccmp<mode>"
478 [(set (match_operand:CCFP 1 "cc_register" "")
480 (match_operator 4 "aarch64_comparison_operator"
481 [(match_operand 0 "cc_register" "")
484 (match_operand:GPF 2 "register_operand" "w")
485 (match_operand:GPF 3 "register_operand" "w"))
486 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
488 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
489 [(set_attr "type" "fccmp<s>")]
492 (define_insn "fccmpe<mode>"
493 [(set (match_operand:CCFPE 1 "cc_register" "")
495 (match_operator 4 "aarch64_comparison_operator"
496 [(match_operand 0 "cc_register" "")
499 (match_operand:GPF 2 "register_operand" "w")
500 (match_operand:GPF 3 "register_operand" "w"))
501 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
503 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
504 [(set_attr "type" "fccmp<s>")]
507 ;; Expansion of signed mod by a power of 2 using CSNEG.
508 ;; For x0 % n where n is a power of 2 produce:
510 ;; and x0, x0, #(n - 1)
511 ;; and x1, x1, #(n - 1)
512 ;; csneg x0, x0, x1, mi
514 (define_expand "mod<mode>3"
515 [(match_operand:GPI 0 "register_operand")
516 (match_operand:GPI 1 "register_operand")
517 (match_operand:GPI 2 "const_int_operand")]
520 HOST_WIDE_INT val = INTVAL (operands[2]);
523 || exact_log2 (val) <= 0
524 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
527 rtx mask = GEN_INT (val - 1);
529 /* In the special case of x0 % 2 we can do the even shorter:
535 rtx masked = gen_reg_rtx (<MODE>mode);
536 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
537 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
538 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
539 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
543 rtx neg_op = gen_reg_rtx (<MODE>mode);
544 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
546 /* Extract the condition register and mode. */
547 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
548 rtx cc_reg = SET_DEST (cmp);
549 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
551 rtx masked_pos = gen_reg_rtx (<MODE>mode);
552 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
554 rtx masked_neg = gen_reg_rtx (<MODE>mode);
555 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
557 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
558 masked_neg, masked_pos));
563 (define_insn "condjump"
564 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
565 [(match_operand 1 "cc_register" "") (const_int 0)])
566 (label_ref (match_operand 2 "" ""))
570 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
571 but the "." is required for SVE conditions. */
572 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
573 if (get_attr_length (insn) == 8)
574 return aarch64_gen_far_branch (operands, 2, "Lbcond",
575 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
577 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
579 [(set_attr "type" "branch")
581 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
582 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
585 (set (attr "far_branch")
586 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
587 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
592 ;; For a 24-bit immediate CST we can optimize the compare for equality
593 ;; and branch sequence from:
595 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
599 ;; sub x0, x1, #(CST & 0xfff000)
600 ;; subs x0, x0, #(CST & 0x000fff)
602 (define_insn_and_split "*compare_condjump<GPI:mode>"
603 [(set (pc) (if_then_else (EQL
604 (match_operand:GPI 0 "register_operand" "r")
605 (match_operand:GPI 1 "aarch64_imm24" "n"))
606 (label_ref:P (match_operand 2 "" ""))
608 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
609 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
610 && !reload_completed"
615 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
616 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
617 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
618 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
619 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
620 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
621 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
623 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
628 (define_expand "casesi"
629 [(match_operand:SI 0 "register_operand") ; Index
630 (match_operand:SI 1 "const_int_operand") ; Lower bound
631 (match_operand:SI 2 "const_int_operand") ; Total range
632 (match_operand:DI 3 "" "") ; Table label
633 (match_operand:DI 4 "" "")] ; Out of range label
636 if (operands[1] != const0_rtx)
638 rtx reg = gen_reg_rtx (SImode);
640 /* Canonical RTL says that if you have:
644 then this should be emitted as:
648 The use of trunc_int_for_mode ensures that the resulting
649 constant can be represented in SImode, this is important
650 for the corner case where operand[1] is INT_MIN. */
652 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
654 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
655 (operands[1], SImode))
656 operands[1] = force_reg (SImode, operands[1]);
657 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
661 if (!aarch64_plus_operand (operands[2], SImode))
662 operands[2] = force_reg (SImode, operands[2]);
663 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
665 operands[0], operands[2], operands[4]));
667 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
669 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
671 operands[2] = gen_rtx_MEM (DImode, operands[2]);
672 MEM_READONLY_P (operands[2]) = 1;
673 MEM_NOTRAP_P (operands[2]) = 1;
674 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
679 (define_expand "casesi_dispatch"
681 [(set (pc) (match_operand:DI 0 ""))
682 (clobber (reg:CC CC_REGNUM))
683 (clobber (match_scratch:DI 2))
684 (clobber (match_scratch:DI 3))
685 (use (label_ref:DI (match_operand 1 "")))])]
688 (define_insn "*casesi_dispatch"
691 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
692 (match_operand:SI 1 "register_operand" "r")]
694 (clobber (reg:CC CC_REGNUM))
695 (clobber (match_scratch:DI 3 "=r"))
696 (clobber (match_scratch:DI 4 "=r"))
697 (use (label_ref:DI (match_operand 2 "" "")))])]
700 return aarch64_output_casesi (operands);
702 [(set_attr "length" "16")
703 (set_attr "type" "branch")]
707 [(unspec[(const_int 0)] UNSPEC_NOP)]
710 [(set_attr "type" "no_insn")]
713 (define_insn "prefetch"
714 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
715 (match_operand:QI 1 "const_int_operand" "")
716 (match_operand:QI 2 "const_int_operand" ""))]
719 const char * pftype[2][4] =
721 {"prfm\\tPLDL1STRM, %0",
722 "prfm\\tPLDL3KEEP, %0",
723 "prfm\\tPLDL2KEEP, %0",
724 "prfm\\tPLDL1KEEP, %0"},
725 {"prfm\\tPSTL1STRM, %0",
726 "prfm\\tPSTL3KEEP, %0",
727 "prfm\\tPSTL2KEEP, %0",
728 "prfm\\tPSTL1KEEP, %0"},
731 int locality = INTVAL (operands[2]);
733 gcc_assert (IN_RANGE (locality, 0, 3));
735 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
736 the address into a DImode MEM so that aarch64_print_operand knows
738 operands[0] = gen_rtx_MEM (DImode, operands[0]);
739 return pftype[INTVAL(operands[1])][locality];
741 [(set_attr "type" "load_4")]
745 [(trap_if (const_int 1) (const_int 8))]
748 [(set_attr "type" "trap")])
750 (define_expand "prologue"
751 [(clobber (const_int 0))]
754 aarch64_expand_prologue ();
759 (define_expand "epilogue"
760 [(clobber (const_int 0))]
763 aarch64_expand_epilogue (false);
768 (define_expand "sibcall_epilogue"
769 [(clobber (const_int 0))]
772 aarch64_expand_epilogue (true);
777 (define_insn "*do_return"
781 if (aarch64_return_address_signing_enabled ()
783 && !crtl->calls_eh_return)
785 if (aarch64_ra_sign_key == AARCH64_KEY_B)
792 [(set_attr "type" "branch")]
795 (define_expand "return"
797 "aarch64_use_return_insn_p ()"
801 (define_insn "simple_return"
805 [(set_attr "type" "branch")]
808 (define_insn "*cb<optab><mode>1"
809 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
811 (label_ref (match_operand 1 "" ""))
813 "!aarch64_track_speculation"
815 if (get_attr_length (insn) == 8)
816 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
818 return "<cbz>\\t%<w>0, %l1";
820 [(set_attr "type" "branch")
822 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
823 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
826 (set (attr "far_branch")
827 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
828 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
833 (define_insn "*tb<optab><mode>1"
834 [(set (pc) (if_then_else
835 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
838 "aarch64_simd_shift_imm_<mode>" "n"))
840 (label_ref (match_operand 2 "" ""))
842 (clobber (reg:CC CC_REGNUM))]
843 "!aarch64_track_speculation"
845 if (get_attr_length (insn) == 8)
847 if (get_attr_far_branch (insn) == 1)
848 return aarch64_gen_far_branch (operands, 2, "Ltb",
849 "<inv_tb>\\t%<w>0, %1, ");
852 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
853 return "tst\t%<w>0, %1\;<bcond>\t%l2";
857 return "<tbz>\t%<w>0, %1, %l2";
859 [(set_attr "type" "branch")
861 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
862 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
865 (set (attr "far_branch")
866 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
867 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
873 (define_insn "*cb<optab><mode>1"
874 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
876 (label_ref (match_operand 1 "" ""))
878 (clobber (reg:CC CC_REGNUM))]
879 "!aarch64_track_speculation"
881 if (get_attr_length (insn) == 8)
883 if (get_attr_far_branch (insn) == 1)
884 return aarch64_gen_far_branch (operands, 1, "Ltb",
885 "<inv_tb>\\t%<w>0, <sizem1>, ");
889 uint64_t val = ((uint64_t) 1)
890 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
891 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
892 output_asm_insn (buf, operands);
893 return "<bcond>\t%l1";
897 return "<tbz>\t%<w>0, <sizem1>, %l1";
899 [(set_attr "type" "branch")
901 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
902 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
905 (set (attr "far_branch")
906 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
907 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
912 ;; -------------------------------------------------------------------
913 ;; Subroutine calls and sibcalls
914 ;; -------------------------------------------------------------------
916 (define_expand "call"
918 [(call (match_operand 0 "memory_operand")
919 (match_operand 1 "general_operand"))
920 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
921 (clobber (reg:DI LR_REGNUM))])]
925 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false);
930 (define_insn "*call_insn"
931 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
932 (match_operand 1 "" ""))
933 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
934 (clobber (reg:DI LR_REGNUM))]
939 [(set_attr "type" "call, call")]
942 (define_expand "call_value"
944 [(set (match_operand 0 "")
945 (call (match_operand 1 "memory_operand")
946 (match_operand 2 "general_operand")))
947 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
948 (clobber (reg:DI LR_REGNUM))])]
952 aarch64_expand_call (operands[0], operands[1], operands[3], false);
957 (define_insn "*call_value_insn"
958 [(set (match_operand 0 "" "")
959 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
960 (match_operand 2 "" "")))
961 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
962 (clobber (reg:DI LR_REGNUM))]
967 [(set_attr "type" "call, call")]
970 (define_expand "sibcall"
972 [(call (match_operand 0 "memory_operand")
973 (match_operand 1 "general_operand"))
974 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
978 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true);
983 (define_expand "sibcall_value"
985 [(set (match_operand 0 "")
986 (call (match_operand 1 "memory_operand")
987 (match_operand 2 "general_operand")))
988 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
992 aarch64_expand_call (operands[0], operands[1], operands[3], true);
997 (define_insn "*sibcall_insn"
998 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
999 (match_operand 1 ""))
1000 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI)
1002 "SIBLING_CALL_P (insn)"
1006 [(set_attr "type" "branch, branch")]
1009 (define_insn "*sibcall_value_insn"
1010 [(set (match_operand 0 "")
1012 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1013 (match_operand 2 "")))
1014 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI)
1016 "SIBLING_CALL_P (insn)"
1020 [(set_attr "type" "branch, branch")]
1023 ;; Call subroutine returning any type.
1025 (define_expand "untyped_call"
1026 [(parallel [(call (match_operand 0 "")
1028 (match_operand 1 "")
1029 (match_operand 2 "")])]
1034 /* Untyped calls always use the default ABI. It's only possible to use
1035 ABI variants if we know the type of the target function. */
1036 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));
1038 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1040 rtx set = XVECEXP (operands[2], 0, i);
1041 emit_move_insn (SET_DEST (set), SET_SRC (set));
1044 /* The optimizer does not know that the call sets the function value
1045 registers we stored in the result block. We avoid problems by
1046 claiming that all hard registers are used and clobbered at this
1048 emit_insn (gen_blockage ());
1052 ;; -------------------------------------------------------------------
1054 ;; -------------------------------------------------------------------
1056 (define_expand "mov<mode>"
1057 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1058 (match_operand:SHORT 1 "general_operand"))]
1061 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1062 operands[1] = force_reg (<MODE>mode, operands[1]);
1064 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1066 aarch64_expand_mov_immediate (operands[0], operands[1]);
1072 (define_insn "*mov<mode>_aarch64"
1073 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1074 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1075 "(register_operand (operands[0], <MODE>mode)
1076 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1078 switch (which_alternative)
1081 return "mov\t%w0, %w1";
1083 return "mov\t%w0, %1";
1085 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1088 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1090 return "ldr<size>\t%w0, %1";
1092 return "ldr\t%<size>0, %1";
1094 return "str<size>\t%w1, %0";
1096 return "str\t%<size>1, %0";
1098 return "umov\t%w0, %1.<v>[0]";
1100 return "dup\t%0.<Vallxd>, %w1";
1102 return "dup\t%<Vetype>0, %1.<v>[0]";
1107 ;; The "mov_imm" type for CNT is just a placeholder.
1108 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1109 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1110 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1113 (define_expand "mov<mode>"
1114 [(set (match_operand:GPI 0 "nonimmediate_operand")
1115 (match_operand:GPI 1 "general_operand"))]
1118 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0])
1119 && CONST_INT_P (operands[1]) && <MODE>mode == DImode
1120 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1123 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1124 operands[1] = force_reg (<MODE>mode, operands[1]);
1126 /* FIXME: RR we still need to fix up what we are doing with
1127 symbol_refs and other types of constants. */
1128 if (CONSTANT_P (operands[1])
1129 && !CONST_INT_P (operands[1]))
1131 aarch64_expand_mov_immediate (operands[0], operands[1]);
1137 (define_insn_and_split "*movsi_aarch64"
1138 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
1139 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1140 "(register_operand (operands[0], SImode)
1141 || aarch64_reg_or_zero (operands[1], SImode))"
1148 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1158 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1159 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1160 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1163 aarch64_expand_mov_immediate (operands[0], operands[1]);
1166 ;; The "mov_imm" type for CNT is just a placeholder.
1167 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1168 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1169 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1172 (define_insn_and_split "*movdi_aarch64"
1173 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1174 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1175 "(register_operand (operands[0], DImode)
1176 || aarch64_reg_or_zero (operands[1], DImode))"
1184 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1194 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1195 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1196 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1199 aarch64_expand_mov_immediate (operands[0], operands[1]);
1202 ;; The "mov_imm" type for CNTD is just a placeholder.
1203 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1204 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1206 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1209 (define_insn "insv_imm<mode>"
1210 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1212 (match_operand:GPI 1 "const_int_operand" "n"))
1213 (match_operand:GPI 2 "const_int_operand" "n"))]
1214 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1215 && UINTVAL (operands[1]) % 16 == 0"
1216 "movk\\t%<w>0, %X2, lsl %1"
1217 [(set_attr "type" "mov_imm")]
1220 (define_expand "movti"
1221 [(set (match_operand:TI 0 "nonimmediate_operand")
1222 (match_operand:TI 1 "general_operand"))]
1225 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1226 operands[1] = force_reg (TImode, operands[1]);
1228 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1230 emit_move_insn (gen_lowpart (DImode, operands[0]),
1231 gen_lowpart (DImode, operands[1]));
1232 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1238 (define_insn "*movti_aarch64"
1239 [(set (match_operand:TI 0
1240 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1242 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1243 "(register_operand (operands[0], TImode)
1244 || aarch64_reg_or_zero (operands[1], TImode))"
1249 mov\\t%0.16b, %1.16b
1255 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1256 load_16,store_16,store_16,\
1258 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1259 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1262 ;; Split a TImode register-register or register-immediate move into
1263 ;; its component DImode pieces, taking care to handle overlapping
1264 ;; source and dest registers.
1266 [(set (match_operand:TI 0 "register_operand" "")
1267 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1268 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1271 aarch64_split_128bit_move (operands[0], operands[1]);
1275 (define_expand "mov<mode>"
1276 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1277 (match_operand:GPF_TF_F16 1 "general_operand"))]
1282 aarch64_err_no_fpadvsimd (<MODE>mode);
1286 if (GET_CODE (operands[0]) == MEM
1287 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1288 && aarch64_float_const_zero_rtx_p (operands[1])))
1289 operands[1] = force_reg (<MODE>mode, operands[1]);
1293 (define_insn "*movhf_aarch64"
1294 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1295 (match_operand:HF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1296 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1297 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1303 mov\\t%0.h[0], %1.h[0]
1305 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1311 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1312 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1313 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1316 (define_insn "*movsf_aarch64"
1317 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1318 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1319 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1320 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1327 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1334 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1335 f_loads,f_stores,load_4,store_4,mov_reg,\
1337 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1340 (define_insn "*movdf_aarch64"
1341 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1342 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1343 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1344 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1351 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1358 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1359 f_loadd,f_stored,load_8,store_8,mov_reg,\
1361 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1365 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1366 (match_operand:GPF_HF 1 "general_operand"))]
1367 "can_create_pseudo_p ()
1368 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1369 && !aarch64_float_const_representable_p (operands[1])
1370 && aarch64_float_const_rtx_p (operands[1])"
1373 unsigned HOST_WIDE_INT ival;
1374 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1377 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1378 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1379 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1384 (define_insn "*movtf_aarch64"
1385 [(set (match_operand:TF 0
1386 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1388 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1389 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1390 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1392 mov\\t%0.16b, %1.16b
1403 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1404 f_loadd,f_stored,load_16,store_16,store_16")
1405 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1406 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1410 [(set (match_operand:TF 0 "register_operand" "")
1411 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1412 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1415 aarch64_split_128bit_move (operands[0], operands[1]);
1422 ;; 2 is size of copy in bytes
1425 (define_expand "cpymemdi"
1426 [(match_operand:BLK 0 "memory_operand")
1427 (match_operand:BLK 1 "memory_operand")
1428 (match_operand:DI 2 "immediate_operand")
1429 (match_operand:DI 3 "immediate_operand")]
1432 if (aarch64_expand_cpymem (operands))
1438 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1439 ;; fairly lax checking on the second memory operation.
1440 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1441 [(set (match_operand:SX 0 "register_operand" "=r,w")
1442 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1443 (set (match_operand:SX2 2 "register_operand" "=r,w")
1444 (match_operand:SX2 3 "memory_operand" "m,m"))]
1445 "rtx_equal_p (XEXP (operands[3], 0),
1446 plus_constant (Pmode,
1447 XEXP (operands[1], 0),
1448 GET_MODE_SIZE (<SX:MODE>mode)))"
1452 [(set_attr "type" "load_8,neon_load1_2reg")
1453 (set_attr "arch" "*,fp")]
1456 ;; Storing different modes that can still be merged
1457 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1458 [(set (match_operand:DX 0 "register_operand" "=r,w")
1459 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1460 (set (match_operand:DX2 2 "register_operand" "=r,w")
1461 (match_operand:DX2 3 "memory_operand" "m,m"))]
1462 "rtx_equal_p (XEXP (operands[3], 0),
1463 plus_constant (Pmode,
1464 XEXP (operands[1], 0),
1465 GET_MODE_SIZE (<DX:MODE>mode)))"
1469 [(set_attr "type" "load_16,neon_load1_2reg")
1470 (set_attr "arch" "*,fp")]
1473 (define_insn "load_pair_dw_tftf"
1474 [(set (match_operand:TF 0 "register_operand" "=w")
1475 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1476 (set (match_operand:TF 2 "register_operand" "=w")
1477 (match_operand:TF 3 "memory_operand" "m"))]
1479 && rtx_equal_p (XEXP (operands[3], 0),
1480 plus_constant (Pmode,
1481 XEXP (operands[1], 0),
1482 GET_MODE_SIZE (TFmode)))"
1483 "ldp\\t%q0, %q2, %1"
1484 [(set_attr "type" "neon_ldp_q")
1485 (set_attr "fp" "yes")]
1488 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1489 ;; fairly lax checking on the second memory operation.
1490 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1491 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1492 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1493 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1494 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1495 "rtx_equal_p (XEXP (operands[2], 0),
1496 plus_constant (Pmode,
1497 XEXP (operands[0], 0),
1498 GET_MODE_SIZE (<SX:MODE>mode)))"
1502 [(set_attr "type" "store_8,neon_store1_2reg")
1503 (set_attr "arch" "*,fp")]
1506 ;; Storing different modes that can still be merged
1507 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1508 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1509 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1510 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1511 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1512 "rtx_equal_p (XEXP (operands[2], 0),
1513 plus_constant (Pmode,
1514 XEXP (operands[0], 0),
1515 GET_MODE_SIZE (<DX:MODE>mode)))"
1519 [(set_attr "type" "store_16,neon_store1_2reg")
1520 (set_attr "arch" "*,fp")]
1523 (define_insn "store_pair_dw_tftf"
1524 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1525 (match_operand:TF 1 "register_operand" "w"))
1526 (set (match_operand:TF 2 "memory_operand" "=m")
1527 (match_operand:TF 3 "register_operand" "w"))]
1529 rtx_equal_p (XEXP (operands[2], 0),
1530 plus_constant (Pmode,
1531 XEXP (operands[0], 0),
1532 GET_MODE_SIZE (TFmode)))"
1533 "stp\\t%q1, %q3, %0"
1534 [(set_attr "type" "neon_stp_q")
1535 (set_attr "fp" "yes")]
1538 ;; Load pair with post-index writeback. This is primarily used in function
1540 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1542 [(set (match_operand:P 0 "register_operand" "=k")
1543 (plus:P (match_operand:P 1 "register_operand" "0")
1544 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1545 (set (match_operand:GPI 2 "register_operand" "=r")
1546 (mem:GPI (match_dup 1)))
1547 (set (match_operand:GPI 3 "register_operand" "=r")
1548 (mem:GPI (plus:P (match_dup 1)
1549 (match_operand:P 5 "const_int_operand" "n"))))])]
1550 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1551 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1552 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1555 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1557 [(set (match_operand:P 0 "register_operand" "=k")
1558 (plus:P (match_operand:P 1 "register_operand" "0")
1559 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1560 (set (match_operand:GPF 2 "register_operand" "=w")
1561 (mem:GPF (match_dup 1)))
1562 (set (match_operand:GPF 3 "register_operand" "=w")
1563 (mem:GPF (plus:P (match_dup 1)
1564 (match_operand:P 5 "const_int_operand" "n"))))])]
1565 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1566 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1567 [(set_attr "type" "neon_load1_2reg")]
1570 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1572 [(set (match_operand:P 0 "register_operand" "=k")
1573 (plus:P (match_operand:P 1 "register_operand" "0")
1574 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1575 (set (match_operand:TX 2 "register_operand" "=w")
1576 (mem:TX (match_dup 1)))
1577 (set (match_operand:TX 3 "register_operand" "=w")
1578 (mem:TX (plus:P (match_dup 1)
1579 (match_operand:P 5 "const_int_operand" "n"))))])]
1580 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1581 "ldp\\t%q2, %q3, [%1], %4"
1582 [(set_attr "type" "neon_ldp_q")]
1585 ;; Store pair with pre-index writeback. This is primarily used in function
1587 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1589 [(set (match_operand:P 0 "register_operand" "=&k")
1590 (plus:P (match_operand:P 1 "register_operand" "0")
1591 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1592 (set (mem:GPI (plus:P (match_dup 0)
1594 (match_operand:GPI 2 "register_operand" "r"))
1595 (set (mem:GPI (plus:P (match_dup 0)
1596 (match_operand:P 5 "const_int_operand" "n")))
1597 (match_operand:GPI 3 "register_operand" "r"))])]
1598 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1599 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1600 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1603 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1605 [(set (match_operand:P 0 "register_operand" "=&k")
1606 (plus:P (match_operand:P 1 "register_operand" "0")
1607 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1608 (set (mem:GPF (plus:P (match_dup 0)
1610 (match_operand:GPF 2 "register_operand" "w"))
1611 (set (mem:GPF (plus:P (match_dup 0)
1612 (match_operand:P 5 "const_int_operand" "n")))
1613 (match_operand:GPF 3 "register_operand" "w"))])]
1614 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1615 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1616 [(set_attr "type" "neon_store1_2reg<q>")]
1619 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1621 [(set (match_operand:P 0 "register_operand" "=&k")
1622 (plus:P (match_operand:P 1 "register_operand" "0")
1623 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1624 (set (mem:TX (plus:P (match_dup 0)
1626 (match_operand:TX 2 "register_operand" "w"))
1627 (set (mem:TX (plus:P (match_dup 0)
1628 (match_operand:P 5 "const_int_operand" "n")))
1629 (match_operand:TX 3 "register_operand" "w"))])]
1631 && INTVAL (operands[5])
1632 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1633 "stp\\t%q2, %q3, [%0, %4]!"
1634 [(set_attr "type" "neon_stp_q")]
1637 ;; -------------------------------------------------------------------
1638 ;; Sign/Zero extension
1639 ;; -------------------------------------------------------------------
1641 (define_expand "<optab>sidi2"
1642 [(set (match_operand:DI 0 "register_operand")
1643 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1647 (define_insn "*extendsidi2_aarch64"
1648 [(set (match_operand:DI 0 "register_operand" "=r,r")
1649 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1654 [(set_attr "type" "extend,load_4")]
1657 (define_insn "*load_pair_extendsidi2_aarch64"
1658 [(set (match_operand:DI 0 "register_operand" "=r")
1659 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1660 (set (match_operand:DI 2 "register_operand" "=r")
1661 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1662 "rtx_equal_p (XEXP (operands[3], 0),
1663 plus_constant (Pmode,
1664 XEXP (operands[1], 0),
1665 GET_MODE_SIZE (SImode)))"
1666 "ldpsw\\t%0, %2, %1"
1667 [(set_attr "type" "load_8")]
1670 (define_insn "*zero_extendsidi2_aarch64"
1671 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1672 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1681 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1682 (set_attr "arch" "*,*,fp,fp,fp,fp")]
1685 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1686 [(set (match_operand:DI 0 "register_operand" "=r,w")
1687 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1688 (set (match_operand:DI 2 "register_operand" "=r,w")
1689 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1690 "rtx_equal_p (XEXP (operands[3], 0),
1691 plus_constant (Pmode,
1692 XEXP (operands[1], 0),
1693 GET_MODE_SIZE (SImode)))"
1697 [(set_attr "type" "load_8,neon_load1_2reg")
1698 (set_attr "arch" "*,fp")]
1701 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1702 [(set (match_operand:GPI 0 "register_operand")
1703 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1707 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1708 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1709 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1712 sxt<SHORT:size>\t%<GPI:w>0, %w1
1713 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1714 [(set_attr "type" "extend,load_4")]
1717 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1718 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1719 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1722 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1723 ldr<SHORT:size>\t%w0, %1
1724 ldr\t%<SHORT:size>0, %1"
1725 [(set_attr "type" "logic_imm,load_4,f_loads")
1726 (set_attr "arch" "*,*,fp")]
1729 (define_expand "<optab>qihi2"
1730 [(set (match_operand:HI 0 "register_operand")
1731 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1735 (define_insn "*extendqihi2_aarch64"
1736 [(set (match_operand:HI 0 "register_operand" "=r,r")
1737 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1742 [(set_attr "type" "extend,load_4")]
1745 (define_insn "*zero_extendqihi2_aarch64"
1746 [(set (match_operand:HI 0 "register_operand" "=r,r")
1747 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1752 [(set_attr "type" "logic_imm,load_4")]
1755 ;; -------------------------------------------------------------------
1756 ;; Simple arithmetic
1757 ;; -------------------------------------------------------------------
1759 (define_expand "add<mode>3"
1761 (match_operand:GPI 0 "register_operand")
1762 (plus:GPI (match_operand:GPI 1 "register_operand")
1763 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1766 /* If operands[1] is a subreg extract the inner RTX. */
1767 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1769 /* If the constant is too large for a single instruction and isn't frame
1770 based, split off the immediate so it is available for CSE. */
1771 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1772 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1773 && can_create_pseudo_p ()
1775 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1776 operands[2] = force_reg (<MODE>mode, operands[2]);
1777 /* Expand polynomial additions now if the destination is the stack
1778 pointer, since we don't want to use that as a temporary. */
1779 else if (operands[0] == stack_pointer_rtx
1780 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1782 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1783 operands[2], NULL_RTX, NULL_RTX);
1788 (define_insn "*add<mode>3_aarch64"
1790 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1792 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1793 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1796 add\\t%<w>0, %<w>1, %2
1797 add\\t%<w>0, %<w>1, %<w>2
1798 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1799 sub\\t%<w>0, %<w>1, #%n2
1801 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1802 * return aarch64_output_sve_addvl_addpl (operands[2]);"
1803 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1804 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1805 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1808 ;; zero_extend version of above
1809 (define_insn "*addsi3_aarch64_uxtw"
1811 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1813 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1814 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1819 sub\\t%w0, %w1, #%n2
1821 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1824 ;; If there's a free register, and we can load the constant with a
1825 ;; single instruction, do so. This has a chance to improve scheduling.
1827 [(match_scratch:GPI 3 "r")
1828 (set (match_operand:GPI 0 "register_operand")
1830 (match_operand:GPI 1 "register_operand")
1831 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1832 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1833 [(set (match_dup 3) (match_dup 2))
1834 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1838 [(match_scratch:SI 3 "r")
1839 (set (match_operand:DI 0 "register_operand")
1842 (match_operand:SI 1 "register_operand")
1843 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1844 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1845 [(set (match_dup 3) (match_dup 2))
1846 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1849 ;; After peephole2 has had a chance to run, split any remaining long
1850 ;; additions into two add immediates.
1852 [(set (match_operand:GPI 0 "register_operand")
1854 (match_operand:GPI 1 "register_operand")
1855 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1856 "epilogue_completed"
1857 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1858 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1860 HOST_WIDE_INT i = INTVAL (operands[2]);
1861 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1862 operands[3] = GEN_INT (i - s);
1863 operands[4] = GEN_INT (s);
1867 ;; Match addition of polynomial offsets that require one temporary, for which
1868 ;; we can use the early-clobbered destination register. This is a separate
1869 ;; pattern so that the early clobber doesn't affect register allocation
1870 ;; for other forms of addition. However, we still need to provide an
1871 ;; all-register alternative, in case the offset goes out of range after
1872 ;; elimination. For completeness we might as well provide all GPR-based
1873 ;; alternatives from the main pattern.
1875 ;; We don't have a pattern for additions requiring two temporaries since at
1876 ;; present LRA doesn't allow new scratches to be added during elimination.
1877 ;; Such offsets should be rare anyway.
1879 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1880 ;; here would go away. We could just handle all polynomial constants in
1882 (define_insn_and_split "*add<mode>3_poly_1"
1884 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
1886 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1887 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
1888 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1890 add\\t%<w>0, %<w>1, %2
1891 add\\t%<w>0, %<w>1, %<w>2
1892 sub\\t%<w>0, %<w>1, #%n2
1894 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1895 * return aarch64_output_sve_addvl_addpl (operands[2]);
1897 "&& epilogue_completed
1898 && !reg_overlap_mentioned_p (operands[0], operands[1])
1899 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1902 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1903 operands[2], operands[0], NULL_RTX);
1906 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1907 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
1911 [(set (match_operand:DI 0 "register_operand")
1914 (match_operand:SI 1 "register_operand")
1915 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1916 "epilogue_completed"
1917 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1918 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1920 HOST_WIDE_INT i = INTVAL (operands[2]);
1921 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1922 operands[3] = GEN_INT (i - s);
1923 operands[4] = GEN_INT (s);
1924 operands[5] = gen_lowpart (SImode, operands[0]);
1928 (define_expand "addv<mode>4"
1929 [(match_operand:GPI 0 "register_operand")
1930 (match_operand:GPI 1 "register_operand")
1931 (match_operand:GPI 2 "aarch64_plus_operand")
1932 (label_ref (match_operand 3 "" ""))]
1935 if (CONST_INT_P (operands[2]))
1936 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1939 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1940 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1945 (define_expand "uaddv<mode>4"
1946 [(match_operand:GPI 0 "register_operand")
1947 (match_operand:GPI 1 "register_operand")
1948 (match_operand:GPI 2 "register_operand")
1949 (label_ref (match_operand 3 "" ""))]
1952 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1953 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1958 (define_expand "addti3"
1959 [(set (match_operand:TI 0 "register_operand")
1960 (plus:TI (match_operand:TI 1 "register_operand")
1961 (match_operand:TI 2 "aarch64_reg_or_imm")))]
1964 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1966 aarch64_addti_scratch_regs (operands[1], operands[2],
1967 &low_dest, &op1_low, &op2_low,
1968 &high_dest, &op1_high, &op2_high);
1970 if (op2_low == const0_rtx)
1973 if (!aarch64_pluslong_operand (op2_high, DImode))
1974 op2_high = force_reg (DImode, op2_high);
1975 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1979 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1980 force_reg (DImode, op2_low)));
1981 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1982 force_reg (DImode, op2_high)));
1985 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1986 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1991 (define_expand "addvti4"
1992 [(match_operand:TI 0 "register_operand")
1993 (match_operand:TI 1 "register_operand")
1994 (match_operand:TI 2 "aarch64_reg_or_imm")
1995 (label_ref (match_operand 3 "" ""))]
1998 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2000 aarch64_addti_scratch_regs (operands[1], operands[2],
2001 &low_dest, &op1_low, &op2_low,
2002 &high_dest, &op1_high, &op2_high);
2004 if (op2_low == const0_rtx)
2007 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
2008 force_reg (DImode, op2_high)));
2012 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2013 force_reg (DImode, op2_low)));
2014 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2015 force_reg (DImode, op2_high)));
2018 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2019 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2021 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2025 (define_expand "uaddvti4"
2026 [(match_operand:TI 0 "register_operand")
2027 (match_operand:TI 1 "register_operand")
2028 (match_operand:TI 2 "aarch64_reg_or_imm")
2029 (label_ref (match_operand 3 "" ""))]
2032 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2034 aarch64_addti_scratch_regs (operands[1], operands[2],
2035 &low_dest, &op1_low, &op2_low,
2036 &high_dest, &op1_high, &op2_high);
2038 if (op2_low == const0_rtx)
2041 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2042 force_reg (DImode, op2_high)));
2046 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2047 force_reg (DImode, op2_low)));
2048 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2049 force_reg (DImode, op2_high)));
2052 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2053 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2055 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2059 (define_insn "add<mode>3_compare0"
2060 [(set (reg:CC_NZ CC_REGNUM)
2062 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2063 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2065 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2066 (plus:GPI (match_dup 1) (match_dup 2)))]
2069 adds\\t%<w>0, %<w>1, %<w>2
2070 adds\\t%<w>0, %<w>1, %2
2071 subs\\t%<w>0, %<w>1, #%n2"
2072 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2075 ;; zero_extend version of above
2076 (define_insn "*addsi3_compare0_uxtw"
2077 [(set (reg:CC_NZ CC_REGNUM)
2079 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2080 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2082 (set (match_operand:DI 0 "register_operand" "=r,r,r")
2083 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2086 adds\\t%w0, %w1, %w2
2088 subs\\t%w0, %w1, #%n2"
2089 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2092 (define_insn "*add<mode>3_compareC_cconly"
2093 [(set (reg:CC_C CC_REGNUM)
2096 (match_operand:GPI 0 "register_operand" "r,r,r")
2097 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2104 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2107 (define_insn "add<mode>3_compareC"
2108 [(set (reg:CC_C CC_REGNUM)
2111 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2112 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2114 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2115 (plus:GPI (match_dup 1) (match_dup 2)))]
2118 adds\\t%<w>0, %<w>1, %<w>2
2119 adds\\t%<w>0, %<w>1, %2
2120 subs\\t%<w>0, %<w>1, #%n2"
2121 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2124 (define_insn "*add<mode>3_compareV_cconly_imm"
2125 [(set (reg:CC_V CC_REGNUM)
2128 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2129 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2133 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2134 "INTVAL (operands[1]) == INTVAL (operands[2])"
2138 [(set_attr "type" "alus_imm")]
2141 (define_insn "*add<mode>3_compareV_cconly"
2142 [(set (reg:CC_V CC_REGNUM)
2145 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2146 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2147 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2149 "cmn\\t%<w>0, %<w>1"
2150 [(set_attr "type" "alus_sreg")]
2153 (define_insn "add<mode>3_compareV_imm"
2154 [(set (reg:CC_V CC_REGNUM)
2158 (match_operand:GPI 1 "register_operand" "rk,rk"))
2159 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2161 (plus:GPI (match_dup 1) (match_dup 2)))))
2162 (set (match_operand:GPI 0 "register_operand" "=r,r")
2163 (plus:GPI (match_dup 1) (match_dup 2)))]
2166 adds\\t%<w>0, %<w>1, %<w>2
2167 subs\\t%<w>0, %<w>1, #%n2"
2168 [(set_attr "type" "alus_imm,alus_imm")]
2171 (define_insn "add<mode>3_compareV"
2172 [(set (reg:CC_V CC_REGNUM)
2175 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2176 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2177 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2178 (set (match_operand:GPI 0 "register_operand" "=r")
2179 (plus:GPI (match_dup 1) (match_dup 2)))]
2181 "adds\\t%<w>0, %<w>1, %<w>2"
2182 [(set_attr "type" "alus_sreg")]
2185 (define_insn "*adds_shift_imm_<mode>"
2186 [(set (reg:CC_NZ CC_REGNUM)
2188 (plus:GPI (ASHIFT:GPI
2189 (match_operand:GPI 1 "register_operand" "r")
2190 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2191 (match_operand:GPI 3 "register_operand" "r"))
2193 (set (match_operand:GPI 0 "register_operand" "=r")
2194 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2197 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2198 [(set_attr "type" "alus_shift_imm")]
2201 (define_insn "*subs_shift_imm_<mode>"
2202 [(set (reg:CC_NZ CC_REGNUM)
2204 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2206 (match_operand:GPI 2 "register_operand" "r")
2207 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2209 (set (match_operand:GPI 0 "register_operand" "=r")
2210 (minus:GPI (match_dup 1)
2211 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2213 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2214 [(set_attr "type" "alus_shift_imm")]
2217 (define_insn "*adds_mul_imm_<mode>"
2218 [(set (reg:CC_NZ CC_REGNUM)
2221 (match_operand:GPI 1 "register_operand" "r")
2222 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2223 (match_operand:GPI 3 "register_operand" "r"))
2225 (set (match_operand:GPI 0 "register_operand" "=r")
2226 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2229 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2230 [(set_attr "type" "alus_shift_imm")]
2233 (define_insn "*subs_mul_imm_<mode>"
2234 [(set (reg:CC_NZ CC_REGNUM)
2236 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2238 (match_operand:GPI 2 "register_operand" "r")
2239 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2241 (set (match_operand:GPI 0 "register_operand" "=r")
2242 (minus:GPI (match_dup 1)
2243 (mult:GPI (match_dup 2) (match_dup 3))))]
2245 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2246 [(set_attr "type" "alus_shift_imm")]
2249 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2250 [(set (reg:CC_NZ CC_REGNUM)
2253 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2254 (match_operand:GPI 2 "register_operand" "rk"))
2256 (set (match_operand:GPI 0 "register_operand" "=r")
2257 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2259 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2260 [(set_attr "type" "alus_ext")]
2263 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2264 [(set (reg:CC_NZ CC_REGNUM)
2266 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2268 (match_operand:ALLX 2 "register_operand" "r")))
2270 (set (match_operand:GPI 0 "register_operand" "=r")
2271 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2273 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2274 [(set_attr "type" "alus_ext")]
2277 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2278 [(set (reg:CC_NZ CC_REGNUM)
2280 (plus:GPI (ashift:GPI
2282 (match_operand:ALLX 1 "register_operand" "r"))
2283 (match_operand 2 "aarch64_imm3" "Ui3"))
2284 (match_operand:GPI 3 "register_operand" "rk"))
2286 (set (match_operand:GPI 0 "register_operand" "=rk")
2287 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2291 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2292 [(set_attr "type" "alus_ext")]
2295 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2296 [(set (reg:CC_NZ CC_REGNUM)
2298 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2301 (match_operand:ALLX 2 "register_operand" "r"))
2302 (match_operand 3 "aarch64_imm3" "Ui3")))
2304 (set (match_operand:GPI 0 "register_operand" "=rk")
2305 (minus:GPI (match_dup 1)
2306 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2309 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2310 [(set_attr "type" "alus_ext")]
2313 (define_insn "*adds_<optab><mode>_multp2"
2314 [(set (reg:CC_NZ CC_REGNUM)
2316 (plus:GPI (ANY_EXTRACT:GPI
2317 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2318 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2319 (match_operand 3 "const_int_operand" "n")
2321 (match_operand:GPI 4 "register_operand" "rk"))
2323 (set (match_operand:GPI 0 "register_operand" "=r")
2324 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2328 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2329 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2330 [(set_attr "type" "alus_ext")]
2333 (define_insn "*subs_<optab><mode>_multp2"
2334 [(set (reg:CC_NZ CC_REGNUM)
2336 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2338 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2339 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2340 (match_operand 3 "const_int_operand" "n")
2343 (set (match_operand:GPI 0 "register_operand" "=r")
2344 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2345 (mult:GPI (match_dup 1) (match_dup 2))
2348 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2349 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2350 [(set_attr "type" "alus_ext")]
2353 (define_insn "*add<mode>3nr_compare0"
2354 [(set (reg:CC_NZ CC_REGNUM)
2356 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2357 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2364 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2367 (define_insn "aarch64_sub<mode>_compare0"
2368 [(set (reg:CC_NZ CC_REGNUM)
2370 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2371 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2374 "cmp\\t%<w>0, %<w>1"
2375 [(set_attr "type" "alus_sreg")]
2378 (define_insn "*compare_neg<mode>"
2379 [(set (reg:CC_Z CC_REGNUM)
2381 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2382 (match_operand:GPI 1 "register_operand" "r")))]
2384 "cmn\\t%<w>1, %<w>0"
2385 [(set_attr "type" "alus_sreg")]
2388 (define_insn "*add_<shift>_<mode>"
2389 [(set (match_operand:GPI 0 "register_operand" "=r")
2390 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2391 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2392 (match_operand:GPI 3 "register_operand" "r")))]
2394 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2395 [(set_attr "type" "alu_shift_imm")]
2398 ;; zero_extend version of above
2399 (define_insn "*add_<shift>_si_uxtw"
2400 [(set (match_operand:DI 0 "register_operand" "=r")
2402 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2403 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2404 (match_operand:SI 3 "register_operand" "r"))))]
2406 "add\\t%w0, %w3, %w1, <shift> %2"
2407 [(set_attr "type" "alu_shift_imm")]
2410 (define_insn "*add_mul_imm_<mode>"
2411 [(set (match_operand:GPI 0 "register_operand" "=r")
2412 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2413 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2414 (match_operand:GPI 3 "register_operand" "r")))]
2416 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2417 [(set_attr "type" "alu_shift_imm")]
2420 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2421 [(set (match_operand:GPI 0 "register_operand" "=rk")
2422 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2423 (match_operand:GPI 2 "register_operand" "r")))]
2425 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2426 [(set_attr "type" "alu_ext")]
2429 ;; zero_extend version of above
2430 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2431 [(set (match_operand:DI 0 "register_operand" "=rk")
2433 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2434 (match_operand:GPI 2 "register_operand" "r"))))]
2436 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2437 [(set_attr "type" "alu_ext")]
2440 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2441 [(set (match_operand:GPI 0 "register_operand" "=rk")
2442 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2443 (match_operand:ALLX 1 "register_operand" "r"))
2444 (match_operand 2 "aarch64_imm3" "Ui3"))
2445 (match_operand:GPI 3 "register_operand" "r")))]
2447 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2448 [(set_attr "type" "alu_ext")]
2451 ;; zero_extend version of above
2452 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2453 [(set (match_operand:DI 0 "register_operand" "=rk")
2455 (plus:SI (ashift:SI (ANY_EXTEND:SI
2456 (match_operand:SHORT 1 "register_operand" "r"))
2457 (match_operand 2 "aarch64_imm3" "Ui3"))
2458 (match_operand:SI 3 "register_operand" "r"))))]
2460 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2461 [(set_attr "type" "alu_ext")]
2464 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2465 [(set (match_operand:GPI 0 "register_operand" "=rk")
2466 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2467 (match_operand:ALLX 1 "register_operand" "r"))
2468 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2469 (match_operand:GPI 3 "register_operand" "r")))]
2471 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2472 [(set_attr "type" "alu_ext")]
2475 ;; zero_extend version of above
2476 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2477 [(set (match_operand:DI 0 "register_operand" "=rk")
2478 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2479 (match_operand:SHORT 1 "register_operand" "r"))
2480 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2481 (match_operand:SI 3 "register_operand" "r"))))]
2483 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2484 [(set_attr "type" "alu_ext")]
2487 (define_insn "*add_<optab><mode>_multp2"
2488 [(set (match_operand:GPI 0 "register_operand" "=rk")
2489 (plus:GPI (ANY_EXTRACT:GPI
2490 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2491 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2492 (match_operand 3 "const_int_operand" "n")
2494 (match_operand:GPI 4 "register_operand" "r")))]
2495 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2496 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2497 [(set_attr "type" "alu_ext")]
2500 ;; zero_extend version of above
2501 (define_insn "*add_<optab>si_multp2_uxtw"
2502 [(set (match_operand:DI 0 "register_operand" "=rk")
2504 (plus:SI (ANY_EXTRACT:SI
2505 (mult:SI (match_operand:SI 1 "register_operand" "r")
2506 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2507 (match_operand 3 "const_int_operand" "n")
2509 (match_operand:SI 4 "register_operand" "r"))))]
2510 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2511 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2512 [(set_attr "type" "alu_ext")]
2515 (define_expand "add<mode>3_carryin"
2516 [(set (match_operand:GPI 0 "register_operand")
2519 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2520 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2521 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2526 ;; Note that add with carry with two zero inputs is matched by cset,
2527 ;; and that add with carry with one zero input is matched by cinc.
2529 (define_insn "*add<mode>3_carryin"
2530 [(set (match_operand:GPI 0 "register_operand" "=r")
2533 (match_operand:GPI 3 "aarch64_carry_operation" "")
2534 (match_operand:GPI 1 "register_operand" "r"))
2535 (match_operand:GPI 2 "register_operand" "r")))]
2537 "adc\\t%<w>0, %<w>1, %<w>2"
2538 [(set_attr "type" "adc_reg")]
2541 ;; zero_extend version of above
2542 (define_insn "*addsi3_carryin_uxtw"
2543 [(set (match_operand:DI 0 "register_operand" "=r")
2547 (match_operand:SI 3 "aarch64_carry_operation" "")
2548 (match_operand:SI 1 "register_operand" "r"))
2549 (match_operand:SI 2 "register_operand" "r"))))]
2551 "adc\\t%w0, %w1, %w2"
2552 [(set_attr "type" "adc_reg")]
2555 (define_expand "add<mode>3_carryinC"
2563 (match_operand:GPI 1 "register_operand")))
2565 (match_operand:GPI 2 "register_operand")))
2567 (set (match_operand:GPI 0 "register_operand")
2569 (plus:GPI (match_dup 5) (match_dup 1))
2573 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2574 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2575 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2576 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2577 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2578 << GET_MODE_BITSIZE (<MODE>mode),
2582 (define_insn "*add<mode>3_carryinC_zero"
2583 [(set (reg:CC_ADC CC_REGNUM)
2586 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2587 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2588 (match_operand 4 "const_scalar_int_operand" "")))
2589 (set (match_operand:GPI 0 "register_operand" "=r")
2590 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2592 "rtx_mode_t (operands[4], <DWI>mode)
2593 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2594 "adcs\\t%<w>0, %<w>1, <w>zr"
2595 [(set_attr "type" "adc_reg")]
2598 (define_insn "*add<mode>3_carryinC"
2599 [(set (reg:CC_ADC CC_REGNUM)
2603 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2604 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2605 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2606 (match_operand 5 "const_scalar_int_operand" "")))
2607 (set (match_operand:GPI 0 "register_operand" "=r")
2609 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2612 "rtx_mode_t (operands[5], <DWI>mode)
2613 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2614 "adcs\\t%<w>0, %<w>1, %<w>2"
2615 [(set_attr "type" "adc_reg")]
2618 (define_expand "add<mode>3_carryinV"
2620 [(set (reg:CC_V CC_REGNUM)
2626 (match_operand:GPI 1 "register_operand")))
2628 (match_operand:GPI 2 "register_operand")))
2631 (plus:GPI (match_dup 4) (match_dup 1))
2633 (set (match_operand:GPI 0 "register_operand")
2635 (plus:GPI (match_dup 4) (match_dup 1))
2639 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2640 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2641 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2644 (define_insn "*add<mode>3_carryinV_zero"
2645 [(set (reg:CC_V CC_REGNUM)
2648 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2649 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2652 (match_operand:GPI 3 "aarch64_carry_operation" "")
2654 (set (match_operand:GPI 0 "register_operand" "=r")
2655 (plus:GPI (match_dup 3) (match_dup 1)))]
2657 "adcs\\t%<w>0, %<w>1, <w>zr"
2658 [(set_attr "type" "adc_reg")]
2661 (define_insn "*add<mode>3_carryinV"
2662 [(set (reg:CC_V CC_REGNUM)
2666 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2667 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2668 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2672 (match_operand:GPI 4 "aarch64_carry_operation" "")
2675 (set (match_operand:GPI 0 "register_operand" "=r")
2677 (plus:GPI (match_dup 4) (match_dup 1))
2680 "adcs\\t%<w>0, %<w>1, %<w>2"
2681 [(set_attr "type" "adc_reg")]
2684 (define_insn "*add_uxt<mode>_shift2"
2685 [(set (match_operand:GPI 0 "register_operand" "=rk")
2687 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2688 (match_operand 2 "aarch64_imm3" "Ui3"))
2689 (match_operand 3 "const_int_operand" "n"))
2690 (match_operand:GPI 4 "register_operand" "r")))]
2691 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2693 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2694 INTVAL (operands[3])));
2695 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2696 [(set_attr "type" "alu_ext")]
2699 ;; zero_extend version of above
2700 (define_insn "*add_uxtsi_shift2_uxtw"
2701 [(set (match_operand:DI 0 "register_operand" "=rk")
2704 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2705 (match_operand 2 "aarch64_imm3" "Ui3"))
2706 (match_operand 3 "const_int_operand" "n"))
2707 (match_operand:SI 4 "register_operand" "r"))))]
2708 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2710 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2711 INTVAL (operands[3])));
2712 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2713 [(set_attr "type" "alu_ext")]
2716 (define_insn "*add_uxt<mode>_multp2"
2717 [(set (match_operand:GPI 0 "register_operand" "=rk")
2719 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2720 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2721 (match_operand 3 "const_int_operand" "n"))
2722 (match_operand:GPI 4 "register_operand" "r")))]
2723 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2725 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2726 INTVAL (operands[3])));
2727 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2728 [(set_attr "type" "alu_ext")]
2731 ;; zero_extend version of above
2732 (define_insn "*add_uxtsi_multp2_uxtw"
2733 [(set (match_operand:DI 0 "register_operand" "=rk")
2736 (mult:SI (match_operand:SI 1 "register_operand" "r")
2737 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2738 (match_operand 3 "const_int_operand" "n"))
2739 (match_operand:SI 4 "register_operand" "r"))))]
2740 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2742 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2743 INTVAL (operands[3])));
2744 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2745 [(set_attr "type" "alu_ext")]
2748 (define_insn "subsi3"
2749 [(set (match_operand:SI 0 "register_operand" "=rk")
2750 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2751 (match_operand:SI 2 "register_operand" "r")))]
2753 "sub\\t%w0, %w1, %w2"
2754 [(set_attr "type" "alu_sreg")]
2757 ;; zero_extend version of above
2758 (define_insn "*subsi3_uxtw"
2759 [(set (match_operand:DI 0 "register_operand" "=rk")
2761 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2762 (match_operand:SI 2 "register_operand" "r"))))]
2764 "sub\\t%w0, %w1, %w2"
2765 [(set_attr "type" "alu_sreg")]
2768 (define_insn "subdi3"
2769 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2770 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2771 (match_operand:DI 2 "register_operand" "r,w")))]
2775 sub\\t%d0, %d1, %d2"
2776 [(set_attr "type" "alu_sreg, neon_sub")
2777 (set_attr "arch" "*,simd")]
2780 (define_expand "subv<GPI:mode>4"
2781 [(match_operand:GPI 0 "register_operand")
2782 (match_operand:GPI 1 "register_operand")
2783 (match_operand:GPI 2 "aarch64_plus_operand")
2784 (label_ref (match_operand 3 "" ""))]
2787 if (CONST_INT_P (operands[2]))
2788 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2790 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2791 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2796 (define_insn "subv<GPI:mode>_insn"
2797 [(set (reg:CC_V CC_REGNUM)
2801 (match_operand:GPI 1 "register_operand" "rk")
2802 (match_operand:GPI 2 "register_operand" "r")))
2803 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2804 (sign_extend:<DWI> (match_dup 2)))))
2805 (set (match_operand:GPI 0 "register_operand" "=r")
2806 (minus:GPI (match_dup 1) (match_dup 2)))]
2808 "subs\\t%<w>0, %<w>1, %<w>2"
2809 [(set_attr "type" "alus_sreg")]
2812 (define_insn "subv<GPI:mode>_imm"
2813 [(set (reg:CC_V CC_REGNUM)
2817 (match_operand:GPI 1 "register_operand" "rk,rk")
2818 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2819 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2821 (set (match_operand:GPI 0 "register_operand" "=r,r")
2822 (minus:GPI (match_dup 1) (match_dup 2)))]
2825 subs\\t%<w>0, %<w>1, %2
2826 adds\\t%<w>0, %<w>1, #%n2"
2827 [(set_attr "type" "alus_sreg")]
2830 (define_expand "negv<GPI:mode>3"
2831 [(match_operand:GPI 0 "register_operand")
2832 (match_operand:GPI 1 "register_operand")
2833 (label_ref (match_operand 2 "" ""))]
2836 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2837 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2843 (define_insn "negv<GPI:mode>_insn"
2844 [(set (reg:CC_V CC_REGNUM)
2847 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2848 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2849 (set (match_operand:GPI 0 "register_operand" "=r")
2850 (neg:GPI (match_dup 1)))]
2852 "negs\\t%<w>0, %<w>1"
2853 [(set_attr "type" "alus_sreg")]
2856 (define_insn "negv<GPI:mode>_cmp_only"
2857 [(set (reg:CC_V CC_REGNUM)
2860 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2861 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2863 "negs\\t%<w>zr, %<w>0"
2864 [(set_attr "type" "alus_sreg")]
2867 (define_insn "*cmpv<GPI:mode>_insn"
2868 [(set (reg:CC_V CC_REGNUM)
2871 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2872 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2873 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2874 (sign_extend:<DWI> (match_dup 1)))))]
2880 [(set_attr "type" "alus_sreg")]
2883 (define_expand "usubv<mode>4"
2884 [(match_operand:GPI 0 "register_operand")
2885 (match_operand:GPI 1 "aarch64_reg_or_zero")
2886 (match_operand:GPI 2 "aarch64_reg_or_zero")
2887 (label_ref (match_operand 3 "" ""))]
2890 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2891 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2896 (define_expand "subti3"
2897 [(set (match_operand:TI 0 "register_operand")
2898 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2899 (match_operand:TI 2 "register_operand")))]
2902 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2904 aarch64_subvti_scratch_regs (operands[1], operands[2],
2905 &low_dest, &op1_low, &op2_low,
2906 &high_dest, &op1_high, &op2_high);
2908 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2909 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2911 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2912 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2916 (define_expand "subvti4"
2917 [(match_operand:TI 0 "register_operand")
2918 (match_operand:TI 1 "register_operand")
2919 (match_operand:TI 2 "aarch64_reg_or_imm")
2920 (label_ref (match_operand 3 "" ""))]
2923 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2925 aarch64_subvti_scratch_regs (operands[1], operands[2],
2926 &low_dest, &op1_low, &op2_low,
2927 &high_dest, &op1_high, &op2_high);
2928 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2929 high_dest, op1_high, op2_high, false);
2931 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2935 (define_expand "usubvti4"
2936 [(match_operand:TI 0 "register_operand")
2937 (match_operand:TI 1 "register_operand")
2938 (match_operand:TI 2 "aarch64_reg_or_imm")
2939 (label_ref (match_operand 3 "" ""))]
2942 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2944 aarch64_subvti_scratch_regs (operands[1], operands[2],
2945 &low_dest, &op1_low, &op2_low,
2946 &high_dest, &op1_high, &op2_high);
2947 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2948 high_dest, op1_high, op2_high, true);
2950 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2954 (define_expand "negvti3"
2955 [(match_operand:TI 0 "register_operand")
2956 (match_operand:TI 1 "register_operand")
2957 (label_ref (match_operand 2 "" ""))]
2960 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2961 gen_lowpart (DImode, operands[1])));
2962 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2963 gen_highpart (DImode, operands[1])));
2964 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2970 (define_insn "negdi_carryout"
2971 [(set (reg:CC CC_REGNUM)
2973 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2974 (set (match_operand:DI 0 "register_operand" "=r")
2975 (neg:DI (match_dup 1)))]
2978 [(set_attr "type" "alus_sreg")]
2981 (define_insn "negvdi_carryinV"
2982 [(set (reg:CC_V CC_REGNUM)
2985 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2986 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2988 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2990 (set (match_operand:DI 0 "register_operand" "=r")
2991 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2995 [(set_attr "type" "alus_sreg")]
2998 (define_insn "*sub<mode>3_compare0"
2999 [(set (reg:CC_NZ CC_REGNUM)
3000 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3001 (match_operand:GPI 2 "register_operand" "r"))
3003 (set (match_operand:GPI 0 "register_operand" "=r")
3004 (minus:GPI (match_dup 1) (match_dup 2)))]
3006 "subs\\t%<w>0, %<w>1, %<w>2"
3007 [(set_attr "type" "alus_sreg")]
3010 ;; zero_extend version of above
3011 (define_insn "*subsi3_compare0_uxtw"
3012 [(set (reg:CC_NZ CC_REGNUM)
3013 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3014 (match_operand:SI 2 "register_operand" "r"))
3016 (set (match_operand:DI 0 "register_operand" "=r")
3017 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3019 "subs\\t%w0, %w1, %w2"
3020 [(set_attr "type" "alus_sreg")]
3023 (define_insn "sub<mode>3_compare1_imm"
3024 [(set (reg:CC CC_REGNUM)
3026 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3027 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3028 (set (match_operand:GPI 0 "register_operand" "=r,r")
3031 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3032 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3034 subs\\t%<w>0, %<w>1, %2
3035 adds\\t%<w>0, %<w>1, #%n2"
3036 [(set_attr "type" "alus_imm")]
3039 (define_insn "sub<mode>3_compare1"
3040 [(set (reg:CC CC_REGNUM)
3042 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3043 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3044 (set (match_operand:GPI 0 "register_operand" "=r")
3045 (minus:GPI (match_dup 1) (match_dup 2)))]
3047 "subs\\t%<w>0, %<w>1, %<w>2"
3048 [(set_attr "type" "alus_sreg")]
3052 [(set (match_operand:GPI 0 "aarch64_general_reg")
3053 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3054 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3055 (set (reg:CC CC_REGNUM)
3059 "!reg_overlap_mentioned_p (operands[0], operands[1])
3060 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3063 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3069 ;; Same as the above peephole but with the compare and minus in
3070 ;; swapped order. The restriction on overlap between operand 0
3071 ;; and operands 1 and 2 doesn't apply here.
3073 [(set (reg:CC CC_REGNUM)
3075 (match_operand:GPI 1 "aarch64_reg_or_zero")
3076 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3077 (set (match_operand:GPI 0 "aarch64_general_reg")
3078 (minus:GPI (match_dup 1)
3083 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3090 [(set (match_operand:GPI 0 "aarch64_general_reg")
3091 (plus:GPI (match_operand:GPI 1 "register_operand")
3092 (match_operand:GPI 2 "aarch64_plus_immediate")))
3093 (set (reg:CC CC_REGNUM)
3096 (match_operand:GPI 3 "const_int_operand")))]
3097 "!reg_overlap_mentioned_p (operands[0], operands[1])
3098 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3101 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3102 operands[3], operands[2]));
3107 ;; Same as the above peephole but with the compare and minus in
3108 ;; swapped order. The restriction on overlap between operand 0
3109 ;; and operands 1 doesn't apply here.
3111 [(set (reg:CC CC_REGNUM)
3113 (match_operand:GPI 1 "register_operand")
3114 (match_operand:GPI 3 "const_int_operand")))
3115 (set (match_operand:GPI 0 "aarch64_general_reg")
3116 (plus:GPI (match_dup 1)
3117 (match_operand:GPI 2 "aarch64_plus_immediate")))]
3118 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3121 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3122 operands[3], operands[2]));
3127 (define_insn "*sub_<shift>_<mode>"
3128 [(set (match_operand:GPI 0 "register_operand" "=r")
3129 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3131 (match_operand:GPI 1 "register_operand" "r")
3132 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3134 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3135 [(set_attr "type" "alu_shift_imm")]
3138 ;; zero_extend version of above
3139 (define_insn "*sub_<shift>_si_uxtw"
3140 [(set (match_operand:DI 0 "register_operand" "=r")
3142 (minus:SI (match_operand:SI 3 "register_operand" "r")
3144 (match_operand:SI 1 "register_operand" "r")
3145 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3147 "sub\\t%w0, %w3, %w1, <shift> %2"
3148 [(set_attr "type" "alu_shift_imm")]
3151 (define_insn "*sub_mul_imm_<mode>"
3152 [(set (match_operand:GPI 0 "register_operand" "=r")
3153 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3155 (match_operand:GPI 1 "register_operand" "r")
3156 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3158 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3159 [(set_attr "type" "alu_shift_imm")]
3162 ;; zero_extend version of above
3163 (define_insn "*sub_mul_imm_si_uxtw"
3164 [(set (match_operand:DI 0 "register_operand" "=r")
3166 (minus:SI (match_operand:SI 3 "register_operand" "r")
3168 (match_operand:SI 1 "register_operand" "r")
3169 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3171 "sub\\t%w0, %w3, %w1, lsl %p2"
3172 [(set_attr "type" "alu_shift_imm")]
3175 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3176 [(set (match_operand:GPI 0 "register_operand" "=rk")
3177 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3179 (match_operand:ALLX 2 "register_operand" "r"))))]
3181 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3182 [(set_attr "type" "alu_ext")]
3185 ;; zero_extend version of above
3186 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3187 [(set (match_operand:DI 0 "register_operand" "=rk")
3189 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3191 (match_operand:SHORT 2 "register_operand" "r")))))]
3193 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3194 [(set_attr "type" "alu_ext")]
3197 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3198 [(set (match_operand:GPI 0 "register_operand" "=rk")
3199 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3200 (ashift:GPI (ANY_EXTEND:GPI
3201 (match_operand:ALLX 2 "register_operand" "r"))
3202 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3204 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3205 [(set_attr "type" "alu_ext")]
3208 ;; zero_extend version of above
3209 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3210 [(set (match_operand:DI 0 "register_operand" "=rk")
3212 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3213 (ashift:SI (ANY_EXTEND:SI
3214 (match_operand:SHORT 2 "register_operand" "r"))
3215 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3217 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3218 [(set_attr "type" "alu_ext")]
3221 (define_insn "*sub_<optab><mode>_multp2"
3222 [(set (match_operand:GPI 0 "register_operand" "=rk")
3223 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3225 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3226 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3227 (match_operand 3 "const_int_operand" "n")
3229 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3230 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3231 [(set_attr "type" "alu_ext")]
3234 ;; zero_extend version of above
3235 (define_insn "*sub_<optab>si_multp2_uxtw"
3236 [(set (match_operand:DI 0 "register_operand" "=rk")
3238 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3240 (mult:SI (match_operand:SI 1 "register_operand" "r")
3241 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3242 (match_operand 3 "const_int_operand" "n")
3244 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3245 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3246 [(set_attr "type" "alu_ext")]
3249 ;; The hardware description is op1 + ~op2 + C.
3250 ;; = op1 + (-op2 + 1) + (1 - !C)
3251 ;; = op1 - op2 - 1 + 1 - !C
3252 ;; = op1 - op2 - !C.
3253 ;; We describe the latter.
3255 (define_insn "*sub<mode>3_carryin0"
3256 [(set (match_operand:GPI 0 "register_operand" "=r")
3258 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3259 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3261 "sbc\\t%<w>0, %<w>1, <w>zr"
3262 [(set_attr "type" "adc_reg")]
3265 ;; zero_extend version of the above
3266 (define_insn "*subsi3_carryin_uxtw"
3267 [(set (match_operand:DI 0 "register_operand" "=r")
3270 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3271 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3273 "sbc\\t%w0, %w1, wzr"
3274 [(set_attr "type" "adc_reg")]
3277 (define_expand "sub<mode>3_carryin"
3278 [(set (match_operand:GPI 0 "register_operand")
3281 (match_operand:GPI 1 "aarch64_reg_or_zero")
3282 (match_operand:GPI 2 "register_operand"))
3283 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3288 (define_insn "*sub<mode>3_carryin"
3289 [(set (match_operand:GPI 0 "register_operand" "=r")
3292 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3293 (match_operand:GPI 2 "register_operand" "r"))
3294 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3297 "sbc\\t%<w>0, %<w>1, %<w>2"
3298 [(set_attr "type" "adc_reg")]
3301 ;; zero_extend version of the above
3302 (define_insn "*subsi3_carryin_uxtw"
3303 [(set (match_operand:DI 0 "register_operand" "=r")
3307 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3308 (match_operand:SI 2 "register_operand" "r"))
3309 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3312 "sbc\\t%w0, %w1, %w2"
3313 [(set_attr "type" "adc_reg")]
3316 (define_insn "*sub<mode>3_carryin_alt"
3317 [(set (match_operand:GPI 0 "register_operand" "=r")
3320 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3321 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3322 (match_operand:GPI 2 "register_operand" "r")))]
3324 "sbc\\t%<w>0, %<w>1, %<w>2"
3325 [(set_attr "type" "adc_reg")]
3328 ;; zero_extend version of the above
3329 (define_insn "*subsi3_carryin_alt_uxtw"
3330 [(set (match_operand:DI 0 "register_operand" "=r")
3334 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3335 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3336 (match_operand:SI 2 "register_operand" "r"))))]
3338 "sbc\\t%w0, %w1, %w2"
3339 [(set_attr "type" "adc_reg")]
3342 (define_expand "usub<GPI:mode>3_carryinC"
3344 [(set (reg:CC CC_REGNUM)
3347 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3350 (match_operand:GPI 2 "register_operand"))
3351 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3352 (set (match_operand:GPI 0 "register_operand")
3354 (minus:GPI (match_dup 1) (match_dup 2))
3355 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3359 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3360 [(set (reg:CC CC_REGNUM)
3365 (match_operand:GPI 1 "register_operand" "r"))
3366 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3367 (set (match_operand:GPI 0 "register_operand" "=r")
3369 (neg:GPI (match_dup 1))
3370 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3372 "sbcs\\t%<w>0, <w>zr, %<w>1"
3373 [(set_attr "type" "adc_reg")]
3376 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3377 [(set (reg:CC CC_REGNUM)
3380 (match_operand:GPI 1 "register_operand" "r"))
3381 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3382 (set (match_operand:GPI 0 "register_operand" "=r")
3385 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3387 "sbcs\\t%<w>0, %<w>1, <w>zr"
3388 [(set_attr "type" "adc_reg")]
3391 (define_insn "*usub<GPI:mode>3_carryinC"
3392 [(set (reg:CC CC_REGNUM)
3395 (match_operand:GPI 1 "register_operand" "r"))
3398 (match_operand:GPI 2 "register_operand" "r"))
3399 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3400 (set (match_operand:GPI 0 "register_operand" "=r")
3402 (minus:GPI (match_dup 1) (match_dup 2))
3403 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3405 "sbcs\\t%<w>0, %<w>1, %<w>2"
3406 [(set_attr "type" "adc_reg")]
3409 (define_expand "sub<GPI:mode>3_carryinV"
3411 [(set (reg:CC_V CC_REGNUM)
3415 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3418 (match_operand:GPI 2 "register_operand"))
3419 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3421 (minus:GPI (match_dup 1)
3422 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3424 (set (match_operand:GPI 0 "register_operand")
3426 (minus:GPI (match_dup 1) (match_dup 2))
3427 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3431 (define_insn "*sub<mode>3_carryinV_z2"
3432 [(set (reg:CC_V CC_REGNUM)
3435 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3436 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3438 (minus:GPI (match_dup 1)
3439 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3440 (set (match_operand:GPI 0 "register_operand" "=r")
3442 (match_dup 1) (match_dup 3)))]
3444 "sbcs\\t%<w>0, %<w>1, <w>zr"
3445 [(set_attr "type" "adc_reg")]
3448 (define_insn "*sub<mode>3_carryinV"
3449 [(set (reg:CC_V CC_REGNUM)
3453 (match_operand:GPI 1 "register_operand" "r"))
3456 (match_operand:GPI 2 "register_operand" "r"))
3457 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3461 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3463 (set (match_operand:GPI 0 "register_operand" "=r")
3465 (minus:GPI (match_dup 1) (match_dup 2))
3468 "sbcs\\t%<w>0, %<w>1, %<w>2"
3469 [(set_attr "type" "adc_reg")]
3472 (define_insn "*sub_uxt<mode>_shift2"
3473 [(set (match_operand:GPI 0 "register_operand" "=rk")
3474 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3476 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3477 (match_operand 2 "aarch64_imm3" "Ui3"))
3478 (match_operand 3 "const_int_operand" "n"))))]
3479 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3481 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3482 INTVAL (operands[3])));
3483 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3484 [(set_attr "type" "alu_ext")]
3487 ;; zero_extend version of above
3488 (define_insn "*sub_uxtsi_shift2_uxtw"
3489 [(set (match_operand:DI 0 "register_operand" "=rk")
3491 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3493 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3494 (match_operand 2 "aarch64_imm3" "Ui3"))
3495 (match_operand 3 "const_int_operand" "n")))))]
3496 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3498 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3499 INTVAL (operands[3])));
3500 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3501 [(set_attr "type" "alu_ext")]
3504 (define_insn "*sub_uxt<mode>_multp2"
3505 [(set (match_operand:GPI 0 "register_operand" "=rk")
3506 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3508 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3509 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3510 (match_operand 3 "const_int_operand" "n"))))]
3511 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3513 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3514 INTVAL (operands[3])));
3515 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3516 [(set_attr "type" "alu_ext")]
3519 ;; zero_extend version of above
3520 (define_insn "*sub_uxtsi_multp2_uxtw"
3521 [(set (match_operand:DI 0 "register_operand" "=rk")
3523 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3525 (mult:SI (match_operand:SI 1 "register_operand" "r")
3526 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3527 (match_operand 3 "const_int_operand" "n")))))]
3528 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3530 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3531 INTVAL (operands[3])));
3532 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3533 [(set_attr "type" "alu_ext")]
3536 (define_expand "abs<mode>2"
3537 [(match_operand:GPI 0 "register_operand")
3538 (match_operand:GPI 1 "register_operand")]
3541 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3542 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3543 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3548 (define_insn "neg<mode>2"
3549 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3550 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3554 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3555 [(set_attr "type" "alu_sreg, neon_neg<q>")
3556 (set_attr "arch" "*,simd")]
3559 ;; zero_extend version of above
3560 (define_insn "*negsi2_uxtw"
3561 [(set (match_operand:DI 0 "register_operand" "=r")
3562 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3565 [(set_attr "type" "alu_sreg")]
3568 (define_insn "*ngc<mode>"
3569 [(set (match_operand:GPI 0 "register_operand" "=r")
3571 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3572 (match_operand:GPI 1 "register_operand" "r")))]
3574 "ngc\\t%<w>0, %<w>1"
3575 [(set_attr "type" "adc_reg")]
3578 (define_insn "*ngcsi_uxtw"
3579 [(set (match_operand:DI 0 "register_operand" "=r")
3582 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3583 (match_operand:SI 1 "register_operand" "r"))))]
3586 [(set_attr "type" "adc_reg")]
3589 (define_insn "neg<mode>2_compare0"
3590 [(set (reg:CC_NZ CC_REGNUM)
3591 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3593 (set (match_operand:GPI 0 "register_operand" "=r")
3594 (neg:GPI (match_dup 1)))]
3596 "negs\\t%<w>0, %<w>1"
3597 [(set_attr "type" "alus_sreg")]
3600 ;; zero_extend version of above
3601 (define_insn "*negsi2_compare0_uxtw"
3602 [(set (reg:CC_NZ CC_REGNUM)
3603 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3605 (set (match_operand:DI 0 "register_operand" "=r")
3606 (zero_extend:DI (neg:SI (match_dup 1))))]
3609 [(set_attr "type" "alus_sreg")]
3612 (define_insn "*neg_<shift><mode>3_compare0"
3613 [(set (reg:CC_NZ CC_REGNUM)
3615 (neg:GPI (ASHIFT:GPI
3616 (match_operand:GPI 1 "register_operand" "r")
3617 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3619 (set (match_operand:GPI 0 "register_operand" "=r")
3620 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3622 "negs\\t%<w>0, %<w>1, <shift> %2"
3623 [(set_attr "type" "alus_shift_imm")]
3626 (define_insn "*neg_<shift>_<mode>2"
3627 [(set (match_operand:GPI 0 "register_operand" "=r")
3628 (neg:GPI (ASHIFT:GPI
3629 (match_operand:GPI 1 "register_operand" "r")
3630 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3632 "neg\\t%<w>0, %<w>1, <shift> %2"
3633 [(set_attr "type" "alu_shift_imm")]
3636 ;; zero_extend version of above
3637 (define_insn "*neg_<shift>_si2_uxtw"
3638 [(set (match_operand:DI 0 "register_operand" "=r")
3641 (match_operand:SI 1 "register_operand" "r")
3642 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3644 "neg\\t%w0, %w1, <shift> %2"
3645 [(set_attr "type" "alu_shift_imm")]
3648 (define_insn "*neg_mul_imm_<mode>2"
3649 [(set (match_operand:GPI 0 "register_operand" "=r")
3651 (match_operand:GPI 1 "register_operand" "r")
3652 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3654 "neg\\t%<w>0, %<w>1, lsl %p2"
3655 [(set_attr "type" "alu_shift_imm")]
3658 ;; zero_extend version of above
3659 (define_insn "*neg_mul_imm_si2_uxtw"
3660 [(set (match_operand:DI 0 "register_operand" "=r")
3663 (match_operand:SI 1 "register_operand" "r")
3664 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3666 "neg\\t%w0, %w1, lsl %p2"
3667 [(set_attr "type" "alu_shift_imm")]
3670 (define_insn "mul<mode>3"
3671 [(set (match_operand:GPI 0 "register_operand" "=r")
3672 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3673 (match_operand:GPI 2 "register_operand" "r")))]
3675 "mul\\t%<w>0, %<w>1, %<w>2"
3676 [(set_attr "type" "mul")]
3679 ;; zero_extend version of above
3680 (define_insn "*mulsi3_uxtw"
3681 [(set (match_operand:DI 0 "register_operand" "=r")
3683 (mult:SI (match_operand:SI 1 "register_operand" "r")
3684 (match_operand:SI 2 "register_operand" "r"))))]
3686 "mul\\t%w0, %w1, %w2"
3687 [(set_attr "type" "mul")]
3690 (define_insn "madd<mode>"
3691 [(set (match_operand:GPI 0 "register_operand" "=r")
3692 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3693 (match_operand:GPI 2 "register_operand" "r"))
3694 (match_operand:GPI 3 "register_operand" "r")))]
3696 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3697 [(set_attr "type" "mla")]
3700 ;; zero_extend version of above
3701 (define_insn "*maddsi_uxtw"
3702 [(set (match_operand:DI 0 "register_operand" "=r")
3704 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3705 (match_operand:SI 2 "register_operand" "r"))
3706 (match_operand:SI 3 "register_operand" "r"))))]
3708 "madd\\t%w0, %w1, %w2, %w3"
3709 [(set_attr "type" "mla")]
3712 (define_insn "*msub<mode>"
3713 [(set (match_operand:GPI 0 "register_operand" "=r")
3714 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3715 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3716 (match_operand:GPI 2 "register_operand" "r"))))]
3719 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3720 [(set_attr "type" "mla")]
3723 ;; zero_extend version of above
3724 (define_insn "*msubsi_uxtw"
3725 [(set (match_operand:DI 0 "register_operand" "=r")
3727 (minus:SI (match_operand:SI 3 "register_operand" "r")
3728 (mult:SI (match_operand:SI 1 "register_operand" "r")
3729 (match_operand:SI 2 "register_operand" "r")))))]
3732 "msub\\t%w0, %w1, %w2, %w3"
3733 [(set_attr "type" "mla")]
3736 (define_insn "*mul<mode>_neg"
3737 [(set (match_operand:GPI 0 "register_operand" "=r")
3738 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3739 (match_operand:GPI 2 "register_operand" "r")))]
3742 "mneg\\t%<w>0, %<w>1, %<w>2"
3743 [(set_attr "type" "mul")]
3746 ;; zero_extend version of above
3747 (define_insn "*mulsi_neg_uxtw"
3748 [(set (match_operand:DI 0 "register_operand" "=r")
3750 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3751 (match_operand:SI 2 "register_operand" "r"))))]
3754 "mneg\\t%w0, %w1, %w2"
3755 [(set_attr "type" "mul")]
3758 (define_insn "<su_optab>mulsidi3"
3759 [(set (match_operand:DI 0 "register_operand" "=r")
3760 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3761 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3763 "<su>mull\\t%0, %w1, %w2"
3764 [(set_attr "type" "<su>mull")]
3767 (define_insn "<su_optab>maddsidi4"
3768 [(set (match_operand:DI 0 "register_operand" "=r")
3770 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3771 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3772 (match_operand:DI 3 "register_operand" "r")))]
3774 "<su>maddl\\t%0, %w1, %w2, %3"
3775 [(set_attr "type" "<su>mlal")]
3778 (define_insn "<su_optab>msubsidi4"
3779 [(set (match_operand:DI 0 "register_operand" "=r")
3781 (match_operand:DI 3 "register_operand" "r")
3782 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3784 (match_operand:SI 2 "register_operand" "r")))))]
3786 "<su>msubl\\t%0, %w1, %w2, %3"
3787 [(set_attr "type" "<su>mlal")]
3790 (define_insn "*<su_optab>mulsidi_neg"
3791 [(set (match_operand:DI 0 "register_operand" "=r")
3793 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3794 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3796 "<su>mnegl\\t%0, %w1, %w2"
3797 [(set_attr "type" "<su>mull")]
3800 (define_expand "<su_optab>mulditi3"
3801 [(set (match_operand:TI 0 "register_operand")
3802 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3803 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3806 rtx low = gen_reg_rtx (DImode);
3807 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3809 rtx high = gen_reg_rtx (DImode);
3810 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3812 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3813 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3817 ;; The default expansion of multi3 using umuldi3_highpart will perform
3818 ;; the additions in an order that fails to combine into two madd insns.
3819 (define_expand "multi3"
3820 [(set (match_operand:TI 0 "register_operand")
3821 (mult:TI (match_operand:TI 1 "register_operand")
3822 (match_operand:TI 2 "register_operand")))]
3825 rtx l0 = gen_reg_rtx (DImode);
3826 rtx l1 = gen_lowpart (DImode, operands[1]);
3827 rtx l2 = gen_lowpart (DImode, operands[2]);
3828 rtx h0 = gen_reg_rtx (DImode);
3829 rtx h1 = gen_highpart (DImode, operands[1]);
3830 rtx h2 = gen_highpart (DImode, operands[2]);
3832 emit_insn (gen_muldi3 (l0, l1, l2));
3833 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3834 emit_insn (gen_madddi (h0, h1, l2, h0));
3835 emit_insn (gen_madddi (h0, l1, h2, h0));
3837 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3838 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3842 (define_insn "<su>muldi3_highpart"
3843 [(set (match_operand:DI 0 "register_operand" "=r")
3847 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3848 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3851 "<su>mulh\\t%0, %1, %2"
3852 [(set_attr "type" "<su>mull")]
3855 (define_insn "<su_optab>div<mode>3"
3856 [(set (match_operand:GPI 0 "register_operand" "=r")
3857 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3858 (match_operand:GPI 2 "register_operand" "r")))]
3860 "<su>div\\t%<w>0, %<w>1, %<w>2"
3861 [(set_attr "type" "<su>div")]
3864 ;; zero_extend version of above
3865 (define_insn "*<su_optab>divsi3_uxtw"
3866 [(set (match_operand:DI 0 "register_operand" "=r")
3868 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3869 (match_operand:SI 2 "register_operand" "r"))))]
3871 "<su>div\\t%w0, %w1, %w2"
3872 [(set_attr "type" "<su>div")]
3875 ;; -------------------------------------------------------------------
3877 ;; -------------------------------------------------------------------
3879 (define_insn "cmp<mode>"
3880 [(set (reg:CC CC_REGNUM)
3881 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3882 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3888 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3891 (define_insn "fcmp<mode>"
3892 [(set (reg:CCFP CC_REGNUM)
3893 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3894 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3898 fcmp\\t%<s>0, %<s>1"
3899 [(set_attr "type" "fcmp<s>")]
3902 (define_insn "fcmpe<mode>"
3903 [(set (reg:CCFPE CC_REGNUM)
3904 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3905 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3909 fcmpe\\t%<s>0, %<s>1"
3910 [(set_attr "type" "fcmp<s>")]
3913 (define_insn "*cmp_swp_<shift>_reg<mode>"
3914 [(set (reg:CC_SWP CC_REGNUM)
3915 (compare:CC_SWP (ASHIFT:GPI
3916 (match_operand:GPI 0 "register_operand" "r")
3917 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3918 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3920 "cmp\\t%<w>2, %<w>0, <shift> %1"
3921 [(set_attr "type" "alus_shift_imm")]
3924 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3925 [(set (reg:CC_SWP CC_REGNUM)
3926 (compare:CC_SWP (ANY_EXTEND:GPI
3927 (match_operand:ALLX 0 "register_operand" "r"))
3928 (match_operand:GPI 1 "register_operand" "r")))]
3930 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3931 [(set_attr "type" "alus_ext")]
3934 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3935 [(set (reg:CC_SWP CC_REGNUM)
3936 (compare:CC_SWP (ashift:GPI
3938 (match_operand:ALLX 0 "register_operand" "r"))
3939 (match_operand 1 "aarch64_imm3" "Ui3"))
3940 (match_operand:GPI 2 "register_operand" "r")))]
3942 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3943 [(set_attr "type" "alus_ext")]
3946 ;; -------------------------------------------------------------------
3947 ;; Store-flag and conditional select insns
3948 ;; -------------------------------------------------------------------
3950 (define_expand "cstore<mode>4"
3951 [(set (match_operand:SI 0 "register_operand")
3952 (match_operator:SI 1 "aarch64_comparison_operator"
3953 [(match_operand:GPI 2 "register_operand")
3954 (match_operand:GPI 3 "aarch64_plus_operand")]))]
3957 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3959 operands[3] = const0_rtx;
3963 (define_expand "cstorecc4"
3964 [(set (match_operand:SI 0 "register_operand")
3965 (match_operator 1 "aarch64_comparison_operator_mode"
3966 [(match_operand 2 "cc_register")
3967 (match_operand 3 "const0_operand")]))]
3970 emit_insn (gen_rtx_SET (operands[0], operands[1]));
3975 (define_expand "cstore<mode>4"
3976 [(set (match_operand:SI 0 "register_operand")
3977 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3978 [(match_operand:GPF 2 "register_operand")
3979 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
3982 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3984 operands[3] = const0_rtx;
3988 (define_insn "aarch64_cstore<mode>"
3989 [(set (match_operand:ALLI 0 "register_operand" "=r")
3990 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3991 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3994 [(set_attr "type" "csel")]
3997 ;; For a 24-bit immediate CST we can optimize the compare for equality
3998 ;; and branch sequence from:
4000 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
4003 ;; into the shorter:
4004 ;; sub x0, x1, #(CST & 0xfff000)
4005 ;; subs x0, x0, #(CST & 0x000fff)
4006 ;; cset x2, <ne, eq>.
4007 (define_insn_and_split "*compare_cstore<mode>_insn"
4008 [(set (match_operand:GPI 0 "register_operand" "=r")
4009 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
4010 (match_operand:GPI 2 "aarch64_imm24" "n")))
4011 (clobber (reg:CC CC_REGNUM))]
4012 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4013 && !aarch64_plus_operand (operands[2], <MODE>mode)
4014 && !reload_completed"
4019 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4020 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4021 rtx tmp = gen_reg_rtx (<MODE>mode);
4022 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4023 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4024 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4025 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4026 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4029 [(set_attr "type" "csel")]
4032 ;; zero_extend version of the above
4033 (define_insn "*cstoresi_insn_uxtw"
4034 [(set (match_operand:DI 0 "register_operand" "=r")
4036 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4037 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4040 [(set_attr "type" "csel")]
4043 (define_insn "cstore<mode>_neg"
4044 [(set (match_operand:ALLI 0 "register_operand" "=r")
4045 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4046 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4048 "csetm\\t%<w>0, %m1"
4049 [(set_attr "type" "csel")]
4052 ;; zero_extend version of the above
4053 (define_insn "*cstoresi_neg_uxtw"
4054 [(set (match_operand:DI 0 "register_operand" "=r")
4056 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4057 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4060 [(set_attr "type" "csel")]
4063 (define_expand "cmov<mode>6"
4064 [(set (match_operand:GPI 0 "register_operand")
4066 (match_operator 1 "aarch64_comparison_operator"
4067 [(match_operand:GPI 2 "register_operand")
4068 (match_operand:GPI 3 "aarch64_plus_operand")])
4069 (match_operand:GPI 4 "register_operand")
4070 (match_operand:GPI 5 "register_operand")))]
4073 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4075 operands[3] = const0_rtx;
4079 (define_expand "cmov<mode>6"
4080 [(set (match_operand:GPF 0 "register_operand")
4082 (match_operator 1 "aarch64_comparison_operator"
4083 [(match_operand:GPF 2 "register_operand")
4084 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4085 (match_operand:GPF 4 "register_operand")
4086 (match_operand:GPF 5 "register_operand")))]
4089 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4091 operands[3] = const0_rtx;
4095 (define_insn "*cmov<mode>_insn"
4096 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4098 (match_operator 1 "aarch64_comparison_operator"
4099 [(match_operand 2 "cc_register" "") (const_int 0)])
4100 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4101 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4102 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4103 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4104 ;; Final two alternatives should be unreachable, but included for completeness
4106 csel\\t%<w>0, %<w>3, %<w>4, %m1
4107 csinv\\t%<w>0, %<w>3, <w>zr, %m1
4108 csinv\\t%<w>0, %<w>4, <w>zr, %M1
4109 csinc\\t%<w>0, %<w>3, <w>zr, %m1
4110 csinc\\t%<w>0, %<w>4, <w>zr, %M1
4113 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4116 ;; zero_extend version of above
4117 (define_insn "*cmovsi_insn_uxtw"
4118 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4121 (match_operator 1 "aarch64_comparison_operator"
4122 [(match_operand 2 "cc_register" "") (const_int 0)])
4123 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4124 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4125 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4126 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4127 ;; Final two alternatives should be unreachable, but included for completeness
4129 csel\\t%w0, %w3, %w4, %m1
4130 csinv\\t%w0, %w3, wzr, %m1
4131 csinv\\t%w0, %w4, wzr, %M1
4132 csinc\\t%w0, %w3, wzr, %m1
4133 csinc\\t%w0, %w4, wzr, %M1
4136 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4139 (define_insn "*cmovdi_insn_uxtw"
4140 [(set (match_operand:DI 0 "register_operand" "=r")
4142 (match_operator 1 "aarch64_comparison_operator"
4143 [(match_operand 2 "cc_register" "") (const_int 0)])
4144 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4145 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4147 "csel\\t%w0, %w3, %w4, %m1"
4148 [(set_attr "type" "csel")]
4151 (define_insn "*cmov<mode>_insn"
4152 [(set (match_operand:GPF 0 "register_operand" "=w")
4154 (match_operator 1 "aarch64_comparison_operator"
4155 [(match_operand 2 "cc_register" "") (const_int 0)])
4156 (match_operand:GPF 3 "register_operand" "w")
4157 (match_operand:GPF 4 "register_operand" "w")))]
4159 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4160 [(set_attr "type" "fcsel")]
4163 (define_expand "mov<mode>cc"
4164 [(set (match_operand:ALLI 0 "register_operand")
4165 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4166 (match_operand:ALLI 2 "register_operand")
4167 (match_operand:ALLI 3 "register_operand")))]
4171 enum rtx_code code = GET_CODE (operands[1]);
4173 if (code == UNEQ || code == LTGT)
4176 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4177 XEXP (operands[1], 1));
4178 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4182 (define_expand "mov<GPF:mode><GPI:mode>cc"
4183 [(set (match_operand:GPI 0 "register_operand")
4184 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4185 (match_operand:GPF 2 "register_operand")
4186 (match_operand:GPF 3 "register_operand")))]
4190 enum rtx_code code = GET_CODE (operands[1]);
4192 if (code == UNEQ || code == LTGT)
4195 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4196 XEXP (operands[1], 1));
4197 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4201 (define_expand "mov<mode>cc"
4202 [(set (match_operand:GPF 0 "register_operand")
4203 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4204 (match_operand:GPF 2 "register_operand")
4205 (match_operand:GPF 3 "register_operand")))]
4209 enum rtx_code code = GET_CODE (operands[1]);
4211 if (code == UNEQ || code == LTGT)
4214 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4215 XEXP (operands[1], 1));
4216 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4220 (define_expand "<neg_not_op><mode>cc"
4221 [(set (match_operand:GPI 0 "register_operand")
4222 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4223 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4224 (match_operand:GPI 3 "register_operand")))]
4228 enum rtx_code code = GET_CODE (operands[1]);
4230 if (code == UNEQ || code == LTGT)
4233 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4234 XEXP (operands[1], 1));
4235 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4239 ;; CRC32 instructions.
4240 (define_insn "aarch64_<crc_variant>"
4241 [(set (match_operand:SI 0 "register_operand" "=r")
4242 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4243 (match_operand:<crc_mode> 2 "register_operand" "r")]
4247 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4248 return "<crc_variant>\\t%w0, %w1, %x2";
4250 return "<crc_variant>\\t%w0, %w1, %w2";
4252 [(set_attr "type" "crc")]
4255 (define_insn "*csinc2<mode>_insn"
4256 [(set (match_operand:GPI 0 "register_operand" "=r")
4257 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4258 (match_operand:GPI 1 "register_operand" "r")))]
4260 "cinc\\t%<w>0, %<w>1, %m2"
4261 [(set_attr "type" "csel")]
4264 (define_insn "csinc3<mode>_insn"
4265 [(set (match_operand:GPI 0 "register_operand" "=r")
4267 (match_operand 1 "aarch64_comparison_operation" "")
4268 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4270 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4272 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4273 [(set_attr "type" "csel")]
4276 (define_insn "*csinv3<mode>_insn"
4277 [(set (match_operand:GPI 0 "register_operand" "=r")
4279 (match_operand 1 "aarch64_comparison_operation" "")
4280 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4281 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4283 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4284 [(set_attr "type" "csel")]
4287 (define_insn "csneg3_uxtw_insn"
4288 [(set (match_operand:DI 0 "register_operand" "=r")
4291 (match_operand 1 "aarch64_comparison_operation" "")
4292 (neg:SI (match_operand:SI 2 "register_operand" "r"))
4293 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4295 "csneg\\t%w0, %w3, %w2, %M1"
4296 [(set_attr "type" "csel")]
4299 (define_insn "csneg3<mode>_insn"
4300 [(set (match_operand:GPI 0 "register_operand" "=r")
4302 (match_operand 1 "aarch64_comparison_operation" "")
4303 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4304 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4306 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4307 [(set_attr "type" "csel")]
4310 ;; If X can be loaded by a single CNT[BHWD] instruction,
4314 ;; is equivalent to:
4316 ;; TMP = UQDEC[BHWD] (B, X)
4319 ;; Defining the pattern this way means that:
4321 ;; A = UMAX (B, X) - X
4325 ;; TMP1 = UQDEC[BHWD] (B, X)
4329 ;; which combine can optimize to:
4331 ;; A = UQDEC[BHWD] (B, X)
4333 ;; We don't use match_operand predicates because the order of the operands
4334 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4335 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4336 (define_expand "umax<mode>3"
4337 [(set (match_operand:GPI 0 "register_operand")
4338 (umax:GPI (match_operand:GPI 1 "")
4339 (match_operand:GPI 2 "")))]
4342 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4343 std::swap (operands[1], operands[2]);
4344 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4346 rtx temp = gen_reg_rtx (<MODE>mode);
4347 operands[1] = force_reg (<MODE>mode, operands[1]);
4348 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4349 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4354 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4355 (define_insn "aarch64_uqdec<mode>"
4356 [(set (match_operand:GPI 0 "register_operand" "=r")
4358 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4359 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4363 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4367 ;; -------------------------------------------------------------------
4368 ;; Logical operations
4369 ;; -------------------------------------------------------------------
4372 (define_insn_and_split "*aarch64_and<mode>_imm2"
4373 [(set (match_operand:GPI 0 "register_operand" "=rk")
4374 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4375 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4381 HOST_WIDE_INT val = INTVAL (operands[2]);
4382 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4383 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4385 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4386 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4391 (define_insn "<optab><mode>3"
4392 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4393 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4394 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4397 <logical>\\t%<w>0, %<w>1, %<w>2
4398 <logical>\\t%<w>0, %<w>1, %2
4399 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4400 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4401 (set_attr "arch" "*,*,simd")]
4404 ;; zero_extend version of above
4405 (define_insn "*<optab>si3_uxtw"
4406 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4408 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4409 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4412 <logical>\\t%w0, %w1, %w2
4413 <logical>\\t%w0, %w1, %2"
4414 [(set_attr "type" "logic_reg,logic_imm")]
4417 (define_insn "*and<mode>3_compare0"
4418 [(set (reg:CC_NZ CC_REGNUM)
4420 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4421 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4423 (set (match_operand:GPI 0 "register_operand" "=r,r")
4424 (and:GPI (match_dup 1) (match_dup 2)))]
4427 ands\\t%<w>0, %<w>1, %<w>2
4428 ands\\t%<w>0, %<w>1, %2"
4429 [(set_attr "type" "logics_reg,logics_imm")]
4432 ;; zero_extend version of above
4433 (define_insn "*andsi3_compare0_uxtw"
4434 [(set (reg:CC_NZ CC_REGNUM)
4436 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4437 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4439 (set (match_operand:DI 0 "register_operand" "=r,r")
4440 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4443 ands\\t%w0, %w1, %w2
4444 ands\\t%w0, %w1, %2"
4445 [(set_attr "type" "logics_reg,logics_imm")]
4448 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4449 [(set (reg:CC_NZ CC_REGNUM)
4452 (match_operand:GPI 1 "register_operand" "r")
4453 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4454 (match_operand:GPI 3 "register_operand" "r"))
4456 (set (match_operand:GPI 0 "register_operand" "=r")
4457 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4459 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4460 [(set_attr "type" "logics_shift_imm")]
4463 ;; zero_extend version of above
4464 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4465 [(set (reg:CC_NZ CC_REGNUM)
4468 (match_operand:SI 1 "register_operand" "r")
4469 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4470 (match_operand:SI 3 "register_operand" "r"))
4472 (set (match_operand:DI 0 "register_operand" "=r")
4473 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4476 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4477 [(set_attr "type" "logics_shift_imm")]
4480 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4481 [(set (match_operand:GPI 0 "register_operand" "=r")
4482 (LOGICAL:GPI (SHIFT:GPI
4483 (match_operand:GPI 1 "register_operand" "r")
4484 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4485 (match_operand:GPI 3 "register_operand" "r")))]
4487 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4488 [(set_attr "type" "logic_shift_imm")]
4491 (define_insn "*<optab>_rol<mode>3"
4492 [(set (match_operand:GPI 0 "register_operand" "=r")
4493 (LOGICAL:GPI (rotate:GPI
4494 (match_operand:GPI 1 "register_operand" "r")
4495 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4496 (match_operand:GPI 3 "register_operand" "r")))]
4498 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4499 [(set_attr "type" "logic_shift_imm")]
4502 ;; zero_extend versions of above
4503 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4504 [(set (match_operand:DI 0 "register_operand" "=r")
4506 (LOGICAL:SI (SHIFT:SI
4507 (match_operand:SI 1 "register_operand" "r")
4508 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4509 (match_operand:SI 3 "register_operand" "r"))))]
4511 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4512 [(set_attr "type" "logic_shift_imm")]
4515 (define_insn "*<optab>_rolsi3_uxtw"
4516 [(set (match_operand:DI 0 "register_operand" "=r")
4518 (LOGICAL:SI (rotate:SI
4519 (match_operand:SI 1 "register_operand" "r")
4520 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4521 (match_operand:SI 3 "register_operand" "r"))))]
4523 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4524 [(set_attr "type" "logic_shift_imm")]
4527 (define_insn "one_cmpl<mode>2"
4528 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4529 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4534 [(set_attr "type" "logic_reg,neon_logic")
4535 (set_attr "arch" "*,simd")]
4538 (define_insn "*one_cmpl_<optab><mode>2"
4539 [(set (match_operand:GPI 0 "register_operand" "=r")
4540 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4541 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4543 "mvn\\t%<w>0, %<w>1, <shift> %2"
4544 [(set_attr "type" "logic_shift_imm")]
4547 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4549 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4550 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4551 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4552 (match_operand:GPI 2 "register_operand" "r,w")))]
4555 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4556 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4557 [(set_attr "type" "logic_reg,neon_logic")
4558 (set_attr "arch" "*,simd")]
4561 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4562 [(set (match_operand:DI 0 "register_operand" "=r")
4564 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4565 (match_operand:SI 2 "register_operand" "r"))))]
4567 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4568 [(set_attr "type" "logic_reg")]
4571 (define_insn "*xor_one_cmplsidi3_ze"
4572 [(set (match_operand:DI 0 "register_operand" "=r")
4574 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4575 (match_operand:SI 2 "register_operand" "r")))))]
4577 "eon\\t%w0, %w1, %w2"
4578 [(set_attr "type" "logic_reg")]
4581 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4582 ;; eon does not operate on SIMD registers so the vector variant must be split.
4583 (define_insn_and_split "*xor_one_cmpl<mode>3"
4584 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4585 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4586 (match_operand:GPI 2 "register_operand" "r,w"))))]
4589 eon\\t%<w>0, %<w>1, %<w>2
4591 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4592 [(set (match_operand:GPI 0 "register_operand" "=w")
4593 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4594 (match_operand:GPI 2 "register_operand" "w")))
4595 (set (match_dup 0) (not:GPI (match_dup 0)))]
4597 [(set_attr "type" "logic_reg,multiple")
4598 (set_attr "arch" "*,simd")]
4601 (define_insn "*and_one_cmpl<mode>3_compare0"
4602 [(set (reg:CC_NZ CC_REGNUM)
4605 (match_operand:GPI 1 "register_operand" "r"))
4606 (match_operand:GPI 2 "register_operand" "r"))
4608 (set (match_operand:GPI 0 "register_operand" "=r")
4609 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4611 "bics\\t%<w>0, %<w>2, %<w>1"
4612 [(set_attr "type" "logics_reg")]
4615 ;; zero_extend version of above
4616 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4617 [(set (reg:CC_NZ CC_REGNUM)
4620 (match_operand:SI 1 "register_operand" "r"))
4621 (match_operand:SI 2 "register_operand" "r"))
4623 (set (match_operand:DI 0 "register_operand" "=r")
4624 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4626 "bics\\t%w0, %w2, %w1"
4627 [(set_attr "type" "logics_reg")]
4630 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4631 [(set (reg:CC_NZ CC_REGNUM)
4634 (match_operand:GPI 0 "register_operand" "r"))
4635 (match_operand:GPI 1 "register_operand" "r"))
4638 "bics\\t<w>zr, %<w>1, %<w>0"
4639 [(set_attr "type" "logics_reg")]
4642 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4643 [(set (match_operand:GPI 0 "register_operand" "=r")
4644 (LOGICAL:GPI (not:GPI
4646 (match_operand:GPI 1 "register_operand" "r")
4647 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4648 (match_operand:GPI 3 "register_operand" "r")))]
4650 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4651 [(set_attr "type" "logic_shift_imm")]
4654 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4655 [(set (match_operand:GPI 0 "register_operand" "=r")
4658 (match_operand:GPI 1 "register_operand" "r")
4659 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4660 (match_operand:GPI 3 "register_operand" "r"))))]
4662 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4663 [(set_attr "type" "logic_shift_imm")]
4666 ;; Zero-extend version of the above.
4667 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4668 [(set (match_operand:DI 0 "register_operand" "=r")
4672 (match_operand:SI 1 "register_operand" "r")
4673 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4674 (match_operand:SI 3 "register_operand" "r")))))]
4676 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4677 [(set_attr "type" "logic_shift_imm")]
4680 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4681 [(set (reg:CC_NZ CC_REGNUM)
4685 (match_operand:GPI 1 "register_operand" "r")
4686 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4687 (match_operand:GPI 3 "register_operand" "r"))
4689 (set (match_operand:GPI 0 "register_operand" "=r")
4692 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4694 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4695 [(set_attr "type" "logics_shift_imm")]
4698 ;; zero_extend version of above
4699 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4700 [(set (reg:CC_NZ CC_REGNUM)
4704 (match_operand:SI 1 "register_operand" "r")
4705 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4706 (match_operand:SI 3 "register_operand" "r"))
4708 (set (match_operand:DI 0 "register_operand" "=r")
4709 (zero_extend:DI (and:SI
4711 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4713 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4714 [(set_attr "type" "logics_shift_imm")]
4717 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4718 [(set (reg:CC_NZ CC_REGNUM)
4722 (match_operand:GPI 0 "register_operand" "r")
4723 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4724 (match_operand:GPI 2 "register_operand" "r"))
4727 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4728 [(set_attr "type" "logics_shift_imm")]
4731 (define_insn "clz<mode>2"
4732 [(set (match_operand:GPI 0 "register_operand" "=r")
4733 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4735 "clz\\t%<w>0, %<w>1"
4736 [(set_attr "type" "clz")]
4739 (define_expand "ffs<mode>2"
4740 [(match_operand:GPI 0 "register_operand")
4741 (match_operand:GPI 1 "register_operand")]
4744 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4745 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4747 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4748 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4749 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4754 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4761 (define_expand "popcount<mode>2"
4762 [(match_operand:GPI 0 "register_operand")
4763 (match_operand:GPI 1 "register_operand")]
4766 rtx v = gen_reg_rtx (V8QImode);
4767 rtx v1 = gen_reg_rtx (V8QImode);
4768 rtx r = gen_reg_rtx (QImode);
4769 rtx in = operands[1];
4770 rtx out = operands[0];
4771 if(<MODE>mode == SImode)
4774 tmp = gen_reg_rtx (DImode);
4775 /* If we have SImode, zero extend to DImode, pop count does
4776 not change if we have extra zeros. */
4777 emit_insn (gen_zero_extendsidi2 (tmp, in));
4780 emit_move_insn (v, gen_lowpart (V8QImode, in));
4781 emit_insn (gen_popcountv8qi2 (v1, v));
4782 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4783 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4787 (define_insn "clrsb<mode>2"
4788 [(set (match_operand:GPI 0 "register_operand" "=r")
4789 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4791 "cls\\t%<w>0, %<w>1"
4792 [(set_attr "type" "clz")]
4795 (define_insn "rbit<mode>2"
4796 [(set (match_operand:GPI 0 "register_operand" "=r")
4797 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4799 "rbit\\t%<w>0, %<w>1"
4800 [(set_attr "type" "rbit")]
4803 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4804 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4805 ;; expression and split after reload to enable scheduling them apart if
4808 (define_insn_and_split "ctz<mode>2"
4809 [(set (match_operand:GPI 0 "register_operand" "=r")
4810 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4816 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4817 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4821 (define_insn "*and<mode>_compare0"
4822 [(set (reg:CC_NZ CC_REGNUM)
4824 (match_operand:SHORT 0 "register_operand" "r")
4827 "tst\\t%<w>0, <short_mask>"
4828 [(set_attr "type" "alus_imm")]
4831 (define_insn "*ands<GPI:mode>_compare0"
4832 [(set (reg:CC_NZ CC_REGNUM)
4834 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4836 (set (match_operand:GPI 0 "register_operand" "=r")
4837 (zero_extend:GPI (match_dup 1)))]
4839 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4840 [(set_attr "type" "alus_imm")]
4843 (define_insn "*and<mode>3nr_compare0"
4844 [(set (reg:CC_NZ CC_REGNUM)
4846 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4847 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4853 [(set_attr "type" "logics_reg,logics_imm")]
4857 [(set (reg:CC_NZ CC_REGNUM)
4859 (and:GPI (match_operand:GPI 0 "register_operand")
4860 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4862 (clobber (match_operand:SI 2 "register_operand"))]
4864 [(set (match_dup 2) (match_dup 1))
4865 (set (reg:CC_NZ CC_REGNUM)
4867 (and:GPI (match_dup 0)
4872 (define_insn "*and<mode>3nr_compare0_zextract"
4873 [(set (reg:CC_NZ CC_REGNUM)
4875 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4876 (match_operand:GPI 1 "const_int_operand" "n")
4877 (match_operand:GPI 2 "const_int_operand" "n"))
4879 "INTVAL (operands[1]) > 0
4880 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4881 <= GET_MODE_BITSIZE (<MODE>mode))
4882 && aarch64_bitmask_imm (
4883 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4888 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4889 return "tst\\t%<w>0, %1";
4891 [(set_attr "type" "logics_shift_imm")]
4894 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4895 [(set (reg:CC_NZ CC_REGNUM)
4898 (match_operand:GPI 0 "register_operand" "r")
4899 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4900 (match_operand:GPI 2 "register_operand" "r"))
4903 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4904 [(set_attr "type" "logics_shift_imm")]
4908 [(set (reg:CC_NZ CC_REGNUM)
4911 (match_operand:GPI 0 "register_operand")
4912 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4913 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4915 (clobber (match_operand:SI 3 "register_operand"))]
4917 [(set (match_dup 3) (match_dup 2))
4918 (set (reg:CC_NZ CC_REGNUM)
4927 ;; -------------------------------------------------------------------
4929 ;; -------------------------------------------------------------------
4931 (define_expand "<optab><mode>3"
4932 [(set (match_operand:GPI 0 "register_operand")
4933 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4934 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4937 if (CONST_INT_P (operands[2]))
4939 operands[2] = GEN_INT (INTVAL (operands[2])
4940 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4942 if (operands[2] == const0_rtx)
4944 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4951 (define_expand "ashl<mode>3"
4952 [(set (match_operand:SHORT 0 "register_operand")
4953 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4954 (match_operand:QI 2 "const_int_operand")))]
4957 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4959 if (operands[2] == const0_rtx)
4961 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4967 (define_expand "rotr<mode>3"
4968 [(set (match_operand:GPI 0 "register_operand")
4969 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4970 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4973 if (CONST_INT_P (operands[2]))
4975 operands[2] = GEN_INT (INTVAL (operands[2])
4976 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4978 if (operands[2] == const0_rtx)
4980 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4987 (define_expand "rotl<mode>3"
4988 [(set (match_operand:GPI 0 "register_operand")
4989 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4990 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4993 /* (SZ - cnt) % SZ == -cnt % SZ */
4994 if (CONST_INT_P (operands[2]))
4996 operands[2] = GEN_INT ((-INTVAL (operands[2]))
4997 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4998 if (operands[2] == const0_rtx)
5000 emit_insn (gen_mov<mode> (operands[0], operands[1]));
5005 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
5010 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5011 ;; they truncate the shift/rotate amount by the size of the registers they
5012 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5013 ;; such redundant masking instructions. GCC can do that automatically when
5014 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5015 ;; because some of the SISD shift alternatives don't perform this truncations.
5016 ;; So this pattern exists to catch such cases.
5018 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5019 [(set (match_operand:GPI 0 "register_operand" "=r")
5021 (match_operand:GPI 1 "register_operand" "r")
5022 (match_operator 4 "subreg_lowpart_operator"
5023 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5024 (match_operand 3 "const_int_operand" "n"))])))]
5025 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5026 "<shift>\t%<w>0, %<w>1, %<w>2"
5027 [(set_attr "type" "shift_reg")]
5030 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5031 [(set (match_operand:GPI 0 "register_operand" "=&r")
5033 (match_operand:GPI 1 "register_operand" "r")
5034 (match_operator 4 "subreg_lowpart_operator"
5035 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5036 (match_operand 3 "const_int_operand" "n")))])))]
5037 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5042 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5043 : lowpart_subreg (SImode, operands[0], <MODE>mode));
5044 emit_insn (gen_negsi2 (tmp, operands[2]));
5046 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5047 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5048 SUBREG_BYTE (operands[4]));
5049 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5054 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5055 [(set (match_operand:GPI 0 "register_operand" "=&r")
5057 (match_operand:GPI 1 "register_operand" "r")
5058 (minus:QI (match_operand 2 "const_int_operand" "n")
5059 (match_operator 5 "subreg_lowpart_operator"
5060 [(and:SI (match_operand:SI 3 "register_operand" "r")
5061 (match_operand 4 "const_int_operand" "n"))]))))]
5062 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5063 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5068 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5071 emit_insn (gen_negsi2 (tmp, operands[3]));
5073 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5074 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5075 SUBREG_BYTE (operands[5]));
5077 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5082 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5083 [(set (match_operand:DI 0 "register_operand" "=r")
5085 (match_operand:DI 1 "register_operand" "r")
5086 (match_operator 4 "subreg_lowpart_operator"
5087 [(and:SI (match_operand:SI 2 "register_operand" "r")
5088 (match_operand 3 "const_int_operand" "n"))])))]
5089 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5092 xop[0] = operands[0];
5093 xop[1] = operands[1];
5094 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5095 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5098 [(set_attr "type" "shift_reg")]
5101 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5102 [(set (match_operand:GPI 0 "register_operand" "=&r")
5104 (match_operand:GPI 1 "register_operand" "r")
5105 (minus:QI (match_operand 2 "const_int_operand" "n")
5106 (match_operand:QI 3 "register_operand" "r"))))]
5107 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5112 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5114 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5115 : gen_lowpart (SImode, operands[0]));
5117 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5119 rtx and_op = gen_rtx_AND (SImode, tmp,
5120 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5122 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5124 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5127 [(set_attr "length" "8")]
5130 ;; Logical left shift using SISD or Integer instruction
5131 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5132 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5134 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5135 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5138 lsl\t%<w>0, %<w>1, %2
5139 lsl\t%<w>0, %<w>1, %<w>2
5140 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5141 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5142 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5143 (set_attr "arch" "*,*,simd,simd")]
5146 ;; Logical right shift using SISD or Integer instruction
5147 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5148 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5150 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5151 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5152 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5155 lsr\t%<w>0, %<w>1, %2
5156 lsr\t%<w>0, %<w>1, %<w>2
5157 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5160 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5161 (set_attr "arch" "*,*,simd,simd,simd")]
5165 [(set (match_operand:DI 0 "aarch64_simd_register")
5167 (match_operand:DI 1 "aarch64_simd_register")
5168 (match_operand:QI 2 "aarch64_simd_register")))]
5169 "TARGET_SIMD && reload_completed"
5171 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5173 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5175 operands[3] = gen_lowpart (QImode, operands[0]);
5180 [(set (match_operand:SI 0 "aarch64_simd_register")
5182 (match_operand:SI 1 "aarch64_simd_register")
5183 (match_operand:QI 2 "aarch64_simd_register")))]
5184 "TARGET_SIMD && reload_completed"
5186 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5188 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5190 operands[3] = gen_lowpart (QImode, operands[0]);
5194 ;; Arithmetic right shift using SISD or Integer instruction
5195 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5196 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5198 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5199 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5200 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5203 asr\t%<w>0, %<w>1, %2
5204 asr\t%<w>0, %<w>1, %<w>2
5205 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5208 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5209 (set_attr "arch" "*,*,simd,simd,simd")]
5213 [(set (match_operand:DI 0 "aarch64_simd_register")
5215 (match_operand:DI 1 "aarch64_simd_register")
5216 (match_operand:QI 2 "aarch64_simd_register")))]
5217 "TARGET_SIMD && reload_completed"
5219 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5221 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5223 operands[3] = gen_lowpart (QImode, operands[0]);
5228 [(set (match_operand:SI 0 "aarch64_simd_register")
5230 (match_operand:SI 1 "aarch64_simd_register")
5231 (match_operand:QI 2 "aarch64_simd_register")))]
5232 "TARGET_SIMD && reload_completed"
5234 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5236 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5238 operands[3] = gen_lowpart (QImode, operands[0]);
5242 (define_insn "*aarch64_sisd_ushl"
5243 [(set (match_operand:DI 0 "register_operand" "=w")
5244 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5245 (match_operand:QI 2 "register_operand" "w")]
5248 "ushl\t%d0, %d1, %d2"
5249 [(set_attr "type" "neon_shift_reg")]
5252 (define_insn "*aarch64_ushl_2s"
5253 [(set (match_operand:SI 0 "register_operand" "=w")
5254 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5255 (match_operand:QI 2 "register_operand" "w")]
5258 "ushl\t%0.2s, %1.2s, %2.2s"
5259 [(set_attr "type" "neon_shift_reg")]
5262 (define_insn "*aarch64_sisd_sshl"
5263 [(set (match_operand:DI 0 "register_operand" "=w")
5264 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5265 (match_operand:QI 2 "register_operand" "w")]
5268 "sshl\t%d0, %d1, %d2"
5269 [(set_attr "type" "neon_shift_reg")]
5272 (define_insn "*aarch64_sshl_2s"
5273 [(set (match_operand:SI 0 "register_operand" "=w")
5274 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5275 (match_operand:QI 2 "register_operand" "w")]
5278 "sshl\t%0.2s, %1.2s, %2.2s"
5279 [(set_attr "type" "neon_shift_reg")]
5282 (define_insn "*aarch64_sisd_neg_qi"
5283 [(set (match_operand:QI 0 "register_operand" "=w")
5284 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5288 [(set_attr "type" "neon_neg")]
5292 (define_insn "*ror<mode>3_insn"
5293 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5295 (match_operand:GPI 1 "register_operand" "r,r")
5296 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5299 ror\\t%<w>0, %<w>1, %2
5300 ror\\t%<w>0, %<w>1, %<w>2"
5301 [(set_attr "type" "rotate_imm,shift_reg")]
5304 ;; zero_extend version of above
5305 (define_insn "*<optab>si3_insn_uxtw"
5306 [(set (match_operand:DI 0 "register_operand" "=r,r")
5307 (zero_extend:DI (SHIFT:SI
5308 (match_operand:SI 1 "register_operand" "r,r")
5309 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5312 <shift>\\t%w0, %w1, %2
5313 <shift>\\t%w0, %w1, %w2"
5314 [(set_attr "type" "bfx,shift_reg")]
5317 (define_insn "*<optab><mode>3_insn"
5318 [(set (match_operand:SHORT 0 "register_operand" "=r")
5319 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5320 (match_operand 2 "const_int_operand" "n")))]
5321 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5323 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5324 return "<bfshift>\t%w0, %w1, %2, %3";
5326 [(set_attr "type" "bfx")]
5329 (define_insn "*extr<mode>5_insn"
5330 [(set (match_operand:GPI 0 "register_operand" "=r")
5331 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5332 (match_operand 3 "const_int_operand" "n"))
5333 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5334 (match_operand 4 "const_int_operand" "n"))))]
5335 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5336 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5337 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5338 [(set_attr "type" "rotate_imm")]
5341 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5342 ;; so we have to match both orderings.
5343 (define_insn "*extr<mode>5_insn_alt"
5344 [(set (match_operand:GPI 0 "register_operand" "=r")
5345 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5346 (match_operand 4 "const_int_operand" "n"))
5347 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5348 (match_operand 3 "const_int_operand" "n"))))]
5349 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5350 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5351 == GET_MODE_BITSIZE (<MODE>mode))"
5352 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5353 [(set_attr "type" "rotate_imm")]
5356 ;; zero_extend version of the above
5357 (define_insn "*extrsi5_insn_uxtw"
5358 [(set (match_operand:DI 0 "register_operand" "=r")
5360 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5361 (match_operand 3 "const_int_operand" "n"))
5362 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5363 (match_operand 4 "const_int_operand" "n")))))]
5364 "UINTVAL (operands[3]) < 32 &&
5365 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5366 "extr\\t%w0, %w1, %w2, %4"
5367 [(set_attr "type" "rotate_imm")]
5370 (define_insn "*extrsi5_insn_uxtw_alt"
5371 [(set (match_operand:DI 0 "register_operand" "=r")
5373 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5374 (match_operand 4 "const_int_operand" "n"))
5375 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5376 (match_operand 3 "const_int_operand" "n")))))]
5377 "UINTVAL (operands[3]) < 32 &&
5378 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5379 "extr\\t%w0, %w1, %w2, %4"
5380 [(set_attr "type" "rotate_imm")]
5383 (define_insn "*ror<mode>3_insn"
5384 [(set (match_operand:GPI 0 "register_operand" "=r")
5385 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5386 (match_operand 2 "const_int_operand" "n")))]
5387 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5389 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5390 return "ror\\t%<w>0, %<w>1, %3";
5392 [(set_attr "type" "rotate_imm")]
5395 ;; zero_extend version of the above
5396 (define_insn "*rorsi3_insn_uxtw"
5397 [(set (match_operand:DI 0 "register_operand" "=r")
5399 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5400 (match_operand 2 "const_int_operand" "n"))))]
5401 "UINTVAL (operands[2]) < 32"
5403 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5404 return "ror\\t%w0, %w1, %3";
5406 [(set_attr "type" "rotate_imm")]
5409 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5410 [(set (match_operand:GPI 0 "register_operand" "=r")
5412 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5413 (match_operand 2 "const_int_operand" "n"))))]
5414 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5416 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5417 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5419 [(set_attr "type" "bfx")]
5422 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5423 [(set (match_operand:GPI 0 "register_operand" "=r")
5425 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5426 (match_operand 2 "const_int_operand" "n"))))]
5427 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5429 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5430 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5432 [(set_attr "type" "bfx")]
5435 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5436 [(set (match_operand:GPI 0 "register_operand" "=r")
5438 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5439 (match_operand 2 "const_int_operand" "n"))))]
5440 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5442 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5443 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5445 [(set_attr "type" "bfx")]
5448 ;; -------------------------------------------------------------------
5450 ;; -------------------------------------------------------------------
5452 (define_expand "<optab>"
5453 [(set (match_operand:DI 0 "register_operand")
5454 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5456 "aarch64_simd_shift_imm_offset_di")
5457 (match_operand 3 "aarch64_simd_shift_imm_di")))]
5460 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5461 1, GET_MODE_BITSIZE (DImode) - 1))
5467 (define_insn "*<optab><mode>"
5468 [(set (match_operand:GPI 0 "register_operand" "=r")
5469 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5471 "aarch64_simd_shift_imm_offset_<mode>" "n")
5473 "aarch64_simd_shift_imm_<mode>" "n")))]
5474 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5475 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5476 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5477 [(set_attr "type" "bfx")]
5480 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5481 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5483 [(set (match_operand:DI 0 "register_operand")
5484 (zero_extract:DI (match_operand:DI 1 "register_operand")
5486 "aarch64_simd_shift_imm_offset_di")
5488 "aarch64_simd_shift_imm_di")))]
5489 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5490 GET_MODE_BITSIZE (DImode) - 1)
5491 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5492 == GET_MODE_BITSIZE (SImode)"
5494 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5496 operands[4] = gen_lowpart (SImode, operands[1]);
5500 ;; Bitfield Insert (insv)
5501 (define_expand "insv<mode>"
5502 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5503 (match_operand 1 "const_int_operand")
5504 (match_operand 2 "const_int_operand"))
5505 (match_operand:GPI 3 "general_operand"))]
5508 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5509 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5510 rtx value = operands[3];
5512 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5515 if (CONST_INT_P (value))
5517 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5519 /* Prefer AND/OR for inserting all zeros or all ones. */
5520 if ((UINTVAL (value) & mask) == 0
5521 || (UINTVAL (value) & mask) == mask)
5524 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5525 if (width == 16 && (pos % 16) == 0)
5528 operands[3] = force_reg (<MODE>mode, value);
5531 (define_insn "*insv_reg<mode>"
5532 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5533 (match_operand 1 "const_int_operand" "n")
5534 (match_operand 2 "const_int_operand" "n"))
5535 (match_operand:GPI 3 "register_operand" "r"))]
5536 "!(UINTVAL (operands[1]) == 0
5537 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5538 > GET_MODE_BITSIZE (<MODE>mode)))"
5539 "bfi\\t%<w>0, %<w>3, %2, %1"
5540 [(set_attr "type" "bfm")]
5543 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5544 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5545 (match_operand 1 "const_int_operand" "n")
5546 (match_operand 2 "const_int_operand" "n"))
5547 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5548 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5549 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5550 [(set_attr "type" "bfm")]
5553 ;; Match a bfi instruction where the shift of OP3 means that we are
5554 ;; actually copying the least significant bits of OP3 into OP0 by way
5555 ;; of the AND masks and the IOR instruction. A similar instruction
5556 ;; with the two parts of the IOR swapped around was never triggered
5557 ;; in a bootstrap build and test of GCC so it was not included.
5559 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5560 [(set (match_operand:GPI 0 "register_operand" "=r")
5561 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5562 (match_operand:GPI 2 "const_int_operand" "n"))
5563 (and:GPI (ashift:GPI
5564 (match_operand:GPI 3 "register_operand" "r")
5565 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5566 (match_operand:GPI 5 "const_int_operand" "n"))))]
5567 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5568 UINTVAL (operands[4]),
5569 UINTVAL(operands[5]))"
5570 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5571 [(set_attr "type" "bfm")]
5574 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5575 [(set (match_operand:GPI 0 "register_operand" "=r")
5576 (ior:GPI (and:GPI (ashift:GPI
5577 (match_operand:GPI 1 "register_operand" "r")
5578 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5579 (match_operand:GPI 3 "const_int_operand" "n"))
5580 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5581 (match_operand:GPI 5 "const_int_operand" "n"))))]
5582 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5583 UINTVAL (operands[2]),
5584 UINTVAL(operands[3]))"
5585 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5586 [(set_attr "type" "bfm")]
5589 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5590 ;; the shift is large enough to remove the need for an AND instruction.
5592 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5593 [(set (match_operand:GPI 0 "register_operand" "=r")
5594 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5595 (match_operand:GPI 2 "const_int_operand" "n"))
5597 (match_operand:GPI 3 "register_operand" "r")
5598 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5599 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5600 UINTVAL (operands[4]),
5601 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5603 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5604 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5606 [(set_attr "type" "bfm")]
5609 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5610 [(set (match_operand:GPI 0 "register_operand" "=r")
5611 (ior:GPI (ashift:GPI
5612 (match_operand:GPI 1 "register_operand" "r")
5613 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5614 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5615 (match_operand:GPI 4 "const_int_operand" "n"))))]
5616 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5617 UINTVAL (operands[2]),
5618 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5620 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5621 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5623 [(set_attr "type" "bfm")]
5626 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5627 ;; copying the least significant bits of OP3 to OP0. We need two versions
5628 ;; of the instruction to handle different checks on the constant values.
5630 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5631 [(set (match_operand:GPI 0 "register_operand" "=r")
5632 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5633 (match_operand:GPI 2 "const_int_operand" "n"))
5634 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5635 (match_operand:GPI 4 "const_int_operand" "n"))))]
5636 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5637 UINTVAL (operands[4]))"
5638 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5639 [(set_attr "type" "bfm")]
5642 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5643 [(set (match_operand:GPI 0 "register_operand" "=r")
5644 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5645 (match_operand:GPI 4 "const_int_operand" "n"))
5646 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5647 (match_operand:GPI 2 "const_int_operand" "n"))))]
5648 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5649 UINTVAL (operands[4]))"
5650 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5651 [(set_attr "type" "bfm")]
5654 (define_insn "*extr_insv_lower_reg<mode>"
5655 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5656 (match_operand 1 "const_int_operand" "n")
5658 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5660 (match_operand 3 "const_int_operand" "n")))]
5661 "!(UINTVAL (operands[1]) == 0
5662 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5663 > GET_MODE_BITSIZE (<MODE>mode)))"
5664 "bfxil\\t%<w>0, %<w>2, %3, %1"
5665 [(set_attr "type" "bfm")]
5668 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5669 [(set (match_operand:GPI 0 "register_operand" "=r")
5670 (ashift:GPI (ANY_EXTEND:GPI
5671 (match_operand:ALLX 1 "register_operand" "r"))
5672 (match_operand 2 "const_int_operand" "n")))]
5673 "UINTVAL (operands[2]) < <GPI:sizen>"
5675 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5676 ? GEN_INT (<ALLX:sizen>)
5677 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5678 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5680 [(set_attr "type" "bfx")]
5683 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5685 (define_insn "*andim_ashift<mode>_bfiz"
5686 [(set (match_operand:GPI 0 "register_operand" "=r")
5687 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5688 (match_operand 2 "const_int_operand" "n"))
5689 (match_operand 3 "const_int_operand" "n")))]
5690 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5691 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5692 [(set_attr "type" "bfx")]
5695 ;; Match sbfiz pattern in a shift left + shift right operation.
5697 (define_insn "*ashift<mode>_extv_bfiz"
5698 [(set (match_operand:GPI 0 "register_operand" "=r")
5699 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5700 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5702 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5703 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5704 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5705 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5706 [(set_attr "type" "bfx")]
5709 ;; When the bit position and width of the equivalent extraction add up to 32
5710 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5711 ;; zero-extension of the X-reg.
5713 [(set (match_operand:DI 0 "register_operand")
5714 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5715 (match_operand 2 "const_int_operand"))
5716 (match_operand 3 "const_int_operand")))]
5717 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5718 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5719 == GET_MODE_BITSIZE (SImode)"
5721 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5723 operands[4] = gen_lowpart (SImode, operands[1]);
5727 (define_insn "bswap<mode>2"
5728 [(set (match_operand:GPI 0 "register_operand" "=r")
5729 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5731 "rev\\t%<w>0, %<w>1"
5732 [(set_attr "type" "rev")]
5735 (define_insn "bswaphi2"
5736 [(set (match_operand:HI 0 "register_operand" "=r")
5737 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5740 [(set_attr "type" "rev")]
5743 (define_insn "*aarch64_bfxil<mode>"
5744 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5745 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5746 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5747 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5748 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5749 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5750 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5751 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5753 switch (which_alternative)
5756 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5757 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5759 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5760 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5765 [(set_attr "type" "bfm")]
5768 ; Zero-extended version of above (aarch64_bfxil)
5769 (define_insn "*aarch64_bfxilsi_uxtw"
5770 [(set (match_operand:DI 0 "register_operand" "=r,r")
5771 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5773 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5774 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5775 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5776 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5777 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5778 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5780 switch (which_alternative)
5783 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5784 return "bfxil\\t%0, %1, 0, %3";
5786 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5787 return "bfxil\\t%0, %2, 0, %3";
5792 [(set_attr "type" "bfm")]
5795 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5796 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5797 ;; each valid permutation.
5799 (define_insn "rev16<mode>2"
5800 [(set (match_operand:GPI 0 "register_operand" "=r")
5801 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5803 (match_operand:GPI 3 "const_int_operand" "n"))
5804 (and:GPI (lshiftrt:GPI (match_dup 1)
5806 (match_operand:GPI 2 "const_int_operand" "n"))))]
5807 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5808 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5809 "rev16\\t%<w>0, %<w>1"
5810 [(set_attr "type" "rev")]
5813 (define_insn "rev16<mode>2_alt"
5814 [(set (match_operand:GPI 0 "register_operand" "=r")
5815 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5817 (match_operand:GPI 2 "const_int_operand" "n"))
5818 (and:GPI (ashift:GPI (match_dup 1)
5820 (match_operand:GPI 3 "const_int_operand" "n"))))]
5821 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5822 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5823 "rev16\\t%<w>0, %<w>1"
5824 [(set_attr "type" "rev")]
5827 ;; zero_extend version of above
5828 (define_insn "*bswapsi2_uxtw"
5829 [(set (match_operand:DI 0 "register_operand" "=r")
5830 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5833 [(set_attr "type" "rev")]
5836 ;; -------------------------------------------------------------------
5837 ;; Floating-point intrinsics
5838 ;; -------------------------------------------------------------------
5840 ;; frint floating-point round to integral standard patterns.
5841 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5843 (define_insn "<frint_pattern><mode>2"
5844 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5845 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5848 "frint<frint_suffix>\\t%<s>0, %<s>1"
5849 [(set_attr "type" "f_rint<stype>")]
5852 ;; frcvt floating-point round to integer and convert standard patterns.
5853 ;; Expands to lbtrunc, lceil, lfloor, lround.
5854 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5855 [(set (match_operand:GPI 0 "register_operand" "=r")
5857 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5860 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5861 [(set_attr "type" "f_cvtf2i")]
5864 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5865 [(set (match_operand:GPI 0 "register_operand" "=r")
5868 (match_operand:GPF 1 "register_operand" "w")
5869 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5871 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5872 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5874 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5876 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5877 output_asm_insn (buf, operands);
5880 [(set_attr "type" "f_cvtf2i")]
5883 ;; fma - expand fma into patterns with the accumulator operand first since
5884 ;; reusing the accumulator results in better register allocation.
5885 ;; The register allocator considers copy preferences in operand order,
5886 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5888 (define_expand "fma<mode>4"
5889 [(set (match_operand:GPF_F16 0 "register_operand")
5890 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5891 (match_operand:GPF_F16 2 "register_operand")
5892 (match_operand:GPF_F16 3 "register_operand")))]
5896 (define_insn "*aarch64_fma<mode>4"
5897 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5898 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5899 (match_operand:GPF_F16 3 "register_operand" "w")
5900 (match_operand:GPF_F16 1 "register_operand" "w")))]
5902 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5903 [(set_attr "type" "fmac<stype>")]
5906 (define_expand "fnma<mode>4"
5907 [(set (match_operand:GPF_F16 0 "register_operand")
5909 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5910 (match_operand:GPF_F16 2 "register_operand")
5911 (match_operand:GPF_F16 3 "register_operand")))]
5915 (define_insn "*aarch64_fnma<mode>4"
5916 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5918 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5919 (match_operand:GPF_F16 3 "register_operand" "w")
5920 (match_operand:GPF_F16 1 "register_operand" "w")))]
5922 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5923 [(set_attr "type" "fmac<stype>")]
5927 (define_expand "fms<mode>4"
5928 [(set (match_operand:GPF 0 "register_operand")
5929 (fma:GPF (match_operand:GPF 1 "register_operand")
5930 (match_operand:GPF 2 "register_operand")
5931 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5935 (define_insn "*aarch64_fms<mode>4"
5936 [(set (match_operand:GPF 0 "register_operand" "=w")
5937 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5938 (match_operand:GPF 3 "register_operand" "w")
5939 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5941 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5942 [(set_attr "type" "fmac<s>")]
5945 (define_expand "fnms<mode>4"
5946 [(set (match_operand:GPF 0 "register_operand")
5947 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5948 (match_operand:GPF 2 "register_operand")
5949 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5953 (define_insn "*aarch64_fnms<mode>4"
5954 [(set (match_operand:GPF 0 "register_operand" "=w")
5955 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5956 (match_operand:GPF 3 "register_operand" "w")
5957 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5959 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5960 [(set_attr "type" "fmac<s>")]
5963 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5964 (define_insn "*aarch64_fnmadd<mode>4"
5965 [(set (match_operand:GPF 0 "register_operand" "=w")
5966 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5967 (match_operand:GPF 3 "register_operand" "w")
5968 (match_operand:GPF 1 "register_operand" "w"))))]
5969 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5970 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5971 [(set_attr "type" "fmac<s>")]
5974 ;; -------------------------------------------------------------------
5975 ;; Floating-point conversions
5976 ;; -------------------------------------------------------------------
5978 (define_insn "extendsfdf2"
5979 [(set (match_operand:DF 0 "register_operand" "=w")
5980 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5983 [(set_attr "type" "f_cvt")]
5986 (define_insn "extendhfsf2"
5987 [(set (match_operand:SF 0 "register_operand" "=w")
5988 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5991 [(set_attr "type" "f_cvt")]
5994 (define_insn "extendhfdf2"
5995 [(set (match_operand:DF 0 "register_operand" "=w")
5996 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5999 [(set_attr "type" "f_cvt")]
6002 (define_insn "truncdfsf2"
6003 [(set (match_operand:SF 0 "register_operand" "=w")
6004 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
6007 [(set_attr "type" "f_cvt")]
6010 (define_insn "truncsfhf2"
6011 [(set (match_operand:HF 0 "register_operand" "=w")
6012 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6015 [(set_attr "type" "f_cvt")]
6018 (define_insn "truncdfhf2"
6019 [(set (match_operand:HF 0 "register_operand" "=w")
6020 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6023 [(set_attr "type" "f_cvt")]
6026 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6027 ;; and making r = w more expensive
6029 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6030 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6031 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6034 fcvtz<su>\t%<s>0, %<s>1
6035 fcvtz<su>\t%<w>0, %<s>1"
6036 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6039 ;; Convert HF -> SI or DI
6041 (define_insn "<optab>_trunchf<GPI:mode>2"
6042 [(set (match_operand:GPI 0 "register_operand" "=r")
6043 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6045 "fcvtz<su>\t%<w>0, %h1"
6046 [(set_attr "type" "f_cvtf2i")]
6049 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6050 ;; input in a fp register and output in a integer register
6052 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6053 [(set (match_operand:GPI 0 "register_operand" "=r")
6054 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6056 "fcvtz<su>\t%<w>0, %<fpw>1"
6057 [(set_attr "type" "f_cvtf2i")]
6060 (define_insn "*fix_to_zero_extend<mode>di2"
6061 [(set (match_operand:DI 0 "register_operand" "=r")
6064 (match_operand:GPF 1 "register_operand" "w"))))]
6066 "fcvtzu\t%w0, %<s>1"
6067 [(set_attr "type" "f_cvtf2i")]
6070 ;; Equal width integer to fp and multiply combine.
6071 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6072 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6073 (mult:GPF (FLOATUORS:GPF
6074 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6075 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6078 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6079 switch (which_alternative)
6082 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6084 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6089 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6090 (set_attr "arch" "simd,fp")]
6093 ;; Unequal width integer to fp and multiply combine.
6094 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6095 [(set (match_operand:GPF 0 "register_operand" "=w")
6096 (mult:GPF (FLOATUORS:GPF
6097 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6098 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6101 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6102 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6104 [(set_attr "type" "f_cvti2f")]
6107 ;; Equal width integer to fp conversion.
6108 (define_insn "<optab><fcvt_target><GPF:mode>2"
6109 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6110 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6113 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6114 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6115 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6116 (set_attr "arch" "simd,fp")]
6119 ;; Unequal width integer to fp conversions.
6120 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6121 [(set (match_operand:GPF 0 "register_operand" "=w")
6122 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6124 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6125 [(set_attr "type" "f_cvti2f")]
6128 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6129 ;; midend will arrange for an SImode conversion to HFmode to first go
6130 ;; through DFmode, then to HFmode. But first it will try converting
6131 ;; to DImode then down, which would match our DImode pattern below and
6132 ;; give very poor code-generation. So, we must provide our own emulation
6133 ;; of the mid-end logic.
6135 (define_insn "aarch64_fp16_<optab><mode>hf2"
6136 [(set (match_operand:HF 0 "register_operand" "=w")
6137 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6139 "<su_optab>cvtf\t%h0, %<w>1"
6140 [(set_attr "type" "f_cvti2f")]
6143 (define_expand "<optab>sihf2"
6144 [(set (match_operand:HF 0 "register_operand")
6145 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6148 if (TARGET_FP_F16INST)
6149 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6152 rtx convert_target = gen_reg_rtx (DFmode);
6153 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6154 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6160 ;; For DImode there is no wide enough floating-point mode that we
6161 ;; can convert through natively (TFmode would work, but requires a library
6162 ;; call). However, we know that any value >= 65504 will be rounded
6163 ;; to infinity on conversion. This is well within the range of SImode, so
6165 ;; Saturate to SImode.
6166 ;; Convert from that to DFmode
6167 ;; Convert from that to HFmode (phew!).
6168 ;; Note that the saturation to SImode requires the SIMD extensions. If
6169 ;; we ever need to provide this pattern where the SIMD extensions are not
6170 ;; available, we would need a different approach.
6172 (define_expand "<optab>dihf2"
6173 [(set (match_operand:HF 0 "register_operand")
6174 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6175 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6177 if (TARGET_FP_F16INST)
6178 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6181 rtx sat_target = gen_reg_rtx (SImode);
6182 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6183 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6190 ;; Convert between fixed-point and floating-point (scalar modes)
6192 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6193 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6194 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6195 (match_operand:SI 2 "immediate_operand" "i, i")]
6199 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6200 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6201 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6202 (set_attr "arch" "fp,simd")]
6205 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6206 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6207 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6208 (match_operand:SI 2 "immediate_operand" "i, i")]
6212 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6213 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6214 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6215 (set_attr "arch" "fp,simd")]
6218 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6219 [(set (match_operand:GPI 0 "register_operand" "=r")
6220 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6221 (match_operand:SI 2 "immediate_operand" "i")]
6224 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6225 [(set_attr "type" "f_cvtf2i")]
6228 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6229 [(set (match_operand:HF 0 "register_operand" "=w")
6230 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6231 (match_operand:SI 2 "immediate_operand" "i")]
6234 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6235 [(set_attr "type" "f_cvti2f")]
6238 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6239 [(set (match_operand:HI 0 "register_operand" "=w")
6240 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6241 (match_operand:SI 2 "immediate_operand" "i")]
6244 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6245 [(set_attr "type" "neon_fp_to_int_s")]
6248 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6249 [(set (match_operand:HF 0 "register_operand" "=w")
6250 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6251 (match_operand:SI 2 "immediate_operand" "i")]
6254 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6255 [(set_attr "type" "neon_int_to_fp_s")]
6258 ;; -------------------------------------------------------------------
6259 ;; Floating-point arithmetic
6260 ;; -------------------------------------------------------------------
6262 (define_insn "add<mode>3"
6263 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6265 (match_operand:GPF_F16 1 "register_operand" "w")
6266 (match_operand:GPF_F16 2 "register_operand" "w")))]
6268 "fadd\\t%<s>0, %<s>1, %<s>2"
6269 [(set_attr "type" "fadd<stype>")]
6272 (define_insn "sub<mode>3"
6273 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6275 (match_operand:GPF_F16 1 "register_operand" "w")
6276 (match_operand:GPF_F16 2 "register_operand" "w")))]
6278 "fsub\\t%<s>0, %<s>1, %<s>2"
6279 [(set_attr "type" "fadd<stype>")]
6282 (define_insn "mul<mode>3"
6283 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6285 (match_operand:GPF_F16 1 "register_operand" "w")
6286 (match_operand:GPF_F16 2 "register_operand" "w")))]
6288 "fmul\\t%<s>0, %<s>1, %<s>2"
6289 [(set_attr "type" "fmul<stype>")]
6292 (define_insn "*fnmul<mode>3"
6293 [(set (match_operand:GPF 0 "register_operand" "=w")
6295 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6296 (match_operand:GPF 2 "register_operand" "w")))]
6297 "TARGET_FLOAT && !flag_rounding_math"
6298 "fnmul\\t%<s>0, %<s>1, %<s>2"
6299 [(set_attr "type" "fmul<s>")]
6302 (define_insn "*fnmul<mode>3"
6303 [(set (match_operand:GPF 0 "register_operand" "=w")
6305 (match_operand:GPF 1 "register_operand" "w")
6306 (match_operand:GPF 2 "register_operand" "w"))))]
6308 "fnmul\\t%<s>0, %<s>1, %<s>2"
6309 [(set_attr "type" "fmul<s>")]
6312 (define_expand "div<mode>3"
6313 [(set (match_operand:GPF_F16 0 "register_operand")
6314 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6315 (match_operand:GPF_F16 2 "register_operand")))]
6318 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6321 operands[1] = force_reg (<MODE>mode, operands[1]);
6324 (define_insn "*div<mode>3"
6325 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6326 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6327 (match_operand:GPF_F16 2 "register_operand" "w")))]
6329 "fdiv\\t%<s>0, %<s>1, %<s>2"
6330 [(set_attr "type" "fdiv<stype>")]
6333 (define_insn "neg<mode>2"
6334 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6335 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6337 "fneg\\t%<s>0, %<s>1"
6338 [(set_attr "type" "ffarith<stype>")]
6341 (define_expand "sqrt<mode>2"
6342 [(set (match_operand:GPF_F16 0 "register_operand")
6343 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6346 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6350 (define_insn "*sqrt<mode>2"
6351 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6352 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6354 "fsqrt\\t%<s>0, %<s>1"
6355 [(set_attr "type" "fsqrt<stype>")]
6358 (define_insn "abs<mode>2"
6359 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6360 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6362 "fabs\\t%<s>0, %<s>1"
6363 [(set_attr "type" "ffarith<stype>")]
6366 ;; Given that smax/smin do not specify the result when either input is NaN,
6367 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6370 (define_insn "smax<mode>3"
6371 [(set (match_operand:GPF 0 "register_operand" "=w")
6372 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6373 (match_operand:GPF 2 "register_operand" "w")))]
6375 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6376 [(set_attr "type" "f_minmax<s>")]
6379 (define_insn "smin<mode>3"
6380 [(set (match_operand:GPF 0 "register_operand" "=w")
6381 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6382 (match_operand:GPF 2 "register_operand" "w")))]
6384 "fminnm\\t%<s>0, %<s>1, %<s>2"
6385 [(set_attr "type" "f_minmax<s>")]
6388 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6389 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6390 ;; which implement the IEEE fmax ()/fmin () functions.
6391 (define_insn "<maxmin_uns><mode>3"
6392 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6393 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6394 (match_operand:GPF_F16 2 "register_operand" "w")]
6397 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6398 [(set_attr "type" "f_minmax<stype>")]
6401 (define_expand "lrint<GPF:mode><GPI:mode>2"
6402 [(match_operand:GPI 0 "register_operand")
6403 (match_operand:GPF 1 "register_operand")]
6405 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6406 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6408 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6409 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6410 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6415 ;; For copysign (x, y), we want to generate:
6417 ;; LDR d2, #(1 << 63)
6418 ;; BSL v2.8b, [y], [x]
6420 ;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6421 ;; we expect these operations to nearly always operate on
6422 ;; floating-point values, we do not want the operation to be
6423 ;; simplified into a bit-field insert operation that operates on the
6424 ;; integer side, since typically that would involve three inter-bank
6425 ;; register copies. As we do not expect copysign to be followed by
6426 ;; other logical operations on the result, it seems preferable to keep
6427 ;; this as an unspec operation, rather than exposing the underlying
6428 ;; logic to the compiler.
6430 (define_expand "copysign<GPF:mode>3"
6431 [(match_operand:GPF 0 "register_operand")
6432 (match_operand:GPF 1 "register_operand")
6433 (match_operand:GPF 2 "register_operand")]
6434 "TARGET_FLOAT && TARGET_SIMD"
6436 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6437 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6438 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6439 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6445 (define_insn "copysign<GPF:mode>3_insn"
6446 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6447 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6448 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6449 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6451 "TARGET_FLOAT && TARGET_SIMD"
6453 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6454 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6455 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6456 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6457 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6461 ;; For xorsign (x, y), we want to generate:
6464 ;; AND v3.8B, v1.8B, v2.8B
6465 ;; EOR v0.8B, v0.8B, v3.8B
6468 (define_expand "xorsign<mode>3"
6469 [(match_operand:GPF 0 "register_operand")
6470 (match_operand:GPF 1 "register_operand")
6471 (match_operand:GPF 2 "register_operand")]
6472 "TARGET_FLOAT && TARGET_SIMD"
6475 machine_mode imode = <V_INT_EQUIV>mode;
6476 rtx mask = gen_reg_rtx (imode);
6477 rtx op1x = gen_reg_rtx (imode);
6478 rtx op2x = gen_reg_rtx (imode);
6480 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6481 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6484 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6485 lowpart_subreg (imode, operands[2],
6487 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6488 lowpart_subreg (imode, operands[1],
6491 emit_move_insn (operands[0],
6492 lowpart_subreg (<MODE>mode, op1x, imode));
6497 ;; -------------------------------------------------------------------
6499 ;; -------------------------------------------------------------------
6500 ;; Reload Scalar Floating point modes from constant pool.
6501 ;; The AArch64 port doesn't have __int128 constant move support.
6502 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6503 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6504 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6505 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6506 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6509 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6510 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6515 ;; Reload Vector modes from constant pool.
6516 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6517 [(set (match_operand:VALL 0 "register_operand" "=w")
6518 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6519 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6522 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6523 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6528 (define_expand "@aarch64_reload_mov<mode>"
6529 [(set (match_operand:TX 0 "register_operand" "=w")
6530 (match_operand:TX 1 "register_operand" "w"))
6531 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6535 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6536 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6537 gen_aarch64_movtilow_tilow (op0, op1);
6538 gen_aarch64_movdi_tihigh (operands[2], op1);
6539 gen_aarch64_movtihigh_di (op0, operands[2]);
6544 ;; The following secondary reload helpers patterns are invoked
6545 ;; after or during reload as we don't want these patterns to start
6546 ;; kicking in during the combiner.
6548 (define_insn "@aarch64_movdi_<mode>low"
6549 [(set (match_operand:DI 0 "register_operand" "=r")
6550 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6551 (const_int 64) (const_int 0)))]
6552 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6554 [(set_attr "type" "f_mrc")
6555 (set_attr "length" "4")
6558 (define_insn "@aarch64_movdi_<mode>high"
6559 [(set (match_operand:DI 0 "register_operand" "=r")
6560 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6561 (const_int 64) (const_int 64)))]
6562 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6563 "fmov\\t%x0, %1.d[1]"
6564 [(set_attr "type" "f_mrc")
6565 (set_attr "length" "4")
6568 (define_insn "@aarch64_mov<mode>high_di"
6569 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6570 (const_int 64) (const_int 64))
6571 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6572 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6573 "fmov\\t%0.d[1], %x1"
6574 [(set_attr "type" "f_mcr")
6575 (set_attr "length" "4")
6578 (define_insn "@aarch64_mov<mode>low_di"
6579 [(set (match_operand:TX 0 "register_operand" "=w")
6580 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6581 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6583 [(set_attr "type" "f_mcr")
6584 (set_attr "length" "4")
6587 (define_insn "aarch64_movtilow_tilow"
6588 [(set (match_operand:TI 0 "register_operand" "=w")
6590 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6591 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6593 [(set_attr "type" "fmov")
6594 (set_attr "length" "4")
6597 ;; There is a deliberate reason why the parameters of high and lo_sum's
6598 ;; don't have modes for ADRP and ADD instructions. This is to allow high
6599 ;; and lo_sum's to be used with the labels defining the jump tables in
6602 (define_expand "add_losym"
6603 [(set (match_operand 0 "register_operand")
6604 (lo_sum (match_operand 1 "register_operand")
6605 (match_operand 2 "aarch64_valid_symref")))]
6608 machine_mode mode = GET_MODE (operands[0]);
6610 emit_insn ((mode == DImode
6612 : gen_add_losym_si) (operands[0],
6618 (define_insn "add_losym_<mode>"
6619 [(set (match_operand:P 0 "register_operand" "=r")
6620 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6621 (match_operand 2 "aarch64_valid_symref" "S")))]
6623 "add\\t%<w>0, %<w>1, :lo12:%c2"
6624 [(set_attr "type" "alu_imm")]
6627 (define_insn "ldr_got_small_<mode>"
6628 [(set (match_operand:PTR 0 "register_operand" "=r")
6629 (unspec:PTR [(mem:PTR (lo_sum:PTR
6630 (match_operand:PTR 1 "register_operand" "r")
6631 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6632 UNSPEC_GOTSMALLPIC))]
6634 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6635 [(set_attr "type" "load_<ldst_sz>")]
6638 (define_insn "ldr_got_small_sidi"
6639 [(set (match_operand:DI 0 "register_operand" "=r")
6641 (unspec:SI [(mem:SI (lo_sum:DI
6642 (match_operand:DI 1 "register_operand" "r")
6643 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6644 UNSPEC_GOTSMALLPIC)))]
6646 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6647 [(set_attr "type" "load_4")]
6650 (define_insn "ldr_got_small_28k_<mode>"
6651 [(set (match_operand:PTR 0 "register_operand" "=r")
6652 (unspec:PTR [(mem:PTR (lo_sum:PTR
6653 (match_operand:PTR 1 "register_operand" "r")
6654 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6655 UNSPEC_GOTSMALLPIC28K))]
6657 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6658 [(set_attr "type" "load_<ldst_sz>")]
6661 (define_insn "ldr_got_small_28k_sidi"
6662 [(set (match_operand:DI 0 "register_operand" "=r")
6664 (unspec:SI [(mem:SI (lo_sum:DI
6665 (match_operand:DI 1 "register_operand" "r")
6666 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6667 UNSPEC_GOTSMALLPIC28K)))]
6669 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6670 [(set_attr "type" "load_4")]
6673 (define_insn "ldr_got_tiny"
6674 [(set (match_operand:DI 0 "register_operand" "=r")
6675 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6676 UNSPEC_GOTTINYPIC))]
6679 [(set_attr "type" "load_8")]
6682 (define_insn "aarch64_load_tp_hard"
6683 [(set (match_operand:DI 0 "register_operand" "=r")
6684 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6686 "mrs\\t%0, tpidr_el0"
6687 [(set_attr "type" "mrs")]
6690 ;; The TLS ABI specifically requires that the compiler does not schedule
6691 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6692 ;; Therefore we treat the stubs as an atomic sequence.
6693 (define_expand "tlsgd_small_<mode>"
6694 [(parallel [(set (match_operand 0 "register_operand")
6695 (call (mem:DI (match_dup 2)) (const_int 1)))
6696 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6697 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6698 (clobber (reg:DI LR_REGNUM))])]
6701 operands[2] = aarch64_tls_get_addr ();
6704 (define_insn "*tlsgd_small_<mode>"
6705 [(set (match_operand 0 "register_operand" "")
6706 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6707 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI)
6708 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6709 (clobber (reg:DI LR_REGNUM))
6712 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6713 [(set_attr "type" "call")
6714 (set_attr "length" "16")])
6716 (define_insn "tlsie_small_<mode>"
6717 [(set (match_operand:PTR 0 "register_operand" "=r")
6718 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6719 UNSPEC_GOTSMALLTLS))]
6721 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6722 [(set_attr "type" "load_4")
6723 (set_attr "length" "8")]
6726 (define_insn "tlsie_small_sidi"
6727 [(set (match_operand:DI 0 "register_operand" "=r")
6729 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6730 UNSPEC_GOTSMALLTLS)))]
6732 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6733 [(set_attr "type" "load_4")
6734 (set_attr "length" "8")]
6737 (define_insn "tlsie_tiny_<mode>"
6738 [(set (match_operand:PTR 0 "register_operand" "=&r")
6739 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6740 (match_operand:PTR 2 "register_operand" "r")]
6741 UNSPEC_GOTTINYTLS))]
6743 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6744 [(set_attr "type" "multiple")
6745 (set_attr "length" "8")]
6748 (define_insn "tlsie_tiny_sidi"
6749 [(set (match_operand:DI 0 "register_operand" "=&r")
6751 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6752 (match_operand:DI 2 "register_operand" "r")
6754 UNSPEC_GOTTINYTLS)))]
6756 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6757 [(set_attr "type" "multiple")
6758 (set_attr "length" "8")]
6761 (define_insn "tlsle12_<mode>"
6762 [(set (match_operand:P 0 "register_operand" "=r")
6763 (unspec:P [(match_operand:P 1 "register_operand" "r")
6764 (match_operand 2 "aarch64_tls_le_symref" "S")]
6767 "add\\t%<w>0, %<w>1, #%L2";
6768 [(set_attr "type" "alu_sreg")
6769 (set_attr "length" "4")]
6772 (define_insn "tlsle24_<mode>"
6773 [(set (match_operand:P 0 "register_operand" "=r")
6774 (unspec:P [(match_operand:P 1 "register_operand" "r")
6775 (match_operand 2 "aarch64_tls_le_symref" "S")]
6778 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6779 [(set_attr "type" "multiple")
6780 (set_attr "length" "8")]
6783 (define_insn "tlsle32_<mode>"
6784 [(set (match_operand:P 0 "register_operand" "=r")
6785 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6788 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6789 [(set_attr "type" "multiple")
6790 (set_attr "length" "8")]
6793 (define_insn "tlsle48_<mode>"
6794 [(set (match_operand:P 0 "register_operand" "=r")
6795 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6798 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6799 [(set_attr "type" "multiple")
6800 (set_attr "length" "12")]
6803 (define_expand "tlsdesc_small_<mode>"
6804 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6809 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode);
6811 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi));
6812 RTL_CONST_CALL_P (call) = 1;
6815 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6820 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6822 (define_insn "tlsdesc_small_advsimd_<mode>"
6823 [(set (reg:PTR R0_REGNUM)
6824 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6826 (clobber (reg:DI LR_REGNUM))
6827 (clobber (reg:CC CC_REGNUM))
6828 (clobber (match_scratch:DI 1 "=r"))
6829 (use (reg:DI FP_REGNUM))]
6830 "TARGET_TLS_DESC && !TARGET_SVE"
6831 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6832 [(set_attr "type" "call")
6833 (set_attr "length" "16")])
6835 ;; For SVE, model tlsdesc calls as normal calls, with the callee ABI
6836 ;; describing the extra call-preserved guarantees. This would work
6837 ;; for non-SVE too, but avoiding a call is probably better if we can.
6838 (define_insn "tlsdesc_small_sve_<mode>"
6839 [(set (reg:PTR R0_REGNUM)
6840 (call (mem:DI (unspec:PTR
6841 [(match_operand 0 "aarch64_valid_symref")]
6844 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI)
6845 (clobber (reg:DI LR_REGNUM))
6846 (clobber (match_scratch:DI 2 "=r"))]
6847 "TARGET_TLS_DESC && TARGET_SVE"
6848 "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2"
6849 [(set_attr "type" "call")
6850 (set_attr "length" "16")])
6852 (define_insn "stack_tie"
6853 [(set (mem:BLK (scratch))
6854 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6855 (match_operand:DI 1 "register_operand" "rk")]
6859 [(set_attr "length" "0")]
6862 (define_insn "aarch64_fjcvtzs"
6863 [(set (match_operand:SI 0 "register_operand" "=r")
6864 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
6867 "fjcvtzs\\t%w0, %d1"
6868 [(set_attr "type" "f_cvtf2i")]
6871 ;; Pointer authentication patterns are always provided. In architecture
6872 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6873 ;; This lets the user write portable software which authenticates pointers
6874 ;; when run on something which implements ARMv8.3-A, and which runs
6875 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6878 ;; Signing/Authenticating R30 using SP as the salt.
6880 (define_insn "<pauth_mnem_prefix>sp"
6881 [(set (reg:DI R30_REGNUM)
6882 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6884 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6887 ;; Signing/Authenticating X17 using X16 as the salt.
6889 (define_insn "<pauth_mnem_prefix>1716"
6890 [(set (reg:DI R17_REGNUM)
6891 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6893 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6896 ;; Stripping the signature in R30.
6898 (define_insn "xpaclri"
6899 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6901 "hint\t7 // xpaclri"
6904 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6905 ;; all of memory. This blocks insns from being moved across this point.
6907 (define_insn "blockage"
6908 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6911 [(set_attr "length" "0")
6912 (set_attr "type" "block")]
6915 (define_insn "probe_stack_range"
6916 [(set (match_operand:DI 0 "register_operand" "=rk")
6917 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6918 (match_operand:DI 2 "register_operand" "r")]
6919 UNSPECV_PROBE_STACK_RANGE))]
6922 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6924 [(set_attr "length" "32")]
6927 ;; This instruction is used to generate the stack clash stack adjustment and
6928 ;; probing loop. We can't change the control flow during prologue and epilogue
6929 ;; code generation. So we must emit a volatile unspec and expand it later on.
6931 (define_insn "@probe_sve_stack_clash_<mode>"
6932 [(set (match_operand:P 0 "register_operand" "=rk")
6933 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6934 (match_operand:P 2 "register_operand" "r")
6935 (match_operand:P 3 "const_int_operand" "n")
6936 (match_operand:P 4 "aarch64_plus_immediate" "L")]
6937 UNSPECV_PROBE_STACK_RANGE))]
6940 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6941 operands[3], operands[4]);
6943 [(set_attr "length" "28")]
6946 ;; Named pattern for expanding thread pointer reference.
6947 (define_expand "get_thread_pointerdi"
6948 [(match_operand:DI 0 "register_operand")]
6951 rtx tmp = aarch64_load_tp (operands[0]);
6952 if (tmp != operands[0])
6953 emit_move_insn (operands[0], tmp);
6957 ;; Named patterns for stack smashing protection.
6958 (define_expand "stack_protect_set"
6959 [(match_operand 0 "memory_operand")
6960 (match_operand 1 "memory_operand")]
6963 machine_mode mode = GET_MODE (operands[0]);
6964 if (aarch64_stack_protector_guard != SSP_GLOBAL)
6966 /* Generate access through the system register. */
6967 rtx tmp_reg = gen_reg_rtx (mode);
6970 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
6971 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
6972 GEN_INT (aarch64_stack_protector_guard_offset)));
6976 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
6977 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
6978 GEN_INT (aarch64_stack_protector_guard_offset)));
6981 operands[1] = gen_rtx_MEM (mode, tmp_reg);
6984 emit_insn ((mode == DImode
6985 ? gen_stack_protect_set_di
6986 : gen_stack_protect_set_si) (operands[0], operands[1]));
6990 (define_insn "reg_stack_protect_address_<mode>"
6991 [(set (match_operand:PTR 0 "register_operand" "=r")
6992 (unspec:PTR [(const_int 0)]
6993 UNSPEC_SSP_SYSREG))]
6994 "aarch64_stack_protector_guard != SSP_GLOBAL"
6997 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
6998 aarch64_stack_protector_guard_reg_str);
6999 output_asm_insn (buf, operands);
7002 [(set_attr "type" "mrs")])
7004 ;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7005 ;; canary value does not live beyond the life of this sequence.
7006 (define_insn "stack_protect_set_<mode>"
7007 [(set (match_operand:PTR 0 "memory_operand" "=m")
7008 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7010 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7012 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7013 [(set_attr "length" "12")
7014 (set_attr "type" "multiple")])
7016 (define_expand "stack_protect_test"
7017 [(match_operand 0 "memory_operand")
7018 (match_operand 1 "memory_operand")
7023 machine_mode mode = GET_MODE (operands[0]);
7025 result = gen_reg_rtx(mode);
7026 if (aarch64_stack_protector_guard != SSP_GLOBAL)
7028 /* Generate access through the system register. The
7029 sequence we want here is the access
7030 of the stack offset to come with
7031 mrs scratch_reg, <system_register>
7032 add scratch_reg, scratch_reg, :lo12:offset. */
7033 rtx tmp_reg = gen_reg_rtx (mode);
7036 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7037 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7038 GEN_INT (aarch64_stack_protector_guard_offset)));
7042 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7043 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7044 GEN_INT (aarch64_stack_protector_guard_offset)));
7047 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7049 emit_insn ((mode == DImode
7050 ? gen_stack_protect_test_di
7051 : gen_stack_protect_test_si) (result,
7056 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7057 result, const0_rtx, operands[2]));
7059 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7060 result, const0_rtx, operands[2]));
7064 (define_insn "stack_protect_test_<mode>"
7065 [(set (match_operand:PTR 0 "register_operand" "=r")
7066 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7067 (match_operand:PTR 2 "memory_operand" "m")]
7069 (clobber (match_scratch:PTR 3 "=&r"))]
7071 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7072 [(set_attr "length" "12")
7073 (set_attr "type" "multiple")])
7075 ;; Write Floating-point Control Register.
7076 (define_insn "set_fpcr"
7077 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7080 [(set_attr "type" "mrs")])
7082 ;; Read Floating-point Control Register.
7083 (define_insn "get_fpcr"
7084 [(set (match_operand:SI 0 "register_operand" "=r")
7085 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7088 [(set_attr "type" "mrs")])
7090 ;; Write Floating-point Status Register.
7091 (define_insn "set_fpsr"
7092 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7095 [(set_attr "type" "mrs")])
7097 ;; Read Floating-point Status Register.
7098 (define_insn "get_fpsr"
7099 [(set (match_operand:SI 0 "register_operand" "=r")
7100 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7103 [(set_attr "type" "mrs")])
7106 ;; Define the subtract-one-and-jump insns so loop.c
7107 ;; knows what to generate.
7108 (define_expand "doloop_end"
7109 [(use (match_operand 0 "" "")) ; loop pseudo
7110 (use (match_operand 1 "" ""))] ; label
7111 "optimize > 0 && flag_modulo_sched"
7120 /* Currently SMS relies on the do-loop pattern to recognize loops
7121 where (1) the control part consists of all insns defining and/or
7122 using a certain 'count' register and (2) the loop count can be
7123 adjusted by modifying this register prior to the loop.
7124 ??? The possible introduction of a new block to initialize the
7125 new IV can potentially affect branch optimizations. */
7127 if (GET_MODE (operands[0]) != DImode)
7131 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7133 cmp = XVECEXP (PATTERN (insn), 0, 0);
7134 cc_reg = SET_DEST (cmp);
7135 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7136 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7137 emit_jump_insn (gen_rtx_SET (pc_rtx,
7138 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7143 ;; Track speculation through conditional branches. We assume that
7144 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7145 (define_insn "speculation_tracker"
7146 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7147 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7148 UNSPEC_SPECULATION_TRACKER))]
7151 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7152 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7155 [(set_attr "type" "csel")]
7158 ;; BTI <target> instructions
7159 (define_insn "bti_noarg"
7160 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7163 [(set_attr "type" "no_insn")]
7166 (define_insn "bti_c"
7167 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7170 [(set_attr "type" "no_insn")]
7173 (define_insn "bti_j"
7174 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7177 [(set_attr "type" "no_insn")]
7180 (define_insn "bti_jc"
7181 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7183 "hint\t38 // bti jc"
7184 [(set_attr "type" "no_insn")]
7187 ;; Hard speculation barrier.
7188 (define_insn "speculation_barrier"
7189 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7192 [(set_attr "length" "8")
7193 (set_attr "type" "block")
7194 (set_attr "speculation_barrier" "true")]
7197 ;; Support for __builtin_speculation_safe_value when we have speculation
7198 ;; tracking enabled. Use the speculation tracker to decide whether to
7199 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7200 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7201 [(set (match_operand:ALLI_TI 0 "register_operand")
7202 (unspec_volatile:ALLI_TI
7203 [(match_operand:ALLI_TI 1 "register_operand")
7204 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7205 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7206 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7210 if (operands[2] == const0_rtx)
7213 if (<MODE>mode == TImode)
7214 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7216 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7218 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7226 ;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7227 ;; encoding for CSDB, but will work in older versions of the assembler.
7228 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7229 [(set (match_operand:ALLI 0 "register_operand" "=r")
7230 (unspec_volatile:ALLI
7231 [(match_operand:ALLI 1 "register_operand" "r")
7232 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7233 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7234 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7237 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7238 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7242 [(set_attr "length" "12")
7243 (set_attr "type" "block")
7244 (set_attr "speculation_barrier" "true")]
7247 ;; Pattern to match despeculate_copyti
7248 (define_insn "*despeculate_copyti_insn"
7249 [(set (match_operand:TI 0 "register_operand" "=r")
7251 [(match_operand:TI 1 "register_operand" "r")
7252 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7253 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7254 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7257 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7259 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7263 [(set_attr "length" "16")
7264 (set_attr "type" "block")
7265 (set_attr "speculation_barrier" "true")]
7268 (define_insn "despeculate_simple<ALLI:mode>"
7269 [(set (match_operand:ALLI 0 "register_operand" "=r")
7270 (unspec_volatile:ALLI
7271 [(match_operand:ALLI 1 "register_operand" "r")
7272 (use (match_operand:ALLI 2 "register_operand" ""))]
7273 UNSPECV_SPECULATION_BARRIER))]
7275 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7276 [(set_attr "type" "block")
7277 (set_attr "length" "8")
7278 (set_attr "speculation_barrier" "true")]
7281 (define_insn "despeculate_simpleti"
7282 [(set (match_operand:TI 0 "register_operand" "=r")
7284 [(match_operand:TI 1 "register_operand" "r")
7285 (use (match_operand:DI 2 "register_operand" ""))]
7286 UNSPECV_SPECULATION_BARRIER))]
7288 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7289 [(set_attr "type" "block")
7290 (set_attr "length" "12")
7291 (set_attr "speculation_barrier" "true")]
7294 (define_insn "aarch64_<frintnzs_op><mode>"
7295 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7296 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7298 "TARGET_FRINT && TARGET_FLOAT
7299 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7300 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7301 [(set_attr "type" "f_rint<stype>")]
7304 ;; Transactional Memory Extension (TME) instructions.
7306 (define_insn "tstart"
7307 [(set (match_operand:DI 0 "register_operand" "=r")
7308 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7309 (clobber (mem:BLK (scratch)))]
7312 [(set_attr "type" "tme")]
7315 (define_insn "ttest"
7316 [(set (match_operand:DI 0 "register_operand" "=r")
7317 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7318 (clobber (mem:BLK (scratch)))]
7321 [(set_attr "type" "tme")]
7324 (define_insn "tcommit"
7325 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7326 (clobber (mem:BLK (scratch)))]
7329 [(set_attr "type" "tme")]
7332 (define_insn "tcancel"
7333 [(unspec_volatile:BLK
7334 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7335 (clobber (mem:BLK (scratch)))]
7336 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7338 [(set_attr "type" "tme")]
7342 (include "aarch64-simd.md")
7344 ;; Atomic Operations
7345 (include "atomics.md")
7347 ;; ldp/stp peephole patterns
7348 (include "aarch64-ldpstp.md")
7351 (include "aarch64-sve.md")
7354 (include "aarch64-sve2.md")