1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
88 (LAST_SAVED_REGNUM 63)
92 ;; Defined only to make the DWARF description simpler.
110 ;; Scratch register used by stack clash protection to calculate
111 ;; SVE CFA offsets during probing.
112 (STACK_CLASH_SVE_CFA_REGNUM 11)
113 ;; Scratch registers for prologue/epilogue use.
116 ;; A couple of call-clobbered registers that we need to reserve when
117 ;; tracking speculation this is not ABI, so is subject to change.
118 (SPECULATION_SCRATCH_REGNUM 14)
119 (SPECULATION_TRACKER_REGNUM 15)
120 ;; Scratch registers used in frame layout.
128 (define_c_enum "unspec" [
161 UNSPEC_GOTSMALLPIC28K
243 UNSPEC_SPECULATION_TRACKER
245 UNSPEC_TTEST ; Represent transaction test.
248 (define_c_enum "unspecv" [
249 UNSPECV_EH_RETURN ; Represent EH_RETURN
250 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
251 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
252 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
253 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
254 UNSPECV_BLOCKAGE ; Represent a blockage
255 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
256 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
257 UNSPECV_BTI_NOARG ; Represent BTI.
258 UNSPECV_BTI_C ; Represent BTI c.
259 UNSPECV_BTI_J ; Represent BTI j.
260 UNSPECV_BTI_JC ; Represent BTI jc.
261 UNSPECV_TSTART ; Represent transaction start.
262 UNSPECV_TCOMMIT ; Represent transaction commit.
263 UNSPECV_TCANCEL ; Represent transaction cancel.
267 ;; These constants are used as a const_int in various SVE unspecs
268 ;; to indicate whether the governing predicate is known to be a PTRUE.
270 [; Indicates that the predicate might not be a PTRUE.
271 (SVE_MAYBE_NOT_PTRUE 0)
273 ; Indicates that the predicate is known to be a PTRUE.
274 (SVE_KNOWN_PTRUE 1)])
276 ;; These constants are used as a const_int in predicated SVE FP arithmetic
277 ;; to indicate whether the operation is allowed to make additional lanes
278 ;; active without worrying about the effect on faulting behavior.
280 [; Indicates either that all lanes are active or that the instruction may
281 ; operate on inactive inputs even if doing so could induce a fault.
284 ; Indicates that some lanes might be inactive and that the instruction
285 ; must not operate on inactive inputs if doing so could induce a fault.
288 ;; If further include files are added the defintion of MD_INCLUDES
291 (include "constraints.md")
292 (include "predicates.md")
293 (include "iterators.md")
295 ;; -------------------------------------------------------------------
296 ;; Instruction types and attributes
297 ;; -------------------------------------------------------------------
299 ; The "type" attribute is included here from AArch32 backend to be able
300 ; to share pipeline descriptions.
301 (include "../arm/types.md")
303 ;; It is important to set the fp or simd attributes to yes when a pattern
304 ;; alternative uses the FP or SIMD register files, usually signified by use of
305 ;; the 'w' constraint. This will ensure that the alternative will be
306 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
307 ;; architecture extensions. If all the alternatives in a pattern use the
308 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
311 ;; Attributes of the architecture required to support the instruction (or
312 ;; alternative). This attribute is used to compute attribute "enabled", use type
313 ;; "any" to enable an alternative in all cases.
315 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
317 (define_enum_attr "arch" "arches" (const_string "any"))
319 ;; [For compatibility with Arm in pipeline models]
320 ;; Attribute that specifies whether or not the instruction touches fp
322 ;; Note that this attribute is not used anywhere in either the arm or aarch64
323 ;; backends except in the scheduling description for xgene1. In that
324 ;; scheduling description this attribute is used to subclass the load_4 and
326 (define_attr "fp" "no,yes"
328 (eq_attr "arch" "fp")
330 (const_string "no")))
332 (define_attr "arch_enabled" "no,yes"
335 (eq_attr "arch" "any")
337 (and (eq_attr "arch" "rcpc8_4")
338 (match_test "AARCH64_ISA_RCPC8_4"))
340 (and (eq_attr "arch" "fp")
341 (match_test "TARGET_FLOAT"))
343 (and (eq_attr "arch" "simd")
344 (match_test "TARGET_SIMD"))
346 (and (eq_attr "arch" "fp16")
347 (match_test "TARGET_FP_F16INST"))
349 (and (eq_attr "arch" "sve")
350 (match_test "TARGET_SVE")))
352 (const_string "no")))
354 ;; Attribute that controls whether an alternative is enabled or not.
355 ;; Currently it is only used to disable alternatives which touch fp or simd
356 ;; registers when -mgeneral-regs-only is specified or to require a special
357 ;; architecture support.
358 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
360 ;; Attribute that specifies whether we are dealing with a branch to a
361 ;; label that is far away, i.e. further away than the maximum/minimum
362 ;; representable in a signed 21-bits number.
365 (define_attr "far_branch" "" (const_int 0))
367 ;; Attribute that specifies whether the alternative uses MOVPRFX.
368 (define_attr "movprfx" "no,yes" (const_string "no"))
370 (define_attr "length" ""
371 (cond [(eq_attr "movprfx" "yes")
375 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
376 ;; no predicated insns.
377 (define_attr "predicated" "yes,no" (const_string "no"))
379 ;; Set to true on an insn that requires the speculation tracking state to be
380 ;; in the tracking register before the insn issues. Otherwise the compiler
381 ;; may chose to hold the tracking state encoded in SP.
382 (define_attr "speculation_barrier" "true,false" (const_string "false"))
384 ;; -------------------------------------------------------------------
385 ;; Pipeline descriptions and scheduling
386 ;; -------------------------------------------------------------------
389 (include "aarch64-tune.md")
392 (include "../arm/cortex-a53.md")
393 (include "../arm/cortex-a57.md")
394 (include "../arm/exynos-m1.md")
395 (include "falkor.md")
396 (include "saphira.md")
397 (include "thunderx.md")
398 (include "../arm/xgene1.md")
399 (include "thunderx2t99.md")
400 (include "tsv110.md")
402 ;; -------------------------------------------------------------------
403 ;; Jumps and other miscellaneous insns
404 ;; -------------------------------------------------------------------
406 (define_insn "indirect_jump"
407 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
410 [(set_attr "type" "branch")]
414 [(set (pc) (label_ref (match_operand 0 "" "")))]
417 [(set_attr "type" "branch")]
420 (define_expand "cbranch<mode>4"
421 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
422 [(match_operand:GPI 1 "register_operand")
423 (match_operand:GPI 2 "aarch64_plus_operand")])
424 (label_ref (match_operand 3 "" ""))
428 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
430 operands[2] = const0_rtx;
434 (define_expand "cbranch<mode>4"
435 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
436 [(match_operand:GPF 1 "register_operand")
437 (match_operand:GPF 2 "aarch64_fp_compare_operand")])
438 (label_ref (match_operand 3 "" ""))
442 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
444 operands[2] = const0_rtx;
448 (define_expand "cbranchcc4"
449 [(set (pc) (if_then_else
450 (match_operator 0 "aarch64_comparison_operator"
451 [(match_operand 1 "cc_register")
452 (match_operand 2 "const0_operand")])
453 (label_ref (match_operand 3 "" ""))
458 (define_insn "ccmp<mode>"
459 [(set (match_operand:CC 1 "cc_register" "")
461 (match_operator 4 "aarch64_comparison_operator"
462 [(match_operand 0 "cc_register" "")
465 (match_operand:GPI 2 "register_operand" "r,r,r")
466 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
467 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
470 ccmp\\t%<w>2, %<w>3, %k5, %m4
471 ccmp\\t%<w>2, %3, %k5, %m4
472 ccmn\\t%<w>2, #%n3, %k5, %m4"
473 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
476 (define_insn "fccmp<mode>"
477 [(set (match_operand:CCFP 1 "cc_register" "")
479 (match_operator 4 "aarch64_comparison_operator"
480 [(match_operand 0 "cc_register" "")
483 (match_operand:GPF 2 "register_operand" "w")
484 (match_operand:GPF 3 "register_operand" "w"))
485 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
487 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
488 [(set_attr "type" "fccmp<s>")]
491 (define_insn "fccmpe<mode>"
492 [(set (match_operand:CCFPE 1 "cc_register" "")
494 (match_operator 4 "aarch64_comparison_operator"
495 [(match_operand 0 "cc_register" "")
498 (match_operand:GPF 2 "register_operand" "w")
499 (match_operand:GPF 3 "register_operand" "w"))
500 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
502 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
503 [(set_attr "type" "fccmp<s>")]
506 ;; Expansion of signed mod by a power of 2 using CSNEG.
507 ;; For x0 % n where n is a power of 2 produce:
509 ;; and x0, x0, #(n - 1)
510 ;; and x1, x1, #(n - 1)
511 ;; csneg x0, x0, x1, mi
513 (define_expand "mod<mode>3"
514 [(match_operand:GPI 0 "register_operand")
515 (match_operand:GPI 1 "register_operand")
516 (match_operand:GPI 2 "const_int_operand")]
519 HOST_WIDE_INT val = INTVAL (operands[2]);
522 || exact_log2 (val) <= 0
523 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
526 rtx mask = GEN_INT (val - 1);
528 /* In the special case of x0 % 2 we can do the even shorter:
534 rtx masked = gen_reg_rtx (<MODE>mode);
535 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
536 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
537 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
538 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
542 rtx neg_op = gen_reg_rtx (<MODE>mode);
543 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
545 /* Extract the condition register and mode. */
546 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
547 rtx cc_reg = SET_DEST (cmp);
548 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
550 rtx masked_pos = gen_reg_rtx (<MODE>mode);
551 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
553 rtx masked_neg = gen_reg_rtx (<MODE>mode);
554 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
556 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
557 masked_neg, masked_pos));
562 (define_insn "condjump"
563 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
564 [(match_operand 1 "cc_register" "") (const_int 0)])
565 (label_ref (match_operand 2 "" ""))
569 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
570 but the "." is required for SVE conditions. */
571 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
572 if (get_attr_length (insn) == 8)
573 return aarch64_gen_far_branch (operands, 2, "Lbcond",
574 use_dot_p ? "b.%M0\\t" : "b%M0\\t");
576 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
578 [(set_attr "type" "branch")
580 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
581 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
584 (set (attr "far_branch")
585 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
586 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
591 ;; For a 24-bit immediate CST we can optimize the compare for equality
592 ;; and branch sequence from:
594 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
598 ;; sub x0, x1, #(CST & 0xfff000)
599 ;; subs x0, x0, #(CST & 0x000fff)
601 (define_insn_and_split "*compare_condjump<GPI:mode>"
602 [(set (pc) (if_then_else (EQL
603 (match_operand:GPI 0 "register_operand" "r")
604 (match_operand:GPI 1 "aarch64_imm24" "n"))
605 (label_ref:P (match_operand 2 "" ""))
607 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
608 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
609 && !reload_completed"
614 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
615 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
616 rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
617 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
618 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
619 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
620 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
622 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
627 (define_expand "casesi"
628 [(match_operand:SI 0 "register_operand") ; Index
629 (match_operand:SI 1 "const_int_operand") ; Lower bound
630 (match_operand:SI 2 "const_int_operand") ; Total range
631 (match_operand:DI 3 "" "") ; Table label
632 (match_operand:DI 4 "" "")] ; Out of range label
635 if (operands[1] != const0_rtx)
637 rtx reg = gen_reg_rtx (SImode);
639 /* Canonical RTL says that if you have:
643 then this should be emitted as:
647 The use of trunc_int_for_mode ensures that the resulting
648 constant can be represented in SImode, this is important
649 for the corner case where operand[1] is INT_MIN. */
651 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
653 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
654 (operands[1], SImode))
655 operands[1] = force_reg (SImode, operands[1]);
656 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
660 if (!aarch64_plus_operand (operands[2], SImode))
661 operands[2] = force_reg (SImode, operands[2]);
662 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
664 operands[0], operands[2], operands[4]));
666 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
668 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
670 operands[2] = gen_rtx_MEM (DImode, operands[2]);
671 MEM_READONLY_P (operands[2]) = 1;
672 MEM_NOTRAP_P (operands[2]) = 1;
673 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
678 (define_expand "casesi_dispatch"
680 [(set (pc) (match_operand:DI 0 ""))
681 (clobber (reg:CC CC_REGNUM))
682 (clobber (match_scratch:DI 2))
683 (clobber (match_scratch:DI 3))
684 (use (label_ref:DI (match_operand 1 "")))])]
687 (define_insn "*casesi_dispatch"
690 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
691 (match_operand:SI 1 "register_operand" "r")]
693 (clobber (reg:CC CC_REGNUM))
694 (clobber (match_scratch:DI 3 "=r"))
695 (clobber (match_scratch:DI 4 "=r"))
696 (use (label_ref:DI (match_operand 2 "" "")))])]
699 return aarch64_output_casesi (operands);
701 [(set_attr "length" "16")
702 (set_attr "type" "branch")]
706 [(unspec[(const_int 0)] UNSPEC_NOP)]
709 [(set_attr "type" "no_insn")]
712 (define_insn "prefetch"
713 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
714 (match_operand:QI 1 "const_int_operand" "")
715 (match_operand:QI 2 "const_int_operand" ""))]
718 const char * pftype[2][4] =
720 {"prfm\\tPLDL1STRM, %0",
721 "prfm\\tPLDL3KEEP, %0",
722 "prfm\\tPLDL2KEEP, %0",
723 "prfm\\tPLDL1KEEP, %0"},
724 {"prfm\\tPSTL1STRM, %0",
725 "prfm\\tPSTL3KEEP, %0",
726 "prfm\\tPSTL2KEEP, %0",
727 "prfm\\tPSTL1KEEP, %0"},
730 int locality = INTVAL (operands[2]);
732 gcc_assert (IN_RANGE (locality, 0, 3));
734 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
735 the address into a DImode MEM so that aarch64_print_operand knows
737 operands[0] = gen_rtx_MEM (DImode, operands[0]);
738 return pftype[INTVAL(operands[1])][locality];
740 [(set_attr "type" "load_4")]
744 [(trap_if (const_int 1) (const_int 8))]
747 [(set_attr "type" "trap")])
749 (define_expand "prologue"
750 [(clobber (const_int 0))]
753 aarch64_expand_prologue ();
758 (define_expand "epilogue"
759 [(clobber (const_int 0))]
762 aarch64_expand_epilogue (false);
767 (define_expand "sibcall_epilogue"
768 [(clobber (const_int 0))]
771 aarch64_expand_epilogue (true);
776 (define_insn "*do_return"
780 if (aarch64_return_address_signing_enabled ()
782 && !crtl->calls_eh_return)
784 if (aarch64_ra_sign_key == AARCH64_KEY_B)
791 [(set_attr "type" "branch")]
794 (define_expand "return"
796 "aarch64_use_return_insn_p ()"
800 (define_insn "simple_return"
802 "aarch64_use_simple_return_insn_p ()"
804 [(set_attr "type" "branch")]
807 (define_insn "*cb<optab><mode>1"
808 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
810 (label_ref (match_operand 1 "" ""))
812 "!aarch64_track_speculation"
814 if (get_attr_length (insn) == 8)
815 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
817 return "<cbz>\\t%<w>0, %l1";
819 [(set_attr "type" "branch")
821 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
822 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
825 (set (attr "far_branch")
826 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
827 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
832 (define_insn "*tb<optab><mode>1"
833 [(set (pc) (if_then_else
834 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
837 "aarch64_simd_shift_imm_<mode>" "n"))
839 (label_ref (match_operand 2 "" ""))
841 (clobber (reg:CC CC_REGNUM))]
842 "!aarch64_track_speculation"
844 if (get_attr_length (insn) == 8)
846 if (get_attr_far_branch (insn) == 1)
847 return aarch64_gen_far_branch (operands, 2, "Ltb",
848 "<inv_tb>\\t%<w>0, %1, ");
851 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
852 return "tst\t%<w>0, %1\;<bcond>\t%l2";
856 return "<tbz>\t%<w>0, %1, %l2";
858 [(set_attr "type" "branch")
860 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
861 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
864 (set (attr "far_branch")
865 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
866 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
872 (define_insn "*cb<optab><mode>1"
873 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
875 (label_ref (match_operand 1 "" ""))
877 (clobber (reg:CC CC_REGNUM))]
878 "!aarch64_track_speculation"
880 if (get_attr_length (insn) == 8)
882 if (get_attr_far_branch (insn) == 1)
883 return aarch64_gen_far_branch (operands, 1, "Ltb",
884 "<inv_tb>\\t%<w>0, <sizem1>, ");
888 uint64_t val = ((uint64_t) 1)
889 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
890 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
891 output_asm_insn (buf, operands);
892 return "<bcond>\t%l1";
896 return "<tbz>\t%<w>0, <sizem1>, %l1";
898 [(set_attr "type" "branch")
900 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
901 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
904 (set (attr "far_branch")
905 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
906 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
911 ;; -------------------------------------------------------------------
912 ;; Subroutine calls and sibcalls
913 ;; -------------------------------------------------------------------
915 (define_expand "call"
916 [(parallel [(call (match_operand 0 "memory_operand")
917 (match_operand 1 "general_operand"))
918 (use (match_operand 2 "" ""))
919 (clobber (reg:DI LR_REGNUM))])]
923 aarch64_expand_call (NULL_RTX, operands[0], false);
928 (define_insn "*call_insn"
929 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
930 (match_operand 1 "" ""))
931 (clobber (reg:DI LR_REGNUM))]
936 [(set_attr "type" "call, call")]
939 (define_expand "call_value"
940 [(parallel [(set (match_operand 0 "" "")
941 (call (match_operand 1 "memory_operand")
942 (match_operand 2 "general_operand")))
943 (use (match_operand 3 "" ""))
944 (clobber (reg:DI LR_REGNUM))])]
948 aarch64_expand_call (operands[0], operands[1], false);
953 (define_insn "*call_value_insn"
954 [(set (match_operand 0 "" "")
955 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
956 (match_operand 2 "" "")))
957 (clobber (reg:DI LR_REGNUM))]
962 [(set_attr "type" "call, call")]
965 (define_expand "sibcall"
966 [(parallel [(call (match_operand 0 "memory_operand")
967 (match_operand 1 "general_operand"))
969 (use (match_operand 2 "" ""))])]
972 aarch64_expand_call (NULL_RTX, operands[0], true);
977 (define_expand "sibcall_value"
978 [(parallel [(set (match_operand 0 "" "")
979 (call (match_operand 1 "memory_operand")
980 (match_operand 2 "general_operand")))
982 (use (match_operand 3 "" ""))])]
985 aarch64_expand_call (operands[0], operands[1], true);
990 (define_insn "*sibcall_insn"
991 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
992 (match_operand 1 "" ""))
994 "SIBLING_CALL_P (insn)"
998 [(set_attr "type" "branch, branch")]
1001 (define_insn "*sibcall_value_insn"
1002 [(set (match_operand 0 "" "")
1004 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1005 (match_operand 2 "" "")))
1007 "SIBLING_CALL_P (insn)"
1011 [(set_attr "type" "branch, branch")]
1014 ;; Call subroutine returning any type.
1016 (define_expand "untyped_call"
1017 [(parallel [(call (match_operand 0 "")
1019 (match_operand 1 "")
1020 (match_operand 2 "")])]
1025 emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
1027 for (i = 0; i < XVECLEN (operands[2], 0); i++)
1029 rtx set = XVECEXP (operands[2], 0, i);
1030 emit_move_insn (SET_DEST (set), SET_SRC (set));
1033 /* The optimizer does not know that the call sets the function value
1034 registers we stored in the result block. We avoid problems by
1035 claiming that all hard registers are used and clobbered at this
1037 emit_insn (gen_blockage ());
1041 ;; -------------------------------------------------------------------
1043 ;; -------------------------------------------------------------------
1045 (define_expand "mov<mode>"
1046 [(set (match_operand:SHORT 0 "nonimmediate_operand")
1047 (match_operand:SHORT 1 "general_operand"))]
1050 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1051 operands[1] = force_reg (<MODE>mode, operands[1]);
1053 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1055 aarch64_expand_mov_immediate (operands[0], operands[1]);
1061 (define_insn "*mov<mode>_aarch64"
1062 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
1063 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1064 "(register_operand (operands[0], <MODE>mode)
1065 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1067 switch (which_alternative)
1070 return "mov\t%w0, %w1";
1072 return "mov\t%w0, %1";
1074 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1077 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1079 return "ldr<size>\t%w0, %1";
1081 return "ldr\t%<size>0, %1";
1083 return "str<size>\t%w1, %0";
1085 return "str\t%<size>1, %0";
1087 return "umov\t%w0, %1.<v>[0]";
1089 return "dup\t%0.<Vallxd>, %w1";
1091 return "dup\t%<Vetype>0, %1.<v>[0]";
1096 ;; The "mov_imm" type for CNT is just a placeholder.
1097 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1098 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1099 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1102 (define_expand "mov<mode>"
1103 [(set (match_operand:GPI 0 "nonimmediate_operand")
1104 (match_operand:GPI 1 "general_operand"))]
1107 if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1108 && <MODE>mode == DImode
1109 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1112 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1113 operands[1] = force_reg (<MODE>mode, operands[1]);
1115 /* FIXME: RR we still need to fix up what we are doing with
1116 symbol_refs and other types of constants. */
1117 if (CONSTANT_P (operands[1])
1118 && !CONST_INT_P (operands[1]))
1120 aarch64_expand_mov_immediate (operands[0], operands[1]);
1126 (define_insn_and_split "*movsi_aarch64"
1127 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
1128 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1129 "(register_operand (operands[0], SImode)
1130 || aarch64_reg_or_zero (operands[1], SImode))"
1137 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1147 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1148 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1149 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1152 aarch64_expand_mov_immediate (operands[0], operands[1]);
1155 ;; The "mov_imm" type for CNT is just a placeholder.
1156 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1157 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1158 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1161 (define_insn_and_split "*movdi_aarch64"
1162 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1163 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1164 "(register_operand (operands[0], DImode)
1165 || aarch64_reg_or_zero (operands[1], DImode))"
1173 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1183 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1184 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1185 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1188 aarch64_expand_mov_immediate (operands[0], operands[1]);
1191 ;; The "mov_imm" type for CNTD is just a placeholder.
1192 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1193 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1195 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1198 (define_insn "insv_imm<mode>"
1199 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1201 (match_operand:GPI 1 "const_int_operand" "n"))
1202 (match_operand:GPI 2 "const_int_operand" "n"))]
1203 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1204 && UINTVAL (operands[1]) % 16 == 0"
1205 "movk\\t%<w>0, %X2, lsl %1"
1206 [(set_attr "type" "mov_imm")]
1209 (define_expand "movti"
1210 [(set (match_operand:TI 0 "nonimmediate_operand")
1211 (match_operand:TI 1 "general_operand"))]
1214 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1215 operands[1] = force_reg (TImode, operands[1]);
1217 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1219 emit_move_insn (gen_lowpart (DImode, operands[0]),
1220 gen_lowpart (DImode, operands[1]));
1221 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1227 (define_insn "*movti_aarch64"
1228 [(set (match_operand:TI 0
1229 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1231 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1232 "(register_operand (operands[0], TImode)
1233 || aarch64_reg_or_zero (operands[1], TImode))"
1238 mov\\t%0.16b, %1.16b
1244 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1245 load_16,store_16,store_16,\
1247 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1248 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1251 ;; Split a TImode register-register or register-immediate move into
1252 ;; its component DImode pieces, taking care to handle overlapping
1253 ;; source and dest registers.
1255 [(set (match_operand:TI 0 "register_operand" "")
1256 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1257 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1260 aarch64_split_128bit_move (operands[0], operands[1]);
1264 (define_expand "mov<mode>"
1265 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1266 (match_operand:GPF_TF_F16 1 "general_operand"))]
1271 aarch64_err_no_fpadvsimd (<MODE>mode);
1275 if (GET_CODE (operands[0]) == MEM
1276 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1277 && aarch64_float_const_zero_rtx_p (operands[1])))
1278 operands[1] = force_reg (<MODE>mode, operands[1]);
1282 (define_insn "*movhf_aarch64"
1283 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1284 (match_operand:HF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1285 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1286 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1292 mov\\t%0.h[0], %1.h[0]
1294 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1300 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1301 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1302 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1305 (define_insn "*movsf_aarch64"
1306 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1307 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1308 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1309 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1316 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1323 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1324 f_loads,f_stores,load_4,store_4,mov_reg,\
1326 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1329 (define_insn "*movdf_aarch64"
1330 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1331 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1332 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1333 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1340 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1347 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1348 f_loadd,f_stored,load_8,store_8,mov_reg,\
1350 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1354 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1355 (match_operand:GPF_HF 1 "general_operand"))]
1356 "can_create_pseudo_p ()
1357 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1358 && !aarch64_float_const_representable_p (operands[1])
1359 && aarch64_float_const_rtx_p (operands[1])"
1362 unsigned HOST_WIDE_INT ival;
1363 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1366 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1367 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1368 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1373 (define_insn "*movtf_aarch64"
1374 [(set (match_operand:TF 0
1375 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1377 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1378 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1379 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1381 mov\\t%0.16b, %1.16b
1392 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1393 f_loadd,f_stored,load_16,store_16,store_16")
1394 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1395 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1399 [(set (match_operand:TF 0 "register_operand" "")
1400 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1401 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1404 aarch64_split_128bit_move (operands[0], operands[1]);
1411 ;; 2 is size of copy in bytes
1414 (define_expand "cpymemdi"
1415 [(match_operand:BLK 0 "memory_operand")
1416 (match_operand:BLK 1 "memory_operand")
1417 (match_operand:DI 2 "immediate_operand")
1418 (match_operand:DI 3 "immediate_operand")]
1421 if (aarch64_expand_cpymem (operands))
1427 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1428 ;; fairly lax checking on the second memory operation.
1429 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1430 [(set (match_operand:SX 0 "register_operand" "=r,w")
1431 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1432 (set (match_operand:SX2 2 "register_operand" "=r,w")
1433 (match_operand:SX2 3 "memory_operand" "m,m"))]
1434 "rtx_equal_p (XEXP (operands[3], 0),
1435 plus_constant (Pmode,
1436 XEXP (operands[1], 0),
1437 GET_MODE_SIZE (<SX:MODE>mode)))"
1441 [(set_attr "type" "load_8,neon_load1_2reg")
1442 (set_attr "arch" "*,fp")]
1445 ;; Storing different modes that can still be merged
1446 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1447 [(set (match_operand:DX 0 "register_operand" "=r,w")
1448 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1449 (set (match_operand:DX2 2 "register_operand" "=r,w")
1450 (match_operand:DX2 3 "memory_operand" "m,m"))]
1451 "rtx_equal_p (XEXP (operands[3], 0),
1452 plus_constant (Pmode,
1453 XEXP (operands[1], 0),
1454 GET_MODE_SIZE (<DX:MODE>mode)))"
1458 [(set_attr "type" "load_16,neon_load1_2reg")
1459 (set_attr "arch" "*,fp")]
1462 (define_insn "load_pair_dw_tftf"
1463 [(set (match_operand:TF 0 "register_operand" "=w")
1464 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1465 (set (match_operand:TF 2 "register_operand" "=w")
1466 (match_operand:TF 3 "memory_operand" "m"))]
1468 && rtx_equal_p (XEXP (operands[3], 0),
1469 plus_constant (Pmode,
1470 XEXP (operands[1], 0),
1471 GET_MODE_SIZE (TFmode)))"
1472 "ldp\\t%q0, %q2, %1"
1473 [(set_attr "type" "neon_ldp_q")
1474 (set_attr "fp" "yes")]
1477 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1478 ;; fairly lax checking on the second memory operation.
1479 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1480 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1481 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1482 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1483 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1484 "rtx_equal_p (XEXP (operands[2], 0),
1485 plus_constant (Pmode,
1486 XEXP (operands[0], 0),
1487 GET_MODE_SIZE (<SX:MODE>mode)))"
1491 [(set_attr "type" "store_8,neon_store1_2reg")
1492 (set_attr "arch" "*,fp")]
1495 ;; Storing different modes that can still be merged
1496 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1497 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1498 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1499 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1500 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1501 "rtx_equal_p (XEXP (operands[2], 0),
1502 plus_constant (Pmode,
1503 XEXP (operands[0], 0),
1504 GET_MODE_SIZE (<DX:MODE>mode)))"
1508 [(set_attr "type" "store_16,neon_store1_2reg")
1509 (set_attr "arch" "*,fp")]
1512 (define_insn "store_pair_dw_tftf"
1513 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1514 (match_operand:TF 1 "register_operand" "w"))
1515 (set (match_operand:TF 2 "memory_operand" "=m")
1516 (match_operand:TF 3 "register_operand" "w"))]
1518 rtx_equal_p (XEXP (operands[2], 0),
1519 plus_constant (Pmode,
1520 XEXP (operands[0], 0),
1521 GET_MODE_SIZE (TFmode)))"
1522 "stp\\t%q1, %q3, %0"
1523 [(set_attr "type" "neon_stp_q")
1524 (set_attr "fp" "yes")]
1527 ;; Load pair with post-index writeback. This is primarily used in function
1529 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1531 [(set (match_operand:P 0 "register_operand" "=k")
1532 (plus:P (match_operand:P 1 "register_operand" "0")
1533 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1534 (set (match_operand:GPI 2 "register_operand" "=r")
1535 (mem:GPI (match_dup 1)))
1536 (set (match_operand:GPI 3 "register_operand" "=r")
1537 (mem:GPI (plus:P (match_dup 1)
1538 (match_operand:P 5 "const_int_operand" "n"))))])]
1539 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1540 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1541 [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1544 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1546 [(set (match_operand:P 0 "register_operand" "=k")
1547 (plus:P (match_operand:P 1 "register_operand" "0")
1548 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1549 (set (match_operand:GPF 2 "register_operand" "=w")
1550 (mem:GPF (match_dup 1)))
1551 (set (match_operand:GPF 3 "register_operand" "=w")
1552 (mem:GPF (plus:P (match_dup 1)
1553 (match_operand:P 5 "const_int_operand" "n"))))])]
1554 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1555 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1556 [(set_attr "type" "neon_load1_2reg")]
1559 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1561 [(set (match_operand:P 0 "register_operand" "=k")
1562 (plus:P (match_operand:P 1 "register_operand" "0")
1563 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1564 (set (match_operand:TX 2 "register_operand" "=w")
1565 (mem:TX (match_dup 1)))
1566 (set (match_operand:TX 3 "register_operand" "=w")
1567 (mem:TX (plus:P (match_dup 1)
1568 (match_operand:P 5 "const_int_operand" "n"))))])]
1569 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1570 "ldp\\t%q2, %q3, [%1], %4"
1571 [(set_attr "type" "neon_ldp_q")]
1574 ;; Store pair with pre-index writeback. This is primarily used in function
1576 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1578 [(set (match_operand:P 0 "register_operand" "=&k")
1579 (plus:P (match_operand:P 1 "register_operand" "0")
1580 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1581 (set (mem:GPI (plus:P (match_dup 0)
1583 (match_operand:GPI 2 "register_operand" "r"))
1584 (set (mem:GPI (plus:P (match_dup 0)
1585 (match_operand:P 5 "const_int_operand" "n")))
1586 (match_operand:GPI 3 "register_operand" "r"))])]
1587 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1588 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1589 [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1592 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1594 [(set (match_operand:P 0 "register_operand" "=&k")
1595 (plus:P (match_operand:P 1 "register_operand" "0")
1596 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1597 (set (mem:GPF (plus:P (match_dup 0)
1599 (match_operand:GPF 2 "register_operand" "w"))
1600 (set (mem:GPF (plus:P (match_dup 0)
1601 (match_operand:P 5 "const_int_operand" "n")))
1602 (match_operand:GPF 3 "register_operand" "w"))])]
1603 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1604 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1605 [(set_attr "type" "neon_store1_2reg<q>")]
1608 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1610 [(set (match_operand:P 0 "register_operand" "=&k")
1611 (plus:P (match_operand:P 1 "register_operand" "0")
1612 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1613 (set (mem:TX (plus:P (match_dup 0)
1615 (match_operand:TX 2 "register_operand" "w"))
1616 (set (mem:TX (plus:P (match_dup 0)
1617 (match_operand:P 5 "const_int_operand" "n")))
1618 (match_operand:TX 3 "register_operand" "w"))])]
1620 && INTVAL (operands[5])
1621 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1622 "stp\\t%q2, %q3, [%0, %4]!"
1623 [(set_attr "type" "neon_stp_q")]
1626 ;; -------------------------------------------------------------------
1627 ;; Sign/Zero extension
1628 ;; -------------------------------------------------------------------
1630 (define_expand "<optab>sidi2"
1631 [(set (match_operand:DI 0 "register_operand")
1632 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1636 (define_insn "*extendsidi2_aarch64"
1637 [(set (match_operand:DI 0 "register_operand" "=r,r")
1638 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1643 [(set_attr "type" "extend,load_4")]
1646 (define_insn "*load_pair_extendsidi2_aarch64"
1647 [(set (match_operand:DI 0 "register_operand" "=r")
1648 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1649 (set (match_operand:DI 2 "register_operand" "=r")
1650 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1651 "rtx_equal_p (XEXP (operands[3], 0),
1652 plus_constant (Pmode,
1653 XEXP (operands[1], 0),
1654 GET_MODE_SIZE (SImode)))"
1655 "ldpsw\\t%0, %2, %1"
1656 [(set_attr "type" "load_8")]
1659 (define_insn "*zero_extendsidi2_aarch64"
1660 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1661 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1670 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1671 (set_attr "arch" "*,*,fp,fp,fp,fp")]
1674 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1675 [(set (match_operand:DI 0 "register_operand" "=r,w")
1676 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1677 (set (match_operand:DI 2 "register_operand" "=r,w")
1678 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1679 "rtx_equal_p (XEXP (operands[3], 0),
1680 plus_constant (Pmode,
1681 XEXP (operands[1], 0),
1682 GET_MODE_SIZE (SImode)))"
1686 [(set_attr "type" "load_8,neon_load1_2reg")
1687 (set_attr "arch" "*,fp")]
1690 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1691 [(set (match_operand:GPI 0 "register_operand")
1692 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1696 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1697 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1698 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1701 sxt<SHORT:size>\t%<GPI:w>0, %w1
1702 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1703 [(set_attr "type" "extend,load_4")]
1706 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1707 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1708 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1711 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1712 ldr<SHORT:size>\t%w0, %1
1713 ldr\t%<SHORT:size>0, %1"
1714 [(set_attr "type" "logic_imm,load_4,f_loads")
1715 (set_attr "arch" "*,*,fp")]
1718 (define_expand "<optab>qihi2"
1719 [(set (match_operand:HI 0 "register_operand")
1720 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1724 (define_insn "*extendqihi2_aarch64"
1725 [(set (match_operand:HI 0 "register_operand" "=r,r")
1726 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1731 [(set_attr "type" "extend,load_4")]
1734 (define_insn "*zero_extendqihi2_aarch64"
1735 [(set (match_operand:HI 0 "register_operand" "=r,r")
1736 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1741 [(set_attr "type" "logic_imm,load_4")]
1744 ;; -------------------------------------------------------------------
1745 ;; Simple arithmetic
1746 ;; -------------------------------------------------------------------
1748 (define_expand "add<mode>3"
1750 (match_operand:GPI 0 "register_operand")
1751 (plus:GPI (match_operand:GPI 1 "register_operand")
1752 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1755 /* If operands[1] is a subreg extract the inner RTX. */
1756 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1758 /* If the constant is too large for a single instruction and isn't frame
1759 based, split off the immediate so it is available for CSE. */
1760 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1761 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1762 && can_create_pseudo_p ()
1764 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1765 operands[2] = force_reg (<MODE>mode, operands[2]);
1766 /* Expand polynomial additions now if the destination is the stack
1767 pointer, since we don't want to use that as a temporary. */
1768 else if (operands[0] == stack_pointer_rtx
1769 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1771 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1772 operands[2], NULL_RTX, NULL_RTX);
1777 (define_insn "*add<mode>3_aarch64"
1779 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1781 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1782 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1785 add\\t%<w>0, %<w>1, %2
1786 add\\t%<w>0, %<w>1, %<w>2
1787 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1788 sub\\t%<w>0, %<w>1, #%n2
1790 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1791 * return aarch64_output_sve_addvl_addpl (operands[2]);"
1792 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1793 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1794 (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1797 ;; zero_extend version of above
1798 (define_insn "*addsi3_aarch64_uxtw"
1800 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1802 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1803 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1808 sub\\t%w0, %w1, #%n2
1810 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1813 ;; If there's a free register, and we can load the constant with a
1814 ;; single instruction, do so. This has a chance to improve scheduling.
1816 [(match_scratch:GPI 3 "r")
1817 (set (match_operand:GPI 0 "register_operand")
1819 (match_operand:GPI 1 "register_operand")
1820 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1821 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1822 [(set (match_dup 3) (match_dup 2))
1823 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1827 [(match_scratch:SI 3 "r")
1828 (set (match_operand:DI 0 "register_operand")
1831 (match_operand:SI 1 "register_operand")
1832 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1833 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1834 [(set (match_dup 3) (match_dup 2))
1835 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1838 ;; After peephole2 has had a chance to run, split any remaining long
1839 ;; additions into two add immediates.
1841 [(set (match_operand:GPI 0 "register_operand")
1843 (match_operand:GPI 1 "register_operand")
1844 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1845 "epilogue_completed"
1846 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1847 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1849 HOST_WIDE_INT i = INTVAL (operands[2]);
1850 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1851 operands[3] = GEN_INT (i - s);
1852 operands[4] = GEN_INT (s);
1856 ;; Match addition of polynomial offsets that require one temporary, for which
1857 ;; we can use the early-clobbered destination register. This is a separate
1858 ;; pattern so that the early clobber doesn't affect register allocation
1859 ;; for other forms of addition. However, we still need to provide an
1860 ;; all-register alternative, in case the offset goes out of range after
1861 ;; elimination. For completeness we might as well provide all GPR-based
1862 ;; alternatives from the main pattern.
1864 ;; We don't have a pattern for additions requiring two temporaries since at
1865 ;; present LRA doesn't allow new scratches to be added during elimination.
1866 ;; Such offsets should be rare anyway.
1868 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1869 ;; here would go away. We could just handle all polynomial constants in
1871 (define_insn_and_split "*add<mode>3_poly_1"
1873 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
1875 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1876 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
1877 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1879 add\\t%<w>0, %<w>1, %2
1880 add\\t%<w>0, %<w>1, %<w>2
1881 sub\\t%<w>0, %<w>1, #%n2
1883 * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1884 * return aarch64_output_sve_addvl_addpl (operands[2]);
1886 "&& epilogue_completed
1887 && !reg_overlap_mentioned_p (operands[0], operands[1])
1888 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1891 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1892 operands[2], operands[0], NULL_RTX);
1895 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1896 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
1900 [(set (match_operand:DI 0 "register_operand")
1903 (match_operand:SI 1 "register_operand")
1904 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1905 "epilogue_completed"
1906 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1907 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1909 HOST_WIDE_INT i = INTVAL (operands[2]);
1910 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1911 operands[3] = GEN_INT (i - s);
1912 operands[4] = GEN_INT (s);
1913 operands[5] = gen_lowpart (SImode, operands[0]);
1917 (define_expand "addv<mode>4"
1918 [(match_operand:GPI 0 "register_operand")
1919 (match_operand:GPI 1 "register_operand")
1920 (match_operand:GPI 2 "aarch64_plus_operand")
1921 (label_ref (match_operand 3 "" ""))]
1924 if (CONST_INT_P (operands[2]))
1925 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1928 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1929 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1934 (define_expand "uaddv<mode>4"
1935 [(match_operand:GPI 0 "register_operand")
1936 (match_operand:GPI 1 "register_operand")
1937 (match_operand:GPI 2 "register_operand")
1938 (label_ref (match_operand 3 "" ""))]
1941 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1942 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1947 (define_expand "addti3"
1948 [(set (match_operand:TI 0 "register_operand")
1949 (plus:TI (match_operand:TI 1 "register_operand")
1950 (match_operand:TI 2 "aarch64_reg_or_imm")))]
1953 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1955 aarch64_addti_scratch_regs (operands[1], operands[2],
1956 &low_dest, &op1_low, &op2_low,
1957 &high_dest, &op1_high, &op2_high);
1959 if (op2_low == const0_rtx)
1962 if (!aarch64_pluslong_operand (op2_high, DImode))
1963 op2_high = force_reg (DImode, op2_high);
1964 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1968 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1969 force_reg (DImode, op2_low)));
1970 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1971 force_reg (DImode, op2_high)));
1974 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1975 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1980 (define_expand "addvti4"
1981 [(match_operand:TI 0 "register_operand")
1982 (match_operand:TI 1 "register_operand")
1983 (match_operand:TI 2 "aarch64_reg_or_imm")
1984 (label_ref (match_operand 3 "" ""))]
1987 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1989 aarch64_addti_scratch_regs (operands[1], operands[2],
1990 &low_dest, &op1_low, &op2_low,
1991 &high_dest, &op1_high, &op2_high);
1993 if (op2_low == const0_rtx)
1996 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1997 force_reg (DImode, op2_high)));
2001 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2002 force_reg (DImode, op2_low)));
2003 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2004 force_reg (DImode, op2_high)));
2007 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2008 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2010 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2014 (define_expand "uaddvti4"
2015 [(match_operand:TI 0 "register_operand")
2016 (match_operand:TI 1 "register_operand")
2017 (match_operand:TI 2 "aarch64_reg_or_imm")
2018 (label_ref (match_operand 3 "" ""))]
2021 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2023 aarch64_addti_scratch_regs (operands[1], operands[2],
2024 &low_dest, &op1_low, &op2_low,
2025 &high_dest, &op1_high, &op2_high);
2027 if (op2_low == const0_rtx)
2030 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2031 force_reg (DImode, op2_high)));
2035 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2036 force_reg (DImode, op2_low)));
2037 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2038 force_reg (DImode, op2_high)));
2041 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2042 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2044 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2048 (define_insn "add<mode>3_compare0"
2049 [(set (reg:CC_NZ CC_REGNUM)
2051 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2052 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2054 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2055 (plus:GPI (match_dup 1) (match_dup 2)))]
2058 adds\\t%<w>0, %<w>1, %<w>2
2059 adds\\t%<w>0, %<w>1, %2
2060 subs\\t%<w>0, %<w>1, #%n2"
2061 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2064 ;; zero_extend version of above
2065 (define_insn "*addsi3_compare0_uxtw"
2066 [(set (reg:CC_NZ CC_REGNUM)
2068 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2069 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2071 (set (match_operand:DI 0 "register_operand" "=r,r,r")
2072 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2075 adds\\t%w0, %w1, %w2
2077 subs\\t%w0, %w1, #%n2"
2078 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2081 (define_insn "*add<mode>3_compareC_cconly"
2082 [(set (reg:CC_C CC_REGNUM)
2085 (match_operand:GPI 0 "register_operand" "r,r,r")
2086 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2093 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2096 (define_insn "add<mode>3_compareC"
2097 [(set (reg:CC_C CC_REGNUM)
2100 (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2101 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2103 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2104 (plus:GPI (match_dup 1) (match_dup 2)))]
2107 adds\\t%<w>0, %<w>1, %<w>2
2108 adds\\t%<w>0, %<w>1, %2
2109 subs\\t%<w>0, %<w>1, #%n2"
2110 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2113 (define_insn "*add<mode>3_compareV_cconly_imm"
2114 [(set (reg:CC_V CC_REGNUM)
2117 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2118 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2122 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2123 "INTVAL (operands[1]) == INTVAL (operands[2])"
2127 [(set_attr "type" "alus_imm")]
2130 (define_insn "*add<mode>3_compareV_cconly"
2131 [(set (reg:CC_V CC_REGNUM)
2134 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2135 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2136 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2138 "cmn\\t%<w>0, %<w>1"
2139 [(set_attr "type" "alus_sreg")]
2142 (define_insn "add<mode>3_compareV_imm"
2143 [(set (reg:CC_V CC_REGNUM)
2147 (match_operand:GPI 1 "register_operand" "rk,rk"))
2148 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2150 (plus:GPI (match_dup 1) (match_dup 2)))))
2151 (set (match_operand:GPI 0 "register_operand" "=r,r")
2152 (plus:GPI (match_dup 1) (match_dup 2)))]
2155 adds\\t%<w>0, %<w>1, %<w>2
2156 subs\\t%<w>0, %<w>1, #%n2"
2157 [(set_attr "type" "alus_imm,alus_imm")]
2160 (define_insn "add<mode>3_compareV"
2161 [(set (reg:CC_V CC_REGNUM)
2164 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2165 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2166 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2167 (set (match_operand:GPI 0 "register_operand" "=r")
2168 (plus:GPI (match_dup 1) (match_dup 2)))]
2170 "adds\\t%<w>0, %<w>1, %<w>2"
2171 [(set_attr "type" "alus_sreg")]
2174 (define_insn "*adds_shift_imm_<mode>"
2175 [(set (reg:CC_NZ CC_REGNUM)
2177 (plus:GPI (ASHIFT:GPI
2178 (match_operand:GPI 1 "register_operand" "r")
2179 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2180 (match_operand:GPI 3 "register_operand" "r"))
2182 (set (match_operand:GPI 0 "register_operand" "=r")
2183 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2186 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2187 [(set_attr "type" "alus_shift_imm")]
2190 (define_insn "*subs_shift_imm_<mode>"
2191 [(set (reg:CC_NZ CC_REGNUM)
2193 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2195 (match_operand:GPI 2 "register_operand" "r")
2196 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2198 (set (match_operand:GPI 0 "register_operand" "=r")
2199 (minus:GPI (match_dup 1)
2200 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2202 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2203 [(set_attr "type" "alus_shift_imm")]
2206 (define_insn "*adds_mul_imm_<mode>"
2207 [(set (reg:CC_NZ CC_REGNUM)
2210 (match_operand:GPI 1 "register_operand" "r")
2211 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2212 (match_operand:GPI 3 "register_operand" "r"))
2214 (set (match_operand:GPI 0 "register_operand" "=r")
2215 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2218 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2219 [(set_attr "type" "alus_shift_imm")]
2222 (define_insn "*subs_mul_imm_<mode>"
2223 [(set (reg:CC_NZ CC_REGNUM)
2225 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2227 (match_operand:GPI 2 "register_operand" "r")
2228 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2230 (set (match_operand:GPI 0 "register_operand" "=r")
2231 (minus:GPI (match_dup 1)
2232 (mult:GPI (match_dup 2) (match_dup 3))))]
2234 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2235 [(set_attr "type" "alus_shift_imm")]
2238 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2239 [(set (reg:CC_NZ CC_REGNUM)
2242 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2243 (match_operand:GPI 2 "register_operand" "rk"))
2245 (set (match_operand:GPI 0 "register_operand" "=r")
2246 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2248 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2249 [(set_attr "type" "alus_ext")]
2252 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2253 [(set (reg:CC_NZ CC_REGNUM)
2255 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2257 (match_operand:ALLX 2 "register_operand" "r")))
2259 (set (match_operand:GPI 0 "register_operand" "=r")
2260 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2262 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2263 [(set_attr "type" "alus_ext")]
2266 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2267 [(set (reg:CC_NZ CC_REGNUM)
2269 (plus:GPI (ashift:GPI
2271 (match_operand:ALLX 1 "register_operand" "r"))
2272 (match_operand 2 "aarch64_imm3" "Ui3"))
2273 (match_operand:GPI 3 "register_operand" "rk"))
2275 (set (match_operand:GPI 0 "register_operand" "=rk")
2276 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2280 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2281 [(set_attr "type" "alus_ext")]
2284 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2285 [(set (reg:CC_NZ CC_REGNUM)
2287 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2290 (match_operand:ALLX 2 "register_operand" "r"))
2291 (match_operand 3 "aarch64_imm3" "Ui3")))
2293 (set (match_operand:GPI 0 "register_operand" "=rk")
2294 (minus:GPI (match_dup 1)
2295 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2298 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2299 [(set_attr "type" "alus_ext")]
2302 (define_insn "*adds_<optab><mode>_multp2"
2303 [(set (reg:CC_NZ CC_REGNUM)
2305 (plus:GPI (ANY_EXTRACT:GPI
2306 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2307 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2308 (match_operand 3 "const_int_operand" "n")
2310 (match_operand:GPI 4 "register_operand" "rk"))
2312 (set (match_operand:GPI 0 "register_operand" "=r")
2313 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2317 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2318 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2319 [(set_attr "type" "alus_ext")]
2322 (define_insn "*subs_<optab><mode>_multp2"
2323 [(set (reg:CC_NZ CC_REGNUM)
2325 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2327 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2328 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2329 (match_operand 3 "const_int_operand" "n")
2332 (set (match_operand:GPI 0 "register_operand" "=r")
2333 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2334 (mult:GPI (match_dup 1) (match_dup 2))
2337 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2338 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2339 [(set_attr "type" "alus_ext")]
2342 (define_insn "*add<mode>3nr_compare0"
2343 [(set (reg:CC_NZ CC_REGNUM)
2345 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2346 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2353 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2356 (define_insn "aarch64_sub<mode>_compare0"
2357 [(set (reg:CC_NZ CC_REGNUM)
2359 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2360 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2363 "cmp\\t%<w>0, %<w>1"
2364 [(set_attr "type" "alus_sreg")]
2367 (define_insn "*compare_neg<mode>"
2368 [(set (reg:CC_Z CC_REGNUM)
2370 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2371 (match_operand:GPI 1 "register_operand" "r")))]
2373 "cmn\\t%<w>1, %<w>0"
2374 [(set_attr "type" "alus_sreg")]
2377 (define_insn "*add_<shift>_<mode>"
2378 [(set (match_operand:GPI 0 "register_operand" "=r")
2379 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2380 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2381 (match_operand:GPI 3 "register_operand" "r")))]
2383 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2384 [(set_attr "type" "alu_shift_imm")]
2387 ;; zero_extend version of above
2388 (define_insn "*add_<shift>_si_uxtw"
2389 [(set (match_operand:DI 0 "register_operand" "=r")
2391 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2392 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2393 (match_operand:SI 3 "register_operand" "r"))))]
2395 "add\\t%w0, %w3, %w1, <shift> %2"
2396 [(set_attr "type" "alu_shift_imm")]
2399 (define_insn "*add_mul_imm_<mode>"
2400 [(set (match_operand:GPI 0 "register_operand" "=r")
2401 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2402 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2403 (match_operand:GPI 3 "register_operand" "r")))]
2405 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2406 [(set_attr "type" "alu_shift_imm")]
2409 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2410 [(set (match_operand:GPI 0 "register_operand" "=rk")
2411 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2412 (match_operand:GPI 2 "register_operand" "r")))]
2414 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2415 [(set_attr "type" "alu_ext")]
2418 ;; zero_extend version of above
2419 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2420 [(set (match_operand:DI 0 "register_operand" "=rk")
2422 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2423 (match_operand:GPI 2 "register_operand" "r"))))]
2425 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2426 [(set_attr "type" "alu_ext")]
2429 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2430 [(set (match_operand:GPI 0 "register_operand" "=rk")
2431 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2432 (match_operand:ALLX 1 "register_operand" "r"))
2433 (match_operand 2 "aarch64_imm3" "Ui3"))
2434 (match_operand:GPI 3 "register_operand" "r")))]
2436 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2437 [(set_attr "type" "alu_ext")]
2440 ;; zero_extend version of above
2441 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2442 [(set (match_operand:DI 0 "register_operand" "=rk")
2444 (plus:SI (ashift:SI (ANY_EXTEND:SI
2445 (match_operand:SHORT 1 "register_operand" "r"))
2446 (match_operand 2 "aarch64_imm3" "Ui3"))
2447 (match_operand:SI 3 "register_operand" "r"))))]
2449 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2450 [(set_attr "type" "alu_ext")]
2453 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2454 [(set (match_operand:GPI 0 "register_operand" "=rk")
2455 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2456 (match_operand:ALLX 1 "register_operand" "r"))
2457 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2458 (match_operand:GPI 3 "register_operand" "r")))]
2460 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2461 [(set_attr "type" "alu_ext")]
2464 ;; zero_extend version of above
2465 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2466 [(set (match_operand:DI 0 "register_operand" "=rk")
2467 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2468 (match_operand:SHORT 1 "register_operand" "r"))
2469 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2470 (match_operand:SI 3 "register_operand" "r"))))]
2472 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2473 [(set_attr "type" "alu_ext")]
2476 (define_insn "*add_<optab><mode>_multp2"
2477 [(set (match_operand:GPI 0 "register_operand" "=rk")
2478 (plus:GPI (ANY_EXTRACT:GPI
2479 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2480 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2481 (match_operand 3 "const_int_operand" "n")
2483 (match_operand:GPI 4 "register_operand" "r")))]
2484 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2485 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2486 [(set_attr "type" "alu_ext")]
2489 ;; zero_extend version of above
2490 (define_insn "*add_<optab>si_multp2_uxtw"
2491 [(set (match_operand:DI 0 "register_operand" "=rk")
2493 (plus:SI (ANY_EXTRACT:SI
2494 (mult:SI (match_operand:SI 1 "register_operand" "r")
2495 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2496 (match_operand 3 "const_int_operand" "n")
2498 (match_operand:SI 4 "register_operand" "r"))))]
2499 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2500 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2501 [(set_attr "type" "alu_ext")]
2504 (define_expand "add<mode>3_carryin"
2505 [(set (match_operand:GPI 0 "register_operand")
2508 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2509 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2510 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2515 ;; Note that add with carry with two zero inputs is matched by cset,
2516 ;; and that add with carry with one zero input is matched by cinc.
2518 (define_insn "*add<mode>3_carryin"
2519 [(set (match_operand:GPI 0 "register_operand" "=r")
2522 (match_operand:GPI 3 "aarch64_carry_operation" "")
2523 (match_operand:GPI 1 "register_operand" "r"))
2524 (match_operand:GPI 2 "register_operand" "r")))]
2526 "adc\\t%<w>0, %<w>1, %<w>2"
2527 [(set_attr "type" "adc_reg")]
2530 ;; zero_extend version of above
2531 (define_insn "*addsi3_carryin_uxtw"
2532 [(set (match_operand:DI 0 "register_operand" "=r")
2536 (match_operand:SI 3 "aarch64_carry_operation" "")
2537 (match_operand:SI 1 "register_operand" "r"))
2538 (match_operand:SI 2 "register_operand" "r"))))]
2540 "adc\\t%w0, %w1, %w2"
2541 [(set_attr "type" "adc_reg")]
2544 (define_expand "add<mode>3_carryinC"
2552 (match_operand:GPI 1 "register_operand")))
2554 (match_operand:GPI 2 "register_operand")))
2556 (set (match_operand:GPI 0 "register_operand")
2558 (plus:GPI (match_dup 5) (match_dup 1))
2562 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2563 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2564 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2565 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2566 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2567 << GET_MODE_BITSIZE (<MODE>mode),
2571 (define_insn "*add<mode>3_carryinC_zero"
2572 [(set (reg:CC_ADC CC_REGNUM)
2575 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2576 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2577 (match_operand 4 "const_scalar_int_operand" "")))
2578 (set (match_operand:GPI 0 "register_operand" "=r")
2579 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2581 "rtx_mode_t (operands[4], <DWI>mode)
2582 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2583 "adcs\\t%<w>0, %<w>1, <w>zr"
2584 [(set_attr "type" "adc_reg")]
2587 (define_insn "*add<mode>3_carryinC"
2588 [(set (reg:CC_ADC CC_REGNUM)
2592 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2593 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2594 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2595 (match_operand 5 "const_scalar_int_operand" "")))
2596 (set (match_operand:GPI 0 "register_operand" "=r")
2598 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2601 "rtx_mode_t (operands[5], <DWI>mode)
2602 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2603 "adcs\\t%<w>0, %<w>1, %<w>2"
2604 [(set_attr "type" "adc_reg")]
2607 (define_expand "add<mode>3_carryinV"
2609 [(set (reg:CC_V CC_REGNUM)
2615 (match_operand:GPI 1 "register_operand")))
2617 (match_operand:GPI 2 "register_operand")))
2620 (plus:GPI (match_dup 4) (match_dup 1))
2622 (set (match_operand:GPI 0 "register_operand")
2624 (plus:GPI (match_dup 4) (match_dup 1))
2628 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2629 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2630 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2633 (define_insn "*add<mode>3_carryinV_zero"
2634 [(set (reg:CC_V CC_REGNUM)
2637 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2638 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2641 (match_operand:GPI 3 "aarch64_carry_operation" "")
2643 (set (match_operand:GPI 0 "register_operand" "=r")
2644 (plus:GPI (match_dup 3) (match_dup 1)))]
2646 "adcs\\t%<w>0, %<w>1, <w>zr"
2647 [(set_attr "type" "adc_reg")]
2650 (define_insn "*add<mode>3_carryinV"
2651 [(set (reg:CC_V CC_REGNUM)
2655 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2656 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2657 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2661 (match_operand:GPI 4 "aarch64_carry_operation" "")
2664 (set (match_operand:GPI 0 "register_operand" "=r")
2666 (plus:GPI (match_dup 4) (match_dup 1))
2669 "adcs\\t%<w>0, %<w>1, %<w>2"
2670 [(set_attr "type" "adc_reg")]
2673 (define_insn "*add_uxt<mode>_shift2"
2674 [(set (match_operand:GPI 0 "register_operand" "=rk")
2676 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2677 (match_operand 2 "aarch64_imm3" "Ui3"))
2678 (match_operand 3 "const_int_operand" "n"))
2679 (match_operand:GPI 4 "register_operand" "r")))]
2680 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2682 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2683 INTVAL (operands[3])));
2684 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2685 [(set_attr "type" "alu_ext")]
2688 ;; zero_extend version of above
2689 (define_insn "*add_uxtsi_shift2_uxtw"
2690 [(set (match_operand:DI 0 "register_operand" "=rk")
2693 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2694 (match_operand 2 "aarch64_imm3" "Ui3"))
2695 (match_operand 3 "const_int_operand" "n"))
2696 (match_operand:SI 4 "register_operand" "r"))))]
2697 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2699 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2700 INTVAL (operands[3])));
2701 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2702 [(set_attr "type" "alu_ext")]
2705 (define_insn "*add_uxt<mode>_multp2"
2706 [(set (match_operand:GPI 0 "register_operand" "=rk")
2708 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2709 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2710 (match_operand 3 "const_int_operand" "n"))
2711 (match_operand:GPI 4 "register_operand" "r")))]
2712 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2714 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2715 INTVAL (operands[3])));
2716 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2717 [(set_attr "type" "alu_ext")]
2720 ;; zero_extend version of above
2721 (define_insn "*add_uxtsi_multp2_uxtw"
2722 [(set (match_operand:DI 0 "register_operand" "=rk")
2725 (mult:SI (match_operand:SI 1 "register_operand" "r")
2726 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2727 (match_operand 3 "const_int_operand" "n"))
2728 (match_operand:SI 4 "register_operand" "r"))))]
2729 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2731 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2732 INTVAL (operands[3])));
2733 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2734 [(set_attr "type" "alu_ext")]
2737 (define_insn "subsi3"
2738 [(set (match_operand:SI 0 "register_operand" "=rk")
2739 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2740 (match_operand:SI 2 "register_operand" "r")))]
2742 "sub\\t%w0, %w1, %w2"
2743 [(set_attr "type" "alu_sreg")]
2746 ;; zero_extend version of above
2747 (define_insn "*subsi3_uxtw"
2748 [(set (match_operand:DI 0 "register_operand" "=rk")
2750 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2751 (match_operand:SI 2 "register_operand" "r"))))]
2753 "sub\\t%w0, %w1, %w2"
2754 [(set_attr "type" "alu_sreg")]
2757 (define_insn "subdi3"
2758 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2759 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2760 (match_operand:DI 2 "register_operand" "r,w")))]
2764 sub\\t%d0, %d1, %d2"
2765 [(set_attr "type" "alu_sreg, neon_sub")
2766 (set_attr "arch" "*,simd")]
2769 (define_expand "subv<GPI:mode>4"
2770 [(match_operand:GPI 0 "register_operand")
2771 (match_operand:GPI 1 "register_operand")
2772 (match_operand:GPI 2 "aarch64_plus_operand")
2773 (label_ref (match_operand 3 "" ""))]
2776 if (CONST_INT_P (operands[2]))
2777 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2779 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2780 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2785 (define_insn "subv<GPI:mode>_insn"
2786 [(set (reg:CC_V CC_REGNUM)
2790 (match_operand:GPI 1 "register_operand" "rk")
2791 (match_operand:GPI 2 "register_operand" "r")))
2792 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2793 (sign_extend:<DWI> (match_dup 2)))))
2794 (set (match_operand:GPI 0 "register_operand" "=r")
2795 (minus:GPI (match_dup 1) (match_dup 2)))]
2797 "subs\\t%<w>0, %<w>1, %<w>2"
2798 [(set_attr "type" "alus_sreg")]
2801 (define_insn "subv<GPI:mode>_imm"
2802 [(set (reg:CC_V CC_REGNUM)
2806 (match_operand:GPI 1 "register_operand" "rk,rk")
2807 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2808 (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2810 (set (match_operand:GPI 0 "register_operand" "=r,r")
2811 (minus:GPI (match_dup 1) (match_dup 2)))]
2814 subs\\t%<w>0, %<w>1, %2
2815 adds\\t%<w>0, %<w>1, #%n2"
2816 [(set_attr "type" "alus_sreg")]
2819 (define_expand "negv<GPI:mode>3"
2820 [(match_operand:GPI 0 "register_operand")
2821 (match_operand:GPI 1 "register_operand")
2822 (label_ref (match_operand 2 "" ""))]
2825 emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2826 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2832 (define_insn "negv<GPI:mode>_insn"
2833 [(set (reg:CC_V CC_REGNUM)
2836 (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2837 (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2838 (set (match_operand:GPI 0 "register_operand" "=r")
2839 (neg:GPI (match_dup 1)))]
2841 "negs\\t%<w>0, %<w>1"
2842 [(set_attr "type" "alus_sreg")]
2845 (define_insn "negv<GPI:mode>_cmp_only"
2846 [(set (reg:CC_V CC_REGNUM)
2849 (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2850 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2852 "negs\\t%<w>zr, %<w>0"
2853 [(set_attr "type" "alus_sreg")]
2856 (define_insn "*cmpv<GPI:mode>_insn"
2857 [(set (reg:CC_V CC_REGNUM)
2860 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2861 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2862 (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2863 (sign_extend:<DWI> (match_dup 1)))))]
2869 [(set_attr "type" "alus_sreg")]
2872 (define_expand "usubv<mode>4"
2873 [(match_operand:GPI 0 "register_operand")
2874 (match_operand:GPI 1 "aarch64_reg_or_zero")
2875 (match_operand:GPI 2 "aarch64_reg_or_zero")
2876 (label_ref (match_operand 3 "" ""))]
2879 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2880 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2885 (define_expand "subti3"
2886 [(set (match_operand:TI 0 "register_operand")
2887 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2888 (match_operand:TI 2 "register_operand")))]
2891 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2893 aarch64_subvti_scratch_regs (operands[1], operands[2],
2894 &low_dest, &op1_low, &op2_low,
2895 &high_dest, &op1_high, &op2_high);
2897 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2898 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2900 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2901 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2905 (define_expand "subvti4"
2906 [(match_operand:TI 0 "register_operand")
2907 (match_operand:TI 1 "register_operand")
2908 (match_operand:TI 2 "aarch64_reg_or_imm")
2909 (label_ref (match_operand 3 "" ""))]
2912 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2914 aarch64_subvti_scratch_regs (operands[1], operands[2],
2915 &low_dest, &op1_low, &op2_low,
2916 &high_dest, &op1_high, &op2_high);
2917 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2918 high_dest, op1_high, op2_high, false);
2920 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2924 (define_expand "usubvti4"
2925 [(match_operand:TI 0 "register_operand")
2926 (match_operand:TI 1 "register_operand")
2927 (match_operand:TI 2 "aarch64_reg_or_imm")
2928 (label_ref (match_operand 3 "" ""))]
2931 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2933 aarch64_subvti_scratch_regs (operands[1], operands[2],
2934 &low_dest, &op1_low, &op2_low,
2935 &high_dest, &op1_high, &op2_high);
2936 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2937 high_dest, op1_high, op2_high, true);
2939 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2943 (define_expand "negvti3"
2944 [(match_operand:TI 0 "register_operand")
2945 (match_operand:TI 1 "register_operand")
2946 (label_ref (match_operand 2 "" ""))]
2949 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2950 gen_lowpart (DImode, operands[1])));
2951 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2952 gen_highpart (DImode, operands[1])));
2953 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2959 (define_insn "negdi_carryout"
2960 [(set (reg:CC CC_REGNUM)
2962 (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2963 (set (match_operand:DI 0 "register_operand" "=r")
2964 (neg:DI (match_dup 1)))]
2967 [(set_attr "type" "alus_sreg")]
2970 (define_insn "negvdi_carryinV"
2971 [(set (reg:CC_V CC_REGNUM)
2974 (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2975 (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2977 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2979 (set (match_operand:DI 0 "register_operand" "=r")
2980 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2984 [(set_attr "type" "alus_sreg")]
2987 (define_insn "*sub<mode>3_compare0"
2988 [(set (reg:CC_NZ CC_REGNUM)
2989 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2990 (match_operand:GPI 2 "register_operand" "r"))
2992 (set (match_operand:GPI 0 "register_operand" "=r")
2993 (minus:GPI (match_dup 1) (match_dup 2)))]
2995 "subs\\t%<w>0, %<w>1, %<w>2"
2996 [(set_attr "type" "alus_sreg")]
2999 ;; zero_extend version of above
3000 (define_insn "*subsi3_compare0_uxtw"
3001 [(set (reg:CC_NZ CC_REGNUM)
3002 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3003 (match_operand:SI 2 "register_operand" "r"))
3005 (set (match_operand:DI 0 "register_operand" "=r")
3006 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3008 "subs\\t%w0, %w1, %w2"
3009 [(set_attr "type" "alus_sreg")]
3012 (define_insn "sub<mode>3_compare1_imm"
3013 [(set (reg:CC CC_REGNUM)
3015 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3016 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3017 (set (match_operand:GPI 0 "register_operand" "=r,r")
3020 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3021 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3023 subs\\t%<w>0, %<w>1, %2
3024 adds\\t%<w>0, %<w>1, #%n2"
3025 [(set_attr "type" "alus_imm")]
3028 (define_insn "sub<mode>3_compare1"
3029 [(set (reg:CC CC_REGNUM)
3031 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3032 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3033 (set (match_operand:GPI 0 "register_operand" "=r")
3034 (minus:GPI (match_dup 1) (match_dup 2)))]
3036 "subs\\t%<w>0, %<w>1, %<w>2"
3037 [(set_attr "type" "alus_sreg")]
3041 [(set (match_operand:GPI 0 "aarch64_general_reg")
3042 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3043 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3044 (set (reg:CC CC_REGNUM)
3048 "!reg_overlap_mentioned_p (operands[0], operands[1])
3049 && !reg_overlap_mentioned_p (operands[0], operands[2])"
3052 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3058 ;; Same as the above peephole but with the compare and minus in
3059 ;; swapped order. The restriction on overlap between operand 0
3060 ;; and operands 1 and 2 doesn't apply here.
3062 [(set (reg:CC CC_REGNUM)
3064 (match_operand:GPI 1 "aarch64_reg_or_zero")
3065 (match_operand:GPI 2 "aarch64_reg_or_zero")))
3066 (set (match_operand:GPI 0 "aarch64_general_reg")
3067 (minus:GPI (match_dup 1)
3072 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3079 [(set (match_operand:GPI 0 "aarch64_general_reg")
3080 (plus:GPI (match_operand:GPI 1 "register_operand")
3081 (match_operand:GPI 2 "aarch64_plus_immediate")))
3082 (set (reg:CC CC_REGNUM)
3085 (match_operand:GPI 3 "const_int_operand")))]
3086 "!reg_overlap_mentioned_p (operands[0], operands[1])
3087 && INTVAL (operands[3]) == -INTVAL (operands[2])"
3090 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3091 operands[3], operands[2]));
3096 ;; Same as the above peephole but with the compare and minus in
3097 ;; swapped order. The restriction on overlap between operand 0
3098 ;; and operands 1 doesn't apply here.
3100 [(set (reg:CC CC_REGNUM)
3102 (match_operand:GPI 1 "register_operand")
3103 (match_operand:GPI 3 "const_int_operand")))
3104 (set (match_operand:GPI 0 "aarch64_general_reg")
3105 (plus:GPI (match_dup 1)
3106 (match_operand:GPI 2 "aarch64_plus_immediate")))]
3107 "INTVAL (operands[3]) == -INTVAL (operands[2])"
3110 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3111 operands[3], operands[2]));
3116 (define_insn "*sub_<shift>_<mode>"
3117 [(set (match_operand:GPI 0 "register_operand" "=r")
3118 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3120 (match_operand:GPI 1 "register_operand" "r")
3121 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3123 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3124 [(set_attr "type" "alu_shift_imm")]
3127 ;; zero_extend version of above
3128 (define_insn "*sub_<shift>_si_uxtw"
3129 [(set (match_operand:DI 0 "register_operand" "=r")
3131 (minus:SI (match_operand:SI 3 "register_operand" "r")
3133 (match_operand:SI 1 "register_operand" "r")
3134 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3136 "sub\\t%w0, %w3, %w1, <shift> %2"
3137 [(set_attr "type" "alu_shift_imm")]
3140 (define_insn "*sub_mul_imm_<mode>"
3141 [(set (match_operand:GPI 0 "register_operand" "=r")
3142 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3144 (match_operand:GPI 1 "register_operand" "r")
3145 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3147 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3148 [(set_attr "type" "alu_shift_imm")]
3151 ;; zero_extend version of above
3152 (define_insn "*sub_mul_imm_si_uxtw"
3153 [(set (match_operand:DI 0 "register_operand" "=r")
3155 (minus:SI (match_operand:SI 3 "register_operand" "r")
3157 (match_operand:SI 1 "register_operand" "r")
3158 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3160 "sub\\t%w0, %w3, %w1, lsl %p2"
3161 [(set_attr "type" "alu_shift_imm")]
3164 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3165 [(set (match_operand:GPI 0 "register_operand" "=rk")
3166 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3168 (match_operand:ALLX 2 "register_operand" "r"))))]
3170 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3171 [(set_attr "type" "alu_ext")]
3174 ;; zero_extend version of above
3175 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3176 [(set (match_operand:DI 0 "register_operand" "=rk")
3178 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3180 (match_operand:SHORT 2 "register_operand" "r")))))]
3182 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3183 [(set_attr "type" "alu_ext")]
3186 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3187 [(set (match_operand:GPI 0 "register_operand" "=rk")
3188 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3189 (ashift:GPI (ANY_EXTEND:GPI
3190 (match_operand:ALLX 2 "register_operand" "r"))
3191 (match_operand 3 "aarch64_imm3" "Ui3"))))]
3193 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3194 [(set_attr "type" "alu_ext")]
3197 ;; zero_extend version of above
3198 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3199 [(set (match_operand:DI 0 "register_operand" "=rk")
3201 (minus:SI (match_operand:SI 1 "register_operand" "rk")
3202 (ashift:SI (ANY_EXTEND:SI
3203 (match_operand:SHORT 2 "register_operand" "r"))
3204 (match_operand 3 "aarch64_imm3" "Ui3")))))]
3206 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3207 [(set_attr "type" "alu_ext")]
3210 (define_insn "*sub_<optab><mode>_multp2"
3211 [(set (match_operand:GPI 0 "register_operand" "=rk")
3212 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3214 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3215 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3216 (match_operand 3 "const_int_operand" "n")
3218 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3219 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3220 [(set_attr "type" "alu_ext")]
3223 ;; zero_extend version of above
3224 (define_insn "*sub_<optab>si_multp2_uxtw"
3225 [(set (match_operand:DI 0 "register_operand" "=rk")
3227 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3229 (mult:SI (match_operand:SI 1 "register_operand" "r")
3230 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3231 (match_operand 3 "const_int_operand" "n")
3233 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3234 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3235 [(set_attr "type" "alu_ext")]
3238 ;; The hardware description is op1 + ~op2 + C.
3239 ;; = op1 + (-op2 + 1) + (1 - !C)
3240 ;; = op1 - op2 - 1 + 1 - !C
3241 ;; = op1 - op2 - !C.
3242 ;; We describe the latter.
3244 (define_insn "*sub<mode>3_carryin0"
3245 [(set (match_operand:GPI 0 "register_operand" "=r")
3247 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3248 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3250 "sbc\\t%<w>0, %<w>1, <w>zr"
3251 [(set_attr "type" "adc_reg")]
3254 ;; zero_extend version of the above
3255 (define_insn "*subsi3_carryin_uxtw"
3256 [(set (match_operand:DI 0 "register_operand" "=r")
3259 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3260 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3262 "sbc\\t%w0, %w1, wzr"
3263 [(set_attr "type" "adc_reg")]
3266 (define_expand "sub<mode>3_carryin"
3267 [(set (match_operand:GPI 0 "register_operand")
3270 (match_operand:GPI 1 "aarch64_reg_or_zero")
3271 (match_operand:GPI 2 "register_operand"))
3272 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3277 (define_insn "*sub<mode>3_carryin"
3278 [(set (match_operand:GPI 0 "register_operand" "=r")
3281 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3282 (match_operand:GPI 2 "register_operand" "r"))
3283 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3286 "sbc\\t%<w>0, %<w>1, %<w>2"
3287 [(set_attr "type" "adc_reg")]
3290 ;; zero_extend version of the above
3291 (define_insn "*subsi3_carryin_uxtw"
3292 [(set (match_operand:DI 0 "register_operand" "=r")
3296 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3297 (match_operand:SI 2 "register_operand" "r"))
3298 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3301 "sbc\\t%w0, %w1, %w2"
3302 [(set_attr "type" "adc_reg")]
3305 (define_insn "*sub<mode>3_carryin_alt"
3306 [(set (match_operand:GPI 0 "register_operand" "=r")
3309 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3310 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3311 (match_operand:GPI 2 "register_operand" "r")))]
3313 "sbc\\t%<w>0, %<w>1, %<w>2"
3314 [(set_attr "type" "adc_reg")]
3317 ;; zero_extend version of the above
3318 (define_insn "*subsi3_carryin_alt_uxtw"
3319 [(set (match_operand:DI 0 "register_operand" "=r")
3323 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3324 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3325 (match_operand:SI 2 "register_operand" "r"))))]
3327 "sbc\\t%w0, %w1, %w2"
3328 [(set_attr "type" "adc_reg")]
3331 (define_expand "usub<GPI:mode>3_carryinC"
3333 [(set (reg:CC CC_REGNUM)
3336 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3339 (match_operand:GPI 2 "register_operand"))
3340 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3341 (set (match_operand:GPI 0 "register_operand")
3343 (minus:GPI (match_dup 1) (match_dup 2))
3344 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3348 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3349 [(set (reg:CC CC_REGNUM)
3354 (match_operand:GPI 1 "register_operand" "r"))
3355 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3356 (set (match_operand:GPI 0 "register_operand" "=r")
3358 (neg:GPI (match_dup 1))
3359 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3361 "sbcs\\t%<w>0, <w>zr, %<w>1"
3362 [(set_attr "type" "adc_reg")]
3365 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3366 [(set (reg:CC CC_REGNUM)
3369 (match_operand:GPI 1 "register_operand" "r"))
3370 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3371 (set (match_operand:GPI 0 "register_operand" "=r")
3374 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3376 "sbcs\\t%<w>0, %<w>1, <w>zr"
3377 [(set_attr "type" "adc_reg")]
3380 (define_insn "*usub<GPI:mode>3_carryinC"
3381 [(set (reg:CC CC_REGNUM)
3384 (match_operand:GPI 1 "register_operand" "r"))
3387 (match_operand:GPI 2 "register_operand" "r"))
3388 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3389 (set (match_operand:GPI 0 "register_operand" "=r")
3391 (minus:GPI (match_dup 1) (match_dup 2))
3392 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3394 "sbcs\\t%<w>0, %<w>1, %<w>2"
3395 [(set_attr "type" "adc_reg")]
3398 (define_expand "sub<GPI:mode>3_carryinV"
3400 [(set (reg:CC_V CC_REGNUM)
3404 (match_operand:GPI 1 "aarch64_reg_or_zero"))
3407 (match_operand:GPI 2 "register_operand"))
3408 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3410 (minus:GPI (match_dup 1)
3411 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3413 (set (match_operand:GPI 0 "register_operand")
3415 (minus:GPI (match_dup 1) (match_dup 2))
3416 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3420 (define_insn "*sub<mode>3_carryinV_z2"
3421 [(set (reg:CC_V CC_REGNUM)
3424 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3425 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3427 (minus:GPI (match_dup 1)
3428 (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3429 (set (match_operand:GPI 0 "register_operand" "=r")
3431 (match_dup 1) (match_dup 3)))]
3433 "sbcs\\t%<w>0, %<w>1, <w>zr"
3434 [(set_attr "type" "adc_reg")]
3437 (define_insn "*sub<mode>3_carryinV"
3438 [(set (reg:CC_V CC_REGNUM)
3442 (match_operand:GPI 1 "register_operand" "r"))
3445 (match_operand:GPI 2 "register_operand" "r"))
3446 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3450 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3452 (set (match_operand:GPI 0 "register_operand" "=r")
3454 (minus:GPI (match_dup 1) (match_dup 2))
3457 "sbcs\\t%<w>0, %<w>1, %<w>2"
3458 [(set_attr "type" "adc_reg")]
3461 (define_insn "*sub_uxt<mode>_shift2"
3462 [(set (match_operand:GPI 0 "register_operand" "=rk")
3463 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3465 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3466 (match_operand 2 "aarch64_imm3" "Ui3"))
3467 (match_operand 3 "const_int_operand" "n"))))]
3468 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3470 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3471 INTVAL (operands[3])));
3472 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3473 [(set_attr "type" "alu_ext")]
3476 ;; zero_extend version of above
3477 (define_insn "*sub_uxtsi_shift2_uxtw"
3478 [(set (match_operand:DI 0 "register_operand" "=rk")
3480 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3482 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3483 (match_operand 2 "aarch64_imm3" "Ui3"))
3484 (match_operand 3 "const_int_operand" "n")))))]
3485 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3487 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3488 INTVAL (operands[3])));
3489 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3490 [(set_attr "type" "alu_ext")]
3493 (define_insn "*sub_uxt<mode>_multp2"
3494 [(set (match_operand:GPI 0 "register_operand" "=rk")
3495 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3497 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3498 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3499 (match_operand 3 "const_int_operand" "n"))))]
3500 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3502 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3503 INTVAL (operands[3])));
3504 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3505 [(set_attr "type" "alu_ext")]
3508 ;; zero_extend version of above
3509 (define_insn "*sub_uxtsi_multp2_uxtw"
3510 [(set (match_operand:DI 0 "register_operand" "=rk")
3512 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3514 (mult:SI (match_operand:SI 1 "register_operand" "r")
3515 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3516 (match_operand 3 "const_int_operand" "n")))))]
3517 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3519 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3520 INTVAL (operands[3])));
3521 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3522 [(set_attr "type" "alu_ext")]
3525 (define_expand "abs<mode>2"
3526 [(match_operand:GPI 0 "register_operand")
3527 (match_operand:GPI 1 "register_operand")]
3530 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3531 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3532 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3537 (define_insn "neg<mode>2"
3538 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3539 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3543 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3544 [(set_attr "type" "alu_sreg, neon_neg<q>")
3545 (set_attr "arch" "*,simd")]
3548 ;; zero_extend version of above
3549 (define_insn "*negsi2_uxtw"
3550 [(set (match_operand:DI 0 "register_operand" "=r")
3551 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3554 [(set_attr "type" "alu_sreg")]
3557 (define_insn "*ngc<mode>"
3558 [(set (match_operand:GPI 0 "register_operand" "=r")
3560 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3561 (match_operand:GPI 1 "register_operand" "r")))]
3563 "ngc\\t%<w>0, %<w>1"
3564 [(set_attr "type" "adc_reg")]
3567 (define_insn "*ngcsi_uxtw"
3568 [(set (match_operand:DI 0 "register_operand" "=r")
3571 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3572 (match_operand:SI 1 "register_operand" "r"))))]
3575 [(set_attr "type" "adc_reg")]
3578 (define_insn "neg<mode>2_compare0"
3579 [(set (reg:CC_NZ CC_REGNUM)
3580 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3582 (set (match_operand:GPI 0 "register_operand" "=r")
3583 (neg:GPI (match_dup 1)))]
3585 "negs\\t%<w>0, %<w>1"
3586 [(set_attr "type" "alus_sreg")]
3589 ;; zero_extend version of above
3590 (define_insn "*negsi2_compare0_uxtw"
3591 [(set (reg:CC_NZ CC_REGNUM)
3592 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3594 (set (match_operand:DI 0 "register_operand" "=r")
3595 (zero_extend:DI (neg:SI (match_dup 1))))]
3598 [(set_attr "type" "alus_sreg")]
3601 (define_insn "*neg_<shift><mode>3_compare0"
3602 [(set (reg:CC_NZ CC_REGNUM)
3604 (neg:GPI (ASHIFT:GPI
3605 (match_operand:GPI 1 "register_operand" "r")
3606 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3608 (set (match_operand:GPI 0 "register_operand" "=r")
3609 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3611 "negs\\t%<w>0, %<w>1, <shift> %2"
3612 [(set_attr "type" "alus_shift_imm")]
3615 (define_insn "*neg_<shift>_<mode>2"
3616 [(set (match_operand:GPI 0 "register_operand" "=r")
3617 (neg:GPI (ASHIFT:GPI
3618 (match_operand:GPI 1 "register_operand" "r")
3619 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3621 "neg\\t%<w>0, %<w>1, <shift> %2"
3622 [(set_attr "type" "alu_shift_imm")]
3625 ;; zero_extend version of above
3626 (define_insn "*neg_<shift>_si2_uxtw"
3627 [(set (match_operand:DI 0 "register_operand" "=r")
3630 (match_operand:SI 1 "register_operand" "r")
3631 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3633 "neg\\t%w0, %w1, <shift> %2"
3634 [(set_attr "type" "alu_shift_imm")]
3637 (define_insn "*neg_mul_imm_<mode>2"
3638 [(set (match_operand:GPI 0 "register_operand" "=r")
3640 (match_operand:GPI 1 "register_operand" "r")
3641 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3643 "neg\\t%<w>0, %<w>1, lsl %p2"
3644 [(set_attr "type" "alu_shift_imm")]
3647 ;; zero_extend version of above
3648 (define_insn "*neg_mul_imm_si2_uxtw"
3649 [(set (match_operand:DI 0 "register_operand" "=r")
3652 (match_operand:SI 1 "register_operand" "r")
3653 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3655 "neg\\t%w0, %w1, lsl %p2"
3656 [(set_attr "type" "alu_shift_imm")]
3659 (define_insn "mul<mode>3"
3660 [(set (match_operand:GPI 0 "register_operand" "=r")
3661 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3662 (match_operand:GPI 2 "register_operand" "r")))]
3664 "mul\\t%<w>0, %<w>1, %<w>2"
3665 [(set_attr "type" "mul")]
3668 ;; zero_extend version of above
3669 (define_insn "*mulsi3_uxtw"
3670 [(set (match_operand:DI 0 "register_operand" "=r")
3672 (mult:SI (match_operand:SI 1 "register_operand" "r")
3673 (match_operand:SI 2 "register_operand" "r"))))]
3675 "mul\\t%w0, %w1, %w2"
3676 [(set_attr "type" "mul")]
3679 (define_insn "madd<mode>"
3680 [(set (match_operand:GPI 0 "register_operand" "=r")
3681 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3682 (match_operand:GPI 2 "register_operand" "r"))
3683 (match_operand:GPI 3 "register_operand" "r")))]
3685 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3686 [(set_attr "type" "mla")]
3689 ;; zero_extend version of above
3690 (define_insn "*maddsi_uxtw"
3691 [(set (match_operand:DI 0 "register_operand" "=r")
3693 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3694 (match_operand:SI 2 "register_operand" "r"))
3695 (match_operand:SI 3 "register_operand" "r"))))]
3697 "madd\\t%w0, %w1, %w2, %w3"
3698 [(set_attr "type" "mla")]
3701 (define_insn "*msub<mode>"
3702 [(set (match_operand:GPI 0 "register_operand" "=r")
3703 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3704 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3705 (match_operand:GPI 2 "register_operand" "r"))))]
3708 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3709 [(set_attr "type" "mla")]
3712 ;; zero_extend version of above
3713 (define_insn "*msubsi_uxtw"
3714 [(set (match_operand:DI 0 "register_operand" "=r")
3716 (minus:SI (match_operand:SI 3 "register_operand" "r")
3717 (mult:SI (match_operand:SI 1 "register_operand" "r")
3718 (match_operand:SI 2 "register_operand" "r")))))]
3721 "msub\\t%w0, %w1, %w2, %w3"
3722 [(set_attr "type" "mla")]
3725 (define_insn "*mul<mode>_neg"
3726 [(set (match_operand:GPI 0 "register_operand" "=r")
3727 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3728 (match_operand:GPI 2 "register_operand" "r")))]
3731 "mneg\\t%<w>0, %<w>1, %<w>2"
3732 [(set_attr "type" "mul")]
3735 ;; zero_extend version of above
3736 (define_insn "*mulsi_neg_uxtw"
3737 [(set (match_operand:DI 0 "register_operand" "=r")
3739 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3740 (match_operand:SI 2 "register_operand" "r"))))]
3743 "mneg\\t%w0, %w1, %w2"
3744 [(set_attr "type" "mul")]
3747 (define_insn "<su_optab>mulsidi3"
3748 [(set (match_operand:DI 0 "register_operand" "=r")
3749 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3750 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3752 "<su>mull\\t%0, %w1, %w2"
3753 [(set_attr "type" "<su>mull")]
3756 (define_insn "<su_optab>maddsidi4"
3757 [(set (match_operand:DI 0 "register_operand" "=r")
3759 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3760 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3761 (match_operand:DI 3 "register_operand" "r")))]
3763 "<su>maddl\\t%0, %w1, %w2, %3"
3764 [(set_attr "type" "<su>mlal")]
3767 (define_insn "<su_optab>msubsidi4"
3768 [(set (match_operand:DI 0 "register_operand" "=r")
3770 (match_operand:DI 3 "register_operand" "r")
3771 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3773 (match_operand:SI 2 "register_operand" "r")))))]
3775 "<su>msubl\\t%0, %w1, %w2, %3"
3776 [(set_attr "type" "<su>mlal")]
3779 (define_insn "*<su_optab>mulsidi_neg"
3780 [(set (match_operand:DI 0 "register_operand" "=r")
3782 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3783 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3785 "<su>mnegl\\t%0, %w1, %w2"
3786 [(set_attr "type" "<su>mull")]
3789 (define_expand "<su_optab>mulditi3"
3790 [(set (match_operand:TI 0 "register_operand")
3791 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3792 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3795 rtx low = gen_reg_rtx (DImode);
3796 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3798 rtx high = gen_reg_rtx (DImode);
3799 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3801 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3802 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3806 ;; The default expansion of multi3 using umuldi3_highpart will perform
3807 ;; the additions in an order that fails to combine into two madd insns.
3808 (define_expand "multi3"
3809 [(set (match_operand:TI 0 "register_operand")
3810 (mult:TI (match_operand:TI 1 "register_operand")
3811 (match_operand:TI 2 "register_operand")))]
3814 rtx l0 = gen_reg_rtx (DImode);
3815 rtx l1 = gen_lowpart (DImode, operands[1]);
3816 rtx l2 = gen_lowpart (DImode, operands[2]);
3817 rtx h0 = gen_reg_rtx (DImode);
3818 rtx h1 = gen_highpart (DImode, operands[1]);
3819 rtx h2 = gen_highpart (DImode, operands[2]);
3821 emit_insn (gen_muldi3 (l0, l1, l2));
3822 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3823 emit_insn (gen_madddi (h0, h1, l2, h0));
3824 emit_insn (gen_madddi (h0, l1, h2, h0));
3826 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3827 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3831 (define_insn "<su>muldi3_highpart"
3832 [(set (match_operand:DI 0 "register_operand" "=r")
3836 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3837 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3840 "<su>mulh\\t%0, %1, %2"
3841 [(set_attr "type" "<su>mull")]
3844 (define_insn "<su_optab>div<mode>3"
3845 [(set (match_operand:GPI 0 "register_operand" "=r")
3846 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3847 (match_operand:GPI 2 "register_operand" "r")))]
3849 "<su>div\\t%<w>0, %<w>1, %<w>2"
3850 [(set_attr "type" "<su>div")]
3853 ;; zero_extend version of above
3854 (define_insn "*<su_optab>divsi3_uxtw"
3855 [(set (match_operand:DI 0 "register_operand" "=r")
3857 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3858 (match_operand:SI 2 "register_operand" "r"))))]
3860 "<su>div\\t%w0, %w1, %w2"
3861 [(set_attr "type" "<su>div")]
3864 ;; -------------------------------------------------------------------
3866 ;; -------------------------------------------------------------------
3868 (define_insn "cmp<mode>"
3869 [(set (reg:CC CC_REGNUM)
3870 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3871 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3877 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3880 (define_insn "fcmp<mode>"
3881 [(set (reg:CCFP CC_REGNUM)
3882 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3883 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3887 fcmp\\t%<s>0, %<s>1"
3888 [(set_attr "type" "fcmp<s>")]
3891 (define_insn "fcmpe<mode>"
3892 [(set (reg:CCFPE CC_REGNUM)
3893 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3894 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3898 fcmpe\\t%<s>0, %<s>1"
3899 [(set_attr "type" "fcmp<s>")]
3902 (define_insn "*cmp_swp_<shift>_reg<mode>"
3903 [(set (reg:CC_SWP CC_REGNUM)
3904 (compare:CC_SWP (ASHIFT:GPI
3905 (match_operand:GPI 0 "register_operand" "r")
3906 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3907 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3909 "cmp\\t%<w>2, %<w>0, <shift> %1"
3910 [(set_attr "type" "alus_shift_imm")]
3913 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3914 [(set (reg:CC_SWP CC_REGNUM)
3915 (compare:CC_SWP (ANY_EXTEND:GPI
3916 (match_operand:ALLX 0 "register_operand" "r"))
3917 (match_operand:GPI 1 "register_operand" "r")))]
3919 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3920 [(set_attr "type" "alus_ext")]
3923 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3924 [(set (reg:CC_SWP CC_REGNUM)
3925 (compare:CC_SWP (ashift:GPI
3927 (match_operand:ALLX 0 "register_operand" "r"))
3928 (match_operand 1 "aarch64_imm3" "Ui3"))
3929 (match_operand:GPI 2 "register_operand" "r")))]
3931 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3932 [(set_attr "type" "alus_ext")]
3935 ;; -------------------------------------------------------------------
3936 ;; Store-flag and conditional select insns
3937 ;; -------------------------------------------------------------------
3939 (define_expand "cstore<mode>4"
3940 [(set (match_operand:SI 0 "register_operand")
3941 (match_operator:SI 1 "aarch64_comparison_operator"
3942 [(match_operand:GPI 2 "register_operand")
3943 (match_operand:GPI 3 "aarch64_plus_operand")]))]
3946 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3948 operands[3] = const0_rtx;
3952 (define_expand "cstorecc4"
3953 [(set (match_operand:SI 0 "register_operand")
3954 (match_operator 1 "aarch64_comparison_operator_mode"
3955 [(match_operand 2 "cc_register")
3956 (match_operand 3 "const0_operand")]))]
3959 emit_insn (gen_rtx_SET (operands[0], operands[1]));
3964 (define_expand "cstore<mode>4"
3965 [(set (match_operand:SI 0 "register_operand")
3966 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3967 [(match_operand:GPF 2 "register_operand")
3968 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
3971 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3973 operands[3] = const0_rtx;
3977 (define_insn "aarch64_cstore<mode>"
3978 [(set (match_operand:ALLI 0 "register_operand" "=r")
3979 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3980 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3983 [(set_attr "type" "csel")]
3986 ;; For a 24-bit immediate CST we can optimize the compare for equality
3987 ;; and branch sequence from:
3989 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
3992 ;; into the shorter:
3993 ;; sub x0, x1, #(CST & 0xfff000)
3994 ;; subs x0, x0, #(CST & 0x000fff)
3995 ;; cset x2, <ne, eq>.
3996 (define_insn_and_split "*compare_cstore<mode>_insn"
3997 [(set (match_operand:GPI 0 "register_operand" "=r")
3998 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3999 (match_operand:GPI 2 "aarch64_imm24" "n")))
4000 (clobber (reg:CC CC_REGNUM))]
4001 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4002 && !aarch64_plus_operand (operands[2], <MODE>mode)
4003 && !reload_completed"
4008 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4009 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4010 rtx tmp = gen_reg_rtx (<MODE>mode);
4011 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4012 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4013 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4014 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4015 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4018 [(set_attr "type" "csel")]
4021 ;; zero_extend version of the above
4022 (define_insn "*cstoresi_insn_uxtw"
4023 [(set (match_operand:DI 0 "register_operand" "=r")
4025 (match_operator:SI 1 "aarch64_comparison_operator_mode"
4026 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4029 [(set_attr "type" "csel")]
4032 (define_insn "cstore<mode>_neg"
4033 [(set (match_operand:ALLI 0 "register_operand" "=r")
4034 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4035 [(match_operand 2 "cc_register" "") (const_int 0)])))]
4037 "csetm\\t%<w>0, %m1"
4038 [(set_attr "type" "csel")]
4041 ;; zero_extend version of the above
4042 (define_insn "*cstoresi_neg_uxtw"
4043 [(set (match_operand:DI 0 "register_operand" "=r")
4045 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4046 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4049 [(set_attr "type" "csel")]
4052 (define_expand "cmov<mode>6"
4053 [(set (match_operand:GPI 0 "register_operand")
4055 (match_operator 1 "aarch64_comparison_operator"
4056 [(match_operand:GPI 2 "register_operand")
4057 (match_operand:GPI 3 "aarch64_plus_operand")])
4058 (match_operand:GPI 4 "register_operand")
4059 (match_operand:GPI 5 "register_operand")))]
4062 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4064 operands[3] = const0_rtx;
4068 (define_expand "cmov<mode>6"
4069 [(set (match_operand:GPF 0 "register_operand")
4071 (match_operator 1 "aarch64_comparison_operator"
4072 [(match_operand:GPF 2 "register_operand")
4073 (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4074 (match_operand:GPF 4 "register_operand")
4075 (match_operand:GPF 5 "register_operand")))]
4078 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4080 operands[3] = const0_rtx;
4084 (define_insn "*cmov<mode>_insn"
4085 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4087 (match_operator 1 "aarch64_comparison_operator"
4088 [(match_operand 2 "cc_register" "") (const_int 0)])
4089 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4090 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4091 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4092 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4093 ;; Final two alternatives should be unreachable, but included for completeness
4095 csel\\t%<w>0, %<w>3, %<w>4, %m1
4096 csinv\\t%<w>0, %<w>3, <w>zr, %m1
4097 csinv\\t%<w>0, %<w>4, <w>zr, %M1
4098 csinc\\t%<w>0, %<w>3, <w>zr, %m1
4099 csinc\\t%<w>0, %<w>4, <w>zr, %M1
4102 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4105 ;; zero_extend version of above
4106 (define_insn "*cmovsi_insn_uxtw"
4107 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4110 (match_operator 1 "aarch64_comparison_operator"
4111 [(match_operand 2 "cc_register" "") (const_int 0)])
4112 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4113 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4114 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4115 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4116 ;; Final two alternatives should be unreachable, but included for completeness
4118 csel\\t%w0, %w3, %w4, %m1
4119 csinv\\t%w0, %w3, wzr, %m1
4120 csinv\\t%w0, %w4, wzr, %M1
4121 csinc\\t%w0, %w3, wzr, %m1
4122 csinc\\t%w0, %w4, wzr, %M1
4125 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4128 (define_insn "*cmovdi_insn_uxtw"
4129 [(set (match_operand:DI 0 "register_operand" "=r")
4131 (match_operator 1 "aarch64_comparison_operator"
4132 [(match_operand 2 "cc_register" "") (const_int 0)])
4133 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4134 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4136 "csel\\t%w0, %w3, %w4, %m1"
4137 [(set_attr "type" "csel")]
4140 (define_insn "*cmov<mode>_insn"
4141 [(set (match_operand:GPF 0 "register_operand" "=w")
4143 (match_operator 1 "aarch64_comparison_operator"
4144 [(match_operand 2 "cc_register" "") (const_int 0)])
4145 (match_operand:GPF 3 "register_operand" "w")
4146 (match_operand:GPF 4 "register_operand" "w")))]
4148 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4149 [(set_attr "type" "fcsel")]
4152 (define_expand "mov<mode>cc"
4153 [(set (match_operand:ALLI 0 "register_operand")
4154 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4155 (match_operand:ALLI 2 "register_operand")
4156 (match_operand:ALLI 3 "register_operand")))]
4160 enum rtx_code code = GET_CODE (operands[1]);
4162 if (code == UNEQ || code == LTGT)
4165 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4166 XEXP (operands[1], 1));
4167 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4171 (define_expand "mov<GPF:mode><GPI:mode>cc"
4172 [(set (match_operand:GPI 0 "register_operand")
4173 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4174 (match_operand:GPF 2 "register_operand")
4175 (match_operand:GPF 3 "register_operand")))]
4179 enum rtx_code code = GET_CODE (operands[1]);
4181 if (code == UNEQ || code == LTGT)
4184 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4185 XEXP (operands[1], 1));
4186 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4190 (define_expand "mov<mode>cc"
4191 [(set (match_operand:GPF 0 "register_operand")
4192 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4193 (match_operand:GPF 2 "register_operand")
4194 (match_operand:GPF 3 "register_operand")))]
4198 enum rtx_code code = GET_CODE (operands[1]);
4200 if (code == UNEQ || code == LTGT)
4203 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4204 XEXP (operands[1], 1));
4205 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4209 (define_expand "<neg_not_op><mode>cc"
4210 [(set (match_operand:GPI 0 "register_operand")
4211 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4212 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4213 (match_operand:GPI 3 "register_operand")))]
4217 enum rtx_code code = GET_CODE (operands[1]);
4219 if (code == UNEQ || code == LTGT)
4222 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4223 XEXP (operands[1], 1));
4224 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4228 ;; CRC32 instructions.
4229 (define_insn "aarch64_<crc_variant>"
4230 [(set (match_operand:SI 0 "register_operand" "=r")
4231 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4232 (match_operand:<crc_mode> 2 "register_operand" "r")]
4236 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4237 return "<crc_variant>\\t%w0, %w1, %x2";
4239 return "<crc_variant>\\t%w0, %w1, %w2";
4241 [(set_attr "type" "crc")]
4244 (define_insn "*csinc2<mode>_insn"
4245 [(set (match_operand:GPI 0 "register_operand" "=r")
4246 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4247 (match_operand:GPI 1 "register_operand" "r")))]
4249 "cinc\\t%<w>0, %<w>1, %m2"
4250 [(set_attr "type" "csel")]
4253 (define_insn "csinc3<mode>_insn"
4254 [(set (match_operand:GPI 0 "register_operand" "=r")
4256 (match_operand 1 "aarch64_comparison_operation" "")
4257 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4259 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4261 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4262 [(set_attr "type" "csel")]
4265 (define_insn "*csinv3<mode>_insn"
4266 [(set (match_operand:GPI 0 "register_operand" "=r")
4268 (match_operand 1 "aarch64_comparison_operation" "")
4269 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4270 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4272 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4273 [(set_attr "type" "csel")]
4276 (define_insn "csneg3_uxtw_insn"
4277 [(set (match_operand:DI 0 "register_operand" "=r")
4280 (match_operand 1 "aarch64_comparison_operation" "")
4281 (neg:SI (match_operand:SI 2 "register_operand" "r"))
4282 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4284 "csneg\\t%w0, %w3, %w2, %M1"
4285 [(set_attr "type" "csel")]
4288 (define_insn "csneg3<mode>_insn"
4289 [(set (match_operand:GPI 0 "register_operand" "=r")
4291 (match_operand 1 "aarch64_comparison_operation" "")
4292 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4293 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4295 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4296 [(set_attr "type" "csel")]
4299 ;; If X can be loaded by a single CNT[BHWD] instruction,
4303 ;; is equivalent to:
4305 ;; TMP = UQDEC[BHWD] (B, X)
4308 ;; Defining the pattern this way means that:
4310 ;; A = UMAX (B, X) - X
4314 ;; TMP1 = UQDEC[BHWD] (B, X)
4318 ;; which combine can optimize to:
4320 ;; A = UQDEC[BHWD] (B, X)
4322 ;; We don't use match_operand predicates because the order of the operands
4323 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4324 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4325 (define_expand "umax<mode>3"
4326 [(set (match_operand:GPI 0 "register_operand")
4327 (umax:GPI (match_operand:GPI 1 "")
4328 (match_operand:GPI 2 "")))]
4331 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4332 std::swap (operands[1], operands[2]);
4333 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4335 rtx temp = gen_reg_rtx (<MODE>mode);
4336 operands[1] = force_reg (<MODE>mode, operands[1]);
4337 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4338 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4343 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4344 (define_insn "aarch64_uqdec<mode>"
4345 [(set (match_operand:GPI 0 "register_operand" "=r")
4347 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4348 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4352 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4356 ;; -------------------------------------------------------------------
4357 ;; Logical operations
4358 ;; -------------------------------------------------------------------
4361 (define_insn_and_split "*aarch64_and<mode>_imm2"
4362 [(set (match_operand:GPI 0 "register_operand" "=rk")
4363 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4364 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4370 HOST_WIDE_INT val = INTVAL (operands[2]);
4371 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4372 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4374 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4375 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4380 (define_insn "<optab><mode>3"
4381 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4382 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4383 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4386 <logical>\\t%<w>0, %<w>1, %<w>2
4387 <logical>\\t%<w>0, %<w>1, %2
4388 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4389 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4390 (set_attr "arch" "*,*,simd")]
4393 ;; zero_extend version of above
4394 (define_insn "*<optab>si3_uxtw"
4395 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4397 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4398 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4401 <logical>\\t%w0, %w1, %w2
4402 <logical>\\t%w0, %w1, %2"
4403 [(set_attr "type" "logic_reg,logic_imm")]
4406 (define_insn "*and<mode>3_compare0"
4407 [(set (reg:CC_NZ CC_REGNUM)
4409 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4410 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4412 (set (match_operand:GPI 0 "register_operand" "=r,r")
4413 (and:GPI (match_dup 1) (match_dup 2)))]
4416 ands\\t%<w>0, %<w>1, %<w>2
4417 ands\\t%<w>0, %<w>1, %2"
4418 [(set_attr "type" "logics_reg,logics_imm")]
4421 ;; zero_extend version of above
4422 (define_insn "*andsi3_compare0_uxtw"
4423 [(set (reg:CC_NZ CC_REGNUM)
4425 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4426 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4428 (set (match_operand:DI 0 "register_operand" "=r,r")
4429 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4432 ands\\t%w0, %w1, %w2
4433 ands\\t%w0, %w1, %2"
4434 [(set_attr "type" "logics_reg,logics_imm")]
4437 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4438 [(set (reg:CC_NZ CC_REGNUM)
4441 (match_operand:GPI 1 "register_operand" "r")
4442 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4443 (match_operand:GPI 3 "register_operand" "r"))
4445 (set (match_operand:GPI 0 "register_operand" "=r")
4446 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4448 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4449 [(set_attr "type" "logics_shift_imm")]
4452 ;; zero_extend version of above
4453 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4454 [(set (reg:CC_NZ CC_REGNUM)
4457 (match_operand:SI 1 "register_operand" "r")
4458 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4459 (match_operand:SI 3 "register_operand" "r"))
4461 (set (match_operand:DI 0 "register_operand" "=r")
4462 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4465 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4466 [(set_attr "type" "logics_shift_imm")]
4469 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4470 [(set (match_operand:GPI 0 "register_operand" "=r")
4471 (LOGICAL:GPI (SHIFT:GPI
4472 (match_operand:GPI 1 "register_operand" "r")
4473 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4474 (match_operand:GPI 3 "register_operand" "r")))]
4476 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4477 [(set_attr "type" "logic_shift_imm")]
4480 (define_insn "*<optab>_rol<mode>3"
4481 [(set (match_operand:GPI 0 "register_operand" "=r")
4482 (LOGICAL:GPI (rotate:GPI
4483 (match_operand:GPI 1 "register_operand" "r")
4484 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4485 (match_operand:GPI 3 "register_operand" "r")))]
4487 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4488 [(set_attr "type" "logic_shift_imm")]
4491 ;; zero_extend versions of above
4492 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4493 [(set (match_operand:DI 0 "register_operand" "=r")
4495 (LOGICAL:SI (SHIFT:SI
4496 (match_operand:SI 1 "register_operand" "r")
4497 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4498 (match_operand:SI 3 "register_operand" "r"))))]
4500 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4501 [(set_attr "type" "logic_shift_imm")]
4504 (define_insn "*<optab>_rolsi3_uxtw"
4505 [(set (match_operand:DI 0 "register_operand" "=r")
4507 (LOGICAL:SI (rotate:SI
4508 (match_operand:SI 1 "register_operand" "r")
4509 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4510 (match_operand:SI 3 "register_operand" "r"))))]
4512 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4513 [(set_attr "type" "logic_shift_imm")]
4516 (define_insn "one_cmpl<mode>2"
4517 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4518 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4523 [(set_attr "type" "logic_reg,neon_logic")
4524 (set_attr "arch" "*,simd")]
4527 (define_insn "*one_cmpl_<optab><mode>2"
4528 [(set (match_operand:GPI 0 "register_operand" "=r")
4529 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4530 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4532 "mvn\\t%<w>0, %<w>1, <shift> %2"
4533 [(set_attr "type" "logic_shift_imm")]
4536 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4538 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4539 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4540 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4541 (match_operand:GPI 2 "register_operand" "r,w")))]
4544 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4545 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4546 [(set_attr "type" "logic_reg,neon_logic")
4547 (set_attr "arch" "*,simd")]
4550 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4551 [(set (match_operand:DI 0 "register_operand" "=r")
4553 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4554 (match_operand:SI 2 "register_operand" "r"))))]
4556 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4557 [(set_attr "type" "logic_reg")]
4560 (define_insn "*xor_one_cmplsidi3_ze"
4561 [(set (match_operand:DI 0 "register_operand" "=r")
4563 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4564 (match_operand:SI 2 "register_operand" "r")))))]
4566 "eon\\t%w0, %w1, %w2"
4567 [(set_attr "type" "logic_reg")]
4570 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4571 ;; eon does not operate on SIMD registers so the vector variant must be split.
4572 (define_insn_and_split "*xor_one_cmpl<mode>3"
4573 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4574 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4575 (match_operand:GPI 2 "register_operand" "r,w"))))]
4578 eon\\t%<w>0, %<w>1, %<w>2
4580 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4581 [(set (match_operand:GPI 0 "register_operand" "=w")
4582 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4583 (match_operand:GPI 2 "register_operand" "w")))
4584 (set (match_dup 0) (not:GPI (match_dup 0)))]
4586 [(set_attr "type" "logic_reg,multiple")
4587 (set_attr "arch" "*,simd")]
4590 (define_insn "*and_one_cmpl<mode>3_compare0"
4591 [(set (reg:CC_NZ CC_REGNUM)
4594 (match_operand:GPI 1 "register_operand" "r"))
4595 (match_operand:GPI 2 "register_operand" "r"))
4597 (set (match_operand:GPI 0 "register_operand" "=r")
4598 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4600 "bics\\t%<w>0, %<w>2, %<w>1"
4601 [(set_attr "type" "logics_reg")]
4604 ;; zero_extend version of above
4605 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4606 [(set (reg:CC_NZ CC_REGNUM)
4609 (match_operand:SI 1 "register_operand" "r"))
4610 (match_operand:SI 2 "register_operand" "r"))
4612 (set (match_operand:DI 0 "register_operand" "=r")
4613 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4615 "bics\\t%w0, %w2, %w1"
4616 [(set_attr "type" "logics_reg")]
4619 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4620 [(set (reg:CC_NZ CC_REGNUM)
4623 (match_operand:GPI 0 "register_operand" "r"))
4624 (match_operand:GPI 1 "register_operand" "r"))
4627 "bics\\t<w>zr, %<w>1, %<w>0"
4628 [(set_attr "type" "logics_reg")]
4631 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4632 [(set (match_operand:GPI 0 "register_operand" "=r")
4633 (LOGICAL:GPI (not:GPI
4635 (match_operand:GPI 1 "register_operand" "r")
4636 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4637 (match_operand:GPI 3 "register_operand" "r")))]
4639 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4640 [(set_attr "type" "logic_shift_imm")]
4643 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4644 [(set (match_operand:GPI 0 "register_operand" "=r")
4647 (match_operand:GPI 1 "register_operand" "r")
4648 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4649 (match_operand:GPI 3 "register_operand" "r"))))]
4651 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4652 [(set_attr "type" "logic_shift_imm")]
4655 ;; Zero-extend version of the above.
4656 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4657 [(set (match_operand:DI 0 "register_operand" "=r")
4661 (match_operand:SI 1 "register_operand" "r")
4662 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4663 (match_operand:SI 3 "register_operand" "r")))))]
4665 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4666 [(set_attr "type" "logic_shift_imm")]
4669 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4670 [(set (reg:CC_NZ CC_REGNUM)
4674 (match_operand:GPI 1 "register_operand" "r")
4675 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4676 (match_operand:GPI 3 "register_operand" "r"))
4678 (set (match_operand:GPI 0 "register_operand" "=r")
4681 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4683 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4684 [(set_attr "type" "logics_shift_imm")]
4687 ;; zero_extend version of above
4688 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4689 [(set (reg:CC_NZ CC_REGNUM)
4693 (match_operand:SI 1 "register_operand" "r")
4694 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4695 (match_operand:SI 3 "register_operand" "r"))
4697 (set (match_operand:DI 0 "register_operand" "=r")
4698 (zero_extend:DI (and:SI
4700 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4702 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4703 [(set_attr "type" "logics_shift_imm")]
4706 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4707 [(set (reg:CC_NZ CC_REGNUM)
4711 (match_operand:GPI 0 "register_operand" "r")
4712 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4713 (match_operand:GPI 2 "register_operand" "r"))
4716 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4717 [(set_attr "type" "logics_shift_imm")]
4720 (define_insn "clz<mode>2"
4721 [(set (match_operand:GPI 0 "register_operand" "=r")
4722 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4724 "clz\\t%<w>0, %<w>1"
4725 [(set_attr "type" "clz")]
4728 (define_expand "ffs<mode>2"
4729 [(match_operand:GPI 0 "register_operand")
4730 (match_operand:GPI 1 "register_operand")]
4733 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4734 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4736 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4737 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4738 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4743 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4750 (define_expand "popcount<mode>2"
4751 [(match_operand:GPI 0 "register_operand")
4752 (match_operand:GPI 1 "register_operand")]
4755 rtx v = gen_reg_rtx (V8QImode);
4756 rtx v1 = gen_reg_rtx (V8QImode);
4757 rtx r = gen_reg_rtx (QImode);
4758 rtx in = operands[1];
4759 rtx out = operands[0];
4760 if(<MODE>mode == SImode)
4763 tmp = gen_reg_rtx (DImode);
4764 /* If we have SImode, zero extend to DImode, pop count does
4765 not change if we have extra zeros. */
4766 emit_insn (gen_zero_extendsidi2 (tmp, in));
4769 emit_move_insn (v, gen_lowpart (V8QImode, in));
4770 emit_insn (gen_popcountv8qi2 (v1, v));
4771 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4772 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4776 (define_insn "clrsb<mode>2"
4777 [(set (match_operand:GPI 0 "register_operand" "=r")
4778 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4780 "cls\\t%<w>0, %<w>1"
4781 [(set_attr "type" "clz")]
4784 (define_insn "rbit<mode>2"
4785 [(set (match_operand:GPI 0 "register_operand" "=r")
4786 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4788 "rbit\\t%<w>0, %<w>1"
4789 [(set_attr "type" "rbit")]
4792 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4793 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4794 ;; expression and split after reload to enable scheduling them apart if
4797 (define_insn_and_split "ctz<mode>2"
4798 [(set (match_operand:GPI 0 "register_operand" "=r")
4799 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4805 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4806 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4810 (define_insn "*and<mode>_compare0"
4811 [(set (reg:CC_NZ CC_REGNUM)
4813 (match_operand:SHORT 0 "register_operand" "r")
4816 "tst\\t%<w>0, <short_mask>"
4817 [(set_attr "type" "alus_imm")]
4820 (define_insn "*ands<GPI:mode>_compare0"
4821 [(set (reg:CC_NZ CC_REGNUM)
4823 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4825 (set (match_operand:GPI 0 "register_operand" "=r")
4826 (zero_extend:GPI (match_dup 1)))]
4828 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4829 [(set_attr "type" "alus_imm")]
4832 (define_insn "*and<mode>3nr_compare0"
4833 [(set (reg:CC_NZ CC_REGNUM)
4835 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4836 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4842 [(set_attr "type" "logics_reg,logics_imm")]
4846 [(set (reg:CC_NZ CC_REGNUM)
4848 (and:GPI (match_operand:GPI 0 "register_operand")
4849 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4851 (clobber (match_operand:SI 2 "register_operand"))]
4853 [(set (match_dup 2) (match_dup 1))
4854 (set (reg:CC_NZ CC_REGNUM)
4856 (and:GPI (match_dup 0)
4861 (define_insn "*and<mode>3nr_compare0_zextract"
4862 [(set (reg:CC_NZ CC_REGNUM)
4864 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4865 (match_operand:GPI 1 "const_int_operand" "n")
4866 (match_operand:GPI 2 "const_int_operand" "n"))
4868 "INTVAL (operands[1]) > 0
4869 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4870 <= GET_MODE_BITSIZE (<MODE>mode))
4871 && aarch64_bitmask_imm (
4872 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4877 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4878 return "tst\\t%<w>0, %1";
4880 [(set_attr "type" "logics_shift_imm")]
4883 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4884 [(set (reg:CC_NZ CC_REGNUM)
4887 (match_operand:GPI 0 "register_operand" "r")
4888 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4889 (match_operand:GPI 2 "register_operand" "r"))
4892 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4893 [(set_attr "type" "logics_shift_imm")]
4897 [(set (reg:CC_NZ CC_REGNUM)
4900 (match_operand:GPI 0 "register_operand")
4901 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4902 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4904 (clobber (match_operand:SI 3 "register_operand"))]
4906 [(set (match_dup 3) (match_dup 2))
4907 (set (reg:CC_NZ CC_REGNUM)
4916 ;; -------------------------------------------------------------------
4918 ;; -------------------------------------------------------------------
4920 (define_expand "<optab><mode>3"
4921 [(set (match_operand:GPI 0 "register_operand")
4922 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4923 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4926 if (CONST_INT_P (operands[2]))
4928 operands[2] = GEN_INT (INTVAL (operands[2])
4929 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4931 if (operands[2] == const0_rtx)
4933 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4940 (define_expand "ashl<mode>3"
4941 [(set (match_operand:SHORT 0 "register_operand")
4942 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4943 (match_operand:QI 2 "const_int_operand")))]
4946 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4948 if (operands[2] == const0_rtx)
4950 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4956 (define_expand "rotr<mode>3"
4957 [(set (match_operand:GPI 0 "register_operand")
4958 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4959 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4962 if (CONST_INT_P (operands[2]))
4964 operands[2] = GEN_INT (INTVAL (operands[2])
4965 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4967 if (operands[2] == const0_rtx)
4969 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4976 (define_expand "rotl<mode>3"
4977 [(set (match_operand:GPI 0 "register_operand")
4978 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4979 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4982 /* (SZ - cnt) % SZ == -cnt % SZ */
4983 if (CONST_INT_P (operands[2]))
4985 operands[2] = GEN_INT ((-INTVAL (operands[2]))
4986 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4987 if (operands[2] == const0_rtx)
4989 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4994 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4999 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5000 ;; they truncate the shift/rotate amount by the size of the registers they
5001 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
5002 ;; such redundant masking instructions. GCC can do that automatically when
5003 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5004 ;; because some of the SISD shift alternatives don't perform this truncations.
5005 ;; So this pattern exists to catch such cases.
5007 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5008 [(set (match_operand:GPI 0 "register_operand" "=r")
5010 (match_operand:GPI 1 "register_operand" "r")
5011 (match_operator 4 "subreg_lowpart_operator"
5012 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5013 (match_operand 3 "const_int_operand" "n"))])))]
5014 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5015 "<shift>\t%<w>0, %<w>1, %<w>2"
5016 [(set_attr "type" "shift_reg")]
5019 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5020 [(set (match_operand:GPI 0 "register_operand" "=&r")
5022 (match_operand:GPI 1 "register_operand" "r")
5023 (match_operator 4 "subreg_lowpart_operator"
5024 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5025 (match_operand 3 "const_int_operand" "n")))])))]
5026 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5031 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5032 : lowpart_subreg (SImode, operands[0], <MODE>mode));
5033 emit_insn (gen_negsi2 (tmp, operands[2]));
5035 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5036 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5037 SUBREG_BYTE (operands[4]));
5038 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5043 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5044 [(set (match_operand:GPI 0 "register_operand" "=&r")
5046 (match_operand:GPI 1 "register_operand" "r")
5047 (minus:QI (match_operand 2 "const_int_operand" "n")
5048 (match_operator 5 "subreg_lowpart_operator"
5049 [(and:SI (match_operand:SI 3 "register_operand" "r")
5050 (match_operand 4 "const_int_operand" "n"))]))))]
5051 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5052 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5057 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5060 emit_insn (gen_negsi2 (tmp, operands[3]));
5062 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5063 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5064 SUBREG_BYTE (operands[5]));
5066 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5071 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5072 [(set (match_operand:DI 0 "register_operand" "=r")
5074 (match_operand:DI 1 "register_operand" "r")
5075 (match_operator 4 "subreg_lowpart_operator"
5076 [(and:SI (match_operand:SI 2 "register_operand" "r")
5077 (match_operand 3 "const_int_operand" "n"))])))]
5078 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5081 xop[0] = operands[0];
5082 xop[1] = operands[1];
5083 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5084 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5087 [(set_attr "type" "shift_reg")]
5090 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5091 [(set (match_operand:GPI 0 "register_operand" "=&r")
5093 (match_operand:GPI 1 "register_operand" "r")
5094 (minus:QI (match_operand 2 "const_int_operand" "n")
5095 (match_operand:QI 3 "register_operand" "r"))))]
5096 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5101 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5103 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5104 : gen_lowpart (SImode, operands[0]));
5106 emit_insn (gen_negsi2 (tmp, subreg_tmp));
5108 rtx and_op = gen_rtx_AND (SImode, tmp,
5109 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5111 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5113 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5116 [(set_attr "length" "8")]
5119 ;; Logical left shift using SISD or Integer instruction
5120 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5121 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5123 (match_operand:GPI 1 "register_operand" "r,r,w,w")
5124 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5127 lsl\t%<w>0, %<w>1, %2
5128 lsl\t%<w>0, %<w>1, %<w>2
5129 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5130 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5131 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5132 (set_attr "arch" "*,*,simd,simd")]
5135 ;; Logical right shift using SISD or Integer instruction
5136 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5137 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5139 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5140 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5141 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5144 lsr\t%<w>0, %<w>1, %2
5145 lsr\t%<w>0, %<w>1, %<w>2
5146 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5149 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5150 (set_attr "arch" "*,*,simd,simd,simd")]
5154 [(set (match_operand:DI 0 "aarch64_simd_register")
5156 (match_operand:DI 1 "aarch64_simd_register")
5157 (match_operand:QI 2 "aarch64_simd_register")))]
5158 "TARGET_SIMD && reload_completed"
5160 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5162 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5164 operands[3] = gen_lowpart (QImode, operands[0]);
5169 [(set (match_operand:SI 0 "aarch64_simd_register")
5171 (match_operand:SI 1 "aarch64_simd_register")
5172 (match_operand:QI 2 "aarch64_simd_register")))]
5173 "TARGET_SIMD && reload_completed"
5175 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5177 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5179 operands[3] = gen_lowpart (QImode, operands[0]);
5183 ;; Arithmetic right shift using SISD or Integer instruction
5184 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5185 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5187 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5188 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5189 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5192 asr\t%<w>0, %<w>1, %2
5193 asr\t%<w>0, %<w>1, %<w>2
5194 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5197 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5198 (set_attr "arch" "*,*,simd,simd,simd")]
5202 [(set (match_operand:DI 0 "aarch64_simd_register")
5204 (match_operand:DI 1 "aarch64_simd_register")
5205 (match_operand:QI 2 "aarch64_simd_register")))]
5206 "TARGET_SIMD && reload_completed"
5208 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5210 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5212 operands[3] = gen_lowpart (QImode, operands[0]);
5217 [(set (match_operand:SI 0 "aarch64_simd_register")
5219 (match_operand:SI 1 "aarch64_simd_register")
5220 (match_operand:QI 2 "aarch64_simd_register")))]
5221 "TARGET_SIMD && reload_completed"
5223 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5225 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5227 operands[3] = gen_lowpart (QImode, operands[0]);
5231 (define_insn "*aarch64_sisd_ushl"
5232 [(set (match_operand:DI 0 "register_operand" "=w")
5233 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5234 (match_operand:QI 2 "register_operand" "w")]
5237 "ushl\t%d0, %d1, %d2"
5238 [(set_attr "type" "neon_shift_reg")]
5241 (define_insn "*aarch64_ushl_2s"
5242 [(set (match_operand:SI 0 "register_operand" "=w")
5243 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5244 (match_operand:QI 2 "register_operand" "w")]
5247 "ushl\t%0.2s, %1.2s, %2.2s"
5248 [(set_attr "type" "neon_shift_reg")]
5251 (define_insn "*aarch64_sisd_sshl"
5252 [(set (match_operand:DI 0 "register_operand" "=w")
5253 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5254 (match_operand:QI 2 "register_operand" "w")]
5257 "sshl\t%d0, %d1, %d2"
5258 [(set_attr "type" "neon_shift_reg")]
5261 (define_insn "*aarch64_sshl_2s"
5262 [(set (match_operand:SI 0 "register_operand" "=w")
5263 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5264 (match_operand:QI 2 "register_operand" "w")]
5267 "sshl\t%0.2s, %1.2s, %2.2s"
5268 [(set_attr "type" "neon_shift_reg")]
5271 (define_insn "*aarch64_sisd_neg_qi"
5272 [(set (match_operand:QI 0 "register_operand" "=w")
5273 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5277 [(set_attr "type" "neon_neg")]
5281 (define_insn "*ror<mode>3_insn"
5282 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5284 (match_operand:GPI 1 "register_operand" "r,r")
5285 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5288 ror\\t%<w>0, %<w>1, %2
5289 ror\\t%<w>0, %<w>1, %<w>2"
5290 [(set_attr "type" "rotate_imm,shift_reg")]
5293 ;; zero_extend version of above
5294 (define_insn "*<optab>si3_insn_uxtw"
5295 [(set (match_operand:DI 0 "register_operand" "=r,r")
5296 (zero_extend:DI (SHIFT:SI
5297 (match_operand:SI 1 "register_operand" "r,r")
5298 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5301 <shift>\\t%w0, %w1, %2
5302 <shift>\\t%w0, %w1, %w2"
5303 [(set_attr "type" "bfx,shift_reg")]
5306 (define_insn "*<optab><mode>3_insn"
5307 [(set (match_operand:SHORT 0 "register_operand" "=r")
5308 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5309 (match_operand 2 "const_int_operand" "n")))]
5310 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5312 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5313 return "<bfshift>\t%w0, %w1, %2, %3";
5315 [(set_attr "type" "bfx")]
5318 (define_insn "*extr<mode>5_insn"
5319 [(set (match_operand:GPI 0 "register_operand" "=r")
5320 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5321 (match_operand 3 "const_int_operand" "n"))
5322 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5323 (match_operand 4 "const_int_operand" "n"))))]
5324 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5325 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5326 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5327 [(set_attr "type" "rotate_imm")]
5330 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5331 ;; so we have to match both orderings.
5332 (define_insn "*extr<mode>5_insn_alt"
5333 [(set (match_operand:GPI 0 "register_operand" "=r")
5334 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5335 (match_operand 4 "const_int_operand" "n"))
5336 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5337 (match_operand 3 "const_int_operand" "n"))))]
5338 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5339 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5340 == GET_MODE_BITSIZE (<MODE>mode))"
5341 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5342 [(set_attr "type" "rotate_imm")]
5345 ;; zero_extend version of the above
5346 (define_insn "*extrsi5_insn_uxtw"
5347 [(set (match_operand:DI 0 "register_operand" "=r")
5349 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5350 (match_operand 3 "const_int_operand" "n"))
5351 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5352 (match_operand 4 "const_int_operand" "n")))))]
5353 "UINTVAL (operands[3]) < 32 &&
5354 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5355 "extr\\t%w0, %w1, %w2, %4"
5356 [(set_attr "type" "rotate_imm")]
5359 (define_insn "*extrsi5_insn_uxtw_alt"
5360 [(set (match_operand:DI 0 "register_operand" "=r")
5362 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5363 (match_operand 4 "const_int_operand" "n"))
5364 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5365 (match_operand 3 "const_int_operand" "n")))))]
5366 "UINTVAL (operands[3]) < 32 &&
5367 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5368 "extr\\t%w0, %w1, %w2, %4"
5369 [(set_attr "type" "rotate_imm")]
5372 (define_insn "*ror<mode>3_insn"
5373 [(set (match_operand:GPI 0 "register_operand" "=r")
5374 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5375 (match_operand 2 "const_int_operand" "n")))]
5376 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5378 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5379 return "ror\\t%<w>0, %<w>1, %3";
5381 [(set_attr "type" "rotate_imm")]
5384 ;; zero_extend version of the above
5385 (define_insn "*rorsi3_insn_uxtw"
5386 [(set (match_operand:DI 0 "register_operand" "=r")
5388 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5389 (match_operand 2 "const_int_operand" "n"))))]
5390 "UINTVAL (operands[2]) < 32"
5392 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5393 return "ror\\t%w0, %w1, %3";
5395 [(set_attr "type" "rotate_imm")]
5398 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5399 [(set (match_operand:GPI 0 "register_operand" "=r")
5401 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5402 (match_operand 2 "const_int_operand" "n"))))]
5403 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5405 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5406 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5408 [(set_attr "type" "bfx")]
5411 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5412 [(set (match_operand:GPI 0 "register_operand" "=r")
5414 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5415 (match_operand 2 "const_int_operand" "n"))))]
5416 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5418 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5419 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5421 [(set_attr "type" "bfx")]
5424 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5425 [(set (match_operand:GPI 0 "register_operand" "=r")
5427 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5428 (match_operand 2 "const_int_operand" "n"))))]
5429 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5431 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5432 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5434 [(set_attr "type" "bfx")]
5437 ;; -------------------------------------------------------------------
5439 ;; -------------------------------------------------------------------
5441 (define_expand "<optab>"
5442 [(set (match_operand:DI 0 "register_operand")
5443 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5445 "aarch64_simd_shift_imm_offset_di")
5446 (match_operand 3 "aarch64_simd_shift_imm_di")))]
5449 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5450 1, GET_MODE_BITSIZE (DImode) - 1))
5456 (define_insn "*<optab><mode>"
5457 [(set (match_operand:GPI 0 "register_operand" "=r")
5458 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5460 "aarch64_simd_shift_imm_offset_<mode>" "n")
5462 "aarch64_simd_shift_imm_<mode>" "n")))]
5463 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5464 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5465 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5466 [(set_attr "type" "bfx")]
5469 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5470 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5472 [(set (match_operand:DI 0 "register_operand")
5473 (zero_extract:DI (match_operand:DI 1 "register_operand")
5475 "aarch64_simd_shift_imm_offset_di")
5477 "aarch64_simd_shift_imm_di")))]
5478 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5479 GET_MODE_BITSIZE (DImode) - 1)
5480 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5481 == GET_MODE_BITSIZE (SImode)"
5483 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5485 operands[4] = gen_lowpart (SImode, operands[1]);
5489 ;; Bitfield Insert (insv)
5490 (define_expand "insv<mode>"
5491 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5492 (match_operand 1 "const_int_operand")
5493 (match_operand 2 "const_int_operand"))
5494 (match_operand:GPI 3 "general_operand"))]
5497 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5498 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5499 rtx value = operands[3];
5501 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5504 if (CONST_INT_P (value))
5506 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5508 /* Prefer AND/OR for inserting all zeros or all ones. */
5509 if ((UINTVAL (value) & mask) == 0
5510 || (UINTVAL (value) & mask) == mask)
5513 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5514 if (width == 16 && (pos % 16) == 0)
5517 operands[3] = force_reg (<MODE>mode, value);
5520 (define_insn "*insv_reg<mode>"
5521 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5522 (match_operand 1 "const_int_operand" "n")
5523 (match_operand 2 "const_int_operand" "n"))
5524 (match_operand:GPI 3 "register_operand" "r"))]
5525 "!(UINTVAL (operands[1]) == 0
5526 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5527 > GET_MODE_BITSIZE (<MODE>mode)))"
5528 "bfi\\t%<w>0, %<w>3, %2, %1"
5529 [(set_attr "type" "bfm")]
5532 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5533 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5534 (match_operand 1 "const_int_operand" "n")
5535 (match_operand 2 "const_int_operand" "n"))
5536 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5537 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5538 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5539 [(set_attr "type" "bfm")]
5542 ;; Match a bfi instruction where the shift of OP3 means that we are
5543 ;; actually copying the least significant bits of OP3 into OP0 by way
5544 ;; of the AND masks and the IOR instruction. A similar instruction
5545 ;; with the two parts of the IOR swapped around was never triggered
5546 ;; in a bootstrap build and test of GCC so it was not included.
5548 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5549 [(set (match_operand:GPI 0 "register_operand" "=r")
5550 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5551 (match_operand:GPI 2 "const_int_operand" "n"))
5552 (and:GPI (ashift:GPI
5553 (match_operand:GPI 3 "register_operand" "r")
5554 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5555 (match_operand:GPI 5 "const_int_operand" "n"))))]
5556 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5557 UINTVAL (operands[4]),
5558 UINTVAL(operands[5]))"
5559 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5560 [(set_attr "type" "bfm")]
5563 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5564 [(set (match_operand:GPI 0 "register_operand" "=r")
5565 (ior:GPI (and:GPI (ashift:GPI
5566 (match_operand:GPI 1 "register_operand" "r")
5567 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5568 (match_operand:GPI 3 "const_int_operand" "n"))
5569 (and:GPI (match_operand:GPI 4 "register_operand" "0")
5570 (match_operand:GPI 5 "const_int_operand" "n"))))]
5571 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5572 UINTVAL (operands[2]),
5573 UINTVAL(operands[3]))"
5574 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5575 [(set_attr "type" "bfm")]
5578 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5579 ;; the shift is large enough to remove the need for an AND instruction.
5581 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5582 [(set (match_operand:GPI 0 "register_operand" "=r")
5583 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5584 (match_operand:GPI 2 "const_int_operand" "n"))
5586 (match_operand:GPI 3 "register_operand" "r")
5587 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5588 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5589 UINTVAL (operands[4]),
5590 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5592 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5593 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5595 [(set_attr "type" "bfm")]
5598 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5599 [(set (match_operand:GPI 0 "register_operand" "=r")
5600 (ior:GPI (ashift:GPI
5601 (match_operand:GPI 1 "register_operand" "r")
5602 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5603 (and:GPI (match_operand:GPI 3 "register_operand" "0")
5604 (match_operand:GPI 4 "const_int_operand" "n"))))]
5605 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5606 UINTVAL (operands[2]),
5607 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5609 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5610 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5612 [(set_attr "type" "bfm")]
5615 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5616 ;; copying the least significant bits of OP3 to OP0. We need two versions
5617 ;; of the instruction to handle different checks on the constant values.
5619 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5620 [(set (match_operand:GPI 0 "register_operand" "=r")
5621 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5622 (match_operand:GPI 2 "const_int_operand" "n"))
5623 (and:GPI (match_operand:GPI 3 "register_operand" "r")
5624 (match_operand:GPI 4 "const_int_operand" "n"))))]
5625 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5626 UINTVAL (operands[4]))"
5627 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5628 [(set_attr "type" "bfm")]
5631 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5632 [(set (match_operand:GPI 0 "register_operand" "=r")
5633 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5634 (match_operand:GPI 4 "const_int_operand" "n"))
5635 (and:GPI (match_operand:GPI 1 "register_operand" "0")
5636 (match_operand:GPI 2 "const_int_operand" "n"))))]
5637 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5638 UINTVAL (operands[4]))"
5639 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5640 [(set_attr "type" "bfm")]
5643 (define_insn "*extr_insv_lower_reg<mode>"
5644 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5645 (match_operand 1 "const_int_operand" "n")
5647 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5649 (match_operand 3 "const_int_operand" "n")))]
5650 "!(UINTVAL (operands[1]) == 0
5651 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5652 > GET_MODE_BITSIZE (<MODE>mode)))"
5653 "bfxil\\t%<w>0, %<w>2, %3, %1"
5654 [(set_attr "type" "bfm")]
5657 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5658 [(set (match_operand:GPI 0 "register_operand" "=r")
5659 (ashift:GPI (ANY_EXTEND:GPI
5660 (match_operand:ALLX 1 "register_operand" "r"))
5661 (match_operand 2 "const_int_operand" "n")))]
5662 "UINTVAL (operands[2]) < <GPI:sizen>"
5664 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5665 ? GEN_INT (<ALLX:sizen>)
5666 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5667 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5669 [(set_attr "type" "bfx")]
5672 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5674 (define_insn "*andim_ashift<mode>_bfiz"
5675 [(set (match_operand:GPI 0 "register_operand" "=r")
5676 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5677 (match_operand 2 "const_int_operand" "n"))
5678 (match_operand 3 "const_int_operand" "n")))]
5679 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5680 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5681 [(set_attr "type" "bfx")]
5684 ;; Match sbfiz pattern in a shift left + shift right operation.
5686 (define_insn "*ashift<mode>_extv_bfiz"
5687 [(set (match_operand:GPI 0 "register_operand" "=r")
5688 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5689 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5691 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5692 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5693 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5694 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5695 [(set_attr "type" "bfx")]
5698 ;; When the bit position and width of the equivalent extraction add up to 32
5699 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5700 ;; zero-extension of the X-reg.
5702 [(set (match_operand:DI 0 "register_operand")
5703 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5704 (match_operand 2 "const_int_operand"))
5705 (match_operand 3 "const_int_operand")))]
5706 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5707 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5708 == GET_MODE_BITSIZE (SImode)"
5710 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5712 operands[4] = gen_lowpart (SImode, operands[1]);
5716 (define_insn "bswap<mode>2"
5717 [(set (match_operand:GPI 0 "register_operand" "=r")
5718 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5720 "rev\\t%<w>0, %<w>1"
5721 [(set_attr "type" "rev")]
5724 (define_insn "bswaphi2"
5725 [(set (match_operand:HI 0 "register_operand" "=r")
5726 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5729 [(set_attr "type" "rev")]
5732 (define_insn "*aarch64_bfxil<mode>"
5733 [(set (match_operand:GPI 0 "register_operand" "=r,r")
5734 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5735 (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5736 (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5737 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5738 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5739 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5740 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5742 switch (which_alternative)
5745 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5746 return "bfxil\\t%<w>0, %<w>1, 0, %3";
5748 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5749 return "bfxil\\t%<w>0, %<w>2, 0, %3";
5754 [(set_attr "type" "bfm")]
5757 ; Zero-extended version of above (aarch64_bfxil)
5758 (define_insn "*aarch64_bfxilsi_uxtw"
5759 [(set (match_operand:DI 0 "register_operand" "=r,r")
5760 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5762 (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5763 (and:SI (match_operand:SI 2 "register_operand" "0,r")
5764 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5765 "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5766 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5767 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5769 switch (which_alternative)
5772 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5773 return "bfxil\\t%0, %1, 0, %3";
5775 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5776 return "bfxil\\t%0, %2, 0, %3";
5781 [(set_attr "type" "bfm")]
5784 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5785 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5786 ;; each valid permutation.
5788 (define_insn "rev16<mode>2"
5789 [(set (match_operand:GPI 0 "register_operand" "=r")
5790 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5792 (match_operand:GPI 3 "const_int_operand" "n"))
5793 (and:GPI (lshiftrt:GPI (match_dup 1)
5795 (match_operand:GPI 2 "const_int_operand" "n"))))]
5796 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5797 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5798 "rev16\\t%<w>0, %<w>1"
5799 [(set_attr "type" "rev")]
5802 (define_insn "rev16<mode>2_alt"
5803 [(set (match_operand:GPI 0 "register_operand" "=r")
5804 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5806 (match_operand:GPI 2 "const_int_operand" "n"))
5807 (and:GPI (ashift:GPI (match_dup 1)
5809 (match_operand:GPI 3 "const_int_operand" "n"))))]
5810 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5811 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5812 "rev16\\t%<w>0, %<w>1"
5813 [(set_attr "type" "rev")]
5816 ;; zero_extend version of above
5817 (define_insn "*bswapsi2_uxtw"
5818 [(set (match_operand:DI 0 "register_operand" "=r")
5819 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5822 [(set_attr "type" "rev")]
5825 ;; -------------------------------------------------------------------
5826 ;; Floating-point intrinsics
5827 ;; -------------------------------------------------------------------
5829 ;; frint floating-point round to integral standard patterns.
5830 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5832 (define_insn "<frint_pattern><mode>2"
5833 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5834 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5837 "frint<frint_suffix>\\t%<s>0, %<s>1"
5838 [(set_attr "type" "f_rint<stype>")]
5841 ;; frcvt floating-point round to integer and convert standard patterns.
5842 ;; Expands to lbtrunc, lceil, lfloor, lround.
5843 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5844 [(set (match_operand:GPI 0 "register_operand" "=r")
5846 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5849 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5850 [(set_attr "type" "f_cvtf2i")]
5853 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5854 [(set (match_operand:GPI 0 "register_operand" "=r")
5857 (match_operand:GPF 1 "register_operand" "w")
5858 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5860 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5861 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5863 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5865 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5866 output_asm_insn (buf, operands);
5869 [(set_attr "type" "f_cvtf2i")]
5872 ;; fma - expand fma into patterns with the accumulator operand first since
5873 ;; reusing the accumulator results in better register allocation.
5874 ;; The register allocator considers copy preferences in operand order,
5875 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5877 (define_expand "fma<mode>4"
5878 [(set (match_operand:GPF_F16 0 "register_operand")
5879 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5880 (match_operand:GPF_F16 2 "register_operand")
5881 (match_operand:GPF_F16 3 "register_operand")))]
5885 (define_insn "*aarch64_fma<mode>4"
5886 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5887 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5888 (match_operand:GPF_F16 3 "register_operand" "w")
5889 (match_operand:GPF_F16 1 "register_operand" "w")))]
5891 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5892 [(set_attr "type" "fmac<stype>")]
5895 (define_expand "fnma<mode>4"
5896 [(set (match_operand:GPF_F16 0 "register_operand")
5898 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5899 (match_operand:GPF_F16 2 "register_operand")
5900 (match_operand:GPF_F16 3 "register_operand")))]
5904 (define_insn "*aarch64_fnma<mode>4"
5905 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5907 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5908 (match_operand:GPF_F16 3 "register_operand" "w")
5909 (match_operand:GPF_F16 1 "register_operand" "w")))]
5911 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5912 [(set_attr "type" "fmac<stype>")]
5916 (define_expand "fms<mode>4"
5917 [(set (match_operand:GPF 0 "register_operand")
5918 (fma:GPF (match_operand:GPF 1 "register_operand")
5919 (match_operand:GPF 2 "register_operand")
5920 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5924 (define_insn "*aarch64_fms<mode>4"
5925 [(set (match_operand:GPF 0 "register_operand" "=w")
5926 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5927 (match_operand:GPF 3 "register_operand" "w")
5928 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5930 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5931 [(set_attr "type" "fmac<s>")]
5934 (define_expand "fnms<mode>4"
5935 [(set (match_operand:GPF 0 "register_operand")
5936 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5937 (match_operand:GPF 2 "register_operand")
5938 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5942 (define_insn "*aarch64_fnms<mode>4"
5943 [(set (match_operand:GPF 0 "register_operand" "=w")
5944 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5945 (match_operand:GPF 3 "register_operand" "w")
5946 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5948 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5949 [(set_attr "type" "fmac<s>")]
5952 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5953 (define_insn "*aarch64_fnmadd<mode>4"
5954 [(set (match_operand:GPF 0 "register_operand" "=w")
5955 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5956 (match_operand:GPF 3 "register_operand" "w")
5957 (match_operand:GPF 1 "register_operand" "w"))))]
5958 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5959 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5960 [(set_attr "type" "fmac<s>")]
5963 ;; -------------------------------------------------------------------
5964 ;; Floating-point conversions
5965 ;; -------------------------------------------------------------------
5967 (define_insn "extendsfdf2"
5968 [(set (match_operand:DF 0 "register_operand" "=w")
5969 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5972 [(set_attr "type" "f_cvt")]
5975 (define_insn "extendhfsf2"
5976 [(set (match_operand:SF 0 "register_operand" "=w")
5977 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5980 [(set_attr "type" "f_cvt")]
5983 (define_insn "extendhfdf2"
5984 [(set (match_operand:DF 0 "register_operand" "=w")
5985 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5988 [(set_attr "type" "f_cvt")]
5991 (define_insn "truncdfsf2"
5992 [(set (match_operand:SF 0 "register_operand" "=w")
5993 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5996 [(set_attr "type" "f_cvt")]
5999 (define_insn "truncsfhf2"
6000 [(set (match_operand:HF 0 "register_operand" "=w")
6001 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6004 [(set_attr "type" "f_cvt")]
6007 (define_insn "truncdfhf2"
6008 [(set (match_operand:HF 0 "register_operand" "=w")
6009 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6012 [(set_attr "type" "f_cvt")]
6015 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6016 ;; and making r = w more expensive
6018 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6019 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6020 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6023 fcvtz<su>\t%<s>0, %<s>1
6024 fcvtz<su>\t%<w>0, %<s>1"
6025 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6028 ;; Convert HF -> SI or DI
6030 (define_insn "<optab>_trunchf<GPI:mode>2"
6031 [(set (match_operand:GPI 0 "register_operand" "=r")
6032 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6034 "fcvtz<su>\t%<w>0, %h1"
6035 [(set_attr "type" "f_cvtf2i")]
6038 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6039 ;; input in a fp register and output in a integer register
6041 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6042 [(set (match_operand:GPI 0 "register_operand" "=r")
6043 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6045 "fcvtz<su>\t%<w>0, %<fpw>1"
6046 [(set_attr "type" "f_cvtf2i")]
6049 (define_insn "*fix_to_zero_extend<mode>di2"
6050 [(set (match_operand:DI 0 "register_operand" "=r")
6053 (match_operand:GPF 1 "register_operand" "w"))))]
6055 "fcvtzu\t%w0, %<s>1"
6056 [(set_attr "type" "f_cvtf2i")]
6059 ;; Equal width integer to fp and multiply combine.
6060 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6061 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6062 (mult:GPF (FLOATUORS:GPF
6063 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6064 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6067 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6068 switch (which_alternative)
6071 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6073 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6078 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6079 (set_attr "arch" "simd,fp")]
6082 ;; Unequal width integer to fp and multiply combine.
6083 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6084 [(set (match_operand:GPF 0 "register_operand" "=w")
6085 (mult:GPF (FLOATUORS:GPF
6086 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6087 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6090 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6091 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6093 [(set_attr "type" "f_cvti2f")]
6096 ;; Equal width integer to fp conversion.
6097 (define_insn "<optab><fcvt_target><GPF:mode>2"
6098 [(set (match_operand:GPF 0 "register_operand" "=w,w")
6099 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6102 <su_optab>cvtf\t%<GPF:s>0, %<s>1
6103 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6104 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6105 (set_attr "arch" "simd,fp")]
6108 ;; Unequal width integer to fp conversions.
6109 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6110 [(set (match_operand:GPF 0 "register_operand" "=w")
6111 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6113 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6114 [(set_attr "type" "f_cvti2f")]
6117 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6118 ;; midend will arrange for an SImode conversion to HFmode to first go
6119 ;; through DFmode, then to HFmode. But first it will try converting
6120 ;; to DImode then down, which would match our DImode pattern below and
6121 ;; give very poor code-generation. So, we must provide our own emulation
6122 ;; of the mid-end logic.
6124 (define_insn "aarch64_fp16_<optab><mode>hf2"
6125 [(set (match_operand:HF 0 "register_operand" "=w")
6126 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6128 "<su_optab>cvtf\t%h0, %<w>1"
6129 [(set_attr "type" "f_cvti2f")]
6132 (define_expand "<optab>sihf2"
6133 [(set (match_operand:HF 0 "register_operand")
6134 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6137 if (TARGET_FP_F16INST)
6138 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6141 rtx convert_target = gen_reg_rtx (DFmode);
6142 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6143 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6149 ;; For DImode there is no wide enough floating-point mode that we
6150 ;; can convert through natively (TFmode would work, but requires a library
6151 ;; call). However, we know that any value >= 65504 will be rounded
6152 ;; to infinity on conversion. This is well within the range of SImode, so
6154 ;; Saturate to SImode.
6155 ;; Convert from that to DFmode
6156 ;; Convert from that to HFmode (phew!).
6157 ;; Note that the saturation to SImode requires the SIMD extensions. If
6158 ;; we ever need to provide this pattern where the SIMD extensions are not
6159 ;; available, we would need a different approach.
6161 (define_expand "<optab>dihf2"
6162 [(set (match_operand:HF 0 "register_operand")
6163 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6164 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6166 if (TARGET_FP_F16INST)
6167 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6170 rtx sat_target = gen_reg_rtx (SImode);
6171 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6172 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6179 ;; Convert between fixed-point and floating-point (scalar modes)
6181 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6182 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6183 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6184 (match_operand:SI 2 "immediate_operand" "i, i")]
6188 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6189 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6190 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6191 (set_attr "arch" "fp,simd")]
6194 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6195 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6196 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6197 (match_operand:SI 2 "immediate_operand" "i, i")]
6201 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6202 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6203 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6204 (set_attr "arch" "fp,simd")]
6207 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6208 [(set (match_operand:GPI 0 "register_operand" "=r")
6209 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6210 (match_operand:SI 2 "immediate_operand" "i")]
6213 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6214 [(set_attr "type" "f_cvtf2i")]
6217 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6218 [(set (match_operand:HF 0 "register_operand" "=w")
6219 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6220 (match_operand:SI 2 "immediate_operand" "i")]
6223 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6224 [(set_attr "type" "f_cvti2f")]
6227 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6228 [(set (match_operand:HI 0 "register_operand" "=w")
6229 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6230 (match_operand:SI 2 "immediate_operand" "i")]
6233 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6234 [(set_attr "type" "neon_fp_to_int_s")]
6237 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6238 [(set (match_operand:HF 0 "register_operand" "=w")
6239 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6240 (match_operand:SI 2 "immediate_operand" "i")]
6243 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6244 [(set_attr "type" "neon_int_to_fp_s")]
6247 ;; -------------------------------------------------------------------
6248 ;; Floating-point arithmetic
6249 ;; -------------------------------------------------------------------
6251 (define_insn "add<mode>3"
6252 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6254 (match_operand:GPF_F16 1 "register_operand" "w")
6255 (match_operand:GPF_F16 2 "register_operand" "w")))]
6257 "fadd\\t%<s>0, %<s>1, %<s>2"
6258 [(set_attr "type" "fadd<stype>")]
6261 (define_insn "sub<mode>3"
6262 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6264 (match_operand:GPF_F16 1 "register_operand" "w")
6265 (match_operand:GPF_F16 2 "register_operand" "w")))]
6267 "fsub\\t%<s>0, %<s>1, %<s>2"
6268 [(set_attr "type" "fadd<stype>")]
6271 (define_insn "mul<mode>3"
6272 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6274 (match_operand:GPF_F16 1 "register_operand" "w")
6275 (match_operand:GPF_F16 2 "register_operand" "w")))]
6277 "fmul\\t%<s>0, %<s>1, %<s>2"
6278 [(set_attr "type" "fmul<stype>")]
6281 (define_insn "*fnmul<mode>3"
6282 [(set (match_operand:GPF 0 "register_operand" "=w")
6284 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6285 (match_operand:GPF 2 "register_operand" "w")))]
6286 "TARGET_FLOAT && !flag_rounding_math"
6287 "fnmul\\t%<s>0, %<s>1, %<s>2"
6288 [(set_attr "type" "fmul<s>")]
6291 (define_insn "*fnmul<mode>3"
6292 [(set (match_operand:GPF 0 "register_operand" "=w")
6294 (match_operand:GPF 1 "register_operand" "w")
6295 (match_operand:GPF 2 "register_operand" "w"))))]
6297 "fnmul\\t%<s>0, %<s>1, %<s>2"
6298 [(set_attr "type" "fmul<s>")]
6301 (define_expand "div<mode>3"
6302 [(set (match_operand:GPF_F16 0 "register_operand")
6303 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6304 (match_operand:GPF_F16 2 "register_operand")))]
6307 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6310 operands[1] = force_reg (<MODE>mode, operands[1]);
6313 (define_insn "*div<mode>3"
6314 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6315 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6316 (match_operand:GPF_F16 2 "register_operand" "w")))]
6318 "fdiv\\t%<s>0, %<s>1, %<s>2"
6319 [(set_attr "type" "fdiv<stype>")]
6322 (define_insn "neg<mode>2"
6323 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6324 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6326 "fneg\\t%<s>0, %<s>1"
6327 [(set_attr "type" "ffarith<stype>")]
6330 (define_expand "sqrt<mode>2"
6331 [(set (match_operand:GPF_F16 0 "register_operand")
6332 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6335 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6339 (define_insn "*sqrt<mode>2"
6340 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6341 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6343 "fsqrt\\t%<s>0, %<s>1"
6344 [(set_attr "type" "fsqrt<stype>")]
6347 (define_insn "abs<mode>2"
6348 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6349 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6351 "fabs\\t%<s>0, %<s>1"
6352 [(set_attr "type" "ffarith<stype>")]
6355 ;; Given that smax/smin do not specify the result when either input is NaN,
6356 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6359 (define_insn "smax<mode>3"
6360 [(set (match_operand:GPF 0 "register_operand" "=w")
6361 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6362 (match_operand:GPF 2 "register_operand" "w")))]
6364 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6365 [(set_attr "type" "f_minmax<s>")]
6368 (define_insn "smin<mode>3"
6369 [(set (match_operand:GPF 0 "register_operand" "=w")
6370 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6371 (match_operand:GPF 2 "register_operand" "w")))]
6373 "fminnm\\t%<s>0, %<s>1, %<s>2"
6374 [(set_attr "type" "f_minmax<s>")]
6377 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6378 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6379 ;; which implement the IEEE fmax ()/fmin () functions.
6380 (define_insn "<maxmin_uns><mode>3"
6381 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6382 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6383 (match_operand:GPF_F16 2 "register_operand" "w")]
6386 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6387 [(set_attr "type" "f_minmax<stype>")]
6390 (define_expand "lrint<GPF:mode><GPI:mode>2"
6391 [(match_operand:GPI 0 "register_operand")
6392 (match_operand:GPF 1 "register_operand")]
6394 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6395 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6397 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6398 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6399 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6404 ;; For copysign (x, y), we want to generate:
6406 ;; LDR d2, #(1 << 63)
6407 ;; BSL v2.8b, [y], [x]
6409 ;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because
6410 ;; we expect these operations to nearly always operate on
6411 ;; floating-point values, we do not want the operation to be
6412 ;; simplified into a bit-field insert operation that operates on the
6413 ;; integer side, since typically that would involve three inter-bank
6414 ;; register copies. As we do not expect copysign to be followed by
6415 ;; other logical operations on the result, it seems preferable to keep
6416 ;; this as an unspec operation, rather than exposing the underlying
6417 ;; logic to the compiler.
6419 (define_expand "copysign<GPF:mode>3"
6420 [(match_operand:GPF 0 "register_operand")
6421 (match_operand:GPF 1 "register_operand")
6422 (match_operand:GPF 2 "register_operand")]
6423 "TARGET_FLOAT && TARGET_SIMD"
6425 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6426 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6427 << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6428 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6434 (define_insn "copysign<GPF:mode>3_insn"
6435 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6436 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6437 (match_operand:GPF 2 "register_operand" "w,w,0,0")
6438 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6440 "TARGET_FLOAT && TARGET_SIMD"
6442 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6443 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6444 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6445 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6446 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6450 ;; For xorsign (x, y), we want to generate:
6453 ;; AND v3.8B, v1.8B, v2.8B
6454 ;; EOR v0.8B, v0.8B, v3.8B
6457 (define_expand "xorsign<mode>3"
6458 [(match_operand:GPF 0 "register_operand")
6459 (match_operand:GPF 1 "register_operand")
6460 (match_operand:GPF 2 "register_operand")]
6461 "TARGET_FLOAT && TARGET_SIMD"
6464 machine_mode imode = <V_INT_EQUIV>mode;
6465 rtx mask = gen_reg_rtx (imode);
6466 rtx op1x = gen_reg_rtx (imode);
6467 rtx op2x = gen_reg_rtx (imode);
6469 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6470 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6473 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6474 lowpart_subreg (imode, operands[2],
6476 emit_insn (gen_xor<v_int_equiv>3 (op1x,
6477 lowpart_subreg (imode, operands[1],
6480 emit_move_insn (operands[0],
6481 lowpart_subreg (<MODE>mode, op1x, imode));
6486 ;; -------------------------------------------------------------------
6488 ;; -------------------------------------------------------------------
6489 ;; Reload Scalar Floating point modes from constant pool.
6490 ;; The AArch64 port doesn't have __int128 constant move support.
6491 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6492 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6493 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6494 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6495 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6498 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6499 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6504 ;; Reload Vector modes from constant pool.
6505 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6506 [(set (match_operand:VALL 0 "register_operand" "=w")
6507 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6508 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6511 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6512 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6517 (define_expand "@aarch64_reload_mov<mode>"
6518 [(set (match_operand:TX 0 "register_operand" "=w")
6519 (match_operand:TX 1 "register_operand" "w"))
6520 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6524 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6525 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6526 gen_aarch64_movtilow_tilow (op0, op1);
6527 gen_aarch64_movdi_tihigh (operands[2], op1);
6528 gen_aarch64_movtihigh_di (op0, operands[2]);
6533 ;; The following secondary reload helpers patterns are invoked
6534 ;; after or during reload as we don't want these patterns to start
6535 ;; kicking in during the combiner.
6537 (define_insn "@aarch64_movdi_<mode>low"
6538 [(set (match_operand:DI 0 "register_operand" "=r")
6539 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6540 (const_int 64) (const_int 0)))]
6541 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6543 [(set_attr "type" "f_mrc")
6544 (set_attr "length" "4")
6547 (define_insn "@aarch64_movdi_<mode>high"
6548 [(set (match_operand:DI 0 "register_operand" "=r")
6549 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6550 (const_int 64) (const_int 64)))]
6551 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6552 "fmov\\t%x0, %1.d[1]"
6553 [(set_attr "type" "f_mrc")
6554 (set_attr "length" "4")
6557 (define_insn "@aarch64_mov<mode>high_di"
6558 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6559 (const_int 64) (const_int 64))
6560 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6561 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6562 "fmov\\t%0.d[1], %x1"
6563 [(set_attr "type" "f_mcr")
6564 (set_attr "length" "4")
6567 (define_insn "@aarch64_mov<mode>low_di"
6568 [(set (match_operand:TX 0 "register_operand" "=w")
6569 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6570 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6572 [(set_attr "type" "f_mcr")
6573 (set_attr "length" "4")
6576 (define_insn "aarch64_movtilow_tilow"
6577 [(set (match_operand:TI 0 "register_operand" "=w")
6579 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6580 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6582 [(set_attr "type" "fmov")
6583 (set_attr "length" "4")
6586 ;; There is a deliberate reason why the parameters of high and lo_sum's
6587 ;; don't have modes for ADRP and ADD instructions. This is to allow high
6588 ;; and lo_sum's to be used with the labels defining the jump tables in
6591 (define_expand "add_losym"
6592 [(set (match_operand 0 "register_operand")
6593 (lo_sum (match_operand 1 "register_operand")
6594 (match_operand 2 "aarch64_valid_symref")))]
6597 machine_mode mode = GET_MODE (operands[0]);
6599 emit_insn ((mode == DImode
6601 : gen_add_losym_si) (operands[0],
6607 (define_insn "add_losym_<mode>"
6608 [(set (match_operand:P 0 "register_operand" "=r")
6609 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6610 (match_operand 2 "aarch64_valid_symref" "S")))]
6612 "add\\t%<w>0, %<w>1, :lo12:%c2"
6613 [(set_attr "type" "alu_imm")]
6616 (define_insn "ldr_got_small_<mode>"
6617 [(set (match_operand:PTR 0 "register_operand" "=r")
6618 (unspec:PTR [(mem:PTR (lo_sum:PTR
6619 (match_operand:PTR 1 "register_operand" "r")
6620 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6621 UNSPEC_GOTSMALLPIC))]
6623 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6624 [(set_attr "type" "load_<ldst_sz>")]
6627 (define_insn "ldr_got_small_sidi"
6628 [(set (match_operand:DI 0 "register_operand" "=r")
6630 (unspec:SI [(mem:SI (lo_sum:DI
6631 (match_operand:DI 1 "register_operand" "r")
6632 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6633 UNSPEC_GOTSMALLPIC)))]
6635 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6636 [(set_attr "type" "load_4")]
6639 (define_insn "ldr_got_small_28k_<mode>"
6640 [(set (match_operand:PTR 0 "register_operand" "=r")
6641 (unspec:PTR [(mem:PTR (lo_sum:PTR
6642 (match_operand:PTR 1 "register_operand" "r")
6643 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6644 UNSPEC_GOTSMALLPIC28K))]
6646 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6647 [(set_attr "type" "load_<ldst_sz>")]
6650 (define_insn "ldr_got_small_28k_sidi"
6651 [(set (match_operand:DI 0 "register_operand" "=r")
6653 (unspec:SI [(mem:SI (lo_sum:DI
6654 (match_operand:DI 1 "register_operand" "r")
6655 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6656 UNSPEC_GOTSMALLPIC28K)))]
6658 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6659 [(set_attr "type" "load_4")]
6662 (define_insn "ldr_got_tiny"
6663 [(set (match_operand:DI 0 "register_operand" "=r")
6664 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6665 UNSPEC_GOTTINYPIC))]
6668 [(set_attr "type" "load_8")]
6671 (define_insn "aarch64_load_tp_hard"
6672 [(set (match_operand:DI 0 "register_operand" "=r")
6673 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6675 "mrs\\t%0, tpidr_el0"
6676 [(set_attr "type" "mrs")]
6679 ;; The TLS ABI specifically requires that the compiler does not schedule
6680 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6681 ;; Therefore we treat the stubs as an atomic sequence.
6682 (define_expand "tlsgd_small_<mode>"
6683 [(parallel [(set (match_operand 0 "register_operand")
6684 (call (mem:DI (match_dup 2)) (const_int 1)))
6685 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6686 (clobber (reg:DI LR_REGNUM))])]
6689 operands[2] = aarch64_tls_get_addr ();
6692 (define_insn "*tlsgd_small_<mode>"
6693 [(set (match_operand 0 "register_operand" "")
6694 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6695 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6696 (clobber (reg:DI LR_REGNUM))
6699 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6700 [(set_attr "type" "call")
6701 (set_attr "length" "16")])
6703 (define_insn "tlsie_small_<mode>"
6704 [(set (match_operand:PTR 0 "register_operand" "=r")
6705 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6706 UNSPEC_GOTSMALLTLS))]
6708 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6709 [(set_attr "type" "load_4")
6710 (set_attr "length" "8")]
6713 (define_insn "tlsie_small_sidi"
6714 [(set (match_operand:DI 0 "register_operand" "=r")
6716 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6717 UNSPEC_GOTSMALLTLS)))]
6719 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6720 [(set_attr "type" "load_4")
6721 (set_attr "length" "8")]
6724 (define_insn "tlsie_tiny_<mode>"
6725 [(set (match_operand:PTR 0 "register_operand" "=&r")
6726 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6727 (match_operand:PTR 2 "register_operand" "r")]
6728 UNSPEC_GOTTINYTLS))]
6730 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6731 [(set_attr "type" "multiple")
6732 (set_attr "length" "8")]
6735 (define_insn "tlsie_tiny_sidi"
6736 [(set (match_operand:DI 0 "register_operand" "=&r")
6738 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6739 (match_operand:DI 2 "register_operand" "r")
6741 UNSPEC_GOTTINYTLS)))]
6743 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6744 [(set_attr "type" "multiple")
6745 (set_attr "length" "8")]
6748 (define_insn "tlsle12_<mode>"
6749 [(set (match_operand:P 0 "register_operand" "=r")
6750 (unspec:P [(match_operand:P 1 "register_operand" "r")
6751 (match_operand 2 "aarch64_tls_le_symref" "S")]
6754 "add\\t%<w>0, %<w>1, #%L2";
6755 [(set_attr "type" "alu_sreg")
6756 (set_attr "length" "4")]
6759 (define_insn "tlsle24_<mode>"
6760 [(set (match_operand:P 0 "register_operand" "=r")
6761 (unspec:P [(match_operand:P 1 "register_operand" "r")
6762 (match_operand 2 "aarch64_tls_le_symref" "S")]
6765 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6766 [(set_attr "type" "multiple")
6767 (set_attr "length" "8")]
6770 (define_insn "tlsle32_<mode>"
6771 [(set (match_operand:P 0 "register_operand" "=r")
6772 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6775 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6776 [(set_attr "type" "multiple")
6777 (set_attr "length" "8")]
6780 (define_insn "tlsle48_<mode>"
6781 [(set (match_operand:P 0 "register_operand" "=r")
6782 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6785 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6786 [(set_attr "type" "multiple")
6787 (set_attr "length" "12")]
6790 (define_expand "tlsdesc_small_<mode>"
6791 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6795 emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6797 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6802 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6804 (define_insn "tlsdesc_small_advsimd_<mode>"
6805 [(set (reg:PTR R0_REGNUM)
6806 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6808 (clobber (reg:DI LR_REGNUM))
6809 (clobber (reg:CC CC_REGNUM))
6810 (clobber (match_scratch:DI 1 "=r"))
6811 (use (reg:DI FP_REGNUM))]
6812 "TARGET_TLS_DESC && !TARGET_SVE"
6813 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6814 [(set_attr "type" "call")
6815 (set_attr "length" "16")])
6817 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6818 ;; all vector registers, and clobber all predicate registers, on
6819 ;; top of the usual R0 and LR.
6820 (define_insn "tlsdesc_small_sve_<mode>"
6821 [(set (reg:PTR R0_REGNUM)
6822 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6824 (clobber (reg:DI LR_REGNUM))
6825 (clobber (reg:CC CC_REGNUM))
6826 (clobber_high (reg:TI V0_REGNUM))
6827 (clobber_high (reg:TI V1_REGNUM))
6828 (clobber_high (reg:TI V2_REGNUM))
6829 (clobber_high (reg:TI V3_REGNUM))
6830 (clobber_high (reg:TI V4_REGNUM))
6831 (clobber_high (reg:TI V5_REGNUM))
6832 (clobber_high (reg:TI V6_REGNUM))
6833 (clobber_high (reg:TI V7_REGNUM))
6834 (clobber_high (reg:TI V8_REGNUM))
6835 (clobber_high (reg:TI V9_REGNUM))
6836 (clobber_high (reg:TI V10_REGNUM))
6837 (clobber_high (reg:TI V11_REGNUM))
6838 (clobber_high (reg:TI V12_REGNUM))
6839 (clobber_high (reg:TI V13_REGNUM))
6840 (clobber_high (reg:TI V14_REGNUM))
6841 (clobber_high (reg:TI V15_REGNUM))
6842 (clobber_high (reg:TI V16_REGNUM))
6843 (clobber_high (reg:TI V17_REGNUM))
6844 (clobber_high (reg:TI V18_REGNUM))
6845 (clobber_high (reg:TI V19_REGNUM))
6846 (clobber_high (reg:TI V20_REGNUM))
6847 (clobber_high (reg:TI V21_REGNUM))
6848 (clobber_high (reg:TI V22_REGNUM))
6849 (clobber_high (reg:TI V23_REGNUM))
6850 (clobber_high (reg:TI V24_REGNUM))
6851 (clobber_high (reg:TI V25_REGNUM))
6852 (clobber_high (reg:TI V26_REGNUM))
6853 (clobber_high (reg:TI V27_REGNUM))
6854 (clobber_high (reg:TI V28_REGNUM))
6855 (clobber_high (reg:TI V29_REGNUM))
6856 (clobber_high (reg:TI V30_REGNUM))
6857 (clobber_high (reg:TI V31_REGNUM))
6858 (clobber (reg:VNx2BI P0_REGNUM))
6859 (clobber (reg:VNx2BI P1_REGNUM))
6860 (clobber (reg:VNx2BI P2_REGNUM))
6861 (clobber (reg:VNx2BI P3_REGNUM))
6862 (clobber (reg:VNx2BI P4_REGNUM))
6863 (clobber (reg:VNx2BI P5_REGNUM))
6864 (clobber (reg:VNx2BI P6_REGNUM))
6865 (clobber (reg:VNx2BI P7_REGNUM))
6866 (clobber (reg:VNx2BI P8_REGNUM))
6867 (clobber (reg:VNx2BI P9_REGNUM))
6868 (clobber (reg:VNx2BI P10_REGNUM))
6869 (clobber (reg:VNx2BI P11_REGNUM))
6870 (clobber (reg:VNx2BI P12_REGNUM))
6871 (clobber (reg:VNx2BI P13_REGNUM))
6872 (clobber (reg:VNx2BI P14_REGNUM))
6873 (clobber (reg:VNx2BI P15_REGNUM))
6874 (clobber (match_scratch:DI 1 "=r"))
6875 (use (reg:DI FP_REGNUM))]
6876 "TARGET_TLS_DESC && TARGET_SVE"
6877 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6878 [(set_attr "type" "call")
6879 (set_attr "length" "16")])
6881 (define_insn "stack_tie"
6882 [(set (mem:BLK (scratch))
6883 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6884 (match_operand:DI 1 "register_operand" "rk")]
6888 [(set_attr "length" "0")]
6891 (define_insn "aarch64_fjcvtzs"
6892 [(set (match_operand:SI 0 "register_operand" "=r")
6893 (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
6896 "fjcvtzs\\t%w0, %d1"
6897 [(set_attr "type" "f_cvtf2i")]
6900 ;; Pointer authentication patterns are always provided. In architecture
6901 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6902 ;; This lets the user write portable software which authenticates pointers
6903 ;; when run on something which implements ARMv8.3-A, and which runs
6904 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6907 ;; Signing/Authenticating R30 using SP as the salt.
6909 (define_insn "<pauth_mnem_prefix>sp"
6910 [(set (reg:DI R30_REGNUM)
6911 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6913 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6916 ;; Signing/Authenticating X17 using X16 as the salt.
6918 (define_insn "<pauth_mnem_prefix>1716"
6919 [(set (reg:DI R17_REGNUM)
6920 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6922 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6925 ;; Stripping the signature in R30.
6927 (define_insn "xpaclri"
6928 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6930 "hint\t7 // xpaclri"
6933 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6934 ;; all of memory. This blocks insns from being moved across this point.
6936 (define_insn "blockage"
6937 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6940 [(set_attr "length" "0")
6941 (set_attr "type" "block")]
6944 (define_insn "probe_stack_range"
6945 [(set (match_operand:DI 0 "register_operand" "=rk")
6946 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6947 (match_operand:DI 2 "register_operand" "r")]
6948 UNSPECV_PROBE_STACK_RANGE))]
6951 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6953 [(set_attr "length" "32")]
6956 ;; This instruction is used to generate the stack clash stack adjustment and
6957 ;; probing loop. We can't change the control flow during prologue and epilogue
6958 ;; code generation. So we must emit a volatile unspec and expand it later on.
6960 (define_insn "@probe_sve_stack_clash_<mode>"
6961 [(set (match_operand:P 0 "register_operand" "=rk")
6962 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6963 (match_operand:P 2 "register_operand" "r")
6964 (match_operand:P 3 "const_int_operand" "n")
6965 (match_operand:P 4 "aarch64_plus_immediate" "L")]
6966 UNSPECV_PROBE_STACK_RANGE))]
6969 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6970 operands[3], operands[4]);
6972 [(set_attr "length" "28")]
6975 ;; Named pattern for expanding thread pointer reference.
6976 (define_expand "get_thread_pointerdi"
6977 [(match_operand:DI 0 "register_operand")]
6980 rtx tmp = aarch64_load_tp (operands[0]);
6981 if (tmp != operands[0])
6982 emit_move_insn (operands[0], tmp);
6986 ;; Named patterns for stack smashing protection.
6987 (define_expand "stack_protect_set"
6988 [(match_operand 0 "memory_operand")
6989 (match_operand 1 "memory_operand")]
6992 machine_mode mode = GET_MODE (operands[0]);
6993 if (aarch64_stack_protector_guard != SSP_GLOBAL)
6995 /* Generate access through the system register. */
6996 rtx tmp_reg = gen_reg_rtx (mode);
6999 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7000 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7001 GEN_INT (aarch64_stack_protector_guard_offset)));
7005 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7006 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7007 GEN_INT (aarch64_stack_protector_guard_offset)));
7010 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7013 emit_insn ((mode == DImode
7014 ? gen_stack_protect_set_di
7015 : gen_stack_protect_set_si) (operands[0], operands[1]));
7019 (define_insn "reg_stack_protect_address_<mode>"
7020 [(set (match_operand:PTR 0 "register_operand" "=r")
7021 (unspec:PTR [(const_int 0)]
7022 UNSPEC_SSP_SYSREG))]
7023 "aarch64_stack_protector_guard != SSP_GLOBAL"
7026 snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7027 aarch64_stack_protector_guard_reg_str);
7028 output_asm_insn (buf, operands);
7031 [(set_attr "type" "mrs")])
7033 ;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the
7034 ;; canary value does not live beyond the life of this sequence.
7035 (define_insn "stack_protect_set_<mode>"
7036 [(set (match_operand:PTR 0 "memory_operand" "=m")
7037 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7039 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7041 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7042 [(set_attr "length" "12")
7043 (set_attr "type" "multiple")])
7045 (define_expand "stack_protect_test"
7046 [(match_operand 0 "memory_operand")
7047 (match_operand 1 "memory_operand")
7052 machine_mode mode = GET_MODE (operands[0]);
7054 result = gen_reg_rtx(mode);
7055 if (aarch64_stack_protector_guard != SSP_GLOBAL)
7057 /* Generate access through the system register. The
7058 sequence we want here is the access
7059 of the stack offset to come with
7060 mrs scratch_reg, <system_register>
7061 add scratch_reg, scratch_reg, :lo12:offset. */
7062 rtx tmp_reg = gen_reg_rtx (mode);
7065 emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7066 emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7067 GEN_INT (aarch64_stack_protector_guard_offset)));
7071 emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7072 emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7073 GEN_INT (aarch64_stack_protector_guard_offset)));
7076 operands[1] = gen_rtx_MEM (mode, tmp_reg);
7078 emit_insn ((mode == DImode
7079 ? gen_stack_protect_test_di
7080 : gen_stack_protect_test_si) (result,
7085 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7086 result, const0_rtx, operands[2]));
7088 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7089 result, const0_rtx, operands[2]));
7093 (define_insn "stack_protect_test_<mode>"
7094 [(set (match_operand:PTR 0 "register_operand" "=r")
7095 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7096 (match_operand:PTR 2 "memory_operand" "m")]
7098 (clobber (match_scratch:PTR 3 "=&r"))]
7100 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7101 [(set_attr "length" "12")
7102 (set_attr "type" "multiple")])
7104 ;; Write Floating-point Control Register.
7105 (define_insn "set_fpcr"
7106 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7109 [(set_attr "type" "mrs")])
7111 ;; Read Floating-point Control Register.
7112 (define_insn "get_fpcr"
7113 [(set (match_operand:SI 0 "register_operand" "=r")
7114 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7117 [(set_attr "type" "mrs")])
7119 ;; Write Floating-point Status Register.
7120 (define_insn "set_fpsr"
7121 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7124 [(set_attr "type" "mrs")])
7126 ;; Read Floating-point Status Register.
7127 (define_insn "get_fpsr"
7128 [(set (match_operand:SI 0 "register_operand" "=r")
7129 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7132 [(set_attr "type" "mrs")])
7135 ;; Define the subtract-one-and-jump insns so loop.c
7136 ;; knows what to generate.
7137 (define_expand "doloop_end"
7138 [(use (match_operand 0 "" "")) ; loop pseudo
7139 (use (match_operand 1 "" ""))] ; label
7140 "optimize > 0 && flag_modulo_sched"
7149 /* Currently SMS relies on the do-loop pattern to recognize loops
7150 where (1) the control part consists of all insns defining and/or
7151 using a certain 'count' register and (2) the loop count can be
7152 adjusted by modifying this register prior to the loop.
7153 ??? The possible introduction of a new block to initialize the
7154 new IV can potentially affect branch optimizations. */
7156 if (GET_MODE (operands[0]) != DImode)
7160 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7162 cmp = XVECEXP (PATTERN (insn), 0, 0);
7163 cc_reg = SET_DEST (cmp);
7164 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7165 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7166 emit_jump_insn (gen_rtx_SET (pc_rtx,
7167 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7172 ;; Track speculation through conditional branches. We assume that
7173 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7174 (define_insn "speculation_tracker"
7175 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7176 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7177 UNSPEC_SPECULATION_TRACKER))]
7180 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7181 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7184 [(set_attr "type" "csel")]
7187 ;; BTI <target> instructions
7188 (define_insn "bti_noarg"
7189 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7192 [(set_attr "type" "no_insn")]
7195 (define_insn "bti_c"
7196 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7199 [(set_attr "type" "no_insn")]
7202 (define_insn "bti_j"
7203 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7206 [(set_attr "type" "no_insn")]
7209 (define_insn "bti_jc"
7210 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7212 "hint\t38 // bti jc"
7213 [(set_attr "type" "no_insn")]
7216 ;; Hard speculation barrier.
7217 (define_insn "speculation_barrier"
7218 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7221 [(set_attr "length" "8")
7222 (set_attr "type" "block")
7223 (set_attr "speculation_barrier" "true")]
7226 ;; Support for __builtin_speculation_safe_value when we have speculation
7227 ;; tracking enabled. Use the speculation tracker to decide whether to
7228 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7229 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7230 [(set (match_operand:ALLI_TI 0 "register_operand")
7231 (unspec_volatile:ALLI_TI
7232 [(match_operand:ALLI_TI 1 "register_operand")
7233 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7234 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7235 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7239 if (operands[2] == const0_rtx)
7242 if (<MODE>mode == TImode)
7243 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7245 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7247 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7255 ;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
7256 ;; encoding for CSDB, but will work in older versions of the assembler.
7257 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7258 [(set (match_operand:ALLI 0 "register_operand" "=r")
7259 (unspec_volatile:ALLI
7260 [(match_operand:ALLI 1 "register_operand" "r")
7261 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7262 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7263 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7266 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7267 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7271 [(set_attr "length" "12")
7272 (set_attr "type" "block")
7273 (set_attr "speculation_barrier" "true")]
7276 ;; Pattern to match despeculate_copyti
7277 (define_insn "*despeculate_copyti_insn"
7278 [(set (match_operand:TI 0 "register_operand" "=r")
7280 [(match_operand:TI 1 "register_operand" "r")
7281 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7282 (use (reg:DI SPECULATION_TRACKER_REGNUM))
7283 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7286 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7288 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7292 [(set_attr "length" "16")
7293 (set_attr "type" "block")
7294 (set_attr "speculation_barrier" "true")]
7297 (define_insn "despeculate_simple<ALLI:mode>"
7298 [(set (match_operand:ALLI 0 "register_operand" "=r")
7299 (unspec_volatile:ALLI
7300 [(match_operand:ALLI 1 "register_operand" "r")
7301 (use (match_operand:ALLI 2 "register_operand" ""))]
7302 UNSPECV_SPECULATION_BARRIER))]
7304 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7305 [(set_attr "type" "block")
7306 (set_attr "length" "8")
7307 (set_attr "speculation_barrier" "true")]
7310 (define_insn "despeculate_simpleti"
7311 [(set (match_operand:TI 0 "register_operand" "=r")
7313 [(match_operand:TI 1 "register_operand" "r")
7314 (use (match_operand:DI 2 "register_operand" ""))]
7315 UNSPECV_SPECULATION_BARRIER))]
7317 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7318 [(set_attr "type" "block")
7319 (set_attr "length" "12")
7320 (set_attr "speculation_barrier" "true")]
7323 (define_insn "aarch64_<frintnzs_op><mode>"
7324 [(set (match_operand:VSFDF 0 "register_operand" "=w")
7325 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7327 "TARGET_FRINT && TARGET_FLOAT
7328 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7329 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7330 [(set_attr "type" "f_rint<stype>")]
7333 ;; Transactional Memory Extension (TME) instructions.
7335 (define_insn "tstart"
7336 [(set (match_operand:DI 0 "register_operand" "=r")
7337 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7338 (clobber (mem:BLK (scratch)))]
7341 [(set_attr "type" "tme")]
7344 (define_insn "ttest"
7345 [(set (match_operand:DI 0 "register_operand" "=r")
7346 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7347 (clobber (mem:BLK (scratch)))]
7350 [(set_attr "type" "tme")]
7353 (define_insn "tcommit"
7354 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7355 (clobber (mem:BLK (scratch)))]
7358 [(set_attr "type" "tme")]
7361 (define_insn "tcancel"
7362 [(unspec_volatile:BLK
7363 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7364 (clobber (mem:BLK (scratch)))]
7365 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7367 [(set_attr "type" "tme")]
7371 (include "aarch64-simd.md")
7373 ;; Atomic Operations
7374 (include "atomics.md")
7376 ;; ldp/stp peephole patterns
7377 (include "aarch64-ldpstp.md")
7380 (include "aarch64-sve.md")
7383 (include "aarch64-sve2.md")