gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     ;; Scratch registers for prologue/epilogue use.
  39     (EP0_REGNUM         12)
  40     (EP1_REGNUM         13)
  41     (R14_REGNUM         14)
  42     (R15_REGNUM         15)
  43     (R16_REGNUM         16)
  44     (IP0_REGNUM         16)
  45     (R17_REGNUM         17)
  46     (IP1_REGNUM         17)
  47     (R18_REGNUM         18)
  48     (R19_REGNUM         19)
  49     (R20_REGNUM         20)
  50     (R21_REGNUM         21)
  51     (R22_REGNUM         22)
  52     (R23_REGNUM         23)
  53     (R24_REGNUM         24)
  54     (R25_REGNUM         25)
  55     (R26_REGNUM         26)
  56     (R27_REGNUM         27)
  57     (R28_REGNUM         28)
  58     (R29_REGNUM         29)
  59     (R30_REGNUM         30)
  60     (LR_REGNUM          30)
  61     (SP_REGNUM          31)
  62     (V0_REGNUM          32)
  63     (V1_REGNUM          33)
  64     (V2_REGNUM          34)
  65     (V3_REGNUM          35)
  66     (V4_REGNUM          36)
  67     (V5_REGNUM          37)
  68     (V6_REGNUM          38)
  69     (V7_REGNUM          39)
  70     (V8_REGNUM          40)
  71     (V9_REGNUM          41)
  72     (V10_REGNUM         42)
  73     (V11_REGNUM         43)
  74     (V12_REGNUM         44)
  75     (V13_REGNUM         45)
  76     (V14_REGNUM         46)
  77     (V15_REGNUM         47)
  78     (V16_REGNUM         48)
  79     (V17_REGNUM         49)
  80     (V18_REGNUM         50)
  81     (V19_REGNUM         51)
  82     (V20_REGNUM         52)
  83     (V21_REGNUM         53)
  84     (V22_REGNUM         54)
  85     (V23_REGNUM         55)
  86     (V24_REGNUM         56)
  87     (V25_REGNUM         57)
  88     (V26_REGNUM         58)
  89     (V27_REGNUM         59)
  90     (V28_REGNUM         60)
  91     (V29_REGNUM         61)
  92     (V30_REGNUM         62)
  93     (V31_REGNUM         63)
  94     (LAST_SAVED_REGNUM  63)
  95     (SFP_REGNUM         64)
  96     (AP_REGNUM          65)
  97     (CC_REGNUM          66)
  98     ;; Defined only to make the DWARF description simpler.
  99     (VG_REGNUM          67)
 100     (P0_REGNUM          68)
 101     (P1_REGNUM          69)
 102     (P2_REGNUM          70)
 103     (P3_REGNUM          71)
 104     (P4_REGNUM          72)
 105     (P5_REGNUM          73)
 106     (P6_REGNUM          74)
 107     (P7_REGNUM          75)
 108     (P8_REGNUM          76)
 109     (P9_REGNUM          77)
 110     (P10_REGNUM         78)
 111     (P11_REGNUM         79)
 112     (P12_REGNUM         80)
 113     (P13_REGNUM         81)
 114     (P14_REGNUM         82)
 115     (P15_REGNUM         83)
 116     ;; A couple of call-clobbered registers that we need to reserve when
 117     ;; tracking speculation this is not ABI, so is subject to change.
 118     (SPECULATION_TRACKER_REGNUM 15)
 119     (SPECULATION_SCRATCH_REGNUM 14)
 120   ]
 121 )
 122
 123 (define_c_enum "unspec" [
 124     UNSPEC_AUTI1716
 125     UNSPEC_AUTISP
 126     UNSPEC_CASESI
 127     UNSPEC_CRC32B
 128     UNSPEC_CRC32CB
 129     UNSPEC_CRC32CH
 130     UNSPEC_CRC32CW
 131     UNSPEC_CRC32CX
 132     UNSPEC_CRC32H
 133     UNSPEC_CRC32W
 134     UNSPEC_CRC32X
 135     UNSPEC_FCVTZS
 136     UNSPEC_FCVTZU
 137     UNSPEC_URECPE
 138     UNSPEC_FRECPE
 139     UNSPEC_FRECPS
 140     UNSPEC_FRECPX
 141     UNSPEC_FRINTA
 142     UNSPEC_FRINTI
 143     UNSPEC_FRINTM
 144     UNSPEC_FRINTN
 145     UNSPEC_FRINTP
 146     UNSPEC_FRINTX
 147     UNSPEC_FRINTZ
 148     UNSPEC_GOTSMALLPIC
 149     UNSPEC_GOTSMALLPIC28K
 150     UNSPEC_GOTSMALLTLS
 151     UNSPEC_GOTTINYPIC
 152     UNSPEC_GOTTINYTLS
 153     UNSPEC_LD1
 154     UNSPEC_LD2
 155     UNSPEC_LD2_DREG
 156     UNSPEC_LD2_DUP
 157     UNSPEC_LD3
 158     UNSPEC_LD3_DREG
 159     UNSPEC_LD3_DUP
 160     UNSPEC_LD4
 161     UNSPEC_LD4_DREG
 162     UNSPEC_LD4_DUP
 163     UNSPEC_LD2_LANE
 164     UNSPEC_LD3_LANE
 165     UNSPEC_LD4_LANE
 166     UNSPEC_MB
 167     UNSPEC_NOP
 168     UNSPEC_PACI1716
 169     UNSPEC_PACISP
 170     UNSPEC_PRLG_STK
 171     UNSPEC_REV
 172     UNSPEC_RBIT
 173     UNSPEC_SABAL
 174     UNSPEC_SABDL2
 175     UNSPEC_SADALP
 176     UNSPEC_SCVTF
 177     UNSPEC_SISD_NEG
 178     UNSPEC_SISD_SSHL
 179     UNSPEC_SISD_USHL
 180     UNSPEC_SSHL_2S
 181     UNSPEC_ST1
 182     UNSPEC_ST2
 183     UNSPEC_ST3
 184     UNSPEC_ST4
 185     UNSPEC_ST2_LANE
 186     UNSPEC_ST3_LANE
 187     UNSPEC_ST4_LANE
 188     UNSPEC_TLS
 189     UNSPEC_TLSDESC
 190     UNSPEC_TLSLE12
 191     UNSPEC_TLSLE24
 192     UNSPEC_TLSLE32
 193     UNSPEC_TLSLE48
 194     UNSPEC_UABAL
 195     UNSPEC_UABDL2
 196     UNSPEC_UADALP
 197     UNSPEC_UCVTF
 198     UNSPEC_USHL_2S
 199     UNSPEC_VSTRUCTDUMMY
 200     UNSPEC_SP_SET
 201     UNSPEC_SP_TEST
 202     UNSPEC_RSQRT
 203     UNSPEC_RSQRTE
 204     UNSPEC_RSQRTS
 205     UNSPEC_NZCV
 206     UNSPEC_XPACLRI
 207     UNSPEC_LD1_SVE
 208     UNSPEC_ST1_SVE
 209     UNSPEC_LD1RQ
 210     UNSPEC_LD1_GATHER
 211     UNSPEC_ST1_SCATTER
 212     UNSPEC_MERGE_PTRUE
 213     UNSPEC_PTEST_PTRUE
 214     UNSPEC_UNPACKSHI
 215     UNSPEC_UNPACKUHI
 216     UNSPEC_UNPACKSLO
 217     UNSPEC_UNPACKULO
 218     UNSPEC_PACK
 219     UNSPEC_FLOAT_CONVERT
 220     UNSPEC_WHILE_LO
 221     UNSPEC_LDN
 222     UNSPEC_STN
 223     UNSPEC_INSR
 224     UNSPEC_CLASTB
 225     UNSPEC_FADDA
 226     UNSPEC_REV_SUBREG
 227     UNSPEC_SPECULATION_TRACKER
 228     UNSPEC_COPYSIGN
 229 ])
 230
 231 (define_c_enum "unspecv" [
 232     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 233     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 234     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 235     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 236     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 237     UNSPECV_BLOCKAGE            ; Represent a blockage
 238     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 239     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 240   ]
 241 )
 242
 243 ;; If further include files are added the defintion of MD_INCLUDES
 244 ;; must be updated.
 245
 246 (include "constraints.md")
 247 (include "predicates.md")
 248 (include "iterators.md")
 249
 250 ;; -------------------------------------------------------------------
 251 ;; Instruction types and attributes
 252 ;; -------------------------------------------------------------------
 253
 254 ; The "type" attribute is included here from AArch32 backend to be able
 255 ; to share pipeline descriptions.
 256 (include "../arm/types.md")
 257
 258 ;; It is important to set the fp or simd attributes to yes when a pattern
 259 ;; alternative uses the FP or SIMD register files, usually signified by use of
 260 ;; the 'w' constraint.  This will ensure that the alternative will be
 261 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 262 ;; architecture extensions.  If all the alternatives in a pattern use the
 263 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 264 ;; or TARGET_SIMD.
 265
 266 ;; Attributes of the architecture required to support the instruction (or
 267 ;; alternative). This attribute is used to compute attribute "enabled", use type
 268 ;; "any" to enable an alternative in all cases.
 269
 270 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 271
 272 (define_enum_attr "arch" "arches" (const_string "any"))
 273
 274 ;; [For compatibility with Arm in pipeline models]
 275 ;; Attribute that specifies whether or not the instruction touches fp
 276 ;; registers.
 277 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 278 ;; backends except in the scheduling description for xgene1.  In that
 279 ;; scheduling description this attribute is used to subclass the load_4 and
 280 ;; load_8 types.
 281 (define_attr "fp" "no,yes"
 282   (if_then_else
 283     (eq_attr "arch" "fp")
 284     (const_string "yes")
 285     (const_string "no")))
 286
 287 (define_attr "arch_enabled" "no,yes"
 288   (if_then_else
 289     (ior
 290         (eq_attr "arch" "any")
 291
 292         (and (eq_attr "arch" "rcpc8_4")
 293              (match_test "AARCH64_ISA_RCPC8_4"))
 294
 295         (and (eq_attr "arch" "fp")
 296              (match_test "TARGET_FLOAT"))
 297
 298         (and (eq_attr "arch" "simd")
 299              (match_test "TARGET_SIMD"))
 300
 301         (and (eq_attr "arch" "fp16")
 302              (match_test "TARGET_FP_F16INST"))
 303
 304         (and (eq_attr "arch" "sve")
 305              (match_test "TARGET_SVE")))
 306     (const_string "yes")
 307     (const_string "no")))
 308
 309 ;; Attribute that controls whether an alternative is enabled or not.
 310 ;; Currently it is only used to disable alternatives which touch fp or simd
 311 ;; registers when -mgeneral-regs-only is specified or to require a special
 312 ;; architecture support.
 313 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 314
 315 ;; Attribute that specifies whether we are dealing with a branch to a
 316 ;; label that is far away, i.e. further away than the maximum/minimum
 317 ;; representable in a signed 21-bits number.
 318 ;; 0 :=: no
 319 ;; 1 :=: yes
 320 (define_attr "far_branch" "" (const_int 0))
 321
 322 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 323 (define_attr "movprfx" "no,yes" (const_string "no"))
 324
 325 (define_attr "length" ""
 326   (cond [(eq_attr "movprfx" "yes")
 327            (const_int 8)
 328         ] (const_int 4)))
 329
 330 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 331 ;; no predicated insns.
 332 (define_attr "predicated" "yes,no" (const_string "no"))
 333
 334 ;; Set to true on an insn that requires the speculation tracking state to be
 335 ;; in the tracking register before the insn issues.  Otherwise the compiler
 336 ;; may chose to hold the tracking state encoded in SP.
 337 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 338
 339 ;; -------------------------------------------------------------------
 340 ;; Pipeline descriptions and scheduling
 341 ;; -------------------------------------------------------------------
 342
 343 ;; Processor types.
 344 (include "aarch64-tune.md")
 345
 346 ;; Scheduling
 347 (include "../arm/cortex-a53.md")
 348 (include "../arm/cortex-a57.md")
 349 (include "../arm/exynos-m1.md")
 350 (include "falkor.md")
 351 (include "saphira.md")
 352 (include "thunderx.md")
 353 (include "../arm/xgene1.md")
 354 (include "thunderx2t99.md")
 355
 356 ;; -------------------------------------------------------------------
 357 ;; Jumps and other miscellaneous insns
 358 ;; -------------------------------------------------------------------
 359
 360 (define_insn "indirect_jump"
 361   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 362   ""
 363   "br\\t%0"
 364   [(set_attr "type" "branch")]
 365 )
 366
 367 (define_insn "jump"
 368   [(set (pc) (label_ref (match_operand 0 "" "")))]
 369   ""
 370   "b\\t%l0"
 371   [(set_attr "type" "branch")]
 372 )
 373
 374 (define_expand "cbranch<mode>4"
 375   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 376                             [(match_operand:GPI 1 "register_operand" "")
 377                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 378                            (label_ref (match_operand 3 "" ""))
 379                            (pc)))]
 380   ""
 381   "
 382   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 383                                          operands[2]);
 384   operands[2] = const0_rtx;
 385   "
 386 )
 387
 388 (define_expand "cbranch<mode>4"
 389   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 390                             [(match_operand:GPF 1 "register_operand" "")
 391                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 392                            (label_ref (match_operand 3 "" ""))
 393                            (pc)))]
 394   ""
 395   "
 396   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 397                                          operands[2]);
 398   operands[2] = const0_rtx;
 399   "
 400 )
 401
 402 (define_expand "cbranchcc4"
 403   [(set (pc) (if_then_else
 404               (match_operator 0 "aarch64_comparison_operator"
 405                [(match_operand 1 "cc_register" "")
 406                 (match_operand 2 "const0_operand")])
 407               (label_ref (match_operand 3 "" ""))
 408               (pc)))]
 409   ""
 410   "")
 411
 412 (define_insn "ccmp<mode>"
 413   [(set (match_operand:CC 1 "cc_register" "")
 414         (if_then_else:CC
 415           (match_operator 4 "aarch64_comparison_operator"
 416            [(match_operand 0 "cc_register" "")
 417             (const_int 0)])
 418           (compare:CC
 419             (match_operand:GPI 2 "register_operand" "r,r,r")
 420             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 421           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 422   ""
 423   "@
 424    ccmp\\t%<w>2, %<w>3, %k5, %m4
 425    ccmp\\t%<w>2, %3, %k5, %m4
 426    ccmn\\t%<w>2, #%n3, %k5, %m4"
 427   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 428 )
 429
 430 (define_insn "fccmp<mode>"
 431   [(set (match_operand:CCFP 1 "cc_register" "")
 432         (if_then_else:CCFP
 433           (match_operator 4 "aarch64_comparison_operator"
 434            [(match_operand 0 "cc_register" "")
 435             (const_int 0)])
 436           (compare:CCFP
 437             (match_operand:GPF 2 "register_operand" "w")
 438             (match_operand:GPF 3 "register_operand" "w"))
 439           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 440   "TARGET_FLOAT"
 441   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 442   [(set_attr "type" "fccmp<s>")]
 443 )
 444
 445 (define_insn "fccmpe<mode>"
 446   [(set (match_operand:CCFPE 1 "cc_register" "")
 447          (if_then_else:CCFPE
 448           (match_operator 4 "aarch64_comparison_operator"
 449            [(match_operand 0 "cc_register" "")
 450           (const_int 0)])
 451            (compare:CCFPE
 452             (match_operand:GPF 2 "register_operand" "w")
 453             (match_operand:GPF 3 "register_operand" "w"))
 454           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 455   "TARGET_FLOAT"
 456   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 457   [(set_attr "type" "fccmp<s>")]
 458 )
 459
 460 ;; Expansion of signed mod by a power of 2 using CSNEG.
 461 ;; For x0 % n where n is a power of 2 produce:
 462 ;; negs   x1, x0
 463 ;; and    x0, x0, #(n - 1)
 464 ;; and    x1, x1, #(n - 1)
 465 ;; csneg  x0, x0, x1, mi
 466
 467 (define_expand "mod<mode>3"
 468   [(match_operand:GPI 0 "register_operand" "")
 469    (match_operand:GPI 1 "register_operand" "")
 470    (match_operand:GPI 2 "const_int_operand" "")]
 471   ""
 472   {
 473     HOST_WIDE_INT val = INTVAL (operands[2]);
 474
 475     if (val <= 0
 476        || exact_log2 (val) <= 0
 477        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 478       FAIL;
 479
 480     rtx mask = GEN_INT (val - 1);
 481
 482     /* In the special case of x0 % 2 we can do the even shorter:
 483         cmp    x0, xzr
 484         and    x0, x0, 1
 485         cneg   x0, x0, lt.  */
 486     if (val == 2)
 487       {
 488         rtx masked = gen_reg_rtx (<MODE>mode);
 489         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 490         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 491         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 492         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 493         DONE;
 494       }
 495
 496     rtx neg_op = gen_reg_rtx (<MODE>mode);
 497     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 498
 499     /* Extract the condition register and mode.  */
 500     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 501     rtx cc_reg = SET_DEST (cmp);
 502     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 503
 504     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 505     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 506
 507     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 508     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 509
 510     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 511                                        masked_neg, masked_pos));
 512     DONE;
 513   }
 514 )
 515
 516 (define_insn "condjump"
 517   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 518                             [(match_operand 1 "cc_register" "") (const_int 0)])
 519                            (label_ref (match_operand 2 "" ""))
 520                            (pc)))]
 521   ""
 522   {
 523     if (get_attr_length (insn) == 8)
 524       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 525     else
 526       return  "b%m0\\t%l2";
 527   }
 528   [(set_attr "type" "branch")
 529    (set (attr "length")
 530         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 531                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 532                       (const_int 4)
 533                       (const_int 8)))
 534    (set (attr "far_branch")
 535         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 536                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 537                       (const_int 0)
 538                       (const_int 1)))]
 539 )
 540
 541 ;; For a 24-bit immediate CST we can optimize the compare for equality
 542 ;; and branch sequence from:
 543 ;;      mov     x0, #imm1
 544 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 545 ;;      cmp     x1, x0
 546 ;;      b<ne,eq> .Label
 547 ;; into the shorter:
 548 ;;      sub     x0, x1, #(CST & 0xfff000)
 549 ;;      subs    x0, x0, #(CST & 0x000fff)
 550 ;;      b<ne,eq> .Label
 551 (define_insn_and_split "*compare_condjump<mode>"
 552   [(set (pc) (if_then_else (EQL
 553                               (match_operand:GPI 0 "register_operand" "r")
 554                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 555                            (label_ref:P (match_operand 2 "" ""))
 556                            (pc)))]
 557   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 558    && !aarch64_plus_operand (operands[1], <MODE>mode)
 559    && !reload_completed"
 560   "#"
 561   "&& true"
 562   [(const_int 0)]
 563   {
 564     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 565     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 566     rtx tmp = gen_reg_rtx (<MODE>mode);
 567     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 568     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 569     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 570     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 571     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 572     DONE;
 573   }
 574 )
 575
 576 (define_expand "casesi"
 577   [(match_operand:SI 0 "register_operand" "")   ; Index
 578    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 579    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 580    (match_operand:DI 3 "" "")                   ; Table label
 581    (match_operand:DI 4 "" "")]                  ; Out of range label
 582   ""
 583   {
 584     if (operands[1] != const0_rtx)
 585       {
 586         rtx reg = gen_reg_rtx (SImode);
 587
 588         /* Canonical RTL says that if you have:
 589
 590            (minus (X) (CONST))
 591
 592            then this should be emitted as:
 593
 594            (plus (X) (-CONST))
 595
 596            The use of trunc_int_for_mode ensures that the resulting
 597            constant can be represented in SImode, this is important
 598            for the corner case where operand[1] is INT_MIN.  */
 599
 600         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 601
 602         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 603               (operands[1], SImode))
 604           operands[1] = force_reg (SImode, operands[1]);
 605         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 606         operands[0] = reg;
 607       }
 608
 609     if (!aarch64_plus_operand (operands[2], SImode))
 610       operands[2] = force_reg (SImode, operands[2]);
 611     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 612                                                  const0_rtx),
 613                                     operands[0], operands[2], operands[4]));
 614
 615     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 616     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 617                                          operands[3]));
 618     DONE;
 619   }
 620 )
 621
 622 (define_insn "casesi_dispatch"
 623   [(parallel
 624     [(set (pc)
 625           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 626                            (match_operand:SI 1 "register_operand" "r")]
 627                         UNSPEC_CASESI)))
 628      (clobber (reg:CC CC_REGNUM))
 629      (clobber (match_scratch:DI 3 "=r"))
 630      (clobber (match_scratch:DI 4 "=r"))
 631      (use (label_ref (match_operand 2 "" "")))])]
 632   ""
 633   "*
 634   return aarch64_output_casesi (operands);
 635   "
 636   [(set_attr "length" "16")
 637    (set_attr "type" "branch")]
 638 )
 639
 640 (define_insn "nop"
 641   [(unspec[(const_int 0)] UNSPEC_NOP)]
 642   ""
 643   "nop"
 644   [(set_attr "type" "no_insn")]
 645 )
 646
 647 (define_insn "prefetch"
 648   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 649             (match_operand:QI 1 "const_int_operand" "")
 650             (match_operand:QI 2 "const_int_operand" ""))]
 651   ""
 652   {
 653     const char * pftype[2][4] =
 654     {
 655       {"prfm\\tPLDL1STRM, %0",
 656        "prfm\\tPLDL3KEEP, %0",
 657        "prfm\\tPLDL2KEEP, %0",
 658        "prfm\\tPLDL1KEEP, %0"},
 659       {"prfm\\tPSTL1STRM, %0",
 660        "prfm\\tPSTL3KEEP, %0",
 661        "prfm\\tPSTL2KEEP, %0",
 662        "prfm\\tPSTL1KEEP, %0"},
 663     };
 664
 665     int locality = INTVAL (operands[2]);
 666
 667     gcc_assert (IN_RANGE (locality, 0, 3));
 668
 669     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 670        the address into a DImode MEM so that aarch64_print_operand knows
 671        how to print it.  */
 672     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 673     return pftype[INTVAL(operands[1])][locality];
 674   }
 675   [(set_attr "type" "load_4")]
 676 )
 677
 678 (define_insn "trap"
 679   [(trap_if (const_int 1) (const_int 8))]
 680   ""
 681   "brk #1000"
 682   [(set_attr "type" "trap")])
 683
 684 (define_expand "prologue"
 685   [(clobber (const_int 0))]
 686   ""
 687   "
 688   aarch64_expand_prologue ();
 689   DONE;
 690   "
 691 )
 692
 693 (define_expand "epilogue"
 694   [(clobber (const_int 0))]
 695   ""
 696   "
 697   aarch64_expand_epilogue (false);
 698   DONE;
 699   "
 700 )
 701
 702 (define_expand "sibcall_epilogue"
 703   [(clobber (const_int 0))]
 704   ""
 705   "
 706   aarch64_expand_epilogue (true);
 707   DONE;
 708   "
 709 )
 710
 711 (define_insn "*do_return"
 712   [(return)]
 713   ""
 714   {
 715     if (aarch64_return_address_signing_enabled ()
 716         && TARGET_ARMV8_3
 717         && !crtl->calls_eh_return)
 718       return "retaa";
 719
 720     return "ret";
 721   }
 722   [(set_attr "type" "branch")]
 723 )
 724
 725 (define_expand "return"
 726   [(simple_return)]
 727   "aarch64_use_return_insn_p ()"
 728   ""
 729 )
 730
 731 (define_insn "simple_return"
 732   [(simple_return)]
 733   "aarch64_use_simple_return_insn_p ()"
 734   "ret"
 735   [(set_attr "type" "branch")]
 736 )
 737
 738 (define_insn "*cb<optab><mode>1"
 739   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 740                                 (const_int 0))
 741                            (label_ref (match_operand 1 "" ""))
 742                            (pc)))]
 743   "!aarch64_track_speculation"
 744   {
 745     if (get_attr_length (insn) == 8)
 746       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 747     else
 748       return "<cbz>\\t%<w>0, %l1";
 749   }
 750   [(set_attr "type" "branch")
 751    (set (attr "length")
 752         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 753                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 754                       (const_int 4)
 755                       (const_int 8)))
 756    (set (attr "far_branch")
 757         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 758                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 759                       (const_int 0)
 760                       (const_int 1)))]
 761 )
 762
 763 (define_insn "*tb<optab><mode>1"
 764   [(set (pc) (if_then_else
 765               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 766                                     (const_int 1)
 767                                     (match_operand 1
 768                                       "aarch64_simd_shift_imm_<mode>" "n"))
 769                    (const_int 0))
 770              (label_ref (match_operand 2 "" ""))
 771              (pc)))
 772    (clobber (reg:CC CC_REGNUM))]
 773   "!aarch64_track_speculation"
 774   {
 775     if (get_attr_length (insn) == 8)
 776       {
 777         if (get_attr_far_branch (insn) == 1)
 778           return aarch64_gen_far_branch (operands, 2, "Ltb",
 779                                          "<inv_tb>\\t%<w>0, %1, ");
 780         else
 781           {
 782             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 783             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 784           }
 785       }
 786     else
 787       return "<tbz>\t%<w>0, %1, %l2";
 788   }
 789   [(set_attr "type" "branch")
 790    (set (attr "length")
 791         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 792                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 793                       (const_int 4)
 794                       (const_int 8)))
 795    (set (attr "far_branch")
 796         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 797                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 798                       (const_int 0)
 799                       (const_int 1)))]
 800
 801 )
 802
 803 (define_insn "*cb<optab><mode>1"
 804   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 805                                  (const_int 0))
 806                            (label_ref (match_operand 1 "" ""))
 807                            (pc)))
 808    (clobber (reg:CC CC_REGNUM))]
 809   "!aarch64_track_speculation"
 810   {
 811     if (get_attr_length (insn) == 8)
 812       {
 813         if (get_attr_far_branch (insn) == 1)
 814           return aarch64_gen_far_branch (operands, 1, "Ltb",
 815                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 816         else
 817           {
 818             char buf[64];
 819             uint64_t val = ((uint64_t) 1)
 820                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 821             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 822             output_asm_insn (buf, operands);
 823             return "<bcond>\t%l1";
 824           }
 825       }
 826     else
 827       return "<tbz>\t%<w>0, <sizem1>, %l1";
 828   }
 829   [(set_attr "type" "branch")
 830    (set (attr "length")
 831         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 832                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 833                       (const_int 4)
 834                       (const_int 8)))
 835    (set (attr "far_branch")
 836         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 837                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 838                       (const_int 0)
 839                       (const_int 1)))]
 840 )
 841
 842 ;; -------------------------------------------------------------------
 843 ;; Subroutine calls and sibcalls
 844 ;; -------------------------------------------------------------------
 845
 846 (define_expand "call"
 847   [(parallel [(call (match_operand 0 "memory_operand" "")
 848                     (match_operand 1 "general_operand" ""))
 849               (use (match_operand 2 "" ""))
 850               (clobber (reg:DI LR_REGNUM))])]
 851   ""
 852   "
 853   {
 854     aarch64_expand_call (NULL_RTX, operands[0], false);
 855     DONE;
 856   }"
 857 )
 858
 859 (define_insn "*call_insn"
 860   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 861          (match_operand 1 "" ""))
 862    (clobber (reg:DI LR_REGNUM))]
 863   ""
 864   "@
 865   blr\\t%0
 866   bl\\t%c0"
 867   [(set_attr "type" "call, call")]
 868 )
 869
 870 (define_expand "call_value"
 871   [(parallel [(set (match_operand 0 "" "")
 872                    (call (match_operand 1 "memory_operand" "")
 873                          (match_operand 2 "general_operand" "")))
 874               (use (match_operand 3 "" ""))
 875               (clobber (reg:DI LR_REGNUM))])]
 876   ""
 877   "
 878   {
 879     aarch64_expand_call (operands[0], operands[1], false);
 880     DONE;
 881   }"
 882 )
 883
 884 (define_insn "*call_value_insn"
 885   [(set (match_operand 0 "" "")
 886         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 887                       (match_operand 2 "" "")))
 888    (clobber (reg:DI LR_REGNUM))]
 889   ""
 890   "@
 891   blr\\t%1
 892   bl\\t%c1"
 893   [(set_attr "type" "call, call")]
 894 )
 895
 896 (define_expand "sibcall"
 897   [(parallel [(call (match_operand 0 "memory_operand" "")
 898                     (match_operand 1 "general_operand" ""))
 899               (return)
 900               (use (match_operand 2 "" ""))])]
 901   ""
 902   {
 903     aarch64_expand_call (NULL_RTX, operands[0], true);
 904     DONE;
 905   }
 906 )
 907
 908 (define_expand "sibcall_value"
 909   [(parallel [(set (match_operand 0 "" "")
 910                    (call (match_operand 1 "memory_operand" "")
 911                          (match_operand 2 "general_operand" "")))
 912               (return)
 913               (use (match_operand 3 "" ""))])]
 914   ""
 915   {
 916     aarch64_expand_call (operands[0], operands[1], true);
 917     DONE;
 918   }
 919 )
 920
 921 (define_insn "*sibcall_insn"
 922   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 923          (match_operand 1 "" ""))
 924    (return)]
 925   "SIBLING_CALL_P (insn)"
 926   "@
 927    br\\t%0
 928    b\\t%c0"
 929   [(set_attr "type" "branch, branch")]
 930 )
 931
 932 (define_insn "*sibcall_value_insn"
 933   [(set (match_operand 0 "" "")
 934         (call (mem:DI
 935                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 936               (match_operand 2 "" "")))
 937    (return)]
 938   "SIBLING_CALL_P (insn)"
 939   "@
 940    br\\t%1
 941    b\\t%c1"
 942   [(set_attr "type" "branch, branch")]
 943 )
 944
 945 ;; Call subroutine returning any type.
 946
 947 (define_expand "untyped_call"
 948   [(parallel [(call (match_operand 0 "")
 949                     (const_int 0))
 950               (match_operand 1 "")
 951               (match_operand 2 "")])]
 952   ""
 953 {
 954   int i;
 955
 956   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 957
 958   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 959     {
 960       rtx set = XVECEXP (operands[2], 0, i);
 961       emit_move_insn (SET_DEST (set), SET_SRC (set));
 962     }
 963
 964   /* The optimizer does not know that the call sets the function value
 965      registers we stored in the result block.  We avoid problems by
 966      claiming that all hard registers are used and clobbered at this
 967      point.  */
 968   emit_insn (gen_blockage ());
 969   DONE;
 970 })
 971
 972 ;; -------------------------------------------------------------------
 973 ;; Moves
 974 ;; -------------------------------------------------------------------
 975
 976 (define_expand "mov<mode>"
 977   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 978         (match_operand:SHORT 1 "general_operand" ""))]
 979   ""
 980   "
 981     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 982       operands[1] = force_reg (<MODE>mode, operands[1]);
 983
 984     if (GET_CODE (operands[1]) == CONST_POLY_INT)
 985       {
 986         aarch64_expand_mov_immediate (operands[0], operands[1]);
 987         DONE;
 988       }
 989   "
 990 )
 991
 992 (define_insn "*mov<mode>_aarch64"
 993   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
 994         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
 995   "(register_operand (operands[0], <MODE>mode)
 996     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 997 {
 998    switch (which_alternative)
 999      {
1000      case 0:
1001        return "mov\t%w0, %w1";
1002      case 1:
1003        return "mov\t%w0, %1";
1004      case 2:
1005        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1006                                                         <MODE>mode);
1007      case 3:
1008        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1009      case 4:
1010        return "ldr<size>\t%w0, %1";
1011      case 5:
1012        return "ldr\t%<size>0, %1";
1013      case 6:
1014        return "str<size>\t%w1, %0";
1015      case 7:
1016        return "str\t%<size>1, %0";
1017      case 8:
1018        return "umov\t%w0, %1.<v>[0]";
1019      case 9:
1020        return "dup\t%0.<Vallxd>, %w1";
1021      case 10:
1022        return "dup\t%<Vetype>0, %1.<v>[0]";
1023      default:
1024        gcc_unreachable ();
1025      }
1026 }
1027   ;; The "mov_imm" type for CNT is just a placeholder.
1028   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1029                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1030    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1031 )
1032
1033 (define_expand "mov<mode>"
1034   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
1035         (match_operand:GPI 1 "general_operand" ""))]
1036   ""
1037   "
1038     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1039         && <MODE>mode == DImode
1040         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1041       DONE;
1042
1043     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1044       operands[1] = force_reg (<MODE>mode, operands[1]);
1045
1046     /* FIXME: RR we still need to fix up what we are doing with
1047        symbol_refs and other types of constants.  */
1048     if (CONSTANT_P (operands[1])
1049         && !CONST_INT_P (operands[1]))
1050      {
1051        aarch64_expand_mov_immediate (operands[0], operands[1]);
1052        DONE;
1053      }
1054   "
1055 )
1056
1057 (define_insn_and_split "*movsi_aarch64"
1058   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1059         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1060   "(register_operand (operands[0], SImode)
1061     || aarch64_reg_or_zero (operands[1], SImode))"
1062   "@
1063    mov\\t%w0, %w1
1064    mov\\t%w0, %w1
1065    mov\\t%w0, %w1
1066    mov\\t%w0, %1
1067    #
1068    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1069    ldr\\t%w0, %1
1070    ldr\\t%s0, %1
1071    str\\t%w1, %0
1072    str\\t%s1, %0
1073    adr\\t%x0, %c1
1074    adrp\\t%x0, %A1
1075    fmov\\t%s0, %w1
1076    fmov\\t%w0, %s1
1077    fmov\\t%s0, %s1
1078    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1079   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1080     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1081    [(const_int 0)]
1082    "{
1083        aarch64_expand_mov_immediate (operands[0], operands[1]);
1084        DONE;
1085     }"
1086   ;; The "mov_imm" type for CNT is just a placeholder.
1087   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1088                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1089    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1090 )
1091
1092 (define_insn_and_split "*movdi_aarch64"
1093   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1094         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1095   "(register_operand (operands[0], DImode)
1096     || aarch64_reg_or_zero (operands[1], DImode))"
1097   "@
1098    mov\\t%x0, %x1
1099    mov\\t%0, %x1
1100    mov\\t%x0, %1
1101    mov\\t%x0, %1
1102    mov\\t%w0, %1
1103    #
1104    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1105    ldr\\t%x0, %1
1106    ldr\\t%d0, %1
1107    str\\t%x1, %0
1108    str\\t%d1, %0
1109    adr\\t%x0, %c1
1110    adrp\\t%x0, %A1
1111    fmov\\t%d0, %x1
1112    fmov\\t%x0, %d1
1113    fmov\\t%d0, %d1
1114    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1115    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1116     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1117    [(const_int 0)]
1118    "{
1119        aarch64_expand_mov_immediate (operands[0], operands[1]);
1120        DONE;
1121     }"
1122   ;; The "mov_imm" type for CNTD is just a placeholder.
1123   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1124                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1125                      neon_move")
1126    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1127 )
1128
1129 (define_insn "insv_imm<mode>"
1130   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1131                           (const_int 16)
1132                           (match_operand:GPI 1 "const_int_operand" "n"))
1133         (match_operand:GPI 2 "const_int_operand" "n"))]
1134   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1135    && UINTVAL (operands[1]) % 16 == 0"
1136   "movk\\t%<w>0, %X2, lsl %1"
1137   [(set_attr "type" "mov_imm")]
1138 )
1139
1140 (define_expand "movti"
1141   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1142         (match_operand:TI 1 "general_operand" ""))]
1143   ""
1144   "
1145     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1146       operands[1] = force_reg (TImode, operands[1]);
1147
1148     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1149       {
1150         emit_move_insn (gen_lowpart (DImode, operands[0]),
1151                         gen_lowpart (DImode, operands[1]));
1152         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1153         DONE;
1154       }
1155   "
1156 )
1157
1158 (define_insn "*movti_aarch64"
1159   [(set (match_operand:TI 0
1160          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1161         (match_operand:TI 1
1162          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1163   "(register_operand (operands[0], TImode)
1164     || aarch64_reg_or_zero (operands[1], TImode))"
1165   "@
1166    #
1167    #
1168    #
1169    mov\\t%0.16b, %1.16b
1170    ldp\\t%0, %H0, %1
1171    stp\\t%1, %H1, %0
1172    stp\\txzr, xzr, %0
1173    ldr\\t%q0, %1
1174    str\\t%q1, %0"
1175   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1176                              load_16,store_16,store_16,\
1177                              load_16,store_16")
1178    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1179    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1180 )
1181
1182 ;; Split a TImode register-register or register-immediate move into
1183 ;; its component DImode pieces, taking care to handle overlapping
1184 ;; source and dest registers.
1185 (define_split
1186    [(set (match_operand:TI 0 "register_operand" "")
1187          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1188   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1189   [(const_int 0)]
1190 {
1191   aarch64_split_128bit_move (operands[0], operands[1]);
1192   DONE;
1193 })
1194
1195 (define_expand "mov<mode>"
1196   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1197         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1198   ""
1199   {
1200     if (!TARGET_FLOAT)
1201       {
1202         aarch64_err_no_fpadvsimd (<MODE>mode);
1203         FAIL;
1204       }
1205
1206     if (GET_CODE (operands[0]) == MEM
1207         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1208               && aarch64_float_const_zero_rtx_p (operands[1])))
1209       operands[1] = force_reg (<MODE>mode, operands[1]);
1210   }
1211 )
1212
1213 (define_insn "*movhf_aarch64"
1214   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1215         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1216   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1217     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1218   "@
1219    movi\\t%0.4h, #0
1220    fmov\\t%h0, %w1
1221    dup\\t%w0.4h, %w1
1222    umov\\t%w0, %1.h[0]
1223    mov\\t%0.h[0], %1.h[0]
1224    fmov\\t%h0, %1
1225    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1226    ldr\\t%h0, %1
1227    str\\t%h1, %0
1228    ldrh\\t%w0, %1
1229    strh\\t%w1, %0
1230    mov\\t%w0, %w1"
1231   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1232                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1233    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1234 )
1235
1236 (define_insn "*movsf_aarch64"
1237   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1238         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1239   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1240     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1241   "@
1242    movi\\t%0.2s, #0
1243    fmov\\t%s0, %w1
1244    fmov\\t%w0, %s1
1245    fmov\\t%s0, %s1
1246    fmov\\t%s0, %1
1247    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1248    ldr\\t%s0, %1
1249    str\\t%s1, %0
1250    ldr\\t%w0, %1
1251    str\\t%w1, %0
1252    mov\\t%w0, %w1
1253    mov\\t%w0, %1"
1254   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1255                      f_loads,f_stores,load_4,store_4,mov_reg,\
1256                      fconsts")
1257    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1258 )
1259
1260 (define_insn "*movdf_aarch64"
1261   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1262         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1263   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1264     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1265   "@
1266    movi\\t%d0, #0
1267    fmov\\t%d0, %x1
1268    fmov\\t%x0, %d1
1269    fmov\\t%d0, %d1
1270    fmov\\t%d0, %1
1271    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1272    ldr\\t%d0, %1
1273    str\\t%d1, %0
1274    ldr\\t%x0, %1
1275    str\\t%x1, %0
1276    mov\\t%x0, %x1
1277    mov\\t%x0, %1"
1278   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1279                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1280                      fconstd")
1281    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1282 )
1283
1284 (define_split
1285   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1286         (match_operand:GPF_HF 1 "general_operand"))]
1287   "can_create_pseudo_p ()
1288    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1289    && !aarch64_float_const_representable_p (operands[1])
1290    &&  aarch64_float_const_rtx_p (operands[1])"
1291   [(const_int 0)]
1292   {
1293     unsigned HOST_WIDE_INT ival;
1294     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1295       FAIL;
1296
1297     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1298     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1299     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1300     DONE;
1301   }
1302 )
1303
1304 (define_insn "*movtf_aarch64"
1305   [(set (match_operand:TF 0
1306          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1307         (match_operand:TF 1
1308          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1309   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1310     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1311   "@
1312    mov\\t%0.16b, %1.16b
1313    #
1314    #
1315    #
1316    movi\\t%0.2d, #0
1317    fmov\\t%s0, wzr
1318    ldr\\t%q0, %1
1319    str\\t%q1, %0
1320    ldp\\t%0, %H0, %1
1321    stp\\t%1, %H1, %0
1322    stp\\txzr, xzr, %0"
1323   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1324                      f_loadd,f_stored,load_16,store_16,store_16")
1325    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1326    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1327 )
1328
1329 (define_split
1330    [(set (match_operand:TF 0 "register_operand" "")
1331          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1332   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1333   [(const_int 0)]
1334   {
1335     aarch64_split_128bit_move (operands[0], operands[1]);
1336     DONE;
1337   }
1338 )
1339
1340 ;; 0 is dst
1341 ;; 1 is src
1342 ;; 2 is size of move in bytes
1343 ;; 3 is alignment
1344
1345 (define_expand "movmemdi"
1346   [(match_operand:BLK 0 "memory_operand")
1347    (match_operand:BLK 1 "memory_operand")
1348    (match_operand:DI 2 "immediate_operand")
1349    (match_operand:DI 3 "immediate_operand")]
1350    "!STRICT_ALIGNMENT"
1351 {
1352   if (aarch64_expand_movmem (operands))
1353     DONE;
1354   FAIL;
1355 }
1356 )
1357
1358 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1359 ;; fairly lax checking on the second memory operation.
1360 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1361   [(set (match_operand:SX 0 "register_operand" "=r,w")
1362         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1363    (set (match_operand:SX2 2 "register_operand" "=r,w")
1364         (match_operand:SX2 3 "memory_operand" "m,m"))]
1365    "rtx_equal_p (XEXP (operands[3], 0),
1366                  plus_constant (Pmode,
1367                                 XEXP (operands[1], 0),
1368                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1369   "@
1370    ldp\\t%w0, %w2, %1
1371    ldp\\t%s0, %s2, %1"
1372   [(set_attr "type" "load_8,neon_load1_2reg")
1373    (set_attr "arch" "*,fp")]
1374 )
1375
1376 ;; Storing different modes that can still be merged
1377 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1378   [(set (match_operand:DX 0 "register_operand" "=r,w")
1379         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1380    (set (match_operand:DX2 2 "register_operand" "=r,w")
1381         (match_operand:DX2 3 "memory_operand" "m,m"))]
1382    "rtx_equal_p (XEXP (operands[3], 0),
1383                  plus_constant (Pmode,
1384                                 XEXP (operands[1], 0),
1385                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1386   "@
1387    ldp\\t%x0, %x2, %1
1388    ldp\\t%d0, %d2, %1"
1389   [(set_attr "type" "load_16,neon_load1_2reg")
1390    (set_attr "arch" "*,fp")]
1391 )
1392
1393 (define_insn "load_pair_dw_tftf"
1394   [(set (match_operand:TF 0 "register_operand" "=w")
1395         (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1396    (set (match_operand:TF 2 "register_operand" "=w")
1397         (match_operand:TF 3 "memory_operand" "m"))]
1398    "TARGET_SIMD
1399     && rtx_equal_p (XEXP (operands[3], 0),
1400                     plus_constant (Pmode,
1401                                    XEXP (operands[1], 0),
1402                                    GET_MODE_SIZE (TFmode)))"
1403   "ldp\\t%q0, %q2, %1"
1404   [(set_attr "type" "neon_ldp_q")
1405    (set_attr "fp" "yes")]
1406 )
1407
1408 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1409 ;; fairly lax checking on the second memory operation.
1410 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1411   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1412         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1413    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1414         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1415    "rtx_equal_p (XEXP (operands[2], 0),
1416                  plus_constant (Pmode,
1417                                 XEXP (operands[0], 0),
1418                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1419   "@
1420    stp\\t%w1, %w3, %0
1421    stp\\t%s1, %s3, %0"
1422   [(set_attr "type" "store_8,neon_store1_2reg")
1423    (set_attr "arch" "*,fp")]
1424 )
1425
1426 ;; Storing different modes that can still be merged
1427 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1428   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1429         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1430    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1431         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1432    "rtx_equal_p (XEXP (operands[2], 0),
1433                  plus_constant (Pmode,
1434                                 XEXP (operands[0], 0),
1435                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1436   "@
1437    stp\\t%x1, %x3, %0
1438    stp\\t%d1, %d3, %0"
1439   [(set_attr "type" "store_16,neon_store1_2reg")
1440    (set_attr "arch" "*,fp")]
1441 )
1442
1443 (define_insn "store_pair_dw_tftf"
1444   [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1445         (match_operand:TF 1 "register_operand" "w"))
1446    (set (match_operand:TF 2 "memory_operand" "=m")
1447         (match_operand:TF 3 "register_operand" "w"))]
1448    "TARGET_SIMD &&
1449     rtx_equal_p (XEXP (operands[2], 0),
1450                  plus_constant (Pmode,
1451                                 XEXP (operands[0], 0),
1452                                 GET_MODE_SIZE (TFmode)))"
1453   "stp\\t%q1, %q3, %0"
1454   [(set_attr "type" "neon_stp_q")
1455    (set_attr "fp" "yes")]
1456 )
1457
1458 ;; Load pair with post-index writeback.  This is primarily used in function
1459 ;; epilogues.
1460 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1461   [(parallel
1462     [(set (match_operand:P 0 "register_operand" "=k")
1463           (plus:P (match_operand:P 1 "register_operand" "0")
1464                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1465      (set (match_operand:GPI 2 "register_operand" "=r")
1466           (mem:GPI (match_dup 1)))
1467      (set (match_operand:GPI 3 "register_operand" "=r")
1468           (mem:GPI (plus:P (match_dup 1)
1469                    (match_operand:P 5 "const_int_operand" "n"))))])]
1470   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1471   "ldp\\t%<w>2, %<w>3, [%1], %4"
1472   [(set_attr "type" "load_<ldpstp_sz>")]
1473 )
1474
1475 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1476   [(parallel
1477     [(set (match_operand:P 0 "register_operand" "=k")
1478           (plus:P (match_operand:P 1 "register_operand" "0")
1479                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1480      (set (match_operand:GPF 2 "register_operand" "=w")
1481           (mem:GPF (match_dup 1)))
1482      (set (match_operand:GPF 3 "register_operand" "=w")
1483           (mem:GPF (plus:P (match_dup 1)
1484                    (match_operand:P 5 "const_int_operand" "n"))))])]
1485   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1486   "ldp\\t%<w>2, %<w>3, [%1], %4"
1487   [(set_attr "type" "neon_load1_2reg")]
1488 )
1489
1490 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1491   [(parallel
1492     [(set (match_operand:P 0 "register_operand" "=k")
1493           (plus:P (match_operand:P 1 "register_operand" "0")
1494                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1495      (set (match_operand:TX 2 "register_operand" "=w")
1496           (mem:TX (match_dup 1)))
1497      (set (match_operand:TX 3 "register_operand" "=w")
1498           (mem:TX (plus:P (match_dup 1)
1499                           (match_operand:P 5 "const_int_operand" "n"))))])]
1500   "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1501   "ldp\\t%q2, %q3, [%1], %4"
1502   [(set_attr "type" "neon_ldp_q")]
1503 )
1504
1505 ;; Store pair with pre-index writeback.  This is primarily used in function
1506 ;; prologues.
1507 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1508   [(parallel
1509     [(set (match_operand:P 0 "register_operand" "=&k")
1510           (plus:P (match_operand:P 1 "register_operand" "0")
1511                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1512      (set (mem:GPI (plus:P (match_dup 0)
1513                    (match_dup 4)))
1514           (match_operand:GPI 2 "register_operand" "r"))
1515      (set (mem:GPI (plus:P (match_dup 0)
1516                    (match_operand:P 5 "const_int_operand" "n")))
1517           (match_operand:GPI 3 "register_operand" "r"))])]
1518   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1519   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1520   [(set_attr "type" "store_<ldpstp_sz>")]
1521 )
1522
1523 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1524   [(parallel
1525     [(set (match_operand:P 0 "register_operand" "=&k")
1526           (plus:P (match_operand:P 1 "register_operand" "0")
1527                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1528      (set (mem:GPF (plus:P (match_dup 0)
1529                    (match_dup 4)))
1530           (match_operand:GPF 2 "register_operand" "w"))
1531      (set (mem:GPF (plus:P (match_dup 0)
1532                    (match_operand:P 5 "const_int_operand" "n")))
1533           (match_operand:GPF 3 "register_operand" "w"))])]
1534   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1535   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1536   [(set_attr "type" "neon_store1_2reg<q>")]
1537 )
1538
1539 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1540   [(parallel
1541     [(set (match_operand:P 0 "register_operand" "=&k")
1542           (plus:P (match_operand:P 1 "register_operand" "0")
1543                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1544      (set (mem:TX (plus:P (match_dup 0)
1545                           (match_dup 4)))
1546           (match_operand:TX 2 "register_operand" "w"))
1547      (set (mem:TX (plus:P (match_dup 0)
1548                           (match_operand:P 5 "const_int_operand" "n")))
1549           (match_operand:TX 3 "register_operand" "w"))])]
1550   "TARGET_SIMD
1551    && INTVAL (operands[5])
1552       == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1553   "stp\\t%q2, %q3, [%0, %4]!"
1554   [(set_attr "type" "neon_stp_q")]
1555 )
1556
1557 ;; -------------------------------------------------------------------
1558 ;; Sign/Zero extension
1559 ;; -------------------------------------------------------------------
1560
1561 (define_expand "<optab>sidi2"
1562   [(set (match_operand:DI 0 "register_operand")
1563         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1564   ""
1565 )
1566
1567 (define_insn "*extendsidi2_aarch64"
1568   [(set (match_operand:DI 0 "register_operand" "=r,r")
1569         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1570   ""
1571   "@
1572    sxtw\t%0, %w1
1573    ldrsw\t%0, %1"
1574   [(set_attr "type" "extend,load_4")]
1575 )
1576
1577 (define_insn "*load_pair_extendsidi2_aarch64"
1578   [(set (match_operand:DI 0 "register_operand" "=r")
1579         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1580    (set (match_operand:DI 2 "register_operand" "=r")
1581         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1582   "rtx_equal_p (XEXP (operands[3], 0),
1583                 plus_constant (Pmode,
1584                                XEXP (operands[1], 0),
1585                                GET_MODE_SIZE (SImode)))"
1586   "ldpsw\\t%0, %2, %1"
1587   [(set_attr "type" "load_8")]
1588 )
1589
1590 (define_insn "*zero_extendsidi2_aarch64"
1591   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1592         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1593   ""
1594   "@
1595    uxtw\t%0, %w1
1596    ldr\t%w0, %1
1597    fmov\t%s0, %w1
1598    ldr\t%s0, %1
1599    fmov\t%w0, %s1
1600    fmov\t%s0, %s1"
1601   [(set_attr "type" "extend,load_4,f_mcr,f_loads,f_mrc,fmov")
1602    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1603 )
1604
1605 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1606   [(set (match_operand:DI 0 "register_operand" "=r,w")
1607         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1608    (set (match_operand:DI 2 "register_operand" "=r,w")
1609         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1610   "rtx_equal_p (XEXP (operands[3], 0),
1611                 plus_constant (Pmode,
1612                                XEXP (operands[1], 0),
1613                                GET_MODE_SIZE (SImode)))"
1614   "@
1615    ldp\t%w0, %w2, %1
1616    ldp\t%s0, %s2, %1"
1617   [(set_attr "type" "load_8,neon_load1_2reg")
1618    (set_attr "arch" "*,fp")]
1619 )
1620
1621 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1622   [(set (match_operand:GPI 0 "register_operand")
1623         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1624   ""
1625 )
1626
1627 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1628   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1629         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1630   ""
1631   "@
1632    sxt<SHORT:size>\t%<GPI:w>0, %w1
1633    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1634   [(set_attr "type" "extend,load_4")]
1635 )
1636
1637 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1638   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1639         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1640   ""
1641   "@
1642    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1643    ldr<SHORT:size>\t%w0, %1
1644    ldr\t%<SHORT:size>0, %1"
1645   [(set_attr "type" "logic_imm,load_4,f_loads")
1646    (set_attr "arch" "*,*,fp")]
1647 )
1648
1649 (define_expand "<optab>qihi2"
1650   [(set (match_operand:HI 0 "register_operand")
1651         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1652   ""
1653 )
1654
1655 (define_insn "*extendqihi2_aarch64"
1656   [(set (match_operand:HI 0 "register_operand" "=r,r")
1657         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1658   ""
1659   "@
1660    sxtb\t%w0, %w1
1661    ldrsb\t%w0, %1"
1662   [(set_attr "type" "extend,load_4")]
1663 )
1664
1665 (define_insn "*zero_extendqihi2_aarch64"
1666   [(set (match_operand:HI 0 "register_operand" "=r,r")
1667         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1668   ""
1669   "@
1670    and\t%w0, %w1, 255
1671    ldrb\t%w0, %1"
1672   [(set_attr "type" "logic_imm,load_4")]
1673 )
1674
1675 ;; -------------------------------------------------------------------
1676 ;; Simple arithmetic
1677 ;; -------------------------------------------------------------------
1678
1679 (define_expand "add<mode>3"
1680   [(set
1681     (match_operand:GPI 0 "register_operand" "")
1682     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1683               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1684   ""
1685 {
1686   /* If operands[1] is a subreg extract the inner RTX.  */
1687   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1688
1689   /* If the constant is too large for a single instruction and isn't frame
1690      based, split off the immediate so it is available for CSE.  */
1691   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1692       && can_create_pseudo_p ()
1693       && (!REG_P (op1)
1694          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1695     operands[2] = force_reg (<MODE>mode, operands[2]);
1696   /* Expand polynomial additions now if the destination is the stack
1697      pointer, since we don't want to use that as a temporary.  */
1698   else if (operands[0] == stack_pointer_rtx
1699            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1700     {
1701       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1702                                 operands[2], NULL_RTX, NULL_RTX);
1703       DONE;
1704     }
1705 })
1706
1707 (define_insn "*add<mode>3_aarch64"
1708   [(set
1709     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1710     (plus:GPI
1711      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1712      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1713   ""
1714   "@
1715   add\\t%<w>0, %<w>1, %2
1716   add\\t%<w>0, %<w>1, %<w>2
1717   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1718   sub\\t%<w>0, %<w>1, #%n2
1719   #
1720   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1721   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1722   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1723    (set_attr "arch" "*,*,simd,*,*,*")]
1724 )
1725
1726 ;; zero_extend version of above
1727 (define_insn "*addsi3_aarch64_uxtw"
1728   [(set
1729     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1730     (zero_extend:DI
1731      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1732               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1733   ""
1734   "@
1735   add\\t%w0, %w1, %2
1736   add\\t%w0, %w1, %w2
1737   sub\\t%w0, %w1, #%n2
1738   #"
1739   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1740 )
1741
1742 ;; If there's a free register, and we can load the constant with a
1743 ;; single instruction, do so.  This has a chance to improve scheduling.
1744 (define_peephole2
1745   [(match_scratch:GPI 3 "r")
1746    (set (match_operand:GPI 0 "register_operand")
1747         (plus:GPI
1748           (match_operand:GPI 1 "register_operand")
1749           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1750   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1751   [(set (match_dup 3) (match_dup 2))
1752    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1753 )
1754
1755 (define_peephole2
1756   [(match_scratch:SI 3 "r")
1757    (set (match_operand:DI 0 "register_operand")
1758         (zero_extend:DI
1759           (plus:SI
1760             (match_operand:SI 1 "register_operand")
1761             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1762   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1763   [(set (match_dup 3) (match_dup 2))
1764    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1765 )
1766
1767 ;; After peephole2 has had a chance to run, split any remaining long
1768 ;; additions into two add immediates.
1769 (define_split
1770   [(set (match_operand:GPI 0 "register_operand")
1771         (plus:GPI
1772           (match_operand:GPI 1 "register_operand")
1773           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1774   "epilogue_completed"
1775   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1776    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1777   {
1778     HOST_WIDE_INT i = INTVAL (operands[2]);
1779     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1780     operands[3] = GEN_INT (i - s);
1781     operands[4] = GEN_INT (s);
1782   }
1783 )
1784
1785 ;; Match addition of polynomial offsets that require one temporary, for which
1786 ;; we can use the early-clobbered destination register.  This is a separate
1787 ;; pattern so that the early clobber doesn't affect register allocation
1788 ;; for other forms of addition.  However, we still need to provide an
1789 ;; all-register alternative, in case the offset goes out of range after
1790 ;; elimination.  For completeness we might as well provide all GPR-based
1791 ;; alternatives from the main pattern.
1792 ;;
1793 ;; We don't have a pattern for additions requiring two temporaries since at
1794 ;; present LRA doesn't allow new scratches to be added during elimination.
1795 ;; Such offsets should be rare anyway.
1796 ;;
1797 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1798 ;; here would go away.  We could just handle all polynomial constants in
1799 ;; this pattern.
1800 (define_insn_and_split "*add<mode>3_poly_1"
1801   [(set
1802     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1803     (plus:GPI
1804      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1805      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1806   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1807   "@
1808   add\\t%<w>0, %<w>1, %2
1809   add\\t%<w>0, %<w>1, %<w>2
1810   sub\\t%<w>0, %<w>1, #%n2
1811   #
1812   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1813   #"
1814   "&& epilogue_completed
1815    && !reg_overlap_mentioned_p (operands[0], operands[1])
1816    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1817   [(const_int 0)]
1818   {
1819     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1820                               operands[2], operands[0], NULL_RTX);
1821     DONE;
1822   }
1823   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1824   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1825 )
1826
1827 (define_split
1828   [(set (match_operand:DI 0 "register_operand")
1829         (zero_extend:DI
1830           (plus:SI
1831             (match_operand:SI 1 "register_operand")
1832             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1833   "epilogue_completed"
1834   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1835    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1836   {
1837     HOST_WIDE_INT i = INTVAL (operands[2]);
1838     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1839     operands[3] = GEN_INT (i - s);
1840     operands[4] = GEN_INT (s);
1841     operands[5] = gen_lowpart (SImode, operands[0]);
1842   }
1843 )
1844
1845 (define_expand "addv<mode>4"
1846   [(match_operand:GPI 0 "register_operand")
1847    (match_operand:GPI 1 "register_operand")
1848    (match_operand:GPI 2 "aarch64_plus_operand")
1849    (label_ref (match_operand 3 "" ""))]
1850   ""
1851 {
1852   if (CONST_INT_P (operands[2]))
1853     emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1854                                             operands[2]));
1855   else
1856     emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1857   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1858
1859   DONE;
1860 })
1861
1862 (define_expand "uaddv<mode>4"
1863   [(match_operand:GPI 0 "register_operand")
1864    (match_operand:GPI 1 "register_operand")
1865    (match_operand:GPI 2 "register_operand")
1866    (label_ref (match_operand 3 "" ""))]
1867   ""
1868 {
1869   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1870   aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1871
1872   DONE;
1873 })
1874
1875 (define_expand "addti3"
1876   [(set (match_operand:TI 0 "register_operand" "")
1877         (plus:TI (match_operand:TI 1 "register_operand" "")
1878                  (match_operand:TI 2 "aarch64_reg_or_imm" "")))]
1879   ""
1880 {
1881   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1882
1883   aarch64_addti_scratch_regs (operands[1], operands[2],
1884                               &low_dest, &op1_low, &op2_low,
1885                               &high_dest, &op1_high, &op2_high);
1886
1887   if (op2_low == const0_rtx)
1888     {
1889       low_dest = op1_low;
1890       if (!aarch64_pluslong_operand (op2_high, DImode))
1891         op2_high = force_reg (DImode, op2_high);
1892       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1893     }
1894   else
1895     {
1896       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1897                                       force_reg (DImode, op2_low)));
1898       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1899                                      force_reg (DImode, op2_high)));
1900     }
1901
1902   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1903   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1904
1905   DONE;
1906 })
1907
1908 (define_expand "addvti4"
1909   [(match_operand:TI 0 "register_operand" "")
1910    (match_operand:TI 1 "register_operand" "")
1911    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1912    (label_ref (match_operand 3 "" ""))]
1913   ""
1914 {
1915   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1916
1917   aarch64_addti_scratch_regs (operands[1], operands[2],
1918                               &low_dest, &op1_low, &op2_low,
1919                               &high_dest, &op1_high, &op2_high);
1920
1921   if (op2_low == const0_rtx)
1922     {
1923       low_dest = op1_low;
1924       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1925                                       force_reg (DImode, op2_high)));
1926     }
1927   else
1928     {
1929       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1930                                       force_reg (DImode, op2_low)));
1931       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
1932                                       force_reg (DImode, op2_high)));
1933     }
1934
1935   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1936   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1937
1938   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1939   DONE;
1940 })
1941
1942 (define_expand "uaddvti4"
1943   [(match_operand:TI 0 "register_operand" "")
1944    (match_operand:TI 1 "register_operand" "")
1945    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1946    (label_ref (match_operand 3 "" ""))]
1947   ""
1948 {
1949   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1950
1951   aarch64_addti_scratch_regs (operands[1], operands[2],
1952                               &low_dest, &op1_low, &op2_low,
1953                               &high_dest, &op1_high, &op2_high);
1954
1955   if (op2_low == const0_rtx)
1956     {
1957       low_dest = op1_low;
1958       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
1959                                       force_reg (DImode, op2_high)));
1960     }
1961   else
1962     {
1963       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1964                                       force_reg (DImode, op2_low)));
1965       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
1966                                       force_reg (DImode, op2_high)));
1967     }
1968
1969   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1970   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1971
1972   aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1973   DONE;
1974  })
1975
1976 (define_insn "add<mode>3_compare0"
1977   [(set (reg:CC_NZ CC_REGNUM)
1978         (compare:CC_NZ
1979          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1980                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1981          (const_int 0)))
1982    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1983         (plus:GPI (match_dup 1) (match_dup 2)))]
1984   ""
1985   "@
1986   adds\\t%<w>0, %<w>1, %<w>2
1987   adds\\t%<w>0, %<w>1, %2
1988   subs\\t%<w>0, %<w>1, #%n2"
1989   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1990 )
1991
1992 ;; zero_extend version of above
1993 (define_insn "*addsi3_compare0_uxtw"
1994   [(set (reg:CC_NZ CC_REGNUM)
1995         (compare:CC_NZ
1996          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1997                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1998          (const_int 0)))
1999    (set (match_operand:DI 0 "register_operand" "=r,r,r")
2000         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2001   ""
2002   "@
2003   adds\\t%w0, %w1, %w2
2004   adds\\t%w0, %w1, %2
2005   subs\\t%w0, %w1, #%n2"
2006   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2007 )
2008
2009 (define_insn "*add<mode>3_compareC_cconly_imm"
2010   [(set (reg:CC_C CC_REGNUM)
2011         (ne:CC_C
2012           (plus:<DWI>
2013             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2014             (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
2015           (zero_extend:<DWI>
2016             (plus:GPI
2017               (match_dup 0)
2018               (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
2019   "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
2020                                  <MODE>mode, operands[1])"
2021   "@
2022   cmn\\t%<w>0, %1
2023   cmp\\t%<w>0, #%n1"
2024   [(set_attr "type" "alus_imm")]
2025 )
2026
2027 (define_insn "*add<mode>3_compareC_cconly"
2028   [(set (reg:CC_C CC_REGNUM)
2029         (ne:CC_C
2030           (plus:<DWI>
2031             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2032             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2033           (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2034   ""
2035   "cmn\\t%<w>0, %<w>1"
2036   [(set_attr "type" "alus_sreg")]
2037 )
2038
2039 (define_insn "*add<mode>3_compareC_imm"
2040   [(set (reg:CC_C CC_REGNUM)
2041         (ne:CC_C
2042           (plus:<DWI>
2043             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
2044             (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
2045           (zero_extend:<DWI>
2046             (plus:GPI
2047               (match_dup 1)
2048               (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
2049    (set (match_operand:GPI 0 "register_operand" "=r,r")
2050         (plus:GPI (match_dup 1) (match_dup 2)))]
2051   "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
2052                                  <MODE>mode, operands[2])"
2053   "@
2054   adds\\t%<w>0, %<w>1, %2
2055   subs\\t%<w>0, %<w>1, #%n2"
2056   [(set_attr "type" "alus_imm")]
2057 )
2058
2059 (define_insn "add<mode>3_compareC"
2060   [(set (reg:CC_C CC_REGNUM)
2061         (compare:CC_C
2062           (plus:<DWI>
2063             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
2064             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2065           (zero_extend:<DWI>
2066             (plus:GPI (match_dup 1) (match_dup 2)))))
2067    (set (match_operand:GPI 0 "register_operand" "=r")
2068         (plus:GPI (match_dup 1) (match_dup 2)))]
2069   ""
2070   "adds\\t%<w>0, %<w>1, %<w>2"
2071   [(set_attr "type" "alus_sreg")]
2072 )
2073
2074 (define_insn "*add<mode>3_compareV_cconly_imm"
2075   [(set (reg:CC_V CC_REGNUM)
2076         (compare:CC_V
2077           (plus:<DWI>
2078             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2079             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2080           (sign_extend:<DWI>
2081            (plus:GPI
2082             (match_dup 0)
2083             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2084   "INTVAL (operands[1]) == INTVAL (operands[2])"
2085   "@
2086   cmn\\t%<w>0, %<w>1
2087   cmp\\t%<w>0, #%n1"
2088   [(set_attr "type" "alus_imm")]
2089 )
2090
2091 (define_insn "*add<mode>3_compareV_cconly"
2092   [(set (reg:CC_V CC_REGNUM)
2093         (compare:CC_V
2094           (plus:<DWI>
2095             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2096             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2097           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2098   ""
2099   "cmn\\t%<w>0, %<w>1"
2100   [(set_attr "type" "alus_sreg")]
2101 )
2102
2103 (define_insn "add<mode>3_compareV_imm"
2104   [(set (reg:CC_V CC_REGNUM)
2105         (compare:CC_V
2106           (plus:<DWI>
2107             (sign_extend:<DWI>
2108               (match_operand:GPI 1 "register_operand" "r,r"))
2109             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2110           (sign_extend:<DWI>
2111             (plus:GPI (match_dup 1) (match_dup 2)))))
2112    (set (match_operand:GPI 0 "register_operand" "=r,r")
2113         (plus:GPI (match_dup 1) (match_dup 2)))]
2114    ""
2115    "@
2116    adds\\t%<w>0, %<w>1, %<w>2
2117    subs\\t%<w>0, %<w>1, #%n2"
2118   [(set_attr "type" "alus_imm,alus_imm")]
2119 )
2120
2121 (define_insn "add<mode>3_compareV"
2122   [(set (reg:CC_V CC_REGNUM)
2123         (compare:CC_V
2124           (plus:<DWI>
2125             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
2126             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2127           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2128    (set (match_operand:GPI 0 "register_operand" "=r")
2129         (plus:GPI (match_dup 1) (match_dup 2)))]
2130   ""
2131   "adds\\t%<w>0, %<w>1, %<w>2"
2132   [(set_attr "type" "alus_sreg")]
2133 )
2134
2135 (define_insn "*adds_shift_imm_<mode>"
2136   [(set (reg:CC_NZ CC_REGNUM)
2137         (compare:CC_NZ
2138          (plus:GPI (ASHIFT:GPI
2139                     (match_operand:GPI 1 "register_operand" "r")
2140                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2141                    (match_operand:GPI 3 "register_operand" "r"))
2142          (const_int 0)))
2143    (set (match_operand:GPI 0 "register_operand" "=r")
2144         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2145                   (match_dup 3)))]
2146   ""
2147   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2148   [(set_attr "type" "alus_shift_imm")]
2149 )
2150
2151 (define_insn "*subs_shift_imm_<mode>"
2152   [(set (reg:CC_NZ CC_REGNUM)
2153         (compare:CC_NZ
2154          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2155                     (ASHIFT:GPI
2156                      (match_operand:GPI 2 "register_operand" "r")
2157                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2158          (const_int 0)))
2159    (set (match_operand:GPI 0 "register_operand" "=r")
2160         (minus:GPI (match_dup 1)
2161                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2162   ""
2163   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2164   [(set_attr "type" "alus_shift_imm")]
2165 )
2166
2167 (define_insn "*adds_mul_imm_<mode>"
2168   [(set (reg:CC_NZ CC_REGNUM)
2169         (compare:CC_NZ
2170          (plus:GPI (mult:GPI
2171                     (match_operand:GPI 1 "register_operand" "r")
2172                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2173                    (match_operand:GPI 3 "register_operand" "r"))
2174          (const_int 0)))
2175    (set (match_operand:GPI 0 "register_operand" "=r")
2176         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2177                   (match_dup 3)))]
2178   ""
2179   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2180   [(set_attr "type" "alus_shift_imm")]
2181 )
2182
2183 (define_insn "*subs_mul_imm_<mode>"
2184   [(set (reg:CC_NZ CC_REGNUM)
2185         (compare:CC_NZ
2186          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2187                     (mult:GPI
2188                      (match_operand:GPI 2 "register_operand" "r")
2189                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2190          (const_int 0)))
2191    (set (match_operand:GPI 0 "register_operand" "=r")
2192         (minus:GPI (match_dup 1)
2193                    (mult:GPI (match_dup 2) (match_dup 3))))]
2194   ""
2195   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2196   [(set_attr "type" "alus_shift_imm")]
2197 )
2198
2199 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2200   [(set (reg:CC_NZ CC_REGNUM)
2201         (compare:CC_NZ
2202          (plus:GPI
2203           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2204           (match_operand:GPI 2 "register_operand" "r"))
2205         (const_int 0)))
2206    (set (match_operand:GPI 0 "register_operand" "=r")
2207         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2208   ""
2209   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2210   [(set_attr "type" "alus_ext")]
2211 )
2212
2213 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2214   [(set (reg:CC_NZ CC_REGNUM)
2215         (compare:CC_NZ
2216          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2217                     (ANY_EXTEND:GPI
2218                      (match_operand:ALLX 2 "register_operand" "r")))
2219         (const_int 0)))
2220    (set (match_operand:GPI 0 "register_operand" "=r")
2221         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2222   ""
2223   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2224   [(set_attr "type" "alus_ext")]
2225 )
2226
2227 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2228   [(set (reg:CC_NZ CC_REGNUM)
2229         (compare:CC_NZ
2230          (plus:GPI (ashift:GPI
2231                     (ANY_EXTEND:GPI
2232                      (match_operand:ALLX 1 "register_operand" "r"))
2233                     (match_operand 2 "aarch64_imm3" "Ui3"))
2234                    (match_operand:GPI 3 "register_operand" "r"))
2235          (const_int 0)))
2236    (set (match_operand:GPI 0 "register_operand" "=rk")
2237         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2238                               (match_dup 2))
2239                   (match_dup 3)))]
2240   ""
2241   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2242   [(set_attr "type" "alus_ext")]
2243 )
2244
2245 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2246   [(set (reg:CC_NZ CC_REGNUM)
2247         (compare:CC_NZ
2248          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2249                     (ashift:GPI
2250                      (ANY_EXTEND:GPI
2251                       (match_operand:ALLX 2 "register_operand" "r"))
2252                      (match_operand 3 "aarch64_imm3" "Ui3")))
2253          (const_int 0)))
2254    (set (match_operand:GPI 0 "register_operand" "=rk")
2255         (minus:GPI (match_dup 1)
2256                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2257                                (match_dup 3))))]
2258   ""
2259   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2260   [(set_attr "type" "alus_ext")]
2261 )
2262
2263 (define_insn "*adds_<optab><mode>_multp2"
2264   [(set (reg:CC_NZ CC_REGNUM)
2265         (compare:CC_NZ
2266          (plus:GPI (ANY_EXTRACT:GPI
2267                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2268                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2269                     (match_operand 3 "const_int_operand" "n")
2270                     (const_int 0))
2271                    (match_operand:GPI 4 "register_operand" "r"))
2272         (const_int 0)))
2273    (set (match_operand:GPI 0 "register_operand" "=r")
2274         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2275                                    (match_dup 3)
2276                                    (const_int 0))
2277                   (match_dup 4)))]
2278   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2279   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2280   [(set_attr "type" "alus_ext")]
2281 )
2282
2283 (define_insn "*subs_<optab><mode>_multp2"
2284   [(set (reg:CC_NZ CC_REGNUM)
2285         (compare:CC_NZ
2286          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2287                     (ANY_EXTRACT:GPI
2288                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2289                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2290                      (match_operand 3 "const_int_operand" "n")
2291                      (const_int 0)))
2292         (const_int 0)))
2293    (set (match_operand:GPI 0 "register_operand" "=r")
2294         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2295                                   (mult:GPI (match_dup 1) (match_dup 2))
2296                                   (match_dup 3)
2297                                   (const_int 0))))]
2298   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2299   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2300   [(set_attr "type" "alus_ext")]
2301 )
2302
2303 (define_insn "*add<mode>3nr_compare0"
2304   [(set (reg:CC_NZ CC_REGNUM)
2305         (compare:CC_NZ
2306          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2307                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2308          (const_int 0)))]
2309   ""
2310   "@
2311   cmn\\t%<w>0, %<w>1
2312   cmn\\t%<w>0, %1
2313   cmp\\t%<w>0, #%n1"
2314   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2315 )
2316
2317 (define_insn "aarch64_sub<mode>_compare0"
2318   [(set (reg:CC_NZ CC_REGNUM)
2319         (compare:CC_NZ
2320          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2321                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2322          (const_int 0)))]
2323   ""
2324   "cmp\\t%<w>0, %<w>1"
2325   [(set_attr "type" "alus_sreg")]
2326 )
2327
2328 (define_insn "*compare_neg<mode>"
2329   [(set (reg:CC_Z CC_REGNUM)
2330         (compare:CC_Z
2331          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2332          (match_operand:GPI 1 "register_operand" "r")))]
2333   ""
2334   "cmn\\t%<w>1, %<w>0"
2335   [(set_attr "type" "alus_sreg")]
2336 )
2337
2338 (define_insn "*add_<shift>_<mode>"
2339   [(set (match_operand:GPI 0 "register_operand" "=r")
2340         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2341                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2342                   (match_operand:GPI 3 "register_operand" "r")))]
2343   ""
2344   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2345   [(set_attr "type" "alu_shift_imm")]
2346 )
2347
2348 ;; zero_extend version of above
2349 (define_insn "*add_<shift>_si_uxtw"
2350   [(set (match_operand:DI 0 "register_operand" "=r")
2351         (zero_extend:DI
2352          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2353                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2354                   (match_operand:SI 3 "register_operand" "r"))))]
2355   ""
2356   "add\\t%w0, %w3, %w1, <shift> %2"
2357   [(set_attr "type" "alu_shift_imm")]
2358 )
2359
2360 (define_insn "*add_mul_imm_<mode>"
2361   [(set (match_operand:GPI 0 "register_operand" "=r")
2362         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2363                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2364                   (match_operand:GPI 3 "register_operand" "r")))]
2365   ""
2366   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2367   [(set_attr "type" "alu_shift_imm")]
2368 )
2369
2370 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2371   [(set (match_operand:GPI 0 "register_operand" "=rk")
2372         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2373                   (match_operand:GPI 2 "register_operand" "r")))]
2374   ""
2375   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2376   [(set_attr "type" "alu_ext")]
2377 )
2378
2379 ;; zero_extend version of above
2380 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2381   [(set (match_operand:DI 0 "register_operand" "=rk")
2382         (zero_extend:DI
2383          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2384                   (match_operand:GPI 2 "register_operand" "r"))))]
2385   ""
2386   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2387   [(set_attr "type" "alu_ext")]
2388 )
2389
2390 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2391   [(set (match_operand:GPI 0 "register_operand" "=rk")
2392         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2393                                (match_operand:ALLX 1 "register_operand" "r"))
2394                               (match_operand 2 "aarch64_imm3" "Ui3"))
2395                   (match_operand:GPI 3 "register_operand" "r")))]
2396   ""
2397   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2398   [(set_attr "type" "alu_ext")]
2399 )
2400
2401 ;; zero_extend version of above
2402 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2403   [(set (match_operand:DI 0 "register_operand" "=rk")
2404         (zero_extend:DI
2405          (plus:SI (ashift:SI (ANY_EXTEND:SI
2406                               (match_operand:SHORT 1 "register_operand" "r"))
2407                              (match_operand 2 "aarch64_imm3" "Ui3"))
2408                   (match_operand:SI 3 "register_operand" "r"))))]
2409   ""
2410   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2411   [(set_attr "type" "alu_ext")]
2412 )
2413
2414 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2415   [(set (match_operand:GPI 0 "register_operand" "=rk")
2416         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2417                              (match_operand:ALLX 1 "register_operand" "r"))
2418                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2419                   (match_operand:GPI 3 "register_operand" "r")))]
2420   ""
2421   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2422   [(set_attr "type" "alu_ext")]
2423 )
2424
2425 ;; zero_extend version of above
2426 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2427   [(set (match_operand:DI 0 "register_operand" "=rk")
2428         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2429                              (match_operand:SHORT 1 "register_operand" "r"))
2430                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2431                   (match_operand:SI 3 "register_operand" "r"))))]
2432   ""
2433   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2434   [(set_attr "type" "alu_ext")]
2435 )
2436
2437 (define_insn "*add_<optab><mode>_multp2"
2438   [(set (match_operand:GPI 0 "register_operand" "=rk")
2439         (plus:GPI (ANY_EXTRACT:GPI
2440                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2441                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2442                    (match_operand 3 "const_int_operand" "n")
2443                    (const_int 0))
2444                   (match_operand:GPI 4 "register_operand" "r")))]
2445   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2446   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2447   [(set_attr "type" "alu_ext")]
2448 )
2449
2450 ;; zero_extend version of above
2451 (define_insn "*add_<optab>si_multp2_uxtw"
2452   [(set (match_operand:DI 0 "register_operand" "=rk")
2453         (zero_extend:DI
2454          (plus:SI (ANY_EXTRACT:SI
2455                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2456                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2457                    (match_operand 3 "const_int_operand" "n")
2458                    (const_int 0))
2459                   (match_operand:SI 4 "register_operand" "r"))))]
2460   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2461   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2462   [(set_attr "type" "alu_ext")]
2463 )
2464
2465 (define_expand "add<mode>3_carryin"
2466   [(set (match_operand:GPI 0 "register_operand")
2467         (plus:GPI
2468           (plus:GPI
2469             (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2470             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2471           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2472    ""
2473    ""
2474 )
2475
2476 ;; Note that add with carry with two zero inputs is matched by cset,
2477 ;; and that add with carry with one zero input is matched by cinc.
2478
2479 (define_insn "*add<mode>3_carryin"
2480   [(set (match_operand:GPI 0 "register_operand" "=r")
2481         (plus:GPI
2482           (plus:GPI
2483             (match_operand:GPI 3 "aarch64_carry_operation" "")
2484             (match_operand:GPI 1 "register_operand" "r"))
2485           (match_operand:GPI 2 "register_operand" "r")))]
2486    ""
2487    "adc\\t%<w>0, %<w>1, %<w>2"
2488   [(set_attr "type" "adc_reg")]
2489 )
2490
2491 ;; zero_extend version of above
2492 (define_insn "*addsi3_carryin_uxtw"
2493   [(set (match_operand:DI 0 "register_operand" "=r")
2494         (zero_extend:DI
2495           (plus:SI
2496             (plus:SI
2497               (match_operand:SI 3 "aarch64_carry_operation" "")
2498               (match_operand:SI 1 "register_operand" "r"))
2499             (match_operand:SI 2 "register_operand" "r"))))]
2500    ""
2501    "adc\\t%w0, %w1, %w2"
2502   [(set_attr "type" "adc_reg")]
2503 )
2504
2505 (define_expand "add<mode>3_carryinC"
2506   [(parallel
2507      [(set (match_dup 3)
2508            (compare:CC_C
2509              (plus:<DWI>
2510                (plus:<DWI>
2511                  (match_dup 4)
2512                  (zero_extend:<DWI>
2513                    (match_operand:GPI 1 "register_operand" "")))
2514                (zero_extend:<DWI>
2515                  (match_operand:GPI 2 "register_operand" "")))
2516            (zero_extend:<DWI>
2517              (plus:GPI
2518                (plus:GPI (match_dup 5) (match_dup 1))
2519                (match_dup 2)))))
2520       (set (match_operand:GPI 0 "register_operand")
2521            (plus:GPI
2522              (plus:GPI (match_dup 5) (match_dup 1))
2523              (match_dup 2)))])]
2524    ""
2525 {
2526   operands[3] = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2527   operands[4] = gen_rtx_NE (<DWI>mode, operands[3], const0_rtx);
2528   operands[5] = gen_rtx_NE (<MODE>mode, operands[3], const0_rtx);
2529 })
2530
2531 (define_insn "*add<mode>3_carryinC_zero"
2532   [(set (reg:CC_C CC_REGNUM)
2533         (compare:CC_C
2534           (plus:<DWI>
2535             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2536             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2537           (zero_extend:<DWI>
2538             (plus:GPI
2539               (match_operand:GPI 3 "aarch64_carry_operation" "")
2540               (match_dup 1)))))
2541    (set (match_operand:GPI 0 "register_operand" "=r")
2542         (plus:GPI (match_dup 3) (match_dup 1)))]
2543    ""
2544    "adcs\\t%<w>0, %<w>1, <w>zr"
2545   [(set_attr "type" "adc_reg")]
2546 )
2547
2548 (define_insn "*add<mode>3_carryinC"
2549   [(set (reg:CC_C CC_REGNUM)
2550         (compare:CC_C
2551           (plus:<DWI>
2552             (plus:<DWI>
2553               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2554               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2555             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2556           (zero_extend:<DWI>
2557             (plus:GPI
2558               (plus:GPI
2559                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2560                 (match_dup 1))
2561               (match_dup 2)))))
2562    (set (match_operand:GPI 0 "register_operand" "=r")
2563         (plus:GPI
2564           (plus:GPI (match_dup 4) (match_dup 1))
2565           (match_dup 2)))]
2566    ""
2567    "adcs\\t%<w>0, %<w>1, %<w>2"
2568   [(set_attr "type" "adc_reg")]
2569 )
2570
2571 (define_expand "add<mode>3_carryinV"
2572   [(parallel
2573      [(set (reg:CC_V CC_REGNUM)
2574            (compare:CC_V
2575              (plus:<DWI>
2576                (plus:<DWI>
2577                  (match_dup 3)
2578                  (sign_extend:<DWI>
2579                    (match_operand:GPI 1 "register_operand" "")))
2580                (sign_extend:<DWI>
2581                  (match_operand:GPI 2 "register_operand" "")))
2582            (sign_extend:<DWI>
2583              (plus:GPI
2584                (plus:GPI (match_dup 4) (match_dup 1))
2585                (match_dup 2)))))
2586       (set (match_operand:GPI 0 "register_operand")
2587            (plus:GPI
2588              (plus:GPI (match_dup 4) (match_dup 1))
2589              (match_dup 2)))])]
2590    ""
2591 {
2592   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2593   operands[3] = gen_rtx_NE (<DWI>mode, cc, const0_rtx);
2594   operands[4] = gen_rtx_NE (<MODE>mode, cc, const0_rtx);
2595 })
2596
2597 (define_insn "*add<mode>3_carryinV_zero"
2598   [(set (reg:CC_V CC_REGNUM)
2599         (compare:CC_V
2600           (plus:<DWI>
2601             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2602             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2603           (sign_extend:<DWI>
2604             (plus:GPI
2605               (match_operand:GPI 3 "aarch64_carry_operation" "")
2606               (match_dup 1)))))
2607    (set (match_operand:GPI 0 "register_operand" "=r")
2608         (plus:GPI (match_dup 3) (match_dup 1)))]
2609    ""
2610    "adcs\\t%<w>0, %<w>1, <w>zr"
2611   [(set_attr "type" "adc_reg")]
2612 )
2613
2614 (define_insn "*add<mode>3_carryinV"
2615   [(set (reg:CC_V CC_REGNUM)
2616         (compare:CC_V
2617           (plus:<DWI>
2618             (plus:<DWI>
2619               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2620               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2621             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2622           (sign_extend:<DWI>
2623             (plus:GPI
2624               (plus:GPI
2625                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2626                 (match_dup 1))
2627               (match_dup 2)))))
2628    (set (match_operand:GPI 0 "register_operand" "=r")
2629         (plus:GPI
2630           (plus:GPI (match_dup 4) (match_dup 1))
2631           (match_dup 2)))]
2632    ""
2633    "adcs\\t%<w>0, %<w>1, %<w>2"
2634   [(set_attr "type" "adc_reg")]
2635 )
2636
2637 (define_insn "*add_uxt<mode>_shift2"
2638   [(set (match_operand:GPI 0 "register_operand" "=rk")
2639         (plus:GPI (and:GPI
2640                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2641                                (match_operand 2 "aarch64_imm3" "Ui3"))
2642                    (match_operand 3 "const_int_operand" "n"))
2643                   (match_operand:GPI 4 "register_operand" "r")))]
2644   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2645   "*
2646   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2647                                            INTVAL (operands[3])));
2648   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2649   [(set_attr "type" "alu_ext")]
2650 )
2651
2652 ;; zero_extend version of above
2653 (define_insn "*add_uxtsi_shift2_uxtw"
2654   [(set (match_operand:DI 0 "register_operand" "=rk")
2655         (zero_extend:DI
2656          (plus:SI (and:SI
2657                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2658                               (match_operand 2 "aarch64_imm3" "Ui3"))
2659                    (match_operand 3 "const_int_operand" "n"))
2660                   (match_operand:SI 4 "register_operand" "r"))))]
2661   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2662   "*
2663   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2664                                            INTVAL (operands[3])));
2665   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2666   [(set_attr "type" "alu_ext")]
2667 )
2668
2669 (define_insn "*add_uxt<mode>_multp2"
2670   [(set (match_operand:GPI 0 "register_operand" "=rk")
2671         (plus:GPI (and:GPI
2672                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2673                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2674                    (match_operand 3 "const_int_operand" "n"))
2675                   (match_operand:GPI 4 "register_operand" "r")))]
2676   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2677   "*
2678   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2679                                            INTVAL (operands[3])));
2680   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2681   [(set_attr "type" "alu_ext")]
2682 )
2683
2684 ;; zero_extend version of above
2685 (define_insn "*add_uxtsi_multp2_uxtw"
2686   [(set (match_operand:DI 0 "register_operand" "=rk")
2687         (zero_extend:DI
2688          (plus:SI (and:SI
2689                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2690                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2691                    (match_operand 3 "const_int_operand" "n"))
2692                   (match_operand:SI 4 "register_operand" "r"))))]
2693   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2694   "*
2695   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2696                                            INTVAL (operands[3])));
2697   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2698   [(set_attr "type" "alu_ext")]
2699 )
2700
2701 (define_insn "subsi3"
2702   [(set (match_operand:SI 0 "register_operand" "=rk")
2703         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2704                   (match_operand:SI 2 "register_operand" "r")))]
2705   ""
2706   "sub\\t%w0, %w1, %w2"
2707   [(set_attr "type" "alu_sreg")]
2708 )
2709
2710 ;; zero_extend version of above
2711 (define_insn "*subsi3_uxtw"
2712   [(set (match_operand:DI 0 "register_operand" "=rk")
2713         (zero_extend:DI
2714          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2715                    (match_operand:SI 2 "register_operand" "r"))))]
2716   ""
2717   "sub\\t%w0, %w1, %w2"
2718   [(set_attr "type" "alu_sreg")]
2719 )
2720
2721 (define_insn "subdi3"
2722   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2723         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2724                   (match_operand:DI 2 "register_operand" "r,w")))]
2725   ""
2726   "@
2727    sub\\t%x0, %x1, %x2
2728    sub\\t%d0, %d1, %d2"
2729   [(set_attr "type" "alu_sreg, neon_sub")
2730    (set_attr "arch" "*,simd")]
2731 )
2732
2733 (define_expand "subv<GPI:mode>4"
2734   [(match_operand:GPI 0 "register_operand")
2735    (match_operand:GPI 1 "register_operand")
2736    (match_operand:GPI 2 "aarch64_plus_operand")
2737    (label_ref (match_operand 3 "" ""))]
2738   ""
2739 {
2740   if (CONST_INT_P (operands[2]))
2741     emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2742   else
2743     emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2744   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2745
2746   DONE;
2747 })
2748
2749 (define_insn "subv<GPI:mode>_insn"
2750   [(set (reg:CC_V CC_REGNUM)
2751         (compare:CC_V
2752          (sign_extend:<DWI>
2753           (minus:GPI
2754            (match_operand:GPI 1 "register_operand" "rk")
2755            (match_operand:GPI 2 "register_operand" "r")))
2756          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2757                       (sign_extend:<DWI> (match_dup 2)))))
2758    (set (match_operand:GPI 0 "register_operand" "=r")
2759         (minus:GPI (match_dup 1) (match_dup 2)))]
2760   ""
2761   "subs\\t%<w>0, %<w>1, %<w>2"
2762   [(set_attr "type" "alus_sreg")]
2763 )
2764
2765 (define_insn "subv<GPI:mode>_imm"
2766   [(set (reg:CC_V CC_REGNUM)
2767         (compare:CC_V
2768          (sign_extend:<DWI>
2769           (minus:GPI
2770            (match_operand:GPI 1 "register_operand" "rk,rk")
2771            (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2772          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2773                       (match_dup 2))))
2774    (set (match_operand:GPI 0 "register_operand" "=r,r")
2775         (minus:GPI (match_dup 1) (match_dup 2)))]
2776   ""
2777   "@
2778    subs\\t%<w>0, %<w>1, %2
2779    adds\\t%<w>0, %<w>1, #%n2"
2780   [(set_attr "type" "alus_sreg")]
2781 )
2782
2783 (define_expand "negv<GPI:mode>3"
2784   [(match_operand:GPI 0 "register_operand")
2785    (match_operand:GPI 1 "register_operand")
2786    (label_ref (match_operand 2 "" ""))]
2787   ""
2788   {
2789     emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2790     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2791
2792     DONE;
2793   }
2794 )
2795
2796 (define_insn "negv<GPI:mode>_insn"
2797   [(set (reg:CC_V CC_REGNUM)
2798         (compare:CC_V
2799          (sign_extend:<DWI>
2800           (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2801          (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2802    (set (match_operand:GPI 0 "register_operand" "=r")
2803         (neg:GPI (match_dup 1)))]
2804   ""
2805   "negs\\t%<w>0, %<w>1"
2806   [(set_attr "type" "alus_sreg")]
2807 )
2808
2809 (define_insn "negv<GPI:mode>_cmp_only"
2810   [(set (reg:CC_V CC_REGNUM)
2811         (compare:CC_V
2812          (sign_extend:<DWI>
2813           (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2814          (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2815   ""
2816   "negs\\t%<w>zr, %<w>0"
2817   [(set_attr "type" "alus_sreg")]
2818 )
2819
2820 (define_insn "*cmpv<GPI:mode>_insn"
2821   [(set (reg:CC_V CC_REGNUM)
2822         (compare:CC_V
2823          (sign_extend:<DWI>
2824           (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2825                      (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2826          (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2827                     (sign_extend:<DWI> (match_dup 1)))))]
2828   ""
2829   "@
2830    cmp\\t%<w>0, %<w>1
2831    cmp\\t%<w>0, %1
2832    cmp\\t%<w>0, #%n1"
2833   [(set_attr "type" "alus_sreg")]
2834 )
2835
2836 (define_expand "usubv<mode>4"
2837   [(match_operand:GPI 0 "register_operand")
2838    (match_operand:GPI 1 "aarch64_reg_or_zero")
2839    (match_operand:GPI 2 "aarch64_reg_or_zero")
2840    (label_ref (match_operand 3 "" ""))]
2841   ""
2842 {
2843   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2844   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2845
2846   DONE;
2847 })
2848
2849 (define_expand "subti3"
2850   [(set (match_operand:TI 0 "register_operand" "")
2851         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero" "")
2852                   (match_operand:TI 2 "register_operand" "")))]
2853   ""
2854 {
2855   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2856
2857   aarch64_subvti_scratch_regs (operands[1], operands[2],
2858                                &low_dest, &op1_low, &op2_low,
2859                                &high_dest, &op1_high, &op2_high);
2860
2861   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2862   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2863
2864   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2865   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2866   DONE;
2867 })
2868
2869 (define_expand "subvti4"
2870   [(match_operand:TI 0 "register_operand")
2871    (match_operand:TI 1 "register_operand")
2872    (match_operand:TI 2 "aarch64_reg_or_imm")
2873    (label_ref (match_operand 3 "" ""))]
2874   ""
2875 {
2876   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2877
2878   aarch64_subvti_scratch_regs (operands[1], operands[2],
2879                                &low_dest, &op1_low, &op2_low,
2880                                &high_dest, &op1_high, &op2_high);
2881   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2882                          high_dest, op1_high, op2_high, false);
2883
2884   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2885   DONE;
2886 })
2887
2888 (define_expand "usubvti4"
2889   [(match_operand:TI 0 "register_operand")
2890    (match_operand:TI 1 "register_operand")
2891    (match_operand:TI 2 "aarch64_reg_or_imm")
2892    (label_ref (match_operand 3 "" ""))]
2893   ""
2894 {
2895   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2896
2897   aarch64_subvti_scratch_regs (operands[1], operands[2],
2898                                     &low_dest, &op1_low, &op2_low,
2899                                &high_dest, &op1_high, &op2_high);
2900   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2901                          high_dest, op1_high, op2_high, true);
2902
2903   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2904   DONE;
2905 })
2906
2907 (define_expand "negvti3"
2908   [(match_operand:TI 0 "register_operand")
2909    (match_operand:TI 1 "register_operand")
2910    (label_ref (match_operand 2 "" ""))]
2911   ""
2912   {
2913     emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2914                                    gen_lowpart (DImode, operands[1])));
2915     emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2916                                     gen_highpart (DImode, operands[1])));
2917     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2918
2919     DONE;
2920   }
2921 )
2922
2923 (define_insn "negdi_carryout"
2924   [(set (reg:CC CC_REGNUM)
2925         (compare:CC
2926          (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2927    (set (match_operand:DI 0 "register_operand" "=r")
2928         (neg:DI (match_dup 1)))]
2929   ""
2930   "negs\\t%0, %1"
2931   [(set_attr "type" "alus_sreg")]
2932 )
2933
2934 (define_insn "negvdi_carryinV"
2935   [(set (reg:CC_V CC_REGNUM)
2936         (compare:CC_V
2937          (neg:TI (plus:TI
2938                   (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2939                   (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2940          (sign_extend:TI
2941           (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2942                            (match_dup 1))))))
2943    (set (match_operand:DI 0 "register_operand" "=r")
2944         (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2945                          (match_dup 1))))]
2946   ""
2947   "ngcs\\t%0, %1"
2948   [(set_attr "type" "alus_sreg")]
2949 )
2950
2951 (define_insn "*sub<mode>3_compare0"
2952   [(set (reg:CC_NZ CC_REGNUM)
2953         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2954                                   (match_operand:GPI 2 "register_operand" "r"))
2955                        (const_int 0)))
2956    (set (match_operand:GPI 0 "register_operand" "=r")
2957         (minus:GPI (match_dup 1) (match_dup 2)))]
2958   ""
2959   "subs\\t%<w>0, %<w>1, %<w>2"
2960   [(set_attr "type" "alus_sreg")]
2961 )
2962
2963 ;; zero_extend version of above
2964 (define_insn "*subsi3_compare0_uxtw"
2965   [(set (reg:CC_NZ CC_REGNUM)
2966         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2967                                  (match_operand:SI 2 "register_operand" "r"))
2968                        (const_int 0)))
2969    (set (match_operand:DI 0 "register_operand" "=r")
2970         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2971   ""
2972   "subs\\t%w0, %w1, %w2"
2973   [(set_attr "type" "alus_sreg")]
2974 )
2975
2976 (define_insn "sub<mode>3_compare1_imm"
2977   [(set (reg:CC CC_REGNUM)
2978         (compare:CC
2979           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ,rZ")
2980           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2981    (set (match_operand:GPI 0 "register_operand" "=r,r")
2982         (plus:GPI
2983           (match_dup 1)
2984           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
2985   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
2986   "@
2987   subs\\t%<w>0, %<w>1, %2
2988   adds\\t%<w>0, %<w>1, #%n2"
2989   [(set_attr "type" "alus_imm")]
2990 )
2991
2992 (define_insn "sub<mode>3_compare1"
2993   [(set (reg:CC CC_REGNUM)
2994         (compare:CC
2995           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2996           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2997    (set (match_operand:GPI 0 "register_operand" "=r")
2998         (minus:GPI (match_dup 1) (match_dup 2)))]
2999   ""
3000   "subs\\t%<w>0, %<w>1, %<w>2"
3001   [(set_attr "type" "alus_sreg")]
3002 )
3003
3004 (define_peephole2
3005   [(set (match_operand:GPI 0 "register_operand")
3006         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3007                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
3008    (set (reg:CC CC_REGNUM)
3009         (compare:CC
3010           (match_dup 1)
3011           (match_dup 2)))]
3012   "!reg_overlap_mentioned_p (operands[0], operands[1])
3013    && !reg_overlap_mentioned_p (operands[0], operands[2])"
3014   [(const_int 0)]
3015   {
3016     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3017                                          operands[2]));
3018     DONE;
3019   }
3020 )
3021
3022 ;; Same as the above peephole but with the compare and minus in
3023 ;; swapped order.  The restriction on overlap between operand 0
3024 ;; and operands 1 and 2 doesn't apply here.
3025 (define_peephole2
3026   [(set (reg:CC CC_REGNUM)
3027         (compare:CC
3028           (match_operand:GPI 1 "aarch64_reg_or_zero")
3029           (match_operand:GPI 2 "aarch64_reg_or_zero")))
3030    (set (match_operand:GPI 0 "register_operand")
3031         (minus:GPI (match_dup 1)
3032                    (match_dup 2)))]
3033   ""
3034   [(const_int 0)]
3035   {
3036     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3037                                          operands[2]));
3038     DONE;
3039   }
3040 )
3041
3042 (define_peephole2
3043   [(set (match_operand:GPI 0 "register_operand")
3044         (plus:GPI (match_operand:GPI 1 "register_operand")
3045                   (match_operand:GPI 2 "aarch64_plus_immediate")))
3046    (set (reg:CC CC_REGNUM)
3047         (compare:CC
3048           (match_dup 1)
3049           (match_operand:GPI 3 "const_int_operand")))]
3050   "!reg_overlap_mentioned_p (operands[0], operands[1])
3051    && INTVAL (operands[3]) == -INTVAL (operands[2])"
3052   [(const_int 0)]
3053   {
3054     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3055                                          operands[3], operands[2]));
3056     DONE;
3057   }
3058 )
3059
3060 ;; Same as the above peephole but with the compare and minus in
3061 ;; swapped order.  The restriction on overlap between operand 0
3062 ;; and operands 1 doesn't apply here.
3063 (define_peephole2
3064   [(set (reg:CC CC_REGNUM)
3065         (compare:CC
3066           (match_operand:GPI 1 "register_operand")
3067           (match_operand:GPI 3 "const_int_operand")))
3068    (set (match_operand:GPI 0 "register_operand")
3069         (plus:GPI (match_dup 1)
3070                   (match_operand:GPI 2 "aarch64_plus_immediate")))]
3071   "INTVAL (operands[3]) == -INTVAL (operands[2])"
3072   [(const_int 0)]
3073   {
3074     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3075                                          operands[3], operands[2]));
3076     DONE;
3077   }
3078 )
3079
3080 (define_insn "*sub_<shift>_<mode>"
3081   [(set (match_operand:GPI 0 "register_operand" "=r")
3082         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3083                    (ASHIFT:GPI
3084                     (match_operand:GPI 1 "register_operand" "r")
3085                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3086   ""
3087   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3088   [(set_attr "type" "alu_shift_imm")]
3089 )
3090
3091 ;; zero_extend version of above
3092 (define_insn "*sub_<shift>_si_uxtw"
3093   [(set (match_operand:DI 0 "register_operand" "=r")
3094         (zero_extend:DI
3095          (minus:SI (match_operand:SI 3 "register_operand" "r")
3096                    (ASHIFT:SI
3097                     (match_operand:SI 1 "register_operand" "r")
3098                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3099   ""
3100   "sub\\t%w0, %w3, %w1, <shift> %2"
3101   [(set_attr "type" "alu_shift_imm")]
3102 )
3103
3104 (define_insn "*sub_mul_imm_<mode>"
3105   [(set (match_operand:GPI 0 "register_operand" "=r")
3106         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3107                    (mult:GPI
3108                     (match_operand:GPI 1 "register_operand" "r")
3109                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3110   ""
3111   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3112   [(set_attr "type" "alu_shift_imm")]
3113 )
3114
3115 ;; zero_extend version of above
3116 (define_insn "*sub_mul_imm_si_uxtw"
3117   [(set (match_operand:DI 0 "register_operand" "=r")
3118         (zero_extend:DI
3119          (minus:SI (match_operand:SI 3 "register_operand" "r")
3120                    (mult:SI
3121                     (match_operand:SI 1 "register_operand" "r")
3122                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3123   ""
3124   "sub\\t%w0, %w3, %w1, lsl %p2"
3125   [(set_attr "type" "alu_shift_imm")]
3126 )
3127
3128 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3129   [(set (match_operand:GPI 0 "register_operand" "=rk")
3130         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3131                    (ANY_EXTEND:GPI
3132                     (match_operand:ALLX 2 "register_operand" "r"))))]
3133   ""
3134   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3135   [(set_attr "type" "alu_ext")]
3136 )
3137
3138 ;; zero_extend version of above
3139 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3140   [(set (match_operand:DI 0 "register_operand" "=rk")
3141         (zero_extend:DI
3142          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3143                    (ANY_EXTEND:SI
3144                     (match_operand:SHORT 2 "register_operand" "r")))))]
3145   ""
3146   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3147   [(set_attr "type" "alu_ext")]
3148 )
3149
3150 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3151   [(set (match_operand:GPI 0 "register_operand" "=rk")
3152         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3153                    (ashift:GPI (ANY_EXTEND:GPI
3154                                 (match_operand:ALLX 2 "register_operand" "r"))
3155                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
3156   ""
3157   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3158   [(set_attr "type" "alu_ext")]
3159 )
3160
3161 ;; zero_extend version of above
3162 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3163   [(set (match_operand:DI 0 "register_operand" "=rk")
3164         (zero_extend:DI
3165          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3166                    (ashift:SI (ANY_EXTEND:SI
3167                                (match_operand:SHORT 2 "register_operand" "r"))
3168                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
3169   ""
3170   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3171   [(set_attr "type" "alu_ext")]
3172 )
3173
3174 (define_insn "*sub_<optab><mode>_multp2"
3175   [(set (match_operand:GPI 0 "register_operand" "=rk")
3176         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3177                    (ANY_EXTRACT:GPI
3178                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3179                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3180                     (match_operand 3 "const_int_operand" "n")
3181                     (const_int 0))))]
3182   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3183   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3184   [(set_attr "type" "alu_ext")]
3185 )
3186
3187 ;; zero_extend version of above
3188 (define_insn "*sub_<optab>si_multp2_uxtw"
3189   [(set (match_operand:DI 0 "register_operand" "=rk")
3190         (zero_extend:DI
3191          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3192                    (ANY_EXTRACT:SI
3193                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3194                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3195                     (match_operand 3 "const_int_operand" "n")
3196                     (const_int 0)))))]
3197   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3198   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3199   [(set_attr "type" "alu_ext")]
3200 )
3201
3202 ;; The hardware description is op1 + ~op2 + C.
3203 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3204 ;;                           = op1 - op2 - 1 + 1 - !C
3205 ;;                           = op1 - op2 - !C.
3206 ;; We describe the latter.
3207
3208 (define_insn "*sub<mode>3_carryin0"
3209   [(set (match_operand:GPI 0 "register_operand" "=r")
3210         (minus:GPI
3211           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3212           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3213    ""
3214    "sbc\\t%<w>0, %<w>1, <w>zr"
3215   [(set_attr "type" "adc_reg")]
3216 )
3217
3218 ;; zero_extend version of the above
3219 (define_insn "*subsi3_carryin_uxtw"
3220   [(set (match_operand:DI 0 "register_operand" "=r")
3221         (zero_extend:DI
3222           (minus:SI
3223             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3224             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3225    ""
3226    "sbc\\t%w0, %w1, wzr"
3227   [(set_attr "type" "adc_reg")]
3228 )
3229
3230 (define_expand "sub<mode>3_carryin"
3231   [(set (match_operand:GPI 0 "register_operand")
3232         (minus:GPI
3233           (minus:GPI
3234             (match_operand:GPI 1 "aarch64_reg_or_zero")
3235             (match_operand:GPI 2 "register_operand"))
3236           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3237    ""
3238    ""
3239 )
3240
3241 (define_insn "*sub<mode>3_carryin"
3242   [(set (match_operand:GPI 0 "register_operand" "=r")
3243         (minus:GPI
3244           (minus:GPI
3245             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3246             (match_operand:GPI 2 "register_operand" "r"))
3247           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3248
3249    ""
3250    "sbc\\t%<w>0, %<w>1, %<w>2"
3251   [(set_attr "type" "adc_reg")]
3252 )
3253
3254 ;; zero_extend version of the above
3255 (define_insn "*subsi3_carryin_uxtw"
3256   [(set (match_operand:DI 0 "register_operand" "=r")
3257         (zero_extend:DI
3258           (minus:SI
3259             (minus:SI
3260               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3261               (match_operand:SI 2 "register_operand" "r"))
3262             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3263
3264    ""
3265    "sbc\\t%w0, %w1, %w2"
3266   [(set_attr "type" "adc_reg")]
3267 )
3268
3269 (define_insn "*sub<mode>3_carryin_alt"
3270   [(set (match_operand:GPI 0 "register_operand" "=r")
3271         (minus:GPI
3272           (minus:GPI
3273             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3274             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3275           (match_operand:GPI 2 "register_operand" "r")))]
3276    ""
3277    "sbc\\t%<w>0, %<w>1, %<w>2"
3278   [(set_attr "type" "adc_reg")]
3279 )
3280
3281 ;; zero_extend version of the above
3282 (define_insn "*subsi3_carryin_alt_uxtw"
3283   [(set (match_operand:DI 0 "register_operand" "=r")
3284         (zero_extend:DI
3285           (minus:SI
3286             (minus:SI
3287               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3288               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3289             (match_operand:SI 2 "register_operand" "r"))))]
3290    ""
3291    "sbc\\t%w0, %w1, %w2"
3292   [(set_attr "type" "adc_reg")]
3293 )
3294
3295 (define_expand "usub<GPI:mode>3_carryinC"
3296   [(parallel
3297      [(set (reg:CC CC_REGNUM)
3298            (compare:CC
3299              (zero_extend:<DWI>
3300                (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3301              (plus:<DWI>
3302                (zero_extend:<DWI>
3303                  (match_operand:GPI 2 "register_operand" ""))
3304                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3305       (set (match_operand:GPI 0 "register_operand" "")
3306            (minus:GPI
3307              (minus:GPI (match_dup 1) (match_dup 2))
3308              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3309    ""
3310 )
3311
3312 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3313   [(set (reg:CC CC_REGNUM)
3314         (compare:CC
3315           (const_int 0)
3316           (plus:<DWI>
3317             (zero_extend:<DWI>
3318               (match_operand:GPI 1 "register_operand" "r"))
3319             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3320    (set (match_operand:GPI 0 "register_operand" "=r")
3321         (minus:GPI
3322           (neg:GPI (match_dup 1))
3323           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3324    ""
3325    "sbcs\\t%<w>0, <w>zr, %<w>1"
3326   [(set_attr "type" "adc_reg")]
3327 )
3328
3329 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3330   [(set (reg:CC CC_REGNUM)
3331         (compare:CC
3332           (zero_extend:<DWI>
3333             (match_operand:GPI 1 "register_operand" "r"))
3334           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3335    (set (match_operand:GPI 0 "register_operand" "=r")
3336         (minus:GPI
3337           (match_dup 1)
3338           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3339    ""
3340    "sbcs\\t%<w>0, %<w>1, <w>zr"
3341   [(set_attr "type" "adc_reg")]
3342 )
3343
3344 (define_insn "*usub<GPI:mode>3_carryinC"
3345   [(set (reg:CC CC_REGNUM)
3346         (compare:CC
3347           (zero_extend:<DWI>
3348             (match_operand:GPI 1 "register_operand" "r"))
3349           (plus:<DWI>
3350             (zero_extend:<DWI>
3351               (match_operand:GPI 2 "register_operand" "r"))
3352             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3353    (set (match_operand:GPI 0 "register_operand" "=r")
3354         (minus:GPI
3355           (minus:GPI (match_dup 1) (match_dup 2))
3356           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3357    ""
3358    "sbcs\\t%<w>0, %<w>1, %<w>2"
3359   [(set_attr "type" "adc_reg")]
3360 )
3361
3362 (define_expand "sub<GPI:mode>3_carryinV"
3363   [(parallel
3364      [(set (reg:CC_V CC_REGNUM)
3365            (compare:CC_V
3366             (minus:<DWI>
3367              (sign_extend:<DWI>
3368                (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3369              (plus:<DWI>
3370                (sign_extend:<DWI>
3371                  (match_operand:GPI 2 "register_operand" ""))
3372                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3373             (sign_extend:<DWI>
3374              (minus:GPI (match_dup 1)
3375                         (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3376                                   (match_dup 2))))))
3377       (set (match_operand:GPI 0 "register_operand" "")
3378            (minus:GPI
3379              (minus:GPI (match_dup 1) (match_dup 2))
3380              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3381    ""
3382 )
3383
3384 (define_insn "*sub<mode>3_carryinV_z2"
3385   [(set (reg:CC_V CC_REGNUM)
3386         (compare:CC_V
3387          (minus:<DWI>
3388           (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3389           (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3390          (sign_extend:<DWI>
3391           (minus:GPI (match_dup 1)
3392                      (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3393    (set (match_operand:GPI 0 "register_operand" "=r")
3394         (minus:GPI
3395          (match_dup 1) (match_dup 3)))]
3396    ""
3397    "sbcs\\t%<w>0, %<w>1, <w>zr"
3398   [(set_attr "type" "adc_reg")]
3399 )
3400
3401 (define_insn "*sub<mode>3_carryinV"
3402   [(set (reg:CC_V CC_REGNUM)
3403         (compare:CC_V
3404          (minus:<DWI>
3405           (sign_extend:<DWI>
3406             (match_operand:GPI 1 "register_operand" "r"))
3407           (plus:<DWI>
3408             (sign_extend:<DWI>
3409               (match_operand:GPI 2 "register_operand" "r"))
3410             (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3411          (sign_extend:<DWI>
3412           (minus:GPI
3413            (match_dup 1)
3414            (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3415                      (match_dup 2))))))
3416    (set (match_operand:GPI 0 "register_operand" "=r")
3417         (minus:GPI
3418           (minus:GPI (match_dup 1) (match_dup 2))
3419           (match_dup 4)))]
3420    ""
3421    "sbcs\\t%<w>0, %<w>1, %<w>2"
3422   [(set_attr "type" "adc_reg")]
3423 )
3424
3425 (define_insn "*sub_uxt<mode>_shift2"
3426   [(set (match_operand:GPI 0 "register_operand" "=rk")
3427         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3428                    (and:GPI
3429                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3430                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3431                     (match_operand 3 "const_int_operand" "n"))))]
3432   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3433   "*
3434   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3435                                            INTVAL (operands[3])));
3436   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3437   [(set_attr "type" "alu_ext")]
3438 )
3439
3440 ;; zero_extend version of above
3441 (define_insn "*sub_uxtsi_shift2_uxtw"
3442   [(set (match_operand:DI 0 "register_operand" "=rk")
3443         (zero_extend:DI
3444          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3445                    (and:SI
3446                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3447                                (match_operand 2 "aarch64_imm3" "Ui3"))
3448                     (match_operand 3 "const_int_operand" "n")))))]
3449   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3450   "*
3451   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3452                                            INTVAL (operands[3])));
3453   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3454   [(set_attr "type" "alu_ext")]
3455 )
3456
3457 (define_insn "*sub_uxt<mode>_multp2"
3458   [(set (match_operand:GPI 0 "register_operand" "=rk")
3459         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3460                    (and:GPI
3461                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3462                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3463                     (match_operand 3 "const_int_operand" "n"))))]
3464   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3465   "*
3466   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3467                                            INTVAL (operands[3])));
3468   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3469   [(set_attr "type" "alu_ext")]
3470 )
3471
3472 ;; zero_extend version of above
3473 (define_insn "*sub_uxtsi_multp2_uxtw"
3474   [(set (match_operand:DI 0 "register_operand" "=rk")
3475         (zero_extend:DI
3476          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3477                    (and:SI
3478                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3479                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3480                     (match_operand 3 "const_int_operand" "n")))))]
3481   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3482   "*
3483   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3484                                            INTVAL (operands[3])));
3485   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3486   [(set_attr "type" "alu_ext")]
3487 )
3488
3489 (define_expand "abs<mode>2"
3490   [(match_operand:GPI 0 "register_operand" "")
3491    (match_operand:GPI 1 "register_operand" "")]
3492   ""
3493   {
3494     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3495     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3496     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3497     DONE;
3498   }
3499 )
3500
3501 (define_insn "neg<mode>2"
3502   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3503         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3504   ""
3505   "@
3506    neg\\t%<w>0, %<w>1
3507    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3508   [(set_attr "type" "alu_sreg, neon_neg<q>")
3509    (set_attr "arch" "*,simd")]
3510 )
3511
3512 ;; zero_extend version of above
3513 (define_insn "*negsi2_uxtw"
3514   [(set (match_operand:DI 0 "register_operand" "=r")
3515         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3516   ""
3517   "neg\\t%w0, %w1"
3518   [(set_attr "type" "alu_sreg")]
3519 )
3520
3521 (define_insn "*ngc<mode>"
3522   [(set (match_operand:GPI 0 "register_operand" "=r")
3523         (minus:GPI
3524           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3525           (match_operand:GPI 1 "register_operand" "r")))]
3526   ""
3527   "ngc\\t%<w>0, %<w>1"
3528   [(set_attr "type" "adc_reg")]
3529 )
3530
3531 (define_insn "*ngcsi_uxtw"
3532   [(set (match_operand:DI 0 "register_operand" "=r")
3533         (zero_extend:DI
3534           (minus:SI
3535             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3536             (match_operand:SI 1 "register_operand" "r"))))]
3537   ""
3538   "ngc\\t%w0, %w1"
3539   [(set_attr "type" "adc_reg")]
3540 )
3541
3542 (define_insn "neg<mode>2_compare0"
3543   [(set (reg:CC_NZ CC_REGNUM)
3544         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3545                        (const_int 0)))
3546    (set (match_operand:GPI 0 "register_operand" "=r")
3547         (neg:GPI (match_dup 1)))]
3548   ""
3549   "negs\\t%<w>0, %<w>1"
3550   [(set_attr "type" "alus_sreg")]
3551 )
3552
3553 ;; zero_extend version of above
3554 (define_insn "*negsi2_compare0_uxtw"
3555   [(set (reg:CC_NZ CC_REGNUM)
3556         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3557                        (const_int 0)))
3558    (set (match_operand:DI 0 "register_operand" "=r")
3559         (zero_extend:DI (neg:SI (match_dup 1))))]
3560   ""
3561   "negs\\t%w0, %w1"
3562   [(set_attr "type" "alus_sreg")]
3563 )
3564
3565 (define_insn "*neg_<shift><mode>3_compare0"
3566   [(set (reg:CC_NZ CC_REGNUM)
3567         (compare:CC_NZ
3568          (neg:GPI (ASHIFT:GPI
3569                    (match_operand:GPI 1 "register_operand" "r")
3570                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3571          (const_int 0)))
3572    (set (match_operand:GPI 0 "register_operand" "=r")
3573         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3574   ""
3575   "negs\\t%<w>0, %<w>1, <shift> %2"
3576   [(set_attr "type" "alus_shift_imm")]
3577 )
3578
3579 (define_insn "*neg_<shift>_<mode>2"
3580   [(set (match_operand:GPI 0 "register_operand" "=r")
3581         (neg:GPI (ASHIFT:GPI
3582                   (match_operand:GPI 1 "register_operand" "r")
3583                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3584   ""
3585   "neg\\t%<w>0, %<w>1, <shift> %2"
3586   [(set_attr "type" "alu_shift_imm")]
3587 )
3588
3589 ;; zero_extend version of above
3590 (define_insn "*neg_<shift>_si2_uxtw"
3591   [(set (match_operand:DI 0 "register_operand" "=r")
3592         (zero_extend:DI
3593          (neg:SI (ASHIFT:SI
3594                   (match_operand:SI 1 "register_operand" "r")
3595                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3596   ""
3597   "neg\\t%w0, %w1, <shift> %2"
3598   [(set_attr "type" "alu_shift_imm")]
3599 )
3600
3601 (define_insn "*neg_mul_imm_<mode>2"
3602   [(set (match_operand:GPI 0 "register_operand" "=r")
3603         (neg:GPI (mult:GPI
3604                   (match_operand:GPI 1 "register_operand" "r")
3605                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3606   ""
3607   "neg\\t%<w>0, %<w>1, lsl %p2"
3608   [(set_attr "type" "alu_shift_imm")]
3609 )
3610
3611 ;; zero_extend version of above
3612 (define_insn "*neg_mul_imm_si2_uxtw"
3613   [(set (match_operand:DI 0 "register_operand" "=r")
3614         (zero_extend:DI
3615          (neg:SI (mult:SI
3616                   (match_operand:SI 1 "register_operand" "r")
3617                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3618   ""
3619   "neg\\t%w0, %w1, lsl %p2"
3620   [(set_attr "type" "alu_shift_imm")]
3621 )
3622
3623 (define_insn "mul<mode>3"
3624   [(set (match_operand:GPI 0 "register_operand" "=r")
3625         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3626                   (match_operand:GPI 2 "register_operand" "r")))]
3627   ""
3628   "mul\\t%<w>0, %<w>1, %<w>2"
3629   [(set_attr "type" "mul")]
3630 )
3631
3632 ;; zero_extend version of above
3633 (define_insn "*mulsi3_uxtw"
3634   [(set (match_operand:DI 0 "register_operand" "=r")
3635         (zero_extend:DI
3636          (mult:SI (match_operand:SI 1 "register_operand" "r")
3637                   (match_operand:SI 2 "register_operand" "r"))))]
3638   ""
3639   "mul\\t%w0, %w1, %w2"
3640   [(set_attr "type" "mul")]
3641 )
3642
3643 (define_insn "madd<mode>"
3644   [(set (match_operand:GPI 0 "register_operand" "=r")
3645         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3646                             (match_operand:GPI 2 "register_operand" "r"))
3647                   (match_operand:GPI 3 "register_operand" "r")))]
3648   ""
3649   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3650   [(set_attr "type" "mla")]
3651 )
3652
3653 ;; zero_extend version of above
3654 (define_insn "*maddsi_uxtw"
3655   [(set (match_operand:DI 0 "register_operand" "=r")
3656         (zero_extend:DI
3657          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3658                            (match_operand:SI 2 "register_operand" "r"))
3659                   (match_operand:SI 3 "register_operand" "r"))))]
3660   ""
3661   "madd\\t%w0, %w1, %w2, %w3"
3662   [(set_attr "type" "mla")]
3663 )
3664
3665 (define_insn "*msub<mode>"
3666   [(set (match_operand:GPI 0 "register_operand" "=r")
3667         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3668                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3669                              (match_operand:GPI 2 "register_operand" "r"))))]
3670
3671   ""
3672   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3673   [(set_attr "type" "mla")]
3674 )
3675
3676 ;; zero_extend version of above
3677 (define_insn "*msubsi_uxtw"
3678   [(set (match_operand:DI 0 "register_operand" "=r")
3679         (zero_extend:DI
3680          (minus:SI (match_operand:SI 3 "register_operand" "r")
3681                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3682                             (match_operand:SI 2 "register_operand" "r")))))]
3683
3684   ""
3685   "msub\\t%w0, %w1, %w2, %w3"
3686   [(set_attr "type" "mla")]
3687 )
3688
3689 (define_insn "*mul<mode>_neg"
3690   [(set (match_operand:GPI 0 "register_operand" "=r")
3691         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3692                   (match_operand:GPI 2 "register_operand" "r")))]
3693
3694   ""
3695   "mneg\\t%<w>0, %<w>1, %<w>2"
3696   [(set_attr "type" "mul")]
3697 )
3698
3699 ;; zero_extend version of above
3700 (define_insn "*mulsi_neg_uxtw"
3701   [(set (match_operand:DI 0 "register_operand" "=r")
3702         (zero_extend:DI
3703          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3704                   (match_operand:SI 2 "register_operand" "r"))))]
3705
3706   ""
3707   "mneg\\t%w0, %w1, %w2"
3708   [(set_attr "type" "mul")]
3709 )
3710
3711 (define_insn "<su_optab>mulsidi3"
3712   [(set (match_operand:DI 0 "register_operand" "=r")
3713         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3714                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3715   ""
3716   "<su>mull\\t%0, %w1, %w2"
3717   [(set_attr "type" "<su>mull")]
3718 )
3719
3720 (define_insn "<su_optab>maddsidi4"
3721   [(set (match_operand:DI 0 "register_operand" "=r")
3722         (plus:DI (mult:DI
3723                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3724                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3725                  (match_operand:DI 3 "register_operand" "r")))]
3726   ""
3727   "<su>maddl\\t%0, %w1, %w2, %3"
3728   [(set_attr "type" "<su>mlal")]
3729 )
3730
3731 (define_insn "<su_optab>msubsidi4"
3732   [(set (match_operand:DI 0 "register_operand" "=r")
3733         (minus:DI
3734          (match_operand:DI 3 "register_operand" "r")
3735          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3736                   (ANY_EXTEND:DI
3737                    (match_operand:SI 2 "register_operand" "r")))))]
3738   ""
3739   "<su>msubl\\t%0, %w1, %w2, %3"
3740   [(set_attr "type" "<su>mlal")]
3741 )
3742
3743 (define_insn "*<su_optab>mulsidi_neg"
3744   [(set (match_operand:DI 0 "register_operand" "=r")
3745         (mult:DI (neg:DI
3746                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3747                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3748   ""
3749   "<su>mnegl\\t%0, %w1, %w2"
3750   [(set_attr "type" "<su>mull")]
3751 )
3752
3753 (define_expand "<su_optab>mulditi3"
3754   [(set (match_operand:TI 0 "register_operand")
3755         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3756                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3757   ""
3758 {
3759   rtx low = gen_reg_rtx (DImode);
3760   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3761
3762   rtx high = gen_reg_rtx (DImode);
3763   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3764
3765   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3766   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3767   DONE;
3768 })
3769
3770 ;; The default expansion of multi3 using umuldi3_highpart will perform
3771 ;; the additions in an order that fails to combine into two madd insns.
3772 (define_expand "multi3"
3773   [(set (match_operand:TI 0 "register_operand")
3774         (mult:TI (match_operand:TI 1 "register_operand")
3775                  (match_operand:TI 2 "register_operand")))]
3776   ""
3777 {
3778   rtx l0 = gen_reg_rtx (DImode);
3779   rtx l1 = gen_lowpart (DImode, operands[1]);
3780   rtx l2 = gen_lowpart (DImode, operands[2]);
3781   rtx h0 = gen_reg_rtx (DImode);
3782   rtx h1 = gen_highpart (DImode, operands[1]);
3783   rtx h2 = gen_highpart (DImode, operands[2]);
3784
3785   emit_insn (gen_muldi3 (l0, l1, l2));
3786   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3787   emit_insn (gen_madddi (h0, h1, l2, h0));
3788   emit_insn (gen_madddi (h0, l1, h2, h0));
3789
3790   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3791   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3792   DONE;
3793 })
3794
3795 (define_insn "<su>muldi3_highpart"
3796   [(set (match_operand:DI 0 "register_operand" "=r")
3797         (truncate:DI
3798          (lshiftrt:TI
3799           (mult:TI
3800            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3801            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3802           (const_int 64))))]
3803   ""
3804   "<su>mulh\\t%0, %1, %2"
3805   [(set_attr "type" "<su>mull")]
3806 )
3807
3808 (define_insn "<su_optab>div<mode>3"
3809   [(set (match_operand:GPI 0 "register_operand" "=r")
3810         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3811                      (match_operand:GPI 2 "register_operand" "r")))]
3812   ""
3813   "<su>div\\t%<w>0, %<w>1, %<w>2"
3814   [(set_attr "type" "<su>div")]
3815 )
3816
3817 ;; zero_extend version of above
3818 (define_insn "*<su_optab>divsi3_uxtw"
3819   [(set (match_operand:DI 0 "register_operand" "=r")
3820         (zero_extend:DI
3821          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3822                      (match_operand:SI 2 "register_operand" "r"))))]
3823   ""
3824   "<su>div\\t%w0, %w1, %w2"
3825   [(set_attr "type" "<su>div")]
3826 )
3827
3828 ;; -------------------------------------------------------------------
3829 ;; Comparison insns
3830 ;; -------------------------------------------------------------------
3831
3832 (define_insn "cmp<mode>"
3833   [(set (reg:CC CC_REGNUM)
3834         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3835                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3836   ""
3837   "@
3838    cmp\\t%<w>0, %<w>1
3839    cmp\\t%<w>0, %1
3840    cmn\\t%<w>0, #%n1"
3841   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3842 )
3843
3844 (define_insn "fcmp<mode>"
3845   [(set (reg:CCFP CC_REGNUM)
3846         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3847                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3848    "TARGET_FLOAT"
3849    "@
3850     fcmp\\t%<s>0, #0.0
3851     fcmp\\t%<s>0, %<s>1"
3852   [(set_attr "type" "fcmp<s>")]
3853 )
3854
3855 (define_insn "fcmpe<mode>"
3856   [(set (reg:CCFPE CC_REGNUM)
3857         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3858                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3859    "TARGET_FLOAT"
3860    "@
3861     fcmpe\\t%<s>0, #0.0
3862     fcmpe\\t%<s>0, %<s>1"
3863   [(set_attr "type" "fcmp<s>")]
3864 )
3865
3866 (define_insn "*cmp_swp_<shift>_reg<mode>"
3867   [(set (reg:CC_SWP CC_REGNUM)
3868         (compare:CC_SWP (ASHIFT:GPI
3869                          (match_operand:GPI 0 "register_operand" "r")
3870                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3871                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3872   ""
3873   "cmp\\t%<w>2, %<w>0, <shift> %1"
3874   [(set_attr "type" "alus_shift_imm")]
3875 )
3876
3877 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3878   [(set (reg:CC_SWP CC_REGNUM)
3879         (compare:CC_SWP (ANY_EXTEND:GPI
3880                          (match_operand:ALLX 0 "register_operand" "r"))
3881                         (match_operand:GPI 1 "register_operand" "r")))]
3882   ""
3883   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3884   [(set_attr "type" "alus_ext")]
3885 )
3886
3887 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3888   [(set (reg:CC_SWP CC_REGNUM)
3889         (compare:CC_SWP (ashift:GPI
3890                          (ANY_EXTEND:GPI
3891                           (match_operand:ALLX 0 "register_operand" "r"))
3892                          (match_operand 1 "aarch64_imm3" "Ui3"))
3893         (match_operand:GPI 2 "register_operand" "r")))]
3894   ""
3895   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3896   [(set_attr "type" "alus_ext")]
3897 )
3898
3899 ;; -------------------------------------------------------------------
3900 ;; Store-flag and conditional select insns
3901 ;; -------------------------------------------------------------------
3902
3903 (define_expand "cstore<mode>4"
3904   [(set (match_operand:SI 0 "register_operand" "")
3905         (match_operator:SI 1 "aarch64_comparison_operator"
3906          [(match_operand:GPI 2 "register_operand" "")
3907           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3908   ""
3909   "
3910   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3911                                       operands[3]);
3912   operands[3] = const0_rtx;
3913   "
3914 )
3915
3916 (define_expand "cstorecc4"
3917   [(set (match_operand:SI 0 "register_operand")
3918        (match_operator 1 "aarch64_comparison_operator_mode"
3919         [(match_operand 2 "cc_register")
3920          (match_operand 3 "const0_operand")]))]
3921   ""
3922 "{
3923   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3924   DONE;
3925 }")
3926
3927
3928 (define_expand "cstore<mode>4"
3929   [(set (match_operand:SI 0 "register_operand" "")
3930         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3931          [(match_operand:GPF 2 "register_operand" "")
3932           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3933   ""
3934   "
3935   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3936                                       operands[3]);
3937   operands[3] = const0_rtx;
3938   "
3939 )
3940
3941 (define_insn "aarch64_cstore<mode>"
3942   [(set (match_operand:ALLI 0 "register_operand" "=r")
3943         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3944          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3945   ""
3946   "cset\\t%<w>0, %m1"
3947   [(set_attr "type" "csel")]
3948 )
3949
3950 ;; For a 24-bit immediate CST we can optimize the compare for equality
3951 ;; and branch sequence from:
3952 ;;      mov     x0, #imm1
3953 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3954 ;;      cmp     x1, x0
3955 ;;      cset    x2, <ne,eq>
3956 ;; into the shorter:
3957 ;;      sub     x0, x1, #(CST & 0xfff000)
3958 ;;      subs    x0, x0, #(CST & 0x000fff)
3959 ;;      cset x2, <ne, eq>.
3960 (define_insn_and_split "*compare_cstore<mode>_insn"
3961   [(set (match_operand:GPI 0 "register_operand" "=r")
3962          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3963                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3964    (clobber (reg:CC CC_REGNUM))]
3965   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3966    && !aarch64_plus_operand (operands[2], <MODE>mode)
3967    && !reload_completed"
3968   "#"
3969   "&& true"
3970   [(const_int 0)]
3971   {
3972     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3973     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3974     rtx tmp = gen_reg_rtx (<MODE>mode);
3975     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3976     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3977     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3978     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3979     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3980     DONE;
3981   }
3982   [(set_attr "type" "csel")]
3983 )
3984
3985 ;; zero_extend version of the above
3986 (define_insn "*cstoresi_insn_uxtw"
3987   [(set (match_operand:DI 0 "register_operand" "=r")
3988         (zero_extend:DI
3989          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3990           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3991   ""
3992   "cset\\t%w0, %m1"
3993   [(set_attr "type" "csel")]
3994 )
3995
3996 (define_insn "cstore<mode>_neg"
3997   [(set (match_operand:ALLI 0 "register_operand" "=r")
3998         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3999                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
4000   ""
4001   "csetm\\t%<w>0, %m1"
4002   [(set_attr "type" "csel")]
4003 )
4004
4005 ;; zero_extend version of the above
4006 (define_insn "*cstoresi_neg_uxtw"
4007   [(set (match_operand:DI 0 "register_operand" "=r")
4008         (zero_extend:DI
4009          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4010                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4011   ""
4012   "csetm\\t%w0, %m1"
4013   [(set_attr "type" "csel")]
4014 )
4015
4016 (define_expand "cmov<mode>6"
4017   [(set (match_operand:GPI 0 "register_operand" "")
4018         (if_then_else:GPI
4019          (match_operator 1 "aarch64_comparison_operator"
4020           [(match_operand:GPI 2 "register_operand" "")
4021            (match_operand:GPI 3 "aarch64_plus_operand" "")])
4022          (match_operand:GPI 4 "register_operand" "")
4023          (match_operand:GPI 5 "register_operand" "")))]
4024   ""
4025   "
4026   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4027                                       operands[3]);
4028   operands[3] = const0_rtx;
4029   "
4030 )
4031
4032 (define_expand "cmov<mode>6"
4033   [(set (match_operand:GPF 0 "register_operand" "")
4034         (if_then_else:GPF
4035          (match_operator 1 "aarch64_comparison_operator"
4036           [(match_operand:GPF 2 "register_operand" "")
4037            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
4038          (match_operand:GPF 4 "register_operand" "")
4039          (match_operand:GPF 5 "register_operand" "")))]
4040   ""
4041   "
4042   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4043                                       operands[3]);
4044   operands[3] = const0_rtx;
4045   "
4046 )
4047
4048 (define_insn "*cmov<mode>_insn"
4049   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4050         (if_then_else:ALLI
4051          (match_operator 1 "aarch64_comparison_operator"
4052           [(match_operand 2 "cc_register" "") (const_int 0)])
4053          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4054          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4055   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4056      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4057   ;; Final two alternatives should be unreachable, but included for completeness
4058   "@
4059    csel\\t%<w>0, %<w>3, %<w>4, %m1
4060    csinv\\t%<w>0, %<w>3, <w>zr, %m1
4061    csinv\\t%<w>0, %<w>4, <w>zr, %M1
4062    csinc\\t%<w>0, %<w>3, <w>zr, %m1
4063    csinc\\t%<w>0, %<w>4, <w>zr, %M1
4064    mov\\t%<w>0, -1
4065    mov\\t%<w>0, 1"
4066   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4067 )
4068
4069 ;; zero_extend version of above
4070 (define_insn "*cmovsi_insn_uxtw"
4071   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4072         (zero_extend:DI
4073          (if_then_else:SI
4074           (match_operator 1 "aarch64_comparison_operator"
4075            [(match_operand 2 "cc_register" "") (const_int 0)])
4076           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4077           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4078   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4079      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4080   ;; Final two alternatives should be unreachable, but included for completeness
4081   "@
4082    csel\\t%w0, %w3, %w4, %m1
4083    csinv\\t%w0, %w3, wzr, %m1
4084    csinv\\t%w0, %w4, wzr, %M1
4085    csinc\\t%w0, %w3, wzr, %m1
4086    csinc\\t%w0, %w4, wzr, %M1
4087    mov\\t%w0, -1
4088    mov\\t%w0, 1"
4089   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4090 )
4091
4092 (define_insn "*cmovdi_insn_uxtw"
4093   [(set (match_operand:DI 0 "register_operand" "=r")
4094         (if_then_else:DI
4095          (match_operator 1 "aarch64_comparison_operator"
4096           [(match_operand 2 "cc_register" "") (const_int 0)])
4097          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4098          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4099   ""
4100   "csel\\t%w0, %w3, %w4, %m1"
4101   [(set_attr "type" "csel")]
4102 )
4103
4104 (define_insn "*cmov<mode>_insn"
4105   [(set (match_operand:GPF 0 "register_operand" "=w")
4106         (if_then_else:GPF
4107          (match_operator 1 "aarch64_comparison_operator"
4108           [(match_operand 2 "cc_register" "") (const_int 0)])
4109          (match_operand:GPF 3 "register_operand" "w")
4110          (match_operand:GPF 4 "register_operand" "w")))]
4111   "TARGET_FLOAT"
4112   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4113   [(set_attr "type" "fcsel")]
4114 )
4115
4116 (define_expand "mov<mode>cc"
4117   [(set (match_operand:ALLI 0 "register_operand" "")
4118         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
4119                            (match_operand:ALLI 2 "register_operand" "")
4120                            (match_operand:ALLI 3 "register_operand" "")))]
4121   ""
4122   {
4123     rtx ccreg;
4124     enum rtx_code code = GET_CODE (operands[1]);
4125
4126     if (code == UNEQ || code == LTGT)
4127       FAIL;
4128
4129     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4130                                      XEXP (operands[1], 1));
4131     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4132   }
4133 )
4134
4135 (define_expand "mov<GPF:mode><GPI:mode>cc"
4136   [(set (match_operand:GPI 0 "register_operand" "")
4137         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
4138                           (match_operand:GPF 2 "register_operand" "")
4139                           (match_operand:GPF 3 "register_operand" "")))]
4140   ""
4141   {
4142     rtx ccreg;
4143     enum rtx_code code = GET_CODE (operands[1]);
4144
4145     if (code == UNEQ || code == LTGT)
4146       FAIL;
4147
4148     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4149                                   XEXP (operands[1], 1));
4150     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4151   }
4152 )
4153
4154 (define_expand "mov<mode>cc"
4155   [(set (match_operand:GPF 0 "register_operand" "")
4156         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
4157                           (match_operand:GPF 2 "register_operand" "")
4158                           (match_operand:GPF 3 "register_operand" "")))]
4159   ""
4160   {
4161     rtx ccreg;
4162     enum rtx_code code = GET_CODE (operands[1]);
4163
4164     if (code == UNEQ || code == LTGT)
4165       FAIL;
4166
4167     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4168                                   XEXP (operands[1], 1));
4169     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4170   }
4171 )
4172
4173 (define_expand "<neg_not_op><mode>cc"
4174   [(set (match_operand:GPI 0 "register_operand" "")
4175         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
4176                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
4177                           (match_operand:GPI 3 "register_operand" "")))]
4178   ""
4179   {
4180     rtx ccreg;
4181     enum rtx_code code = GET_CODE (operands[1]);
4182
4183     if (code == UNEQ || code == LTGT)
4184       FAIL;
4185
4186     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4187                                       XEXP (operands[1], 1));
4188     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4189   }
4190 )
4191
4192 ;; CRC32 instructions.
4193 (define_insn "aarch64_<crc_variant>"
4194   [(set (match_operand:SI 0 "register_operand" "=r")
4195         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4196                     (match_operand:<crc_mode> 2 "register_operand" "r")]
4197          CRC))]
4198   "TARGET_CRC32"
4199   {
4200     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4201       return "<crc_variant>\\t%w0, %w1, %x2";
4202     else
4203       return "<crc_variant>\\t%w0, %w1, %w2";
4204   }
4205   [(set_attr "type" "crc")]
4206 )
4207
4208 (define_insn "*csinc2<mode>_insn"
4209   [(set (match_operand:GPI 0 "register_operand" "=r")
4210         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4211                   (match_operand:GPI 1 "register_operand" "r")))]
4212   ""
4213   "cinc\\t%<w>0, %<w>1, %m2"
4214   [(set_attr "type" "csel")]
4215 )
4216
4217 (define_insn "csinc3<mode>_insn"
4218   [(set (match_operand:GPI 0 "register_operand" "=r")
4219         (if_then_else:GPI
4220           (match_operand 1 "aarch64_comparison_operation" "")
4221           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4222                     (const_int 1))
4223           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4224   ""
4225   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4226   [(set_attr "type" "csel")]
4227 )
4228
4229 (define_insn "*csinv3<mode>_insn"
4230   [(set (match_operand:GPI 0 "register_operand" "=r")
4231         (if_then_else:GPI
4232           (match_operand 1 "aarch64_comparison_operation" "")
4233           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4234           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4235   ""
4236   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4237   [(set_attr "type" "csel")]
4238 )
4239
4240 (define_insn "csneg3_uxtw_insn"
4241   [(set (match_operand:DI 0 "register_operand" "=r")
4242         (zero_extend:DI
4243           (if_then_else:SI
4244             (match_operand 1 "aarch64_comparison_operation" "")
4245             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4246             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4247   ""
4248   "csneg\\t%w0, %w3, %w2, %M1"
4249   [(set_attr "type" "csel")]
4250 )
4251
4252 (define_insn "csneg3<mode>_insn"
4253   [(set (match_operand:GPI 0 "register_operand" "=r")
4254         (if_then_else:GPI
4255           (match_operand 1 "aarch64_comparison_operation" "")
4256           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4257           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4258   ""
4259   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4260   [(set_attr "type" "csel")]
4261 )
4262
4263 ;; If X can be loaded by a single CNT[BHWD] instruction,
4264 ;;
4265 ;;    A = UMAX (B, X)
4266 ;;
4267 ;; is equivalent to:
4268 ;;
4269 ;;    TMP = UQDEC[BHWD] (B, X)
4270 ;;    A = TMP + X
4271 ;;
4272 ;; Defining the pattern this way means that:
4273 ;;
4274 ;;    A = UMAX (B, X) - X
4275 ;;
4276 ;; becomes:
4277 ;;
4278 ;;    TMP1 = UQDEC[BHWD] (B, X)
4279 ;;    TMP2 = TMP1 + X
4280 ;;    A = TMP2 - X
4281 ;;
4282 ;; which combine can optimize to:
4283 ;;
4284 ;;    A = UQDEC[BHWD] (B, X)
4285 ;;
4286 ;; We don't use match_operand predicates because the order of the operands
4287 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4288 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4289 (define_expand "umax<mode>3"
4290   [(set (match_operand:GPI 0 "register_operand")
4291         (umax:GPI (match_operand:GPI 1 "")
4292                   (match_operand:GPI 2 "")))]
4293   "TARGET_SVE"
4294   {
4295     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4296       std::swap (operands[1], operands[2]);
4297     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4298       FAIL;
4299     rtx temp = gen_reg_rtx (<MODE>mode);
4300     operands[1] = force_reg (<MODE>mode, operands[1]);
4301     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4302     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4303     DONE;
4304   }
4305 )
4306
4307 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4308 (define_insn "aarch64_uqdec<mode>"
4309   [(set (match_operand:GPI 0 "register_operand" "=r")
4310         (minus:GPI
4311          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4312                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4313          (match_dup 2)))]
4314   "TARGET_SVE"
4315   {
4316     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4317   }
4318 )
4319
4320 ;; -------------------------------------------------------------------
4321 ;; Logical operations
4322 ;; -------------------------------------------------------------------
4323
4324
4325 (define_insn_and_split "*aarch64_and<mode>_imm2"
4326   [(set (match_operand:GPI 0 "register_operand" "=rk")
4327         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4328                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4329   ""
4330   "#"
4331   "true"
4332   [(const_int 0)]
4333   {
4334      HOST_WIDE_INT val = INTVAL (operands[2]);
4335      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4336      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4337
4338      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4339      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4340      DONE;
4341   }
4342 )
4343
4344 (define_insn "<optab><mode>3"
4345   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4346         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4347                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4348   ""
4349   "@
4350   <logical>\\t%<w>0, %<w>1, %<w>2
4351   <logical>\\t%<w>0, %<w>1, %2
4352   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4353   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4354    (set_attr "arch" "*,*,simd")]
4355 )
4356
4357 ;; zero_extend version of above
4358 (define_insn "*<optab>si3_uxtw"
4359   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4360         (zero_extend:DI
4361          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4362                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4363   ""
4364   "@
4365    <logical>\\t%w0, %w1, %w2
4366    <logical>\\t%w0, %w1, %2"
4367   [(set_attr "type" "logic_reg,logic_imm")]
4368 )
4369
4370 (define_insn "*and<mode>3_compare0"
4371   [(set (reg:CC_NZ CC_REGNUM)
4372         (compare:CC_NZ
4373          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4374                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4375          (const_int 0)))
4376    (set (match_operand:GPI 0 "register_operand" "=r,r")
4377         (and:GPI (match_dup 1) (match_dup 2)))]
4378   ""
4379   "@
4380    ands\\t%<w>0, %<w>1, %<w>2
4381    ands\\t%<w>0, %<w>1, %2"
4382   [(set_attr "type" "logics_reg,logics_imm")]
4383 )
4384
4385 ;; zero_extend version of above
4386 (define_insn "*andsi3_compare0_uxtw"
4387   [(set (reg:CC_NZ CC_REGNUM)
4388         (compare:CC_NZ
4389          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4390                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4391          (const_int 0)))
4392    (set (match_operand:DI 0 "register_operand" "=r,r")
4393         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4394   ""
4395   "@
4396    ands\\t%w0, %w1, %w2
4397    ands\\t%w0, %w1, %2"
4398   [(set_attr "type" "logics_reg,logics_imm")]
4399 )
4400
4401 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4402   [(set (reg:CC_NZ CC_REGNUM)
4403         (compare:CC_NZ
4404          (and:GPI (SHIFT:GPI
4405                    (match_operand:GPI 1 "register_operand" "r")
4406                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4407                   (match_operand:GPI 3 "register_operand" "r"))
4408          (const_int 0)))
4409    (set (match_operand:GPI 0 "register_operand" "=r")
4410         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4411   ""
4412   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4413   [(set_attr "type" "logics_shift_imm")]
4414 )
4415
4416 ;; zero_extend version of above
4417 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4418   [(set (reg:CC_NZ CC_REGNUM)
4419         (compare:CC_NZ
4420          (and:SI (SHIFT:SI
4421                   (match_operand:SI 1 "register_operand" "r")
4422                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4423                  (match_operand:SI 3 "register_operand" "r"))
4424          (const_int 0)))
4425    (set (match_operand:DI 0 "register_operand" "=r")
4426         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4427                                 (match_dup 3))))]
4428   ""
4429   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4430   [(set_attr "type" "logics_shift_imm")]
4431 )
4432
4433 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4434   [(set (match_operand:GPI 0 "register_operand" "=r")
4435         (LOGICAL:GPI (SHIFT:GPI
4436                       (match_operand:GPI 1 "register_operand" "r")
4437                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4438                      (match_operand:GPI 3 "register_operand" "r")))]
4439   ""
4440   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4441   [(set_attr "type" "logic_shift_imm")]
4442 )
4443
4444 (define_insn "*<optab>_rol<mode>3"
4445   [(set (match_operand:GPI 0 "register_operand" "=r")
4446         (LOGICAL:GPI (rotate:GPI
4447                       (match_operand:GPI 1 "register_operand" "r")
4448                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4449                      (match_operand:GPI 3 "register_operand" "r")))]
4450   ""
4451   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4452   [(set_attr "type" "logic_shift_imm")]
4453 )
4454
4455 ;; zero_extend versions of above
4456 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4457   [(set (match_operand:DI 0 "register_operand" "=r")
4458         (zero_extend:DI
4459          (LOGICAL:SI (SHIFT:SI
4460                       (match_operand:SI 1 "register_operand" "r")
4461                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4462                      (match_operand:SI 3 "register_operand" "r"))))]
4463   ""
4464   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4465   [(set_attr "type" "logic_shift_imm")]
4466 )
4467
4468 (define_insn "*<optab>_rolsi3_uxtw"
4469   [(set (match_operand:DI 0 "register_operand" "=r")
4470         (zero_extend:DI
4471          (LOGICAL:SI (rotate:SI
4472                       (match_operand:SI 1 "register_operand" "r")
4473                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4474                      (match_operand:SI 3 "register_operand" "r"))))]
4475   ""
4476   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4477   [(set_attr "type" "logic_shift_imm")]
4478 )
4479
4480 (define_insn "one_cmpl<mode>2"
4481   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4482         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4483   ""
4484   "@
4485   mvn\\t%<w>0, %<w>1
4486   mvn\\t%0.8b, %1.8b"
4487   [(set_attr "type" "logic_reg,neon_logic")
4488    (set_attr "arch" "*,simd")]
4489 )
4490
4491 (define_insn "*one_cmpl_<optab><mode>2"
4492   [(set (match_operand:GPI 0 "register_operand" "=r")
4493         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4494                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4495   ""
4496   "mvn\\t%<w>0, %<w>1, <shift> %2"
4497   [(set_attr "type" "logic_shift_imm")]
4498 )
4499
4500 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4501
4502 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4503   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4504         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4505                      (match_operand:GPI 2 "register_operand" "r,w")))]
4506   ""
4507   "@
4508   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4509   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4510   [(set_attr "type" "logic_reg,neon_logic")
4511    (set_attr "arch" "*,simd")]
4512 )
4513
4514 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4515   [(set (match_operand:DI 0 "register_operand" "=r")
4516         (zero_extend:DI
4517           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4518                        (match_operand:SI 2 "register_operand" "r"))))]
4519   ""
4520   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4521   [(set_attr "type" "logic_reg")]
4522 )
4523
4524 (define_insn "*xor_one_cmplsidi3_ze"
4525   [(set (match_operand:DI 0 "register_operand" "=r")
4526         (zero_extend:DI
4527           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4528                           (match_operand:SI 2 "register_operand" "r")))))]
4529   ""
4530   "eon\\t%w0, %w1, %w2"
4531   [(set_attr "type" "logic_reg")]
4532 )
4533
4534 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4535 ;; eon does not operate on SIMD registers so the vector variant must be split.
4536 (define_insn_and_split "*xor_one_cmpl<mode>3"
4537   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4538         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4539                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4540   ""
4541   "@
4542   eon\\t%<w>0, %<w>1, %<w>2
4543   #"
4544   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4545   [(set (match_operand:GPI 0 "register_operand" "=w")
4546         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4547                  (match_operand:GPI 2 "register_operand" "w")))
4548    (set (match_dup 0) (not:GPI (match_dup 0)))]
4549   ""
4550   [(set_attr "type" "logic_reg,multiple")
4551    (set_attr "arch" "*,simd")]
4552 )
4553
4554 (define_insn "*and_one_cmpl<mode>3_compare0"
4555   [(set (reg:CC_NZ CC_REGNUM)
4556         (compare:CC_NZ
4557          (and:GPI (not:GPI
4558                    (match_operand:GPI 1 "register_operand" "r"))
4559                   (match_operand:GPI 2 "register_operand" "r"))
4560          (const_int 0)))
4561    (set (match_operand:GPI 0 "register_operand" "=r")
4562         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4563   ""
4564   "bics\\t%<w>0, %<w>2, %<w>1"
4565   [(set_attr "type" "logics_reg")]
4566 )
4567
4568 ;; zero_extend version of above
4569 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4570   [(set (reg:CC_NZ CC_REGNUM)
4571         (compare:CC_NZ
4572          (and:SI (not:SI
4573                   (match_operand:SI 1 "register_operand" "r"))
4574                  (match_operand:SI 2 "register_operand" "r"))
4575          (const_int 0)))
4576    (set (match_operand:DI 0 "register_operand" "=r")
4577         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4578   ""
4579   "bics\\t%w0, %w2, %w1"
4580   [(set_attr "type" "logics_reg")]
4581 )
4582
4583 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4584   [(set (reg:CC_NZ CC_REGNUM)
4585     (compare:CC_NZ
4586      (and:GPI (not:GPI
4587            (match_operand:GPI 0 "register_operand" "r"))
4588           (match_operand:GPI 1 "register_operand" "r"))
4589      (const_int 0)))]
4590   ""
4591   "bics\\t<w>zr, %<w>1, %<w>0"
4592   [(set_attr "type" "logics_reg")]
4593 )
4594
4595 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4596   [(set (match_operand:GPI 0 "register_operand" "=r")
4597         (LOGICAL:GPI (not:GPI
4598                       (SHIFT:GPI
4599                        (match_operand:GPI 1 "register_operand" "r")
4600                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4601                      (match_operand:GPI 3 "register_operand" "r")))]
4602   ""
4603   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4604   [(set_attr "type" "logic_shift_imm")]
4605 )
4606
4607 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4608   [(set (match_operand:GPI 0 "register_operand" "=r")
4609         (not:GPI (xor:GPI
4610                       (SHIFT:GPI
4611                        (match_operand:GPI 1 "register_operand" "r")
4612                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4613                      (match_operand:GPI 3 "register_operand" "r"))))]
4614   ""
4615   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4616   [(set_attr "type" "logic_shift_imm")]
4617 )
4618
4619 ;; Zero-extend version of the above.
4620 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4621   [(set (match_operand:DI 0 "register_operand" "=r")
4622         (zero_extend:DI
4623           (not:SI (xor:SI
4624                     (SHIFT:SI
4625                       (match_operand:SI 1 "register_operand" "r")
4626                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4627                     (match_operand:SI 3 "register_operand" "r")))))]
4628   ""
4629   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4630   [(set_attr "type" "logic_shift_imm")]
4631 )
4632
4633 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4634   [(set (reg:CC_NZ CC_REGNUM)
4635         (compare:CC_NZ
4636          (and:GPI (not:GPI
4637                    (SHIFT:GPI
4638                     (match_operand:GPI 1 "register_operand" "r")
4639                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4640                   (match_operand:GPI 3 "register_operand" "r"))
4641          (const_int 0)))
4642    (set (match_operand:GPI 0 "register_operand" "=r")
4643         (and:GPI (not:GPI
4644                   (SHIFT:GPI
4645                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4646   ""
4647   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4648   [(set_attr "type" "logics_shift_imm")]
4649 )
4650
4651 ;; zero_extend version of above
4652 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4653   [(set (reg:CC_NZ CC_REGNUM)
4654         (compare:CC_NZ
4655          (and:SI (not:SI
4656                   (SHIFT:SI
4657                    (match_operand:SI 1 "register_operand" "r")
4658                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4659                  (match_operand:SI 3 "register_operand" "r"))
4660          (const_int 0)))
4661    (set (match_operand:DI 0 "register_operand" "=r")
4662         (zero_extend:DI (and:SI
4663                          (not:SI
4664                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4665   ""
4666   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4667   [(set_attr "type" "logics_shift_imm")]
4668 )
4669
4670 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4671   [(set (reg:CC_NZ CC_REGNUM)
4672     (compare:CC_NZ
4673      (and:GPI (not:GPI
4674            (SHIFT:GPI
4675             (match_operand:GPI 0 "register_operand" "r")
4676             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4677           (match_operand:GPI 2 "register_operand" "r"))
4678      (const_int 0)))]
4679   ""
4680   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4681   [(set_attr "type" "logics_shift_imm")]
4682 )
4683
4684 (define_insn "clz<mode>2"
4685   [(set (match_operand:GPI 0 "register_operand" "=r")
4686         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4687   ""
4688   "clz\\t%<w>0, %<w>1"
4689   [(set_attr "type" "clz")]
4690 )
4691
4692 (define_expand "ffs<mode>2"
4693   [(match_operand:GPI 0 "register_operand")
4694    (match_operand:GPI 1 "register_operand")]
4695   ""
4696   {
4697     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4698     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4699
4700     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4701     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4702     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4703     DONE;
4704   }
4705 )
4706
4707 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4708 ;;
4709 ;; MOV  v.1d, x0
4710 ;; CNT  v1.8b, v.8b
4711 ;; ADDV b2, v1.8b
4712 ;; MOV  w0, v2.b[0]
4713
4714 (define_expand "popcount<mode>2"
4715   [(match_operand:GPI 0 "register_operand")
4716    (match_operand:GPI 1 "register_operand")]
4717   "TARGET_SIMD"
4718 {
4719   rtx v = gen_reg_rtx (V8QImode);
4720   rtx v1 = gen_reg_rtx (V8QImode);
4721   rtx r = gen_reg_rtx (QImode);
4722   rtx in = operands[1];
4723   rtx out = operands[0];
4724   if(<MODE>mode == SImode)
4725     {
4726       rtx tmp;
4727       tmp = gen_reg_rtx (DImode);
4728       /* If we have SImode, zero extend to DImode, pop count does
4729          not change if we have extra zeros. */
4730       emit_insn (gen_zero_extendsidi2 (tmp, in));
4731       in = tmp;
4732     }
4733   emit_move_insn (v, gen_lowpart (V8QImode, in));
4734   emit_insn (gen_popcountv8qi2 (v1, v));
4735   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4736   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4737   DONE;
4738 })
4739
4740 (define_insn "clrsb<mode>2"
4741   [(set (match_operand:GPI 0 "register_operand" "=r")
4742         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4743   ""
4744   "cls\\t%<w>0, %<w>1"
4745   [(set_attr "type" "clz")]
4746 )
4747
4748 (define_insn "rbit<mode>2"
4749   [(set (match_operand:GPI 0 "register_operand" "=r")
4750         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4751   ""
4752   "rbit\\t%<w>0, %<w>1"
4753   [(set_attr "type" "rbit")]
4754 )
4755
4756 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4757 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4758 ;; expression and split after reload to enable scheduling them apart if
4759 ;; needed.
4760
4761 (define_insn_and_split "ctz<mode>2"
4762  [(set (match_operand:GPI           0 "register_operand" "=r")
4763        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4764   ""
4765   "#"
4766   "reload_completed"
4767   [(const_int 0)]
4768   "
4769   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4770   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4771   DONE;
4772 ")
4773
4774 (define_insn "*and<mode>_compare0"
4775   [(set (reg:CC_NZ CC_REGNUM)
4776         (compare:CC_NZ
4777          (match_operand:SHORT 0 "register_operand" "r")
4778          (const_int 0)))]
4779   ""
4780   "tst\\t%<w>0, <short_mask>"
4781   [(set_attr "type" "alus_imm")]
4782 )
4783
4784 (define_insn "*ands<mode>_compare0"
4785   [(set (reg:CC_NZ CC_REGNUM)
4786         (compare:CC_NZ
4787          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4788          (const_int 0)))
4789    (set (match_operand:GPI 0 "register_operand" "=r")
4790         (zero_extend:GPI (match_dup 1)))]
4791   ""
4792   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4793   [(set_attr "type" "alus_imm")]
4794 )
4795
4796 (define_insn "*and<mode>3nr_compare0"
4797   [(set (reg:CC_NZ CC_REGNUM)
4798         (compare:CC_NZ
4799          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4800                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4801          (const_int 0)))]
4802   ""
4803   "@
4804    tst\\t%<w>0, %<w>1
4805    tst\\t%<w>0, %1"
4806   [(set_attr "type" "logics_reg,logics_imm")]
4807 )
4808
4809 (define_split
4810   [(set (reg:CC_NZ CC_REGNUM)
4811         (compare:CC_NZ
4812          (and:GPI (match_operand:GPI 0 "register_operand")
4813                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4814          (const_int 0)))
4815    (clobber (match_operand:SI 2 "register_operand"))]
4816   ""
4817   [(set (match_dup 2) (match_dup 1))
4818    (set (reg:CC_NZ CC_REGNUM)
4819         (compare:CC_NZ
4820          (and:GPI (match_dup 0)
4821                   (match_dup 2))
4822          (const_int 0)))]
4823 )
4824
4825 (define_insn "*and<mode>3nr_compare0_zextract"
4826   [(set (reg:CC_NZ CC_REGNUM)
4827         (compare:CC_NZ
4828          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4829                   (match_operand:GPI 1 "const_int_operand" "n")
4830                   (match_operand:GPI 2 "const_int_operand" "n"))
4831          (const_int 0)))]
4832   "INTVAL (operands[1]) > 0
4833    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4834         <= GET_MODE_BITSIZE (<MODE>mode))
4835    && aarch64_bitmask_imm (
4836         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4837                                                  operands[2])),
4838         <MODE>mode)"
4839   {
4840     operands[1]
4841       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4842     return "tst\\t%<w>0, %1";
4843   }
4844   [(set_attr "type" "logics_shift_imm")]
4845 )
4846
4847 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4848   [(set (reg:CC_NZ CC_REGNUM)
4849         (compare:CC_NZ
4850          (and:GPI (SHIFT:GPI
4851                    (match_operand:GPI 0 "register_operand" "r")
4852                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4853                   (match_operand:GPI 2 "register_operand" "r"))
4854         (const_int 0)))]
4855   ""
4856   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4857   [(set_attr "type" "logics_shift_imm")]
4858 )
4859
4860 (define_split
4861   [(set (reg:CC_NZ CC_REGNUM)
4862         (compare:CC_NZ
4863          (and:GPI (SHIFT:GPI
4864                    (match_operand:GPI 0 "register_operand")
4865                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4866                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4867         (const_int 0)))
4868     (clobber (match_operand:SI 3 "register_operand"))]
4869   ""
4870   [(set (match_dup 3) (match_dup 2))
4871    (set (reg:CC_NZ CC_REGNUM)
4872         (compare:CC_NZ
4873          (and:GPI (SHIFT:GPI
4874                    (match_dup 0)
4875                    (match_dup 1))
4876                   (match_dup 3))
4877          (const_int 0)))]
4878 )
4879
4880 ;; -------------------------------------------------------------------
4881 ;; Shifts
4882 ;; -------------------------------------------------------------------
4883
4884 (define_expand "<optab><mode>3"
4885   [(set (match_operand:GPI 0 "register_operand")
4886         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4887                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4888   ""
4889   {
4890     if (CONST_INT_P (operands[2]))
4891       {
4892         operands[2] = GEN_INT (INTVAL (operands[2])
4893                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4894
4895         if (operands[2] == const0_rtx)
4896           {
4897             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4898             DONE;
4899           }
4900       }
4901   }
4902 )
4903
4904 (define_expand "ashl<mode>3"
4905   [(set (match_operand:SHORT 0 "register_operand")
4906         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4907                       (match_operand:QI 2 "const_int_operand")))]
4908   ""
4909   {
4910     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4911
4912     if (operands[2] == const0_rtx)
4913       {
4914         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4915         DONE;
4916       }
4917   }
4918 )
4919
4920 (define_expand "rotr<mode>3"
4921   [(set (match_operand:GPI 0 "register_operand")
4922         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4923                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4924   ""
4925   {
4926     if (CONST_INT_P (operands[2]))
4927       {
4928         operands[2] = GEN_INT (INTVAL (operands[2])
4929                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4930
4931         if (operands[2] == const0_rtx)
4932           {
4933             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4934             DONE;
4935           }
4936       }
4937   }
4938 )
4939
4940 (define_expand "rotl<mode>3"
4941   [(set (match_operand:GPI 0 "register_operand")
4942         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4943                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4944   ""
4945   {
4946     /* (SZ - cnt) % SZ == -cnt % SZ */
4947     if (CONST_INT_P (operands[2]))
4948       {
4949         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4950                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4951         if (operands[2] == const0_rtx)
4952           {
4953             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4954             DONE;
4955           }
4956       }
4957     else
4958       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4959                                         NULL_RTX, 1);
4960   }
4961 )
4962
4963 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4964 ;; they truncate the shift/rotate amount by the size of the registers they
4965 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4966 ;; such redundant masking instructions.  GCC can do that automatically when
4967 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4968 ;; because some of the SISD shift alternatives don't perform this truncations.
4969 ;; So this pattern exists to catch such cases.
4970
4971 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4972   [(set (match_operand:GPI 0 "register_operand" "=r")
4973         (SHIFT:GPI
4974           (match_operand:GPI 1 "register_operand" "r")
4975           (match_operator 4 "subreg_lowpart_operator"
4976            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4977                      (match_operand 3 "const_int_operand" "n"))])))]
4978   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4979   "<shift>\t%<w>0, %<w>1, %<w>2"
4980   [(set_attr "type" "shift_reg")]
4981 )
4982
4983 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4984   [(set (match_operand:GPI 0 "register_operand" "=&r")
4985         (SHIFT:GPI
4986           (match_operand:GPI 1 "register_operand" "r")
4987           (match_operator 4 "subreg_lowpart_operator"
4988           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4989                            (match_operand 3 "const_int_operand" "n")))])))]
4990   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4991   "#"
4992   "&& true"
4993   [(const_int 0)]
4994   {
4995     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4996                : lowpart_subreg (SImode, operands[0], <MODE>mode));
4997     emit_insn (gen_negsi2 (tmp, operands[2]));
4998
4999     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5000     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5001                                      SUBREG_BYTE (operands[4]));
5002     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5003     DONE;
5004   }
5005 )
5006
5007 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5008   [(set (match_operand:GPI 0 "register_operand" "=&r")
5009         (ashift:GPI
5010           (match_operand:GPI 1 "register_operand" "r")
5011           (minus:QI (match_operand 2 "const_int_operand" "n")
5012                     (match_operator 5 "subreg_lowpart_operator"
5013                     [(and:SI (match_operand:SI 3 "register_operand" "r")
5014                              (match_operand 4 "const_int_operand" "n"))]))))]
5015   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5016    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5017   "#"
5018   "&& true"
5019   [(const_int 0)]
5020   {
5021     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5022                : operands[0]);
5023
5024     emit_insn (gen_negsi2 (tmp, operands[3]));
5025
5026     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5027     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5028                                      SUBREG_BYTE (operands[5]));
5029
5030     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5031     DONE;
5032   }
5033 )
5034
5035 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5036   [(set (match_operand:DI 0 "register_operand" "=r")
5037         (SHIFT:DI
5038           (match_operand:DI 1 "register_operand" "r")
5039           (match_operator 4 "subreg_lowpart_operator"
5040            [(and:SI (match_operand:SI 2 "register_operand" "r")
5041                     (match_operand 3 "const_int_operand" "n"))])))]
5042   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5043 {
5044   rtx xop[3];
5045   xop[0] = operands[0];
5046   xop[1] = operands[1];
5047   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5048   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5049   return "";
5050 }
5051   [(set_attr "type" "shift_reg")]
5052 )
5053
5054 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5055   [(set (match_operand:GPI 0 "register_operand" "=&r")
5056         (ASHIFT:GPI
5057           (match_operand:GPI 1 "register_operand" "r")
5058           (minus:QI (match_operand 2 "const_int_operand" "n")
5059                     (match_operand:QI 3 "register_operand" "r"))))]
5060   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5061   "#"
5062   "&& true"
5063   [(const_int 0)]
5064   {
5065     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5066
5067     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5068                : gen_lowpart (SImode, operands[0]));
5069
5070     emit_insn (gen_negsi2 (tmp, subreg_tmp));
5071
5072     rtx and_op = gen_rtx_AND (SImode, tmp,
5073                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5074
5075     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5076
5077     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5078     DONE;
5079   }
5080   [(set_attr "length" "8")]
5081 )
5082
5083 ;; Logical left shift using SISD or Integer instruction
5084 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5085   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5086         (ashift:GPI
5087           (match_operand:GPI 1 "register_operand" "r,r,w,w")
5088           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5089   ""
5090   "@
5091    lsl\t%<w>0, %<w>1, %2
5092    lsl\t%<w>0, %<w>1, %<w>2
5093    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5094    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5095   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5096    (set_attr "arch" "*,*,simd,simd")]
5097 )
5098
5099 ;; Logical right shift using SISD or Integer instruction
5100 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5101   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5102         (lshiftrt:GPI
5103          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5104          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5105                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5106   ""
5107   "@
5108    lsr\t%<w>0, %<w>1, %2
5109    lsr\t%<w>0, %<w>1, %<w>2
5110    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5111    #
5112    #"
5113   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5114    (set_attr "arch" "*,*,simd,simd,simd")]
5115 )
5116
5117 (define_split
5118   [(set (match_operand:DI 0 "aarch64_simd_register")
5119         (lshiftrt:DI
5120            (match_operand:DI 1 "aarch64_simd_register")
5121            (match_operand:QI 2 "aarch64_simd_register")))]
5122   "TARGET_SIMD && reload_completed"
5123   [(set (match_dup 3)
5124         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5125    (set (match_dup 0)
5126         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5127   {
5128     operands[3] = gen_lowpart (QImode, operands[0]);
5129   }
5130 )
5131
5132 (define_split
5133   [(set (match_operand:SI 0 "aarch64_simd_register")
5134         (lshiftrt:SI
5135            (match_operand:SI 1 "aarch64_simd_register")
5136            (match_operand:QI 2 "aarch64_simd_register")))]
5137   "TARGET_SIMD && reload_completed"
5138   [(set (match_dup 3)
5139         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5140    (set (match_dup 0)
5141         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5142   {
5143     operands[3] = gen_lowpart (QImode, operands[0]);
5144   }
5145 )
5146
5147 ;; Arithmetic right shift using SISD or Integer instruction
5148 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5149   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5150         (ashiftrt:GPI
5151           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5152           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5153                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5154   ""
5155   "@
5156    asr\t%<w>0, %<w>1, %2
5157    asr\t%<w>0, %<w>1, %<w>2
5158    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5159    #
5160    #"
5161   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5162    (set_attr "arch" "*,*,simd,simd,simd")]
5163 )
5164
5165 (define_split
5166   [(set (match_operand:DI 0 "aarch64_simd_register")
5167         (ashiftrt:DI
5168            (match_operand:DI 1 "aarch64_simd_register")
5169            (match_operand:QI 2 "aarch64_simd_register")))]
5170   "TARGET_SIMD && reload_completed"
5171   [(set (match_dup 3)
5172         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5173    (set (match_dup 0)
5174         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5175 {
5176   operands[3] = gen_lowpart (QImode, operands[0]);
5177 }
5178 )
5179
5180 (define_split
5181   [(set (match_operand:SI 0 "aarch64_simd_register")
5182         (ashiftrt:SI
5183            (match_operand:SI 1 "aarch64_simd_register")
5184            (match_operand:QI 2 "aarch64_simd_register")))]
5185   "TARGET_SIMD && reload_completed"
5186   [(set (match_dup 3)
5187         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5188    (set (match_dup 0)
5189         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5190 {
5191   operands[3] = gen_lowpart (QImode, operands[0]);
5192 }
5193 )
5194
5195 (define_insn "*aarch64_sisd_ushl"
5196   [(set (match_operand:DI 0 "register_operand" "=w")
5197         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5198                     (match_operand:QI 2 "register_operand" "w")]
5199                    UNSPEC_SISD_USHL))]
5200   "TARGET_SIMD"
5201   "ushl\t%d0, %d1, %d2"
5202   [(set_attr "type" "neon_shift_reg")]
5203 )
5204
5205 (define_insn "*aarch64_ushl_2s"
5206   [(set (match_operand:SI 0 "register_operand" "=w")
5207         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5208                     (match_operand:QI 2 "register_operand" "w")]
5209                    UNSPEC_USHL_2S))]
5210   "TARGET_SIMD"
5211   "ushl\t%0.2s, %1.2s, %2.2s"
5212   [(set_attr "type" "neon_shift_reg")]
5213 )
5214
5215 (define_insn "*aarch64_sisd_sshl"
5216   [(set (match_operand:DI 0 "register_operand" "=w")
5217         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5218                     (match_operand:QI 2 "register_operand" "w")]
5219                    UNSPEC_SISD_SSHL))]
5220   "TARGET_SIMD"
5221   "sshl\t%d0, %d1, %d2"
5222   [(set_attr "type" "neon_shift_reg")]
5223 )
5224
5225 (define_insn "*aarch64_sshl_2s"
5226   [(set (match_operand:SI 0 "register_operand" "=w")
5227         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5228                     (match_operand:QI 2 "register_operand" "w")]
5229                    UNSPEC_SSHL_2S))]
5230   "TARGET_SIMD"
5231   "sshl\t%0.2s, %1.2s, %2.2s"
5232   [(set_attr "type" "neon_shift_reg")]
5233 )
5234
5235 (define_insn "*aarch64_sisd_neg_qi"
5236   [(set (match_operand:QI 0 "register_operand" "=w")
5237         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5238                    UNSPEC_SISD_NEG))]
5239   "TARGET_SIMD"
5240   "neg\t%d0, %d1"
5241   [(set_attr "type" "neon_neg")]
5242 )
5243
5244 ;; Rotate right
5245 (define_insn "*ror<mode>3_insn"
5246   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5247      (rotatert:GPI
5248        (match_operand:GPI 1 "register_operand" "r,r")
5249        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5250   ""
5251   "@
5252    ror\\t%<w>0, %<w>1, %2
5253    ror\\t%<w>0, %<w>1, %<w>2"
5254   [(set_attr "type" "rotate_imm,shift_reg")]
5255 )
5256
5257 ;; zero_extend version of above
5258 (define_insn "*<optab>si3_insn_uxtw"
5259   [(set (match_operand:DI 0 "register_operand" "=r,r")
5260         (zero_extend:DI (SHIFT:SI
5261          (match_operand:SI 1 "register_operand" "r,r")
5262          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5263   ""
5264   "@
5265    <shift>\\t%w0, %w1, %2
5266    <shift>\\t%w0, %w1, %w2"
5267   [(set_attr "type" "bfx,shift_reg")]
5268 )
5269
5270 (define_insn "*<optab><mode>3_insn"
5271   [(set (match_operand:SHORT 0 "register_operand" "=r")
5272         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5273                       (match_operand 2 "const_int_operand" "n")))]
5274   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5275 {
5276   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5277   return "<bfshift>\t%w0, %w1, %2, %3";
5278 }
5279   [(set_attr "type" "bfx")]
5280 )
5281
5282 (define_insn "*extr<mode>5_insn"
5283   [(set (match_operand:GPI 0 "register_operand" "=r")
5284         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5285                              (match_operand 3 "const_int_operand" "n"))
5286                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5287                                (match_operand 4 "const_int_operand" "n"))))]
5288   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5289    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5290   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5291   [(set_attr "type" "rotate_imm")]
5292 )
5293
5294 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5295 ;; so we have to match both orderings.
5296 (define_insn "*extr<mode>5_insn_alt"
5297   [(set (match_operand:GPI 0 "register_operand" "=r")
5298         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5299                                 (match_operand 4 "const_int_operand" "n"))
5300                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5301                               (match_operand 3 "const_int_operand" "n"))))]
5302   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5303    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5304        == GET_MODE_BITSIZE (<MODE>mode))"
5305   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5306   [(set_attr "type" "rotate_imm")]
5307 )
5308
5309 ;; zero_extend version of the above
5310 (define_insn "*extrsi5_insn_uxtw"
5311   [(set (match_operand:DI 0 "register_operand" "=r")
5312         (zero_extend:DI
5313          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5314                             (match_operand 3 "const_int_operand" "n"))
5315                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5316                               (match_operand 4 "const_int_operand" "n")))))]
5317   "UINTVAL (operands[3]) < 32 &&
5318    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5319   "extr\\t%w0, %w1, %w2, %4"
5320   [(set_attr "type" "rotate_imm")]
5321 )
5322
5323 (define_insn "*extrsi5_insn_uxtw_alt"
5324   [(set (match_operand:DI 0 "register_operand" "=r")
5325         (zero_extend:DI
5326          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5327                                (match_operand 4 "const_int_operand" "n"))
5328                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5329                             (match_operand 3 "const_int_operand" "n")))))]
5330   "UINTVAL (operands[3]) < 32 &&
5331    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5332   "extr\\t%w0, %w1, %w2, %4"
5333   [(set_attr "type" "rotate_imm")]
5334 )
5335
5336 (define_insn "*ror<mode>3_insn"
5337   [(set (match_operand:GPI 0 "register_operand" "=r")
5338         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5339                     (match_operand 2 "const_int_operand" "n")))]
5340   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5341 {
5342   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5343   return "ror\\t%<w>0, %<w>1, %3";
5344 }
5345   [(set_attr "type" "rotate_imm")]
5346 )
5347
5348 ;; zero_extend version of the above
5349 (define_insn "*rorsi3_insn_uxtw"
5350   [(set (match_operand:DI 0 "register_operand" "=r")
5351         (zero_extend:DI
5352          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5353                     (match_operand 2 "const_int_operand" "n"))))]
5354   "UINTVAL (operands[2]) < 32"
5355 {
5356   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5357   return "ror\\t%w0, %w1, %3";
5358 }
5359   [(set_attr "type" "rotate_imm")]
5360 )
5361
5362 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5363   [(set (match_operand:GPI 0 "register_operand" "=r")
5364         (ANY_EXTEND:GPI
5365          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5366                        (match_operand 2 "const_int_operand" "n"))))]
5367   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5368 {
5369   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5370   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5371 }
5372   [(set_attr "type" "bfx")]
5373 )
5374
5375 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5376   [(set (match_operand:GPI 0 "register_operand" "=r")
5377         (zero_extend:GPI
5378          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5379                          (match_operand 2 "const_int_operand" "n"))))]
5380   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5381 {
5382   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5383   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5384 }
5385   [(set_attr "type" "bfx")]
5386 )
5387
5388 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5389   [(set (match_operand:GPI 0 "register_operand" "=r")
5390         (sign_extend:GPI
5391          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5392                          (match_operand 2 "const_int_operand" "n"))))]
5393   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5394 {
5395   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5396   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5397 }
5398   [(set_attr "type" "bfx")]
5399 )
5400
5401 ;; -------------------------------------------------------------------
5402 ;; Bitfields
5403 ;; -------------------------------------------------------------------
5404
5405 (define_expand "<optab>"
5406   [(set (match_operand:DI 0 "register_operand" "=r")
5407         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5408                         (match_operand 2
5409                           "aarch64_simd_shift_imm_offset_di")
5410                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5411   ""
5412   {
5413     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5414                    1, GET_MODE_BITSIZE (DImode) - 1))
5415      FAIL;
5416   }
5417 )
5418
5419
5420 (define_insn "*<optab><mode>"
5421   [(set (match_operand:GPI 0 "register_operand" "=r")
5422         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5423                          (match_operand 2
5424                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5425                          (match_operand 3
5426                            "aarch64_simd_shift_imm_<mode>" "n")))]
5427   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5428              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5429   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5430   [(set_attr "type" "bfx")]
5431 )
5432
5433 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5434 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5435 (define_split
5436   [(set (match_operand:DI 0 "register_operand")
5437         (zero_extract:DI (match_operand:DI 1 "register_operand")
5438                          (match_operand 2
5439                            "aarch64_simd_shift_imm_offset_di")
5440                          (match_operand 3
5441                            "aarch64_simd_shift_imm_di")))]
5442   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5443              GET_MODE_BITSIZE (DImode) - 1)
5444    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5445        == GET_MODE_BITSIZE (SImode)"
5446   [(set (match_dup 0)
5447         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5448   {
5449     operands[4] = gen_lowpart (SImode, operands[1]);
5450   }
5451 )
5452
5453 ;; Bitfield Insert (insv)
5454 (define_expand "insv<mode>"
5455   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5456                           (match_operand 1 "const_int_operand")
5457                           (match_operand 2 "const_int_operand"))
5458         (match_operand:GPI 3 "general_operand"))]
5459   ""
5460 {
5461   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5462   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5463   rtx value = operands[3];
5464
5465   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5466     FAIL;
5467
5468   if (CONST_INT_P (value))
5469     {
5470       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5471
5472       /* Prefer AND/OR for inserting all zeros or all ones.  */
5473       if ((UINTVAL (value) & mask) == 0
5474            || (UINTVAL (value) & mask) == mask)
5475         FAIL;
5476
5477       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5478       if (width == 16 && (pos % 16) == 0)
5479         DONE;
5480     }
5481   operands[3] = force_reg (<MODE>mode, value);
5482 })
5483
5484 (define_insn "*insv_reg<mode>"
5485   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5486                           (match_operand 1 "const_int_operand" "n")
5487                           (match_operand 2 "const_int_operand" "n"))
5488         (match_operand:GPI 3 "register_operand" "r"))]
5489   "!(UINTVAL (operands[1]) == 0
5490      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5491          > GET_MODE_BITSIZE (<MODE>mode)))"
5492   "bfi\\t%<w>0, %<w>3, %2, %1"
5493   [(set_attr "type" "bfm")]
5494 )
5495
5496 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5497   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5498                           (match_operand 1 "const_int_operand" "n")
5499                           (match_operand 2 "const_int_operand" "n"))
5500         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5501   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5502   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5503   [(set_attr "type" "bfm")]
5504 )
5505
5506 (define_insn "*extr_insv_lower_reg<mode>"
5507   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5508                           (match_operand 1 "const_int_operand" "n")
5509                           (const_int 0))
5510         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5511                           (match_dup 1)
5512                           (match_operand 3 "const_int_operand" "n")))]
5513   "!(UINTVAL (operands[1]) == 0
5514      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5515          > GET_MODE_BITSIZE (<MODE>mode)))"
5516   "bfxil\\t%<w>0, %<w>2, %3, %1"
5517   [(set_attr "type" "bfm")]
5518 )
5519
5520 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5521   [(set (match_operand:GPI 0 "register_operand" "=r")
5522         (ashift:GPI (ANY_EXTEND:GPI
5523                      (match_operand:ALLX 1 "register_operand" "r"))
5524                     (match_operand 2 "const_int_operand" "n")))]
5525   "UINTVAL (operands[2]) < <GPI:sizen>"
5526 {
5527   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5528               ? GEN_INT (<ALLX:sizen>)
5529               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5530   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5531 }
5532   [(set_attr "type" "bfx")]
5533 )
5534
5535 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5536
5537 (define_insn "*andim_ashift<mode>_bfiz"
5538   [(set (match_operand:GPI 0 "register_operand" "=r")
5539         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5540                              (match_operand 2 "const_int_operand" "n"))
5541                  (match_operand 3 "const_int_operand" "n")))]
5542   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5543   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5544   [(set_attr "type" "bfx")]
5545 )
5546
5547 ;; Match sbfiz pattern in a shift left + shift right operation.
5548
5549 (define_insn "*ashift<mode>_extv_bfiz"
5550   [(set (match_operand:GPI 0 "register_operand" "=r")
5551         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5552                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5553                                       (const_int 0))
5554                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5555   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5556              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5557   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5558   [(set_attr "type" "bfx")]
5559 )
5560
5561 ;; When the bit position and width of the equivalent extraction add up to 32
5562 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5563 ;; zero-extension of the X-reg.
5564 (define_split
5565   [(set (match_operand:DI 0 "register_operand")
5566         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5567                              (match_operand 2 "const_int_operand"))
5568                  (match_operand 3 "const_int_operand")))]
5569  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5570   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5571       == GET_MODE_BITSIZE (SImode)"
5572   [(set (match_dup 0)
5573         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5574   {
5575     operands[4] = gen_lowpart (SImode, operands[1]);
5576   }
5577 )
5578
5579 (define_insn "bswap<mode>2"
5580   [(set (match_operand:GPI 0 "register_operand" "=r")
5581         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5582   ""
5583   "rev\\t%<w>0, %<w>1"
5584   [(set_attr "type" "rev")]
5585 )
5586
5587 (define_insn "bswaphi2"
5588   [(set (match_operand:HI 0 "register_operand" "=r")
5589         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5590   ""
5591   "rev16\\t%w0, %w1"
5592   [(set_attr "type" "rev")]
5593 )
5594
5595 (define_insn "*aarch64_bfxil<mode>"
5596   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5597     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5598                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5599             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5600                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5601   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5602   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5603     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5604   {
5605     switch (which_alternative)
5606     {
5607       case 0:
5608         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5609         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5610       case 1:
5611         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5612         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5613       default:
5614         gcc_unreachable ();
5615     }
5616   }
5617   [(set_attr "type" "bfm")]
5618 )
5619
5620 ; Zero-extended version of above (aarch64_bfxil)
5621 (define_insn "*aarch64_bfxilsi_uxtw"
5622   [(set (match_operand:DI 0 "register_operand" "=r,r")
5623         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5624                                         "r,0")
5625                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5626             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5627                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5628   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5629   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5630     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5631   {
5632     switch (which_alternative)
5633     {
5634       case 0:
5635         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5636         return "bfxil\\t%0, %1, 0, %3";
5637       case 1:
5638         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5639         return "bfxil\\t%0, %2, 0, %3";
5640       default:
5641         gcc_unreachable ();
5642     }
5643   }
5644   [(set_attr "type" "bfm")]
5645 )
5646
5647 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5648 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5649 ;; each valid permutation.
5650
5651 (define_insn "rev16<mode>2"
5652   [(set (match_operand:GPI 0 "register_operand" "=r")
5653         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5654                                       (const_int 8))
5655                           (match_operand:GPI 3 "const_int_operand" "n"))
5656                  (and:GPI (lshiftrt:GPI (match_dup 1)
5657                                         (const_int 8))
5658                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5659   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5660    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5661   "rev16\\t%<w>0, %<w>1"
5662   [(set_attr "type" "rev")]
5663 )
5664
5665 (define_insn "rev16<mode>2_alt"
5666   [(set (match_operand:GPI 0 "register_operand" "=r")
5667         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5668                                         (const_int 8))
5669                           (match_operand:GPI 2 "const_int_operand" "n"))
5670                  (and:GPI (ashift:GPI (match_dup 1)
5671                                       (const_int 8))
5672                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5673   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5674    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5675   "rev16\\t%<w>0, %<w>1"
5676   [(set_attr "type" "rev")]
5677 )
5678
5679 ;; zero_extend version of above
5680 (define_insn "*bswapsi2_uxtw"
5681   [(set (match_operand:DI 0 "register_operand" "=r")
5682         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5683   ""
5684   "rev\\t%w0, %w1"
5685   [(set_attr "type" "rev")]
5686 )
5687
5688 ;; -------------------------------------------------------------------
5689 ;; Floating-point intrinsics
5690 ;; -------------------------------------------------------------------
5691
5692 ;; frint floating-point round to integral standard patterns.
5693 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5694
5695 (define_insn "<frint_pattern><mode>2"
5696   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5697         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5698          FRINT))]
5699   "TARGET_FLOAT"
5700   "frint<frint_suffix>\\t%<s>0, %<s>1"
5701   [(set_attr "type" "f_rint<stype>")]
5702 )
5703
5704 ;; frcvt floating-point round to integer and convert standard patterns.
5705 ;; Expands to lbtrunc, lceil, lfloor, lround.
5706 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5707   [(set (match_operand:GPI 0 "register_operand" "=r")
5708         (FIXUORS:GPI
5709           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5710            FCVT)))]
5711   "TARGET_FLOAT"
5712   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5713   [(set_attr "type" "f_cvtf2i")]
5714 )
5715
5716 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5717   [(set (match_operand:GPI 0 "register_operand" "=r")
5718         (FIXUORS:GPI
5719           (mult:GPF
5720             (match_operand:GPF 1 "register_operand" "w")
5721             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5722   "TARGET_FLOAT
5723    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5724                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5725   {
5726     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5727     char buf[64];
5728     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5729     output_asm_insn (buf, operands);
5730     return "";
5731   }
5732   [(set_attr "type" "f_cvtf2i")]
5733 )
5734
5735 ;; fma - expand fma into patterns with the accumulator operand first since
5736 ;; reusing the accumulator results in better register allocation.
5737 ;; The register allocator considers copy preferences in operand order,
5738 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5739
5740 (define_expand "fma<mode>4"
5741   [(set (match_operand:GPF_F16 0 "register_operand")
5742         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5743                      (match_operand:GPF_F16 2 "register_operand")
5744                      (match_operand:GPF_F16 3 "register_operand")))]
5745   "TARGET_FLOAT"
5746 )
5747
5748 (define_insn "*aarch64_fma<mode>4"
5749   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5750         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5751                      (match_operand:GPF_F16 3 "register_operand" "w")
5752                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5753   "TARGET_FLOAT"
5754   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5755   [(set_attr "type" "fmac<stype>")]
5756 )
5757
5758 (define_expand "fnma<mode>4"
5759   [(set (match_operand:GPF_F16 0 "register_operand")
5760         (fma:GPF_F16
5761           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5762           (match_operand:GPF_F16 2 "register_operand")
5763           (match_operand:GPF_F16 3 "register_operand")))]
5764   "TARGET_FLOAT"
5765 )
5766
5767 (define_insn "*aarch64_fnma<mode>4"
5768   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5769         (fma:GPF_F16
5770           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5771           (match_operand:GPF_F16 3 "register_operand" "w")
5772           (match_operand:GPF_F16 1 "register_operand" "w")))]
5773   "TARGET_FLOAT"
5774   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5775   [(set_attr "type" "fmac<stype>")]
5776 )
5777
5778
5779 (define_expand "fms<mode>4"
5780   [(set (match_operand:GPF 0 "register_operand")
5781         (fma:GPF (match_operand:GPF 1 "register_operand")
5782                  (match_operand:GPF 2 "register_operand")
5783                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5784   "TARGET_FLOAT"
5785 )
5786
5787 (define_insn "*aarch64_fms<mode>4"
5788   [(set (match_operand:GPF 0 "register_operand" "=w")
5789         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5790                  (match_operand:GPF 3 "register_operand" "w")
5791                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5792   "TARGET_FLOAT"
5793   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5794   [(set_attr "type" "fmac<s>")]
5795 )
5796
5797 (define_expand "fnms<mode>4"
5798   [(set (match_operand:GPF 0 "register_operand")
5799         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5800                  (match_operand:GPF 2 "register_operand")
5801                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5802   "TARGET_FLOAT"
5803 )
5804
5805 (define_insn "*aarch64_fnms<mode>4"
5806   [(set (match_operand:GPF 0 "register_operand" "=w")
5807         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5808                  (match_operand:GPF 3 "register_operand" "w")
5809                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5810   "TARGET_FLOAT"
5811   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5812   [(set_attr "type" "fmac<s>")]
5813 )
5814
5815 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5816 (define_insn "*aarch64_fnmadd<mode>4"
5817   [(set (match_operand:GPF 0 "register_operand" "=w")
5818         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5819                           (match_operand:GPF 3 "register_operand" "w")
5820                           (match_operand:GPF 1 "register_operand" "w"))))]
5821   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5822   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5823   [(set_attr "type" "fmac<s>")]
5824 )
5825
5826 ;; -------------------------------------------------------------------
5827 ;; Floating-point conversions
5828 ;; -------------------------------------------------------------------
5829
5830 (define_insn "extendsfdf2"
5831   [(set (match_operand:DF 0 "register_operand" "=w")
5832         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5833   "TARGET_FLOAT"
5834   "fcvt\\t%d0, %s1"
5835   [(set_attr "type" "f_cvt")]
5836 )
5837
5838 (define_insn "extendhfsf2"
5839   [(set (match_operand:SF 0 "register_operand" "=w")
5840         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5841   "TARGET_FLOAT"
5842   "fcvt\\t%s0, %h1"
5843   [(set_attr "type" "f_cvt")]
5844 )
5845
5846 (define_insn "extendhfdf2"
5847   [(set (match_operand:DF 0 "register_operand" "=w")
5848         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5849   "TARGET_FLOAT"
5850   "fcvt\\t%d0, %h1"
5851   [(set_attr "type" "f_cvt")]
5852 )
5853
5854 (define_insn "truncdfsf2"
5855   [(set (match_operand:SF 0 "register_operand" "=w")
5856         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5857   "TARGET_FLOAT"
5858   "fcvt\\t%s0, %d1"
5859   [(set_attr "type" "f_cvt")]
5860 )
5861
5862 (define_insn "truncsfhf2"
5863   [(set (match_operand:HF 0 "register_operand" "=w")
5864         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5865   "TARGET_FLOAT"
5866   "fcvt\\t%h0, %s1"
5867   [(set_attr "type" "f_cvt")]
5868 )
5869
5870 (define_insn "truncdfhf2"
5871   [(set (match_operand:HF 0 "register_operand" "=w")
5872         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5873   "TARGET_FLOAT"
5874   "fcvt\\t%h0, %d1"
5875   [(set_attr "type" "f_cvt")]
5876 )
5877
5878 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5879 ;; and making r = w more expensive
5880
5881 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5882   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5883         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5884   "TARGET_FLOAT"
5885   "@
5886    fcvtz<su>\t%<s>0, %<s>1
5887    fcvtz<su>\t%<w>0, %<s>1"
5888   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5889 )
5890
5891 ;; Convert HF -> SI or DI
5892
5893 (define_insn "<optab>_trunchf<GPI:mode>2"
5894   [(set (match_operand:GPI 0 "register_operand" "=r")
5895         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5896   "TARGET_FP_F16INST"
5897   "fcvtz<su>\t%<w>0, %h1"
5898   [(set_attr "type" "f_cvtf2i")]
5899 )
5900
5901 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5902 ;; input in a fp register and output in a integer register
5903
5904 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5905   [(set (match_operand:GPI 0 "register_operand" "=r")
5906         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5907   "TARGET_FLOAT"
5908   "fcvtz<su>\t%<w>0, %<fpw>1"
5909   [(set_attr "type" "f_cvtf2i")]
5910 )
5911
5912 (define_insn "*fix_to_zero_extend<mode>di2"
5913   [(set (match_operand:DI 0 "register_operand" "=r")
5914         (zero_extend:DI
5915          (unsigned_fix:SI
5916           (match_operand:GPF 1 "register_operand" "w"))))]
5917   "TARGET_FLOAT"
5918   "fcvtzu\t%w0, %<s>1"
5919   [(set_attr "type" "f_cvtf2i")]
5920 )
5921
5922 (define_insn "<optab><fcvt_target><GPF:mode>2"
5923   [(set (match_operand:GPF 0 "register_operand" "=w,w")
5924         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
5925   "TARGET_FLOAT"
5926   "@
5927    <su_optab>cvtf\t%<GPF:s>0, %<s>1
5928    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5929   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
5930    (set_attr "arch" "simd,fp")]
5931 )
5932
5933 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5934   [(set (match_operand:GPF 0 "register_operand" "=w")
5935         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5936   "TARGET_FLOAT"
5937   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5938   [(set_attr "type" "f_cvti2f")]
5939 )
5940
5941 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5942 ;; midend will arrange for an SImode conversion to HFmode to first go
5943 ;; through DFmode, then to HFmode.  But first it will try converting
5944 ;; to DImode then down, which would match our DImode pattern below and
5945 ;; give very poor code-generation.  So, we must provide our own emulation
5946 ;; of the mid-end logic.
5947
5948 (define_insn "aarch64_fp16_<optab><mode>hf2"
5949   [(set (match_operand:HF 0 "register_operand" "=w")
5950         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5951   "TARGET_FP_F16INST"
5952   "<su_optab>cvtf\t%h0, %<w>1"
5953   [(set_attr "type" "f_cvti2f")]
5954 )
5955
5956 (define_expand "<optab>sihf2"
5957   [(set (match_operand:HF 0 "register_operand")
5958         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5959   "TARGET_FLOAT"
5960 {
5961   if (TARGET_FP_F16INST)
5962     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5963   else
5964     {
5965       rtx convert_target = gen_reg_rtx (DFmode);
5966       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5967       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5968     }
5969   DONE;
5970 }
5971 )
5972
5973 ;; For DImode there is no wide enough floating-point mode that we
5974 ;; can convert through natively (TFmode would work, but requires a library
5975 ;; call).  However, we know that any value >= 65504 will be rounded
5976 ;; to infinity on conversion.  This is well within the range of SImode, so
5977 ;; we can:
5978 ;;   Saturate to SImode.
5979 ;;   Convert from that to DFmode
5980 ;;   Convert from that to HFmode (phew!).
5981 ;; Note that the saturation to SImode requires the SIMD extensions.  If
5982 ;; we ever need to provide this pattern where the SIMD extensions are not
5983 ;; available, we would need a different approach.
5984
5985 (define_expand "<optab>dihf2"
5986   [(set (match_operand:HF 0 "register_operand")
5987         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5988   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5989 {
5990   if (TARGET_FP_F16INST)
5991     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5992   else
5993     {
5994       rtx sat_target = gen_reg_rtx (SImode);
5995       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5996       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5997     }
5998
5999   DONE;
6000 }
6001 )
6002
6003 ;; Convert between fixed-point and floating-point (scalar modes)
6004
6005 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6006   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6007         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6008                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6009          FCVT_F2FIXED))]
6010   ""
6011   "@
6012    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6013    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6014   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6015    (set_attr "arch" "fp,simd")]
6016 )
6017
6018 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6019   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6020         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6021                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6022          FCVT_FIXED2F))]
6023   ""
6024   "@
6025    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6026    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6027   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6028    (set_attr "arch" "fp,simd")]
6029 )
6030
6031 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6032   [(set (match_operand:GPI 0 "register_operand" "=r")
6033         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6034                      (match_operand:SI 2 "immediate_operand" "i")]
6035          FCVT_F2FIXED))]
6036   "TARGET_FP_F16INST"
6037    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6038   [(set_attr "type" "f_cvtf2i")]
6039 )
6040
6041 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6042   [(set (match_operand:HF 0 "register_operand" "=w")
6043         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6044                     (match_operand:SI 2 "immediate_operand" "i")]
6045          FCVT_FIXED2F))]
6046   "TARGET_FP_F16INST"
6047   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6048   [(set_attr "type" "f_cvti2f")]
6049 )
6050
6051 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6052   [(set (match_operand:HI 0 "register_operand" "=w")
6053         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6054                     (match_operand:SI 2 "immediate_operand" "i")]
6055          FCVT_F2FIXED))]
6056   "TARGET_SIMD"
6057   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6058   [(set_attr "type" "neon_fp_to_int_s")]
6059 )
6060
6061 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6062   [(set (match_operand:HF 0 "register_operand" "=w")
6063         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6064                     (match_operand:SI 2 "immediate_operand" "i")]
6065          FCVT_FIXED2F))]
6066   "TARGET_SIMD"
6067   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6068   [(set_attr "type" "neon_int_to_fp_s")]
6069 )
6070
6071 ;; -------------------------------------------------------------------
6072 ;; Floating-point arithmetic
6073 ;; -------------------------------------------------------------------
6074
6075 (define_insn "add<mode>3"
6076   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6077         (plus:GPF_F16
6078          (match_operand:GPF_F16 1 "register_operand" "w")
6079          (match_operand:GPF_F16 2 "register_operand" "w")))]
6080   "TARGET_FLOAT"
6081   "fadd\\t%<s>0, %<s>1, %<s>2"
6082   [(set_attr "type" "fadd<stype>")]
6083 )
6084
6085 (define_insn "sub<mode>3"
6086   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6087         (minus:GPF_F16
6088          (match_operand:GPF_F16 1 "register_operand" "w")
6089          (match_operand:GPF_F16 2 "register_operand" "w")))]
6090   "TARGET_FLOAT"
6091   "fsub\\t%<s>0, %<s>1, %<s>2"
6092   [(set_attr "type" "fadd<stype>")]
6093 )
6094
6095 (define_insn "mul<mode>3"
6096   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6097         (mult:GPF_F16
6098          (match_operand:GPF_F16 1 "register_operand" "w")
6099          (match_operand:GPF_F16 2 "register_operand" "w")))]
6100   "TARGET_FLOAT"
6101   "fmul\\t%<s>0, %<s>1, %<s>2"
6102   [(set_attr "type" "fmul<stype>")]
6103 )
6104
6105 (define_insn "*fnmul<mode>3"
6106   [(set (match_operand:GPF 0 "register_operand" "=w")
6107         (mult:GPF
6108                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6109                  (match_operand:GPF 2 "register_operand" "w")))]
6110   "TARGET_FLOAT && !flag_rounding_math"
6111   "fnmul\\t%<s>0, %<s>1, %<s>2"
6112   [(set_attr "type" "fmul<s>")]
6113 )
6114
6115 (define_insn "*fnmul<mode>3"
6116   [(set (match_operand:GPF 0 "register_operand" "=w")
6117         (neg:GPF (mult:GPF
6118                  (match_operand:GPF 1 "register_operand" "w")
6119                  (match_operand:GPF 2 "register_operand" "w"))))]
6120   "TARGET_FLOAT"
6121   "fnmul\\t%<s>0, %<s>1, %<s>2"
6122   [(set_attr "type" "fmul<s>")]
6123 )
6124
6125 (define_expand "div<mode>3"
6126  [(set (match_operand:GPF_F16 0 "register_operand")
6127        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6128                     (match_operand:GPF_F16 2 "register_operand")))]
6129  "TARGET_FLOAT"
6130 {
6131   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6132     DONE;
6133
6134   operands[1] = force_reg (<MODE>mode, operands[1]);
6135 })
6136
6137 (define_insn "*div<mode>3"
6138   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6139         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6140                      (match_operand:GPF_F16 2 "register_operand" "w")))]
6141   "TARGET_FLOAT"
6142   "fdiv\\t%<s>0, %<s>1, %<s>2"
6143   [(set_attr "type" "fdiv<stype>")]
6144 )
6145
6146 (define_insn "neg<mode>2"
6147   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6148         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6149   "TARGET_FLOAT"
6150   "fneg\\t%<s>0, %<s>1"
6151   [(set_attr "type" "ffarith<stype>")]
6152 )
6153
6154 (define_expand "sqrt<mode>2"
6155   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6156         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6157   "TARGET_FLOAT"
6158 {
6159   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6160     DONE;
6161 })
6162
6163 (define_insn "*sqrt<mode>2"
6164   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6165         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6166   "TARGET_FLOAT"
6167   "fsqrt\\t%<s>0, %<s>1"
6168   [(set_attr "type" "fsqrt<stype>")]
6169 )
6170
6171 (define_insn "abs<mode>2"
6172   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6173         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6174   "TARGET_FLOAT"
6175   "fabs\\t%<s>0, %<s>1"
6176   [(set_attr "type" "ffarith<stype>")]
6177 )
6178
6179 ;; Given that smax/smin do not specify the result when either input is NaN,
6180 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6181 ;; for smin.
6182
6183 (define_insn "smax<mode>3"
6184   [(set (match_operand:GPF 0 "register_operand" "=w")
6185         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6186                   (match_operand:GPF 2 "register_operand" "w")))]
6187   "TARGET_FLOAT"
6188   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6189   [(set_attr "type" "f_minmax<s>")]
6190 )
6191
6192 (define_insn "smin<mode>3"
6193   [(set (match_operand:GPF 0 "register_operand" "=w")
6194         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6195                   (match_operand:GPF 2 "register_operand" "w")))]
6196   "TARGET_FLOAT"
6197   "fminnm\\t%<s>0, %<s>1, %<s>2"
6198   [(set_attr "type" "f_minmax<s>")]
6199 )
6200
6201 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6202 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6203 ;; which implement the IEEE fmax ()/fmin () functions.
6204 (define_insn "<maxmin_uns><mode>3"
6205   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6206         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6207                      (match_operand:GPF_F16 2 "register_operand" "w")]
6208                      FMAXMIN_UNS))]
6209   "TARGET_FLOAT"
6210   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6211   [(set_attr "type" "f_minmax<stype>")]
6212 )
6213
6214 (define_expand "lrint<GPF:mode><GPI:mode>2"
6215   [(match_operand:GPI 0 "register_operand")
6216    (match_operand:GPF 1 "register_operand")]
6217   "TARGET_FLOAT
6218    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
6219    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6220 {
6221   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6222   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6223   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6224   DONE;
6225 }
6226 )
6227
6228 ;; For copysign (x, y), we want to generate:
6229 ;;
6230 ;;   LDR d2, #(1 << 63)
6231 ;;   BSL v2.8b, [y], [x]
6232 ;;
6233 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.  Because
6234 ;; we expect these operations to nearly always operate on
6235 ;; floating-point values, we do not want the operation to be
6236 ;; simplified into a bit-field insert operation that operates on the
6237 ;; integer side, since typically that would involve three inter-bank
6238 ;; register copies.  As we do not expect copysign to be followed by
6239 ;; other logical operations on the result, it seems preferable to keep
6240 ;; this as an unspec operation, rather than exposing the underlying
6241 ;; logic to the compiler.
6242
6243 (define_expand "copysign<GPF:mode>3"
6244   [(match_operand:GPF 0 "register_operand")
6245    (match_operand:GPF 1 "register_operand")
6246    (match_operand:GPF 2 "register_operand")]
6247   "TARGET_FLOAT && TARGET_SIMD"
6248 {
6249   rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6250   emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6251                                     << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6252   emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6253                                        bitmask));
6254   DONE;
6255 }
6256 )
6257
6258 (define_insn "copysign<GPF:mode>3_insn"
6259   [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6260         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6261                      (match_operand:GPF 2 "register_operand" "w,w,0,0")
6262                      (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6263          UNSPEC_COPYSIGN))]
6264   "TARGET_FLOAT && TARGET_SIMD"
6265   "@
6266    bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6267    bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6268    bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6269    bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6270   [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6271 )
6272
6273
6274 ;; For xorsign (x, y), we want to generate:
6275 ;;
6276 ;; LDR   d2, #1<<63
6277 ;; AND   v3.8B, v1.8B, v2.8B
6278 ;; EOR   v0.8B, v0.8B, v3.8B
6279 ;;
6280
6281 (define_expand "xorsign<mode>3"
6282   [(match_operand:GPF 0 "register_operand")
6283    (match_operand:GPF 1 "register_operand")
6284    (match_operand:GPF 2 "register_operand")]
6285   "TARGET_FLOAT && TARGET_SIMD"
6286 {
6287
6288   machine_mode imode = <V_INT_EQUIV>mode;
6289   rtx mask = gen_reg_rtx (imode);
6290   rtx op1x = gen_reg_rtx (imode);
6291   rtx op2x = gen_reg_rtx (imode);
6292
6293   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6294   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6295                                                      imode)));
6296
6297   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6298                                     lowpart_subreg (imode, operands[2],
6299                                                     <MODE>mode)));
6300   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6301                                     lowpart_subreg (imode, operands[1],
6302                                                     <MODE>mode),
6303                                     op2x));
6304   emit_move_insn (operands[0],
6305                   lowpart_subreg (<MODE>mode, op1x, imode));
6306   DONE;
6307 }
6308 )
6309
6310 ;; -------------------------------------------------------------------
6311 ;; Reload support
6312 ;; -------------------------------------------------------------------
6313 ;; Reload Scalar Floating point modes from constant pool.
6314 ;; The AArch64 port doesn't have __int128 constant move support.
6315 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6316  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6317        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6318   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6319  "TARGET_FLOAT"
6320  {
6321    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6322    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6323    DONE;
6324  }
6325 )
6326
6327 ;; Reload Vector modes from constant pool.
6328 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6329  [(set (match_operand:VALL 0 "register_operand" "=w")
6330        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6331   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6332  "TARGET_FLOAT"
6333  {
6334    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6335    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6336    DONE;
6337  }
6338 )
6339
6340 (define_expand "@aarch64_reload_mov<mode>"
6341   [(set (match_operand:TX 0 "register_operand" "=w")
6342         (match_operand:TX 1 "register_operand" "w"))
6343    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6344   ]
6345   "TARGET_FLOAT"
6346   {
6347     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6348     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6349     gen_aarch64_movtilow_tilow (op0, op1);
6350     gen_aarch64_movdi_tihigh (operands[2], op1);
6351     gen_aarch64_movtihigh_di (op0, operands[2]);
6352     DONE;
6353   }
6354 )
6355
6356 ;; The following secondary reload helpers patterns are invoked
6357 ;; after or during reload as we don't want these patterns to start
6358 ;; kicking in during the combiner.
6359
6360 (define_insn "@aarch64_movdi_<mode>low"
6361   [(set (match_operand:DI 0 "register_operand" "=r")
6362         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6363                          (const_int 64) (const_int 0)))]
6364   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6365   "fmov\\t%x0, %d1"
6366   [(set_attr "type" "f_mrc")
6367    (set_attr "length" "4")
6368   ])
6369
6370 (define_insn "@aarch64_movdi_<mode>high"
6371   [(set (match_operand:DI 0 "register_operand" "=r")
6372         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6373                          (const_int 64) (const_int 64)))]
6374   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6375   "fmov\\t%x0, %1.d[1]"
6376   [(set_attr "type" "f_mrc")
6377    (set_attr "length" "4")
6378   ])
6379
6380 (define_insn "@aarch64_mov<mode>high_di"
6381   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6382                          (const_int 64) (const_int 64))
6383         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6384   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6385   "fmov\\t%0.d[1], %x1"
6386   [(set_attr "type" "f_mcr")
6387    (set_attr "length" "4")
6388   ])
6389
6390 (define_insn "@aarch64_mov<mode>low_di"
6391   [(set (match_operand:TX 0 "register_operand" "=w")
6392         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6393   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6394   "fmov\\t%d0, %x1"
6395   [(set_attr "type" "f_mcr")
6396    (set_attr "length" "4")
6397   ])
6398
6399 (define_insn "aarch64_movtilow_tilow"
6400   [(set (match_operand:TI 0 "register_operand" "=w")
6401         (zero_extend:TI
6402           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6403   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6404   "fmov\\t%d0, %d1"
6405   [(set_attr "type" "fmov")
6406    (set_attr "length" "4")
6407   ])
6408
6409 ;; There is a deliberate reason why the parameters of high and lo_sum's
6410 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6411 ;; and lo_sum's to be used with the labels defining the jump tables in
6412 ;; rodata section.
6413
6414 (define_expand "add_losym"
6415   [(set (match_operand 0 "register_operand" "=r")
6416         (lo_sum (match_operand 1 "register_operand" "r")
6417                 (match_operand 2 "aarch64_valid_symref" "S")))]
6418   ""
6419 {
6420   machine_mode mode = GET_MODE (operands[0]);
6421
6422   emit_insn ((mode == DImode
6423               ? gen_add_losym_di
6424               : gen_add_losym_si) (operands[0],
6425                                    operands[1],
6426                                    operands[2]));
6427   DONE;
6428 })
6429
6430 (define_insn "add_losym_<mode>"
6431   [(set (match_operand:P 0 "register_operand" "=r")
6432         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6433                   (match_operand 2 "aarch64_valid_symref" "S")))]
6434   ""
6435   "add\\t%<w>0, %<w>1, :lo12:%c2"
6436   [(set_attr "type" "alu_imm")]
6437 )
6438
6439 (define_insn "ldr_got_small_<mode>"
6440   [(set (match_operand:PTR 0 "register_operand" "=r")
6441         (unspec:PTR [(mem:PTR (lo_sum:PTR
6442                               (match_operand:PTR 1 "register_operand" "r")
6443                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6444                     UNSPEC_GOTSMALLPIC))]
6445   ""
6446   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6447   [(set_attr "type" "load_<ldst_sz>")]
6448 )
6449
6450 (define_insn "ldr_got_small_sidi"
6451   [(set (match_operand:DI 0 "register_operand" "=r")
6452         (zero_extend:DI
6453          (unspec:SI [(mem:SI (lo_sum:DI
6454                              (match_operand:DI 1 "register_operand" "r")
6455                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6456                     UNSPEC_GOTSMALLPIC)))]
6457   "TARGET_ILP32"
6458   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6459   [(set_attr "type" "load_4")]
6460 )
6461
6462 (define_insn "ldr_got_small_28k_<mode>"
6463   [(set (match_operand:PTR 0 "register_operand" "=r")
6464         (unspec:PTR [(mem:PTR (lo_sum:PTR
6465                               (match_operand:PTR 1 "register_operand" "r")
6466                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6467                     UNSPEC_GOTSMALLPIC28K))]
6468   ""
6469   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6470   [(set_attr "type" "load_<ldst_sz>")]
6471 )
6472
6473 (define_insn "ldr_got_small_28k_sidi"
6474   [(set (match_operand:DI 0 "register_operand" "=r")
6475         (zero_extend:DI
6476          (unspec:SI [(mem:SI (lo_sum:DI
6477                              (match_operand:DI 1 "register_operand" "r")
6478                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6479                     UNSPEC_GOTSMALLPIC28K)))]
6480   "TARGET_ILP32"
6481   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6482   [(set_attr "type" "load_4")]
6483 )
6484
6485 (define_insn "ldr_got_tiny"
6486   [(set (match_operand:DI 0 "register_operand" "=r")
6487         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6488                    UNSPEC_GOTTINYPIC))]
6489   ""
6490   "ldr\\t%0, %L1"
6491   [(set_attr "type" "load_8")]
6492 )
6493
6494 (define_insn "aarch64_load_tp_hard"
6495   [(set (match_operand:DI 0 "register_operand" "=r")
6496         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6497   ""
6498   "mrs\\t%0, tpidr_el0"
6499   [(set_attr "type" "mrs")]
6500 )
6501
6502 ;; The TLS ABI specifically requires that the compiler does not schedule
6503 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6504 ;; Therefore we treat the stubs as an atomic sequence.
6505 (define_expand "tlsgd_small_<mode>"
6506  [(parallel [(set (match_operand 0 "register_operand" "")
6507                   (call (mem:DI (match_dup 2)) (const_int 1)))
6508              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
6509              (clobber (reg:DI LR_REGNUM))])]
6510  ""
6511 {
6512   operands[2] = aarch64_tls_get_addr ();
6513 })
6514
6515 (define_insn "*tlsgd_small_<mode>"
6516   [(set (match_operand 0 "register_operand" "")
6517         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6518    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6519    (clobber (reg:DI LR_REGNUM))
6520   ]
6521   ""
6522   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6523   [(set_attr "type" "call")
6524    (set_attr "length" "16")])
6525
6526 (define_insn "tlsie_small_<mode>"
6527   [(set (match_operand:PTR 0 "register_operand" "=r")
6528         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6529                    UNSPEC_GOTSMALLTLS))]
6530   ""
6531   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6532   [(set_attr "type" "load_4")
6533    (set_attr "length" "8")]
6534 )
6535
6536 (define_insn "tlsie_small_sidi"
6537   [(set (match_operand:DI 0 "register_operand" "=r")
6538         (zero_extend:DI
6539           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6540                       UNSPEC_GOTSMALLTLS)))]
6541   ""
6542   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6543   [(set_attr "type" "load_4")
6544    (set_attr "length" "8")]
6545 )
6546
6547 (define_insn "tlsie_tiny_<mode>"
6548   [(set (match_operand:PTR 0 "register_operand" "=&r")
6549         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6550                      (match_operand:PTR 2 "register_operand" "r")]
6551                    UNSPEC_GOTTINYTLS))]
6552   ""
6553   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6554   [(set_attr "type" "multiple")
6555    (set_attr "length" "8")]
6556 )
6557
6558 (define_insn "tlsie_tiny_sidi"
6559   [(set (match_operand:DI 0 "register_operand" "=&r")
6560         (zero_extend:DI
6561           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6562                       (match_operand:DI 2 "register_operand" "r")
6563                       ]
6564                       UNSPEC_GOTTINYTLS)))]
6565   ""
6566   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6567   [(set_attr "type" "multiple")
6568    (set_attr "length" "8")]
6569 )
6570
6571 (define_insn "tlsle12_<mode>"
6572   [(set (match_operand:P 0 "register_operand" "=r")
6573         (unspec:P [(match_operand:P 1 "register_operand" "r")
6574                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6575                    UNSPEC_TLSLE12))]
6576   ""
6577   "add\\t%<w>0, %<w>1, #%L2";
6578   [(set_attr "type" "alu_sreg")
6579    (set_attr "length" "4")]
6580 )
6581
6582 (define_insn "tlsle24_<mode>"
6583   [(set (match_operand:P 0 "register_operand" "=r")
6584         (unspec:P [(match_operand:P 1 "register_operand" "r")
6585                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6586                    UNSPEC_TLSLE24))]
6587   ""
6588   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6589   [(set_attr "type" "multiple")
6590    (set_attr "length" "8")]
6591 )
6592
6593 (define_insn "tlsle32_<mode>"
6594   [(set (match_operand:P 0 "register_operand" "=r")
6595         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6596                    UNSPEC_TLSLE32))]
6597   ""
6598   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6599   [(set_attr "type" "multiple")
6600    (set_attr "length" "8")]
6601 )
6602
6603 (define_insn "tlsle48_<mode>"
6604   [(set (match_operand:P 0 "register_operand" "=r")
6605         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6606                    UNSPEC_TLSLE48))]
6607   ""
6608   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6609   [(set_attr "type" "multiple")
6610    (set_attr "length" "12")]
6611 )
6612
6613 (define_expand "tlsdesc_small_<mode>"
6614   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6615   "TARGET_TLS_DESC"
6616   {
6617     if (TARGET_SVE)
6618       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6619     else
6620       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6621     DONE;
6622   }
6623 )
6624
6625 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6626 ;; R0 and LR.
6627 (define_insn "tlsdesc_small_advsimd_<mode>"
6628   [(set (reg:PTR R0_REGNUM)
6629         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6630                     UNSPEC_TLSDESC))
6631    (clobber (reg:DI LR_REGNUM))
6632    (clobber (reg:CC CC_REGNUM))
6633    (clobber (match_scratch:DI 1 "=r"))]
6634   "TARGET_TLS_DESC && !TARGET_SVE"
6635   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6636   [(set_attr "type" "call")
6637    (set_attr "length" "16")])
6638
6639 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6640 ;; all vector registers, and clobber all predicate registers, on
6641 ;; top of the usual R0 and LR.
6642 (define_insn "tlsdesc_small_sve_<mode>"
6643   [(set (reg:PTR R0_REGNUM)
6644         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6645                     UNSPEC_TLSDESC))
6646    (clobber (reg:DI LR_REGNUM))
6647    (clobber (reg:CC CC_REGNUM))
6648    (clobber_high (reg:TI V0_REGNUM))
6649    (clobber_high (reg:TI V1_REGNUM))
6650    (clobber_high (reg:TI V2_REGNUM))
6651    (clobber_high (reg:TI V3_REGNUM))
6652    (clobber_high (reg:TI V4_REGNUM))
6653    (clobber_high (reg:TI V5_REGNUM))
6654    (clobber_high (reg:TI V6_REGNUM))
6655    (clobber_high (reg:TI V7_REGNUM))
6656    (clobber_high (reg:TI V8_REGNUM))
6657    (clobber_high (reg:TI V9_REGNUM))
6658    (clobber_high (reg:TI V10_REGNUM))
6659    (clobber_high (reg:TI V11_REGNUM))
6660    (clobber_high (reg:TI V12_REGNUM))
6661    (clobber_high (reg:TI V13_REGNUM))
6662    (clobber_high (reg:TI V14_REGNUM))
6663    (clobber_high (reg:TI V15_REGNUM))
6664    (clobber_high (reg:TI V16_REGNUM))
6665    (clobber_high (reg:TI V17_REGNUM))
6666    (clobber_high (reg:TI V18_REGNUM))
6667    (clobber_high (reg:TI V19_REGNUM))
6668    (clobber_high (reg:TI V20_REGNUM))
6669    (clobber_high (reg:TI V21_REGNUM))
6670    (clobber_high (reg:TI V22_REGNUM))
6671    (clobber_high (reg:TI V23_REGNUM))
6672    (clobber_high (reg:TI V24_REGNUM))
6673    (clobber_high (reg:TI V25_REGNUM))
6674    (clobber_high (reg:TI V26_REGNUM))
6675    (clobber_high (reg:TI V27_REGNUM))
6676    (clobber_high (reg:TI V28_REGNUM))
6677    (clobber_high (reg:TI V29_REGNUM))
6678    (clobber_high (reg:TI V30_REGNUM))
6679    (clobber_high (reg:TI V31_REGNUM))
6680    (clobber (reg:VNx2BI P0_REGNUM))
6681    (clobber (reg:VNx2BI P1_REGNUM))
6682    (clobber (reg:VNx2BI P2_REGNUM))
6683    (clobber (reg:VNx2BI P3_REGNUM))
6684    (clobber (reg:VNx2BI P4_REGNUM))
6685    (clobber (reg:VNx2BI P5_REGNUM))
6686    (clobber (reg:VNx2BI P6_REGNUM))
6687    (clobber (reg:VNx2BI P7_REGNUM))
6688    (clobber (reg:VNx2BI P8_REGNUM))
6689    (clobber (reg:VNx2BI P9_REGNUM))
6690    (clobber (reg:VNx2BI P10_REGNUM))
6691    (clobber (reg:VNx2BI P11_REGNUM))
6692    (clobber (reg:VNx2BI P12_REGNUM))
6693    (clobber (reg:VNx2BI P13_REGNUM))
6694    (clobber (reg:VNx2BI P14_REGNUM))
6695    (clobber (reg:VNx2BI P15_REGNUM))
6696    (clobber (match_scratch:DI 1 "=r"))]
6697   "TARGET_TLS_DESC && TARGET_SVE"
6698   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6699   [(set_attr "type" "call")
6700    (set_attr "length" "16")])
6701
6702 (define_insn "stack_tie"
6703   [(set (mem:BLK (scratch))
6704         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6705                      (match_operand:DI 1 "register_operand" "rk")]
6706                     UNSPEC_PRLG_STK))]
6707   ""
6708   ""
6709   [(set_attr "length" "0")]
6710 )
6711
6712 ;; Pointer authentication patterns are always provided.  In architecture
6713 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6714 ;; This lets the user write portable software which authenticates pointers
6715 ;; when run on something which implements ARMv8.3-A, and which runs
6716 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6717 ;; implemented.
6718
6719 ;; Signing/Authenticating R30 using SP as the salt.
6720
6721 (define_insn "<pauth_mnem_prefix>sp"
6722   [(set (reg:DI R30_REGNUM)
6723         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6724   ""
6725   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
6726 )
6727
6728 ;; Signing/Authenticating X17 using X16 as the salt.
6729
6730 (define_insn "<pauth_mnem_prefix>1716"
6731   [(set (reg:DI R17_REGNUM)
6732         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6733   ""
6734   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
6735 )
6736
6737 ;; Stripping the signature in R30.
6738
6739 (define_insn "xpaclri"
6740   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6741   ""
6742   "hint\t7 // xpaclri"
6743 )
6744
6745 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6746 ;; all of memory.  This blocks insns from being moved across this point.
6747
6748 (define_insn "blockage"
6749   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6750   ""
6751   ""
6752   [(set_attr "length" "0")
6753    (set_attr "type" "block")]
6754 )
6755
6756 (define_insn "probe_stack_range"
6757   [(set (match_operand:DI 0 "register_operand" "=rk")
6758         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6759                              (match_operand:DI 2 "register_operand" "r")]
6760                               UNSPECV_PROBE_STACK_RANGE))]
6761   ""
6762 {
6763   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6764 }
6765   [(set_attr "length" "32")]
6766 )
6767
6768 ;; This instruction is used to generate the stack clash stack adjustment and
6769 ;; probing loop.  We can't change the control flow during prologue and epilogue
6770 ;; code generation.  So we must emit a volatile unspec and expand it later on.
6771
6772 (define_insn "@probe_sve_stack_clash_<mode>"
6773   [(set (match_operand:P 0 "register_operand" "=rk")
6774         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6775                             (match_operand:P 2 "register_operand" "r")
6776                             (match_operand:P 3 "const_int_operand" "n")
6777                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
6778                              UNSPECV_PROBE_STACK_RANGE))]
6779   "TARGET_SVE"
6780 {
6781   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6782                                                operands[3], operands[4]);
6783 }
6784   [(set_attr "length" "28")]
6785 )
6786
6787 ;; Named pattern for expanding thread pointer reference.
6788 (define_expand "get_thread_pointerdi"
6789   [(match_operand:DI 0 "register_operand" "=r")]
6790   ""
6791 {
6792   rtx tmp = aarch64_load_tp (operands[0]);
6793   if (tmp != operands[0])
6794     emit_move_insn (operands[0], tmp);
6795   DONE;
6796 })
6797
6798 ;; Named patterns for stack smashing protection.
6799 (define_expand "stack_protect_set"
6800   [(match_operand 0 "memory_operand")
6801    (match_operand 1 "memory_operand")]
6802   ""
6803 {
6804   machine_mode mode = GET_MODE (operands[0]);
6805
6806   emit_insn ((mode == DImode
6807               ? gen_stack_protect_set_di
6808               : gen_stack_protect_set_si) (operands[0], operands[1]));
6809   DONE;
6810 })
6811
6812 (define_insn "stack_protect_set_<mode>"
6813   [(set (match_operand:PTR 0 "memory_operand" "=m")
6814         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6815          UNSPEC_SP_SET))
6816    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6817   ""
6818   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6819   [(set_attr "length" "12")
6820    (set_attr "type" "multiple")])
6821
6822 (define_expand "stack_protect_test"
6823   [(match_operand 0 "memory_operand")
6824    (match_operand 1 "memory_operand")
6825    (match_operand 2)]
6826   ""
6827 {
6828   rtx result;
6829   machine_mode mode = GET_MODE (operands[0]);
6830
6831   result = gen_reg_rtx(mode);
6832
6833   emit_insn ((mode == DImode
6834               ? gen_stack_protect_test_di
6835               : gen_stack_protect_test_si) (result,
6836                                             operands[0],
6837                                             operands[1]));
6838
6839   if (mode == DImode)
6840     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6841                                     result, const0_rtx, operands[2]));
6842   else
6843     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6844                                     result, const0_rtx, operands[2]));
6845   DONE;
6846 })
6847
6848 (define_insn "stack_protect_test_<mode>"
6849   [(set (match_operand:PTR 0 "register_operand" "=r")
6850         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
6851                      (match_operand:PTR 2 "memory_operand" "m")]
6852          UNSPEC_SP_TEST))
6853    (clobber (match_scratch:PTR 3 "=&r"))]
6854   ""
6855   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
6856   [(set_attr "length" "12")
6857    (set_attr "type" "multiple")])
6858
6859 ;; Write Floating-point Control Register.
6860 (define_insn "set_fpcr"
6861   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
6862   ""
6863   "msr\\tfpcr, %0"
6864   [(set_attr "type" "mrs")])
6865
6866 ;; Read Floating-point Control Register.
6867 (define_insn "get_fpcr"
6868   [(set (match_operand:SI 0 "register_operand" "=r")
6869         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
6870   ""
6871   "mrs\\t%0, fpcr"
6872   [(set_attr "type" "mrs")])
6873
6874 ;; Write Floating-point Status Register.
6875 (define_insn "set_fpsr"
6876   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
6877   ""
6878   "msr\\tfpsr, %0"
6879   [(set_attr "type" "mrs")])
6880
6881 ;; Read Floating-point Status Register.
6882 (define_insn "get_fpsr"
6883   [(set (match_operand:SI 0 "register_operand" "=r")
6884         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
6885   ""
6886   "mrs\\t%0, fpsr"
6887   [(set_attr "type" "mrs")])
6888
6889
6890 ;; Define the subtract-one-and-jump insns so loop.c
6891 ;; knows what to generate.
6892 (define_expand "doloop_end"
6893   [(use (match_operand 0 "" ""))      ; loop pseudo
6894    (use (match_operand 1 "" ""))]     ; label
6895   "optimize > 0 && flag_modulo_sched"
6896 {
6897   rtx s0;
6898   rtx bcomp;
6899   rtx loc_ref;
6900   rtx cc_reg;
6901   rtx insn;
6902   rtx cmp;
6903
6904   /* Currently SMS relies on the do-loop pattern to recognize loops
6905      where (1) the control part consists of all insns defining and/or
6906      using a certain 'count' register and (2) the loop count can be
6907      adjusted by modifying this register prior to the loop.
6908      ??? The possible introduction of a new block to initialize the
6909      new IV can potentially affect branch optimizations.  */
6910
6911   if (GET_MODE (operands[0]) != DImode)
6912     FAIL;
6913
6914   s0 = operands [0];
6915   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
6916
6917   cmp = XVECEXP (PATTERN (insn), 0, 0);
6918   cc_reg = SET_DEST (cmp);
6919   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6920   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6921   emit_jump_insn (gen_rtx_SET (pc_rtx,
6922                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6923                                                      loc_ref, pc_rtx)));
6924   DONE;
6925 })
6926
6927 ;; Track speculation through conditional branches.  We assume that
6928 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
6929 (define_insn "speculation_tracker"
6930   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
6931         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
6932          UNSPEC_SPECULATION_TRACKER))]
6933   ""
6934   {
6935     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6936     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
6937     return "";
6938   }
6939   [(set_attr "type" "csel")]
6940 )
6941
6942 ;; Helper for aarch64.c code.
6943 (define_expand "set_clobber_cc"
6944   [(parallel [(set (match_operand 0)
6945                    (match_operand 1))
6946               (clobber (reg:CC CC_REGNUM))])])
6947
6948 ;; Hard speculation barrier.
6949 (define_insn "speculation_barrier"
6950   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
6951   ""
6952   "isb\;dsb\\tsy"
6953   [(set_attr "length" "8")
6954    (set_attr "type" "block")
6955    (set_attr "speculation_barrier" "true")]
6956 )
6957
6958 ;; Support for __builtin_speculation_safe_value when we have speculation
6959 ;; tracking enabled.  Use the speculation tracker to decide whether to
6960 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
6961 (define_expand "@despeculate_copy<ALLI_TI:mode>"
6962   [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
6963         (unspec_volatile:ALLI_TI
6964          [(match_operand:ALLI_TI 1 "register_operand" "r")
6965           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
6966           (use (reg:DI SPECULATION_TRACKER_REGNUM))
6967           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6968   ""
6969   "
6970   {
6971     if (operands[2] == const0_rtx)
6972       {
6973         rtx tracker;
6974         if (<MODE>mode == TImode)
6975           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6976         else
6977           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
6978
6979         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
6980                                                  tracker));
6981         DONE;
6982       }
6983   }
6984   "
6985 )
6986
6987 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
6988 ;; encoding for CSDB, but will work in older versions of the assembler.
6989 (define_insn "*despeculate_copy<ALLI:mode>_insn"
6990   [(set (match_operand:ALLI 0 "register_operand" "=r")
6991         (unspec_volatile:ALLI
6992          [(match_operand:ALLI 1 "register_operand" "r")
6993           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
6994           (use (reg:DI SPECULATION_TRACKER_REGNUM))
6995           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6996   ""
6997   {
6998     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6999     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7000                      operands);
7001     return "";
7002   }
7003   [(set_attr "length" "12")
7004    (set_attr "type" "block")
7005    (set_attr "speculation_barrier" "true")]
7006 )
7007
7008 ;; Pattern to match despeculate_copyti
7009 (define_insn "*despeculate_copyti_insn"
7010   [(set (match_operand:TI 0 "register_operand" "=r")
7011         (unspec_volatile:TI
7012          [(match_operand:TI 1 "register_operand" "r")
7013           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7014           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7015           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7016   ""
7017   {
7018     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7019     output_asm_insn
7020       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7021        operands);
7022     return "";
7023   }
7024   [(set_attr "length" "16")
7025    (set_attr "type" "block")
7026    (set_attr "speculation_barrier" "true")]
7027 )
7028
7029 (define_insn "despeculate_simple<ALLI:mode>"
7030   [(set (match_operand:ALLI 0 "register_operand" "=r")
7031         (unspec_volatile:ALLI
7032          [(match_operand:ALLI 1 "register_operand" "r")
7033           (use (match_operand:ALLI 2 "register_operand" ""))]
7034          UNSPECV_SPECULATION_BARRIER))]
7035   ""
7036   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7037   [(set_attr "type" "block")
7038    (set_attr "length" "8")
7039    (set_attr "speculation_barrier" "true")]
7040 )
7041
7042 (define_insn "despeculate_simpleti"
7043   [(set (match_operand:TI 0 "register_operand" "=r")
7044         (unspec_volatile:TI
7045          [(match_operand:TI 1 "register_operand" "r")
7046           (use (match_operand:DI 2 "register_operand" ""))]
7047          UNSPECV_SPECULATION_BARRIER))]
7048   ""
7049   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7050   [(set_attr "type" "block")
7051    (set_attr "length" "12")
7052    (set_attr "speculation_barrier" "true")]
7053 )
7054
7055 ;; AdvSIMD Stuff
7056 (include "aarch64-simd.md")
7057
7058 ;; Atomic Operations
7059 (include "atomics.md")
7060
7061 ;; ldp/stp peephole patterns
7062 (include "aarch64-ldpstp.md")
7063
7064 ;; SVE.
7065 (include "aarch64-sve.md")