gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (R17_REGNUM         17)
  42     (R18_REGNUM         18)
  43     (R19_REGNUM         19)
  44     (R20_REGNUM         20)
  45     (R21_REGNUM         21)
  46     (R22_REGNUM         22)
  47     (R23_REGNUM         23)
  48     (R24_REGNUM         24)
  49     (R25_REGNUM         25)
  50     (R26_REGNUM         26)
  51     (R27_REGNUM         27)
  52     (R28_REGNUM         28)
  53     (R29_REGNUM         29)
  54     (R30_REGNUM         30)
  55     (SP_REGNUM          31)
  56     (V0_REGNUM          32)
  57     (V1_REGNUM          33)
  58     (V2_REGNUM          34)
  59     (V3_REGNUM          35)
  60     (V4_REGNUM          36)
  61     (V5_REGNUM          37)
  62     (V6_REGNUM          38)
  63     (V7_REGNUM          39)
  64     (V8_REGNUM          40)
  65     (V9_REGNUM          41)
  66     (V10_REGNUM         42)
  67     (V11_REGNUM         43)
  68     (V12_REGNUM         44)
  69     (V13_REGNUM         45)
  70     (V14_REGNUM         46)
  71     (V15_REGNUM         47)
  72     (V16_REGNUM         48)
  73     (V17_REGNUM         49)
  74     (V18_REGNUM         50)
  75     (V19_REGNUM         51)
  76     (V20_REGNUM         52)
  77     (V21_REGNUM         53)
  78     (V22_REGNUM         54)
  79     (V23_REGNUM         55)
  80     (V24_REGNUM         56)
  81     (V25_REGNUM         57)
  82     (V26_REGNUM         58)
  83     (V27_REGNUM         59)
  84     (V28_REGNUM         60)
  85     (V29_REGNUM         61)
  86     (V30_REGNUM         62)
  87     (V31_REGNUM         63)
  88     (LAST_SAVED_REGNUM  63)
  89     (SFP_REGNUM         64)
  90     (AP_REGNUM          65)
  91     (CC_REGNUM          66)
  92     ;; Defined only to make the DWARF description simpler.
  93     (VG_REGNUM          67)
  94     (P0_REGNUM          68)
  95     (P1_REGNUM          69)
  96     (P2_REGNUM          70)
  97     (P3_REGNUM          71)
  98     (P4_REGNUM          72)
  99     (P5_REGNUM          73)
 100     (P6_REGNUM          74)
 101     (P7_REGNUM          75)
 102     (P8_REGNUM          76)
 103     (P9_REGNUM          77)
 104     (P10_REGNUM         78)
 105     (P11_REGNUM         79)
 106     (P12_REGNUM         80)
 107     (P13_REGNUM         81)
 108     (P14_REGNUM         82)
 109     (P15_REGNUM         83)
 110     ;; Scratch register used by stack clash protection to calculate
 111     ;; SVE CFA offsets during probing.
 112     (STACK_CLASH_SVE_CFA_REGNUM 11)
 113     ;; Scratch registers for prologue/epilogue use.
 114     (EP0_REGNUM         12)
 115     (EP1_REGNUM         13)
 116     ;; A couple of call-clobbered registers that we need to reserve when
 117     ;; tracking speculation this is not ABI, so is subject to change.
 118     (SPECULATION_SCRATCH_REGNUM 14)
 119     (SPECULATION_TRACKER_REGNUM 15)
 120     ;; Scratch registers used in frame layout.
 121     (IP0_REGNUM         16)
 122     (IP1_REGNUM         17)
 123     (FP_REGNUM          29)
 124     (LR_REGNUM          30)
 125   ]
 126 )
 127
 128 (define_c_enum "unspec" [
 129     UNSPEC_AUTIA1716
 130     UNSPEC_AUTIB1716
 131     UNSPEC_AUTIASP
 132     UNSPEC_AUTIBSP
 133     UNSPEC_CASESI
 134     UNSPEC_CRC32B
 135     UNSPEC_CRC32CB
 136     UNSPEC_CRC32CH
 137     UNSPEC_CRC32CW
 138     UNSPEC_CRC32CX
 139     UNSPEC_CRC32H
 140     UNSPEC_CRC32W
 141     UNSPEC_CRC32X
 142     UNSPEC_FCVTZS
 143     UNSPEC_FCVTZU
 144     UNSPEC_FJCVTZS
 145     UNSPEC_FRINT32Z
 146     UNSPEC_FRINT32X
 147     UNSPEC_FRINT64Z
 148     UNSPEC_FRINT64X
 149     UNSPEC_URECPE
 150     UNSPEC_FRECPE
 151     UNSPEC_FRECPS
 152     UNSPEC_FRECPX
 153     UNSPEC_FRINTA
 154     UNSPEC_FRINTI
 155     UNSPEC_FRINTM
 156     UNSPEC_FRINTN
 157     UNSPEC_FRINTP
 158     UNSPEC_FRINTX
 159     UNSPEC_FRINTZ
 160     UNSPEC_GOTSMALLPIC
 161     UNSPEC_GOTSMALLPIC28K
 162     UNSPEC_GOTSMALLTLS
 163     UNSPEC_GOTTINYPIC
 164     UNSPEC_GOTTINYTLS
 165     UNSPEC_LD1
 166     UNSPEC_LD2
 167     UNSPEC_LD2_DREG
 168     UNSPEC_LD2_DUP
 169     UNSPEC_LD3
 170     UNSPEC_LD3_DREG
 171     UNSPEC_LD3_DUP
 172     UNSPEC_LD4
 173     UNSPEC_LD4_DREG
 174     UNSPEC_LD4_DUP
 175     UNSPEC_LD2_LANE
 176     UNSPEC_LD3_LANE
 177     UNSPEC_LD4_LANE
 178     UNSPEC_MB
 179     UNSPEC_NOP
 180     UNSPEC_PACIA1716
 181     UNSPEC_PACIB1716
 182     UNSPEC_PACIASP
 183     UNSPEC_PACIBSP
 184     UNSPEC_PRLG_STK
 185     UNSPEC_REV
 186     UNSPEC_RBIT
 187     UNSPEC_SABAL
 188     UNSPEC_SABDL2
 189     UNSPEC_SADALP
 190     UNSPEC_SCVTF
 191     UNSPEC_SISD_NEG
 192     UNSPEC_SISD_SSHL
 193     UNSPEC_SISD_USHL
 194     UNSPEC_SSHL_2S
 195     UNSPEC_ST1
 196     UNSPEC_ST2
 197     UNSPEC_ST3
 198     UNSPEC_ST4
 199     UNSPEC_ST2_LANE
 200     UNSPEC_ST3_LANE
 201     UNSPEC_ST4_LANE
 202     UNSPEC_TLS
 203     UNSPEC_TLSDESC
 204     UNSPEC_TLSLE12
 205     UNSPEC_TLSLE24
 206     UNSPEC_TLSLE32
 207     UNSPEC_TLSLE48
 208     UNSPEC_UABAL
 209     UNSPEC_UABDL2
 210     UNSPEC_UADALP
 211     UNSPEC_UCVTF
 212     UNSPEC_USHL_2S
 213     UNSPEC_VSTRUCTDUMMY
 214     UNSPEC_SSP_SYSREG
 215     UNSPEC_SP_SET
 216     UNSPEC_SP_TEST
 217     UNSPEC_RSQRT
 218     UNSPEC_RSQRTE
 219     UNSPEC_RSQRTS
 220     UNSPEC_NZCV
 221     UNSPEC_XPACLRI
 222     UNSPEC_LD1_SVE
 223     UNSPEC_ST1_SVE
 224     UNSPEC_LD1RQ
 225     UNSPEC_LD1_GATHER
 226     UNSPEC_ST1_SCATTER
 227     UNSPEC_PRED_X
 228     UNSPEC_PRED_Z
 229     UNSPEC_PTEST
 230     UNSPEC_UNPACKSHI
 231     UNSPEC_UNPACKUHI
 232     UNSPEC_UNPACKSLO
 233     UNSPEC_UNPACKULO
 234     UNSPEC_PACK
 235     UNSPEC_WHILE_LO
 236     UNSPEC_LDN
 237     UNSPEC_STN
 238     UNSPEC_INSR
 239     UNSPEC_CLASTB
 240     UNSPEC_FADDA
 241     UNSPEC_REV_SUBREG
 242     UNSPEC_REINTERPRET
 243     UNSPEC_SPECULATION_TRACKER
 244     UNSPEC_COPYSIGN
 245     UNSPEC_TTEST                ; Represent transaction test.
 246 ])
 247
 248 (define_c_enum "unspecv" [
 249     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 250     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 251     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 252     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 253     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 254     UNSPECV_BLOCKAGE            ; Represent a blockage
 255     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 256     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 257     UNSPECV_BTI_NOARG           ; Represent BTI.
 258     UNSPECV_BTI_C               ; Represent BTI c.
 259     UNSPECV_BTI_J               ; Represent BTI j.
 260     UNSPECV_BTI_JC              ; Represent BTI jc.
 261     UNSPECV_TSTART              ; Represent transaction start.
 262     UNSPECV_TCOMMIT             ; Represent transaction commit.
 263     UNSPECV_TCANCEL             ; Represent transaction cancel.
 264   ]
 265 )
 266
 267 ;; These constants are used as a const_int in various SVE unspecs
 268 ;; to indicate whether the governing predicate is known to be a PTRUE.
 269 (define_constants
 270   [; Indicates that the predicate might not be a PTRUE.
 271    (SVE_MAYBE_NOT_PTRUE 0)
 272
 273    ; Indicates that the predicate is known to be a PTRUE.
 274    (SVE_KNOWN_PTRUE 1)])
 275
 276 ;; These constants are used as a const_int in predicated SVE FP arithmetic
 277 ;; to indicate whether the operation is allowed to make additional lanes
 278 ;; active without worrying about the effect on faulting behavior.
 279 (define_constants
 280   [; Indicates either that all lanes are active or that the instruction may
 281    ; operate on inactive inputs even if doing so could induce a fault.
 282    (SVE_RELAXED_GP 0)
 283
 284    ; Indicates that some lanes might be inactive and that the instruction
 285    ; must not operate on inactive inputs if doing so could induce a fault.
 286    (SVE_STRICT_GP 1)])
 287
 288 ;; If further include files are added the defintion of MD_INCLUDES
 289 ;; must be updated.
 290
 291 (include "constraints.md")
 292 (include "predicates.md")
 293 (include "iterators.md")
 294
 295 ;; -------------------------------------------------------------------
 296 ;; Instruction types and attributes
 297 ;; -------------------------------------------------------------------
 298
 299 ; The "type" attribute is included here from AArch32 backend to be able
 300 ; to share pipeline descriptions.
 301 (include "../arm/types.md")
 302
 303 ;; It is important to set the fp or simd attributes to yes when a pattern
 304 ;; alternative uses the FP or SIMD register files, usually signified by use of
 305 ;; the 'w' constraint.  This will ensure that the alternative will be
 306 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 307 ;; architecture extensions.  If all the alternatives in a pattern use the
 308 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 309 ;; or TARGET_SIMD.
 310
 311 ;; Attributes of the architecture required to support the instruction (or
 312 ;; alternative). This attribute is used to compute attribute "enabled", use type
 313 ;; "any" to enable an alternative in all cases.
 314
 315 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 316
 317 (define_enum_attr "arch" "arches" (const_string "any"))
 318
 319 ;; [For compatibility with Arm in pipeline models]
 320 ;; Attribute that specifies whether or not the instruction touches fp
 321 ;; registers.
 322 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 323 ;; backends except in the scheduling description for xgene1.  In that
 324 ;; scheduling description this attribute is used to subclass the load_4 and
 325 ;; load_8 types.
 326 (define_attr "fp" "no,yes"
 327   (if_then_else
 328     (eq_attr "arch" "fp")
 329     (const_string "yes")
 330     (const_string "no")))
 331
 332 (define_attr "arch_enabled" "no,yes"
 333   (if_then_else
 334     (ior
 335         (eq_attr "arch" "any")
 336
 337         (and (eq_attr "arch" "rcpc8_4")
 338              (match_test "AARCH64_ISA_RCPC8_4"))
 339
 340         (and (eq_attr "arch" "fp")
 341              (match_test "TARGET_FLOAT"))
 342
 343         (and (eq_attr "arch" "simd")
 344              (match_test "TARGET_SIMD"))
 345
 346         (and (eq_attr "arch" "fp16")
 347              (match_test "TARGET_FP_F16INST"))
 348
 349         (and (eq_attr "arch" "sve")
 350              (match_test "TARGET_SVE")))
 351     (const_string "yes")
 352     (const_string "no")))
 353
 354 ;; Attribute that controls whether an alternative is enabled or not.
 355 ;; Currently it is only used to disable alternatives which touch fp or simd
 356 ;; registers when -mgeneral-regs-only is specified or to require a special
 357 ;; architecture support.
 358 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 359
 360 ;; Attribute that specifies whether we are dealing with a branch to a
 361 ;; label that is far away, i.e. further away than the maximum/minimum
 362 ;; representable in a signed 21-bits number.
 363 ;; 0 :=: no
 364 ;; 1 :=: yes
 365 (define_attr "far_branch" "" (const_int 0))
 366
 367 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 368 (define_attr "movprfx" "no,yes" (const_string "no"))
 369
 370 (define_attr "length" ""
 371   (cond [(eq_attr "movprfx" "yes")
 372            (const_int 8)
 373         ] (const_int 4)))
 374
 375 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 376 ;; no predicated insns.
 377 (define_attr "predicated" "yes,no" (const_string "no"))
 378
 379 ;; Set to true on an insn that requires the speculation tracking state to be
 380 ;; in the tracking register before the insn issues.  Otherwise the compiler
 381 ;; may chose to hold the tracking state encoded in SP.
 382 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 383
 384 ;; -------------------------------------------------------------------
 385 ;; Pipeline descriptions and scheduling
 386 ;; -------------------------------------------------------------------
 387
 388 ;; Processor types.
 389 (include "aarch64-tune.md")
 390
 391 ;; Scheduling
 392 (include "../arm/cortex-a53.md")
 393 (include "../arm/cortex-a57.md")
 394 (include "../arm/exynos-m1.md")
 395 (include "falkor.md")
 396 (include "saphira.md")
 397 (include "thunderx.md")
 398 (include "../arm/xgene1.md")
 399 (include "thunderx2t99.md")
 400 (include "tsv110.md")
 401
 402 ;; -------------------------------------------------------------------
 403 ;; Jumps and other miscellaneous insns
 404 ;; -------------------------------------------------------------------
 405
 406 (define_insn "indirect_jump"
 407   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 408   ""
 409   "br\\t%0"
 410   [(set_attr "type" "branch")]
 411 )
 412
 413 (define_insn "jump"
 414   [(set (pc) (label_ref (match_operand 0 "" "")))]
 415   ""
 416   "b\\t%l0"
 417   [(set_attr "type" "branch")]
 418 )
 419
 420 (define_expand "cbranch<mode>4"
 421   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 422                             [(match_operand:GPI 1 "register_operand")
 423                              (match_operand:GPI 2 "aarch64_plus_operand")])
 424                            (label_ref (match_operand 3 "" ""))
 425                            (pc)))]
 426   ""
 427   "
 428   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 429                                          operands[2]);
 430   operands[2] = const0_rtx;
 431   "
 432 )
 433
 434 (define_expand "cbranch<mode>4"
 435   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 436                             [(match_operand:GPF 1 "register_operand")
 437                              (match_operand:GPF 2 "aarch64_fp_compare_operand")])
 438                            (label_ref (match_operand 3 "" ""))
 439                            (pc)))]
 440   ""
 441   "
 442   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 443                                          operands[2]);
 444   operands[2] = const0_rtx;
 445   "
 446 )
 447
 448 (define_expand "cbranchcc4"
 449   [(set (pc) (if_then_else
 450               (match_operator 0 "aarch64_comparison_operator"
 451                [(match_operand 1 "cc_register")
 452                 (match_operand 2 "const0_operand")])
 453               (label_ref (match_operand 3 "" ""))
 454               (pc)))]
 455   ""
 456   "")
 457
 458 (define_insn "ccmp<mode>"
 459   [(set (match_operand:CC 1 "cc_register" "")
 460         (if_then_else:CC
 461           (match_operator 4 "aarch64_comparison_operator"
 462            [(match_operand 0 "cc_register" "")
 463             (const_int 0)])
 464           (compare:CC
 465             (match_operand:GPI 2 "register_operand" "r,r,r")
 466             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 467           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 468   ""
 469   "@
 470    ccmp\\t%<w>2, %<w>3, %k5, %m4
 471    ccmp\\t%<w>2, %3, %k5, %m4
 472    ccmn\\t%<w>2, #%n3, %k5, %m4"
 473   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 474 )
 475
 476 (define_insn "fccmp<mode>"
 477   [(set (match_operand:CCFP 1 "cc_register" "")
 478         (if_then_else:CCFP
 479           (match_operator 4 "aarch64_comparison_operator"
 480            [(match_operand 0 "cc_register" "")
 481             (const_int 0)])
 482           (compare:CCFP
 483             (match_operand:GPF 2 "register_operand" "w")
 484             (match_operand:GPF 3 "register_operand" "w"))
 485           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 486   "TARGET_FLOAT"
 487   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 488   [(set_attr "type" "fccmp<s>")]
 489 )
 490
 491 (define_insn "fccmpe<mode>"
 492   [(set (match_operand:CCFPE 1 "cc_register" "")
 493          (if_then_else:CCFPE
 494           (match_operator 4 "aarch64_comparison_operator"
 495            [(match_operand 0 "cc_register" "")
 496           (const_int 0)])
 497            (compare:CCFPE
 498             (match_operand:GPF 2 "register_operand" "w")
 499             (match_operand:GPF 3 "register_operand" "w"))
 500           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 501   "TARGET_FLOAT"
 502   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 503   [(set_attr "type" "fccmp<s>")]
 504 )
 505
 506 ;; Expansion of signed mod by a power of 2 using CSNEG.
 507 ;; For x0 % n where n is a power of 2 produce:
 508 ;; negs   x1, x0
 509 ;; and    x0, x0, #(n - 1)
 510 ;; and    x1, x1, #(n - 1)
 511 ;; csneg  x0, x0, x1, mi
 512
 513 (define_expand "mod<mode>3"
 514   [(match_operand:GPI 0 "register_operand")
 515    (match_operand:GPI 1 "register_operand")
 516    (match_operand:GPI 2 "const_int_operand")]
 517   ""
 518   {
 519     HOST_WIDE_INT val = INTVAL (operands[2]);
 520
 521     if (val <= 0
 522        || exact_log2 (val) <= 0
 523        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 524       FAIL;
 525
 526     rtx mask = GEN_INT (val - 1);
 527
 528     /* In the special case of x0 % 2 we can do the even shorter:
 529         cmp    x0, xzr
 530         and    x0, x0, 1
 531         cneg   x0, x0, lt.  */
 532     if (val == 2)
 533       {
 534         rtx masked = gen_reg_rtx (<MODE>mode);
 535         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 536         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 537         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 538         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 539         DONE;
 540       }
 541
 542     rtx neg_op = gen_reg_rtx (<MODE>mode);
 543     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 544
 545     /* Extract the condition register and mode.  */
 546     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 547     rtx cc_reg = SET_DEST (cmp);
 548     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 549
 550     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 551     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 552
 553     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 554     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 555
 556     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 557                                        masked_neg, masked_pos));
 558     DONE;
 559   }
 560 )
 561
 562 (define_insn "condjump"
 563   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 564                             [(match_operand 1 "cc_register" "") (const_int 0)])
 565                            (label_ref (match_operand 2 "" ""))
 566                            (pc)))]
 567   ""
 568   {
 569     /* GCC's traditional style has been to use "beq" instead of "b.eq", etc.,
 570        but the "." is required for SVE conditions.  */
 571     bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode;
 572     if (get_attr_length (insn) == 8)
 573       return aarch64_gen_far_branch (operands, 2, "Lbcond",
 574                                      use_dot_p ? "b.%M0\\t" : "b%M0\\t");
 575     else
 576       return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2";
 577   }
 578   [(set_attr "type" "branch")
 579    (set (attr "length")
 580         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 581                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 582                       (const_int 4)
 583                       (const_int 8)))
 584    (set (attr "far_branch")
 585         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 586                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 587                       (const_int 0)
 588                       (const_int 1)))]
 589 )
 590
 591 ;; For a 24-bit immediate CST we can optimize the compare for equality
 592 ;; and branch sequence from:
 593 ;;      mov     x0, #imm1
 594 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 595 ;;      cmp     x1, x0
 596 ;;      b<ne,eq> .Label
 597 ;; into the shorter:
 598 ;;      sub     x0, x1, #(CST & 0xfff000)
 599 ;;      subs    x0, x0, #(CST & 0x000fff)
 600 ;;      b<ne,eq> .Label
 601 (define_insn_and_split "*compare_condjump<GPI:mode>"
 602   [(set (pc) (if_then_else (EQL
 603                               (match_operand:GPI 0 "register_operand" "r")
 604                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 605                            (label_ref:P (match_operand 2 "" ""))
 606                            (pc)))]
 607   "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode)
 608    && !aarch64_plus_operand (operands[1], <GPI:MODE>mode)
 609    && !reload_completed"
 610   "#"
 611   "&& true"
 612   [(const_int 0)]
 613   {
 614     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 615     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 616     rtx tmp = gen_reg_rtx (<GPI:MODE>mode);
 617     emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 618     emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 619     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 620     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode,
 621                                   cc_reg, const0_rtx);
 622     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 623     DONE;
 624   }
 625 )
 626
 627 (define_expand "casesi"
 628   [(match_operand:SI 0 "register_operand")      ; Index
 629    (match_operand:SI 1 "const_int_operand")     ; Lower bound
 630    (match_operand:SI 2 "const_int_operand")     ; Total range
 631    (match_operand:DI 3 "" "")                   ; Table label
 632    (match_operand:DI 4 "" "")]                  ; Out of range label
 633   ""
 634   {
 635     if (operands[1] != const0_rtx)
 636       {
 637         rtx reg = gen_reg_rtx (SImode);
 638
 639         /* Canonical RTL says that if you have:
 640
 641            (minus (X) (CONST))
 642
 643            then this should be emitted as:
 644
 645            (plus (X) (-CONST))
 646
 647            The use of trunc_int_for_mode ensures that the resulting
 648            constant can be represented in SImode, this is important
 649            for the corner case where operand[1] is INT_MIN.  */
 650
 651         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 652
 653         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 654               (operands[1], SImode))
 655           operands[1] = force_reg (SImode, operands[1]);
 656         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 657         operands[0] = reg;
 658       }
 659
 660     if (!aarch64_plus_operand (operands[2], SImode))
 661       operands[2] = force_reg (SImode, operands[2]);
 662     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 663                                                  const0_rtx),
 664                                     operands[0], operands[2], operands[4]));
 665
 666     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 667     operands[2]
 668       = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]),
 669                         UNSPEC_CASESI);
 670     operands[2] = gen_rtx_MEM (DImode, operands[2]);
 671     MEM_READONLY_P (operands[2]) = 1;
 672     MEM_NOTRAP_P (operands[2]) = 1;
 673     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3]));
 674     DONE;
 675   }
 676 )
 677
 678 (define_expand "casesi_dispatch"
 679   [(parallel
 680     [(set (pc) (match_operand:DI 0 ""))
 681      (clobber (reg:CC CC_REGNUM))
 682      (clobber (match_scratch:DI 2))
 683      (clobber (match_scratch:DI 3))
 684      (use (label_ref:DI (match_operand 1 "")))])]
 685   "")
 686
 687 (define_insn "*casesi_dispatch"
 688   [(parallel
 689     [(set (pc)
 690           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 691                            (match_operand:SI 1 "register_operand" "r")]
 692                         UNSPEC_CASESI)))
 693      (clobber (reg:CC CC_REGNUM))
 694      (clobber (match_scratch:DI 3 "=r"))
 695      (clobber (match_scratch:DI 4 "=r"))
 696      (use (label_ref:DI (match_operand 2 "" "")))])]
 697   ""
 698   "*
 699   return aarch64_output_casesi (operands);
 700   "
 701   [(set_attr "length" "16")
 702    (set_attr "type" "branch")]
 703 )
 704
 705 (define_insn "nop"
 706   [(unspec[(const_int 0)] UNSPEC_NOP)]
 707   ""
 708   "nop"
 709   [(set_attr "type" "no_insn")]
 710 )
 711
 712 (define_insn "prefetch"
 713   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 714             (match_operand:QI 1 "const_int_operand" "")
 715             (match_operand:QI 2 "const_int_operand" ""))]
 716   ""
 717   {
 718     const char * pftype[2][4] =
 719     {
 720       {"prfm\\tPLDL1STRM, %0",
 721        "prfm\\tPLDL3KEEP, %0",
 722        "prfm\\tPLDL2KEEP, %0",
 723        "prfm\\tPLDL1KEEP, %0"},
 724       {"prfm\\tPSTL1STRM, %0",
 725        "prfm\\tPSTL3KEEP, %0",
 726        "prfm\\tPSTL2KEEP, %0",
 727        "prfm\\tPSTL1KEEP, %0"},
 728     };
 729
 730     int locality = INTVAL (operands[2]);
 731
 732     gcc_assert (IN_RANGE (locality, 0, 3));
 733
 734     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 735        the address into a DImode MEM so that aarch64_print_operand knows
 736        how to print it.  */
 737     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 738     return pftype[INTVAL(operands[1])][locality];
 739   }
 740   [(set_attr "type" "load_4")]
 741 )
 742
 743 (define_insn "trap"
 744   [(trap_if (const_int 1) (const_int 8))]
 745   ""
 746   "brk #1000"
 747   [(set_attr "type" "trap")])
 748
 749 (define_expand "prologue"
 750   [(clobber (const_int 0))]
 751   ""
 752   "
 753   aarch64_expand_prologue ();
 754   DONE;
 755   "
 756 )
 757
 758 (define_expand "epilogue"
 759   [(clobber (const_int 0))]
 760   ""
 761   "
 762   aarch64_expand_epilogue (false);
 763   DONE;
 764   "
 765 )
 766
 767 (define_expand "sibcall_epilogue"
 768   [(clobber (const_int 0))]
 769   ""
 770   "
 771   aarch64_expand_epilogue (true);
 772   DONE;
 773   "
 774 )
 775
 776 (define_insn "*do_return"
 777   [(return)]
 778   ""
 779   {
 780     if (aarch64_return_address_signing_enabled ()
 781         && TARGET_ARMV8_3
 782         && !crtl->calls_eh_return)
 783       {
 784         if (aarch64_ra_sign_key == AARCH64_KEY_B)
 785           return "retab";
 786         else
 787           return "retaa";
 788       }
 789     return "ret";
 790   }
 791   [(set_attr "type" "branch")]
 792 )
 793
 794 (define_expand "return"
 795   [(simple_return)]
 796   "aarch64_use_return_insn_p ()"
 797   ""
 798 )
 799
 800 (define_insn "simple_return"
 801   [(simple_return)]
 802   "aarch64_use_simple_return_insn_p ()"
 803   "ret"
 804   [(set_attr "type" "branch")]
 805 )
 806
 807 (define_insn "*cb<optab><mode>1"
 808   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 809                                 (const_int 0))
 810                            (label_ref (match_operand 1 "" ""))
 811                            (pc)))]
 812   "!aarch64_track_speculation"
 813   {
 814     if (get_attr_length (insn) == 8)
 815       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 816     else
 817       return "<cbz>\\t%<w>0, %l1";
 818   }
 819   [(set_attr "type" "branch")
 820    (set (attr "length")
 821         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 822                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 823                       (const_int 4)
 824                       (const_int 8)))
 825    (set (attr "far_branch")
 826         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 827                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 828                       (const_int 0)
 829                       (const_int 1)))]
 830 )
 831
 832 (define_insn "*tb<optab><mode>1"
 833   [(set (pc) (if_then_else
 834               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 835                                     (const_int 1)
 836                                     (match_operand 1
 837                                       "aarch64_simd_shift_imm_<mode>" "n"))
 838                    (const_int 0))
 839              (label_ref (match_operand 2 "" ""))
 840              (pc)))
 841    (clobber (reg:CC CC_REGNUM))]
 842   "!aarch64_track_speculation"
 843   {
 844     if (get_attr_length (insn) == 8)
 845       {
 846         if (get_attr_far_branch (insn) == 1)
 847           return aarch64_gen_far_branch (operands, 2, "Ltb",
 848                                          "<inv_tb>\\t%<w>0, %1, ");
 849         else
 850           {
 851             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 852             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 853           }
 854       }
 855     else
 856       return "<tbz>\t%<w>0, %1, %l2";
 857   }
 858   [(set_attr "type" "branch")
 859    (set (attr "length")
 860         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 861                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 862                       (const_int 4)
 863                       (const_int 8)))
 864    (set (attr "far_branch")
 865         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 866                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 867                       (const_int 0)
 868                       (const_int 1)))]
 869
 870 )
 871
 872 (define_insn "*cb<optab><mode>1"
 873   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 874                                  (const_int 0))
 875                            (label_ref (match_operand 1 "" ""))
 876                            (pc)))
 877    (clobber (reg:CC CC_REGNUM))]
 878   "!aarch64_track_speculation"
 879   {
 880     if (get_attr_length (insn) == 8)
 881       {
 882         if (get_attr_far_branch (insn) == 1)
 883           return aarch64_gen_far_branch (operands, 1, "Ltb",
 884                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 885         else
 886           {
 887             char buf[64];
 888             uint64_t val = ((uint64_t) 1)
 889                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 890             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 891             output_asm_insn (buf, operands);
 892             return "<bcond>\t%l1";
 893           }
 894       }
 895     else
 896       return "<tbz>\t%<w>0, <sizem1>, %l1";
 897   }
 898   [(set_attr "type" "branch")
 899    (set (attr "length")
 900         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 901                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 902                       (const_int 4)
 903                       (const_int 8)))
 904    (set (attr "far_branch")
 905         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 906                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 907                       (const_int 0)
 908                       (const_int 1)))]
 909 )
 910
 911 ;; -------------------------------------------------------------------
 912 ;; Subroutine calls and sibcalls
 913 ;; -------------------------------------------------------------------
 914
 915 (define_expand "call"
 916   [(parallel [(call (match_operand 0 "memory_operand")
 917                     (match_operand 1 "general_operand"))
 918               (use (match_operand 2 "" ""))
 919               (clobber (reg:DI LR_REGNUM))])]
 920   ""
 921   "
 922   {
 923     aarch64_expand_call (NULL_RTX, operands[0], false);
 924     DONE;
 925   }"
 926 )
 927
 928 (define_insn "*call_insn"
 929   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 930          (match_operand 1 "" ""))
 931    (clobber (reg:DI LR_REGNUM))]
 932   ""
 933   "@
 934   blr\\t%0
 935   bl\\t%c0"
 936   [(set_attr "type" "call, call")]
 937 )
 938
 939 (define_expand "call_value"
 940   [(parallel [(set (match_operand 0 "" "")
 941                    (call (match_operand 1 "memory_operand")
 942                          (match_operand 2 "general_operand")))
 943               (use (match_operand 3 "" ""))
 944               (clobber (reg:DI LR_REGNUM))])]
 945   ""
 946   "
 947   {
 948     aarch64_expand_call (operands[0], operands[1], false);
 949     DONE;
 950   }"
 951 )
 952
 953 (define_insn "*call_value_insn"
 954   [(set (match_operand 0 "" "")
 955         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 956                       (match_operand 2 "" "")))
 957    (clobber (reg:DI LR_REGNUM))]
 958   ""
 959   "@
 960   blr\\t%1
 961   bl\\t%c1"
 962   [(set_attr "type" "call, call")]
 963 )
 964
 965 (define_expand "sibcall"
 966   [(parallel [(call (match_operand 0 "memory_operand")
 967                     (match_operand 1 "general_operand"))
 968               (return)
 969               (use (match_operand 2 "" ""))])]
 970   ""
 971   {
 972     aarch64_expand_call (NULL_RTX, operands[0], true);
 973     DONE;
 974   }
 975 )
 976
 977 (define_expand "sibcall_value"
 978   [(parallel [(set (match_operand 0 "" "")
 979                    (call (match_operand 1 "memory_operand")
 980                          (match_operand 2 "general_operand")))
 981               (return)
 982               (use (match_operand 3 "" ""))])]
 983   ""
 984   {
 985     aarch64_expand_call (operands[0], operands[1], true);
 986     DONE;
 987   }
 988 )
 989
 990 (define_insn "*sibcall_insn"
 991   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 992          (match_operand 1 "" ""))
 993    (return)]
 994   "SIBLING_CALL_P (insn)"
 995   "@
 996    br\\t%0
 997    b\\t%c0"
 998   [(set_attr "type" "branch, branch")]
 999 )
1000
1001 (define_insn "*sibcall_value_insn"
1002   [(set (match_operand 0 "" "")
1003         (call (mem:DI
1004                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
1005               (match_operand 2 "" "")))
1006    (return)]
1007   "SIBLING_CALL_P (insn)"
1008   "@
1009    br\\t%1
1010    b\\t%c1"
1011   [(set_attr "type" "branch, branch")]
1012 )
1013
1014 ;; Call subroutine returning any type.
1015
1016 (define_expand "untyped_call"
1017   [(parallel [(call (match_operand 0 "")
1018                     (const_int 0))
1019               (match_operand 1 "")
1020               (match_operand 2 "")])]
1021   ""
1022 {
1023   int i;
1024
1025   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
1026
1027   for (i = 0; i < XVECLEN (operands[2], 0); i++)
1028     {
1029       rtx set = XVECEXP (operands[2], 0, i);
1030       emit_move_insn (SET_DEST (set), SET_SRC (set));
1031     }
1032
1033   /* The optimizer does not know that the call sets the function value
1034      registers we stored in the result block.  We avoid problems by
1035      claiming that all hard registers are used and clobbered at this
1036      point.  */
1037   emit_insn (gen_blockage ());
1038   DONE;
1039 })
1040
1041 ;; -------------------------------------------------------------------
1042 ;; Moves
1043 ;; -------------------------------------------------------------------
1044
1045 (define_expand "mov<mode>"
1046   [(set (match_operand:SHORT 0 "nonimmediate_operand")
1047         (match_operand:SHORT 1 "general_operand"))]
1048   ""
1049   "
1050     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1051       operands[1] = force_reg (<MODE>mode, operands[1]);
1052
1053     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1054       {
1055         aarch64_expand_mov_immediate (operands[0], operands[1]);
1056         DONE;
1057       }
1058   "
1059 )
1060
1061 (define_insn "*mov<mode>_aarch64"
1062   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
1063         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
1064   "(register_operand (operands[0], <MODE>mode)
1065     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
1066 {
1067    switch (which_alternative)
1068      {
1069      case 0:
1070        return "mov\t%w0, %w1";
1071      case 1:
1072        return "mov\t%w0, %1";
1073      case 2:
1074        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1075                                                         <MODE>mode);
1076      case 3:
1077        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1078      case 4:
1079        return "ldr<size>\t%w0, %1";
1080      case 5:
1081        return "ldr\t%<size>0, %1";
1082      case 6:
1083        return "str<size>\t%w1, %0";
1084      case 7:
1085        return "str\t%<size>1, %0";
1086      case 8:
1087        return "umov\t%w0, %1.<v>[0]";
1088      case 9:
1089        return "dup\t%0.<Vallxd>, %w1";
1090      case 10:
1091        return "dup\t%<Vetype>0, %1.<v>[0]";
1092      default:
1093        gcc_unreachable ();
1094      }
1095 }
1096   ;; The "mov_imm" type for CNT is just a placeholder.
1097   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1098                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1099    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1100 )
1101
1102 (define_expand "mov<mode>"
1103   [(set (match_operand:GPI 0 "nonimmediate_operand")
1104         (match_operand:GPI 1 "general_operand"))]
1105   ""
1106   "
1107     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1108         && <MODE>mode == DImode
1109         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1110       DONE;
1111
1112     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1113       operands[1] = force_reg (<MODE>mode, operands[1]);
1114
1115     /* FIXME: RR we still need to fix up what we are doing with
1116        symbol_refs and other types of constants.  */
1117     if (CONSTANT_P (operands[1])
1118         && !CONST_INT_P (operands[1]))
1119      {
1120        aarch64_expand_mov_immediate (operands[0], operands[1]);
1121        DONE;
1122      }
1123   "
1124 )
1125
1126 (define_insn_and_split "*movsi_aarch64"
1127   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1128         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1129   "(register_operand (operands[0], SImode)
1130     || aarch64_reg_or_zero (operands[1], SImode))"
1131   "@
1132    mov\\t%w0, %w1
1133    mov\\t%w0, %w1
1134    mov\\t%w0, %w1
1135    mov\\t%w0, %1
1136    #
1137    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1138    ldr\\t%w0, %1
1139    ldr\\t%s0, %1
1140    str\\t%w1, %0
1141    str\\t%s1, %0
1142    adr\\t%x0, %c1
1143    adrp\\t%x0, %A1
1144    fmov\\t%s0, %w1
1145    fmov\\t%w0, %s1
1146    fmov\\t%s0, %s1
1147    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1148   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1149     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1150    [(const_int 0)]
1151    "{
1152        aarch64_expand_mov_immediate (operands[0], operands[1]);
1153        DONE;
1154     }"
1155   ;; The "mov_imm" type for CNT is just a placeholder.
1156   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1157                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1158    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1159 )
1160
1161 (define_insn_and_split "*movdi_aarch64"
1162   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1163         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1164   "(register_operand (operands[0], DImode)
1165     || aarch64_reg_or_zero (operands[1], DImode))"
1166   "@
1167    mov\\t%x0, %x1
1168    mov\\t%0, %x1
1169    mov\\t%x0, %1
1170    mov\\t%x0, %1
1171    mov\\t%w0, %1
1172    #
1173    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1174    ldr\\t%x0, %1
1175    ldr\\t%d0, %1
1176    str\\t%x1, %0
1177    str\\t%d1, %0
1178    adr\\t%x0, %c1
1179    adrp\\t%x0, %A1
1180    fmov\\t%d0, %x1
1181    fmov\\t%x0, %d1
1182    fmov\\t%d0, %d1
1183    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1184    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1185     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1186    [(const_int 0)]
1187    "{
1188        aarch64_expand_mov_immediate (operands[0], operands[1]);
1189        DONE;
1190     }"
1191   ;; The "mov_imm" type for CNTD is just a placeholder.
1192   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1193                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1194                      neon_move")
1195    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1196 )
1197
1198 (define_insn "insv_imm<mode>"
1199   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1200                           (const_int 16)
1201                           (match_operand:GPI 1 "const_int_operand" "n"))
1202         (match_operand:GPI 2 "const_int_operand" "n"))]
1203   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1204    && UINTVAL (operands[1]) % 16 == 0"
1205   "movk\\t%<w>0, %X2, lsl %1"
1206   [(set_attr "type" "mov_imm")]
1207 )
1208
1209 (define_expand "movti"
1210   [(set (match_operand:TI 0 "nonimmediate_operand")
1211         (match_operand:TI 1 "general_operand"))]
1212   ""
1213   "
1214     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1215       operands[1] = force_reg (TImode, operands[1]);
1216
1217     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1218       {
1219         emit_move_insn (gen_lowpart (DImode, operands[0]),
1220                         gen_lowpart (DImode, operands[1]));
1221         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1222         DONE;
1223       }
1224   "
1225 )
1226
1227 (define_insn "*movti_aarch64"
1228   [(set (match_operand:TI 0
1229          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1230         (match_operand:TI 1
1231          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1232   "(register_operand (operands[0], TImode)
1233     || aarch64_reg_or_zero (operands[1], TImode))"
1234   "@
1235    #
1236    #
1237    #
1238    mov\\t%0.16b, %1.16b
1239    ldp\\t%0, %H0, %1
1240    stp\\t%1, %H1, %0
1241    stp\\txzr, xzr, %0
1242    ldr\\t%q0, %1
1243    str\\t%q1, %0"
1244   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1245                              load_16,store_16,store_16,\
1246                              load_16,store_16")
1247    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1248    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1249 )
1250
1251 ;; Split a TImode register-register or register-immediate move into
1252 ;; its component DImode pieces, taking care to handle overlapping
1253 ;; source and dest registers.
1254 (define_split
1255    [(set (match_operand:TI 0 "register_operand" "")
1256          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1257   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1258   [(const_int 0)]
1259 {
1260   aarch64_split_128bit_move (operands[0], operands[1]);
1261   DONE;
1262 })
1263
1264 (define_expand "mov<mode>"
1265   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand")
1266         (match_operand:GPF_TF_F16 1 "general_operand"))]
1267   ""
1268   {
1269     if (!TARGET_FLOAT)
1270       {
1271         aarch64_err_no_fpadvsimd (<MODE>mode);
1272         FAIL;
1273       }
1274
1275     if (GET_CODE (operands[0]) == MEM
1276         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1277               && aarch64_float_const_zero_rtx_p (operands[1])))
1278       operands[1] = force_reg (<MODE>mode, operands[1]);
1279   }
1280 )
1281
1282 (define_insn "*movhf_aarch64"
1283   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1284         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1285   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1286     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1287   "@
1288    movi\\t%0.4h, #0
1289    fmov\\t%h0, %w1
1290    dup\\t%w0.4h, %w1
1291    umov\\t%w0, %1.h[0]
1292    mov\\t%0.h[0], %1.h[0]
1293    fmov\\t%h0, %1
1294    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1295    ldr\\t%h0, %1
1296    str\\t%h1, %0
1297    ldrh\\t%w0, %1
1298    strh\\t%w1, %0
1299    mov\\t%w0, %w1"
1300   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1301                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1302    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1303 )
1304
1305 (define_insn "*movsf_aarch64"
1306   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1307         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1308   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1309     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1310   "@
1311    movi\\t%0.2s, #0
1312    fmov\\t%s0, %w1
1313    fmov\\t%w0, %s1
1314    fmov\\t%s0, %s1
1315    fmov\\t%s0, %1
1316    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1317    ldr\\t%s0, %1
1318    str\\t%s1, %0
1319    ldr\\t%w0, %1
1320    str\\t%w1, %0
1321    mov\\t%w0, %w1
1322    mov\\t%w0, %1"
1323   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1324                      f_loads,f_stores,load_4,store_4,mov_reg,\
1325                      fconsts")
1326    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1327 )
1328
1329 (define_insn "*movdf_aarch64"
1330   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1331         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1332   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1333     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1334   "@
1335    movi\\t%d0, #0
1336    fmov\\t%d0, %x1
1337    fmov\\t%x0, %d1
1338    fmov\\t%d0, %d1
1339    fmov\\t%d0, %1
1340    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1341    ldr\\t%d0, %1
1342    str\\t%d1, %0
1343    ldr\\t%x0, %1
1344    str\\t%x1, %0
1345    mov\\t%x0, %x1
1346    mov\\t%x0, %1"
1347   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1348                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1349                      fconstd")
1350    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1351 )
1352
1353 (define_split
1354   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1355         (match_operand:GPF_HF 1 "general_operand"))]
1356   "can_create_pseudo_p ()
1357    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1358    && !aarch64_float_const_representable_p (operands[1])
1359    &&  aarch64_float_const_rtx_p (operands[1])"
1360   [(const_int 0)]
1361   {
1362     unsigned HOST_WIDE_INT ival;
1363     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1364       FAIL;
1365
1366     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1367     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1368     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1369     DONE;
1370   }
1371 )
1372
1373 (define_insn "*movtf_aarch64"
1374   [(set (match_operand:TF 0
1375          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1376         (match_operand:TF 1
1377          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1378   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1379     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1380   "@
1381    mov\\t%0.16b, %1.16b
1382    #
1383    #
1384    #
1385    movi\\t%0.2d, #0
1386    fmov\\t%s0, wzr
1387    ldr\\t%q0, %1
1388    str\\t%q1, %0
1389    ldp\\t%0, %H0, %1
1390    stp\\t%1, %H1, %0
1391    stp\\txzr, xzr, %0"
1392   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1393                      f_loadd,f_stored,load_16,store_16,store_16")
1394    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1395    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1396 )
1397
1398 (define_split
1399    [(set (match_operand:TF 0 "register_operand" "")
1400          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1401   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1402   [(const_int 0)]
1403   {
1404     aarch64_split_128bit_move (operands[0], operands[1]);
1405     DONE;
1406   }
1407 )
1408
1409 ;; 0 is dst
1410 ;; 1 is src
1411 ;; 2 is size of copy in bytes
1412 ;; 3 is alignment
1413
1414 (define_expand "cpymemdi"
1415   [(match_operand:BLK 0 "memory_operand")
1416    (match_operand:BLK 1 "memory_operand")
1417    (match_operand:DI 2 "immediate_operand")
1418    (match_operand:DI 3 "immediate_operand")]
1419    "!STRICT_ALIGNMENT"
1420 {
1421   if (aarch64_expand_cpymem (operands))
1422     DONE;
1423   FAIL;
1424 }
1425 )
1426
1427 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1428 ;; fairly lax checking on the second memory operation.
1429 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1430   [(set (match_operand:SX 0 "register_operand" "=r,w")
1431         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1432    (set (match_operand:SX2 2 "register_operand" "=r,w")
1433         (match_operand:SX2 3 "memory_operand" "m,m"))]
1434    "rtx_equal_p (XEXP (operands[3], 0),
1435                  plus_constant (Pmode,
1436                                 XEXP (operands[1], 0),
1437                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1438   "@
1439    ldp\\t%w0, %w2, %1
1440    ldp\\t%s0, %s2, %1"
1441   [(set_attr "type" "load_8,neon_load1_2reg")
1442    (set_attr "arch" "*,fp")]
1443 )
1444
1445 ;; Storing different modes that can still be merged
1446 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1447   [(set (match_operand:DX 0 "register_operand" "=r,w")
1448         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1449    (set (match_operand:DX2 2 "register_operand" "=r,w")
1450         (match_operand:DX2 3 "memory_operand" "m,m"))]
1451    "rtx_equal_p (XEXP (operands[3], 0),
1452                  plus_constant (Pmode,
1453                                 XEXP (operands[1], 0),
1454                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1455   "@
1456    ldp\\t%x0, %x2, %1
1457    ldp\\t%d0, %d2, %1"
1458   [(set_attr "type" "load_16,neon_load1_2reg")
1459    (set_attr "arch" "*,fp")]
1460 )
1461
1462 (define_insn "load_pair_dw_tftf"
1463   [(set (match_operand:TF 0 "register_operand" "=w")
1464         (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump"))
1465    (set (match_operand:TF 2 "register_operand" "=w")
1466         (match_operand:TF 3 "memory_operand" "m"))]
1467    "TARGET_SIMD
1468     && rtx_equal_p (XEXP (operands[3], 0),
1469                     plus_constant (Pmode,
1470                                    XEXP (operands[1], 0),
1471                                    GET_MODE_SIZE (TFmode)))"
1472   "ldp\\t%q0, %q2, %1"
1473   [(set_attr "type" "neon_ldp_q")
1474    (set_attr "fp" "yes")]
1475 )
1476
1477 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1478 ;; fairly lax checking on the second memory operation.
1479 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1480   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1481         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1482    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1483         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1484    "rtx_equal_p (XEXP (operands[2], 0),
1485                  plus_constant (Pmode,
1486                                 XEXP (operands[0], 0),
1487                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1488   "@
1489    stp\\t%w1, %w3, %0
1490    stp\\t%s1, %s3, %0"
1491   [(set_attr "type" "store_8,neon_store1_2reg")
1492    (set_attr "arch" "*,fp")]
1493 )
1494
1495 ;; Storing different modes that can still be merged
1496 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1497   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1498         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1499    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1500         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1501    "rtx_equal_p (XEXP (operands[2], 0),
1502                  plus_constant (Pmode,
1503                                 XEXP (operands[0], 0),
1504                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1505   "@
1506    stp\\t%x1, %x3, %0
1507    stp\\t%d1, %d3, %0"
1508   [(set_attr "type" "store_16,neon_store1_2reg")
1509    (set_attr "arch" "*,fp")]
1510 )
1511
1512 (define_insn "store_pair_dw_tftf"
1513   [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump")
1514         (match_operand:TF 1 "register_operand" "w"))
1515    (set (match_operand:TF 2 "memory_operand" "=m")
1516         (match_operand:TF 3 "register_operand" "w"))]
1517    "TARGET_SIMD &&
1518     rtx_equal_p (XEXP (operands[2], 0),
1519                  plus_constant (Pmode,
1520                                 XEXP (operands[0], 0),
1521                                 GET_MODE_SIZE (TFmode)))"
1522   "stp\\t%q1, %q3, %0"
1523   [(set_attr "type" "neon_stp_q")
1524    (set_attr "fp" "yes")]
1525 )
1526
1527 ;; Load pair with post-index writeback.  This is primarily used in function
1528 ;; epilogues.
1529 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1530   [(parallel
1531     [(set (match_operand:P 0 "register_operand" "=k")
1532           (plus:P (match_operand:P 1 "register_operand" "0")
1533                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1534      (set (match_operand:GPI 2 "register_operand" "=r")
1535           (mem:GPI (match_dup 1)))
1536      (set (match_operand:GPI 3 "register_operand" "=r")
1537           (mem:GPI (plus:P (match_dup 1)
1538                    (match_operand:P 5 "const_int_operand" "n"))))])]
1539   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1540   "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4"
1541   [(set_attr "type" "load_<GPI:ldpstp_sz>")]
1542 )
1543
1544 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1545   [(parallel
1546     [(set (match_operand:P 0 "register_operand" "=k")
1547           (plus:P (match_operand:P 1 "register_operand" "0")
1548                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1549      (set (match_operand:GPF 2 "register_operand" "=w")
1550           (mem:GPF (match_dup 1)))
1551      (set (match_operand:GPF 3 "register_operand" "=w")
1552           (mem:GPF (plus:P (match_dup 1)
1553                    (match_operand:P 5 "const_int_operand" "n"))))])]
1554   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1555   "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4"
1556   [(set_attr "type" "neon_load1_2reg")]
1557 )
1558
1559 (define_insn "loadwb_pair<TX:mode>_<P:mode>"
1560   [(parallel
1561     [(set (match_operand:P 0 "register_operand" "=k")
1562           (plus:P (match_operand:P 1 "register_operand" "0")
1563                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1564      (set (match_operand:TX 2 "register_operand" "=w")
1565           (mem:TX (match_dup 1)))
1566      (set (match_operand:TX 3 "register_operand" "=w")
1567           (mem:TX (plus:P (match_dup 1)
1568                           (match_operand:P 5 "const_int_operand" "n"))))])]
1569   "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
1570   "ldp\\t%q2, %q3, [%1], %4"
1571   [(set_attr "type" "neon_ldp_q")]
1572 )
1573
1574 ;; Store pair with pre-index writeback.  This is primarily used in function
1575 ;; prologues.
1576 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1577   [(parallel
1578     [(set (match_operand:P 0 "register_operand" "=&k")
1579           (plus:P (match_operand:P 1 "register_operand" "0")
1580                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1581      (set (mem:GPI (plus:P (match_dup 0)
1582                    (match_dup 4)))
1583           (match_operand:GPI 2 "register_operand" "r"))
1584      (set (mem:GPI (plus:P (match_dup 0)
1585                    (match_operand:P 5 "const_int_operand" "n")))
1586           (match_operand:GPI 3 "register_operand" "r"))])]
1587   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1588   "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!"
1589   [(set_attr "type" "store_<GPI:ldpstp_sz>")]
1590 )
1591
1592 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1593   [(parallel
1594     [(set (match_operand:P 0 "register_operand" "=&k")
1595           (plus:P (match_operand:P 1 "register_operand" "0")
1596                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1597      (set (mem:GPF (plus:P (match_dup 0)
1598                    (match_dup 4)))
1599           (match_operand:GPF 2 "register_operand" "w"))
1600      (set (mem:GPF (plus:P (match_dup 0)
1601                    (match_operand:P 5 "const_int_operand" "n")))
1602           (match_operand:GPF 3 "register_operand" "w"))])]
1603   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1604   "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!"
1605   [(set_attr "type" "neon_store1_2reg<q>")]
1606 )
1607
1608 (define_insn "storewb_pair<TX:mode>_<P:mode>"
1609   [(parallel
1610     [(set (match_operand:P 0 "register_operand" "=&k")
1611           (plus:P (match_operand:P 1 "register_operand" "0")
1612                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1613      (set (mem:TX (plus:P (match_dup 0)
1614                           (match_dup 4)))
1615           (match_operand:TX 2 "register_operand" "w"))
1616      (set (mem:TX (plus:P (match_dup 0)
1617                           (match_operand:P 5 "const_int_operand" "n")))
1618           (match_operand:TX 3 "register_operand" "w"))])]
1619   "TARGET_SIMD
1620    && INTVAL (operands[5])
1621       == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
1622   "stp\\t%q2, %q3, [%0, %4]!"
1623   [(set_attr "type" "neon_stp_q")]
1624 )
1625
1626 ;; -------------------------------------------------------------------
1627 ;; Sign/Zero extension
1628 ;; -------------------------------------------------------------------
1629
1630 (define_expand "<optab>sidi2"
1631   [(set (match_operand:DI 0 "register_operand")
1632         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1633   ""
1634 )
1635
1636 (define_insn "*extendsidi2_aarch64"
1637   [(set (match_operand:DI 0 "register_operand" "=r,r")
1638         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1639   ""
1640   "@
1641    sxtw\t%0, %w1
1642    ldrsw\t%0, %1"
1643   [(set_attr "type" "extend,load_4")]
1644 )
1645
1646 (define_insn "*load_pair_extendsidi2_aarch64"
1647   [(set (match_operand:DI 0 "register_operand" "=r")
1648         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1649    (set (match_operand:DI 2 "register_operand" "=r")
1650         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1651   "rtx_equal_p (XEXP (operands[3], 0),
1652                 plus_constant (Pmode,
1653                                XEXP (operands[1], 0),
1654                                GET_MODE_SIZE (SImode)))"
1655   "ldpsw\\t%0, %2, %1"
1656   [(set_attr "type" "load_8")]
1657 )
1658
1659 (define_insn "*zero_extendsidi2_aarch64"
1660   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1661         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1662   ""
1663   "@
1664    uxtw\t%0, %w1
1665    ldr\t%w0, %1
1666    fmov\t%s0, %w1
1667    ldr\t%s0, %1
1668    fmov\t%w0, %s1
1669    fmov\t%s0, %s1"
1670   [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov")
1671    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1672 )
1673
1674 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1675   [(set (match_operand:DI 0 "register_operand" "=r,w")
1676         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1677    (set (match_operand:DI 2 "register_operand" "=r,w")
1678         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1679   "rtx_equal_p (XEXP (operands[3], 0),
1680                 plus_constant (Pmode,
1681                                XEXP (operands[1], 0),
1682                                GET_MODE_SIZE (SImode)))"
1683   "@
1684    ldp\t%w0, %w2, %1
1685    ldp\t%s0, %s2, %1"
1686   [(set_attr "type" "load_8,neon_load1_2reg")
1687    (set_attr "arch" "*,fp")]
1688 )
1689
1690 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1691   [(set (match_operand:GPI 0 "register_operand")
1692         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1693   ""
1694 )
1695
1696 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1697   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1698         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1699   ""
1700   "@
1701    sxt<SHORT:size>\t%<GPI:w>0, %w1
1702    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1703   [(set_attr "type" "extend,load_4")]
1704 )
1705
1706 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1707   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1708         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1709   ""
1710   "@
1711    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1712    ldr<SHORT:size>\t%w0, %1
1713    ldr\t%<SHORT:size>0, %1"
1714   [(set_attr "type" "logic_imm,load_4,f_loads")
1715    (set_attr "arch" "*,*,fp")]
1716 )
1717
1718 (define_expand "<optab>qihi2"
1719   [(set (match_operand:HI 0 "register_operand")
1720         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1721   ""
1722 )
1723
1724 (define_insn "*extendqihi2_aarch64"
1725   [(set (match_operand:HI 0 "register_operand" "=r,r")
1726         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1727   ""
1728   "@
1729    sxtb\t%w0, %w1
1730    ldrsb\t%w0, %1"
1731   [(set_attr "type" "extend,load_4")]
1732 )
1733
1734 (define_insn "*zero_extendqihi2_aarch64"
1735   [(set (match_operand:HI 0 "register_operand" "=r,r")
1736         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1737   ""
1738   "@
1739    and\t%w0, %w1, 255
1740    ldrb\t%w0, %1"
1741   [(set_attr "type" "logic_imm,load_4")]
1742 )
1743
1744 ;; -------------------------------------------------------------------
1745 ;; Simple arithmetic
1746 ;; -------------------------------------------------------------------
1747
1748 (define_expand "add<mode>3"
1749   [(set
1750     (match_operand:GPI 0 "register_operand")
1751     (plus:GPI (match_operand:GPI 1 "register_operand")
1752               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))]
1753   ""
1754 {
1755   /* If operands[1] is a subreg extract the inner RTX.  */
1756   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1757
1758   /* If the constant is too large for a single instruction and isn't frame
1759      based, split off the immediate so it is available for CSE.  */
1760   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1761       && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode))
1762       && can_create_pseudo_p ()
1763       && (!REG_P (op1)
1764          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1765     operands[2] = force_reg (<MODE>mode, operands[2]);
1766   /* Expand polynomial additions now if the destination is the stack
1767      pointer, since we don't want to use that as a temporary.  */
1768   else if (operands[0] == stack_pointer_rtx
1769            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1770     {
1771       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1772                                 operands[2], NULL_RTX, NULL_RTX);
1773       DONE;
1774     }
1775 })
1776
1777 (define_insn "*add<mode>3_aarch64"
1778   [(set
1779     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk")
1780     (plus:GPI
1781      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk")
1782      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))]
1783   ""
1784   "@
1785   add\\t%<w>0, %<w>1, %2
1786   add\\t%<w>0, %<w>1, %<w>2
1787   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1788   sub\\t%<w>0, %<w>1, #%n2
1789   #
1790   * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1791   * return aarch64_output_sve_addvl_addpl (operands[2]);"
1792   ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1793   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm")
1794    (set_attr "arch" "*,*,simd,*,*,sve,sve")]
1795 )
1796
1797 ;; zero_extend version of above
1798 (define_insn "*addsi3_aarch64_uxtw"
1799   [(set
1800     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1801     (zero_extend:DI
1802      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1803               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1804   ""
1805   "@
1806   add\\t%w0, %w1, %2
1807   add\\t%w0, %w1, %w2
1808   sub\\t%w0, %w1, #%n2
1809   #"
1810   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1811 )
1812
1813 ;; If there's a free register, and we can load the constant with a
1814 ;; single instruction, do so.  This has a chance to improve scheduling.
1815 (define_peephole2
1816   [(match_scratch:GPI 3 "r")
1817    (set (match_operand:GPI 0 "register_operand")
1818         (plus:GPI
1819           (match_operand:GPI 1 "register_operand")
1820           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1821   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1822   [(set (match_dup 3) (match_dup 2))
1823    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1824 )
1825
1826 (define_peephole2
1827   [(match_scratch:SI 3 "r")
1828    (set (match_operand:DI 0 "register_operand")
1829         (zero_extend:DI
1830           (plus:SI
1831             (match_operand:SI 1 "register_operand")
1832             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1833   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1834   [(set (match_dup 3) (match_dup 2))
1835    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1836 )
1837
1838 ;; After peephole2 has had a chance to run, split any remaining long
1839 ;; additions into two add immediates.
1840 (define_split
1841   [(set (match_operand:GPI 0 "register_operand")
1842         (plus:GPI
1843           (match_operand:GPI 1 "register_operand")
1844           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1845   "epilogue_completed"
1846   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1847    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1848   {
1849     HOST_WIDE_INT i = INTVAL (operands[2]);
1850     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1851     operands[3] = GEN_INT (i - s);
1852     operands[4] = GEN_INT (s);
1853   }
1854 )
1855
1856 ;; Match addition of polynomial offsets that require one temporary, for which
1857 ;; we can use the early-clobbered destination register.  This is a separate
1858 ;; pattern so that the early clobber doesn't affect register allocation
1859 ;; for other forms of addition.  However, we still need to provide an
1860 ;; all-register alternative, in case the offset goes out of range after
1861 ;; elimination.  For completeness we might as well provide all GPR-based
1862 ;; alternatives from the main pattern.
1863 ;;
1864 ;; We don't have a pattern for additions requiring two temporaries since at
1865 ;; present LRA doesn't allow new scratches to be added during elimination.
1866 ;; Such offsets should be rare anyway.
1867 ;;
1868 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1869 ;; here would go away.  We could just handle all polynomial constants in
1870 ;; this pattern.
1871 (define_insn_and_split "*add<mode>3_poly_1"
1872   [(set
1873     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r")
1874     (plus:GPI
1875      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,0,rk")
1876      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uai,Uat")))]
1877   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1878   "@
1879   add\\t%<w>0, %<w>1, %2
1880   add\\t%<w>0, %<w>1, %<w>2
1881   sub\\t%<w>0, %<w>1, #%n2
1882   #
1883   * return aarch64_output_sve_scalar_inc_dec (operands[2]);
1884   * return aarch64_output_sve_addvl_addpl (operands[2]);
1885   #"
1886   "&& epilogue_completed
1887    && !reg_overlap_mentioned_p (operands[0], operands[1])
1888    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1889   [(const_int 0)]
1890   {
1891     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1892                               operands[2], operands[0], NULL_RTX);
1893     DONE;
1894   }
1895   ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders.
1896   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")]
1897 )
1898
1899 (define_split
1900   [(set (match_operand:DI 0 "register_operand")
1901         (zero_extend:DI
1902           (plus:SI
1903             (match_operand:SI 1 "register_operand")
1904             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1905   "epilogue_completed"
1906   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1907    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1908   {
1909     HOST_WIDE_INT i = INTVAL (operands[2]);
1910     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1911     operands[3] = GEN_INT (i - s);
1912     operands[4] = GEN_INT (s);
1913     operands[5] = gen_lowpart (SImode, operands[0]);
1914   }
1915 )
1916
1917 (define_expand "addv<mode>4"
1918   [(match_operand:GPI 0 "register_operand")
1919    (match_operand:GPI 1 "register_operand")
1920    (match_operand:GPI 2 "aarch64_plus_operand")
1921    (label_ref (match_operand 3 "" ""))]
1922   ""
1923 {
1924   if (CONST_INT_P (operands[2]))
1925     emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1],
1926                                             operands[2]));
1927   else
1928     emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1929   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1930
1931   DONE;
1932 })
1933
1934 (define_expand "uaddv<mode>4"
1935   [(match_operand:GPI 0 "register_operand")
1936    (match_operand:GPI 1 "register_operand")
1937    (match_operand:GPI 2 "register_operand")
1938    (label_ref (match_operand 3 "" ""))]
1939   ""
1940 {
1941   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1942   aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]);
1943
1944   DONE;
1945 })
1946
1947 (define_expand "addti3"
1948   [(set (match_operand:TI 0 "register_operand")
1949         (plus:TI (match_operand:TI 1 "register_operand")
1950                  (match_operand:TI 2 "aarch64_reg_or_imm")))]
1951   ""
1952 {
1953   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1954
1955   aarch64_addti_scratch_regs (operands[1], operands[2],
1956                               &low_dest, &op1_low, &op2_low,
1957                               &high_dest, &op1_high, &op2_high);
1958
1959   if (op2_low == const0_rtx)
1960     {
1961       low_dest = op1_low;
1962       if (!aarch64_pluslong_operand (op2_high, DImode))
1963         op2_high = force_reg (DImode, op2_high);
1964       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1965     }
1966   else
1967     {
1968       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1969                                       force_reg (DImode, op2_low)));
1970       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1971                                      force_reg (DImode, op2_high)));
1972     }
1973
1974   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1975   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1976
1977   DONE;
1978 })
1979
1980 (define_expand "addvti4"
1981   [(match_operand:TI 0 "register_operand")
1982    (match_operand:TI 1 "register_operand")
1983    (match_operand:TI 2 "aarch64_reg_or_imm")
1984    (label_ref (match_operand 3 "" ""))]
1985   ""
1986 {
1987   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1988
1989   aarch64_addti_scratch_regs (operands[1], operands[2],
1990                               &low_dest, &op1_low, &op2_low,
1991                               &high_dest, &op1_high, &op2_high);
1992
1993   if (op2_low == const0_rtx)
1994     {
1995       low_dest = op1_low;
1996       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1997                                       force_reg (DImode, op2_high)));
1998     }
1999   else
2000     {
2001       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2002                                       force_reg (DImode, op2_low)));
2003       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
2004                                       force_reg (DImode, op2_high)));
2005     }
2006
2007   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2008   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2009
2010   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2011   DONE;
2012 })
2013
2014 (define_expand "uaddvti4"
2015   [(match_operand:TI 0 "register_operand")
2016    (match_operand:TI 1 "register_operand")
2017    (match_operand:TI 2 "aarch64_reg_or_imm")
2018    (label_ref (match_operand 3 "" ""))]
2019   ""
2020 {
2021   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2022
2023   aarch64_addti_scratch_regs (operands[1], operands[2],
2024                               &low_dest, &op1_low, &op2_low,
2025                               &high_dest, &op1_high, &op2_high);
2026
2027   if (op2_low == const0_rtx)
2028     {
2029       low_dest = op1_low;
2030       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
2031                                       force_reg (DImode, op2_high)));
2032     }
2033   else
2034     {
2035       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
2036                                       force_reg (DImode, op2_low)));
2037       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
2038                                       force_reg (DImode, op2_high)));
2039     }
2040
2041   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2042   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2043
2044   aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]);
2045   DONE;
2046  })
2047
2048 (define_insn "add<mode>3_compare0"
2049   [(set (reg:CC_NZ CC_REGNUM)
2050         (compare:CC_NZ
2051          (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk")
2052                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2053          (const_int 0)))
2054    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2055         (plus:GPI (match_dup 1) (match_dup 2)))]
2056   ""
2057   "@
2058   adds\\t%<w>0, %<w>1, %<w>2
2059   adds\\t%<w>0, %<w>1, %2
2060   subs\\t%<w>0, %<w>1, #%n2"
2061   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2062 )
2063
2064 ;; zero_extend version of above
2065 (define_insn "*addsi3_compare0_uxtw"
2066   [(set (reg:CC_NZ CC_REGNUM)
2067         (compare:CC_NZ
2068          (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
2069                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
2070          (const_int 0)))
2071    (set (match_operand:DI 0 "register_operand" "=r,r,r")
2072         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
2073   ""
2074   "@
2075   adds\\t%w0, %w1, %w2
2076   adds\\t%w0, %w1, %2
2077   subs\\t%w0, %w1, #%n2"
2078   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2079 )
2080
2081 (define_insn "*add<mode>3_compareC_cconly"
2082   [(set (reg:CC_C CC_REGNUM)
2083         (compare:CC_C
2084           (plus:GPI
2085             (match_operand:GPI 0 "register_operand" "r,r,r")
2086             (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2087           (match_dup 0)))]
2088   ""
2089   "@
2090   cmn\\t%<w>0, %<w>1
2091   cmn\\t%<w>0, %1
2092   cmp\\t%<w>0, #%n1"
2093   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2094 )
2095
2096 (define_insn "add<mode>3_compareC"
2097   [(set (reg:CC_C CC_REGNUM)
2098         (compare:CC_C
2099           (plus:GPI
2100             (match_operand:GPI 1 "register_operand" "rk,rk,rk")
2101             (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
2102           (match_dup 1)))
2103    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
2104         (plus:GPI (match_dup 1) (match_dup 2)))]
2105   ""
2106   "@
2107   adds\\t%<w>0, %<w>1, %<w>2
2108   adds\\t%<w>0, %<w>1, %2
2109   subs\\t%<w>0, %<w>1, #%n2"
2110   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2111 )
2112
2113 (define_insn "*add<mode>3_compareV_cconly_imm"
2114   [(set (reg:CC_V CC_REGNUM)
2115         (compare:CC_V
2116           (plus:<DWI>
2117             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2118             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2119           (sign_extend:<DWI>
2120            (plus:GPI
2121             (match_dup 0)
2122             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2123   "INTVAL (operands[1]) == INTVAL (operands[2])"
2124   "@
2125   cmn\\t%<w>0, %<w>1
2126   cmp\\t%<w>0, #%n1"
2127   [(set_attr "type" "alus_imm")]
2128 )
2129
2130 (define_insn "*add<mode>3_compareV_cconly"
2131   [(set (reg:CC_V CC_REGNUM)
2132         (compare:CC_V
2133           (plus:<DWI>
2134             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2135             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2136           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2137   ""
2138   "cmn\\t%<w>0, %<w>1"
2139   [(set_attr "type" "alus_sreg")]
2140 )
2141
2142 (define_insn "add<mode>3_compareV_imm"
2143   [(set (reg:CC_V CC_REGNUM)
2144         (compare:CC_V
2145           (plus:<DWI>
2146             (sign_extend:<DWI>
2147               (match_operand:GPI 1 "register_operand" "rk,rk"))
2148             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2149           (sign_extend:<DWI>
2150             (plus:GPI (match_dup 1) (match_dup 2)))))
2151    (set (match_operand:GPI 0 "register_operand" "=r,r")
2152         (plus:GPI (match_dup 1) (match_dup 2)))]
2153    ""
2154    "@
2155    adds\\t%<w>0, %<w>1, %<w>2
2156    subs\\t%<w>0, %<w>1, #%n2"
2157   [(set_attr "type" "alus_imm,alus_imm")]
2158 )
2159
2160 (define_insn "add<mode>3_compareV"
2161   [(set (reg:CC_V CC_REGNUM)
2162         (compare:CC_V
2163           (plus:<DWI>
2164             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk"))
2165             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2166           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2167    (set (match_operand:GPI 0 "register_operand" "=r")
2168         (plus:GPI (match_dup 1) (match_dup 2)))]
2169   ""
2170   "adds\\t%<w>0, %<w>1, %<w>2"
2171   [(set_attr "type" "alus_sreg")]
2172 )
2173
2174 (define_insn "*adds_shift_imm_<mode>"
2175   [(set (reg:CC_NZ CC_REGNUM)
2176         (compare:CC_NZ
2177          (plus:GPI (ASHIFT:GPI
2178                     (match_operand:GPI 1 "register_operand" "r")
2179                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2180                    (match_operand:GPI 3 "register_operand" "r"))
2181          (const_int 0)))
2182    (set (match_operand:GPI 0 "register_operand" "=r")
2183         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2184                   (match_dup 3)))]
2185   ""
2186   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2187   [(set_attr "type" "alus_shift_imm")]
2188 )
2189
2190 (define_insn "*subs_shift_imm_<mode>"
2191   [(set (reg:CC_NZ CC_REGNUM)
2192         (compare:CC_NZ
2193          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2194                     (ASHIFT:GPI
2195                      (match_operand:GPI 2 "register_operand" "r")
2196                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2197          (const_int 0)))
2198    (set (match_operand:GPI 0 "register_operand" "=r")
2199         (minus:GPI (match_dup 1)
2200                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2201   ""
2202   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2203   [(set_attr "type" "alus_shift_imm")]
2204 )
2205
2206 (define_insn "*adds_mul_imm_<mode>"
2207   [(set (reg:CC_NZ CC_REGNUM)
2208         (compare:CC_NZ
2209          (plus:GPI (mult:GPI
2210                     (match_operand:GPI 1 "register_operand" "r")
2211                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2212                    (match_operand:GPI 3 "register_operand" "r"))
2213          (const_int 0)))
2214    (set (match_operand:GPI 0 "register_operand" "=r")
2215         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2216                   (match_dup 3)))]
2217   ""
2218   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2219   [(set_attr "type" "alus_shift_imm")]
2220 )
2221
2222 (define_insn "*subs_mul_imm_<mode>"
2223   [(set (reg:CC_NZ CC_REGNUM)
2224         (compare:CC_NZ
2225          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2226                     (mult:GPI
2227                      (match_operand:GPI 2 "register_operand" "r")
2228                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2229          (const_int 0)))
2230    (set (match_operand:GPI 0 "register_operand" "=r")
2231         (minus:GPI (match_dup 1)
2232                    (mult:GPI (match_dup 2) (match_dup 3))))]
2233   ""
2234   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2235   [(set_attr "type" "alus_shift_imm")]
2236 )
2237
2238 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2239   [(set (reg:CC_NZ CC_REGNUM)
2240         (compare:CC_NZ
2241          (plus:GPI
2242           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2243           (match_operand:GPI 2 "register_operand" "rk"))
2244         (const_int 0)))
2245    (set (match_operand:GPI 0 "register_operand" "=r")
2246         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2247   ""
2248   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2249   [(set_attr "type" "alus_ext")]
2250 )
2251
2252 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2253   [(set (reg:CC_NZ CC_REGNUM)
2254         (compare:CC_NZ
2255          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2256                     (ANY_EXTEND:GPI
2257                      (match_operand:ALLX 2 "register_operand" "r")))
2258         (const_int 0)))
2259    (set (match_operand:GPI 0 "register_operand" "=r")
2260         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2261   ""
2262   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2263   [(set_attr "type" "alus_ext")]
2264 )
2265
2266 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2267   [(set (reg:CC_NZ CC_REGNUM)
2268         (compare:CC_NZ
2269          (plus:GPI (ashift:GPI
2270                     (ANY_EXTEND:GPI
2271                      (match_operand:ALLX 1 "register_operand" "r"))
2272                     (match_operand 2 "aarch64_imm3" "Ui3"))
2273                    (match_operand:GPI 3 "register_operand" "rk"))
2274          (const_int 0)))
2275    (set (match_operand:GPI 0 "register_operand" "=rk")
2276         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2277                               (match_dup 2))
2278                   (match_dup 3)))]
2279   ""
2280   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2281   [(set_attr "type" "alus_ext")]
2282 )
2283
2284 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2285   [(set (reg:CC_NZ CC_REGNUM)
2286         (compare:CC_NZ
2287          (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2288                     (ashift:GPI
2289                      (ANY_EXTEND:GPI
2290                       (match_operand:ALLX 2 "register_operand" "r"))
2291                      (match_operand 3 "aarch64_imm3" "Ui3")))
2292          (const_int 0)))
2293    (set (match_operand:GPI 0 "register_operand" "=rk")
2294         (minus:GPI (match_dup 1)
2295                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2296                                (match_dup 3))))]
2297   ""
2298   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2299   [(set_attr "type" "alus_ext")]
2300 )
2301
2302 (define_insn "*adds_<optab><mode>_multp2"
2303   [(set (reg:CC_NZ CC_REGNUM)
2304         (compare:CC_NZ
2305          (plus:GPI (ANY_EXTRACT:GPI
2306                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2307                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2308                     (match_operand 3 "const_int_operand" "n")
2309                     (const_int 0))
2310                    (match_operand:GPI 4 "register_operand" "rk"))
2311         (const_int 0)))
2312    (set (match_operand:GPI 0 "register_operand" "=r")
2313         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2314                                    (match_dup 3)
2315                                    (const_int 0))
2316                   (match_dup 4)))]
2317   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2318   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2319   [(set_attr "type" "alus_ext")]
2320 )
2321
2322 (define_insn "*subs_<optab><mode>_multp2"
2323   [(set (reg:CC_NZ CC_REGNUM)
2324         (compare:CC_NZ
2325          (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2326                     (ANY_EXTRACT:GPI
2327                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2328                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2329                      (match_operand 3 "const_int_operand" "n")
2330                      (const_int 0)))
2331         (const_int 0)))
2332    (set (match_operand:GPI 0 "register_operand" "=r")
2333         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2334                                   (mult:GPI (match_dup 1) (match_dup 2))
2335                                   (match_dup 3)
2336                                   (const_int 0))))]
2337   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2338   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2339   [(set_attr "type" "alus_ext")]
2340 )
2341
2342 (define_insn "*add<mode>3nr_compare0"
2343   [(set (reg:CC_NZ CC_REGNUM)
2344         (compare:CC_NZ
2345          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2346                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2347          (const_int 0)))]
2348   ""
2349   "@
2350   cmn\\t%<w>0, %<w>1
2351   cmn\\t%<w>0, %1
2352   cmp\\t%<w>0, #%n1"
2353   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2354 )
2355
2356 (define_insn "aarch64_sub<mode>_compare0"
2357   [(set (reg:CC_NZ CC_REGNUM)
2358         (compare:CC_NZ
2359          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2360                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2361          (const_int 0)))]
2362   ""
2363   "cmp\\t%<w>0, %<w>1"
2364   [(set_attr "type" "alus_sreg")]
2365 )
2366
2367 (define_insn "*compare_neg<mode>"
2368   [(set (reg:CC_Z CC_REGNUM)
2369         (compare:CC_Z
2370          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2371          (match_operand:GPI 1 "register_operand" "r")))]
2372   ""
2373   "cmn\\t%<w>1, %<w>0"
2374   [(set_attr "type" "alus_sreg")]
2375 )
2376
2377 (define_insn "*add_<shift>_<mode>"
2378   [(set (match_operand:GPI 0 "register_operand" "=r")
2379         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2380                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2381                   (match_operand:GPI 3 "register_operand" "r")))]
2382   ""
2383   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2384   [(set_attr "type" "alu_shift_imm")]
2385 )
2386
2387 ;; zero_extend version of above
2388 (define_insn "*add_<shift>_si_uxtw"
2389   [(set (match_operand:DI 0 "register_operand" "=r")
2390         (zero_extend:DI
2391          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2392                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2393                   (match_operand:SI 3 "register_operand" "r"))))]
2394   ""
2395   "add\\t%w0, %w3, %w1, <shift> %2"
2396   [(set_attr "type" "alu_shift_imm")]
2397 )
2398
2399 (define_insn "*add_mul_imm_<mode>"
2400   [(set (match_operand:GPI 0 "register_operand" "=r")
2401         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2402                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2403                   (match_operand:GPI 3 "register_operand" "r")))]
2404   ""
2405   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2406   [(set_attr "type" "alu_shift_imm")]
2407 )
2408
2409 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2410   [(set (match_operand:GPI 0 "register_operand" "=rk")
2411         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2412                   (match_operand:GPI 2 "register_operand" "r")))]
2413   ""
2414   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2415   [(set_attr "type" "alu_ext")]
2416 )
2417
2418 ;; zero_extend version of above
2419 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2420   [(set (match_operand:DI 0 "register_operand" "=rk")
2421         (zero_extend:DI
2422          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2423                   (match_operand:GPI 2 "register_operand" "r"))))]
2424   ""
2425   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2426   [(set_attr "type" "alu_ext")]
2427 )
2428
2429 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2430   [(set (match_operand:GPI 0 "register_operand" "=rk")
2431         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2432                                (match_operand:ALLX 1 "register_operand" "r"))
2433                               (match_operand 2 "aarch64_imm3" "Ui3"))
2434                   (match_operand:GPI 3 "register_operand" "r")))]
2435   ""
2436   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2437   [(set_attr "type" "alu_ext")]
2438 )
2439
2440 ;; zero_extend version of above
2441 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2442   [(set (match_operand:DI 0 "register_operand" "=rk")
2443         (zero_extend:DI
2444          (plus:SI (ashift:SI (ANY_EXTEND:SI
2445                               (match_operand:SHORT 1 "register_operand" "r"))
2446                              (match_operand 2 "aarch64_imm3" "Ui3"))
2447                   (match_operand:SI 3 "register_operand" "r"))))]
2448   ""
2449   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2450   [(set_attr "type" "alu_ext")]
2451 )
2452
2453 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2454   [(set (match_operand:GPI 0 "register_operand" "=rk")
2455         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2456                              (match_operand:ALLX 1 "register_operand" "r"))
2457                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2458                   (match_operand:GPI 3 "register_operand" "r")))]
2459   ""
2460   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2461   [(set_attr "type" "alu_ext")]
2462 )
2463
2464 ;; zero_extend version of above
2465 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2466   [(set (match_operand:DI 0 "register_operand" "=rk")
2467         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2468                              (match_operand:SHORT 1 "register_operand" "r"))
2469                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2470                   (match_operand:SI 3 "register_operand" "r"))))]
2471   ""
2472   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2473   [(set_attr "type" "alu_ext")]
2474 )
2475
2476 (define_insn "*add_<optab><mode>_multp2"
2477   [(set (match_operand:GPI 0 "register_operand" "=rk")
2478         (plus:GPI (ANY_EXTRACT:GPI
2479                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2480                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2481                    (match_operand 3 "const_int_operand" "n")
2482                    (const_int 0))
2483                   (match_operand:GPI 4 "register_operand" "r")))]
2484   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2485   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2486   [(set_attr "type" "alu_ext")]
2487 )
2488
2489 ;; zero_extend version of above
2490 (define_insn "*add_<optab>si_multp2_uxtw"
2491   [(set (match_operand:DI 0 "register_operand" "=rk")
2492         (zero_extend:DI
2493          (plus:SI (ANY_EXTRACT:SI
2494                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2495                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2496                    (match_operand 3 "const_int_operand" "n")
2497                    (const_int 0))
2498                   (match_operand:SI 4 "register_operand" "r"))))]
2499   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2500   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2501   [(set_attr "type" "alu_ext")]
2502 )
2503
2504 (define_expand "add<mode>3_carryin"
2505   [(set (match_operand:GPI 0 "register_operand")
2506         (plus:GPI
2507           (plus:GPI
2508             (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2509             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2510           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2511    ""
2512    ""
2513 )
2514
2515 ;; Note that add with carry with two zero inputs is matched by cset,
2516 ;; and that add with carry with one zero input is matched by cinc.
2517
2518 (define_insn "*add<mode>3_carryin"
2519   [(set (match_operand:GPI 0 "register_operand" "=r")
2520         (plus:GPI
2521           (plus:GPI
2522             (match_operand:GPI 3 "aarch64_carry_operation" "")
2523             (match_operand:GPI 1 "register_operand" "r"))
2524           (match_operand:GPI 2 "register_operand" "r")))]
2525    ""
2526    "adc\\t%<w>0, %<w>1, %<w>2"
2527   [(set_attr "type" "adc_reg")]
2528 )
2529
2530 ;; zero_extend version of above
2531 (define_insn "*addsi3_carryin_uxtw"
2532   [(set (match_operand:DI 0 "register_operand" "=r")
2533         (zero_extend:DI
2534           (plus:SI
2535             (plus:SI
2536               (match_operand:SI 3 "aarch64_carry_operation" "")
2537               (match_operand:SI 1 "register_operand" "r"))
2538             (match_operand:SI 2 "register_operand" "r"))))]
2539    ""
2540    "adc\\t%w0, %w1, %w2"
2541   [(set_attr "type" "adc_reg")]
2542 )
2543
2544 (define_expand "add<mode>3_carryinC"
2545   [(parallel
2546      [(set (match_dup 3)
2547            (compare:CC_ADC
2548              (plus:<DWI>
2549                (plus:<DWI>
2550                  (match_dup 4)
2551                  (zero_extend:<DWI>
2552                    (match_operand:GPI 1 "register_operand")))
2553                (zero_extend:<DWI>
2554                  (match_operand:GPI 2 "register_operand")))
2555              (match_dup 6)))
2556       (set (match_operand:GPI 0 "register_operand")
2557            (plus:GPI
2558              (plus:GPI (match_dup 5) (match_dup 1))
2559              (match_dup 2)))])]
2560    ""
2561 {
2562   operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM);
2563   rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2564   operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx);
2565   operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx);
2566   operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode)
2567                                       << GET_MODE_BITSIZE (<MODE>mode),
2568                                       TImode);
2569 })
2570
2571 (define_insn "*add<mode>3_carryinC_zero"
2572   [(set (reg:CC_ADC CC_REGNUM)
2573         (compare:CC_ADC
2574           (plus:<DWI>
2575             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2576             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2577           (match_operand 4 "const_scalar_int_operand" "")))
2578    (set (match_operand:GPI 0 "register_operand" "=r")
2579         (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "")
2580                   (match_dup 1)))]
2581   "rtx_mode_t (operands[4], <DWI>mode)
2582    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2583    "adcs\\t%<w>0, %<w>1, <w>zr"
2584   [(set_attr "type" "adc_reg")]
2585 )
2586
2587 (define_insn "*add<mode>3_carryinC"
2588   [(set (reg:CC_ADC CC_REGNUM)
2589         (compare:CC_ADC
2590           (plus:<DWI>
2591             (plus:<DWI>
2592               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2593               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2594             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2595           (match_operand 5 "const_scalar_int_operand" "")))
2596    (set (match_operand:GPI 0 "register_operand" "=r")
2597         (plus:GPI
2598           (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "")
2599                     (match_dup 1))
2600           (match_dup 2)))]
2601   "rtx_mode_t (operands[5], <DWI>mode)
2602    == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))"
2603    "adcs\\t%<w>0, %<w>1, %<w>2"
2604   [(set_attr "type" "adc_reg")]
2605 )
2606
2607 (define_expand "add<mode>3_carryinV"
2608   [(parallel
2609      [(set (reg:CC_V CC_REGNUM)
2610            (compare:CC_V
2611              (plus:<DWI>
2612                (plus:<DWI>
2613                  (match_dup 3)
2614                  (sign_extend:<DWI>
2615                    (match_operand:GPI 1 "register_operand")))
2616                (sign_extend:<DWI>
2617                  (match_operand:GPI 2 "register_operand")))
2618            (sign_extend:<DWI>
2619              (plus:GPI
2620                (plus:GPI (match_dup 4) (match_dup 1))
2621                (match_dup 2)))))
2622       (set (match_operand:GPI 0 "register_operand")
2623            (plus:GPI
2624              (plus:GPI (match_dup 4) (match_dup 1))
2625              (match_dup 2)))])]
2626    ""
2627 {
2628   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2629   operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx);
2630   operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx);
2631 })
2632
2633 (define_insn "*add<mode>3_carryinV_zero"
2634   [(set (reg:CC_V CC_REGNUM)
2635         (compare:CC_V
2636           (plus:<DWI>
2637             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2638             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2639           (sign_extend:<DWI>
2640             (plus:GPI
2641               (match_operand:GPI 3 "aarch64_carry_operation" "")
2642               (match_dup 1)))))
2643    (set (match_operand:GPI 0 "register_operand" "=r")
2644         (plus:GPI (match_dup 3) (match_dup 1)))]
2645    ""
2646    "adcs\\t%<w>0, %<w>1, <w>zr"
2647   [(set_attr "type" "adc_reg")]
2648 )
2649
2650 (define_insn "*add<mode>3_carryinV"
2651   [(set (reg:CC_V CC_REGNUM)
2652         (compare:CC_V
2653           (plus:<DWI>
2654             (plus:<DWI>
2655               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2656               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2657             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2658           (sign_extend:<DWI>
2659             (plus:GPI
2660               (plus:GPI
2661                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2662                 (match_dup 1))
2663               (match_dup 2)))))
2664    (set (match_operand:GPI 0 "register_operand" "=r")
2665         (plus:GPI
2666           (plus:GPI (match_dup 4) (match_dup 1))
2667           (match_dup 2)))]
2668    ""
2669    "adcs\\t%<w>0, %<w>1, %<w>2"
2670   [(set_attr "type" "adc_reg")]
2671 )
2672
2673 (define_insn "*add_uxt<mode>_shift2"
2674   [(set (match_operand:GPI 0 "register_operand" "=rk")
2675         (plus:GPI (and:GPI
2676                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2677                                (match_operand 2 "aarch64_imm3" "Ui3"))
2678                    (match_operand 3 "const_int_operand" "n"))
2679                   (match_operand:GPI 4 "register_operand" "r")))]
2680   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2681   "*
2682   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2683                                            INTVAL (operands[3])));
2684   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2685   [(set_attr "type" "alu_ext")]
2686 )
2687
2688 ;; zero_extend version of above
2689 (define_insn "*add_uxtsi_shift2_uxtw"
2690   [(set (match_operand:DI 0 "register_operand" "=rk")
2691         (zero_extend:DI
2692          (plus:SI (and:SI
2693                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2694                               (match_operand 2 "aarch64_imm3" "Ui3"))
2695                    (match_operand 3 "const_int_operand" "n"))
2696                   (match_operand:SI 4 "register_operand" "r"))))]
2697   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2698   "*
2699   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2700                                            INTVAL (operands[3])));
2701   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2702   [(set_attr "type" "alu_ext")]
2703 )
2704
2705 (define_insn "*add_uxt<mode>_multp2"
2706   [(set (match_operand:GPI 0 "register_operand" "=rk")
2707         (plus:GPI (and:GPI
2708                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2709                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2710                    (match_operand 3 "const_int_operand" "n"))
2711                   (match_operand:GPI 4 "register_operand" "r")))]
2712   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2713   "*
2714   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2715                                            INTVAL (operands[3])));
2716   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2717   [(set_attr "type" "alu_ext")]
2718 )
2719
2720 ;; zero_extend version of above
2721 (define_insn "*add_uxtsi_multp2_uxtw"
2722   [(set (match_operand:DI 0 "register_operand" "=rk")
2723         (zero_extend:DI
2724          (plus:SI (and:SI
2725                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2726                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2727                    (match_operand 3 "const_int_operand" "n"))
2728                   (match_operand:SI 4 "register_operand" "r"))))]
2729   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2730   "*
2731   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2732                                            INTVAL (operands[3])));
2733   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2734   [(set_attr "type" "alu_ext")]
2735 )
2736
2737 (define_insn "subsi3"
2738   [(set (match_operand:SI 0 "register_operand" "=rk")
2739         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2740                   (match_operand:SI 2 "register_operand" "r")))]
2741   ""
2742   "sub\\t%w0, %w1, %w2"
2743   [(set_attr "type" "alu_sreg")]
2744 )
2745
2746 ;; zero_extend version of above
2747 (define_insn "*subsi3_uxtw"
2748   [(set (match_operand:DI 0 "register_operand" "=rk")
2749         (zero_extend:DI
2750          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2751                    (match_operand:SI 2 "register_operand" "r"))))]
2752   ""
2753   "sub\\t%w0, %w1, %w2"
2754   [(set_attr "type" "alu_sreg")]
2755 )
2756
2757 (define_insn "subdi3"
2758   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2759         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2760                   (match_operand:DI 2 "register_operand" "r,w")))]
2761   ""
2762   "@
2763    sub\\t%x0, %x1, %x2
2764    sub\\t%d0, %d1, %d2"
2765   [(set_attr "type" "alu_sreg, neon_sub")
2766    (set_attr "arch" "*,simd")]
2767 )
2768
2769 (define_expand "subv<GPI:mode>4"
2770   [(match_operand:GPI 0 "register_operand")
2771    (match_operand:GPI 1 "register_operand")
2772    (match_operand:GPI 2 "aarch64_plus_operand")
2773    (label_ref (match_operand 3 "" ""))]
2774   ""
2775 {
2776   if (CONST_INT_P (operands[2]))
2777     emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2]));
2778   else
2779     emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2]));
2780   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2781
2782   DONE;
2783 })
2784
2785 (define_insn "subv<GPI:mode>_insn"
2786   [(set (reg:CC_V CC_REGNUM)
2787         (compare:CC_V
2788          (sign_extend:<DWI>
2789           (minus:GPI
2790            (match_operand:GPI 1 "register_operand" "rk")
2791            (match_operand:GPI 2 "register_operand" "r")))
2792          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2793                       (sign_extend:<DWI> (match_dup 2)))))
2794    (set (match_operand:GPI 0 "register_operand" "=r")
2795         (minus:GPI (match_dup 1) (match_dup 2)))]
2796   ""
2797   "subs\\t%<w>0, %<w>1, %<w>2"
2798   [(set_attr "type" "alus_sreg")]
2799 )
2800
2801 (define_insn "subv<GPI:mode>_imm"
2802   [(set (reg:CC_V CC_REGNUM)
2803         (compare:CC_V
2804          (sign_extend:<DWI>
2805           (minus:GPI
2806            (match_operand:GPI 1 "register_operand" "rk,rk")
2807            (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2808          (minus:<DWI> (sign_extend:<DWI> (match_dup 1))
2809                       (match_dup 2))))
2810    (set (match_operand:GPI 0 "register_operand" "=r,r")
2811         (minus:GPI (match_dup 1) (match_dup 2)))]
2812   ""
2813   "@
2814    subs\\t%<w>0, %<w>1, %2
2815    adds\\t%<w>0, %<w>1, #%n2"
2816   [(set_attr "type" "alus_sreg")]
2817 )
2818
2819 (define_expand "negv<GPI:mode>3"
2820   [(match_operand:GPI 0 "register_operand")
2821    (match_operand:GPI 1 "register_operand")
2822    (label_ref (match_operand 2 "" ""))]
2823   ""
2824   {
2825     emit_insn (gen_negv<mode>_insn (operands[0], operands[1]));
2826     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2827
2828     DONE;
2829   }
2830 )
2831
2832 (define_insn "negv<GPI:mode>_insn"
2833   [(set (reg:CC_V CC_REGNUM)
2834         (compare:CC_V
2835          (sign_extend:<DWI>
2836           (neg:GPI (match_operand:GPI 1 "register_operand" "r")))
2837          (neg:<DWI> (sign_extend:<DWI> (match_dup 1)))))
2838    (set (match_operand:GPI 0 "register_operand" "=r")
2839         (neg:GPI (match_dup 1)))]
2840   ""
2841   "negs\\t%<w>0, %<w>1"
2842   [(set_attr "type" "alus_sreg")]
2843 )
2844
2845 (define_insn "negv<GPI:mode>_cmp_only"
2846   [(set (reg:CC_V CC_REGNUM)
2847         (compare:CC_V
2848          (sign_extend:<DWI>
2849           (neg:GPI (match_operand:GPI 0 "register_operand" "r")))
2850          (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))]
2851   ""
2852   "negs\\t%<w>zr, %<w>0"
2853   [(set_attr "type" "alus_sreg")]
2854 )
2855
2856 (define_insn "*cmpv<GPI:mode>_insn"
2857   [(set (reg:CC_V CC_REGNUM)
2858         (compare:CC_V
2859          (sign_extend:<DWI>
2860           (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r")
2861                      (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))
2862          (minus:<DWI> (sign_extend:<DWI> (match_dup 0))
2863                     (sign_extend:<DWI> (match_dup 1)))))]
2864   ""
2865   "@
2866    cmp\\t%<w>0, %<w>1
2867    cmp\\t%<w>0, %1
2868    cmp\\t%<w>0, #%n1"
2869   [(set_attr "type" "alus_sreg")]
2870 )
2871
2872 (define_expand "usubv<mode>4"
2873   [(match_operand:GPI 0 "register_operand")
2874    (match_operand:GPI 1 "aarch64_reg_or_zero")
2875    (match_operand:GPI 2 "aarch64_reg_or_zero")
2876    (label_ref (match_operand 3 "" ""))]
2877   ""
2878 {
2879   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2880   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2881
2882   DONE;
2883 })
2884
2885 (define_expand "subti3"
2886   [(set (match_operand:TI 0 "register_operand")
2887         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero")
2888                   (match_operand:TI 2 "register_operand")))]
2889   ""
2890 {
2891   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2892
2893   aarch64_subvti_scratch_regs (operands[1], operands[2],
2894                                &low_dest, &op1_low, &op2_low,
2895                                &high_dest, &op1_high, &op2_high);
2896
2897   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2898   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2899
2900   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2901   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2902   DONE;
2903 })
2904
2905 (define_expand "subvti4"
2906   [(match_operand:TI 0 "register_operand")
2907    (match_operand:TI 1 "register_operand")
2908    (match_operand:TI 2 "aarch64_reg_or_imm")
2909    (label_ref (match_operand 3 "" ""))]
2910   ""
2911 {
2912   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2913
2914   aarch64_subvti_scratch_regs (operands[1], operands[2],
2915                                &low_dest, &op1_low, &op2_low,
2916                                &high_dest, &op1_high, &op2_high);
2917   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2918                          high_dest, op1_high, op2_high, false);
2919
2920   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2921   DONE;
2922 })
2923
2924 (define_expand "usubvti4"
2925   [(match_operand:TI 0 "register_operand")
2926    (match_operand:TI 1 "register_operand")
2927    (match_operand:TI 2 "aarch64_reg_or_imm")
2928    (label_ref (match_operand 3 "" ""))]
2929   ""
2930 {
2931   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2932
2933   aarch64_subvti_scratch_regs (operands[1], operands[2],
2934                                     &low_dest, &op1_low, &op2_low,
2935                                &high_dest, &op1_high, &op2_high);
2936   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2937                          high_dest, op1_high, op2_high, true);
2938
2939   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2940   DONE;
2941 })
2942
2943 (define_expand "negvti3"
2944   [(match_operand:TI 0 "register_operand")
2945    (match_operand:TI 1 "register_operand")
2946    (label_ref (match_operand 2 "" ""))]
2947   ""
2948   {
2949     emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]),
2950                                    gen_lowpart (DImode, operands[1])));
2951     emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]),
2952                                     gen_highpart (DImode, operands[1])));
2953     aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]);
2954
2955     DONE;
2956   }
2957 )
2958
2959 (define_insn "negdi_carryout"
2960   [(set (reg:CC CC_REGNUM)
2961         (compare:CC
2962          (const_int 0) (match_operand:DI 1 "register_operand" "r")))
2963    (set (match_operand:DI 0 "register_operand" "=r")
2964         (neg:DI (match_dup 1)))]
2965   ""
2966   "negs\\t%0, %1"
2967   [(set_attr "type" "alus_sreg")]
2968 )
2969
2970 (define_insn "negvdi_carryinV"
2971   [(set (reg:CC_V CC_REGNUM)
2972         (compare:CC_V
2973          (neg:TI (plus:TI
2974                   (ltu:TI (reg:CC CC_REGNUM) (const_int 0))
2975                   (sign_extend:TI (match_operand:DI 1 "register_operand" "r"))))
2976          (sign_extend:TI
2977           (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2978                            (match_dup 1))))))
2979    (set (match_operand:DI 0 "register_operand" "=r")
2980         (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0))
2981                          (match_dup 1))))]
2982   ""
2983   "ngcs\\t%0, %1"
2984   [(set_attr "type" "alus_sreg")]
2985 )
2986
2987 (define_insn "*sub<mode>3_compare0"
2988   [(set (reg:CC_NZ CC_REGNUM)
2989         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2990                                   (match_operand:GPI 2 "register_operand" "r"))
2991                        (const_int 0)))
2992    (set (match_operand:GPI 0 "register_operand" "=r")
2993         (minus:GPI (match_dup 1) (match_dup 2)))]
2994   ""
2995   "subs\\t%<w>0, %<w>1, %<w>2"
2996   [(set_attr "type" "alus_sreg")]
2997 )
2998
2999 ;; zero_extend version of above
3000 (define_insn "*subsi3_compare0_uxtw"
3001   [(set (reg:CC_NZ CC_REGNUM)
3002         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk")
3003                                  (match_operand:SI 2 "register_operand" "r"))
3004                        (const_int 0)))
3005    (set (match_operand:DI 0 "register_operand" "=r")
3006         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
3007   ""
3008   "subs\\t%w0, %w1, %w2"
3009   [(set_attr "type" "alus_sreg")]
3010 )
3011
3012 (define_insn "sub<mode>3_compare1_imm"
3013   [(set (reg:CC CC_REGNUM)
3014         (compare:CC
3015           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ")
3016           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
3017    (set (match_operand:GPI 0 "register_operand" "=r,r")
3018         (plus:GPI
3019           (match_dup 1)
3020           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
3021   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
3022   "@
3023   subs\\t%<w>0, %<w>1, %2
3024   adds\\t%<w>0, %<w>1, #%n2"
3025   [(set_attr "type" "alus_imm")]
3026 )
3027
3028 (define_insn "sub<mode>3_compare1"
3029   [(set (reg:CC CC_REGNUM)
3030         (compare:CC
3031           (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ")
3032           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
3033    (set (match_operand:GPI 0 "register_operand" "=r")
3034         (minus:GPI (match_dup 1) (match_dup 2)))]
3035   ""
3036   "subs\\t%<w>0, %<w>1, %<w>2"
3037   [(set_attr "type" "alus_sreg")]
3038 )
3039
3040 (define_peephole2
3041   [(set (match_operand:GPI 0 "aarch64_general_reg")
3042         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
3043                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
3044    (set (reg:CC CC_REGNUM)
3045         (compare:CC
3046           (match_dup 1)
3047           (match_dup 2)))]
3048   "!reg_overlap_mentioned_p (operands[0], operands[1])
3049    && !reg_overlap_mentioned_p (operands[0], operands[2])"
3050   [(const_int 0)]
3051   {
3052     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3053                                          operands[2]));
3054     DONE;
3055   }
3056 )
3057
3058 ;; Same as the above peephole but with the compare and minus in
3059 ;; swapped order.  The restriction on overlap between operand 0
3060 ;; and operands 1 and 2 doesn't apply here.
3061 (define_peephole2
3062   [(set (reg:CC CC_REGNUM)
3063         (compare:CC
3064           (match_operand:GPI 1 "aarch64_reg_or_zero")
3065           (match_operand:GPI 2 "aarch64_reg_or_zero")))
3066    (set (match_operand:GPI 0 "aarch64_general_reg")
3067         (minus:GPI (match_dup 1)
3068                    (match_dup 2)))]
3069   ""
3070   [(const_int 0)]
3071   {
3072     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
3073                                          operands[2]));
3074     DONE;
3075   }
3076 )
3077
3078 (define_peephole2
3079   [(set (match_operand:GPI 0 "aarch64_general_reg")
3080         (plus:GPI (match_operand:GPI 1 "register_operand")
3081                   (match_operand:GPI 2 "aarch64_plus_immediate")))
3082    (set (reg:CC CC_REGNUM)
3083         (compare:CC
3084           (match_dup 1)
3085           (match_operand:GPI 3 "const_int_operand")))]
3086   "!reg_overlap_mentioned_p (operands[0], operands[1])
3087    && INTVAL (operands[3]) == -INTVAL (operands[2])"
3088   [(const_int 0)]
3089   {
3090     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3091                                          operands[3], operands[2]));
3092     DONE;
3093   }
3094 )
3095
3096 ;; Same as the above peephole but with the compare and minus in
3097 ;; swapped order.  The restriction on overlap between operand 0
3098 ;; and operands 1 doesn't apply here.
3099 (define_peephole2
3100   [(set (reg:CC CC_REGNUM)
3101         (compare:CC
3102           (match_operand:GPI 1 "register_operand")
3103           (match_operand:GPI 3 "const_int_operand")))
3104    (set (match_operand:GPI 0 "aarch64_general_reg")
3105         (plus:GPI (match_dup 1)
3106                   (match_operand:GPI 2 "aarch64_plus_immediate")))]
3107   "INTVAL (operands[3]) == -INTVAL (operands[2])"
3108   [(const_int 0)]
3109   {
3110     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
3111                                          operands[3], operands[2]));
3112     DONE;
3113   }
3114 )
3115
3116 (define_insn "*sub_<shift>_<mode>"
3117   [(set (match_operand:GPI 0 "register_operand" "=r")
3118         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3119                    (ASHIFT:GPI
3120                     (match_operand:GPI 1 "register_operand" "r")
3121                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3122   ""
3123   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
3124   [(set_attr "type" "alu_shift_imm")]
3125 )
3126
3127 ;; zero_extend version of above
3128 (define_insn "*sub_<shift>_si_uxtw"
3129   [(set (match_operand:DI 0 "register_operand" "=r")
3130         (zero_extend:DI
3131          (minus:SI (match_operand:SI 3 "register_operand" "r")
3132                    (ASHIFT:SI
3133                     (match_operand:SI 1 "register_operand" "r")
3134                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3135   ""
3136   "sub\\t%w0, %w3, %w1, <shift> %2"
3137   [(set_attr "type" "alu_shift_imm")]
3138 )
3139
3140 (define_insn "*sub_mul_imm_<mode>"
3141   [(set (match_operand:GPI 0 "register_operand" "=r")
3142         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3143                    (mult:GPI
3144                     (match_operand:GPI 1 "register_operand" "r")
3145                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3146   ""
3147   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
3148   [(set_attr "type" "alu_shift_imm")]
3149 )
3150
3151 ;; zero_extend version of above
3152 (define_insn "*sub_mul_imm_si_uxtw"
3153   [(set (match_operand:DI 0 "register_operand" "=r")
3154         (zero_extend:DI
3155          (minus:SI (match_operand:SI 3 "register_operand" "r")
3156                    (mult:SI
3157                     (match_operand:SI 1 "register_operand" "r")
3158                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3159   ""
3160   "sub\\t%w0, %w3, %w1, lsl %p2"
3161   [(set_attr "type" "alu_shift_imm")]
3162 )
3163
3164 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
3165   [(set (match_operand:GPI 0 "register_operand" "=rk")
3166         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3167                    (ANY_EXTEND:GPI
3168                     (match_operand:ALLX 2 "register_operand" "r"))))]
3169   ""
3170   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
3171   [(set_attr "type" "alu_ext")]
3172 )
3173
3174 ;; zero_extend version of above
3175 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
3176   [(set (match_operand:DI 0 "register_operand" "=rk")
3177         (zero_extend:DI
3178          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3179                    (ANY_EXTEND:SI
3180                     (match_operand:SHORT 2 "register_operand" "r")))))]
3181   ""
3182   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
3183   [(set_attr "type" "alu_ext")]
3184 )
3185
3186 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
3187   [(set (match_operand:GPI 0 "register_operand" "=rk")
3188         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
3189                    (ashift:GPI (ANY_EXTEND:GPI
3190                                 (match_operand:ALLX 2 "register_operand" "r"))
3191                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
3192   ""
3193   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
3194   [(set_attr "type" "alu_ext")]
3195 )
3196
3197 ;; zero_extend version of above
3198 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
3199   [(set (match_operand:DI 0 "register_operand" "=rk")
3200         (zero_extend:DI
3201          (minus:SI (match_operand:SI 1 "register_operand" "rk")
3202                    (ashift:SI (ANY_EXTEND:SI
3203                                (match_operand:SHORT 2 "register_operand" "r"))
3204                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
3205   ""
3206   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
3207   [(set_attr "type" "alu_ext")]
3208 )
3209
3210 (define_insn "*sub_<optab><mode>_multp2"
3211   [(set (match_operand:GPI 0 "register_operand" "=rk")
3212         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3213                    (ANY_EXTRACT:GPI
3214                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3215                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3216                     (match_operand 3 "const_int_operand" "n")
3217                     (const_int 0))))]
3218   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
3219   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
3220   [(set_attr "type" "alu_ext")]
3221 )
3222
3223 ;; zero_extend version of above
3224 (define_insn "*sub_<optab>si_multp2_uxtw"
3225   [(set (match_operand:DI 0 "register_operand" "=rk")
3226         (zero_extend:DI
3227          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3228                    (ANY_EXTRACT:SI
3229                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3230                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3231                     (match_operand 3 "const_int_operand" "n")
3232                     (const_int 0)))))]
3233   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3234   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3235   [(set_attr "type" "alu_ext")]
3236 )
3237
3238 ;; The hardware description is op1 + ~op2 + C.
3239 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3240 ;;                           = op1 - op2 - 1 + 1 - !C
3241 ;;                           = op1 - op2 - !C.
3242 ;; We describe the latter.
3243
3244 (define_insn "*sub<mode>3_carryin0"
3245   [(set (match_operand:GPI 0 "register_operand" "=r")
3246         (minus:GPI
3247           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3248           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3249    ""
3250    "sbc\\t%<w>0, %<w>1, <w>zr"
3251   [(set_attr "type" "adc_reg")]
3252 )
3253
3254 ;; zero_extend version of the above
3255 (define_insn "*subsi3_carryin_uxtw"
3256   [(set (match_operand:DI 0 "register_operand" "=r")
3257         (zero_extend:DI
3258           (minus:SI
3259             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3260             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3261    ""
3262    "sbc\\t%w0, %w1, wzr"
3263   [(set_attr "type" "adc_reg")]
3264 )
3265
3266 (define_expand "sub<mode>3_carryin"
3267   [(set (match_operand:GPI 0 "register_operand")
3268         (minus:GPI
3269           (minus:GPI
3270             (match_operand:GPI 1 "aarch64_reg_or_zero")
3271             (match_operand:GPI 2 "register_operand"))
3272           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3273    ""
3274    ""
3275 )
3276
3277 (define_insn "*sub<mode>3_carryin"
3278   [(set (match_operand:GPI 0 "register_operand" "=r")
3279         (minus:GPI
3280           (minus:GPI
3281             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3282             (match_operand:GPI 2 "register_operand" "r"))
3283           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3284
3285    ""
3286    "sbc\\t%<w>0, %<w>1, %<w>2"
3287   [(set_attr "type" "adc_reg")]
3288 )
3289
3290 ;; zero_extend version of the above
3291 (define_insn "*subsi3_carryin_uxtw"
3292   [(set (match_operand:DI 0 "register_operand" "=r")
3293         (zero_extend:DI
3294           (minus:SI
3295             (minus:SI
3296               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3297               (match_operand:SI 2 "register_operand" "r"))
3298             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3299
3300    ""
3301    "sbc\\t%w0, %w1, %w2"
3302   [(set_attr "type" "adc_reg")]
3303 )
3304
3305 (define_insn "*sub<mode>3_carryin_alt"
3306   [(set (match_operand:GPI 0 "register_operand" "=r")
3307         (minus:GPI
3308           (minus:GPI
3309             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3310             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3311           (match_operand:GPI 2 "register_operand" "r")))]
3312    ""
3313    "sbc\\t%<w>0, %<w>1, %<w>2"
3314   [(set_attr "type" "adc_reg")]
3315 )
3316
3317 ;; zero_extend version of the above
3318 (define_insn "*subsi3_carryin_alt_uxtw"
3319   [(set (match_operand:DI 0 "register_operand" "=r")
3320         (zero_extend:DI
3321           (minus:SI
3322             (minus:SI
3323               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3324               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3325             (match_operand:SI 2 "register_operand" "r"))))]
3326    ""
3327    "sbc\\t%w0, %w1, %w2"
3328   [(set_attr "type" "adc_reg")]
3329 )
3330
3331 (define_expand "usub<GPI:mode>3_carryinC"
3332   [(parallel
3333      [(set (reg:CC CC_REGNUM)
3334            (compare:CC
3335              (zero_extend:<DWI>
3336                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3337              (plus:<DWI>
3338                (zero_extend:<DWI>
3339                  (match_operand:GPI 2 "register_operand"))
3340                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3341       (set (match_operand:GPI 0 "register_operand")
3342            (minus:GPI
3343              (minus:GPI (match_dup 1) (match_dup 2))
3344              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3345    ""
3346 )
3347
3348 (define_insn "*usub<GPI:mode>3_carryinC_z1"
3349   [(set (reg:CC CC_REGNUM)
3350         (compare:CC
3351           (const_int 0)
3352           (plus:<DWI>
3353             (zero_extend:<DWI>
3354               (match_operand:GPI 1 "register_operand" "r"))
3355             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3356    (set (match_operand:GPI 0 "register_operand" "=r")
3357         (minus:GPI
3358           (neg:GPI (match_dup 1))
3359           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3360    ""
3361    "sbcs\\t%<w>0, <w>zr, %<w>1"
3362   [(set_attr "type" "adc_reg")]
3363 )
3364
3365 (define_insn "*usub<GPI:mode>3_carryinC_z2"
3366   [(set (reg:CC CC_REGNUM)
3367         (compare:CC
3368           (zero_extend:<DWI>
3369             (match_operand:GPI 1 "register_operand" "r"))
3370           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3371    (set (match_operand:GPI 0 "register_operand" "=r")
3372         (minus:GPI
3373           (match_dup 1)
3374           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3375    ""
3376    "sbcs\\t%<w>0, %<w>1, <w>zr"
3377   [(set_attr "type" "adc_reg")]
3378 )
3379
3380 (define_insn "*usub<GPI:mode>3_carryinC"
3381   [(set (reg:CC CC_REGNUM)
3382         (compare:CC
3383           (zero_extend:<DWI>
3384             (match_operand:GPI 1 "register_operand" "r"))
3385           (plus:<DWI>
3386             (zero_extend:<DWI>
3387               (match_operand:GPI 2 "register_operand" "r"))
3388             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3389    (set (match_operand:GPI 0 "register_operand" "=r")
3390         (minus:GPI
3391           (minus:GPI (match_dup 1) (match_dup 2))
3392           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3393    ""
3394    "sbcs\\t%<w>0, %<w>1, %<w>2"
3395   [(set_attr "type" "adc_reg")]
3396 )
3397
3398 (define_expand "sub<GPI:mode>3_carryinV"
3399   [(parallel
3400      [(set (reg:CC_V CC_REGNUM)
3401            (compare:CC_V
3402             (minus:<DWI>
3403              (sign_extend:<DWI>
3404                (match_operand:GPI 1 "aarch64_reg_or_zero"))
3405              (plus:<DWI>
3406                (sign_extend:<DWI>
3407                  (match_operand:GPI 2 "register_operand"))
3408                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))
3409             (sign_extend:<DWI>
3410              (minus:GPI (match_dup 1)
3411                         (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))
3412                                   (match_dup 2))))))
3413       (set (match_operand:GPI 0 "register_operand")
3414            (minus:GPI
3415              (minus:GPI (match_dup 1) (match_dup 2))
3416              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3417    ""
3418 )
3419
3420 (define_insn "*sub<mode>3_carryinV_z2"
3421   [(set (reg:CC_V CC_REGNUM)
3422         (compare:CC_V
3423          (minus:<DWI>
3424           (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
3425           (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))
3426          (sign_extend:<DWI>
3427           (minus:GPI (match_dup 1)
3428                      (match_operand:GPI 3 "aarch64_borrow_operation" "")))))
3429    (set (match_operand:GPI 0 "register_operand" "=r")
3430         (minus:GPI
3431          (match_dup 1) (match_dup 3)))]
3432    ""
3433    "sbcs\\t%<w>0, %<w>1, <w>zr"
3434   [(set_attr "type" "adc_reg")]
3435 )
3436
3437 (define_insn "*sub<mode>3_carryinV"
3438   [(set (reg:CC_V CC_REGNUM)
3439         (compare:CC_V
3440          (minus:<DWI>
3441           (sign_extend:<DWI>
3442             (match_operand:GPI 1 "register_operand" "r"))
3443           (plus:<DWI>
3444             (sign_extend:<DWI>
3445               (match_operand:GPI 2 "register_operand" "r"))
3446             (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))
3447          (sign_extend:<DWI>
3448           (minus:GPI
3449            (match_dup 1)
3450            (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "")
3451                      (match_dup 2))))))
3452    (set (match_operand:GPI 0 "register_operand" "=r")
3453         (minus:GPI
3454           (minus:GPI (match_dup 1) (match_dup 2))
3455           (match_dup 4)))]
3456    ""
3457    "sbcs\\t%<w>0, %<w>1, %<w>2"
3458   [(set_attr "type" "adc_reg")]
3459 )
3460
3461 (define_insn "*sub_uxt<mode>_shift2"
3462   [(set (match_operand:GPI 0 "register_operand" "=rk")
3463         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3464                    (and:GPI
3465                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3466                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3467                     (match_operand 3 "const_int_operand" "n"))))]
3468   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3469   "*
3470   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3471                                            INTVAL (operands[3])));
3472   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3473   [(set_attr "type" "alu_ext")]
3474 )
3475
3476 ;; zero_extend version of above
3477 (define_insn "*sub_uxtsi_shift2_uxtw"
3478   [(set (match_operand:DI 0 "register_operand" "=rk")
3479         (zero_extend:DI
3480          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3481                    (and:SI
3482                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3483                                (match_operand 2 "aarch64_imm3" "Ui3"))
3484                     (match_operand 3 "const_int_operand" "n")))))]
3485   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3486   "*
3487   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3488                                            INTVAL (operands[3])));
3489   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3490   [(set_attr "type" "alu_ext")]
3491 )
3492
3493 (define_insn "*sub_uxt<mode>_multp2"
3494   [(set (match_operand:GPI 0 "register_operand" "=rk")
3495         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3496                    (and:GPI
3497                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3498                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3499                     (match_operand 3 "const_int_operand" "n"))))]
3500   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3501   "*
3502   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3503                                            INTVAL (operands[3])));
3504   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3505   [(set_attr "type" "alu_ext")]
3506 )
3507
3508 ;; zero_extend version of above
3509 (define_insn "*sub_uxtsi_multp2_uxtw"
3510   [(set (match_operand:DI 0 "register_operand" "=rk")
3511         (zero_extend:DI
3512          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3513                    (and:SI
3514                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3515                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3516                     (match_operand 3 "const_int_operand" "n")))))]
3517   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3518   "*
3519   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3520                                            INTVAL (operands[3])));
3521   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3522   [(set_attr "type" "alu_ext")]
3523 )
3524
3525 (define_expand "abs<mode>2"
3526   [(match_operand:GPI 0 "register_operand")
3527    (match_operand:GPI 1 "register_operand")]
3528   ""
3529   {
3530     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3531     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3532     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3533     DONE;
3534   }
3535 )
3536
3537 (define_insn "neg<mode>2"
3538   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3539         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3540   ""
3541   "@
3542    neg\\t%<w>0, %<w>1
3543    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3544   [(set_attr "type" "alu_sreg, neon_neg<q>")
3545    (set_attr "arch" "*,simd")]
3546 )
3547
3548 ;; zero_extend version of above
3549 (define_insn "*negsi2_uxtw"
3550   [(set (match_operand:DI 0 "register_operand" "=r")
3551         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3552   ""
3553   "neg\\t%w0, %w1"
3554   [(set_attr "type" "alu_sreg")]
3555 )
3556
3557 (define_insn "*ngc<mode>"
3558   [(set (match_operand:GPI 0 "register_operand" "=r")
3559         (minus:GPI
3560           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3561           (match_operand:GPI 1 "register_operand" "r")))]
3562   ""
3563   "ngc\\t%<w>0, %<w>1"
3564   [(set_attr "type" "adc_reg")]
3565 )
3566
3567 (define_insn "*ngcsi_uxtw"
3568   [(set (match_operand:DI 0 "register_operand" "=r")
3569         (zero_extend:DI
3570           (minus:SI
3571             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3572             (match_operand:SI 1 "register_operand" "r"))))]
3573   ""
3574   "ngc\\t%w0, %w1"
3575   [(set_attr "type" "adc_reg")]
3576 )
3577
3578 (define_insn "neg<mode>2_compare0"
3579   [(set (reg:CC_NZ CC_REGNUM)
3580         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3581                        (const_int 0)))
3582    (set (match_operand:GPI 0 "register_operand" "=r")
3583         (neg:GPI (match_dup 1)))]
3584   ""
3585   "negs\\t%<w>0, %<w>1"
3586   [(set_attr "type" "alus_sreg")]
3587 )
3588
3589 ;; zero_extend version of above
3590 (define_insn "*negsi2_compare0_uxtw"
3591   [(set (reg:CC_NZ CC_REGNUM)
3592         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3593                        (const_int 0)))
3594    (set (match_operand:DI 0 "register_operand" "=r")
3595         (zero_extend:DI (neg:SI (match_dup 1))))]
3596   ""
3597   "negs\\t%w0, %w1"
3598   [(set_attr "type" "alus_sreg")]
3599 )
3600
3601 (define_insn "*neg_<shift><mode>3_compare0"
3602   [(set (reg:CC_NZ CC_REGNUM)
3603         (compare:CC_NZ
3604          (neg:GPI (ASHIFT:GPI
3605                    (match_operand:GPI 1 "register_operand" "r")
3606                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3607          (const_int 0)))
3608    (set (match_operand:GPI 0 "register_operand" "=r")
3609         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3610   ""
3611   "negs\\t%<w>0, %<w>1, <shift> %2"
3612   [(set_attr "type" "alus_shift_imm")]
3613 )
3614
3615 (define_insn "*neg_<shift>_<mode>2"
3616   [(set (match_operand:GPI 0 "register_operand" "=r")
3617         (neg:GPI (ASHIFT:GPI
3618                   (match_operand:GPI 1 "register_operand" "r")
3619                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3620   ""
3621   "neg\\t%<w>0, %<w>1, <shift> %2"
3622   [(set_attr "type" "alu_shift_imm")]
3623 )
3624
3625 ;; zero_extend version of above
3626 (define_insn "*neg_<shift>_si2_uxtw"
3627   [(set (match_operand:DI 0 "register_operand" "=r")
3628         (zero_extend:DI
3629          (neg:SI (ASHIFT:SI
3630                   (match_operand:SI 1 "register_operand" "r")
3631                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3632   ""
3633   "neg\\t%w0, %w1, <shift> %2"
3634   [(set_attr "type" "alu_shift_imm")]
3635 )
3636
3637 (define_insn "*neg_mul_imm_<mode>2"
3638   [(set (match_operand:GPI 0 "register_operand" "=r")
3639         (neg:GPI (mult:GPI
3640                   (match_operand:GPI 1 "register_operand" "r")
3641                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3642   ""
3643   "neg\\t%<w>0, %<w>1, lsl %p2"
3644   [(set_attr "type" "alu_shift_imm")]
3645 )
3646
3647 ;; zero_extend version of above
3648 (define_insn "*neg_mul_imm_si2_uxtw"
3649   [(set (match_operand:DI 0 "register_operand" "=r")
3650         (zero_extend:DI
3651          (neg:SI (mult:SI
3652                   (match_operand:SI 1 "register_operand" "r")
3653                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3654   ""
3655   "neg\\t%w0, %w1, lsl %p2"
3656   [(set_attr "type" "alu_shift_imm")]
3657 )
3658
3659 (define_insn "mul<mode>3"
3660   [(set (match_operand:GPI 0 "register_operand" "=r")
3661         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3662                   (match_operand:GPI 2 "register_operand" "r")))]
3663   ""
3664   "mul\\t%<w>0, %<w>1, %<w>2"
3665   [(set_attr "type" "mul")]
3666 )
3667
3668 ;; zero_extend version of above
3669 (define_insn "*mulsi3_uxtw"
3670   [(set (match_operand:DI 0 "register_operand" "=r")
3671         (zero_extend:DI
3672          (mult:SI (match_operand:SI 1 "register_operand" "r")
3673                   (match_operand:SI 2 "register_operand" "r"))))]
3674   ""
3675   "mul\\t%w0, %w1, %w2"
3676   [(set_attr "type" "mul")]
3677 )
3678
3679 (define_insn "madd<mode>"
3680   [(set (match_operand:GPI 0 "register_operand" "=r")
3681         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3682                             (match_operand:GPI 2 "register_operand" "r"))
3683                   (match_operand:GPI 3 "register_operand" "r")))]
3684   ""
3685   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3686   [(set_attr "type" "mla")]
3687 )
3688
3689 ;; zero_extend version of above
3690 (define_insn "*maddsi_uxtw"
3691   [(set (match_operand:DI 0 "register_operand" "=r")
3692         (zero_extend:DI
3693          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3694                            (match_operand:SI 2 "register_operand" "r"))
3695                   (match_operand:SI 3 "register_operand" "r"))))]
3696   ""
3697   "madd\\t%w0, %w1, %w2, %w3"
3698   [(set_attr "type" "mla")]
3699 )
3700
3701 (define_insn "*msub<mode>"
3702   [(set (match_operand:GPI 0 "register_operand" "=r")
3703         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3704                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3705                              (match_operand:GPI 2 "register_operand" "r"))))]
3706
3707   ""
3708   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3709   [(set_attr "type" "mla")]
3710 )
3711
3712 ;; zero_extend version of above
3713 (define_insn "*msubsi_uxtw"
3714   [(set (match_operand:DI 0 "register_operand" "=r")
3715         (zero_extend:DI
3716          (minus:SI (match_operand:SI 3 "register_operand" "r")
3717                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3718                             (match_operand:SI 2 "register_operand" "r")))))]
3719
3720   ""
3721   "msub\\t%w0, %w1, %w2, %w3"
3722   [(set_attr "type" "mla")]
3723 )
3724
3725 (define_insn "*mul<mode>_neg"
3726   [(set (match_operand:GPI 0 "register_operand" "=r")
3727         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3728                   (match_operand:GPI 2 "register_operand" "r")))]
3729
3730   ""
3731   "mneg\\t%<w>0, %<w>1, %<w>2"
3732   [(set_attr "type" "mul")]
3733 )
3734
3735 ;; zero_extend version of above
3736 (define_insn "*mulsi_neg_uxtw"
3737   [(set (match_operand:DI 0 "register_operand" "=r")
3738         (zero_extend:DI
3739          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3740                   (match_operand:SI 2 "register_operand" "r"))))]
3741
3742   ""
3743   "mneg\\t%w0, %w1, %w2"
3744   [(set_attr "type" "mul")]
3745 )
3746
3747 (define_insn "<su_optab>mulsidi3"
3748   [(set (match_operand:DI 0 "register_operand" "=r")
3749         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3750                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3751   ""
3752   "<su>mull\\t%0, %w1, %w2"
3753   [(set_attr "type" "<su>mull")]
3754 )
3755
3756 (define_insn "<su_optab>maddsidi4"
3757   [(set (match_operand:DI 0 "register_operand" "=r")
3758         (plus:DI (mult:DI
3759                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3760                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3761                  (match_operand:DI 3 "register_operand" "r")))]
3762   ""
3763   "<su>maddl\\t%0, %w1, %w2, %3"
3764   [(set_attr "type" "<su>mlal")]
3765 )
3766
3767 (define_insn "<su_optab>msubsidi4"
3768   [(set (match_operand:DI 0 "register_operand" "=r")
3769         (minus:DI
3770          (match_operand:DI 3 "register_operand" "r")
3771          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3772                   (ANY_EXTEND:DI
3773                    (match_operand:SI 2 "register_operand" "r")))))]
3774   ""
3775   "<su>msubl\\t%0, %w1, %w2, %3"
3776   [(set_attr "type" "<su>mlal")]
3777 )
3778
3779 (define_insn "*<su_optab>mulsidi_neg"
3780   [(set (match_operand:DI 0 "register_operand" "=r")
3781         (mult:DI (neg:DI
3782                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3783                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3784   ""
3785   "<su>mnegl\\t%0, %w1, %w2"
3786   [(set_attr "type" "<su>mull")]
3787 )
3788
3789 (define_expand "<su_optab>mulditi3"
3790   [(set (match_operand:TI 0 "register_operand")
3791         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3792                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3793   ""
3794 {
3795   rtx low = gen_reg_rtx (DImode);
3796   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3797
3798   rtx high = gen_reg_rtx (DImode);
3799   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3800
3801   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3802   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3803   DONE;
3804 })
3805
3806 ;; The default expansion of multi3 using umuldi3_highpart will perform
3807 ;; the additions in an order that fails to combine into two madd insns.
3808 (define_expand "multi3"
3809   [(set (match_operand:TI 0 "register_operand")
3810         (mult:TI (match_operand:TI 1 "register_operand")
3811                  (match_operand:TI 2 "register_operand")))]
3812   ""
3813 {
3814   rtx l0 = gen_reg_rtx (DImode);
3815   rtx l1 = gen_lowpart (DImode, operands[1]);
3816   rtx l2 = gen_lowpart (DImode, operands[2]);
3817   rtx h0 = gen_reg_rtx (DImode);
3818   rtx h1 = gen_highpart (DImode, operands[1]);
3819   rtx h2 = gen_highpart (DImode, operands[2]);
3820
3821   emit_insn (gen_muldi3 (l0, l1, l2));
3822   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3823   emit_insn (gen_madddi (h0, h1, l2, h0));
3824   emit_insn (gen_madddi (h0, l1, h2, h0));
3825
3826   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3827   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3828   DONE;
3829 })
3830
3831 (define_insn "<su>muldi3_highpart"
3832   [(set (match_operand:DI 0 "register_operand" "=r")
3833         (truncate:DI
3834          (lshiftrt:TI
3835           (mult:TI
3836            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3837            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3838           (const_int 64))))]
3839   ""
3840   "<su>mulh\\t%0, %1, %2"
3841   [(set_attr "type" "<su>mull")]
3842 )
3843
3844 (define_insn "<su_optab>div<mode>3"
3845   [(set (match_operand:GPI 0 "register_operand" "=r")
3846         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3847                      (match_operand:GPI 2 "register_operand" "r")))]
3848   ""
3849   "<su>div\\t%<w>0, %<w>1, %<w>2"
3850   [(set_attr "type" "<su>div")]
3851 )
3852
3853 ;; zero_extend version of above
3854 (define_insn "*<su_optab>divsi3_uxtw"
3855   [(set (match_operand:DI 0 "register_operand" "=r")
3856         (zero_extend:DI
3857          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3858                      (match_operand:SI 2 "register_operand" "r"))))]
3859   ""
3860   "<su>div\\t%w0, %w1, %w2"
3861   [(set_attr "type" "<su>div")]
3862 )
3863
3864 ;; -------------------------------------------------------------------
3865 ;; Comparison insns
3866 ;; -------------------------------------------------------------------
3867
3868 (define_insn "cmp<mode>"
3869   [(set (reg:CC CC_REGNUM)
3870         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3871                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3872   ""
3873   "@
3874    cmp\\t%<w>0, %<w>1
3875    cmp\\t%<w>0, %1
3876    cmn\\t%<w>0, #%n1"
3877   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3878 )
3879
3880 (define_insn "fcmp<mode>"
3881   [(set (reg:CCFP CC_REGNUM)
3882         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3883                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3884    "TARGET_FLOAT"
3885    "@
3886     fcmp\\t%<s>0, #0.0
3887     fcmp\\t%<s>0, %<s>1"
3888   [(set_attr "type" "fcmp<s>")]
3889 )
3890
3891 (define_insn "fcmpe<mode>"
3892   [(set (reg:CCFPE CC_REGNUM)
3893         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3894                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3895    "TARGET_FLOAT"
3896    "@
3897     fcmpe\\t%<s>0, #0.0
3898     fcmpe\\t%<s>0, %<s>1"
3899   [(set_attr "type" "fcmp<s>")]
3900 )
3901
3902 (define_insn "*cmp_swp_<shift>_reg<mode>"
3903   [(set (reg:CC_SWP CC_REGNUM)
3904         (compare:CC_SWP (ASHIFT:GPI
3905                          (match_operand:GPI 0 "register_operand" "r")
3906                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3907                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3908   ""
3909   "cmp\\t%<w>2, %<w>0, <shift> %1"
3910   [(set_attr "type" "alus_shift_imm")]
3911 )
3912
3913 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3914   [(set (reg:CC_SWP CC_REGNUM)
3915         (compare:CC_SWP (ANY_EXTEND:GPI
3916                          (match_operand:ALLX 0 "register_operand" "r"))
3917                         (match_operand:GPI 1 "register_operand" "r")))]
3918   ""
3919   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3920   [(set_attr "type" "alus_ext")]
3921 )
3922
3923 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3924   [(set (reg:CC_SWP CC_REGNUM)
3925         (compare:CC_SWP (ashift:GPI
3926                          (ANY_EXTEND:GPI
3927                           (match_operand:ALLX 0 "register_operand" "r"))
3928                          (match_operand 1 "aarch64_imm3" "Ui3"))
3929         (match_operand:GPI 2 "register_operand" "r")))]
3930   ""
3931   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3932   [(set_attr "type" "alus_ext")]
3933 )
3934
3935 ;; -------------------------------------------------------------------
3936 ;; Store-flag and conditional select insns
3937 ;; -------------------------------------------------------------------
3938
3939 (define_expand "cstore<mode>4"
3940   [(set (match_operand:SI 0 "register_operand")
3941         (match_operator:SI 1 "aarch64_comparison_operator"
3942          [(match_operand:GPI 2 "register_operand")
3943           (match_operand:GPI 3 "aarch64_plus_operand")]))]
3944   ""
3945   "
3946   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3947                                       operands[3]);
3948   operands[3] = const0_rtx;
3949   "
3950 )
3951
3952 (define_expand "cstorecc4"
3953   [(set (match_operand:SI 0 "register_operand")
3954        (match_operator 1 "aarch64_comparison_operator_mode"
3955         [(match_operand 2 "cc_register")
3956          (match_operand 3 "const0_operand")]))]
3957   ""
3958 "{
3959   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3960   DONE;
3961 }")
3962
3963
3964 (define_expand "cstore<mode>4"
3965   [(set (match_operand:SI 0 "register_operand")
3966         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3967          [(match_operand:GPF 2 "register_operand")
3968           (match_operand:GPF 3 "aarch64_fp_compare_operand")]))]
3969   ""
3970   "
3971   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3972                                       operands[3]);
3973   operands[3] = const0_rtx;
3974   "
3975 )
3976
3977 (define_insn "aarch64_cstore<mode>"
3978   [(set (match_operand:ALLI 0 "register_operand" "=r")
3979         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3980          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3981   ""
3982   "cset\\t%<w>0, %m1"
3983   [(set_attr "type" "csel")]
3984 )
3985
3986 ;; For a 24-bit immediate CST we can optimize the compare for equality
3987 ;; and branch sequence from:
3988 ;;      mov     x0, #imm1
3989 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3990 ;;      cmp     x1, x0
3991 ;;      cset    x2, <ne,eq>
3992 ;; into the shorter:
3993 ;;      sub     x0, x1, #(CST & 0xfff000)
3994 ;;      subs    x0, x0, #(CST & 0x000fff)
3995 ;;      cset x2, <ne, eq>.
3996 (define_insn_and_split "*compare_cstore<mode>_insn"
3997   [(set (match_operand:GPI 0 "register_operand" "=r")
3998          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3999                   (match_operand:GPI 2 "aarch64_imm24" "n")))
4000    (clobber (reg:CC CC_REGNUM))]
4001   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
4002    && !aarch64_plus_operand (operands[2], <MODE>mode)
4003    && !reload_completed"
4004   "#"
4005   "&& true"
4006   [(const_int 0)]
4007   {
4008     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
4009     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
4010     rtx tmp = gen_reg_rtx (<MODE>mode);
4011     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
4012     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
4013     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
4014     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
4015     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
4016     DONE;
4017   }
4018   [(set_attr "type" "csel")]
4019 )
4020
4021 ;; zero_extend version of the above
4022 (define_insn "*cstoresi_insn_uxtw"
4023   [(set (match_operand:DI 0 "register_operand" "=r")
4024         (zero_extend:DI
4025          (match_operator:SI 1 "aarch64_comparison_operator_mode"
4026           [(match_operand 2 "cc_register" "") (const_int 0)])))]
4027   ""
4028   "cset\\t%w0, %m1"
4029   [(set_attr "type" "csel")]
4030 )
4031
4032 (define_insn "cstore<mode>_neg"
4033   [(set (match_operand:ALLI 0 "register_operand" "=r")
4034         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
4035                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
4036   ""
4037   "csetm\\t%<w>0, %m1"
4038   [(set_attr "type" "csel")]
4039 )
4040
4041 ;; zero_extend version of the above
4042 (define_insn "*cstoresi_neg_uxtw"
4043   [(set (match_operand:DI 0 "register_operand" "=r")
4044         (zero_extend:DI
4045          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
4046                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
4047   ""
4048   "csetm\\t%w0, %m1"
4049   [(set_attr "type" "csel")]
4050 )
4051
4052 (define_expand "cmov<mode>6"
4053   [(set (match_operand:GPI 0 "register_operand")
4054         (if_then_else:GPI
4055          (match_operator 1 "aarch64_comparison_operator"
4056           [(match_operand:GPI 2 "register_operand")
4057            (match_operand:GPI 3 "aarch64_plus_operand")])
4058          (match_operand:GPI 4 "register_operand")
4059          (match_operand:GPI 5 "register_operand")))]
4060   ""
4061   "
4062   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4063                                       operands[3]);
4064   operands[3] = const0_rtx;
4065   "
4066 )
4067
4068 (define_expand "cmov<mode>6"
4069   [(set (match_operand:GPF 0 "register_operand")
4070         (if_then_else:GPF
4071          (match_operator 1 "aarch64_comparison_operator"
4072           [(match_operand:GPF 2 "register_operand")
4073            (match_operand:GPF 3 "aarch64_fp_compare_operand")])
4074          (match_operand:GPF 4 "register_operand")
4075          (match_operand:GPF 5 "register_operand")))]
4076   ""
4077   "
4078   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
4079                                       operands[3]);
4080   operands[3] = const0_rtx;
4081   "
4082 )
4083
4084 (define_insn "*cmov<mode>_insn"
4085   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
4086         (if_then_else:ALLI
4087          (match_operator 1 "aarch64_comparison_operator"
4088           [(match_operand 2 "cc_register" "") (const_int 0)])
4089          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4090          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
4091   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4092      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4093   ;; Final two alternatives should be unreachable, but included for completeness
4094   "@
4095    csel\\t%<w>0, %<w>3, %<w>4, %m1
4096    csinv\\t%<w>0, %<w>3, <w>zr, %m1
4097    csinv\\t%<w>0, %<w>4, <w>zr, %M1
4098    csinc\\t%<w>0, %<w>3, <w>zr, %m1
4099    csinc\\t%<w>0, %<w>4, <w>zr, %M1
4100    mov\\t%<w>0, -1
4101    mov\\t%<w>0, 1"
4102   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4103 )
4104
4105 ;; zero_extend version of above
4106 (define_insn "*cmovsi_insn_uxtw"
4107   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
4108         (zero_extend:DI
4109          (if_then_else:SI
4110           (match_operator 1 "aarch64_comparison_operator"
4111            [(match_operand 2 "cc_register" "") (const_int 0)])
4112           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
4113           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
4114   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
4115      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
4116   ;; Final two alternatives should be unreachable, but included for completeness
4117   "@
4118    csel\\t%w0, %w3, %w4, %m1
4119    csinv\\t%w0, %w3, wzr, %m1
4120    csinv\\t%w0, %w4, wzr, %M1
4121    csinc\\t%w0, %w3, wzr, %m1
4122    csinc\\t%w0, %w4, wzr, %M1
4123    mov\\t%w0, -1
4124    mov\\t%w0, 1"
4125   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
4126 )
4127
4128 (define_insn "*cmovdi_insn_uxtw"
4129   [(set (match_operand:DI 0 "register_operand" "=r")
4130         (if_then_else:DI
4131          (match_operator 1 "aarch64_comparison_operator"
4132           [(match_operand 2 "cc_register" "") (const_int 0)])
4133          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
4134          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
4135   ""
4136   "csel\\t%w0, %w3, %w4, %m1"
4137   [(set_attr "type" "csel")]
4138 )
4139
4140 (define_insn "*cmov<mode>_insn"
4141   [(set (match_operand:GPF 0 "register_operand" "=w")
4142         (if_then_else:GPF
4143          (match_operator 1 "aarch64_comparison_operator"
4144           [(match_operand 2 "cc_register" "") (const_int 0)])
4145          (match_operand:GPF 3 "register_operand" "w")
4146          (match_operand:GPF 4 "register_operand" "w")))]
4147   "TARGET_FLOAT"
4148   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
4149   [(set_attr "type" "fcsel")]
4150 )
4151
4152 (define_expand "mov<mode>cc"
4153   [(set (match_operand:ALLI 0 "register_operand")
4154         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator")
4155                            (match_operand:ALLI 2 "register_operand")
4156                            (match_operand:ALLI 3 "register_operand")))]
4157   ""
4158   {
4159     rtx ccreg;
4160     enum rtx_code code = GET_CODE (operands[1]);
4161
4162     if (code == UNEQ || code == LTGT)
4163       FAIL;
4164
4165     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4166                                      XEXP (operands[1], 1));
4167     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4168   }
4169 )
4170
4171 (define_expand "mov<GPF:mode><GPI:mode>cc"
4172   [(set (match_operand:GPI 0 "register_operand")
4173         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4174                           (match_operand:GPF 2 "register_operand")
4175                           (match_operand:GPF 3 "register_operand")))]
4176   ""
4177   {
4178     rtx ccreg;
4179     enum rtx_code code = GET_CODE (operands[1]);
4180
4181     if (code == UNEQ || code == LTGT)
4182       FAIL;
4183
4184     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4185                                   XEXP (operands[1], 1));
4186     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4187   }
4188 )
4189
4190 (define_expand "mov<mode>cc"
4191   [(set (match_operand:GPF 0 "register_operand")
4192         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator")
4193                           (match_operand:GPF 2 "register_operand")
4194                           (match_operand:GPF 3 "register_operand")))]
4195   ""
4196   {
4197     rtx ccreg;
4198     enum rtx_code code = GET_CODE (operands[1]);
4199
4200     if (code == UNEQ || code == LTGT)
4201       FAIL;
4202
4203     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4204                                   XEXP (operands[1], 1));
4205     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4206   }
4207 )
4208
4209 (define_expand "<neg_not_op><mode>cc"
4210   [(set (match_operand:GPI 0 "register_operand")
4211         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator")
4212                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand"))
4213                           (match_operand:GPI 3 "register_operand")))]
4214   ""
4215   {
4216     rtx ccreg;
4217     enum rtx_code code = GET_CODE (operands[1]);
4218
4219     if (code == UNEQ || code == LTGT)
4220       FAIL;
4221
4222     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
4223                                       XEXP (operands[1], 1));
4224     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
4225   }
4226 )
4227
4228 ;; CRC32 instructions.
4229 (define_insn "aarch64_<crc_variant>"
4230   [(set (match_operand:SI 0 "register_operand" "=r")
4231         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
4232                     (match_operand:<crc_mode> 2 "register_operand" "r")]
4233          CRC))]
4234   "TARGET_CRC32"
4235   {
4236     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
4237       return "<crc_variant>\\t%w0, %w1, %x2";
4238     else
4239       return "<crc_variant>\\t%w0, %w1, %w2";
4240   }
4241   [(set_attr "type" "crc")]
4242 )
4243
4244 (define_insn "*csinc2<mode>_insn"
4245   [(set (match_operand:GPI 0 "register_operand" "=r")
4246         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
4247                   (match_operand:GPI 1 "register_operand" "r")))]
4248   ""
4249   "cinc\\t%<w>0, %<w>1, %m2"
4250   [(set_attr "type" "csel")]
4251 )
4252
4253 (define_insn "csinc3<mode>_insn"
4254   [(set (match_operand:GPI 0 "register_operand" "=r")
4255         (if_then_else:GPI
4256           (match_operand 1 "aarch64_comparison_operation" "")
4257           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
4258                     (const_int 1))
4259           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4260   ""
4261   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
4262   [(set_attr "type" "csel")]
4263 )
4264
4265 (define_insn "*csinv3<mode>_insn"
4266   [(set (match_operand:GPI 0 "register_operand" "=r")
4267         (if_then_else:GPI
4268           (match_operand 1 "aarch64_comparison_operation" "")
4269           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
4270           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4271   ""
4272   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
4273   [(set_attr "type" "csel")]
4274 )
4275
4276 (define_insn "csneg3_uxtw_insn"
4277   [(set (match_operand:DI 0 "register_operand" "=r")
4278         (zero_extend:DI
4279           (if_then_else:SI
4280             (match_operand 1 "aarch64_comparison_operation" "")
4281             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4282             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4283   ""
4284   "csneg\\t%w0, %w3, %w2, %M1"
4285   [(set_attr "type" "csel")]
4286 )
4287
4288 (define_insn "csneg3<mode>_insn"
4289   [(set (match_operand:GPI 0 "register_operand" "=r")
4290         (if_then_else:GPI
4291           (match_operand 1 "aarch64_comparison_operation" "")
4292           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4293           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4294   ""
4295   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4296   [(set_attr "type" "csel")]
4297 )
4298
4299 ;; If X can be loaded by a single CNT[BHWD] instruction,
4300 ;;
4301 ;;    A = UMAX (B, X)
4302 ;;
4303 ;; is equivalent to:
4304 ;;
4305 ;;    TMP = UQDEC[BHWD] (B, X)
4306 ;;    A = TMP + X
4307 ;;
4308 ;; Defining the pattern this way means that:
4309 ;;
4310 ;;    A = UMAX (B, X) - X
4311 ;;
4312 ;; becomes:
4313 ;;
4314 ;;    TMP1 = UQDEC[BHWD] (B, X)
4315 ;;    TMP2 = TMP1 + X
4316 ;;    A = TMP2 - X
4317 ;;
4318 ;; which combine can optimize to:
4319 ;;
4320 ;;    A = UQDEC[BHWD] (B, X)
4321 ;;
4322 ;; We don't use match_operand predicates because the order of the operands
4323 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4324 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4325 (define_expand "umax<mode>3"
4326   [(set (match_operand:GPI 0 "register_operand")
4327         (umax:GPI (match_operand:GPI 1 "")
4328                   (match_operand:GPI 2 "")))]
4329   "TARGET_SVE"
4330   {
4331     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4332       std::swap (operands[1], operands[2]);
4333     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4334       FAIL;
4335     rtx temp = gen_reg_rtx (<MODE>mode);
4336     operands[1] = force_reg (<MODE>mode, operands[1]);
4337     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4338     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4339     DONE;
4340   }
4341 )
4342
4343 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4344 (define_insn "aarch64_uqdec<mode>"
4345   [(set (match_operand:GPI 0 "register_operand" "=r")
4346         (minus:GPI
4347          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4348                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4349          (match_dup 2)))]
4350   "TARGET_SVE"
4351   {
4352     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4353   }
4354 )
4355
4356 ;; -------------------------------------------------------------------
4357 ;; Logical operations
4358 ;; -------------------------------------------------------------------
4359
4360
4361 (define_insn_and_split "*aarch64_and<mode>_imm2"
4362   [(set (match_operand:GPI 0 "register_operand" "=rk")
4363         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4364                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4365   ""
4366   "#"
4367   "true"
4368   [(const_int 0)]
4369   {
4370      HOST_WIDE_INT val = INTVAL (operands[2]);
4371      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4372      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4373
4374      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4375      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4376      DONE;
4377   }
4378 )
4379
4380 (define_insn "<optab><mode>3"
4381   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4382         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4383                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4384   ""
4385   "@
4386   <logical>\\t%<w>0, %<w>1, %<w>2
4387   <logical>\\t%<w>0, %<w>1, %2
4388   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4389   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4390    (set_attr "arch" "*,*,simd")]
4391 )
4392
4393 ;; zero_extend version of above
4394 (define_insn "*<optab>si3_uxtw"
4395   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4396         (zero_extend:DI
4397          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4398                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4399   ""
4400   "@
4401    <logical>\\t%w0, %w1, %w2
4402    <logical>\\t%w0, %w1, %2"
4403   [(set_attr "type" "logic_reg,logic_imm")]
4404 )
4405
4406 (define_insn "*and<mode>3_compare0"
4407   [(set (reg:CC_NZ CC_REGNUM)
4408         (compare:CC_NZ
4409          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4410                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4411          (const_int 0)))
4412    (set (match_operand:GPI 0 "register_operand" "=r,r")
4413         (and:GPI (match_dup 1) (match_dup 2)))]
4414   ""
4415   "@
4416    ands\\t%<w>0, %<w>1, %<w>2
4417    ands\\t%<w>0, %<w>1, %2"
4418   [(set_attr "type" "logics_reg,logics_imm")]
4419 )
4420
4421 ;; zero_extend version of above
4422 (define_insn "*andsi3_compare0_uxtw"
4423   [(set (reg:CC_NZ CC_REGNUM)
4424         (compare:CC_NZ
4425          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4426                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4427          (const_int 0)))
4428    (set (match_operand:DI 0 "register_operand" "=r,r")
4429         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4430   ""
4431   "@
4432    ands\\t%w0, %w1, %w2
4433    ands\\t%w0, %w1, %2"
4434   [(set_attr "type" "logics_reg,logics_imm")]
4435 )
4436
4437 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4438   [(set (reg:CC_NZ CC_REGNUM)
4439         (compare:CC_NZ
4440          (and:GPI (SHIFT:GPI
4441                    (match_operand:GPI 1 "register_operand" "r")
4442                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4443                   (match_operand:GPI 3 "register_operand" "r"))
4444          (const_int 0)))
4445    (set (match_operand:GPI 0 "register_operand" "=r")
4446         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4447   ""
4448   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4449   [(set_attr "type" "logics_shift_imm")]
4450 )
4451
4452 ;; zero_extend version of above
4453 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4454   [(set (reg:CC_NZ CC_REGNUM)
4455         (compare:CC_NZ
4456          (and:SI (SHIFT:SI
4457                   (match_operand:SI 1 "register_operand" "r")
4458                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4459                  (match_operand:SI 3 "register_operand" "r"))
4460          (const_int 0)))
4461    (set (match_operand:DI 0 "register_operand" "=r")
4462         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4463                                 (match_dup 3))))]
4464   ""
4465   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4466   [(set_attr "type" "logics_shift_imm")]
4467 )
4468
4469 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4470   [(set (match_operand:GPI 0 "register_operand" "=r")
4471         (LOGICAL:GPI (SHIFT:GPI
4472                       (match_operand:GPI 1 "register_operand" "r")
4473                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4474                      (match_operand:GPI 3 "register_operand" "r")))]
4475   ""
4476   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4477   [(set_attr "type" "logic_shift_imm")]
4478 )
4479
4480 (define_insn "*<optab>_rol<mode>3"
4481   [(set (match_operand:GPI 0 "register_operand" "=r")
4482         (LOGICAL:GPI (rotate:GPI
4483                       (match_operand:GPI 1 "register_operand" "r")
4484                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4485                      (match_operand:GPI 3 "register_operand" "r")))]
4486   ""
4487   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4488   [(set_attr "type" "logic_shift_imm")]
4489 )
4490
4491 ;; zero_extend versions of above
4492 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4493   [(set (match_operand:DI 0 "register_operand" "=r")
4494         (zero_extend:DI
4495          (LOGICAL:SI (SHIFT:SI
4496                       (match_operand:SI 1 "register_operand" "r")
4497                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4498                      (match_operand:SI 3 "register_operand" "r"))))]
4499   ""
4500   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4501   [(set_attr "type" "logic_shift_imm")]
4502 )
4503
4504 (define_insn "*<optab>_rolsi3_uxtw"
4505   [(set (match_operand:DI 0 "register_operand" "=r")
4506         (zero_extend:DI
4507          (LOGICAL:SI (rotate:SI
4508                       (match_operand:SI 1 "register_operand" "r")
4509                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4510                      (match_operand:SI 3 "register_operand" "r"))))]
4511   ""
4512   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4513   [(set_attr "type" "logic_shift_imm")]
4514 )
4515
4516 (define_insn "one_cmpl<mode>2"
4517   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4518         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4519   ""
4520   "@
4521   mvn\\t%<w>0, %<w>1
4522   mvn\\t%0.8b, %1.8b"
4523   [(set_attr "type" "logic_reg,neon_logic")
4524    (set_attr "arch" "*,simd")]
4525 )
4526
4527 (define_insn "*one_cmpl_<optab><mode>2"
4528   [(set (match_operand:GPI 0 "register_operand" "=r")
4529         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4530                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4531   ""
4532   "mvn\\t%<w>0, %<w>1, <shift> %2"
4533   [(set_attr "type" "logic_shift_imm")]
4534 )
4535
4536 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4537
4538 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4539   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4540         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4541                      (match_operand:GPI 2 "register_operand" "r,w")))]
4542   ""
4543   "@
4544   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4545   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4546   [(set_attr "type" "logic_reg,neon_logic")
4547    (set_attr "arch" "*,simd")]
4548 )
4549
4550 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4551   [(set (match_operand:DI 0 "register_operand" "=r")
4552         (zero_extend:DI
4553           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4554                        (match_operand:SI 2 "register_operand" "r"))))]
4555   ""
4556   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4557   [(set_attr "type" "logic_reg")]
4558 )
4559
4560 (define_insn "*xor_one_cmplsidi3_ze"
4561   [(set (match_operand:DI 0 "register_operand" "=r")
4562         (zero_extend:DI
4563           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4564                           (match_operand:SI 2 "register_operand" "r")))))]
4565   ""
4566   "eon\\t%w0, %w1, %w2"
4567   [(set_attr "type" "logic_reg")]
4568 )
4569
4570 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4571 ;; eon does not operate on SIMD registers so the vector variant must be split.
4572 (define_insn_and_split "*xor_one_cmpl<mode>3"
4573   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4574         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4575                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4576   ""
4577   "@
4578   eon\\t%<w>0, %<w>1, %<w>2
4579   #"
4580   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4581   [(set (match_operand:GPI 0 "register_operand" "=w")
4582         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4583                  (match_operand:GPI 2 "register_operand" "w")))
4584    (set (match_dup 0) (not:GPI (match_dup 0)))]
4585   ""
4586   [(set_attr "type" "logic_reg,multiple")
4587    (set_attr "arch" "*,simd")]
4588 )
4589
4590 (define_insn "*and_one_cmpl<mode>3_compare0"
4591   [(set (reg:CC_NZ CC_REGNUM)
4592         (compare:CC_NZ
4593          (and:GPI (not:GPI
4594                    (match_operand:GPI 1 "register_operand" "r"))
4595                   (match_operand:GPI 2 "register_operand" "r"))
4596          (const_int 0)))
4597    (set (match_operand:GPI 0 "register_operand" "=r")
4598         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4599   ""
4600   "bics\\t%<w>0, %<w>2, %<w>1"
4601   [(set_attr "type" "logics_reg")]
4602 )
4603
4604 ;; zero_extend version of above
4605 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4606   [(set (reg:CC_NZ CC_REGNUM)
4607         (compare:CC_NZ
4608          (and:SI (not:SI
4609                   (match_operand:SI 1 "register_operand" "r"))
4610                  (match_operand:SI 2 "register_operand" "r"))
4611          (const_int 0)))
4612    (set (match_operand:DI 0 "register_operand" "=r")
4613         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4614   ""
4615   "bics\\t%w0, %w2, %w1"
4616   [(set_attr "type" "logics_reg")]
4617 )
4618
4619 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4620   [(set (reg:CC_NZ CC_REGNUM)
4621     (compare:CC_NZ
4622      (and:GPI (not:GPI
4623            (match_operand:GPI 0 "register_operand" "r"))
4624           (match_operand:GPI 1 "register_operand" "r"))
4625      (const_int 0)))]
4626   ""
4627   "bics\\t<w>zr, %<w>1, %<w>0"
4628   [(set_attr "type" "logics_reg")]
4629 )
4630
4631 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4632   [(set (match_operand:GPI 0 "register_operand" "=r")
4633         (LOGICAL:GPI (not:GPI
4634                       (SHIFT:GPI
4635                        (match_operand:GPI 1 "register_operand" "r")
4636                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4637                      (match_operand:GPI 3 "register_operand" "r")))]
4638   ""
4639   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4640   [(set_attr "type" "logic_shift_imm")]
4641 )
4642
4643 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4644   [(set (match_operand:GPI 0 "register_operand" "=r")
4645         (not:GPI (xor:GPI
4646                       (SHIFT:GPI
4647                        (match_operand:GPI 1 "register_operand" "r")
4648                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4649                      (match_operand:GPI 3 "register_operand" "r"))))]
4650   ""
4651   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4652   [(set_attr "type" "logic_shift_imm")]
4653 )
4654
4655 ;; Zero-extend version of the above.
4656 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4657   [(set (match_operand:DI 0 "register_operand" "=r")
4658         (zero_extend:DI
4659           (not:SI (xor:SI
4660                     (SHIFT:SI
4661                       (match_operand:SI 1 "register_operand" "r")
4662                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4663                     (match_operand:SI 3 "register_operand" "r")))))]
4664   ""
4665   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4666   [(set_attr "type" "logic_shift_imm")]
4667 )
4668
4669 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4670   [(set (reg:CC_NZ CC_REGNUM)
4671         (compare:CC_NZ
4672          (and:GPI (not:GPI
4673                    (SHIFT:GPI
4674                     (match_operand:GPI 1 "register_operand" "r")
4675                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4676                   (match_operand:GPI 3 "register_operand" "r"))
4677          (const_int 0)))
4678    (set (match_operand:GPI 0 "register_operand" "=r")
4679         (and:GPI (not:GPI
4680                   (SHIFT:GPI
4681                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4682   ""
4683   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4684   [(set_attr "type" "logics_shift_imm")]
4685 )
4686
4687 ;; zero_extend version of above
4688 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4689   [(set (reg:CC_NZ CC_REGNUM)
4690         (compare:CC_NZ
4691          (and:SI (not:SI
4692                   (SHIFT:SI
4693                    (match_operand:SI 1 "register_operand" "r")
4694                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4695                  (match_operand:SI 3 "register_operand" "r"))
4696          (const_int 0)))
4697    (set (match_operand:DI 0 "register_operand" "=r")
4698         (zero_extend:DI (and:SI
4699                          (not:SI
4700                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4701   ""
4702   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4703   [(set_attr "type" "logics_shift_imm")]
4704 )
4705
4706 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4707   [(set (reg:CC_NZ CC_REGNUM)
4708     (compare:CC_NZ
4709      (and:GPI (not:GPI
4710            (SHIFT:GPI
4711             (match_operand:GPI 0 "register_operand" "r")
4712             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4713           (match_operand:GPI 2 "register_operand" "r"))
4714      (const_int 0)))]
4715   ""
4716   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4717   [(set_attr "type" "logics_shift_imm")]
4718 )
4719
4720 (define_insn "clz<mode>2"
4721   [(set (match_operand:GPI 0 "register_operand" "=r")
4722         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4723   ""
4724   "clz\\t%<w>0, %<w>1"
4725   [(set_attr "type" "clz")]
4726 )
4727
4728 (define_expand "ffs<mode>2"
4729   [(match_operand:GPI 0 "register_operand")
4730    (match_operand:GPI 1 "register_operand")]
4731   ""
4732   {
4733     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4734     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4735
4736     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4737     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4738     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4739     DONE;
4740   }
4741 )
4742
4743 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4744 ;;
4745 ;; MOV  v.1d, x0
4746 ;; CNT  v1.8b, v.8b
4747 ;; ADDV b2, v1.8b
4748 ;; MOV  w0, v2.b[0]
4749
4750 (define_expand "popcount<mode>2"
4751   [(match_operand:GPI 0 "register_operand")
4752    (match_operand:GPI 1 "register_operand")]
4753   "TARGET_SIMD"
4754 {
4755   rtx v = gen_reg_rtx (V8QImode);
4756   rtx v1 = gen_reg_rtx (V8QImode);
4757   rtx r = gen_reg_rtx (QImode);
4758   rtx in = operands[1];
4759   rtx out = operands[0];
4760   if(<MODE>mode == SImode)
4761     {
4762       rtx tmp;
4763       tmp = gen_reg_rtx (DImode);
4764       /* If we have SImode, zero extend to DImode, pop count does
4765          not change if we have extra zeros. */
4766       emit_insn (gen_zero_extendsidi2 (tmp, in));
4767       in = tmp;
4768     }
4769   emit_move_insn (v, gen_lowpart (V8QImode, in));
4770   emit_insn (gen_popcountv8qi2 (v1, v));
4771   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4772   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4773   DONE;
4774 })
4775
4776 (define_insn "clrsb<mode>2"
4777   [(set (match_operand:GPI 0 "register_operand" "=r")
4778         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4779   ""
4780   "cls\\t%<w>0, %<w>1"
4781   [(set_attr "type" "clz")]
4782 )
4783
4784 (define_insn "rbit<mode>2"
4785   [(set (match_operand:GPI 0 "register_operand" "=r")
4786         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4787   ""
4788   "rbit\\t%<w>0, %<w>1"
4789   [(set_attr "type" "rbit")]
4790 )
4791
4792 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4793 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4794 ;; expression and split after reload to enable scheduling them apart if
4795 ;; needed.
4796
4797 (define_insn_and_split "ctz<mode>2"
4798  [(set (match_operand:GPI           0 "register_operand" "=r")
4799        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4800   ""
4801   "#"
4802   "reload_completed"
4803   [(const_int 0)]
4804   "
4805   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4806   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4807   DONE;
4808 ")
4809
4810 (define_insn "*and<mode>_compare0"
4811   [(set (reg:CC_NZ CC_REGNUM)
4812         (compare:CC_NZ
4813          (match_operand:SHORT 0 "register_operand" "r")
4814          (const_int 0)))]
4815   ""
4816   "tst\\t%<w>0, <short_mask>"
4817   [(set_attr "type" "alus_imm")]
4818 )
4819
4820 (define_insn "*ands<GPI:mode>_compare0"
4821   [(set (reg:CC_NZ CC_REGNUM)
4822         (compare:CC_NZ
4823          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4824          (const_int 0)))
4825    (set (match_operand:GPI 0 "register_operand" "=r")
4826         (zero_extend:GPI (match_dup 1)))]
4827   ""
4828   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4829   [(set_attr "type" "alus_imm")]
4830 )
4831
4832 (define_insn "*and<mode>3nr_compare0"
4833   [(set (reg:CC_NZ CC_REGNUM)
4834         (compare:CC_NZ
4835          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4836                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4837          (const_int 0)))]
4838   ""
4839   "@
4840    tst\\t%<w>0, %<w>1
4841    tst\\t%<w>0, %1"
4842   [(set_attr "type" "logics_reg,logics_imm")]
4843 )
4844
4845 (define_split
4846   [(set (reg:CC_NZ CC_REGNUM)
4847         (compare:CC_NZ
4848          (and:GPI (match_operand:GPI 0 "register_operand")
4849                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4850          (const_int 0)))
4851    (clobber (match_operand:SI 2 "register_operand"))]
4852   ""
4853   [(set (match_dup 2) (match_dup 1))
4854    (set (reg:CC_NZ CC_REGNUM)
4855         (compare:CC_NZ
4856          (and:GPI (match_dup 0)
4857                   (match_dup 2))
4858          (const_int 0)))]
4859 )
4860
4861 (define_insn "*and<mode>3nr_compare0_zextract"
4862   [(set (reg:CC_NZ CC_REGNUM)
4863         (compare:CC_NZ
4864          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4865                   (match_operand:GPI 1 "const_int_operand" "n")
4866                   (match_operand:GPI 2 "const_int_operand" "n"))
4867          (const_int 0)))]
4868   "INTVAL (operands[1]) > 0
4869    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4870         <= GET_MODE_BITSIZE (<MODE>mode))
4871    && aarch64_bitmask_imm (
4872         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4873                                                  operands[2])),
4874         <MODE>mode)"
4875   {
4876     operands[1]
4877       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4878     return "tst\\t%<w>0, %1";
4879   }
4880   [(set_attr "type" "logics_shift_imm")]
4881 )
4882
4883 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4884   [(set (reg:CC_NZ CC_REGNUM)
4885         (compare:CC_NZ
4886          (and:GPI (SHIFT:GPI
4887                    (match_operand:GPI 0 "register_operand" "r")
4888                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4889                   (match_operand:GPI 2 "register_operand" "r"))
4890         (const_int 0)))]
4891   ""
4892   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4893   [(set_attr "type" "logics_shift_imm")]
4894 )
4895
4896 (define_split
4897   [(set (reg:CC_NZ CC_REGNUM)
4898         (compare:CC_NZ
4899          (and:GPI (SHIFT:GPI
4900                    (match_operand:GPI 0 "register_operand")
4901                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4902                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4903         (const_int 0)))
4904     (clobber (match_operand:SI 3 "register_operand"))]
4905   ""
4906   [(set (match_dup 3) (match_dup 2))
4907    (set (reg:CC_NZ CC_REGNUM)
4908         (compare:CC_NZ
4909          (and:GPI (SHIFT:GPI
4910                    (match_dup 0)
4911                    (match_dup 1))
4912                   (match_dup 3))
4913          (const_int 0)))]
4914 )
4915
4916 ;; -------------------------------------------------------------------
4917 ;; Shifts
4918 ;; -------------------------------------------------------------------
4919
4920 (define_expand "<optab><mode>3"
4921   [(set (match_operand:GPI 0 "register_operand")
4922         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4923                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4924   ""
4925   {
4926     if (CONST_INT_P (operands[2]))
4927       {
4928         operands[2] = GEN_INT (INTVAL (operands[2])
4929                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4930
4931         if (operands[2] == const0_rtx)
4932           {
4933             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4934             DONE;
4935           }
4936       }
4937   }
4938 )
4939
4940 (define_expand "ashl<mode>3"
4941   [(set (match_operand:SHORT 0 "register_operand")
4942         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4943                       (match_operand:QI 2 "const_int_operand")))]
4944   ""
4945   {
4946     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4947
4948     if (operands[2] == const0_rtx)
4949       {
4950         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4951         DONE;
4952       }
4953   }
4954 )
4955
4956 (define_expand "rotr<mode>3"
4957   [(set (match_operand:GPI 0 "register_operand")
4958         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4959                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4960   ""
4961   {
4962     if (CONST_INT_P (operands[2]))
4963       {
4964         operands[2] = GEN_INT (INTVAL (operands[2])
4965                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4966
4967         if (operands[2] == const0_rtx)
4968           {
4969             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4970             DONE;
4971           }
4972       }
4973   }
4974 )
4975
4976 (define_expand "rotl<mode>3"
4977   [(set (match_operand:GPI 0 "register_operand")
4978         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4979                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4980   ""
4981   {
4982     /* (SZ - cnt) % SZ == -cnt % SZ */
4983     if (CONST_INT_P (operands[2]))
4984       {
4985         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4986                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4987         if (operands[2] == const0_rtx)
4988           {
4989             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4990             DONE;
4991           }
4992       }
4993     else
4994       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4995                                         NULL_RTX, 1);
4996   }
4997 )
4998
4999 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
5000 ;; they truncate the shift/rotate amount by the size of the registers they
5001 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
5002 ;; such redundant masking instructions.  GCC can do that automatically when
5003 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
5004 ;; because some of the SISD shift alternatives don't perform this truncations.
5005 ;; So this pattern exists to catch such cases.
5006
5007 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
5008   [(set (match_operand:GPI 0 "register_operand" "=r")
5009         (SHIFT:GPI
5010           (match_operand:GPI 1 "register_operand" "r")
5011           (match_operator 4 "subreg_lowpart_operator"
5012            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
5013                      (match_operand 3 "const_int_operand" "n"))])))]
5014   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
5015   "<shift>\t%<w>0, %<w>1, %<w>2"
5016   [(set_attr "type" "shift_reg")]
5017 )
5018
5019 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
5020   [(set (match_operand:GPI 0 "register_operand" "=&r")
5021         (SHIFT:GPI
5022           (match_operand:GPI 1 "register_operand" "r")
5023           (match_operator 4 "subreg_lowpart_operator"
5024           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
5025                            (match_operand 3 "const_int_operand" "n")))])))]
5026   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
5027   "#"
5028   "&& true"
5029   [(const_int 0)]
5030   {
5031     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5032                : lowpart_subreg (SImode, operands[0], <MODE>mode));
5033     emit_insn (gen_negsi2 (tmp, operands[2]));
5034
5035     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
5036     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
5037                                      SUBREG_BYTE (operands[4]));
5038     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
5039     DONE;
5040   }
5041 )
5042
5043 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
5044   [(set (match_operand:GPI 0 "register_operand" "=&r")
5045         (ashift:GPI
5046           (match_operand:GPI 1 "register_operand" "r")
5047           (minus:QI (match_operand 2 "const_int_operand" "n")
5048                     (match_operator 5 "subreg_lowpart_operator"
5049                     [(and:SI (match_operand:SI 3 "register_operand" "r")
5050                              (match_operand 4 "const_int_operand" "n"))]))))]
5051   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
5052    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5053   "#"
5054   "&& true"
5055   [(const_int 0)]
5056   {
5057     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5058                : operands[0]);
5059
5060     emit_insn (gen_negsi2 (tmp, operands[3]));
5061
5062     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
5063     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
5064                                      SUBREG_BYTE (operands[5]));
5065
5066     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
5067     DONE;
5068   }
5069 )
5070
5071 (define_insn "*aarch64_<optab>_reg_di3_mask2"
5072   [(set (match_operand:DI 0 "register_operand" "=r")
5073         (SHIFT:DI
5074           (match_operand:DI 1 "register_operand" "r")
5075           (match_operator 4 "subreg_lowpart_operator"
5076            [(and:SI (match_operand:SI 2 "register_operand" "r")
5077                     (match_operand 3 "const_int_operand" "n"))])))]
5078   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
5079 {
5080   rtx xop[3];
5081   xop[0] = operands[0];
5082   xop[1] = operands[1];
5083   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
5084   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
5085   return "";
5086 }
5087   [(set_attr "type" "shift_reg")]
5088 )
5089
5090 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
5091   [(set (match_operand:GPI 0 "register_operand" "=&r")
5092         (ASHIFT:GPI
5093           (match_operand:GPI 1 "register_operand" "r")
5094           (minus:QI (match_operand 2 "const_int_operand" "n")
5095                     (match_operand:QI 3 "register_operand" "r"))))]
5096   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
5097   "#"
5098   "&& true"
5099   [(const_int 0)]
5100   {
5101     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
5102
5103     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
5104                : gen_lowpart (SImode, operands[0]));
5105
5106     emit_insn (gen_negsi2 (tmp, subreg_tmp));
5107
5108     rtx and_op = gen_rtx_AND (SImode, tmp,
5109                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
5110
5111     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
5112
5113     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
5114     DONE;
5115   }
5116   [(set_attr "length" "8")]
5117 )
5118
5119 ;; Logical left shift using SISD or Integer instruction
5120 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
5121   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
5122         (ashift:GPI
5123           (match_operand:GPI 1 "register_operand" "r,r,w,w")
5124           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
5125   ""
5126   "@
5127    lsl\t%<w>0, %<w>1, %2
5128    lsl\t%<w>0, %<w>1, %<w>2
5129    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5130    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
5131   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
5132    (set_attr "arch" "*,*,simd,simd")]
5133 )
5134
5135 ;; Logical right shift using SISD or Integer instruction
5136 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
5137   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5138         (lshiftrt:GPI
5139          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5140          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
5141                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5142   ""
5143   "@
5144    lsr\t%<w>0, %<w>1, %2
5145    lsr\t%<w>0, %<w>1, %<w>2
5146    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5147    #
5148    #"
5149   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5150    (set_attr "arch" "*,*,simd,simd,simd")]
5151 )
5152
5153 (define_split
5154   [(set (match_operand:DI 0 "aarch64_simd_register")
5155         (lshiftrt:DI
5156            (match_operand:DI 1 "aarch64_simd_register")
5157            (match_operand:QI 2 "aarch64_simd_register")))]
5158   "TARGET_SIMD && reload_completed"
5159   [(set (match_dup 3)
5160         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5161    (set (match_dup 0)
5162         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
5163   {
5164     operands[3] = gen_lowpart (QImode, operands[0]);
5165   }
5166 )
5167
5168 (define_split
5169   [(set (match_operand:SI 0 "aarch64_simd_register")
5170         (lshiftrt:SI
5171            (match_operand:SI 1 "aarch64_simd_register")
5172            (match_operand:QI 2 "aarch64_simd_register")))]
5173   "TARGET_SIMD && reload_completed"
5174   [(set (match_dup 3)
5175         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5176    (set (match_dup 0)
5177         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
5178   {
5179     operands[3] = gen_lowpart (QImode, operands[0]);
5180   }
5181 )
5182
5183 ;; Arithmetic right shift using SISD or Integer instruction
5184 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
5185   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
5186         (ashiftrt:GPI
5187           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
5188           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
5189                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
5190   ""
5191   "@
5192    asr\t%<w>0, %<w>1, %2
5193    asr\t%<w>0, %<w>1, %<w>2
5194    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
5195    #
5196    #"
5197   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
5198    (set_attr "arch" "*,*,simd,simd,simd")]
5199 )
5200
5201 (define_split
5202   [(set (match_operand:DI 0 "aarch64_simd_register")
5203         (ashiftrt:DI
5204            (match_operand:DI 1 "aarch64_simd_register")
5205            (match_operand:QI 2 "aarch64_simd_register")))]
5206   "TARGET_SIMD && reload_completed"
5207   [(set (match_dup 3)
5208         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5209    (set (match_dup 0)
5210         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
5211 {
5212   operands[3] = gen_lowpart (QImode, operands[0]);
5213 }
5214 )
5215
5216 (define_split
5217   [(set (match_operand:SI 0 "aarch64_simd_register")
5218         (ashiftrt:SI
5219            (match_operand:SI 1 "aarch64_simd_register")
5220            (match_operand:QI 2 "aarch64_simd_register")))]
5221   "TARGET_SIMD && reload_completed"
5222   [(set (match_dup 3)
5223         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
5224    (set (match_dup 0)
5225         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
5226 {
5227   operands[3] = gen_lowpart (QImode, operands[0]);
5228 }
5229 )
5230
5231 (define_insn "*aarch64_sisd_ushl"
5232   [(set (match_operand:DI 0 "register_operand" "=w")
5233         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5234                     (match_operand:QI 2 "register_operand" "w")]
5235                    UNSPEC_SISD_USHL))]
5236   "TARGET_SIMD"
5237   "ushl\t%d0, %d1, %d2"
5238   [(set_attr "type" "neon_shift_reg")]
5239 )
5240
5241 (define_insn "*aarch64_ushl_2s"
5242   [(set (match_operand:SI 0 "register_operand" "=w")
5243         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5244                     (match_operand:QI 2 "register_operand" "w")]
5245                    UNSPEC_USHL_2S))]
5246   "TARGET_SIMD"
5247   "ushl\t%0.2s, %1.2s, %2.2s"
5248   [(set_attr "type" "neon_shift_reg")]
5249 )
5250
5251 (define_insn "*aarch64_sisd_sshl"
5252   [(set (match_operand:DI 0 "register_operand" "=w")
5253         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
5254                     (match_operand:QI 2 "register_operand" "w")]
5255                    UNSPEC_SISD_SSHL))]
5256   "TARGET_SIMD"
5257   "sshl\t%d0, %d1, %d2"
5258   [(set_attr "type" "neon_shift_reg")]
5259 )
5260
5261 (define_insn "*aarch64_sshl_2s"
5262   [(set (match_operand:SI 0 "register_operand" "=w")
5263         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
5264                     (match_operand:QI 2 "register_operand" "w")]
5265                    UNSPEC_SSHL_2S))]
5266   "TARGET_SIMD"
5267   "sshl\t%0.2s, %1.2s, %2.2s"
5268   [(set_attr "type" "neon_shift_reg")]
5269 )
5270
5271 (define_insn "*aarch64_sisd_neg_qi"
5272   [(set (match_operand:QI 0 "register_operand" "=w")
5273         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
5274                    UNSPEC_SISD_NEG))]
5275   "TARGET_SIMD"
5276   "neg\t%d0, %d1"
5277   [(set_attr "type" "neon_neg")]
5278 )
5279
5280 ;; Rotate right
5281 (define_insn "*ror<mode>3_insn"
5282   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5283      (rotatert:GPI
5284        (match_operand:GPI 1 "register_operand" "r,r")
5285        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5286   ""
5287   "@
5288    ror\\t%<w>0, %<w>1, %2
5289    ror\\t%<w>0, %<w>1, %<w>2"
5290   [(set_attr "type" "rotate_imm,shift_reg")]
5291 )
5292
5293 ;; zero_extend version of above
5294 (define_insn "*<optab>si3_insn_uxtw"
5295   [(set (match_operand:DI 0 "register_operand" "=r,r")
5296         (zero_extend:DI (SHIFT:SI
5297          (match_operand:SI 1 "register_operand" "r,r")
5298          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5299   ""
5300   "@
5301    <shift>\\t%w0, %w1, %2
5302    <shift>\\t%w0, %w1, %w2"
5303   [(set_attr "type" "bfx,shift_reg")]
5304 )
5305
5306 (define_insn "*<optab><mode>3_insn"
5307   [(set (match_operand:SHORT 0 "register_operand" "=r")
5308         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5309                       (match_operand 2 "const_int_operand" "n")))]
5310   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5311 {
5312   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5313   return "<bfshift>\t%w0, %w1, %2, %3";
5314 }
5315   [(set_attr "type" "bfx")]
5316 )
5317
5318 (define_insn "*extr<mode>5_insn"
5319   [(set (match_operand:GPI 0 "register_operand" "=r")
5320         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5321                              (match_operand 3 "const_int_operand" "n"))
5322                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5323                                (match_operand 4 "const_int_operand" "n"))))]
5324   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5325    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5326   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5327   [(set_attr "type" "rotate_imm")]
5328 )
5329
5330 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5331 ;; so we have to match both orderings.
5332 (define_insn "*extr<mode>5_insn_alt"
5333   [(set (match_operand:GPI 0 "register_operand" "=r")
5334         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5335                                 (match_operand 4 "const_int_operand" "n"))
5336                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5337                               (match_operand 3 "const_int_operand" "n"))))]
5338   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5339    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5340        == GET_MODE_BITSIZE (<MODE>mode))"
5341   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5342   [(set_attr "type" "rotate_imm")]
5343 )
5344
5345 ;; zero_extend version of the above
5346 (define_insn "*extrsi5_insn_uxtw"
5347   [(set (match_operand:DI 0 "register_operand" "=r")
5348         (zero_extend:DI
5349          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5350                             (match_operand 3 "const_int_operand" "n"))
5351                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5352                               (match_operand 4 "const_int_operand" "n")))))]
5353   "UINTVAL (operands[3]) < 32 &&
5354    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5355   "extr\\t%w0, %w1, %w2, %4"
5356   [(set_attr "type" "rotate_imm")]
5357 )
5358
5359 (define_insn "*extrsi5_insn_uxtw_alt"
5360   [(set (match_operand:DI 0 "register_operand" "=r")
5361         (zero_extend:DI
5362          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5363                                (match_operand 4 "const_int_operand" "n"))
5364                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5365                             (match_operand 3 "const_int_operand" "n")))))]
5366   "UINTVAL (operands[3]) < 32 &&
5367    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5368   "extr\\t%w0, %w1, %w2, %4"
5369   [(set_attr "type" "rotate_imm")]
5370 )
5371
5372 (define_insn "*ror<mode>3_insn"
5373   [(set (match_operand:GPI 0 "register_operand" "=r")
5374         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5375                     (match_operand 2 "const_int_operand" "n")))]
5376   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5377 {
5378   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5379   return "ror\\t%<w>0, %<w>1, %3";
5380 }
5381   [(set_attr "type" "rotate_imm")]
5382 )
5383
5384 ;; zero_extend version of the above
5385 (define_insn "*rorsi3_insn_uxtw"
5386   [(set (match_operand:DI 0 "register_operand" "=r")
5387         (zero_extend:DI
5388          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5389                     (match_operand 2 "const_int_operand" "n"))))]
5390   "UINTVAL (operands[2]) < 32"
5391 {
5392   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5393   return "ror\\t%w0, %w1, %3";
5394 }
5395   [(set_attr "type" "rotate_imm")]
5396 )
5397
5398 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5399   [(set (match_operand:GPI 0 "register_operand" "=r")
5400         (ANY_EXTEND:GPI
5401          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5402                        (match_operand 2 "const_int_operand" "n"))))]
5403   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5404 {
5405   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5406   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5407 }
5408   [(set_attr "type" "bfx")]
5409 )
5410
5411 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5412   [(set (match_operand:GPI 0 "register_operand" "=r")
5413         (zero_extend:GPI
5414          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5415                          (match_operand 2 "const_int_operand" "n"))))]
5416   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5417 {
5418   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5419   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5420 }
5421   [(set_attr "type" "bfx")]
5422 )
5423
5424 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5425   [(set (match_operand:GPI 0 "register_operand" "=r")
5426         (sign_extend:GPI
5427          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5428                          (match_operand 2 "const_int_operand" "n"))))]
5429   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5430 {
5431   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5432   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5433 }
5434   [(set_attr "type" "bfx")]
5435 )
5436
5437 ;; -------------------------------------------------------------------
5438 ;; Bitfields
5439 ;; -------------------------------------------------------------------
5440
5441 (define_expand "<optab>"
5442   [(set (match_operand:DI 0 "register_operand")
5443         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5444                         (match_operand 2
5445                           "aarch64_simd_shift_imm_offset_di")
5446                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5447   ""
5448   {
5449     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5450                    1, GET_MODE_BITSIZE (DImode) - 1))
5451      FAIL;
5452   }
5453 )
5454
5455
5456 (define_insn "*<optab><mode>"
5457   [(set (match_operand:GPI 0 "register_operand" "=r")
5458         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5459                          (match_operand 2
5460                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5461                          (match_operand 3
5462                            "aarch64_simd_shift_imm_<mode>" "n")))]
5463   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5464              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5465   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5466   [(set_attr "type" "bfx")]
5467 )
5468
5469 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5470 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5471 (define_split
5472   [(set (match_operand:DI 0 "register_operand")
5473         (zero_extract:DI (match_operand:DI 1 "register_operand")
5474                          (match_operand 2
5475                            "aarch64_simd_shift_imm_offset_di")
5476                          (match_operand 3
5477                            "aarch64_simd_shift_imm_di")))]
5478   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5479              GET_MODE_BITSIZE (DImode) - 1)
5480    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5481        == GET_MODE_BITSIZE (SImode)"
5482   [(set (match_dup 0)
5483         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5484   {
5485     operands[4] = gen_lowpart (SImode, operands[1]);
5486   }
5487 )
5488
5489 ;; Bitfield Insert (insv)
5490 (define_expand "insv<mode>"
5491   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5492                           (match_operand 1 "const_int_operand")
5493                           (match_operand 2 "const_int_operand"))
5494         (match_operand:GPI 3 "general_operand"))]
5495   ""
5496 {
5497   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5498   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5499   rtx value = operands[3];
5500
5501   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5502     FAIL;
5503
5504   if (CONST_INT_P (value))
5505     {
5506       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5507
5508       /* Prefer AND/OR for inserting all zeros or all ones.  */
5509       if ((UINTVAL (value) & mask) == 0
5510            || (UINTVAL (value) & mask) == mask)
5511         FAIL;
5512
5513       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5514       if (width == 16 && (pos % 16) == 0)
5515         DONE;
5516     }
5517   operands[3] = force_reg (<MODE>mode, value);
5518 })
5519
5520 (define_insn "*insv_reg<mode>"
5521   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5522                           (match_operand 1 "const_int_operand" "n")
5523                           (match_operand 2 "const_int_operand" "n"))
5524         (match_operand:GPI 3 "register_operand" "r"))]
5525   "!(UINTVAL (operands[1]) == 0
5526      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5527          > GET_MODE_BITSIZE (<MODE>mode)))"
5528   "bfi\\t%<w>0, %<w>3, %2, %1"
5529   [(set_attr "type" "bfm")]
5530 )
5531
5532 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5533   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5534                           (match_operand 1 "const_int_operand" "n")
5535                           (match_operand 2 "const_int_operand" "n"))
5536         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5537   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5538   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5539   [(set_attr "type" "bfm")]
5540 )
5541
5542 ;;  Match a bfi instruction where the shift of OP3 means that we are
5543 ;;  actually copying the least significant bits of OP3 into OP0 by way
5544 ;;  of the AND masks and the IOR instruction.  A similar instruction
5545 ;;  with the two parts of the IOR swapped around was never triggered
5546 ;;  in a bootstrap build and test of GCC so it was not included.
5547
5548 (define_insn "*aarch64_bfi<GPI:mode>5_shift"
5549   [(set (match_operand:GPI 0 "register_operand" "=r")
5550         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5551                           (match_operand:GPI 2 "const_int_operand" "n"))
5552                  (and:GPI (ashift:GPI
5553                            (match_operand:GPI 3 "register_operand" "r")
5554                            (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))
5555                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5556   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5557                                       UINTVAL (operands[4]),
5558                                       UINTVAL(operands[5]))"
5559   "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5"
5560   [(set_attr "type" "bfm")]
5561 )
5562
5563 (define_insn "*aarch64_bfi<GPI:mode>5_shift_alt"
5564   [(set (match_operand:GPI 0 "register_operand" "=r")
5565         (ior:GPI (and:GPI (ashift:GPI
5566                            (match_operand:GPI 1 "register_operand" "r")
5567                            (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5568                           (match_operand:GPI 3 "const_int_operand" "n"))
5569                  (and:GPI (match_operand:GPI 4 "register_operand" "0")
5570                           (match_operand:GPI 5 "const_int_operand" "n"))))]
5571   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]),
5572                                       UINTVAL (operands[2]),
5573                                       UINTVAL(operands[3]))"
5574   "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3"
5575   [(set_attr "type" "bfm")]
5576 )
5577
5578 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because
5579 ;; the shift is large enough to remove the need for an AND instruction.
5580
5581 (define_insn "*aarch64_bfi<GPI:mode>4_noand"
5582   [(set (match_operand:GPI 0 "register_operand" "=r")
5583         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5584                           (match_operand:GPI 2 "const_int_operand" "n"))
5585                  (ashift:GPI
5586                           (match_operand:GPI 3 "register_operand" "r")
5587                           (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))]
5588   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]),
5589                                       UINTVAL (operands[4]),
5590                                       HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )"
5591 {
5592   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4]));
5593   return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5";
5594 }
5595   [(set_attr "type" "bfm")]
5596 )
5597
5598 (define_insn "*aarch64_bfi<GPI:mode>4_noand_alt"
5599   [(set (match_operand:GPI 0 "register_operand" "=r")
5600         (ior:GPI (ashift:GPI
5601                           (match_operand:GPI 1 "register_operand" "r")
5602                           (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n"))
5603                  (and:GPI (match_operand:GPI 3 "register_operand" "0")
5604                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5605   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]),
5606                                       UINTVAL (operands[2]),
5607                                       HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )"
5608 {
5609   operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2]));
5610   return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5";
5611 }
5612   [(set_attr "type" "bfm")]
5613 )
5614
5615 ;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just
5616 ;; copying the least significant bits of OP3 to OP0.  We need two versions
5617 ;; of the instruction to handle different checks on the constant values.
5618
5619 (define_insn "*aarch64_bfi<GPI:mode>4_noshift"
5620   [(set (match_operand:GPI 0 "register_operand" "=r")
5621         (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0")
5622                           (match_operand:GPI 2 "const_int_operand" "n"))
5623                  (and:GPI (match_operand:GPI 3 "register_operand" "r")
5624                           (match_operand:GPI 4 "const_int_operand" "n"))))]
5625   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5626                                       UINTVAL (operands[4]))"
5627   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5628   [(set_attr "type" "bfm")]
5629 )
5630
5631 (define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt"
5632   [(set (match_operand:GPI 0 "register_operand" "=r")
5633         (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r")
5634                           (match_operand:GPI 4 "const_int_operand" "n"))
5635                  (and:GPI (match_operand:GPI 1 "register_operand" "0")
5636                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5637   "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0,
5638                                       UINTVAL (operands[4]))"
5639   "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4"
5640   [(set_attr "type" "bfm")]
5641 )
5642
5643 (define_insn "*extr_insv_lower_reg<mode>"
5644   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5645                           (match_operand 1 "const_int_operand" "n")
5646                           (const_int 0))
5647         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5648                           (match_dup 1)
5649                           (match_operand 3 "const_int_operand" "n")))]
5650   "!(UINTVAL (operands[1]) == 0
5651      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5652          > GET_MODE_BITSIZE (<MODE>mode)))"
5653   "bfxil\\t%<w>0, %<w>2, %3, %1"
5654   [(set_attr "type" "bfm")]
5655 )
5656
5657 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5658   [(set (match_operand:GPI 0 "register_operand" "=r")
5659         (ashift:GPI (ANY_EXTEND:GPI
5660                      (match_operand:ALLX 1 "register_operand" "r"))
5661                     (match_operand 2 "const_int_operand" "n")))]
5662   "UINTVAL (operands[2]) < <GPI:sizen>"
5663 {
5664   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5665               ? GEN_INT (<ALLX:sizen>)
5666               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5667   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5668 }
5669   [(set_attr "type" "bfx")]
5670 )
5671
5672 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5673
5674 (define_insn "*andim_ashift<mode>_bfiz"
5675   [(set (match_operand:GPI 0 "register_operand" "=r")
5676         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5677                              (match_operand 2 "const_int_operand" "n"))
5678                  (match_operand 3 "const_int_operand" "n")))]
5679   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5680   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5681   [(set_attr "type" "bfx")]
5682 )
5683
5684 ;; Match sbfiz pattern in a shift left + shift right operation.
5685
5686 (define_insn "*ashift<mode>_extv_bfiz"
5687   [(set (match_operand:GPI 0 "register_operand" "=r")
5688         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5689                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5690                                       (const_int 0))
5691                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5692   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5693              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5694   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5695   [(set_attr "type" "bfx")]
5696 )
5697
5698 ;; When the bit position and width of the equivalent extraction add up to 32
5699 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5700 ;; zero-extension of the X-reg.
5701 (define_split
5702   [(set (match_operand:DI 0 "register_operand")
5703         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5704                              (match_operand 2 "const_int_operand"))
5705                  (match_operand 3 "const_int_operand")))]
5706  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5707   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5708       == GET_MODE_BITSIZE (SImode)"
5709   [(set (match_dup 0)
5710         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5711   {
5712     operands[4] = gen_lowpart (SImode, operands[1]);
5713   }
5714 )
5715
5716 (define_insn "bswap<mode>2"
5717   [(set (match_operand:GPI 0 "register_operand" "=r")
5718         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5719   ""
5720   "rev\\t%<w>0, %<w>1"
5721   [(set_attr "type" "rev")]
5722 )
5723
5724 (define_insn "bswaphi2"
5725   [(set (match_operand:HI 0 "register_operand" "=r")
5726         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5727   ""
5728   "rev16\\t%w0, %w1"
5729   [(set_attr "type" "rev")]
5730 )
5731
5732 (define_insn "*aarch64_bfxil<mode>"
5733   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5734     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5735                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5736             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5737                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5738   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5739   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5740     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5741   {
5742     switch (which_alternative)
5743     {
5744       case 0:
5745         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5746         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5747       case 1:
5748         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5749         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5750       default:
5751         gcc_unreachable ();
5752     }
5753   }
5754   [(set_attr "type" "bfm")]
5755 )
5756
5757 ; Zero-extended version of above (aarch64_bfxil)
5758 (define_insn "*aarch64_bfxilsi_uxtw"
5759   [(set (match_operand:DI 0 "register_operand" "=r,r")
5760         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5761                                         "r,0")
5762                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5763             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5764                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5765   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5766   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5767     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5768   {
5769     switch (which_alternative)
5770     {
5771       case 0:
5772         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5773         return "bfxil\\t%0, %1, 0, %3";
5774       case 1:
5775         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5776         return "bfxil\\t%0, %2, 0, %3";
5777       default:
5778         gcc_unreachable ();
5779     }
5780   }
5781   [(set_attr "type" "bfm")]
5782 )
5783
5784 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5785 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5786 ;; each valid permutation.
5787
5788 (define_insn "rev16<mode>2"
5789   [(set (match_operand:GPI 0 "register_operand" "=r")
5790         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5791                                       (const_int 8))
5792                           (match_operand:GPI 3 "const_int_operand" "n"))
5793                  (and:GPI (lshiftrt:GPI (match_dup 1)
5794                                         (const_int 8))
5795                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5796   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5797    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5798   "rev16\\t%<w>0, %<w>1"
5799   [(set_attr "type" "rev")]
5800 )
5801
5802 (define_insn "rev16<mode>2_alt"
5803   [(set (match_operand:GPI 0 "register_operand" "=r")
5804         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5805                                         (const_int 8))
5806                           (match_operand:GPI 2 "const_int_operand" "n"))
5807                  (and:GPI (ashift:GPI (match_dup 1)
5808                                       (const_int 8))
5809                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5810   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5811    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5812   "rev16\\t%<w>0, %<w>1"
5813   [(set_attr "type" "rev")]
5814 )
5815
5816 ;; zero_extend version of above
5817 (define_insn "*bswapsi2_uxtw"
5818   [(set (match_operand:DI 0 "register_operand" "=r")
5819         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5820   ""
5821   "rev\\t%w0, %w1"
5822   [(set_attr "type" "rev")]
5823 )
5824
5825 ;; -------------------------------------------------------------------
5826 ;; Floating-point intrinsics
5827 ;; -------------------------------------------------------------------
5828
5829 ;; frint floating-point round to integral standard patterns.
5830 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5831
5832 (define_insn "<frint_pattern><mode>2"
5833   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5834         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5835          FRINT))]
5836   "TARGET_FLOAT"
5837   "frint<frint_suffix>\\t%<s>0, %<s>1"
5838   [(set_attr "type" "f_rint<stype>")]
5839 )
5840
5841 ;; frcvt floating-point round to integer and convert standard patterns.
5842 ;; Expands to lbtrunc, lceil, lfloor, lround.
5843 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5844   [(set (match_operand:GPI 0 "register_operand" "=r")
5845         (FIXUORS:GPI
5846           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5847            FCVT)))]
5848   "TARGET_FLOAT"
5849   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5850   [(set_attr "type" "f_cvtf2i")]
5851 )
5852
5853 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5854   [(set (match_operand:GPI 0 "register_operand" "=r")
5855         (FIXUORS:GPI
5856           (mult:GPF
5857             (match_operand:GPF 1 "register_operand" "w")
5858             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5859   "TARGET_FLOAT
5860    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5861                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5862   {
5863     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5864     char buf[64];
5865     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5866     output_asm_insn (buf, operands);
5867     return "";
5868   }
5869   [(set_attr "type" "f_cvtf2i")]
5870 )
5871
5872 ;; fma - expand fma into patterns with the accumulator operand first since
5873 ;; reusing the accumulator results in better register allocation.
5874 ;; The register allocator considers copy preferences in operand order,
5875 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5876
5877 (define_expand "fma<mode>4"
5878   [(set (match_operand:GPF_F16 0 "register_operand")
5879         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5880                      (match_operand:GPF_F16 2 "register_operand")
5881                      (match_operand:GPF_F16 3 "register_operand")))]
5882   "TARGET_FLOAT"
5883 )
5884
5885 (define_insn "*aarch64_fma<mode>4"
5886   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5887         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5888                      (match_operand:GPF_F16 3 "register_operand" "w")
5889                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5890   "TARGET_FLOAT"
5891   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5892   [(set_attr "type" "fmac<stype>")]
5893 )
5894
5895 (define_expand "fnma<mode>4"
5896   [(set (match_operand:GPF_F16 0 "register_operand")
5897         (fma:GPF_F16
5898           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5899           (match_operand:GPF_F16 2 "register_operand")
5900           (match_operand:GPF_F16 3 "register_operand")))]
5901   "TARGET_FLOAT"
5902 )
5903
5904 (define_insn "*aarch64_fnma<mode>4"
5905   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5906         (fma:GPF_F16
5907           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5908           (match_operand:GPF_F16 3 "register_operand" "w")
5909           (match_operand:GPF_F16 1 "register_operand" "w")))]
5910   "TARGET_FLOAT"
5911   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5912   [(set_attr "type" "fmac<stype>")]
5913 )
5914
5915
5916 (define_expand "fms<mode>4"
5917   [(set (match_operand:GPF 0 "register_operand")
5918         (fma:GPF (match_operand:GPF 1 "register_operand")
5919                  (match_operand:GPF 2 "register_operand")
5920                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5921   "TARGET_FLOAT"
5922 )
5923
5924 (define_insn "*aarch64_fms<mode>4"
5925   [(set (match_operand:GPF 0 "register_operand" "=w")
5926         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5927                  (match_operand:GPF 3 "register_operand" "w")
5928                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5929   "TARGET_FLOAT"
5930   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5931   [(set_attr "type" "fmac<s>")]
5932 )
5933
5934 (define_expand "fnms<mode>4"
5935   [(set (match_operand:GPF 0 "register_operand")
5936         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5937                  (match_operand:GPF 2 "register_operand")
5938                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5939   "TARGET_FLOAT"
5940 )
5941
5942 (define_insn "*aarch64_fnms<mode>4"
5943   [(set (match_operand:GPF 0 "register_operand" "=w")
5944         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5945                  (match_operand:GPF 3 "register_operand" "w")
5946                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5947   "TARGET_FLOAT"
5948   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5949   [(set_attr "type" "fmac<s>")]
5950 )
5951
5952 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5953 (define_insn "*aarch64_fnmadd<mode>4"
5954   [(set (match_operand:GPF 0 "register_operand" "=w")
5955         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5956                           (match_operand:GPF 3 "register_operand" "w")
5957                           (match_operand:GPF 1 "register_operand" "w"))))]
5958   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5959   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5960   [(set_attr "type" "fmac<s>")]
5961 )
5962
5963 ;; -------------------------------------------------------------------
5964 ;; Floating-point conversions
5965 ;; -------------------------------------------------------------------
5966
5967 (define_insn "extendsfdf2"
5968   [(set (match_operand:DF 0 "register_operand" "=w")
5969         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5970   "TARGET_FLOAT"
5971   "fcvt\\t%d0, %s1"
5972   [(set_attr "type" "f_cvt")]
5973 )
5974
5975 (define_insn "extendhfsf2"
5976   [(set (match_operand:SF 0 "register_operand" "=w")
5977         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5978   "TARGET_FLOAT"
5979   "fcvt\\t%s0, %h1"
5980   [(set_attr "type" "f_cvt")]
5981 )
5982
5983 (define_insn "extendhfdf2"
5984   [(set (match_operand:DF 0 "register_operand" "=w")
5985         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5986   "TARGET_FLOAT"
5987   "fcvt\\t%d0, %h1"
5988   [(set_attr "type" "f_cvt")]
5989 )
5990
5991 (define_insn "truncdfsf2"
5992   [(set (match_operand:SF 0 "register_operand" "=w")
5993         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5994   "TARGET_FLOAT"
5995   "fcvt\\t%s0, %d1"
5996   [(set_attr "type" "f_cvt")]
5997 )
5998
5999 (define_insn "truncsfhf2"
6000   [(set (match_operand:HF 0 "register_operand" "=w")
6001         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
6002   "TARGET_FLOAT"
6003   "fcvt\\t%h0, %s1"
6004   [(set_attr "type" "f_cvt")]
6005 )
6006
6007 (define_insn "truncdfhf2"
6008   [(set (match_operand:HF 0 "register_operand" "=w")
6009         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
6010   "TARGET_FLOAT"
6011   "fcvt\\t%h0, %d1"
6012   [(set_attr "type" "f_cvt")]
6013 )
6014
6015 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
6016 ;; and making r = w more expensive
6017
6018 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
6019   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
6020         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
6021   "TARGET_FLOAT"
6022   "@
6023    fcvtz<su>\t%<s>0, %<s>1
6024    fcvtz<su>\t%<w>0, %<s>1"
6025   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
6026 )
6027
6028 ;; Convert HF -> SI or DI
6029
6030 (define_insn "<optab>_trunchf<GPI:mode>2"
6031   [(set (match_operand:GPI 0 "register_operand" "=r")
6032         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
6033   "TARGET_FP_F16INST"
6034   "fcvtz<su>\t%<w>0, %h1"
6035   [(set_attr "type" "f_cvtf2i")]
6036 )
6037
6038 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
6039 ;; input in a fp register and output in a integer register
6040
6041 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
6042   [(set (match_operand:GPI 0 "register_operand" "=r")
6043         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
6044   "TARGET_FLOAT"
6045   "fcvtz<su>\t%<w>0, %<fpw>1"
6046   [(set_attr "type" "f_cvtf2i")]
6047 )
6048
6049 (define_insn "*fix_to_zero_extend<mode>di2"
6050   [(set (match_operand:DI 0 "register_operand" "=r")
6051         (zero_extend:DI
6052          (unsigned_fix:SI
6053           (match_operand:GPF 1 "register_operand" "w"))))]
6054   "TARGET_FLOAT"
6055   "fcvtzu\t%w0, %<s>1"
6056   [(set_attr "type" "f_cvtf2i")]
6057 )
6058
6059 ;; Equal width integer to fp and multiply combine.
6060 (define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult"
6061   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6062         (mult:GPF (FLOATUORS:GPF
6063                    (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r"))
6064                    (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))]
6065   "TARGET_FLOAT"
6066   {
6067     operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6068     switch (which_alternative)
6069     {
6070       case 0:
6071         return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2";
6072       case 1:
6073         return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2";
6074       default:
6075         gcc_unreachable ();
6076     }
6077   }
6078   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6079    (set_attr "arch" "simd,fp")]
6080 )
6081
6082 ;; Unequal width integer to fp and multiply combine.
6083 (define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult"
6084   [(set (match_operand:GPF 0 "register_operand" "=w")
6085         (mult:GPF (FLOATUORS:GPF
6086                    (match_operand:<FCVT_IESIZE> 1 "register_operand" "r"))
6087                    (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))]
6088   "TARGET_FLOAT"
6089   {
6090     operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2]));
6091     return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2";
6092   }
6093   [(set_attr "type" "f_cvti2f")]
6094 )
6095
6096 ;; Equal width integer to fp conversion.
6097 (define_insn "<optab><fcvt_target><GPF:mode>2"
6098   [(set (match_operand:GPF 0 "register_operand" "=w,w")
6099         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
6100   "TARGET_FLOAT"
6101   "@
6102    <su_optab>cvtf\t%<GPF:s>0, %<s>1
6103    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
6104   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
6105    (set_attr "arch" "simd,fp")]
6106 )
6107
6108 ;; Unequal width integer to fp conversions.
6109 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
6110   [(set (match_operand:GPF 0 "register_operand" "=w")
6111         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
6112   "TARGET_FLOAT"
6113   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
6114   [(set_attr "type" "f_cvti2f")]
6115 )
6116
6117 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
6118 ;; midend will arrange for an SImode conversion to HFmode to first go
6119 ;; through DFmode, then to HFmode.  But first it will try converting
6120 ;; to DImode then down, which would match our DImode pattern below and
6121 ;; give very poor code-generation.  So, we must provide our own emulation
6122 ;; of the mid-end logic.
6123
6124 (define_insn "aarch64_fp16_<optab><mode>hf2"
6125   [(set (match_operand:HF 0 "register_operand" "=w")
6126         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
6127   "TARGET_FP_F16INST"
6128   "<su_optab>cvtf\t%h0, %<w>1"
6129   [(set_attr "type" "f_cvti2f")]
6130 )
6131
6132 (define_expand "<optab>sihf2"
6133   [(set (match_operand:HF 0 "register_operand")
6134         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
6135   "TARGET_FLOAT"
6136 {
6137   if (TARGET_FP_F16INST)
6138     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
6139   else
6140     {
6141       rtx convert_target = gen_reg_rtx (DFmode);
6142       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
6143       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
6144     }
6145   DONE;
6146 }
6147 )
6148
6149 ;; For DImode there is no wide enough floating-point mode that we
6150 ;; can convert through natively (TFmode would work, but requires a library
6151 ;; call).  However, we know that any value >= 65504 will be rounded
6152 ;; to infinity on conversion.  This is well within the range of SImode, so
6153 ;; we can:
6154 ;;   Saturate to SImode.
6155 ;;   Convert from that to DFmode
6156 ;;   Convert from that to HFmode (phew!).
6157 ;; Note that the saturation to SImode requires the SIMD extensions.  If
6158 ;; we ever need to provide this pattern where the SIMD extensions are not
6159 ;; available, we would need a different approach.
6160
6161 (define_expand "<optab>dihf2"
6162   [(set (match_operand:HF 0 "register_operand")
6163         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
6164   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
6165 {
6166   if (TARGET_FP_F16INST)
6167     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
6168   else
6169     {
6170       rtx sat_target = gen_reg_rtx (SImode);
6171       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
6172       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
6173     }
6174
6175   DONE;
6176 }
6177 )
6178
6179 ;; Convert between fixed-point and floating-point (scalar modes)
6180
6181 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
6182   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
6183         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
6184                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6185          FCVT_F2FIXED))]
6186   ""
6187   "@
6188    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
6189    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
6190   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
6191    (set_attr "arch" "fp,simd")]
6192 )
6193
6194 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
6195   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
6196         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
6197                                    (match_operand:SI 2 "immediate_operand" "i, i")]
6198          FCVT_FIXED2F))]
6199   ""
6200   "@
6201    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
6202    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
6203   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
6204    (set_attr "arch" "fp,simd")]
6205 )
6206
6207 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
6208   [(set (match_operand:GPI 0 "register_operand" "=r")
6209         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
6210                      (match_operand:SI 2 "immediate_operand" "i")]
6211          FCVT_F2FIXED))]
6212   "TARGET_FP_F16INST"
6213    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
6214   [(set_attr "type" "f_cvtf2i")]
6215 )
6216
6217 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
6218   [(set (match_operand:HF 0 "register_operand" "=w")
6219         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
6220                     (match_operand:SI 2 "immediate_operand" "i")]
6221          FCVT_FIXED2F))]
6222   "TARGET_FP_F16INST"
6223   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
6224   [(set_attr "type" "f_cvti2f")]
6225 )
6226
6227 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
6228   [(set (match_operand:HI 0 "register_operand" "=w")
6229         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
6230                     (match_operand:SI 2 "immediate_operand" "i")]
6231          FCVT_F2FIXED))]
6232   "TARGET_SIMD"
6233   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
6234   [(set_attr "type" "neon_fp_to_int_s")]
6235 )
6236
6237 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
6238   [(set (match_operand:HF 0 "register_operand" "=w")
6239         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
6240                     (match_operand:SI 2 "immediate_operand" "i")]
6241          FCVT_FIXED2F))]
6242   "TARGET_SIMD"
6243   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
6244   [(set_attr "type" "neon_int_to_fp_s")]
6245 )
6246
6247 ;; -------------------------------------------------------------------
6248 ;; Floating-point arithmetic
6249 ;; -------------------------------------------------------------------
6250
6251 (define_insn "add<mode>3"
6252   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6253         (plus:GPF_F16
6254          (match_operand:GPF_F16 1 "register_operand" "w")
6255          (match_operand:GPF_F16 2 "register_operand" "w")))]
6256   "TARGET_FLOAT"
6257   "fadd\\t%<s>0, %<s>1, %<s>2"
6258   [(set_attr "type" "fadd<stype>")]
6259 )
6260
6261 (define_insn "sub<mode>3"
6262   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6263         (minus:GPF_F16
6264          (match_operand:GPF_F16 1 "register_operand" "w")
6265          (match_operand:GPF_F16 2 "register_operand" "w")))]
6266   "TARGET_FLOAT"
6267   "fsub\\t%<s>0, %<s>1, %<s>2"
6268   [(set_attr "type" "fadd<stype>")]
6269 )
6270
6271 (define_insn "mul<mode>3"
6272   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6273         (mult:GPF_F16
6274          (match_operand:GPF_F16 1 "register_operand" "w")
6275          (match_operand:GPF_F16 2 "register_operand" "w")))]
6276   "TARGET_FLOAT"
6277   "fmul\\t%<s>0, %<s>1, %<s>2"
6278   [(set_attr "type" "fmul<stype>")]
6279 )
6280
6281 (define_insn "*fnmul<mode>3"
6282   [(set (match_operand:GPF 0 "register_operand" "=w")
6283         (mult:GPF
6284                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
6285                  (match_operand:GPF 2 "register_operand" "w")))]
6286   "TARGET_FLOAT && !flag_rounding_math"
6287   "fnmul\\t%<s>0, %<s>1, %<s>2"
6288   [(set_attr "type" "fmul<s>")]
6289 )
6290
6291 (define_insn "*fnmul<mode>3"
6292   [(set (match_operand:GPF 0 "register_operand" "=w")
6293         (neg:GPF (mult:GPF
6294                  (match_operand:GPF 1 "register_operand" "w")
6295                  (match_operand:GPF 2 "register_operand" "w"))))]
6296   "TARGET_FLOAT"
6297   "fnmul\\t%<s>0, %<s>1, %<s>2"
6298   [(set_attr "type" "fmul<s>")]
6299 )
6300
6301 (define_expand "div<mode>3"
6302  [(set (match_operand:GPF_F16 0 "register_operand")
6303        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
6304                     (match_operand:GPF_F16 2 "register_operand")))]
6305  "TARGET_FLOAT"
6306 {
6307   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
6308     DONE;
6309
6310   operands[1] = force_reg (<MODE>mode, operands[1]);
6311 })
6312
6313 (define_insn "*div<mode>3"
6314   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6315         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
6316                      (match_operand:GPF_F16 2 "register_operand" "w")))]
6317   "TARGET_FLOAT"
6318   "fdiv\\t%<s>0, %<s>1, %<s>2"
6319   [(set_attr "type" "fdiv<stype>")]
6320 )
6321
6322 (define_insn "neg<mode>2"
6323   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6324         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6325   "TARGET_FLOAT"
6326   "fneg\\t%<s>0, %<s>1"
6327   [(set_attr "type" "ffarith<stype>")]
6328 )
6329
6330 (define_expand "sqrt<mode>2"
6331   [(set (match_operand:GPF_F16 0 "register_operand")
6332         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))]
6333   "TARGET_FLOAT"
6334 {
6335   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
6336     DONE;
6337 })
6338
6339 (define_insn "*sqrt<mode>2"
6340   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6341         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6342   "TARGET_FLOAT"
6343   "fsqrt\\t%<s>0, %<s>1"
6344   [(set_attr "type" "fsqrt<stype>")]
6345 )
6346
6347 (define_insn "abs<mode>2"
6348   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6349         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
6350   "TARGET_FLOAT"
6351   "fabs\\t%<s>0, %<s>1"
6352   [(set_attr "type" "ffarith<stype>")]
6353 )
6354
6355 ;; Given that smax/smin do not specify the result when either input is NaN,
6356 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
6357 ;; for smin.
6358
6359 (define_insn "smax<mode>3"
6360   [(set (match_operand:GPF 0 "register_operand" "=w")
6361         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
6362                   (match_operand:GPF 2 "register_operand" "w")))]
6363   "TARGET_FLOAT"
6364   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
6365   [(set_attr "type" "f_minmax<s>")]
6366 )
6367
6368 (define_insn "smin<mode>3"
6369   [(set (match_operand:GPF 0 "register_operand" "=w")
6370         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
6371                   (match_operand:GPF 2 "register_operand" "w")))]
6372   "TARGET_FLOAT"
6373   "fminnm\\t%<s>0, %<s>1, %<s>2"
6374   [(set_attr "type" "f_minmax<s>")]
6375 )
6376
6377 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
6378 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
6379 ;; which implement the IEEE fmax ()/fmin () functions.
6380 (define_insn "<maxmin_uns><mode>3"
6381   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
6382         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
6383                      (match_operand:GPF_F16 2 "register_operand" "w")]
6384                      FMAXMIN_UNS))]
6385   "TARGET_FLOAT"
6386   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
6387   [(set_attr "type" "f_minmax<stype>")]
6388 )
6389
6390 (define_expand "lrint<GPF:mode><GPI:mode>2"
6391   [(match_operand:GPI 0 "register_operand")
6392    (match_operand:GPF 1 "register_operand")]
6393   "TARGET_FLOAT
6394    && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE)
6395    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
6396 {
6397   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
6398   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
6399   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
6400   DONE;
6401 }
6402 )
6403
6404 ;; For copysign (x, y), we want to generate:
6405 ;;
6406 ;;   LDR d2, #(1 << 63)
6407 ;;   BSL v2.8b, [y], [x]
6408 ;;
6409 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.  Because
6410 ;; we expect these operations to nearly always operate on
6411 ;; floating-point values, we do not want the operation to be
6412 ;; simplified into a bit-field insert operation that operates on the
6413 ;; integer side, since typically that would involve three inter-bank
6414 ;; register copies.  As we do not expect copysign to be followed by
6415 ;; other logical operations on the result, it seems preferable to keep
6416 ;; this as an unspec operation, rather than exposing the underlying
6417 ;; logic to the compiler.
6418
6419 (define_expand "copysign<GPF:mode>3"
6420   [(match_operand:GPF 0 "register_operand")
6421    (match_operand:GPF 1 "register_operand")
6422    (match_operand:GPF 2 "register_operand")]
6423   "TARGET_FLOAT && TARGET_SIMD"
6424 {
6425   rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
6426   emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U
6427                                     << (GET_MODE_BITSIZE (<MODE>mode) - 1)));
6428   emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2],
6429                                        bitmask));
6430   DONE;
6431 }
6432 )
6433
6434 (define_insn "copysign<GPF:mode>3_insn"
6435   [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r")
6436         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r")
6437                      (match_operand:GPF 2 "register_operand" "w,w,0,0")
6438                      (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")]
6439          UNSPEC_COPYSIGN))]
6440   "TARGET_FLOAT && TARGET_SIMD"
6441   "@
6442    bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
6443    bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
6444    bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype>
6445    bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>"
6446   [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")]
6447 )
6448
6449
6450 ;; For xorsign (x, y), we want to generate:
6451 ;;
6452 ;; LDR   d2, #1<<63
6453 ;; AND   v3.8B, v1.8B, v2.8B
6454 ;; EOR   v0.8B, v0.8B, v3.8B
6455 ;;
6456
6457 (define_expand "xorsign<mode>3"
6458   [(match_operand:GPF 0 "register_operand")
6459    (match_operand:GPF 1 "register_operand")
6460    (match_operand:GPF 2 "register_operand")]
6461   "TARGET_FLOAT && TARGET_SIMD"
6462 {
6463
6464   machine_mode imode = <V_INT_EQUIV>mode;
6465   rtx mask = gen_reg_rtx (imode);
6466   rtx op1x = gen_reg_rtx (imode);
6467   rtx op2x = gen_reg_rtx (imode);
6468
6469   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6470   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6471                                                      imode)));
6472
6473   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6474                                     lowpart_subreg (imode, operands[2],
6475                                                     <MODE>mode)));
6476   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6477                                     lowpart_subreg (imode, operands[1],
6478                                                     <MODE>mode),
6479                                     op2x));
6480   emit_move_insn (operands[0],
6481                   lowpart_subreg (<MODE>mode, op1x, imode));
6482   DONE;
6483 }
6484 )
6485
6486 ;; -------------------------------------------------------------------
6487 ;; Reload support
6488 ;; -------------------------------------------------------------------
6489 ;; Reload Scalar Floating point modes from constant pool.
6490 ;; The AArch64 port doesn't have __int128 constant move support.
6491 ;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook.
6492 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6493  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6494        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6495   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6496  "TARGET_FLOAT"
6497  {
6498    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6499    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6500    DONE;
6501  }
6502 )
6503
6504 ;; Reload Vector modes from constant pool.
6505 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6506  [(set (match_operand:VALL 0 "register_operand" "=w")
6507        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6508   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6509  "TARGET_FLOAT"
6510  {
6511    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6512    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6513    DONE;
6514  }
6515 )
6516
6517 (define_expand "@aarch64_reload_mov<mode>"
6518   [(set (match_operand:TX 0 "register_operand" "=w")
6519         (match_operand:TX 1 "register_operand" "w"))
6520    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6521   ]
6522   "TARGET_FLOAT"
6523   {
6524     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6525     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6526     gen_aarch64_movtilow_tilow (op0, op1);
6527     gen_aarch64_movdi_tihigh (operands[2], op1);
6528     gen_aarch64_movtihigh_di (op0, operands[2]);
6529     DONE;
6530   }
6531 )
6532
6533 ;; The following secondary reload helpers patterns are invoked
6534 ;; after or during reload as we don't want these patterns to start
6535 ;; kicking in during the combiner.
6536
6537 (define_insn "@aarch64_movdi_<mode>low"
6538   [(set (match_operand:DI 0 "register_operand" "=r")
6539         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6540                          (const_int 64) (const_int 0)))]
6541   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6542   "fmov\\t%x0, %d1"
6543   [(set_attr "type" "f_mrc")
6544    (set_attr "length" "4")
6545   ])
6546
6547 (define_insn "@aarch64_movdi_<mode>high"
6548   [(set (match_operand:DI 0 "register_operand" "=r")
6549         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6550                          (const_int 64) (const_int 64)))]
6551   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6552   "fmov\\t%x0, %1.d[1]"
6553   [(set_attr "type" "f_mrc")
6554    (set_attr "length" "4")
6555   ])
6556
6557 (define_insn "@aarch64_mov<mode>high_di"
6558   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6559                          (const_int 64) (const_int 64))
6560         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6561   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6562   "fmov\\t%0.d[1], %x1"
6563   [(set_attr "type" "f_mcr")
6564    (set_attr "length" "4")
6565   ])
6566
6567 (define_insn "@aarch64_mov<mode>low_di"
6568   [(set (match_operand:TX 0 "register_operand" "=w")
6569         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6570   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6571   "fmov\\t%d0, %x1"
6572   [(set_attr "type" "f_mcr")
6573    (set_attr "length" "4")
6574   ])
6575
6576 (define_insn "aarch64_movtilow_tilow"
6577   [(set (match_operand:TI 0 "register_operand" "=w")
6578         (zero_extend:TI
6579           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6580   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6581   "fmov\\t%d0, %d1"
6582   [(set_attr "type" "fmov")
6583    (set_attr "length" "4")
6584   ])
6585
6586 ;; There is a deliberate reason why the parameters of high and lo_sum's
6587 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6588 ;; and lo_sum's to be used with the labels defining the jump tables in
6589 ;; rodata section.
6590
6591 (define_expand "add_losym"
6592   [(set (match_operand 0 "register_operand")
6593         (lo_sum (match_operand 1 "register_operand")
6594                 (match_operand 2 "aarch64_valid_symref")))]
6595   ""
6596 {
6597   machine_mode mode = GET_MODE (operands[0]);
6598
6599   emit_insn ((mode == DImode
6600               ? gen_add_losym_di
6601               : gen_add_losym_si) (operands[0],
6602                                    operands[1],
6603                                    operands[2]));
6604   DONE;
6605 })
6606
6607 (define_insn "add_losym_<mode>"
6608   [(set (match_operand:P 0 "register_operand" "=r")
6609         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6610                   (match_operand 2 "aarch64_valid_symref" "S")))]
6611   ""
6612   "add\\t%<w>0, %<w>1, :lo12:%c2"
6613   [(set_attr "type" "alu_imm")]
6614 )
6615
6616 (define_insn "ldr_got_small_<mode>"
6617   [(set (match_operand:PTR 0 "register_operand" "=r")
6618         (unspec:PTR [(mem:PTR (lo_sum:PTR
6619                               (match_operand:PTR 1 "register_operand" "r")
6620                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6621                     UNSPEC_GOTSMALLPIC))]
6622   ""
6623   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6624   [(set_attr "type" "load_<ldst_sz>")]
6625 )
6626
6627 (define_insn "ldr_got_small_sidi"
6628   [(set (match_operand:DI 0 "register_operand" "=r")
6629         (zero_extend:DI
6630          (unspec:SI [(mem:SI (lo_sum:DI
6631                              (match_operand:DI 1 "register_operand" "r")
6632                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6633                     UNSPEC_GOTSMALLPIC)))]
6634   "TARGET_ILP32"
6635   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6636   [(set_attr "type" "load_4")]
6637 )
6638
6639 (define_insn "ldr_got_small_28k_<mode>"
6640   [(set (match_operand:PTR 0 "register_operand" "=r")
6641         (unspec:PTR [(mem:PTR (lo_sum:PTR
6642                               (match_operand:PTR 1 "register_operand" "r")
6643                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6644                     UNSPEC_GOTSMALLPIC28K))]
6645   ""
6646   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6647   [(set_attr "type" "load_<ldst_sz>")]
6648 )
6649
6650 (define_insn "ldr_got_small_28k_sidi"
6651   [(set (match_operand:DI 0 "register_operand" "=r")
6652         (zero_extend:DI
6653          (unspec:SI [(mem:SI (lo_sum:DI
6654                              (match_operand:DI 1 "register_operand" "r")
6655                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6656                     UNSPEC_GOTSMALLPIC28K)))]
6657   "TARGET_ILP32"
6658   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6659   [(set_attr "type" "load_4")]
6660 )
6661
6662 (define_insn "ldr_got_tiny"
6663   [(set (match_operand:DI 0 "register_operand" "=r")
6664         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6665                    UNSPEC_GOTTINYPIC))]
6666   ""
6667   "ldr\\t%0, %L1"
6668   [(set_attr "type" "load_8")]
6669 )
6670
6671 (define_insn "aarch64_load_tp_hard"
6672   [(set (match_operand:DI 0 "register_operand" "=r")
6673         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6674   ""
6675   "mrs\\t%0, tpidr_el0"
6676   [(set_attr "type" "mrs")]
6677 )
6678
6679 ;; The TLS ABI specifically requires that the compiler does not schedule
6680 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6681 ;; Therefore we treat the stubs as an atomic sequence.
6682 (define_expand "tlsgd_small_<mode>"
6683  [(parallel [(set (match_operand 0 "register_operand")
6684                   (call (mem:DI (match_dup 2)) (const_int 1)))
6685              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS)
6686              (clobber (reg:DI LR_REGNUM))])]
6687  ""
6688 {
6689   operands[2] = aarch64_tls_get_addr ();
6690 })
6691
6692 (define_insn "*tlsgd_small_<mode>"
6693   [(set (match_operand 0 "register_operand" "")
6694         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6695    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6696    (clobber (reg:DI LR_REGNUM))
6697   ]
6698   ""
6699   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6700   [(set_attr "type" "call")
6701    (set_attr "length" "16")])
6702
6703 (define_insn "tlsie_small_<mode>"
6704   [(set (match_operand:PTR 0 "register_operand" "=r")
6705         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6706                    UNSPEC_GOTSMALLTLS))]
6707   ""
6708   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6709   [(set_attr "type" "load_4")
6710    (set_attr "length" "8")]
6711 )
6712
6713 (define_insn "tlsie_small_sidi"
6714   [(set (match_operand:DI 0 "register_operand" "=r")
6715         (zero_extend:DI
6716           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6717                       UNSPEC_GOTSMALLTLS)))]
6718   ""
6719   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6720   [(set_attr "type" "load_4")
6721    (set_attr "length" "8")]
6722 )
6723
6724 (define_insn "tlsie_tiny_<mode>"
6725   [(set (match_operand:PTR 0 "register_operand" "=&r")
6726         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6727                      (match_operand:PTR 2 "register_operand" "r")]
6728                    UNSPEC_GOTTINYTLS))]
6729   ""
6730   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6731   [(set_attr "type" "multiple")
6732    (set_attr "length" "8")]
6733 )
6734
6735 (define_insn "tlsie_tiny_sidi"
6736   [(set (match_operand:DI 0 "register_operand" "=&r")
6737         (zero_extend:DI
6738           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6739                       (match_operand:DI 2 "register_operand" "r")
6740                       ]
6741                       UNSPEC_GOTTINYTLS)))]
6742   ""
6743   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6744   [(set_attr "type" "multiple")
6745    (set_attr "length" "8")]
6746 )
6747
6748 (define_insn "tlsle12_<mode>"
6749   [(set (match_operand:P 0 "register_operand" "=r")
6750         (unspec:P [(match_operand:P 1 "register_operand" "r")
6751                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6752                    UNSPEC_TLSLE12))]
6753   ""
6754   "add\\t%<w>0, %<w>1, #%L2";
6755   [(set_attr "type" "alu_sreg")
6756    (set_attr "length" "4")]
6757 )
6758
6759 (define_insn "tlsle24_<mode>"
6760   [(set (match_operand:P 0 "register_operand" "=r")
6761         (unspec:P [(match_operand:P 1 "register_operand" "r")
6762                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6763                    UNSPEC_TLSLE24))]
6764   ""
6765   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6766   [(set_attr "type" "multiple")
6767    (set_attr "length" "8")]
6768 )
6769
6770 (define_insn "tlsle32_<mode>"
6771   [(set (match_operand:P 0 "register_operand" "=r")
6772         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6773                    UNSPEC_TLSLE32))]
6774   ""
6775   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6776   [(set_attr "type" "multiple")
6777    (set_attr "length" "8")]
6778 )
6779
6780 (define_insn "tlsle48_<mode>"
6781   [(set (match_operand:P 0 "register_operand" "=r")
6782         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6783                    UNSPEC_TLSLE48))]
6784   ""
6785   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6786   [(set_attr "type" "multiple")
6787    (set_attr "length" "12")]
6788 )
6789
6790 (define_expand "tlsdesc_small_<mode>"
6791   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6792   "TARGET_TLS_DESC"
6793   {
6794     if (TARGET_SVE)
6795       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6796     else
6797       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6798     DONE;
6799   }
6800 )
6801
6802 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6803 ;; R0 and LR.
6804 (define_insn "tlsdesc_small_advsimd_<mode>"
6805   [(set (reg:PTR R0_REGNUM)
6806         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6807                     UNSPEC_TLSDESC))
6808    (clobber (reg:DI LR_REGNUM))
6809    (clobber (reg:CC CC_REGNUM))
6810    (clobber (match_scratch:DI 1 "=r"))
6811    (use (reg:DI FP_REGNUM))]
6812   "TARGET_TLS_DESC && !TARGET_SVE"
6813   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6814   [(set_attr "type" "call")
6815    (set_attr "length" "16")])
6816
6817 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6818 ;; all vector registers, and clobber all predicate registers, on
6819 ;; top of the usual R0 and LR.
6820 (define_insn "tlsdesc_small_sve_<mode>"
6821   [(set (reg:PTR R0_REGNUM)
6822         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6823                     UNSPEC_TLSDESC))
6824    (clobber (reg:DI LR_REGNUM))
6825    (clobber (reg:CC CC_REGNUM))
6826    (clobber_high (reg:TI V0_REGNUM))
6827    (clobber_high (reg:TI V1_REGNUM))
6828    (clobber_high (reg:TI V2_REGNUM))
6829    (clobber_high (reg:TI V3_REGNUM))
6830    (clobber_high (reg:TI V4_REGNUM))
6831    (clobber_high (reg:TI V5_REGNUM))
6832    (clobber_high (reg:TI V6_REGNUM))
6833    (clobber_high (reg:TI V7_REGNUM))
6834    (clobber_high (reg:TI V8_REGNUM))
6835    (clobber_high (reg:TI V9_REGNUM))
6836    (clobber_high (reg:TI V10_REGNUM))
6837    (clobber_high (reg:TI V11_REGNUM))
6838    (clobber_high (reg:TI V12_REGNUM))
6839    (clobber_high (reg:TI V13_REGNUM))
6840    (clobber_high (reg:TI V14_REGNUM))
6841    (clobber_high (reg:TI V15_REGNUM))
6842    (clobber_high (reg:TI V16_REGNUM))
6843    (clobber_high (reg:TI V17_REGNUM))
6844    (clobber_high (reg:TI V18_REGNUM))
6845    (clobber_high (reg:TI V19_REGNUM))
6846    (clobber_high (reg:TI V20_REGNUM))
6847    (clobber_high (reg:TI V21_REGNUM))
6848    (clobber_high (reg:TI V22_REGNUM))
6849    (clobber_high (reg:TI V23_REGNUM))
6850    (clobber_high (reg:TI V24_REGNUM))
6851    (clobber_high (reg:TI V25_REGNUM))
6852    (clobber_high (reg:TI V26_REGNUM))
6853    (clobber_high (reg:TI V27_REGNUM))
6854    (clobber_high (reg:TI V28_REGNUM))
6855    (clobber_high (reg:TI V29_REGNUM))
6856    (clobber_high (reg:TI V30_REGNUM))
6857    (clobber_high (reg:TI V31_REGNUM))
6858    (clobber (reg:VNx2BI P0_REGNUM))
6859    (clobber (reg:VNx2BI P1_REGNUM))
6860    (clobber (reg:VNx2BI P2_REGNUM))
6861    (clobber (reg:VNx2BI P3_REGNUM))
6862    (clobber (reg:VNx2BI P4_REGNUM))
6863    (clobber (reg:VNx2BI P5_REGNUM))
6864    (clobber (reg:VNx2BI P6_REGNUM))
6865    (clobber (reg:VNx2BI P7_REGNUM))
6866    (clobber (reg:VNx2BI P8_REGNUM))
6867    (clobber (reg:VNx2BI P9_REGNUM))
6868    (clobber (reg:VNx2BI P10_REGNUM))
6869    (clobber (reg:VNx2BI P11_REGNUM))
6870    (clobber (reg:VNx2BI P12_REGNUM))
6871    (clobber (reg:VNx2BI P13_REGNUM))
6872    (clobber (reg:VNx2BI P14_REGNUM))
6873    (clobber (reg:VNx2BI P15_REGNUM))
6874    (clobber (match_scratch:DI 1 "=r"))
6875    (use (reg:DI FP_REGNUM))]
6876   "TARGET_TLS_DESC && TARGET_SVE"
6877   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6878   [(set_attr "type" "call")
6879    (set_attr "length" "16")])
6880
6881 (define_insn "stack_tie"
6882   [(set (mem:BLK (scratch))
6883         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6884                      (match_operand:DI 1 "register_operand" "rk")]
6885                     UNSPEC_PRLG_STK))]
6886   ""
6887   ""
6888   [(set_attr "length" "0")]
6889 )
6890
6891 (define_insn "aarch64_fjcvtzs"
6892   [(set (match_operand:SI 0 "register_operand" "=r")
6893         (unspec:SI [(match_operand:DF 1 "register_operand" "w")]
6894                    UNSPEC_FJCVTZS))]
6895   "TARGET_JSCVT"
6896   "fjcvtzs\\t%w0, %d1"
6897   [(set_attr "type" "f_cvtf2i")]
6898 )
6899
6900 ;; Pointer authentication patterns are always provided.  In architecture
6901 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6902 ;; This lets the user write portable software which authenticates pointers
6903 ;; when run on something which implements ARMv8.3-A, and which runs
6904 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6905 ;; implemented.
6906
6907 ;; Signing/Authenticating R30 using SP as the salt.
6908
6909 (define_insn "<pauth_mnem_prefix>sp"
6910   [(set (reg:DI R30_REGNUM)
6911         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6912   ""
6913   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp";
6914 )
6915
6916 ;; Signing/Authenticating X17 using X16 as the salt.
6917
6918 (define_insn "<pauth_mnem_prefix>1716"
6919   [(set (reg:DI R17_REGNUM)
6920         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6921   ""
6922   "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716";
6923 )
6924
6925 ;; Stripping the signature in R30.
6926
6927 (define_insn "xpaclri"
6928   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6929   ""
6930   "hint\t7 // xpaclri"
6931 )
6932
6933 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6934 ;; all of memory.  This blocks insns from being moved across this point.
6935
6936 (define_insn "blockage"
6937   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6938   ""
6939   ""
6940   [(set_attr "length" "0")
6941    (set_attr "type" "block")]
6942 )
6943
6944 (define_insn "probe_stack_range"
6945   [(set (match_operand:DI 0 "register_operand" "=rk")
6946         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6947                              (match_operand:DI 2 "register_operand" "r")]
6948                               UNSPECV_PROBE_STACK_RANGE))]
6949   ""
6950 {
6951   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6952 }
6953   [(set_attr "length" "32")]
6954 )
6955
6956 ;; This instruction is used to generate the stack clash stack adjustment and
6957 ;; probing loop.  We can't change the control flow during prologue and epilogue
6958 ;; code generation.  So we must emit a volatile unspec and expand it later on.
6959
6960 (define_insn "@probe_sve_stack_clash_<mode>"
6961   [(set (match_operand:P 0 "register_operand" "=rk")
6962         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6963                             (match_operand:P 2 "register_operand" "r")
6964                             (match_operand:P 3 "const_int_operand" "n")
6965                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
6966                              UNSPECV_PROBE_STACK_RANGE))]
6967   "TARGET_SVE"
6968 {
6969   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6970                                                operands[3], operands[4]);
6971 }
6972   [(set_attr "length" "28")]
6973 )
6974
6975 ;; Named pattern for expanding thread pointer reference.
6976 (define_expand "get_thread_pointerdi"
6977   [(match_operand:DI 0 "register_operand")]
6978   ""
6979 {
6980   rtx tmp = aarch64_load_tp (operands[0]);
6981   if (tmp != operands[0])
6982     emit_move_insn (operands[0], tmp);
6983   DONE;
6984 })
6985
6986 ;; Named patterns for stack smashing protection.
6987 (define_expand "stack_protect_set"
6988   [(match_operand 0 "memory_operand")
6989    (match_operand 1 "memory_operand")]
6990   ""
6991 {
6992   machine_mode mode = GET_MODE (operands[0]);
6993   if (aarch64_stack_protector_guard != SSP_GLOBAL)
6994   {
6995     /* Generate access through the system register.  */
6996     rtx tmp_reg = gen_reg_rtx (mode);
6997     if (mode == DImode)
6998     {
6999         emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7000         emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7001                                GEN_INT (aarch64_stack_protector_guard_offset)));
7002     }
7003     else
7004     {
7005         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7006         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7007                                GEN_INT (aarch64_stack_protector_guard_offset)));
7008
7009     }
7010     operands[1] = gen_rtx_MEM (mode, tmp_reg);
7011   }
7012
7013   emit_insn ((mode == DImode
7014               ? gen_stack_protect_set_di
7015               : gen_stack_protect_set_si) (operands[0], operands[1]));
7016   DONE;
7017 })
7018
7019 (define_insn "reg_stack_protect_address_<mode>"
7020  [(set (match_operand:PTR 0 "register_operand" "=r")
7021        (unspec:PTR [(const_int 0)]
7022         UNSPEC_SSP_SYSREG))]
7023  "aarch64_stack_protector_guard != SSP_GLOBAL"
7024  {
7025    char buf[150];
7026    snprintf (buf, 150, "mrs\\t%%<w>0, %s",
7027             aarch64_stack_protector_guard_reg_str);
7028    output_asm_insn (buf, operands);
7029    return "";
7030  }
7031  [(set_attr "type" "mrs")])
7032
7033 ;; DO NOT SPLIT THIS PATTERN.  It is important for security reasons that the
7034 ;; canary value does not live beyond the life of this sequence.
7035 (define_insn "stack_protect_set_<mode>"
7036   [(set (match_operand:PTR 0 "memory_operand" "=m")
7037         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
7038          UNSPEC_SP_SET))
7039    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
7040   ""
7041   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0"
7042   [(set_attr "length" "12")
7043    (set_attr "type" "multiple")])
7044
7045 (define_expand "stack_protect_test"
7046   [(match_operand 0 "memory_operand")
7047    (match_operand 1 "memory_operand")
7048    (match_operand 2)]
7049   ""
7050 {
7051   rtx result;
7052   machine_mode mode = GET_MODE (operands[0]);
7053
7054   result = gen_reg_rtx(mode);
7055   if (aarch64_stack_protector_guard != SSP_GLOBAL)
7056   {
7057     /* Generate access through the system register. The
7058        sequence we want here is the access
7059        of the stack offset to come with
7060        mrs scratch_reg, <system_register>
7061        add scratch_reg, scratch_reg, :lo12:offset. */
7062     rtx tmp_reg = gen_reg_rtx (mode);
7063     if (mode == DImode)
7064     {
7065        emit_insn (gen_reg_stack_protect_address_di (tmp_reg));
7066        emit_insn (gen_adddi3 (tmp_reg, tmp_reg,
7067                               GEN_INT (aarch64_stack_protector_guard_offset)));
7068     }
7069     else
7070     {
7071         emit_insn (gen_reg_stack_protect_address_si (tmp_reg));
7072         emit_insn (gen_addsi3 (tmp_reg, tmp_reg,
7073                                GEN_INT (aarch64_stack_protector_guard_offset)));
7074
7075     }
7076     operands[1] = gen_rtx_MEM (mode, tmp_reg);
7077   }
7078   emit_insn ((mode == DImode
7079                   ? gen_stack_protect_test_di
7080                   : gen_stack_protect_test_si) (result,
7081                                                 operands[0],
7082                                                 operands[1]));
7083
7084   if (mode == DImode)
7085     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7086                                     result, const0_rtx, operands[2]));
7087   else
7088     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
7089                                     result, const0_rtx, operands[2]));
7090   DONE;
7091 })
7092
7093 (define_insn "stack_protect_test_<mode>"
7094   [(set (match_operand:PTR 0 "register_operand" "=r")
7095         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
7096                      (match_operand:PTR 2 "memory_operand" "m")]
7097          UNSPEC_SP_TEST))
7098    (clobber (match_scratch:PTR 3 "=&r"))]
7099   ""
7100   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
7101   [(set_attr "length" "12")
7102    (set_attr "type" "multiple")])
7103
7104 ;; Write Floating-point Control Register.
7105 (define_insn "set_fpcr"
7106   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
7107   ""
7108   "msr\\tfpcr, %0"
7109   [(set_attr "type" "mrs")])
7110
7111 ;; Read Floating-point Control Register.
7112 (define_insn "get_fpcr"
7113   [(set (match_operand:SI 0 "register_operand" "=r")
7114         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
7115   ""
7116   "mrs\\t%0, fpcr"
7117   [(set_attr "type" "mrs")])
7118
7119 ;; Write Floating-point Status Register.
7120 (define_insn "set_fpsr"
7121   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
7122   ""
7123   "msr\\tfpsr, %0"
7124   [(set_attr "type" "mrs")])
7125
7126 ;; Read Floating-point Status Register.
7127 (define_insn "get_fpsr"
7128   [(set (match_operand:SI 0 "register_operand" "=r")
7129         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
7130   ""
7131   "mrs\\t%0, fpsr"
7132   [(set_attr "type" "mrs")])
7133
7134
7135 ;; Define the subtract-one-and-jump insns so loop.c
7136 ;; knows what to generate.
7137 (define_expand "doloop_end"
7138   [(use (match_operand 0 "" ""))      ; loop pseudo
7139    (use (match_operand 1 "" ""))]     ; label
7140   "optimize > 0 && flag_modulo_sched"
7141 {
7142   rtx s0;
7143   rtx bcomp;
7144   rtx loc_ref;
7145   rtx cc_reg;
7146   rtx insn;
7147   rtx cmp;
7148
7149   /* Currently SMS relies on the do-loop pattern to recognize loops
7150      where (1) the control part consists of all insns defining and/or
7151      using a certain 'count' register and (2) the loop count can be
7152      adjusted by modifying this register prior to the loop.
7153      ??? The possible introduction of a new block to initialize the
7154      new IV can potentially affect branch optimizations.  */
7155
7156   if (GET_MODE (operands[0]) != DImode)
7157     FAIL;
7158
7159   s0 = operands [0];
7160   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
7161
7162   cmp = XVECEXP (PATTERN (insn), 0, 0);
7163   cc_reg = SET_DEST (cmp);
7164   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
7165   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
7166   emit_jump_insn (gen_rtx_SET (pc_rtx,
7167                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
7168                                                      loc_ref, pc_rtx)));
7169   DONE;
7170 })
7171
7172 ;; Track speculation through conditional branches.  We assume that
7173 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
7174 (define_insn "speculation_tracker"
7175   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
7176         (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
7177          UNSPEC_SPECULATION_TRACKER))]
7178   ""
7179   {
7180     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7181     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
7182     return "";
7183   }
7184   [(set_attr "type" "csel")]
7185 )
7186
7187 ;; BTI <target> instructions
7188 (define_insn "bti_noarg"
7189   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)]
7190   ""
7191   "hint\t32 // bti"
7192   [(set_attr "type" "no_insn")]
7193 )
7194
7195 (define_insn "bti_c"
7196   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)]
7197   ""
7198   "hint\t34 // bti c"
7199   [(set_attr "type" "no_insn")]
7200 )
7201
7202 (define_insn "bti_j"
7203   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)]
7204   ""
7205   "hint\t36 // bti j"
7206   [(set_attr "type" "no_insn")]
7207 )
7208
7209 (define_insn "bti_jc"
7210   [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)]
7211   ""
7212   "hint\t38 // bti jc"
7213   [(set_attr "type" "no_insn")]
7214 )
7215
7216 ;; Hard speculation barrier.
7217 (define_insn "speculation_barrier"
7218   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
7219   ""
7220   "isb\;dsb\\tsy"
7221   [(set_attr "length" "8")
7222    (set_attr "type" "block")
7223    (set_attr "speculation_barrier" "true")]
7224 )
7225
7226 ;; Support for __builtin_speculation_safe_value when we have speculation
7227 ;; tracking enabled.  Use the speculation tracker to decide whether to
7228 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
7229 (define_expand "@despeculate_copy<ALLI_TI:mode>"
7230   [(set (match_operand:ALLI_TI 0 "register_operand")
7231         (unspec_volatile:ALLI_TI
7232          [(match_operand:ALLI_TI 1 "register_operand")
7233           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero")
7234           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7235           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7236   ""
7237   "
7238   {
7239     if (operands[2] == const0_rtx)
7240       {
7241         rtx tracker;
7242         if (<MODE>mode == TImode)
7243           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7244         else
7245           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
7246
7247         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
7248                                                  tracker));
7249         DONE;
7250       }
7251   }
7252   "
7253 )
7254
7255 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
7256 ;; encoding for CSDB, but will work in older versions of the assembler.
7257 (define_insn "*despeculate_copy<ALLI:mode>_insn"
7258   [(set (match_operand:ALLI 0 "register_operand" "=r")
7259         (unspec_volatile:ALLI
7260          [(match_operand:ALLI 1 "register_operand" "r")
7261           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
7262           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7263           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7264   ""
7265   {
7266     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7267     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
7268                      operands);
7269     return "";
7270   }
7271   [(set_attr "length" "12")
7272    (set_attr "type" "block")
7273    (set_attr "speculation_barrier" "true")]
7274 )
7275
7276 ;; Pattern to match despeculate_copyti
7277 (define_insn "*despeculate_copyti_insn"
7278   [(set (match_operand:TI 0 "register_operand" "=r")
7279         (unspec_volatile:TI
7280          [(match_operand:TI 1 "register_operand" "r")
7281           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
7282           (use (reg:DI SPECULATION_TRACKER_REGNUM))
7283           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
7284   ""
7285   {
7286     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
7287     output_asm_insn
7288       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
7289        operands);
7290     return "";
7291   }
7292   [(set_attr "length" "16")
7293    (set_attr "type" "block")
7294    (set_attr "speculation_barrier" "true")]
7295 )
7296
7297 (define_insn "despeculate_simple<ALLI:mode>"
7298   [(set (match_operand:ALLI 0 "register_operand" "=r")
7299         (unspec_volatile:ALLI
7300          [(match_operand:ALLI 1 "register_operand" "r")
7301           (use (match_operand:ALLI 2 "register_operand" ""))]
7302          UNSPECV_SPECULATION_BARRIER))]
7303   ""
7304   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
7305   [(set_attr "type" "block")
7306    (set_attr "length" "8")
7307    (set_attr "speculation_barrier" "true")]
7308 )
7309
7310 (define_insn "despeculate_simpleti"
7311   [(set (match_operand:TI 0 "register_operand" "=r")
7312         (unspec_volatile:TI
7313          [(match_operand:TI 1 "register_operand" "r")
7314           (use (match_operand:DI 2 "register_operand" ""))]
7315          UNSPECV_SPECULATION_BARRIER))]
7316   ""
7317   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
7318   [(set_attr "type" "block")
7319    (set_attr "length" "12")
7320    (set_attr "speculation_barrier" "true")]
7321 )
7322
7323 (define_insn "aarch64_<frintnzs_op><mode>"
7324   [(set (match_operand:VSFDF 0 "register_operand" "=w")
7325         (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")]
7326                       FRINTNZX))]
7327   "TARGET_FRINT && TARGET_FLOAT
7328    && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)"
7329   "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
7330   [(set_attr "type" "f_rint<stype>")]
7331 )
7332
7333 ;; Transactional Memory Extension (TME) instructions.
7334
7335 (define_insn "tstart"
7336   [(set (match_operand:DI 0 "register_operand" "=r")
7337         (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART))
7338    (clobber (mem:BLK (scratch)))]
7339   "TARGET_TME"
7340   "tstart\\t%0"
7341   [(set_attr "type" "tme")]
7342 )
7343
7344 (define_insn "ttest"
7345   [(set (match_operand:DI 0 "register_operand" "=r")
7346         (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST))
7347    (clobber (mem:BLK (scratch)))]
7348   "TARGET_TME"
7349   "ttest\\t%0"
7350   [(set_attr "type" "tme")]
7351 )
7352
7353 (define_insn "tcommit"
7354   [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT)
7355    (clobber (mem:BLK (scratch)))]
7356   "TARGET_TME"
7357   "tcommit"
7358   [(set_attr "type" "tme")]
7359 )
7360
7361 (define_insn "tcancel"
7362   [(unspec_volatile:BLK
7363      [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL)
7364    (clobber (mem:BLK (scratch)))]
7365   "TARGET_TME && (UINTVAL (operands[0]) <= 65535)"
7366   "tcancel\\t#%0"
7367   [(set_attr "type" "tme")]
7368 )
7369
7370 ;; AdvSIMD Stuff
7371 (include "aarch64-simd.md")
7372
7373 ;; Atomic Operations
7374 (include "atomics.md")
7375
7376 ;; ldp/stp peephole patterns
7377 (include "aarch64-ldpstp.md")
7378
7379 ;; SVE.
7380 (include "aarch64-sve.md")
7381
7382 ;; SVE2.
7383 (include "aarch64-sve2.md")