gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V4_REGNUM          36)
  61     (V8_REGNUM          40)
  62     (V12_REGNUM         44)
  63     (V15_REGNUM         47)
  64     (V16_REGNUM         48)
  65     (V20_REGNUM         52)
  66     (V24_REGNUM         56)
  67     (V28_REGNUM         60)
  68     (V31_REGNUM         63)
  69     (LAST_SAVED_REGNUM  63)
  70     (SFP_REGNUM         64)
  71     (AP_REGNUM          65)
  72     (CC_REGNUM          66)
  73     ;; Defined only to make the DWARF description simpler.
  74     (VG_REGNUM          67)
  75     (P0_REGNUM          68)
  76     (P1_REGNUM          69)
  77     (P2_REGNUM          70)
  78     (P3_REGNUM          71)
  79     (P4_REGNUM          72)
  80     (P5_REGNUM          73)
  81     (P6_REGNUM          74)
  82     (P7_REGNUM          75)
  83     (P8_REGNUM          76)
  84     (P9_REGNUM          77)
  85     (P10_REGNUM         78)
  86     (P11_REGNUM         79)
  87     (P12_REGNUM         80)
  88     (P13_REGNUM         81)
  89     (P14_REGNUM         82)
  90     (P15_REGNUM         83)
  91   ]
  92 )
  93
  94 (define_c_enum "unspec" [
  95     UNSPEC_AUTI1716
  96     UNSPEC_AUTISP
  97     UNSPEC_CASESI
  98     UNSPEC_CRC32B
  99     UNSPEC_CRC32CB
 100     UNSPEC_CRC32CH
 101     UNSPEC_CRC32CW
 102     UNSPEC_CRC32CX
 103     UNSPEC_CRC32H
 104     UNSPEC_CRC32W
 105     UNSPEC_CRC32X
 106     UNSPEC_FCVTZS
 107     UNSPEC_FCVTZU
 108     UNSPEC_URECPE
 109     UNSPEC_FRECPE
 110     UNSPEC_FRECPS
 111     UNSPEC_FRECPX
 112     UNSPEC_FRINTA
 113     UNSPEC_FRINTI
 114     UNSPEC_FRINTM
 115     UNSPEC_FRINTN
 116     UNSPEC_FRINTP
 117     UNSPEC_FRINTX
 118     UNSPEC_FRINTZ
 119     UNSPEC_GOTSMALLPIC
 120     UNSPEC_GOTSMALLPIC28K
 121     UNSPEC_GOTSMALLTLS
 122     UNSPEC_GOTTINYPIC
 123     UNSPEC_GOTTINYTLS
 124     UNSPEC_LD1
 125     UNSPEC_LD2
 126     UNSPEC_LD2_DREG
 127     UNSPEC_LD2_DUP
 128     UNSPEC_LD3
 129     UNSPEC_LD3_DREG
 130     UNSPEC_LD3_DUP
 131     UNSPEC_LD4
 132     UNSPEC_LD4_DREG
 133     UNSPEC_LD4_DUP
 134     UNSPEC_LD2_LANE
 135     UNSPEC_LD3_LANE
 136     UNSPEC_LD4_LANE
 137     UNSPEC_MB
 138     UNSPEC_NOP
 139     UNSPEC_PACI1716
 140     UNSPEC_PACISP
 141     UNSPEC_PRLG_STK
 142     UNSPEC_REV
 143     UNSPEC_RBIT
 144     UNSPEC_SABAL
 145     UNSPEC_SABDL2
 146     UNSPEC_SADALP
 147     UNSPEC_SCVTF
 148     UNSPEC_SISD_NEG
 149     UNSPEC_SISD_SSHL
 150     UNSPEC_SISD_USHL
 151     UNSPEC_SSHL_2S
 152     UNSPEC_ST1
 153     UNSPEC_ST2
 154     UNSPEC_ST3
 155     UNSPEC_ST4
 156     UNSPEC_ST2_LANE
 157     UNSPEC_ST3_LANE
 158     UNSPEC_ST4_LANE
 159     UNSPEC_TLS
 160     UNSPEC_TLSDESC
 161     UNSPEC_TLSLE12
 162     UNSPEC_TLSLE24
 163     UNSPEC_TLSLE32
 164     UNSPEC_TLSLE48
 165     UNSPEC_UABAL
 166     UNSPEC_UABDL2
 167     UNSPEC_UADALP
 168     UNSPEC_UCVTF
 169     UNSPEC_USHL_2S
 170     UNSPEC_VSTRUCTDUMMY
 171     UNSPEC_SP_SET
 172     UNSPEC_SP_TEST
 173     UNSPEC_RSQRT
 174     UNSPEC_RSQRTE
 175     UNSPEC_RSQRTS
 176     UNSPEC_NZCV
 177     UNSPEC_XPACLRI
 178     UNSPEC_LD1_SVE
 179     UNSPEC_ST1_SVE
 180     UNSPEC_LD1RQ
 181     UNSPEC_LD1_GATHER
 182     UNSPEC_ST1_SCATTER
 183     UNSPEC_MERGE_PTRUE
 184     UNSPEC_PTEST_PTRUE
 185     UNSPEC_UNPACKSHI
 186     UNSPEC_UNPACKUHI
 187     UNSPEC_UNPACKSLO
 188     UNSPEC_UNPACKULO
 189     UNSPEC_PACK
 190     UNSPEC_FLOAT_CONVERT
 191     UNSPEC_WHILE_LO
 192     UNSPEC_LDN
 193     UNSPEC_STN
 194     UNSPEC_INSR
 195     UNSPEC_CLASTB
 196     UNSPEC_FADDA
 197     UNSPEC_REV_SUBREG
 198 ])
 199
 200 (define_c_enum "unspecv" [
 201     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 202     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 203     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 204     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 205     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 206     UNSPECV_BLOCKAGE            ; Represent a blockage
 207     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 208   ]
 209 )
 210
 211 ;; If further include files are added the defintion of MD_INCLUDES
 212 ;; must be updated.
 213
 214 (include "constraints.md")
 215 (include "predicates.md")
 216 (include "iterators.md")
 217
 218 ;; -------------------------------------------------------------------
 219 ;; Instruction types and attributes
 220 ;; -------------------------------------------------------------------
 221
 222 ; The "type" attribute is included here from AArch32 backend to be able
 223 ; to share pipeline descriptions.
 224 (include "../arm/types.md")
 225
 226 ;; It is important to set the fp or simd attributes to yes when a pattern
 227 ;; alternative uses the FP or SIMD register files, usually signified by use of
 228 ;; the 'w' constraint.  This will ensure that the alternative will be
 229 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 230 ;; architecture extensions.  If all the alternatives in a pattern use the
 231 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 232 ;; or TARGET_SIMD.
 233
 234 ;; Attribute that specifies whether or not the instruction touches fp
 235 ;; registers.  When this is set to yes for an alternative, that alternative
 236 ;; will be disabled when !TARGET_FLOAT.
 237 (define_attr "fp" "no,yes" (const_string "no"))
 238
 239 ;; Attribute that specifies whether or not the instruction touches half
 240 ;; precision fp registers.  When this is set to yes for an alternative,
 241 ;; that alternative will be disabled when !TARGET_FP_F16INST.
 242 (define_attr "fp16" "no,yes" (const_string "no"))
 243
 244 ;; Attribute that specifies whether or not the instruction touches simd
 245 ;; registers.  When this is set to yes for an alternative, that alternative
 246 ;; will be disabled when !TARGET_SIMD.
 247 (define_attr "simd" "no,yes" (const_string "no"))
 248
 249 ;; Attribute that specifies whether or not the instruction uses SVE.
 250 ;; When this is set to yes for an alternative, that alternative
 251 ;; will be disabled when !TARGET_SVE.
 252 (define_attr "sve" "no,yes" (const_string "no"))
 253
 254 ;; Attribute that controls whether an alternative is enabled or not.
 255 ;; Currently it is only used to disable alternatives which touch fp or simd
 256 ;; registers when -mgeneral-regs-only is specified.
 257 (define_attr "enabled" "no,yes"
 258   (cond [(ior
 259             (and (eq_attr "fp" "yes")
 260                  (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 261             (and (eq_attr "simd" "yes")
 262                  (eq (symbol_ref "TARGET_SIMD") (const_int 0)))
 263             (and (eq_attr "fp16" "yes")
 264                  (eq (symbol_ref "TARGET_FP_F16INST") (const_int 0)))
 265             (and (eq_attr "sve" "yes")
 266                  (eq (symbol_ref "TARGET_SVE") (const_int 0))))
 267             (const_string "no")
 268         ] (const_string "yes")))
 269
 270 ;; Attribute that specifies whether we are dealing with a branch to a
 271 ;; label that is far away, i.e. further away than the maximum/minimum
 272 ;; representable in a signed 21-bits number.
 273 ;; 0 :=: no
 274 ;; 1 :=: yes
 275 (define_attr "far_branch" "" (const_int 0))
 276
 277 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 278 (define_attr "movprfx" "no,yes" (const_string "no"))
 279
 280 (define_attr "length" ""
 281   (cond [(eq_attr "movprfx" "yes")
 282            (const_int 8)
 283         ] (const_int 4)))
 284
 285 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 286 ;; no predicated insns.
 287 (define_attr "predicated" "yes,no" (const_string "no"))
 288
 289 ;; -------------------------------------------------------------------
 290 ;; Pipeline descriptions and scheduling
 291 ;; -------------------------------------------------------------------
 292
 293 ;; Processor types.
 294 (include "aarch64-tune.md")
 295
 296 ;; Scheduling
 297 (include "../arm/cortex-a53.md")
 298 (include "../arm/cortex-a57.md")
 299 (include "../arm/exynos-m1.md")
 300 (include "falkor.md")
 301 (include "thunderx.md")
 302 (include "../arm/xgene1.md")
 303 (include "thunderx2t99.md")
 304
 305 ;; -------------------------------------------------------------------
 306 ;; Jumps and other miscellaneous insns
 307 ;; -------------------------------------------------------------------
 308
 309 (define_insn "indirect_jump"
 310   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 311   ""
 312   "br\\t%0"
 313   [(set_attr "type" "branch")]
 314 )
 315
 316 (define_insn "jump"
 317   [(set (pc) (label_ref (match_operand 0 "" "")))]
 318   ""
 319   "b\\t%l0"
 320   [(set_attr "type" "branch")]
 321 )
 322
 323 (define_expand "cbranch<mode>4"
 324   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 325                             [(match_operand:GPI 1 "register_operand" "")
 326                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 327                            (label_ref (match_operand 3 "" ""))
 328                            (pc)))]
 329   ""
 330   "
 331   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 332                                          operands[2]);
 333   operands[2] = const0_rtx;
 334   "
 335 )
 336
 337 (define_expand "cbranch<mode>4"
 338   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 339                             [(match_operand:GPF 1 "register_operand" "")
 340                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 341                            (label_ref (match_operand 3 "" ""))
 342                            (pc)))]
 343   ""
 344   "
 345   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 346                                          operands[2]);
 347   operands[2] = const0_rtx;
 348   "
 349 )
 350
 351 (define_expand "cbranchcc4"
 352   [(set (pc) (if_then_else
 353               (match_operator 0 "aarch64_comparison_operator"
 354                [(match_operand 1 "cc_register" "")
 355                 (match_operand 2 "const0_operand")])
 356               (label_ref (match_operand 3 "" ""))
 357               (pc)))]
 358   ""
 359   "")
 360
 361 (define_insn "ccmp<mode>"
 362   [(set (match_operand:CC 1 "cc_register" "")
 363         (if_then_else:CC
 364           (match_operator 4 "aarch64_comparison_operator"
 365            [(match_operand 0 "cc_register" "")
 366             (const_int 0)])
 367           (compare:CC
 368             (match_operand:GPI 2 "register_operand" "r,r,r")
 369             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 370           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 371   ""
 372   "@
 373    ccmp\\t%<w>2, %<w>3, %k5, %m4
 374    ccmp\\t%<w>2, %3, %k5, %m4
 375    ccmn\\t%<w>2, #%n3, %k5, %m4"
 376   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 377 )
 378
 379 (define_insn "fccmp<mode>"
 380   [(set (match_operand:CCFP 1 "cc_register" "")
 381         (if_then_else:CCFP
 382           (match_operator 4 "aarch64_comparison_operator"
 383            [(match_operand 0 "cc_register" "")
 384             (const_int 0)])
 385           (compare:CCFP
 386             (match_operand:GPF 2 "register_operand" "w")
 387             (match_operand:GPF 3 "register_operand" "w"))
 388           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 389   "TARGET_FLOAT"
 390   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 391   [(set_attr "type" "fccmp<s>")]
 392 )
 393
 394 (define_insn "fccmpe<mode>"
 395   [(set (match_operand:CCFPE 1 "cc_register" "")
 396          (if_then_else:CCFPE
 397           (match_operator 4 "aarch64_comparison_operator"
 398            [(match_operand 0 "cc_register" "")
 399           (const_int 0)])
 400            (compare:CCFPE
 401             (match_operand:GPF 2 "register_operand" "w")
 402             (match_operand:GPF 3 "register_operand" "w"))
 403           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 404   "TARGET_FLOAT"
 405   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 406   [(set_attr "type" "fccmp<s>")]
 407 )
 408
 409 ;; Expansion of signed mod by a power of 2 using CSNEG.
 410 ;; For x0 % n where n is a power of 2 produce:
 411 ;; negs   x1, x0
 412 ;; and    x0, x0, #(n - 1)
 413 ;; and    x1, x1, #(n - 1)
 414 ;; csneg  x0, x0, x1, mi
 415
 416 (define_expand "mod<mode>3"
 417   [(match_operand:GPI 0 "register_operand" "")
 418    (match_operand:GPI 1 "register_operand" "")
 419    (match_operand:GPI 2 "const_int_operand" "")]
 420   ""
 421   {
 422     HOST_WIDE_INT val = INTVAL (operands[2]);
 423
 424     if (val <= 0
 425        || exact_log2 (val) <= 0
 426        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 427       FAIL;
 428
 429     rtx mask = GEN_INT (val - 1);
 430
 431     /* In the special case of x0 % 2 we can do the even shorter:
 432         cmp    x0, xzr
 433         and    x0, x0, 1
 434         cneg   x0, x0, lt.  */
 435     if (val == 2)
 436       {
 437         rtx masked = gen_reg_rtx (<MODE>mode);
 438         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 439         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 440         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 441         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 442         DONE;
 443       }
 444
 445     rtx neg_op = gen_reg_rtx (<MODE>mode);
 446     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 447
 448     /* Extract the condition register and mode.  */
 449     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 450     rtx cc_reg = SET_DEST (cmp);
 451     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 452
 453     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 454     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 455
 456     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 457     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 458
 459     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 460                                        masked_neg, masked_pos));
 461     DONE;
 462   }
 463 )
 464
 465 (define_insn "condjump"
 466   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 467                             [(match_operand 1 "cc_register" "") (const_int 0)])
 468                            (label_ref (match_operand 2 "" ""))
 469                            (pc)))]
 470   ""
 471   {
 472     if (get_attr_length (insn) == 8)
 473       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 474     else
 475       return  "b%m0\\t%l2";
 476   }
 477   [(set_attr "type" "branch")
 478    (set (attr "length")
 479         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 480                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 481                       (const_int 4)
 482                       (const_int 8)))
 483    (set (attr "far_branch")
 484         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 485                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 486                       (const_int 0)
 487                       (const_int 1)))]
 488 )
 489
 490 ;; For a 24-bit immediate CST we can optimize the compare for equality
 491 ;; and branch sequence from:
 492 ;;      mov     x0, #imm1
 493 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 494 ;;      cmp     x1, x0
 495 ;;      b<ne,eq> .Label
 496 ;; into the shorter:
 497 ;;      sub     x0, x1, #(CST & 0xfff000)
 498 ;;      subs    x0, x0, #(CST & 0x000fff)
 499 ;;      b<ne,eq> .Label
 500 (define_insn_and_split "*compare_condjump<mode>"
 501   [(set (pc) (if_then_else (EQL
 502                               (match_operand:GPI 0 "register_operand" "r")
 503                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 504                            (label_ref:P (match_operand 2 "" ""))
 505                            (pc)))]
 506   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 507    && !aarch64_plus_operand (operands[1], <MODE>mode)
 508    && !reload_completed"
 509   "#"
 510   "&& true"
 511   [(const_int 0)]
 512   {
 513     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 514     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 515     rtx tmp = gen_reg_rtx (<MODE>mode);
 516     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 517     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 518     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 519     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 520     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 521     DONE;
 522   }
 523 )
 524
 525 (define_expand "casesi"
 526   [(match_operand:SI 0 "register_operand" "")   ; Index
 527    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 528    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 529    (match_operand:DI 3 "" "")                   ; Table label
 530    (match_operand:DI 4 "" "")]                  ; Out of range label
 531   ""
 532   {
 533     if (operands[1] != const0_rtx)
 534       {
 535         rtx reg = gen_reg_rtx (SImode);
 536
 537         /* Canonical RTL says that if you have:
 538
 539            (minus (X) (CONST))
 540
 541            then this should be emitted as:
 542
 543            (plus (X) (-CONST))
 544
 545            The use of trunc_int_for_mode ensures that the resulting
 546            constant can be represented in SImode, this is important
 547            for the corner case where operand[1] is INT_MIN.  */
 548
 549         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 550
 551         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 552               (operands[1], SImode))
 553           operands[1] = force_reg (SImode, operands[1]);
 554         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 555         operands[0] = reg;
 556       }
 557
 558     if (!aarch64_plus_operand (operands[2], SImode))
 559       operands[2] = force_reg (SImode, operands[2]);
 560     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 561                                                  const0_rtx),
 562                                     operands[0], operands[2], operands[4]));
 563
 564     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 565     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 566                                          operands[3]));
 567     DONE;
 568   }
 569 )
 570
 571 (define_insn "casesi_dispatch"
 572   [(parallel
 573     [(set (pc)
 574           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 575                            (match_operand:SI 1 "register_operand" "r")]
 576                         UNSPEC_CASESI)))
 577      (clobber (reg:CC CC_REGNUM))
 578      (clobber (match_scratch:DI 3 "=r"))
 579      (clobber (match_scratch:DI 4 "=r"))
 580      (use (label_ref (match_operand 2 "" "")))])]
 581   ""
 582   "*
 583   return aarch64_output_casesi (operands);
 584   "
 585   [(set_attr "length" "16")
 586    (set_attr "type" "branch")]
 587 )
 588
 589 (define_insn "nop"
 590   [(unspec[(const_int 0)] UNSPEC_NOP)]
 591   ""
 592   "nop"
 593   [(set_attr "type" "no_insn")]
 594 )
 595
 596 (define_insn "prefetch"
 597   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 598             (match_operand:QI 1 "const_int_operand" "")
 599             (match_operand:QI 2 "const_int_operand" ""))]
 600   ""
 601   {
 602     const char * pftype[2][4] =
 603     {
 604       {"prfm\\tPLDL1STRM, %0",
 605        "prfm\\tPLDL3KEEP, %0",
 606        "prfm\\tPLDL2KEEP, %0",
 607        "prfm\\tPLDL1KEEP, %0"},
 608       {"prfm\\tPSTL1STRM, %0",
 609        "prfm\\tPSTL3KEEP, %0",
 610        "prfm\\tPSTL2KEEP, %0",
 611        "prfm\\tPSTL1KEEP, %0"},
 612     };
 613
 614     int locality = INTVAL (operands[2]);
 615
 616     gcc_assert (IN_RANGE (locality, 0, 3));
 617
 618     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 619        the address into a DImode MEM so that aarch64_print_operand knows
 620        how to print it.  */
 621     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 622     return pftype[INTVAL(operands[1])][locality];
 623   }
 624   [(set_attr "type" "load_4")]
 625 )
 626
 627 (define_insn "trap"
 628   [(trap_if (const_int 1) (const_int 8))]
 629   ""
 630   "brk #1000"
 631   [(set_attr "type" "trap")])
 632
 633 (define_expand "prologue"
 634   [(clobber (const_int 0))]
 635   ""
 636   "
 637   aarch64_expand_prologue ();
 638   DONE;
 639   "
 640 )
 641
 642 (define_expand "epilogue"
 643   [(clobber (const_int 0))]
 644   ""
 645   "
 646   aarch64_expand_epilogue (false);
 647   DONE;
 648   "
 649 )
 650
 651 (define_expand "sibcall_epilogue"
 652   [(clobber (const_int 0))]
 653   ""
 654   "
 655   aarch64_expand_epilogue (true);
 656   DONE;
 657   "
 658 )
 659
 660 (define_insn "*do_return"
 661   [(return)]
 662   ""
 663   {
 664     if (aarch64_return_address_signing_enabled ()
 665         && TARGET_ARMV8_3
 666         && !crtl->calls_eh_return)
 667       return "retaa";
 668
 669     return "ret";
 670   }
 671   [(set_attr "type" "branch")]
 672 )
 673
 674 (define_expand "return"
 675   [(simple_return)]
 676   "aarch64_use_return_insn_p ()"
 677   ""
 678 )
 679
 680 (define_insn "simple_return"
 681   [(simple_return)]
 682   ""
 683   "ret"
 684   [(set_attr "type" "branch")]
 685 )
 686
 687 (define_insn "*cb<optab><mode>1"
 688   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 689                                 (const_int 0))
 690                            (label_ref (match_operand 1 "" ""))
 691                            (pc)))]
 692   ""
 693   {
 694     if (get_attr_length (insn) == 8)
 695       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 696     else
 697       return "<cbz>\\t%<w>0, %l1";
 698   }
 699   [(set_attr "type" "branch")
 700    (set (attr "length")
 701         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 702                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 703                       (const_int 4)
 704                       (const_int 8)))
 705    (set (attr "far_branch")
 706         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 707                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 708                       (const_int 0)
 709                       (const_int 1)))]
 710 )
 711
 712 (define_insn "*tb<optab><mode>1"
 713   [(set (pc) (if_then_else
 714               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 715                                     (const_int 1)
 716                                     (match_operand 1
 717                                       "aarch64_simd_shift_imm_<mode>" "n"))
 718                    (const_int 0))
 719              (label_ref (match_operand 2 "" ""))
 720              (pc)))
 721    (clobber (reg:CC CC_REGNUM))]
 722   ""
 723   {
 724     if (get_attr_length (insn) == 8)
 725       {
 726         if (get_attr_far_branch (insn) == 1)
 727           return aarch64_gen_far_branch (operands, 2, "Ltb",
 728                                          "<inv_tb>\\t%<w>0, %1, ");
 729         else
 730           {
 731             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 732             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 733           }
 734       }
 735     else
 736       return "<tbz>\t%<w>0, %1, %l2";
 737   }
 738   [(set_attr "type" "branch")
 739    (set (attr "length")
 740         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 741                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 742                       (const_int 4)
 743                       (const_int 8)))
 744    (set (attr "far_branch")
 745         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 746                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 747                       (const_int 0)
 748                       (const_int 1)))]
 749
 750 )
 751
 752 (define_insn "*cb<optab><mode>1"
 753   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 754                                  (const_int 0))
 755                            (label_ref (match_operand 1 "" ""))
 756                            (pc)))
 757    (clobber (reg:CC CC_REGNUM))]
 758   ""
 759   {
 760     if (get_attr_length (insn) == 8)
 761       {
 762         if (get_attr_far_branch (insn) == 1)
 763           return aarch64_gen_far_branch (operands, 1, "Ltb",
 764                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 765         else
 766           {
 767             char buf[64];
 768             uint64_t val = ((uint64_t) 1)
 769                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 770             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 771             output_asm_insn (buf, operands);
 772             return "<bcond>\t%l1";
 773           }
 774       }
 775     else
 776       return "<tbz>\t%<w>0, <sizem1>, %l1";
 777   }
 778   [(set_attr "type" "branch")
 779    (set (attr "length")
 780         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 781                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 782                       (const_int 4)
 783                       (const_int 8)))
 784    (set (attr "far_branch")
 785         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 786                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 787                       (const_int 0)
 788                       (const_int 1)))]
 789 )
 790
 791 ;; -------------------------------------------------------------------
 792 ;; Subroutine calls and sibcalls
 793 ;; -------------------------------------------------------------------
 794
 795 (define_expand "call"
 796   [(parallel [(call (match_operand 0 "memory_operand" "")
 797                     (match_operand 1 "general_operand" ""))
 798               (use (match_operand 2 "" ""))
 799               (clobber (reg:DI LR_REGNUM))])]
 800   ""
 801   "
 802   {
 803     aarch64_expand_call (NULL_RTX, operands[0], false);
 804     DONE;
 805   }"
 806 )
 807
 808 (define_insn "*call_insn"
 809   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 810          (match_operand 1 "" ""))
 811    (clobber (reg:DI LR_REGNUM))]
 812   ""
 813   "@
 814   blr\\t%0
 815   bl\\t%c0"
 816   [(set_attr "type" "call, call")]
 817 )
 818
 819 (define_expand "call_value"
 820   [(parallel [(set (match_operand 0 "" "")
 821                    (call (match_operand 1 "memory_operand" "")
 822                          (match_operand 2 "general_operand" "")))
 823               (use (match_operand 3 "" ""))
 824               (clobber (reg:DI LR_REGNUM))])]
 825   ""
 826   "
 827   {
 828     aarch64_expand_call (operands[0], operands[1], false);
 829     DONE;
 830   }"
 831 )
 832
 833 (define_insn "*call_value_insn"
 834   [(set (match_operand 0 "" "")
 835         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 836                       (match_operand 2 "" "")))
 837    (clobber (reg:DI LR_REGNUM))]
 838   ""
 839   "@
 840   blr\\t%1
 841   bl\\t%c1"
 842   [(set_attr "type" "call, call")]
 843 )
 844
 845 (define_expand "sibcall"
 846   [(parallel [(call (match_operand 0 "memory_operand" "")
 847                     (match_operand 1 "general_operand" ""))
 848               (return)
 849               (use (match_operand 2 "" ""))])]
 850   ""
 851   {
 852     aarch64_expand_call (NULL_RTX, operands[0], true);
 853     DONE;
 854   }
 855 )
 856
 857 (define_expand "sibcall_value"
 858   [(parallel [(set (match_operand 0 "" "")
 859                    (call (match_operand 1 "memory_operand" "")
 860                          (match_operand 2 "general_operand" "")))
 861               (return)
 862               (use (match_operand 3 "" ""))])]
 863   ""
 864   {
 865     aarch64_expand_call (operands[0], operands[1], true);
 866     DONE;
 867   }
 868 )
 869
 870 (define_insn "*sibcall_insn"
 871   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 872          (match_operand 1 "" ""))
 873    (return)]
 874   "SIBLING_CALL_P (insn)"
 875   "@
 876    br\\t%0
 877    b\\t%c0"
 878   [(set_attr "type" "branch, branch")]
 879 )
 880
 881 (define_insn "*sibcall_value_insn"
 882   [(set (match_operand 0 "" "")
 883         (call (mem:DI
 884                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 885               (match_operand 2 "" "")))
 886    (return)]
 887   "SIBLING_CALL_P (insn)"
 888   "@
 889    br\\t%1
 890    b\\t%c1"
 891   [(set_attr "type" "branch, branch")]
 892 )
 893
 894 ;; Call subroutine returning any type.
 895
 896 (define_expand "untyped_call"
 897   [(parallel [(call (match_operand 0 "")
 898                     (const_int 0))
 899               (match_operand 1 "")
 900               (match_operand 2 "")])]
 901   ""
 902 {
 903   int i;
 904
 905   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 906
 907   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 908     {
 909       rtx set = XVECEXP (operands[2], 0, i);
 910       emit_move_insn (SET_DEST (set), SET_SRC (set));
 911     }
 912
 913   /* The optimizer does not know that the call sets the function value
 914      registers we stored in the result block.  We avoid problems by
 915      claiming that all hard registers are used and clobbered at this
 916      point.  */
 917   emit_insn (gen_blockage ());
 918   DONE;
 919 })
 920
 921 ;; -------------------------------------------------------------------
 922 ;; Moves
 923 ;; -------------------------------------------------------------------
 924
 925 (define_expand "mov<mode>"
 926   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 927         (match_operand:SHORT 1 "general_operand" ""))]
 928   ""
 929   "
 930     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 931       operands[1] = force_reg (<MODE>mode, operands[1]);
 932
 933     if (GET_CODE (operands[1]) == CONST_POLY_INT)
 934       {
 935         aarch64_expand_mov_immediate (operands[0], operands[1]);
 936         DONE;
 937       }
 938   "
 939 )
 940
 941 (define_insn "*mov<mode>_aarch64"
 942   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
 943         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
 944   "(register_operand (operands[0], <MODE>mode)
 945     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 946 {
 947    switch (which_alternative)
 948      {
 949      case 0:
 950        return "mov\t%w0, %w1";
 951      case 1:
 952        return "mov\t%w0, %1";
 953      case 2:
 954        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 955                                                         <MODE>mode);
 956      case 3:
 957        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
 958      case 4:
 959        return "ldr<size>\t%w0, %1";
 960      case 5:
 961        return "ldr\t%<size>0, %1";
 962      case 6:
 963        return "str<size>\t%w1, %0";
 964      case 7:
 965        return "str\t%<size>1, %0";
 966      case 8:
 967        return "umov\t%w0, %1.<v>[0]";
 968      case 9:
 969        return "dup\t%0.<Vallxd>, %w1";
 970      case 10:
 971        return "dup\t%<Vetype>0, %1.<v>[0]";
 972      default:
 973        gcc_unreachable ();
 974      }
 975 }
 976   ;; The "mov_imm" type for CNT is just a placeholder.
 977   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
 978                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
 979    (set_attr "simd" "*,*,yes,*,*,*,*,*,yes,yes,yes")
 980    (set_attr "sve" "*,*,*,yes,*,*,*,*,*,*,*")]
 981 )
 982
 983 (define_expand "mov<mode>"
 984   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
 985         (match_operand:GPI 1 "general_operand" ""))]
 986   ""
 987   "
 988     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
 989         && <MODE>mode == DImode
 990         && aarch64_split_dimode_const_store (operands[0], operands[1]))
 991       DONE;
 992
 993     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 994       operands[1] = force_reg (<MODE>mode, operands[1]);
 995
 996     /* FIXME: RR we still need to fix up what we are doing with
 997        symbol_refs and other types of constants.  */
 998     if (CONSTANT_P (operands[1])
 999         && !CONST_INT_P (operands[1]))
1000      {
1001        aarch64_expand_mov_immediate (operands[0], operands[1]);
1002        DONE;
1003      }
1004   "
1005 )
1006
1007 (define_insn_and_split "*movsi_aarch64"
1008   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1009         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1010   "(register_operand (operands[0], SImode)
1011     || aarch64_reg_or_zero (operands[1], SImode))"
1012   "@
1013    mov\\t%w0, %w1
1014    mov\\t%w0, %w1
1015    mov\\t%w0, %w1
1016    mov\\t%w0, %1
1017    #
1018    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1019    ldr\\t%w0, %1
1020    ldr\\t%s0, %1
1021    str\\t%w1, %0
1022    str\\t%s1, %0
1023    adr\\t%x0, %c1
1024    adrp\\t%x0, %A1
1025    fmov\\t%s0, %w1
1026    fmov\\t%w0, %s1
1027    fmov\\t%s0, %s1
1028    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1029   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1030     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1031    [(const_int 0)]
1032    "{
1033        aarch64_expand_mov_immediate (operands[0], operands[1]);
1034        DONE;
1035     }"
1036   ;; The "mov_imm" type for CNT is just a placeholder.
1037   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1038                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1039    (set_attr "fp" "*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1040    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1041    (set_attr "sve" "*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1042 )
1043
1044 (define_insn_and_split "*movdi_aarch64"
1045   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1046         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1047   "(register_operand (operands[0], DImode)
1048     || aarch64_reg_or_zero (operands[1], DImode))"
1049   "@
1050    mov\\t%x0, %x1
1051    mov\\t%0, %x1
1052    mov\\t%x0, %1
1053    mov\\t%x0, %1
1054    mov\\t%w0, %1
1055    #
1056    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1057    ldr\\t%x0, %1
1058    ldr\\t%d0, %1
1059    str\\t%x1, %0
1060    str\\t%d1, %0
1061    adr\\t%x0, %c1
1062    adrp\\t%x0, %A1
1063    fmov\\t%d0, %x1
1064    fmov\\t%x0, %d1
1065    fmov\\t%d0, %d1
1066    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1067    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1068     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1069    [(const_int 0)]
1070    "{
1071        aarch64_expand_mov_immediate (operands[0], operands[1]);
1072        DONE;
1073     }"
1074   ;; The "mov_imm" type for CNTD is just a placeholder.
1075   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1076                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1077                      neon_move")
1078    (set_attr "fp" "*,*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1079    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1080    (set_attr "sve" "*,*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1081 )
1082
1083 (define_insn "insv_imm<mode>"
1084   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1085                           (const_int 16)
1086                           (match_operand:GPI 1 "const_int_operand" "n"))
1087         (match_operand:GPI 2 "const_int_operand" "n"))]
1088   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1089    && UINTVAL (operands[1]) % 16 == 0"
1090   "movk\\t%<w>0, %X2, lsl %1"
1091   [(set_attr "type" "mov_imm")]
1092 )
1093
1094 (define_expand "movti"
1095   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1096         (match_operand:TI 1 "general_operand" ""))]
1097   ""
1098   "
1099     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1100       operands[1] = force_reg (TImode, operands[1]);
1101
1102     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1103       {
1104         emit_move_insn (gen_lowpart (DImode, operands[0]),
1105                         gen_lowpart (DImode, operands[1]));
1106         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1107         DONE;
1108       }
1109   "
1110 )
1111
1112 (define_insn "*movti_aarch64"
1113   [(set (match_operand:TI 0
1114          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1115         (match_operand:TI 1
1116          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1117   "(register_operand (operands[0], TImode)
1118     || aarch64_reg_or_zero (operands[1], TImode))"
1119   "@
1120    #
1121    #
1122    #
1123    mov\\t%0.16b, %1.16b
1124    ldp\\t%0, %H0, %1
1125    stp\\t%1, %H1, %0
1126    stp\\txzr, xzr, %0
1127    ldr\\t%q0, %1
1128    str\\t%q1, %0"
1129   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1130                              load_16,store_16,store_16,\
1131                              load_16,store_16")
1132    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1133    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1134    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1135 )
1136
1137 ;; Split a TImode register-register or register-immediate move into
1138 ;; its component DImode pieces, taking care to handle overlapping
1139 ;; source and dest registers.
1140 (define_split
1141    [(set (match_operand:TI 0 "register_operand" "")
1142          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1143   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1144   [(const_int 0)]
1145 {
1146   aarch64_split_128bit_move (operands[0], operands[1]);
1147   DONE;
1148 })
1149
1150 (define_expand "mov<mode>"
1151   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1152         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1153   ""
1154   {
1155     if (!TARGET_FLOAT)
1156       {
1157         aarch64_err_no_fpadvsimd (<MODE>mode);
1158         FAIL;
1159       }
1160
1161     if (GET_CODE (operands[0]) == MEM
1162         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1163               && aarch64_float_const_zero_rtx_p (operands[1])))
1164       operands[1] = force_reg (<MODE>mode, operands[1]);
1165   }
1166 )
1167
1168 (define_insn "*movhf_aarch64"
1169   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1170         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1171   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1172     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1173   "@
1174    movi\\t%0.4h, #0
1175    fmov\\t%h0, %w1
1176    dup\\t%w0.4h, %w1
1177    umov\\t%w0, %1.h[0]
1178    mov\\t%0.h[0], %1.h[0]
1179    fmov\\t%h0, %1
1180    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1181    ldr\\t%h0, %1
1182    str\\t%h1, %0
1183    ldrh\\t%w0, %1
1184    strh\\t%w1, %0
1185    mov\\t%w0, %w1"
1186   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1187                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1188    (set_attr "simd" "yes,*,yes,yes,yes,*,yes,*,*,*,*,*")
1189    (set_attr "fp16"   "*,yes,*,*,*,yes,*,*,*,*,*,*")]
1190 )
1191
1192 (define_insn "*movsf_aarch64"
1193   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1194         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1195   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1196     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1197   "@
1198    movi\\t%0.2s, #0
1199    fmov\\t%s0, %w1
1200    fmov\\t%w0, %s1
1201    fmov\\t%s0, %s1
1202    fmov\\t%s0, %1
1203    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1204    ldr\\t%s0, %1
1205    str\\t%s1, %0
1206    ldr\\t%w0, %1
1207    str\\t%w1, %0
1208    mov\\t%w0, %w1
1209    mov\\t%w0, %1"
1210   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1211                      f_loads,f_stores,load_4,store_4,mov_reg,\
1212                      fconsts")
1213    (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1214 )
1215
1216 (define_insn "*movdf_aarch64"
1217   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1218         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1219   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1220     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1221   "@
1222    movi\\t%d0, #0
1223    fmov\\t%d0, %x1
1224    fmov\\t%x0, %d1
1225    fmov\\t%d0, %d1
1226    fmov\\t%d0, %1
1227    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1228    ldr\\t%d0, %1
1229    str\\t%d1, %0
1230    ldr\\t%x0, %1
1231    str\\t%x1, %0
1232    mov\\t%x0, %x1
1233    mov\\t%x0, %1"
1234   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1235                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1236                      fconstd")
1237    (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1238 )
1239
1240 (define_split
1241   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1242         (match_operand:GPF_HF 1 "general_operand"))]
1243   "can_create_pseudo_p ()
1244    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1245    && !aarch64_float_const_representable_p (operands[1])
1246    &&  aarch64_float_const_rtx_p (operands[1])"
1247   [(const_int 0)]
1248   {
1249     unsigned HOST_WIDE_INT ival;
1250     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1251       FAIL;
1252
1253     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1254     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1255     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1256     DONE;
1257   }
1258 )
1259
1260 (define_insn "*movtf_aarch64"
1261   [(set (match_operand:TF 0
1262          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1263         (match_operand:TF 1
1264          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1265   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1266     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1267   "@
1268    mov\\t%0.16b, %1.16b
1269    #
1270    #
1271    #
1272    movi\\t%0.2d, #0
1273    fmov\\t%s0, wzr
1274    ldr\\t%q0, %1
1275    str\\t%q1, %0
1276    ldp\\t%0, %H0, %1
1277    stp\\t%1, %H1, %0
1278    stp\\txzr, xzr, %0"
1279   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1280                      f_loadd,f_stored,load_16,store_16,store_16")
1281    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1282    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1283 )
1284
1285 (define_split
1286    [(set (match_operand:TF 0 "register_operand" "")
1287          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1288   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1289   [(const_int 0)]
1290   {
1291     aarch64_split_128bit_move (operands[0], operands[1]);
1292     DONE;
1293   }
1294 )
1295
1296 ;; 0 is dst
1297 ;; 1 is src
1298 ;; 2 is size of move in bytes
1299 ;; 3 is alignment
1300
1301 (define_expand "movmemdi"
1302   [(match_operand:BLK 0 "memory_operand")
1303    (match_operand:BLK 1 "memory_operand")
1304    (match_operand:DI 2 "immediate_operand")
1305    (match_operand:DI 3 "immediate_operand")]
1306    "!STRICT_ALIGNMENT"
1307 {
1308   if (aarch64_expand_movmem (operands))
1309     DONE;
1310   FAIL;
1311 }
1312 )
1313
1314 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1315 ;; fairly lax checking on the second memory operation.
1316 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1317   [(set (match_operand:SX 0 "register_operand" "=r,w")
1318         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1319    (set (match_operand:SX2 2 "register_operand" "=r,w")
1320         (match_operand:SX2 3 "memory_operand" "m,m"))]
1321    "rtx_equal_p (XEXP (operands[3], 0),
1322                  plus_constant (Pmode,
1323                                 XEXP (operands[1], 0),
1324                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1325   "@
1326    ldp\\t%w0, %w2, %1
1327    ldp\\t%s0, %s2, %1"
1328   [(set_attr "type" "load_8,neon_load1_2reg")
1329    (set_attr "fp" "*,yes")]
1330 )
1331
1332 ;; Storing different modes that can still be merged
1333 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1334   [(set (match_operand:DX 0 "register_operand" "=r,w")
1335         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1336    (set (match_operand:DX2 2 "register_operand" "=r,w")
1337         (match_operand:DX2 3 "memory_operand" "m,m"))]
1338    "rtx_equal_p (XEXP (operands[3], 0),
1339                  plus_constant (Pmode,
1340                                 XEXP (operands[1], 0),
1341                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1342   "@
1343    ldp\\t%x0, %x2, %1
1344    ldp\\t%d0, %d2, %1"
1345   [(set_attr "type" "load_16,neon_load1_2reg")
1346    (set_attr "fp" "*,yes")]
1347 )
1348
1349 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1350 ;; fairly lax checking on the second memory operation.
1351 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1352   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1353         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1354    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1355         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1356    "rtx_equal_p (XEXP (operands[2], 0),
1357                  plus_constant (Pmode,
1358                                 XEXP (operands[0], 0),
1359                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1360   "@
1361    stp\\t%w1, %w3, %0
1362    stp\\t%s1, %s3, %0"
1363   [(set_attr "type" "store_8,neon_store1_2reg")
1364    (set_attr "fp" "*,yes")]
1365 )
1366
1367 ;; Storing different modes that can still be merged
1368 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1369   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1370         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1371    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1372         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1373    "rtx_equal_p (XEXP (operands[2], 0),
1374                  plus_constant (Pmode,
1375                                 XEXP (operands[0], 0),
1376                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1377   "@
1378    stp\\t%x1, %x3, %0
1379    stp\\t%d1, %d3, %0"
1380   [(set_attr "type" "store_16,neon_store1_2reg")
1381    (set_attr "fp" "*,yes")]
1382 )
1383
1384 ;; Load pair with post-index writeback.  This is primarily used in function
1385 ;; epilogues.
1386 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1387   [(parallel
1388     [(set (match_operand:P 0 "register_operand" "=k")
1389           (plus:P (match_operand:P 1 "register_operand" "0")
1390                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1391      (set (match_operand:GPI 2 "register_operand" "=r")
1392           (mem:GPI (match_dup 1)))
1393      (set (match_operand:GPI 3 "register_operand" "=r")
1394           (mem:GPI (plus:P (match_dup 1)
1395                    (match_operand:P 5 "const_int_operand" "n"))))])]
1396   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1397   "ldp\\t%<w>2, %<w>3, [%1], %4"
1398   [(set_attr "type" "load_<ldpstp_sz>")]
1399 )
1400
1401 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1402   [(parallel
1403     [(set (match_operand:P 0 "register_operand" "=k")
1404           (plus:P (match_operand:P 1 "register_operand" "0")
1405                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1406      (set (match_operand:GPF 2 "register_operand" "=w")
1407           (mem:GPF (match_dup 1)))
1408      (set (match_operand:GPF 3 "register_operand" "=w")
1409           (mem:GPF (plus:P (match_dup 1)
1410                    (match_operand:P 5 "const_int_operand" "n"))))])]
1411   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1412   "ldp\\t%<w>2, %<w>3, [%1], %4"
1413   [(set_attr "type" "neon_load1_2reg")]
1414 )
1415
1416 ;; Store pair with pre-index writeback.  This is primarily used in function
1417 ;; prologues.
1418 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1419   [(parallel
1420     [(set (match_operand:P 0 "register_operand" "=&k")
1421           (plus:P (match_operand:P 1 "register_operand" "0")
1422                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1423      (set (mem:GPI (plus:P (match_dup 0)
1424                    (match_dup 4)))
1425           (match_operand:GPI 2 "register_operand" "r"))
1426      (set (mem:GPI (plus:P (match_dup 0)
1427                    (match_operand:P 5 "const_int_operand" "n")))
1428           (match_operand:GPI 3 "register_operand" "r"))])]
1429   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1430   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1431   [(set_attr "type" "store_<ldpstp_sz>")]
1432 )
1433
1434 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1435   [(parallel
1436     [(set (match_operand:P 0 "register_operand" "=&k")
1437           (plus:P (match_operand:P 1 "register_operand" "0")
1438                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1439      (set (mem:GPF (plus:P (match_dup 0)
1440                    (match_dup 4)))
1441           (match_operand:GPF 2 "register_operand" "w"))
1442      (set (mem:GPF (plus:P (match_dup 0)
1443                    (match_operand:P 5 "const_int_operand" "n")))
1444           (match_operand:GPF 3 "register_operand" "w"))])]
1445   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1446   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1447   [(set_attr "type" "neon_store1_2reg<q>")]
1448 )
1449
1450 ;; -------------------------------------------------------------------
1451 ;; Sign/Zero extension
1452 ;; -------------------------------------------------------------------
1453
1454 (define_expand "<optab>sidi2"
1455   [(set (match_operand:DI 0 "register_operand")
1456         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1457   ""
1458 )
1459
1460 (define_insn "*extendsidi2_aarch64"
1461   [(set (match_operand:DI 0 "register_operand" "=r,r")
1462         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1463   ""
1464   "@
1465    sxtw\t%0, %w1
1466    ldrsw\t%0, %1"
1467   [(set_attr "type" "extend,load_4")]
1468 )
1469
1470 (define_insn "*load_pair_extendsidi2_aarch64"
1471   [(set (match_operand:DI 0 "register_operand" "=r")
1472         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1473    (set (match_operand:DI 2 "register_operand" "=r")
1474         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1475   "rtx_equal_p (XEXP (operands[3], 0),
1476                 plus_constant (Pmode,
1477                                XEXP (operands[1], 0),
1478                                GET_MODE_SIZE (SImode)))"
1479   "ldpsw\\t%0, %2, %1"
1480   [(set_attr "type" "load_8")]
1481 )
1482
1483 (define_insn "*zero_extendsidi2_aarch64"
1484   [(set (match_operand:DI 0 "register_operand" "=r,r")
1485         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1486   ""
1487   "@
1488    uxtw\t%0, %w1
1489    ldr\t%w0, %1"
1490   [(set_attr "type" "extend,load_4")]
1491 )
1492
1493 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1494   [(set (match_operand:DI 0 "register_operand" "=r")
1495         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1496    (set (match_operand:DI 2 "register_operand" "=r")
1497         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1498   "rtx_equal_p (XEXP (operands[3], 0),
1499                 plus_constant (Pmode,
1500                                XEXP (operands[1], 0),
1501                                GET_MODE_SIZE (SImode)))"
1502   "ldp\\t%w0, %w2, %1"
1503   [(set_attr "type" "load_8")]
1504 )
1505
1506 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1507   [(set (match_operand:GPI 0 "register_operand")
1508         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1509   ""
1510 )
1511
1512 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1513   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1514         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1515   ""
1516   "@
1517    sxt<SHORT:size>\t%<GPI:w>0, %w1
1518    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1519   [(set_attr "type" "extend,load_4")]
1520 )
1521
1522 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1523   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1524         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1525   ""
1526   "@
1527    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1528    ldr<SHORT:size>\t%w0, %1
1529    ldr\t%<SHORT:size>0, %1"
1530   [(set_attr "type" "logic_imm,load_4,load_4")]
1531 )
1532
1533 (define_expand "<optab>qihi2"
1534   [(set (match_operand:HI 0 "register_operand")
1535         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1536   ""
1537 )
1538
1539 (define_insn "*extendqihi2_aarch64"
1540   [(set (match_operand:HI 0 "register_operand" "=r,r")
1541         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1542   ""
1543   "@
1544    sxtb\t%w0, %w1
1545    ldrsb\t%w0, %1"
1546   [(set_attr "type" "extend,load_4")]
1547 )
1548
1549 (define_insn "*zero_extendqihi2_aarch64"
1550   [(set (match_operand:HI 0 "register_operand" "=r,r")
1551         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1552   ""
1553   "@
1554    and\t%w0, %w1, 255
1555    ldrb\t%w0, %1"
1556   [(set_attr "type" "logic_imm,load_4")]
1557 )
1558
1559 ;; -------------------------------------------------------------------
1560 ;; Simple arithmetic
1561 ;; -------------------------------------------------------------------
1562
1563 (define_expand "add<mode>3"
1564   [(set
1565     (match_operand:GPI 0 "register_operand" "")
1566     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1567               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1568   ""
1569 {
1570   /* If operands[1] is a subreg extract the inner RTX.  */
1571   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1572
1573   /* If the constant is too large for a single instruction and isn't frame
1574      based, split off the immediate so it is available for CSE.  */
1575   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1576       && can_create_pseudo_p ()
1577       && (!REG_P (op1)
1578          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1579     operands[2] = force_reg (<MODE>mode, operands[2]);
1580   /* Expand polynomial additions now if the destination is the stack
1581      pointer, since we don't want to use that as a temporary.  */
1582   else if (operands[0] == stack_pointer_rtx
1583            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1584     {
1585       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1586                                 operands[2], NULL_RTX, NULL_RTX);
1587       DONE;
1588     }
1589 })
1590
1591 (define_insn "*add<mode>3_aarch64"
1592   [(set
1593     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1594     (plus:GPI
1595      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1596      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1597   ""
1598   "@
1599   add\\t%<w>0, %<w>1, %2
1600   add\\t%<w>0, %<w>1, %<w>2
1601   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1602   sub\\t%<w>0, %<w>1, #%n2
1603   #
1604   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1605   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1606   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1607    (set_attr "simd" "*,*,yes,*,*,*")]
1608 )
1609
1610 ;; zero_extend version of above
1611 (define_insn "*addsi3_aarch64_uxtw"
1612   [(set
1613     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1614     (zero_extend:DI
1615      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1616               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1617   ""
1618   "@
1619   add\\t%w0, %w1, %2
1620   add\\t%w0, %w1, %w2
1621   sub\\t%w0, %w1, #%n2
1622   #"
1623   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1624 )
1625
1626 ;; If there's a free register, and we can load the constant with a
1627 ;; single instruction, do so.  This has a chance to improve scheduling.
1628 (define_peephole2
1629   [(match_scratch:GPI 3 "r")
1630    (set (match_operand:GPI 0 "register_operand")
1631         (plus:GPI
1632           (match_operand:GPI 1 "register_operand")
1633           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1634   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1635   [(set (match_dup 3) (match_dup 2))
1636    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1637 )
1638
1639 (define_peephole2
1640   [(match_scratch:SI 3 "r")
1641    (set (match_operand:DI 0 "register_operand")
1642         (zero_extend:DI
1643           (plus:SI
1644             (match_operand:SI 1 "register_operand")
1645             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1646   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1647   [(set (match_dup 3) (match_dup 2))
1648    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1649 )
1650
1651 ;; After peephole2 has had a chance to run, split any remaining long
1652 ;; additions into two add immediates.
1653 (define_split
1654   [(set (match_operand:GPI 0 "register_operand")
1655         (plus:GPI
1656           (match_operand:GPI 1 "register_operand")
1657           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1658   "epilogue_completed"
1659   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1660    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1661   {
1662     HOST_WIDE_INT i = INTVAL (operands[2]);
1663     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1664     operands[3] = GEN_INT (i - s);
1665     operands[4] = GEN_INT (s);
1666   }
1667 )
1668
1669 ;; Match addition of polynomial offsets that require one temporary, for which
1670 ;; we can use the early-clobbered destination register.  This is a separate
1671 ;; pattern so that the early clobber doesn't affect register allocation
1672 ;; for other forms of addition.  However, we still need to provide an
1673 ;; all-register alternative, in case the offset goes out of range after
1674 ;; elimination.  For completeness we might as well provide all GPR-based
1675 ;; alternatives from the main pattern.
1676 ;;
1677 ;; We don't have a pattern for additions requiring two temporaries since at
1678 ;; present LRA doesn't allow new scratches to be added during elimination.
1679 ;; Such offsets should be rare anyway.
1680 ;;
1681 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1682 ;; here would go away.  We could just handle all polynomial constants in
1683 ;; this pattern.
1684 (define_insn_and_split "*add<mode>3_poly_1"
1685   [(set
1686     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1687     (plus:GPI
1688      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1689      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1690   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1691   "@
1692   add\\t%<w>0, %<w>1, %2
1693   add\\t%<w>0, %<w>1, %<w>2
1694   sub\\t%<w>0, %<w>1, #%n2
1695   #
1696   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1697   #"
1698   "&& epilogue_completed
1699    && !reg_overlap_mentioned_p (operands[0], operands[1])
1700    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1701   [(const_int 0)]
1702   {
1703     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1704                               operands[2], operands[0], NULL_RTX);
1705     DONE;
1706   }
1707   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1708   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1709 )
1710
1711 (define_split
1712   [(set (match_operand:DI 0 "register_operand")
1713         (zero_extend:DI
1714           (plus:SI
1715             (match_operand:SI 1 "register_operand")
1716             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1717   "epilogue_completed"
1718   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1719    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1720   {
1721     HOST_WIDE_INT i = INTVAL (operands[2]);
1722     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1723     operands[3] = GEN_INT (i - s);
1724     operands[4] = GEN_INT (s);
1725     operands[5] = gen_lowpart (SImode, operands[0]);
1726   }
1727 )
1728
1729 (define_expand "addti3"
1730   [(set (match_operand:TI 0 "register_operand" "")
1731         (plus:TI (match_operand:TI 1 "register_operand" "")
1732                  (match_operand:TI 2 "register_operand" "")))]
1733   ""
1734 {
1735   rtx low = gen_reg_rtx (DImode);
1736   emit_insn (gen_adddi3_compareC (low, gen_lowpart (DImode, operands[1]),
1737                                   gen_lowpart (DImode, operands[2])));
1738
1739   rtx high = gen_reg_rtx (DImode);
1740   emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1741                                  gen_highpart (DImode, operands[2])));
1742
1743   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1744   emit_move_insn (gen_highpart (DImode, operands[0]), high);
1745   DONE;
1746 })
1747
1748 (define_insn "add<mode>3_compare0"
1749   [(set (reg:CC_NZ CC_REGNUM)
1750         (compare:CC_NZ
1751          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1752                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1753          (const_int 0)))
1754    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1755         (plus:GPI (match_dup 1) (match_dup 2)))]
1756   ""
1757   "@
1758   adds\\t%<w>0, %<w>1, %<w>2
1759   adds\\t%<w>0, %<w>1, %2
1760   subs\\t%<w>0, %<w>1, #%n2"
1761   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1762 )
1763
1764 ;; zero_extend version of above
1765 (define_insn "*addsi3_compare0_uxtw"
1766   [(set (reg:CC_NZ CC_REGNUM)
1767         (compare:CC_NZ
1768          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1769                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1770          (const_int 0)))
1771    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1772         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1773   ""
1774   "@
1775   adds\\t%w0, %w1, %w2
1776   adds\\t%w0, %w1, %2
1777   subs\\t%w0, %w1, #%n2"
1778   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1779 )
1780
1781 (define_insn "*add<mode>3_compareC_cconly_imm"
1782   [(set (reg:CC_C CC_REGNUM)
1783         (ne:CC_C
1784           (plus:<DWI>
1785             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1786             (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1787           (zero_extend:<DWI>
1788             (plus:GPI
1789               (match_dup 0)
1790               (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1791   "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1792                                  <MODE>mode, operands[1])"
1793   "@
1794   cmn\\t%<w>0, %1
1795   cmp\\t%<w>0, #%n1"
1796   [(set_attr "type" "alus_imm")]
1797 )
1798
1799 (define_insn "*add<mode>3_compareC_cconly"
1800   [(set (reg:CC_C CC_REGNUM)
1801         (ne:CC_C
1802           (plus:<DWI>
1803             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1804             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1805           (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1806   ""
1807   "cmn\\t%<w>0, %<w>1"
1808   [(set_attr "type" "alus_sreg")]
1809 )
1810
1811 (define_insn "*add<mode>3_compareC_imm"
1812   [(set (reg:CC_C CC_REGNUM)
1813         (ne:CC_C
1814           (plus:<DWI>
1815             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1816             (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1817           (zero_extend:<DWI>
1818             (plus:GPI
1819               (match_dup 1)
1820               (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1821    (set (match_operand:GPI 0 "register_operand" "=r,r")
1822         (plus:GPI (match_dup 1) (match_dup 2)))]
1823   "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1824                                  <MODE>mode, operands[2])"
1825   "@
1826   adds\\t%<w>0, %<w>1, %2
1827   subs\\t%<w>0, %<w>1, #%n2"
1828   [(set_attr "type" "alus_imm")]
1829 )
1830
1831 (define_insn "add<mode>3_compareC"
1832   [(set (reg:CC_C CC_REGNUM)
1833         (ne:CC_C
1834           (plus:<DWI>
1835             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1836             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1837           (zero_extend:<DWI>
1838             (plus:GPI (match_dup 1) (match_dup 2)))))
1839    (set (match_operand:GPI 0 "register_operand" "=r")
1840         (plus:GPI (match_dup 1) (match_dup 2)))]
1841   ""
1842   "adds\\t%<w>0, %<w>1, %<w>2"
1843   [(set_attr "type" "alus_sreg")]
1844 )
1845
1846 (define_insn "*adds_shift_imm_<mode>"
1847   [(set (reg:CC_NZ CC_REGNUM)
1848         (compare:CC_NZ
1849          (plus:GPI (ASHIFT:GPI
1850                     (match_operand:GPI 1 "register_operand" "r")
1851                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1852                    (match_operand:GPI 3 "register_operand" "r"))
1853          (const_int 0)))
1854    (set (match_operand:GPI 0 "register_operand" "=r")
1855         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1856                   (match_dup 3)))]
1857   ""
1858   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1859   [(set_attr "type" "alus_shift_imm")]
1860 )
1861
1862 (define_insn "*subs_shift_imm_<mode>"
1863   [(set (reg:CC_NZ CC_REGNUM)
1864         (compare:CC_NZ
1865          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1866                     (ASHIFT:GPI
1867                      (match_operand:GPI 2 "register_operand" "r")
1868                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1869          (const_int 0)))
1870    (set (match_operand:GPI 0 "register_operand" "=r")
1871         (minus:GPI (match_dup 1)
1872                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1873   ""
1874   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1875   [(set_attr "type" "alus_shift_imm")]
1876 )
1877
1878 (define_insn "*adds_mul_imm_<mode>"
1879   [(set (reg:CC_NZ CC_REGNUM)
1880         (compare:CC_NZ
1881          (plus:GPI (mult:GPI
1882                     (match_operand:GPI 1 "register_operand" "r")
1883                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1884                    (match_operand:GPI 3 "register_operand" "r"))
1885          (const_int 0)))
1886    (set (match_operand:GPI 0 "register_operand" "=r")
1887         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1888                   (match_dup 3)))]
1889   ""
1890   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1891   [(set_attr "type" "alus_shift_imm")]
1892 )
1893
1894 (define_insn "*subs_mul_imm_<mode>"
1895   [(set (reg:CC_NZ CC_REGNUM)
1896         (compare:CC_NZ
1897          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1898                     (mult:GPI
1899                      (match_operand:GPI 2 "register_operand" "r")
1900                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1901          (const_int 0)))
1902    (set (match_operand:GPI 0 "register_operand" "=r")
1903         (minus:GPI (match_dup 1)
1904                    (mult:GPI (match_dup 2) (match_dup 3))))]
1905   ""
1906   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1907   [(set_attr "type" "alus_shift_imm")]
1908 )
1909
1910 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1911   [(set (reg:CC_NZ CC_REGNUM)
1912         (compare:CC_NZ
1913          (plus:GPI
1914           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1915           (match_operand:GPI 2 "register_operand" "r"))
1916         (const_int 0)))
1917    (set (match_operand:GPI 0 "register_operand" "=r")
1918         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1919   ""
1920   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1921   [(set_attr "type" "alus_ext")]
1922 )
1923
1924 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1925   [(set (reg:CC_NZ CC_REGNUM)
1926         (compare:CC_NZ
1927          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1928                     (ANY_EXTEND:GPI
1929                      (match_operand:ALLX 2 "register_operand" "r")))
1930         (const_int 0)))
1931    (set (match_operand:GPI 0 "register_operand" "=r")
1932         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1933   ""
1934   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1935   [(set_attr "type" "alus_ext")]
1936 )
1937
1938 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1939   [(set (reg:CC_NZ CC_REGNUM)
1940         (compare:CC_NZ
1941          (plus:GPI (ashift:GPI
1942                     (ANY_EXTEND:GPI
1943                      (match_operand:ALLX 1 "register_operand" "r"))
1944                     (match_operand 2 "aarch64_imm3" "Ui3"))
1945                    (match_operand:GPI 3 "register_operand" "r"))
1946          (const_int 0)))
1947    (set (match_operand:GPI 0 "register_operand" "=rk")
1948         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1949                               (match_dup 2))
1950                   (match_dup 3)))]
1951   ""
1952   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1953   [(set_attr "type" "alus_ext")]
1954 )
1955
1956 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1957   [(set (reg:CC_NZ CC_REGNUM)
1958         (compare:CC_NZ
1959          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1960                     (ashift:GPI
1961                      (ANY_EXTEND:GPI
1962                       (match_operand:ALLX 2 "register_operand" "r"))
1963                      (match_operand 3 "aarch64_imm3" "Ui3")))
1964          (const_int 0)))
1965    (set (match_operand:GPI 0 "register_operand" "=rk")
1966         (minus:GPI (match_dup 1)
1967                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
1968                                (match_dup 3))))]
1969   ""
1970   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
1971   [(set_attr "type" "alus_ext")]
1972 )
1973
1974 (define_insn "*adds_<optab><mode>_multp2"
1975   [(set (reg:CC_NZ CC_REGNUM)
1976         (compare:CC_NZ
1977          (plus:GPI (ANY_EXTRACT:GPI
1978                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1979                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1980                     (match_operand 3 "const_int_operand" "n")
1981                     (const_int 0))
1982                    (match_operand:GPI 4 "register_operand" "r"))
1983         (const_int 0)))
1984    (set (match_operand:GPI 0 "register_operand" "=r")
1985         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
1986                                    (match_dup 3)
1987                                    (const_int 0))
1988                   (match_dup 4)))]
1989   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1990   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1991   [(set_attr "type" "alus_ext")]
1992 )
1993
1994 (define_insn "*subs_<optab><mode>_multp2"
1995   [(set (reg:CC_NZ CC_REGNUM)
1996         (compare:CC_NZ
1997          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
1998                     (ANY_EXTRACT:GPI
1999                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2000                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2001                      (match_operand 3 "const_int_operand" "n")
2002                      (const_int 0)))
2003         (const_int 0)))
2004    (set (match_operand:GPI 0 "register_operand" "=r")
2005         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2006                                   (mult:GPI (match_dup 1) (match_dup 2))
2007                                   (match_dup 3)
2008                                   (const_int 0))))]
2009   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2010   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2011   [(set_attr "type" "alus_ext")]
2012 )
2013
2014 (define_insn "*add<mode>3nr_compare0"
2015   [(set (reg:CC_NZ CC_REGNUM)
2016         (compare:CC_NZ
2017          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2018                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2019          (const_int 0)))]
2020   ""
2021   "@
2022   cmn\\t%<w>0, %<w>1
2023   cmn\\t%<w>0, %1
2024   cmp\\t%<w>0, #%n1"
2025   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2026 )
2027
2028 (define_insn "aarch64_sub<mode>_compare0"
2029   [(set (reg:CC_NZ CC_REGNUM)
2030         (compare:CC_NZ
2031          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2032                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2033          (const_int 0)))]
2034   ""
2035   "cmp\\t%<w>0, %<w>1"
2036   [(set_attr "type" "alus_sreg")]
2037 )
2038
2039 (define_insn "*compare_neg<mode>"
2040   [(set (reg:CC_Z CC_REGNUM)
2041         (compare:CC_Z
2042          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2043          (match_operand:GPI 1 "register_operand" "r")))]
2044   ""
2045   "cmn\\t%<w>1, %<w>0"
2046   [(set_attr "type" "alus_sreg")]
2047 )
2048
2049 (define_insn "*add_<shift>_<mode>"
2050   [(set (match_operand:GPI 0 "register_operand" "=r")
2051         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2052                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2053                   (match_operand:GPI 3 "register_operand" "r")))]
2054   ""
2055   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2056   [(set_attr "type" "alu_shift_imm")]
2057 )
2058
2059 ;; zero_extend version of above
2060 (define_insn "*add_<shift>_si_uxtw"
2061   [(set (match_operand:DI 0 "register_operand" "=r")
2062         (zero_extend:DI
2063          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2064                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2065                   (match_operand:SI 3 "register_operand" "r"))))]
2066   ""
2067   "add\\t%w0, %w3, %w1, <shift> %2"
2068   [(set_attr "type" "alu_shift_imm")]
2069 )
2070
2071 (define_insn "*add_mul_imm_<mode>"
2072   [(set (match_operand:GPI 0 "register_operand" "=r")
2073         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2074                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2075                   (match_operand:GPI 3 "register_operand" "r")))]
2076   ""
2077   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2078   [(set_attr "type" "alu_shift_imm")]
2079 )
2080
2081 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2082   [(set (match_operand:GPI 0 "register_operand" "=rk")
2083         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2084                   (match_operand:GPI 2 "register_operand" "r")))]
2085   ""
2086   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2087   [(set_attr "type" "alu_ext")]
2088 )
2089
2090 ;; zero_extend version of above
2091 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2092   [(set (match_operand:DI 0 "register_operand" "=rk")
2093         (zero_extend:DI
2094          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2095                   (match_operand:GPI 2 "register_operand" "r"))))]
2096   ""
2097   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2098   [(set_attr "type" "alu_ext")]
2099 )
2100
2101 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2102   [(set (match_operand:GPI 0 "register_operand" "=rk")
2103         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2104                                (match_operand:ALLX 1 "register_operand" "r"))
2105                               (match_operand 2 "aarch64_imm3" "Ui3"))
2106                   (match_operand:GPI 3 "register_operand" "r")))]
2107   ""
2108   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2109   [(set_attr "type" "alu_ext")]
2110 )
2111
2112 ;; zero_extend version of above
2113 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2114   [(set (match_operand:DI 0 "register_operand" "=rk")
2115         (zero_extend:DI
2116          (plus:SI (ashift:SI (ANY_EXTEND:SI
2117                               (match_operand:SHORT 1 "register_operand" "r"))
2118                              (match_operand 2 "aarch64_imm3" "Ui3"))
2119                   (match_operand:SI 3 "register_operand" "r"))))]
2120   ""
2121   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2122   [(set_attr "type" "alu_ext")]
2123 )
2124
2125 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2126   [(set (match_operand:GPI 0 "register_operand" "=rk")
2127         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2128                              (match_operand:ALLX 1 "register_operand" "r"))
2129                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2130                   (match_operand:GPI 3 "register_operand" "r")))]
2131   ""
2132   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2133   [(set_attr "type" "alu_ext")]
2134 )
2135
2136 ;; zero_extend version of above
2137 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2138   [(set (match_operand:DI 0 "register_operand" "=rk")
2139         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2140                              (match_operand:SHORT 1 "register_operand" "r"))
2141                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2142                   (match_operand:SI 3 "register_operand" "r"))))]
2143   ""
2144   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2145   [(set_attr "type" "alu_ext")]
2146 )
2147
2148 (define_insn "*add_<optab><mode>_multp2"
2149   [(set (match_operand:GPI 0 "register_operand" "=rk")
2150         (plus:GPI (ANY_EXTRACT:GPI
2151                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2152                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2153                    (match_operand 3 "const_int_operand" "n")
2154                    (const_int 0))
2155                   (match_operand:GPI 4 "register_operand" "r")))]
2156   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2157   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2158   [(set_attr "type" "alu_ext")]
2159 )
2160
2161 ;; zero_extend version of above
2162 (define_insn "*add_<optab>si_multp2_uxtw"
2163   [(set (match_operand:DI 0 "register_operand" "=rk")
2164         (zero_extend:DI
2165          (plus:SI (ANY_EXTRACT:SI
2166                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2167                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2168                    (match_operand 3 "const_int_operand" "n")
2169                    (const_int 0))
2170                   (match_operand:SI 4 "register_operand" "r"))))]
2171   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2172   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2173   [(set_attr "type" "alu_ext")]
2174 )
2175
2176 (define_expand "add<mode>3_carryin"
2177   [(set (match_operand:GPI 0 "register_operand")
2178         (plus:GPI
2179           (plus:GPI
2180             (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2181             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2182           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2183    ""
2184    ""
2185 )
2186
2187 ;; Note that add with carry with two zero inputs is matched by cset,
2188 ;; and that add with carry with one zero input is matched by cinc.
2189
2190 (define_insn "*add<mode>3_carryin"
2191   [(set (match_operand:GPI 0 "register_operand" "=r")
2192         (plus:GPI
2193           (plus:GPI
2194             (match_operand:GPI 3 "aarch64_carry_operation" "")
2195             (match_operand:GPI 1 "register_operand" "r"))
2196           (match_operand:GPI 2 "register_operand" "r")))]
2197    ""
2198    "adc\\t%<w>0, %<w>1, %<w>2"
2199   [(set_attr "type" "adc_reg")]
2200 )
2201
2202 ;; zero_extend version of above
2203 (define_insn "*addsi3_carryin_uxtw"
2204   [(set (match_operand:DI 0 "register_operand" "=r")
2205         (zero_extend:DI
2206           (plus:SI
2207             (plus:SI
2208               (match_operand:SI 3 "aarch64_carry_operation" "")
2209               (match_operand:SI 1 "register_operand" "r"))
2210             (match_operand:SI 2 "register_operand" "r"))))]
2211    ""
2212    "adc\\t%w0, %w1, %w2"
2213   [(set_attr "type" "adc_reg")]
2214 )
2215
2216 (define_insn "*add_uxt<mode>_shift2"
2217   [(set (match_operand:GPI 0 "register_operand" "=rk")
2218         (plus:GPI (and:GPI
2219                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2220                                (match_operand 2 "aarch64_imm3" "Ui3"))
2221                    (match_operand 3 "const_int_operand" "n"))
2222                   (match_operand:GPI 4 "register_operand" "r")))]
2223   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2224   "*
2225   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2226                                            INTVAL (operands[3])));
2227   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2228   [(set_attr "type" "alu_ext")]
2229 )
2230
2231 ;; zero_extend version of above
2232 (define_insn "*add_uxtsi_shift2_uxtw"
2233   [(set (match_operand:DI 0 "register_operand" "=rk")
2234         (zero_extend:DI
2235          (plus:SI (and:SI
2236                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2237                               (match_operand 2 "aarch64_imm3" "Ui3"))
2238                    (match_operand 3 "const_int_operand" "n"))
2239                   (match_operand:SI 4 "register_operand" "r"))))]
2240   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2241   "*
2242   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2243                                            INTVAL (operands[3])));
2244   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2245   [(set_attr "type" "alu_ext")]
2246 )
2247
2248 (define_insn "*add_uxt<mode>_multp2"
2249   [(set (match_operand:GPI 0 "register_operand" "=rk")
2250         (plus:GPI (and:GPI
2251                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2252                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2253                    (match_operand 3 "const_int_operand" "n"))
2254                   (match_operand:GPI 4 "register_operand" "r")))]
2255   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2256   "*
2257   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2258                                            INTVAL (operands[3])));
2259   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2260   [(set_attr "type" "alu_ext")]
2261 )
2262
2263 ;; zero_extend version of above
2264 (define_insn "*add_uxtsi_multp2_uxtw"
2265   [(set (match_operand:DI 0 "register_operand" "=rk")
2266         (zero_extend:DI
2267          (plus:SI (and:SI
2268                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2269                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2270                    (match_operand 3 "const_int_operand" "n"))
2271                   (match_operand:SI 4 "register_operand" "r"))))]
2272   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2273   "*
2274   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2275                                            INTVAL (operands[3])));
2276   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2277   [(set_attr "type" "alu_ext")]
2278 )
2279
2280 (define_insn "subsi3"
2281   [(set (match_operand:SI 0 "register_operand" "=rk")
2282         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2283                   (match_operand:SI 2 "register_operand" "r")))]
2284   ""
2285   "sub\\t%w0, %w1, %w2"
2286   [(set_attr "type" "alu_sreg")]
2287 )
2288
2289 ;; zero_extend version of above
2290 (define_insn "*subsi3_uxtw"
2291   [(set (match_operand:DI 0 "register_operand" "=rk")
2292         (zero_extend:DI
2293          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2294                    (match_operand:SI 2 "register_operand" "r"))))]
2295   ""
2296   "sub\\t%w0, %w1, %w2"
2297   [(set_attr "type" "alu_sreg")]
2298 )
2299
2300 (define_insn "subdi3"
2301   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2302         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2303                   (match_operand:DI 2 "register_operand" "r,w")))]
2304   ""
2305   "@
2306    sub\\t%x0, %x1, %x2
2307    sub\\t%d0, %d1, %d2"
2308   [(set_attr "type" "alu_sreg, neon_sub")
2309    (set_attr "simd" "*,yes")]
2310 )
2311
2312 (define_expand "subti3"
2313   [(set (match_operand:TI 0 "register_operand" "")
2314         (minus:TI (match_operand:TI 1 "register_operand" "")
2315                   (match_operand:TI 2 "register_operand" "")))]
2316   ""
2317 {
2318   rtx low = gen_reg_rtx (DImode);
2319   emit_insn (gen_subdi3_compare1 (low, gen_lowpart (DImode, operands[1]),
2320                                   gen_lowpart (DImode, operands[2])));
2321
2322   rtx high = gen_reg_rtx (DImode);
2323   emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2324                                  gen_highpart (DImode, operands[2])));
2325
2326   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2327   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2328   DONE;
2329 })
2330
2331 (define_insn "*sub<mode>3_compare0"
2332   [(set (reg:CC_NZ CC_REGNUM)
2333         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2334                                   (match_operand:GPI 2 "register_operand" "r"))
2335                        (const_int 0)))
2336    (set (match_operand:GPI 0 "register_operand" "=r")
2337         (minus:GPI (match_dup 1) (match_dup 2)))]
2338   ""
2339   "subs\\t%<w>0, %<w>1, %<w>2"
2340   [(set_attr "type" "alus_sreg")]
2341 )
2342
2343 ;; zero_extend version of above
2344 (define_insn "*subsi3_compare0_uxtw"
2345   [(set (reg:CC_NZ CC_REGNUM)
2346         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2347                                  (match_operand:SI 2 "register_operand" "r"))
2348                        (const_int 0)))
2349    (set (match_operand:DI 0 "register_operand" "=r")
2350         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2351   ""
2352   "subs\\t%w0, %w1, %w2"
2353   [(set_attr "type" "alus_sreg")]
2354 )
2355
2356 (define_insn "sub<mode>3_compare1"
2357   [(set (reg:CC CC_REGNUM)
2358         (compare:CC
2359           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2360           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2361    (set (match_operand:GPI 0 "register_operand" "=r")
2362         (minus:GPI (match_dup 1) (match_dup 2)))]
2363   ""
2364   "subs\\t%<w>0, %<w>1, %<w>2"
2365   [(set_attr "type" "alus_sreg")]
2366 )
2367
2368 (define_insn "sub<mode>3_compare1_imm"
2369   [(set (reg:CC CC_REGNUM)
2370         (compare:CC
2371           (match_operand:GPI 1 "register_operand" "r")
2372           (match_operand:GPI 3 "const_int_operand" "n")))
2373    (set (match_operand:GPI 0 "register_operand" "=r")
2374         (plus:GPI (match_dup 1)
2375                   (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2376   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2377   "subs\\t%<w>0, %<w>1, #%n2"
2378   [(set_attr "type" "alus_sreg")]
2379 )
2380
2381 (define_peephole2
2382   [(set (match_operand:GPI 0 "register_operand")
2383         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2384                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
2385    (set (reg:CC CC_REGNUM)
2386         (compare:CC
2387           (match_dup 1)
2388           (match_dup 2)))]
2389   "!reg_overlap_mentioned_p (operands[0], operands[1])
2390    && !reg_overlap_mentioned_p (operands[0], operands[2])"
2391   [(const_int 0)]
2392   {
2393     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2394                                          operands[2]));
2395     DONE;
2396   }
2397 )
2398
2399 ;; Same as the above peephole but with the compare and minus in
2400 ;; swapped order.  The restriction on overlap between operand 0
2401 ;; and operands 1 and 2 doesn't apply here.
2402 (define_peephole2
2403   [(set (reg:CC CC_REGNUM)
2404         (compare:CC
2405           (match_operand:GPI 1 "aarch64_reg_or_zero")
2406           (match_operand:GPI 2 "aarch64_reg_or_zero")))
2407    (set (match_operand:GPI 0 "register_operand")
2408         (minus:GPI (match_dup 1)
2409                    (match_dup 2)))]
2410   ""
2411   [(const_int 0)]
2412   {
2413     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2414                                          operands[2]));
2415     DONE;
2416   }
2417 )
2418
2419 (define_peephole2
2420   [(set (match_operand:GPI 0 "register_operand")
2421         (plus:GPI (match_operand:GPI 1 "register_operand")
2422                   (match_operand:GPI 2 "aarch64_sub_immediate")))
2423    (set (reg:CC CC_REGNUM)
2424         (compare:CC
2425           (match_dup 1)
2426           (match_operand:GPI 3 "const_int_operand")))]
2427   "!reg_overlap_mentioned_p (operands[0], operands[1])
2428    && INTVAL (operands[3]) == -INTVAL (operands[2])"
2429   [(const_int 0)]
2430   {
2431     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2432                                          operands[2], operands[3]));
2433     DONE;
2434   }
2435 )
2436
2437 ;; Same as the above peephole but with the compare and minus in
2438 ;; swapped order.  The restriction on overlap between operand 0
2439 ;; and operands 1 doesn't apply here.
2440 (define_peephole2
2441   [(set (reg:CC CC_REGNUM)
2442         (compare:CC
2443           (match_operand:GPI 1 "register_operand")
2444           (match_operand:GPI 3 "const_int_operand")))
2445    (set (match_operand:GPI 0 "register_operand")
2446         (plus:GPI (match_dup 1)
2447                   (match_operand:GPI 2 "aarch64_sub_immediate")))]
2448   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2449   [(const_int 0)]
2450   {
2451     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2452                                          operands[2], operands[3]));
2453     DONE;
2454   }
2455 )
2456
2457 (define_insn "*sub_<shift>_<mode>"
2458   [(set (match_operand:GPI 0 "register_operand" "=r")
2459         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2460                    (ASHIFT:GPI
2461                     (match_operand:GPI 1 "register_operand" "r")
2462                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2463   ""
2464   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2465   [(set_attr "type" "alu_shift_imm")]
2466 )
2467
2468 ;; zero_extend version of above
2469 (define_insn "*sub_<shift>_si_uxtw"
2470   [(set (match_operand:DI 0 "register_operand" "=r")
2471         (zero_extend:DI
2472          (minus:SI (match_operand:SI 3 "register_operand" "r")
2473                    (ASHIFT:SI
2474                     (match_operand:SI 1 "register_operand" "r")
2475                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2476   ""
2477   "sub\\t%w0, %w3, %w1, <shift> %2"
2478   [(set_attr "type" "alu_shift_imm")]
2479 )
2480
2481 (define_insn "*sub_mul_imm_<mode>"
2482   [(set (match_operand:GPI 0 "register_operand" "=r")
2483         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2484                    (mult:GPI
2485                     (match_operand:GPI 1 "register_operand" "r")
2486                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2487   ""
2488   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2489   [(set_attr "type" "alu_shift_imm")]
2490 )
2491
2492 ;; zero_extend version of above
2493 (define_insn "*sub_mul_imm_si_uxtw"
2494   [(set (match_operand:DI 0 "register_operand" "=r")
2495         (zero_extend:DI
2496          (minus:SI (match_operand:SI 3 "register_operand" "r")
2497                    (mult:SI
2498                     (match_operand:SI 1 "register_operand" "r")
2499                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2500   ""
2501   "sub\\t%w0, %w3, %w1, lsl %p2"
2502   [(set_attr "type" "alu_shift_imm")]
2503 )
2504
2505 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2506   [(set (match_operand:GPI 0 "register_operand" "=rk")
2507         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2508                    (ANY_EXTEND:GPI
2509                     (match_operand:ALLX 2 "register_operand" "r"))))]
2510   ""
2511   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2512   [(set_attr "type" "alu_ext")]
2513 )
2514
2515 ;; zero_extend version of above
2516 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2517   [(set (match_operand:DI 0 "register_operand" "=rk")
2518         (zero_extend:DI
2519          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2520                    (ANY_EXTEND:SI
2521                     (match_operand:SHORT 2 "register_operand" "r")))))]
2522   ""
2523   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2524   [(set_attr "type" "alu_ext")]
2525 )
2526
2527 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2528   [(set (match_operand:GPI 0 "register_operand" "=rk")
2529         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2530                    (ashift:GPI (ANY_EXTEND:GPI
2531                                 (match_operand:ALLX 2 "register_operand" "r"))
2532                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2533   ""
2534   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2535   [(set_attr "type" "alu_ext")]
2536 )
2537
2538 ;; zero_extend version of above
2539 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2540   [(set (match_operand:DI 0 "register_operand" "=rk")
2541         (zero_extend:DI
2542          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2543                    (ashift:SI (ANY_EXTEND:SI
2544                                (match_operand:SHORT 2 "register_operand" "r"))
2545                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2546   ""
2547   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2548   [(set_attr "type" "alu_ext")]
2549 )
2550
2551 (define_insn "*sub_<optab><mode>_multp2"
2552   [(set (match_operand:GPI 0 "register_operand" "=rk")
2553         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2554                    (ANY_EXTRACT:GPI
2555                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2556                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2557                     (match_operand 3 "const_int_operand" "n")
2558                     (const_int 0))))]
2559   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2560   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2561   [(set_attr "type" "alu_ext")]
2562 )
2563
2564 ;; zero_extend version of above
2565 (define_insn "*sub_<optab>si_multp2_uxtw"
2566   [(set (match_operand:DI 0 "register_operand" "=rk")
2567         (zero_extend:DI
2568          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2569                    (ANY_EXTRACT:SI
2570                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2571                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2572                     (match_operand 3 "const_int_operand" "n")
2573                     (const_int 0)))))]
2574   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2575   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2576   [(set_attr "type" "alu_ext")]
2577 )
2578
2579 ;; The hardware description is op1 + ~op2 + C.
2580 ;;                           = op1 + (-op2 + 1) + (1 - !C)
2581 ;;                           = op1 - op2 - 1 + 1 - !C
2582 ;;                           = op1 - op2 - !C.
2583 ;; We describe the latter.
2584
2585 (define_insn "*sub<mode>3_carryin0"
2586   [(set (match_operand:GPI 0 "register_operand" "=r")
2587         (minus:GPI
2588           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2589           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
2590    ""
2591    "sbc\\t%<w>0, %<w>1, <w>zr"
2592   [(set_attr "type" "adc_reg")]
2593 )
2594
2595 ;; zero_extend version of the above
2596 (define_insn "*subsi3_carryin_uxtw"
2597   [(set (match_operand:DI 0 "register_operand" "=r")
2598         (zero_extend:DI
2599           (minus:SI
2600             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2601             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
2602    ""
2603    "sbc\\t%w0, %w1, wzr"
2604   [(set_attr "type" "adc_reg")]
2605 )
2606
2607 (define_expand "sub<mode>3_carryin"
2608   [(set (match_operand:GPI 0 "register_operand")
2609         (minus:GPI
2610           (minus:GPI
2611             (match_operand:GPI 1 "aarch64_reg_or_zero")
2612             (match_operand:GPI 2 "register_operand"))
2613           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2614    ""
2615    ""
2616 )
2617
2618 (define_insn "*sub<mode>3_carryin"
2619   [(set (match_operand:GPI 0 "register_operand" "=r")
2620         (minus:GPI
2621           (minus:GPI
2622             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2623             (match_operand:GPI 2 "register_operand" "r"))
2624           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
2625
2626    ""
2627    "sbc\\t%<w>0, %<w>1, %<w>2"
2628   [(set_attr "type" "adc_reg")]
2629 )
2630
2631 ;; zero_extend version of the above
2632 (define_insn "*subsi3_carryin_uxtw"
2633   [(set (match_operand:DI 0 "register_operand" "=r")
2634         (zero_extend:DI
2635           (minus:SI
2636             (minus:SI
2637               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2638               (match_operand:SI 2 "register_operand" "r"))
2639             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
2640
2641    ""
2642    "sbc\\t%w0, %w1, %w2"
2643   [(set_attr "type" "adc_reg")]
2644 )
2645
2646 (define_insn "*sub<mode>3_carryin_alt"
2647   [(set (match_operand:GPI 0 "register_operand" "=r")
2648         (minus:GPI
2649           (minus:GPI
2650             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2651             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
2652           (match_operand:GPI 2 "register_operand" "r")))]
2653    ""
2654    "sbc\\t%<w>0, %<w>1, %<w>2"
2655   [(set_attr "type" "adc_reg")]
2656 )
2657
2658 ;; zero_extend version of the above
2659 (define_insn "*subsi3_carryin_alt_uxtw"
2660   [(set (match_operand:DI 0 "register_operand" "=r")
2661         (zero_extend:DI
2662           (minus:SI
2663             (minus:SI
2664               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2665               (match_operand:SI 3 "aarch64_borrow_operation" ""))
2666             (match_operand:SI 2 "register_operand" "r"))))]
2667    ""
2668    "sbc\\t%w0, %w1, %w2"
2669   [(set_attr "type" "adc_reg")]
2670 )
2671
2672 (define_insn "*sub_uxt<mode>_shift2"
2673   [(set (match_operand:GPI 0 "register_operand" "=rk")
2674         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2675                    (and:GPI
2676                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2677                                 (match_operand 2 "aarch64_imm3" "Ui3"))
2678                     (match_operand 3 "const_int_operand" "n"))))]
2679   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2680   "*
2681   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2682                                            INTVAL (operands[3])));
2683   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2684   [(set_attr "type" "alu_ext")]
2685 )
2686
2687 ;; zero_extend version of above
2688 (define_insn "*sub_uxtsi_shift2_uxtw"
2689   [(set (match_operand:DI 0 "register_operand" "=rk")
2690         (zero_extend:DI
2691          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2692                    (and:SI
2693                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
2694                                (match_operand 2 "aarch64_imm3" "Ui3"))
2695                     (match_operand 3 "const_int_operand" "n")))))]
2696   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2697   "*
2698   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2699                                            INTVAL (operands[3])));
2700   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2701   [(set_attr "type" "alu_ext")]
2702 )
2703
2704 (define_insn "*sub_uxt<mode>_multp2"
2705   [(set (match_operand:GPI 0 "register_operand" "=rk")
2706         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2707                    (and:GPI
2708                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2709                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2710                     (match_operand 3 "const_int_operand" "n"))))]
2711   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2712   "*
2713   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2714                                            INTVAL (operands[3])));
2715   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2716   [(set_attr "type" "alu_ext")]
2717 )
2718
2719 ;; zero_extend version of above
2720 (define_insn "*sub_uxtsi_multp2_uxtw"
2721   [(set (match_operand:DI 0 "register_operand" "=rk")
2722         (zero_extend:DI
2723          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2724                    (and:SI
2725                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2726                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2727                     (match_operand 3 "const_int_operand" "n")))))]
2728   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2729   "*
2730   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2731                                            INTVAL (operands[3])));
2732   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2733   [(set_attr "type" "alu_ext")]
2734 )
2735
2736 (define_expand "abs<mode>2"
2737   [(match_operand:GPI 0 "register_operand" "")
2738    (match_operand:GPI 1 "register_operand" "")]
2739   ""
2740   {
2741     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2742     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2743     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2744     DONE;
2745   }
2746 )
2747
2748 (define_insn "neg<mode>2"
2749   [(set (match_operand:GPI 0 "register_operand" "=r,w")
2750         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2751   ""
2752   "@
2753    neg\\t%<w>0, %<w>1
2754    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2755   [(set_attr "type" "alu_sreg, neon_neg<q>")
2756    (set_attr "simd" "*,yes")]
2757 )
2758
2759 ;; zero_extend version of above
2760 (define_insn "*negsi2_uxtw"
2761   [(set (match_operand:DI 0 "register_operand" "=r")
2762         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2763   ""
2764   "neg\\t%w0, %w1"
2765   [(set_attr "type" "alu_sreg")]
2766 )
2767
2768 (define_insn "*ngc<mode>"
2769   [(set (match_operand:GPI 0 "register_operand" "=r")
2770         (minus:GPI
2771           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
2772           (match_operand:GPI 1 "register_operand" "r")))]
2773   ""
2774   "ngc\\t%<w>0, %<w>1"
2775   [(set_attr "type" "adc_reg")]
2776 )
2777
2778 (define_insn "*ngcsi_uxtw"
2779   [(set (match_operand:DI 0 "register_operand" "=r")
2780         (zero_extend:DI
2781           (minus:SI
2782             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
2783             (match_operand:SI 1 "register_operand" "r"))))]
2784   ""
2785   "ngc\\t%w0, %w1"
2786   [(set_attr "type" "adc_reg")]
2787 )
2788
2789 (define_insn "neg<mode>2_compare0"
2790   [(set (reg:CC_NZ CC_REGNUM)
2791         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2792                        (const_int 0)))
2793    (set (match_operand:GPI 0 "register_operand" "=r")
2794         (neg:GPI (match_dup 1)))]
2795   ""
2796   "negs\\t%<w>0, %<w>1"
2797   [(set_attr "type" "alus_sreg")]
2798 )
2799
2800 ;; zero_extend version of above
2801 (define_insn "*negsi2_compare0_uxtw"
2802   [(set (reg:CC_NZ CC_REGNUM)
2803         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2804                        (const_int 0)))
2805    (set (match_operand:DI 0 "register_operand" "=r")
2806         (zero_extend:DI (neg:SI (match_dup 1))))]
2807   ""
2808   "negs\\t%w0, %w1"
2809   [(set_attr "type" "alus_sreg")]
2810 )
2811
2812 (define_insn "*neg_<shift><mode>3_compare0"
2813   [(set (reg:CC_NZ CC_REGNUM)
2814         (compare:CC_NZ
2815          (neg:GPI (ASHIFT:GPI
2816                    (match_operand:GPI 1 "register_operand" "r")
2817                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2818          (const_int 0)))
2819    (set (match_operand:GPI 0 "register_operand" "=r")
2820         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2821   ""
2822   "negs\\t%<w>0, %<w>1, <shift> %2"
2823   [(set_attr "type" "alus_shift_imm")]
2824 )
2825
2826 (define_insn "*neg_<shift>_<mode>2"
2827   [(set (match_operand:GPI 0 "register_operand" "=r")
2828         (neg:GPI (ASHIFT:GPI
2829                   (match_operand:GPI 1 "register_operand" "r")
2830                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2831   ""
2832   "neg\\t%<w>0, %<w>1, <shift> %2"
2833   [(set_attr "type" "alu_shift_imm")]
2834 )
2835
2836 ;; zero_extend version of above
2837 (define_insn "*neg_<shift>_si2_uxtw"
2838   [(set (match_operand:DI 0 "register_operand" "=r")
2839         (zero_extend:DI
2840          (neg:SI (ASHIFT:SI
2841                   (match_operand:SI 1 "register_operand" "r")
2842                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2843   ""
2844   "neg\\t%w0, %w1, <shift> %2"
2845   [(set_attr "type" "alu_shift_imm")]
2846 )
2847
2848 (define_insn "*neg_mul_imm_<mode>2"
2849   [(set (match_operand:GPI 0 "register_operand" "=r")
2850         (neg:GPI (mult:GPI
2851                   (match_operand:GPI 1 "register_operand" "r")
2852                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2853   ""
2854   "neg\\t%<w>0, %<w>1, lsl %p2"
2855   [(set_attr "type" "alu_shift_imm")]
2856 )
2857
2858 ;; zero_extend version of above
2859 (define_insn "*neg_mul_imm_si2_uxtw"
2860   [(set (match_operand:DI 0 "register_operand" "=r")
2861         (zero_extend:DI
2862          (neg:SI (mult:SI
2863                   (match_operand:SI 1 "register_operand" "r")
2864                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2865   ""
2866   "neg\\t%w0, %w1, lsl %p2"
2867   [(set_attr "type" "alu_shift_imm")]
2868 )
2869
2870 (define_insn "mul<mode>3"
2871   [(set (match_operand:GPI 0 "register_operand" "=r")
2872         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2873                   (match_operand:GPI 2 "register_operand" "r")))]
2874   ""
2875   "mul\\t%<w>0, %<w>1, %<w>2"
2876   [(set_attr "type" "mul")]
2877 )
2878
2879 ;; zero_extend version of above
2880 (define_insn "*mulsi3_uxtw"
2881   [(set (match_operand:DI 0 "register_operand" "=r")
2882         (zero_extend:DI
2883          (mult:SI (match_operand:SI 1 "register_operand" "r")
2884                   (match_operand:SI 2 "register_operand" "r"))))]
2885   ""
2886   "mul\\t%w0, %w1, %w2"
2887   [(set_attr "type" "mul")]
2888 )
2889
2890 (define_insn "madd<mode>"
2891   [(set (match_operand:GPI 0 "register_operand" "=r")
2892         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2893                             (match_operand:GPI 2 "register_operand" "r"))
2894                   (match_operand:GPI 3 "register_operand" "r")))]
2895   ""
2896   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2897   [(set_attr "type" "mla")]
2898 )
2899
2900 ;; zero_extend version of above
2901 (define_insn "*maddsi_uxtw"
2902   [(set (match_operand:DI 0 "register_operand" "=r")
2903         (zero_extend:DI
2904          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2905                            (match_operand:SI 2 "register_operand" "r"))
2906                   (match_operand:SI 3 "register_operand" "r"))))]
2907   ""
2908   "madd\\t%w0, %w1, %w2, %w3"
2909   [(set_attr "type" "mla")]
2910 )
2911
2912 (define_insn "*msub<mode>"
2913   [(set (match_operand:GPI 0 "register_operand" "=r")
2914         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2915                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2916                              (match_operand:GPI 2 "register_operand" "r"))))]
2917
2918   ""
2919   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2920   [(set_attr "type" "mla")]
2921 )
2922
2923 ;; zero_extend version of above
2924 (define_insn "*msubsi_uxtw"
2925   [(set (match_operand:DI 0 "register_operand" "=r")
2926         (zero_extend:DI
2927          (minus:SI (match_operand:SI 3 "register_operand" "r")
2928                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2929                             (match_operand:SI 2 "register_operand" "r")))))]
2930
2931   ""
2932   "msub\\t%w0, %w1, %w2, %w3"
2933   [(set_attr "type" "mla")]
2934 )
2935
2936 (define_insn "*mul<mode>_neg"
2937   [(set (match_operand:GPI 0 "register_operand" "=r")
2938         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2939                   (match_operand:GPI 2 "register_operand" "r")))]
2940
2941   ""
2942   "mneg\\t%<w>0, %<w>1, %<w>2"
2943   [(set_attr "type" "mul")]
2944 )
2945
2946 ;; zero_extend version of above
2947 (define_insn "*mulsi_neg_uxtw"
2948   [(set (match_operand:DI 0 "register_operand" "=r")
2949         (zero_extend:DI
2950          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2951                   (match_operand:SI 2 "register_operand" "r"))))]
2952
2953   ""
2954   "mneg\\t%w0, %w1, %w2"
2955   [(set_attr "type" "mul")]
2956 )
2957
2958 (define_insn "<su_optab>mulsidi3"
2959   [(set (match_operand:DI 0 "register_operand" "=r")
2960         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2961                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2962   ""
2963   "<su>mull\\t%0, %w1, %w2"
2964   [(set_attr "type" "<su>mull")]
2965 )
2966
2967 (define_insn "<su_optab>maddsidi4"
2968   [(set (match_operand:DI 0 "register_operand" "=r")
2969         (plus:DI (mult:DI
2970                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2971                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
2972                  (match_operand:DI 3 "register_operand" "r")))]
2973   ""
2974   "<su>maddl\\t%0, %w1, %w2, %3"
2975   [(set_attr "type" "<su>mlal")]
2976 )
2977
2978 (define_insn "<su_optab>msubsidi4"
2979   [(set (match_operand:DI 0 "register_operand" "=r")
2980         (minus:DI
2981          (match_operand:DI 3 "register_operand" "r")
2982          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2983                   (ANY_EXTEND:DI
2984                    (match_operand:SI 2 "register_operand" "r")))))]
2985   ""
2986   "<su>msubl\\t%0, %w1, %w2, %3"
2987   [(set_attr "type" "<su>mlal")]
2988 )
2989
2990 (define_insn "*<su_optab>mulsidi_neg"
2991   [(set (match_operand:DI 0 "register_operand" "=r")
2992         (mult:DI (neg:DI
2993                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
2994                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2995   ""
2996   "<su>mnegl\\t%0, %w1, %w2"
2997   [(set_attr "type" "<su>mull")]
2998 )
2999
3000 (define_expand "<su_optab>mulditi3"
3001   [(set (match_operand:TI 0 "register_operand")
3002         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3003                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3004   ""
3005 {
3006   rtx low = gen_reg_rtx (DImode);
3007   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3008
3009   rtx high = gen_reg_rtx (DImode);
3010   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3011
3012   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3013   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3014   DONE;
3015 })
3016
3017 ;; The default expansion of multi3 using umuldi3_highpart will perform
3018 ;; the additions in an order that fails to combine into two madd insns.
3019 (define_expand "multi3"
3020   [(set (match_operand:TI 0 "register_operand")
3021         (mult:TI (match_operand:TI 1 "register_operand")
3022                  (match_operand:TI 2 "register_operand")))]
3023   ""
3024 {
3025   rtx l0 = gen_reg_rtx (DImode);
3026   rtx l1 = gen_lowpart (DImode, operands[1]);
3027   rtx l2 = gen_lowpart (DImode, operands[2]);
3028   rtx h0 = gen_reg_rtx (DImode);
3029   rtx h1 = gen_highpart (DImode, operands[1]);
3030   rtx h2 = gen_highpart (DImode, operands[2]);
3031
3032   emit_insn (gen_muldi3 (l0, l1, l2));
3033   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3034   emit_insn (gen_madddi (h0, h1, l2, h0));
3035   emit_insn (gen_madddi (h0, l1, h2, h0));
3036
3037   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3038   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3039   DONE;
3040 })
3041
3042 (define_insn "<su>muldi3_highpart"
3043   [(set (match_operand:DI 0 "register_operand" "=r")
3044         (truncate:DI
3045          (lshiftrt:TI
3046           (mult:TI
3047            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3048            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3049           (const_int 64))))]
3050   ""
3051   "<su>mulh\\t%0, %1, %2"
3052   [(set_attr "type" "<su>mull")]
3053 )
3054
3055 (define_insn "<su_optab>div<mode>3"
3056   [(set (match_operand:GPI 0 "register_operand" "=r")
3057         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3058                      (match_operand:GPI 2 "register_operand" "r")))]
3059   ""
3060   "<su>div\\t%<w>0, %<w>1, %<w>2"
3061   [(set_attr "type" "<su>div")]
3062 )
3063
3064 ;; zero_extend version of above
3065 (define_insn "*<su_optab>divsi3_uxtw"
3066   [(set (match_operand:DI 0 "register_operand" "=r")
3067         (zero_extend:DI
3068          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3069                      (match_operand:SI 2 "register_operand" "r"))))]
3070   ""
3071   "<su>div\\t%w0, %w1, %w2"
3072   [(set_attr "type" "<su>div")]
3073 )
3074
3075 ;; -------------------------------------------------------------------
3076 ;; Comparison insns
3077 ;; -------------------------------------------------------------------
3078
3079 (define_insn "cmp<mode>"
3080   [(set (reg:CC CC_REGNUM)
3081         (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
3082                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3083   ""
3084   "@
3085    cmp\\t%<w>0, %<w>1
3086    cmp\\t%<w>0, %1
3087    cmn\\t%<w>0, #%n1"
3088   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3089 )
3090
3091 (define_insn "fcmp<mode>"
3092   [(set (reg:CCFP CC_REGNUM)
3093         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3094                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3095    "TARGET_FLOAT"
3096    "@
3097     fcmp\\t%<s>0, #0.0
3098     fcmp\\t%<s>0, %<s>1"
3099   [(set_attr "type" "fcmp<s>")]
3100 )
3101
3102 (define_insn "fcmpe<mode>"
3103   [(set (reg:CCFPE CC_REGNUM)
3104         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3105                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3106    "TARGET_FLOAT"
3107    "@
3108     fcmpe\\t%<s>0, #0.0
3109     fcmpe\\t%<s>0, %<s>1"
3110   [(set_attr "type" "fcmp<s>")]
3111 )
3112
3113 (define_insn "*cmp_swp_<shift>_reg<mode>"
3114   [(set (reg:CC_SWP CC_REGNUM)
3115         (compare:CC_SWP (ASHIFT:GPI
3116                          (match_operand:GPI 0 "register_operand" "r")
3117                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3118                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3119   ""
3120   "cmp\\t%<w>2, %<w>0, <shift> %1"
3121   [(set_attr "type" "alus_shift_imm")]
3122 )
3123
3124 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3125   [(set (reg:CC_SWP CC_REGNUM)
3126         (compare:CC_SWP (ANY_EXTEND:GPI
3127                          (match_operand:ALLX 0 "register_operand" "r"))
3128                         (match_operand:GPI 1 "register_operand" "r")))]
3129   ""
3130   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3131   [(set_attr "type" "alus_ext")]
3132 )
3133
3134 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3135   [(set (reg:CC_SWP CC_REGNUM)
3136         (compare:CC_SWP (ashift:GPI
3137                          (ANY_EXTEND:GPI
3138                           (match_operand:ALLX 0 "register_operand" "r"))
3139                          (match_operand 1 "aarch64_imm3" "Ui3"))
3140         (match_operand:GPI 2 "register_operand" "r")))]
3141   ""
3142   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3143   [(set_attr "type" "alus_ext")]
3144 )
3145
3146 ;; -------------------------------------------------------------------
3147 ;; Store-flag and conditional select insns
3148 ;; -------------------------------------------------------------------
3149
3150 (define_expand "cstore<mode>4"
3151   [(set (match_operand:SI 0 "register_operand" "")
3152         (match_operator:SI 1 "aarch64_comparison_operator"
3153          [(match_operand:GPI 2 "register_operand" "")
3154           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3155   ""
3156   "
3157   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3158                                       operands[3]);
3159   operands[3] = const0_rtx;
3160   "
3161 )
3162
3163 (define_expand "cstorecc4"
3164   [(set (match_operand:SI 0 "register_operand")
3165        (match_operator 1 "aarch64_comparison_operator_mode"
3166         [(match_operand 2 "cc_register")
3167          (match_operand 3 "const0_operand")]))]
3168   ""
3169 "{
3170   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3171   DONE;
3172 }")
3173
3174
3175 (define_expand "cstore<mode>4"
3176   [(set (match_operand:SI 0 "register_operand" "")
3177         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3178          [(match_operand:GPF 2 "register_operand" "")
3179           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3180   ""
3181   "
3182   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3183                                       operands[3]);
3184   operands[3] = const0_rtx;
3185   "
3186 )
3187
3188 (define_insn "aarch64_cstore<mode>"
3189   [(set (match_operand:ALLI 0 "register_operand" "=r")
3190         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3191          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3192   ""
3193   "cset\\t%<w>0, %m1"
3194   [(set_attr "type" "csel")]
3195 )
3196
3197 ;; For a 24-bit immediate CST we can optimize the compare for equality
3198 ;; and branch sequence from:
3199 ;;      mov     x0, #imm1
3200 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3201 ;;      cmp     x1, x0
3202 ;;      cset    x2, <ne,eq>
3203 ;; into the shorter:
3204 ;;      sub     x0, x1, #(CST & 0xfff000)
3205 ;;      subs    x0, x0, #(CST & 0x000fff)
3206 ;;      cset x2, <ne, eq>.
3207 (define_insn_and_split "*compare_cstore<mode>_insn"
3208   [(set (match_operand:GPI 0 "register_operand" "=r")
3209          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3210                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3211    (clobber (reg:CC CC_REGNUM))]
3212   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3213    && !aarch64_plus_operand (operands[2], <MODE>mode)
3214    && !reload_completed"
3215   "#"
3216   "&& true"
3217   [(const_int 0)]
3218   {
3219     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3220     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3221     rtx tmp = gen_reg_rtx (<MODE>mode);
3222     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3223     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3224     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3225     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3226     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3227     DONE;
3228   }
3229   [(set_attr "type" "csel")]
3230 )
3231
3232 ;; zero_extend version of the above
3233 (define_insn "*cstoresi_insn_uxtw"
3234   [(set (match_operand:DI 0 "register_operand" "=r")
3235         (zero_extend:DI
3236          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3237           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3238   ""
3239   "cset\\t%w0, %m1"
3240   [(set_attr "type" "csel")]
3241 )
3242
3243 (define_insn "cstore<mode>_neg"
3244   [(set (match_operand:ALLI 0 "register_operand" "=r")
3245         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3246                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
3247   ""
3248   "csetm\\t%<w>0, %m1"
3249   [(set_attr "type" "csel")]
3250 )
3251
3252 ;; zero_extend version of the above
3253 (define_insn "*cstoresi_neg_uxtw"
3254   [(set (match_operand:DI 0 "register_operand" "=r")
3255         (zero_extend:DI
3256          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3257                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3258   ""
3259   "csetm\\t%w0, %m1"
3260   [(set_attr "type" "csel")]
3261 )
3262
3263 (define_expand "cmov<mode>6"
3264   [(set (match_operand:GPI 0 "register_operand" "")
3265         (if_then_else:GPI
3266          (match_operator 1 "aarch64_comparison_operator"
3267           [(match_operand:GPI 2 "register_operand" "")
3268            (match_operand:GPI 3 "aarch64_plus_operand" "")])
3269          (match_operand:GPI 4 "register_operand" "")
3270          (match_operand:GPI 5 "register_operand" "")))]
3271   ""
3272   "
3273   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3274                                       operands[3]);
3275   operands[3] = const0_rtx;
3276   "
3277 )
3278
3279 (define_expand "cmov<mode>6"
3280   [(set (match_operand:GPF 0 "register_operand" "")
3281         (if_then_else:GPF
3282          (match_operator 1 "aarch64_comparison_operator"
3283           [(match_operand:GPF 2 "register_operand" "")
3284            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3285          (match_operand:GPF 4 "register_operand" "")
3286          (match_operand:GPF 5 "register_operand" "")))]
3287   ""
3288   "
3289   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3290                                       operands[3]);
3291   operands[3] = const0_rtx;
3292   "
3293 )
3294
3295 (define_insn "*cmov<mode>_insn"
3296   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3297         (if_then_else:ALLI
3298          (match_operator 1 "aarch64_comparison_operator"
3299           [(match_operand 2 "cc_register" "") (const_int 0)])
3300          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3301          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3302   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3303      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3304   ;; Final two alternatives should be unreachable, but included for completeness
3305   "@
3306    csel\\t%<w>0, %<w>3, %<w>4, %m1
3307    csinv\\t%<w>0, %<w>3, <w>zr, %m1
3308    csinv\\t%<w>0, %<w>4, <w>zr, %M1
3309    csinc\\t%<w>0, %<w>3, <w>zr, %m1
3310    csinc\\t%<w>0, %<w>4, <w>zr, %M1
3311    mov\\t%<w>0, -1
3312    mov\\t%<w>0, 1"
3313   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3314 )
3315
3316 ;; zero_extend version of above
3317 (define_insn "*cmovsi_insn_uxtw"
3318   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3319         (zero_extend:DI
3320          (if_then_else:SI
3321           (match_operator 1 "aarch64_comparison_operator"
3322            [(match_operand 2 "cc_register" "") (const_int 0)])
3323           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3324           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3325   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3326      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3327   ;; Final two alternatives should be unreachable, but included for completeness
3328   "@
3329    csel\\t%w0, %w3, %w4, %m1
3330    csinv\\t%w0, %w3, wzr, %m1
3331    csinv\\t%w0, %w4, wzr, %M1
3332    csinc\\t%w0, %w3, wzr, %m1
3333    csinc\\t%w0, %w4, wzr, %M1
3334    mov\\t%w0, -1
3335    mov\\t%w0, 1"
3336   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3337 )
3338
3339 (define_insn "*cmovdi_insn_uxtw"
3340   [(set (match_operand:DI 0 "register_operand" "=r")
3341         (if_then_else:DI
3342          (match_operator 1 "aarch64_comparison_operator"
3343           [(match_operand 2 "cc_register" "") (const_int 0)])
3344          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3345          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3346   ""
3347   "csel\\t%w0, %w3, %w4, %m1"
3348   [(set_attr "type" "csel")]
3349 )
3350
3351 (define_insn "*cmov<mode>_insn"
3352   [(set (match_operand:GPF 0 "register_operand" "=w")
3353         (if_then_else:GPF
3354          (match_operator 1 "aarch64_comparison_operator"
3355           [(match_operand 2 "cc_register" "") (const_int 0)])
3356          (match_operand:GPF 3 "register_operand" "w")
3357          (match_operand:GPF 4 "register_operand" "w")))]
3358   "TARGET_FLOAT"
3359   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3360   [(set_attr "type" "fcsel")]
3361 )
3362
3363 (define_expand "mov<mode>cc"
3364   [(set (match_operand:ALLI 0 "register_operand" "")
3365         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3366                            (match_operand:ALLI 2 "register_operand" "")
3367                            (match_operand:ALLI 3 "register_operand" "")))]
3368   ""
3369   {
3370     rtx ccreg;
3371     enum rtx_code code = GET_CODE (operands[1]);
3372
3373     if (code == UNEQ || code == LTGT)
3374       FAIL;
3375
3376     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3377                                      XEXP (operands[1], 1));
3378     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3379   }
3380 )
3381
3382 (define_expand "mov<GPF:mode><GPI:mode>cc"
3383   [(set (match_operand:GPI 0 "register_operand" "")
3384         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3385                           (match_operand:GPF 2 "register_operand" "")
3386                           (match_operand:GPF 3 "register_operand" "")))]
3387   ""
3388   {
3389     rtx ccreg;
3390     enum rtx_code code = GET_CODE (operands[1]);
3391
3392     if (code == UNEQ || code == LTGT)
3393       FAIL;
3394
3395     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3396                                   XEXP (operands[1], 1));
3397     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3398   }
3399 )
3400
3401 (define_expand "mov<mode>cc"
3402   [(set (match_operand:GPF 0 "register_operand" "")
3403         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3404                           (match_operand:GPF 2 "register_operand" "")
3405                           (match_operand:GPF 3 "register_operand" "")))]
3406   ""
3407   {
3408     rtx ccreg;
3409     enum rtx_code code = GET_CODE (operands[1]);
3410
3411     if (code == UNEQ || code == LTGT)
3412       FAIL;
3413
3414     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3415                                   XEXP (operands[1], 1));
3416     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3417   }
3418 )
3419
3420 (define_expand "<neg_not_op><mode>cc"
3421   [(set (match_operand:GPI 0 "register_operand" "")
3422         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3423                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3424                           (match_operand:GPI 3 "register_operand" "")))]
3425   ""
3426   {
3427     rtx ccreg;
3428     enum rtx_code code = GET_CODE (operands[1]);
3429
3430     if (code == UNEQ || code == LTGT)
3431       FAIL;
3432
3433     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3434                                       XEXP (operands[1], 1));
3435     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3436   }
3437 )
3438
3439 ;; CRC32 instructions.
3440 (define_insn "aarch64_<crc_variant>"
3441   [(set (match_operand:SI 0 "register_operand" "=r")
3442         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3443                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3444          CRC))]
3445   "TARGET_CRC32"
3446   {
3447     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3448       return "<crc_variant>\\t%w0, %w1, %x2";
3449     else
3450       return "<crc_variant>\\t%w0, %w1, %w2";
3451   }
3452   [(set_attr "type" "crc")]
3453 )
3454
3455 (define_insn "*csinc2<mode>_insn"
3456   [(set (match_operand:GPI 0 "register_operand" "=r")
3457         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3458                   (match_operand:GPI 1 "register_operand" "r")))]
3459   ""
3460   "cinc\\t%<w>0, %<w>1, %m2"
3461   [(set_attr "type" "csel")]
3462 )
3463
3464 (define_insn "csinc3<mode>_insn"
3465   [(set (match_operand:GPI 0 "register_operand" "=r")
3466         (if_then_else:GPI
3467           (match_operand 1 "aarch64_comparison_operation" "")
3468           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3469                     (const_int 1))
3470           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3471   ""
3472   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3473   [(set_attr "type" "csel")]
3474 )
3475
3476 (define_insn "*csinv3<mode>_insn"
3477   [(set (match_operand:GPI 0 "register_operand" "=r")
3478         (if_then_else:GPI
3479           (match_operand 1 "aarch64_comparison_operation" "")
3480           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3481           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3482   ""
3483   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3484   [(set_attr "type" "csel")]
3485 )
3486
3487 (define_insn "csneg3_uxtw_insn"
3488   [(set (match_operand:DI 0 "register_operand" "=r")
3489         (zero_extend:DI
3490           (if_then_else:SI
3491             (match_operand 1 "aarch64_comparison_operation" "")
3492             (neg:SI (match_operand:SI 2 "register_operand" "r"))
3493             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3494   ""
3495   "csneg\\t%w0, %w3, %w2, %M1"
3496   [(set_attr "type" "csel")]
3497 )
3498
3499 (define_insn "csneg3<mode>_insn"
3500   [(set (match_operand:GPI 0 "register_operand" "=r")
3501         (if_then_else:GPI
3502           (match_operand 1 "aarch64_comparison_operation" "")
3503           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3504           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3505   ""
3506   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3507   [(set_attr "type" "csel")]
3508 )
3509
3510 ;; If X can be loaded by a single CNT[BHWD] instruction,
3511 ;;
3512 ;;    A = UMAX (B, X)
3513 ;;
3514 ;; is equivalent to:
3515 ;;
3516 ;;    TMP = UQDEC[BHWD] (B, X)
3517 ;;    A = TMP + X
3518 ;;
3519 ;; Defining the pattern this way means that:
3520 ;;
3521 ;;    A = UMAX (B, X) - X
3522 ;;
3523 ;; becomes:
3524 ;;
3525 ;;    TMP1 = UQDEC[BHWD] (B, X)
3526 ;;    TMP2 = TMP1 + X
3527 ;;    A = TMP2 - X
3528 ;;
3529 ;; which combine can optimize to:
3530 ;;
3531 ;;    A = UQDEC[BHWD] (B, X)
3532 ;;
3533 ;; We don't use match_operand predicates because the order of the operands
3534 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
3535 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
3536 (define_expand "umax<mode>3"
3537   [(set (match_operand:GPI 0 "register_operand")
3538         (umax:GPI (match_operand:GPI 1 "")
3539                   (match_operand:GPI 2 "")))]
3540   "TARGET_SVE"
3541   {
3542     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
3543       std::swap (operands[1], operands[2]);
3544     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
3545       FAIL;
3546     rtx temp = gen_reg_rtx (<MODE>mode);
3547     operands[1] = force_reg (<MODE>mode, operands[1]);
3548     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
3549     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
3550     DONE;
3551   }
3552 )
3553
3554 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
3555 (define_insn "aarch64_uqdec<mode>"
3556   [(set (match_operand:GPI 0 "register_operand" "=r")
3557         (minus:GPI
3558          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
3559                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
3560          (match_dup 2)))]
3561   "TARGET_SVE"
3562   {
3563     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
3564   }
3565 )
3566
3567 ;; -------------------------------------------------------------------
3568 ;; Logical operations
3569 ;; -------------------------------------------------------------------
3570
3571
3572 (define_insn_and_split "*aarch64_and<mode>_imm2"
3573   [(set (match_operand:GPI 0 "register_operand" "=rk")
3574         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
3575                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
3576   ""
3577   "#"
3578   "true"
3579   [(const_int 0)]
3580   {
3581      HOST_WIDE_INT val = INTVAL (operands[2]);
3582      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
3583      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
3584
3585      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
3586      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
3587      DONE;
3588   }
3589 )
3590
3591 (define_insn "<optab><mode>3"
3592   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3593         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3594                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3595   ""
3596   "@
3597   <logical>\\t%<w>0, %<w>1, %<w>2
3598   <logical>\\t%<w>0, %<w>1, %2
3599   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3600   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3601    (set_attr "simd" "*,*,yes")]
3602 )
3603
3604 ;; zero_extend version of above
3605 (define_insn "*<optab>si3_uxtw"
3606   [(set (match_operand:DI 0 "register_operand" "=r,rk")
3607         (zero_extend:DI
3608          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3609                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3610   ""
3611   "@
3612    <logical>\\t%w0, %w1, %w2
3613    <logical>\\t%w0, %w1, %2"
3614   [(set_attr "type" "logic_reg,logic_imm")]
3615 )
3616
3617 (define_insn "*and<mode>3_compare0"
3618   [(set (reg:CC_NZ CC_REGNUM)
3619         (compare:CC_NZ
3620          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3621                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3622          (const_int 0)))
3623    (set (match_operand:GPI 0 "register_operand" "=r,r")
3624         (and:GPI (match_dup 1) (match_dup 2)))]
3625   ""
3626   "@
3627    ands\\t%<w>0, %<w>1, %<w>2
3628    ands\\t%<w>0, %<w>1, %2"
3629   [(set_attr "type" "logics_reg,logics_imm")]
3630 )
3631
3632 ;; zero_extend version of above
3633 (define_insn "*andsi3_compare0_uxtw"
3634   [(set (reg:CC_NZ CC_REGNUM)
3635         (compare:CC_NZ
3636          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3637                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3638          (const_int 0)))
3639    (set (match_operand:DI 0 "register_operand" "=r,r")
3640         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3641   ""
3642   "@
3643    ands\\t%w0, %w1, %w2
3644    ands\\t%w0, %w1, %2"
3645   [(set_attr "type" "logics_reg,logics_imm")]
3646 )
3647
3648 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3649   [(set (reg:CC_NZ CC_REGNUM)
3650         (compare:CC_NZ
3651          (and:GPI (SHIFT:GPI
3652                    (match_operand:GPI 1 "register_operand" "r")
3653                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3654                   (match_operand:GPI 3 "register_operand" "r"))
3655          (const_int 0)))
3656    (set (match_operand:GPI 0 "register_operand" "=r")
3657         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3658   ""
3659   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3660   [(set_attr "type" "logics_shift_imm")]
3661 )
3662
3663 ;; zero_extend version of above
3664 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3665   [(set (reg:CC_NZ CC_REGNUM)
3666         (compare:CC_NZ
3667          (and:SI (SHIFT:SI
3668                   (match_operand:SI 1 "register_operand" "r")
3669                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3670                  (match_operand:SI 3 "register_operand" "r"))
3671          (const_int 0)))
3672    (set (match_operand:DI 0 "register_operand" "=r")
3673         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3674                                 (match_dup 3))))]
3675   ""
3676   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3677   [(set_attr "type" "logics_shift_imm")]
3678 )
3679
3680 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3681   [(set (match_operand:GPI 0 "register_operand" "=r")
3682         (LOGICAL:GPI (SHIFT:GPI
3683                       (match_operand:GPI 1 "register_operand" "r")
3684                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3685                      (match_operand:GPI 3 "register_operand" "r")))]
3686   ""
3687   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3688   [(set_attr "type" "logic_shift_imm")]
3689 )
3690
3691 (define_insn "*<optab>_rol<mode>3"
3692   [(set (match_operand:GPI 0 "register_operand" "=r")
3693         (LOGICAL:GPI (rotate:GPI
3694                       (match_operand:GPI 1 "register_operand" "r")
3695                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3696                      (match_operand:GPI 3 "register_operand" "r")))]
3697   ""
3698   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3699   [(set_attr "type" "logic_shift_imm")]
3700 )
3701
3702 ;; zero_extend versions of above
3703 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3704   [(set (match_operand:DI 0 "register_operand" "=r")
3705         (zero_extend:DI
3706          (LOGICAL:SI (SHIFT:SI
3707                       (match_operand:SI 1 "register_operand" "r")
3708                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3709                      (match_operand:SI 3 "register_operand" "r"))))]
3710   ""
3711   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3712   [(set_attr "type" "logic_shift_imm")]
3713 )
3714
3715 (define_insn "*<optab>_rolsi3_uxtw"
3716   [(set (match_operand:DI 0 "register_operand" "=r")
3717         (zero_extend:DI
3718          (LOGICAL:SI (rotate:SI
3719                       (match_operand:SI 1 "register_operand" "r")
3720                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3721                      (match_operand:SI 3 "register_operand" "r"))))]
3722   ""
3723   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3724   [(set_attr "type" "logic_shift_imm")]
3725 )
3726
3727 (define_insn "one_cmpl<mode>2"
3728   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3729         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3730   ""
3731   "@
3732   mvn\\t%<w>0, %<w>1
3733   mvn\\t%0.8b, %1.8b"
3734   [(set_attr "type" "logic_reg,neon_logic")
3735    (set_attr "simd" "*,yes")]
3736 )
3737
3738 (define_insn "*one_cmpl_<optab><mode>2"
3739   [(set (match_operand:GPI 0 "register_operand" "=r")
3740         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3741                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3742   ""
3743   "mvn\\t%<w>0, %<w>1, <shift> %2"
3744   [(set_attr "type" "logic_shift_imm")]
3745 )
3746
3747 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3748
3749 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3750   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3751         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3752                      (match_operand:GPI 2 "register_operand" "r,w")))]
3753   ""
3754   "@
3755   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3756   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3757   [(set_attr "type" "logic_reg,neon_logic")
3758    (set_attr "simd" "*,yes")]
3759 )
3760
3761 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3762   [(set (match_operand:DI 0 "register_operand" "=r")
3763         (zero_extend:DI
3764           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3765                        (match_operand:SI 2 "register_operand" "r"))))]
3766   ""
3767   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3768   [(set_attr "type" "logic_reg")]
3769 )
3770
3771 (define_insn "*xor_one_cmplsidi3_ze"
3772   [(set (match_operand:DI 0 "register_operand" "=r")
3773         (zero_extend:DI
3774           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3775                           (match_operand:SI 2 "register_operand" "r")))))]
3776   ""
3777   "eon\\t%w0, %w1, %w2"
3778   [(set_attr "type" "logic_reg")]
3779 )
3780
3781 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3782 ;; eon does not operate on SIMD registers so the vector variant must be split.
3783 (define_insn_and_split "*xor_one_cmpl<mode>3"
3784   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3785         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3786                           (match_operand:GPI 2 "register_operand" "r,w"))))]
3787   ""
3788   "@
3789   eon\\t%<w>0, %<w>1, %<w>2
3790   #"
3791   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3792   [(set (match_operand:GPI 0 "register_operand" "=w")
3793         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3794                  (match_operand:GPI 2 "register_operand" "w")))
3795    (set (match_dup 0) (not:GPI (match_dup 0)))]
3796   ""
3797   [(set_attr "type" "logic_reg,multiple")
3798    (set_attr "simd" "*,yes")]
3799 )
3800
3801 (define_insn "*and_one_cmpl<mode>3_compare0"
3802   [(set (reg:CC_NZ CC_REGNUM)
3803         (compare:CC_NZ
3804          (and:GPI (not:GPI
3805                    (match_operand:GPI 1 "register_operand" "r"))
3806                   (match_operand:GPI 2 "register_operand" "r"))
3807          (const_int 0)))
3808    (set (match_operand:GPI 0 "register_operand" "=r")
3809         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3810   ""
3811   "bics\\t%<w>0, %<w>2, %<w>1"
3812   [(set_attr "type" "logics_reg")]
3813 )
3814
3815 ;; zero_extend version of above
3816 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3817   [(set (reg:CC_NZ CC_REGNUM)
3818         (compare:CC_NZ
3819          (and:SI (not:SI
3820                   (match_operand:SI 1 "register_operand" "r"))
3821                  (match_operand:SI 2 "register_operand" "r"))
3822          (const_int 0)))
3823    (set (match_operand:DI 0 "register_operand" "=r")
3824         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3825   ""
3826   "bics\\t%w0, %w2, %w1"
3827   [(set_attr "type" "logics_reg")]
3828 )
3829
3830 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3831   [(set (reg:CC_NZ CC_REGNUM)
3832     (compare:CC_NZ
3833      (and:GPI (not:GPI
3834            (match_operand:GPI 0 "register_operand" "r"))
3835           (match_operand:GPI 1 "register_operand" "r"))
3836      (const_int 0)))]
3837   ""
3838   "bics\\t<w>zr, %<w>1, %<w>0"
3839   [(set_attr "type" "logics_reg")]
3840 )
3841
3842 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3843   [(set (match_operand:GPI 0 "register_operand" "=r")
3844         (LOGICAL:GPI (not:GPI
3845                       (SHIFT:GPI
3846                        (match_operand:GPI 1 "register_operand" "r")
3847                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3848                      (match_operand:GPI 3 "register_operand" "r")))]
3849   ""
3850   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3851   [(set_attr "type" "logic_shift_imm")]
3852 )
3853
3854 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3855   [(set (match_operand:GPI 0 "register_operand" "=r")
3856         (not:GPI (xor:GPI
3857                       (SHIFT:GPI
3858                        (match_operand:GPI 1 "register_operand" "r")
3859                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3860                      (match_operand:GPI 3 "register_operand" "r"))))]
3861   ""
3862   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3863   [(set_attr "type" "logic_shift_imm")]
3864 )
3865
3866 ;; Zero-extend version of the above.
3867 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3868   [(set (match_operand:DI 0 "register_operand" "=r")
3869         (zero_extend:DI
3870           (not:SI (xor:SI
3871                     (SHIFT:SI
3872                       (match_operand:SI 1 "register_operand" "r")
3873                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3874                     (match_operand:SI 3 "register_operand" "r")))))]
3875   ""
3876   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3877   [(set_attr "type" "logic_shift_imm")]
3878 )
3879
3880 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3881   [(set (reg:CC_NZ CC_REGNUM)
3882         (compare:CC_NZ
3883          (and:GPI (not:GPI
3884                    (SHIFT:GPI
3885                     (match_operand:GPI 1 "register_operand" "r")
3886                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3887                   (match_operand:GPI 3 "register_operand" "r"))
3888          (const_int 0)))
3889    (set (match_operand:GPI 0 "register_operand" "=r")
3890         (and:GPI (not:GPI
3891                   (SHIFT:GPI
3892                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
3893   ""
3894   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3895   [(set_attr "type" "logics_shift_imm")]
3896 )
3897
3898 ;; zero_extend version of above
3899 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3900   [(set (reg:CC_NZ CC_REGNUM)
3901         (compare:CC_NZ
3902          (and:SI (not:SI
3903                   (SHIFT:SI
3904                    (match_operand:SI 1 "register_operand" "r")
3905                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3906                  (match_operand:SI 3 "register_operand" "r"))
3907          (const_int 0)))
3908    (set (match_operand:DI 0 "register_operand" "=r")
3909         (zero_extend:DI (and:SI
3910                          (not:SI
3911                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3912   ""
3913   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3914   [(set_attr "type" "logics_shift_imm")]
3915 )
3916
3917 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3918   [(set (reg:CC_NZ CC_REGNUM)
3919     (compare:CC_NZ
3920      (and:GPI (not:GPI
3921            (SHIFT:GPI
3922             (match_operand:GPI 0 "register_operand" "r")
3923             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3924           (match_operand:GPI 2 "register_operand" "r"))
3925      (const_int 0)))]
3926   ""
3927   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3928   [(set_attr "type" "logics_shift_imm")]
3929 )
3930
3931 (define_insn "clz<mode>2"
3932   [(set (match_operand:GPI 0 "register_operand" "=r")
3933         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3934   ""
3935   "clz\\t%<w>0, %<w>1"
3936   [(set_attr "type" "clz")]
3937 )
3938
3939 (define_expand "ffs<mode>2"
3940   [(match_operand:GPI 0 "register_operand")
3941    (match_operand:GPI 1 "register_operand")]
3942   ""
3943   {
3944     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3945     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3946
3947     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3948     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3949     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3950     DONE;
3951   }
3952 )
3953
3954 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
3955 ;;
3956 ;; MOV  v.1d, x0
3957 ;; CNT  v1.8b, v.8b
3958 ;; ADDV b2, v1.8b
3959 ;; MOV  w0, v2.b[0]
3960
3961 (define_expand "popcount<mode>2"
3962   [(match_operand:GPI 0 "register_operand")
3963    (match_operand:GPI 1 "register_operand")]
3964   "TARGET_SIMD"
3965 {
3966   rtx v = gen_reg_rtx (V8QImode);
3967   rtx v1 = gen_reg_rtx (V8QImode);
3968   rtx r = gen_reg_rtx (QImode);
3969   rtx in = operands[1];
3970   rtx out = operands[0];
3971   if(<MODE>mode == SImode)
3972     {
3973       rtx tmp;
3974       tmp = gen_reg_rtx (DImode);
3975       /* If we have SImode, zero extend to DImode, pop count does
3976          not change if we have extra zeros. */
3977       emit_insn (gen_zero_extendsidi2 (tmp, in));
3978       in = tmp;
3979     }
3980   emit_move_insn (v, gen_lowpart (V8QImode, in));
3981   emit_insn (gen_popcountv8qi2 (v1, v));
3982   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
3983   emit_insn (gen_zero_extendqi<mode>2 (out, r));
3984   DONE;
3985 })
3986
3987 (define_insn "clrsb<mode>2"
3988   [(set (match_operand:GPI 0 "register_operand" "=r")
3989         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
3990   ""
3991   "cls\\t%<w>0, %<w>1"
3992   [(set_attr "type" "clz")]
3993 )
3994
3995 (define_insn "rbit<mode>2"
3996   [(set (match_operand:GPI 0 "register_operand" "=r")
3997         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
3998   ""
3999   "rbit\\t%<w>0, %<w>1"
4000   [(set_attr "type" "rbit")]
4001 )
4002
4003 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4004 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4005 ;; expression and split after reload to enable scheduling them apart if
4006 ;; needed.
4007
4008 (define_insn_and_split "ctz<mode>2"
4009  [(set (match_operand:GPI           0 "register_operand" "=r")
4010        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4011   ""
4012   "#"
4013   "reload_completed"
4014   [(const_int 0)]
4015   "
4016   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4017   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4018   DONE;
4019 ")
4020
4021 (define_insn "*and<mode>_compare0"
4022   [(set (reg:CC_NZ CC_REGNUM)
4023         (compare:CC_NZ
4024          (match_operand:SHORT 0 "register_operand" "r")
4025          (const_int 0)))]
4026   ""
4027   "tst\\t%<w>0, <short_mask>"
4028   [(set_attr "type" "alus_imm")]
4029 )
4030
4031 (define_insn "*ands<mode>_compare0"
4032   [(set (reg:CC_NZ CC_REGNUM)
4033         (compare:CC_NZ
4034          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4035          (const_int 0)))
4036    (set (match_operand:GPI 0 "register_operand" "=r")
4037         (zero_extend:GPI (match_dup 1)))]
4038   ""
4039   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4040   [(set_attr "type" "alus_imm")]
4041 )
4042
4043 (define_insn "*and<mode>3nr_compare0"
4044   [(set (reg:CC_NZ CC_REGNUM)
4045         (compare:CC_NZ
4046          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4047                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4048          (const_int 0)))]
4049   ""
4050   "@
4051    tst\\t%<w>0, %<w>1
4052    tst\\t%<w>0, %1"
4053   [(set_attr "type" "logics_reg,logics_imm")]
4054 )
4055
4056 (define_split
4057   [(set (reg:CC_NZ CC_REGNUM)
4058         (compare:CC_NZ
4059          (and:GPI (match_operand:GPI 0 "register_operand")
4060                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4061          (const_int 0)))
4062    (clobber (match_operand:SI 2 "register_operand"))]
4063   ""
4064   [(set (match_dup 2) (match_dup 1))
4065    (set (reg:CC_NZ CC_REGNUM)
4066         (compare:CC_NZ
4067          (and:GPI (match_dup 0)
4068                   (match_dup 2))
4069          (const_int 0)))]
4070 )
4071
4072 (define_insn "*and<mode>3nr_compare0_zextract"
4073   [(set (reg:CC_NZ CC_REGNUM)
4074         (compare:CC_NZ
4075          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4076                   (match_operand:GPI 1 "const_int_operand" "n")
4077                   (match_operand:GPI 2 "const_int_operand" "n"))
4078          (const_int 0)))]
4079   "INTVAL (operands[1]) > 0
4080    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4081         <= GET_MODE_BITSIZE (<MODE>mode))
4082    && aarch64_bitmask_imm (
4083         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4084                                                  operands[2])),
4085         <MODE>mode)"
4086   {
4087     operands[1]
4088       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4089     return "tst\\t%<w>0, %1";
4090   }
4091   [(set_attr "type" "logics_shift_imm")]
4092 )
4093
4094 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4095   [(set (reg:CC_NZ CC_REGNUM)
4096         (compare:CC_NZ
4097          (and:GPI (SHIFT:GPI
4098                    (match_operand:GPI 0 "register_operand" "r")
4099                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4100                   (match_operand:GPI 2 "register_operand" "r"))
4101         (const_int 0)))]
4102   ""
4103   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4104   [(set_attr "type" "logics_shift_imm")]
4105 )
4106
4107 (define_split
4108   [(set (reg:CC_NZ CC_REGNUM)
4109         (compare:CC_NZ
4110          (and:GPI (SHIFT:GPI
4111                    (match_operand:GPI 0 "register_operand")
4112                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4113                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4114         (const_int 0)))
4115     (clobber (match_operand:SI 3 "register_operand"))]
4116   ""
4117   [(set (match_dup 3) (match_dup 2))
4118    (set (reg:CC_NZ CC_REGNUM)
4119         (compare:CC_NZ
4120          (and:GPI (SHIFT:GPI
4121                    (match_dup 0)
4122                    (match_dup 1))
4123                   (match_dup 3))
4124          (const_int 0)))]
4125 )
4126
4127 ;; -------------------------------------------------------------------
4128 ;; Shifts
4129 ;; -------------------------------------------------------------------
4130
4131 (define_expand "<optab><mode>3"
4132   [(set (match_operand:GPI 0 "register_operand")
4133         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4134                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4135   ""
4136   {
4137     if (CONST_INT_P (operands[2]))
4138       {
4139         operands[2] = GEN_INT (INTVAL (operands[2])
4140                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4141
4142         if (operands[2] == const0_rtx)
4143           {
4144             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4145             DONE;
4146           }
4147       }
4148   }
4149 )
4150
4151 (define_expand "ashl<mode>3"
4152   [(set (match_operand:SHORT 0 "register_operand")
4153         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4154                       (match_operand:QI 2 "const_int_operand")))]
4155   ""
4156   {
4157     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4158
4159     if (operands[2] == const0_rtx)
4160       {
4161         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4162         DONE;
4163       }
4164   }
4165 )
4166
4167 (define_expand "rotr<mode>3"
4168   [(set (match_operand:GPI 0 "register_operand")
4169         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4170                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4171   ""
4172   {
4173     if (CONST_INT_P (operands[2]))
4174       {
4175         operands[2] = GEN_INT (INTVAL (operands[2])
4176                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4177
4178         if (operands[2] == const0_rtx)
4179           {
4180             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4181             DONE;
4182           }
4183       }
4184   }
4185 )
4186
4187 (define_expand "rotl<mode>3"
4188   [(set (match_operand:GPI 0 "register_operand")
4189         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4190                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4191   ""
4192   {
4193     /* (SZ - cnt) % SZ == -cnt % SZ */
4194     if (CONST_INT_P (operands[2]))
4195       {
4196         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4197                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4198         if (operands[2] == const0_rtx)
4199           {
4200             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4201             DONE;
4202           }
4203       }
4204     else
4205       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4206                                         NULL_RTX, 1);
4207   }
4208 )
4209
4210 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4211 ;; they truncate the shift/rotate amount by the size of the registers they
4212 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4213 ;; such redundant masking instructions.  GCC can do that automatically when
4214 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4215 ;; because some of the SISD shift alternatives don't perform this truncations.
4216 ;; So this pattern exists to catch such cases.
4217
4218 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4219   [(set (match_operand:GPI 0 "register_operand" "=r")
4220         (SHIFT:GPI
4221           (match_operand:GPI 1 "register_operand" "r")
4222           (match_operator 4 "subreg_lowpart_operator"
4223            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4224                      (match_operand 3 "const_int_operand" "n"))])))]
4225   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4226   "<shift>\t%<w>0, %<w>1, %<w>2"
4227   [(set_attr "type" "shift_reg")]
4228 )
4229
4230 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4231   [(set (match_operand:GPI 0 "register_operand" "=&r")
4232         (SHIFT:GPI
4233           (match_operand:GPI 1 "register_operand" "r")
4234           (match_operator 4 "subreg_lowpart_operator"
4235           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4236                            (match_operand 3 "const_int_operand" "n")))])))]
4237   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4238   "#"
4239   "&& true"
4240   [(const_int 0)]
4241   {
4242     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4243                : lowpart_subreg (SImode, operands[0], <MODE>mode));
4244     emit_insn (gen_negsi2 (tmp, operands[2]));
4245
4246     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4247     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4248                                      SUBREG_BYTE (operands[4]));
4249     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4250     DONE;
4251   }
4252 )
4253
4254 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
4255   [(set (match_operand:GPI 0 "register_operand" "=&r")
4256         (ashift:GPI
4257           (match_operand:GPI 1 "register_operand" "r")
4258           (minus:QI (match_operand 2 "const_int_operand" "n")
4259                     (match_operator 5 "subreg_lowpart_operator"
4260                     [(and:SI (match_operand:SI 3 "register_operand" "r")
4261                              (match_operand 4 "const_int_operand" "n"))]))))]
4262   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4263    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4264   "#"
4265   "&& true"
4266   [(const_int 0)]
4267   {
4268     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4269                : operands[0]);
4270
4271     emit_insn (gen_negsi2 (tmp, operands[3]));
4272
4273     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4274     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4275                                      SUBREG_BYTE (operands[5]));
4276
4277     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4278     DONE;
4279   }
4280 )
4281
4282 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4283   [(set (match_operand:DI 0 "register_operand" "=r")
4284         (SHIFT:DI
4285           (match_operand:DI 1 "register_operand" "r")
4286           (match_operator 4 "subreg_lowpart_operator"
4287            [(and:SI (match_operand:SI 2 "register_operand" "r")
4288                     (match_operand 3 "const_int_operand" "n"))])))]
4289   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
4290 {
4291   rtx xop[3];
4292   xop[0] = operands[0];
4293   xop[1] = operands[1];
4294   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4295   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4296   return "";
4297 }
4298   [(set_attr "type" "shift_reg")]
4299 )
4300
4301 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
4302   [(set (match_operand:GPI 0 "register_operand" "=&r")
4303         (ASHIFT:GPI
4304           (match_operand:GPI 1 "register_operand" "r")
4305           (minus:QI (match_operand 2 "const_int_operand" "n")
4306                     (match_operand:QI 3 "register_operand" "r"))))]
4307   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4308   "#"
4309   "&& true"
4310   [(const_int 0)]
4311   {
4312     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4313
4314     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4315                : gen_lowpart (SImode, operands[0]));
4316
4317     emit_insn (gen_negsi2 (tmp, subreg_tmp));
4318
4319     rtx and_op = gen_rtx_AND (SImode, tmp,
4320                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4321
4322     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4323
4324     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4325     DONE;
4326   }
4327   [(set_attr "length" "8")]
4328 )
4329
4330 ;; Logical left shift using SISD or Integer instruction
4331 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4332   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4333         (ashift:GPI
4334           (match_operand:GPI 1 "register_operand" "r,r,w,w")
4335           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4336   ""
4337   "@
4338    lsl\t%<w>0, %<w>1, %2
4339    lsl\t%<w>0, %<w>1, %<w>2
4340    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4341    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4342   [(set_attr "simd" "no,no,yes,yes")
4343    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
4344 )
4345
4346 ;; Logical right shift using SISD or Integer instruction
4347 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4348   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4349         (lshiftrt:GPI
4350          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4351          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
4352                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4353   ""
4354   "@
4355    lsr\t%<w>0, %<w>1, %2
4356    lsr\t%<w>0, %<w>1, %<w>2
4357    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4358    #
4359    #"
4360   [(set_attr "simd" "no,no,yes,yes,yes")
4361    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4362 )
4363
4364 (define_split
4365   [(set (match_operand:DI 0 "aarch64_simd_register")
4366         (lshiftrt:DI
4367            (match_operand:DI 1 "aarch64_simd_register")
4368            (match_operand:QI 2 "aarch64_simd_register")))]
4369   "TARGET_SIMD && reload_completed"
4370   [(set (match_dup 3)
4371         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4372    (set (match_dup 0)
4373         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4374   {
4375     operands[3] = gen_lowpart (QImode, operands[0]);
4376   }
4377 )
4378
4379 (define_split
4380   [(set (match_operand:SI 0 "aarch64_simd_register")
4381         (lshiftrt:SI
4382            (match_operand:SI 1 "aarch64_simd_register")
4383            (match_operand:QI 2 "aarch64_simd_register")))]
4384   "TARGET_SIMD && reload_completed"
4385   [(set (match_dup 3)
4386         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4387    (set (match_dup 0)
4388         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4389   {
4390     operands[3] = gen_lowpart (QImode, operands[0]);
4391   }
4392 )
4393
4394 ;; Arithmetic right shift using SISD or Integer instruction
4395 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4396   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4397         (ashiftrt:GPI
4398           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4399           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
4400                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4401   ""
4402   "@
4403    asr\t%<w>0, %<w>1, %2
4404    asr\t%<w>0, %<w>1, %<w>2
4405    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4406    #
4407    #"
4408   [(set_attr "simd" "no,no,yes,yes,yes")
4409    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4410 )
4411
4412 (define_split
4413   [(set (match_operand:DI 0 "aarch64_simd_register")
4414         (ashiftrt:DI
4415            (match_operand:DI 1 "aarch64_simd_register")
4416            (match_operand:QI 2 "aarch64_simd_register")))]
4417   "TARGET_SIMD && reload_completed"
4418   [(set (match_dup 3)
4419         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4420    (set (match_dup 0)
4421         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4422 {
4423   operands[3] = gen_lowpart (QImode, operands[0]);
4424 }
4425 )
4426
4427 (define_split
4428   [(set (match_operand:SI 0 "aarch64_simd_register")
4429         (ashiftrt:SI
4430            (match_operand:SI 1 "aarch64_simd_register")
4431            (match_operand:QI 2 "aarch64_simd_register")))]
4432   "TARGET_SIMD && reload_completed"
4433   [(set (match_dup 3)
4434         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4435    (set (match_dup 0)
4436         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4437 {
4438   operands[3] = gen_lowpart (QImode, operands[0]);
4439 }
4440 )
4441
4442 (define_insn "*aarch64_sisd_ushl"
4443   [(set (match_operand:DI 0 "register_operand" "=w")
4444         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4445                     (match_operand:QI 2 "register_operand" "w")]
4446                    UNSPEC_SISD_USHL))]
4447   "TARGET_SIMD"
4448   "ushl\t%d0, %d1, %d2"
4449   [(set_attr "simd" "yes")
4450    (set_attr "type" "neon_shift_reg")]
4451 )
4452
4453 (define_insn "*aarch64_ushl_2s"
4454   [(set (match_operand:SI 0 "register_operand" "=w")
4455         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4456                     (match_operand:QI 2 "register_operand" "w")]
4457                    UNSPEC_USHL_2S))]
4458   "TARGET_SIMD"
4459   "ushl\t%0.2s, %1.2s, %2.2s"
4460   [(set_attr "simd" "yes")
4461    (set_attr "type" "neon_shift_reg")]
4462 )
4463
4464 (define_insn "*aarch64_sisd_sshl"
4465   [(set (match_operand:DI 0 "register_operand" "=w")
4466         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4467                     (match_operand:QI 2 "register_operand" "w")]
4468                    UNSPEC_SISD_SSHL))]
4469   "TARGET_SIMD"
4470   "sshl\t%d0, %d1, %d2"
4471   [(set_attr "simd" "yes")
4472    (set_attr "type" "neon_shift_reg")]
4473 )
4474
4475 (define_insn "*aarch64_sshl_2s"
4476   [(set (match_operand:SI 0 "register_operand" "=w")
4477         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4478                     (match_operand:QI 2 "register_operand" "w")]
4479                    UNSPEC_SSHL_2S))]
4480   "TARGET_SIMD"
4481   "sshl\t%0.2s, %1.2s, %2.2s"
4482   [(set_attr "simd" "yes")
4483    (set_attr "type" "neon_shift_reg")]
4484 )
4485
4486 (define_insn "*aarch64_sisd_neg_qi"
4487   [(set (match_operand:QI 0 "register_operand" "=w")
4488         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4489                    UNSPEC_SISD_NEG))]
4490   "TARGET_SIMD"
4491   "neg\t%d0, %d1"
4492   [(set_attr "simd" "yes")
4493    (set_attr "type" "neon_neg")]
4494 )
4495
4496 ;; Rotate right
4497 (define_insn "*ror<mode>3_insn"
4498   [(set (match_operand:GPI 0 "register_operand" "=r,r")
4499      (rotatert:GPI
4500        (match_operand:GPI 1 "register_operand" "r,r")
4501        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4502   ""
4503   "@
4504    ror\\t%<w>0, %<w>1, %2
4505    ror\\t%<w>0, %<w>1, %<w>2"
4506   [(set_attr "type" "rotate_imm,shift_reg")]
4507 )
4508
4509 ;; zero_extend version of above
4510 (define_insn "*<optab>si3_insn_uxtw"
4511   [(set (match_operand:DI 0 "register_operand" "=r,r")
4512         (zero_extend:DI (SHIFT:SI
4513          (match_operand:SI 1 "register_operand" "r,r")
4514          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
4515   ""
4516   "@
4517    <shift>\\t%w0, %w1, %2
4518    <shift>\\t%w0, %w1, %w2"
4519   [(set_attr "type" "bfx,shift_reg")]
4520 )
4521
4522 (define_insn "*<optab><mode>3_insn"
4523   [(set (match_operand:SHORT 0 "register_operand" "=r")
4524         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
4525                       (match_operand 2 "const_int_operand" "n")))]
4526   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4527 {
4528   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4529   return "<bfshift>\t%w0, %w1, %2, %3";
4530 }
4531   [(set_attr "type" "bfx")]
4532 )
4533
4534 (define_insn "*extr<mode>5_insn"
4535   [(set (match_operand:GPI 0 "register_operand" "=r")
4536         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4537                              (match_operand 3 "const_int_operand" "n"))
4538                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4539                                (match_operand 4 "const_int_operand" "n"))))]
4540   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
4541    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
4542   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4543   [(set_attr "type" "rotate_imm")]
4544 )
4545
4546 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
4547 ;; so we have to match both orderings.
4548 (define_insn "*extr<mode>5_insn_alt"
4549   [(set (match_operand:GPI 0 "register_operand" "=r")
4550         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4551                                 (match_operand 4 "const_int_operand" "n"))
4552                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4553                               (match_operand 3 "const_int_operand" "n"))))]
4554   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
4555    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
4556        == GET_MODE_BITSIZE (<MODE>mode))"
4557   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4558   [(set_attr "type" "rotate_imm")]
4559 )
4560
4561 ;; zero_extend version of the above
4562 (define_insn "*extrsi5_insn_uxtw"
4563   [(set (match_operand:DI 0 "register_operand" "=r")
4564         (zero_extend:DI
4565          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
4566                             (match_operand 3 "const_int_operand" "n"))
4567                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4568                               (match_operand 4 "const_int_operand" "n")))))]
4569   "UINTVAL (operands[3]) < 32 &&
4570    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4571   "extr\\t%w0, %w1, %w2, %4"
4572   [(set_attr "type" "rotate_imm")]
4573 )
4574
4575 (define_insn "*extrsi5_insn_uxtw_alt"
4576   [(set (match_operand:DI 0 "register_operand" "=r")
4577         (zero_extend:DI
4578          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4579                                (match_operand 4 "const_int_operand" "n"))
4580                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
4581                             (match_operand 3 "const_int_operand" "n")))))]
4582   "UINTVAL (operands[3]) < 32 &&
4583    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4584   "extr\\t%w0, %w1, %w2, %4"
4585   [(set_attr "type" "rotate_imm")]
4586 )
4587
4588 (define_insn "*ror<mode>3_insn"
4589   [(set (match_operand:GPI 0 "register_operand" "=r")
4590         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
4591                     (match_operand 2 "const_int_operand" "n")))]
4592   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4593 {
4594   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4595   return "ror\\t%<w>0, %<w>1, %3";
4596 }
4597   [(set_attr "type" "rotate_imm")]
4598 )
4599
4600 ;; zero_extend version of the above
4601 (define_insn "*rorsi3_insn_uxtw"
4602   [(set (match_operand:DI 0 "register_operand" "=r")
4603         (zero_extend:DI
4604          (rotate:SI (match_operand:SI 1 "register_operand" "r")
4605                     (match_operand 2 "const_int_operand" "n"))))]
4606   "UINTVAL (operands[2]) < 32"
4607 {
4608   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
4609   return "ror\\t%w0, %w1, %3";
4610 }
4611   [(set_attr "type" "rotate_imm")]
4612 )
4613
4614 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
4615   [(set (match_operand:GPI 0 "register_operand" "=r")
4616         (ANY_EXTEND:GPI
4617          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
4618                        (match_operand 2 "const_int_operand" "n"))))]
4619   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4620 {
4621   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4622   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4623 }
4624   [(set_attr "type" "bfx")]
4625 )
4626
4627 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
4628   [(set (match_operand:GPI 0 "register_operand" "=r")
4629         (zero_extend:GPI
4630          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4631                          (match_operand 2 "const_int_operand" "n"))))]
4632   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4633 {
4634   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4635   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4636 }
4637   [(set_attr "type" "bfx")]
4638 )
4639
4640 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
4641   [(set (match_operand:GPI 0 "register_operand" "=r")
4642         (sign_extend:GPI
4643          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4644                          (match_operand 2 "const_int_operand" "n"))))]
4645   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4646 {
4647   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4648   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4649 }
4650   [(set_attr "type" "bfx")]
4651 )
4652
4653 ;; -------------------------------------------------------------------
4654 ;; Bitfields
4655 ;; -------------------------------------------------------------------
4656
4657 (define_expand "<optab>"
4658   [(set (match_operand:DI 0 "register_operand" "=r")
4659         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
4660                         (match_operand 2
4661                           "aarch64_simd_shift_imm_offset_di")
4662                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
4663   ""
4664   {
4665     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4666                    1, GET_MODE_BITSIZE (DImode) - 1))
4667      FAIL;
4668   }
4669 )
4670
4671
4672 (define_insn "*<optab><mode>"
4673   [(set (match_operand:GPI 0 "register_operand" "=r")
4674         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4675                          (match_operand 2
4676                            "aarch64_simd_shift_imm_offset_<mode>" "n")
4677                          (match_operand 3
4678                            "aarch64_simd_shift_imm_<mode>" "n")))]
4679   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4680              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
4681   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4682   [(set_attr "type" "bfx")]
4683 )
4684
4685 ;; When the bit position and width add up to 32 we can use a W-reg LSR
4686 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
4687 (define_split
4688   [(set (match_operand:DI 0 "register_operand")
4689         (zero_extract:DI (match_operand:DI 1 "register_operand")
4690                          (match_operand 2
4691                            "aarch64_simd_shift_imm_offset_di")
4692                          (match_operand 3
4693                            "aarch64_simd_shift_imm_di")))]
4694   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
4695              GET_MODE_BITSIZE (DImode) - 1)
4696    && (INTVAL (operands[2]) + INTVAL (operands[3]))
4697        == GET_MODE_BITSIZE (SImode)"
4698   [(set (match_dup 0)
4699         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
4700   {
4701     operands[4] = gen_lowpart (SImode, operands[1]);
4702   }
4703 )
4704
4705 ;; Bitfield Insert (insv)
4706 (define_expand "insv<mode>"
4707   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4708                           (match_operand 1 "const_int_operand")
4709                           (match_operand 2 "const_int_operand"))
4710         (match_operand:GPI 3 "general_operand"))]
4711   ""
4712 {
4713   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4714   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4715   rtx value = operands[3];
4716
4717   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4718     FAIL;
4719
4720   if (CONST_INT_P (value))
4721     {
4722       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4723
4724       /* Prefer AND/OR for inserting all zeros or all ones.  */
4725       if ((UINTVAL (value) & mask) == 0
4726            || (UINTVAL (value) & mask) == mask)
4727         FAIL;
4728
4729       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
4730       if (width == 16 && (pos % 16) == 0)
4731         DONE;
4732     }
4733   operands[3] = force_reg (<MODE>mode, value);
4734 })
4735
4736 (define_insn "*insv_reg<mode>"
4737   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4738                           (match_operand 1 "const_int_operand" "n")
4739                           (match_operand 2 "const_int_operand" "n"))
4740         (match_operand:GPI 3 "register_operand" "r"))]
4741   "!(UINTVAL (operands[1]) == 0
4742      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4743          > GET_MODE_BITSIZE (<MODE>mode)))"
4744   "bfi\\t%<w>0, %<w>3, %2, %1"
4745   [(set_attr "type" "bfm")]
4746 )
4747
4748 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4749   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4750                           (match_operand 1 "const_int_operand" "n")
4751                           (match_operand 2 "const_int_operand" "n"))
4752         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
4753   "UINTVAL (operands[1]) <= <ALLX:sizen>"
4754   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4755   [(set_attr "type" "bfm")]
4756 )
4757
4758 (define_insn "*extr_insv_lower_reg<mode>"
4759   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4760                           (match_operand 1 "const_int_operand" "n")
4761                           (const_int 0))
4762         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4763                           (match_dup 1)
4764                           (match_operand 3 "const_int_operand" "n")))]
4765   "!(UINTVAL (operands[1]) == 0
4766      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4767          > GET_MODE_BITSIZE (<MODE>mode)))"
4768   "bfxil\\t%<w>0, %<w>2, %3, %1"
4769   [(set_attr "type" "bfm")]
4770 )
4771
4772 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4773   [(set (match_operand:GPI 0 "register_operand" "=r")
4774         (ashift:GPI (ANY_EXTEND:GPI
4775                      (match_operand:ALLX 1 "register_operand" "r"))
4776                     (match_operand 2 "const_int_operand" "n")))]
4777   "UINTVAL (operands[2]) < <GPI:sizen>"
4778 {
4779   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4780               ? GEN_INT (<ALLX:sizen>)
4781               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4782   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4783 }
4784   [(set_attr "type" "bfx")]
4785 )
4786
4787 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4788
4789 (define_insn "*andim_ashift<mode>_bfiz"
4790   [(set (match_operand:GPI 0 "register_operand" "=r")
4791         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4792                              (match_operand 2 "const_int_operand" "n"))
4793                  (match_operand 3 "const_int_operand" "n")))]
4794   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
4795   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4796   [(set_attr "type" "bfx")]
4797 )
4798
4799 ;; Match sbfiz pattern in a shift left + shift right operation.
4800
4801 (define_insn "*ashift<mode>_extv_bfiz"
4802   [(set (match_operand:GPI 0 "register_operand" "=r")
4803         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
4804                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
4805                                       (const_int 0))
4806                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
4807   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4808              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
4809   "sbfiz\\t%<w>0, %<w>1, %3, %2"
4810   [(set_attr "type" "bfx")]
4811 )
4812
4813 ;; When the bit position and width of the equivalent extraction add up to 32
4814 ;; we can use a W-reg LSL instruction taking advantage of the implicit
4815 ;; zero-extension of the X-reg.
4816 (define_split
4817   [(set (match_operand:DI 0 "register_operand")
4818         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
4819                              (match_operand 2 "const_int_operand"))
4820                  (match_operand 3 "const_int_operand")))]
4821  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
4822   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
4823       == GET_MODE_BITSIZE (SImode)"
4824   [(set (match_dup 0)
4825         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
4826   {
4827     operands[4] = gen_lowpart (SImode, operands[1]);
4828   }
4829 )
4830
4831 (define_insn "bswap<mode>2"
4832   [(set (match_operand:GPI 0 "register_operand" "=r")
4833         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4834   ""
4835   "rev\\t%<w>0, %<w>1"
4836   [(set_attr "type" "rev")]
4837 )
4838
4839 (define_insn "bswaphi2"
4840   [(set (match_operand:HI 0 "register_operand" "=r")
4841         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4842   ""
4843   "rev16\\t%w0, %w1"
4844   [(set_attr "type" "rev")]
4845 )
4846
4847 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4848 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4849 ;; each valid permutation.
4850
4851 (define_insn "rev16<mode>2"
4852   [(set (match_operand:GPI 0 "register_operand" "=r")
4853         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4854                                       (const_int 8))
4855                           (match_operand:GPI 3 "const_int_operand" "n"))
4856                  (and:GPI (lshiftrt:GPI (match_dup 1)
4857                                         (const_int 8))
4858                           (match_operand:GPI 2 "const_int_operand" "n"))))]
4859   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4860    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4861   "rev16\\t%<w>0, %<w>1"
4862   [(set_attr "type" "rev")]
4863 )
4864
4865 (define_insn "rev16<mode>2_alt"
4866   [(set (match_operand:GPI 0 "register_operand" "=r")
4867         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4868                                         (const_int 8))
4869                           (match_operand:GPI 2 "const_int_operand" "n"))
4870                  (and:GPI (ashift:GPI (match_dup 1)
4871                                       (const_int 8))
4872                           (match_operand:GPI 3 "const_int_operand" "n"))))]
4873   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4874    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4875   "rev16\\t%<w>0, %<w>1"
4876   [(set_attr "type" "rev")]
4877 )
4878
4879 ;; zero_extend version of above
4880 (define_insn "*bswapsi2_uxtw"
4881   [(set (match_operand:DI 0 "register_operand" "=r")
4882         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4883   ""
4884   "rev\\t%w0, %w1"
4885   [(set_attr "type" "rev")]
4886 )
4887
4888 ;; -------------------------------------------------------------------
4889 ;; Floating-point intrinsics
4890 ;; -------------------------------------------------------------------
4891
4892 ;; frint floating-point round to integral standard patterns.
4893 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4894
4895 (define_insn "<frint_pattern><mode>2"
4896   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4897         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4898          FRINT))]
4899   "TARGET_FLOAT"
4900   "frint<frint_suffix>\\t%<s>0, %<s>1"
4901   [(set_attr "type" "f_rint<stype>")]
4902 )
4903
4904 ;; frcvt floating-point round to integer and convert standard patterns.
4905 ;; Expands to lbtrunc, lceil, lfloor, lround.
4906 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
4907   [(set (match_operand:GPI 0 "register_operand" "=r")
4908         (FIXUORS:GPI
4909           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4910            FCVT)))]
4911   "TARGET_FLOAT"
4912   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
4913   [(set_attr "type" "f_cvtf2i")]
4914 )
4915
4916 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4917   [(set (match_operand:GPI 0 "register_operand" "=r")
4918         (FIXUORS:GPI
4919           (mult:GPF
4920             (match_operand:GPF 1 "register_operand" "w")
4921             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4922   "TARGET_FLOAT
4923    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4924                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4925   {
4926     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4927     char buf[64];
4928     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4929     output_asm_insn (buf, operands);
4930     return "";
4931   }
4932   [(set_attr "type" "f_cvtf2i")]
4933 )
4934
4935 ;; fma - expand fma into patterns with the accumulator operand first since
4936 ;; reusing the accumulator results in better register allocation.
4937 ;; The register allocator considers copy preferences in operand order,
4938 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
4939
4940 (define_expand "fma<mode>4"
4941   [(set (match_operand:GPF_F16 0 "register_operand")
4942         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
4943                      (match_operand:GPF_F16 2 "register_operand")
4944                      (match_operand:GPF_F16 3 "register_operand")))]
4945   "TARGET_FLOAT"
4946 )
4947
4948 (define_insn "*aarch64_fma<mode>4"
4949   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4950         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
4951                      (match_operand:GPF_F16 3 "register_operand" "w")
4952                      (match_operand:GPF_F16 1 "register_operand" "w")))]
4953   "TARGET_FLOAT"
4954   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
4955   [(set_attr "type" "fmac<stype>")]
4956 )
4957
4958 (define_expand "fnma<mode>4"
4959   [(set (match_operand:GPF_F16 0 "register_operand")
4960         (fma:GPF_F16
4961           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
4962           (match_operand:GPF_F16 2 "register_operand")
4963           (match_operand:GPF_F16 3 "register_operand")))]
4964   "TARGET_FLOAT"
4965 )
4966
4967 (define_insn "*aarch64_fnma<mode>4"
4968   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4969         (fma:GPF_F16
4970           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
4971           (match_operand:GPF_F16 3 "register_operand" "w")
4972           (match_operand:GPF_F16 1 "register_operand" "w")))]
4973   "TARGET_FLOAT"
4974   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
4975   [(set_attr "type" "fmac<stype>")]
4976 )
4977
4978
4979 (define_expand "fms<mode>4"
4980   [(set (match_operand:GPF 0 "register_operand")
4981         (fma:GPF (match_operand:GPF 1 "register_operand")
4982                  (match_operand:GPF 2 "register_operand")
4983                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
4984   "TARGET_FLOAT"
4985 )
4986
4987 (define_insn "*aarch64_fms<mode>4"
4988   [(set (match_operand:GPF 0 "register_operand" "=w")
4989         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
4990                  (match_operand:GPF 3 "register_operand" "w")
4991                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
4992   "TARGET_FLOAT"
4993   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
4994   [(set_attr "type" "fmac<s>")]
4995 )
4996
4997 (define_expand "fnms<mode>4"
4998   [(set (match_operand:GPF 0 "register_operand")
4999         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5000                  (match_operand:GPF 2 "register_operand")
5001                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5002   "TARGET_FLOAT"
5003 )
5004
5005 (define_insn "*aarch64_fnms<mode>4"
5006   [(set (match_operand:GPF 0 "register_operand" "=w")
5007         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5008                  (match_operand:GPF 3 "register_operand" "w")
5009                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5010   "TARGET_FLOAT"
5011   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5012   [(set_attr "type" "fmac<s>")]
5013 )
5014
5015 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5016 (define_insn "*aarch64_fnmadd<mode>4"
5017   [(set (match_operand:GPF 0 "register_operand" "=w")
5018         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5019                           (match_operand:GPF 3 "register_operand" "w")
5020                           (match_operand:GPF 1 "register_operand" "w"))))]
5021   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5022   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5023   [(set_attr "type" "fmac<s>")]
5024 )
5025
5026 ;; -------------------------------------------------------------------
5027 ;; Floating-point conversions
5028 ;; -------------------------------------------------------------------
5029
5030 (define_insn "extendsfdf2"
5031   [(set (match_operand:DF 0 "register_operand" "=w")
5032         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5033   "TARGET_FLOAT"
5034   "fcvt\\t%d0, %s1"
5035   [(set_attr "type" "f_cvt")]
5036 )
5037
5038 (define_insn "extendhfsf2"
5039   [(set (match_operand:SF 0 "register_operand" "=w")
5040         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5041   "TARGET_FLOAT"
5042   "fcvt\\t%s0, %h1"
5043   [(set_attr "type" "f_cvt")]
5044 )
5045
5046 (define_insn "extendhfdf2"
5047   [(set (match_operand:DF 0 "register_operand" "=w")
5048         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5049   "TARGET_FLOAT"
5050   "fcvt\\t%d0, %h1"
5051   [(set_attr "type" "f_cvt")]
5052 )
5053
5054 (define_insn "truncdfsf2"
5055   [(set (match_operand:SF 0 "register_operand" "=w")
5056         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5057   "TARGET_FLOAT"
5058   "fcvt\\t%s0, %d1"
5059   [(set_attr "type" "f_cvt")]
5060 )
5061
5062 (define_insn "truncsfhf2"
5063   [(set (match_operand:HF 0 "register_operand" "=w")
5064         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5065   "TARGET_FLOAT"
5066   "fcvt\\t%h0, %s1"
5067   [(set_attr "type" "f_cvt")]
5068 )
5069
5070 (define_insn "truncdfhf2"
5071   [(set (match_operand:HF 0 "register_operand" "=w")
5072         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5073   "TARGET_FLOAT"
5074   "fcvt\\t%h0, %d1"
5075   [(set_attr "type" "f_cvt")]
5076 )
5077
5078 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5079 ;; and making r = w more expensive
5080
5081 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5082   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5083         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5084   "TARGET_FLOAT"
5085   "@
5086    fcvtz<su>\t%<s>0, %<s>1
5087    fcvtz<su>\t%<w>0, %<s>1"
5088   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5089 )
5090
5091 ;; Convert HF -> SI or DI
5092
5093 (define_insn "<optab>_trunchf<GPI:mode>2"
5094   [(set (match_operand:GPI 0 "register_operand" "=r")
5095         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5096   "TARGET_FP_F16INST"
5097   "fcvtz<su>\t%<w>0, %h1"
5098   [(set_attr "type" "f_cvtf2i")]
5099 )
5100
5101 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5102 ;; input in a fp register and output in a integer register
5103
5104 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5105   [(set (match_operand:GPI 0 "register_operand" "=r")
5106         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5107   "TARGET_FLOAT"
5108   "fcvtz<su>\t%<w>0, %<fpw>1"
5109   [(set_attr "type" "f_cvtf2i")]
5110 )
5111
5112 (define_insn "*fix_to_zero_extend<mode>di2"
5113   [(set (match_operand:DI 0 "register_operand" "=r")
5114         (zero_extend:DI
5115          (unsigned_fix:SI
5116           (match_operand:GPF 1 "register_operand" "w"))))]
5117   "TARGET_FLOAT"
5118   "fcvtzu\t%w0, %<s>1"
5119   [(set_attr "type" "f_cvtf2i")]
5120 )
5121
5122 (define_insn "<optab><fcvt_target><GPF:mode>2"
5123   [(set (match_operand:GPF 0 "register_operand" "=w,w")
5124         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
5125   "TARGET_FLOAT"
5126   "@
5127    <su_optab>cvtf\t%<GPF:s>0, %<s>1
5128    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5129   [(set_attr "simd" "yes,no")
5130    (set_attr "fp" "no,yes")
5131    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
5132 )
5133
5134 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5135   [(set (match_operand:GPF 0 "register_operand" "=w")
5136         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5137   "TARGET_FLOAT"
5138   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5139   [(set_attr "type" "f_cvti2f")]
5140 )
5141
5142 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5143 ;; midend will arrange for an SImode conversion to HFmode to first go
5144 ;; through DFmode, then to HFmode.  But first it will try converting
5145 ;; to DImode then down, which would match our DImode pattern below and
5146 ;; give very poor code-generation.  So, we must provide our own emulation
5147 ;; of the mid-end logic.
5148
5149 (define_insn "aarch64_fp16_<optab><mode>hf2"
5150   [(set (match_operand:HF 0 "register_operand" "=w")
5151         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5152   "TARGET_FP_F16INST"
5153   "<su_optab>cvtf\t%h0, %<w>1"
5154   [(set_attr "type" "f_cvti2f")]
5155 )
5156
5157 (define_expand "<optab>sihf2"
5158   [(set (match_operand:HF 0 "register_operand")
5159         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5160   "TARGET_FLOAT"
5161 {
5162   if (TARGET_FP_F16INST)
5163     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5164   else
5165     {
5166       rtx convert_target = gen_reg_rtx (DFmode);
5167       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5168       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5169     }
5170   DONE;
5171 }
5172 )
5173
5174 ;; For DImode there is no wide enough floating-point mode that we
5175 ;; can convert through natively (TFmode would work, but requires a library
5176 ;; call).  However, we know that any value >= 65504 will be rounded
5177 ;; to infinity on conversion.  This is well within the range of SImode, so
5178 ;; we can:
5179 ;;   Saturate to SImode.
5180 ;;   Convert from that to DFmode
5181 ;;   Convert from that to HFmode (phew!).
5182 ;; Note that the saturation to SImode requires the SIMD extensions.  If
5183 ;; we ever need to provide this pattern where the SIMD extensions are not
5184 ;; available, we would need a different approach.
5185
5186 (define_expand "<optab>dihf2"
5187   [(set (match_operand:HF 0 "register_operand")
5188         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5189   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5190 {
5191   if (TARGET_FP_F16INST)
5192     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5193   else
5194     {
5195       rtx sat_target = gen_reg_rtx (SImode);
5196       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5197       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5198     }
5199
5200   DONE;
5201 }
5202 )
5203
5204 ;; Convert between fixed-point and floating-point (scalar modes)
5205
5206 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5207   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5208         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5209                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5210          FCVT_F2FIXED))]
5211   ""
5212   "@
5213    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5214    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5215   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5216    (set_attr "fp" "yes, *")
5217    (set_attr "simd" "*, yes")]
5218 )
5219
5220 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5221   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5222         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5223                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5224          FCVT_FIXED2F))]
5225   ""
5226   "@
5227    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5228    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5229   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5230    (set_attr "fp" "yes, *")
5231    (set_attr "simd" "*, yes")]
5232 )
5233
5234 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5235   [(set (match_operand:GPI 0 "register_operand" "=r")
5236         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5237                      (match_operand:SI 2 "immediate_operand" "i")]
5238          FCVT_F2FIXED))]
5239   "TARGET_FP_F16INST"
5240    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5241   [(set_attr "type" "f_cvtf2i")]
5242 )
5243
5244 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5245   [(set (match_operand:HF 0 "register_operand" "=w")
5246         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5247                     (match_operand:SI 2 "immediate_operand" "i")]
5248          FCVT_FIXED2F))]
5249   "TARGET_FP_F16INST"
5250   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5251   [(set_attr "type" "f_cvti2f")]
5252 )
5253
5254 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5255   [(set (match_operand:HI 0 "register_operand" "=w")
5256         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5257                     (match_operand:SI 2 "immediate_operand" "i")]
5258          FCVT_F2FIXED))]
5259   "TARGET_SIMD"
5260   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5261   [(set_attr "type" "neon_fp_to_int_s")]
5262 )
5263
5264 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5265   [(set (match_operand:HF 0 "register_operand" "=w")
5266         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5267                     (match_operand:SI 2 "immediate_operand" "i")]
5268          FCVT_FIXED2F))]
5269   "TARGET_SIMD"
5270   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5271   [(set_attr "type" "neon_int_to_fp_s")]
5272 )
5273
5274 ;; -------------------------------------------------------------------
5275 ;; Floating-point arithmetic
5276 ;; -------------------------------------------------------------------
5277
5278 (define_insn "add<mode>3"
5279   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5280         (plus:GPF_F16
5281          (match_operand:GPF_F16 1 "register_operand" "w")
5282          (match_operand:GPF_F16 2 "register_operand" "w")))]
5283   "TARGET_FLOAT"
5284   "fadd\\t%<s>0, %<s>1, %<s>2"
5285   [(set_attr "type" "fadd<stype>")]
5286 )
5287
5288 (define_insn "sub<mode>3"
5289   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5290         (minus:GPF_F16
5291          (match_operand:GPF_F16 1 "register_operand" "w")
5292          (match_operand:GPF_F16 2 "register_operand" "w")))]
5293   "TARGET_FLOAT"
5294   "fsub\\t%<s>0, %<s>1, %<s>2"
5295   [(set_attr "type" "fadd<stype>")]
5296 )
5297
5298 (define_insn "mul<mode>3"
5299   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5300         (mult:GPF_F16
5301          (match_operand:GPF_F16 1 "register_operand" "w")
5302          (match_operand:GPF_F16 2 "register_operand" "w")))]
5303   "TARGET_FLOAT"
5304   "fmul\\t%<s>0, %<s>1, %<s>2"
5305   [(set_attr "type" "fmul<stype>")]
5306 )
5307
5308 (define_insn "*fnmul<mode>3"
5309   [(set (match_operand:GPF 0 "register_operand" "=w")
5310         (mult:GPF
5311                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5312                  (match_operand:GPF 2 "register_operand" "w")))]
5313   "TARGET_FLOAT && !flag_rounding_math"
5314   "fnmul\\t%<s>0, %<s>1, %<s>2"
5315   [(set_attr "type" "fmul<s>")]
5316 )
5317
5318 (define_insn "*fnmul<mode>3"
5319   [(set (match_operand:GPF 0 "register_operand" "=w")
5320         (neg:GPF (mult:GPF
5321                  (match_operand:GPF 1 "register_operand" "w")
5322                  (match_operand:GPF 2 "register_operand" "w"))))]
5323   "TARGET_FLOAT"
5324   "fnmul\\t%<s>0, %<s>1, %<s>2"
5325   [(set_attr "type" "fmul<s>")]
5326 )
5327
5328 (define_expand "div<mode>3"
5329  [(set (match_operand:GPF_F16 0 "register_operand")
5330        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5331                     (match_operand:GPF_F16 2 "register_operand")))]
5332  "TARGET_FLOAT"
5333 {
5334   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5335     DONE;
5336
5337   operands[1] = force_reg (<MODE>mode, operands[1]);
5338 })
5339
5340 (define_insn "*div<mode>3"
5341   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5342         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5343                      (match_operand:GPF_F16 2 "register_operand" "w")))]
5344   "TARGET_FLOAT"
5345   "fdiv\\t%<s>0, %<s>1, %<s>2"
5346   [(set_attr "type" "fdiv<stype>")]
5347 )
5348
5349 (define_insn "neg<mode>2"
5350   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5351         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5352   "TARGET_FLOAT"
5353   "fneg\\t%<s>0, %<s>1"
5354   [(set_attr "type" "ffarith<stype>")]
5355 )
5356
5357 (define_expand "sqrt<mode>2"
5358   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5359         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5360   "TARGET_FLOAT"
5361 {
5362   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5363     DONE;
5364 })
5365
5366 (define_insn "*sqrt<mode>2"
5367   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5368         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5369   "TARGET_FLOAT"
5370   "fsqrt\\t%<s>0, %<s>1"
5371   [(set_attr "type" "fsqrt<stype>")]
5372 )
5373
5374 (define_insn "abs<mode>2"
5375   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5376         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5377   "TARGET_FLOAT"
5378   "fabs\\t%<s>0, %<s>1"
5379   [(set_attr "type" "ffarith<stype>")]
5380 )
5381
5382 ;; Given that smax/smin do not specify the result when either input is NaN,
5383 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5384 ;; for smin.
5385
5386 (define_insn "smax<mode>3"
5387   [(set (match_operand:GPF 0 "register_operand" "=w")
5388         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5389                   (match_operand:GPF 2 "register_operand" "w")))]
5390   "TARGET_FLOAT"
5391   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5392   [(set_attr "type" "f_minmax<s>")]
5393 )
5394
5395 (define_insn "smin<mode>3"
5396   [(set (match_operand:GPF 0 "register_operand" "=w")
5397         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5398                   (match_operand:GPF 2 "register_operand" "w")))]
5399   "TARGET_FLOAT"
5400   "fminnm\\t%<s>0, %<s>1, %<s>2"
5401   [(set_attr "type" "f_minmax<s>")]
5402 )
5403
5404 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5405 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5406 ;; which implement the IEEE fmax ()/fmin () functions.
5407 (define_insn "<maxmin_uns><mode>3"
5408   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5409         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5410                      (match_operand:GPF_F16 2 "register_operand" "w")]
5411                      FMAXMIN_UNS))]
5412   "TARGET_FLOAT"
5413   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5414   [(set_attr "type" "f_minmax<stype>")]
5415 )
5416
5417 (define_expand "lrint<GPF:mode><GPI:mode>2"
5418   [(match_operand:GPI 0 "register_operand")
5419    (match_operand:GPF 1 "register_operand")]
5420   "TARGET_FLOAT
5421    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5422    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5423 {
5424   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5425   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5426   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5427   DONE;
5428 }
5429 )
5430
5431 ;; For copysign (x, y), we want to generate:
5432 ;;
5433 ;;   LDR d2, #(1 << 63)
5434 ;;   BSL v2.8b, [y], [x]
5435 ;;
5436 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5437 ;; aarch64_simd_bsldf will select the best suited of these instructions
5438 ;; to generate based on register allocation, and knows how to partially
5439 ;; constant fold based on the values of X and Y, so expand through that.
5440
5441 (define_expand "copysigndf3"
5442   [(match_operand:DF 0 "register_operand")
5443    (match_operand:DF 1 "register_operand")
5444    (match_operand:DF 2 "register_operand")]
5445   "TARGET_FLOAT && TARGET_SIMD"
5446 {
5447   rtx mask = gen_reg_rtx (DImode);
5448   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
5449   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
5450                                      operands[2], operands[1]));
5451   DONE;
5452 }
5453 )
5454
5455 ;; As above, but we must first get to a 64-bit value if we wish to use
5456 ;; aarch64_simd_bslv2sf.
5457
5458 (define_expand "copysignsf3"
5459   [(match_operand:SF 0 "register_operand")
5460    (match_operand:SF 1 "register_operand")
5461    (match_operand:SF 2 "register_operand")]
5462   "TARGET_FLOAT && TARGET_SIMD"
5463 {
5464   rtx v_bitmask = gen_reg_rtx (V2SImode);
5465
5466   /* Juggle modes to get us in to a vector mode for BSL.  */
5467   rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
5468   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
5469   rtx tmp = gen_reg_rtx (V2SFmode);
5470   emit_move_insn (v_bitmask,
5471                   aarch64_simd_gen_const_vector_dup (V2SImode,
5472                                                      HOST_WIDE_INT_M1U << 31));
5473   emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
5474   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
5475   DONE;
5476 }
5477 )
5478
5479 ;; For xorsign (x, y), we want to generate:
5480 ;;
5481 ;; LDR   d2, #1<<63
5482 ;; AND   v3.8B, v1.8B, v2.8B
5483 ;; EOR   v0.8B, v0.8B, v3.8B
5484 ;;
5485
5486 (define_expand "xorsign<mode>3"
5487   [(match_operand:GPF 0 "register_operand")
5488    (match_operand:GPF 1 "register_operand")
5489    (match_operand:GPF 2 "register_operand")]
5490   "TARGET_FLOAT && TARGET_SIMD"
5491 {
5492
5493   machine_mode imode = <V_INT_EQUIV>mode;
5494   rtx mask = gen_reg_rtx (imode);
5495   rtx op1x = gen_reg_rtx (imode);
5496   rtx op2x = gen_reg_rtx (imode);
5497
5498   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
5499   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
5500                                                      imode)));
5501
5502   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
5503                                     lowpart_subreg (imode, operands[2],
5504                                                     <MODE>mode)));
5505   emit_insn (gen_xor<v_int_equiv>3 (op1x,
5506                                     lowpart_subreg (imode, operands[1],
5507                                                     <MODE>mode),
5508                                     op2x));
5509   emit_move_insn (operands[0],
5510                   lowpart_subreg (<MODE>mode, op1x, imode));
5511   DONE;
5512 }
5513 )
5514
5515 ;; -------------------------------------------------------------------
5516 ;; Reload support
5517 ;; -------------------------------------------------------------------
5518 ;; Reload Scalar Floating point modes from constant pool.
5519 ;; The AArch64 port doesn't have __int128 constant move support.
5520 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
5521  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
5522        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
5523   (clobber (match_operand:P 2 "register_operand" "=&r"))]
5524  "TARGET_FLOAT"
5525  {
5526    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5527    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
5528    DONE;
5529  }
5530 )
5531
5532 ;; Reload Vector modes from constant pool.
5533 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
5534  [(set (match_operand:VALL 0 "register_operand" "=w")
5535        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
5536   (clobber (match_operand:P 2 "register_operand" "=&r"))]
5537  "TARGET_FLOAT"
5538  {
5539    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5540    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
5541    DONE;
5542  }
5543 )
5544
5545 (define_expand "aarch64_reload_mov<mode>"
5546   [(set (match_operand:TX 0 "register_operand" "=w")
5547         (match_operand:TX 1 "register_operand" "w"))
5548    (clobber (match_operand:DI 2 "register_operand" "=&r"))
5549   ]
5550   "TARGET_FLOAT"
5551   {
5552     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
5553     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
5554     gen_aarch64_movtilow_tilow (op0, op1);
5555     gen_aarch64_movdi_tihigh (operands[2], op1);
5556     gen_aarch64_movtihigh_di (op0, operands[2]);
5557     DONE;
5558   }
5559 )
5560
5561 ;; The following secondary reload helpers patterns are invoked
5562 ;; after or during reload as we don't want these patterns to start
5563 ;; kicking in during the combiner.
5564
5565 (define_insn "aarch64_movdi_<mode>low"
5566   [(set (match_operand:DI 0 "register_operand" "=r")
5567         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5568                          (const_int 64) (const_int 0)))]
5569   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5570   "fmov\\t%x0, %d1"
5571   [(set_attr "type" "f_mrc")
5572    (set_attr "length" "4")
5573   ])
5574
5575 (define_insn "aarch64_movdi_<mode>high"
5576   [(set (match_operand:DI 0 "register_operand" "=r")
5577         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5578                          (const_int 64) (const_int 64)))]
5579   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5580   "fmov\\t%x0, %1.d[1]"
5581   [(set_attr "type" "f_mrc")
5582    (set_attr "length" "4")
5583   ])
5584
5585 (define_insn "aarch64_mov<mode>high_di"
5586   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
5587                          (const_int 64) (const_int 64))
5588         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5589   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5590   "fmov\\t%0.d[1], %x1"
5591   [(set_attr "type" "f_mcr")
5592    (set_attr "length" "4")
5593   ])
5594
5595 (define_insn "aarch64_mov<mode>low_di"
5596   [(set (match_operand:TX 0 "register_operand" "=w")
5597         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5598   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5599   "fmov\\t%d0, %x1"
5600   [(set_attr "type" "f_mcr")
5601    (set_attr "length" "4")
5602   ])
5603
5604 (define_insn "aarch64_movtilow_tilow"
5605   [(set (match_operand:TI 0 "register_operand" "=w")
5606         (zero_extend:TI
5607           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
5608   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5609   "fmov\\t%d0, %d1"
5610   [(set_attr "type" "fmov")
5611    (set_attr "length" "4")
5612   ])
5613
5614 ;; There is a deliberate reason why the parameters of high and lo_sum's
5615 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
5616 ;; and lo_sum's to be used with the labels defining the jump tables in
5617 ;; rodata section.
5618
5619 (define_expand "add_losym"
5620   [(set (match_operand 0 "register_operand" "=r")
5621         (lo_sum (match_operand 1 "register_operand" "r")
5622                 (match_operand 2 "aarch64_valid_symref" "S")))]
5623   ""
5624 {
5625   machine_mode mode = GET_MODE (operands[0]);
5626
5627   emit_insn ((mode == DImode
5628               ? gen_add_losym_di
5629               : gen_add_losym_si) (operands[0],
5630                                    operands[1],
5631                                    operands[2]));
5632   DONE;
5633 })
5634
5635 (define_insn "add_losym_<mode>"
5636   [(set (match_operand:P 0 "register_operand" "=r")
5637         (lo_sum:P (match_operand:P 1 "register_operand" "r")
5638                   (match_operand 2 "aarch64_valid_symref" "S")))]
5639   ""
5640   "add\\t%<w>0, %<w>1, :lo12:%c2"
5641   [(set_attr "type" "alu_imm")]
5642 )
5643
5644 (define_insn "ldr_got_small_<mode>"
5645   [(set (match_operand:PTR 0 "register_operand" "=r")
5646         (unspec:PTR [(mem:PTR (lo_sum:PTR
5647                               (match_operand:PTR 1 "register_operand" "r")
5648                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5649                     UNSPEC_GOTSMALLPIC))]
5650   ""
5651   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
5652   [(set_attr "type" "load_<ldst_sz>")]
5653 )
5654
5655 (define_insn "ldr_got_small_sidi"
5656   [(set (match_operand:DI 0 "register_operand" "=r")
5657         (zero_extend:DI
5658          (unspec:SI [(mem:SI (lo_sum:DI
5659                              (match_operand:DI 1 "register_operand" "r")
5660                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5661                     UNSPEC_GOTSMALLPIC)))]
5662   "TARGET_ILP32"
5663   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
5664   [(set_attr "type" "load_4")]
5665 )
5666
5667 (define_insn "ldr_got_small_28k_<mode>"
5668   [(set (match_operand:PTR 0 "register_operand" "=r")
5669         (unspec:PTR [(mem:PTR (lo_sum:PTR
5670                               (match_operand:PTR 1 "register_operand" "r")
5671                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5672                     UNSPEC_GOTSMALLPIC28K))]
5673   ""
5674   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
5675   [(set_attr "type" "load_<ldst_sz>")]
5676 )
5677
5678 (define_insn "ldr_got_small_28k_sidi"
5679   [(set (match_operand:DI 0 "register_operand" "=r")
5680         (zero_extend:DI
5681          (unspec:SI [(mem:SI (lo_sum:DI
5682                              (match_operand:DI 1 "register_operand" "r")
5683                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5684                     UNSPEC_GOTSMALLPIC28K)))]
5685   "TARGET_ILP32"
5686   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
5687   [(set_attr "type" "load_4")]
5688 )
5689
5690 (define_insn "ldr_got_tiny"
5691   [(set (match_operand:DI 0 "register_operand" "=r")
5692         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
5693                    UNSPEC_GOTTINYPIC))]
5694   ""
5695   "ldr\\t%0, %L1"
5696   [(set_attr "type" "load_8")]
5697 )
5698
5699 (define_insn "aarch64_load_tp_hard"
5700   [(set (match_operand:DI 0 "register_operand" "=r")
5701         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
5702   ""
5703   "mrs\\t%0, tpidr_el0"
5704   [(set_attr "type" "mrs")]
5705 )
5706
5707 ;; The TLS ABI specifically requires that the compiler does not schedule
5708 ;; instructions in the TLS stubs, in order to enable linker relaxation.
5709 ;; Therefore we treat the stubs as an atomic sequence.
5710 (define_expand "tlsgd_small_<mode>"
5711  [(parallel [(set (match_operand 0 "register_operand" "")
5712                   (call (mem:DI (match_dup 2)) (const_int 1)))
5713              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
5714              (clobber (reg:DI LR_REGNUM))])]
5715  ""
5716 {
5717   operands[2] = aarch64_tls_get_addr ();
5718 })
5719
5720 (define_insn "*tlsgd_small_<mode>"
5721   [(set (match_operand 0 "register_operand" "")
5722         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
5723    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
5724    (clobber (reg:DI LR_REGNUM))
5725   ]
5726   ""
5727   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
5728   [(set_attr "type" "call")
5729    (set_attr "length" "16")])
5730
5731 (define_insn "tlsie_small_<mode>"
5732   [(set (match_operand:PTR 0 "register_operand" "=r")
5733         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5734                    UNSPEC_GOTSMALLTLS))]
5735   ""
5736   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
5737   [(set_attr "type" "load_4")
5738    (set_attr "length" "8")]
5739 )
5740
5741 (define_insn "tlsie_small_sidi"
5742   [(set (match_operand:DI 0 "register_operand" "=r")
5743         (zero_extend:DI
5744           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5745                       UNSPEC_GOTSMALLTLS)))]
5746   ""
5747   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
5748   [(set_attr "type" "load_4")
5749    (set_attr "length" "8")]
5750 )
5751
5752 (define_insn "tlsie_tiny_<mode>"
5753   [(set (match_operand:PTR 0 "register_operand" "=&r")
5754         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
5755                      (match_operand:PTR 2 "register_operand" "r")]
5756                    UNSPEC_GOTTINYTLS))]
5757   ""
5758   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
5759   [(set_attr "type" "multiple")
5760    (set_attr "length" "8")]
5761 )
5762
5763 (define_insn "tlsie_tiny_sidi"
5764   [(set (match_operand:DI 0 "register_operand" "=&r")
5765         (zero_extend:DI
5766           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
5767                       (match_operand:DI 2 "register_operand" "r")
5768                       ]
5769                       UNSPEC_GOTTINYTLS)))]
5770   ""
5771   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5772   [(set_attr "type" "multiple")
5773    (set_attr "length" "8")]
5774 )
5775
5776 (define_insn "tlsle12_<mode>"
5777   [(set (match_operand:P 0 "register_operand" "=r")
5778         (unspec:P [(match_operand:P 1 "register_operand" "r")
5779                    (match_operand 2 "aarch64_tls_le_symref" "S")]
5780                    UNSPEC_TLSLE12))]
5781   ""
5782   "add\\t%<w>0, %<w>1, #%L2";
5783   [(set_attr "type" "alu_sreg")
5784    (set_attr "length" "4")]
5785 )
5786
5787 (define_insn "tlsle24_<mode>"
5788   [(set (match_operand:P 0 "register_operand" "=r")
5789         (unspec:P [(match_operand:P 1 "register_operand" "r")
5790                    (match_operand 2 "aarch64_tls_le_symref" "S")]
5791                    UNSPEC_TLSLE24))]
5792   ""
5793   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
5794   [(set_attr "type" "multiple")
5795    (set_attr "length" "8")]
5796 )
5797
5798 (define_insn "tlsle32_<mode>"
5799   [(set (match_operand:P 0 "register_operand" "=r")
5800         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5801                    UNSPEC_TLSLE32))]
5802   ""
5803   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5804   [(set_attr "type" "multiple")
5805    (set_attr "length" "8")]
5806 )
5807
5808 (define_insn "tlsle48_<mode>"
5809   [(set (match_operand:P 0 "register_operand" "=r")
5810         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5811                    UNSPEC_TLSLE48))]
5812   ""
5813   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5814   [(set_attr "type" "multiple")
5815    (set_attr "length" "12")]
5816 )
5817
5818 (define_expand "tlsdesc_small_<mode>"
5819   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
5820   "TARGET_TLS_DESC"
5821   {
5822     if (TARGET_SVE)
5823       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
5824     else
5825       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
5826     DONE;
5827   }
5828 )
5829
5830 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
5831 ;; R0 and LR.
5832 (define_insn "tlsdesc_small_advsimd_<mode>"
5833   [(set (reg:PTR R0_REGNUM)
5834         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5835                     UNSPEC_TLSDESC))
5836    (clobber (reg:DI LR_REGNUM))
5837    (clobber (reg:CC CC_REGNUM))
5838    (clobber (match_scratch:DI 1 "=r"))]
5839   "TARGET_TLS_DESC && !TARGET_SVE"
5840   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5841   [(set_attr "type" "call")
5842    (set_attr "length" "16")])
5843
5844 ;; For SVE, model tlsdesc calls as clobbering all vector and predicate
5845 ;; registers, on top of the usual R0 and LR.  In reality the calls
5846 ;; preserve the low 128 bits of the vector registers, but we don't
5847 ;; yet have a way of representing that in the instruction pattern.
5848 (define_insn "tlsdesc_small_sve_<mode>"
5849   [(set (reg:PTR R0_REGNUM)
5850         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5851                     UNSPEC_TLSDESC))
5852    (clobber (reg:DI LR_REGNUM))
5853    (clobber (reg:CC CC_REGNUM))
5854    (clobber (reg:XI V0_REGNUM))
5855    (clobber (reg:XI V4_REGNUM))
5856    (clobber (reg:XI V8_REGNUM))
5857    (clobber (reg:XI V12_REGNUM))
5858    (clobber (reg:XI V16_REGNUM))
5859    (clobber (reg:XI V20_REGNUM))
5860    (clobber (reg:XI V24_REGNUM))
5861    (clobber (reg:XI V28_REGNUM))
5862    (clobber (reg:VNx2BI P0_REGNUM))
5863    (clobber (reg:VNx2BI P1_REGNUM))
5864    (clobber (reg:VNx2BI P2_REGNUM))
5865    (clobber (reg:VNx2BI P3_REGNUM))
5866    (clobber (reg:VNx2BI P4_REGNUM))
5867    (clobber (reg:VNx2BI P5_REGNUM))
5868    (clobber (reg:VNx2BI P6_REGNUM))
5869    (clobber (reg:VNx2BI P7_REGNUM))
5870    (clobber (reg:VNx2BI P8_REGNUM))
5871    (clobber (reg:VNx2BI P9_REGNUM))
5872    (clobber (reg:VNx2BI P10_REGNUM))
5873    (clobber (reg:VNx2BI P11_REGNUM))
5874    (clobber (reg:VNx2BI P12_REGNUM))
5875    (clobber (reg:VNx2BI P13_REGNUM))
5876    (clobber (reg:VNx2BI P14_REGNUM))
5877    (clobber (reg:VNx2BI P15_REGNUM))
5878    (clobber (match_scratch:DI 1 "=r"))]
5879   "TARGET_TLS_DESC && TARGET_SVE"
5880   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5881   [(set_attr "type" "call")
5882    (set_attr "length" "16")])
5883
5884 (define_insn "stack_tie"
5885   [(set (mem:BLK (scratch))
5886         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
5887                      (match_operand:DI 1 "register_operand" "rk")]
5888                     UNSPEC_PRLG_STK))]
5889   ""
5890   ""
5891   [(set_attr "length" "0")]
5892 )
5893
5894 ;; Pointer authentication patterns are always provided.  In architecture
5895 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
5896 ;; This lets the user write portable software which authenticates pointers
5897 ;; when run on something which implements ARMv8.3-A, and which runs
5898 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
5899 ;; implemented.
5900
5901 ;; Signing/Authenticating R30 using SP as the salt.
5902
5903 (define_insn "<pauth_mnem_prefix>sp"
5904   [(set (reg:DI R30_REGNUM)
5905         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
5906   ""
5907   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
5908 )
5909
5910 ;; Signing/Authenticating X17 using X16 as the salt.
5911
5912 (define_insn "<pauth_mnem_prefix>1716"
5913   [(set (reg:DI R17_REGNUM)
5914         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
5915   ""
5916   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
5917 )
5918
5919 ;; Stripping the signature in R30.
5920
5921 (define_insn "xpaclri"
5922   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
5923   ""
5924   "hint\t7 // xpaclri"
5925 )
5926
5927 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
5928 ;; all of memory.  This blocks insns from being moved across this point.
5929
5930 (define_insn "blockage"
5931   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
5932   ""
5933   ""
5934   [(set_attr "length" "0")
5935    (set_attr "type" "block")]
5936 )
5937
5938 (define_insn "probe_stack_range"
5939   [(set (match_operand:DI 0 "register_operand" "=r")
5940         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
5941                              (match_operand:DI 2 "register_operand" "r")]
5942                               UNSPECV_PROBE_STACK_RANGE))]
5943   ""
5944 {
5945   return aarch64_output_probe_stack_range (operands[0], operands[2]);
5946 }
5947   [(set_attr "length" "32")]
5948 )
5949
5950 ;; Named pattern for expanding thread pointer reference.
5951 (define_expand "get_thread_pointerdi"
5952   [(match_operand:DI 0 "register_operand" "=r")]
5953   ""
5954 {
5955   rtx tmp = aarch64_load_tp (operands[0]);
5956   if (tmp != operands[0])
5957     emit_move_insn (operands[0], tmp);
5958   DONE;
5959 })
5960
5961 ;; Named patterns for stack smashing protection.
5962 (define_expand "stack_protect_set"
5963   [(match_operand 0 "memory_operand")
5964    (match_operand 1 "memory_operand")]
5965   ""
5966 {
5967   machine_mode mode = GET_MODE (operands[0]);
5968
5969   emit_insn ((mode == DImode
5970               ? gen_stack_protect_set_di
5971               : gen_stack_protect_set_si) (operands[0], operands[1]));
5972   DONE;
5973 })
5974
5975 (define_insn "stack_protect_set_<mode>"
5976   [(set (match_operand:PTR 0 "memory_operand" "=m")
5977         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
5978          UNSPEC_SP_SET))
5979    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
5980   ""
5981   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
5982   [(set_attr "length" "12")
5983    (set_attr "type" "multiple")])
5984
5985 (define_expand "stack_protect_test"
5986   [(match_operand 0 "memory_operand")
5987    (match_operand 1 "memory_operand")
5988    (match_operand 2)]
5989   ""
5990 {
5991   rtx result;
5992   machine_mode mode = GET_MODE (operands[0]);
5993
5994   result = gen_reg_rtx(mode);
5995
5996   emit_insn ((mode == DImode
5997               ? gen_stack_protect_test_di
5998               : gen_stack_protect_test_si) (result,
5999                                             operands[0],
6000                                             operands[1]));
6001
6002   if (mode == DImode)
6003     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6004                                     result, const0_rtx, operands[2]));
6005   else
6006     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6007                                     result, const0_rtx, operands[2]));
6008   DONE;
6009 })
6010
6011 (define_insn "stack_protect_test_<mode>"
6012   [(set (match_operand:PTR 0 "register_operand" "=r")
6013         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
6014                      (match_operand:PTR 2 "memory_operand" "m")]
6015          UNSPEC_SP_TEST))
6016    (clobber (match_scratch:PTR 3 "=&r"))]
6017   ""
6018   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
6019   [(set_attr "length" "12")
6020    (set_attr "type" "multiple")])
6021
6022 ;; Write Floating-point Control Register.
6023 (define_insn "set_fpcr"
6024   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
6025   ""
6026   "msr\\tfpcr, %0"
6027   [(set_attr "type" "mrs")])
6028
6029 ;; Read Floating-point Control Register.
6030 (define_insn "get_fpcr"
6031   [(set (match_operand:SI 0 "register_operand" "=r")
6032         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
6033   ""
6034   "mrs\\t%0, fpcr"
6035   [(set_attr "type" "mrs")])
6036
6037 ;; Write Floating-point Status Register.
6038 (define_insn "set_fpsr"
6039   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
6040   ""
6041   "msr\\tfpsr, %0"
6042   [(set_attr "type" "mrs")])
6043
6044 ;; Read Floating-point Status Register.
6045 (define_insn "get_fpsr"
6046   [(set (match_operand:SI 0 "register_operand" "=r")
6047         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
6048   ""
6049   "mrs\\t%0, fpsr"
6050   [(set_attr "type" "mrs")])
6051
6052
6053 ;; Define the subtract-one-and-jump insns so loop.c
6054 ;; knows what to generate.
6055 (define_expand "doloop_end"
6056   [(use (match_operand 0 "" ""))      ; loop pseudo
6057    (use (match_operand 1 "" ""))]     ; label
6058   "optimize > 0 && flag_modulo_sched"
6059 {
6060   rtx s0;
6061   rtx bcomp;
6062   rtx loc_ref;
6063   rtx cc_reg;
6064   rtx insn;
6065   rtx cmp;
6066
6067   /* Currently SMS relies on the do-loop pattern to recognize loops
6068      where (1) the control part consists of all insns defining and/or
6069      using a certain 'count' register and (2) the loop count can be
6070      adjusted by modifying this register prior to the loop.
6071      ??? The possible introduction of a new block to initialize the
6072      new IV can potentially affect branch optimizations.  */
6073
6074   if (GET_MODE (operands[0]) != DImode)
6075     FAIL;
6076
6077   s0 = operands [0];
6078   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
6079
6080   cmp = XVECEXP (PATTERN (insn), 0, 0);
6081   cc_reg = SET_DEST (cmp);
6082   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6083   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6084   emit_jump_insn (gen_rtx_SET (pc_rtx,
6085                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6086                                                      loc_ref, pc_rtx)));
6087   DONE;
6088 })
6089
6090 ;; Helper for aarch64.c code.
6091 (define_expand "set_clobber_cc"
6092   [(parallel [(set (match_operand 0)
6093                    (match_operand 1))
6094               (clobber (reg:CC CC_REGNUM))])])
6095
6096 ;; AdvSIMD Stuff
6097 (include "aarch64-simd.md")
6098
6099 ;; Atomic Operations
6100 (include "atomics.md")
6101
6102 ;; ldp/stp peephole patterns
6103 (include "aarch64-ldpstp.md")
6104
6105 ;; SVE.
6106 (include "aarch64-sve.md")