gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2015 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V15_REGNUM         47)
  61     (V31_REGNUM         63)
  62     (SFP_REGNUM         64)
  63     (AP_REGNUM          65)
  64     (CC_REGNUM          66)
  65   ]
  66 )
  67
  68 (define_c_enum "unspec" [
  69     UNSPEC_CASESI
  70     UNSPEC_CRC32B
  71     UNSPEC_CRC32CB
  72     UNSPEC_CRC32CH
  73     UNSPEC_CRC32CW
  74     UNSPEC_CRC32CX
  75     UNSPEC_CRC32H
  76     UNSPEC_CRC32W
  77     UNSPEC_CRC32X
  78     UNSPEC_URECPE
  79     UNSPEC_FRECPE
  80     UNSPEC_FRECPS
  81     UNSPEC_FRECPX
  82     UNSPEC_FRINTA
  83     UNSPEC_FRINTI
  84     UNSPEC_FRINTM
  85     UNSPEC_FRINTN
  86     UNSPEC_FRINTP
  87     UNSPEC_FRINTX
  88     UNSPEC_FRINTZ
  89     UNSPEC_GOTSMALLPIC
  90     UNSPEC_GOTSMALLTLS
  91     UNSPEC_GOTTINYPIC
  92     UNSPEC_LD1
  93     UNSPEC_LD2
  94     UNSPEC_LD2_DUP
  95     UNSPEC_LD3
  96     UNSPEC_LD3_DUP
  97     UNSPEC_LD4
  98     UNSPEC_LD4_DUP
  99     UNSPEC_LD2_LANE
 100     UNSPEC_LD3_LANE
 101     UNSPEC_LD4_LANE
 102     UNSPEC_MB
 103     UNSPEC_NOP
 104     UNSPEC_PRLG_STK
 105     UNSPEC_RBIT
 106     UNSPEC_SISD_NEG
 107     UNSPEC_SISD_SSHL
 108     UNSPEC_SISD_USHL
 109     UNSPEC_SSHL_2S
 110     UNSPEC_ST1
 111     UNSPEC_ST2
 112     UNSPEC_ST3
 113     UNSPEC_ST4
 114     UNSPEC_ST2_LANE
 115     UNSPEC_ST3_LANE
 116     UNSPEC_ST4_LANE
 117     UNSPEC_TLS
 118     UNSPEC_TLSDESC
 119     UNSPEC_USHL_2S
 120     UNSPEC_VSTRUCTDUMMY
 121     UNSPEC_SP_SET
 122     UNSPEC_SP_TEST
 123 ])
 124
 125 (define_c_enum "unspecv" [
 126     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 127     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 128     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 129     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 130     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 131   ]
 132 )
 133
 134 ;; If further include files are added the defintion of MD_INCLUDES
 135 ;; must be updated.
 136
 137 (include "constraints.md")
 138 (include "predicates.md")
 139 (include "iterators.md")
 140
 141 ;; -------------------------------------------------------------------
 142 ;; Instruction types and attributes
 143 ;; -------------------------------------------------------------------
 144
 145 ; The "type" attribute is is included here from AArch32 backend to be able
 146 ; to share pipeline descriptions.
 147 (include "../arm/types.md")
 148
 149 ;; It is important to set the fp or simd attributes to yes when a pattern
 150 ;; alternative uses the FP or SIMD register files, usually signified by use of
 151 ;; the 'w' constraint.  This will ensure that the alternative will be
 152 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 153 ;; architecture extensions.  If all the alternatives in a pattern use the
 154 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 155 ;; or TARGET_SIMD.
 156
 157 ;; Attribute that specifies whether or not the instruction touches fp
 158 ;; registers.  When this is set to yes for an alternative, that alternative
 159 ;; will be disabled when !TARGET_FLOAT.
 160 (define_attr "fp" "no,yes" (const_string "no"))
 161
 162 ;; Attribute that specifies whether or not the instruction touches simd
 163 ;; registers.  When this is set to yes for an alternative, that alternative
 164 ;; will be disabled when !TARGET_SIMD.
 165 (define_attr "simd" "no,yes" (const_string "no"))
 166
 167 (define_attr "length" ""
 168   (const_int 4))
 169
 170 ;; Attribute that controls whether an alternative is enabled or not.
 171 ;; Currently it is only used to disable alternatives which touch fp or simd
 172 ;; registers when -mgeneral-regs-only is specified.
 173 (define_attr "enabled" "no,yes"
 174   (cond [(ior
 175         (and (eq_attr "fp" "yes")
 176              (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 177         (and (eq_attr "simd" "yes")
 178              (eq (symbol_ref "TARGET_SIMD") (const_int 0))))
 179              (const_string "no")
 180         ] (const_string "yes")))
 181
 182 ;; -------------------------------------------------------------------
 183 ;; Pipeline descriptions and scheduling
 184 ;; -------------------------------------------------------------------
 185
 186 ;; Processor types.
 187 (include "aarch64-tune.md")
 188
 189 ;; Scheduling
 190 (include "../arm/cortex-a53.md")
 191 (include "../arm/cortex-a57.md")
 192 (include "thunderx.md")
 193 (include "../arm/xgene1.md")
 194
 195 ;; -------------------------------------------------------------------
 196 ;; Jumps and other miscellaneous insns
 197 ;; -------------------------------------------------------------------
 198
 199 (define_insn "indirect_jump"
 200   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 201   ""
 202   "br\\t%0"
 203   [(set_attr "type" "branch")]
 204 )
 205
 206 (define_insn "jump"
 207   [(set (pc) (label_ref (match_operand 0 "" "")))]
 208   ""
 209   "b\\t%l0"
 210   [(set_attr "type" "branch")]
 211 )
 212
 213 (define_expand "cbranch<mode>4"
 214   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 215                             [(match_operand:GPI 1 "register_operand" "")
 216                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 217                            (label_ref (match_operand 3 "" ""))
 218                            (pc)))]
 219   ""
 220   "
 221   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 222                                          operands[2]);
 223   operands[2] = const0_rtx;
 224   "
 225 )
 226
 227 (define_expand "cbranch<mode>4"
 228   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 229                             [(match_operand:GPF 1 "register_operand" "")
 230                              (match_operand:GPF 2 "aarch64_reg_or_zero" "")])
 231                            (label_ref (match_operand 3 "" ""))
 232                            (pc)))]
 233   ""
 234   "
 235   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 236                                          operands[2]);
 237   operands[2] = const0_rtx;
 238   "
 239 )
 240
 241 (define_expand "cbranchcc4"
 242   [(set (pc) (if_then_else
 243               (match_operator 0 "aarch64_comparison_operator"
 244                [(match_operand 1 "cc_register" "")
 245                 (match_operand 2 "const0_operand")])
 246               (label_ref (match_operand 3 "" ""))
 247               (pc)))]
 248   ""
 249   "")
 250
 251 (define_insn "ccmp_and<mode>"
 252   [(set (match_operand 1 "ccmp_cc_register" "")
 253         (compare
 254          (and:SI
 255           (match_operator 4 "aarch64_comparison_operator"
 256            [(match_operand 0 "ccmp_cc_register" "")
 257             (const_int 0)])
 258           (match_operator 5 "aarch64_comparison_operator"
 259            [(match_operand:GPI 2 "register_operand" "r,r,r")
 260             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")]))
 261          (const_int 0)))]
 262   "aarch64_ccmp_mode_to_code (GET_MODE (operands[1])) == GET_CODE (operands[5])"
 263   "@
 264    ccmp\\t%<w>2, %<w>3, %k5, %m4
 265    ccmp\\t%<w>2, %<w>3, %k5, %m4
 266    ccmn\\t%<w>2, #%n3, %k5, %m4"
 267   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 268 )
 269
 270 (define_insn "ccmp_ior<mode>"
 271   [(set (match_operand 1 "ccmp_cc_register" "")
 272         (compare
 273          (ior:SI
 274           (match_operator 4 "aarch64_comparison_operator"
 275            [(match_operand 0 "ccmp_cc_register" "")
 276             (const_int 0)])
 277           (match_operator 5 "aarch64_comparison_operator"
 278            [(match_operand:GPI 2 "register_operand" "r,r,r")
 279             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")]))
 280          (const_int 0)))]
 281   "aarch64_ccmp_mode_to_code (GET_MODE (operands[1])) == GET_CODE (operands[5])"
 282   "@
 283    ccmp\\t%<w>2, %<w>3, %K5, %M4
 284    ccmp\\t%<w>2, %<w>3, %K5, %M4
 285    ccmn\\t%<w>2, #%n3, %K5, %M4"
 286   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 287 )
 288
 289 (define_expand "cmp<mode>"
 290   [(set (match_operand 0 "cc_register" "")
 291         (match_operator:CC 1 "aarch64_comparison_operator"
 292          [(match_operand:GPI 2 "register_operand" "")
 293           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
 294   ""
 295   {
 296     operands[1] = gen_rtx_fmt_ee (COMPARE,
 297                                   SELECT_CC_MODE (GET_CODE (operands[1]),
 298                                                   operands[2], operands[3]),
 299                                   operands[2], operands[3]);
 300   }
 301 )
 302
 303 (define_insn "*condjump"
 304   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 305                             [(match_operand 1 "cc_register" "") (const_int 0)])
 306                            (label_ref (match_operand 2 "" ""))
 307                            (pc)))]
 308   ""
 309   "b%m0\\t%l2"
 310   [(set_attr "type" "branch")]
 311 )
 312
 313 (define_expand "casesi"
 314   [(match_operand:SI 0 "register_operand" "")   ; Index
 315    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 316    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 317    (match_operand:DI 3 "" "")                   ; Table label
 318    (match_operand:DI 4 "" "")]                  ; Out of range label
 319   ""
 320   {
 321     if (operands[1] != const0_rtx)
 322       {
 323         rtx reg = gen_reg_rtx (SImode);
 324
 325         /* Canonical RTL says that if you have:
 326
 327            (minus (X) (CONST))
 328
 329            then this should be emitted as:
 330
 331            (plus (X) (-CONST))
 332
 333            The use of trunc_int_for_mode ensures that the resulting
 334            constant can be represented in SImode, this is important
 335            for the corner case where operand[1] is INT_MIN.  */
 336
 337         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 338
 339         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 340               (operands[1], SImode))
 341           operands[1] = force_reg (SImode, operands[1]);
 342         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 343         operands[0] = reg;
 344       }
 345
 346     if (!aarch64_plus_operand (operands[2], SImode))
 347       operands[2] = force_reg (SImode, operands[2]);
 348     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 349                                                  const0_rtx),
 350                                     operands[0], operands[2], operands[4]));
 351
 352     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (VOIDmode, operands[3]));
 353     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 354                                          operands[3]));
 355     DONE;
 356   }
 357 )
 358
 359 (define_insn "casesi_dispatch"
 360   [(parallel
 361     [(set (pc)
 362           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 363                            (match_operand:SI 1 "register_operand" "r")]
 364                         UNSPEC_CASESI)))
 365      (clobber (reg:CC CC_REGNUM))
 366      (clobber (match_scratch:DI 3 "=r"))
 367      (clobber (match_scratch:DI 4 "=r"))
 368      (use (label_ref (match_operand 2 "" "")))])]
 369   ""
 370   "*
 371   return aarch64_output_casesi (operands);
 372   "
 373   [(set_attr "length" "16")
 374    (set_attr "type" "branch")]
 375 )
 376
 377 (define_insn "nop"
 378   [(unspec[(const_int 0)] UNSPEC_NOP)]
 379   ""
 380   "nop"
 381   [(set_attr "type" "no_insn")]
 382 )
 383
 384 (define_insn "prefetch"
 385   [(prefetch (match_operand:DI 0 "address_operand" "r")
 386             (match_operand:QI 1 "const_int_operand" "")
 387             (match_operand:QI 2 "const_int_operand" ""))]
 388   ""
 389   {
 390     const char * pftype[2][4] =
 391     {
 392       {"prfm\\tPLDL1STRM, %a0",
 393        "prfm\\tPLDL3KEEP, %a0",
 394        "prfm\\tPLDL2KEEP, %a0",
 395        "prfm\\tPLDL1KEEP, %a0"},
 396       {"prfm\\tPSTL1STRM, %a0",
 397        "prfm\\tPSTL3KEEP, %a0",
 398        "prfm\\tPSTL2KEEP, %a0",
 399        "prfm\\tPSTL1KEEP, %a0"},
 400     };
 401
 402     int locality = INTVAL (operands[2]);
 403
 404     gcc_assert (IN_RANGE (locality, 0, 3));
 405
 406     return pftype[INTVAL(operands[1])][locality];
 407   }
 408   [(set_attr "type" "load1")]
 409 )
 410
 411 (define_insn "trap"
 412   [(trap_if (const_int 1) (const_int 8))]
 413   ""
 414   "brk #1000"
 415   [(set_attr "type" "trap")])
 416
 417 (define_expand "prologue"
 418   [(clobber (const_int 0))]
 419   ""
 420   "
 421   aarch64_expand_prologue ();
 422   DONE;
 423   "
 424 )
 425
 426 (define_expand "epilogue"
 427   [(clobber (const_int 0))]
 428   ""
 429   "
 430   aarch64_expand_epilogue (false);
 431   DONE;
 432   "
 433 )
 434
 435 (define_expand "sibcall_epilogue"
 436   [(clobber (const_int 0))]
 437   ""
 438   "
 439   aarch64_expand_epilogue (true);
 440   DONE;
 441   "
 442 )
 443
 444 (define_insn "*do_return"
 445   [(return)]
 446   ""
 447   "ret"
 448   [(set_attr "type" "branch")]
 449 )
 450
 451 (define_expand "return"
 452   [(simple_return)]
 453   "aarch64_use_return_insn_p ()"
 454   ""
 455 )
 456
 457 (define_insn "simple_return"
 458   [(simple_return)]
 459   ""
 460   "ret"
 461   [(set_attr "type" "branch")]
 462 )
 463
 464 (define_insn "eh_return"
 465   [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")]
 466     UNSPECV_EH_RETURN)]
 467   ""
 468   "#"
 469   [(set_attr "type" "branch")]
 470
 471 )
 472
 473 (define_split
 474   [(unspec_volatile [(match_operand:DI 0 "register_operand" "")]
 475     UNSPECV_EH_RETURN)]
 476   "reload_completed"
 477   [(set (match_dup 1) (match_dup 0))]
 478   {
 479     operands[1] = aarch64_final_eh_return_addr ();
 480   }
 481 )
 482
 483 (define_insn "*cb<optab><mode>1"
 484   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 485                                 (const_int 0))
 486                            (label_ref (match_operand 1 "" ""))
 487                            (pc)))]
 488   ""
 489   "<cbz>\\t%<w>0, %l1"
 490   [(set_attr "type" "branch")]
 491
 492 )
 493
 494 (define_insn "*tb<optab><mode>1"
 495   [(set (pc) (if_then_else
 496               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 497                                     (const_int 1)
 498                                     (match_operand 1 "const_int_operand" "n"))
 499                    (const_int 0))
 500              (label_ref (match_operand 2 "" ""))
 501              (pc)))
 502    (clobber (reg:CC CC_REGNUM))]
 503   ""
 504   {
 505     if (get_attr_length (insn) == 8)
 506       {
 507         operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 508         return "tst\t%<w>0, %1\;<bcond>\t%l2";
 509       }
 510     else
 511       return "<tbz>\t%<w>0, %1, %l2";
 512   }
 513   [(set_attr "type" "branch")
 514    (set (attr "length")
 515         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 516                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 517                       (const_int 4)
 518                       (const_int 8)))]
 519 )
 520
 521 (define_insn "*cb<optab><mode>1"
 522   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 523                                  (const_int 0))
 524                            (label_ref (match_operand 1 "" ""))
 525                            (pc)))
 526    (clobber (reg:CC CC_REGNUM))]
 527   ""
 528   {
 529     if (get_attr_length (insn) == 8)
 530       {
 531         char buf[64];
 532         uint64_t val = ((uint64_t ) 1)
 533                         << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 534         sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 535         output_asm_insn (buf, operands);
 536         return "<bcond>\t%l1";
 537       }
 538     else
 539       return "<tbz>\t%<w>0, <sizem1>, %l1";
 540   }
 541   [(set_attr "type" "branch")
 542    (set (attr "length")
 543         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 544                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 545                       (const_int 4)
 546                       (const_int 8)))]
 547 )
 548
 549 ;; -------------------------------------------------------------------
 550 ;; Subroutine calls and sibcalls
 551 ;; -------------------------------------------------------------------
 552
 553 (define_expand "call_internal"
 554   [(parallel [(call (match_operand 0 "memory_operand" "")
 555                     (match_operand 1 "general_operand" ""))
 556               (use (match_operand 2 "" ""))
 557               (clobber (reg:DI LR_REGNUM))])])
 558
 559 (define_expand "call"
 560   [(parallel [(call (match_operand 0 "memory_operand" "")
 561                     (match_operand 1 "general_operand" ""))
 562               (use (match_operand 2 "" ""))
 563               (clobber (reg:DI LR_REGNUM))])]
 564   ""
 565   "
 566   {
 567     rtx callee, pat;
 568
 569     /* In an untyped call, we can get NULL for operand 2.  */
 570     if (operands[2] == NULL)
 571       operands[2] = const0_rtx;
 572
 573     /* Decide if we should generate indirect calls by loading the
 574        64-bit address of the callee into a register before performing
 575        the branch-and-link.  */
 576     callee = XEXP (operands[0], 0);
 577     if (GET_CODE (callee) == SYMBOL_REF
 578         ? aarch64_is_long_call_p (callee)
 579         : !REG_P (callee))
 580       XEXP (operands[0], 0) = force_reg (Pmode, callee);
 581
 582     pat = gen_call_internal (operands[0], operands[1], operands[2]);
 583     aarch64_emit_call_insn (pat);
 584     DONE;
 585   }"
 586 )
 587
 588 (define_insn "*call_reg"
 589   [(call (mem:DI (match_operand:DI 0 "register_operand" "r"))
 590          (match_operand 1 "" ""))
 591    (use (match_operand 2 "" ""))
 592    (clobber (reg:DI LR_REGNUM))]
 593   ""
 594   "blr\\t%0"
 595   [(set_attr "type" "call")]
 596 )
 597
 598 (define_insn "*call_symbol"
 599   [(call (mem:DI (match_operand:DI 0 "" ""))
 600          (match_operand 1 "" ""))
 601    (use (match_operand 2 "" ""))
 602    (clobber (reg:DI LR_REGNUM))]
 603   "GET_CODE (operands[0]) == SYMBOL_REF
 604    && !aarch64_is_long_call_p (operands[0])"
 605   "bl\\t%a0"
 606   [(set_attr "type" "call")]
 607 )
 608
 609 (define_expand "call_value_internal"
 610   [(parallel [(set (match_operand 0 "" "")
 611                    (call (match_operand 1 "memory_operand" "")
 612                          (match_operand 2 "general_operand" "")))
 613               (use (match_operand 3 "" ""))
 614               (clobber (reg:DI LR_REGNUM))])])
 615
 616 (define_expand "call_value"
 617   [(parallel [(set (match_operand 0 "" "")
 618                    (call (match_operand 1 "memory_operand" "")
 619                          (match_operand 2 "general_operand" "")))
 620               (use (match_operand 3 "" ""))
 621               (clobber (reg:DI LR_REGNUM))])]
 622   ""
 623   "
 624   {
 625     rtx callee, pat;
 626
 627     /* In an untyped call, we can get NULL for operand 3.  */
 628     if (operands[3] == NULL)
 629       operands[3] = const0_rtx;
 630
 631     /* Decide if we should generate indirect calls by loading the
 632        64-bit address of the callee into a register before performing
 633        the branch-and-link.  */
 634     callee = XEXP (operands[1], 0);
 635     if (GET_CODE (callee) == SYMBOL_REF
 636         ? aarch64_is_long_call_p (callee)
 637         : !REG_P (callee))
 638       XEXP (operands[1], 0) = force_reg (Pmode, callee);
 639
 640     pat = gen_call_value_internal (operands[0], operands[1], operands[2],
 641                                    operands[3]);
 642     aarch64_emit_call_insn (pat);
 643     DONE;
 644   }"
 645 )
 646
 647 (define_insn "*call_value_reg"
 648   [(set (match_operand 0 "" "")
 649         (call (mem:DI (match_operand:DI 1 "register_operand" "r"))
 650                       (match_operand 2 "" "")))
 651    (use (match_operand 3 "" ""))
 652    (clobber (reg:DI LR_REGNUM))]
 653   ""
 654   "blr\\t%1"
 655   [(set_attr "type" "call")]
 656
 657 )
 658
 659 (define_insn "*call_value_symbol"
 660   [(set (match_operand 0 "" "")
 661         (call (mem:DI (match_operand:DI 1 "" ""))
 662               (match_operand 2 "" "")))
 663    (use (match_operand 3 "" ""))
 664    (clobber (reg:DI LR_REGNUM))]
 665   "GET_CODE (operands[1]) == SYMBOL_REF
 666    && !aarch64_is_long_call_p (operands[1])"
 667   "bl\\t%a1"
 668   [(set_attr "type" "call")]
 669 )
 670
 671 (define_expand "sibcall_internal"
 672   [(parallel [(call (match_operand 0 "memory_operand" "")
 673                     (match_operand 1 "general_operand" ""))
 674               (return)
 675               (use (match_operand 2 "" ""))])])
 676
 677 (define_expand "sibcall"
 678   [(parallel [(call (match_operand 0 "memory_operand" "")
 679                     (match_operand 1 "general_operand" ""))
 680               (return)
 681               (use (match_operand 2 "" ""))])]
 682   ""
 683   {
 684     rtx pat;
 685
 686     if (!REG_P (XEXP (operands[0], 0))
 687        && (GET_CODE (XEXP (operands[0], 0)) != SYMBOL_REF))
 688      XEXP (operands[0], 0) = force_reg (Pmode, XEXP (operands[0], 0));
 689
 690     if (operands[2] == NULL_RTX)
 691       operands[2] = const0_rtx;
 692
 693     pat = gen_sibcall_internal (operands[0], operands[1], operands[2]);
 694     aarch64_emit_call_insn (pat);
 695     DONE;
 696   }
 697 )
 698
 699 (define_expand "sibcall_value_internal"
 700   [(parallel [(set (match_operand 0 "" "")
 701                    (call (match_operand 1 "memory_operand" "")
 702                          (match_operand 2 "general_operand" "")))
 703               (return)
 704               (use (match_operand 3 "" ""))])])
 705
 706 (define_expand "sibcall_value"
 707   [(parallel [(set (match_operand 0 "" "")
 708                    (call (match_operand 1 "memory_operand" "")
 709                          (match_operand 2 "general_operand" "")))
 710               (return)
 711               (use (match_operand 3 "" ""))])]
 712   ""
 713   {
 714     rtx pat;
 715
 716     if (!REG_P (XEXP (operands[1], 0))
 717        && (GET_CODE (XEXP (operands[1], 0)) != SYMBOL_REF))
 718      XEXP (operands[1], 0) = force_reg (Pmode, XEXP (operands[1], 0));
 719
 720     if (operands[3] == NULL_RTX)
 721       operands[3] = const0_rtx;
 722
 723     pat = gen_sibcall_value_internal (operands[0], operands[1], operands[2],
 724                                       operands[3]);
 725     aarch64_emit_call_insn (pat);
 726     DONE;
 727   }
 728 )
 729
 730 (define_insn "*sibcall_insn"
 731   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 732          (match_operand 1 "" ""))
 733    (return)
 734    (use (match_operand 2 "" ""))]
 735   "SIBLING_CALL_P (insn)"
 736   "@
 737    br\\t%0
 738    b\\t%a0"
 739   [(set_attr "type" "branch, branch")]
 740 )
 741
 742 (define_insn "*sibcall_value_insn"
 743   [(set (match_operand 0 "" "")
 744         (call (mem:DI
 745                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 746               (match_operand 2 "" "")))
 747    (return)
 748    (use (match_operand 3 "" ""))]
 749   "SIBLING_CALL_P (insn)"
 750   "@
 751    br\\t%1
 752    b\\t%a1"
 753   [(set_attr "type" "branch, branch")]
 754 )
 755
 756 ;; Call subroutine returning any type.
 757
 758 (define_expand "untyped_call"
 759   [(parallel [(call (match_operand 0 "")
 760                     (const_int 0))
 761               (match_operand 1 "")
 762               (match_operand 2 "")])]
 763   ""
 764 {
 765   int i;
 766
 767   emit_call_insn (GEN_CALL (operands[0], const0_rtx, NULL, const0_rtx));
 768
 769   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 770     {
 771       rtx set = XVECEXP (operands[2], 0, i);
 772       emit_move_insn (SET_DEST (set), SET_SRC (set));
 773     }
 774
 775   /* The optimizer does not know that the call sets the function value
 776      registers we stored in the result block.  We avoid problems by
 777      claiming that all hard registers are used and clobbered at this
 778      point.  */
 779   emit_insn (gen_blockage ());
 780   DONE;
 781 })
 782
 783 ;; -------------------------------------------------------------------
 784 ;; Moves
 785 ;; -------------------------------------------------------------------
 786
 787 (define_expand "mov<mode>"
 788   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 789         (match_operand:SHORT 1 "general_operand" ""))]
 790   ""
 791   "
 792     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 793       operands[1] = force_reg (<MODE>mode, operands[1]);
 794   "
 795 )
 796
 797 (define_insn "*mov<mode>_aarch64"
 798   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,   *w,r,*w, m, m, r,*w,*w")
 799         (match_operand:SHORT 1 "general_operand"      " r,M,D<hq>,m, m,rZ,*w,*w, r,*w"))]
 800   "(register_operand (operands[0], <MODE>mode)
 801     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 802 {
 803    switch (which_alternative)
 804      {
 805      case 0:
 806        return "mov\t%w0, %w1";
 807      case 1:
 808        return "mov\t%w0, %1";
 809      case 2:
 810        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 811                                                         <MODE>mode);
 812      case 3:
 813        return "ldr<size>\t%w0, %1";
 814      case 4:
 815        return "ldr\t%<size>0, %1";
 816      case 5:
 817        return "str<size>\t%w1, %0";
 818      case 6:
 819        return "str\t%<size>1, %0";
 820      case 7:
 821        return "umov\t%w0, %1.<v>[0]";
 822      case 8:
 823        return "dup\t%0.<Vallxd>, %w1";
 824      case 9:
 825        return "dup\t%<Vetype>0, %1.<v>[0]";
 826      default:
 827        gcc_unreachable ();
 828      }
 829 }
 830   [(set_attr "type" "mov_reg,mov_imm,neon_move,load1,load1,store1,store1,\
 831                      neon_to_gp<q>,neon_from_gp<q>,neon_dup")
 832    (set_attr "simd" "*,*,yes,*,*,*,*,yes,yes,yes")]
 833 )
 834
 835 (define_expand "mov<mode>"
 836   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
 837         (match_operand:GPI 1 "general_operand" ""))]
 838   ""
 839   "
 840     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 841       operands[1] = force_reg (<MODE>mode, operands[1]);
 842
 843     /* FIXME: RR we still need to fix up what we are doing with
 844        symbol_refs and other types of constants.  */
 845     if (CONSTANT_P (operands[1])
 846         && !CONST_INT_P (operands[1]))
 847      {
 848        aarch64_expand_mov_immediate (operands[0], operands[1]);
 849        DONE;
 850      }
 851   "
 852 )
 853
 854 (define_insn_and_split "*movsi_aarch64"
 855   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r  ,*w, r,*w")
 856         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,m, m,rZ,*w,S,Ush,rZ,*w,*w"))]
 857   "(register_operand (operands[0], SImode)
 858     || aarch64_reg_or_zero (operands[1], SImode))"
 859   "@
 860    mov\\t%w0, %w1
 861    mov\\t%w0, %w1
 862    mov\\t%w0, %w1
 863    mov\\t%w0, %1
 864    #
 865    ldr\\t%w0, %1
 866    ldr\\t%s0, %1
 867    str\\t%w1, %0
 868    str\\t%s1, %0
 869    adr\\t%x0, %a1
 870    adrp\\t%x0, %A1
 871    fmov\\t%s0, %w1
 872    fmov\\t%w0, %s1
 873    fmov\\t%s0, %s1"
 874    "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
 875     && GP_REGNUM_P (REGNO (operands[0]))"
 876    [(const_int 0)]
 877    "{
 878        aarch64_expand_mov_immediate (operands[0], operands[1]);
 879        DONE;
 880     }"
 881   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
 882                      adr,adr,f_mcr,f_mrc,fmov")
 883    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes")]
 884 )
 885
 886 (define_insn_and_split "*movdi_aarch64"
 887   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,*w,m,  m,r,r,  *w, r,*w,w")
 888         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,n,m, m,rZ,*w,S,Ush,rZ,*w,*w,Dd"))]
 889   "(register_operand (operands[0], DImode)
 890     || aarch64_reg_or_zero (operands[1], DImode))"
 891   "@
 892    mov\\t%x0, %x1
 893    mov\\t%0, %x1
 894    mov\\t%x0, %1
 895    mov\\t%x0, %1
 896    #
 897    ldr\\t%x0, %1
 898    ldr\\t%d0, %1
 899    str\\t%x1, %0
 900    str\\t%d1, %0
 901    adr\\t%x0, %a1
 902    adrp\\t%x0, %A1
 903    fmov\\t%d0, %x1
 904    fmov\\t%x0, %d1
 905    fmov\\t%d0, %d1
 906    movi\\t%d0, %1"
 907    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
 908     && GP_REGNUM_P (REGNO (operands[0]))"
 909    [(const_int 0)]
 910    "{
 911        aarch64_expand_mov_immediate (operands[0], operands[1]);
 912        DONE;
 913     }"
 914   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,load1,load1,store1,store1,\
 915                      adr,adr,f_mcr,f_mrc,fmov,neon_move")
 916    (set_attr "fp" "*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
 917    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")]
 918 )
 919
 920 (define_insn "insv_imm<mode>"
 921   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
 922                           (const_int 16)
 923                           (match_operand:GPI 1 "const_int_operand" "n"))
 924         (match_operand:GPI 2 "const_int_operand" "n"))]
 925   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
 926    && UINTVAL (operands[1]) % 16 == 0"
 927   "movk\\t%<w>0, %X2, lsl %1"
 928   [(set_attr "type" "mov_imm")]
 929 )
 930
 931 (define_expand "movti"
 932   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 933         (match_operand:TI 1 "general_operand" ""))]
 934   ""
 935   "
 936     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 937       operands[1] = force_reg (TImode, operands[1]);
 938   "
 939 )
 940
 941 (define_insn "*movti_aarch64"
 942   [(set (match_operand:TI 0
 943          "nonimmediate_operand"  "=r, *w,r ,*w,r  ,Ump,Ump,*w,m")
 944         (match_operand:TI 1
 945          "aarch64_movti_operand" " rn,r ,*w,*w,Ump,r  ,Z  , m,*w"))]
 946   "(register_operand (operands[0], TImode)
 947     || aarch64_reg_or_zero (operands[1], TImode))"
 948   "@
 949    #
 950    #
 951    #
 952    orr\\t%0.16b, %1.16b, %1.16b
 953    ldp\\t%0, %H0, %1
 954    stp\\t%1, %H1, %0
 955    stp\\txzr, xzr, %0
 956    ldr\\t%q0, %1
 957    str\\t%q1, %0"
 958   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
 959                              load2,store2,store2,f_loadd,f_stored")
 960    (set_attr "length" "8,8,8,4,4,4,4,4,4")
 961    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
 962    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
 963 )
 964
 965 ;; Split a TImode register-register or register-immediate move into
 966 ;; its component DImode pieces, taking care to handle overlapping
 967 ;; source and dest registers.
 968 (define_split
 969    [(set (match_operand:TI 0 "register_operand" "")
 970          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
 971   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
 972   [(const_int 0)]
 973 {
 974   aarch64_split_128bit_move (operands[0], operands[1]);
 975   DONE;
 976 })
 977
 978 (define_expand "mov<mode>"
 979   [(set (match_operand:GPF 0 "nonimmediate_operand" "")
 980         (match_operand:GPF 1 "general_operand" ""))]
 981   ""
 982   {
 983     if (!TARGET_FLOAT)
 984       {
 985         aarch64_err_no_fpadvsimd (<MODE>mode, "code");
 986         FAIL;
 987       }
 988
 989     if (GET_CODE (operands[0]) == MEM
 990         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
 991               && aarch64_float_const_zero_rtx_p (operands[1])))
 992       operands[1] = force_reg (<MODE>mode, operands[1]);
 993   }
 994 )
 995
 996 (define_insn "*movsf_aarch64"
 997   [(set (match_operand:SF 0 "nonimmediate_operand" "=w, ?r,w,w  ,w,m,r,m ,r")
 998         (match_operand:SF 1 "general_operand"      "?rY, w,w,Ufc,m,w,m,rY,r"))]
 999   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1000     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1001   "@
1002    fmov\\t%s0, %w1
1003    fmov\\t%w0, %s1
1004    fmov\\t%s0, %s1
1005    fmov\\t%s0, %1
1006    ldr\\t%s0, %1
1007    str\\t%s1, %0
1008    ldr\\t%w0, %1
1009    str\\t%w1, %0
1010    mov\\t%w0, %w1"
1011   [(set_attr "type" "f_mcr,f_mrc,fmov,fconsts,\
1012                      f_loads,f_stores,load1,store1,mov_reg")]
1013 )
1014
1015 (define_insn "*movdf_aarch64"
1016   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, ?r,w,w  ,w,m,r,m ,r")
1017         (match_operand:DF 1 "general_operand"      "?rY, w,w,Ufc,m,w,m,rY,r"))]
1018   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1019     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1020   "@
1021    fmov\\t%d0, %x1
1022    fmov\\t%x0, %d1
1023    fmov\\t%d0, %d1
1024    fmov\\t%d0, %1
1025    ldr\\t%d0, %1
1026    str\\t%d1, %0
1027    ldr\\t%x0, %1
1028    str\\t%x1, %0
1029    mov\\t%x0, %x1"
1030   [(set_attr "type" "f_mcr,f_mrc,fmov,fconstd,\
1031                      f_loadd,f_stored,load1,store1,mov_reg")]
1032 )
1033
1034 (define_expand "movtf"
1035   [(set (match_operand:TF 0 "nonimmediate_operand" "")
1036         (match_operand:TF 1 "general_operand" ""))]
1037   ""
1038   {
1039     if (!TARGET_FLOAT)
1040       {
1041         aarch64_err_no_fpadvsimd (TFmode, "code");
1042         FAIL;
1043       }
1044
1045     if (GET_CODE (operands[0]) == MEM
1046         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1047               && aarch64_float_const_zero_rtx_p (operands[1])))
1048       operands[1] = force_reg (TFmode, operands[1]);
1049   }
1050 )
1051
1052 (define_insn "*movtf_aarch64"
1053   [(set (match_operand:TF 0
1054          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r ,Ump,Ump")
1055         (match_operand:TF 1
1056          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,Ump,?r ,Y"))]
1057   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1058     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1059   "@
1060    orr\\t%0.16b, %1.16b, %1.16b
1061    #
1062    #
1063    #
1064    movi\\t%0.2d, #0
1065    fmov\\t%s0, wzr
1066    ldr\\t%q0, %1
1067    str\\t%q1, %0
1068    ldp\\t%0, %H0, %1
1069    stp\\t%1, %H1, %0
1070    stp\\txzr, xzr, %0"
1071   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,fconstd,\
1072                      f_loadd,f_stored,load2,store2,store2")
1073    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1074    (set_attr "fp" "*,*,yes,yes,*,yes,yes,yes,*,*,*")
1075    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1076 )
1077
1078 (define_split
1079    [(set (match_operand:TF 0 "register_operand" "")
1080          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1081   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1082   [(const_int 0)]
1083   {
1084     aarch64_split_128bit_move (operands[0], operands[1]);
1085     DONE;
1086   }
1087 )
1088
1089 ;; 0 is dst
1090 ;; 1 is src
1091 ;; 2 is size of move in bytes
1092 ;; 3 is alignment
1093
1094 (define_expand "movmemdi"
1095   [(match_operand:BLK 0 "memory_operand")
1096    (match_operand:BLK 1 "memory_operand")
1097    (match_operand:DI 2 "immediate_operand")
1098    (match_operand:DI 3 "immediate_operand")]
1099    "!STRICT_ALIGNMENT"
1100 {
1101   if (aarch64_expand_movmem (operands))
1102     DONE;
1103   FAIL;
1104 }
1105 )
1106
1107 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1108 ;; fairly lax checking on the second memory operation.
1109 (define_insn "load_pairsi"
1110   [(set (match_operand:SI 0 "register_operand" "=r,*w")
1111         (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1112    (set (match_operand:SI 2 "register_operand" "=r,*w")
1113         (match_operand:SI 3 "memory_operand" "m,m"))]
1114   "rtx_equal_p (XEXP (operands[3], 0),
1115                 plus_constant (Pmode,
1116                                XEXP (operands[1], 0),
1117                                GET_MODE_SIZE (SImode)))"
1118   "@
1119    ldp\\t%w0, %w2, %1
1120    ldp\\t%s0, %s2, %1"
1121   [(set_attr "type" "load2,neon_load1_2reg")
1122    (set_attr "fp" "*,yes")]
1123 )
1124
1125 (define_insn "load_pairdi"
1126   [(set (match_operand:DI 0 "register_operand" "=r,*w")
1127         (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1128    (set (match_operand:DI 2 "register_operand" "=r,*w")
1129         (match_operand:DI 3 "memory_operand" "m,m"))]
1130   "rtx_equal_p (XEXP (operands[3], 0),
1131                 plus_constant (Pmode,
1132                                XEXP (operands[1], 0),
1133                                GET_MODE_SIZE (DImode)))"
1134   "@
1135    ldp\\t%x0, %x2, %1
1136    ldp\\t%d0, %d2, %1"
1137   [(set_attr "type" "load2,neon_load1_2reg")
1138    (set_attr "fp" "*,yes")]
1139 )
1140
1141
1142 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1143 ;; fairly lax checking on the second memory operation.
1144 (define_insn "store_pairsi"
1145   [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1146         (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,*w"))
1147    (set (match_operand:SI 2 "memory_operand" "=m,m")
1148         (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1149   "rtx_equal_p (XEXP (operands[2], 0),
1150                 plus_constant (Pmode,
1151                                XEXP (operands[0], 0),
1152                                GET_MODE_SIZE (SImode)))"
1153   "@
1154    stp\\t%w1, %w3, %0
1155    stp\\t%s1, %s3, %0"
1156   [(set_attr "type" "store2,neon_store1_2reg")
1157    (set_attr "fp" "*,yes")]
1158 )
1159
1160 (define_insn "store_pairdi"
1161   [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1162         (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,*w"))
1163    (set (match_operand:DI 2 "memory_operand" "=m,m")
1164         (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1165   "rtx_equal_p (XEXP (operands[2], 0),
1166                 plus_constant (Pmode,
1167                                XEXP (operands[0], 0),
1168                                GET_MODE_SIZE (DImode)))"
1169   "@
1170    stp\\t%x1, %x3, %0
1171    stp\\t%d1, %d3, %0"
1172   [(set_attr "type" "store2,neon_store1_2reg")
1173    (set_attr "fp" "*,yes")]
1174 )
1175
1176 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1177 ;; fairly lax checking on the second memory operation.
1178 (define_insn "load_pairsf"
1179   [(set (match_operand:SF 0 "register_operand" "=w,*r")
1180         (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1181    (set (match_operand:SF 2 "register_operand" "=w,*r")
1182         (match_operand:SF 3 "memory_operand" "m,m"))]
1183   "rtx_equal_p (XEXP (operands[3], 0),
1184                 plus_constant (Pmode,
1185                                XEXP (operands[1], 0),
1186                                GET_MODE_SIZE (SFmode)))"
1187   "@
1188    ldp\\t%s0, %s2, %1
1189    ldp\\t%w0, %w2, %1"
1190   [(set_attr "type" "neon_load1_2reg,load2")
1191    (set_attr "fp" "yes,*")]
1192 )
1193
1194 (define_insn "load_pairdf"
1195   [(set (match_operand:DF 0 "register_operand" "=w,*r")
1196         (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1197    (set (match_operand:DF 2 "register_operand" "=w,*r")
1198         (match_operand:DF 3 "memory_operand" "m,m"))]
1199   "rtx_equal_p (XEXP (operands[3], 0),
1200                 plus_constant (Pmode,
1201                                XEXP (operands[1], 0),
1202                                GET_MODE_SIZE (DFmode)))"
1203   "@
1204    ldp\\t%d0, %d2, %1
1205    ldp\\t%x0, %x2, %1"
1206   [(set_attr "type" "neon_load1_2reg,load2")
1207    (set_attr "fp" "yes,*")]
1208 )
1209
1210 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1211 ;; fairly lax checking on the second memory operation.
1212 (define_insn "store_pairsf"
1213   [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1214         (match_operand:SF 1 "register_operand" "w,*r"))
1215    (set (match_operand:SF 2 "memory_operand" "=m,m")
1216         (match_operand:SF 3 "register_operand" "w,*r"))]
1217   "rtx_equal_p (XEXP (operands[2], 0),
1218                 plus_constant (Pmode,
1219                                XEXP (operands[0], 0),
1220                                GET_MODE_SIZE (SFmode)))"
1221   "@
1222    stp\\t%s1, %s3, %0
1223    stp\\t%w1, %w3, %0"
1224   [(set_attr "type" "neon_store1_2reg,store2")
1225    (set_attr "fp" "yes,*")]
1226 )
1227
1228 (define_insn "store_pairdf"
1229   [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1230         (match_operand:DF 1 "register_operand" "w,*r"))
1231    (set (match_operand:DF 2 "memory_operand" "=m,m")
1232         (match_operand:DF 3 "register_operand" "w,*r"))]
1233   "rtx_equal_p (XEXP (operands[2], 0),
1234                 plus_constant (Pmode,
1235                                XEXP (operands[0], 0),
1236                                GET_MODE_SIZE (DFmode)))"
1237   "@
1238    stp\\t%d1, %d3, %0
1239    stp\\t%x1, %x3, %0"
1240   [(set_attr "type" "neon_store1_2reg,store2")
1241    (set_attr "fp" "yes,*")]
1242 )
1243
1244 ;; Load pair with post-index writeback.  This is primarily used in function
1245 ;; epilogues.
1246 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1247   [(parallel
1248     [(set (match_operand:P 0 "register_operand" "=k")
1249           (plus:P (match_operand:P 1 "register_operand" "0")
1250                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1251      (set (match_operand:GPI 2 "register_operand" "=r")
1252           (mem:GPI (match_dup 1)))
1253      (set (match_operand:GPI 3 "register_operand" "=r")
1254           (mem:GPI (plus:P (match_dup 1)
1255                    (match_operand:P 5 "const_int_operand" "n"))))])]
1256   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1257   "ldp\\t%<w>2, %<w>3, [%1], %4"
1258   [(set_attr "type" "load2")]
1259 )
1260
1261 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1262   [(parallel
1263     [(set (match_operand:P 0 "register_operand" "=k")
1264           (plus:P (match_operand:P 1 "register_operand" "0")
1265                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1266      (set (match_operand:GPF 2 "register_operand" "=w")
1267           (mem:GPF (match_dup 1)))
1268      (set (match_operand:GPF 3 "register_operand" "=w")
1269           (mem:GPF (plus:P (match_dup 1)
1270                    (match_operand:P 5 "const_int_operand" "n"))))])]
1271   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1272   "ldp\\t%<w>2, %<w>3, [%1], %4"
1273   [(set_attr "type" "neon_load1_2reg")]
1274 )
1275
1276 ;; Store pair with pre-index writeback.  This is primarily used in function
1277 ;; prologues.
1278 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1279   [(parallel
1280     [(set (match_operand:P 0 "register_operand" "=&k")
1281           (plus:P (match_operand:P 1 "register_operand" "0")
1282                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1283      (set (mem:GPI (plus:P (match_dup 0)
1284                    (match_dup 4)))
1285           (match_operand:GPI 2 "register_operand" "r"))
1286      (set (mem:GPI (plus:P (match_dup 0)
1287                    (match_operand:P 5 "const_int_operand" "n")))
1288           (match_operand:GPI 3 "register_operand" "r"))])]
1289   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1290   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1291   [(set_attr "type" "store2")]
1292 )
1293
1294 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1295   [(parallel
1296     [(set (match_operand:P 0 "register_operand" "=&k")
1297           (plus:P (match_operand:P 1 "register_operand" "0")
1298                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1299      (set (mem:GPF (plus:P (match_dup 0)
1300                    (match_dup 4)))
1301           (match_operand:GPF 2 "register_operand" "w"))
1302      (set (mem:GPF (plus:P (match_dup 0)
1303                    (match_operand:P 5 "const_int_operand" "n")))
1304           (match_operand:GPF 3 "register_operand" "w"))])]
1305   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1306   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1307   [(set_attr "type" "neon_store1_2reg<q>")]
1308 )
1309
1310 ;; -------------------------------------------------------------------
1311 ;; Sign/Zero extension
1312 ;; -------------------------------------------------------------------
1313
1314 (define_expand "<optab>sidi2"
1315   [(set (match_operand:DI 0 "register_operand")
1316         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1317   ""
1318 )
1319
1320 (define_insn "*extendsidi2_aarch64"
1321   [(set (match_operand:DI 0 "register_operand" "=r,r")
1322         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1323   ""
1324   "@
1325    sxtw\t%0, %w1
1326    ldrsw\t%0, %1"
1327   [(set_attr "type" "extend,load1")]
1328 )
1329
1330 (define_insn "*load_pair_extendsidi2_aarch64"
1331   [(set (match_operand:DI 0 "register_operand" "=r")
1332         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1333    (set (match_operand:DI 2 "register_operand" "=r")
1334         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1335   "rtx_equal_p (XEXP (operands[3], 0),
1336                 plus_constant (Pmode,
1337                                XEXP (operands[1], 0),
1338                                GET_MODE_SIZE (SImode)))"
1339   "ldpsw\\t%0, %2, %1"
1340   [(set_attr "type" "load2")]
1341 )
1342
1343 (define_insn "*zero_extendsidi2_aarch64"
1344   [(set (match_operand:DI 0 "register_operand" "=r,r")
1345         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1346   ""
1347   "@
1348    uxtw\t%0, %w1
1349    ldr\t%w0, %1"
1350   [(set_attr "type" "extend,load1")]
1351 )
1352
1353 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1354   [(set (match_operand:DI 0 "register_operand" "=r")
1355         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1356    (set (match_operand:DI 2 "register_operand" "=r")
1357         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1358   "rtx_equal_p (XEXP (operands[3], 0),
1359                 plus_constant (Pmode,
1360                                XEXP (operands[1], 0),
1361                                GET_MODE_SIZE (SImode)))"
1362   "ldp\\t%w0, %w2, %1"
1363   [(set_attr "type" "load2")]
1364 )
1365
1366 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1367   [(set (match_operand:GPI 0 "register_operand")
1368         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1369   ""
1370 )
1371
1372 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1373   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1374         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1375   ""
1376   "@
1377    sxt<SHORT:size>\t%<GPI:w>0, %w1
1378    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1379   [(set_attr "type" "extend,load1")]
1380 )
1381
1382 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1383   [(set (match_operand:GPI 0 "register_operand" "=r,r,*w")
1384         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1385   ""
1386   "@
1387    uxt<SHORT:size>\t%<GPI:w>0, %w1
1388    ldr<SHORT:size>\t%w0, %1
1389    ldr\t%<SHORT:size>0, %1"
1390   [(set_attr "type" "extend,load1,load1")]
1391 )
1392
1393 (define_expand "<optab>qihi2"
1394   [(set (match_operand:HI 0 "register_operand")
1395         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1396   ""
1397 )
1398
1399 (define_insn "*<optab>qihi2_aarch64"
1400   [(set (match_operand:HI 0 "register_operand" "=r,r")
1401         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1402   ""
1403   "@
1404    <su>xtb\t%w0, %w1
1405    <ldrxt>b\t%w0, %1"
1406   [(set_attr "type" "extend,load1")]
1407 )
1408
1409 ;; -------------------------------------------------------------------
1410 ;; Simple arithmetic
1411 ;; -------------------------------------------------------------------
1412
1413 (define_expand "add<mode>3"
1414   [(set
1415     (match_operand:GPI 0 "register_operand" "")
1416     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1417               (match_operand:GPI 2 "aarch64_pluslong_operand" "")))]
1418   ""
1419   "
1420   if (! aarch64_plus_operand (operands[2], VOIDmode))
1421     {
1422       HOST_WIDE_INT imm = INTVAL (operands[2]);
1423
1424       if (aarch64_move_imm (imm, <MODE>mode) && can_create_pseudo_p ())
1425         {
1426           rtx tmp = gen_reg_rtx (<MODE>mode);
1427           emit_move_insn (tmp, operands[2]);
1428           operands[2] = tmp;
1429         }
1430       else
1431         {
1432           rtx subtarget = ((optimize && can_create_pseudo_p ())
1433                            ? gen_reg_rtx (<MODE>mode) : operands[0]);
1434
1435           if (imm < 0)
1436             imm = -(-imm & ~0xfff);
1437           else
1438             imm &= ~0xfff;
1439
1440           emit_insn (gen_add<mode>3 (subtarget, operands[1], GEN_INT (imm)));
1441           operands[1] = subtarget;
1442           operands[2] = GEN_INT (INTVAL (operands[2]) - imm);
1443         }
1444     }
1445   "
1446 )
1447
1448 (define_insn "*addsi3_aarch64"
1449   [(set
1450     (match_operand:SI 0 "register_operand" "=rk,rk,w,rk")
1451     (plus:SI
1452      (match_operand:SI 1 "register_operand" "%rk,rk,w,rk")
1453      (match_operand:SI 2 "aarch64_plus_operand" "I,r,w,J")))]
1454   ""
1455   "@
1456   add\\t%w0, %w1, %2
1457   add\\t%w0, %w1, %w2
1458   add\\t%0.2s, %1.2s, %2.2s
1459   sub\\t%w0, %w1, #%n2"
1460   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm")
1461    (set_attr "simd" "*,*,yes,*")]
1462 )
1463
1464 ;; zero_extend version of above
1465 (define_insn "*addsi3_aarch64_uxtw"
1466   [(set
1467     (match_operand:DI 0 "register_operand" "=rk,rk,rk")
1468     (zero_extend:DI
1469      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk")
1470               (match_operand:SI 2 "aarch64_plus_operand" "I,r,J"))))]
1471   ""
1472   "@
1473   add\\t%w0, %w1, %2
1474   add\\t%w0, %w1, %w2
1475   sub\\t%w0, %w1, #%n2"
1476   [(set_attr "type" "alu_imm,alu_sreg,alu_imm")]
1477 )
1478
1479 (define_insn "*adddi3_aarch64"
1480   [(set
1481     (match_operand:DI 0 "register_operand" "=rk,rk,rk,w")
1482     (plus:DI
1483      (match_operand:DI 1 "register_operand" "%rk,rk,rk,w")
1484      (match_operand:DI 2 "aarch64_plus_operand" "I,r,J,w")))]
1485   ""
1486   "@
1487   add\\t%x0, %x1, %2
1488   add\\t%x0, %x1, %x2
1489   sub\\t%x0, %x1, #%n2
1490   add\\t%d0, %d1, %d2"
1491   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,neon_add")
1492    (set_attr "simd" "*,*,*,yes")]
1493 )
1494
1495 (define_expand "addti3"
1496   [(set (match_operand:TI 0 "register_operand" "")
1497         (plus:TI (match_operand:TI 1 "register_operand" "")
1498                  (match_operand:TI 2 "register_operand" "")))]
1499   ""
1500 {
1501   rtx low = gen_reg_rtx (DImode);
1502   emit_insn (gen_adddi3_compare0 (low, gen_lowpart (DImode, operands[1]),
1503                                   gen_lowpart (DImode, operands[2])));
1504
1505   rtx high = gen_reg_rtx (DImode);
1506   emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1507                                  gen_highpart (DImode, operands[2])));
1508
1509   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1510   emit_move_insn (gen_highpart (DImode, operands[0]), high);
1511   DONE;
1512 })
1513
1514 (define_insn "add<mode>3_compare0"
1515   [(set (reg:CC_NZ CC_REGNUM)
1516         (compare:CC_NZ
1517          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1518                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1519          (const_int 0)))
1520    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1521         (plus:GPI (match_dup 1) (match_dup 2)))]
1522   ""
1523   "@
1524   adds\\t%<w>0, %<w>1, %<w>2
1525   adds\\t%<w>0, %<w>1, %<w>2
1526   subs\\t%<w>0, %<w>1, #%n2"
1527   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1528 )
1529
1530 ;; zero_extend version of above
1531 (define_insn "*addsi3_compare0_uxtw"
1532   [(set (reg:CC_NZ CC_REGNUM)
1533         (compare:CC_NZ
1534          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1535                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1536          (const_int 0)))
1537    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1538         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1539   ""
1540   "@
1541   adds\\t%w0, %w1, %w2
1542   adds\\t%w0, %w1, %w2
1543   subs\\t%w0, %w1, #%n2"
1544   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1545 )
1546
1547 (define_insn "*adds_shift_imm_<mode>"
1548   [(set (reg:CC_NZ CC_REGNUM)
1549         (compare:CC_NZ
1550          (plus:GPI (ASHIFT:GPI
1551                     (match_operand:GPI 1 "register_operand" "r")
1552                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1553                    (match_operand:GPI 3 "register_operand" "r"))
1554          (const_int 0)))
1555    (set (match_operand:GPI 0 "register_operand" "=r")
1556         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1557                   (match_dup 3)))]
1558   ""
1559   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1560   [(set_attr "type" "alus_shift_imm")]
1561 )
1562
1563 (define_insn "*subs_shift_imm_<mode>"
1564   [(set (reg:CC_NZ CC_REGNUM)
1565         (compare:CC_NZ
1566          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1567                     (ASHIFT:GPI
1568                      (match_operand:GPI 2 "register_operand" "r")
1569                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1570          (const_int 0)))
1571    (set (match_operand:GPI 0 "register_operand" "=r")
1572         (minus:GPI (match_dup 1)
1573                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1574   ""
1575   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1576   [(set_attr "type" "alus_shift_imm")]
1577 )
1578
1579 (define_insn "*adds_mul_imm_<mode>"
1580   [(set (reg:CC_NZ CC_REGNUM)
1581         (compare:CC_NZ
1582          (plus:GPI (mult:GPI
1583                     (match_operand:GPI 1 "register_operand" "r")
1584                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1585                    (match_operand:GPI 3 "register_operand" "r"))
1586          (const_int 0)))
1587    (set (match_operand:GPI 0 "register_operand" "=r")
1588         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1589                   (match_dup 3)))]
1590   ""
1591   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1592   [(set_attr "type" "alus_shift_imm")]
1593 )
1594
1595 (define_insn "*subs_mul_imm_<mode>"
1596   [(set (reg:CC_NZ CC_REGNUM)
1597         (compare:CC_NZ
1598          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1599                     (mult:GPI
1600                      (match_operand:GPI 2 "register_operand" "r")
1601                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1602          (const_int 0)))
1603    (set (match_operand:GPI 0 "register_operand" "=r")
1604         (minus:GPI (match_dup 1)
1605                    (mult:GPI (match_dup 2) (match_dup 3))))]
1606   ""
1607   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1608   [(set_attr "type" "alus_shift_imm")]
1609 )
1610
1611 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1612   [(set (reg:CC_NZ CC_REGNUM)
1613         (compare:CC_NZ
1614          (plus:GPI
1615           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1616           (match_operand:GPI 2 "register_operand" "r"))
1617         (const_int 0)))
1618    (set (match_operand:GPI 0 "register_operand" "=r")
1619         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1620   ""
1621   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1622   [(set_attr "type" "alus_ext")]
1623 )
1624
1625 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1626   [(set (reg:CC_NZ CC_REGNUM)
1627         (compare:CC_NZ
1628          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1629                     (ANY_EXTEND:GPI
1630                      (match_operand:ALLX 2 "register_operand" "r")))
1631         (const_int 0)))
1632    (set (match_operand:GPI 0 "register_operand" "=r")
1633         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1634   ""
1635   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1636   [(set_attr "type" "alus_ext")]
1637 )
1638
1639 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1640   [(set (reg:CC_NZ CC_REGNUM)
1641         (compare:CC_NZ
1642          (plus:GPI (ashift:GPI
1643                     (ANY_EXTEND:GPI
1644                      (match_operand:ALLX 1 "register_operand" "r"))
1645                     (match_operand 2 "aarch64_imm3" "Ui3"))
1646                    (match_operand:GPI 3 "register_operand" "r"))
1647          (const_int 0)))
1648    (set (match_operand:GPI 0 "register_operand" "=rk")
1649         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1650                               (match_dup 2))
1651                   (match_dup 3)))]
1652   ""
1653   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1654   [(set_attr "type" "alus_ext")]
1655 )
1656
1657 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1658   [(set (reg:CC_NZ CC_REGNUM)
1659         (compare:CC_NZ
1660          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1661                     (ashift:GPI
1662                      (ANY_EXTEND:GPI
1663                       (match_operand:ALLX 2 "register_operand" "r"))
1664                      (match_operand 3 "aarch64_imm3" "Ui3")))
1665          (const_int 0)))
1666    (set (match_operand:GPI 0 "register_operand" "=rk")
1667         (minus:GPI (match_dup 1)
1668                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
1669                                (match_dup 3))))]
1670   ""
1671   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
1672   [(set_attr "type" "alus_ext")]
1673 )
1674
1675 (define_insn "*adds_<optab><mode>_multp2"
1676   [(set (reg:CC_NZ CC_REGNUM)
1677         (compare:CC_NZ
1678          (plus:GPI (ANY_EXTRACT:GPI
1679                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1680                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1681                     (match_operand 3 "const_int_operand" "n")
1682                     (const_int 0))
1683                    (match_operand:GPI 4 "register_operand" "r"))
1684         (const_int 0)))
1685    (set (match_operand:GPI 0 "register_operand" "=r")
1686         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
1687                                    (match_dup 3)
1688                                    (const_int 0))
1689                   (match_dup 4)))]
1690   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1691   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1692   [(set_attr "type" "alus_ext")]
1693 )
1694
1695 (define_insn "*subs_<optab><mode>_multp2"
1696   [(set (reg:CC_NZ CC_REGNUM)
1697         (compare:CC_NZ
1698          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
1699                     (ANY_EXTRACT:GPI
1700                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1701                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1702                      (match_operand 3 "const_int_operand" "n")
1703                      (const_int 0)))
1704         (const_int 0)))
1705    (set (match_operand:GPI 0 "register_operand" "=r")
1706         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
1707                                   (mult:GPI (match_dup 1) (match_dup 2))
1708                                   (match_dup 3)
1709                                   (const_int 0))))]
1710   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1711   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1712   [(set_attr "type" "alus_ext")]
1713 )
1714
1715 (define_insn "*add<mode>3nr_compare0"
1716   [(set (reg:CC_NZ CC_REGNUM)
1717         (compare:CC_NZ
1718          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
1719                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
1720          (const_int 0)))]
1721   ""
1722   "@
1723   cmn\\t%<w>0, %<w>1
1724   cmn\\t%<w>0, %<w>1
1725   cmp\\t%<w>0, #%n1"
1726   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1727 )
1728
1729 (define_insn "*compare_neg<mode>"
1730   [(set (reg:CC_Z CC_REGNUM)
1731         (compare:CC_Z
1732          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
1733          (match_operand:GPI 1 "register_operand" "r")))]
1734   ""
1735   "cmn\\t%<w>1, %<w>0"
1736   [(set_attr "type" "alus_sreg")]
1737 )
1738
1739 (define_insn "*add_<shift>_<mode>"
1740   [(set (match_operand:GPI 0 "register_operand" "=r")
1741         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
1742                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1743                   (match_operand:GPI 3 "register_operand" "r")))]
1744   ""
1745   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1746   [(set_attr "type" "alu_shift_imm")]
1747 )
1748
1749 ;; zero_extend version of above
1750 (define_insn "*add_<shift>_si_uxtw"
1751   [(set (match_operand:DI 0 "register_operand" "=r")
1752         (zero_extend:DI
1753          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
1754                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
1755                   (match_operand:SI 3 "register_operand" "r"))))]
1756   ""
1757   "add\\t%w0, %w3, %w1, <shift> %2"
1758   [(set_attr "type" "alu_shift_imm")]
1759 )
1760
1761 (define_insn "*add_mul_imm_<mode>"
1762   [(set (match_operand:GPI 0 "register_operand" "=r")
1763         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1764                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1765                   (match_operand:GPI 3 "register_operand" "r")))]
1766   ""
1767   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1768   [(set_attr "type" "alu_shift_imm")]
1769 )
1770
1771 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
1772   [(set (match_operand:GPI 0 "register_operand" "=rk")
1773         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1774                   (match_operand:GPI 2 "register_operand" "r")))]
1775   ""
1776   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1777   [(set_attr "type" "alu_ext")]
1778 )
1779
1780 ;; zero_extend version of above
1781 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
1782   [(set (match_operand:DI 0 "register_operand" "=rk")
1783         (zero_extend:DI
1784          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
1785                   (match_operand:GPI 2 "register_operand" "r"))))]
1786   ""
1787   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
1788   [(set_attr "type" "alu_ext")]
1789 )
1790
1791 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
1792   [(set (match_operand:GPI 0 "register_operand" "=rk")
1793         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
1794                                (match_operand:ALLX 1 "register_operand" "r"))
1795                               (match_operand 2 "aarch64_imm3" "Ui3"))
1796                   (match_operand:GPI 3 "register_operand" "r")))]
1797   ""
1798   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1799   [(set_attr "type" "alu_ext")]
1800 )
1801
1802 ;; zero_extend version of above
1803 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
1804   [(set (match_operand:DI 0 "register_operand" "=rk")
1805         (zero_extend:DI
1806          (plus:SI (ashift:SI (ANY_EXTEND:SI
1807                               (match_operand:SHORT 1 "register_operand" "r"))
1808                              (match_operand 2 "aarch64_imm3" "Ui3"))
1809                   (match_operand:SI 3 "register_operand" "r"))))]
1810   ""
1811   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
1812   [(set_attr "type" "alu_ext")]
1813 )
1814
1815 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
1816   [(set (match_operand:GPI 0 "register_operand" "=rk")
1817         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
1818                              (match_operand:ALLX 1 "register_operand" "r"))
1819                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1820                   (match_operand:GPI 3 "register_operand" "r")))]
1821   ""
1822   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
1823   [(set_attr "type" "alu_ext")]
1824 )
1825
1826 ;; zero_extend version of above
1827 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
1828   [(set (match_operand:DI 0 "register_operand" "=rk")
1829         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
1830                              (match_operand:SHORT 1 "register_operand" "r"))
1831                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1832                   (match_operand:SI 3 "register_operand" "r"))))]
1833   ""
1834   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
1835   [(set_attr "type" "alu_ext")]
1836 )
1837
1838 (define_insn "*add_<optab><mode>_multp2"
1839   [(set (match_operand:GPI 0 "register_operand" "=rk")
1840         (plus:GPI (ANY_EXTRACT:GPI
1841                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
1842                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1843                    (match_operand 3 "const_int_operand" "n")
1844                    (const_int 0))
1845                   (match_operand:GPI 4 "register_operand" "r")))]
1846   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
1847   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
1848   [(set_attr "type" "alu_ext")]
1849 )
1850
1851 ;; zero_extend version of above
1852 (define_insn "*add_<optab>si_multp2_uxtw"
1853   [(set (match_operand:DI 0 "register_operand" "=rk")
1854         (zero_extend:DI
1855          (plus:SI (ANY_EXTRACT:SI
1856                    (mult:SI (match_operand:SI 1 "register_operand" "r")
1857                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
1858                    (match_operand 3 "const_int_operand" "n")
1859                    (const_int 0))
1860                   (match_operand:SI 4 "register_operand" "r"))))]
1861   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
1862   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
1863   [(set_attr "type" "alu_ext")]
1864 )
1865
1866 (define_insn "add<mode>3_carryin"
1867   [(set
1868     (match_operand:GPI 0 "register_operand" "=r")
1869     (plus:GPI (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
1870               (plus:GPI
1871                 (match_operand:GPI 1 "register_operand" "r")
1872                 (match_operand:GPI 2 "register_operand" "r"))))]
1873    ""
1874    "adc\\t%<w>0, %<w>1, %<w>2"
1875   [(set_attr "type" "adc_reg")]
1876 )
1877
1878 ;; zero_extend version of above
1879 (define_insn "*addsi3_carryin_uxtw"
1880   [(set
1881     (match_operand:DI 0 "register_operand" "=r")
1882     (zero_extend:DI
1883      (plus:SI (geu:SI (reg:CC CC_REGNUM) (const_int 0))
1884               (plus:SI
1885                (match_operand:SI 1 "register_operand" "r")
1886                (match_operand:SI 2 "register_operand" "r")))))]
1887    ""
1888    "adc\\t%w0, %w1, %w2"
1889   [(set_attr "type" "adc_reg")]
1890 )
1891
1892 (define_insn "*add<mode>3_carryin_alt1"
1893   [(set
1894     (match_operand:GPI 0 "register_operand" "=r")
1895     (plus:GPI (plus:GPI
1896                 (match_operand:GPI 1 "register_operand" "r")
1897                 (match_operand:GPI 2 "register_operand" "r"))
1898               (geu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
1899    ""
1900    "adc\\t%<w>0, %<w>1, %<w>2"
1901   [(set_attr "type" "adc_reg")]
1902 )
1903
1904 ;; zero_extend version of above
1905 (define_insn "*addsi3_carryin_alt1_uxtw"
1906   [(set
1907     (match_operand:DI 0 "register_operand" "=r")
1908     (zero_extend:DI
1909      (plus:SI (plus:SI
1910                (match_operand:SI 1 "register_operand" "r")
1911                (match_operand:SI 2 "register_operand" "r"))
1912               (geu:SI (reg:CC CC_REGNUM) (const_int 0)))))]
1913    ""
1914    "adc\\t%w0, %w1, %w2"
1915   [(set_attr "type" "adc_reg")]
1916 )
1917
1918 (define_insn "*add<mode>3_carryin_alt2"
1919   [(set
1920     (match_operand:GPI 0 "register_operand" "=r")
1921     (plus:GPI (plus:GPI
1922                 (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
1923                 (match_operand:GPI 1 "register_operand" "r"))
1924               (match_operand:GPI 2 "register_operand" "r")))]
1925    ""
1926    "adc\\t%<w>0, %<w>1, %<w>2"
1927   [(set_attr "type" "adc_reg")]
1928 )
1929
1930 ;; zero_extend version of above
1931 (define_insn "*addsi3_carryin_alt2_uxtw"
1932   [(set
1933     (match_operand:DI 0 "register_operand" "=r")
1934     (zero_extend:DI
1935      (plus:SI (plus:SI
1936                (geu:SI (reg:CC CC_REGNUM) (const_int 0))
1937                (match_operand:SI 1 "register_operand" "r"))
1938               (match_operand:SI 2 "register_operand" "r"))))]
1939    ""
1940    "adc\\t%w0, %w1, %w2"
1941   [(set_attr "type" "adc_reg")]
1942 )
1943
1944 (define_insn "*add<mode>3_carryin_alt3"
1945   [(set
1946     (match_operand:GPI 0 "register_operand" "=r")
1947     (plus:GPI (plus:GPI
1948                 (geu:GPI (reg:CC CC_REGNUM) (const_int 0))
1949                 (match_operand:GPI 2 "register_operand" "r"))
1950               (match_operand:GPI 1 "register_operand" "r")))]
1951    ""
1952    "adc\\t%<w>0, %<w>1, %<w>2"
1953   [(set_attr "type" "adc_reg")]
1954 )
1955
1956 ;; zero_extend version of above
1957 (define_insn "*addsi3_carryin_alt3_uxtw"
1958   [(set
1959     (match_operand:DI 0 "register_operand" "=r")
1960     (zero_extend:DI
1961      (plus:SI (plus:SI
1962                (geu:SI (reg:CC CC_REGNUM) (const_int 0))
1963                (match_operand:SI 2 "register_operand" "r"))
1964               (match_operand:SI 1 "register_operand" "r"))))]
1965    ""
1966    "adc\\t%w0, %w1, %w2"
1967   [(set_attr "type" "adc_reg")]
1968 )
1969
1970 (define_insn "*add_uxt<mode>_shift2"
1971   [(set (match_operand:GPI 0 "register_operand" "=rk")
1972         (plus:GPI (and:GPI
1973                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
1974                                (match_operand 2 "aarch64_imm3" "Ui3"))
1975                    (match_operand 3 "const_int_operand" "n"))
1976                   (match_operand:GPI 4 "register_operand" "r")))]
1977   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
1978   "*
1979   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
1980                                            INTVAL (operands[3])));
1981   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
1982   [(set_attr "type" "alu_ext")]
1983 )
1984
1985 ;; zero_extend version of above
1986 (define_insn "*add_uxtsi_shift2_uxtw"
1987   [(set (match_operand:DI 0 "register_operand" "=rk")
1988         (zero_extend:DI
1989          (plus:SI (and:SI
1990                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
1991                               (match_operand 2 "aarch64_imm3" "Ui3"))
1992                    (match_operand 3 "const_int_operand" "n"))
1993                   (match_operand:SI 4 "register_operand" "r"))))]
1994   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
1995   "*
1996   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
1997                                            INTVAL (operands[3])));
1998   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
1999   [(set_attr "type" "alu_ext")]
2000 )
2001
2002 (define_insn "*add_uxt<mode>_multp2"
2003   [(set (match_operand:GPI 0 "register_operand" "=rk")
2004         (plus:GPI (and:GPI
2005                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2006                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2007                    (match_operand 3 "const_int_operand" "n"))
2008                   (match_operand:GPI 4 "register_operand" "r")))]
2009   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2010   "*
2011   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2012                                            INTVAL (operands[3])));
2013   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2014   [(set_attr "type" "alu_ext")]
2015 )
2016
2017 ;; zero_extend version of above
2018 (define_insn "*add_uxtsi_multp2_uxtw"
2019   [(set (match_operand:DI 0 "register_operand" "=rk")
2020         (zero_extend:DI
2021          (plus:SI (and:SI
2022                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2023                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2024                    (match_operand 3 "const_int_operand" "n"))
2025                   (match_operand:SI 4 "register_operand" "r"))))]
2026   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2027   "*
2028   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2029                                            INTVAL (operands[3])));
2030   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2031   [(set_attr "type" "alu_ext")]
2032 )
2033
2034 (define_insn "subsi3"
2035   [(set (match_operand:SI 0 "register_operand" "=rk")
2036         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2037                   (match_operand:SI 2 "register_operand" "r")))]
2038   ""
2039   "sub\\t%w0, %w1, %w2"
2040   [(set_attr "type" "alu_sreg")]
2041 )
2042
2043 ;; zero_extend version of above
2044 (define_insn "*subsi3_uxtw"
2045   [(set (match_operand:DI 0 "register_operand" "=rk")
2046         (zero_extend:DI
2047          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2048                    (match_operand:SI 2 "register_operand" "r"))))]
2049   ""
2050   "sub\\t%w0, %w1, %w2"
2051   [(set_attr "type" "alu_sreg")]
2052 )
2053
2054 (define_insn "subdi3"
2055   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2056         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2057                   (match_operand:DI 2 "register_operand" "r,w")))]
2058   ""
2059   "@
2060    sub\\t%x0, %x1, %x2
2061    sub\\t%d0, %d1, %d2"
2062   [(set_attr "type" "alu_sreg, neon_sub")
2063    (set_attr "simd" "*,yes")]
2064 )
2065
2066 (define_expand "subti3"
2067   [(set (match_operand:TI 0 "register_operand" "")
2068         (minus:TI (match_operand:TI 1 "register_operand" "")
2069                   (match_operand:TI 2 "register_operand" "")))]
2070   ""
2071 {
2072   rtx low = gen_reg_rtx (DImode);
2073   emit_insn (gen_subdi3_compare0 (low, gen_lowpart (DImode, operands[1]),
2074                                   gen_lowpart (DImode, operands[2])));
2075
2076   rtx high = gen_reg_rtx (DImode);
2077   emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2078                                  gen_highpart (DImode, operands[2])));
2079
2080   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2081   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2082   DONE;
2083 })
2084
2085 (define_insn "sub<mode>3_compare0"
2086   [(set (reg:CC_NZ CC_REGNUM)
2087         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2088                                   (match_operand:GPI 2 "register_operand" "r"))
2089                        (const_int 0)))
2090    (set (match_operand:GPI 0 "register_operand" "=r")
2091         (minus:GPI (match_dup 1) (match_dup 2)))]
2092   ""
2093   "subs\\t%<w>0, %<w>1, %<w>2"
2094   [(set_attr "type" "alus_sreg")]
2095 )
2096
2097 ;; zero_extend version of above
2098 (define_insn "*subsi3_compare0_uxtw"
2099   [(set (reg:CC_NZ CC_REGNUM)
2100         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2101                                  (match_operand:SI 2 "register_operand" "r"))
2102                        (const_int 0)))
2103    (set (match_operand:DI 0 "register_operand" "=r")
2104         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2105   ""
2106   "subs\\t%w0, %w1, %w2"
2107   [(set_attr "type" "alus_sreg")]
2108 )
2109
2110 (define_insn "*sub_<shift>_<mode>"
2111   [(set (match_operand:GPI 0 "register_operand" "=r")
2112         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2113                    (ASHIFT:GPI
2114                     (match_operand:GPI 1 "register_operand" "r")
2115                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2116   ""
2117   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2118   [(set_attr "type" "alu_shift_imm")]
2119 )
2120
2121 ;; zero_extend version of above
2122 (define_insn "*sub_<shift>_si_uxtw"
2123   [(set (match_operand:DI 0 "register_operand" "=r")
2124         (zero_extend:DI
2125          (minus:SI (match_operand:SI 3 "register_operand" "r")
2126                    (ASHIFT:SI
2127                     (match_operand:SI 1 "register_operand" "r")
2128                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2129   ""
2130   "sub\\t%w0, %w3, %w1, <shift> %2"
2131   [(set_attr "type" "alu_shift_imm")]
2132 )
2133
2134 (define_insn "*sub_mul_imm_<mode>"
2135   [(set (match_operand:GPI 0 "register_operand" "=r")
2136         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2137                    (mult:GPI
2138                     (match_operand:GPI 1 "register_operand" "r")
2139                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2140   ""
2141   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2142   [(set_attr "type" "alu_shift_imm")]
2143 )
2144
2145 ;; zero_extend version of above
2146 (define_insn "*sub_mul_imm_si_uxtw"
2147   [(set (match_operand:DI 0 "register_operand" "=r")
2148         (zero_extend:DI
2149          (minus:SI (match_operand:SI 3 "register_operand" "r")
2150                    (mult:SI
2151                     (match_operand:SI 1 "register_operand" "r")
2152                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2153   ""
2154   "sub\\t%w0, %w3, %w1, lsl %p2"
2155   [(set_attr "type" "alu_shift_imm")]
2156 )
2157
2158 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2159   [(set (match_operand:GPI 0 "register_operand" "=rk")
2160         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2161                    (ANY_EXTEND:GPI
2162                     (match_operand:ALLX 2 "register_operand" "r"))))]
2163   ""
2164   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2165   [(set_attr "type" "alu_ext")]
2166 )
2167
2168 ;; zero_extend version of above
2169 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2170   [(set (match_operand:DI 0 "register_operand" "=rk")
2171         (zero_extend:DI
2172          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2173                    (ANY_EXTEND:SI
2174                     (match_operand:SHORT 2 "register_operand" "r")))))]
2175   ""
2176   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2177   [(set_attr "type" "alu_ext")]
2178 )
2179
2180 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2181   [(set (match_operand:GPI 0 "register_operand" "=rk")
2182         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2183                    (ashift:GPI (ANY_EXTEND:GPI
2184                                 (match_operand:ALLX 2 "register_operand" "r"))
2185                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2186   ""
2187   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2188   [(set_attr "type" "alu_ext")]
2189 )
2190
2191 ;; zero_extend version of above
2192 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2193   [(set (match_operand:DI 0 "register_operand" "=rk")
2194         (zero_extend:DI
2195          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2196                    (ashift:SI (ANY_EXTEND:SI
2197                                (match_operand:SHORT 2 "register_operand" "r"))
2198                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2199   ""
2200   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2201   [(set_attr "type" "alu_ext")]
2202 )
2203
2204 (define_insn "*sub_<optab><mode>_multp2"
2205   [(set (match_operand:GPI 0 "register_operand" "=rk")
2206         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2207                    (ANY_EXTRACT:GPI
2208                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2209                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2210                     (match_operand 3 "const_int_operand" "n")
2211                     (const_int 0))))]
2212   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2213   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2214   [(set_attr "type" "alu_ext")]
2215 )
2216
2217 ;; zero_extend version of above
2218 (define_insn "*sub_<optab>si_multp2_uxtw"
2219   [(set (match_operand:DI 0 "register_operand" "=rk")
2220         (zero_extend:DI
2221          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2222                    (ANY_EXTRACT:SI
2223                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2224                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2225                     (match_operand 3 "const_int_operand" "n")
2226                     (const_int 0)))))]
2227   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2228   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2229   [(set_attr "type" "alu_ext")]
2230 )
2231
2232 (define_insn "sub<mode>3_carryin"
2233   [(set
2234     (match_operand:GPI 0 "register_operand" "=r")
2235     (minus:GPI (minus:GPI
2236                 (match_operand:GPI 1 "register_operand" "r")
2237                 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)))
2238                (match_operand:GPI 2 "register_operand" "r")))]
2239    ""
2240    "sbc\\t%<w>0, %<w>1, %<w>2"
2241   [(set_attr "type" "adc_reg")]
2242 )
2243
2244 ;; zero_extend version of the above
2245 (define_insn "*subsi3_carryin_uxtw"
2246   [(set
2247     (match_operand:DI 0 "register_operand" "=r")
2248     (zero_extend:DI
2249      (minus:SI (minus:SI
2250                 (match_operand:SI 1 "register_operand" "r")
2251                 (ltu:SI (reg:CC CC_REGNUM) (const_int 0)))
2252                (match_operand:SI 2 "register_operand" "r"))))]
2253    ""
2254    "sbc\\t%w0, %w1, %w2"
2255   [(set_attr "type" "adc_reg")]
2256 )
2257
2258 (define_insn "*sub_uxt<mode>_shift2"
2259   [(set (match_operand:GPI 0 "register_operand" "=rk")
2260         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2261                    (and:GPI
2262                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2263                                 (match_operand 2 "aarch64_imm3" "Ui3"))
2264                     (match_operand 3 "const_int_operand" "n"))))]
2265   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2266   "*
2267   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2268                                            INTVAL (operands[3])));
2269   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2270   [(set_attr "type" "alu_ext")]
2271 )
2272
2273 ;; zero_extend version of above
2274 (define_insn "*sub_uxtsi_shift2_uxtw"
2275   [(set (match_operand:DI 0 "register_operand" "=rk")
2276         (zero_extend:DI
2277          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2278                    (and:SI
2279                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
2280                                (match_operand 2 "aarch64_imm3" "Ui3"))
2281                     (match_operand 3 "const_int_operand" "n")))))]
2282   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2283   "*
2284   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2285                                            INTVAL (operands[3])));
2286   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2287   [(set_attr "type" "alu_ext")]
2288 )
2289
2290 (define_insn "*sub_uxt<mode>_multp2"
2291   [(set (match_operand:GPI 0 "register_operand" "=rk")
2292         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2293                    (and:GPI
2294                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2295                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2296                     (match_operand 3 "const_int_operand" "n"))))]
2297   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2298   "*
2299   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2300                                            INTVAL (operands[3])));
2301   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2302   [(set_attr "type" "alu_ext")]
2303 )
2304
2305 ;; zero_extend version of above
2306 (define_insn "*sub_uxtsi_multp2_uxtw"
2307   [(set (match_operand:DI 0 "register_operand" "=rk")
2308         (zero_extend:DI
2309          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2310                    (and:SI
2311                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2312                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2313                     (match_operand 3 "const_int_operand" "n")))))]
2314   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2315   "*
2316   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2317                                            INTVAL (operands[3])));
2318   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2319   [(set_attr "type" "alu_ext")]
2320 )
2321
2322 (define_expand "abs<mode>2"
2323   [(match_operand:GPI 0 "register_operand" "")
2324    (match_operand:GPI 1 "register_operand" "")]
2325   ""
2326   {
2327     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2328     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2329     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2330     DONE;
2331   }
2332 )
2333
2334 (define_insn "neg<mode>2"
2335   [(set (match_operand:GPI 0 "register_operand" "=r,w")
2336         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2337   ""
2338   "@
2339    neg\\t%<w>0, %<w>1
2340    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2341   [(set_attr "type" "alu_sreg, neon_neg<q>")
2342    (set_attr "simd" "*,yes")]
2343 )
2344
2345 ;; zero_extend version of above
2346 (define_insn "*negsi2_uxtw"
2347   [(set (match_operand:DI 0 "register_operand" "=r")
2348         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2349   ""
2350   "neg\\t%w0, %w1"
2351   [(set_attr "type" "alu_sreg")]
2352 )
2353
2354 (define_insn "*ngc<mode>"
2355   [(set (match_operand:GPI 0 "register_operand" "=r")
2356         (minus:GPI (neg:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)))
2357                    (match_operand:GPI 1 "register_operand" "r")))]
2358   ""
2359   "ngc\\t%<w>0, %<w>1"
2360   [(set_attr "type" "adc_reg")]
2361 )
2362
2363 (define_insn "*ngcsi_uxtw"
2364   [(set (match_operand:DI 0 "register_operand" "=r")
2365         (zero_extend:DI
2366          (minus:SI (neg:SI (ltu:SI (reg:CC CC_REGNUM) (const_int 0)))
2367                    (match_operand:SI 1 "register_operand" "r"))))]
2368   ""
2369   "ngc\\t%w0, %w1"
2370   [(set_attr "type" "adc_reg")]
2371 )
2372
2373 (define_insn "*neg<mode>2_compare0"
2374   [(set (reg:CC_NZ CC_REGNUM)
2375         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2376                        (const_int 0)))
2377    (set (match_operand:GPI 0 "register_operand" "=r")
2378         (neg:GPI (match_dup 1)))]
2379   ""
2380   "negs\\t%<w>0, %<w>1"
2381   [(set_attr "type" "alus_sreg")]
2382 )
2383
2384 ;; zero_extend version of above
2385 (define_insn "*negsi2_compare0_uxtw"
2386   [(set (reg:CC_NZ CC_REGNUM)
2387         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2388                        (const_int 0)))
2389    (set (match_operand:DI 0 "register_operand" "=r")
2390         (zero_extend:DI (neg:SI (match_dup 1))))]
2391   ""
2392   "negs\\t%w0, %w1"
2393   [(set_attr "type" "alus_sreg")]
2394 )
2395
2396 (define_insn "*neg_<shift><mode>3_compare0"
2397   [(set (reg:CC_NZ CC_REGNUM)
2398         (compare:CC_NZ
2399          (neg:GPI (ASHIFT:GPI
2400                    (match_operand:GPI 1 "register_operand" "r")
2401                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2402          (const_int 0)))
2403    (set (match_operand:GPI 0 "register_operand" "=r")
2404         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2405   ""
2406   "negs\\t%<w>0, %<w>1, <shift> %2"
2407   [(set_attr "type" "alus_shift_imm")]
2408 )
2409
2410 (define_insn "*neg_<shift>_<mode>2"
2411   [(set (match_operand:GPI 0 "register_operand" "=r")
2412         (neg:GPI (ASHIFT:GPI
2413                   (match_operand:GPI 1 "register_operand" "r")
2414                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2415   ""
2416   "neg\\t%<w>0, %<w>1, <shift> %2"
2417   [(set_attr "type" "alu_shift_imm")]
2418 )
2419
2420 ;; zero_extend version of above
2421 (define_insn "*neg_<shift>_si2_uxtw"
2422   [(set (match_operand:DI 0 "register_operand" "=r")
2423         (zero_extend:DI
2424          (neg:SI (ASHIFT:SI
2425                   (match_operand:SI 1 "register_operand" "r")
2426                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2427   ""
2428   "neg\\t%w0, %w1, <shift> %2"
2429   [(set_attr "type" "alu_shift_imm")]
2430 )
2431
2432 (define_insn "*neg_mul_imm_<mode>2"
2433   [(set (match_operand:GPI 0 "register_operand" "=r")
2434         (neg:GPI (mult:GPI
2435                   (match_operand:GPI 1 "register_operand" "r")
2436                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2437   ""
2438   "neg\\t%<w>0, %<w>1, lsl %p2"
2439   [(set_attr "type" "alu_shift_imm")]
2440 )
2441
2442 ;; zero_extend version of above
2443 (define_insn "*neg_mul_imm_si2_uxtw"
2444   [(set (match_operand:DI 0 "register_operand" "=r")
2445         (zero_extend:DI
2446          (neg:SI (mult:SI
2447                   (match_operand:SI 1 "register_operand" "r")
2448                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2449   ""
2450   "neg\\t%w0, %w1, lsl %p2"
2451   [(set_attr "type" "alu_shift_imm")]
2452 )
2453
2454 (define_insn "mul<mode>3"
2455   [(set (match_operand:GPI 0 "register_operand" "=r")
2456         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2457                   (match_operand:GPI 2 "register_operand" "r")))]
2458   ""
2459   "mul\\t%<w>0, %<w>1, %<w>2"
2460   [(set_attr "type" "mul")]
2461 )
2462
2463 ;; zero_extend version of above
2464 (define_insn "*mulsi3_uxtw"
2465   [(set (match_operand:DI 0 "register_operand" "=r")
2466         (zero_extend:DI
2467          (mult:SI (match_operand:SI 1 "register_operand" "r")
2468                   (match_operand:SI 2 "register_operand" "r"))))]
2469   ""
2470   "mul\\t%w0, %w1, %w2"
2471   [(set_attr "type" "mul")]
2472 )
2473
2474 (define_insn "madd<mode>"
2475   [(set (match_operand:GPI 0 "register_operand" "=r")
2476         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2477                             (match_operand:GPI 2 "register_operand" "r"))
2478                   (match_operand:GPI 3 "register_operand" "r")))]
2479   ""
2480   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2481   [(set_attr "type" "mla")]
2482 )
2483
2484 ;; zero_extend version of above
2485 (define_insn "*maddsi_uxtw"
2486   [(set (match_operand:DI 0 "register_operand" "=r")
2487         (zero_extend:DI
2488          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2489                            (match_operand:SI 2 "register_operand" "r"))
2490                   (match_operand:SI 3 "register_operand" "r"))))]
2491   ""
2492   "madd\\t%w0, %w1, %w2, %w3"
2493   [(set_attr "type" "mla")]
2494 )
2495
2496 (define_insn "*msub<mode>"
2497   [(set (match_operand:GPI 0 "register_operand" "=r")
2498         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2499                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2500                              (match_operand:GPI 2 "register_operand" "r"))))]
2501
2502   ""
2503   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2504   [(set_attr "type" "mla")]
2505 )
2506
2507 ;; zero_extend version of above
2508 (define_insn "*msubsi_uxtw"
2509   [(set (match_operand:DI 0 "register_operand" "=r")
2510         (zero_extend:DI
2511          (minus:SI (match_operand:SI 3 "register_operand" "r")
2512                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2513                             (match_operand:SI 2 "register_operand" "r")))))]
2514
2515   ""
2516   "msub\\t%w0, %w1, %w2, %w3"
2517   [(set_attr "type" "mla")]
2518 )
2519
2520 (define_insn "*mul<mode>_neg"
2521   [(set (match_operand:GPI 0 "register_operand" "=r")
2522         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2523                   (match_operand:GPI 2 "register_operand" "r")))]
2524
2525   ""
2526   "mneg\\t%<w>0, %<w>1, %<w>2"
2527   [(set_attr "type" "mul")]
2528 )
2529
2530 ;; zero_extend version of above
2531 (define_insn "*mulsi_neg_uxtw"
2532   [(set (match_operand:DI 0 "register_operand" "=r")
2533         (zero_extend:DI
2534          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2535                   (match_operand:SI 2 "register_operand" "r"))))]
2536
2537   ""
2538   "mneg\\t%w0, %w1, %w2"
2539   [(set_attr "type" "mul")]
2540 )
2541
2542 (define_insn "<su_optab>mulsidi3"
2543   [(set (match_operand:DI 0 "register_operand" "=r")
2544         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2545                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2546   ""
2547   "<su>mull\\t%0, %w1, %w2"
2548   [(set_attr "type" "<su>mull")]
2549 )
2550
2551 (define_insn "<su_optab>maddsidi4"
2552   [(set (match_operand:DI 0 "register_operand" "=r")
2553         (plus:DI (mult:DI
2554                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2555                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
2556                  (match_operand:DI 3 "register_operand" "r")))]
2557   ""
2558   "<su>maddl\\t%0, %w1, %w2, %3"
2559   [(set_attr "type" "<su>mlal")]
2560 )
2561
2562 (define_insn "<su_optab>msubsidi4"
2563   [(set (match_operand:DI 0 "register_operand" "=r")
2564         (minus:DI
2565          (match_operand:DI 3 "register_operand" "r")
2566          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2567                   (ANY_EXTEND:DI
2568                    (match_operand:SI 2 "register_operand" "r")))))]
2569   ""
2570   "<su>msubl\\t%0, %w1, %w2, %3"
2571   [(set_attr "type" "<su>mlal")]
2572 )
2573
2574 (define_insn "*<su_optab>mulsidi_neg"
2575   [(set (match_operand:DI 0 "register_operand" "=r")
2576         (mult:DI (neg:DI
2577                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
2578                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2579   ""
2580   "<su>mnegl\\t%0, %w1, %w2"
2581   [(set_attr "type" "<su>mull")]
2582 )
2583
2584 (define_expand "<su_optab>mulditi3"
2585   [(set (match_operand:TI 0 "register_operand")
2586         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
2587                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
2588   ""
2589 {
2590   rtx low = gen_reg_rtx (DImode);
2591   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
2592
2593   rtx high = gen_reg_rtx (DImode);
2594   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
2595
2596   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2597   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2598   DONE;
2599 })
2600
2601 ;; The default expansion of multi3 using umuldi3_highpart will perform
2602 ;; the additions in an order that fails to combine into two madd insns.
2603 (define_expand "multi3"
2604   [(set (match_operand:TI 0 "register_operand")
2605         (mult:TI (match_operand:TI 1 "register_operand")
2606                  (match_operand:TI 2 "register_operand")))]
2607   ""
2608 {
2609   rtx l0 = gen_reg_rtx (DImode);
2610   rtx l1 = gen_lowpart (DImode, operands[1]);
2611   rtx l2 = gen_lowpart (DImode, operands[2]);
2612   rtx h0 = gen_reg_rtx (DImode);
2613   rtx h1 = gen_highpart (DImode, operands[1]);
2614   rtx h2 = gen_highpart (DImode, operands[2]);
2615
2616   emit_insn (gen_muldi3 (l0, l1, l2));
2617   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
2618   emit_insn (gen_madddi (h0, h1, l2, h0));
2619   emit_insn (gen_madddi (h0, l1, h2, h0));
2620
2621   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
2622   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
2623   DONE;
2624 })
2625
2626 (define_insn "<su>muldi3_highpart"
2627   [(set (match_operand:DI 0 "register_operand" "=r")
2628         (truncate:DI
2629          (lshiftrt:TI
2630           (mult:TI
2631            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
2632            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
2633           (const_int 64))))]
2634   ""
2635   "<su>mulh\\t%0, %1, %2"
2636   [(set_attr "type" "<su>mull")]
2637 )
2638
2639 (define_insn "<su_optab>div<mode>3"
2640   [(set (match_operand:GPI 0 "register_operand" "=r")
2641         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
2642                      (match_operand:GPI 2 "register_operand" "r")))]
2643   ""
2644   "<su>div\\t%<w>0, %<w>1, %<w>2"
2645   [(set_attr "type" "<su>div")]
2646 )
2647
2648 ;; zero_extend version of above
2649 (define_insn "*<su_optab>divsi3_uxtw"
2650   [(set (match_operand:DI 0 "register_operand" "=r")
2651         (zero_extend:DI
2652          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
2653                      (match_operand:SI 2 "register_operand" "r"))))]
2654   ""
2655   "<su>div\\t%w0, %w1, %w2"
2656   [(set_attr "type" "<su>div")]
2657 )
2658
2659 ;; -------------------------------------------------------------------
2660 ;; Comparison insns
2661 ;; -------------------------------------------------------------------
2662
2663 (define_insn "*cmp<mode>"
2664   [(set (reg:CC CC_REGNUM)
2665         (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
2666                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
2667   ""
2668   "@
2669    cmp\\t%<w>0, %<w>1
2670    cmp\\t%<w>0, %<w>1
2671    cmn\\t%<w>0, #%n1"
2672   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2673 )
2674
2675 (define_insn "*cmp<mode>"
2676   [(set (reg:CCFP CC_REGNUM)
2677         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
2678                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2679    "TARGET_FLOAT"
2680    "@
2681     fcmp\\t%<s>0, #0.0
2682     fcmp\\t%<s>0, %<s>1"
2683   [(set_attr "type" "fcmp<s>")]
2684 )
2685
2686 (define_insn "*cmpe<mode>"
2687   [(set (reg:CCFPE CC_REGNUM)
2688         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
2689                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
2690    "TARGET_FLOAT"
2691    "@
2692     fcmpe\\t%<s>0, #0.0
2693     fcmpe\\t%<s>0, %<s>1"
2694   [(set_attr "type" "fcmp<s>")]
2695 )
2696
2697 (define_insn "*cmp_swp_<shift>_reg<mode>"
2698   [(set (reg:CC_SWP CC_REGNUM)
2699         (compare:CC_SWP (ASHIFT:GPI
2700                          (match_operand:GPI 0 "register_operand" "r")
2701                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
2702                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
2703   ""
2704   "cmp\\t%<w>2, %<w>0, <shift> %1"
2705   [(set_attr "type" "alus_shift_imm")]
2706 )
2707
2708 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
2709   [(set (reg:CC_SWP CC_REGNUM)
2710         (compare:CC_SWP (ANY_EXTEND:GPI
2711                          (match_operand:ALLX 0 "register_operand" "r"))
2712                         (match_operand:GPI 1 "register_operand" "r")))]
2713   ""
2714   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
2715   [(set_attr "type" "alus_ext")]
2716 )
2717
2718 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
2719   [(set (reg:CC_SWP CC_REGNUM)
2720         (compare:CC_SWP (ashift:GPI
2721                          (ANY_EXTEND:GPI
2722                           (match_operand:ALLX 0 "register_operand" "r"))
2723                          (match_operand 1 "aarch64_imm3" "Ui3"))
2724         (match_operand:GPI 2 "register_operand" "r")))]
2725   ""
2726   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
2727   [(set_attr "type" "alus_ext")]
2728 )
2729
2730 ;; -------------------------------------------------------------------
2731 ;; Store-flag and conditional select insns
2732 ;; -------------------------------------------------------------------
2733
2734 (define_expand "cstore<mode>4"
2735   [(set (match_operand:SI 0 "register_operand" "")
2736         (match_operator:SI 1 "aarch64_comparison_operator"
2737          [(match_operand:GPI 2 "register_operand" "")
2738           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
2739   ""
2740   "
2741   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2742                                       operands[3]);
2743   operands[3] = const0_rtx;
2744   "
2745 )
2746
2747 (define_expand "cstorecc4"
2748   [(set (match_operand:SI 0 "register_operand")
2749        (match_operator 1 "aarch64_comparison_operator"
2750         [(match_operand 2 "ccmp_cc_register")
2751          (match_operand 3 "const0_operand")]))]
2752   ""
2753 "{
2754   emit_insn (gen_rtx_SET (operands[0], operands[1]));
2755   DONE;
2756 }")
2757
2758
2759 (define_expand "cstore<mode>4"
2760   [(set (match_operand:SI 0 "register_operand" "")
2761         (match_operator:SI 1 "aarch64_comparison_operator"
2762          [(match_operand:GPF 2 "register_operand" "")
2763           (match_operand:GPF 3 "register_operand" "")]))]
2764   ""
2765   "
2766   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2767                                       operands[3]);
2768   operands[3] = const0_rtx;
2769   "
2770 )
2771
2772 (define_insn "*cstore<mode>_insn"
2773   [(set (match_operand:ALLI 0 "register_operand" "=r")
2774         (match_operator:ALLI 1 "aarch64_comparison_operator"
2775          [(match_operand 2 "cc_register" "") (const_int 0)]))]
2776   ""
2777   "cset\\t%<w>0, %m1"
2778   [(set_attr "type" "csel")]
2779 )
2780
2781 ;; zero_extend version of the above
2782 (define_insn "*cstoresi_insn_uxtw"
2783   [(set (match_operand:DI 0 "register_operand" "=r")
2784         (zero_extend:DI
2785          (match_operator:SI 1 "aarch64_comparison_operator"
2786           [(match_operand 2 "cc_register" "") (const_int 0)])))]
2787   ""
2788   "cset\\t%w0, %m1"
2789   [(set_attr "type" "csel")]
2790 )
2791
2792 (define_insn "cstore<mode>_neg"
2793   [(set (match_operand:ALLI 0 "register_operand" "=r")
2794         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator"
2795                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
2796   ""
2797   "csetm\\t%<w>0, %m1"
2798   [(set_attr "type" "csel")]
2799 )
2800
2801 ;; zero_extend version of the above
2802 (define_insn "*cstoresi_neg_uxtw"
2803   [(set (match_operand:DI 0 "register_operand" "=r")
2804         (zero_extend:DI
2805          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator"
2806                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
2807   ""
2808   "csetm\\t%w0, %m1"
2809   [(set_attr "type" "csel")]
2810 )
2811
2812 (define_expand "cmov<mode>6"
2813   [(set (match_operand:GPI 0 "register_operand" "")
2814         (if_then_else:GPI
2815          (match_operator 1 "aarch64_comparison_operator"
2816           [(match_operand:GPI 2 "register_operand" "")
2817            (match_operand:GPI 3 "aarch64_plus_operand" "")])
2818          (match_operand:GPI 4 "register_operand" "")
2819          (match_operand:GPI 5 "register_operand" "")))]
2820   ""
2821   "
2822   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2823                                       operands[3]);
2824   operands[3] = const0_rtx;
2825   "
2826 )
2827
2828 (define_expand "cmov<mode>6"
2829   [(set (match_operand:GPF 0 "register_operand" "")
2830         (if_then_else:GPF
2831          (match_operator 1 "aarch64_comparison_operator"
2832           [(match_operand:GPF 2 "register_operand" "")
2833            (match_operand:GPF 3 "register_operand" "")])
2834          (match_operand:GPF 4 "register_operand" "")
2835          (match_operand:GPF 5 "register_operand" "")))]
2836   ""
2837   "
2838   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
2839                                       operands[3]);
2840   operands[3] = const0_rtx;
2841   "
2842 )
2843
2844 (define_insn "*cmov<mode>_insn"
2845   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
2846         (if_then_else:ALLI
2847          (match_operator 1 "aarch64_comparison_operator"
2848           [(match_operand 2 "cc_register" "") (const_int 0)])
2849          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
2850          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
2851   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
2852      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
2853   ;; Final two alternatives should be unreachable, but included for completeness
2854   "@
2855    csel\\t%<w>0, %<w>3, %<w>4, %m1
2856    csinv\\t%<w>0, %<w>3, <w>zr, %m1
2857    csinv\\t%<w>0, %<w>4, <w>zr, %M1
2858    csinc\\t%<w>0, %<w>3, <w>zr, %m1
2859    csinc\\t%<w>0, %<w>4, <w>zr, %M1
2860    mov\\t%<w>0, -1
2861    mov\\t%<w>0, 1"
2862   [(set_attr "type" "csel")]
2863 )
2864
2865 ;; zero_extend version of above
2866 (define_insn "*cmovsi_insn_uxtw"
2867   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
2868         (zero_extend:DI
2869          (if_then_else:SI
2870           (match_operator 1 "aarch64_comparison_operator"
2871            [(match_operand 2 "cc_register" "") (const_int 0)])
2872           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
2873           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
2874   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
2875      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
2876   ;; Final two alternatives should be unreachable, but included for completeness
2877   "@
2878    csel\\t%w0, %w3, %w4, %m1
2879    csinv\\t%w0, %w3, wzr, %m1
2880    csinv\\t%w0, %w4, wzr, %M1
2881    csinc\\t%w0, %w3, wzr, %m1
2882    csinc\\t%w0, %w4, wzr, %M1
2883    mov\\t%w0, -1
2884    mov\\t%w0, 1"
2885   [(set_attr "type" "csel")]
2886 )
2887
2888 (define_insn "*cmov<mode>_insn"
2889   [(set (match_operand:GPF 0 "register_operand" "=w")
2890         (if_then_else:GPF
2891          (match_operator 1 "aarch64_comparison_operator"
2892           [(match_operand 2 "cc_register" "") (const_int 0)])
2893          (match_operand:GPF 3 "register_operand" "w")
2894          (match_operand:GPF 4 "register_operand" "w")))]
2895   "TARGET_FLOAT"
2896   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
2897   [(set_attr "type" "fcsel")]
2898 )
2899
2900 (define_expand "mov<mode>cc"
2901   [(set (match_operand:ALLI 0 "register_operand" "")
2902         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
2903                            (match_operand:ALLI 2 "register_operand" "")
2904                            (match_operand:ALLI 3 "register_operand" "")))]
2905   ""
2906   {
2907     enum rtx_code code = GET_CODE (operands[1]);
2908
2909     if (code == UNEQ || code == LTGT)
2910       FAIL;
2911
2912     if (!ccmp_cc_register (XEXP (operands[1], 0),
2913                            GET_MODE (XEXP (operands[1], 0))))
2914       {
2915         rtx ccreg;
2916         ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
2917                                          XEXP (operands[1], 1));
2918         operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
2919       }
2920   }
2921 )
2922
2923 (define_expand "mov<GPF:mode><GPI:mode>cc"
2924   [(set (match_operand:GPI 0 "register_operand" "")
2925         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
2926                           (match_operand:GPF 2 "register_operand" "")
2927                           (match_operand:GPF 3 "register_operand" "")))]
2928   ""
2929   {
2930     rtx ccreg;
2931     enum rtx_code code = GET_CODE (operands[1]);
2932
2933     if (code == UNEQ || code == LTGT)
2934       FAIL;
2935
2936     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
2937                                   XEXP (operands[1], 1));
2938     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
2939   }
2940 )
2941
2942 (define_expand "mov<mode>cc"
2943   [(set (match_operand:GPF 0 "register_operand" "")
2944         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
2945                           (match_operand:GPF 2 "register_operand" "")
2946                           (match_operand:GPF 3 "register_operand" "")))]
2947   ""
2948   {
2949     rtx ccreg;
2950     enum rtx_code code = GET_CODE (operands[1]);
2951
2952     if (code == UNEQ || code == LTGT)
2953       FAIL;
2954
2955     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
2956                                   XEXP (operands[1], 1));
2957     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
2958   }
2959 )
2960
2961
2962 ;; CRC32 instructions.
2963 (define_insn "aarch64_<crc_variant>"
2964   [(set (match_operand:SI 0 "register_operand" "=r")
2965         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
2966                     (match_operand:<crc_mode> 2 "register_operand" "r")]
2967          CRC))]
2968   "TARGET_CRC32"
2969   {
2970     if (GET_MODE_BITSIZE (GET_MODE (operands[2])) >= 64)
2971       return "<crc_variant>\\t%w0, %w1, %x2";
2972     else
2973       return "<crc_variant>\\t%w0, %w1, %w2";
2974   }
2975   [(set_attr "type" "crc")]
2976 )
2977
2978 (define_insn "*csinc2<mode>_insn"
2979   [(set (match_operand:GPI 0 "register_operand" "=r")
2980         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
2981                   (match_operand:GPI 1 "register_operand" "r")))]
2982   ""
2983   "csinc\\t%<w>0, %<w>1, %<w>1, %M2"
2984   [(set_attr "type" "csel")]
2985 )
2986
2987 (define_insn "csinc3<mode>_insn"
2988   [(set (match_operand:GPI 0 "register_operand" "=r")
2989         (if_then_else:GPI
2990           (match_operand 1 "aarch64_comparison_operation" "")
2991           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
2992                     (const_int 1))
2993           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
2994   ""
2995   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
2996   [(set_attr "type" "csel")]
2997 )
2998
2999 (define_insn "*csinv3<mode>_insn"
3000   [(set (match_operand:GPI 0 "register_operand" "=r")
3001         (if_then_else:GPI
3002           (match_operand 1 "aarch64_comparison_operation" "")
3003           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3004           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3005   ""
3006   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3007   [(set_attr "type" "csel")]
3008 )
3009
3010 (define_insn "csneg3<mode>_insn"
3011   [(set (match_operand:GPI 0 "register_operand" "=r")
3012         (if_then_else:GPI
3013           (match_operand 1 "aarch64_comparison_operation" "")
3014           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3015           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3016   ""
3017   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3018   [(set_attr "type" "csel")]
3019 )
3020
3021 ;; -------------------------------------------------------------------
3022 ;; Logical operations
3023 ;; -------------------------------------------------------------------
3024
3025 (define_insn "<optab><mode>3"
3026   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3027         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3028                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3029   ""
3030   "@
3031   <logical>\\t%<w>0, %<w>1, %<w>2
3032   <logical>\\t%<w>0, %<w>1, %<w>2
3033   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3034   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3035    (set_attr "simd" "*,*,yes")]
3036 )
3037
3038 ;; zero_extend version of above
3039 (define_insn "*<optab>si3_uxtw"
3040   [(set (match_operand:DI 0 "register_operand" "=r,rk")
3041         (zero_extend:DI
3042          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3043                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3044   ""
3045   "<logical>\\t%w0, %w1, %w2"
3046   [(set_attr "type" "logic_reg,logic_imm")]
3047 )
3048
3049 (define_insn "*and<mode>3_compare0"
3050   [(set (reg:CC_NZ CC_REGNUM)
3051         (compare:CC_NZ
3052          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3053                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3054          (const_int 0)))
3055    (set (match_operand:GPI 0 "register_operand" "=r,r")
3056         (and:GPI (match_dup 1) (match_dup 2)))]
3057   ""
3058   "ands\\t%<w>0, %<w>1, %<w>2"
3059   [(set_attr "type" "logics_reg,logics_imm")]
3060 )
3061
3062 ;; zero_extend version of above
3063 (define_insn "*andsi3_compare0_uxtw"
3064   [(set (reg:CC_NZ CC_REGNUM)
3065         (compare:CC_NZ
3066          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3067                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3068          (const_int 0)))
3069    (set (match_operand:DI 0 "register_operand" "=r,r")
3070         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3071   ""
3072   "ands\\t%w0, %w1, %w2"
3073   [(set_attr "type" "logics_reg,logics_imm")]
3074 )
3075
3076 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3077   [(set (reg:CC_NZ CC_REGNUM)
3078         (compare:CC_NZ
3079          (and:GPI (SHIFT:GPI
3080                    (match_operand:GPI 1 "register_operand" "r")
3081                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3082                   (match_operand:GPI 3 "register_operand" "r"))
3083          (const_int 0)))
3084    (set (match_operand:GPI 0 "register_operand" "=r")
3085         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3086   ""
3087   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3088   [(set_attr "type" "logics_shift_imm")]
3089 )
3090
3091 ;; zero_extend version of above
3092 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3093   [(set (reg:CC_NZ CC_REGNUM)
3094         (compare:CC_NZ
3095          (and:SI (SHIFT:SI
3096                   (match_operand:SI 1 "register_operand" "r")
3097                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3098                  (match_operand:SI 3 "register_operand" "r"))
3099          (const_int 0)))
3100    (set (match_operand:DI 0 "register_operand" "=r")
3101         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3102                                 (match_dup 3))))]
3103   ""
3104   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3105   [(set_attr "type" "logics_shift_imm")]
3106 )
3107
3108 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3109   [(set (match_operand:GPI 0 "register_operand" "=r")
3110         (LOGICAL:GPI (SHIFT:GPI
3111                       (match_operand:GPI 1 "register_operand" "r")
3112                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3113                      (match_operand:GPI 3 "register_operand" "r")))]
3114   ""
3115   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3116   [(set_attr "type" "logic_shift_imm")]
3117 )
3118
3119 (define_insn "*<optab>_rol<mode>3"
3120   [(set (match_operand:GPI 0 "register_operand" "=r")
3121         (LOGICAL:GPI (rotate:GPI
3122                       (match_operand:GPI 1 "register_operand" "r")
3123                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3124                      (match_operand:GPI 3 "register_operand" "r")))]
3125   ""
3126   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3127   [(set_attr "type" "logic_shift_imm")]
3128 )
3129
3130 ;; zero_extend versions of above
3131 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3132   [(set (match_operand:DI 0 "register_operand" "=r")
3133         (zero_extend:DI
3134          (LOGICAL:SI (SHIFT:SI
3135                       (match_operand:SI 1 "register_operand" "r")
3136                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3137                      (match_operand:SI 3 "register_operand" "r"))))]
3138   ""
3139   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3140   [(set_attr "type" "logic_shift_imm")]
3141 )
3142
3143 (define_insn "*<optab>_rolsi3_uxtw"
3144   [(set (match_operand:DI 0 "register_operand" "=r")
3145         (zero_extend:DI
3146          (LOGICAL:SI (rotate:SI
3147                       (match_operand:SI 1 "register_operand" "r")
3148                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3149                      (match_operand:SI 3 "register_operand" "r"))))]
3150   ""
3151   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3152   [(set_attr "type" "logic_shift_imm")]
3153 )
3154
3155 (define_insn "one_cmpl<mode>2"
3156   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3157         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3158   ""
3159   "@
3160   mvn\\t%<w>0, %<w>1
3161   mvn\\t%0.8b, %1.8b"
3162   [(set_attr "type" "logic_reg,neon_logic")
3163    (set_attr "simd" "*,yes")]
3164 )
3165
3166 (define_insn "*one_cmpl_<optab><mode>2"
3167   [(set (match_operand:GPI 0 "register_operand" "=r")
3168         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3169                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3170   ""
3171   "mvn\\t%<w>0, %<w>1, <shift> %2"
3172   [(set_attr "type" "logic_shift_imm")]
3173 )
3174
3175 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3176
3177 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3178   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3179         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3180                      (match_operand:GPI 2 "register_operand" "r,w")))]
3181   ""
3182   "@
3183   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3184   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3185   [(set_attr "type" "logic_reg,neon_logic")
3186    (set_attr "simd" "*,yes")]
3187 )
3188
3189 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3190   [(set (match_operand:DI 0 "register_operand" "=r")
3191         (zero_extend:DI
3192           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3193                        (match_operand:SI 2 "register_operand" "r"))))]
3194   ""
3195   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3196   [(set_attr "type" "logic_reg")]
3197 )
3198
3199 (define_insn "*xor_one_cmplsidi3_ze"
3200   [(set (match_operand:DI 0 "register_operand" "=r")
3201         (zero_extend:DI
3202           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3203                           (match_operand:SI 2 "register_operand" "r")))))]
3204   ""
3205   "eon\\t%w0, %w1, %w2"
3206   [(set_attr "type" "logic_reg")]
3207 )
3208
3209 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3210 ;; eon does not operate on SIMD registers so the vector variant must be split.
3211 (define_insn_and_split "*xor_one_cmpl<mode>3"
3212   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3213         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3214                           (match_operand:GPI 2 "register_operand" "r,w"))))]
3215   ""
3216   "@
3217   eon\\t%<w>0, %<w>1, %<w>2
3218   #"
3219   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3220   [(set (match_operand:GPI 0 "register_operand" "=w")
3221         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3222                  (match_operand:GPI 2 "register_operand" "w")))
3223    (set (match_dup 0) (not:GPI (match_dup 0)))]
3224   ""
3225   [(set_attr "type" "logic_reg,multiple")
3226    (set_attr "simd" "*,yes")]
3227 )
3228
3229 (define_insn "*and_one_cmpl<mode>3_compare0"
3230   [(set (reg:CC_NZ CC_REGNUM)
3231         (compare:CC_NZ
3232          (and:GPI (not:GPI
3233                    (match_operand:GPI 1 "register_operand" "r"))
3234                   (match_operand:GPI 2 "register_operand" "r"))
3235          (const_int 0)))
3236    (set (match_operand:GPI 0 "register_operand" "=r")
3237         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3238   ""
3239   "bics\\t%<w>0, %<w>2, %<w>1"
3240   [(set_attr "type" "logics_reg")]
3241 )
3242
3243 ;; zero_extend version of above
3244 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3245   [(set (reg:CC_NZ CC_REGNUM)
3246         (compare:CC_NZ
3247          (and:SI (not:SI
3248                   (match_operand:SI 1 "register_operand" "r"))
3249                  (match_operand:SI 2 "register_operand" "r"))
3250          (const_int 0)))
3251    (set (match_operand:DI 0 "register_operand" "=r")
3252         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3253   ""
3254   "bics\\t%w0, %w2, %w1"
3255   [(set_attr "type" "logics_reg")]
3256 )
3257
3258 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3259   [(set (reg:CC_NZ CC_REGNUM)
3260     (compare:CC_NZ
3261      (and:GPI (not:GPI
3262            (match_operand:GPI 0 "register_operand" "r"))
3263           (match_operand:GPI 1 "register_operand" "r"))
3264      (const_int 0)))]
3265   ""
3266   "bics\\t<w>zr, %<w>1, %<w>0"
3267   [(set_attr "type" "logics_reg")]
3268 )
3269
3270 (define_insn "*<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3271   [(set (match_operand:GPI 0 "register_operand" "=r")
3272         (LOGICAL:GPI (not:GPI
3273                       (SHIFT:GPI
3274                        (match_operand:GPI 1 "register_operand" "r")
3275                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3276                      (match_operand:GPI 3 "register_operand" "r")))]
3277   ""
3278   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3279   [(set_attr "type" "logic_shift_imm")]
3280 )
3281
3282 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3283   [(set (match_operand:GPI 0 "register_operand" "=r")
3284         (not:GPI (xor:GPI
3285                       (SHIFT:GPI
3286                        (match_operand:GPI 1 "register_operand" "r")
3287                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3288                      (match_operand:GPI 3 "register_operand" "r"))))]
3289   ""
3290   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3291   [(set_attr "type" "logic_shift_imm")]
3292 )
3293
3294 ;; Zero-extend version of the above.
3295 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3296   [(set (match_operand:DI 0 "register_operand" "=r")
3297         (zero_extend:DI
3298           (not:SI (xor:SI
3299                     (SHIFT:SI
3300                       (match_operand:SI 1 "register_operand" "r")
3301                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3302                     (match_operand:SI 3 "register_operand" "r")))))]
3303   ""
3304   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3305   [(set_attr "type" "logic_shift_imm")]
3306 )
3307
3308 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3309   [(set (reg:CC_NZ CC_REGNUM)
3310         (compare:CC_NZ
3311          (and:GPI (not:GPI
3312                    (SHIFT:GPI
3313                     (match_operand:GPI 1 "register_operand" "r")
3314                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3315                   (match_operand:GPI 3 "register_operand" "r"))
3316          (const_int 0)))
3317    (set (match_operand:GPI 0 "register_operand" "=r")
3318         (and:GPI (not:GPI
3319                   (SHIFT:GPI
3320                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
3321   ""
3322   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3323   [(set_attr "type" "logics_shift_imm")]
3324 )
3325
3326 ;; zero_extend version of above
3327 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3328   [(set (reg:CC_NZ CC_REGNUM)
3329         (compare:CC_NZ
3330          (and:SI (not:SI
3331                   (SHIFT:SI
3332                    (match_operand:SI 1 "register_operand" "r")
3333                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3334                  (match_operand:SI 3 "register_operand" "r"))
3335          (const_int 0)))
3336    (set (match_operand:DI 0 "register_operand" "=r")
3337         (zero_extend:DI (and:SI
3338                          (not:SI
3339                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3340   ""
3341   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3342   [(set_attr "type" "logics_shift_imm")]
3343 )
3344
3345 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3346   [(set (reg:CC_NZ CC_REGNUM)
3347     (compare:CC_NZ
3348      (and:GPI (not:GPI
3349            (SHIFT:GPI
3350             (match_operand:GPI 0 "register_operand" "r")
3351             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3352           (match_operand:GPI 2 "register_operand" "r"))
3353      (const_int 0)))]
3354   ""
3355   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3356   [(set_attr "type" "logics_shift_imm")]
3357 )
3358
3359 (define_insn "clz<mode>2"
3360   [(set (match_operand:GPI 0 "register_operand" "=r")
3361         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3362   ""
3363   "clz\\t%<w>0, %<w>1"
3364   [(set_attr "type" "clz")]
3365 )
3366
3367 (define_expand "ffs<mode>2"
3368   [(match_operand:GPI 0 "register_operand")
3369    (match_operand:GPI 1 "register_operand")]
3370   ""
3371   {
3372     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3373     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3374
3375     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3376     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3377     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3378     DONE;
3379   }
3380 )
3381
3382 (define_insn "clrsb<mode>2"
3383   [(set (match_operand:GPI 0 "register_operand" "=r")
3384         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
3385   ""
3386   "cls\\t%<w>0, %<w>1"
3387   [(set_attr "type" "clz")]
3388 )
3389
3390 (define_insn "rbit<mode>2"
3391   [(set (match_operand:GPI 0 "register_operand" "=r")
3392         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
3393   ""
3394   "rbit\\t%<w>0, %<w>1"
3395   [(set_attr "type" "rbit")]
3396 )
3397
3398 (define_expand "ctz<mode>2"
3399   [(match_operand:GPI 0 "register_operand")
3400    (match_operand:GPI 1 "register_operand")]
3401   ""
3402   {
3403     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3404     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3405     DONE;
3406   }
3407 )
3408
3409 (define_insn "*and<mode>3nr_compare0"
3410   [(set (reg:CC_NZ CC_REGNUM)
3411         (compare:CC_NZ
3412          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
3413                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
3414          (const_int 0)))]
3415   ""
3416   "tst\\t%<w>0, %<w>1"
3417   [(set_attr "type" "logics_reg")]
3418 )
3419
3420 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
3421   [(set (reg:CC_NZ CC_REGNUM)
3422         (compare:CC_NZ
3423          (and:GPI (SHIFT:GPI
3424                    (match_operand:GPI 0 "register_operand" "r")
3425                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3426                   (match_operand:GPI 2 "register_operand" "r"))
3427         (const_int 0)))]
3428   ""
3429   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
3430   [(set_attr "type" "logics_shift_imm")]
3431 )
3432
3433 ;; -------------------------------------------------------------------
3434 ;; Shifts
3435 ;; -------------------------------------------------------------------
3436
3437 (define_expand "<optab><mode>3"
3438   [(set (match_operand:GPI 0 "register_operand")
3439         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
3440                     (match_operand:QI 2 "nonmemory_operand")))]
3441   ""
3442   {
3443     if (CONST_INT_P (operands[2]))
3444       {
3445         operands[2] = GEN_INT (INTVAL (operands[2])
3446                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3447
3448         if (operands[2] == const0_rtx)
3449           {
3450             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3451             DONE;
3452           }
3453       }
3454   }
3455 )
3456
3457 (define_expand "ashl<mode>3"
3458   [(set (match_operand:SHORT 0 "register_operand")
3459         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
3460                       (match_operand:QI 2 "nonmemory_operand")))]
3461   ""
3462   {
3463     if (CONST_INT_P (operands[2]))
3464       {
3465         operands[2] = GEN_INT (INTVAL (operands[2])
3466                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3467
3468         if (operands[2] == const0_rtx)
3469           {
3470             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3471             DONE;
3472           }
3473       }
3474     else
3475       FAIL;
3476   }
3477 )
3478
3479 (define_expand "rotr<mode>3"
3480   [(set (match_operand:GPI 0 "register_operand")
3481         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3482                       (match_operand:QI 2 "nonmemory_operand")))]
3483   ""
3484   {
3485     if (CONST_INT_P (operands[2]))
3486       {
3487         operands[2] = GEN_INT (INTVAL (operands[2])
3488                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3489
3490         if (operands[2] == const0_rtx)
3491           {
3492             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3493             DONE;
3494           }
3495       }
3496   }
3497 )
3498
3499 (define_expand "rotl<mode>3"
3500   [(set (match_operand:GPI 0 "register_operand")
3501         (rotatert:GPI (match_operand:GPI 1 "register_operand")
3502                       (match_operand:QI 2 "nonmemory_operand")))]
3503   ""
3504   {
3505     /* (SZ - cnt) % SZ == -cnt % SZ */
3506     if (CONST_INT_P (operands[2]))
3507       {
3508         operands[2] = GEN_INT ((-INTVAL (operands[2]))
3509                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
3510         if (operands[2] == const0_rtx)
3511           {
3512             emit_insn (gen_mov<mode> (operands[0], operands[1]));
3513             DONE;
3514           }
3515       }
3516     else
3517       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
3518                                         NULL_RTX, 1);
3519   }
3520 )
3521
3522 ;; Logical left shift using SISD or Integer instruction
3523 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
3524   [(set (match_operand:GPI 0 "register_operand" "=w,w,r")
3525         (ashift:GPI
3526           (match_operand:GPI 1 "register_operand" "w,w,r")
3527           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,w,rUs<cmode>")))]
3528   ""
3529   "@
3530    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3531    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
3532    lsl\t%<w>0, %<w>1, %<w>2"
3533   [(set_attr "simd" "yes,yes,no")
3534    (set_attr "type" "neon_shift_imm<q>, neon_shift_reg<q>,shift_reg")]
3535 )
3536
3537 ;; Logical right shift using SISD or Integer instruction
3538 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
3539   [(set (match_operand:GPI 0 "register_operand" "=w,&w,r")
3540         (lshiftrt:GPI
3541           (match_operand:GPI 1 "register_operand" "w,w,r")
3542           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,w,rUs<cmode>")))]
3543   ""
3544   "@
3545    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3546    #
3547    lsr\t%<w>0, %<w>1, %<w>2"
3548   [(set_attr "simd" "yes,yes,no")
3549    (set_attr "type" "neon_shift_imm<q>,neon_shift_reg<q>,shift_reg")]
3550 )
3551
3552 (define_split
3553   [(set (match_operand:DI 0 "aarch64_simd_register")
3554         (lshiftrt:DI
3555            (match_operand:DI 1 "aarch64_simd_register")
3556            (match_operand:QI 2 "aarch64_simd_register")))]
3557   "TARGET_SIMD && reload_completed"
3558   [(set (match_dup 3)
3559         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3560    (set (match_dup 0)
3561         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
3562   {
3563     operands[3] = gen_lowpart (QImode, operands[0]);
3564   }
3565 )
3566
3567 (define_split
3568   [(set (match_operand:SI 0 "aarch64_simd_register")
3569         (lshiftrt:SI
3570            (match_operand:SI 1 "aarch64_simd_register")
3571            (match_operand:QI 2 "aarch64_simd_register")))]
3572   "TARGET_SIMD && reload_completed"
3573   [(set (match_dup 3)
3574         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3575    (set (match_dup 0)
3576         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
3577   {
3578     operands[3] = gen_lowpart (QImode, operands[0]);
3579   }
3580 )
3581
3582 ;; Arithmetic right shift using SISD or Integer instruction
3583 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
3584   [(set (match_operand:GPI 0 "register_operand" "=w,&w,&w,r")
3585         (ashiftrt:GPI
3586           (match_operand:GPI 1 "register_operand" "w,w,w,r")
3587           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" "Us<cmode>,w,0,rUs<cmode>")))]
3588   ""
3589   "@
3590    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
3591    #
3592    #
3593    asr\t%<w>0, %<w>1, %<w>2"
3594   [(set_attr "simd" "yes,yes,yes,no")
3595    (set_attr "type" "neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>,shift_reg")]
3596 )
3597
3598 (define_split
3599   [(set (match_operand:DI 0 "aarch64_simd_register")
3600         (ashiftrt:DI
3601            (match_operand:DI 1 "aarch64_simd_register")
3602            (match_operand:QI 2 "aarch64_simd_register")))]
3603   "TARGET_SIMD && reload_completed"
3604   [(set (match_dup 3)
3605         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3606    (set (match_dup 0)
3607         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
3608 {
3609   operands[3] = gen_lowpart (QImode, operands[0]);
3610 }
3611 )
3612
3613 (define_split
3614   [(set (match_operand:SI 0 "aarch64_simd_register")
3615         (ashiftrt:SI
3616            (match_operand:SI 1 "aarch64_simd_register")
3617            (match_operand:QI 2 "aarch64_simd_register")))]
3618   "TARGET_SIMD && reload_completed"
3619   [(set (match_dup 3)
3620         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
3621    (set (match_dup 0)
3622         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
3623 {
3624   operands[3] = gen_lowpart (QImode, operands[0]);
3625 }
3626 )
3627
3628 (define_insn "*aarch64_sisd_ushl"
3629   [(set (match_operand:DI 0 "register_operand" "=w")
3630         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
3631                     (match_operand:QI 2 "register_operand" "w")]
3632                    UNSPEC_SISD_USHL))]
3633   "TARGET_SIMD"
3634   "ushl\t%d0, %d1, %d2"
3635   [(set_attr "simd" "yes")
3636    (set_attr "type" "neon_shift_reg")]
3637 )
3638
3639 (define_insn "*aarch64_ushl_2s"
3640   [(set (match_operand:SI 0 "register_operand" "=w")
3641         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
3642                     (match_operand:QI 2 "register_operand" "w")]
3643                    UNSPEC_USHL_2S))]
3644   "TARGET_SIMD"
3645   "ushl\t%0.2s, %1.2s, %2.2s"
3646   [(set_attr "simd" "yes")
3647    (set_attr "type" "neon_shift_reg")]
3648 )
3649
3650 (define_insn "*aarch64_sisd_sshl"
3651   [(set (match_operand:DI 0 "register_operand" "=w")
3652         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
3653                     (match_operand:QI 2 "register_operand" "w")]
3654                    UNSPEC_SISD_SSHL))]
3655   "TARGET_SIMD"
3656   "sshl\t%d0, %d1, %d2"
3657   [(set_attr "simd" "yes")
3658    (set_attr "type" "neon_shift_reg")]
3659 )
3660
3661 (define_insn "*aarch64_sshl_2s"
3662   [(set (match_operand:SI 0 "register_operand" "=w")
3663         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
3664                     (match_operand:QI 2 "register_operand" "w")]
3665                    UNSPEC_SSHL_2S))]
3666   "TARGET_SIMD"
3667   "sshl\t%0.2s, %1.2s, %2.2s"
3668   [(set_attr "simd" "yes")
3669    (set_attr "type" "neon_shift_reg")]
3670 )
3671
3672 (define_insn "*aarch64_sisd_neg_qi"
3673   [(set (match_operand:QI 0 "register_operand" "=w")
3674         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
3675                    UNSPEC_SISD_NEG))]
3676   "TARGET_SIMD"
3677   "neg\t%d0, %d1"
3678   [(set_attr "simd" "yes")
3679    (set_attr "type" "neon_neg")]
3680 )
3681
3682 ;; Rotate right
3683 (define_insn "*ror<mode>3_insn"
3684   [(set (match_operand:GPI 0 "register_operand" "=r")
3685         (rotatert:GPI
3686           (match_operand:GPI 1 "register_operand" "r")
3687           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "rUs<cmode>")))]
3688   ""
3689   "ror\\t%<w>0, %<w>1, %<w>2"
3690   [(set_attr "type" "shift_reg")]
3691 )
3692
3693 ;; zero_extend version of above
3694 (define_insn "*<optab>si3_insn_uxtw"
3695   [(set (match_operand:DI 0 "register_operand" "=r")
3696         (zero_extend:DI (SHIFT:SI
3697          (match_operand:SI 1 "register_operand" "r")
3698          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "rUss"))))]
3699   ""
3700   "<shift>\\t%w0, %w1, %w2"
3701   [(set_attr "type" "shift_reg")]
3702 )
3703
3704 (define_insn "*<optab><mode>3_insn"
3705   [(set (match_operand:SHORT 0 "register_operand" "=r")
3706         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
3707                       (match_operand 2 "const_int_operand" "n")))]
3708   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
3709 {
3710   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
3711   return "<bfshift>\t%w0, %w1, %2, %3";
3712 }
3713   [(set_attr "type" "bfm")]
3714 )
3715
3716 (define_insn "*extr<mode>5_insn"
3717   [(set (match_operand:GPI 0 "register_operand" "=r")
3718         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3719                              (match_operand 3 "const_int_operand" "n"))
3720                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
3721                                (match_operand 4 "const_int_operand" "n"))))]
3722   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
3723    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
3724   "extr\\t%<w>0, %<w>1, %<w>2, %4"
3725   [(set_attr "type" "shift_imm")]
3726 )
3727
3728 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
3729 ;; so we have to match both orderings.
3730 (define_insn "*extr<mode>5_insn_alt"
3731   [(set (match_operand:GPI 0 "register_operand" "=r")
3732         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
3733                                 (match_operand 4 "const_int_operand" "n"))
3734                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3735                               (match_operand 3 "const_int_operand" "n"))))]
3736   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
3737    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
3738        == GET_MODE_BITSIZE (<MODE>mode))"
3739   "extr\\t%<w>0, %<w>1, %<w>2, %4"
3740   [(set_attr "type" "shift_imm")]
3741 )
3742
3743 ;; zero_extend version of the above
3744 (define_insn "*extrsi5_insn_uxtw"
3745   [(set (match_operand:DI 0 "register_operand" "=r")
3746         (zero_extend:DI
3747          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
3748                             (match_operand 3 "const_int_operand" "n"))
3749                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
3750                               (match_operand 4 "const_int_operand" "n")))))]
3751   "UINTVAL (operands[3]) < 32 &&
3752    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
3753   "extr\\t%w0, %w1, %w2, %4"
3754   [(set_attr "type" "shift_imm")]
3755 )
3756
3757 (define_insn "*extrsi5_insn_uxtw_alt"
3758   [(set (match_operand:DI 0 "register_operand" "=r")
3759         (zero_extend:DI
3760          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
3761                                (match_operand 4 "const_int_operand" "n"))
3762                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
3763                             (match_operand 3 "const_int_operand" "n")))))]
3764   "UINTVAL (operands[3]) < 32 &&
3765    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
3766   "extr\\t%w0, %w1, %w2, %4"
3767   [(set_attr "type" "shift_imm")]
3768 )
3769
3770 (define_insn "*ror<mode>3_insn"
3771   [(set (match_operand:GPI 0 "register_operand" "=r")
3772         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
3773                     (match_operand 2 "const_int_operand" "n")))]
3774   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
3775 {
3776   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
3777   return "ror\\t%<w>0, %<w>1, %3";
3778 }
3779   [(set_attr "type" "shift_imm")]
3780 )
3781
3782 ;; zero_extend version of the above
3783 (define_insn "*rorsi3_insn_uxtw"
3784   [(set (match_operand:DI 0 "register_operand" "=r")
3785         (zero_extend:DI
3786          (rotate:SI (match_operand:SI 1 "register_operand" "r")
3787                     (match_operand 2 "const_int_operand" "n"))))]
3788   "UINTVAL (operands[2]) < 32"
3789 {
3790   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
3791   return "ror\\t%w0, %w1, %3";
3792 }
3793   [(set_attr "type" "shift_imm")]
3794 )
3795
3796 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
3797   [(set (match_operand:GPI 0 "register_operand" "=r")
3798         (ANY_EXTEND:GPI
3799          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
3800                        (match_operand 2 "const_int_operand" "n"))))]
3801   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
3802 {
3803   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
3804   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3805 }
3806   [(set_attr "type" "bfm")]
3807 )
3808
3809 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
3810   [(set (match_operand:GPI 0 "register_operand" "=r")
3811         (zero_extend:GPI
3812          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
3813                          (match_operand 2 "const_int_operand" "n"))))]
3814   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
3815 {
3816   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
3817   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3818 }
3819   [(set_attr "type" "bfm")]
3820 )
3821
3822 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
3823   [(set (match_operand:GPI 0 "register_operand" "=r")
3824         (sign_extend:GPI
3825          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
3826                          (match_operand 2 "const_int_operand" "n"))))]
3827   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
3828 {
3829   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
3830   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3831 }
3832   [(set_attr "type" "bfm")]
3833 )
3834
3835 ;; -------------------------------------------------------------------
3836 ;; Bitfields
3837 ;; -------------------------------------------------------------------
3838
3839 (define_expand "<optab>"
3840   [(set (match_operand:DI 0 "register_operand" "=r")
3841         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand" "r")
3842                         (match_operand 2 "const_int_operand" "n")
3843                         (match_operand 3 "const_int_operand" "n")))]
3844   ""
3845   ""
3846 )
3847
3848 (define_insn "*<optab><mode>"
3849   [(set (match_operand:GPI 0 "register_operand" "=r")
3850         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
3851                          (match_operand 2 "const_int_operand" "n")
3852                          (match_operand 3 "const_int_operand" "n")))]
3853   ""
3854   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
3855   [(set_attr "type" "bfm")]
3856 )
3857
3858 ;; Bitfield Insert (insv)
3859 (define_expand "insv<mode>"
3860   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
3861                           (match_operand 1 "const_int_operand")
3862                           (match_operand 2 "const_int_operand"))
3863         (match_operand:GPI 3 "general_operand"))]
3864   ""
3865 {
3866   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
3867   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
3868   rtx value = operands[3];
3869
3870   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
3871     FAIL;
3872
3873   if (CONST_INT_P (value))
3874     {
3875       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
3876
3877       /* Prefer AND/OR for inserting all zeros or all ones.  */
3878       if ((UINTVAL (value) & mask) == 0
3879            || (UINTVAL (value) & mask) == mask)
3880         FAIL;
3881
3882       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
3883       if (width == 16 && (pos % 16) == 0)
3884         DONE;
3885     }
3886   operands[3] = force_reg (<MODE>mode, value);
3887 })
3888
3889 (define_insn "*insv_reg<mode>"
3890   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
3891                           (match_operand 1 "const_int_operand" "n")
3892                           (match_operand 2 "const_int_operand" "n"))
3893         (match_operand:GPI 3 "register_operand" "r"))]
3894   "!(UINTVAL (operands[1]) == 0
3895      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
3896          > GET_MODE_BITSIZE (<MODE>mode)))"
3897   "bfi\\t%<w>0, %<w>3, %2, %1"
3898   [(set_attr "type" "bfm")]
3899 )
3900
3901 (define_insn "*extr_insv_lower_reg<mode>"
3902   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
3903                           (match_operand 1 "const_int_operand" "n")
3904                           (const_int 0))
3905         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
3906                           (match_dup 1)
3907                           (match_operand 3 "const_int_operand" "n")))]
3908   "!(UINTVAL (operands[1]) == 0
3909      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
3910          > GET_MODE_BITSIZE (<MODE>mode)))"
3911   "bfxil\\t%<w>0, %<w>2, %3, %1"
3912   [(set_attr "type" "bfm")]
3913 )
3914
3915 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
3916   [(set (match_operand:GPI 0 "register_operand" "=r")
3917         (ashift:GPI (ANY_EXTEND:GPI
3918                      (match_operand:ALLX 1 "register_operand" "r"))
3919                     (match_operand 2 "const_int_operand" "n")))]
3920   "UINTVAL (operands[2]) < <GPI:sizen>"
3921 {
3922   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
3923               ? GEN_INT (<ALLX:sizen>)
3924               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
3925   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
3926 }
3927   [(set_attr "type" "bfm")]
3928 )
3929
3930 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
3931
3932 (define_insn "*andim_ashift<mode>_bfiz"
3933   [(set (match_operand:GPI 0 "register_operand" "=r")
3934         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3935                              (match_operand 2 "const_int_operand" "n"))
3936                  (match_operand 3 "const_int_operand" "n")))]
3937   "(INTVAL (operands[2]) < (<GPI:sizen>))
3938    && exact_log2 ((INTVAL (operands[3]) >> INTVAL (operands[2])) + 1) >= 0
3939    && (INTVAL (operands[3]) & ((1 << INTVAL (operands[2])) - 1)) == 0"
3940   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
3941   [(set_attr "type" "bfm")]
3942 )
3943
3944 (define_insn "bswap<mode>2"
3945   [(set (match_operand:GPI 0 "register_operand" "=r")
3946         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
3947   ""
3948   "rev\\t%<w>0, %<w>1"
3949   [(set_attr "type" "rev")]
3950 )
3951
3952 (define_insn "bswaphi2"
3953   [(set (match_operand:HI 0 "register_operand" "=r")
3954         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
3955   ""
3956   "rev16\\t%w0, %w1"
3957   [(set_attr "type" "rev")]
3958 )
3959
3960 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
3961 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
3962 ;; each valid permutation.
3963
3964 (define_insn "rev16<mode>2"
3965   [(set (match_operand:GPI 0 "register_operand" "=r")
3966         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3967                                       (const_int 8))
3968                           (match_operand:GPI 3 "const_int_operand" "n"))
3969                  (and:GPI (lshiftrt:GPI (match_dup 1)
3970                                         (const_int 8))
3971                           (match_operand:GPI 2 "const_int_operand" "n"))))]
3972   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
3973    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
3974   "rev16\\t%<w>0, %<w>1"
3975   [(set_attr "type" "rev")]
3976 )
3977
3978 (define_insn "rev16<mode>2_alt"
3979   [(set (match_operand:GPI 0 "register_operand" "=r")
3980         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
3981                                         (const_int 8))
3982                           (match_operand:GPI 2 "const_int_operand" "n"))
3983                  (and:GPI (ashift:GPI (match_dup 1)
3984                                       (const_int 8))
3985                           (match_operand:GPI 3 "const_int_operand" "n"))))]
3986   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
3987    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
3988   "rev16\\t%<w>0, %<w>1"
3989   [(set_attr "type" "rev")]
3990 )
3991
3992 ;; zero_extend version of above
3993 (define_insn "*bswapsi2_uxtw"
3994   [(set (match_operand:DI 0 "register_operand" "=r")
3995         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
3996   ""
3997   "rev\\t%w0, %w1"
3998   [(set_attr "type" "rev")]
3999 )
4000
4001 ;; -------------------------------------------------------------------
4002 ;; Floating-point intrinsics
4003 ;; -------------------------------------------------------------------
4004
4005 ;; frint floating-point round to integral standard patterns.
4006 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4007
4008 (define_insn "<frint_pattern><mode>2"
4009   [(set (match_operand:GPF 0 "register_operand" "=w")
4010         (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4011          FRINT))]
4012   "TARGET_FLOAT"
4013   "frint<frint_suffix>\\t%<s>0, %<s>1"
4014   [(set_attr "type" "f_rint<s>")]
4015 )
4016
4017 ;; frcvt floating-point round to integer and convert standard patterns.
4018 ;; Expands to lbtrunc, lceil, lfloor, lround.
4019 (define_insn "l<fcvt_pattern><su_optab><GPF:mode><GPI:mode>2"
4020   [(set (match_operand:GPI 0 "register_operand" "=r")
4021         (FIXUORS:GPI (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4022                       FCVT)))]
4023   "TARGET_FLOAT"
4024   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF:s>1"
4025   [(set_attr "type" "f_cvtf2i")]
4026 )
4027
4028 ;; fma - no throw
4029
4030 (define_insn "fma<mode>4"
4031   [(set (match_operand:GPF 0 "register_operand" "=w")
4032         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4033                  (match_operand:GPF 2 "register_operand" "w")
4034                  (match_operand:GPF 3 "register_operand" "w")))]
4035   "TARGET_FLOAT"
4036   "fmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4037   [(set_attr "type" "fmac<s>")]
4038 )
4039
4040 (define_insn "fnma<mode>4"
4041   [(set (match_operand:GPF 0 "register_operand" "=w")
4042         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4043                  (match_operand:GPF 2 "register_operand" "w")
4044                  (match_operand:GPF 3 "register_operand" "w")))]
4045   "TARGET_FLOAT"
4046   "fmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4047   [(set_attr "type" "fmac<s>")]
4048 )
4049
4050 (define_insn "fms<mode>4"
4051   [(set (match_operand:GPF 0 "register_operand" "=w")
4052         (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4053                  (match_operand:GPF 2 "register_operand" "w")
4054                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4055   "TARGET_FLOAT"
4056   "fnmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4057   [(set_attr "type" "fmac<s>")]
4058 )
4059
4060 (define_insn "fnms<mode>4"
4061   [(set (match_operand:GPF 0 "register_operand" "=w")
4062         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4063                  (match_operand:GPF 2 "register_operand" "w")
4064                  (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4065   "TARGET_FLOAT"
4066   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4067   [(set_attr "type" "fmac<s>")]
4068 )
4069
4070 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
4071 (define_insn "*fnmadd<mode>4"
4072   [(set (match_operand:GPF 0 "register_operand" "=w")
4073         (neg:GPF (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4074                           (match_operand:GPF 2 "register_operand" "w")
4075                           (match_operand:GPF 3 "register_operand" "w"))))]
4076   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
4077   "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4078   [(set_attr "type" "fmac<s>")]
4079 )
4080
4081 ;; -------------------------------------------------------------------
4082 ;; Floating-point conversions
4083 ;; -------------------------------------------------------------------
4084
4085 (define_insn "extendsfdf2"
4086   [(set (match_operand:DF 0 "register_operand" "=w")
4087         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
4088   "TARGET_FLOAT"
4089   "fcvt\\t%d0, %s1"
4090   [(set_attr "type" "f_cvt")]
4091 )
4092
4093 (define_insn "truncdfsf2"
4094   [(set (match_operand:SF 0 "register_operand" "=w")
4095         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
4096   "TARGET_FLOAT"
4097   "fcvt\\t%s0, %d1"
4098   [(set_attr "type" "f_cvt")]
4099 )
4100
4101 (define_insn "fix_trunc<GPF:mode><GPI:mode>2"
4102   [(set (match_operand:GPI 0 "register_operand" "=r")
4103         (fix:GPI (match_operand:GPF 1 "register_operand" "w")))]
4104   "TARGET_FLOAT"
4105   "fcvtzs\\t%<GPI:w>0, %<GPF:s>1"
4106   [(set_attr "type" "f_cvtf2i")]
4107 )
4108
4109 (define_insn "fixuns_trunc<GPF:mode><GPI:mode>2"
4110   [(set (match_operand:GPI 0 "register_operand" "=r")
4111         (unsigned_fix:GPI (match_operand:GPF 1 "register_operand" "w")))]
4112   "TARGET_FLOAT"
4113   "fcvtzu\\t%<GPI:w>0, %<GPF:s>1"
4114   [(set_attr "type" "f_cvtf2i")]
4115 )
4116
4117 (define_insn "<optab><fcvt_target><GPF:mode>2"
4118   [(set (match_operand:GPF 0 "register_operand" "=w,w")
4119         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,r")))]
4120   ""
4121   "@
4122    <su_optab>cvtf\t%<GPF:s>0, %<s>1
4123    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
4124   [(set_attr "simd" "yes,no")
4125    (set_attr "fp" "no,yes")
4126    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
4127 )
4128
4129 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
4130   [(set (match_operand:GPF 0 "register_operand" "=w")
4131         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
4132   "TARGET_FLOAT"
4133   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
4134   [(set_attr "type" "f_cvti2f")]
4135 )
4136
4137 ;; -------------------------------------------------------------------
4138 ;; Floating-point arithmetic
4139 ;; -------------------------------------------------------------------
4140
4141 (define_insn "add<mode>3"
4142   [(set (match_operand:GPF 0 "register_operand" "=w")
4143         (plus:GPF
4144          (match_operand:GPF 1 "register_operand" "w")
4145          (match_operand:GPF 2 "register_operand" "w")))]
4146   "TARGET_FLOAT"
4147   "fadd\\t%<s>0, %<s>1, %<s>2"
4148   [(set_attr "type" "fadd<s>")]
4149 )
4150
4151 (define_insn "sub<mode>3"
4152   [(set (match_operand:GPF 0 "register_operand" "=w")
4153         (minus:GPF
4154          (match_operand:GPF 1 "register_operand" "w")
4155          (match_operand:GPF 2 "register_operand" "w")))]
4156   "TARGET_FLOAT"
4157   "fsub\\t%<s>0, %<s>1, %<s>2"
4158   [(set_attr "type" "fadd<s>")]
4159 )
4160
4161 (define_insn "mul<mode>3"
4162   [(set (match_operand:GPF 0 "register_operand" "=w")
4163         (mult:GPF
4164          (match_operand:GPF 1 "register_operand" "w")
4165          (match_operand:GPF 2 "register_operand" "w")))]
4166   "TARGET_FLOAT"
4167   "fmul\\t%<s>0, %<s>1, %<s>2"
4168   [(set_attr "type" "fmul<s>")]
4169 )
4170
4171 (define_insn "*fnmul<mode>3"
4172   [(set (match_operand:GPF 0 "register_operand" "=w")
4173         (mult:GPF
4174                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4175                  (match_operand:GPF 2 "register_operand" "w")))]
4176   "TARGET_FLOAT"
4177   "fnmul\\t%<s>0, %<s>1, %<s>2"
4178   [(set_attr "type" "fmul<s>")]
4179 )
4180
4181 (define_insn "div<mode>3"
4182   [(set (match_operand:GPF 0 "register_operand" "=w")
4183         (div:GPF
4184          (match_operand:GPF 1 "register_operand" "w")
4185          (match_operand:GPF 2 "register_operand" "w")))]
4186   "TARGET_FLOAT"
4187   "fdiv\\t%<s>0, %<s>1, %<s>2"
4188   [(set_attr "type" "fdiv<s>")]
4189 )
4190
4191 (define_insn "neg<mode>2"
4192   [(set (match_operand:GPF 0 "register_operand" "=w")
4193         (neg:GPF (match_operand:GPF 1 "register_operand" "w")))]
4194   "TARGET_FLOAT"
4195   "fneg\\t%<s>0, %<s>1"
4196   [(set_attr "type" "ffarith<s>")]
4197 )
4198
4199 (define_insn "sqrt<mode>2"
4200   [(set (match_operand:GPF 0 "register_operand" "=w")
4201         (sqrt:GPF (match_operand:GPF 1 "register_operand" "w")))]
4202   "TARGET_FLOAT"
4203   "fsqrt\\t%<s>0, %<s>1"
4204   [(set_attr "type" "fsqrt<s>")]
4205 )
4206
4207 (define_insn "abs<mode>2"
4208   [(set (match_operand:GPF 0 "register_operand" "=w")
4209         (abs:GPF (match_operand:GPF 1 "register_operand" "w")))]
4210   "TARGET_FLOAT"
4211   "fabs\\t%<s>0, %<s>1"
4212   [(set_attr "type" "ffarith<s>")]
4213 )
4214
4215 ;; Given that smax/smin do not specify the result when either input is NaN,
4216 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
4217 ;; for smin.
4218
4219 (define_insn "smax<mode>3"
4220   [(set (match_operand:GPF 0 "register_operand" "=w")
4221         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
4222                   (match_operand:GPF 2 "register_operand" "w")))]
4223   "TARGET_FLOAT"
4224   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
4225   [(set_attr "type" "f_minmax<s>")]
4226 )
4227
4228 (define_insn "smin<mode>3"
4229   [(set (match_operand:GPF 0 "register_operand" "=w")
4230         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
4231                   (match_operand:GPF 2 "register_operand" "w")))]
4232   "TARGET_FLOAT"
4233   "fminnm\\t%<s>0, %<s>1, %<s>2"
4234   [(set_attr "type" "f_minmax<s>")]
4235 )
4236
4237 ;; -------------------------------------------------------------------
4238 ;; Reload support
4239 ;; -------------------------------------------------------------------
4240
4241 (define_expand "aarch64_reload_mov<mode>"
4242   [(set (match_operand:TX 0 "register_operand" "=w")
4243         (match_operand:TX 1 "register_operand" "w"))
4244    (clobber (match_operand:DI 2 "register_operand" "=&r"))
4245   ]
4246   "TARGET_FLOAT"
4247   {
4248     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
4249     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
4250     gen_aarch64_movtilow_tilow (op0, op1);
4251     gen_aarch64_movdi_tihigh (operands[2], op1);
4252     gen_aarch64_movtihigh_di (op0, operands[2]);
4253     DONE;
4254   }
4255 )
4256
4257 ;; The following secondary reload helpers patterns are invoked
4258 ;; after or during reload as we don't want these patterns to start
4259 ;; kicking in during the combiner.
4260
4261 (define_insn "aarch64_movdi_<mode>low"
4262   [(set (match_operand:DI 0 "register_operand" "=r")
4263         (truncate:DI (match_operand:TX 1 "register_operand" "w")))]
4264   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4265   "fmov\\t%x0, %d1"
4266   [(set_attr "type" "f_mrc")
4267    (set_attr "length" "4")
4268   ])
4269
4270 (define_insn "aarch64_movdi_<mode>high"
4271   [(set (match_operand:DI 0 "register_operand" "=r")
4272         (truncate:DI
4273           (lshiftrt:TX (match_operand:TX 1 "register_operand" "w")
4274                        (const_int 64))))]
4275   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4276   "fmov\\t%x0, %1.d[1]"
4277   [(set_attr "type" "f_mrc")
4278    (set_attr "length" "4")
4279   ])
4280
4281 (define_insn "aarch64_mov<mode>high_di"
4282   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
4283                          (const_int 64) (const_int 64))
4284         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
4285   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4286   "fmov\\t%0.d[1], %x1"
4287   [(set_attr "type" "f_mcr")
4288    (set_attr "length" "4")
4289   ])
4290
4291 (define_insn "aarch64_mov<mode>low_di"
4292   [(set (match_operand:TX 0 "register_operand" "=w")
4293         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
4294   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4295   "fmov\\t%d0, %x1"
4296   [(set_attr "type" "f_mcr")
4297    (set_attr "length" "4")
4298   ])
4299
4300 (define_insn "aarch64_movtilow_tilow"
4301   [(set (match_operand:TI 0 "register_operand" "=w")
4302         (zero_extend:TI
4303           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
4304   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
4305   "fmov\\t%d0, %d1"
4306   [(set_attr "type" "fmov")
4307    (set_attr "length" "4")
4308   ])
4309
4310 ;; There is a deliberate reason why the parameters of high and lo_sum's
4311 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
4312 ;; and lo_sum's to be used with the labels defining the jump tables in
4313 ;; rodata section.
4314
4315 (define_expand "add_losym"
4316   [(set (match_operand 0 "register_operand" "=r")
4317         (lo_sum (match_operand 1 "register_operand" "r")
4318                 (match_operand 2 "aarch64_valid_symref" "S")))]
4319   ""
4320 {
4321   machine_mode mode = GET_MODE (operands[0]);
4322
4323   emit_insn ((mode == DImode
4324               ? gen_add_losym_di
4325               : gen_add_losym_si) (operands[0],
4326                                    operands[1],
4327                                    operands[2]));
4328   DONE;
4329 })
4330
4331 (define_insn "add_losym_<mode>"
4332   [(set (match_operand:P 0 "register_operand" "=r")
4333         (lo_sum:P (match_operand:P 1 "register_operand" "r")
4334                   (match_operand 2 "aarch64_valid_symref" "S")))]
4335   ""
4336   "add\\t%<w>0, %<w>1, :lo12:%a2"
4337   [(set_attr "type" "alu_imm")]
4338 )
4339
4340 (define_insn "ldr_got_small_<mode>"
4341   [(set (match_operand:PTR 0 "register_operand" "=r")
4342         (unspec:PTR [(mem:PTR (lo_sum:PTR
4343                               (match_operand:PTR 1 "register_operand" "r")
4344                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
4345                     UNSPEC_GOTSMALLPIC))]
4346   ""
4347   "ldr\\t%<w>0, [%1, #:got_lo12:%a2]"
4348   [(set_attr "type" "load1")]
4349 )
4350
4351 (define_insn "ldr_got_small_sidi"
4352   [(set (match_operand:DI 0 "register_operand" "=r")
4353         (zero_extend:DI
4354          (unspec:SI [(mem:SI (lo_sum:DI
4355                              (match_operand:DI 1 "register_operand" "r")
4356                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
4357                     UNSPEC_GOTSMALLPIC)))]
4358   "TARGET_ILP32"
4359   "ldr\\t%w0, [%1, #:got_lo12:%a2]"
4360   [(set_attr "type" "load1")]
4361 )
4362
4363 (define_insn "ldr_got_tiny"
4364   [(set (match_operand:DI 0 "register_operand" "=r")
4365         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
4366                    UNSPEC_GOTTINYPIC))]
4367   ""
4368   "ldr\\t%0, %L1"
4369   [(set_attr "type" "load1")]
4370 )
4371
4372 (define_insn "aarch64_load_tp_hard"
4373   [(set (match_operand:DI 0 "register_operand" "=r")
4374         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
4375   ""
4376   "mrs\\t%0, tpidr_el0"
4377   [(set_attr "type" "mrs")]
4378 )
4379
4380 ;; The TLS ABI specifically requires that the compiler does not schedule
4381 ;; instructions in the TLS stubs, in order to enable linker relaxation.
4382 ;; Therefore we treat the stubs as an atomic sequence.
4383 (define_expand "tlsgd_small"
4384  [(parallel [(set (match_operand 0 "register_operand" "")
4385                   (call (mem:DI (match_dup 2)) (const_int 1)))
4386              (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
4387              (clobber (reg:DI LR_REGNUM))])]
4388  ""
4389 {
4390   operands[2] = aarch64_tls_get_addr ();
4391 })
4392
4393 (define_insn "*tlsgd_small"
4394   [(set (match_operand 0 "register_operand" "")
4395         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
4396    (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
4397    (clobber (reg:DI LR_REGNUM))
4398   ]
4399   ""
4400   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
4401   [(set_attr "type" "call")
4402    (set_attr "length" "16")])
4403
4404 (define_insn "tlsie_small_<mode>"
4405   [(set (match_operand:PTR 0 "register_operand" "=r")
4406         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
4407                    UNSPEC_GOTSMALLTLS))]
4408   ""
4409   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
4410   [(set_attr "type" "load1")
4411    (set_attr "length" "8")]
4412 )
4413
4414 (define_insn "tlsie_small_sidi"
4415   [(set (match_operand:DI 0 "register_operand" "=r")
4416         (zero_extend:DI
4417           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
4418                       UNSPEC_GOTSMALLTLS)))]
4419   ""
4420   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
4421   [(set_attr "type" "load1")
4422    (set_attr "length" "8")]
4423 )
4424
4425 (define_expand "tlsle_small"
4426   [(set (match_operand 0 "register_operand" "=r")
4427         (unspec [(match_operand 1 "register_operand" "r")
4428                    (match_operand 2 "aarch64_tls_le_symref" "S")]
4429                    UNSPEC_GOTSMALLTLS))]
4430   ""
4431 {
4432   machine_mode mode = GET_MODE (operands[0]);
4433   emit_insn ((mode == DImode
4434               ? gen_tlsle_small_di
4435               : gen_tlsle_small_si) (operands[0],
4436                                      operands[1],
4437                                      operands[2]));
4438   DONE;
4439 })
4440
4441 (define_insn "tlsle_small_<mode>"
4442   [(set (match_operand:P 0 "register_operand" "=r")
4443         (unspec:P [(match_operand:P 1 "register_operand" "r")
4444                    (match_operand 2 "aarch64_tls_le_symref" "S")]
4445                    UNSPEC_GOTSMALLTLS))]
4446   ""
4447   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
4448   [(set_attr "type" "alu_sreg")
4449    (set_attr "length" "8")]
4450 )
4451
4452 (define_insn "tlsdesc_small_<mode>"
4453   [(set (reg:PTR R0_REGNUM)
4454         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
4455                    UNSPEC_TLSDESC))
4456    (clobber (reg:DI LR_REGNUM))
4457    (clobber (reg:CC CC_REGNUM))
4458    (clobber (match_scratch:DI 1 "=r"))]
4459   "TARGET_TLS_DESC"
4460   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
4461   [(set_attr "type" "call")
4462    (set_attr "length" "16")])
4463
4464 (define_insn "stack_tie"
4465   [(set (mem:BLK (scratch))
4466         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
4467                      (match_operand:DI 1 "register_operand" "rk")]
4468                     UNSPEC_PRLG_STK))]
4469   ""
4470   ""
4471   [(set_attr "length" "0")]
4472 )
4473
4474 ;; Named pattern for expanding thread pointer reference.
4475 (define_expand "get_thread_pointerdi"
4476   [(match_operand:DI 0 "register_operand" "=r")]
4477   ""
4478 {
4479   rtx tmp = aarch64_load_tp (operands[0]);
4480   if (tmp != operands[0])
4481     emit_move_insn (operands[0], tmp);
4482   DONE;
4483 })
4484
4485 ;; Named patterns for stack smashing protection.
4486 (define_expand "stack_protect_set"
4487   [(match_operand 0 "memory_operand")
4488    (match_operand 1 "memory_operand")]
4489   ""
4490 {
4491   machine_mode mode = GET_MODE (operands[0]);
4492
4493   emit_insn ((mode == DImode
4494               ? gen_stack_protect_set_di
4495               : gen_stack_protect_set_si) (operands[0], operands[1]));
4496   DONE;
4497 })
4498
4499 (define_insn "stack_protect_set_<mode>"
4500   [(set (match_operand:PTR 0 "memory_operand" "=m")
4501         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
4502          UNSPEC_SP_SET))
4503    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
4504   ""
4505   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
4506   [(set_attr "length" "12")
4507    (set_attr "type" "multiple")])
4508
4509 (define_expand "stack_protect_test"
4510   [(match_operand 0 "memory_operand")
4511    (match_operand 1 "memory_operand")
4512    (match_operand 2)]
4513   ""
4514 {
4515   rtx result;
4516   machine_mode mode = GET_MODE (operands[0]);
4517
4518   result = gen_reg_rtx(mode);
4519
4520   emit_insn ((mode == DImode
4521               ? gen_stack_protect_test_di
4522               : gen_stack_protect_test_si) (result,
4523                                             operands[0],
4524                                             operands[1]));
4525
4526   if (mode == DImode)
4527     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
4528                                     result, const0_rtx, operands[2]));
4529   else
4530     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
4531                                     result, const0_rtx, operands[2]));
4532   DONE;
4533 })
4534
4535 (define_insn "stack_protect_test_<mode>"
4536   [(set (match_operand:PTR 0 "register_operand" "=r")
4537         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
4538                      (match_operand:PTR 2 "memory_operand" "m")]
4539          UNSPEC_SP_TEST))
4540    (clobber (match_scratch:PTR 3 "=&r"))]
4541   ""
4542   "ldr\t%<w>3, %x1\;ldr\t%<w>0, %x2\;eor\t%<w>0, %<w>3, %<w>0"
4543   [(set_attr "length" "12")
4544    (set_attr "type" "multiple")])
4545
4546 ;; Write Floating-point Control Register.
4547 (define_insn "set_fpcr"
4548   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
4549   ""
4550   "msr\\tfpcr, %0"
4551   [(set_attr "type" "mrs")])
4552
4553 ;; Read Floating-point Control Register.
4554 (define_insn "get_fpcr"
4555   [(set (match_operand:SI 0 "register_operand" "=r")
4556         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
4557   ""
4558   "mrs\\t%0, fpcr"
4559   [(set_attr "type" "mrs")])
4560
4561 ;; Write Floating-point Status Register.
4562 (define_insn "set_fpsr"
4563   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
4564   ""
4565   "msr\\tfpsr, %0"
4566   [(set_attr "type" "mrs")])
4567
4568 ;; Read Floating-point Status Register.
4569 (define_insn "get_fpsr"
4570   [(set (match_operand:SI 0 "register_operand" "=r")
4571         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
4572   ""
4573   "mrs\\t%0, fpsr"
4574   [(set_attr "type" "mrs")])
4575
4576
4577 ;; Define the subtract-one-and-jump insns so loop.c
4578 ;; knows what to generate.
4579 (define_expand "doloop_end"
4580   [(use (match_operand 0 "" ""))      ; loop pseudo
4581    (use (match_operand 1 "" ""))]     ; label
4582   "optimize > 0 && flag_modulo_sched"
4583 {
4584   rtx s0;
4585   rtx bcomp;
4586   rtx loc_ref;
4587   rtx cc_reg;
4588   rtx insn;
4589   rtx cmp;
4590
4591   /* Currently SMS relies on the do-loop pattern to recognize loops
4592      where (1) the control part consists of all insns defining and/or
4593      using a certain 'count' register and (2) the loop count can be
4594      adjusted by modifying this register prior to the loop.
4595      ??? The possible introduction of a new block to initialize the
4596      new IV can potentially affect branch optimizations.  */
4597
4598   if (GET_MODE (operands[0]) != DImode)
4599     FAIL;
4600
4601   s0 = operands [0];
4602   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
4603
4604   cmp = XVECEXP (PATTERN (insn), 0, 0);
4605   cc_reg = SET_DEST (cmp);
4606   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
4607   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
4608   emit_jump_insn (gen_rtx_SET (pc_rtx,
4609                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
4610                                                      loc_ref, pc_rtx)));
4611   DONE;
4612 })
4613
4614 ;; AdvSIMD Stuff
4615 (include "aarch64-simd.md")
4616
4617 ;; Atomic Operations
4618 (include "atomics.md")
4619
4620 ;; ldp/stp peephole patterns
4621 (include "aarch64-ldpstp.md")