S/390 zvector builtin support.

[gcc.git] / gcc / config / s390 / s390.md

;;- Machine description for GNU compiler -- S/390 / zSeries version.
;;  Copyright (C) 1999-2015 Free Software Foundation, Inc.
;;  Contributed by Hartmut Penner (hpenner@de.ibm.com) and
;;                 Ulrich Weigand (uweigand@de.ibm.com) and
;;                 Andreas Krebbel (Andreas.Krebbel@de.ibm.com)

;; This file is part of GCC.

;; GCC is free software; you can redistribute it and/or modify it under
;; the terms of the GNU General Public License as published by the Free
;; Software Foundation; either version 3, or (at your option) any later
;; version.

;; GCC is distributed in the hope that it will be useful, but WITHOUT ANY
;; WARRANTY; without even the implied warranty of MERCHANTABILITY or
;; FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
;; for more details.

;; You should have received a copy of the GNU General Public License
;; along with GCC; see the file COPYING3.  If not see
;; <http://www.gnu.org/licenses/>.

;;
;; See constraints.md for a description of constraints specific to s390.
;;

;; Special formats used for outputting 390 instructions.
;;
;;     %C: print opcode suffix for branch condition.
;;     %D: print opcode suffix for inverse branch condition.
;;     %J: print tls_load/tls_gdcall/tls_ldcall suffix
;;     %G: print the size of the operand in bytes.
;;     %O: print only the displacement of a memory reference.
;;     %R: print only the base register of a memory reference.
;;     %S: print S-type memory reference (base+displacement).
;;     %N: print the second word of a DImode operand.
;;     %M: print the second word of a TImode operand.
;;     %Y: print shift count operand.
;;
;;     %b: print integer X as if it's an unsigned byte.
;;     %c: print integer X as if it's an signed byte.
;;     %x: print integer X as if it's an unsigned halfword.
;;     %h: print integer X as if it's a signed halfword.
;;     %i: print the first nonzero HImode part of X.
;;     %j: print the first HImode part unequal to -1 of X.
;;     %k: print the first nonzero SImode part of X.
;;     %m: print the first SImode part unequal to -1 of X.
;;     %o: print integer X as if it's an unsigned 32bit word.
;;
;; We have a special constraint for pattern matching.
;;
;;   s_operand -- Matches a valid S operand in a RS, SI or SS type instruction.
;;

;;
;; UNSPEC usage
;;

(define_c_enum "unspec" [
   ; Miscellaneous
   UNSPEC_ROUND
   UNSPEC_ICM
   UNSPEC_TIE

   ; Convert CC into a str comparison result and copy it into an
   ; integer register
   ; cc0->0, cc1->1, cc2->-1, (cc3->-1)
   UNSPEC_STRCMPCC_TO_INT

   ; Copy CC as is into the lower 2 bits of an integer register
   UNSPEC_CC_TO_INT

   ; GOT/PLT and lt-relative accesses
   UNSPEC_LTREL_OFFSET
   UNSPEC_LTREL_BASE
   UNSPEC_POOL_OFFSET
   UNSPEC_GOTENT
   UNSPEC_GOT
   UNSPEC_GOTOFF
   UNSPEC_PLT
   UNSPEC_PLTOFF

   ; Literal pool
   UNSPEC_RELOAD_BASE
   UNSPEC_MAIN_BASE
   UNSPEC_LTREF
   UNSPEC_INSN
   UNSPEC_EXECUTE

   ; Atomic Support
   UNSPEC_MB
   UNSPEC_MOVA

   ; TLS relocation specifiers
   UNSPEC_TLSGD
   UNSPEC_TLSLDM
   UNSPEC_NTPOFF
   UNSPEC_DTPOFF
   UNSPEC_GOTNTPOFF
   UNSPEC_INDNTPOFF

   ; TLS support
   UNSPEC_TLSLDM_NTPOFF
   UNSPEC_TLS_LOAD

   ; String Functions
   UNSPEC_SRST
   UNSPEC_MVST

   ; Stack Smashing Protector
   UNSPEC_SP_SET
   UNSPEC_SP_TEST

   ; Test Data Class (TDC)
   UNSPEC_TDC_INSN

   ; Population Count
   UNSPEC_POPCNT
   UNSPEC_COPYSIGN

   ; Load FP Integer
   UNSPEC_FPINT_FLOOR
   UNSPEC_FPINT_BTRUNC
   UNSPEC_FPINT_ROUND
   UNSPEC_FPINT_CEIL
   UNSPEC_FPINT_NEARBYINT
   UNSPEC_FPINT_RINT

   UNSPEC_LCBB

   ; Vector
   UNSPEC_VEC_SMULT_HI
   UNSPEC_VEC_UMULT_HI
   UNSPEC_VEC_SMULT_LO
   UNSPEC_VEC_SMULT_EVEN
   UNSPEC_VEC_UMULT_EVEN
   UNSPEC_VEC_SMULT_ODD
   UNSPEC_VEC_UMULT_ODD

   UNSPEC_VEC_VMAL
   UNSPEC_VEC_VMAH
   UNSPEC_VEC_VMALH
   UNSPEC_VEC_VMAE
   UNSPEC_VEC_VMALE
   UNSPEC_VEC_VMAO
   UNSPEC_VEC_VMALO

   UNSPEC_VEC_GATHER
   UNSPEC_VEC_EXTRACT
   UNSPEC_VEC_INSERT_AND_ZERO
   UNSPEC_VEC_LOAD_BNDRY
   UNSPEC_VEC_LOAD_LEN
   UNSPEC_VEC_MERGEH
   UNSPEC_VEC_MERGEL
   UNSPEC_VEC_PACK
   UNSPEC_VEC_PACK_SATURATE
   UNSPEC_VEC_PACK_SATURATE_CC
   UNSPEC_VEC_PACK_SATURATE_GENCC
   UNSPEC_VEC_PACK_UNSIGNED_SATURATE
   UNSPEC_VEC_PACK_UNSIGNED_SATURATE_CC
   UNSPEC_VEC_PACK_UNSIGNED_SATURATE_GENCC
   UNSPEC_VEC_PERM
   UNSPEC_VEC_PERMI
   UNSPEC_VEC_EXTEND
   UNSPEC_VEC_STORE_LEN
   UNSPEC_VEC_UNPACKH
   UNSPEC_VEC_UNPACKH_L
   UNSPEC_VEC_UNPACKL
   UNSPEC_VEC_UNPACKL_L
   UNSPEC_VEC_ADDC
   UNSPEC_VEC_ADDC_U128
   UNSPEC_VEC_ADDE_U128
   UNSPEC_VEC_ADDEC_U128
   UNSPEC_VEC_AVG
   UNSPEC_VEC_AVGU
   UNSPEC_VEC_CHECKSUM
   UNSPEC_VEC_GFMSUM
   UNSPEC_VEC_GFMSUM_128
   UNSPEC_VEC_GFMSUM_ACCUM
   UNSPEC_VEC_GFMSUM_ACCUM_128
   UNSPEC_VEC_SET

   UNSPEC_VEC_VSUMG
   UNSPEC_VEC_VSUMQ
   UNSPEC_VEC_VSUM
   UNSPEC_VEC_RL_MASK
   UNSPEC_VEC_SLL
   UNSPEC_VEC_SLB
   UNSPEC_VEC_SLDB
   UNSPEC_VEC_SRAL
   UNSPEC_VEC_SRAB
   UNSPEC_VEC_SRL
   UNSPEC_VEC_SRLB

   UNSPEC_VEC_SUB_U128
   UNSPEC_VEC_SUBC
   UNSPEC_VEC_SUBC_U128
   UNSPEC_VEC_SUBE_U128
   UNSPEC_VEC_SUBEC_U128

   UNSPEC_VEC_TEST_MASK

   UNSPEC_VEC_VFAE
   UNSPEC_VEC_VFAECC

   UNSPEC_VEC_VFEE
   UNSPEC_VEC_VFEECC
   UNSPEC_VEC_VFENE
   UNSPEC_VEC_VFENECC

   UNSPEC_VEC_VISTR
   UNSPEC_VEC_VISTRCC

   UNSPEC_VEC_VSTRC
   UNSPEC_VEC_VSTRCCC

   UNSPEC_VEC_VCDGB
   UNSPEC_VEC_VCDLGB

   UNSPEC_VEC_VCGDB
   UNSPEC_VEC_VCLGDB

   UNSPEC_VEC_VFIDB

   UNSPEC_VEC_VLDEB
   UNSPEC_VEC_VLEDB

   UNSPEC_VEC_VFTCIDB
   UNSPEC_VEC_VFTCIDBCC
])

;;
;; UNSPEC_VOLATILE usage
;;

(define_c_enum "unspecv" [
   ; Blockage
   UNSPECV_BLOCKAGE

   ; TPF Support
   UNSPECV_TPF_PROLOGUE
   UNSPECV_TPF_EPILOGUE

   ; Literal pool
   UNSPECV_POOL
   UNSPECV_POOL_SECTION
   UNSPECV_POOL_ALIGN
   UNSPECV_POOL_ENTRY
   UNSPECV_MAIN_POOL

   ; TLS support
   UNSPECV_SET_TP

   ; Atomic Support
   UNSPECV_CAS
   UNSPECV_ATOMIC_OP

   ; Hotpatching (unremovable NOPs)
   UNSPECV_NOP_2_BYTE
   UNSPECV_NOP_4_BYTE
   UNSPECV_NOP_6_BYTE

   ; Transactional Execution support
   UNSPECV_TBEGIN
   UNSPECV_TBEGIN_TDB
   UNSPECV_TBEGINC
   UNSPECV_TEND
   UNSPECV_TABORT
   UNSPECV_ETND
   UNSPECV_NTSTG
   UNSPECV_PPA

   ; Set and get floating point control register
   UNSPECV_SFPC
   UNSPECV_EFPC
  ])

;;
;; Registers
;;

; Registers with special meaning

(define_constants
  [
   ; Sibling call register.
   (SIBCALL_REGNUM               1)
   ; Literal pool base register.
   (BASE_REGNUM                 13)
   ; Return address register.
   (RETURN_REGNUM               14)
   ; Condition code register.
   (CC_REGNUM                   33)
   ; Thread local storage pointer register.
   (TP_REGNUM                   36)
  ])

; Hardware register names

(define_constants
  [
   ; General purpose registers
   (GPR0_REGNUM                  0)
   ; Floating point registers.
   (FPR0_REGNUM                 16)
   (FPR1_REGNUM                 20)
   (FPR2_REGNUM                 17)
   (FPR3_REGNUM                 21)
   (FPR4_REGNUM                 18)
   (FPR5_REGNUM                 22)
   (FPR6_REGNUM                 19)
   (FPR7_REGNUM                 23)
   (FPR8_REGNUM                 24)
   (FPR9_REGNUM                 28)
   (FPR10_REGNUM                25)
   (FPR11_REGNUM                29)
   (FPR12_REGNUM                26)
   (FPR13_REGNUM                30)
   (FPR14_REGNUM                27)
   (FPR15_REGNUM                31)
   (VR0_REGNUM                  16)
   (VR16_REGNUM                 38)
   (VR23_REGNUM                 45)
   (VR24_REGNUM                 46)
   (VR31_REGNUM                 53)
  ])

;;
;; PFPO GPR0 argument format
;;

(define_constants
  [
   ; PFPO operation type
   (PFPO_CONVERT          0x1000000)
   ; PFPO operand types
   (PFPO_OP_TYPE_SF             0x5)
   (PFPO_OP_TYPE_DF             0x6)
   (PFPO_OP_TYPE_TF             0x7)
   (PFPO_OP_TYPE_SD             0x8)
   (PFPO_OP_TYPE_DD             0x9)
   (PFPO_OP_TYPE_TD             0xa)
   ; Bitposition of operand types
   (PFPO_OP0_TYPE_SHIFT          16)
   (PFPO_OP1_TYPE_SHIFT           8)
  ])

; Immediate operands for tbegin and tbeginc
(define_constants [(TBEGIN_MASK  65292)]) ; 0xff0c
(define_constants [(TBEGINC_MASK 65288)]) ; 0xff08

;; Instruction operand type as used in the Principles of Operation.
;; Used to determine defaults for length and other attribute values.

(define_attr "op_type"
  "NN,E,RR,RRE,RX,RS,RSI,RI,SI,S,SS,SSE,RXE,RSE,RIL,RIE,RXY,RSY,SIY,RRF,RRR,SIL,RRS,RIS,VRI,VRR,VRS,VRV,VRX"
  (const_string "NN"))

;; Instruction type attribute used for scheduling.

(define_attr "type" "none,integer,load,lr,la,larl,lm,stm,
                     cs,vs,store,sem,idiv,
                     imulhi,imulsi,imuldi,
                     branch,jsr,fsimptf,fsimpdf,fsimpsf,fhex,
                     floadtf,floaddf,floadsf,fstoredf,fstoresf,
                     fmultf,fmuldf,fmulsf,fdivtf,fdivdf,fdivsf,
                     ftoi,fsqrttf,fsqrtdf,fsqrtsf,
                     fmadddf,fmaddsf,
                     ftrunctf,ftruncdf, ftruncsd, ftruncdd,
                     itoftf, itofdf, itofsf, itofdd, itoftd,
                     fdivdd, fdivtd, floaddd, floadsd, fmuldd, fmultd,
                     fsimpdd, fsimpsd, fsimptd, fstoredd, fstoresd,
                     ftoidfp, other"
  (cond [(eq_attr "op_type" "NN")  (const_string "other")
         (eq_attr "op_type" "SS")  (const_string "cs")]
    (const_string "integer")))

;; Another attribute used for scheduling purposes:
;;   agen: Instruction uses the address generation unit
;;   reg: Instruction does not use the agen unit

(define_attr "atype" "agen,reg"
  (if_then_else (eq_attr "op_type" "E,RR,RI,RRE,RSI,RIL,RIE,RRF,RRR")
                (const_string "reg")
                (const_string "agen")))

;; Properties concerning Z10 execution grouping and value forwarding.
;; z10_super: instruction is superscalar.
;; z10_super_c: instruction is superscalar and meets the condition of z10_c.
;; z10_fwd: The instruction reads the value of an operand and stores it into a
;;   target register.  It can forward this value to a second instruction that reads
;;   the same register if that second instruction is issued in the same group.
;; z10_rec: The instruction is in the T pipeline and reads a register. If the
;;   instruction in the S pipe writes to the register, then the T instruction
;;   can immediately read the new value.
;; z10_fr: union of Z10_fwd and z10_rec.
;; z10_c: second operand of instruction is a register and read with complemented bits.
;;
;; An additional suffix A1, A3, or E1 indicates the respective AGI bypass.


(define_attr "z10prop" "none,
                        z10_super, z10_super_E1, z10_super_A1, z10_super_c, z10_super_c_E1,
                        z10_fwd, z10_fwd_A1, z10_fwd_A3, z10_fwd_E1,
                        z10_rec,
                        z10_fr, z10_fr_A3, z10_fr_E1,
                        z10_c"
             (const_string "none"))

;; Properties concerning Z196 decoding
;; z196_alone: must group alone
;; z196_end: ends a group
;; z196_cracked: instruction is cracked or expanded
(define_attr "z196prop" "none,
                         z196_alone, z196_ends,
                         z196_cracked"
             (const_string "none"))

(define_attr "mnemonic" "bcr_flush,unknown" (const_string "unknown"))

;; Length in bytes.

(define_attr "length" ""
  (cond [(eq_attr "op_type" "E,RR")                           (const_int 2)
         (eq_attr "op_type" "RX,RI,RRE,RS,RSI,S,SI,RRF,RRR")  (const_int 4)]
    (const_int 6)))


;; Processor type.  This attribute must exactly match the processor_type
;; enumeration in s390.h.  The current machine description does not
;; distinguish between g5 and g6, but there are differences between the two
;; CPUs could in theory be modeled.

(define_attr "cpu" "g5,g6,z900,z990,z9_109,z9_ec,z10,z196,zEC12,z13"
  (const (symbol_ref "s390_tune_attr")))

(define_attr "cpu_facility"
  "standard,ieee,zarch,cpu_zarch,longdisp,extimm,dfp,z10,z196,zEC12,vec"
  (const_string "standard"))

(define_attr "enabled" ""
  (cond [(eq_attr "cpu_facility" "standard")
         (const_int 1)

         (and (eq_attr "cpu_facility" "ieee")
              (match_test "TARGET_CPU_IEEE_FLOAT"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "zarch")
              (match_test "TARGET_ZARCH"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "longdisp")
              (match_test "TARGET_LONG_DISPLACEMENT"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "extimm")
              (match_test "TARGET_EXTIMM"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "dfp")
              (match_test "TARGET_DFP"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "cpu_zarch")
              (match_test "TARGET_CPU_ZARCH"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "z10")
              (match_test "TARGET_Z10"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "z196")
              (match_test "TARGET_Z196"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "zEC12")
              (match_test "TARGET_ZEC12"))
         (const_int 1)

         (and (eq_attr "cpu_facility" "vec")
              (match_test "TARGET_VX"))
         (const_int 1)]
        (const_int 0)))

;; Pipeline description for z900.  For lack of anything better,
;; this description is also used for the g5 and g6.
(include "2064.md")

;; Pipeline description for z990, z9-109 and z9-ec.
(include "2084.md")

;; Pipeline description for z10
(include "2097.md")

;; Pipeline description for z196
(include "2817.md")

;; Pipeline description for zEC12
(include "2827.md")

;; Predicates
(include "predicates.md")

;; Constraint definitions
(include "constraints.md")

;; Other includes
(include "tpf.md")

;; Iterators

(define_mode_iterator ALL [TI DI SI HI QI TF DF SF TD DD SD V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI V1SF V2SF V4SF V1TI V1DF V2DF V1TF])

;; These mode iterators allow floating point patterns to be generated from the
;; same template.
(define_mode_iterator FP_ALL [TF DF SF (TD "TARGET_HARD_DFP") (DD "TARGET_HARD_DFP")
                              (SD "TARGET_HARD_DFP")])
(define_mode_iterator FP [TF DF SF (TD "TARGET_HARD_DFP") (DD "TARGET_HARD_DFP")])
(define_mode_iterator BFP [TF DF SF])
(define_mode_iterator DFP [TD DD])
(define_mode_iterator DFP_ALL [TD DD SD])
(define_mode_iterator DSF [DF SF])
(define_mode_iterator SD_SF [SF SD])
(define_mode_iterator DD_DF [DF DD])
(define_mode_iterator TD_TF [TF TD])

;; These mode iterators allow 31-bit and 64-bit GPR patterns to be generated
;; from the same template.
(define_mode_iterator GPR [(DI "TARGET_ZARCH") SI])
(define_mode_iterator DGPR [(TI "TARGET_ZARCH") DI SI])
(define_mode_iterator DSI [DI SI])
(define_mode_iterator TDI [TI DI])

;; These mode iterators allow :P to be used for patterns that operate on
;; pointer-sized quantities.  Exactly one of the two alternatives will match.
(define_mode_iterator P [(DI "TARGET_64BIT") (SI "!TARGET_64BIT")])

;; These macros refer to the actual word_mode of the configuration.
;; This is equal to Pmode except on 31-bit machines in zarch mode.
(define_mode_iterator DW [(TI "TARGET_ZARCH") (DI "!TARGET_ZARCH")])
(define_mode_iterator W  [(DI "TARGET_ZARCH") (SI "!TARGET_ZARCH")])

;; Used by the umul pattern to express modes having half the size.
(define_mode_attr DWH [(TI "DI") (DI "SI")])
(define_mode_attr dwh [(TI "di") (DI "si")])

;; This mode iterator allows the QI and HI patterns to be defined from
;; the same template.
(define_mode_iterator HQI [HI QI])

;; This mode iterator allows the integer patterns to be defined from the
;; same template.
(define_mode_iterator INT [(DI "TARGET_ZARCH") SI HI QI])
(define_mode_iterator DINT [(TI "TARGET_ZARCH") DI SI HI QI])

;; This iterator allows some 'ashift' and 'lshiftrt' pattern to be defined from
;; the same template.
(define_code_iterator SHIFT [ashift lshiftrt])

;; This iterator allows r[ox]sbg to be defined with the same template
(define_code_iterator IXOR [ior xor])

;; This iterator is used to expand the patterns for the nearest
;; integer functions.
(define_int_iterator FPINT [UNSPEC_FPINT_FLOOR UNSPEC_FPINT_BTRUNC
                            UNSPEC_FPINT_ROUND UNSPEC_FPINT_CEIL
                            UNSPEC_FPINT_NEARBYINT])
(define_int_attr fpint_name [(UNSPEC_FPINT_FLOOR "floor")
                             (UNSPEC_FPINT_BTRUNC "btrunc")
                             (UNSPEC_FPINT_ROUND "round")
                             (UNSPEC_FPINT_CEIL "ceil")
                             (UNSPEC_FPINT_NEARBYINT "nearbyint")])
(define_int_attr fpint_roundingmode [(UNSPEC_FPINT_FLOOR "7")
                                     (UNSPEC_FPINT_BTRUNC "5")
                                     (UNSPEC_FPINT_ROUND "1")
                                     (UNSPEC_FPINT_CEIL "6")
                                     (UNSPEC_FPINT_NEARBYINT "0")])

;; This iterator and attribute allow to combine most atomic operations.
(define_code_iterator ATOMIC [and ior xor plus minus mult])
(define_code_iterator ATOMIC_Z196 [and ior xor plus])
(define_code_attr atomic [(and "and") (ior "or") (xor "xor")
                          (plus "add") (minus "sub") (mult "nand")])
(define_code_attr noxa [(and "n") (ior "o") (xor "x") (plus "a")])

;; In FP templates, a string like "lt<de>br" will expand to "ltxbr" in
;; TF/TDmode, "ltdbr" in DF/DDmode, and "ltebr" in SF/SDmode.
(define_mode_attr xde [(TF "x") (DF "d") (SF "e") (TD "x") (DD "d") (SD "e")])

;; In FP templates, a <dee> in "m<dee><bt>r" will expand to "mx<bt>r" in
;; TF/TDmode, "md<bt>r" in DF/DDmode, "mee<bt>r" in SFmode and "me<bt>r in
;; SDmode.
(define_mode_attr xdee [(TF "x") (DF "d") (SF "ee") (TD "x") (DD "d") (SD "e")])

;; In FP templates, "<RRe>" will expand to "RRE" in TFmode and "RR" otherwise.
;; Likewise for "<RXe>".
(define_mode_attr RRe [(TF "RRE") (DF "RR") (SF "RR")])
(define_mode_attr RXe [(TF "RXE") (DF "RX") (SF "RX")])

;; The decimal floating point variants of add, sub, div and mul support 3
;; fp register operands.  The following attributes allow to merge the bfp and
;; dfp variants in a single insn definition.

;; This attribute is used to set op_type accordingly.
(define_mode_attr RRer [(TF "RRE") (DF "RRE") (SF "RRE") (TD "RRR")
                        (DD "RRR") (SD "RRR")])

;; This attribute is used in the operand constraint list in order to have the
;; first and the second operand match for bfp modes.
(define_mode_attr f0 [(TF "0") (DF "0") (SF "0") (TD "f") (DD "f") (DD "f")])

;; This attribute is used to merge the scalar vector instructions into
;; the FP patterns.  For non-supported modes (all but DF) it expands
;; to constraints which are supposed to be matched by an earlier
;; variant.
(define_mode_attr v0      [(TF "0") (DF "v") (SF "0") (TD "0") (DD "0") (DD "0") (TI "0") (DI "v") (SI "0")])
(define_mode_attr vf      [(TF "f") (DF "v") (SF "f") (TD "f") (DD "f") (DD "f") (TI "f") (DI "v") (SI "f")])
(define_mode_attr vd      [(TF "d") (DF "v") (SF "d") (TD "d") (DD "d") (DD "d") (TI "d") (DI "v") (SI "d")])

;; This attribute is used in the operand list of the instruction to have an
;; additional operand for the dfp instructions.
(define_mode_attr op1 [(TF "") (DF "") (SF "")
                       (TD "%1,") (DD "%1,") (SD "%1,")])


;; This attribute is used in the operand constraint list
;; for instructions dealing with the sign bit of 32 or 64bit fp values.
;; TFmode values are represented by a fp register pair.  Since the
;; sign bit instructions only handle single source and target fp registers
;; these instructions can only be used for TFmode values if the source and
;; target operand uses the same fp register.
(define_mode_attr fT0 [(TF "0") (DF "f") (SF "f")])

;; In FP templates, "<Rf>" will expand to "f" in TFmode and "R" otherwise.
;; This is used to disable the memory alternative in TFmode patterns.
(define_mode_attr Rf [(TF "f") (DF "R") (SF "R") (TD "f") (DD "f") (SD "f")])

;; This attribute adds b for bfp instructions and t for dfp instructions and is used
;; within instruction mnemonics.
(define_mode_attr bt [(TF "b") (DF "b") (SF "b") (TD "t") (DD "t") (SD "t")])

;; This attribute is used within instruction mnemonics.  It evaluates to d for dfp
;; modes and to an empty string for bfp modes.
(define_mode_attr _d [(TF "") (DF "") (SF "") (TD "d") (DD "d") (SD "d")])

;; In GPR and P templates, a constraint like "<d0>" will expand to "d" in DImode
;; and "0" in SImode. This allows to combine instructions of which the 31bit
;; version only operates on one register.
(define_mode_attr d0 [(DI "d") (SI "0")])

;; In combination with d0 this allows to combine instructions of which the 31bit
;; version only operates on one register. The DImode version needs an additional
;; register for the assembler output.
(define_mode_attr 1 [(DI "%1,") (SI "")])

;; In SHIFT templates, a string like "s<lr>dl" will expand to "sldl" in
;; 'ashift' and "srdl" in 'lshiftrt'.
(define_code_attr lr [(ashift "l") (lshiftrt "r")])

;; In SHIFT templates, this attribute holds the correct standard name for the
;; pattern itself and the corresponding function calls.
(define_code_attr shift [(ashift "ashl") (lshiftrt "lshr")])

;; This attribute handles differences in the instruction 'type' and will result
;; in "RRE" for DImode and "RR" for SImode.
(define_mode_attr E [(DI "E") (SI "")])

;; This attribute handles differences in the instruction 'type' and makes RX<Y>
;; to result in "RXY" for DImode and "RX" for SImode.
(define_mode_attr Y [(DI "Y") (SI "")])

;; This attribute handles differences in the instruction 'type' and will result
;; in "RSE" for TImode and "RS" for DImode.
(define_mode_attr TE [(TI "E") (DI "")])

;; In GPR templates, a string like "lc<g>r" will expand to "lcgr" in DImode
;; and "lcr" in SImode.
(define_mode_attr g [(DI "g") (SI "")])

;; In GPR templates, a string like "sl<y>" will expand to "slg" in DImode
;; and "sly" in SImode. This is useful because on 64bit the ..g instructions
;; were enhanced with long displacements whereas 31bit instructions got a ..y
;; variant for long displacements.
(define_mode_attr y [(DI "g") (SI "y")])

;; In DW templates, a string like "cds<g>" will expand to "cdsg" in TImode
;; and "cds" in DImode.
(define_mode_attr tg [(TI "g") (DI "")])

;; In TDI templates, a string like "c<d>sg".
(define_mode_attr td [(TI "d") (DI "")])

;; In GPR templates, a string like "c<gf>dbr" will expand to "cgdbr" in DImode
;; and "cfdbr" in SImode.
(define_mode_attr gf [(DI "g") (SI "f")])

;; In GPR templates, a string like sll<gk> will expand to sllg for DI
;; and sllk for SI.  This way it is possible to merge the new z196 SI
;; 3 operands shift instructions into the existing patterns.
(define_mode_attr gk [(DI "g") (SI "k")])

;; ICM mask required to load MODE value into the lowest subreg
;; of a SImode register.
(define_mode_attr icm_lo [(HI "3") (QI "1")])

;; In HQI templates, a string like "llg<hc>" will expand to "llgh" in
;; HImode and "llgc" in QImode.
(define_mode_attr hc [(HI "h") (QI "c")])

;; In P templates, the mode <DBL> will expand to "TI" in DImode and "DI"
;; in SImode.
(define_mode_attr DBL [(DI "TI") (SI "DI")])

;; This attribute expands to DF for TFmode and to DD for TDmode .  It is
;; used for Txmode splitters splitting a Txmode copy into 2 Dxmode copies.
(define_mode_attr HALF_TMODE [(TF "DF") (TD "DD")])

;; Maximum unsigned integer that fits in MODE.
(define_mode_attr max_uint [(HI "65535") (QI "255")])

;; Start and end field computations for RISBG et al.
(define_mode_attr bfstart [(DI "s") (SI "t")])
(define_mode_attr bfend   [(DI "e") (SI "f")])

;; In place of GET_MODE_BITSIZE (<MODE>mode)
(define_mode_attr bitsize [(DI "64") (SI "32") (HI "16") (QI "8")])

;; Allow return and simple_return to be defined from a single template.
(define_code_iterator ANY_RETURN [return simple_return])



; Condition code modes generated by vector fp comparisons.  These will
; be used also in single element mode.
(define_mode_iterator VFCMP [CCVEQ CCVFH CCVFHE])
; Used with VFCMP to expand part of the mnemonic
; For fp we have a mismatch: eq in the insn name - e in asm
(define_mode_attr asm_fcmp [(CCVEQ "e") (CCVFH "h") (CCVFHE "he")])
(define_mode_attr insn_cmp [(CCVEQ "eq") (CCVH "h") (CCVHU "hl") (CCVFH "h") (CCVFHE "he")])


(include "vector.md")

;;
;;- Compare instructions.
;;

; Test-under-Mask instructions

(define_insn "*tmqi_mem"
  [(set (reg CC_REGNUM)
        (compare (and:QI (match_operand:QI 0 "memory_operand" "Q,S")
                         (match_operand:QI 1 "immediate_operand" "n,n"))
                 (match_operand:QI 2 "immediate_operand" "n,n")))]
  "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], false))"
  "@
   tm\t%S0,%b1
   tmy\t%S0,%b1"
  [(set_attr "op_type" "SI,SIY")
   (set_attr "z10prop" "z10_super,z10_super")])

(define_insn "*tmdi_reg"
  [(set (reg CC_REGNUM)
        (compare (and:DI (match_operand:DI 0 "nonimmediate_operand" "d,d,d,d")
                         (match_operand:DI 1 "immediate_operand"
                                             "N0HD0,N1HD0,N2HD0,N3HD0"))
                 (match_operand:DI 2 "immediate_operand" "n,n,n,n")))]
  "TARGET_ZARCH
   && s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], true))
   && s390_single_part (operands[1], DImode, HImode, 0) >= 0"
  "@
   tmhh\t%0,%i1
   tmhl\t%0,%i1
   tmlh\t%0,%i1
   tmll\t%0,%i1"
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super,z10_super,z10_super,z10_super")])

(define_insn "*tmsi_reg"
  [(set (reg CC_REGNUM)
        (compare (and:SI (match_operand:SI 0 "nonimmediate_operand" "d,d")
                         (match_operand:SI 1 "immediate_operand" "N0HS0,N1HS0"))
                 (match_operand:SI 2 "immediate_operand" "n,n")))]
  "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], true))
   && s390_single_part (operands[1], SImode, HImode, 0) >= 0"
  "@
   tmh\t%0,%i1
   tml\t%0,%i1"
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super,z10_super")])

(define_insn "*tm<mode>_full"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "register_operand" "d")
                 (match_operand:HQI 1 "immediate_operand" "n")))]
  "s390_match_ccmode (insn, s390_tm_ccmode (constm1_rtx, operands[1], true))"
  "tml\t%0,<max_uint>"
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super")])


;
; Load-and-Test instructions
;

; tst(di|si) instruction pattern(s).

(define_insn "*tstdi_sign"
  [(set (reg CC_REGNUM)
        (compare
          (ashiftrt:DI
            (ashift:DI
              (subreg:DI (match_operand:SI 0 "nonimmediate_operand" "d,RT") 0)
              (const_int 32)) (const_int 32))
          (match_operand:DI 1 "const0_operand" "")))
   (set (match_operand:DI 2 "register_operand" "=d,d")
        (sign_extend:DI (match_dup 0)))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_ZARCH"
  "ltgfr\t%2,%0
   ltgf\t%2,%0"
  [(set_attr "op_type"      "RRE,RXY")
   (set_attr "cpu_facility" "*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1") ])

; ltr, lt, ltgr, ltg
(define_insn "*tst<mode>_extimm"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "nonimmediate_operand" "d,RT")
                 (match_operand:GPR 1 "const0_operand" "")))
   (set (match_operand:GPR 2 "register_operand" "=d,d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
  "@
   lt<g>r\t%2,%0
   lt<g>\t%2,%0"
  [(set_attr "op_type" "RR<E>,RXY")
   (set_attr "z10prop" "z10_fr_E1,z10_fwd_A3") ])

; ltr, lt, ltgr, ltg
(define_insn "*tst<mode>_cconly_extimm"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "nonimmediate_operand" "d,RT")
                 (match_operand:GPR 1 "const0_operand" "")))
   (clobber (match_scratch:GPR 2 "=X,d"))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
  "@
   lt<g>r\t%0,%0
   lt<g>\t%2,%0"
  [(set_attr "op_type" "RR<E>,RXY")
   (set_attr "z10prop" "z10_fr_E1,z10_fwd_A3")])

(define_insn "*tstdi"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "register_operand" "d")
                 (match_operand:DI 1 "const0_operand" "")))
   (set (match_operand:DI 2 "register_operand" "=d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_ZARCH && !TARGET_EXTIMM"
  "ltgr\t%2,%0"
  [(set_attr "op_type" "RRE")
   (set_attr "z10prop" "z10_fr_E1")])

(define_insn "*tstsi"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
                 (match_operand:SI 1 "const0_operand" "")))
   (set (match_operand:SI 2 "register_operand" "=d,d,d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode) && !TARGET_EXTIMM"
  "@
   ltr\t%2,%0
   icm\t%2,15,%S0
   icmy\t%2,15,%S0"
  [(set_attr "op_type" "RR,RS,RSY")
   (set_attr "z10prop" "z10_fr_E1,z10_super_E1,z10_super_E1")])

(define_insn "*tstsi_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
                 (match_operand:SI 1 "const0_operand" "")))
   (clobber (match_scratch:SI 2 "=X,d,d"))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   ltr\t%0,%0
   icm\t%2,15,%S0
   icmy\t%2,15,%S0"
  [(set_attr "op_type" "RR,RS,RSY")
   (set_attr "z10prop" "z10_fr_E1,z10_super_E1,z10_super_E1")])

(define_insn "*tstdi_cconly_31"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "register_operand" "d")
                 (match_operand:DI 1 "const0_operand" "")))]
  "s390_match_ccmode(insn, CCSmode) && !TARGET_ZARCH"
  "srda\t%0,0"
  [(set_attr "op_type" "RS")
   (set_attr "atype"   "reg")])

; ltr, ltgr
(define_insn "*tst<mode>_cconly2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "register_operand" "d")
                 (match_operand:GPR 1 "const0_operand" "")))]
  "s390_match_ccmode(insn, CCSmode)"
  "lt<g>r\t%0,%0"
  [(set_attr "op_type" "RR<E>")
   (set_attr "z10prop" "z10_fr_E1")])

; tst(hi|qi) instruction pattern(s).

(define_insn "*tst<mode>CCT"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "nonimmediate_operand" "?Q,?S,d")
                 (match_operand:HQI 1 "const0_operand" "")))
   (set (match_operand:HQI 2 "register_operand" "=d,d,0")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   icm\t%2,<icm_lo>,%S0
   icmy\t%2,<icm_lo>,%S0
   tml\t%0,<max_uint>"
  [(set_attr "op_type" "RS,RSY,RI")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super")])

(define_insn "*tsthiCCT_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HI 0 "nonimmediate_operand" "Q,S,d")
                 (match_operand:HI 1 "const0_operand" "")))
   (clobber (match_scratch:HI 2 "=d,d,X"))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   icm\t%2,3,%S0
   icmy\t%2,3,%S0
   tml\t%0,65535"
  [(set_attr "op_type" "RS,RSY,RI")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super")])

(define_insn "*tstqiCCT_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:QI 0 "nonimmediate_operand" "?Q,?S,d")
                 (match_operand:QI 1 "const0_operand" "")))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   cli\t%S0,0
   cliy\t%S0,0
   tml\t%0,255"
  [(set_attr "op_type" "SI,SIY,RI")
   (set_attr "z10prop" "z10_super,z10_super,z10_super")])

(define_insn "*tst<mode>"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "s_operand" "Q,S")
                 (match_operand:HQI 1 "const0_operand" "")))
   (set (match_operand:HQI 2 "register_operand" "=d,d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   icm\t%2,<icm_lo>,%S0
   icmy\t%2,<icm_lo>,%S0"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*tst<mode>_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "s_operand" "Q,S")
                 (match_operand:HQI 1 "const0_operand" "")))
   (clobber (match_scratch:HQI 2 "=d,d"))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   icm\t%2,<icm_lo>,%S0
   icmy\t%2,<icm_lo>,%S0"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])


; Compare (equality) instructions

(define_insn "*cmpdi_cct"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "nonimmediate_operand" "%d,d,d,d,Q")
                 (match_operand:DI 1 "general_operand" "d,K,Os,RT,BQ")))]
  "s390_match_ccmode (insn, CCTmode) && TARGET_ZARCH"
  "@
   cgr\t%0,%1
   cghi\t%0,%h1
   cgfi\t%0,%1
   cg\t%0,%1
   #"
  [(set_attr "op_type" "RRE,RI,RIL,RXY,SS")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,*")])

(define_insn "*cmpsi_cct"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "%d,d,d,d,d,Q")
                 (match_operand:SI 1 "general_operand" "d,K,Os,R,T,BQ")))]
  "s390_match_ccmode (insn, CCTmode)"
  "@
   cr\t%0,%1
   chi\t%0,%h1
   cfi\t%0,%1
   c\t%0,%1
   cy\t%0,%1
   #"
  [(set_attr "op_type" "RR,RI,RIL,RX,RXY,SS")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,*")])

; Compare (signed) instructions

(define_insn "*cmpdi_ccs_sign"
  [(set (reg CC_REGNUM)
        (compare (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand"
                                                     "d,RT,b"))
                 (match_operand:DI 0 "register_operand" "d, d,d")))]
  "s390_match_ccmode(insn, CCSRmode) && TARGET_ZARCH"
  "@
   cgfr\t%0,%1
   cgf\t%0,%1
   cgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "z10prop" "z10_c,*,*")
   (set_attr "type"         "*,*,larl")])



(define_insn "*cmpsi_ccs_sign"
  [(set (reg CC_REGNUM)
        (compare (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T,b"))
                 (match_operand:SI 0 "register_operand" "d,d,d")))]
  "s390_match_ccmode(insn, CCSRmode)"
  "@
   ch\t%0,%1
   chy\t%0,%1
   chrl\t%0,%1"
  [(set_attr "op_type"      "RX,RXY,RIL")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "type"         "*,*,larl")
   (set_attr "z196prop" "z196_cracked,z196_cracked,z196_cracked")])

(define_insn "*cmphi_ccs_z10"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HI 0 "s_operand"         "Q")
                 (match_operand:HI 1 "immediate_operand" "K")))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_Z10"
  "chhsi\t%0,%1"
  [(set_attr "op_type" "SIL")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*cmpdi_ccs_signhi_rl"
  [(set (reg CC_REGNUM)
        (compare (sign_extend:DI (match_operand:HI 1 "memory_operand" "RT,b"))
                 (match_operand:GPR 0 "register_operand"  "d,d")))]
  "s390_match_ccmode(insn, CCSRmode) && TARGET_Z10"
  "@
   cgh\t%0,%1
   cghrl\t%0,%1"
  [(set_attr "op_type" "RXY,RIL")
   (set_attr "type"    "*,larl")])

; cr, chi, cfi, c, cy, cgr, cghi, cgfi, cg, chsi, cghsi, crl, cgrl
(define_insn "*cmp<mode>_ccs"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "nonimmediate_operand"
                                      "d,d,Q, d,d,d,d")
                 (match_operand:GPR 1 "general_operand"
                                      "d,K,K,Os,R,T,b")))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   c<g>r\t%0,%1
   c<g>hi\t%0,%h1
   c<g>hsi\t%0,%h1
   c<g>fi\t%0,%1
   c<g>\t%0,%1
   c<y>\t%0,%1
   c<g>rl\t%0,%1"
  [(set_attr "op_type" "RR<E>,RI,SIL,RIL,RX<Y>,RXY,RIL")
   (set_attr "cpu_facility" "*,*,z10,extimm,*,*,z10")
   (set_attr "type" "*,*,*,*,*,*,larl")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,z10_super,z10_super")])


; Compare (unsigned) instructions

(define_insn "*cmpsi_ccu_zerohi_rlsi"
  [(set (reg CC_REGNUM)
        (compare (zero_extend:SI (mem:HI (match_operand:SI 1
                                          "larl_operand" "X")))
                 (match_operand:SI 0 "register_operand" "d")))]
  "s390_match_ccmode(insn, CCURmode) && TARGET_Z10"
  "clhrl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_super")])

; clhrl, clghrl
(define_insn "*cmp<GPR:mode>_ccu_zerohi_rldi"
  [(set (reg CC_REGNUM)
        (compare (zero_extend:GPR (mem:HI (match_operand:DI 1
                                          "larl_operand" "X")))
                 (match_operand:GPR 0 "register_operand" "d")))]
  "s390_match_ccmode(insn, CCURmode) && TARGET_Z10"
  "cl<g>hrl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_super")])

(define_insn "*cmpdi_ccu_zero"
  [(set (reg CC_REGNUM)
        (compare (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand"
                                                        "d,RT,b"))
                 (match_operand:DI 0 "register_operand" "d, d,d")))]
  "s390_match_ccmode (insn, CCURmode) && TARGET_ZARCH"
  "@
   clgfr\t%0,%1
   clgf\t%0,%1
   clgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "type"         "*,*,larl")
   (set_attr "z10prop" "z10_super_c,z10_super_E1,z10_super")])

(define_insn "*cmpdi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "nonimmediate_operand"
                                     "d, d,d,Q, d, Q,BQ")
                 (match_operand:DI 1 "general_operand"
                                     "d,Op,b,D,RT,BQ,Q")))]
  "s390_match_ccmode (insn, CCUmode) && TARGET_ZARCH"
  "@
   clgr\t%0,%1
   clgfi\t%0,%1
   clgrl\t%0,%1
   clghsi\t%0,%x1
   clg\t%0,%1
   #
   #"
  [(set_attr "op_type" "RRE,RIL,RIL,SIL,RXY,SS,SS")
   (set_attr "cpu_facility" "*,extimm,z10,z10,*,*,*")
   (set_attr "type"         "*,*,larl,*,*,*,*")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,*,*")])

(define_insn "*cmpsi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "d, d,d,Q,d,d, Q,BQ")
                 (match_operand:SI 1 "general_operand"      "d,Os,b,D,R,T,BQ, Q")))]
  "s390_match_ccmode (insn, CCUmode)"
  "@
   clr\t%0,%1
   clfi\t%0,%o1
   clrl\t%0,%1
   clfhsi\t%0,%x1
   cl\t%0,%1
   cly\t%0,%1
   #
   #"
  [(set_attr "op_type" "RR,RIL,RIL,SIL,RX,RXY,SS,SS")
   (set_attr "cpu_facility" "*,extimm,z10,z10,*,*,*,*")
   (set_attr "type"         "*,*,larl,*,*,*,*,*")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,z10_super,*,*")])

(define_insn "*cmphi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HI 0 "nonimmediate_operand" "d,d,Q,Q,BQ")
                 (match_operand:HI 1 "general_operand"      "Q,S,D,BQ,Q")))]
  "s390_match_ccmode (insn, CCUmode)
   && !register_operand (operands[1], HImode)"
  "@
   clm\t%0,3,%S1
   clmy\t%0,3,%S1
   clhhsi\t%0,%1
   #
   #"
  [(set_attr "op_type" "RS,RSY,SIL,SS,SS")
   (set_attr "cpu_facility" "*,*,z10,*,*")
   (set_attr "z10prop" "*,*,z10_super,*,*")])

(define_insn "*cmpqi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:QI 0 "nonimmediate_operand" "d,d,Q,S,Q,BQ")
                 (match_operand:QI 1 "general_operand" "Q,S,n,n,BQ,Q")))]
  "s390_match_ccmode (insn, CCUmode)
   && !register_operand (operands[1], QImode)"
  "@
   clm\t%0,1,%S1
   clmy\t%0,1,%S1
   cli\t%S0,%b1
   cliy\t%S0,%b1
   #
   #"
  [(set_attr "op_type" "RS,RSY,SI,SIY,SS,SS")
   (set_attr "z10prop" "*,*,z10_super,z10_super,*,*")])


; Block compare (CLC) instruction patterns.

(define_insn "*clc"
  [(set (reg CC_REGNUM)
        (compare (match_operand:BLK 0 "memory_operand" "Q")
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))]
  "s390_match_ccmode (insn, CCUmode)
   && INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "clc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")])

(define_split
  [(set (reg CC_REGNUM)
        (compare (match_operand 0 "memory_operand" "")
                 (match_operand 1 "memory_operand" "")))]
  "reload_completed
   && s390_match_ccmode (insn, CCUmode)
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (match_dup 2))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);

  operands[1] = gen_rtx_COMPARE (GET_MODE (SET_DEST (PATTERN (curr_insn))),
                                 operands[0], operands[1]);
  operands[0] = SET_DEST (PATTERN (curr_insn));
})


; (TF|DF|SF|TD|DD|SD) instructions

; ltxbr, ltdbr, ltebr, ltxtr, ltdtr
(define_insn "*cmp<mode>_ccs_0"
  [(set (reg CC_REGNUM)
        (compare (match_operand:FP 0 "register_operand" "f")
                 (match_operand:FP 1 "const0_operand"   "")))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT"
  "lt<xde><bt>r\t%0,%0"
   [(set_attr "op_type" "RRE")
    (set_attr "type"  "fsimp<mode>")])

; cxtr, cxbr, cdtr, cdbr, cebr, cdb, ceb
(define_insn "*cmp<mode>_ccs"
  [(set (reg CC_REGNUM)
        (compare (match_operand:FP 0 "register_operand" "f,f")
                 (match_operand:FP 1 "general_operand"  "f,<Rf>")))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   c<xde><bt>r\t%0,%1
   c<xde>b\t%0,%1"
   [(set_attr "op_type" "RRE,RXE")
    (set_attr "type"  "fsimp<mode>")])

; wfcedbs, wfchdbs, wfchedbs
(define_insn "*vec_cmp<insn_cmp>df_cconly"
  [(set (reg:VFCMP CC_REGNUM)
        (compare:VFCMP (match_operand:DF 0 "register_operand" "v")
                       (match_operand:DF 1 "register_operand" "v")))
   (clobber (match_scratch:V2DI 2 "=v"))]
  "TARGET_Z13 && TARGET_HARD_FLOAT"
  "wfc<asm_fcmp>dbs\t%v2,%v0,%v1"
  [(set_attr "op_type" "VRR")])

; Compare and Branch instructions

; cij, cgij, crj, cgrj, cfi, cgfi, cr, cgr
; The following instructions do a complementary access of their second
; operand (z01 only): crj_c, cgrjc, cr, cgr
(define_insn "*cmp_and_br_signed_<mode>"
  [(set (pc)
        (if_then_else (match_operator 0 "s390_signed_integer_comparison"
                        [(match_operand:GPR 1 "register_operand"  "d,d")
                         (match_operand:GPR 2 "nonmemory_operand" "d,C")])
                      (label_ref (match_operand 3 "" ""))
                      (pc)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "c<g>ij%C0\t%1,%c2,%l3" : "c<g>rj%C0\t%1,%2,%l3";
  else
    return which_alternative ?
      "c<g>fi\t%1,%c2\;jg%C0\t%l3" : "c<g>r\t%1,%2\;jg%C0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for cr/jg
                                                       ; 10 byte for cgr/jg

; clij, clgij, clrj, clgrj, clfi, clgfi, clr, clgr
; The following instructions do a complementary access of their second
; operand (z10 only): clrj, clgrj, clr, clgr
(define_insn "*cmp_and_br_unsigned_<mode>"
  [(set (pc)
        (if_then_else (match_operator 0 "s390_unsigned_integer_comparison"
                        [(match_operand:GPR 1 "register_operand"  "d,d")
                         (match_operand:GPR 2 "nonmemory_operand" "d,I")])
                      (label_ref (match_operand 3 "" ""))
                      (pc)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "cl<g>ij%C0\t%1,%b2,%l3" : "cl<g>rj%C0\t%1,%2,%l3";
  else
    return which_alternative ?
      "cl<g>fi\t%1,%b2\;jg%C0\t%l3" : "cl<g>r\t%1,%2\;jg%C0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for clr/jg
                                                       ; 10 byte for clgr/jg

; And now the same two patterns as above but with a negated CC mask.

; cij, cgij, crj, cgrj, cfi, cgfi, cr, cgr
; The following instructions do a complementary access of their second
; operand (z01 only): crj_c, cgrjc, cr, cgr
(define_insn "*icmp_and_br_signed_<mode>"
  [(set (pc)
        (if_then_else (match_operator 0 "s390_signed_integer_comparison"
                        [(match_operand:GPR 1 "register_operand"  "d,d")
                         (match_operand:GPR 2 "nonmemory_operand" "d,C")])
                      (pc)
                      (label_ref (match_operand 3 "" ""))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "c<g>ij%D0\t%1,%c2,%l3" : "c<g>rj%D0\t%1,%2,%l3";
  else
    return which_alternative ?
      "c<g>fi\t%1,%c2\;jg%D0\t%l3" : "c<g>r\t%1,%2\;jg%D0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for cr/jg
                                                       ; 10 byte for cgr/jg

; clij, clgij, clrj, clgrj, clfi, clgfi, clr, clgr
; The following instructions do a complementary access of their second
; operand (z10 only): clrj, clgrj, clr, clgr
(define_insn "*icmp_and_br_unsigned_<mode>"
  [(set (pc)
        (if_then_else (match_operator 0 "s390_unsigned_integer_comparison"
                        [(match_operand:GPR 1 "register_operand"  "d,d")
                         (match_operand:GPR 2 "nonmemory_operand" "d,I")])
                      (pc)
                      (label_ref (match_operand 3 "" ""))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "cl<g>ij%D0\t%1,%b2,%l3" : "cl<g>rj%D0\t%1,%2,%l3";
  else
    return which_alternative ?
      "cl<g>fi\t%1,%b2\;jg%D0\t%l3" : "cl<g>r\t%1,%2\;jg%D0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for clr/jg
                                                       ; 10 byte for clgr/jg

;;
;;- Move instructions.
;;

;
; movti instruction pattern(s).
;

; FIXME: More constants are possible by enabling jxx, jyy constraints
; for TImode (use double-int for the calculations)
(define_insn "movti"
  [(set (match_operand:TI 0 "nonimmediate_operand" "=d,QS,v,  v,  v,v,d, v,QR,   d,o")
        (match_operand:TI 1 "general_operand"      "QS, d,v,j00,jm1,d,v,QR, v,dPRT,d"))]
  "TARGET_ZARCH"
  "@
   lmg\t%0,%N0,%S1
   stmg\t%1,%N1,%S0
   vlr\t%v0,%v1
   vzero\t%v0
   vone\t%v0
   vlvgp\t%v0,%1,%N1
   #
   vl\t%v0,%1
   vst\t%v1,%0
   #
   #"
  [(set_attr "op_type" "RSY,RSY,VRR,VRI,VRI,VRR,*,VRX,VRX,*,*")
   (set_attr "type" "lm,stm,*,*,*,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,vec,vec,vec,vec,vec,vec,vec,*,*")])

(define_split
  [(set (match_operand:TI 0 "nonimmediate_operand" "")
        (match_operand:TI 1 "general_operand" ""))]
  "TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], TImode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, TImode);
  operands[3] = operand_subword (operands[0], 1, 0, TImode);
  operands[4] = operand_subword (operands[1], 0, 0, TImode);
  operands[5] = operand_subword (operands[1], 1, 0, TImode);
})

(define_split
  [(set (match_operand:TI 0 "nonimmediate_operand" "")
        (match_operand:TI 1 "general_operand" ""))]
  "TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], TImode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, TImode);
  operands[3] = operand_subword (operands[0], 0, 0, TImode);
  operands[4] = operand_subword (operands[1], 1, 0, TImode);
  operands[5] = operand_subword (operands[1], 0, 0, TImode);
})

; Use part of the TImode target reg to perform the address
; calculation.  If the TImode value is supposed to be copied into a VR
; this splitter is not necessary.
(define_split
  [(set (match_operand:TI 0 "register_operand" "")
        (match_operand:TI 1 "memory_operand" ""))]
  "TARGET_ZARCH && reload_completed
   && !VECTOR_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, TImode);
  addr = gen_lowpart (Pmode, addr);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})


; Split a VR -> GPR TImode move into 2 vector load GR from VR element.
; For the higher order bits we do simply a DImode move while the
; second part is done via vec extract.  Both will end up as vlgvg.
(define_split
  [(set (match_operand:TI 0 "register_operand" "")
        (match_operand:TI 1 "register_operand" ""))]
  "TARGET_VX && reload_completed
   && GENERAL_REG_P (operands[0])
   && VECTOR_REG_P (operands[1])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (unspec:DI [(match_dup 5) (const_int 1)]
                                 UNSPEC_VEC_EXTRACT))]
{
  operands[2] = operand_subword (operands[0], 0, 0, TImode);
  operands[3] = operand_subword (operands[0], 1, 0, TImode);
  operands[4] = gen_rtx_REG (DImode, REGNO (operands[1]));
  operands[5] = gen_rtx_REG (V2DImode, REGNO (operands[1]));
})

;
; Patterns used for secondary reloads
;

; z10 provides move instructions accepting larl memory operands.
; Unfortunately there is no such variant for QI, TI and FP mode moves.
; These patterns are also used for unaligned SI and DI accesses.

(define_expand "reload<ALL:mode><P:mode>_tomem_z10"
  [(parallel [(match_operand:ALL 0 "memory_operand"   "")
              (match_operand:ALL 1 "register_operand" "=d")
              (match_operand:P   2 "register_operand" "=&a")])]
  "TARGET_Z10"
{
  s390_reload_symref_address (operands[1], operands[0], operands[2], 1);
  DONE;
})

(define_expand "reload<ALL:mode><P:mode>_toreg_z10"
  [(parallel [(match_operand:ALL 0 "register_operand" "=d")
              (match_operand:ALL 1 "memory_operand"   "")
              (match_operand:P   2 "register_operand" "=a")])]
  "TARGET_Z10"
{
  s390_reload_symref_address (operands[0], operands[1], operands[2], 0);
  DONE;
})

(define_expand "reload<P:mode>_larl_odd_addend_z10"
  [(parallel [(match_operand:P 0 "register_operand" "=d")
              (match_operand:P 1 "larl_operand"     "")
              (match_operand:P 2 "register_operand" "=a")])]
  "TARGET_Z10"
{
  s390_reload_larl_operand (operands[0], operands[1], operands[2]);
  DONE;
})

; Handles loading a PLUS (load address) expression

(define_expand "reload<mode>_plus"
  [(parallel [(match_operand:P 0 "register_operand"  "=a")
              (match_operand:P 1 "s390_plus_operand" "")
              (match_operand:P 2 "register_operand"  "=&a")])]
  ""
{
  s390_expand_plus_operand (operands[0], operands[1], operands[2]);
  DONE;
})

; Not all the indirect memory access instructions support the full
; format (long disp + index + base).  So whenever a move from/to such
; an address is required and the instruction cannot deal with it we do
; a load address into a scratch register first and use this as the new
; base register.
; This in particular is used for:
; - non-offsetable memory accesses for multiword moves
; - full vector reg moves with long displacements

(define_expand "reload<mode>_la_in"
  [(parallel [(match_operand 0   "register_operand" "")
              (match_operand 1   "" "")
              (match_operand:P 2 "register_operand" "=&a")])]
  ""
{
  gcc_assert (MEM_P (operands[1]));
  s390_load_address (operands[2], find_replacement (&XEXP (operands[1], 0)));
  operands[1] = replace_equiv_address (operands[1], operands[2]);
  emit_move_insn (operands[0], operands[1]);
  DONE;
})

(define_expand "reload<mode>_la_out"
  [(parallel [(match_operand   0 "" "")
              (match_operand   1 "register_operand" "")
              (match_operand:P 2 "register_operand" "=&a")])]
  ""
{
  gcc_assert (MEM_P (operands[0]));
  s390_load_address (operands[2], find_replacement (&XEXP (operands[0], 0)));
  operands[0] = replace_equiv_address (operands[0], operands[2]);
  emit_move_insn (operands[0], operands[1]);
  DONE;
})

(define_expand "reload<mode>_PIC_addr"
  [(parallel [(match_operand   0 "register_operand" "=d")
              (match_operand   1 "larl_operand"     "")
              (match_operand:P 2 "register_operand" "=a")])]
  ""
{
  rtx new_rtx = legitimize_pic_address (operands[1], operands[2]);
  emit_move_insn (operands[0], new_rtx);
})

;
; movdi instruction pattern(s).
;

(define_expand "movdi"
  [(set (match_operand:DI 0 "general_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  ""
{
  /* Handle symbolic constants.  */
  if (TARGET_64BIT
      && (SYMBOLIC_CONST (operands[1])
          || (GET_CODE (operands[1]) == PLUS
              && XEXP (operands[1], 0) == pic_offset_table_rtx
              && SYMBOLIC_CONST (XEXP (operands[1], 1)))))
    emit_symbolic_move (operands);
})

(define_insn "*movdi_larl"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (match_operand:DI 1 "larl_operand" "X"))]
  "TARGET_64BIT
   && !FP_REG_P (operands[0])"
  "larl\t%0,%1"
   [(set_attr "op_type" "RIL")
    (set_attr "type"    "larl")
    (set_attr "z10prop" "z10_super_A1")])

(define_insn "*movdi_64"
  [(set (match_operand:DI 0 "nonimmediate_operand"
         "=d,    d,    d,    d,    d, d,    d,    d,f,d,d,d,d, d,RT,!*f,!*f,!*f,!R,!T,b,Q,d,t,Q,t,v,v,v,d, v,QR")
        (match_operand:DI 1 "general_operand"
         " K,N0HD0,N1HD0,N2HD0,N3HD0,Os,N0SD0,N1SD0,d,f,L,b,d,RT, d, *f,  R,  T,*f,*f,d,K,t,d,t,Q,K,v,d,v,QR, v"))]
  "TARGET_ZARCH"
  "@
   lghi\t%0,%h1
   llihh\t%0,%i1
   llihl\t%0,%i1
   llilh\t%0,%i1
   llill\t%0,%i1
   lgfi\t%0,%1
   llihf\t%0,%k1
   llilf\t%0,%k1
   ldgr\t%0,%1
   lgdr\t%0,%1
   lay\t%0,%a1
   lgrl\t%0,%1
   lgr\t%0,%1
   lg\t%0,%1
   stg\t%1,%0
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   stgrl\t%1,%0
   mvghi\t%0,%1
   #
   #
   stam\t%1,%N1,%S0
   lam\t%0,%N0,%S1
   vleig\t%v0,%h1,0
   vlr\t%v0,%v1
   vlvgg\t%v0,%1,0
   vlgvg\t%0,%v1,0
   vleg\t%v0,%1,0
   vsteg\t%v1,%0,0"
  [(set_attr "op_type" "RI,RI,RI,RI,RI,RIL,RIL,RIL,RRE,RRE,RXY,RIL,RRE,RXY,
                        RXY,RR,RX,RXY,RX,RXY,RIL,SIL,*,*,RS,RS,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type" "*,*,*,*,*,*,*,*,floaddf,floaddf,la,larl,lr,load,store,
                     floaddf,floaddf,floaddf,fstoredf,fstoredf,larl,*,*,*,*,
                     *,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,*,*,extimm,extimm,extimm,dfp,dfp,longdisp,
                             z10,*,*,*,*,*,longdisp,*,longdisp,
                             z10,z10,*,*,*,*,vec,vec,vec,vec,vec,vec")
   (set_attr "z10prop" "z10_fwd_A1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_A1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        *,
                        *,
                        z10_fwd_A1,
                        z10_fwd_A3,
                        z10_fr_E1,
                        z10_fwd_A3,
                        z10_rec,
                        *,
                        *,
                        *,
                        *,
                        *,
                        z10_rec,
                        z10_super,
                        *,
                        *,
                        *,
                        *,*,*,*,*,*,*")
])

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))]
  "TARGET_ZARCH && ACCESS_REG_P (operands[1])"
  [(set (match_dup 2) (match_dup 3))
   (set (match_dup 0) (ashift:DI (match_dup 0) (const_int 32)))
   (set (strict_low_part (match_dup 2)) (match_dup 4))]
  "operands[2] = gen_lowpart (SImode, operands[0]);
   s390_split_access_reg (operands[1], &operands[4], &operands[3]);")

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))]
  "TARGET_ZARCH && ACCESS_REG_P (operands[0])
   && dead_or_set_p (insn, operands[1])"
  [(set (match_dup 3) (match_dup 2))
   (set (match_dup 1) (lshiftrt:DI (match_dup 1) (const_int 32)))
   (set (match_dup 4) (match_dup 2))]
  "operands[2] = gen_lowpart (SImode, operands[1]);
   s390_split_access_reg (operands[0], &operands[3], &operands[4]);")

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))]
  "TARGET_ZARCH && ACCESS_REG_P (operands[0])
   && !dead_or_set_p (insn, operands[1])"
  [(set (match_dup 3) (match_dup 2))
   (set (match_dup 1) (rotate:DI (match_dup 1) (const_int 32)))
   (set (match_dup 4) (match_dup 2))
   (set (match_dup 1) (rotate:DI (match_dup 1) (const_int 32)))]
  "operands[2] = gen_lowpart (SImode, operands[1]);
   s390_split_access_reg (operands[0], &operands[3], &operands[4]);")

(define_insn "*movdi_31"
  [(set (match_operand:DI 0 "nonimmediate_operand"
                            "=d,d,Q,S,d   ,o,!*f,!*f,!*f,!R,!T,d")
        (match_operand:DI 1 "general_operand"
                            " Q,S,d,d,dPRT,d, *f,  R,  T,*f,*f,b"))]
  "!TARGET_ZARCH"
  "@
   lm\t%0,%N0,%S1
   lmy\t%0,%N0,%S1
   stm\t%1,%N1,%S0
   stmy\t%1,%N1,%S0
   #
   #
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   #"
  [(set_attr "op_type" "RS,RSY,RS,RSY,*,*,RR,RX,RXY,RX,RXY,*")
   (set_attr "type" "lm,lm,stm,stm,*,*,floaddf,floaddf,floaddf,fstoredf,fstoredf,*")
   (set_attr "cpu_facility" "*,*,*,*,*,*,*,*,*,*,*,z10")])

; For a load from a symbol ref we can use one of the target registers
; together with larl to load the address.
(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "memory_operand" ""))]
  "!TARGET_ZARCH && reload_completed && TARGET_Z10
   && larl_operand (XEXP (operands[1], 0), SImode)"
  [(set (match_dup 2) (match_dup 3))
   (set (match_dup 0) (match_dup 1))]
{
  operands[2] = operand_subword (operands[0], 1, 0, DImode);
  operands[3] = XEXP (operands[1], 0);
  operands[1] = replace_equiv_address (operands[1], operands[2]);
})

(define_split
  [(set (match_operand:DI 0 "nonimmediate_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], DImode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, DImode);
  operands[3] = operand_subword (operands[0], 1, 0, DImode);
  operands[4] = operand_subword (operands[1], 0, 0, DImode);
  operands[5] = operand_subword (operands[1], 1, 0, DImode);
})

(define_split
  [(set (match_operand:DI 0 "nonimmediate_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], DImode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, DImode);
  operands[3] = operand_subword (operands[0], 0, 0, DImode);
  operands[4] = operand_subword (operands[1], 1, 0, DImode);
  operands[5] = operand_subword (operands[1], 0, 0, DImode);
})

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "memory_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !FP_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, DImode);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})

(define_peephole2
  [(set (match_operand:DI 0 "register_operand" "")
        (mem:DI (match_operand 1 "address_operand" "")))]
  "TARGET_ZARCH
   && !FP_REG_P (operands[0])
   && GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == DImode
   && legitimate_reload_constant_p (get_pool_constant (operands[1]))"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

(define_insn "*la_64"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (match_operand:QI 1 "address_operand" "ZQZR,ZSZT"))]
  "TARGET_64BIT"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type" "RX,RXY")
   (set_attr "type"    "la")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

(define_peephole2
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
          (match_operand:QI 1 "address_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_64BIT
   && preferred_la_operand_p (operands[1], const0_rtx)"
  [(set (match_dup 0) (match_dup 1))]
  "")

(define_peephole2
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))
   (parallel
    [(set (match_dup 0)
          (plus:DI (match_dup 0)
                   (match_operand:DI 2 "nonmemory_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_64BIT
   && !reg_overlap_mentioned_p (operands[0], operands[2])
   && preferred_la_operand_p (operands[1], operands[2])"
  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))]
  "")

;
; movsi instruction pattern(s).
;

(define_expand "movsi"
  [(set (match_operand:SI 0 "general_operand" "")
        (match_operand:SI 1 "general_operand" ""))]
  ""
{
  /* Handle symbolic constants.  */
  if (!TARGET_64BIT
      && (SYMBOLIC_CONST (operands[1])
          || (GET_CODE (operands[1]) == PLUS
              && XEXP (operands[1], 0) == pic_offset_table_rtx
              && SYMBOLIC_CONST (XEXP(operands[1], 1)))))
    emit_symbolic_move (operands);
})

(define_insn "*movsi_larl"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (match_operand:SI 1 "larl_operand" "X"))]
  "!TARGET_64BIT && TARGET_CPU_ZARCH
   && !FP_REG_P (operands[0])"
  "larl\t%0,%1"
   [(set_attr "op_type" "RIL")
    (set_attr "type"    "larl")
    (set_attr "z10prop" "z10_fwd_A1")])

(define_insn "*movsi_zarch"
  [(set (match_operand:SI 0 "nonimmediate_operand"
         "=d,    d,    d, d,d,d,d,d,d,R,T,!*f,!*f,!*f,!*f,!*f,!R,!T,d,t,Q,b,Q,t,v,v,v,d, v,QR")
        (match_operand:SI 1 "general_operand"
         " K,N0HS0,N1HS0,Os,L,b,d,R,T,d,d, *f, *f,  R,  R,  T,*f,*f,t,d,t,d,K,Q,K,v,d,v,QR, v"))]
  "TARGET_ZARCH"
  "@
   lhi\t%0,%h1
   llilh\t%0,%i1
   llill\t%0,%i1
   iilf\t%0,%o1
   lay\t%0,%a1
   lrl\t%0,%1
   lr\t%0,%1
   l\t%0,%1
   ly\t%0,%1
   st\t%1,%0
   sty\t%1,%0
   lder\t%0,%1
   ler\t%0,%1
   lde\t%0,%1
   le\t%0,%1
   ley\t%0,%1
   ste\t%1,%0
   stey\t%1,%0
   ear\t%0,%1
   sar\t%0,%1
   stam\t%1,%1,%S0
   strl\t%1,%0
   mvhi\t%0,%1
   lam\t%0,%0,%S1
   vleif\t%v0,%h1,0
   vlr\t%v0,%v1
   vlvgf\t%v0,%1,0
   vlgvf\t%0,%v1,0
   vlef\t%v0,%1,0
   vstef\t%v1,%0,0"
  [(set_attr "op_type" "RI,RI,RI,RIL,RXY,RIL,RR,RX,RXY,RX,RXY,
                        RRE,RR,RXE,RX,RXY,RX,RXY,RRE,RRE,RS,RIL,SIL,RS,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type" "*,
                     *,
                     *,
                     *,
                     la,
                     larl,
                     lr,
                     load,
                     load,
                     store,
                     store,
                     floadsf,
                     floadsf,
                     floadsf,
                     floadsf,
                     floadsf,
                     fstoresf,
                     fstoresf,
                     *,
                     *,
                     *,
                     larl,
                     *,
                     *,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,extimm,longdisp,z10,*,*,longdisp,*,longdisp,
                             vec,*,vec,*,longdisp,*,longdisp,*,*,*,z10,z10,*,vec,vec,vec,vec,vec,vec")
   (set_attr "z10prop" "z10_fwd_A1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_A1,
                        z10_fwd_A1,
                        z10_fwd_A3,
                        z10_fr_E1,
                        z10_fwd_A3,
                        z10_fwd_A3,
                        z10_rec,
                        z10_rec,
                        *,
                        *,
                        *,
                        *,
                        *,
                        *,
                        *,
                        z10_super_E1,
                        z10_super,
                        *,
                        z10_rec,
                        z10_super,
                        *,*,*,*,*,*,*")])

(define_insn "*movsi_esa"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,d,d,R,!*f,!*f,!*f,!*f,!R,d,t,Q,t")
        (match_operand:SI 1 "general_operand"       "K,d,R,d, *f, *f,  R,  R,*f,t,d,t,Q"))]
  "!TARGET_ZARCH"
  "@
   lhi\t%0,%h1
   lr\t%0,%1
   l\t%0,%1
   st\t%1,%0
   lder\t%0,%1
   ler\t%0,%1
   lde\t%0,%1
   le\t%0,%1
   ste\t%1,%0
   ear\t%0,%1
   sar\t%0,%1
   stam\t%1,%1,%S0
   lam\t%0,%0,%S1"
  [(set_attr "op_type" "RI,RR,RX,RX,RRE,RR,RXE,RX,RX,RRE,RRE,RS,RS")
   (set_attr "type" "*,lr,load,store,floadsf,floadsf,floadsf,floadsf,fstoresf,*,*,*,*")
   (set_attr "z10prop" "z10_fwd_A1,z10_fr_E1,z10_fwd_A3,z10_rec,*,*,*,*,*,z10_super_E1,
                        z10_super,*,*")
   (set_attr "cpu_facility" "*,*,*,*,vec,*,vec,*,*,*,*,*,*")
])

(define_peephole2
  [(set (match_operand:SI 0 "register_operand" "")
        (mem:SI (match_operand 1 "address_operand" "")))]
  "!FP_REG_P (operands[0])
   && GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == SImode
   && legitimate_reload_constant_p (get_pool_constant (operands[1]))"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

(define_insn "*la_31"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (match_operand:QI 1 "address_operand" "ZQZR,ZSZT"))]
  "!TARGET_64BIT && legitimate_la_operand_p (operands[1])"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "type"     "la")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

(define_peephole2
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (match_operand:QI 1 "address_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "!TARGET_64BIT
   && preferred_la_operand_p (operands[1], const0_rtx)"
  [(set (match_dup 0) (match_dup 1))]
  "")

(define_peephole2
  [(set (match_operand:SI 0 "register_operand" "")
        (match_operand:SI 1 "register_operand" ""))
   (parallel
    [(set (match_dup 0)
          (plus:SI (match_dup 0)
                   (match_operand:SI 2 "nonmemory_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  "!TARGET_64BIT
   && !reg_overlap_mentioned_p (operands[0], operands[2])
   && preferred_la_operand_p (operands[1], operands[2])"
  [(set (match_dup 0) (plus:SI (match_dup 1) (match_dup 2)))]
  "")

(define_insn "*la_31_and"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (and:SI (match_operand:QI 1 "address_operand" "ZQZR,ZSZT")
                (const_int 2147483647)))]
  "!TARGET_64BIT"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "type"     "la")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

(define_insn_and_split "*la_31_and_cc"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (and:SI (match_operand:QI 1 "address_operand" "p")
                (const_int 2147483647)))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_64BIT"
  "#"
  "&& reload_completed"
  [(set (match_dup 0)
        (and:SI (match_dup 1) (const_int 2147483647)))]
  ""
  [(set_attr "op_type"  "RX")
   (set_attr "type"     "la")])

(define_insn "force_la_31"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (match_operand:QI 1 "address_operand" "ZQZR,ZSZT"))
   (use (const_int 0))]
  "!TARGET_64BIT"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type"  "RX")
   (set_attr "type"     "la")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

;
; movhi instruction pattern(s).
;

(define_expand "movhi"
  [(set (match_operand:HI 0 "nonimmediate_operand" "")
        (match_operand:HI 1 "general_operand" ""))]
  ""
{
  /* Make it explicit that loading a register from memory
     always sign-extends (at least) to SImode.  */
  if (optimize && can_create_pseudo_p ()
      && register_operand (operands[0], VOIDmode)
      && GET_CODE (operands[1]) == MEM)
    {
      rtx tmp = gen_reg_rtx (SImode);
      rtx ext = gen_rtx_SIGN_EXTEND (SImode, operands[1]);
      emit_insn (gen_rtx_SET (tmp, ext));
      operands[1] = gen_lowpart (HImode, tmp);
    }
})

(define_insn "*movhi"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,d,d,d,d,R,T,b,Q,v,v,v,d, v,QR")
        (match_operand:HI 1 "general_operand"      " d,n,R,T,b,d,d,d,K,K,v,d,v,QR, v"))]
  ""
  "@
   lr\t%0,%1
   lhi\t%0,%h1
   lh\t%0,%1
   lhy\t%0,%1
   lhrl\t%0,%1
   sth\t%1,%0
   sthy\t%1,%0
   sthrl\t%1,%0
   mvhhi\t%0,%1
   vleih\t%v0,%h1,0
   vlr\t%v0,%v1
   vlvgh\t%v0,%1,0
   vlgvh\t%0,%v1,0
   vleh\t%v0,%1,0
   vsteh\t%v1,%0,0"
  [(set_attr "op_type"      "RR,RI,RX,RXY,RIL,RX,RXY,RIL,SIL,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type"         "lr,*,*,*,larl,store,store,store,*,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,*,z10,*,*,z10,z10,vec,vec,vec,vec,vec,vec")
   (set_attr "z10prop" "z10_fr_E1,
                       z10_fwd_A1,
                       z10_super_E1,
                       z10_super_E1,
                       z10_super_E1,
                       z10_rec,
                       z10_rec,
                       z10_rec,
                       z10_super,*,*,*,*,*,*")])

(define_peephole2
  [(set (match_operand:HI 0 "register_operand" "")
        (mem:HI (match_operand 1 "address_operand" "")))]
  "GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == HImode
   && GET_CODE (get_pool_constant (operands[1])) == CONST_INT"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

;
; movqi instruction pattern(s).
;

(define_expand "movqi"
  [(set (match_operand:QI 0 "nonimmediate_operand" "")
        (match_operand:QI 1 "general_operand" ""))]
  ""
{
  /* On z/Architecture, zero-extending from memory to register
     is just as fast as a QImode load.  */
  if (TARGET_ZARCH && optimize && can_create_pseudo_p ()
      && register_operand (operands[0], VOIDmode)
      && GET_CODE (operands[1]) == MEM)
    {
      rtx tmp = gen_reg_rtx (DImode);
      rtx ext = gen_rtx_ZERO_EXTEND (DImode, operands[1]);
      emit_insn (gen_rtx_SET (tmp, ext));
      operands[1] = gen_lowpart (QImode, tmp);
    }
})

(define_insn "*movqi"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,d,d,d,R,T,Q,S,?Q,v,v,v,d, v,QR")
        (match_operand:QI 1 "general_operand"      " d,n,R,T,d,d,n,n,?Q,K,v,d,v,QR, v"))]
  ""
  "@
   lr\t%0,%1
   lhi\t%0,%b1
   ic\t%0,%1
   icy\t%0,%1
   stc\t%1,%0
   stcy\t%1,%0
   mvi\t%S0,%b1
   mviy\t%S0,%b1
   #
   vleib\t%v0,%b1,0
   vlr\t%v0,%v1
   vlvgb\t%v0,%1,0
   vlgvb\t%0,%v1,0
   vleb\t%v0,%1,0
   vsteb\t%v1,%0,0"
  [(set_attr "op_type" "RR,RI,RX,RXY,RX,RXY,SI,SIY,SS,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type" "lr,*,*,*,store,store,store,store,*,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,*,*,*,*,*,*,vec,vec,vec,vec,vec,vec")
   (set_attr "z10prop" "z10_fr_E1,
                        z10_fwd_A1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_rec,
                        z10_rec,
                        z10_super,
                        z10_super,
                        *,*,*,*,*,*,*")])

(define_peephole2
  [(set (match_operand:QI 0 "nonimmediate_operand" "")
        (mem:QI (match_operand 1 "address_operand" "")))]
  "GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == QImode
   && GET_CODE (get_pool_constant (operands[1])) == CONST_INT"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

;
; movstrictqi instruction pattern(s).
;

(define_insn "*movstrictqi"
  [(set (strict_low_part (match_operand:QI 0 "register_operand" "+d,d"))
                         (match_operand:QI 1 "memory_operand" "R,T"))]
  ""
  "@
   ic\t%0,%1
   icy\t%0,%1"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; movstricthi instruction pattern(s).
;

(define_insn "*movstricthi"
  [(set (strict_low_part (match_operand:HI 0 "register_operand" "+d,d"))
                         (match_operand:HI 1 "memory_operand" "Q,S"))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   icm\t%0,3,%S1
   icmy\t%0,3,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; movstrictsi instruction pattern(s).
;

(define_insn "movstrictsi"
  [(set (strict_low_part (match_operand:SI 0 "register_operand" "+d,d,d,d"))
                         (match_operand:SI 1 "general_operand" "d,R,T,t"))]
  "TARGET_ZARCH"
  "@
   lr\t%0,%1
   l\t%0,%1
   ly\t%0,%1
   ear\t%0,%1"
  [(set_attr "op_type" "RR,RX,RXY,RRE")
   (set_attr "type" "lr,load,load,*")
   (set_attr "z10prop" "z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_super_E1")])

;
; mov(tf|td) instruction pattern(s).
;

(define_expand "mov<mode>"
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "")
        (match_operand:TD_TF 1 "general_operand"      ""))]
  ""
  "")

(define_insn "*mov<mode>_64"
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "=f,f,f,o, d,QS,  d,o")
        (match_operand:TD_TF 1 "general_operand"      " G,f,o,f,QS, d,dRT,d"))]
  "TARGET_ZARCH"
  "@
   lzxr\t%0
   lxr\t%0,%1
   #
   #
   lmg\t%0,%N0,%S1
   stmg\t%1,%N1,%S0
   #
   #"
  [(set_attr "op_type"      "RRE,RRE,*,*,RSY,RSY,*,*")
   (set_attr "type"         "fsimptf,fsimptf,*,*,lm,stm,*,*")
   (set_attr "cpu_facility" "z196,*,*,*,*,*,*,*")])

(define_insn "*mov<mode>_31"
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "=f,f,f,o")
        (match_operand:TD_TF 1 "general_operand"      " G,f,o,f"))]
  "!TARGET_ZARCH"
  "@
   lzxr\t%0
   lxr\t%0,%1
   #
   #"
  [(set_attr "op_type"      "RRE,RRE,*,*")
   (set_attr "type"         "fsimptf,fsimptf,*,*")
   (set_attr "cpu_facility" "z196,*,*,*")])

; TFmode in GPRs splitters

(define_split
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "")
        (match_operand:TD_TF 1 "general_operand"      ""))]
  "TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 0, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 1, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "")
        (match_operand:TD_TF 1 "general_operand"      ""))]
  "TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 1, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 0, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:TD_TF 0 "register_operand" "")
        (match_operand:TD_TF 1 "memory_operand"   ""))]
  "TARGET_ZARCH && reload_completed
   && GENERAL_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, <MODE>mode);
  addr = gen_lowpart (Pmode, addr);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})

; TFmode in BFPs splitters

(define_split
  [(set (match_operand:TD_TF 0 "register_operand" "")
        (match_operand:TD_TF 1 "memory_operand" ""))]
  "reload_completed && offsettable_memref_p (operands[1])
   && FP_REG_P (operands[0])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = simplify_gen_subreg (<HALF_TMODE>mode, operands[0],
                                     <MODE>mode, 0);
  operands[3] = simplify_gen_subreg (<HALF_TMODE>mode, operands[0],
                                     <MODE>mode, 8);
  operands[4] = adjust_address_nv (operands[1], <HALF_TMODE>mode, 0);
  operands[5] = adjust_address_nv (operands[1], <HALF_TMODE>mode, 8);
})

(define_split
  [(set (match_operand:TD_TF 0 "memory_operand" "")
        (match_operand:TD_TF 1 "register_operand" ""))]
  "reload_completed && offsettable_memref_p (operands[0])
   && FP_REG_P (operands[1])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = adjust_address_nv (operands[0], <HALF_TMODE>mode, 0);
  operands[3] = adjust_address_nv (operands[0], <HALF_TMODE>mode, 8);
  operands[4] = simplify_gen_subreg (<HALF_TMODE>mode, operands[1],
                                     <MODE>mode, 0);
  operands[5] = simplify_gen_subreg (<HALF_TMODE>mode, operands[1],
                                     <MODE>mode, 8);
})

;
; mov(df|dd) instruction pattern(s).
;

(define_expand "mov<mode>"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "")
        (match_operand:DD_DF 1 "general_operand"  ""))]
  ""
  "")

(define_insn "*mov<mode>_64dfp"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand"
                               "=f,f,f,d,f,f,R,T,d,d,d, d,b,RT,v,v,d,v,QR")
        (match_operand:DD_DF 1 "general_operand"
                               " G,f,d,f,R,T,f,f,G,d,b,RT,d, d,v,d,v,QR,v"))]
  "TARGET_DFP"
  "@
   lzdr\t%0
   ldr\t%0,%1
   ldgr\t%0,%1
   lgdr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   lghi\t%0,0
   lgr\t%0,%1
   lgrl\t%0,%1
   lg\t%0,%1
   stgrl\t%1,%0
   stg\t%1,%0
   vlr\t%v0,%v1
   vlvgg\t%v0,%1,0
   vlgvg\t%0,%v1,0
   vleg\t%0,%1,0
   vsteg\t%1,%0,0"
  [(set_attr "op_type" "RRE,RR,RRE,RRE,RX,RXY,RX,RXY,RI,RRE,RIL,RXY,RIL,RXY,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type" "fsimpdf,floaddf,floaddf,floaddf,floaddf,floaddf,
                     fstoredf,fstoredf,*,lr,load,load,store,store,*,*,*,load,store")
   (set_attr "z10prop" "*,*,*,*,*,*,*,*,z10_fwd_A1,z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec,*,*,*,*,*")
   (set_attr "cpu_facility" "z196,*,*,*,*,*,*,*,*,*,z10,*,z10,*,vec,vec,vec,vec,vec")])

(define_insn "*mov<mode>_64"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "=f,f,f,f,R,T,d,d,d, d,b,RT,v,v,QR")
        (match_operand:DD_DF 1 "general_operand"      " G,f,R,T,f,f,G,d,b,RT,d, d,v,QR,v"))]
  "TARGET_ZARCH"
  "@
   lzdr\t%0
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   lghi\t%0,0
   lgr\t%0,%1
   lgrl\t%0,%1
   lg\t%0,%1
   stgrl\t%1,%0
   stg\t%1,%0
   vlr\t%v0,%v1
   vleg\t%v0,%1,0
   vsteg\t%v1,%0,0"
  [(set_attr "op_type" "RRE,RR,RX,RXY,RX,RXY,RI,RRE,RIL,RXY,RIL,RXY,VRR,VRX,VRX")
   (set_attr "type"    "fsimpdf,fload<mode>,fload<mode>,fload<mode>,
                        fstore<mode>,fstore<mode>,*,lr,load,load,store,store,*,load,store")
   (set_attr "z10prop" "*,*,*,*,*,*,z10_fwd_A1,z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec,*,*,*")
   (set_attr "cpu_facility" "z196,*,*,*,*,*,*,*,z10,*,z10,*,vec,vec,vec")])

(define_insn "*mov<mode>_31"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand"
                               "=f,f,f,f,R,T,d,d,Q,S,   d,o")
        (match_operand:DD_DF 1 "general_operand"
                               " G,f,R,T,f,f,Q,S,d,d,dPRT,d"))]
  "!TARGET_ZARCH"
  "@
   lzdr\t%0
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   lm\t%0,%N0,%S1
   lmy\t%0,%N0,%S1
   stm\t%1,%N1,%S0
   stmy\t%1,%N1,%S0
   #
   #"
  [(set_attr "op_type" "RRE,RR,RX,RXY,RX,RXY,RS,RSY,RS,RSY,*,*")
   (set_attr "type"    "fsimpdf,fload<mode>,fload<mode>,fload<mode>,
                        fstore<mode>,fstore<mode>,lm,lm,stm,stm,*,*")
   (set_attr "cpu_facility" "z196,*,*,*,*,*,*,*,*,*,*,*")])

(define_split
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "")
        (match_operand:DD_DF 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 0, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 1, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "")
        (match_operand:DD_DF 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 1, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 0, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:DD_DF 0 "register_operand" "")
        (match_operand:DD_DF 1 "memory_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !FP_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, <MODE>mode);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})

;
; mov(sf|sd) instruction pattern(s).
;

(define_insn "mov<mode>"
  [(set (match_operand:SD_SF 0 "nonimmediate_operand"
                               "=f,f,f,f,f,f,R,T,d,d,d,d,d,b,R,T,v,v,v,d,v,QR")
        (match_operand:SD_SF 1 "general_operand"
                               " G,f,f,R,R,T,f,f,G,d,b,R,T,d,d,d,v,G,d,v,QR,v"))]
  ""
  "@
   lzer\t%0
   lder\t%0,%1
   ler\t%0,%1
   lde\t%0,%1
   le\t%0,%1
   ley\t%0,%1
   ste\t%1,%0
   stey\t%1,%0
   lhi\t%0,0
   lr\t%0,%1
   lrl\t%0,%1
   l\t%0,%1
   ly\t%0,%1
   strl\t%1,%0
   st\t%1,%0
   sty\t%1,%0
   vlr\t%v0,%v1
   vleif\t%v0,0
   vlvgf\t%v0,%1,0
   vlgvf\t%0,%v1,0
   vleg\t%0,%1,0
   vsteg\t%1,%0,0"
  [(set_attr "op_type" "RRE,RRE,RR,RXE,RX,RXY,RX,RXY,RI,RR,RIL,RX,RXY,RIL,RX,RXY,VRR,VRI,VRS,VRS,VRX,VRX")
   (set_attr "type"    "fsimpsf,fsimpsf,fload<mode>,fload<mode>,fload<mode>,fload<mode>,
                        fstore<mode>,fstore<mode>,*,lr,load,load,load,store,store,store,*,*,*,*,load,store")
   (set_attr "z10prop" "*,*,*,*,*,*,*,*,z10_fwd_A1,z10_fr_E1,z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec,z10_rec,*,*,*,*,*,*")
   (set_attr "cpu_facility" "z196,vec,*,vec,*,*,*,*,*,*,z10,*,*,z10,*,*,vec,vec,vec,vec,vec,vec")])

;
; movcc instruction pattern
;

(define_insn "movcc"
  [(set (match_operand:CC 0 "nonimmediate_operand" "=d,c,d,d,d,R,T")
        (match_operand:CC 1 "nonimmediate_operand" " d,d,c,R,T,d,d"))]
  ""
  "@
   lr\t%0,%1
   tmh\t%1,12288
   ipm\t%0
   l\t%0,%1
   ly\t%0,%1
   st\t%1,%0
   sty\t%1,%0"
  [(set_attr "op_type" "RR,RI,RRE,RX,RXY,RX,RXY")
   (set_attr "type" "lr,*,*,load,load,store,store")
   (set_attr "z10prop" "z10_fr_E1,z10_super,*,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec")
   (set_attr "z196prop" "*,*,z196_ends,*,*,*,*")])

;
; Block move (MVC) patterns.
;

(define_insn "*mvc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (match_operand:BLK 1 "memory_operand" "Q"))
   (use (match_operand 2 "const_int_operand" "n"))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "mvc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")])

; This splitter converts a QI to QI mode copy into a BLK mode copy in
; order to have it implemented with mvc.

(define_split
  [(set (match_operand:QI 0 "memory_operand" "")
        (match_operand:QI 1 "memory_operand" ""))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
{
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})


(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (match_operand:BLK 1 "memory_operand" ""))
     (use (match_operand 2 "const_int_operand" ""))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (match_operand:BLK 4 "memory_operand" ""))
     (use (match_operand 5 "const_int_operand" ""))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (match_dup 7))
     (use (match_dup 8))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")


;
; load_multiple pattern(s).
;
; ??? Due to reload problems with replacing registers inside match_parallel
; we currently support load_multiple/store_multiple only after reload.
;

(define_expand "load_multiple"
  [(match_par_dup 3 [(set (match_operand 0 "" "")
                          (match_operand 1 "" ""))
                     (use (match_operand 2 "" ""))])]
  "reload_completed"
{
  machine_mode mode;
  int regno;
  int count;
  rtx from;
  int i, off;

  /* Support only loading a constant number of fixed-point registers from
     memory and only bother with this if more than two */
  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) < 2
      || INTVAL (operands[2]) > 16
      || GET_CODE (operands[1]) != MEM
      || GET_CODE (operands[0]) != REG
      || REGNO (operands[0]) >= 16)
    FAIL;

  count = INTVAL (operands[2]);
  regno = REGNO (operands[0]);
  mode = GET_MODE (operands[0]);
  if (mode != SImode && (!TARGET_ZARCH || mode != DImode))
    FAIL;

  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
  if (!can_create_pseudo_p ())
    {
      if (GET_CODE (XEXP (operands[1], 0)) == REG)
        {
          from = XEXP (operands[1], 0);
          off = 0;
        }
      else if (GET_CODE (XEXP (operands[1], 0)) == PLUS
               && GET_CODE (XEXP (XEXP (operands[1], 0), 0)) == REG
               && GET_CODE (XEXP (XEXP (operands[1], 0), 1)) == CONST_INT)
        {
          from = XEXP (XEXP (operands[1], 0), 0);
          off = INTVAL (XEXP (XEXP (operands[1], 0), 1));
        }
      else
        FAIL;
    }
  else
    {
      from = force_reg (Pmode, XEXP (operands[1], 0));
      off = 0;
    }

  for (i = 0; i < count; i++)
    XVECEXP (operands[3], 0, i)
      = gen_rtx_SET (gen_rtx_REG (mode, regno + i),
                     change_address (operands[1], mode,
                       plus_constant (Pmode, from,
                                      off + i * GET_MODE_SIZE (mode))));
})

(define_insn "*load_multiple_di"
  [(match_parallel 0 "load_multiple_operation"
                   [(set (match_operand:DI 1 "register_operand" "=r")
                         (match_operand:DI 2 "s_operand" "QS"))])]
  "reload_completed && TARGET_ZARCH"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (DImode, REGNO (operands[1]) + words - 1);
  return "lmg\t%1,%0,%S2";
}
   [(set_attr "op_type" "RSY")
    (set_attr "type"    "lm")])

(define_insn "*load_multiple_si"
  [(match_parallel 0 "load_multiple_operation"
                   [(set (match_operand:SI 1 "register_operand" "=r,r")
                         (match_operand:SI 2 "s_operand" "Q,S"))])]
  "reload_completed"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (SImode, REGNO (operands[1]) + words - 1);
  return which_alternative == 0 ? "lm\t%1,%0,%S2" : "lmy\t%1,%0,%S2";
}
   [(set_attr "op_type" "RS,RSY")
    (set_attr "type"    "lm")])

;
; store multiple pattern(s).
;

(define_expand "store_multiple"
  [(match_par_dup 3 [(set (match_operand 0 "" "")
                          (match_operand 1 "" ""))
                     (use (match_operand 2 "" ""))])]
  "reload_completed"
{
  machine_mode mode;
  int regno;
  int count;
  rtx to;
  int i, off;

  /* Support only storing a constant number of fixed-point registers to
     memory and only bother with this if more than two.  */
  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) < 2
      || INTVAL (operands[2]) > 16
      || GET_CODE (operands[0]) != MEM
      || GET_CODE (operands[1]) != REG
      || REGNO (operands[1]) >= 16)
    FAIL;

  count = INTVAL (operands[2]);
  regno = REGNO (operands[1]);
  mode = GET_MODE (operands[1]);
  if (mode != SImode && (!TARGET_ZARCH || mode != DImode))
    FAIL;

  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));

  if (!can_create_pseudo_p ())
    {
      if (GET_CODE (XEXP (operands[0], 0)) == REG)
        {
          to = XEXP (operands[0], 0);
          off = 0;
        }
      else if (GET_CODE (XEXP (operands[0], 0)) == PLUS
               && GET_CODE (XEXP (XEXP (operands[0], 0), 0)) == REG
               && GET_CODE (XEXP (XEXP (operands[0], 0), 1)) == CONST_INT)
        {
          to = XEXP (XEXP (operands[0], 0), 0);
          off = INTVAL (XEXP (XEXP (operands[0], 0), 1));
        }
      else
        FAIL;
    }
  else
    {
      to = force_reg (Pmode, XEXP (operands[0], 0));
      off = 0;
    }

  for (i = 0; i < count; i++)
    XVECEXP (operands[3], 0, i)
      = gen_rtx_SET (change_address (operands[0], mode,
                       plus_constant (Pmode, to,
                                      off + i * GET_MODE_SIZE (mode))),
                     gen_rtx_REG (mode, regno + i));
})

(define_insn "*store_multiple_di"
  [(match_parallel 0 "store_multiple_operation"
                   [(set (match_operand:DI 1 "s_operand" "=QS")
                         (match_operand:DI 2 "register_operand" "r"))])]
  "reload_completed && TARGET_ZARCH"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (DImode, REGNO (operands[2]) + words - 1);
  return "stmg\t%2,%0,%S1";
}
   [(set_attr "op_type" "RSY")
    (set_attr "type"    "stm")])


(define_insn "*store_multiple_si"
  [(match_parallel 0 "store_multiple_operation"
                   [(set (match_operand:SI 1 "s_operand" "=Q,S")
                         (match_operand:SI 2 "register_operand" "r,r"))])]
  "reload_completed"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (SImode, REGNO (operands[2]) + words - 1);
  return which_alternative == 0 ? "stm\t%2,%0,%S1" : "stmy\t%2,%0,%S1";
}
   [(set_attr "op_type" "RS,RSY")
    (set_attr "type"    "stm")])

;;
;; String instructions.
;;

(define_insn "*execute_rl"
  [(match_parallel 0 "execute_operation"
    [(unspec [(match_operand 1    "register_operand" "a")
              (match_operand 2    "" "")
              (match_operand:SI 3 "larl_operand" "X")] UNSPEC_EXECUTE)])]
  "TARGET_Z10 && GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
   && GET_MODE_SIZE (GET_MODE (operands[1])) <= UNITS_PER_WORD"
  "exrl\t%1,%3"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "cs")])

(define_insn "*execute"
  [(match_parallel 0 "execute_operation"
    [(unspec [(match_operand 1 "register_operand" "a")
              (match_operand:BLK 2 "memory_operand" "R")
              (match_operand 3 "" "")] UNSPEC_EXECUTE)])]
  "GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
   && GET_MODE_SIZE (GET_MODE (operands[1])) <= UNITS_PER_WORD"
  "ex\t%1,%2"
  [(set_attr "op_type" "RX")
   (set_attr "type" "cs")])


;
; strlenM instruction pattern(s).
;

(define_expand "strlen<mode>"
  [(match_operand:P   0 "register_operand" "")  ; result
   (match_operand:BLK 1 "memory_operand" "")    ; input string
   (match_operand:SI  2 "immediate_operand" "") ; search character
   (match_operand:SI  3 "immediate_operand" "")] ; known alignment
  ""
{
  if (!TARGET_VX || operands[2] != const0_rtx)
    emit_insn (gen_strlen_srst<mode> (operands[0], operands[1],
                                      operands[2], operands[3]));
  else
    s390_expand_vec_strlen (operands[0], operands[1], operands[3]);

  DONE;
})

(define_expand "strlen_srst<mode>"
  [(set (reg:SI 0) (match_operand:SI 2 "immediate_operand" ""))
   (parallel
    [(set (match_dup 4)
          (unspec:P [(const_int 0)
                      (match_operand:BLK 1 "memory_operand" "")
                      (reg:SI 0)
                      (match_operand 3 "immediate_operand" "")] UNSPEC_SRST))
     (clobber (scratch:P))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:P 0 "register_operand" "")
          (minus:P (match_dup 4) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  operands[4] = gen_reg_rtx (Pmode);
  operands[5] = gen_reg_rtx (Pmode);
  emit_move_insn (operands[5], force_operand (XEXP (operands[1], 0), NULL_RTX));
  operands[1] = replace_equiv_address (operands[1], operands[5]);
})

(define_insn "*strlen<mode>"
  [(set (match_operand:P 0 "register_operand" "=a")
        (unspec:P [(match_operand:P 2 "general_operand" "0")
                    (mem:BLK (match_operand:P 3 "register_operand" "1"))
                    (reg:SI 0)
                    (match_operand 4 "immediate_operand" "")] UNSPEC_SRST))
   (clobber (match_scratch:P 1 "=a"))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "srst\t%0,%1\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

;
; cmpstrM instruction pattern(s).
;

(define_expand "cmpstrsi"
  [(set (reg:SI 0) (const_int 0))
   (parallel
    [(clobber (match_operand 3 "" ""))
     (clobber (match_dup 4))
     (set (reg:CCU CC_REGNUM)
          (compare:CCU (match_operand:BLK 1 "memory_operand" "")
                       (match_operand:BLK 2 "memory_operand" "")))
     (use (reg:SI 0))])
   (parallel
    [(set (match_operand:SI 0 "register_operand" "=d")
          (unspec:SI [(reg:CCU CC_REGNUM)] UNSPEC_STRCMPCC_TO_INT))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  /* As the result of CMPINT is inverted compared to what we need,
     we have to swap the operands.  */
  rtx op1 = operands[2];
  rtx op2 = operands[1];
  rtx addr1 = gen_reg_rtx (Pmode);
  rtx addr2 = gen_reg_rtx (Pmode);

  emit_move_insn (addr1, force_operand (XEXP (op1, 0), NULL_RTX));
  emit_move_insn (addr2, force_operand (XEXP (op2, 0), NULL_RTX));
  operands[1] = replace_equiv_address_nv (op1, addr1);
  operands[2] = replace_equiv_address_nv (op2, addr2);
  operands[3] = addr1;
  operands[4] = addr2;
})

(define_insn "*cmpstr<mode>"
  [(clobber (match_operand:P 0 "register_operand" "=d"))
   (clobber (match_operand:P 1 "register_operand" "=d"))
   (set (reg:CCU CC_REGNUM)
        (compare:CCU (mem:BLK (match_operand:P 2 "register_operand" "0"))
                     (mem:BLK (match_operand:P 3 "register_operand" "1"))))
   (use (reg:SI 0))]
  ""
  "clst\t%0,%1\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

;
; movstr instruction pattern.
;

(define_expand "movstr"
  [(set (reg:SI 0) (const_int 0))
   (parallel
    [(clobber (match_dup 3))
     (set (match_operand:BLK 1 "memory_operand" "")
          (match_operand:BLK 2 "memory_operand" ""))
     (set (match_operand 0 "register_operand" "")
          (unspec [(match_dup 1)
                   (match_dup 2)
                   (reg:SI 0)] UNSPEC_MVST))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  rtx addr1 = gen_reg_rtx (Pmode);
  rtx addr2 = gen_reg_rtx (Pmode);

  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
  emit_move_insn (addr2, force_operand (XEXP (operands[2], 0), NULL_RTX));
  operands[1] = replace_equiv_address_nv (operands[1], addr1);
  operands[2] = replace_equiv_address_nv (operands[2], addr2);
  operands[3] = addr2;
})

(define_insn "*movstr"
  [(clobber (match_operand:P 2 "register_operand" "=d"))
   (set (mem:BLK (match_operand:P 1 "register_operand" "0"))
        (mem:BLK (match_operand:P 3 "register_operand" "2")))
   (set (match_operand:P 0 "register_operand" "=d")
        (unspec [(mem:BLK (match_dup 1))
                 (mem:BLK (match_dup 3))
                 (reg:SI 0)] UNSPEC_MVST))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "mvst\t%1,%2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])


;
; movmemM instruction pattern(s).
;

(define_expand "movmem<mode>"
  [(set (match_operand:BLK 0 "memory_operand" "")   ; destination
        (match_operand:BLK 1 "memory_operand" ""))  ; source
   (use (match_operand:GPR 2 "general_operand" "")) ; count
   (match_operand 3 "" "")]
  ""
{
  if (s390_expand_movmem (operands[0], operands[1], operands[2]))
    DONE;
  else
    FAIL;
})

; Move a block that is up to 256 bytes in length.
; The block length is taken as (operands[2] % 256) + 1.

(define_expand "movmem_short"
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (match_operand:BLK 1 "memory_operand" ""))
     (use (match_operand 2 "nonmemory_operand" ""))
     (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
     (clobber (match_dup 3))])]
  ""
  "operands[3] = gen_rtx_SCRATCH (Pmode);")

(define_insn "*movmem_short"
  [(set (match_operand:BLK 0 "memory_operand" "=Q,Q,Q,Q")
        (match_operand:BLK 1 "memory_operand" "Q,Q,Q,Q"))
   (use (match_operand 2 "nonmemory_operand" "n,a,a,a"))
   (use (match_operand 3 "immediate_operand" "X,R,X,X"))
   (clobber (match_scratch:P 4 "=X,X,X,&a"))]
  "(GET_MODE (operands[2]) == Pmode || GET_MODE (operands[2]) == VOIDmode)"
  "#"
  [(set_attr "type"         "cs")
   (set_attr "cpu_facility" "*,*,z10,cpu_zarch")])

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "const_int_operand" ""))
   (use (match_operand 3 "immediate_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (match_dup 2))])]
  "operands[2] = GEN_INT ((INTVAL (operands[2]) & 0xff) + 1);")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "register_operand" ""))
   (use (match_operand 3 "memory_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (match_dup 3)
              (const_int 0)] UNSPEC_EXECUTE)
     (set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
  "")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (scratch))]
  "TARGET_Z10 && reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (const_int 0)
              (label_ref (match_dup 3))] UNSPEC_EXECUTE)
     (set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
  "operands[3] = gen_label_rtx ();")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (match_operand 3 "register_operand" ""))]
  "reload_completed && TARGET_CPU_ZARCH"
  [(set (match_dup 3) (label_ref (match_dup 4)))
   (parallel
    [(unspec [(match_dup 2) (mem:BLK (match_dup 3))
              (label_ref (match_dup 4))] UNSPEC_EXECUTE)
     (set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
  "operands[4] = gen_label_rtx ();")

; Move a block of arbitrary length.

(define_expand "movmem_long"
  [(parallel
    [(clobber (match_dup 2))
     (clobber (match_dup 3))
     (set (match_operand:BLK 0 "memory_operand" "")
          (match_operand:BLK 1 "memory_operand" ""))
     (use (match_operand 2 "general_operand" ""))
     (use (match_dup 3))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  machine_mode sreg_mode = TARGET_ZARCH ? DImode : SImode;
  machine_mode dreg_mode = TARGET_ZARCH ? TImode : DImode;
  rtx reg0 = gen_reg_rtx (dreg_mode);
  rtx reg1 = gen_reg_rtx (dreg_mode);
  rtx addr0 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg0));
  rtx addr1 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg1));
  rtx len0 = gen_lowpart (Pmode, reg0);
  rtx len1 = gen_lowpart (Pmode, reg1);

  emit_clobber (reg0);
  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
  emit_move_insn (len0, operands[2]);

  emit_clobber (reg1);
  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
  emit_move_insn (len1, operands[2]);

  operands[0] = replace_equiv_address_nv (operands[0], addr0);
  operands[1] = replace_equiv_address_nv (operands[1], addr1);
  operands[2] = reg0;
  operands[3] = reg1;
})

(define_insn "*movmem_long"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (clobber (match_operand:<DBL> 1 "register_operand" "=d"))
   (set (mem:BLK (subreg:P (match_operand:<DBL> 2 "register_operand" "0") 0))
        (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "1") 0)))
   (use (match_dup 2))
   (use (match_dup 3))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_64BIT || !TARGET_ZARCH"
  "mvcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*movmem_long_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (clobber (match_operand:TI 1 "register_operand" "=d"))
   (set (mem:BLK (subreg:SI (match_operand:TI 2 "register_operand" "0") 4))
        (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "1") 4)))
   (use (match_dup 2))
   (use (match_dup 3))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_64BIT && TARGET_ZARCH"
  "mvcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])


;
; Test data class.
;

(define_expand "signbit<mode>2"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:FP_ALL 1 "register_operand" "f")
                     (match_dup 2)]
                     UNSPEC_TDC_INSN))
   (set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(reg:CCZ CC_REGNUM)] UNSPEC_CC_TO_INT))]
  "TARGET_HARD_FLOAT"
{
  operands[2] = GEN_INT (S390_TDC_SIGNBIT_SET);
})

(define_expand "isinf<mode>2"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:FP_ALL 1 "register_operand" "f")
                     (match_dup 2)]
                     UNSPEC_TDC_INSN))
   (set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(reg:CCZ CC_REGNUM)] UNSPEC_CC_TO_INT))]
  "TARGET_HARD_FLOAT"
{
  operands[2] = GEN_INT (S390_TDC_INFINITY);
})

; This extracts CC into a GPR properly shifted.  The actual IPM
; instruction will be issued by reload.  The constraint of operand 1
; forces reload to use a GPR.  So reload will issue a movcc insn for
; copying CC into a GPR first.
(define_insn_and_split "*cc_to_int"
  [(set (match_operand:SI 0 "nonimmediate_operand"     "=d")
        (unspec:SI [(match_operand 1 "register_operand" "0")]
                   UNSPEC_CC_TO_INT))]
  "operands != NULL"
  "#"
  "reload_completed"
  [(set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 28)))])

; This insn is used to generate all variants of the Test Data Class
; instruction, namely tcxb, tcdb, and tceb.  The insn's first operand
; is the register to be tested and the second one is the bit mask
; specifying the required test(s).
;
; tcxb, tcdb, tceb, tdcxt, tdcdt, tdcet
(define_insn "*TDC_insn_<mode>"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:FP_ALL 0 "register_operand" "f")
                     (match_operand:SI 1 "const_int_operand")] UNSPEC_TDC_INSN))]
  "TARGET_HARD_FLOAT"
  "t<_d>c<xde><bt>\t%0,%1"
   [(set_attr "op_type" "RXE")
    (set_attr "type"  "fsimp<mode>")])



;
; setmemM instruction pattern(s).
;

(define_expand "setmem<mode>"
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:QI 2 "general_operand" ""))
   (use (match_operand:GPR 1 "general_operand" ""))
   (match_operand 3 "" "")]
  ""
  "s390_expand_setmem (operands[0], operands[1], operands[2]); DONE;")

; Clear a block that is up to 256 bytes in length.
; The block length is taken as (operands[1] % 256) + 1.

(define_expand "clrmem_short"
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (const_int 0))
     (use (match_operand 1 "nonmemory_operand" ""))
     (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
     (clobber (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "operands[2] = gen_rtx_SCRATCH (Pmode);")

(define_insn "*clrmem_short"
  [(set (match_operand:BLK 0 "memory_operand" "=Q,Q,Q,Q")
        (const_int 0))
   (use (match_operand 1 "nonmemory_operand" "n,a,a,a"))
   (use (match_operand 2 "immediate_operand" "X,R,X,X"))
   (clobber (match_scratch:P 3 "=X,X,X,&a"))
   (clobber (reg:CC CC_REGNUM))]
  "(GET_MODE (operands[1]) == Pmode || GET_MODE (operands[1]) == VOIDmode)"
  "#"
  [(set_attr "type" "cs")
   (set_attr "cpu_facility" "*,*,z10,cpu_zarch")])

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "const_int_operand" ""))
   (use (match_operand 2 "immediate_operand" ""))
   (clobber (scratch))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (const_int 0))
     (use (match_dup 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[1] = GEN_INT ((INTVAL (operands[1]) & 0xff) + 1);")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "register_operand" ""))
   (use (match_operand 2 "memory_operand" ""))
   (clobber (scratch))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(unspec [(match_dup 1) (match_dup 2)
              (const_int 0)] UNSPEC_EXECUTE)
     (set (match_dup 0) (const_int 0))
     (use (const_int 1))
     (clobber (reg:CC CC_REGNUM))])]
  "")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (scratch))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && reload_completed"
  [(parallel
    [(unspec [(match_dup 1) (const_int 0)
              (label_ref (match_dup 3))] UNSPEC_EXECUTE)
     (set (match_dup 0) (const_int 0))
     (use (const_int 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = gen_label_rtx ();")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (match_operand 2 "register_operand" ""))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed && TARGET_CPU_ZARCH"
  [(set (match_dup 2) (label_ref (match_dup 3)))
   (parallel
    [(unspec [(match_dup 1) (mem:BLK (match_dup 2))
              (label_ref (match_dup 3))] UNSPEC_EXECUTE)
     (set (match_dup 0) (const_int 0))
     (use (const_int 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = gen_label_rtx ();")

; Initialize a block of arbitrary length with (operands[2] % 256).

(define_expand "setmem_long"
  [(parallel
    [(clobber (match_dup 1))
     (set (match_operand:BLK 0 "memory_operand" "")
          (match_operand 2 "shift_count_or_setmem_operand" ""))
     (use (match_operand 1 "general_operand" ""))
     (use (match_dup 3))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  machine_mode sreg_mode = TARGET_ZARCH ? DImode : SImode;
  machine_mode dreg_mode = TARGET_ZARCH ? TImode : DImode;
  rtx reg0 = gen_reg_rtx (dreg_mode);
  rtx reg1 = gen_reg_rtx (dreg_mode);
  rtx addr0 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg0));
  rtx len0 = gen_lowpart (Pmode, reg0);

  emit_clobber (reg0);
  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
  emit_move_insn (len0, operands[1]);

  emit_move_insn (reg1, const0_rtx);

  operands[0] = replace_equiv_address_nv (operands[0], addr0);
  operands[1] = reg0;
  operands[3] = reg1;
})

(define_insn "*setmem_long"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "0") 0))
        (match_operand 2 "shift_count_or_setmem_operand" "Y"))
   (use (match_dup 3))
   (use (match_operand:<DBL> 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_64BIT || !TARGET_ZARCH"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*setmem_long_and"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "0") 0))
        (and (match_operand 2 "shift_count_or_setmem_operand" "Y")
             (match_operand 4 "const_int_operand"             "n")))
   (use (match_dup 3))
   (use (match_operand:<DBL> 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "(TARGET_64BIT || !TARGET_ZARCH) &&
   (INTVAL (operands[4]) & 255) == 255"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*setmem_long_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "0") 4))
        (match_operand 2 "shift_count_or_setmem_operand" "Y"))
   (use (match_dup 3))
   (use (match_operand:TI 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_64BIT && TARGET_ZARCH"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

;
; cmpmemM instruction pattern(s).
;

(define_expand "cmpmemsi"
  [(set (match_operand:SI 0 "register_operand" "")
        (compare:SI (match_operand:BLK 1 "memory_operand" "")
                    (match_operand:BLK 2 "memory_operand" "") ) )
   (use (match_operand:SI 3 "general_operand" ""))
   (use (match_operand:SI 4 "" ""))]
  ""
{
  if (s390_expand_cmpmem (operands[0], operands[1],
                          operands[2], operands[3]))
    DONE;
  else
    FAIL;
})

; Compare a block that is up to 256 bytes in length.
; The block length is taken as (operands[2] % 256) + 1.

(define_expand "cmpmem_short"
  [(parallel
    [(set (reg:CCU CC_REGNUM)
          (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                       (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "nonmemory_operand" ""))
     (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
     (clobber (match_dup 3))])]
  ""
  "operands[3] = gen_rtx_SCRATCH (Pmode);")

(define_insn "*cmpmem_short"
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "Q,Q,Q,Q")
                     (match_operand:BLK 1 "memory_operand" "Q,Q,Q,Q")))
   (use (match_operand 2 "nonmemory_operand" "n,a,a,a"))
   (use (match_operand 3 "immediate_operand" "X,R,X,X"))
   (clobber (match_scratch:P 4 "=X,X,X,&a"))]
  "(GET_MODE (operands[2]) == Pmode || GET_MODE (operands[2]) == VOIDmode)"
  "#"
  [(set_attr "type" "cs")
   (set_attr "cpu_facility" "*,*,z10,cpu_zarch")])

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "const_int_operand" ""))
   (use (match_operand 3 "immediate_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (match_dup 2))])]
  "operands[2] = GEN_INT ((INTVAL (operands[2]) & 0xff) + 1);")

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "register_operand" ""))
   (use (match_operand 3 "memory_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (match_dup 3)
              (const_int 0)] UNSPEC_EXECUTE)
     (set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (const_int 1))])]
  "")

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (scratch))]
  "TARGET_Z10 && reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (const_int 0)
              (label_ref (match_dup 4))] UNSPEC_EXECUTE)
     (set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (const_int 1))])]
  "operands[4] = gen_label_rtx ();")

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (match_operand 3 "register_operand" ""))]
  "reload_completed && TARGET_CPU_ZARCH"
  [(set (match_dup 3) (label_ref (match_dup 4)))
   (parallel
    [(unspec [(match_dup 2) (mem:BLK (match_dup 3))
              (label_ref (match_dup 4))] UNSPEC_EXECUTE)
     (set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (const_int 1))])]
  "operands[4] = gen_label_rtx ();")

; Compare a block of arbitrary length.

(define_expand "cmpmem_long"
  [(parallel
    [(clobber (match_dup 2))
     (clobber (match_dup 3))
     (set (reg:CCU CC_REGNUM)
          (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                       (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "general_operand" ""))
     (use (match_dup 3))])]
  ""
{
  machine_mode sreg_mode = TARGET_ZARCH ? DImode : SImode;
  machine_mode dreg_mode = TARGET_ZARCH ? TImode : DImode;
  rtx reg0 = gen_reg_rtx (dreg_mode);
  rtx reg1 = gen_reg_rtx (dreg_mode);
  rtx addr0 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg0));
  rtx addr1 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg1));
  rtx len0 = gen_lowpart (Pmode, reg0);
  rtx len1 = gen_lowpart (Pmode, reg1);

  emit_clobber (reg0);
  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
  emit_move_insn (len0, operands[2]);

  emit_clobber (reg1);
  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
  emit_move_insn (len1, operands[2]);

  operands[0] = replace_equiv_address_nv (operands[0], addr0);
  operands[1] = replace_equiv_address_nv (operands[1], addr1);
  operands[2] = reg0;
  operands[3] = reg1;
})

(define_insn "*cmpmem_long"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (clobber (match_operand:<DBL> 1 "register_operand" "=d"))
   (set (reg:CCU CC_REGNUM)
        (compare:CCU (mem:BLK (subreg:P (match_operand:<DBL> 2 "register_operand" "0") 0))
                     (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "1") 0))))
   (use (match_dup 2))
   (use (match_dup 3))]
  "TARGET_64BIT || !TARGET_ZARCH"
  "clcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*cmpmem_long_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (clobber (match_operand:TI 1 "register_operand" "=d"))
   (set (reg:CCU CC_REGNUM)
        (compare:CCU (mem:BLK (subreg:SI (match_operand:TI 2 "register_operand" "0") 4))
                     (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "1") 4))))
   (use (match_dup 2))
   (use (match_dup 3))]
  "!TARGET_64BIT && TARGET_ZARCH"
  "clcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "op_type" "NN")
   (set_attr "type"    "vs")
   (set_attr "length"  "8")])

; Convert CCUmode condition code to integer.
; Result is zero if EQ, positive if LTU, negative if GTU.

(define_insn_and_split "cmpint"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                   UNSPEC_STRCMPCC_TO_INT))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "reload_completed"
  [(set (match_dup 0) (ashift:SI (match_dup 0) (const_int 2)))
   (parallel
    [(set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 30)))
     (clobber (reg:CC CC_REGNUM))])])

(define_insn_and_split "*cmpint_cc"
  [(set (reg CC_REGNUM)
        (compare (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                            UNSPEC_STRCMPCC_TO_INT)
                 (const_int 0)))
   (set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(match_dup 1)] UNSPEC_STRCMPCC_TO_INT))]
  "s390_match_ccmode (insn, CCSmode)"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (ashift:SI (match_dup 0) (const_int 2)))
   (parallel
    [(set (match_dup 2) (match_dup 3))
     (set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 30)))])]
{
  rtx result = gen_rtx_ASHIFTRT (SImode, operands[0], GEN_INT (30));
  operands[2] = SET_DEST (XVECEXP (PATTERN (curr_insn), 0, 0));
  operands[3] = gen_rtx_COMPARE (GET_MODE (operands[2]), result, const0_rtx);
})

(define_insn_and_split "*cmpint_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (sign_extend:DI (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                                   UNSPEC_STRCMPCC_TO_INT)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (ashift:DI (match_dup 0) (const_int 34)))
   (parallel
    [(set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 62)))
     (clobber (reg:CC CC_REGNUM))])])

(define_insn_and_split "*cmpint_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:DI (ashift:DI (subreg:DI
                   (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                              UNSPEC_STRCMPCC_TO_INT) 0)
                   (const_int 32)) (const_int 32))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (sign_extend:DI (unspec:SI [(match_dup 1)] UNSPEC_STRCMPCC_TO_INT)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (ashift:DI (match_dup 0) (const_int 34)))
   (parallel
    [(set (match_dup 2) (match_dup 3))
     (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 62)))])]
{
  rtx result = gen_rtx_ASHIFTRT (DImode, operands[0], GEN_INT (62));
  operands[2] = SET_DEST (XVECEXP (PATTERN (curr_insn), 0, 0));
  operands[3] = gen_rtx_COMPARE (GET_MODE (operands[2]), result, const0_rtx);
})


;;
;;- Conversion instructions.
;;

(define_insn "*sethighpartsi"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (unspec:SI [(match_operand:BLK 1 "s_operand" "Q,S")
                    (match_operand 2 "const_int_operand" "n,n")] UNSPEC_ICM))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   icm\t%0,%2,%S1
   icmy\t%0,%2,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*sethighpartdi_64"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (unspec:DI [(match_operand:BLK 1 "s_operand" "QS")
                    (match_operand 2 "const_int_operand" "n")] UNSPEC_ICM))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "icmh\t%0,%2,%S1"
  [(set_attr "op_type" "RSY")
   (set_attr "z10prop" "z10_super")])

(define_insn "*sethighpartdi_31"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (unspec:DI [(match_operand:BLK 1 "s_operand" "Q,S")
                    (match_operand 2 "const_int_operand" "n,n")] UNSPEC_ICM))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "@
   icm\t%0,%2,%S1
   icmy\t%0,%2,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; extv instruction patterns
;

; FIXME: This expander needs to be converted from DI to GPR as well
; after resolving some issues with it.

(define_expand "extzv"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "=d")
        (zero_extract:DI
         (match_operand:DI 1 "register_operand" "d")
         (match_operand 2 "const_int_operand" "")   ; size
         (match_operand 3 "const_int_operand" ""))) ; start
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z10"
{
  /* Starting with zEC12 there is risbgn not clobbering CC.  */
  if (TARGET_ZEC12)
    {
      emit_move_insn (operands[0],
                    gen_rtx_ZERO_EXTRACT (DImode,
                                          operands[1],
                                          operands[2],
                                          operands[3]));
      DONE;
    }
})

(define_insn "*extzv<mode>_zEC12"
  [(set (match_operand:GPR 0 "register_operand" "=d")
      (zero_extract:GPR
        (match_operand:GPR 1 "register_operand" "d")
        (match_operand 2 "const_int_operand" "")   ; size
        (match_operand 3 "const_int_operand" "")))] ; start]
  "TARGET_ZEC12"
  "risbgn\t%0,%1,64-%2,128+63,<bitsize>+%3+%2" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")])

(define_insn "*extzv<mode>_z10"
  [(set (match_operand:GPR 0 "register_operand" "=d")
      (zero_extract:GPR
       (match_operand:GPR 1 "register_operand" "d")
       (match_operand 2 "const_int_operand" "")   ; size
       (match_operand 3 "const_int_operand" ""))) ; start
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  "risbg\t%0,%1,64-%2,128+63,<bitsize>+%3+%2" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn_and_split "*pre_z10_extzv<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (zero_extract:GPR (match_operand:QI 1 "s_operand" "QS")
                          (match_operand 2 "nonzero_shift_count_operand" "")
                          (const_int 0)))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_Z10"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (match_dup 3)] UNSPEC_ICM))
     (clobber (reg:CC CC_REGNUM))])
   (set (match_dup 0) (lshiftrt:GPR (match_dup 0) (match_dup 2)))]
{
  int bitsize = INTVAL (operands[2]);
  int size = (bitsize - 1) / BITS_PER_UNIT + 1; /* round up */
  int mask = ((1ul << size) - 1) << (GET_MODE_SIZE (SImode) - size);

  operands[1] = adjust_address (operands[1], BLKmode, 0);
  set_mem_size (operands[1], size);
  operands[2] = GEN_INT (<GPR:bitsize> - bitsize);
  operands[3] = GEN_INT (mask);
})

(define_insn_and_split "*pre_z10_extv<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (sign_extract:GPR (match_operand:QI 1 "s_operand" "QS")
                          (match_operand 2 "nonzero_shift_count_operand" "")
                          (const_int 0)))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (match_dup 3)] UNSPEC_ICM))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_dup 0) (ashiftrt:GPR (match_dup 0) (match_dup 2)))
     (clobber (reg:CC CC_REGNUM))])]
{
  int bitsize = INTVAL (operands[2]);
  int size = (bitsize - 1) / BITS_PER_UNIT + 1; /* round up */
  int mask = ((1ul << size) - 1) << (GET_MODE_SIZE (SImode) - size);

  operands[1] = adjust_address (operands[1], BLKmode, 0);
  set_mem_size (operands[1], size);
  operands[2] = GEN_INT (<GPR:bitsize> - bitsize);
  operands[3] = GEN_INT (mask);
})

;
; insv instruction patterns
;

(define_expand "insv"
  [(set (zero_extract (match_operand 0 "nonimmediate_operand" "")
                      (match_operand 1 "const_int_operand" "")
                      (match_operand 2 "const_int_operand" ""))
        (match_operand 3 "general_operand" ""))]
  ""
{
  if (s390_expand_insv (operands[0], operands[1], operands[2], operands[3]))
    DONE;
  FAIL;
})


; The normal RTL expansion will never generate a zero_extract where
; the location operand isn't word mode.  However, we do this in the
; back-end when generating atomic operations. See s390_two_part_insv.
(define_insn "*insv<mode>_zEC12"
  [(set (zero_extract:GPR (match_operand:GPR 0 "nonimmediate_operand" "+d")
                          (match_operand 1 "const_int_operand"    "I")  ; size
                          (match_operand 2 "const_int_operand"    "I")) ; pos
        (match_operand:GPR 3 "nonimmediate_operand" "d"))]
  "TARGET_ZEC12
   && (INTVAL (operands[1]) + INTVAL (operands[2])) <= <bitsize>"
  "risbgn\t%0,%3,64-<bitsize>+%2,64-<bitsize>+%2+%1-1,<bitsize>-%2-%1"
  [(set_attr "op_type" "RIE")])

(define_insn "*insv<mode>_z10"
  [(set (zero_extract:GPR (match_operand:GPR 0 "nonimmediate_operand" "+d")
                          (match_operand 1 "const_int_operand"    "I")  ; size
                          (match_operand 2 "const_int_operand"    "I")) ; pos
        (match_operand:GPR 3 "nonimmediate_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && (INTVAL (operands[1]) + INTVAL (operands[2])) <= <bitsize>"
  "risbg\t%0,%3,64-<bitsize>+%2,64-<bitsize>+%2+%1-1,<bitsize>-%2-%1"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; and op1 with a mask being 1 for the selected bits and 0 for the rest
; and op3=op0 with a mask being 0 for the selected bits and 1 for the rest
(define_insn "*insv<mode>_zEC12_noshift"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
        (ior:GPR (and:GPR (match_operand:GPR 1 "nonimmediate_operand" "d")
                          (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
                 (and:GPR (match_operand:GPR 3 "nonimmediate_operand" "0")
                          (match_operand:GPR 4 "const_int_operand" ""))))]
  "TARGET_ZEC12 && INTVAL (operands[2]) == ~INTVAL (operands[4])"
  "risbgn\t%0,%1,%<bfstart>2,%<bfend>2,0"
  [(set_attr "op_type" "RIE")])

(define_insn "*insv<mode>_z10_noshift"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
        (ior:GPR (and:GPR (match_operand:GPR 1 "nonimmediate_operand" "d")
                          (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
                 (and:GPR (match_operand:GPR 3 "nonimmediate_operand" "0")
                          (match_operand:GPR 4 "const_int_operand" ""))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && INTVAL (operands[2]) == ~INTVAL (operands[4])"
  "risbg\t%0,%1,%<bfstart>2,%<bfend>2,0"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*r<noxa>sbg_<mode>_noshift"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
        (IXOR:GPR
          (and:GPR (match_operand:GPR 1 "nonimmediate_operand" "d")
                   (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
          (match_operand:GPR 3 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,0"
  [(set_attr "op_type" "RIE")])

(define_insn "*r<noxa>sbg_di_rotl"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=d")
        (IXOR:DI
          (and:DI
            (rotate:DI
              (match_operand:DI 1 "nonimmediate_operand" "d")
              (match_operand:DI 3 "const_int_operand" ""))
            (match_operand:DI 2 "contiguous_bitmask_operand" ""))
          (match_operand:DI 4 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,%b3"
  [(set_attr "op_type" "RIE")])

(define_insn "*r<noxa>sbg_<mode>_srl"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
        (IXOR:GPR
          (and:GPR
            (lshiftrt:GPR
              (match_operand:GPR 1 "nonimmediate_operand" "d")
              (match_operand:GPR 3 "nonzero_shift_count_operand" ""))
            (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
          (match_operand:GPR 4 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && s390_extzv_shift_ok (<bitsize>, 64 - INTVAL (operands[3]),
                           INTVAL (operands[2]))"
  "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,64-%3"
  [(set_attr "op_type" "RIE")])

(define_insn "*r<noxa>sbg_<mode>_sll"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
        (IXOR:GPR
          (and:GPR
            (ashift:GPR
              (match_operand:GPR 1 "nonimmediate_operand" "d")
              (match_operand:GPR 3 "nonzero_shift_count_operand" ""))
            (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
          (match_operand:GPR 4 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && s390_extzv_shift_ok (<bitsize>, INTVAL (operands[3]),
                           INTVAL (operands[2]))"
  "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,%3"
  [(set_attr "op_type" "RIE")])

;; These two are generated by combine for s.bf &= val.
;; ??? For bitfields smaller than 32-bits, we wind up with SImode
;; shifts and ands, which results in some truly awful patterns
;; including subregs of operations.  Rather unnecessisarily, IMO.
;; Instead of
;;
;; (set (zero_extract:DI (reg/v:DI 50 [ s ])
;;        (const_int 24 [0x18])
;;        (const_int 0 [0]))
;;    (subreg:DI (and:SI (subreg:SI (lshiftrt:DI (reg/v:DI 50 [ s ])
;;                    (const_int 40 [0x28])) 4)
;;            (reg:SI 4 %r4 [ y+4 ])) 0))
;;
;; we should instead generate
;;
;; (set (zero_extract:DI (reg/v:DI 50 [ s ])
;;        (const_int 24 [0x18])
;;        (const_int 0 [0]))
;;    (and:DI (lshiftrt:DI (reg/v:DI 50 [ s ])
;;                    (const_int 40 [0x28]))
;;            (subreg:DI (reg:SI 4 %r4 [ y+4 ]) 0)))
;;
;; by noticing that we can push down the outer paradoxical subreg
;; into the operation.

(define_insn "*insv_rnsbg_noshift"
  [(set (zero_extract:DI
          (match_operand:DI 0 "nonimmediate_operand" "+d")
          (match_operand 1 "const_int_operand" "")
          (match_operand 2 "const_int_operand" ""))
        (and:DI
          (match_dup 0)
          (match_operand:DI 3 "nonimmediate_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && INTVAL (operands[1]) + INTVAL (operands[2]) == 64"
  "rnsbg\t%0,%3,%2,63,0"
  [(set_attr "op_type" "RIE")])

(define_insn "*insv_rnsbg_srl"
  [(set (zero_extract:DI
          (match_operand:DI 0 "nonimmediate_operand" "+d")
          (match_operand 1 "const_int_operand" "")
          (match_operand 2 "const_int_operand" ""))
        (and:DI
          (lshiftrt:DI
            (match_dup 0)
            (match_operand 3 "const_int_operand" ""))
          (match_operand:DI 4 "nonimmediate_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && INTVAL (operands[3]) == 64 - INTVAL (operands[1]) - INTVAL (operands[2])"
  "rnsbg\t%0,%4,%2,%2+%1-1,%3"
  [(set_attr "op_type" "RIE")])

(define_insn "*insv<mode>_mem_reg"
  [(set (zero_extract:W (match_operand:QI 0 "memory_operand" "+Q,S")
                        (match_operand 1 "const_int_operand" "n,n")
                        (const_int 0))
        (match_operand:W 2 "register_operand" "d,d"))]
  "INTVAL (operands[1]) > 0
   && INTVAL (operands[1]) <= GET_MODE_BITSIZE (SImode)
   && INTVAL (operands[1]) % BITS_PER_UNIT == 0"
{
    int size = INTVAL (operands[1]) / BITS_PER_UNIT;

    operands[1] = GEN_INT ((1ul << size) - 1);
    return (which_alternative == 0) ? "stcm\t%2,%1,%S0"
                                    : "stcmy\t%2,%1,%S0";
}
  [(set_attr "op_type" "RS,RSY")
   (set_attr "z10prop" "z10_super,z10_super")])

(define_insn "*insvdi_mem_reghigh"
  [(set (zero_extract:DI (match_operand:QI 0 "memory_operand" "+QS")
                         (match_operand 1 "const_int_operand" "n")
                         (const_int 0))
        (lshiftrt:DI (match_operand:DI 2 "register_operand" "d")
                     (const_int 32)))]
  "TARGET_ZARCH
   && INTVAL (operands[1]) > 0
   && INTVAL (operands[1]) <= GET_MODE_BITSIZE (SImode)
   && INTVAL (operands[1]) % BITS_PER_UNIT == 0"
{
    int size = INTVAL (operands[1]) / BITS_PER_UNIT;

    operands[1] = GEN_INT ((1ul << size) - 1);
    return "stcmh\t%2,%1,%S0";
}
[(set_attr "op_type" "RSY")
 (set_attr "z10prop" "z10_super")])

(define_insn "*insvdi_reg_imm"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
                         (const_int 16)
                         (match_operand 1 "const_int_operand" "n"))
        (match_operand:DI 2 "const_int_operand" "n"))]
  "TARGET_ZARCH
   && INTVAL (operands[1]) >= 0
   && INTVAL (operands[1]) < BITS_PER_WORD
   && INTVAL (operands[1]) % 16 == 0"
{
  switch (BITS_PER_WORD - INTVAL (operands[1]))
    {
      case 64: return "iihh\t%0,%x2"; break;
      case 48: return "iihl\t%0,%x2"; break;
      case 32: return "iilh\t%0,%x2"; break;
      case 16: return "iill\t%0,%x2"; break;
      default: gcc_unreachable();
    }
}
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super_E1")])

; Update the left-most 32 bit of a DI.
(define_insn "*insv_h_di_reg_extimm"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
                         (const_int 32)
                         (const_int 0))
        (match_operand:DI 1 "const_int_operand" "n"))]
  "TARGET_EXTIMM"
  "iihf\t%0,%o1"
  [(set_attr "op_type" "RIL")
   (set_attr "z10prop" "z10_fwd_E1")])

; Update the right-most 32 bit of a DI.
(define_insn "*insv_l_di_reg_extimm"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
                         (const_int 32)
                         (const_int 32))
        (match_operand:DI 1 "const_int_operand" "n"))]
  "TARGET_EXTIMM"
  "iilf\t%0,%o1"
  [(set_attr "op_type" "RIL")
   (set_attr "z10prop" "z10_fwd_A1")])

;
; extendsidi2 instruction pattern(s).
;

(define_expand "extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "")
        (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_ZARCH)
    {
      emit_clobber (operands[0]);
      emit_move_insn (gen_highpart (SImode, operands[0]), operands[1]);
      emit_move_insn (gen_lowpart (SImode, operands[0]), const0_rtx);
      emit_insn (gen_ashrdi3 (operands[0], operands[0], GEN_INT (32)));
      DONE;
    }
})

(define_insn "*extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "d,RT,b")))]
  "TARGET_ZARCH"
  "@
   lgfr\t%0,%1
   lgf\t%0,%1
   lgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1")])

;
; extend(hi|qi)(si|di)2 instruction pattern(s).
;

(define_expand "extend<HQI:mode><DSI:mode>2"
  [(set (match_operand:DSI 0 "register_operand" "")
        (sign_extend:DSI (match_operand:HQI 1 "nonimmediate_operand" "")))]
  ""
{
  if (<DSI:MODE>mode == DImode && !TARGET_ZARCH)
    {
      rtx tmp = gen_reg_rtx (SImode);
      emit_insn (gen_extend<HQI:mode>si2 (tmp, operands[1]));
      emit_insn (gen_extendsidi2 (operands[0], tmp));
      DONE;
    }
  else if (!TARGET_EXTIMM)
    {
      rtx bitcount = GEN_INT (<DSI:bitsize> - <HQI:bitsize>);

      operands[1] = gen_lowpart (<DSI:MODE>mode, operands[1]);
      emit_insn (gen_ashl<DSI:mode>3 (operands[0], operands[1], bitcount));
      emit_insn (gen_ashr<DSI:mode>3 (operands[0], operands[0], bitcount));
      DONE;
    }
})

;
; extendhidi2 instruction pattern(s).
;

(define_insn "*extendhidi2_extimm"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (sign_extend:DI (match_operand:HI 1 "general_operand" "d,RT,b")))]
  "TARGET_ZARCH && TARGET_EXTIMM"
  "@
   lghr\t%0,%1
   lgh\t%0,%1
   lghrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "extimm,extimm,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1")])

(define_insn "*extendhidi2"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (sign_extend:DI (match_operand:HI 1 "memory_operand" "RT")))]
  "TARGET_ZARCH"
  "lgh\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_super_E1")])

;
; extendhisi2 instruction pattern(s).
;

(define_insn "*extendhisi2_extimm"
  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d")
        (sign_extend:SI (match_operand:HI 1 "nonimmediate_operand" " d,R,T,b")))]
  "TARGET_EXTIMM"
  "@
   lhr\t%0,%1
   lh\t%0,%1
   lhy\t%0,%1
   lhrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RX,RXY,RIL")
   (set_attr "type"         "*,*,*,larl")
   (set_attr "cpu_facility" "extimm,extimm,extimm,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1,z10_super_E1")])

(define_insn "*extendhisi2"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T")))]
  "!TARGET_EXTIMM"
  "@
   lh\t%0,%1
   lhy\t%0,%1"
  [(set_attr "op_type" "RX,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; extendqi(si|di)2 instruction pattern(s).
;

; lbr, lgbr, lb, lgb
(define_insn "*extendqi<mode>2_extimm"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (sign_extend:GPR (match_operand:QI 1 "nonimmediate_operand" "d,RT")))]
  "TARGET_EXTIMM"
  "@
   l<g>br\t%0,%1
   l<g>b\t%0,%1"
  [(set_attr "op_type" "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

; lb, lgb
(define_insn "*extendqi<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (sign_extend:GPR (match_operand:QI 1 "memory_operand" "RT")))]
  "!TARGET_EXTIMM && TARGET_LONG_DISPLACEMENT"
  "l<g>b\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn_and_split "*extendqi<mode>2_short_displ"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (sign_extend:GPR (match_operand:QI 1 "s_operand" "Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_EXTIMM && !TARGET_LONG_DISPLACEMENT"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (const_int 8)] UNSPEC_ICM))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_dup 0) (ashiftrt:GPR (match_dup 0) (match_dup 2)))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[1] = adjust_address (operands[1], BLKmode, 0);
  set_mem_size (operands[1], GET_MODE_SIZE (QImode));
  operands[2] = GEN_INT (<GPR:bitsize> - BITS_PER_UNIT);
})

;
; zero_extendsidi2 instruction pattern(s).
;

(define_expand "zero_extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "")
        (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_ZARCH)
    {
      emit_clobber (operands[0]);
      emit_move_insn (gen_lowpart (SImode, operands[0]), operands[1]);
      emit_move_insn (gen_highpart (SImode, operands[0]), const0_rtx);
      DONE;
    }
})

(define_insn "*zero_extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "d,RT,b")))]
  "TARGET_ZARCH"
  "@
   llgfr\t%0,%1
   llgf\t%0,%1
   llgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "z10prop" "z10_fwd_E1,z10_fwd_A3,z10_fwd_A3")])

;
; LLGT-type instructions (zero-extend from 31 bit to 64 bit).
;

(define_insn "*llgt_sidi"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (and:DI (subreg:DI (match_operand:SI 1 "memory_operand" "RT") 0)
                (const_int 2147483647)))]
  "TARGET_ZARCH"
  "llgt\t%0,%1"
  [(set_attr "op_type"  "RXE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn_and_split "*llgt_sidi_split"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (and:DI (subreg:DI (match_operand:SI 1 "memory_operand" "RT") 0)
                (const_int 2147483647)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0)
        (and:DI (subreg:DI (match_dup 1) 0)
                (const_int 2147483647)))]
  "")

(define_insn "*llgt_sisi"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (and:SI (match_operand:SI 1 "nonimmediate_operand" "d,RT")
                (const_int 2147483647)))]
  "TARGET_ZARCH"
  "@
   llgtr\t%0,%1
   llgt\t%0,%1"
  [(set_attr "op_type"  "RRE,RXE")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*llgt_didi"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (and:DI (match_operand:DI 1 "nonimmediate_operand" "d,o")
                (const_int 2147483647)))]
  "TARGET_ZARCH"
  "@
   llgtr\t%0,%1
   llgt\t%0,%N1"
  [(set_attr "op_type"  "RRE,RXE")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_split
  [(set (match_operand:DSI 0 "register_operand" "")
        (and:DSI (match_operand:DSI 1 "nonimmediate_operand" "")
                 (const_int 2147483647)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && reload_completed"
  [(set (match_dup 0)
        (and:DSI (match_dup 1)
                 (const_int 2147483647)))]
  "")

;
; zero_extend(hi|qi)(si|di)2 instruction pattern(s).
;

(define_expand "zero_extend<mode>di2"
  [(set (match_operand:DI 0 "register_operand" "")
        (zero_extend:DI (match_operand:HQI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_ZARCH)
    {
      rtx tmp = gen_reg_rtx (SImode);
      emit_insn (gen_zero_extend<mode>si2 (tmp, operands[1]));
      emit_insn (gen_zero_extendsidi2 (operands[0], tmp));
      DONE;
    }
  else if (!TARGET_EXTIMM)
    {
      rtx bitcount = GEN_INT (64 - <HQI:bitsize>);
      operands[1] = gen_lowpart (DImode, operands[1]);
      emit_insn (gen_ashldi3 (operands[0], operands[1], bitcount));
      emit_insn (gen_lshrdi3 (operands[0], operands[0], bitcount));
      DONE;
    }
})

(define_expand "zero_extend<mode>si2"
  [(set (match_operand:SI 0 "register_operand" "")
        (zero_extend:SI (match_operand:HQI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_EXTIMM)
    {
      operands[1] = gen_lowpart (SImode, operands[1]);
      emit_insn (gen_andsi3 (operands[0], operands[1],
                             GEN_INT ((1 << <HQI:bitsize>) - 1)));
      DONE;
    }
})

; llhrl, llghrl
(define_insn "*zero_extendhi<mode>2_z10"
  [(set (match_operand:GPR 0 "register_operand" "=d,d,d")
        (zero_extend:GPR (match_operand:HI 1 "nonimmediate_operand" "d,RT,b")))]
  "TARGET_Z10"
  "@
   ll<g>hr\t%0,%1
   ll<g>h\t%0,%1
   ll<g>hrl\t%0,%1"
  [(set_attr "op_type"      "RXY,RRE,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_fwd_A3,z10_fwd_A3")])

; llhr, llcr, llghr, llgcr, llh, llc, llgh, llgc
(define_insn "*zero_extend<HQI:mode><GPR:mode>2_extimm"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (zero_extend:GPR (match_operand:HQI 1 "nonimmediate_operand" "d,RT")))]
  "TARGET_EXTIMM"
  "@
   ll<g><hc>r\t%0,%1
   ll<g><hc>\t%0,%1"
  [(set_attr "op_type" "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_fwd_A3")])

; llgh, llgc
(define_insn "*zero_extend<HQI:mode><GPR:mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (zero_extend:GPR (match_operand:HQI 1 "memory_operand" "RT")))]
  "TARGET_ZARCH && !TARGET_EXTIMM"
  "llg<hc>\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_fwd_A3")])

(define_insn_and_split "*zero_extendhisi2_31"
  [(set (match_operand:SI 0 "register_operand" "=&d")
        (zero_extend:SI (match_operand:HI 1 "s_operand" "QS")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (parallel
    [(set (strict_low_part (match_dup 2)) (match_dup 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[2] = gen_lowpart (HImode, operands[0]);")

(define_insn_and_split "*zero_extendqisi2_31"
  [(set (match_operand:SI 0 "register_operand" "=&d")
        (zero_extend:SI (match_operand:QI 1 "memory_operand" "RT")))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (set (strict_low_part (match_dup 2)) (match_dup 1))]
  "operands[2] = gen_lowpart (QImode, operands[0]);")

;
; zero_extendqihi2 instruction pattern(s).
;

(define_expand "zero_extendqihi2"
  [(set (match_operand:HI 0 "register_operand" "")
        (zero_extend:HI (match_operand:QI 1 "register_operand" "")))]
  "TARGET_ZARCH && !TARGET_EXTIMM"
{
  operands[1] = gen_lowpart (HImode, operands[1]);
  emit_insn (gen_andhi3 (operands[0], operands[1], GEN_INT (0xff)));
  DONE;
})

(define_insn "*zero_extendqihi2_64"
  [(set (match_operand:HI 0 "register_operand" "=d")
        (zero_extend:HI (match_operand:QI 1 "memory_operand" "RT")))]
  "TARGET_ZARCH && !TARGET_EXTIMM"
  "llgc\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_fwd_A3")])

(define_insn_and_split "*zero_extendqihi2_31"
  [(set (match_operand:HI 0 "register_operand" "=&d")
        (zero_extend:HI (match_operand:QI 1 "memory_operand" "RT")))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (set (strict_low_part (match_dup 2)) (match_dup 1))]
  "operands[2] = gen_lowpart (QImode, operands[0]);")

;
; fixuns_trunc(dd|td)di2 instruction pattern(s).
;

(define_expand "fixuns_truncdddi2"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
          (unsigned_fix:DI (match_operand:DD 1 "register_operand" "")))
     (unspec:DI [(const_int 5)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]

  "TARGET_HARD_DFP"
{
  if (!TARGET_Z196)
    {
      rtx_code_label *label1 = gen_label_rtx ();
      rtx_code_label *label2 = gen_label_rtx ();
      rtx temp = gen_reg_rtx (TDmode);
      REAL_VALUE_TYPE cmp, sub;

      decimal_real_from_string (&cmp, "9223372036854775808.0");  /* 2^63 */
      decimal_real_from_string (&sub, "18446744073709551616.0"); /* 2^64 */

      /* 2^63 can't be represented as 64bit DFP number with full precision.  The
         solution is doing the check and the subtraction in TD mode and using a
         TD -> DI convert afterwards.  */
      emit_insn (gen_extendddtd2 (temp, operands[1]));
      temp = force_reg (TDmode, temp);
      emit_cmp_and_jump_insns (temp,
            CONST_DOUBLE_FROM_REAL_VALUE (cmp, TDmode),
            LT, NULL_RTX, VOIDmode, 0, label1);
      emit_insn (gen_subtd3 (temp, temp,
            CONST_DOUBLE_FROM_REAL_VALUE (sub, TDmode)));
      emit_insn (gen_fix_trunctddi2_dfp (operands[0], temp, GEN_INT (11)));
      emit_jump (label2);

      emit_label (label1);
      emit_insn (gen_fix_truncdddi2_dfp (operands[0], operands[1], GEN_INT (9)));
      emit_label (label2);
      DONE;
    }
})

(define_expand "fixuns_trunctddi2"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
          (unsigned_fix:DI (match_operand:TD 1 "register_operand" "")))
     (unspec:DI [(const_int 5)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]

  "TARGET_HARD_DFP"
{
  if (!TARGET_Z196)
    {
      rtx_code_label *label1 = gen_label_rtx ();
      rtx_code_label *label2 = gen_label_rtx ();
      rtx temp = gen_reg_rtx (TDmode);
      REAL_VALUE_TYPE cmp, sub;

      operands[1] = force_reg (TDmode, operands[1]);
      decimal_real_from_string (&cmp, "9223372036854775808.0");  /* 2^63 */
      decimal_real_from_string (&sub, "18446744073709551616.0"); /* 2^64 */

      emit_cmp_and_jump_insns (operands[1],
            CONST_DOUBLE_FROM_REAL_VALUE (cmp, TDmode),
            LT, NULL_RTX, VOIDmode, 0, label1);
      emit_insn (gen_subtd3 (temp, operands[1],
            CONST_DOUBLE_FROM_REAL_VALUE (sub, TDmode)));
      emit_insn (gen_fix_trunctddi2_dfp (operands[0], temp, GEN_INT (11)));
      emit_jump (label2);

      emit_label (label1);
      emit_insn (gen_fix_trunctddi2_dfp (operands[0], operands[1], GEN_INT (9)));
      emit_label (label2);
      DONE;
    }
})

;
; fixuns_trunc(sf|df|tf)(si|di)2 and fix_trunc(sf|df|tf)(si|di)2
; instruction pattern(s).
;

(define_expand "fixuns_trunc<BFP:mode><GPR:mode>2"
  [(parallel
    [(set (match_operand:GPR 0 "register_operand" "")
          (unsigned_fix:GPR (match_operand:BFP 1 "register_operand" "")))
     (unspec:GPR [(const_int 5)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
{
  if (!TARGET_Z196)
    {
      rtx_code_label *label1 = gen_label_rtx ();
      rtx_code_label *label2 = gen_label_rtx ();
      rtx temp = gen_reg_rtx (<BFP:MODE>mode);
      REAL_VALUE_TYPE cmp, sub;

      operands[1] = force_reg (<BFP:MODE>mode, operands[1]);
      real_2expN (&cmp, <GPR:bitsize> - 1, <BFP:MODE>mode);
      real_2expN (&sub, <GPR:bitsize>, <BFP:MODE>mode);

      emit_cmp_and_jump_insns (operands[1],
            CONST_DOUBLE_FROM_REAL_VALUE (cmp, <BFP:MODE>mode),
            LT, NULL_RTX, VOIDmode, 0, label1);
      emit_insn (gen_sub<BFP:mode>3 (temp, operands[1],
            CONST_DOUBLE_FROM_REAL_VALUE (sub, <BFP:MODE>mode)));
      emit_insn (gen_fix_trunc<BFP:mode><GPR:mode>2_bfp (operands[0], temp,
            GEN_INT (7)));
      emit_jump (label2);

      emit_label (label1);
      emit_insn (gen_fix_trunc<BFP:mode><GPR:mode>2_bfp (operands[0],
            operands[1], GEN_INT (5)));
      emit_label (label2);
      DONE;
    }
})

; fixuns_trunc(td|dd)si2 expander
(define_expand "fixuns_trunc<mode>si2"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (unsigned_fix:SI (match_operand:DFP 1 "register_operand" "")))
     (unspec:SI [(const_int 5)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z196 && TARGET_HARD_DFP"
  "")

; fixuns_trunc(tf|df|sf|td|dd)(di|si)2 instruction patterns.

(define_insn "*fixuns_truncdfdi2_z13"
  [(set (match_operand:DI                  0 "register_operand" "=d,v")
        (unsigned_fix:DI (match_operand:DF 1 "register_operand"  "f,v")))
   (unspec:DI [(match_operand:DI           2 "immediate_operand" "K,K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
   "TARGET_Z13 && TARGET_HARD_FLOAT"
   "@
    clgdbr\t%0,%h2,%1,0
    wclgdb\t%v0,%v1,0,%h2"
   [(set_attr "op_type" "RRF,VRR")
    (set_attr "type"    "ftoi")])

; clfebr, clfdbr, clfxbr, clgebr, clgdbr, clgxbr
;         clfdtr, clfxtr,         clgdtr, clgxtr
(define_insn "*fixuns_trunc<FP:mode><GPR:mode>2_z196"
  [(set (match_operand:GPR                  0 "register_operand" "=d")
        (unsigned_fix:GPR (match_operand:FP 1 "register_operand"  "f")))
   (unspec:GPR [(match_operand:GPR          2 "immediate_operand" "K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
   "TARGET_Z196 && TARGET_HARD_FLOAT
    && (!TARGET_Z13 || <GPR:MODE>mode != DImode || <FP:MODE>mode != DFmode)"
   "cl<GPR:gf><FP:xde><FP:bt>r\t%0,%h2,%1,0"
   [(set_attr "op_type" "RRF")
    (set_attr "type"    "ftoi")])

(define_expand "fix_trunc<DSF:mode><GPR:mode>2"
  [(set (match_operand:GPR 0 "register_operand" "")
        (fix:GPR (match_operand:DSF 1 "register_operand" "")))]
  "TARGET_HARD_FLOAT"
{
  emit_insn (gen_fix_trunc<DSF:mode><GPR:mode>2_bfp (operands[0], operands[1],
             GEN_INT (5)));
  DONE;
})

(define_insn "*fix_truncdfdi2_bfp_z13"
  [(set (match_operand:DI         0 "register_operand" "=d,v")
        (fix:DI (match_operand:DF 1 "register_operand"  "f,v")))
   (unspec:DI [(match_operand:DI  2 "immediate_operand" "K,K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z13 && TARGET_HARD_FLOAT"
  "@
   cgdbr\t%0,%h2,%1
   wcgdb\t%v0,%v1,0,%h2"
  [(set_attr "op_type" "RRE,VRR")
   (set_attr "type"    "ftoi")])

; cgxbr, cgdbr, cgebr, cfxbr, cfdbr, cfebr
(define_insn "*fix_trunc<BFP:mode><GPR:mode>2_bfp"
  [(set (match_operand:GPR          0 "register_operand" "=d")
        (fix:GPR (match_operand:BFP 1 "register_operand"  "f")))
   (unspec:GPR [(match_operand:GPR  2 "immediate_operand" "K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT
    && (!TARGET_VX || <GPR:MODE>mode != DImode || <BFP:MODE>mode != DFmode)"
  "c<GPR:gf><BFP:xde>br\t%0,%h2,%1"
  [(set_attr "op_type" "RRE")
   (set_attr "type"    "ftoi")])

(define_expand "fix_trunc<BFP:mode><GPR:mode>2_bfp"
  [(parallel
    [(set (match_operand:GPR          0 "register_operand" "=d")
          (fix:GPR (match_operand:BFP 1 "register_operand"  "f")))
     (unspec:GPR [(match_operand:GPR  2 "immediate_operand" "K")] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT")
;
; fix_trunc(td|dd)di2 instruction pattern(s).
;

(define_expand "fix_trunc<mode>di2"
  [(set (match_operand:DI 0 "register_operand" "")
        (fix:DI (match_operand:DFP 1 "nonimmediate_operand" "")))]
  "TARGET_ZARCH && TARGET_HARD_DFP"
{
  operands[1] = force_reg (<MODE>mode, operands[1]);
  emit_insn (gen_fix_trunc<mode>di2_dfp (operands[0], operands[1],
      GEN_INT (9)));
  DONE;
})

; cgxtr, cgdtr
(define_insn "fix_trunc<DFP:mode>di2_dfp"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (fix:DI (match_operand:DFP 1 "register_operand" "f")))
   (unspec:DI [(match_operand:DI 2 "immediate_operand" "K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && TARGET_HARD_DFP"
  "cg<DFP:xde>tr\t%0,%h2,%1"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "ftoidfp")])


;
; fix_trunctf(si|di)2 instruction pattern(s).
;

(define_expand "fix_trunctf<mode>2"
  [(parallel [(set (match_operand:GPR 0 "register_operand" "")
                   (fix:GPR (match_operand:TF 1 "register_operand" "")))
              (unspec:GPR [(const_int 5)] UNSPEC_ROUND)
              (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
  "")


;
; float(si|di)(tf|df|sf|td|dd)2 instruction pattern(s).
;

; cxgbr, cdgbr, cegbr, cxgtr, cdgtr
(define_insn "floatdi<mode>2"
  [(set (match_operand:FP           0 "register_operand" "=f,<vf>")
        (float:FP (match_operand:DI 1 "register_operand"  "d,<vd>")))]
  "TARGET_ZARCH && TARGET_HARD_FLOAT"
  "@
   c<xde>g<bt>r\t%0,%1
   wcdgb\t%v0,%v1,0,0"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "type"         "itof<mode>" )
   (set_attr "cpu_facility" "*,vec")])

; cxfbr, cdfbr, cefbr
(define_insn "floatsi<mode>2"
  [(set (match_operand:BFP 0 "register_operand" "=f")
        (float:BFP (match_operand:SI 1 "register_operand" "d")))]
  "TARGET_HARD_FLOAT"
  "c<xde>fbr\t%0,%1"
  [(set_attr "op_type" "RRE")
   (set_attr "type"   "itof<mode>" )])

; cxftr, cdftr
(define_insn "floatsi<mode>2"
  [(set (match_operand:DFP 0 "register_operand" "=f")
        (float:DFP (match_operand:SI 1 "register_operand" "d")))]
  "TARGET_Z196 && TARGET_HARD_FLOAT"
  "c<xde>ftr\t%0,0,%1,0"
  [(set_attr "op_type" "RRE")
   (set_attr "type"   "itof<mode>" )])

;
; floatuns(si|di)(tf|df|sf|td|dd)2 instruction pattern(s).
;

(define_insn "*floatunsdidf2_z13"
  [(set (match_operand:DF                    0 "register_operand" "=f,v")
        (unsigned_float:DF (match_operand:DI 1 "register_operand"  "d,v")))]
  "TARGET_Z13 && TARGET_HARD_FLOAT"
  "@
   cdlgbr\t%0,0,%1,0
   wcdlgb\t%v0,%v1,0,0"
  [(set_attr "op_type" "RRE,VRR")
   (set_attr "type"    "itofdf")])

; cxlgbr, cdlgbr, celgbr, cxlgtr, cdlgtr
; cxlfbr, cdlfbr, celfbr, cxlftr, cdlftr
(define_insn "*floatuns<GPR:mode><FP:mode>2"
  [(set (match_operand:FP                     0 "register_operand" "=f")
        (unsigned_float:FP (match_operand:GPR 1 "register_operand"  "d")))]
  "TARGET_Z196 && TARGET_HARD_FLOAT
   && (!TARGET_VX || <FP:MODE>mode != DFmode || <GPR:MODE>mode != DImode)"
  "c<FP:xde>l<GPR:gf><FP:bt>r\t%0,0,%1,0"
  [(set_attr "op_type" "RRE")
   (set_attr "type"    "itof<FP:mode>")])

(define_expand "floatuns<GPR:mode><FP:mode>2"
  [(set (match_operand:FP                     0 "register_operand" "")
        (unsigned_float:FP (match_operand:GPR 1 "register_operand" "")))]
  "TARGET_Z196 && TARGET_HARD_FLOAT")

;
; truncdfsf2 instruction pattern(s).
;

(define_insn "truncdfsf2"
  [(set (match_operand:SF                    0 "register_operand" "=f,v")
        (float_truncate:SF (match_operand:DF 1 "register_operand"  "f,v")))]
  "TARGET_HARD_FLOAT"
  "@
   ledbr\t%0,%1
   wledb\t%v0,%v1,0,0" ; IEEE inexact exception not suppressed
                       ; According to BFP rounding mode
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "type"         "ftruncdf")
   (set_attr "cpu_facility" "*,vec")])

;
; trunctf(df|sf)2 instruction pattern(s).
;

; ldxbr, lexbr
(define_insn "trunctf<mode>2"
  [(set (match_operand:DSF 0 "register_operand" "=f")
        (float_truncate:DSF (match_operand:TF 1 "register_operand" "f")))
   (clobber (match_scratch:TF 2 "=f"))]
  "TARGET_HARD_FLOAT"
  "l<xde>xbr\t%2,%1\;l<xde>r\t%0,%2"
  [(set_attr "length" "6")
   (set_attr "type"   "ftrunctf")])

;
; trunctddd2 and truncddsd2 instruction pattern(s).
;

(define_insn "trunctddd2"
  [(set (match_operand:DD 0 "register_operand" "=f")
        (float_truncate:DD (match_operand:TD 1 "register_operand" "f")))
   (clobber (match_scratch:TD 2 "=f"))]
  "TARGET_HARD_DFP"
  "ldxtr\t%2,0,%1,0\;ldr\t%0,%2"
  [(set_attr "length"  "6")
   (set_attr "type"    "ftruncdd")])

(define_insn "truncddsd2"
  [(set (match_operand:SD 0 "register_operand" "=f")
        (float_truncate:SD (match_operand:DD 1 "register_operand" "f")))]
  "TARGET_HARD_DFP"
  "ledtr\t%0,0,%1,0"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "ftruncsd")])

(define_expand "trunctdsd2"
  [(parallel
    [(set (match_dup 3)
          (float_truncate:DD (match_operand:TD 1 "register_operand" "")))
     (clobber (match_scratch:TD 2 ""))])
   (set (match_operand:SD 0 "register_operand" "")
        (float_truncate:SD (match_dup 3)))]
  "TARGET_HARD_DFP"
{
  operands[3] = gen_reg_rtx (DDmode);
})

;
; extend(sf|df)(df|tf)2 instruction pattern(s).
;

(define_insn "*extendsfdf2_z13"
  [(set (match_operand:DF                  0 "register_operand"     "=f,f,v")
        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand"  "f,R,v")))]
  "TARGET_Z13 && TARGET_HARD_FLOAT"
  "@
   ldebr\t%0,%1
   ldeb\t%0,%1
   wldeb\t%v0,%v1"
  [(set_attr "op_type" "RRE,RXE,VRR")
   (set_attr "type"    "fsimpdf, floaddf,fsimpdf")])

; ldebr, ldeb, lxdbr, lxdb, lxebr, lxeb
(define_insn "*extend<DSF:mode><BFP:mode>2"
  [(set (match_operand:BFP                   0 "register_operand"     "=f,f")
        (float_extend:BFP (match_operand:DSF 1 "nonimmediate_operand"  "f,R")))]
  "TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<BFP:MODE>mode) > GET_MODE_SIZE (<DSF:MODE>mode)
   && (!TARGET_VX || <BFP:MODE>mode != DFmode || <DSF:MODE>mode != SFmode)"
  "@
   l<BFP:xde><DSF:xde>br\t%0,%1
   l<BFP:xde><DSF:xde>b\t%0,%1"
  [(set_attr "op_type" "RRE,RXE")
   (set_attr "type"    "fsimp<BFP:mode>, fload<BFP:mode>")])

(define_expand "extend<DSF:mode><BFP:mode>2"
  [(set (match_operand:BFP                   0 "register_operand"     "")
        (float_extend:BFP (match_operand:DSF 1 "nonimmediate_operand" "")))]
  "TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<BFP:MODE>mode) > GET_MODE_SIZE (<DSF:MODE>mode)")

;
; extendddtd2 and extendsddd2 instruction pattern(s).
;

(define_insn "extendddtd2"
  [(set (match_operand:TD 0 "register_operand" "=f")
        (float_extend:TD (match_operand:DD 1 "register_operand" "f")))]
  "TARGET_HARD_DFP"
  "lxdtr\t%0,%1,0"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "fsimptf")])

(define_insn "extendsddd2"
  [(set (match_operand:DD 0 "register_operand" "=f")
        (float_extend:DD (match_operand:SD 1 "register_operand" "f")))]
  "TARGET_HARD_DFP"
  "ldetr\t%0,%1,0"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "fsimptf")])

(define_expand "extendsdtd2"
  [(set (match_dup 2)
        (float_extend:DD (match_operand:SD 1 "register_operand" "")))
   (set (match_operand:TD 0 "register_operand" "")
        (float_extend:TD (match_dup 2)))]
  "TARGET_HARD_DFP"
{
  operands[2] = gen_reg_rtx (DDmode);
})

; Binary Floating Point - load fp integer

; Expanders for: floor, btrunc, round, ceil, and nearbyint
; For all of them the inexact exceptions are suppressed.

; fiebra, fidbra, fixbra
(define_insn "<FPINT:fpint_name><BFP:mode>2"
  [(set (match_operand:BFP 0 "register_operand" "=f")
        (unspec:BFP [(match_operand:BFP 1 "register_operand" "f")]
                    FPINT))]
  "TARGET_Z196"
  "fi<BFP:xde>bra\t%0,<FPINT:fpint_roundingmode>,%1,4"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<BFP:mode>")])

; rint is supposed to raise an inexact exception so we can use the
; older instructions.

; fiebr, fidbr, fixbr
(define_insn "rint<BFP:mode>2"
  [(set (match_operand:BFP 0 "register_operand" "=f")
        (unspec:BFP [(match_operand:BFP 1 "register_operand" "f")]
                    UNSPEC_FPINT_RINT))]
  ""
  "fi<BFP:xde>br\t%0,0,%1"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<BFP:mode>")])


; Decimal Floating Point - load fp integer

; fidtr, fixtr
(define_insn "<FPINT:fpint_name><DFP:mode>2"
  [(set (match_operand:DFP 0 "register_operand" "=f")
        (unspec:DFP [(match_operand:DFP 1 "register_operand" "f")]
                    FPINT))]
  "TARGET_HARD_DFP"
  "fi<DFP:xde>tr\t%0,<FPINT:fpint_roundingmode>,%1,4"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<DFP:mode>")])

; fidtr, fixtr
(define_insn "rint<DFP:mode>2"
  [(set (match_operand:DFP 0 "register_operand" "=f")
        (unspec:DFP [(match_operand:DFP 1 "register_operand" "f")]
                    UNSPEC_FPINT_RINT))]
  "TARGET_HARD_DFP"
  "fi<DFP:xde>tr\t%0,0,%1,0"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<DFP:mode>")])

;
; Binary <-> Decimal floating point trunc patterns
;

(define_insn "*trunc<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:DFP_ALL FPR0_REGNUM)
        (float_truncate:DFP_ALL (reg:BFP FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_insn "*trunc<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:BFP FPR0_REGNUM)
        (float_truncate:BFP (reg:DFP_ALL FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_expand "trunc<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:BFP FPR4_REGNUM) (match_operand:BFP 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:DFP_ALL FPR0_REGNUM)
          (float_truncate:DFP_ALL (reg:BFP FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))])
   (set (match_operand:DFP_ALL 0 "nonimmediate_operand" "")
        (reg:DFP_ALL FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<BFP:MODE>mode) > GET_MODE_SIZE (<DFP_ALL:MODE>mode)"
{
  HOST_WIDE_INT flags;

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP1_TYPE_SHIFT);

  operands[2] = GEN_INT (flags);
})

(define_expand "trunc<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:DFP_ALL FPR4_REGNUM)
        (match_operand:DFP_ALL 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:BFP FPR0_REGNUM) (float_truncate:BFP (reg:DFP_ALL FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))])
   (set (match_operand:BFP 0 "nonimmediate_operand" "") (reg:BFP FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<DFP_ALL:MODE>mode) >= GET_MODE_SIZE (<BFP:MODE>mode)"
{
  HOST_WIDE_INT flags;

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP1_TYPE_SHIFT);

  operands[2] = GEN_INT (flags);
})

;
; Binary <-> Decimal floating point extend patterns
;

(define_insn "*extend<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:DFP_ALL FPR0_REGNUM) (float_extend:DFP_ALL (reg:BFP FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_insn "*extend<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:BFP FPR0_REGNUM) (float_extend:BFP (reg:DFP_ALL FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_expand "extend<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:BFP FPR4_REGNUM) (match_operand:BFP 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:DFP_ALL FPR0_REGNUM)
          (float_extend:DFP_ALL (reg:BFP FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))])
   (set (match_operand:DFP_ALL 0 "nonimmediate_operand" "")
        (reg:DFP_ALL FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<BFP:MODE>mode) <= GET_MODE_SIZE (<DFP_ALL:MODE>mode)"
{
  HOST_WIDE_INT flags;

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP1_TYPE_SHIFT);

  operands[2] = GEN_INT (flags);
})

(define_expand "extend<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:DFP_ALL FPR4_REGNUM)
        (match_operand:DFP_ALL 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:BFP FPR0_REGNUM) (float_extend:BFP (reg:DFP_ALL FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))])
   (set (match_operand:BFP 0 "nonimmediate_operand" "") (reg:BFP FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<DFP_ALL:MODE>mode) < GET_MODE_SIZE (<BFP:MODE>mode)"
{
  HOST_WIDE_INT flags;

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP1_TYPE_SHIFT);

  operands[2] = GEN_INT (flags);
})


;;
;; ARITHMETIC OPERATIONS
;;
;  arithmetic operations set the ConditionCode,
;  because of unpredictable Bits in Register for Halfword and Byte
;  the ConditionCode can be set wrong in operations for Halfword and Byte

;;
;;- Add instructions.
;;

;
; addti3 instruction pattern(s).
;

(define_expand "addti3"
  [(parallel
    [(set (match_operand:TI          0 "register_operand"     "")
          (plus:TI (match_operand:TI 1 "nonimmediate_operand" "")
                   (match_operand:TI 2 "general_operand"      "") ) )
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_ZARCH"
{
  /* For z13 we have vaq which doesn't set CC.  */
  if (TARGET_VX)
    {
      emit_insn (gen_rtx_SET (operands[0],
                              gen_rtx_PLUS (TImode,
                                            copy_to_mode_reg (TImode, operands[1]),
                                            copy_to_mode_reg (TImode, operands[2]))));
      DONE;
    }
})

(define_insn_and_split "*addti3"
  [(set (match_operand:TI          0 "register_operand"    "=&d")
        (plus:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
                 (match_operand:TI 2 "general_operand"      "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL1 CC_REGNUM)
          (compare:CCL1 (plus:DI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (plus:DI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (plus:DI
                          (plus:DI (ltu:DI (reg:CCL1 CC_REGNUM) (const_int 0))
                                   (match_dup 4)) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, TImode);
   operands[4] = operand_subword (operands[1], 0, 0, TImode);
   operands[5] = operand_subword (operands[2], 0, 0, TImode);
   operands[6] = operand_subword (operands[0], 1, 0, TImode);
   operands[7] = operand_subword (operands[1], 1, 0, TImode);
   operands[8] = operand_subword (operands[2], 1, 0, TImode);"
  [(set_attr "op_type"  "*")
   (set_attr "cpu_facility" "*")])

;
; adddi3 instruction pattern(s).
;

(define_expand "adddi3"
  [(parallel
    [(set (match_operand:DI 0 "nonimmediate_operand" "")
          (plus:DI (match_operand:DI 1 "nonimmediate_operand" "")
                   (match_operand:DI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*adddi3_sign"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (plus:DI (sign_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))
                 (match_operand:DI 1 "register_operand" "0,0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   agfr\t%0,%2
   agf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z196prop" "z196_cracked,z196_cracked")])

(define_insn "*adddi3_zero_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))
                          (match_operand:DI 1 "register_operand" "0,0"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d,d")
        (plus:DI (zero_extend:DI (match_dup 2)) (match_dup 1)))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   algfr\t%0,%2
   algf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*adddi3_zero_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))
                          (match_operand:DI 1 "register_operand" "0,0"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d,d"))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   algfr\t%0,%2
   algf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*adddi3_zero"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))
                 (match_operand:DI 1 "register_operand" "0,0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   algfr\t%0,%2
   algf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn_and_split "*adddi3_31z"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=&d")
        (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
                 (match_operand:DI 2 "general_operand" "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && TARGET_CPU_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL1 CC_REGNUM)
          (compare:CCL1 (plus:SI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (plus:SI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (plus:SI
                          (plus:SI (ltu:SI (reg:CCL1 CC_REGNUM) (const_int 0))
                                   (match_dup 4)) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
   operands[4] = operand_subword (operands[1], 0, 0, DImode);
   operands[5] = operand_subword (operands[2], 0, 0, DImode);
   operands[6] = operand_subword (operands[0], 1, 0, DImode);
   operands[7] = operand_subword (operands[1], 1, 0, DImode);
   operands[8] = operand_subword (operands[2], 1, 0, DImode);")

(define_insn_and_split "*adddi3_31"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=&d")
        (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
                 (match_operand:DI 2 "general_operand" "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_CPU_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 3) (plus:SI (match_dup 4) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (reg:CCL1 CC_REGNUM)
          (compare:CCL1 (plus:SI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (plus:SI (match_dup 7) (match_dup 8)))])
   (set (pc)
        (if_then_else (ltu (reg:CCL1 CC_REGNUM) (const_int 0))
                      (pc)
                      (label_ref (match_dup 9))))
   (parallel
    [(set (match_dup 3) (plus:SI (match_dup 3) (const_int 1)))
     (clobber (reg:CC CC_REGNUM))])
   (match_dup 9)]
  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
   operands[4] = operand_subword (operands[1], 0, 0, DImode);
   operands[5] = operand_subword (operands[2], 0, 0, DImode);
   operands[6] = operand_subword (operands[0], 1, 0, DImode);
   operands[7] = operand_subword (operands[1], 1, 0, DImode);
   operands[8] = operand_subword (operands[2], 1, 0, DImode);
   operands[9] = gen_label_rtx ();")

;
; addsi3 instruction pattern(s).
;

(define_expand "addsi3"
  [(parallel
    [(set (match_operand:SI 0 "nonimmediate_operand" "")
          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "")
                   (match_operand:SI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*addsi3_sign"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (plus:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))
                 (match_operand:SI 1 "register_operand" "0,0")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   ah\t%0,%2
   ahy\t%0,%2"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "z196prop" "z196_cracked,z196_cracked")])

;
; add(di|si)3 instruction pattern(s).
;

; ark, agrk, ar, ahi, ahik, aghik, alfi, slfi, a, ay, agr, aghi, algfi, slgfi, ag, asi, agsi
(define_insn "*add<mode>3"
  [(set (match_operand:GPR 0 "nonimmediate_operand"           "=d,d,d,d, d, d,d,d,QS")
        (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,d, 0, 0,0,0, 0")
                  (match_operand:GPR 2 "general_operand"      " d,d,K,K,Op,On,R,T, C") ) )
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   a<g>r\t%0,%2
   a<g>rk\t%0,%1,%2
   a<g>hi\t%0,%h2
   a<g>hik\t%0,%1,%h2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   a<g>\t%0,%2
   a<y>\t%0,%2
   a<g>si\t%0,%c2"
  [(set_attr "op_type"  "RR<E>,RRF,RI,RIE,RIL,RIL,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,*,z196,extimm,extimm,*,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,*,z10_super_E1,z10_super_E1,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, alfi, slfi, al, aly, alrk, alhsik, algr, algfi, slgfi, alg, alsi, algsi, algrk, alghsik
(define_insn "*add<mode>3_carry1_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d, 0, 0,d,0,0,0")
                           (match_operand:GPR 2 "general_operand"      " d,d,Op,On,K,R,T,C"))
                 (match_dup 1)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d,d, d, d,d,d,d,d")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   al<g>hsik\t%0,%1,%h2
   al<g>\t%0,%2
   al<y>\t%0,%2
   al<g>si\t%0,%c2"
  [(set_attr "op_type"      "RR<E>,RRF,RIL,RIL,RIE,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,extimm,z196,*,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_carry1_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,0")
                           (match_operand:GPR 2 "general_operand"       "d,d,R,T"))
                 (match_dup 1)))
   (clobber (match_scratch:GPR 0                                       "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; alr, alfi, slfi, al, aly, algr, algfi, slgfi, alg, alsi, algsi, alrk, algrk, alhsik, alghsik
(define_insn "*add<mode>3_carry2_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d, 0, 0,d,0,0, 0")
                           (match_operand:GPR 2 "general_operand"      " d,d,Op,On,K,R,T, C"))
                 (match_dup 2)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d,d, d, d,d,d,d,RS")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   al<g>hsik\t%0,%1,%h2
   al<g>\t%0,%2
   al<y>\t%0,%2
   al<g>si\t%0,%c2"
  [(set_attr "op_type"  "RR<E>,RRF,RIL,RIL,RIE,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,extimm,z196,*,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_carry2_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,0")
                           (match_operand:GPR 2 "general_operand"       "d,d,R,T"))
                 (match_dup 2)))
   (clobber (match_scratch:GPR 0                                       "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; alr, alfi, slfi, al, aly, algr, algfi, slgfi, alg, alsi, algsi, alrk, algrk, alhsik, alghsik
(define_insn "*add<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d, 0, 0,d,0,0, 0")
                           (match_operand:GPR 2 "general_operand"      " d,d,Op,On,K,R,T, C"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d,d, d, d,d,d,d,RS")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   al<g>hsik\t%0,%1,%h2
   al<g>\t%0,%2
   al<y>\t%0,%2
   al<g>si\t%0,%c2"
  [(set_attr "op_type"  "RR<E>,RRF,RIL,RIL,RIE,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,extimm,z196,*,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1,
                        *,z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,0")
                           (match_operand:GPR 2 "general_operand"       "d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0                                       "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_cconly2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 1 "nonimmediate_operand"    "%0,d,0,0")
                 (neg:GPR (match_operand:GPR 2 "general_operand" "d,d,R,T"))))
   (clobber (match_scratch:GPR 0                                "=d,d,d,d"))]
  "s390_match_ccmode(insn, CCLmode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; ahi, afi, aghi, agfi, asi, agsi
(define_insn "*add<mode>3_imm_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" " 0, d,0, 0")
                           (match_operand:GPR 2 "const_int_operand"    " K, K,Os, C"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d, d,d,QS")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCAmode)
   && (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")
       || (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'O', \"Os\")
           /* Avoid INT32_MIN on 32 bit.  */
           && (!TARGET_ZARCH || INTVAL (operands[2]) != -0x7fffffff - 1)))"
  "@
   a<g>hi\t%0,%h2
   a<g>hik\t%0,%1,%h2
   a<g>fi\t%0,%2
   a<g>si\t%0,%c2"
  [(set_attr "op_type"      "RI,RIE,RIL,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

;
; add(tf|df|sf|td|dd)3 instruction pattern(s).
;

; axbr, adbr, aebr, axb, adb, aeb, adtr, axtr
; FIXME: wfadb does not clobber cc
(define_insn "add<mode>3"
  [(set (match_operand:FP 0 "register_operand"                 "=f,   f,<vf>")
        (plus:FP (match_operand:FP 1 "nonimmediate_operand" "%<f0>,   0,<v0>")
                 (match_operand:FP 2 "general_operand"          "f,<Rf>,<vf>")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   a<xde><bt>r\t%0,<op1>%2
   a<xde>b\t%0,%2
   wfadb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "<RRer>,RXE,VRR")
   (set_attr "type"         "fsimp<mode>")
   (set_attr "cpu_facility" "*,*,vec")])

; axbr, adbr, aebr, axb, adb, aeb, adtr, axtr
(define_insn "*add<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:FP (match_operand:FP 1 "nonimmediate_operand" "%<f0>,0")
                          (match_operand:FP 2 "general_operand"      " f,<Rf>"))
                 (match_operand:FP 3 "const0_operand" "")))
   (set (match_operand:FP 0 "register_operand" "=f,f")
        (plus:FP (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   a<xde><bt>r\t%0,<op1>%2
   a<xde>b\t%0,%2"
  [(set_attr "op_type"  "<RRer>,RXE")
   (set_attr "type"     "fsimp<mode>")])

; axbr, adbr, aebr, axb, adb, aeb, adtr, axtr
(define_insn "*add<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:FP (match_operand:FP 1 "nonimmediate_operand" "%<f0>,0")
                           (match_operand:FP 2 "general_operand"      " f,<Rf>"))
                 (match_operand:FP 3 "const0_operand" "")))
   (clobber (match_scratch:FP 0 "=f,f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   a<xde><bt>r\t%0,<op1>%2
   a<xde>b\t%0,%2"
  [(set_attr "op_type"  "<RRer>,RXE")
   (set_attr "type"     "fsimp<mode>")])

;
; Pointer add instruction patterns
;

; This will match "*la_64"
(define_expand "addptrdi3"
  [(set (match_operand:DI 0 "register_operand" "")
        (plus:DI (match_operand:DI 1 "register_operand" "")
                 (match_operand:DI 2 "nonmemory_operand" "")))]
  "TARGET_64BIT"
{
  if (GET_CODE (operands[2]) == CONST_INT)
    {
      HOST_WIDE_INT c = INTVAL (operands[2]);

      if (!CONST_OK_FOR_CONSTRAINT_P (c, 'K', "K")
          && !CONST_OK_FOR_CONSTRAINT_P (c, 'O', "Os"))
        {
          operands[2] = force_const_mem (DImode, operands[2]);
          operands[2] = force_reg (DImode, operands[2]);
        }
      else if (!DISP_IN_RANGE (INTVAL (operands[2])))
        operands[2] = force_reg (DImode, operands[2]);
    }
})

; For 31 bit we have to prevent the generated pattern from matching
; normal ADDs since la only does a 31 bit add.  This is supposed to
; match "force_la_31".
(define_expand "addptrsi3"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (plus:SI (match_operand:SI 1 "register_operand" "")
                   (match_operand:SI 2 "nonmemory_operand" "")))
                   (use (const_int 0))])]
  "!TARGET_64BIT"
{
  if (GET_CODE (operands[2]) == CONST_INT)
    {
      HOST_WIDE_INT c = INTVAL (operands[2]);

      if (!CONST_OK_FOR_CONSTRAINT_P (c, 'K', "K")
          && !CONST_OK_FOR_CONSTRAINT_P (c, 'O', "Os"))
        {
          operands[2] = force_const_mem (SImode, operands[2]);
          operands[2] = force_reg (SImode, operands[2]);
        }
      else if (!DISP_IN_RANGE (INTVAL (operands[2])))
        operands[2] = force_reg (SImode, operands[2]);
    }
})

;;
;;- Subtract instructions.
;;

;
; subti3 instruction pattern(s).
;

(define_expand "subti3"
  [(parallel
    [(set (match_operand:TI           0 "register_operand" "")
          (minus:TI (match_operand:TI 1 "register_operand" "")
                    (match_operand:TI 2 "general_operand"  "") ) )
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_ZARCH"
{
  /* For z13 we have vaq which doesn't set CC.  */
  if (TARGET_VX)
    {
      emit_insn (gen_rtx_SET (operands[0],
                              gen_rtx_MINUS (TImode,
                                            operands[1],
                                            copy_to_mode_reg (TImode, operands[2]))));
      DONE;
    }
})

(define_insn_and_split "*subti3"
  [(set (match_operand:TI           0 "register_operand" "=&d")
        (minus:TI (match_operand:TI 1 "register_operand"   "0")
                  (match_operand:TI 2 "general_operand"   "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL2 CC_REGNUM)
          (compare:CCL2 (minus:DI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (minus:DI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (minus:DI (minus:DI (match_dup 4) (match_dup 5))
                                  (gtu:DI (reg:CCL2 CC_REGNUM) (const_int 0))))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, TImode);
   operands[4] = operand_subword (operands[1], 0, 0, TImode);
   operands[5] = operand_subword (operands[2], 0, 0, TImode);
   operands[6] = operand_subword (operands[0], 1, 0, TImode);
   operands[7] = operand_subword (operands[1], 1, 0, TImode);
   operands[8] = operand_subword (operands[2], 1, 0, TImode);"
  [(set_attr "op_type"      "*")
   (set_attr "cpu_facility" "*")])

;
; subdi3 instruction pattern(s).
;

(define_expand "subdi3"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
          (minus:DI (match_operand:DI 1 "register_operand" "")
                    (match_operand:DI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*subdi3_sign"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                  (sign_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   sgfr\t%0,%2
   sgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_c,*")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*subdi3_zero_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                           (zero_extend:DI (match_operand:SI 2 "general_operand" "d,RT")))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d,d")
        (minus:DI (match_dup 1) (zero_extend:DI (match_dup 2))))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   slgfr\t%0,%2
   slgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_c_E1,z10_super_E1")])

(define_insn "*subdi3_zero_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                           (zero_extend:DI (match_operand:SI 2 "general_operand" "d,RT")))
                 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d,d"))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   slgfr\t%0,%2
   slgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_c_E1,z10_super_E1")])

(define_insn "*subdi3_zero"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                  (zero_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   slgfr\t%0,%2
   slgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_c_E1,z10_super_E1")])

(define_insn_and_split "*subdi3_31z"
  [(set (match_operand:DI 0 "register_operand" "=&d")
        (minus:DI (match_operand:DI 1 "register_operand" "0")
                  (match_operand:DI 2 "general_operand" "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && TARGET_CPU_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL2 CC_REGNUM)
          (compare:CCL2 (minus:SI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (minus:SI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (minus:SI (minus:SI (match_dup 4) (match_dup 5))
                                  (gtu:SI (reg:CCL2 CC_REGNUM) (const_int 0))))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
   operands[4] = operand_subword (operands[1], 0, 0, DImode);
   operands[5] = operand_subword (operands[2], 0, 0, DImode);
   operands[6] = operand_subword (operands[0], 1, 0, DImode);
   operands[7] = operand_subword (operands[1], 1, 0, DImode);
   operands[8] = operand_subword (operands[2], 1, 0, DImode);")

(define_insn_and_split "*subdi3_31"
  [(set (match_operand:DI 0 "register_operand" "=&d")
        (minus:DI (match_operand:DI 1 "register_operand" "0")
                  (match_operand:DI 2 "general_operand" "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_CPU_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 3) (minus:SI (match_dup 4) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (reg:CCL2 CC_REGNUM)
          (compare:CCL2 (minus:SI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (minus:SI (match_dup 7) (match_dup 8)))])
   (set (pc)
        (if_then_else (gtu (reg:CCL2 CC_REGNUM) (const_int 0))
                      (pc)
                      (label_ref (match_dup 9))))
   (parallel
    [(set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])
   (match_dup 9)]
  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
   operands[4] = operand_subword (operands[1], 0, 0, DImode);
   operands[5] = operand_subword (operands[2], 0, 0, DImode);
   operands[6] = operand_subword (operands[0], 1, 0, DImode);
   operands[7] = operand_subword (operands[1], 1, 0, DImode);
   operands[8] = operand_subword (operands[2], 1, 0, DImode);
   operands[9] = gen_label_rtx ();")

;
; subsi3 instruction pattern(s).
;

(define_expand "subsi3"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (minus:SI (match_operand:SI 1 "register_operand" "")
                    (match_operand:SI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*subsi3_sign"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (minus:SI (match_operand:SI 1 "register_operand" "0,0")
                  (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   sh\t%0,%2
   shy\t%0,%2"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "z196prop" "z196_cracked,z196_cracked")])

;
; sub(di|si)3 instruction pattern(s).
;

; sr, s, sy, sgr, sg, srk, sgrk
(define_insn "*sub<mode>3"
  [(set (match_operand:GPR 0 "register_operand"           "=d,d,d,d")
        (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
                   (match_operand:GPR 2 "general_operand"  "d,d,R,T") ) )
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   s<g>r\t%0,%2
   s<g>rk\t%0,%1,%2
   s<g>\t%0,%2
   s<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_borrow_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
                            (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (match_dup 1)))
   (set (match_operand:GPR 0 "register_operand"                    "=d,d,d,d")
        (minus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL2mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_borrow_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
                            (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (match_dup 1)))
   (clobber (match_scratch:GPR 0                                   "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL2mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
                            (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand"                    "=d,d,d,d")
        (minus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cc2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 1 "register_operand" "0,d,0,0")
                 (match_operand:GPR 2 "general_operand"  "d,d,R,T")))
   (set (match_operand:GPR 0 "register_operand"         "=d,d,d,d")
        (minus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL3mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
                            (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0                                   "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])


; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cconly2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 1 "register_operand" "0,d,0,0")
                 (match_operand:GPR 2 "general_operand"  "d,d,R,T")))
   (clobber (match_scratch:GPR 0                        "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL3mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,*")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])


;
; sub(tf|df|sf|td|dd)3 instruction pattern(s).
;

; sxbr, sdbr, sebr, sdb, seb, sxtr, sdtr
(define_insn "sub<mode>3"
  [(set (match_operand:FP           0 "register_operand"   "=f,   f,<vf>")
        (minus:FP (match_operand:FP 1 "register_operand" "<f0>,   0,<v0>")
                  (match_operand:FP 2 "general_operand"     "f,<Rf>,<vf>")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   s<xde><bt>r\t%0,<op1>%2
   s<xde>b\t%0,%2
   wfsdb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "<RRer>,RXE,VRR")
   (set_attr "type"         "fsimp<mode>")
   (set_attr "cpu_facility" "*,*,vec")])

; sxbr, sdbr, sebr, sdb, seb, sxtr, sdtr
(define_insn "*sub<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:FP (match_operand:FP 1 "nonimmediate_operand" "<f0>,0")
                           (match_operand:FP 2 "general_operand"      "f,<Rf>"))
                 (match_operand:FP 3 "const0_operand" "")))
   (set (match_operand:FP 0 "register_operand" "=f,f")
        (minus:FP (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   s<xde><bt>r\t%0,<op1>%2
   s<xde>b\t%0,%2"
  [(set_attr "op_type"  "<RRer>,RXE")
   (set_attr "type"     "fsimp<mode>")])

; sxbr, sdbr, sebr, sdb, seb, sxtr, sdtr
(define_insn "*sub<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:FP (match_operand:FP 1 "nonimmediate_operand" "<f0>,0")
                           (match_operand:FP 2 "general_operand"      "f,<Rf>"))
                 (match_operand:FP 3 "const0_operand" "")))
   (clobber (match_scratch:FP 0 "=f,f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   s<xde><bt>r\t%0,<op1>%2
   s<xde>b\t%0,%2"
  [(set_attr "op_type"  "<RRer>,RXE")
   (set_attr "type"     "fsimp<mode>")])


;;
;;- Conditional add/subtract instructions.
;;

;
; add(di|si)cc instruction pattern(s).
;

; the following 4 patterns are used when the result of an add with
; carry is checked for an overflow condition

; op1 + op2 + c < op1

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry1_cc"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,RT"))
          (match_dup 1)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_dup 3) (match_dup 1)) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode) && TARGET_CPU_ZARCH"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z196prop" "z196_alone,z196_alone")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry1_cconly"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,RT"))
          (match_dup 1)))
   (clobber (match_scratch:GPR 0 "=d,d"))]
  "s390_match_ccmode (insn, CCL1mode) && TARGET_CPU_ZARCH"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z196prop" "z196_alone,z196_alone")])

; op1 + op2 + c < op2

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry2_cc"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,RT"))
          (match_dup 2)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_dup 3) (match_dup 1)) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode) && TARGET_CPU_ZARCH"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry2_cconly"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,RT"))
          (match_dup 2)))
   (clobber (match_scratch:GPR 0 "=d,d"))]
  "s390_match_ccmode (insn, CCL1mode) && TARGET_CPU_ZARCH"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_cc"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,RT"))
          (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_dup 3) (match_dup 1)) (match_dup 2)))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_CPU_ZARCH"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                            (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                  (match_operand:GPR 2 "general_operand" "d,RT")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_CPU_ZARCH"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; slbr, slb, slbgr, slbg
(define_insn "*sub<mode>3_slb_cc"
  [(set (reg CC_REGNUM)
        (compare
          (minus:GPR (minus:GPR (match_operand:GPR 1 "nonimmediate_operand" "0,0")
                                (match_operand:GPR 2 "general_operand" "d,RT"))
                     (match_operand:GPR 3 "s390_slb_comparison" ""))
          (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (minus:GPR (minus:GPR (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_CPU_ZARCH"
  "@
   slb<g>r\t%0,%2
   slb<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_c,*")])

; slbr, slb, slbgr, slbg
(define_insn "*sub<mode>3_slb"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (minus:GPR (minus:GPR (match_operand:GPR 1 "nonimmediate_operand" "0,0")
                              (match_operand:GPR 2 "general_operand" "d,RT"))
                   (match_operand:GPR 3 "s390_slb_comparison" "")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_CPU_ZARCH"
  "@
   slb<g>r\t%0,%2
   slb<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_c,*")])

(define_expand "add<mode>cc"
  [(match_operand:GPR 0 "register_operand" "")
   (match_operand 1 "comparison_operator" "")
   (match_operand:GPR 2 "register_operand" "")
   (match_operand:GPR 3 "const_int_operand" "")]
  "TARGET_CPU_ZARCH"
  "if (!s390_expand_addcc (GET_CODE (operands[1]),
                           XEXP (operands[1], 0), XEXP (operands[1], 1),
                           operands[0], operands[2],
                           operands[3])) FAIL; DONE;")

;
; scond instruction pattern(s).
;

(define_insn_and_split "*scond<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=&d")
        (match_operand:GPR 1 "s390_alc_comparison" ""))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_CPU_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (parallel
    [(set (match_dup 0) (plus:GPR (plus:GPR (match_dup 1) (match_dup 0))
                                  (match_dup 0)))
     (clobber (reg:CC CC_REGNUM))])]
  "")

(define_insn_and_split "*scond<mode>_neg"
  [(set (match_operand:GPR 0 "register_operand" "=&d")
        (match_operand:GPR 1 "s390_slb_comparison" ""))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_CPU_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (parallel
    [(set (match_dup 0) (minus:GPR (minus:GPR (match_dup 0) (match_dup 0))
                                   (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_dup 0) (neg:GPR (match_dup 0)))
     (clobber (reg:CC CC_REGNUM))])]
  "")


(define_expand "cstore<mode>4"
  [(set (match_operand:SI 0 "register_operand" "")
        (match_operator:SI 1 "s390_scond_operator"
         [(match_operand:GPR 2 "register_operand" "")
          (match_operand:GPR 3 "general_operand" "")]))]
  "TARGET_CPU_ZARCH"
  "if (!s390_expand_addcc (GET_CODE (operands[1]), operands[2], operands[3],
                           operands[0], const0_rtx, const1_rtx)) FAIL; DONE;")

(define_expand "cstorecc4"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (match_operator:SI 1 "s390_eqne_operator"
           [(match_operand:CCZ1 2 "register_operand")
            (match_operand 3 "const0_operand")]))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "emit_insn (gen_sne (operands[0], operands[2]));
   if (GET_CODE (operands[1]) == EQ)
     emit_insn (gen_xorsi3 (operands[0], operands[0], const1_rtx));
   DONE;")

(define_insn_and_split "sne"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (ne:SI (match_operand:CCZ1 1 "register_operand" "0")
               (const_int 0)))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 28)))
     (clobber (reg:CC CC_REGNUM))])])


;;
;; - Conditional move instructions (introduced with z196)
;;

(define_expand "mov<mode>cc"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "")
        (if_then_else:GPR (match_operand 1 "comparison_operator" "")
                          (match_operand:GPR 2 "nonimmediate_operand" "")
                          (match_operand:GPR 3 "nonimmediate_operand" "")))]
  "TARGET_Z196"
  "operands[1] = s390_emit_compare (GET_CODE (operands[1]),
                                    XEXP (operands[1], 0), XEXP (operands[1], 1));")

; locr, loc, stoc, locgr, locg, stocg
(define_insn_and_split "*mov<mode>cc"
  [(set (match_operand:GPR 0 "nonimmediate_operand"   "=d,d, d, d,QS,QS,&d")
        (if_then_else:GPR
          (match_operator 1 "s390_comparison"
            [(match_operand 2 "cc_reg_operand"        " c,c, c, c, c, c, c")
             (match_operand 5 "const_int_operand"     "")])
          (match_operand:GPR 3 "nonimmediate_operand" " d,0,QS, 0, d, 0,QS")
          (match_operand:GPR 4 "nonimmediate_operand" " 0,d, 0,QS, 0, d,QS")))]
  "TARGET_Z196"
  "@
   loc<g>r%C1\t%0,%3
   loc<g>r%D1\t%0,%4
   loc<g>%C1\t%0,%3
   loc<g>%D1\t%0,%4
   stoc<g>%C1\t%3,%0
   stoc<g>%D1\t%4,%0
   #"
  "&& reload_completed
   && MEM_P (operands[3]) && MEM_P (operands[4])"
  [(set (match_dup 0)
        (if_then_else:GPR
         (match_op_dup 1 [(match_dup 2) (const_int 0)])
         (match_dup 3)
         (match_dup 0)))
   (set (match_dup 0)
        (if_then_else:GPR
         (match_op_dup 1 [(match_dup 2) (const_int 0)])
         (match_dup 0)
         (match_dup 4)))]
  ""
  [(set_attr "op_type" "RRF,RRF,RSY,RSY,RSY,RSY,*")])

;;
;;- Multiply instructions.
;;

;
; muldi3 instruction pattern(s).
;

(define_insn "*muldi3_sign"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (mult:DI (sign_extend:DI (match_operand:SI 2 "general_operand" "d,RT"))
                 (match_operand:DI 1 "register_operand" "0,0")))]
  "TARGET_ZARCH"
  "@
   msgfr\t%0,%2
   msgf\t%0,%2"
  [(set_attr "op_type"      "RRE,RXY")
   (set_attr "type"         "imuldi")])

(define_insn "muldi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d,d")
        (mult:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,0")
                 (match_operand:DI 2 "general_operand" "d,K,RT,Os")))]
  "TARGET_ZARCH"
  "@
   msgr\t%0,%2
   mghi\t%0,%h2
   msg\t%0,%2
   msgfi\t%0,%2"
  [(set_attr "op_type"      "RRE,RI,RXY,RIL")
   (set_attr "type"         "imuldi")
   (set_attr "cpu_facility" "*,*,*,z10")])

;
; mulsi3 instruction pattern(s).
;

(define_insn "*mulsi3_sign"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (mult:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))
                 (match_operand:SI 1 "register_operand" "0,0")))]
  ""
  "@
   mh\t%0,%2
   mhy\t%0,%2"
  [(set_attr "op_type"      "RX,RXY")
   (set_attr "type"         "imulhi")
   (set_attr "cpu_facility" "*,z10")])

(define_insn "mulsi3"
  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d,d")
        (mult:SI  (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0,0")
                  (match_operand:SI 2 "general_operand" "d,K,R,T,Os")))]
  ""
  "@
   msr\t%0,%2
   mhi\t%0,%h2
   ms\t%0,%2
   msy\t%0,%2
   msfi\t%0,%2"
  [(set_attr "op_type"      "RRE,RI,RX,RXY,RIL")
   (set_attr "type"         "imulsi,imulhi,imulsi,imulsi,imulsi")
   (set_attr "cpu_facility" "*,*,*,*,z10")])

;
; mulsidi3 instruction pattern(s).
;

(define_insn "mulsidi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (mult:DI (sign_extend:DI
                   (match_operand:SI 1 "register_operand" "%0,0,0"))
                 (sign_extend:DI
                   (match_operand:SI 2 "nonimmediate_operand" "d,R,T"))))]
  "!TARGET_ZARCH"
  "@
   mr\t%0,%2
   m\t%0,%2
   mfy\t%0,%2"
  [(set_attr "op_type"      "RR,RX,RXY")
   (set_attr "type"         "imulsi")
   (set_attr "cpu_facility" "*,*,z10")])

;
; umul instruction pattern(s).
;

; mlr, ml, mlgr, mlg
(define_insn "umul<dwh><mode>3"
  [(set (match_operand:DW 0 "register_operand"                   "=d, d")
        (mult:DW (zero_extend:DW
                   (match_operand:<DWH> 1 "register_operand"     "%0, 0"))
                 (zero_extend:DW
                   (match_operand:<DWH> 2 "nonimmediate_operand" " d,RT"))))]
  "TARGET_CPU_ZARCH"
  "@
   ml<tg>r\t%0,%2
   ml<tg>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "imul<dwh>")])

;
; mul(tf|df|sf|td|dd)3 instruction pattern(s).
;

; mxbr, mdbr, meebr, mxb, mxb, meeb, mdtr, mxtr
(define_insn "mul<mode>3"
  [(set (match_operand:FP          0 "register_operand"        "=f,   f,<vf>")
        (mult:FP (match_operand:FP 1 "nonimmediate_operand" "%<f0>,   0,<v0>")
                 (match_operand:FP 2 "general_operand"          "f,<Rf>,<vf>")))]
  "TARGET_HARD_FLOAT"
  "@
   m<xdee><bt>r\t%0,<op1>%2
   m<xdee>b\t%0,%2
   wfmdb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "<RRer>,RXE,VRR")
   (set_attr "type"         "fmul<mode>")
   (set_attr "cpu_facility" "*,*,vec")])

; madbr, maebr, maxb, madb, maeb
(define_insn "fma<mode>4"
  [(set (match_operand:DSF          0 "register_operand"     "=f,f,<vf>")
        (fma:DSF (match_operand:DSF 1 "nonimmediate_operand" "%f,f,<vf>")
                 (match_operand:DSF 2 "nonimmediate_operand"  "f,R,<vf>")
                 (match_operand:DSF 3 "register_operand"      "0,0,<v0>")))]
  "TARGET_HARD_FLOAT"
  "@
   ma<xde>br\t%0,%1,%2
   ma<xde>b\t%0,%1,%2
   wfmadb\t%v0,%v1,%v2,%v3"
  [(set_attr "op_type"      "RRE,RXE,VRR")
   (set_attr "type"         "fmadd<mode>")
   (set_attr "cpu_facility" "*,*,vec")])

; msxbr, msdbr, msebr, msxb, msdb, mseb
(define_insn "fms<mode>4"
  [(set (match_operand:DSF                   0 "register_operand"     "=f,f,<vf>")
        (fma:DSF (match_operand:DSF          1 "nonimmediate_operand" "%f,f,<vf>")
                 (match_operand:DSF          2 "nonimmediate_operand"  "f,R,<vf>")
                 (neg:DSF (match_operand:DSF 3 "register_operand"      "0,0,<v0>"))))]
  "TARGET_HARD_FLOAT"
  "@
   ms<xde>br\t%0,%1,%2
   ms<xde>b\t%0,%1,%2
   wfmsdb\t%v0,%v1,%v2,%v3"
  [(set_attr "op_type"      "RRE,RXE,VRR")
   (set_attr "type"         "fmadd<mode>")
   (set_attr "cpu_facility" "*,*,vec")])

;;
;;- Divide and modulo instructions.
;;

;
; divmoddi4 instruction pattern(s).
;

(define_expand "divmoddi4"
  [(parallel [(set (match_operand:DI 0 "general_operand" "")
                   (div:DI (match_operand:DI 1 "register_operand" "")
                           (match_operand:DI 2 "general_operand" "")))
              (set (match_operand:DI 3 "general_operand" "")
                   (mod:DI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "TARGET_ZARCH"
{
  rtx insn, div_equal, mod_equal;

  div_equal = gen_rtx_DIV (DImode, operands[1], operands[2]);
  mod_equal = gen_rtx_MOD (DImode, operands[1], operands[2]);

  operands[4] = gen_reg_rtx(TImode);
  emit_insn (gen_divmodtidi3 (operands[4], operands[1], operands[2]));

  insn = emit_move_insn (operands[0], gen_lowpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "divmodtidi3"
  [(set (match_operand:TI 0 "register_operand" "=d,d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (mod:DI (match_operand:DI 1 "register_operand" "0,0")
                      (match_operand:DI 2 "general_operand" "d,RT")))
            (const_int 64))
          (zero_extend:TI (div:DI (match_dup 1) (match_dup 2)))))]
  "TARGET_ZARCH"
  "@
   dsgr\t%0,%2
   dsg\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

(define_insn "divmodtisi3"
  [(set (match_operand:TI 0 "register_operand" "=d,d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (mod:DI (match_operand:DI 1 "register_operand" "0,0")
                      (sign_extend:DI
                        (match_operand:SI 2 "nonimmediate_operand" "d,RT"))))
            (const_int 64))
          (zero_extend:TI
            (div:DI (match_dup 1) (sign_extend:DI (match_dup 2))))))]
  "TARGET_ZARCH"
  "@
   dsgfr\t%0,%2
   dsgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

;
; udivmoddi4 instruction pattern(s).
;

(define_expand "udivmoddi4"
  [(parallel [(set (match_operand:DI 0 "general_operand" "")
                   (udiv:DI (match_operand:DI 1 "general_operand" "")
                            (match_operand:DI 2 "nonimmediate_operand" "")))
              (set (match_operand:DI 3 "general_operand" "")
                   (umod:DI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "TARGET_ZARCH"
{
  rtx insn, div_equal, mod_equal, equal;

  div_equal = gen_rtx_UDIV (DImode, operands[1], operands[2]);
  mod_equal = gen_rtx_UMOD (DImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (TImode,
                       gen_rtx_ASHIFT (TImode,
                                       gen_rtx_ZERO_EXTEND (TImode, mod_equal),
                                       GEN_INT (64)),
                       gen_rtx_ZERO_EXTEND (TImode, div_equal));

  operands[4] = gen_reg_rtx(TImode);
  emit_clobber (operands[4]);
  emit_move_insn (gen_lowpart (DImode, operands[4]), operands[1]);
  emit_move_insn (gen_highpart (DImode, operands[4]), const0_rtx);

  insn = emit_insn (gen_udivmodtidi3 (operands[4], operands[4], operands[2]));
  set_unique_reg_note (insn, REG_EQUAL, equal);

  insn = emit_move_insn (operands[0], gen_lowpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "udivmodtidi3"
  [(set (match_operand:TI 0 "register_operand" "=d,d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (truncate:DI
                (umod:TI (match_operand:TI 1 "register_operand" "0,0")
                         (zero_extend:TI
                           (match_operand:DI 2 "nonimmediate_operand" "d,RT")))))
            (const_int 64))
          (zero_extend:TI
            (truncate:DI
              (udiv:TI (match_dup 1) (zero_extend:TI (match_dup 2)))))))]
  "TARGET_ZARCH"
  "@
   dlgr\t%0,%2
   dlg\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

;
; divmodsi4 instruction pattern(s).
;

(define_expand "divmodsi4"
  [(parallel [(set (match_operand:SI 0 "general_operand" "")
                   (div:SI (match_operand:SI 1 "general_operand" "")
                           (match_operand:SI 2 "nonimmediate_operand" "")))
              (set (match_operand:SI 3 "general_operand" "")
                   (mod:SI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "!TARGET_ZARCH"
{
  rtx insn, div_equal, mod_equal, equal;

  div_equal = gen_rtx_DIV (SImode, operands[1], operands[2]);
  mod_equal = gen_rtx_MOD (SImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (DImode,
                       gen_rtx_ASHIFT (DImode,
                                       gen_rtx_ZERO_EXTEND (DImode, mod_equal),
                                       GEN_INT (32)),
                       gen_rtx_ZERO_EXTEND (DImode, div_equal));

  operands[4] = gen_reg_rtx(DImode);
  emit_insn (gen_extendsidi2 (operands[4], operands[1]));

  insn = emit_insn (gen_divmoddisi3 (operands[4], operands[4], operands[2]));
  set_unique_reg_note (insn, REG_EQUAL, equal);

  insn = emit_move_insn (operands[0], gen_lowpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "divmoddisi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (ior:DI
          (ashift:DI
            (zero_extend:DI
              (truncate:SI
                (mod:DI (match_operand:DI 1 "register_operand" "0,0")
                        (sign_extend:DI
                          (match_operand:SI 2 "nonimmediate_operand" "d,R")))))
            (const_int 32))
          (zero_extend:DI
            (truncate:SI
              (div:DI (match_dup 1) (sign_extend:DI (match_dup 2)))))))]
  "!TARGET_ZARCH"
  "@
   dr\t%0,%2
   d\t%0,%2"
  [(set_attr "op_type"  "RR,RX")
   (set_attr "type"     "idiv")])

;
; udivsi3 and umodsi3 instruction pattern(s).
;

(define_expand "udivmodsi4"
  [(parallel [(set (match_operand:SI 0 "general_operand" "")
                   (udiv:SI (match_operand:SI 1 "general_operand" "")
                            (match_operand:SI 2 "nonimmediate_operand" "")))
              (set (match_operand:SI 3 "general_operand" "")
                   (umod:SI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "!TARGET_ZARCH && TARGET_CPU_ZARCH"
{
  rtx insn, div_equal, mod_equal, equal;

  div_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
  mod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (DImode,
                       gen_rtx_ASHIFT (DImode,
                                       gen_rtx_ZERO_EXTEND (DImode, mod_equal),
                                       GEN_INT (32)),
                       gen_rtx_ZERO_EXTEND (DImode, div_equal));

  operands[4] = gen_reg_rtx(DImode);
  emit_clobber (operands[4]);
  emit_move_insn (gen_lowpart (SImode, operands[4]), operands[1]);
  emit_move_insn (gen_highpart (SImode, operands[4]), const0_rtx);

  insn = emit_insn (gen_udivmoddisi3 (operands[4], operands[4], operands[2]));
  set_unique_reg_note (insn, REG_EQUAL, equal);

  insn = emit_move_insn (operands[0], gen_lowpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "udivmoddisi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (ior:DI
          (ashift:DI
            (zero_extend:DI
              (truncate:SI
                (umod:DI (match_operand:DI 1 "register_operand" "0,0")
                         (zero_extend:DI
                           (match_operand:SI 2 "nonimmediate_operand" "d,RT")))))
            (const_int 32))
          (zero_extend:DI
            (truncate:SI
              (udiv:DI (match_dup 1) (zero_extend:DI (match_dup 2)))))))]
  "!TARGET_ZARCH && TARGET_CPU_ZARCH"
  "@
   dlr\t%0,%2
   dl\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

(define_expand "udivsi3"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (udiv:SI (match_operand:SI 1 "general_operand" "")
                 (match_operand:SI 2 "general_operand" "")))
   (clobber (match_dup 3))]
  "!TARGET_ZARCH && !TARGET_CPU_ZARCH"
{
  rtx insn, udiv_equal, umod_equal, equal;

  udiv_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
  umod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (DImode,
                       gen_rtx_ASHIFT (DImode,
                                       gen_rtx_ZERO_EXTEND (DImode, umod_equal),
                                       GEN_INT (32)),
                       gen_rtx_ZERO_EXTEND (DImode, udiv_equal));

  operands[3] = gen_reg_rtx (DImode);

  if (CONSTANT_P (operands[2]))
    {
      if (GET_CODE (operands[2]) == CONST_INT && INTVAL (operands[2]) < 0)
        {
          rtx_code_label *label1 = gen_label_rtx ();

          operands[1] = make_safe_from (operands[1], operands[0]);
          emit_move_insn (operands[0], const0_rtx);
          emit_cmp_and_jump_insns (operands[1], operands[2], LT, NULL_RTX,
                                   SImode, 1, label1);
          emit_move_insn (operands[0], const1_rtx);
          emit_label (label1);
        }
      else
        {
          operands[2] = force_reg (SImode, operands[2]);
          operands[2] = make_safe_from (operands[2], operands[0]);

          emit_insn (gen_zero_extendsidi2 (operands[3], operands[1]));
          insn = emit_insn (gen_divmoddisi3 (operands[3], operands[3],
                                             operands[2]));
          set_unique_reg_note (insn, REG_EQUAL, equal);

          insn = emit_move_insn (operands[0],
                                 gen_lowpart (SImode, operands[3]));
          set_unique_reg_note (insn, REG_EQUAL, udiv_equal);
        }
    }
  else
    {
      rtx_code_label *label1 = gen_label_rtx ();
      rtx_code_label *label2 = gen_label_rtx ();
      rtx_code_label *label3 = gen_label_rtx ();

      operands[1] = force_reg (SImode, operands[1]);
      operands[1] = make_safe_from (operands[1], operands[0]);
      operands[2] = force_reg (SImode, operands[2]);
      operands[2] = make_safe_from (operands[2], operands[0]);

      emit_move_insn (operands[0], const0_rtx);
      emit_cmp_and_jump_insns (operands[2], operands[1], GT, NULL_RTX,
                               SImode, 1, label3);
      emit_cmp_and_jump_insns (operands[2], const0_rtx, LT, NULL_RTX,
                               SImode, 0, label2);
      emit_cmp_and_jump_insns (operands[2], const1_rtx, EQ, NULL_RTX,
                               SImode, 0, label1);
      emit_insn (gen_zero_extendsidi2 (operands[3], operands[1]));
      insn = emit_insn (gen_divmoddisi3 (operands[3], operands[3],
                                         operands[2]));
      set_unique_reg_note (insn, REG_EQUAL, equal);

      insn = emit_move_insn (operands[0],
                             gen_lowpart (SImode, operands[3]));
      set_unique_reg_note (insn, REG_EQUAL, udiv_equal);

      emit_jump (label3);
      emit_label (label1);
      emit_move_insn (operands[0], operands[1]);
      emit_jump (label3);
      emit_label (label2);
      emit_move_insn (operands[0], const1_rtx);
      emit_label (label3);
    }
  emit_move_insn (operands[0], operands[0]);
  DONE;
})

(define_expand "umodsi3"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (umod:SI (match_operand:SI 1 "nonimmediate_operand" "")
                 (match_operand:SI 2 "nonimmediate_operand" "")))
   (clobber (match_dup 3))]
  "!TARGET_ZARCH && !TARGET_CPU_ZARCH"
{
  rtx insn, udiv_equal, umod_equal, equal;

  udiv_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
  umod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (DImode,
                       gen_rtx_ASHIFT (DImode,
                                       gen_rtx_ZERO_EXTEND (DImode, umod_equal),
                                       GEN_INT (32)),
                       gen_rtx_ZERO_EXTEND (DImode, udiv_equal));

  operands[3] = gen_reg_rtx (DImode);

  if (CONSTANT_P (operands[2]))
    {
      if (GET_CODE (operands[2]) == CONST_INT && INTVAL (operands[2]) <= 0)
        {
          rtx_code_label *label1 = gen_label_rtx ();

          operands[1] = make_safe_from (operands[1], operands[0]);
          emit_move_insn (operands[0], operands[1]);
          emit_cmp_and_jump_insns (operands[0], operands[2], LT, NULL_RTX,
                                   SImode, 1, label1);
          emit_insn (gen_abssi2 (operands[0], operands[2]));
          emit_insn (gen_addsi3 (operands[0], operands[0], operands[1]));
          emit_label (label1);
        }
      else
        {
          operands[2] = force_reg (SImode, operands[2]);
          operands[2] = make_safe_from (operands[2], operands[0]);

          emit_insn (gen_zero_extendsidi2 (operands[3], operands[1]));
          insn = emit_insn (gen_divmoddisi3 (operands[3], operands[3],
                                             operands[2]));
          set_unique_reg_note (insn, REG_EQUAL, equal);

          insn = emit_move_insn (operands[0],
                                 gen_highpart (SImode, operands[3]));
          set_unique_reg_note (insn, REG_EQUAL, umod_equal);
        }
    }
  else
    {
      rtx_code_label *label1 = gen_label_rtx ();
      rtx_code_label *label2 = gen_label_rtx ();
      rtx_code_label *label3 = gen_label_rtx ();

      operands[1] = force_reg (SImode, operands[1]);
      operands[1] = make_safe_from (operands[1], operands[0]);
      operands[2] = force_reg (SImode, operands[2]);
      operands[2] = make_safe_from (operands[2], operands[0]);

      emit_move_insn(operands[0], operands[1]);
      emit_cmp_and_jump_insns (operands[2], operands[1], GT, NULL_RTX,
                               SImode, 1, label3);
      emit_cmp_and_jump_insns (operands[2], const0_rtx, LT, NULL_RTX,
                               SImode, 0, label2);
      emit_cmp_and_jump_insns (operands[2], const1_rtx, EQ, NULL_RTX,
                               SImode, 0, label1);
      emit_insn (gen_zero_extendsidi2 (operands[3], operands[1]));
      insn = emit_insn (gen_divmoddisi3 (operands[3], operands[3],
                                         operands[2]));
      set_unique_reg_note (insn, REG_EQUAL, equal);

      insn = emit_move_insn (operands[0],
                             gen_highpart (SImode, operands[3]));
      set_unique_reg_note (insn, REG_EQUAL, umod_equal);

      emit_jump (label3);
      emit_label (label1);
      emit_move_insn (operands[0], const0_rtx);
      emit_jump (label3);
      emit_label (label2);
      emit_insn (gen_subsi3 (operands[0], operands[0], operands[2]));
      emit_label (label3);
    }
  DONE;
})

;
; div(df|sf)3 instruction pattern(s).
;

; dxbr, ddbr, debr, dxb, ddb, deb, ddtr, dxtr
(define_insn "div<mode>3"
  [(set (match_operand:FP         0 "register_operand"   "=f,   f,<vf>")
        (div:FP (match_operand:FP 1 "register_operand" "<f0>,   0,<v0>")
                (match_operand:FP 2 "general_operand"     "f,<Rf>,<vf>")))]
  "TARGET_HARD_FLOAT"
  "@
   d<xde><bt>r\t%0,<op1>%2
   d<xde>b\t%0,%2
   wfddb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "<RRer>,RXE,VRR")
   (set_attr "type"         "fdiv<mode>")
   (set_attr "cpu_facility" "*,*,vec")])


;;
;;- And instructions.
;;

(define_expand "and<mode>3"
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (and:INT (match_operand:INT 1 "nonimmediate_operand" "")
                 (match_operand:INT 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "s390_expand_logical_operator (AND, <MODE>mode, operands); DONE;")

;
; anddi3 instruction pattern(s).
;

(define_insn "*anddi3_cc"
  [(set (reg CC_REGNUM)
        (compare
          (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d, 0,    d")
                  (match_operand:DI 2 "general_operand"      " d,d,RT,NxxDq"))
          (const_int 0)))
   (set (match_operand:DI 0 "register_operand"               "=d,d, d,    d")
        (and:DI (match_dup 1) (match_dup 2)))]
  "TARGET_ZARCH && s390_match_ccmode(insn, CCTmode)"
  "@
   ngr\t%0,%2
   ngrk\t%0,%1,%2
   ng\t%0,%2
   risbg\t%0,%1,%s2,128+%e2,0"
  [(set_attr "op_type"  "RRE,RRF,RXY,RIE")
   (set_attr "cpu_facility" "*,z196,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*anddi3_cconly"
  [(set (reg CC_REGNUM)
        (compare
          (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d, 0,    d")
                  (match_operand:DI 2 "general_operand"      " d,d,RT,NxxDq"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0                              "=d,d, d,    d"))]
  "TARGET_ZARCH
   && s390_match_ccmode(insn, CCTmode)
   /* Do not steal TM patterns.  */
   && s390_single_part (operands[2], DImode, HImode, 0) < 0"
  "@
   ngr\t%0,%2
   ngrk\t%0,%1,%2
   ng\t%0,%2
   risbg\t%0,%1,%s2,128+%e2,0"
  [(set_attr "op_type"  "RRE,RRF,RXY,RIE")
   (set_attr "cpu_facility" "*,z196,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*anddi3"
  [(set (match_operand:DI 0 "nonimmediate_operand"
            "=d,d,    d,    d,    d,    d,    d,    d,d,d, d,    d,   AQ,Q")
        (and:DI
          (match_operand:DI 1 "nonimmediate_operand"
            "%d,o,    0,    0,    0,    0,    0,    0,0,d, 0,    d,    0,0")
          (match_operand:DI 2 "general_operand"
            "M, M,N0HDF,N1HDF,N2HDF,N3HDF,N0SDF,N1SDF,d,d,RT,NxxDq,NxQDF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   #
   #
   nihh\t%0,%j2
   nihl\t%0,%j2
   nilh\t%0,%j2
   nill\t%0,%j2
   nihf\t%0,%m2
   nilf\t%0,%m2
   ngr\t%0,%2
   ngrk\t%0,%1,%2
   ng\t%0,%2
   risbg\t%0,%1,%s2,128+%e2,0
   #
   #"
  [(set_attr "op_type" "RRE,RXE,RI,RI,RI,RI,RIL,RIL,RRE,RRF,RXY,RIE,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,*,*,extimm,extimm,*,z196,*,z10,*,*")
   (set_attr "z10prop" "*,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        *")])

(define_split
  [(set (match_operand:DI 0 "s_operand" "")
        (and:DI (match_dup 0) (match_operand:DI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (and:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (AND, &operands[0], &operands[1]);")

;; These two are what combine generates for (ashift (zero_extract)).
(define_insn "*extzv_<mode>_srl"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (and:GPR (lshiftrt:GPR
                   (match_operand:GPR 1 "register_operand" "d")
                   (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
                (match_operand:GPR 3 "contiguous_bitmask_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   /* Note that even for the SImode pattern, the rotate is always DImode.  */
   && s390_extzv_shift_ok (<bitsize>, -INTVAL (operands[2]),
                           INTVAL (operands[3]))"
  "risbg\t%0,%1,%<bfstart>3,128+%<bfend>3,64-%2"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*extzv_<mode>_sll"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (and:GPR (ashift:GPR
                  (match_operand:GPR 1 "register_operand" "d")
                  (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
                (match_operand:GPR 3 "contiguous_bitmask_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && s390_extzv_shift_ok (<bitsize>, INTVAL (operands[2]),
                           INTVAL (operands[3]))"
  "risbg\t%0,%1,%<bfstart>3,128+%<bfend>3,%2"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])


;
; andsi3 instruction pattern(s).
;

(define_insn "*andsi3_cc"
  [(set (reg CC_REGNUM)
        (compare
          (and:SI
            (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0,    d")
            (match_operand:SI 2 "general_operand"      "Os,d,d,R,T,NxxSq"))
          (const_int 0)))
   (set (match_operand:SI 0 "register_operand"         "=d,d,d,d,d,    d")
        (and:SI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   nilf\t%0,%o2
   nr\t%0,%2
   nrk\t%0,%1,%2
   n\t%0,%2
   ny\t%0,%2
   risbg\t%0,%1,%t2,128+%f2,0"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY,RIE")
   (set_attr "cpu_facility" "*,*,z196,*,*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

(define_insn "*andsi3_cconly"
  [(set (reg CC_REGNUM)
        (compare
          (and:SI
            (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0,    d")
            (match_operand:SI 2 "general_operand"      "Os,d,d,R,T,NxxSq"))
          (const_int 0)))
   (clobber (match_scratch:SI 0                        "=d,d,d,d,d,    d"))]
  "s390_match_ccmode(insn, CCTmode)
   /* Do not steal TM patterns.  */
   && s390_single_part (operands[2], SImode, HImode, 0) < 0"
  "@
   nilf\t%0,%o2
   nr\t%0,%2
   nrk\t%0,%1,%2
   n\t%0,%2
   ny\t%0,%2
   risbg\t%0,%1,%t2,128+%f2,0"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY,RIE")
   (set_attr "cpu_facility" "*,*,z196,*,*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

(define_insn "*andsi3_zarch"
  [(set (match_operand:SI 0 "nonimmediate_operand"
                            "=d,d,    d,    d, d,d,d,d,d,    d,   AQ,Q")
        (and:SI (match_operand:SI 1 "nonimmediate_operand"
                            "%d,o,    0,    0, 0,0,d,0,0,    d,    0,0")
                (match_operand:SI 2 "general_operand"
                            " M,M,N0HSF,N1HSF,Os,d,d,R,T,NxxSq,NxQSF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   #
   #
   nilh\t%0,%j2
   nill\t%0,%j2
   nilf\t%0,%o2
   nr\t%0,%2
   nrk\t%0,%1,%2
   n\t%0,%2
   ny\t%0,%2
   risbg\t%0,%1,%t2,128+%f2,0
   #
   #"
  [(set_attr "op_type"  "RRE,RXE,RI,RI,RIL,RR,RRF,RX,RXY,RIE,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,*,*,z196,*,*,z10,*,*")
   (set_attr "z10prop" "*,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        *")])

(define_insn "*andsi3_esa"
  [(set (match_operand:SI 0 "nonimmediate_operand"         "=d,d,   AQ,Q")
        (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,    0,0")
                (match_operand:SI 2 "general_operand"      " d,R,NxQSF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   n\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,RX,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,*")])


(define_split
  [(set (match_operand:SI 0 "s_operand" "")
        (and:SI (match_dup 0) (match_operand:SI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (and:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (AND, &operands[0], &operands[1]);")

;
; andhi3 instruction pattern(s).
;

(define_insn "*andhi3_zarch"
  [(set (match_operand:HI 0 "nonimmediate_operand"         "=d,d,d,   AQ,Q")
        (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,d,0,    0,0")
                (match_operand:HI 2 "general_operand"      " d,d,n,NxQHF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   nrk\t%0,%1,%2
   nill\t%0,%x2
   #
   #"
  [(set_attr "op_type"  "RR,RRF,RI,SI,SS")
   (set_attr "cpu_facility" "*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,*,*")
])

(define_insn "*andhi3_esa"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,AQ,Q")
        (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:HI 2 "general_operand" "d,NxQHF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,*,*")
])

(define_split
  [(set (match_operand:HI 0 "s_operand" "")
        (and:HI (match_dup 0) (match_operand:HI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (and:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (AND, &operands[0], &operands[1]);")

;
; andqi3 instruction pattern(s).
;

(define_insn "*andqi3_zarch"
  [(set (match_operand:QI 0 "nonimmediate_operand"         "=d,d,d,Q,S,Q")
        (and:QI (match_operand:QI 1 "nonimmediate_operand" "%0,d,0,0,0,0")
                (match_operand:QI 2 "general_operand"      " d,d,n,n,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   nrk\t%0,%1,%2
   nill\t%0,%b2
   ni\t%S0,%b2
   niy\t%S0,%b2
   #"
  [(set_attr "op_type"  "RR,RRF,RI,SI,SIY,SS")
   (set_attr "cpu_facility" "*,z196,*,*,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super,z10_super,*")])

(define_insn "*andqi3_esa"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,Q,Q")
        (and:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:QI 2 "general_operand" "d,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   ni\t%S0,%b2
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super,*")])

;
; Block and (NC) patterns.
;

(define_insn "*nc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (and:BLK (match_dup 0)
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "nc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")
   (set_attr "z196prop" "z196_cracked")])

(define_split
  [(set (match_operand 0 "memory_operand" "")
        (and (match_dup 0)
             (match_operand 1 "memory_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (and:BLK (match_dup 0) (match_dup 1)))
     (use (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (and:BLK (match_dup 0)
                   (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (and:BLK (match_dup 3)
                   (match_operand:BLK 4 "memory_operand" "")))
     (use (match_operand 5 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (and:BLK (match_dup 6) (match_dup 7)))
     (use (match_dup 8))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")


;;
;;- Bit set (inclusive or) instructions.
;;

(define_expand "ior<mode>3"
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (ior:INT (match_operand:INT 1 "nonimmediate_operand" "")
                 (match_operand:INT 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "s390_expand_logical_operator (IOR, <MODE>mode, operands); DONE;")

;
; iordi3 instruction pattern(s).
;

(define_insn "*iordi3_cc"
  [(set (reg CC_REGNUM)
        (compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d, 0")
                         (match_operand:DI 2 "general_operand"      " d,d,RT"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand"                      "=d,d, d")
        (ior:DI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   ogr\t%0,%2
   ogrk\t%0,%1,%2
   og\t%0,%2"
  [(set_attr "op_type"  "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*iordi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0")
                         (match_operand:DI 2 "general_operand"      " d,d,RT"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0                                     "=d,d,d"))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   ogr\t%0,%2
   ogrk\t%0,%1,%2
   og\t%0,%2"
  [(set_attr "op_type"  "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*iordi3"
  [(set (match_operand:DI 0 "nonimmediate_operand"
                               "=d,    d,    d,    d,    d,    d,d,d, d,   AQ,Q")
        (ior:DI (match_operand:DI 1 "nonimmediate_operand"
                            "   %0,    0,    0,    0,    0,    0,0,d, 0,    0,0")
                (match_operand:DI 2 "general_operand"
                            "N0HD0,N1HD0,N2HD0,N3HD0,N0SD0,N1SD0,d,d,RT,NxQD0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   oihh\t%0,%i2
   oihl\t%0,%i2
   oilh\t%0,%i2
   oill\t%0,%i2
   oihf\t%0,%k2
   oilf\t%0,%k2
   ogr\t%0,%2
   ogrk\t%0,%1,%2
   og\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RI,RI,RI,RI,RIL,RIL,RRE,RRF,RXY,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,extimm,extimm,*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        *,
                        *")])

(define_split
  [(set (match_operand:DI 0 "s_operand" "")
        (ior:DI (match_dup 0) (match_operand:DI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ior:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (IOR, &operands[0], &operands[1]);")

;
; iorsi3 instruction pattern(s).
;

(define_insn "*iorsi3_cc"
  [(set (reg CC_REGNUM)
        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (set (match_operand:SI 0 "register_operand"                      "=d,d,d,d,d")
        (ior:SI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   oilf\t%0,%o2
   or\t%0,%2
   ork\t%0,%1,%2
   o\t%0,%2
   oy\t%0,%2"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*iorsi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:SI 0                                     "=d,d,d,d,d"))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   oilf\t%0,%o2
   or\t%0,%2
   ork\t%0,%1,%2
   o\t%0,%2
   oy\t%0,%2"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*iorsi3_zarch"
  [(set (match_operand:SI 0 "nonimmediate_operand"         "=d,    d, d,d,d,d,d,   AQ,Q")
        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,    0, 0,0,d,0,0,    0,0")
                (match_operand:SI 2 "general_operand"   "N0HS0,N1HS0,Os,d,d,R,T,NxQS0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   oilh\t%0,%i2
   oill\t%0,%i2
   oilf\t%0,%o2
   or\t%0,%2
   ork\t%0,%1,%2
   o\t%0,%2
   oy\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RI,RI,RIL,RR,RRF,RX,RXY,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,z196,*,*,*,*")
   (set_attr "z10prop" "z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        *")])

(define_insn "*iorsi3_esa"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,d,AQ,Q")
        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
                (match_operand:SI 2 "general_operand" "d,R,NxQS0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   o\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,RX,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,*")])

(define_split
  [(set (match_operand:SI 0 "s_operand" "")
        (ior:SI (match_dup 0) (match_operand:SI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ior:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (IOR, &operands[0], &operands[1]);")

;
; iorhi3 instruction pattern(s).
;

(define_insn "*iorhi3_zarch"
  [(set (match_operand:HI 0 "nonimmediate_operand"         "=d,d,d,   AQ,Q")
        (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,d,0,    0,0")
                (match_operand:HI 2 "general_operand"      " d,d,n,NxQH0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   ork\t%0,%1,%2
   oill\t%0,%x2
   #
   #"
  [(set_attr "op_type"  "RR,RRF,RI,SI,SS")
   (set_attr "cpu_facility" "*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,*,*")])

(define_insn "*iorhi3_esa"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,AQ,Q")
        (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:HI 2 "general_operand" "d,NxQH0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,*,*")])

(define_split
  [(set (match_operand:HI 0 "s_operand" "")
        (ior:HI (match_dup 0) (match_operand:HI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ior:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (IOR, &operands[0], &operands[1]);")

;
; iorqi3 instruction pattern(s).
;

(define_insn "*iorqi3_zarch"
  [(set (match_operand:QI 0 "nonimmediate_operand"         "=d,d,d,Q,S,Q")
        (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,d,0,0,0,0")
                (match_operand:QI 2 "general_operand"      " d,d,n,n,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   ork\t%0,%1,%2
   oill\t%0,%b2
   oi\t%S0,%b2
   oiy\t%S0,%b2
   #"
  [(set_attr "op_type" "RR,RRF,RI,SI,SIY,SS")
   (set_attr "cpu_facility" "*,z196,*,*,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,
                        z10_super,z10_super,*")])

(define_insn "*iorqi3_esa"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,Q,Q")
        (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:QI 2 "general_operand" "d,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   oi\t%S0,%b2
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super,*")])

;
; Block inclusive or (OC) patterns.
;

(define_insn "*oc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (ior:BLK (match_dup 0)
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "oc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")
   (set_attr "z196prop" "z196_cracked")])

(define_split
  [(set (match_operand 0 "memory_operand" "")
        (ior (match_dup 0)
             (match_operand 1 "memory_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (ior:BLK (match_dup 0) (match_dup 1)))
     (use (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (ior:BLK (match_dup 0)
                   (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (ior:BLK (match_dup 3)
                   (match_operand:BLK 4 "memory_operand" "")))
     (use (match_operand 5 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (ior:BLK (match_dup 6) (match_dup 7)))
     (use (match_dup 8))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")


;;
;;- Xor instructions.
;;

(define_expand "xor<mode>3"
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (xor:INT (match_operand:INT 1 "nonimmediate_operand" "")
                 (match_operand:INT 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "s390_expand_logical_operator (XOR, <MODE>mode, operands); DONE;")

; Combine replaces (xor (x) (const_int -1)) with (not (x)) when doing
; simplifications.  So its better to have something matching.
(define_split
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (not:INT (match_operand:INT 1 "nonimmediate_operand" "")))]
  ""
  [(parallel
    [(set (match_dup 0) (xor:INT (match_dup 1) (match_dup 2)))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = constm1_rtx;
  if (!s390_logical_operator_ok_p (operands))
    FAIL;
})

;
; xordi3 instruction pattern(s).
;

(define_insn "*xordi3_cc"
  [(set (reg CC_REGNUM)
        (compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d, 0")
                         (match_operand:DI 2 "general_operand"      " d,d,RT"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand"                      "=d,d, d")
        (xor:DI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   xgr\t%0,%2
   xgrk\t%0,%1,%2
   xg\t%0,%2"
  [(set_attr "op_type" "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*xordi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d, 0")
                         (match_operand:DI 2 "general_operand"      " d,d,RT"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0                                     "=d,d, d"))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   xgr\t%0,%2
   xgrk\t%0,%1,%2
   xg\t%0,%2"
  [(set_attr "op_type" "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*xordi3"
  [(set (match_operand:DI 0 "nonimmediate_operand"         "=d,    d,d,d, d,   AQ,Q")
        (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,    0,0,d, 0,    0,0")
                (match_operand:DI 2 "general_operand"   "N0SD0,N1SD0,d,d,RT,NxQD0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   xihf\t%0,%k2
   xilf\t%0,%k2
   xgr\t%0,%2
   xgrk\t%0,%1,%2
   xg\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RIL,RIL,RRE,RRF,RXY,SI,SS")
   (set_attr "cpu_facility" "extimm,extimm,*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1,
                        *,z10_super_E1,*,*")])

(define_split
  [(set (match_operand:DI 0 "s_operand" "")
        (xor:DI (match_dup 0) (match_operand:DI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (xor:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (XOR, &operands[0], &operands[1]);")

;
; xorsi3 instruction pattern(s).
;

(define_insn "*xorsi3_cc"
  [(set (reg CC_REGNUM)
        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (set (match_operand:SI 0 "register_operand"                      "=d,d,d,d,d")
        (xor:SI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   xilf\t%0,%o2
   xr\t%0,%2
   xrk\t%0,%1,%2
   x\t%0,%2
   xy\t%0,%2"
  [(set_attr "op_type" "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1")])

(define_insn "*xorsi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:SI 0                                     "=d,d,d,d,d"))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   xilf\t%0,%o2
   xr\t%0,%2
   xrk\t%0,%1,%2
   x\t%0,%2
   xy\t%0,%2"
  [(set_attr "op_type" "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1")])

(define_insn "*xorsi3"
  [(set (match_operand:SI 0 "nonimmediate_operand"         "=d,d,d,d,d,   AQ,Q")
        (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0,    0,0")
                (match_operand:SI 2 "general_operand"      "Os,d,d,R,T,NxQS0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "s390_logical_operator_ok_p (operands)"
  "@
   xilf\t%0,%o2
   xr\t%0,%2
   xrk\t%0,%1,%2
   x\t%0,%2
   xy\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY,SI,SS")
   (set_attr "cpu_facility" "*,*,z196,*,*,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,*,*")])

(define_split
  [(set (match_operand:SI 0 "s_operand" "")
        (xor:SI (match_dup 0) (match_operand:SI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (xor:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (XOR, &operands[0], &operands[1]);")

;
; xorhi3 instruction pattern(s).
;

(define_insn "*xorhi3"
  [(set (match_operand:HI 0 "nonimmediate_operand"         "=d,d,d,   AQ,Q")
        (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,d,    0,0")
                (match_operand:HI 2 "general_operand"      "Os,d,d,NxQH0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "s390_logical_operator_ok_p (operands)"
  "@
   xilf\t%0,%x2
   xr\t%0,%2
   xrk\t%0,%1,%2
   #
   #"
  [(set_attr "op_type"  "RIL,RR,RRF,SI,SS")
   (set_attr "cpu_facility" "*,*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,*,*")])

(define_split
  [(set (match_operand:HI 0 "s_operand" "")
        (xor:HI (match_dup 0) (match_operand:HI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (xor:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (XOR, &operands[0], &operands[1]);")

;
; xorqi3 instruction pattern(s).
;

(define_insn "*xorqi3"
  [(set (match_operand:QI 0 "nonimmediate_operand"         "=d,d,d,Q,S,Q")
        (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,d,0,0,0")
                (match_operand:QI 2 "general_operand"      "Os,d,d,n,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "s390_logical_operator_ok_p (operands)"
  "@
   xilf\t%0,%b2
   xr\t%0,%2
   xrk\t%0,%1,%2
   xi\t%S0,%b2
   xiy\t%S0,%b2
   #"
  [(set_attr "op_type"  "RIL,RR,RRF,SI,SIY,SS")
   (set_attr "cpu_facility" "*,*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,z10_super,z10_super,*")])


;
; Block exclusive or (XC) patterns.
;

(define_insn "*xc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (xor:BLK (match_dup 0)
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "xc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")])

(define_split
  [(set (match_operand 0 "memory_operand" "")
        (xor (match_dup 0)
             (match_operand 1 "memory_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (xor:BLK (match_dup 0) (match_dup 1)))
     (use (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (xor:BLK (match_dup 0)
                   (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (xor:BLK (match_dup 3)
                   (match_operand:BLK 4 "memory_operand" "")))
     (use (match_operand 5 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (xor:BLK (match_dup 6) (match_dup 7)))
     (use (match_dup 8))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")

;
; Block xor (XC) patterns with src == dest.
;

(define_insn "*xc_zero"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (const_int 0))
   (use (match_operand 1 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[1]) >= 1 && INTVAL (operands[1]) <= 256"
  "xc\t%O0(%1,%R0),%S0"
  [(set_attr "op_type" "SS")
   (set_attr "z196prop" "z196_cracked")])

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (const_int 0))
     (use (match_operand 1 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 2 "memory_operand" "")
          (const_int 0))
     (use (match_operand 3 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[2], operands[1])
   && INTVAL (operands[1]) + INTVAL (operands[3]) <= 256"
  [(parallel
    [(set (match_dup 4) (const_int 0))
     (use (match_dup 5))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[4] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[5] = GEN_INT (INTVAL (operands[1]) + INTVAL (operands[3]));")


;;
;;- Negate instructions.
;;

;
; neg(di|si)2 instruction pattern(s).
;

(define_expand "neg<mode>2"
  [(parallel
    [(set (match_operand:DSI 0 "register_operand" "=d")
          (neg:DSI (match_operand:DSI 1 "register_operand" "d")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*negdi2_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:DI (ashiftrt:DI (ashift:DI (subreg:DI
                           (match_operand:SI 1 "register_operand" "d") 0)
                           (const_int 32)) (const_int 32)))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (sign_extend:DI (match_dup 1))))]
  "TARGET_ZARCH && s390_match_ccmode (insn, CCAmode)"
  "lcgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

(define_insn "*negdi2_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (sign_extend:DI (match_operand:SI 1 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "lcgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

; lcr, lcgr
(define_insn "*neg<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (match_operand:GPR 1 "register_operand" "d"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (match_dup 1)))]
  "s390_match_ccmode (insn, CCAmode)"
  "lc<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_super_c_E1")])

; lcr, lcgr
(define_insn "*neg<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (match_operand:GPR 1 "register_operand" "d"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "s390_match_ccmode (insn, CCAmode)"
  "lc<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_super_c_E1")])

; lcr, lcgr
(define_insn "*neg<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (match_operand:GPR 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "lc<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_super_c_E1")])

(define_insn "*negdi2_31"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (match_operand:DI 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "#")

; Split a DImode NEG on 31bit into 2 SImode NEGs

; Doing the twos complement separately on the SImode parts does an
; unwanted +1 on the high part which needs to be subtracted afterwards
; ... unless the +1 on the low part created an overflow.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (neg:DI (match_operand:DI 1 "register_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH
   && (REGNO (operands[0]) == REGNO (operands[1])
      || s390_split_ok_p (operands[0], operands[1], DImode, 0))
   && reload_completed"
  [(parallel
    [(set (match_dup 2) (neg:SI (match_dup 3)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (reg:CCAP CC_REGNUM)
          (compare:CCAP (neg:SI (match_dup 5)) (const_int 0)))
     (set (match_dup 4) (neg:SI (match_dup 5)))])
   (set (pc)
        (if_then_else (ne (reg:CCAP CC_REGNUM) (const_int 0))
                      (pc)
                      (label_ref (match_dup 6))))
   (parallel
    [(set (match_dup 2) (plus:SI (match_dup 2) (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])
   (match_dup 6)]
  "operands[2] = operand_subword (operands[0], 0, 0, DImode);
   operands[3] = operand_subword (operands[1], 0, 0, DImode);
   operands[4] = operand_subword (operands[0], 1, 0, DImode);
   operands[5] = operand_subword (operands[1], 1, 0, DImode);
   operands[6] = gen_label_rtx ();")

; Like above but first make a copy of the low part of the src operand
; since it might overlap with the high part of the destination.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (neg:DI (match_operand:DI 1 "register_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH
   && s390_split_ok_p (operands[0], operands[1], DImode, 1)
   && reload_completed"
  [; Make a backup of op5 first
   (set (match_dup 4) (match_dup 5))
   ; Setting op2 here might clobber op5
   (parallel
    [(set (match_dup 2) (neg:SI (match_dup 3)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (reg:CCAP CC_REGNUM)
          (compare:CCAP (neg:SI (match_dup 4)) (const_int 0)))
     (set (match_dup 4) (neg:SI (match_dup 4)))])
   (set (pc)
        (if_then_else (ne (reg:CCAP CC_REGNUM) (const_int 0))
                      (pc)
                      (label_ref (match_dup 6))))
   (parallel
    [(set (match_dup 2) (plus:SI (match_dup 2) (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])
   (match_dup 6)]
  "operands[2] = operand_subword (operands[0], 0, 0, DImode);
   operands[3] = operand_subword (operands[1], 0, 0, DImode);
   operands[4] = operand_subword (operands[0], 1, 0, DImode);
   operands[5] = operand_subword (operands[1], 1, 0, DImode);
   operands[6] = gen_label_rtx ();")

;
; neg(df|sf)2 instruction pattern(s).
;

(define_expand "neg<mode>2"
  [(parallel
    [(set (match_operand:BFP 0 "register_operand" "=f")
          (neg:BFP (match_operand:BFP 1 "register_operand" "f")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
  "")

; lcxbr, lcdbr, lcebr
(define_insn "*neg<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (set (match_operand:BFP 0 "register_operand" "=f")
        (neg:BFP (match_dup 1)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lc<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lcxbr, lcdbr, lcebr
(define_insn "*neg<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (clobber (match_scratch:BFP 0 "=f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lc<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lcdfr
(define_insn "*neg<mode>2_nocc"
  [(set (match_operand:FP 0 "register_operand"         "=f")
        (neg:FP (match_operand:FP 1 "register_operand" "<fT0>")))]
  "TARGET_DFP"
  "lcdfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lcxbr, lcdbr, lcebr
; FIXME: wflcdb does not clobber cc
(define_insn "*neg<mode>2"
  [(set (match_operand:BFP          0 "register_operand" "=f,<vf>")
        (neg:BFP (match_operand:BFP 1 "register_operand"  "f,<vf>")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   lc<xde>br\t%0,%1
   wflcdb\t%0,%1"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "cpu_facility" "*,vec")
   (set_attr "type"         "fsimp<mode>,*")])


;;
;;- Absolute value instructions.
;;

;
; abs(di|si)2 instruction pattern(s).
;

(define_insn "*absdi2_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (abs:DI (ashiftrt:DI (ashift:DI (subreg:DI
                           (match_operand:SI 1 "register_operand" "d") 0)
                           (const_int 32)) (const_int 32)))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (abs:DI (sign_extend:DI (match_dup 1))))]
  "TARGET_ZARCH && s390_match_ccmode (insn, CCAmode)"
  "lpgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

(define_insn "*absdi2_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (abs:DI (sign_extend:DI (match_operand:SI 1 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "lpgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

; lpr, lpgr
(define_insn "*abs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (abs:GPR (match_operand:DI 1 "register_operand" "d"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
        (abs:GPR (match_dup 1)))]
  "s390_match_ccmode (insn, CCAmode)"
  "lp<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lpr, lpgr
(define_insn "*abs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (abs:GPR (match_operand:GPR 1 "register_operand" "d"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "s390_match_ccmode (insn, CCAmode)"
  "lp<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lpr, lpgr
(define_insn "abs<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (abs:GPR (match_operand:GPR 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "lp<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

;
; abs(df|sf)2 instruction pattern(s).
;

(define_expand "abs<mode>2"
  [(parallel
    [(set (match_operand:BFP 0 "register_operand" "=f")
          (abs:BFP (match_operand:BFP 1 "register_operand" "f")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
  "")

; lpxbr, lpdbr, lpebr
(define_insn "*abs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (abs:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (set (match_operand:BFP 0 "register_operand" "=f")
        (abs:BFP (match_dup 1)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lp<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lpxbr, lpdbr, lpebr
(define_insn "*abs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (abs:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (clobber (match_scratch:BFP 0 "=f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lp<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lpdfr
(define_insn "*abs<mode>2_nocc"
  [(set (match_operand:FP 0 "register_operand"         "=f")
        (abs:FP (match_operand:FP 1 "register_operand" "<fT0>")))]
  "TARGET_DFP"
  "lpdfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lpxbr, lpdbr, lpebr
; FIXME: wflpdb does not clobber cc
(define_insn "*abs<mode>2"
  [(set (match_operand:BFP          0 "register_operand" "=f,<vf>")
        (abs:BFP (match_operand:BFP 1 "register_operand"  "f,<vf>")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
    lp<xde>br\t%0,%1
    wflpdb\t%0,%1"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "cpu_facility" "*,vec")
   (set_attr "type"         "fsimp<mode>,*")])


;;
;;- Negated absolute value instructions
;;

;
; Integer
;

(define_insn "*negabsdi2_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:DI (abs:DI (ashiftrt:DI (ashift:DI (subreg:DI
                           (match_operand:SI 1 "register_operand" "d") 0)
                           (const_int 32)) (const_int 32))))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (abs:DI (sign_extend:DI (match_dup 1)))))]
  "TARGET_ZARCH && s390_match_ccmode (insn, CCAmode)"
  "lngfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

(define_insn "*negabsdi2_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (abs:DI (sign_extend:DI
                          (match_operand:SI 1 "register_operand" "d")))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "lngfr\t%0,%1"
  [(set_attr "op_type" "RRE")
   (set_attr "z10prop" "z10_c")])

; lnr, lngr
(define_insn "*negabs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (abs:GPR (match_operand:GPR 1 "register_operand" "d")))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (abs:GPR (match_dup 1))))]
  "s390_match_ccmode (insn, CCAmode)"
  "ln<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lnr, lngr
(define_insn "*negabs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (abs:GPR (match_operand:GPR 1 "register_operand" "d")))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "s390_match_ccmode (insn, CCAmode)"
  "ln<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lnr, lngr
(define_insn "*negabs<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (abs:GPR (match_operand:GPR 1 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "ln<g>r\t%0,%1"
  [(set_attr "op_type" "RR<E>")
   (set_attr "z10prop" "z10_c")])

;
; Floating point
;

; lnxbr, lndbr, lnebr
(define_insn "*negabs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (abs:BFP (match_operand:BFP 1 "register_operand" "f")))
                 (match_operand:BFP 2 "const0_operand" "")))
   (set (match_operand:BFP 0 "register_operand" "=f")
        (neg:BFP (abs:BFP (match_dup 1))))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "ln<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lnxbr, lndbr, lnebr
(define_insn "*negabs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (abs:BFP (match_operand:BFP 1 "register_operand" "f")))
                 (match_operand:BFP 2 "const0_operand" "")))
   (clobber (match_scratch:BFP 0 "=f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "ln<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lndfr
(define_insn "*negabs<mode>2_nocc"
  [(set (match_operand:FP 0 "register_operand"                  "=f")
        (neg:FP (abs:FP (match_operand:BFP 1 "register_operand" "<fT0>"))))]
  "TARGET_DFP"
  "lndfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lnxbr, lndbr, lnebr
; FIXME: wflndb does not clobber cc
(define_insn "*negabs<mode>2"
  [(set (match_operand:BFP                   0 "register_operand" "=f,<vf>")
        (neg:BFP (abs:BFP (match_operand:BFP 1 "register_operand"  "f,<vf>"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   ln<xde>br\t%0,%1
   wflndb\t%0,%1"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "cpu_facility" "*,vec")
   (set_attr "type"         "fsimp<mode>,*")])

;;
;;- Square root instructions.
;;

;
; sqrt(df|sf)2 instruction pattern(s).
;

; sqxbr, sqdbr, sqebr, sqdb, sqeb
(define_insn "sqrt<mode>2"
  [(set (match_operand:BFP           0 "register_operand" "=f,   f,<vf>")
        (sqrt:BFP (match_operand:BFP 1 "general_operand"   "f,<Rf>,<vf>")))]
  "TARGET_HARD_FLOAT"
  "@
   sq<xde>br\t%0,%1
   sq<xde>b\t%0,%1
   wfsqdb\t%v0,%v1"
  [(set_attr "op_type"      "RRE,RXE,VRR")
   (set_attr "type"         "fsqrt<mode>")
   (set_attr "cpu_facility" "*,*,vec")])


;;
;;- One complement instructions.
;;

;
; one_cmpl(di|si|hi|qi)2 instruction pattern(s).
;

(define_expand "one_cmpl<mode>2"
  [(parallel
    [(set (match_operand:INT 0 "register_operand" "")
          (xor:INT (match_operand:INT 1 "register_operand" "")
                   (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")


;;
;; Find leftmost bit instructions.
;;

(define_expand "clzdi2"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (clz:DI (match_operand:DI 1 "register_operand" "d")))]
  "TARGET_EXTIMM && TARGET_ZARCH"
{
  rtx insn, clz_equal;
  rtx wide_reg = gen_reg_rtx (TImode);
  rtx msb = gen_rtx_CONST_INT (DImode, (unsigned HOST_WIDE_INT) 1 << 63);

  clz_equal = gen_rtx_CLZ (DImode, operands[1]);

  emit_insn (gen_clztidi2 (wide_reg, operands[1], msb));

  insn = emit_move_insn (operands[0], gen_highpart (DImode, wide_reg));
  set_unique_reg_note (insn, REG_EQUAL, clz_equal);

  DONE;
})

(define_insn "clztidi2"
  [(set (match_operand:TI 0 "register_operand" "=d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (xor:DI (match_operand:DI 1 "register_operand" "d")
                      (lshiftrt (match_operand:DI 2 "const_int_operand" "")
                                (subreg:SI (clz:DI (match_dup 1)) 4))))

            (const_int 64))
          (zero_extend:TI (clz:DI (match_dup 1)))))
   (clobber (reg:CC CC_REGNUM))]
  "(unsigned HOST_WIDE_INT) INTVAL (operands[2])
   == (unsigned HOST_WIDE_INT) 1 << 63
   && TARGET_EXTIMM && TARGET_ZARCH"
  "flogr\t%0,%1"
  [(set_attr "op_type"  "RRE")])


;;
;;- Rotate instructions.
;;

;
; rotl(di|si)3 instruction pattern(s).
;

; rll, rllg
(define_insn "rotl<mode>3"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (rotate:GPR (match_operand:GPR 1 "register_operand" "d")
                    (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))]
  "TARGET_CPU_ZARCH"
  "rll<g>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RSE")
   (set_attr "atype"    "reg")
   (set_attr "z10prop" "z10_super_E1")])

; rll, rllg
(define_insn "*rotl<mode>3_and"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (rotate:GPR (match_operand:GPR 1 "register_operand" "d")
                    (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
                            (match_operand:SI 3 "const_int_operand"   "n"))))]
  "TARGET_CPU_ZARCH && (INTVAL (operands[3]) & 63) == 63"
  "rll<g>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RSE")
   (set_attr "atype"    "reg")
   (set_attr "z10prop" "z10_super_E1")])


;;
;;- Shift instructions.
;;

;
; (ashl|lshr)(di|si)3 instruction pattern(s).
; Left shifts and logical right shifts

(define_expand "<shift><mode>3"
  [(set (match_operand:DSI 0 "register_operand" "")
        (SHIFT:DSI (match_operand:DSI 1 "register_operand" "")
                   (match_operand:SI 2 "shift_count_or_setmem_operand" "")))]
  ""
  "")

; sldl, srdl
(define_insn "*<shift>di3_31"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (SHIFT:DI (match_operand:DI 1 "register_operand" "0")
                  (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))]
  "!TARGET_ZARCH"
  "s<lr>dl\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")
   (set_attr "z196prop" "z196_cracked")])

; sll, srl, sllg, srlg, sllk, srlk
(define_insn "*<shift><mode>3"
  [(set (match_operand:GPR 0 "register_operand"                          "=d,d")
        (SHIFT:GPR (match_operand:GPR 1 "register_operand"             "<d0>,d")
                   (match_operand:SI 2 "shift_count_or_setmem_operand"    "Y,Y")))]
  ""
  "@
   s<lr>l<g>\t%0,<1>%Y2
   s<lr>l<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])

; sldl, srdl
(define_insn "*<shift>di3_31_and"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (SHIFT:DI (match_operand:DI 1 "register_operand" "0")
                  (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
                          (match_operand:SI 3 "const_int_operand"   "n"))))]
  "!TARGET_ZARCH && (INTVAL (operands[3]) & 63) == 63"
  "s<lr>dl\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

; sll, srl, sllg, srlg, sllk, srlk
(define_insn "*<shift><mode>3_and"
  [(set (match_operand:GPR 0 "register_operand"                                 "=d,d")
        (SHIFT:GPR (match_operand:GPR 1 "register_operand"                    "<d0>,d")
                   (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand"   "Y,Y")
                           (match_operand:SI 3 "const_int_operand"               "n,n"))))]
  "(INTVAL (operands[3]) & 63) == 63"
  "@
   s<lr>l<g>\t%0,<1>%Y2
   s<lr>l<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])

;
; ashr(di|si)3 instruction pattern(s).
; Arithmetic right shifts

(define_expand "ashr<mode>3"
  [(parallel
    [(set (match_operand:DSI 0 "register_operand" "")
          (ashiftrt:DSI (match_operand:DSI 1 "register_operand" "")
                        (match_operand:SI 2 "shift_count_or_setmem_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*ashrdi3_cc_31"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
                              (match_operand:SI 2 "shift_count_or_setmem_operand" "Y"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (ashiftrt:DI (match_dup 1) (match_dup 2)))]
  "!TARGET_ZARCH && s390_match_ccmode(insn, CCSmode)"
  "srda\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

(define_insn "*ashrdi3_cconly_31"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
                              (match_operand:SI 2 "shift_count_or_setmem_operand" "Y"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d"))]
  "!TARGET_ZARCH && s390_match_ccmode(insn, CCSmode)"
  "srda\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

(define_insn "*ashrdi3_31"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
                     (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "srda\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

; sra, srag, srak
(define_insn "*ashr<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand"          "<d0>,d")
                               (match_operand:SI 2 "shift_count_or_setmem_operand" "Y,Y"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand"                                   "=d,d")
        (ashiftrt:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   sra<g>\t%0,<1>%Y2
   sra<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])

; sra, srag, srak
(define_insn "*ashr<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand"          "<d0>,d")
                               (match_operand:SI 2 "shift_count_or_setmem_operand" "Y,Y"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0                                                  "=d,d"))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   sra<g>\t%0,<1>%Y2
   sra<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])

; sra, srag
(define_insn "*ashr<mode>3"
  [(set (match_operand:GPR 0 "register_operand"                          "=d,d")
        (ashiftrt:GPR (match_operand:GPR 1 "register_operand"          "<d0>,d")
                      (match_operand:SI 2 "shift_count_or_setmem_operand" "Y,Y")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   sra<g>\t%0,<1>%Y2
   sra<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])


; shift pattern with implicit ANDs

(define_insn "*ashrdi3_cc_31_and"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
                              (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
                                      (match_operand:SI 3 "const_int_operand"   "n")))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (ashiftrt:DI (match_dup 1) (and:SI (match_dup 2) (match_dup 3))))]
  "!TARGET_ZARCH && s390_match_ccmode(insn, CCSmode)
   && (INTVAL (operands[3]) & 63) == 63"
  "srda\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

(define_insn "*ashrdi3_cconly_31_and"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
                              (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
                                      (match_operand:SI 3 "const_int_operand"   "n")))
                 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d"))]
  "!TARGET_ZARCH && s390_match_ccmode(insn, CCSmode)
   && (INTVAL (operands[3]) & 63) == 63"
  "srda\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

(define_insn "*ashrdi3_31_and"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
                     (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
                             (match_operand:SI 3 "const_int_operand"   "n"))))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && (INTVAL (operands[3]) & 63) == 63"
  "srda\t%0,%Y2"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")])

; sra, srag, srak
(define_insn "*ashr<mode>3_cc_and"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand"                  "<d0>,d")
                               (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y,Y")
                                       (match_operand:SI 3 "const_int_operand"             "n,n")))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand"                                           "=d,d")
        (ashiftrt:GPR (match_dup 1) (and:SI (match_dup 2) (match_dup 3))))]
  "s390_match_ccmode(insn, CCSmode) && (INTVAL (operands[3]) & 63) == 63"
  "@
   sra<g>\t%0,<1>%Y2
   sra<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])

; sra, srag, srak
(define_insn "*ashr<mode>3_cconly_and"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand"                  "<d0>,d")
                               (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y,Y")
                                       (match_operand:SI 3 "const_int_operand"             "n,n")))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0                                                          "=d,d"))]
  "s390_match_ccmode(insn, CCSmode) && (INTVAL (operands[3]) & 63) == 63"
  "@
   sra<g>\t%0,<1>%Y2
   sra<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])

; sra, srag, srak
(define_insn "*ashr<mode>3_and"
  [(set (match_operand:GPR 0 "register_operand"                                  "=d,d")
        (ashiftrt:GPR (match_operand:GPR 1 "register_operand"                  "<d0>,d")
                      (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y,Y")
                              (match_operand:SI 3 "const_int_operand"             "n,n"))))
   (clobber (reg:CC CC_REGNUM))]
  "(INTVAL (operands[3]) & 63) == 63"
  "@
   sra<g>\t%0,<1>%Y2
   sra<gk>\t%0,%1,%Y2"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])


;;
;; Branch instruction patterns.
;;

(define_expand "cbranch<mode>4"
  [(set (pc)
        (if_then_else (match_operator 0 "comparison_operator"
                       [(match_operand:GPR 1 "register_operand" "")
                        (match_operand:GPR 2 "general_operand" "")])
                      (label_ref (match_operand 3 "" ""))
                      (pc)))]
  ""
  "s390_emit_jump (operands[3],
    s390_emit_compare (GET_CODE (operands[0]), operands[1], operands[2]));
   DONE;")

(define_expand "cbranch<mode>4"
  [(set (pc)
        (if_then_else (match_operator 0 "comparison_operator"
                       [(match_operand:FP 1 "register_operand" "")
                        (match_operand:FP 2 "general_operand" "")])
                      (label_ref (match_operand 3 "" ""))
                      (pc)))]
  "TARGET_HARD_FLOAT"
  "s390_emit_jump (operands[3],
    s390_emit_compare (GET_CODE (operands[0]), operands[1], operands[2]));
   DONE;")

(define_expand "cbranchcc4"
  [(set (pc)
        (if_then_else (match_operator 0 "s390_comparison"
                       [(match_operand 1 "cc_reg_operand" "")
                        (match_operand 2 "const_int_operand" "")])
                      (label_ref (match_operand 3 "" ""))
                      (pc)))]
  ""
  "")


;;
;;- Conditional jump instructions.
;;

(define_insn "*cjump_64"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM)
                                               (match_operand 2 "const_int_operand" "")])
          (label_ref (match_operand 0 "" ""))
          (pc)))]
  "TARGET_CPU_ZARCH"
{
  if (get_attr_length (insn) == 4)
    return "j%C1\t%l0";
  else
    return "jg%C1\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 6)))])

(define_insn "*cjump_31"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM)
                                               (match_operand 2 "const_int_operand" "")])
          (label_ref (match_operand 0 "" ""))
          (pc)))]
  "!TARGET_CPU_ZARCH"
{
  gcc_assert (get_attr_length (insn) == 4);
  return "j%C1\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")
   (set (attr "length")
        (if_then_else (not (match_test "flag_pic"))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 6))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 8))))])

(define_insn "*cjump_long"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (match_operand 0 "address_operand" "ZQZR")
          (pc)))]
  ""
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "b%C1r\t%0";
  else
    return "b%C1\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

;; A conditional return instruction.
(define_insn "*c<code>"
  [(set (pc)
        (if_then_else
          (match_operator 0 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (ANY_RETURN)
          (pc)))]
  "s390_can_use_<code>_insn ()"
  "b%C0r\t%%r14"
  [(set_attr "op_type" "RR")
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")])

;;
;;- Negated conditional jump instructions.
;;

(define_insn "*icjump_64"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (pc)
          (label_ref (match_operand 0 "" ""))))]
  "TARGET_CPU_ZARCH"
{
  if (get_attr_length (insn) == 4)
    return "j%D1\t%l0";
  else
    return "jg%D1\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 6)))])

(define_insn "*icjump_31"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (pc)
          (label_ref (match_operand 0 "" ""))))]
  "!TARGET_CPU_ZARCH"
{
  gcc_assert (get_attr_length (insn) == 4);
  return "j%D1\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")
   (set (attr "length")
        (if_then_else (not (match_test "flag_pic"))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 6))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 8))))])

(define_insn "*icjump_long"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (pc)
          (match_operand 0 "address_operand" "ZQZR")))]
  ""
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "b%D1r\t%0";
  else
    return "b%D1\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

;;
;;- Trap instructions.
;;

(define_insn "trap"
  [(trap_if (const_int 1) (const_int 0))]
  ""
  "j\t.+2"
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")])

(define_expand "ctrap<mode>4"
  [(trap_if (match_operator 0 "comparison_operator"
             [(match_operand:GPR 1 "register_operand" "")
              (match_operand:GPR 2 "general_operand" "")])
             (match_operand 3 "const0_operand" ""))]
  ""
  {
    rtx cond = s390_emit_compare (GET_CODE (operands[0]),
                                  operands[1], operands[2]);
    emit_insn (gen_condtrap (cond, XEXP (cond, 0)));
    DONE;
  })

(define_expand "ctrap<mode>4"
  [(trap_if (match_operator 0 "comparison_operator"
             [(match_operand:FP 1 "register_operand" "")
              (match_operand:FP 2 "general_operand" "")])
             (match_operand 3 "const0_operand" ""))]
  ""
  {
    rtx cond = s390_emit_compare (GET_CODE (operands[0]),
                                  operands[1], operands[2]);
    emit_insn (gen_condtrap (cond, XEXP (cond, 0)));
    DONE;
  })

(define_insn "condtrap"
  [(trap_if (match_operator 0 "s390_comparison"
             [(match_operand 1 "cc_reg_operand" "c")
              (const_int 0)])
            (const_int 0))]
  ""
  "j%C0\t.+2";
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")])

; crt, cgrt, cit, cgit
(define_insn "*cmp_and_trap_signed_int<mode>"
  [(trap_if (match_operator 0 "s390_signed_integer_comparison"
               [(match_operand:GPR 1 "register_operand"  "d,d")
                (match_operand:GPR 2 "nonmemory_operand" "d,K")])
            (const_int 0))]
  "TARGET_Z10"
  "@
   c<g>rt%C0\t%1,%2
   c<g>it%C0\t%1,%h2"
  [(set_attr "op_type" "RRF,RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")])

; clrt, clgrt, clfit, clgit, clt, clgt
(define_insn "*cmp_and_trap_unsigned_int<mode>"
  [(trap_if (match_operator 0 "s390_unsigned_integer_comparison"
               [(match_operand:GPR 1 "register_operand" "d,d, d")
                (match_operand:GPR 2 "general_operand"  "d,D,RT")])
            (const_int 0))]
  "TARGET_Z10"
  "@
   cl<g>rt%C0\t%1,%2
   cl<gf>it%C0\t%1,%x2
   cl<g>t%C0\t%1,%2"
  [(set_attr "op_type"      "RRF,RIE,RSY")
   (set_attr "type"         "branch")
   (set_attr "z10prop"      "z10_super_c,z10_super,*")
   (set_attr "cpu_facility" "z10,z10,zEC12")])

; lat, lgat
(define_insn "*load_and_trap<mode>"
  [(trap_if (eq (match_operand:GPR 0 "memory_operand"  "RT")
                (const_int 0))
            (const_int 0))
   (set (match_operand:GPR 1 "register_operand" "=d")
        (match_dup 0))]
  "TARGET_ZEC12"
  "l<g>at\t%1,%0"
  [(set_attr "op_type" "RXY")])


;;
;;- Loop instructions.
;;
;;  This is all complicated by the fact that since this is a jump insn
;;  we must handle our own output reloads.

;; branch on index

; This splitter will be matched by combine and has to add the 2 moves
; necessary to load the compare and the increment values into a
; register pair as needed by brxle.

(define_insn_and_split "*brx_stage1_<GPR:mode>"
  [(set (pc)
        (if_then_else
         (match_operator 6 "s390_brx_operator"
            [(plus:GPR (match_operand:GPR 1 "register_operand" "")
                       (match_operand:GPR 2 "general_operand"  ""))
             (match_operand:GPR 3 "register_operand" "")])
         (label_ref (match_operand 0 "" ""))
         (pc)))
   (set (match_operand:GPR 4 "nonimmediate_operand" "")
        (plus:GPR (match_dup 1) (match_dup 2)))
   (clobber (match_scratch:GPR 5 ""))]
  "TARGET_CPU_ZARCH"
  "#"
  "!reload_completed && !reload_in_progress"
  [(set (match_dup 7) (match_dup 2)) ; the increment
   (set (match_dup 8) (match_dup 3)) ; the comparison value
   (parallel [(set (pc)
                   (if_then_else
                    (match_op_dup 6
                       [(plus:GPR (match_dup 1) (match_dup 7))
                        (match_dup 8)])
                    (label_ref (match_dup 0))
                    (pc)))
              (set (match_dup 4)
                   (plus:GPR (match_dup 1) (match_dup 7)))
              (clobber (match_dup 5))
              (clobber (reg:CC CC_REGNUM))])]
  {
    rtx dreg = gen_reg_rtx (word_mode == DImode ? TImode : DImode);
    operands[7] = gen_lowpart (<GPR:MODE>mode,
                               gen_highpart (word_mode, dreg));
    operands[8] = gen_lowpart (<GPR:MODE>mode,
                               gen_lowpart (word_mode, dreg));
  })

; brxlg, brxhg

(define_insn_and_split "*brxg_64bit"
  [(set (pc)
        (if_then_else
          (match_operator 5 "s390_brx_operator"
             [(plus:DI (match_operand:DI 1 "register_operand" "d,d,d")
                       (subreg:DI (match_operand:TI 2 "register_operand" "d,d,d") 0))
              (subreg:DI (match_dup 2) 8)])
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:DI 3 "nonimmediate_operand" "=1,?X,?X")
        (plus:DI (match_dup 1)
                 (subreg:DI (match_dup 2) 0)))
   (clobber (match_scratch:DI 4 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 6)
    return "brx%E5g\t%1,%2,%l0";
  else
    return "agr\t%1,%2\;cgr\t%1,%M2\;jg%C5\t%l0";
}
  "&& reload_completed
   && (!REG_P (operands[3])
       || !rtx_equal_p (operands[1], operands[3]))"
  [(set (match_dup 4) (match_dup 1))
   (parallel [(set (match_dup 4) (plus:DI (match_dup 4) (subreg:DI (match_dup 2) 0)))
              (clobber (reg:CC CC_REGNUM))])
   (set (reg:CCS CC_REGNUM) (compare:CCS (match_dup 4) (subreg:DI (match_dup 2) 8)))
   (set (match_dup 3) (match_dup 4))
   (set (pc) (if_then_else (match_op_dup 5 [(reg:CCS CC_REGNUM) (const_int 0)])
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RIE")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 6) (const_int 16)))])

; brxle, brxh

(define_insn_and_split "*brx_64bit"
  [(set (pc)
        (if_then_else
          (match_operator 5 "s390_brx_operator"
             [(plus:SI (match_operand:SI 1 "register_operand" "d,d,d")
                       (subreg:SI (match_operand:TI 2 "register_operand" "d,d,d") 4))
              (subreg:SI (match_dup 2) 12)])
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 3 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1)
                 (subreg:SI (match_dup 2) 4)))
   (clobber (match_scratch:SI 4 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 6)
    return "brx%C5\t%1,%2,%l0";
  else
    return "ar\t%1,%2\;cr\t%1,%M2\;jg%C5\t%l0";
}
  "&& reload_completed
   && (!REG_P (operands[3])
       || !rtx_equal_p (operands[1], operands[3]))"
  [(set (match_dup 4) (match_dup 1))
   (parallel [(set (match_dup 4) (plus:SI (match_dup 4) (subreg:SI (match_dup 2) 4)))
              (clobber (reg:CC CC_REGNUM))])
   (set (reg:CCS CC_REGNUM) (compare:CCS (match_dup 4) (subreg:SI (match_dup 2) 12)))
   (set (match_dup 3) (match_dup 4))
   (set (pc) (if_then_else (match_op_dup 5 [(reg:CCS CC_REGNUM) (const_int 0)])
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RSI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 6) (const_int 14)))])

; brxle, brxh

(define_insn_and_split "*brx_31bit"
  [(set (pc)
        (if_then_else
          (match_operator 5 "s390_brx_operator"
            [(plus:SI (match_operand:SI 1 "register_operand" "d,d,d")
                      (subreg:SI (match_operand:DI 2 "register_operand" "d,d,d") 0))
             (subreg:SI (match_dup 2) 4)])
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 3 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1)
                 (subreg:SI (match_dup 2) 0)))
   (clobber (match_scratch:SI 4 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && TARGET_CPU_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 6)
    return "brx%C5\t%1,%2,%l0";
  else
    return "ar\t%1,%2\;cr\t%1,%M2\;jg%C5\t%l0";
}
  "&& reload_completed
   && (!REG_P (operands[3])
       || !rtx_equal_p (operands[1], operands[3]))"
  [(set (match_dup 4) (match_dup 1))
   (parallel [(set (match_dup 4) (plus:SI (match_dup 4) (subreg:SI (match_dup 2) 0)))
              (clobber (reg:CC CC_REGNUM))])
   (set (reg:CCS CC_REGNUM) (compare:CCS (match_dup 4) (subreg:SI (match_dup 2) 4)))
   (set (match_dup 3) (match_dup 4))
   (set (pc) (if_then_else (match_op_dup 5 [(reg:CCS CC_REGNUM) (const_int 0)])
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RSI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 6) (const_int 14)))])


;; branch on count

(define_expand "doloop_end"
  [(use (match_operand 0 "" ""))        ; loop pseudo
   (use (match_operand 1 "" ""))]       ; label
  ""
{
  if (GET_MODE (operands[0]) == SImode && !TARGET_CPU_ZARCH)
    emit_jump_insn (gen_doloop_si31 (operands[1], operands[0], operands[0]));
  else if (GET_MODE (operands[0]) == SImode && TARGET_CPU_ZARCH)
    emit_jump_insn (gen_doloop_si64 (operands[1], operands[0], operands[0]));
  else if (GET_MODE (operands[0]) == DImode && TARGET_ZARCH)
    emit_jump_insn (gen_doloop_di (operands[1], operands[0], operands[0]));
  else
    FAIL;

  DONE;
})

(define_insn_and_split "doloop_si64"
  [(set (pc)
        (if_then_else
          (ne (match_operand:SI 1 "register_operand" "d,d,d")
              (const_int 1))
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 2 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1) (const_int -1)))
   (clobber (match_scratch:SI 3 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_CPU_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 4)
    return "brct\t%1,%l0";
  else
    return "ahi\t%1,-1\;jgne\t%l0";
}
  "&& reload_completed
   && (! REG_P (operands[2])
       || ! rtx_equal_p (operands[1], operands[2]))"
  [(set (match_dup 3) (match_dup 1))
   (parallel [(set (reg:CCAN CC_REGNUM)
                   (compare:CCAN (plus:SI (match_dup 3) (const_int -1))
                                 (const_int 0)))
              (set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))])
   (set (match_dup 2) (match_dup 3))
   (set (pc) (if_then_else (ne (reg:CCAN CC_REGNUM) (const_int 0))
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RI")
   ; Strictly speaking, the z10 properties are valid for brct only, however, it does not
   ; hurt us in the (rare) case of ahi.
   (set_attr "z10prop"  "z10_super_E1")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 10)))])

(define_insn_and_split "doloop_si31"
  [(set (pc)
        (if_then_else
          (ne (match_operand:SI 1 "register_operand" "d,d,d")
              (const_int 1))
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 2 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1) (const_int -1)))
   (clobber (match_scratch:SI 3 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_CPU_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 4)
    return "brct\t%1,%l0";
  else
    gcc_unreachable ();
}
  "&& reload_completed
   && (! REG_P (operands[2])
       || ! rtx_equal_p (operands[1], operands[2]))"
  [(set (match_dup 3) (match_dup 1))
   (parallel [(set (reg:CCAN CC_REGNUM)
                   (compare:CCAN (plus:SI (match_dup 3) (const_int -1))
                                 (const_int 0)))
              (set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))])
   (set (match_dup 2) (match_dup 3))
   (set (pc) (if_then_else (ne (reg:CCAN CC_REGNUM) (const_int 0))
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RI")
   ; Strictly speaking, the z10 properties are valid for brct only, however, it does not
   ; hurt us in the (rare) case of ahi.
   (set_attr "z10prop"  "z10_super_E1")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (not (match_test "flag_pic"))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 6))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 8))))])

(define_insn "*doloop_si_long"
  [(set (pc)
        (if_then_else
          (ne (match_operand:SI 1 "register_operand" "d")
              (const_int 1))
          (match_operand 0 "address_operand" "ZQZR")
          (pc)))
   (set (match_operand:SI 2 "register_operand" "=1")
        (plus:SI (match_dup 1) (const_int -1)))
   (clobber (match_scratch:SI 3 "=X"))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_CPU_ZARCH"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "bctr\t%1,%0";
  else
    return "bct\t%1,%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")
   (set_attr "z10prop"  "z10_c")
   (set_attr "z196prop" "z196_cracked")])

(define_insn_and_split "doloop_di"
  [(set (pc)
        (if_then_else
          (ne (match_operand:DI 1 "register_operand" "d,d,d")
              (const_int 1))
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:DI 2 "nonimmediate_operand" "=1,?X,?X")
        (plus:DI (match_dup 1) (const_int -1)))
   (clobber (match_scratch:DI 3 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 4)
    return "brctg\t%1,%l0";
  else
    return "aghi\t%1,-1\;jgne\t%l0";
}
  "&& reload_completed
   && (! REG_P (operands[2])
       || ! rtx_equal_p (operands[1], operands[2]))"
  [(set (match_dup 3) (match_dup 1))
   (parallel [(set (reg:CCAN CC_REGNUM)
                   (compare:CCAN (plus:DI (match_dup 3) (const_int -1))
                                 (const_int 0)))
              (set (match_dup 3) (plus:DI (match_dup 3) (const_int -1)))])
   (set (match_dup 2) (match_dup 3))
   (set (pc) (if_then_else (ne (reg:CCAN CC_REGNUM) (const_int 0))
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RI")
   ; Strictly speaking, the z10 properties are valid for brct only, however, it does not
   ; hurt us in the (rare) case of ahi.
   (set_attr "z10prop"  "z10_super_E1")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 10)))])

;;
;;- Unconditional jump instructions.
;;

;
; jump instruction pattern(s).
;

(define_expand "jump"
  [(match_operand 0 "" "")]
  ""
  "s390_emit_jump (operands[0], NULL_RTX); DONE;")

(define_insn "*jump64"
  [(set (pc) (label_ref (match_operand 0 "" "")))]
  "TARGET_CPU_ZARCH"
{
  if (get_attr_length (insn) == 4)
    return "j\t%l0";
  else
    return "jg\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 6)))])

(define_insn "*jump31"
  [(set (pc) (label_ref (match_operand 0 "" "")))]
  "!TARGET_CPU_ZARCH"
{
  gcc_assert (get_attr_length (insn) == 4);
  return "j\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (not (match_test "flag_pic"))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 6))
          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                        (const_int 4) (const_int 8))))])

;
; indirect-jump instruction pattern(s).
;

(define_insn "indirect_jump"
 [(set (pc) (match_operand 0 "address_operand" "ZQZR"))]
  ""
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "br\t%0";
  else
    return "b\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

;
; casesi instruction pattern(s).
;

(define_insn "casesi_jump"
 [(set (pc) (match_operand 0 "address_operand" "ZQZR"))
   (use (label_ref (match_operand 1 "" "")))]
  ""
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "br\t%0";
  else
    return "b\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_expand "casesi"
  [(match_operand:SI 0 "general_operand" "")
   (match_operand:SI 1 "general_operand" "")
   (match_operand:SI 2 "general_operand" "")
   (label_ref (match_operand 3 "" ""))
   (label_ref (match_operand 4 "" ""))]
  ""
{
   rtx index  = gen_reg_rtx (SImode);
   rtx base   = gen_reg_rtx (Pmode);
   rtx target = gen_reg_rtx (Pmode);

   emit_move_insn (index, operands[0]);
   emit_insn (gen_subsi3 (index, index, operands[1]));
   emit_cmp_and_jump_insns (index, operands[2], GTU, NULL_RTX, SImode, 1,
                            operands[4]);

   if (Pmode != SImode)
     index = convert_to_mode (Pmode, index, 1);
   if (GET_CODE (index) != REG)
     index = copy_to_mode_reg (Pmode, index);

   if (TARGET_64BIT)
       emit_insn (gen_ashldi3 (index, index, GEN_INT (3)));
   else
       emit_insn (gen_ashlsi3 (index, index, const2_rtx));

   emit_move_insn (base, gen_rtx_LABEL_REF (Pmode, operands[3]));

   index = gen_const_mem (Pmode, gen_rtx_PLUS (Pmode, base, index));
   emit_move_insn (target, index);

   if (flag_pic)
     target = gen_rtx_PLUS (Pmode, base, target);
   emit_jump_insn (gen_casesi_jump (target, operands[3]));

   DONE;
})


;;
;;- Jump to subroutine.
;;
;;

;
; untyped call instruction pattern(s).
;

;; Call subroutine returning any type.
(define_expand "untyped_call"
  [(parallel [(call (match_operand 0 "" "")
                    (const_int 0))
              (match_operand 1 "" "")
              (match_operand 2 "" "")])]
  ""
{
  int i;

  emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));

  for (i = 0; i < XVECLEN (operands[2], 0); i++)
    {
      rtx set = XVECEXP (operands[2], 0, i);
      emit_move_insn (SET_DEST (set), SET_SRC (set));
    }

  /* The optimizer does not know that the call sets the function value
     registers we stored in the result block.  We avoid problems by
     claiming that all hard registers are used and clobbered at this
     point.  */
  emit_insn (gen_blockage ());

  DONE;
})

;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
;; all of memory.  This blocks insns from being moved across this point.

(define_insn "blockage"
  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
  ""
  ""
  [(set_attr "type"    "none")
   (set_attr "length"  "0")])

;
; sibcall patterns
;

(define_expand "sibcall"
  [(call (match_operand 0 "" "")
         (match_operand 1 "" ""))]
  ""
{
  s390_emit_call (XEXP (operands[0], 0), NULL_RTX, NULL_RTX, NULL_RTX);
  DONE;
})

(define_insn "*sibcall_br"
  [(call (mem:QI (reg SIBCALL_REGNUM))
         (match_operand 0 "const_int_operand" "n"))]
  "SIBLING_CALL_P (insn)
   && GET_MODE (XEXP (XEXP (PATTERN (insn), 0), 0)) == Pmode"
  "br\t%%r1"
  [(set_attr "op_type" "RR")
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_insn "*sibcall_brc"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))]
  "SIBLING_CALL_P (insn) && TARGET_SMALL_EXEC"
  "j\t%0"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")])

(define_insn "*sibcall_brcl"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))]
  "SIBLING_CALL_P (insn) && TARGET_CPU_ZARCH"
  "jg\t%0"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "branch")])

;
; sibcall_value patterns
;

(define_expand "sibcall_value"
  [(set (match_operand 0 "" "")
        (call (match_operand 1 "" "")
              (match_operand 2 "" "")))]
  ""
{
  s390_emit_call (XEXP (operands[1], 0), NULL_RTX, operands[0], NULL_RTX);
  DONE;
})

(define_insn "*sibcall_value_br"
  [(set (match_operand 0 "" "")
        (call (mem:QI (reg SIBCALL_REGNUM))
              (match_operand 1 "const_int_operand" "n")))]
  "SIBLING_CALL_P (insn)
   && GET_MODE (XEXP (XEXP (XEXP (PATTERN (insn), 1), 0), 0)) == Pmode"
  "br\t%%r1"
  [(set_attr "op_type" "RR")
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_insn "*sibcall_value_brc"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))]
  "SIBLING_CALL_P (insn) && TARGET_SMALL_EXEC"
  "j\t%1"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")])

(define_insn "*sibcall_value_brcl"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))]
  "SIBLING_CALL_P (insn) && TARGET_CPU_ZARCH"
  "jg\t%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "branch")])


;
; call instruction pattern(s).
;

(define_expand "call"
  [(call (match_operand 0 "" "")
         (match_operand 1 "" ""))
   (use (match_operand 2 "" ""))]
  ""
{
  s390_emit_call (XEXP (operands[0], 0), NULL_RTX, NULL_RTX,
                  gen_rtx_REG (Pmode, RETURN_REGNUM));
  DONE;
})

(define_insn "*bras"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))
   (clobber (match_operand 2 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)
   && TARGET_SMALL_EXEC
   && GET_MODE (operands[2]) == Pmode"
  "bras\t%2,%0"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*brasl"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))
   (clobber (match_operand 2 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)
   && TARGET_CPU_ZARCH
   && GET_MODE (operands[2]) == Pmode"
  "brasl\t%2,%0"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr"
  [(call (mem:QI (match_operand 0 "address_operand" "ZQZR"))
         (match_operand 1 "const_int_operand" "n"))
   (clobber (match_operand 2 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn) && GET_MODE (operands[2]) == Pmode"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "basr\t%2,%0";
  else
    return "bas\t%2,%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

;
; call_value instruction pattern(s).
;

(define_expand "call_value"
  [(set (match_operand 0 "" "")
        (call (match_operand 1 "" "")
              (match_operand 2 "" "")))
   (use (match_operand 3 "" ""))]
  ""
{
  s390_emit_call (XEXP (operands[1], 0), NULL_RTX, operands[0],
                  gen_rtx_REG (Pmode, RETURN_REGNUM));
  DONE;
})

(define_insn "*bras_r"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand:SI 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)
   && TARGET_SMALL_EXEC
   && GET_MODE (operands[3]) == Pmode"
  "bras\t%3,%1"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*brasl_r"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)
   && TARGET_CPU_ZARCH
   && GET_MODE (operands[3]) == Pmode"
  "brasl\t%3,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr_r"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "address_operand" "ZQZR"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn) && GET_MODE (operands[3]) == Pmode"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "basr\t%3,%1";
  else
    return "bas\t%3,%a1";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 1 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

;;
;;- Thread-local storage support.
;;

(define_expand "get_thread_pointer<mode>"
  [(set (match_operand:P 0 "nonimmediate_operand" "") (reg:P TP_REGNUM))]
  ""
  "")

(define_expand "set_thread_pointer<mode>"
  [(set (reg:P TP_REGNUM) (match_operand:P 0 "nonimmediate_operand" ""))
   (set (reg:P TP_REGNUM) (unspec_volatile:P [(reg:P TP_REGNUM)] UNSPECV_SET_TP))]
  ""
  "")

(define_insn "*set_tp"
  [(set (reg TP_REGNUM) (unspec_volatile [(reg TP_REGNUM)] UNSPECV_SET_TP))]
  ""
  ""
  [(set_attr "type" "none")
   (set_attr "length" "0")])

(define_insn "*tls_load_64"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "RT")
                    (match_operand:DI 2 "" "")]
                   UNSPEC_TLS_LOAD))]
  "TARGET_64BIT"
  "lg\t%0,%1%J2"
  [(set_attr "op_type" "RXE")
   (set_attr "z10prop" "z10_fwd_A3")])

(define_insn "*tls_load_31"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "R,T")
                    (match_operand:SI 2 "" "")]
                   UNSPEC_TLS_LOAD))]
  "!TARGET_64BIT"
  "@
   l\t%0,%1%J2
   ly\t%0,%1%J2"
  [(set_attr "op_type" "RX,RXY")
   (set_attr "type" "load")
   (set_attr "z10prop" "z10_fwd_A3,z10_fwd_A3")])

(define_insn "*bras_tls"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))
   (use (match_operand 4 "" ""))]
  "!SIBLING_CALL_P (insn)
   && TARGET_SMALL_EXEC
   && GET_MODE (operands[3]) == Pmode"
  "bras\t%3,%1%J4"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*brasl_tls"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))
   (use (match_operand 4 "" ""))]
  "!SIBLING_CALL_P (insn)
   && TARGET_CPU_ZARCH
   && GET_MODE (operands[3]) == Pmode"
  "brasl\t%3,%1%J4"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr_tls"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "address_operand" "ZQZR"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))
   (use (match_operand 4 "" ""))]
  "!SIBLING_CALL_P (insn) && GET_MODE (operands[3]) == Pmode"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "basr\t%3,%1%J4";
  else
    return "bas\t%3,%a1%J4";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 1 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

;;
;;- Atomic operations
;;

;
; memory barrier patterns.
;

(define_expand "mem_signal_fence"
  [(match_operand:SI 0 "const_int_operand")]            ;; model
  ""
{
  /* The s390 memory model is strong enough not to require any
     barrier in order to synchronize a thread with itself.  */
  DONE;
})

(define_expand "mem_thread_fence"
  [(match_operand:SI 0 "const_int_operand")]            ;; model
  ""
{
  /* Unless this is a SEQ_CST fence, the s390 memory model is strong
     enough not to require barriers of any kind.  */
  if (is_mm_seq_cst (memmodel_from_int (INTVAL (operands[0]))))
    {
      rtx mem = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
      MEM_VOLATILE_P (mem) = 1;
      emit_insn (gen_mem_thread_fence_1 (mem));
    }
  DONE;
})

; Although bcr is superscalar on Z10, this variant will never
; become part of an execution group.
; With z196 we can make use of the fast-BCR-serialization facility.
; This allows for a slightly faster sync which is sufficient for our
; purposes.
(define_insn "mem_thread_fence_1"
  [(set (match_operand:BLK 0 "" "")
        (unspec:BLK [(match_dup 0)] UNSPEC_MB))]
  ""
{
  if (TARGET_Z196)
    return "bcr\t14,0";
  else
    return "bcr\t15,0";
}
  [(set_attr "op_type" "RR")
   (set_attr "mnemonic" "bcr_flush")
   (set_attr "z196prop" "z196_alone")])

;
; atomic load/store operations
;

; Atomic loads need not examine the memory model at all.
(define_expand "atomic_load<mode>"
  [(match_operand:DINT 0 "register_operand")    ;; output
   (match_operand:DINT 1 "memory_operand")      ;; memory
   (match_operand:SI 2 "const_int_operand")]    ;; model
  ""
{
  if (MEM_ALIGN (operands[1]) < GET_MODE_BITSIZE (GET_MODE (operands[1])))
    FAIL;

  if (<MODE>mode == TImode)
    emit_insn (gen_atomic_loadti_1 (operands[0], operands[1]));
  else if (<MODE>mode == DImode && !TARGET_ZARCH)
    emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
  else
    emit_move_insn (operands[0], operands[1]);
  DONE;
})

; Different from movdi_31 in that we want no splitters.
(define_insn "atomic_loaddi_1"
  [(set (match_operand:DI 0 "register_operand" "=d,d,!*f,!*f")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "Q,S,R,T")]
                   UNSPEC_MOVA))]
  "!TARGET_ZARCH"
  "@
   lm\t%0,%M0,%S1
   lmy\t%0,%M0,%S1
   ld\t%0,%1
   ldy\t%0,%1"
  [(set_attr "op_type" "RS,RSY,RS,RSY")
   (set_attr "type" "lm,lm,floaddf,floaddf")])

(define_insn "atomic_loadti_1"
  [(set (match_operand:TI 0 "register_operand" "=r")
        (unspec:TI [(match_operand:TI 1 "memory_operand" "RT")]
                   UNSPEC_MOVA))]
  "TARGET_ZARCH"
  "lpq\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "type" "other")])

; Atomic stores must(?) enforce sequential consistency.
(define_expand "atomic_store<mode>"
  [(match_operand:DINT 0 "memory_operand")      ;; memory
   (match_operand:DINT 1 "register_operand")    ;; input
   (match_operand:SI 2 "const_int_operand")]    ;; model
  ""
{
  enum memmodel model = memmodel_from_int (INTVAL (operands[2]));

  if (MEM_ALIGN (operands[0]) < GET_MODE_BITSIZE (GET_MODE (operands[0])))
    FAIL;

  if (<MODE>mode == TImode)
    emit_insn (gen_atomic_storeti_1 (operands[0], operands[1]));
  else if (<MODE>mode == DImode && !TARGET_ZARCH)
    emit_insn (gen_atomic_storedi_1 (operands[0], operands[1]));
  else
    emit_move_insn (operands[0], operands[1]);
  if (is_mm_seq_cst (model))
    emit_insn (gen_mem_thread_fence (operands[2]));
  DONE;
})

; Different from movdi_31 in that we want no splitters.
(define_insn "atomic_storedi_1"
  [(set (match_operand:DI 0 "memory_operand" "=Q,S,R,T")
        (unspec:DI [(match_operand:DI 1 "register_operand" "d,d,!*f,!*f")]
                   UNSPEC_MOVA))]
  "!TARGET_ZARCH"
  "@
   stm\t%1,%N1,%S0
   stmy\t%1,%N1,%S0
   std %1,%0
   stdy %1,%0"
  [(set_attr "op_type" "RS,RSY,RS,RSY")
   (set_attr "type" "stm,stm,fstoredf,fstoredf")])

(define_insn "atomic_storeti_1"
  [(set (match_operand:TI 0 "memory_operand" "=RT")
        (unspec:TI [(match_operand:TI 1 "register_operand" "r")]
                   UNSPEC_MOVA))]
  "TARGET_ZARCH"
  "stpq\t%1,%0"
  [(set_attr "op_type" "RXY")
   (set_attr "type" "other")])

;
; compare and swap patterns.
;

(define_expand "atomic_compare_and_swap<mode>"
  [(match_operand:SI 0 "register_operand")      ;; bool success output
   (match_operand:DGPR 1 "nonimmediate_operand");; oldval output
   (match_operand:DGPR 2 "memory_operand")      ;; memory
   (match_operand:DGPR 3 "register_operand")    ;; expected intput
   (match_operand:DGPR 4 "register_operand")    ;; newval intput
   (match_operand:SI 5 "const_int_operand")     ;; is_weak
   (match_operand:SI 6 "const_int_operand")     ;; success model
   (match_operand:SI 7 "const_int_operand")]    ;; failure model
  ""
{
  rtx cc, cmp, output = operands[1];

  if (!register_operand (output, <MODE>mode))
    output = gen_reg_rtx (<MODE>mode);

  if (MEM_ALIGN (operands[2]) < GET_MODE_BITSIZE (GET_MODE (operands[2])))
    FAIL;

  emit_insn (gen_atomic_compare_and_swap<mode>_internal
             (output, operands[2], operands[3], operands[4]));

  /* We deliberately accept non-register operands in the predicate
     to ensure the write back to the output operand happens *before*
     the store-flags code below.  This makes it easier for combine
     to merge the store-flags code with a potential test-and-branch
     pattern following (immediately!) afterwards.  */
  if (output != operands[1])
    emit_move_insn (operands[1], output);

  cc = gen_rtx_REG (CCZ1mode, CC_REGNUM);
  cmp = gen_rtx_EQ (SImode, cc, const0_rtx);
  emit_insn (gen_cstorecc4 (operands[0], cmp, cc, const0_rtx));
  DONE;
})

(define_expand "atomic_compare_and_swap<mode>"
  [(match_operand:SI 0 "register_operand")      ;; bool success output
   (match_operand:HQI 1 "nonimmediate_operand") ;; oldval output
   (match_operand:HQI 2 "memory_operand")       ;; memory
   (match_operand:HQI 3 "general_operand")      ;; expected intput
   (match_operand:HQI 4 "general_operand")      ;; newval intput
   (match_operand:SI 5 "const_int_operand")     ;; is_weak
   (match_operand:SI 6 "const_int_operand")     ;; success model
   (match_operand:SI 7 "const_int_operand")]    ;; failure model
  ""
{
  s390_expand_cs_hqi (<MODE>mode, operands[0], operands[1], operands[2],
                      operands[3], operands[4], INTVAL (operands[5]));
  DONE;
})

(define_expand "atomic_compare_and_swap<mode>_internal"
  [(parallel
     [(set (match_operand:DGPR 0 "register_operand")
           (match_operand:DGPR 1 "memory_operand"))
      (set (match_dup 1)
           (unspec_volatile:DGPR
             [(match_dup 1)
              (match_operand:DGPR 2 "register_operand")
              (match_operand:DGPR 3 "register_operand")]
             UNSPECV_CAS))
      (set (reg:CCZ1 CC_REGNUM)
           (compare:CCZ1 (match_dup 1) (match_dup 2)))])]
  "")

; cdsg, csg
(define_insn "*atomic_compare_and_swap<mode>_1"
  [(set (match_operand:TDI 0 "register_operand" "=r")
        (match_operand:TDI 1 "memory_operand" "+QS"))
   (set (match_dup 1)
        (unspec_volatile:TDI
          [(match_dup 1)
           (match_operand:TDI 2 "register_operand" "0")
           (match_operand:TDI 3 "register_operand" "r")]
          UNSPECV_CAS))
   (set (reg:CCZ1 CC_REGNUM)
        (compare:CCZ1 (match_dup 1) (match_dup 2)))]
  "TARGET_ZARCH"
  "c<td>sg\t%0,%3,%S1"
  [(set_attr "op_type" "RSY")
   (set_attr "type"   "sem")])

; cds, cdsy
(define_insn "*atomic_compare_and_swapdi_2"
  [(set (match_operand:DI 0 "register_operand" "=r,r")
        (match_operand:DI 1 "memory_operand" "+Q,S"))
   (set (match_dup 1)
        (unspec_volatile:DI
          [(match_dup 1)
           (match_operand:DI 2 "register_operand" "0,0")
           (match_operand:DI 3 "register_operand" "r,r")]
          UNSPECV_CAS))
   (set (reg:CCZ1 CC_REGNUM)
        (compare:CCZ1 (match_dup 1) (match_dup 2)))]
  "!TARGET_ZARCH"
  "@
   cds\t%0,%3,%S1
   cdsy\t%0,%3,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "type" "sem")])

; cs, csy
(define_insn "*atomic_compare_and_swapsi_3"
  [(set (match_operand:SI 0 "register_operand" "=r,r")
        (match_operand:SI 1 "memory_operand" "+Q,S"))
   (set (match_dup 1)
        (unspec_volatile:SI
          [(match_dup 1)
           (match_operand:SI 2 "register_operand" "0,0")
           (match_operand:SI 3 "register_operand" "r,r")]
          UNSPECV_CAS))
   (set (reg:CCZ1 CC_REGNUM)
        (compare:CCZ1 (match_dup 1) (match_dup 2)))]
  ""
  "@
   cs\t%0,%3,%S1
   csy\t%0,%3,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "type"   "sem")])

;
; Other atomic instruction patterns.
;

; z196 load and add, xor, or and and instructions

(define_expand "atomic_fetch_<atomic><mode>"
  [(match_operand:GPR 0 "register_operand")             ;; val out
   (ATOMIC_Z196:GPR
     (match_operand:GPR 1 "memory_operand")             ;; memory
     (match_operand:GPR 2 "register_operand"))          ;; val in
   (match_operand:SI 3 "const_int_operand")]            ;; model
  "TARGET_Z196"
{
  if (MEM_ALIGN (operands[1]) < GET_MODE_BITSIZE (GET_MODE (operands[1])))
    FAIL;

  emit_insn (gen_atomic_fetch_<atomic><mode>_iaf
             (operands[0], operands[1], operands[2]));
  DONE;
})

; lan, lang, lao, laog, lax, laxg, laa, laag
(define_insn "atomic_fetch_<atomic><mode>_iaf"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (match_operand:GPR 1 "memory_operand" "+QS"))
   (set (match_dup 1)
        (unspec_volatile:GPR
         [(ATOMIC_Z196:GPR (match_dup 1)
                           (match_operand:GPR 2 "general_operand" "d"))]
         UNSPECV_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z196"
  "la<noxa><g>\t%0,%2,%1"
  [(set_attr "op_type" "RSY")
   (set_attr "type" "sem")])

;; For SImode and larger, the optabs.c code will do just fine in
;; expanding a compare-and-swap loop.  For QI/HImode, we can do
;; better by expanding our own loop.

(define_expand "atomic_<atomic><mode>"
  [(ATOMIC:HQI
     (match_operand:HQI 0 "memory_operand")             ;; memory
     (match_operand:HQI 1 "general_operand"))           ;; val in
   (match_operand:SI 2 "const_int_operand")]            ;; model
  ""
{
  s390_expand_atomic (<MODE>mode, <CODE>, NULL_RTX, operands[0],
                       operands[1], false);
  DONE;
})

(define_expand "atomic_fetch_<atomic><mode>"
  [(match_operand:HQI 0 "register_operand")             ;; val out
   (ATOMIC:HQI
     (match_operand:HQI 1 "memory_operand")             ;; memory
     (match_operand:HQI 2 "general_operand"))           ;; val in
   (match_operand:SI 3 "const_int_operand")]            ;; model
  ""
{
  s390_expand_atomic (<MODE>mode, <CODE>, operands[0], operands[1],
                      operands[2], false);
  DONE;
})

(define_expand "atomic_<atomic>_fetch<mode>"
  [(match_operand:HQI 0 "register_operand")             ;; val out
   (ATOMIC:HQI
     (match_operand:HQI 1 "memory_operand")             ;; memory
     (match_operand:HQI 2 "general_operand"))           ;; val in
   (match_operand:SI 3 "const_int_operand")]            ;; model
  ""
{
  s390_expand_atomic (<MODE>mode, <CODE>, operands[0], operands[1],
                      operands[2], true);
  DONE;
})

(define_expand "atomic_exchange<mode>"
  [(match_operand:HQI 0 "register_operand")             ;; val out
   (match_operand:HQI 1 "memory_operand")               ;; memory
   (match_operand:HQI 2 "general_operand")              ;; val in
   (match_operand:SI 3 "const_int_operand")]            ;; model
  ""
{
  s390_expand_atomic (<MODE>mode, SET, operands[0], operands[1],
                      operands[2], false);
  DONE;
})

;;
;;- Miscellaneous instructions.
;;

;
; allocate stack instruction pattern(s).
;

(define_expand "allocate_stack"
  [(match_operand 0 "general_operand" "")
   (match_operand 1 "general_operand" "")]
 "TARGET_BACKCHAIN"
{
  rtx temp = gen_reg_rtx (Pmode);

  emit_move_insn (temp, s390_back_chain_rtx ());
  anti_adjust_stack (operands[1]);
  emit_move_insn (s390_back_chain_rtx (), temp);

  emit_move_insn (operands[0], virtual_stack_dynamic_rtx);
  DONE;
})


;
; setjmp instruction pattern.
;

(define_expand "builtin_setjmp_receiver"
  [(match_operand 0 "" "")]
  "flag_pic"
{
  emit_insn (s390_load_got ());
  emit_use (pic_offset_table_rtx);
  DONE;
})

;; These patterns say how to save and restore the stack pointer.  We need not
;; save the stack pointer at function level since we are careful to
;; preserve the backchain.  At block level, we have to restore the backchain
;; when we restore the stack pointer.
;;
;; For nonlocal gotos, we must save both the stack pointer and its
;; backchain and restore both.  Note that in the nonlocal case, the
;; save area is a memory location.

(define_expand "save_stack_function"
  [(match_operand 0 "general_operand" "")
   (match_operand 1 "general_operand" "")]
  ""
  "DONE;")

(define_expand "restore_stack_function"
  [(match_operand 0 "general_operand" "")
   (match_operand 1 "general_operand" "")]
  ""
  "DONE;")

(define_expand "restore_stack_block"
  [(match_operand 0 "register_operand" "")
   (match_operand 1 "register_operand" "")]
  "TARGET_BACKCHAIN"
{
  rtx temp = gen_reg_rtx (Pmode);

  emit_move_insn (temp, s390_back_chain_rtx ());
  emit_move_insn (operands[0], operands[1]);
  emit_move_insn (s390_back_chain_rtx (), temp);

  DONE;
})

(define_expand "save_stack_nonlocal"
  [(match_operand 0 "memory_operand" "")
   (match_operand 1 "register_operand" "")]
  ""
{
  rtx base = gen_rtx_REG (Pmode, BASE_REGNUM);

  /* Copy the backchain to the first word, sp to the second and the
     literal pool base to the third.  */

  rtx save_bc = adjust_address (operands[0], Pmode, 0);
  rtx save_sp = adjust_address (operands[0], Pmode, GET_MODE_SIZE (Pmode));
  rtx save_bp = adjust_address (operands[0], Pmode, 2 * GET_MODE_SIZE (Pmode));

  if (TARGET_BACKCHAIN)
    emit_move_insn (save_bc, force_reg (Pmode, s390_back_chain_rtx ()));

  emit_move_insn (save_sp, operands[1]);
  emit_move_insn (save_bp, base);

  DONE;
})

(define_expand "restore_stack_nonlocal"
  [(match_operand 0 "register_operand" "")
   (match_operand 1 "memory_operand" "")]
  ""
{
  rtx base = gen_rtx_REG (Pmode, BASE_REGNUM);
  rtx temp = NULL_RTX;

  /* Restore the backchain from the first word, sp from the second and the
     literal pool base from the third.  */

  rtx save_bc = adjust_address (operands[1], Pmode, 0);
  rtx save_sp = adjust_address (operands[1], Pmode, GET_MODE_SIZE (Pmode));
  rtx save_bp = adjust_address (operands[1], Pmode, 2 * GET_MODE_SIZE (Pmode));

  if (TARGET_BACKCHAIN)
    temp = force_reg (Pmode, save_bc);

  emit_move_insn (base, save_bp);
  emit_move_insn (operands[0], save_sp);

  if (temp)
    emit_move_insn (s390_back_chain_rtx (), temp);

  emit_use (base);
  DONE;
})

(define_expand "exception_receiver"
  [(const_int 0)]
  ""
{
  s390_set_has_landing_pad_p (true);
  DONE;
})

;
; nop instruction pattern(s).
;

(define_insn "nop"
  [(const_int 0)]
  ""
  "lr\t0,0"
  [(set_attr "op_type" "RR")
   (set_attr "z10prop"  "z10_fr_E1")])

(define_insn "nop1"
  [(const_int 1)]
  ""
  "lr\t1,1"
  [(set_attr "op_type" "RR")])

;;- Undeletable nops (used for hotpatching)

(define_insn "nop_2_byte"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_2_BYTE)]
  ""
  "nopr\t%%r7"
  [(set_attr "op_type" "RR")])

(define_insn "nop_4_byte"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_4_BYTE)]
  ""
  "nop\t0"
  [(set_attr "op_type" "RX")])

(define_insn "nop_6_byte"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_6_BYTE)]
  "TARGET_CPU_ZARCH"
  "brcl\t0, 0"
  [(set_attr "op_type" "RIL")])


;
; Special literal pool access instruction pattern(s).
;

(define_insn "*pool_entry"
  [(unspec_volatile [(match_operand 0 "consttable_operand" "X")]
                    UNSPECV_POOL_ENTRY)]
  ""
{
  machine_mode mode = GET_MODE (PATTERN (insn));
  unsigned int align = GET_MODE_BITSIZE (mode);
  s390_output_pool_entry (operands[0], mode, align);
  return "";
}
  [(set (attr "length")
        (symbol_ref "GET_MODE_SIZE (GET_MODE (PATTERN (insn)))"))])

(define_insn "pool_align"
  [(unspec_volatile [(match_operand 0 "const_int_operand" "n")]
                    UNSPECV_POOL_ALIGN)]
  ""
  ".align\t%0"
  [(set (attr "length") (symbol_ref "INTVAL (operands[0])"))])

(define_insn "pool_section_start"
  [(unspec_volatile [(const_int 1)] UNSPECV_POOL_SECTION)]
  ""
  ".section\t.rodata"
  [(set_attr "length" "0")])

(define_insn "pool_section_end"
  [(unspec_volatile [(const_int 0)] UNSPECV_POOL_SECTION)]
  ""
  ".previous"
  [(set_attr "length" "0")])

(define_insn "main_base_31_small"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))]
  "!TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
  "basr\t%0,0"
  [(set_attr "op_type" "RR")
   (set_attr "type"    "la")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "main_base_31_large"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))
   (set (pc) (label_ref (match_operand 2 "" "")))]
  "!TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
  "bras\t%0,%2"
  [(set_attr "op_type" "RI")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "main_base_64"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))]
  "TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
  "larl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_fwd_A1")])

(define_insn "main_pool"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec_volatile [(const_int 0)] UNSPECV_MAIN_POOL))]
  "GET_MODE (operands[0]) == Pmode"
{
  gcc_unreachable ();
}
  [(set (attr "type")
        (if_then_else (match_test "TARGET_CPU_ZARCH")
                      (const_string "larl") (const_string "la")))])

(define_insn "reload_base_31"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_RELOAD_BASE))]
  "!TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
  "basr\t%0,0\;la\t%0,%1-.(%0)"
  [(set_attr "length" "6")
   (set_attr "type" "la")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "reload_base_64"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_RELOAD_BASE))]
  "TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
  "larl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_fwd_A1")])

(define_insn "pool"
  [(unspec_volatile [(match_operand 0 "const_int_operand" "n")] UNSPECV_POOL)]
  ""
{
  gcc_unreachable ();
}
  [(set (attr "length") (symbol_ref "INTVAL (operands[0])"))])

;;
;; Insns related to generating the function prologue and epilogue.
;;


(define_expand "prologue"
  [(use (const_int 0))]
  ""
  "s390_emit_prologue (); DONE;")

(define_expand "epilogue"
  [(use (const_int 1))]
  ""
  "s390_emit_epilogue (false); DONE;")

(define_expand "sibcall_epilogue"
  [(use (const_int 0))]
  ""
  "s390_emit_epilogue (true); DONE;")

;; A direct return instruction, without using an epilogue.
(define_insn "<code>"
  [(ANY_RETURN)]
  "s390_can_use_<code>_insn ()"
  "br\t%%r14"
  [(set_attr "op_type" "RR")
   (set_attr "type"    "jsr")
   (set_attr "atype"   "agen")])

(define_insn "*return"
  [(return)
   (use (match_operand 0 "register_operand" "a"))]
  "GET_MODE (operands[0]) == Pmode"
  "br\t%0"
  [(set_attr "op_type" "RR")
   (set_attr "type"    "jsr")
   (set_attr "atype"   "agen")])


;; Instruction definition to extend a 31-bit pointer into a 64-bit
;; pointer. This is used for compatibility.

(define_expand "ptr_extend"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (match_operand:SI 1 "register_operand" "r"))]
  "TARGET_64BIT"
{
  emit_insn (gen_anddi3 (operands[0],
                         gen_lowpart (DImode, operands[1]),
                         GEN_INT (0x7fffffff)));
  DONE;
})

;; Instruction definition to expand eh_return macro to support
;; swapping in special linkage return addresses.

(define_expand "eh_return"
  [(use (match_operand 0 "register_operand" ""))]
  "TARGET_TPF"
{
  s390_emit_tpf_eh_return (operands[0]);
  DONE;
})

;
; Stack Protector Patterns
;

(define_expand "stack_protect_set"
  [(set (match_operand 0 "memory_operand" "")
        (match_operand 1 "memory_operand" ""))]
  ""
{
#ifdef TARGET_THREAD_SSP_OFFSET
  operands[1]
    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
#endif
  if (TARGET_64BIT)
    emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
  else
    emit_insn (gen_stack_protect_setsi (operands[0], operands[1]));

  DONE;
})

(define_insn "stack_protect_set<mode>"
  [(set (match_operand:DSI 0 "memory_operand" "=Q")
        (unspec:DSI [(match_operand:DSI 1 "memory_operand" "Q")] UNSPEC_SP_SET))]
  ""
  "mvc\t%O0(%G0,%R0),%S1"
  [(set_attr "op_type" "SS")])

(define_expand "stack_protect_test"
  [(set (reg:CC CC_REGNUM)
        (compare (match_operand 0 "memory_operand" "")
                 (match_operand 1 "memory_operand" "")))
   (match_operand 2 "" "")]
  ""
{
  rtx cc_reg, test;
#ifdef TARGET_THREAD_SSP_OFFSET
  operands[1]
    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
#endif
  if (TARGET_64BIT)
    emit_insn (gen_stack_protect_testdi (operands[0], operands[1]));
  else
    emit_insn (gen_stack_protect_testsi (operands[0], operands[1]));

  cc_reg = gen_rtx_REG (CCZmode, CC_REGNUM);
  test = gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx);
  emit_jump_insn (gen_cbranchcc4 (test, cc_reg, const0_rtx, operands[2]));
  DONE;
})

(define_insn "stack_protect_test<mode>"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:DSI 0 "memory_operand" "Q")
                     (match_operand:DSI 1 "memory_operand" "Q")] UNSPEC_SP_TEST))]
  ""
  "clc\t%O0(%G0,%R0),%S1"
  [(set_attr "op_type" "SS")])

; This is used in s390_emit_prologue in order to prevent insns
; adjusting the stack pointer to be moved over insns writing stack
; slots using a copy of the stack pointer in a different register.
(define_insn "stack_tie"
  [(set (match_operand:BLK 0 "memory_operand" "+m")
        (unspec:BLK [(match_dup 0)] UNSPEC_TIE))]
  ""
  ""
  [(set_attr "length" "0")])


;
; Data prefetch patterns
;

(define_insn "prefetch"
  [(prefetch (match_operand 0    "address_operand"   "ZQZRZSZT,X")
             (match_operand:SI 1 "const_int_operand" "       n,n")
             (match_operand:SI 2 "const_int_operand" "       n,n"))]
  "TARGET_Z10"
{
  switch (which_alternative)
    {
      case 0:
        return INTVAL (operands[1]) == 1 ? "pfd\t2,%a0" : "pfd\t1,%a0";
      case 1:
        if (larl_operand (operands[0], Pmode))
          return INTVAL (operands[1]) == 1 ? "pfdrl\t2,%a0" : "pfdrl\t1,%a0";
      default:

        /* This might be reached for symbolic operands with an odd
           addend.  We simply omit the prefetch for such rare cases.  */

        return "";
     }
}
  [(set_attr "type" "load,larl")
   (set_attr "op_type" "RXY,RIL")
   (set_attr "z10prop" "z10_super")
   (set_attr "z196prop" "z196_alone")])


;
; Byte swap instructions
;

(define_insn "bswap<mode>2"
  [(set (match_operand:GPR 0            "register_operand"     "=d, d")
        (bswap:GPR (match_operand:GPR 1 "nonimmediate_operand" " d,RT")))]
  "TARGET_CPU_ZARCH"
  "@
   lrv<g>r\t%0,%1
   lrv<g>\t%0,%1"
  [(set_attr "type" "*,load")
   (set_attr "op_type" "RRE,RXY")
   (set_attr "z10prop" "z10_super")])


;
; Population count instruction
;

; The S/390 popcount instruction counts the bits of op1 in 8 byte
; portions and stores the result in the corresponding bytes in op0.
(define_insn "*popcount<mode>"
  [(set (match_operand:INT 0 "register_operand" "=d")
        (unspec:INT [(match_operand:INT 1 "register_operand" "d")] UNSPEC_POPCNT))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z196"
  "popcnt\t%0,%1"
  [(set_attr "op_type" "RRE")])

(define_expand "popcountdi2"
  [; popcnt op0, op1
   (parallel [(set (match_operand:DI 0 "register_operand" "")
                   (unspec:DI [(match_operand:DI 1 "register_operand")]
                              UNSPEC_POPCNT))
              (clobber (reg:CC CC_REGNUM))])
   ; sllg op2, op0, 32
   (set (match_dup 2) (ashift:DI (match_dup 0) (const_int 32)))
   ; agr op0, op2
   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (match_dup 2)))
              (clobber (reg:CC CC_REGNUM))])
   ; sllg op2, op0, 16
   (set (match_dup 2)
        (ashift:DI (match_dup 0) (const_int 16)))
   ; agr op0, op2
   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (match_dup 2)))
              (clobber (reg:CC CC_REGNUM))])
   ; sllg op2, op0, 8
   (set (match_dup 2) (ashift:DI (match_dup 0) (const_int 8)))
   ; agr op0, op2
   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (match_dup 2)))
              (clobber (reg:CC CC_REGNUM))])
   ; srlg op0, op0, 56
   (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))]
  "TARGET_Z196 && TARGET_64BIT"
  "operands[2] = gen_reg_rtx (DImode);")

(define_expand "popcountsi2"
  [; popcnt op0, op1
   (parallel [(set (match_operand:SI 0 "register_operand" "")
                   (unspec:SI [(match_operand:SI 1 "register_operand")]
                              UNSPEC_POPCNT))
              (clobber (reg:CC CC_REGNUM))])
   ; sllk op2, op0, 16
   (set (match_dup 2)
        (ashift:SI (match_dup 0) (const_int 16)))
   ; ar op0, op2
   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (match_dup 2)))
              (clobber (reg:CC CC_REGNUM))])
   ; sllk op2, op0, 8
   (set (match_dup 2) (ashift:SI (match_dup 0) (const_int 8)))
   ; ar op0, op2
   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (match_dup 2)))
              (clobber (reg:CC CC_REGNUM))])
   ; srl op0, op0, 24
   (set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 24)))]
  "TARGET_Z196"
  "operands[2] = gen_reg_rtx (SImode);")

(define_expand "popcounthi2"
  [; popcnt op0, op1
   (parallel [(set (match_operand:HI 0 "register_operand" "")
                   (unspec:HI [(match_operand:HI 1 "register_operand")]
                              UNSPEC_POPCNT))
              (clobber (reg:CC CC_REGNUM))])
   ; sllk op2, op0, 8
   (set (match_dup 2)
        (ashift:SI (match_dup 0) (const_int 8)))
   ; ar op0, op2
   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (match_dup 2)))
              (clobber (reg:CC CC_REGNUM))])
   ; srl op0, op0, 8
   (set (match_dup 0) (lshiftrt:HI (match_dup 0) (const_int 8)))]
  "TARGET_Z196"
  "operands[2] = gen_reg_rtx (SImode);")

(define_expand "popcountqi2"
  [; popcnt op0, op1
   (parallel [(set (match_operand:QI 0 "register_operand" "")
                   (unspec:QI [(match_operand:QI 1 "register_operand")]
                              UNSPEC_POPCNT))
              (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z196"
  "")

;;
;;- Copy sign instructions
;;

(define_insn "copysign<mode>3"
  [(set (match_operand:FP 0 "register_operand" "=f")
      (unspec:FP [(match_operand:FP 1 "register_operand" "<fT0>")
                  (match_operand:FP 2 "register_operand" "f")]
                  UNSPEC_COPYSIGN))]
  "TARGET_Z196"
  "cpsdr\t%0,%2,%1"
  [(set_attr "op_type"  "RRF")
   (set_attr "type"     "fsimp<mode>")])


;;
;;- Transactional execution instructions
;;

; This splitter helps combine to make use of CC directly when
; comparing the integer result of a tbegin builtin with a constant.
; The unspec is already removed by canonicalize_comparison. So this
; splitters only job is to turn the PARALLEL into separate insns
; again.  Unfortunately this only works with the very first cc/int
; compare since combine is not able to deal with data flow across
; basic block boundaries.

; It needs to be an insn pattern as well since combine does not apply
; the splitter directly.  Combine would only use it if it actually
; would reduce the number of instructions.
(define_insn_and_split "*ccraw_to_int"
  [(set (pc)
        (if_then_else
         (match_operator 0 "s390_eqne_operator"
                         [(reg:CCRAW CC_REGNUM)
                          (match_operand 1 "const_int_operand" "")])
         (label_ref (match_operand 2 "" ""))
         (pc)))
   (set (match_operand:SI 3 "register_operand" "=d")
        (unspec:SI [(reg:CCRAW CC_REGNUM)] UNSPEC_CC_TO_INT))]
  ""
  "#"
  ""
  [(set (match_dup 3)
        (unspec:SI [(reg:CCRAW CC_REGNUM)] UNSPEC_CC_TO_INT))
   (set (pc)
        (if_then_else (match_op_dup 0 [(reg:CCRAW CC_REGNUM) (match_dup 1)])
                      (label_ref (match_dup 2))
                      (pc)))]
  "")

; Non-constrained transaction begin

(define_expand "tbegin"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], NULL_RTX, true);
  DONE;
})

(define_expand "tbegin_nofloat"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], NULL_RTX, false);
  DONE;
})

(define_expand "tbegin_retry"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")
   (match_operand:SI 2 "general_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], operands[2], true);
  DONE;
})

(define_expand "tbegin_retry_nofloat"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")
   (match_operand:SI 2 "general_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], operands[2], false);
  DONE;
})

(define_insn "tbegin_1"
  [(set (reg:CCRAW CC_REGNUM)
        (unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" "D")]
                               UNSPECV_TBEGIN))
   (set (match_operand:BLK 1 "memory_operand" "=Q")
        (unspec_volatile:BLK [(match_dup 0)] UNSPECV_TBEGIN_TDB))
   (clobber (reg:DF 16))
   (clobber (reg:DF 17))
   (clobber (reg:DF 18))
   (clobber (reg:DF 19))
   (clobber (reg:DF 20))
   (clobber (reg:DF 21))
   (clobber (reg:DF 22))
   (clobber (reg:DF 23))
   (clobber (reg:DF 24))
   (clobber (reg:DF 25))
   (clobber (reg:DF 26))
   (clobber (reg:DF 27))
   (clobber (reg:DF 28))
   (clobber (reg:DF 29))
   (clobber (reg:DF 30))
   (clobber (reg:DF 31))]
; CONST_OK_FOR_CONSTRAINT_P does not work with D constraint since D is
; not supposed to be used for immediates (see genpreds.c).
  "TARGET_HTM && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbegin\t%1,%x0"
  [(set_attr "op_type" "SIL")])

; Same as above but without the FPR clobbers
(define_insn "tbegin_nofloat_1"
  [(set (reg:CCRAW CC_REGNUM)
        (unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" "D")]
                               UNSPECV_TBEGIN))
   (set (match_operand:BLK 1 "memory_operand" "=Q")
        (unspec_volatile:BLK [(match_dup 0)] UNSPECV_TBEGIN_TDB))]
  "TARGET_HTM && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbegin\t%1,%x0"
  [(set_attr "op_type" "SIL")])


; Constrained transaction begin

(define_expand "tbeginc"
  [(set (reg:CCRAW CC_REGNUM)
        (unspec_volatile:CCRAW [(const_int TBEGINC_MASK)]
                               UNSPECV_TBEGINC))]
  "TARGET_HTM"
  "")

(define_insn "*tbeginc_1"
  [(set (reg:CCRAW CC_REGNUM)
        (unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" " D")]
                               UNSPECV_TBEGINC))]
  "TARGET_HTM && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbeginc\t0,%x0"
  [(set_attr "op_type" "SIL")])

; Transaction end

(define_expand "tend"
  [(set (reg:CCRAW CC_REGNUM)
        (unspec_volatile:CCRAW [(const_int 0)] UNSPECV_TEND))
   (set (match_operand:SI 0 "register_operand" "")
        (unspec:SI [(reg:CCRAW CC_REGNUM)] UNSPEC_CC_TO_INT))]
  "TARGET_HTM"
  "")

(define_insn "*tend_1"
  [(set (reg:CCRAW CC_REGNUM)
        (unspec_volatile:CCRAW [(const_int 0)] UNSPECV_TEND))]
  "TARGET_HTM"
  "tend"
  [(set_attr "op_type" "S")])

; Transaction abort

(define_expand "tabort"
  [(unspec_volatile [(match_operand:SI 0 "shift_count_or_setmem_operand" "")]
                    UNSPECV_TABORT)]
  "TARGET_HTM && operands != NULL"
{
  if (CONST_INT_P (operands[0])
      && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 255)
    {
      error ("Invalid transaction abort code: " HOST_WIDE_INT_PRINT_DEC
             ".  Values in range 0 through 255 are reserved.",
             INTVAL (operands[0]));
      FAIL;
    }
})

(define_insn "*tabort_1"
  [(unspec_volatile [(match_operand:SI 0 "shift_count_or_setmem_operand" "Y")]
                    UNSPECV_TABORT)]
  "TARGET_HTM && operands != NULL"
  "tabort\t%Y0"
  [(set_attr "op_type" "S")])

; Transaction extract nesting depth

(define_insn "etnd"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (unspec_volatile:SI [(const_int 0)] UNSPECV_ETND))]
  "TARGET_HTM"
  "etnd\t%0"
  [(set_attr "op_type" "RRE")])

; Non-transactional store

(define_insn "ntstg"
  [(set (match_operand:DI 0 "memory_operand" "=RT")
        (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "d")]
                            UNSPECV_NTSTG))]
  "TARGET_HTM"
  "ntstg\t%1,%0"
  [(set_attr "op_type" "RXY")])

; Transaction perform processor assist

(define_expand "tx_assist"
  [(unspec_volatile [(match_operand:SI 0 "register_operand" "")
                     (reg:SI GPR0_REGNUM)
                     (const_int 1)]
                    UNSPECV_PPA)]
  "TARGET_HTM"
  "")

(define_insn "*ppa"
  [(unspec_volatile [(match_operand:SI 0 "register_operand" "d")
                     (match_operand:SI 1 "register_operand" "d")
                     (match_operand 2 "const_int_operand" "I")]
                    UNSPECV_PPA)]
  "TARGET_HTM && INTVAL (operands[2]) < 16"
  "ppa\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])


; Set and get floating point control register

(define_insn "sfpc"
  [(unspec_volatile [(match_operand:SI 0 "register_operand" "d")]
                    UNSPECV_SFPC)]
  "TARGET_HARD_FLOAT"
  "sfpc\t%0")

(define_insn "efpc"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (unspec_volatile:SI [(const_int 0)] UNSPECV_EFPC))]
  "TARGET_HARD_FLOAT"
  "efpc\t%0")


; Load count to block boundary

(define_insn "lcbb"
  [(set (match_operand:SI             0 "register_operand"  "=d")
        (unspec:SI [(match_operand:SI 1 "address_operand" "ZQZR")
                    (match_operand:SI 2 "immediate_operand"  "C")] UNSPEC_LCBB))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z13"
  "lcbb\t%0,%1,%b2"
  [(set_attr "op_type" "VRX")])