gcc/config/arm/mve.md

   1 ;; Arm M-profile Vector Extension Machine Description
   2 ;; Copyright (C) 2019-2020 Free Software Foundation, Inc.
   3 ;;
   4 ;; This file is part of GCC.
   5 ;;
   6 ;; GCC is free software; you can redistribute it and/or modify it
   7 ;; under the terms of the GNU General Public License as published by
   8 ;; the Free Software Foundation; either version 3, or (at your option)
   9 ;; any later version.
  10 ;;
  11 ;; GCC is distributed in the hope that it will be useful, but
  12 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 ;; General Public License for more details.
  15 ;;
  16 ;; You should have received a copy of the GNU General Public License
  17 ;; along with GCC; see the file COPYING3.  If not see
  18 ;; <http://www.gnu.org/licenses/>.
  19
  20 (define_mode_attr V_sz_elem2 [(V16QI "s8") (V8HI "u16") (V4SI "u32")
  21                               (V2DI "u64")])
  22 (define_mode_iterator MVE_types [V16QI V8HI V4SI V2DI TI V8HF V4SF V2DF])
  23 (define_mode_iterator MVE_VLD_ST [V16QI V8HI V4SI V8HF V4SF])
  24 (define_mode_iterator MVE_0 [V8HF V4SF])
  25 (define_mode_iterator MVE_1 [V16QI V8HI V4SI V2DI])
  26 (define_mode_iterator MVE_3 [V16QI V8HI])
  27 (define_mode_iterator MVE_2 [V16QI V8HI V4SI])
  28 (define_mode_iterator MVE_5 [V8HI V4SI])
  29 (define_mode_iterator MVE_6 [V8HI V4SI])
  30
  31 (define_c_enum "unspec" [VST4Q VRNDXQ_F VRNDQ_F VRNDPQ_F VRNDNQ_F VRNDMQ_F
  32                          VRNDAQ_F VREV64Q_F VNEGQ_F VDUPQ_N_F VABSQ_F VREV32Q_F
  33                          VCVTTQ_F32_F16 VCVTBQ_F32_F16 VCVTQ_TO_F_S VQNEGQ_S
  34                          VCVTQ_TO_F_U VREV16Q_S VREV16Q_U VADDLVQ_S VMVNQ_N_S
  35                          VMVNQ_N_U VCVTAQ_S VCVTAQ_U VREV64Q_S VREV64Q_U
  36                          VQABSQ_S VNEGQ_S VMVNQ_S VMVNQ_U VDUPQ_N_U VDUPQ_N_S
  37                          VCLZQ_U VCLZQ_S VCLSQ_S VADDVQ_S VADDVQ_U VABSQ_S
  38                          VREV32Q_U VREV32Q_S VMOVLTQ_U VMOVLTQ_S VMOVLBQ_S
  39                          VMOVLBQ_U VCVTQ_FROM_F_S VCVTQ_FROM_F_U VCVTPQ_S
  40                          VCVTPQ_U VCVTNQ_S VCVTNQ_U VCVTMQ_S VCVTMQ_U
  41                          VADDLVQ_U VCTP8Q VCTP16Q VCTP32Q VCTP64Q VPNOT
  42                          VCREATEQ_F VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U VBRSRQ_N_F
  43                          VSUBQ_N_F VCREATEQ_U VCREATEQ_S VSHRQ_N_S VSHRQ_N_U
  44                          VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U VADDLVQ_P_S
  45                          VADDLVQ_P_U VCMPNEQ_U VCMPNEQ_S VSHLQ_S VSHLQ_U VABDQ_S
  46                          VADDQ_N_S VADDVAQ_S VADDVQ_P_S VANDQ_S VBICQ_S
  47                          VBRSRQ_N_S VCADDQ_ROT270_S VCADDQ_ROT90_S VCMPEQQ_S
  48                          VCMPEQQ_N_S VCMPNEQ_N_S VEORQ_S VHADDQ_S VHADDQ_N_S
  49                          VHSUBQ_S VHSUBQ_N_S VMAXQ_S VMAXVQ_S VMINQ_S VMINVQ_S
  50                          VMLADAVQ_S VMULHQ_S VMULLBQ_INT_S VMULLTQ_INT_S VMULQ_S
  51                          VMULQ_N_S VORNQ_S VORRQ_S VQADDQ_S VQADDQ_N_S VQRSHLQ_S
  52                          VQRSHLQ_N_S VQSHLQ_S VQSHLQ_N_S VQSHLQ_R_S VQSUBQ_S
  53                          VQSUBQ_N_S VRHADDQ_S VRMULHQ_S VRSHLQ_S VRSHLQ_N_S
  54                          VRSHRQ_N_S VSHLQ_N_S VSHLQ_R_S VSUBQ_S VSUBQ_N_S
  55                          VABDQ_U VADDQ_N_U VADDVAQ_U VADDVQ_P_U VANDQ_U VBICQ_U
  56                          VBRSRQ_N_U VCADDQ_ROT270_U VCADDQ_ROT90_U VCMPEQQ_U
  57                          VCMPEQQ_N_U VCMPNEQ_N_U VEORQ_U VHADDQ_U VHADDQ_N_U
  58                          VHSUBQ_U VHSUBQ_N_U VMAXQ_U VMAXVQ_U VMINQ_U VMINVQ_U
  59                          VMLADAVQ_U VMULHQ_U VMULLBQ_INT_U VMULLTQ_INT_U VMULQ_U
  60                          VMULQ_N_U VORNQ_U VORRQ_U VQADDQ_U VQADDQ_N_U VQRSHLQ_U
  61                          VQRSHLQ_N_U VQSHLQ_U VQSHLQ_N_U VQSHLQ_R_U VQSUBQ_U
  62                          VQSUBQ_N_U VRHADDQ_U VRMULHQ_U VRSHLQ_U VRSHLQ_N_U
  63                          VRSHRQ_N_U VSHLQ_N_U VSHLQ_R_U VSUBQ_U VSUBQ_N_U
  64                          VCMPGEQ_N_S VCMPGEQ_S VCMPGTQ_N_S VCMPGTQ_S VCMPLEQ_N_S
  65                          VCMPLEQ_S VCMPLTQ_N_S VCMPLTQ_S VHCADDQ_ROT270_S
  66                          VHCADDQ_ROT90_S VMAXAQ_S VMAXAVQ_S VMINAQ_S VMINAVQ_S
  67                          VMLADAVXQ_S VMLSDAVQ_S VMLSDAVXQ_S VQDMULHQ_N_S
  68                          VQDMULHQ_S VQRDMULHQ_N_S VQRDMULHQ_S VQSHLUQ_N_S
  69                          VCMPCSQ_N_U VCMPCSQ_U VCMPHIQ_N_U VCMPHIQ_U VABDQ_M_S
  70                          VABDQ_M_U VABDQ_F VADDQ_N_F VANDQ_F VBICQ_F
  71                          VCADDQ_ROT270_F VCADDQ_ROT90_F VCMPEQQ_F VCMPEQQ_N_F
  72                          VCMPGEQ_F VCMPGEQ_N_F VCMPGTQ_F VCMPGTQ_N_F VCMPLEQ_F
  73                          VCMPLEQ_N_F VCMPLTQ_F VCMPLTQ_N_F VCMPNEQ_F VCMPNEQ_N_F
  74                          VCMULQ_F VCMULQ_ROT180_F VCMULQ_ROT270_F VCMULQ_ROT90_F
  75                          VEORQ_F VMAXNMAQ_F VMAXNMAVQ_F VMAXNMQ_F VMAXNMVQ_F
  76                          VMINNMAQ_F VMINNMAVQ_F VMINNMQ_F VMINNMVQ_F VMULQ_F
  77                          VMULQ_N_F VORNQ_F VORRQ_F VSUBQ_F VADDLVAQ_U
  78                          VADDLVAQ_S VBICQ_N_U VBICQ_N_S VCTP8Q_M VCTP16Q_M
  79                          VCTP32Q_M VCTP64Q_M VCVTBQ_F16_F32 VCVTTQ_F16_F32
  80                          VMLALDAVQ_U VMLALDAVXQ_U VMLALDAVXQ_S VMLALDAVQ_S
  81                          VMLSLDAVQ_S VMLSLDAVXQ_S VMOVNBQ_U VMOVNBQ_S
  82                          VMOVNTQ_U VMOVNTQ_S VORRQ_N_S VORRQ_N_U VQDMULLBQ_N_S
  83                          VQDMULLBQ_S VQDMULLTQ_N_S VQDMULLTQ_S VQMOVNBQ_U
  84                          VQMOVNBQ_S VQMOVUNBQ_S VQMOVUNTQ_S VRMLALDAVHXQ_S
  85                          VRMLSLDAVHQ_S VRMLSLDAVHXQ_S VSHLLBQ_S
  86                          VSHLLBQ_U VSHLLTQ_U VSHLLTQ_S VQMOVNTQ_U VQMOVNTQ_S
  87                          VSHLLBQ_N_S VSHLLBQ_N_U VSHLLTQ_N_U VSHLLTQ_N_S
  88                          VRMLALDAVHQ_U VRMLALDAVHQ_S VMULLTQ_POLY_P
  89                          VMULLBQ_POLY_P VBICQ_M_N_S VBICQ_M_N_U VCMPEQQ_M_F
  90                          VCVTAQ_M_S VCVTAQ_M_U VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U
  91                          VQRSHRNBQ_N_U VQRSHRNBQ_N_S VQRSHRUNBQ_N_S
  92                          VRMLALDAVHAQ_S VABAVQ_S VABAVQ_U VSHLCQ_S VSHLCQ_U
  93                          VRMLALDAVHAQ_U VABSQ_M_S VADDVAQ_P_S VADDVAQ_P_U
  94                          VCLSQ_M_S VCLZQ_M_S VCLZQ_M_U VCMPCSQ_M_N_U
  95                          VCMPCSQ_M_U VCMPEQQ_M_N_S VCMPEQQ_M_N_U VCMPEQQ_M_S
  96                          VCMPEQQ_M_U VCMPGEQ_M_N_S VCMPGEQ_M_S VCMPGTQ_M_N_S
  97                          VCMPGTQ_M_S VCMPHIQ_M_N_U VCMPHIQ_M_U VCMPLEQ_M_N_S
  98                          VCMPLEQ_M_S VCMPLTQ_M_N_S VCMPLTQ_M_S VCMPNEQ_M_N_S
  99                          VCMPNEQ_M_N_U VCMPNEQ_M_S VCMPNEQ_M_U VDUPQ_M_N_S
 100                          VDUPQ_M_N_U VDWDUPQ_N_U VDWDUPQ_WB_U VIWDUPQ_N_U
 101                          VIWDUPQ_WB_U VMAXAQ_M_S VMAXAVQ_P_S VMAXVQ_P_S
 102                          VMAXVQ_P_U VMINAQ_M_S VMINAVQ_P_S VMINVQ_P_S VMINVQ_P_U
 103                          VMLADAVAQ_S VMLADAVAQ_U VMLADAVQ_P_S VMLADAVQ_P_U
 104                          VMLADAVXQ_P_S VMLAQ_N_S VMLAQ_N_U VMLASQ_N_S VMLASQ_N_U
 105                          VMLSDAVQ_P_S VMLSDAVXQ_P_S VMVNQ_M_S VMVNQ_M_U
 106                          VNEGQ_M_S VPSELQ_S VPSELQ_U VQABSQ_M_S VQDMLAHQ_N_S
 107                          VQDMLAHQ_N_U VQNEGQ_M_S VQRDMLADHQ_S VQRDMLADHXQ_S
 108                          VQRDMLAHQ_N_S VQRDMLAHQ_N_U VQRDMLASHQ_N_S
 109                          VQRDMLASHQ_N_U VQRDMLSDHQ_S VQRDMLSDHXQ_S VQRSHLQ_M_N_S
 110                          VQRSHLQ_M_N_U VQSHLQ_M_R_S VQSHLQ_M_R_U VREV64Q_M_S
 111                          VREV64Q_M_U VRSHLQ_M_N_S VRSHLQ_M_N_U VSHLQ_M_R_S
 112                          VSHLQ_M_R_U VSLIQ_N_S VSLIQ_N_U VSRIQ_N_S VSRIQ_N_U
 113                          VQDMLSDHXQ_S VQDMLSDHQ_S VQDMLADHXQ_S VQDMLADHQ_S
 114                          VMLSDAVAXQ_S VMLSDAVAQ_S VMLADAVAXQ_S
 115                          VCMPGEQ_M_F VCMPGTQ_M_N_F VMLSLDAVQ_P_S VRMLALDAVHAXQ_S
 116                          VMLSLDAVXQ_P_S VFMAQ_F VMLSLDAVAQ_S VQSHRUNBQ_N_S
 117                          VQRSHRUNTQ_N_S VCMLAQ_F VMINNMAQ_M_F VFMASQ_N_F
 118                          VDUPQ_M_N_F VCMPGTQ_M_F VCMPLTQ_M_F VRMLSLDAVHQ_P_S
 119                          VQSHRUNTQ_N_S VABSQ_M_F VMAXNMAVQ_P_F VFMAQ_N_F
 120                          VRMLSLDAVHXQ_P_S VREV32Q_M_F VRMLSLDAVHAQ_S
 121                          VRMLSLDAVHAXQ_S VCMPLTQ_M_N_F VCMPNEQ_M_F VRNDAQ_M_F
 122                          VRNDPQ_M_F VADDLVAQ_P_S VQMOVUNBQ_M_S VCMPLEQ_M_F
 123                          VCMLAQ_ROT180_F VMLSLDAVAXQ_S VRNDXQ_M_F VFMSQ_F
 124                          VMINNMVQ_P_F VMAXNMVQ_P_F VPSELQ_F VCMLAQ_ROT90_F
 125                          VQMOVUNTQ_M_S VREV64Q_M_F VNEGQ_M_F VRNDMQ_M_F
 126                          VCMPLEQ_M_N_F VCMPGEQ_M_N_F VRNDNQ_M_F VMINNMAVQ_P_F
 127                          VCMPNEQ_M_N_F VRMLALDAVHQ_P_S VRMLALDAVHXQ_P_S
 128                          VCMPEQQ_M_N_F VCMLAQ_ROT270_F VMAXNMAQ_M_F VRNDQ_M_F
 129                          VMLALDAVQ_P_U VMLALDAVQ_P_S VQMOVNBQ_M_S VQMOVNBQ_M_U
 130                          VMOVLTQ_M_U VMOVLTQ_M_S VMOVNBQ_M_U VMOVNBQ_M_S
 131                          VRSHRNTQ_N_U VRSHRNTQ_N_S VORRQ_M_N_S VORRQ_M_N_U
 132                          VREV32Q_M_S VREV32Q_M_U VQRSHRNTQ_N_U VQRSHRNTQ_N_S
 133                          VMOVNTQ_M_U VMOVNTQ_M_S VMOVLBQ_M_U VMOVLBQ_M_S
 134                          VMLALDAVAQ_S VMLALDAVAQ_U VQSHRNBQ_N_U VQSHRNBQ_N_S
 135                          VSHRNBQ_N_U VSHRNBQ_N_S VRSHRNBQ_N_S VRSHRNBQ_N_U
 136                          VMLALDAVXQ_P_U VMLALDAVXQ_P_S VQMOVNTQ_M_U VQMOVNTQ_M_S
 137                          VMVNQ_M_N_U VMVNQ_M_N_S VQSHRNTQ_N_U VQSHRNTQ_N_S
 138                          VMLALDAVAXQ_S VMLALDAVAXQ_U VSHRNTQ_N_S VSHRNTQ_N_U
 139                          VCVTBQ_M_F16_F32 VCVTBQ_M_F32_F16 VCVTTQ_M_F16_F32
 140                          VCVTTQ_M_F32_F16 VCVTMQ_M_S VCVTMQ_M_U VCVTNQ_M_S
 141                          VCVTPQ_M_S VCVTPQ_M_U VCVTQ_M_N_FROM_F_S VCVTNQ_M_U
 142                          VREV16Q_M_S VREV16Q_M_U VREV32Q_M VCVTQ_M_FROM_F_U
 143                          VCVTQ_M_FROM_F_S VRMLALDAVHQ_P_U VADDLVAQ_P_U
 144                          VCVTQ_M_N_FROM_F_U VQSHLUQ_M_N_S VABAVQ_P_S
 145                          VABAVQ_P_U VSHLQ_M_S VSHLQ_M_U VSRIQ_M_N_S
 146                          VSRIQ_M_N_U VSUBQ_M_U VSUBQ_M_S VCVTQ_M_N_TO_F_U
 147                          VCVTQ_M_N_TO_F_S VQADDQ_M_U VQADDQ_M_S
 148                          VRSHRQ_M_N_S VSUBQ_M_N_S VSUBQ_M_N_U VBRSRQ_M_N_S
 149                          VSUBQ_M_N_F VBICQ_M_F VHADDQ_M_U VBICQ_M_U VBICQ_M_S
 150                          VMULQ_M_N_U VHADDQ_M_S VORNQ_M_F VMLAQ_M_N_S VQSUBQ_M_U
 151                          VQSUBQ_M_S VMLAQ_M_N_U VQSUBQ_M_N_U VQSUBQ_M_N_S
 152                          VMULLTQ_INT_M_S VMULLTQ_INT_M_U VMULQ_M_N_S VMULQ_M_N_F
 153                          VMLASQ_M_N_U VMLASQ_M_N_S VMAXQ_M_U VQRDMLAHQ_M_N_U
 154                          VCADDQ_ROT270_M_F VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S
 155                          VQRSHLQ_M_S VMULQ_M_F VRHADDQ_M_U VSHRQ_M_N_U
 156                          VRHADDQ_M_S VMULQ_M_S VMULQ_M_U VQRDMLASHQ_M_N_S
 157                          VRSHLQ_M_S VRSHLQ_M_U VRSHRQ_M_N_U VADDQ_M_N_F
 158                          VADDQ_M_N_S VADDQ_M_N_U VQRDMLASHQ_M_N_U VMAXQ_M_S
 159                          VQRDMLAHQ_M_N_S VORRQ_M_S VORRQ_M_U VORRQ_M_F
 160                          VQRSHLQ_M_U VRMULHQ_M_U VRMULHQ_M_S VMINQ_M_S VMINQ_M_U
 161                          VANDQ_M_F VANDQ_M_U VANDQ_M_S VHSUBQ_M_N_S VHSUBQ_M_N_U
 162                          VMULHQ_M_S VMULHQ_M_U VMULLBQ_INT_M_U
 163                          VMULLBQ_INT_M_S VCADDQ_ROT90_M_F
 164                          VSHRQ_M_N_S VADDQ_M_U VSLIQ_M_N_U
 165                          VQADDQ_M_N_S VBRSRQ_M_N_F VABDQ_M_F VBRSRQ_M_N_U
 166                          VEORQ_M_F VSHLQ_M_N_S VQDMLAHQ_M_N_U VQDMLAHQ_M_N_S
 167                          VSHLQ_M_N_U VMLADAVAQ_P_U VMLADAVAQ_P_S VSLIQ_M_N_S
 168                          VQSHLQ_M_U VQSHLQ_M_S VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S
 169                          VORNQ_M_U VORNQ_M_S VQSHLQ_M_N_S VQSHLQ_M_N_U VADDQ_M_S
 170                          VHADDQ_M_N_S VADDQ_M_F VQADDQ_M_N_U VEORQ_M_S VEORQ_M_U
 171                          VHSUBQ_M_S VHSUBQ_M_U VHADDQ_M_N_U VHCADDQ_ROT90_M_S
 172                          VQRDMLSDHQ_M_S VQRDMLSDHXQ_M_S VQRDMLADHXQ_M_S
 173                          VQDMULHQ_M_S VMLADAVAXQ_P_S VQDMLADHXQ_M_S
 174                          VQRDMULHQ_M_S VMLSDAVAXQ_P_S VQDMULHQ_M_N_S
 175                          VHCADDQ_ROT270_M_S VQDMLSDHQ_M_S VQDMLSDHXQ_M_S
 176                          VMLSDAVAQ_P_S VQRDMLADHQ_M_S VQDMLADHQ_M_S
 177                          VMLALDAVAQ_P_U VMLALDAVAQ_P_S VMLALDAVAXQ_P_U
 178                          VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S VQRSHRNTQ_M_N_S
 179                          VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S VQSHRNTQ_M_N_S
 180                          VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S VRSHRNTQ_M_N_U
 181                          VSHLLBQ_M_N_U VSHLLBQ_M_N_S VSHLLTQ_M_N_U VSHLLTQ_M_N_S
 182                          VSHRNBQ_M_N_S VSHRNBQ_M_N_U VSHRNTQ_M_N_S VSHRNTQ_M_N_U
 183                          VMLALDAVAXQ_P_S VQRSHRNTQ_M_N_U VQSHRNTQ_M_N_U
 184                          VRSHRNTQ_M_N_S VQRDMULHQ_M_N_S VRMLALDAVHAQ_P_S
 185                          VMLSLDAVAQ_P_S VMLSLDAVAXQ_P_S VMULLBQ_POLY_M_P
 186                          VMULLTQ_POLY_M_P VQDMULLBQ_M_N_S VQDMULLBQ_M_S
 187                          VQDMULLTQ_M_N_S VQDMULLTQ_M_S VQRSHRUNBQ_M_N_S
 188                          VQRSHRUNTQ_M_N_SVQSHRUNBQ_M_N_S VQSHRUNTQ_M_N_S
 189                          VRMLALDAVHAQ_P_U VRMLALDAVHAXQ_P_S VRMLSLDAVHAQ_P_S
 190                          VRMLSLDAVHAXQ_P_S VQRSHRUNTQ_M_N_S VQSHRUNBQ_M_N_S
 191                          VCMLAQ_M_F VCMLAQ_ROT180_M_F VCMLAQ_ROT270_M_F
 192                          VCMLAQ_ROT90_M_F VCMULQ_M_F VCMULQ_ROT180_M_F
 193                          VCMULQ_ROT270_M_F VCMULQ_ROT90_M_F VFMAQ_M_F
 194                          VFMAQ_M_N_F VFMASQ_M_N_F VFMSQ_M_F VMAXNMQ_M_F
 195                          VMINNMQ_M_F VSUBQ_M_F VSTRWQSB_S VSTRWQSB_U
 196                          VSTRBQSO_S VSTRBQSO_U VSTRBQ_S VSTRBQ_U VLDRBQGO_S
 197                          VLDRBQGO_U VLDRBQ_S VLDRBQ_U VLDRWQGB_S VLDRWQGB_U
 198                          VLD1Q_F VLD1Q_S VLD1Q_U VLDRHQ_F VLDRHQGO_S
 199                          VLDRHQGO_U VLDRHQGSO_S VLDRHQGSO_U VLDRHQ_S VLDRHQ_U
 200                          VLDRWQ_F VLDRWQ_S VLDRWQ_U VLDRDQGB_S VLDRDQGB_U
 201                          VLDRDQGO_S VLDRDQGO_U VLDRDQGSO_S VLDRDQGSO_U
 202                          VLDRHQGO_F VLDRHQGSO_F VLDRWQGB_F VLDRWQGO_F
 203                          VLDRWQGO_S VLDRWQGO_U VLDRWQGSO_F VLDRWQGSO_S
 204                          VLDRWQGSO_U VSTRHQ_F VST1Q_S VST1Q_U VSTRHQSO_S
 205                          VSTRHQSO_U VSTRHQSSO_S VSTRHQSSO_U VSTRHQ_S
 206                          VSTRHQ_U VSTRWQ_S VSTRWQ_U VSTRWQ_F VST1Q_F VSTRDQSB_S
 207                          VSTRDQSB_U VSTRDQSO_S VSTRDQSO_U VSTRDQSSO_S
 208                          VSTRDQSSO_U VSTRWQSO_S VSTRWQSO_U VSTRWQSSO_S
 209                          VSTRWQSSO_U VSTRHQSO_F VSTRHQSSO_F VSTRWQSB_F
 210                          VSTRWQSO_F VSTRWQSSO_F VDDUPQ VDDUPQ_M VDWDUPQ
 211                          VDWDUPQ_M VIDUPQ VIDUPQ_M VIWDUPQ VIWDUPQ_M
 212                          VSTRWQSBWB_S VSTRWQSBWB_U VLDRWQGBWB_S VLDRWQGBWB_U
 213                          VSTRWQSBWB_F VLDRWQGBWB_F VSTRDQSBWB_S VSTRDQSBWB_U
 214                          VLDRDQGBWB_S VLDRDQGBWB_U VADCQ_U VADCQ_M_U VADCQ_S
 215                          VADCQ_M_S VSBCIQ_U VSBCIQ_S VSBCIQ_M_U VSBCIQ_M_S
 216                          VSBCQ_U VSBCQ_S VSBCQ_M_U VSBCQ_M_S VADCIQ_U VADCIQ_M_U
 217                          VADCIQ_S VADCIQ_M_S VLD2Q VLD4Q VST2Q SRSHRL SRSHR
 218                          URSHR URSHRL SQRSHR UQRSHL UQRSHLL_64 VSHLCQ_M_U
 219                          UQRSHLL_48 SQRSHRL_64 SQRSHRL_48 VSHLCQ_M_S])
 220
 221 (define_mode_attr MVE_CNVT [(V8HI "V8HF") (V4SI "V4SF") (V8HF "V8HI")
 222                             (V4SF "V4SI")])
 223
 224 (define_int_attr supf [(VCVTQ_TO_F_S "s") (VCVTQ_TO_F_U "u") (VREV16Q_S "s")
 225                        (VREV16Q_U "u") (VMVNQ_N_S "s") (VMVNQ_N_U "u")
 226                        (VCVTAQ_U "u") (VCVTAQ_S "s") (VREV64Q_S "s")
 227                        (VREV64Q_U "u") (VMVNQ_S "s") (VMVNQ_U "u")
 228                        (VDUPQ_N_U "u") (VDUPQ_N_S"s") (VADDVQ_S "s")
 229                        (VADDVQ_U "u") (VADDVQ_S "s") (VADDVQ_U "u")
 230                        (VMOVLTQ_U "u") (VMOVLTQ_S "s") (VMOVLBQ_S "s")
 231                        (VMOVLBQ_U "u") (VCVTQ_FROM_F_S "s") (VCVTQ_FROM_F_U "u")
 232                        (VCVTPQ_S "s") (VCVTPQ_U "u") (VCVTNQ_S "s")
 233                        (VCVTNQ_U "u") (VCVTMQ_S "s") (VCVTMQ_U "u")
 234                        (VCLZQ_U "u") (VCLZQ_S "s") (VREV32Q_U "u")
 235                        (VREV32Q_S "s") (VADDLVQ_U "u") (VADDLVQ_S "s")
 236                        (VCVTQ_N_TO_F_S "s") (VCVTQ_N_TO_F_U "u")
 237                        (VCREATEQ_U "u") (VCREATEQ_S "s") (VSHRQ_N_S "s")
 238                        (VSHRQ_N_U "u") (VCVTQ_N_FROM_F_S "s") (VSHLQ_U "u")
 239                        (VCVTQ_N_FROM_F_U "u") (VADDLVQ_P_S "s") (VSHLQ_S "s")
 240                        (VADDLVQ_P_U "u") (VCMPNEQ_U "u") (VCMPNEQ_S "s")
 241                        (VABDQ_M_S "s") (VABDQ_M_U "u") (VABDQ_S "s")
 242                        (VABDQ_U "u") (VADDQ_N_S "s") (VADDQ_N_U "u")
 243                        (VADDVQ_P_S "s") (VADDVQ_P_U "u") (VANDQ_S "s")
 244                        (VANDQ_U "u") (VBICQ_S "s") (VBICQ_U "u")
 245                        (VBRSRQ_N_S "s") (VBRSRQ_N_U "u") (VCADDQ_ROT270_S "s")
 246                        (VCADDQ_ROT270_U "u") (VCADDQ_ROT90_S "s")
 247                        (VCMPEQQ_S "s") (VCMPEQQ_U "u") (VCADDQ_ROT90_U "u")
 248                        (VCMPEQQ_N_S "s") (VCMPEQQ_N_U "u") (VCMPNEQ_N_S "s")
 249                        (VCMPNEQ_N_U "u") (VEORQ_S "s") (VEORQ_U "u")
 250                        (VHADDQ_N_S "s") (VHADDQ_N_U "u") (VHADDQ_S "s")
 251                        (VHADDQ_U "u") (VHSUBQ_N_S "s")  (VHSUBQ_N_U "u")
 252                        (VHSUBQ_S "s") (VMAXQ_S "s") (VMAXQ_U "u") (VHSUBQ_U "u")
 253                        (VMAXVQ_S "s") (VMAXVQ_U "u") (VMINQ_S "s") (VMINQ_U "u")
 254                        (VMINVQ_S "s") (VMINVQ_U "u") (VMLADAVQ_S "s")
 255                        (VMLADAVQ_U "u") (VMULHQ_S "s") (VMULHQ_U "u")
 256                        (VMULLBQ_INT_S "s") (VMULLBQ_INT_U "u") (VQADDQ_S "s")
 257                        (VMULLTQ_INT_S "s") (VMULLTQ_INT_U "u") (VQADDQ_U "u")
 258                        (VMULQ_N_S "s") (VMULQ_N_U "u") (VMULQ_S "s")
 259                        (VMULQ_U "u") (VORNQ_S "s") (VORNQ_U "u") (VORRQ_S "s")
 260                        (VORRQ_U "u") (VQADDQ_N_S "s") (VQADDQ_N_U "u")
 261                        (VQRSHLQ_N_S "s") (VQRSHLQ_N_U "u") (VQRSHLQ_S "s")
 262                        (VQRSHLQ_U "u") (VQSHLQ_N_S "s") (VQSHLQ_N_U "u")
 263                        (VQSHLQ_R_S "s") (VQSHLQ_R_U "u") (VQSHLQ_S "s")
 264                        (VQSHLQ_U "u") (VQSUBQ_N_S "s") (VQSUBQ_N_U "u")
 265                        (VQSUBQ_S "s") (VQSUBQ_U "u") (VRHADDQ_S "s")
 266                        (VRHADDQ_U "u") (VRMULHQ_S "s") (VRMULHQ_U "u")
 267                        (VRSHLQ_N_S "s") (VRSHLQ_N_U "u") (VRSHLQ_S "s")
 268                        (VRSHLQ_U "u") (VRSHRQ_N_S "s") (VRSHRQ_N_U "u")
 269                        (VSHLQ_N_S "s") (VSHLQ_N_U "u") (VSHLQ_R_S "s")
 270                        (VSHLQ_R_U "u") (VSUBQ_N_S "s") (VSUBQ_N_U "u")
 271                        (VSUBQ_S "s") (VSUBQ_U "u") (VADDVAQ_S "s")
 272                        (VADDVAQ_U "u") (VADDLVAQ_S "s") (VADDLVAQ_U "u")
 273                        (VBICQ_N_S "s") (VBICQ_N_U "u") (VMLALDAVQ_U "u")
 274                        (VMLALDAVQ_S "s") (VMLALDAVXQ_U "u") (VMLALDAVXQ_S "s")
 275                        (VMOVNBQ_U "u") (VMOVNBQ_S "s") (VMOVNTQ_U "u")
 276                        (VMOVNTQ_S "s") (VORRQ_N_S "s") (VORRQ_N_U "u")
 277                        (VQMOVNBQ_U "u") (VQMOVNBQ_S "s") (VQMOVNTQ_S "s")
 278                        (VQMOVNTQ_U "u") (VSHLLBQ_N_U "u") (VSHLLBQ_N_S "s")
 279                        (VSHLLTQ_N_U "u") (VSHLLTQ_N_S "s") (VRMLALDAVHQ_U "u")
 280                        (VRMLALDAVHQ_S "s") (VBICQ_M_N_S "s") (VBICQ_M_N_U "u")
 281                        (VCVTAQ_M_S "s") (VCVTAQ_M_U "u") (VCVTQ_M_TO_F_S "s")
 282                        (VCVTQ_M_TO_F_U "u") (VQRSHRNBQ_N_S "s")
 283                        (VQRSHRNBQ_N_U "u") (VABAVQ_S "s") (VABAVQ_U "u")
 284                        (VRMLALDAVHAQ_U "u") (VRMLALDAVHAQ_S "s") (VSHLCQ_S "s")
 285                        (VSHLCQ_U "u") (VADDVAQ_P_S "s") (VADDVAQ_P_U "u")
 286                        (VCLZQ_M_S "s") (VCLZQ_M_U "u") (VCMPEQQ_M_N_S "s")
 287                        (VCMPEQQ_M_N_U "u") (VCMPEQQ_M_S "s") (VCMPEQQ_M_U "u")
 288                        (VCMPNEQ_M_N_S "s") (VCMPNEQ_M_N_U "u") (VCMPNEQ_M_S "s")
 289                        (VCMPNEQ_M_U "u") (VDUPQ_M_N_S "s") (VDUPQ_M_N_U "u")
 290                        (VMAXVQ_P_S "s") (VMAXVQ_P_U "u") (VMINVQ_P_S "s")
 291                        (VMINVQ_P_U "u") (VMLADAVAQ_S "s") (VMLADAVAQ_U "u")
 292                        (VMLADAVQ_P_S "s") (VMLADAVQ_P_U "u") (VMLAQ_N_S "s")
 293                        (VMLAQ_N_U "u") (VMLASQ_N_S "s") (VMLASQ_N_U "u")
 294                        (VMVNQ_M_S "s") (VMVNQ_M_U "u") (VPSELQ_S "s")
 295                        (VPSELQ_U "u") (VQDMLAHQ_N_S "s") (VQDMLAHQ_N_U "u")
 296                        (VQRDMLAHQ_N_S "s") (VQRDMLAHQ_N_U "u")
 297                        (VQRDMLASHQ_N_S "s") (VQRDMLASHQ_N_U "u")
 298                        (VQRSHLQ_M_N_S "s") (VQRSHLQ_M_N_U "u")
 299                        (VQSHLQ_M_R_S "s") (VQSHLQ_M_R_U "u") (VSRIQ_N_S "s")
 300                        (VREV64Q_M_S "s") (VREV64Q_M_U "u") (VSRIQ_N_U "u")
 301                        (VRSHLQ_M_N_S "s") (VRSHLQ_M_N_U "u") (VSHLQ_M_R_S "s")
 302                        (VSHLQ_M_R_U "u") (VSLIQ_N_S "s") (VSLIQ_N_U "u")
 303                        (VMLALDAVQ_P_S "s") (VQMOVNBQ_M_S "s") (VMOVLTQ_M_S "s")
 304                        (VMOVNBQ_M_S "s") (VRSHRNTQ_N_S "s") (VORRQ_M_N_S "s")
 305                        (VREV32Q_M_S "s") (VQRSHRNTQ_N_S "s") (VMOVNTQ_M_S "s")
 306                        (VMOVLBQ_M_S "s") (VMLALDAVAQ_S "s") (VQSHRNBQ_N_S "s")
 307                        (VSHRNBQ_N_S "s") (VRSHRNBQ_N_S "s") (VMLALDAVXQ_P_S "s")
 308                        (VQMOVNTQ_M_S "s") (VMVNQ_M_N_S "s") (VQSHRNTQ_N_S "s")
 309                        (VMLALDAVAXQ_S "s") (VSHRNTQ_N_S "s") (VMLALDAVQ_P_U "u")
 310                        (VQMOVNBQ_M_U "u") (VMOVLTQ_M_U "u") (VMOVNBQ_M_U "u")
 311                        (VRSHRNTQ_N_U "u") (VORRQ_M_N_U "u") (VREV32Q_M_U "u")
 312                        (VREV16Q_M_S "s") (VREV16Q_M_U "u")
 313                        (VQRSHRNTQ_N_U "u") (VMOVNTQ_M_U "u") (VMOVLBQ_M_U "u")
 314                        (VMLALDAVAQ_U "u") (VQSHRNBQ_N_U "u") (VSHRNBQ_N_U "u")
 315                        (VRSHRNBQ_N_U "u") (VMLALDAVXQ_P_U "u")
 316                        (VMVNQ_M_N_U "u") (VQSHRNTQ_N_U "u") (VMLALDAVAXQ_U "u")
 317                        (VQMOVNTQ_M_U "u") (VSHRNTQ_N_U "u") (VCVTMQ_M_S "s")
 318                        (VCVTMQ_M_U "u") (VCVTNQ_M_S "s") (VCVTNQ_M_U "u")
 319                        (VCVTPQ_M_S "s") (VCVTPQ_M_U "u") (VADDLVAQ_P_S "s")
 320                        (VCVTQ_M_N_FROM_F_U "u") (VCVTQ_M_FROM_F_S "s")
 321                        (VCVTQ_M_FROM_F_U "u") (VRMLALDAVHQ_P_U "u")
 322                        (VRMLALDAVHQ_P_S "s") (VADDLVAQ_P_U "u")
 323                        (VCVTQ_M_N_FROM_F_S "s") (VABAVQ_P_U "u")
 324                        (VABAVQ_P_S "s") (VSHLQ_M_S "s") (VSHLQ_M_U "u")
 325                        (VSRIQ_M_N_S "s") (VSRIQ_M_N_U "u") (VSUBQ_M_S "s")
 326                        (VSUBQ_M_U "u") (VCVTQ_M_N_TO_F_S "s")
 327                        (VCVTQ_M_N_TO_F_U "u") (VADDQ_M_N_U "u")
 328                        (VSHLQ_M_N_S "s") (VMAXQ_M_U "u") (VHSUBQ_M_N_U "u")
 329                        (VMULQ_M_N_S "s") (VQSHLQ_M_U "u") (VRHADDQ_M_S "s")
 330                        (VEORQ_M_U "u") (VSHRQ_M_N_U "u") (VCADDQ_ROT90_M_U "u")
 331                        (VMLADAVAQ_P_U "u") (VEORQ_M_S "s") (VBRSRQ_M_N_S "s")
 332                        (VMULQ_M_U "u") (VQRDMLAHQ_M_N_S "s") (VHSUBQ_M_N_S "s")
 333                        (VQRSHLQ_M_S "s") (VMULQ_M_N_U "u")
 334                        (VMULQ_M_S "s") (VQSHLQ_M_N_U "u") (VSLIQ_M_N_U "u")
 335                        (VMLADAVAQ_P_S "s") (VQRSHLQ_M_U "u")
 336                        (VMULLBQ_INT_M_U "u") (VSHLQ_M_N_U "u") (VQSUBQ_M_U "u")
 337                        (VQRDMLASHQ_M_N_U "u") (VRSHRQ_M_N_S "s")
 338                        (VORNQ_M_S "s") (VCADDQ_ROT270_M_S "s") (VRHADDQ_M_U "u")
 339                        (VRSHRQ_M_N_U "u") (VMLASQ_M_N_U "u") (VHSUBQ_M_U "u")
 340                        (VQSUBQ_M_N_S "s") (VMULLTQ_INT_M_S "s")
 341                        (VORRQ_M_S "s") (VQDMLAHQ_M_N_U "u") (VRSHLQ_M_S "s")
 342                        (VHADDQ_M_U "u") (VHADDQ_M_N_S "s") (VMULLTQ_INT_M_U "u")
 343                        (VORRQ_M_U "u") (VHADDQ_M_S "s") (VHADDQ_M_N_U "u")
 344                        (VQDMLAHQ_M_N_S "s") (VMAXQ_M_S "s") (VORNQ_M_U "u")
 345                        (VCADDQ_ROT270_M_U "u") (VQADDQ_M_U "u")
 346                        (VQRDMLASHQ_M_N_S "s") (VBICQ_M_U "u") (VMINQ_M_U "u")
 347                        (VSUBQ_M_N_S "s") (VMULLBQ_INT_M_S "s") (VQSUBQ_M_S "s")
 348                        (VCADDQ_ROT90_M_S "s") (VRMULHQ_M_S "s") (VANDQ_M_U "u")
 349                        (VMULHQ_M_S "s") (VADDQ_M_S "s") (VQRDMLAHQ_M_N_U "u")
 350                        (VMLASQ_M_N_S "s") (VHSUBQ_M_S "s") (VRMULHQ_M_U "u")
 351                        (VQADDQ_M_N_S "s") (VSHRQ_M_N_S "s") (VANDQ_M_S "s")
 352                        (VABDQ_M_U "u") (VQSHLQ_M_S "s") (VABDQ_M_S "s")
 353                        (VSUBQ_M_N_U "u") (VMLAQ_M_N_S "s") (VBRSRQ_M_N_U "u")
 354                        (VADDQ_M_U "u") (VRSHLQ_M_U "u") (VSLIQ_M_N_S "s")
 355                        (VQADDQ_M_N_U "u") (VADDQ_M_N_S "s") (VQSUBQ_M_N_U "u")
 356                        (VMLAQ_M_N_U "u") (VMINQ_M_S "s") (VMULHQ_M_U "u")
 357                        (VQADDQ_M_S "s") (VBICQ_M_S "s") (VQSHLQ_M_N_S "s")
 358                        (VQSHRNTQ_M_N_S "s") (VQSHRNTQ_M_N_U "u")
 359                        (VSHRNTQ_M_N_U "u") (VSHRNTQ_M_N_S "s")
 360                        (VSHRNBQ_M_N_S "s") (VSHRNBQ_M_N_U "u")
 361                        (VSHLLTQ_M_N_S "s") (VSHLLTQ_M_N_U "u")
 362                        (VSHLLBQ_M_N_S "s") (VSHLLBQ_M_N_U "u")
 363                        (VRSHRNTQ_M_N_S "s") (VRSHRNTQ_M_N_U "u")
 364                        (VRSHRNBQ_M_N_U "u") (VRSHRNBQ_M_N_S "s")
 365                        (VQSHRNTQ_M_N_U "u") (VQSHRNTQ_M_N_S "s")
 366                        (VQSHRNBQ_M_N_S "s") (VQSHRNBQ_M_N_U "u")
 367                        (VQRSHRNTQ_M_N_S "s") (VQRSHRNTQ_M_N_U "u")
 368                        (VQRSHRNBQ_M_N_S "s") (VQRSHRNBQ_M_N_U "u")
 369                        (VMLALDAVAXQ_P_S "s") (VMLALDAVAXQ_P_U "u")
 370                        (VMLALDAVAQ_P_S "s") (VMLALDAVAQ_P_U "u")
 371                        (VSTRWQSB_S "s") (VSTRWQSB_U "u") (VSTRBQSO_S "s")
 372                        (VSTRBQSO_U "u") (VSTRBQ_S "s") (VSTRBQ_U "u")
 373                        (VLDRBQGO_S "s") (VLDRBQGO_U "u") (VLDRBQ_S "s")
 374                        (VLDRBQ_U "u") (VLDRWQGB_S "s") (VLDRWQGB_U "u")
 375                        (VLD1Q_S "s") (VLD1Q_U "u") (VLDRHQGO_S "s")
 376                        (VLDRHQGO_U "u") (VLDRHQGSO_S "s") (VLDRHQGSO_U "u")
 377                        (VLDRHQ_S "s") (VLDRHQ_U "u") (VLDRWQ_S "s")
 378                        (VLDRWQ_U "u") (VLDRDQGB_S "s") (VLDRDQGB_U "u")
 379                        (VLDRDQGO_S "s") (VLDRDQGO_U "u") (VLDRDQGSO_S "s")
 380                        (VLDRDQGSO_U "u") (VLDRWQGO_S "s") (VLDRWQGO_U "u")
 381                        (VLDRWQGSO_S "s") (VLDRWQGSO_U "u") (VST1Q_S "s")
 382                        (VST1Q_U "u") (VSTRHQSO_S "s") (VSTRHQSO_U "u")
 383                        (VSTRHQSSO_S "s") (VSTRHQSSO_U "u") (VSTRHQ_S "s")
 384                        (VSTRHQ_U "u") (VSTRWQ_S "s") (VSTRWQ_U "u")
 385                        (VSTRDQSB_S "s") (VSTRDQSB_U "u") (VSTRDQSO_S "s")
 386                        (VSTRDQSO_U "u") (VSTRDQSSO_S "s") (VSTRDQSSO_U "u")
 387                        (VSTRWQSO_U "u") (VSTRWQSO_S "s") (VSTRWQSSO_U "u")
 388                        (VSTRWQSSO_S "s") (VSTRWQSBWB_S "s") (VSTRWQSBWB_U "u")
 389                        (VLDRWQGBWB_S "s") (VLDRWQGBWB_U "u") (VLDRDQGBWB_S "s")
 390                        (VLDRDQGBWB_U "u") (VSTRDQSBWB_S "s") (VADCQ_M_S "s")
 391                        (VSTRDQSBWB_U "u") (VSBCQ_U "u")  (VSBCQ_M_U "u")
 392                        (VSBCQ_S "s")  (VSBCQ_M_S "s") (VSBCIQ_U "u")
 393                        (VSBCIQ_M_U "u") (VSBCIQ_S "s") (VSBCIQ_M_S "s")
 394                        (VADCQ_U "u")  (VADCQ_M_U "u") (VADCQ_S "s")
 395                        (VADCIQ_U "u") (VADCIQ_M_U "u") (VADCIQ_S "s")
 396                        (VADCIQ_M_S "s") (SQRSHRL_64 "64") (SQRSHRL_48 "48")
 397                        (UQRSHLL_64 "64") (UQRSHLL_48 "48") (VSHLCQ_M_S "s")
 398                        (VSHLCQ_M_U "u")])
 399
 400 (define_int_attr mode1 [(VCTP8Q "8") (VCTP16Q "16") (VCTP32Q "32")
 401                         (VCTP64Q "64") (VCTP8Q_M "8") (VCTP16Q_M "16")
 402                         (VCTP32Q_M "32") (VCTP64Q_M "64")])
 403 (define_mode_attr MVE_pred2 [(V16QI "mve_imm_8") (V8HI "mve_imm_16")
 404                              (V4SI "mve_imm_32")
 405                              (V8HF "mve_imm_16") (V4SF "mve_imm_32")])
 406 (define_mode_attr MVE_constraint2 [(V16QI "Rb") (V8HI "Rd") (V4SI "Rf")
 407                                     (V8HF "Rd") (V4SF "Rf")])
 408 (define_mode_attr MVE_LANES [(V16QI "16") (V8HI "8") (V4SI "4")])
 409 (define_mode_attr MVE_constraint [ (V16QI "Ra") (V8HI "Rc") (V4SI "Re")])
 410 (define_mode_attr MVE_pred [ (V16QI "mve_imm_7") (V8HI "mve_imm_15")
 411                                    (V4SI "mve_imm_31")])
 412 (define_mode_attr MVE_constraint3 [ (V8HI "Rb") (V4SI "Rd")])
 413 (define_mode_attr MVE_pred3 [ (V8HI "mve_imm_8") (V4SI "mve_imm_16")])
 414 (define_mode_attr MVE_constraint1 [ (V8HI "Ra") (V4SI "Rc")])
 415 (define_mode_attr MVE_pred1 [ (V8HI "mve_imm_7") (V4SI "mve_imm_15")])
 416 (define_mode_attr MVE_B_ELEM [ (V16QI "V16QI") (V8HI "V8QI") (V4SI "V4QI")])
 417 (define_mode_attr MVE_H_ELEM [ (V8HI "V8HI") (V4SI "V4HI")])
 418 (define_mode_attr V_sz_elem1 [(V16QI "b") (V8HI  "h") (V4SI "w") (V8HF "h")
 419                               (V4SF "w")])
 420 (define_mode_attr V_extr_elem [(V16QI "u8") (V8HI "u16") (V4SI "32")
 421                               (V8HF "u16") (V4SF "32")])
 422
 423 (define_mode_attr earlyclobber_32 [(V16QI "=w") (V8HI "=w") (V4SI "=&w")
 424                                                 (V8HF "=w") (V4SF "=&w")])
 425
 426 (define_int_iterator VCVTQ_TO_F [VCVTQ_TO_F_S VCVTQ_TO_F_U])
 427 (define_int_iterator VMVNQ_N [VMVNQ_N_U VMVNQ_N_S])
 428 (define_int_iterator VREV64Q [VREV64Q_S VREV64Q_U])
 429 (define_int_iterator VCVTQ_FROM_F [VCVTQ_FROM_F_S VCVTQ_FROM_F_U])
 430 (define_int_iterator VREV16Q [VREV16Q_U VREV16Q_S])
 431 (define_int_iterator VCVTAQ [VCVTAQ_U VCVTAQ_S])
 432 (define_int_iterator VMVNQ [VMVNQ_U VMVNQ_S])
 433 (define_int_iterator VDUPQ_N [VDUPQ_N_U VDUPQ_N_S])
 434 (define_int_iterator VCLZQ [VCLZQ_U VCLZQ_S])
 435 (define_int_iterator VADDVQ [VADDVQ_U VADDVQ_S])
 436 (define_int_iterator VREV32Q [VREV32Q_U VREV32Q_S])
 437 (define_int_iterator VMOVLBQ [VMOVLBQ_S VMOVLBQ_U])
 438 (define_int_iterator VMOVLTQ [VMOVLTQ_U VMOVLTQ_S])
 439 (define_int_iterator VCVTPQ [VCVTPQ_S VCVTPQ_U])
 440 (define_int_iterator VCVTNQ [VCVTNQ_S VCVTNQ_U])
 441 (define_int_iterator VCVTMQ [VCVTMQ_S VCVTMQ_U])
 442 (define_int_iterator VADDLVQ [VADDLVQ_U VADDLVQ_S])
 443 (define_int_iterator VCTPQ [VCTP8Q VCTP16Q VCTP32Q VCTP64Q])
 444 (define_int_iterator VCTPQ_M [VCTP8Q_M VCTP16Q_M VCTP32Q_M VCTP64Q_M])
 445 (define_int_iterator VCVTQ_N_TO_F [VCVTQ_N_TO_F_S VCVTQ_N_TO_F_U])
 446 (define_int_iterator VCREATEQ [VCREATEQ_U VCREATEQ_S])
 447 (define_int_iterator VSHRQ_N [VSHRQ_N_S VSHRQ_N_U])
 448 (define_int_iterator VCVTQ_N_FROM_F [VCVTQ_N_FROM_F_S VCVTQ_N_FROM_F_U])
 449 (define_int_iterator VADDLVQ_P [VADDLVQ_P_S VADDLVQ_P_U])
 450 (define_int_iterator VCMPNEQ [VCMPNEQ_U VCMPNEQ_S])
 451 (define_int_iterator VSHLQ [VSHLQ_S VSHLQ_U])
 452 (define_int_iterator VABDQ [VABDQ_S VABDQ_U])
 453 (define_int_iterator VADDQ_N [VADDQ_N_S VADDQ_N_U])
 454 (define_int_iterator VADDVAQ [VADDVAQ_S VADDVAQ_U])
 455 (define_int_iterator VADDVQ_P [VADDVQ_P_U VADDVQ_P_S])
 456 (define_int_iterator VANDQ [VANDQ_U VANDQ_S])
 457 (define_int_iterator VBICQ [VBICQ_S VBICQ_U])
 458 (define_int_iterator VBRSRQ_N [VBRSRQ_N_U VBRSRQ_N_S])
 459 (define_int_iterator VCADDQ_ROT270 [VCADDQ_ROT270_S VCADDQ_ROT270_U])
 460 (define_int_iterator VCADDQ_ROT90 [VCADDQ_ROT90_U VCADDQ_ROT90_S])
 461 (define_int_iterator VCMPEQQ [VCMPEQQ_U VCMPEQQ_S])
 462 (define_int_iterator VCMPEQQ_N [VCMPEQQ_N_S VCMPEQQ_N_U])
 463 (define_int_iterator VCMPNEQ_N [VCMPNEQ_N_U VCMPNEQ_N_S])
 464 (define_int_iterator VEORQ [VEORQ_U VEORQ_S])
 465 (define_int_iterator VHADDQ [VHADDQ_S VHADDQ_U])
 466 (define_int_iterator VHADDQ_N [VHADDQ_N_U VHADDQ_N_S])
 467 (define_int_iterator VHSUBQ [VHSUBQ_S VHSUBQ_U])
 468 (define_int_iterator VHSUBQ_N [VHSUBQ_N_U VHSUBQ_N_S])
 469 (define_int_iterator VMAXQ [VMAXQ_U VMAXQ_S])
 470 (define_int_iterator VMAXVQ [VMAXVQ_U VMAXVQ_S])
 471 (define_int_iterator VMINQ [VMINQ_S VMINQ_U])
 472 (define_int_iterator VMINVQ [VMINVQ_U VMINVQ_S])
 473 (define_int_iterator VMLADAVQ [VMLADAVQ_U VMLADAVQ_S])
 474 (define_int_iterator VMULHQ [VMULHQ_S VMULHQ_U])
 475 (define_int_iterator VMULLBQ_INT [VMULLBQ_INT_U VMULLBQ_INT_S])
 476 (define_int_iterator VMULLTQ_INT [VMULLTQ_INT_U VMULLTQ_INT_S])
 477 (define_int_iterator VMULQ [VMULQ_U VMULQ_S])
 478 (define_int_iterator VMULQ_N [VMULQ_N_U VMULQ_N_S])
 479 (define_int_iterator VORNQ [VORNQ_U VORNQ_S])
 480 (define_int_iterator VORRQ [VORRQ_S VORRQ_U])
 481 (define_int_iterator VQADDQ [VQADDQ_U VQADDQ_S])
 482 (define_int_iterator VQADDQ_N [VQADDQ_N_S VQADDQ_N_U])
 483 (define_int_iterator VQRSHLQ [VQRSHLQ_S VQRSHLQ_U])
 484 (define_int_iterator VQRSHLQ_N [VQRSHLQ_N_S VQRSHLQ_N_U])
 485 (define_int_iterator VQSHLQ [VQSHLQ_S VQSHLQ_U])
 486 (define_int_iterator VQSHLQ_N [VQSHLQ_N_S VQSHLQ_N_U])
 487 (define_int_iterator VQSHLQ_R [VQSHLQ_R_U VQSHLQ_R_S])
 488 (define_int_iterator VQSUBQ [VQSUBQ_U VQSUBQ_S])
 489 (define_int_iterator VQSUBQ_N [VQSUBQ_N_S VQSUBQ_N_U])
 490 (define_int_iterator VRHADDQ [VRHADDQ_S VRHADDQ_U])
 491 (define_int_iterator VRMULHQ [VRMULHQ_S VRMULHQ_U])
 492 (define_int_iterator VRSHLQ [VRSHLQ_S VRSHLQ_U])
 493 (define_int_iterator VRSHLQ_N [VRSHLQ_N_U VRSHLQ_N_S])
 494 (define_int_iterator VRSHRQ_N [VRSHRQ_N_S VRSHRQ_N_U])
 495 (define_int_iterator VSHLQ_N [VSHLQ_N_U VSHLQ_N_S])
 496 (define_int_iterator VSHLQ_R [VSHLQ_R_S VSHLQ_R_U])
 497 (define_int_iterator VSUBQ [VSUBQ_S VSUBQ_U])
 498 (define_int_iterator VSUBQ_N [VSUBQ_N_S VSUBQ_N_U])
 499 (define_int_iterator VADDLVAQ [VADDLVAQ_S VADDLVAQ_U])
 500 (define_int_iterator VBICQ_N [VBICQ_N_S VBICQ_N_U])
 501 (define_int_iterator VMLALDAVQ [VMLALDAVQ_U VMLALDAVQ_S])
 502 (define_int_iterator VMLALDAVXQ [VMLALDAVXQ_U VMLALDAVXQ_S])
 503 (define_int_iterator VMOVNBQ [VMOVNBQ_U VMOVNBQ_S])
 504 (define_int_iterator VMOVNTQ [VMOVNTQ_S VMOVNTQ_U])
 505 (define_int_iterator VORRQ_N [VORRQ_N_U VORRQ_N_S])
 506 (define_int_iterator VQMOVNBQ [VQMOVNBQ_U VQMOVNBQ_S])
 507 (define_int_iterator VQMOVNTQ [VQMOVNTQ_U VQMOVNTQ_S])
 508 (define_int_iterator VSHLLBQ_N [VSHLLBQ_N_S VSHLLBQ_N_U])
 509 (define_int_iterator VSHLLTQ_N [VSHLLTQ_N_U VSHLLTQ_N_S])
 510 (define_int_iterator VRMLALDAVHQ [VRMLALDAVHQ_U VRMLALDAVHQ_S])
 511 (define_int_iterator VBICQ_M_N [VBICQ_M_N_S VBICQ_M_N_U])
 512 (define_int_iterator VCVTAQ_M [VCVTAQ_M_S VCVTAQ_M_U])
 513 (define_int_iterator VCVTQ_M_TO_F [VCVTQ_M_TO_F_S VCVTQ_M_TO_F_U])
 514 (define_int_iterator VQRSHRNBQ_N [VQRSHRNBQ_N_U VQRSHRNBQ_N_S])
 515 (define_int_iterator VABAVQ [VABAVQ_S VABAVQ_U])
 516 (define_int_iterator VSHLCQ [VSHLCQ_S VSHLCQ_U])
 517 (define_int_iterator VRMLALDAVHAQ [VRMLALDAVHAQ_S VRMLALDAVHAQ_U])
 518 (define_int_iterator VADDVAQ_P [VADDVAQ_P_S VADDVAQ_P_U])
 519 (define_int_iterator VCLZQ_M [VCLZQ_M_S VCLZQ_M_U])
 520 (define_int_iterator VCMPEQQ_M_N [VCMPEQQ_M_N_S VCMPEQQ_M_N_U])
 521 (define_int_iterator VCMPEQQ_M [VCMPEQQ_M_S VCMPEQQ_M_U])
 522 (define_int_iterator VCMPNEQ_M_N [VCMPNEQ_M_N_S VCMPNEQ_M_N_U])
 523 (define_int_iterator VCMPNEQ_M [VCMPNEQ_M_S VCMPNEQ_M_U])
 524 (define_int_iterator VDUPQ_M_N [VDUPQ_M_N_S VDUPQ_M_N_U])
 525 (define_int_iterator VMAXVQ_P [VMAXVQ_P_S VMAXVQ_P_U])
 526 (define_int_iterator VMINVQ_P [VMINVQ_P_S VMINVQ_P_U])
 527 (define_int_iterator VMLADAVAQ [VMLADAVAQ_S VMLADAVAQ_U])
 528 (define_int_iterator VMLADAVQ_P [VMLADAVQ_P_S VMLADAVQ_P_U])
 529 (define_int_iterator VMLAQ_N [VMLAQ_N_S VMLAQ_N_U])
 530 (define_int_iterator VMLASQ_N [VMLASQ_N_S VMLASQ_N_U])
 531 (define_int_iterator VMVNQ_M [VMVNQ_M_S VMVNQ_M_U])
 532 (define_int_iterator VPSELQ [VPSELQ_S VPSELQ_U])
 533 (define_int_iterator VQDMLAHQ_N [VQDMLAHQ_N_S VQDMLAHQ_N_U])
 534 (define_int_iterator VQRDMLAHQ_N [VQRDMLAHQ_N_S VQRDMLAHQ_N_U])
 535 (define_int_iterator VQRDMLASHQ_N [VQRDMLASHQ_N_S VQRDMLASHQ_N_U])
 536 (define_int_iterator VQRSHLQ_M_N [VQRSHLQ_M_N_S VQRSHLQ_M_N_U])
 537 (define_int_iterator VQSHLQ_M_R [VQSHLQ_M_R_S VQSHLQ_M_R_U])
 538 (define_int_iterator VREV64Q_M [VREV64Q_M_S VREV64Q_M_U])
 539 (define_int_iterator VRSHLQ_M_N [VRSHLQ_M_N_S VRSHLQ_M_N_U])
 540 (define_int_iterator VSHLQ_M_R [VSHLQ_M_R_S VSHLQ_M_R_U])
 541 (define_int_iterator VSLIQ_N [VSLIQ_N_S VSLIQ_N_U])
 542 (define_int_iterator VSRIQ_N [VSRIQ_N_S VSRIQ_N_U])
 543 (define_int_iterator VMLALDAVQ_P [VMLALDAVQ_P_U VMLALDAVQ_P_S])
 544 (define_int_iterator VQMOVNBQ_M [VQMOVNBQ_M_S VQMOVNBQ_M_U])
 545 (define_int_iterator VMOVLTQ_M [VMOVLTQ_M_U VMOVLTQ_M_S])
 546 (define_int_iterator VMOVNBQ_M [VMOVNBQ_M_U VMOVNBQ_M_S])
 547 (define_int_iterator VRSHRNTQ_N [VRSHRNTQ_N_U VRSHRNTQ_N_S])
 548 (define_int_iterator VORRQ_M_N [VORRQ_M_N_S VORRQ_M_N_U])
 549 (define_int_iterator VREV32Q_M [VREV32Q_M_S VREV32Q_M_U])
 550 (define_int_iterator VREV16Q_M [VREV16Q_M_S VREV16Q_M_U])
 551 (define_int_iterator VQRSHRNTQ_N [VQRSHRNTQ_N_U VQRSHRNTQ_N_S])
 552 (define_int_iterator VMOVNTQ_M [VMOVNTQ_M_U VMOVNTQ_M_S])
 553 (define_int_iterator VMOVLBQ_M [VMOVLBQ_M_U VMOVLBQ_M_S])
 554 (define_int_iterator VMLALDAVAQ [VMLALDAVAQ_S VMLALDAVAQ_U])
 555 (define_int_iterator VQSHRNBQ_N [VQSHRNBQ_N_U VQSHRNBQ_N_S])
 556 (define_int_iterator VSHRNBQ_N [VSHRNBQ_N_U VSHRNBQ_N_S])
 557 (define_int_iterator VRSHRNBQ_N [VRSHRNBQ_N_S VRSHRNBQ_N_U])
 558 (define_int_iterator VMLALDAVXQ_P [VMLALDAVXQ_P_U VMLALDAVXQ_P_S])
 559 (define_int_iterator VQMOVNTQ_M [VQMOVNTQ_M_U VQMOVNTQ_M_S])
 560 (define_int_iterator VMVNQ_M_N [VMVNQ_M_N_U VMVNQ_M_N_S])
 561 (define_int_iterator VQSHRNTQ_N [VQSHRNTQ_N_U VQSHRNTQ_N_S])
 562 (define_int_iterator VMLALDAVAXQ [VMLALDAVAXQ_S VMLALDAVAXQ_U])
 563 (define_int_iterator VSHRNTQ_N [VSHRNTQ_N_S VSHRNTQ_N_U])
 564 (define_int_iterator VCVTMQ_M [VCVTMQ_M_S VCVTMQ_M_U])
 565 (define_int_iterator VCVTNQ_M [VCVTNQ_M_S VCVTNQ_M_U])
 566 (define_int_iterator VCVTPQ_M [VCVTPQ_M_S VCVTPQ_M_U])
 567 (define_int_iterator VCVTQ_M_N_FROM_F [VCVTQ_M_N_FROM_F_S VCVTQ_M_N_FROM_F_U])
 568 (define_int_iterator VCVTQ_M_FROM_F [VCVTQ_M_FROM_F_U VCVTQ_M_FROM_F_S])
 569 (define_int_iterator VRMLALDAVHQ_P [VRMLALDAVHQ_P_S VRMLALDAVHQ_P_U])
 570 (define_int_iterator VADDLVAQ_P [VADDLVAQ_P_U VADDLVAQ_P_S])
 571 (define_int_iterator VABAVQ_P [VABAVQ_P_S VABAVQ_P_U])
 572 (define_int_iterator VSHLQ_M [VSHLQ_M_S VSHLQ_M_U])
 573 (define_int_iterator VSRIQ_M_N [VSRIQ_M_N_S VSRIQ_M_N_U])
 574 (define_int_iterator VSUBQ_M [VSUBQ_M_U VSUBQ_M_S])
 575 (define_int_iterator VCVTQ_M_N_TO_F [VCVTQ_M_N_TO_F_U VCVTQ_M_N_TO_F_S])
 576 (define_int_iterator VHSUBQ_M [VHSUBQ_M_S VHSUBQ_M_U])
 577 (define_int_iterator VSLIQ_M_N [VSLIQ_M_N_U VSLIQ_M_N_S])
 578 (define_int_iterator VRSHLQ_M [VRSHLQ_M_S VRSHLQ_M_U])
 579 (define_int_iterator VMINQ_M [VMINQ_M_S VMINQ_M_U])
 580 (define_int_iterator VMULLBQ_INT_M [VMULLBQ_INT_M_U VMULLBQ_INT_M_S])
 581 (define_int_iterator VMULHQ_M [VMULHQ_M_S VMULHQ_M_U])
 582 (define_int_iterator VMULQ_M [VMULQ_M_S VMULQ_M_U])
 583 (define_int_iterator VHSUBQ_M_N [VHSUBQ_M_N_S VHSUBQ_M_N_U])
 584 (define_int_iterator VHADDQ_M_N [VHADDQ_M_N_S VHADDQ_M_N_U])
 585 (define_int_iterator VORRQ_M [VORRQ_M_S VORRQ_M_U])
 586 (define_int_iterator VRMULHQ_M [VRMULHQ_M_U VRMULHQ_M_S])
 587 (define_int_iterator VQADDQ_M [VQADDQ_M_U VQADDQ_M_S])
 588 (define_int_iterator VRSHRQ_M_N [VRSHRQ_M_N_S VRSHRQ_M_N_U])
 589 (define_int_iterator VQSUBQ_M_N [VQSUBQ_M_N_U VQSUBQ_M_N_S])
 590 (define_int_iterator VADDQ_M [VADDQ_M_U VADDQ_M_S])
 591 (define_int_iterator VORNQ_M [VORNQ_M_U VORNQ_M_S])
 592 (define_int_iterator VRHADDQ_M [VRHADDQ_M_U VRHADDQ_M_S])
 593 (define_int_iterator VQSHLQ_M [VQSHLQ_M_U VQSHLQ_M_S])
 594 (define_int_iterator VANDQ_M [VANDQ_M_U VANDQ_M_S])
 595 (define_int_iterator VBICQ_M [VBICQ_M_U VBICQ_M_S])
 596 (define_int_iterator VSHLQ_M_N [VSHLQ_M_N_S VSHLQ_M_N_U])
 597 (define_int_iterator VCADDQ_ROT270_M [VCADDQ_ROT270_M_U VCADDQ_ROT270_M_S])
 598 (define_int_iterator VQRSHLQ_M [VQRSHLQ_M_U VQRSHLQ_M_S])
 599 (define_int_iterator VQADDQ_M_N [VQADDQ_M_N_U VQADDQ_M_N_S])
 600 (define_int_iterator VADDQ_M_N [VADDQ_M_N_S VADDQ_M_N_U])
 601 (define_int_iterator VMAXQ_M [VMAXQ_M_S VMAXQ_M_U])
 602 (define_int_iterator VQSUBQ_M [VQSUBQ_M_U VQSUBQ_M_S])
 603 (define_int_iterator VMLASQ_M_N [VMLASQ_M_N_U VMLASQ_M_N_S])
 604 (define_int_iterator VMLADAVAQ_P [VMLADAVAQ_P_U VMLADAVAQ_P_S])
 605 (define_int_iterator VBRSRQ_M_N [VBRSRQ_M_N_U VBRSRQ_M_N_S])
 606 (define_int_iterator VMULQ_M_N [VMULQ_M_N_U VMULQ_M_N_S])
 607 (define_int_iterator VCADDQ_ROT90_M [VCADDQ_ROT90_M_U VCADDQ_ROT90_M_S])
 608 (define_int_iterator VMULLTQ_INT_M [VMULLTQ_INT_M_S VMULLTQ_INT_M_U])
 609 (define_int_iterator VEORQ_M [VEORQ_M_S VEORQ_M_U])
 610 (define_int_iterator VSHRQ_M_N [VSHRQ_M_N_S VSHRQ_M_N_U])
 611 (define_int_iterator VSUBQ_M_N [VSUBQ_M_N_S VSUBQ_M_N_U])
 612 (define_int_iterator VHADDQ_M [VHADDQ_M_S VHADDQ_M_U])
 613 (define_int_iterator VABDQ_M [VABDQ_M_S VABDQ_M_U])
 614 (define_int_iterator VMLAQ_M_N [VMLAQ_M_N_S VMLAQ_M_N_U])
 615 (define_int_iterator VQSHLQ_M_N [VQSHLQ_M_N_S VQSHLQ_M_N_U])
 616 (define_int_iterator VMLALDAVAQ_P [VMLALDAVAQ_P_U VMLALDAVAQ_P_S])
 617 (define_int_iterator VMLALDAVAXQ_P [VMLALDAVAXQ_P_U VMLALDAVAXQ_P_S])
 618 (define_int_iterator VQRSHRNBQ_M_N [VQRSHRNBQ_M_N_U VQRSHRNBQ_M_N_S])
 619 (define_int_iterator VQRSHRNTQ_M_N [VQRSHRNTQ_M_N_S VQRSHRNTQ_M_N_U])
 620 (define_int_iterator VQSHRNBQ_M_N [VQSHRNBQ_M_N_U VQSHRNBQ_M_N_S])
 621 (define_int_iterator VQSHRNTQ_M_N [VQSHRNTQ_M_N_S VQSHRNTQ_M_N_U])
 622 (define_int_iterator VRSHRNBQ_M_N [VRSHRNBQ_M_N_U VRSHRNBQ_M_N_S])
 623 (define_int_iterator VRSHRNTQ_M_N [VRSHRNTQ_M_N_U VRSHRNTQ_M_N_S])
 624 (define_int_iterator VSHLLBQ_M_N [VSHLLBQ_M_N_U VSHLLBQ_M_N_S])
 625 (define_int_iterator VSHLLTQ_M_N [VSHLLTQ_M_N_U VSHLLTQ_M_N_S])
 626 (define_int_iterator VSHRNBQ_M_N [VSHRNBQ_M_N_S VSHRNBQ_M_N_U])
 627 (define_int_iterator VSHRNTQ_M_N [VSHRNTQ_M_N_S VSHRNTQ_M_N_U])
 628 (define_int_iterator VSTRWSBQ [VSTRWQSB_S VSTRWQSB_U])
 629 (define_int_iterator VSTRBSOQ [VSTRBQSO_S VSTRBQSO_U])
 630 (define_int_iterator VSTRBQ [VSTRBQ_S VSTRBQ_U])
 631 (define_int_iterator VLDRBGOQ [VLDRBQGO_S VLDRBQGO_U])
 632 (define_int_iterator VLDRBQ [VLDRBQ_S VLDRBQ_U])
 633 (define_int_iterator VLDRWGBQ [VLDRWQGB_S VLDRWQGB_U])
 634 (define_int_iterator VLD1Q [VLD1Q_S VLD1Q_U])
 635 (define_int_iterator VLDRHGOQ [VLDRHQGO_S VLDRHQGO_U])
 636 (define_int_iterator VLDRHGSOQ [VLDRHQGSO_S VLDRHQGSO_U])
 637 (define_int_iterator VLDRHQ [VLDRHQ_S VLDRHQ_U])
 638 (define_int_iterator VLDRWQ [VLDRWQ_S VLDRWQ_U])
 639 (define_int_iterator VLDRDGBQ [VLDRDQGB_S VLDRDQGB_U])
 640 (define_int_iterator VLDRDGOQ [VLDRDQGO_S VLDRDQGO_U])
 641 (define_int_iterator VLDRDGSOQ [VLDRDQGSO_S VLDRDQGSO_U])
 642 (define_int_iterator VLDRWGOQ [VLDRWQGO_S VLDRWQGO_U])
 643 (define_int_iterator VLDRWGSOQ [VLDRWQGSO_S VLDRWQGSO_U])
 644 (define_int_iterator VST1Q [VST1Q_S VST1Q_U])
 645 (define_int_iterator VSTRHSOQ [VSTRHQSO_S VSTRHQSO_U])
 646 (define_int_iterator VSTRHSSOQ [VSTRHQSSO_S VSTRHQSSO_U])
 647 (define_int_iterator VSTRHQ [VSTRHQ_S VSTRHQ_U])
 648 (define_int_iterator VSTRWQ [VSTRWQ_S VSTRWQ_U])
 649 (define_int_iterator VSTRDSBQ [VSTRDQSB_S VSTRDQSB_U])
 650 (define_int_iterator VSTRDSOQ [VSTRDQSO_S VSTRDQSO_U])
 651 (define_int_iterator VSTRDSSOQ [VSTRDQSSO_S VSTRDQSSO_U])
 652 (define_int_iterator VSTRWSOQ [VSTRWQSO_S VSTRWQSO_U])
 653 (define_int_iterator VSTRWSSOQ [VSTRWQSSO_S VSTRWQSSO_U])
 654 (define_int_iterator VSTRWSBWBQ [VSTRWQSBWB_S VSTRWQSBWB_U])
 655 (define_int_iterator VLDRWGBWBQ [VLDRWQGBWB_S VLDRWQGBWB_U])
 656 (define_int_iterator VSTRDSBWBQ [VSTRDQSBWB_S VSTRDQSBWB_U])
 657 (define_int_iterator VLDRDGBWBQ [VLDRDQGBWB_S VLDRDQGBWB_U])
 658 (define_int_iterator VADCIQ [VADCIQ_U VADCIQ_S])
 659 (define_int_iterator VADCIQ_M [VADCIQ_M_U VADCIQ_M_S])
 660 (define_int_iterator VSBCQ [VSBCQ_U VSBCQ_S])
 661 (define_int_iterator VSBCQ_M [VSBCQ_M_U VSBCQ_M_S])
 662 (define_int_iterator VSBCIQ [VSBCIQ_U VSBCIQ_S])
 663 (define_int_iterator VSBCIQ_M [VSBCIQ_M_U VSBCIQ_M_S])
 664 (define_int_iterator VADCQ [VADCQ_U VADCQ_S])
 665 (define_int_iterator VADCQ_M [VADCQ_M_U VADCQ_M_S])
 666 (define_int_iterator UQRSHLLQ [UQRSHLL_64 UQRSHLL_48])
 667 (define_int_iterator SQRSHRLQ [SQRSHRL_64 SQRSHRL_48])
 668 (define_int_iterator VSHLCQ_M [VSHLCQ_M_S VSHLCQ_M_U])
 669
 670 (define_insn "*mve_mov<mode>"
 671   [(set (match_operand:MVE_types 0 "nonimmediate_operand" "=w,w,r,w,w,r,w,Us")
 672         (match_operand:MVE_types 1 "general_operand" "w,r,w,Dn,Usi,r,Dm,w"))]
 673   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 674 {
 675   if (which_alternative == 3 || which_alternative == 6)
 676     {
 677       int width, is_valid;
 678       static char templ[40];
 679
 680       is_valid = simd_immediate_valid_for_move (operands[1], <MODE>mode,
 681         &operands[1], &width);
 682
 683       gcc_assert (is_valid != 0);
 684
 685       if (width == 0)
 686         return "vmov.f32\t%q0, %1  @ <mode>";
 687       else
 688         sprintf (templ, "vmov.i%d\t%%q0, %%x1  @ <mode>", width);
 689       return templ;
 690     }
 691   switch (which_alternative)
 692     {
 693     case 0:
 694       return "vmov\t%q0, %q1";
 695     case 1:
 696       return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 697     case 2:
 698       return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 699     case 4:
 700       if (MEM_P (operands[1])
 701           && (GET_CODE (XEXP (operands[1], 0)) == LABEL_REF
 702               || GET_CODE (XEXP (operands[1], 0)) == CONST))
 703         return output_move_neon (operands);
 704       else
 705         return "vldrb.8 %q0, %E1";
 706     case 5:
 707       return output_move_quad (operands);
 708     case 7:
 709       return "vstrb.8 %q1, %E0";
 710     default:
 711       gcc_unreachable ();
 712       return "";
 713     }
 714 }
 715   [(set_attr "type" "mve_move,mve_move,mve_move,mve_move,mve_load,multiple,mve_move,mve_store")
 716    (set_attr "length" "4,8,8,4,8,8,4,4")
 717    (set_attr "thumb2_pool_range" "*,*,*,*,1018,*,*,*")
 718    (set_attr "neg_pool_range" "*,*,*,*,996,*,*,*")])
 719
 720 (define_insn "*mve_mov<mode>"
 721   [(set (match_operand:MVE_types 0 "s_register_operand" "=w,w")
 722         (vec_duplicate:MVE_types
 723           (match_operand:SI 1 "nonmemory_operand" "r,i")))]
 724   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
 725 {
 726   if (which_alternative == 0)
 727     return "vdup.<V_sz_elem>\t%q0, %1";
 728   return "vmov.<V_sz_elem>\t%q0, %1";
 729 }
 730   [(set_attr "length" "4,4")
 731    (set_attr "type" "mve_move,mve_move")])
 732
 733 ;;
 734 ;; [vst4q])
 735 ;;
 736 (define_insn "mve_vst4q<mode>"
 737   [(set (match_operand:XI 0 "neon_struct_operand" "=Um")
 738         (unspec:XI [(match_operand:XI 1 "s_register_operand" "w")
 739                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
 740          VST4Q))
 741   ]
 742   "TARGET_HAVE_MVE"
 743 {
 744    rtx ops[6];
 745    int regno = REGNO (operands[1]);
 746    ops[0] = gen_rtx_REG (TImode, regno);
 747    ops[1] = gen_rtx_REG (TImode, regno+4);
 748    ops[2] = gen_rtx_REG (TImode, regno+8);
 749    ops[3] = gen_rtx_REG (TImode, regno+12);
 750    rtx reg  = operands[0];
 751    while (reg && !REG_P (reg))
 752     reg = XEXP (reg, 0);
 753    gcc_assert (REG_P (reg));
 754    ops[4] = reg;
 755    ops[5] = operands[0];
 756    /* Here in first three instructions data is stored to ops[4]'s location but
 757       in the fourth instruction data is stored to operands[0], this is to
 758       support the writeback.  */
 759    output_asm_insn ("vst40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 760                     "vst41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 761                     "vst42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
 762                     "vst43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
 763    return "";
 764 }
 765   [(set_attr "length" "16")])
 766
 767 ;;
 768 ;; [vrndq_m_f])
 769 ;;
 770 (define_insn "mve_vrndq_m_f<mode>"
 771   [
 772    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 773         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
 774                        (match_operand:MVE_0 2 "s_register_operand" "w")
 775                        (match_operand:HI 3 "vpr_register_operand" "Up")]
 776          VRNDQ_M_F))
 777   ]
 778   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 779   "vpst\;vrintzt.f%#<V_sz_elem> %q0, %q2"
 780   [(set_attr "type" "mve_move")
 781    (set_attr "length""8")])
 782
 783 ;;
 784 ;; [vrndxq_f])
 785 ;;
 786 (define_insn "mve_vrndxq_f<mode>"
 787   [
 788    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 789         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 790          VRNDXQ_F))
 791   ]
 792   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 793   "vrintx.f%#<V_sz_elem>        %q0, %q1"
 794   [(set_attr "type" "mve_move")
 795 ])
 796
 797 ;;
 798 ;; [vrndq_f])
 799 ;;
 800 (define_insn "mve_vrndq_f<mode>"
 801   [
 802    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 803         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 804          VRNDQ_F))
 805   ]
 806   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 807   "vrintz.f%#<V_sz_elem>        %q0, %q1"
 808   [(set_attr "type" "mve_move")
 809 ])
 810
 811 ;;
 812 ;; [vrndpq_f])
 813 ;;
 814 (define_insn "mve_vrndpq_f<mode>"
 815   [
 816    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 817         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 818          VRNDPQ_F))
 819   ]
 820   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 821   "vrintp.f%#<V_sz_elem>        %q0, %q1"
 822   [(set_attr "type" "mve_move")
 823 ])
 824
 825 ;;
 826 ;; [vrndnq_f])
 827 ;;
 828 (define_insn "mve_vrndnq_f<mode>"
 829   [
 830    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 831         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 832          VRNDNQ_F))
 833   ]
 834   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 835   "vrintn.f%#<V_sz_elem>        %q0, %q1"
 836   [(set_attr "type" "mve_move")
 837 ])
 838
 839 ;;
 840 ;; [vrndmq_f])
 841 ;;
 842 (define_insn "mve_vrndmq_f<mode>"
 843   [
 844    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 845         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 846          VRNDMQ_F))
 847   ]
 848   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 849   "vrintm.f%#<V_sz_elem>        %q0, %q1"
 850   [(set_attr "type" "mve_move")
 851 ])
 852
 853 ;;
 854 ;; [vrndaq_f])
 855 ;;
 856 (define_insn "mve_vrndaq_f<mode>"
 857   [
 858    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 859         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 860          VRNDAQ_F))
 861   ]
 862   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 863   "vrinta.f%#<V_sz_elem>        %q0, %q1"
 864   [(set_attr "type" "mve_move")
 865 ])
 866
 867 ;;
 868 ;; [vrev64q_f])
 869 ;;
 870 (define_insn "mve_vrev64q_f<mode>"
 871   [
 872    (set (match_operand:MVE_0 0 "s_register_operand" "=&w")
 873         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 874          VREV64Q_F))
 875   ]
 876   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 877   "vrev64.%#<V_sz_elem> %q0, %q1"
 878   [(set_attr "type" "mve_move")
 879 ])
 880
 881 ;;
 882 ;; [vnegq_f])
 883 ;;
 884 (define_insn "mve_vnegq_f<mode>"
 885   [
 886    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 887         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 888          VNEGQ_F))
 889   ]
 890   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 891   "vneg.f%#<V_sz_elem>  %q0, %q1"
 892   [(set_attr "type" "mve_move")
 893 ])
 894
 895 ;;
 896 ;; [vdupq_n_f])
 897 ;;
 898 (define_insn "mve_vdupq_n_f<mode>"
 899   [
 900    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 901         (unspec:MVE_0 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
 902          VDUPQ_N_F))
 903   ]
 904   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 905   "vdup.%#<V_sz_elem>   %q0, %1"
 906   [(set_attr "type" "mve_move")
 907 ])
 908
 909 ;;
 910 ;; [vabsq_f])
 911 ;;
 912 (define_insn "mve_vabsq_f<mode>"
 913   [
 914    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 915         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")]
 916          VABSQ_F))
 917   ]
 918   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 919   "vabs.f%#<V_sz_elem>  %q0, %q1"
 920   [(set_attr "type" "mve_move")
 921 ])
 922
 923 ;;
 924 ;; [vrev32q_f])
 925 ;;
 926 (define_insn "mve_vrev32q_fv8hf"
 927   [
 928    (set (match_operand:V8HF 0 "s_register_operand" "=w")
 929         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "w")]
 930          VREV32Q_F))
 931   ]
 932   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 933   "vrev32.16 %q0, %q1"
 934   [(set_attr "type" "mve_move")
 935 ])
 936 ;;
 937 ;; [vcvttq_f32_f16])
 938 ;;
 939 (define_insn "mve_vcvttq_f32_f16v4sf"
 940   [
 941    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 942         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 943          VCVTTQ_F32_F16))
 944   ]
 945   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 946   "vcvtt.f32.f16 %q0, %q1"
 947   [(set_attr "type" "mve_move")
 948 ])
 949
 950 ;;
 951 ;; [vcvtbq_f32_f16])
 952 ;;
 953 (define_insn "mve_vcvtbq_f32_f16v4sf"
 954   [
 955    (set (match_operand:V4SF 0 "s_register_operand" "=w")
 956         (unspec:V4SF [(match_operand:V8HF 1 "s_register_operand" "w")]
 957          VCVTBQ_F32_F16))
 958   ]
 959   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 960   "vcvtb.f32.f16 %q0, %q1"
 961   [(set_attr "type" "mve_move")
 962 ])
 963
 964 ;;
 965 ;; [vcvtq_to_f_s, vcvtq_to_f_u])
 966 ;;
 967 (define_insn "mve_vcvtq_to_f_<supf><mode>"
 968   [
 969    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
 970         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 971          VCVTQ_TO_F))
 972   ]
 973   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
 974   "vcvt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>       %q0, %q1"
 975   [(set_attr "type" "mve_move")
 976 ])
 977
 978 ;;
 979 ;; [vrev64q_u, vrev64q_s])
 980 ;;
 981 (define_insn "mve_vrev64q_<supf><mode>"
 982   [
 983    (set (match_operand:MVE_2 0 "s_register_operand" "=&w")
 984         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
 985          VREV64Q))
 986   ]
 987   "TARGET_HAVE_MVE"
 988   "vrev64.%#<V_sz_elem> %q0, %q1"
 989   [(set_attr "type" "mve_move")
 990 ])
 991
 992 ;;
 993 ;; [vcvtq_from_f_s, vcvtq_from_f_u])
 994 ;;
 995 (define_insn "mve_vcvtq_from_f_<supf><mode>"
 996   [
 997    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
 998         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
 999          VCVTQ_FROM_F))
1000   ]
1001   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1002   "vcvt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>       %q0, %q1"
1003   [(set_attr "type" "mve_move")
1004 ])
1005 ;; [vqnegq_s])
1006 ;;
1007 (define_insn "mve_vqnegq_s<mode>"
1008   [
1009    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1010         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1011          VQNEGQ_S))
1012   ]
1013   "TARGET_HAVE_MVE"
1014   "vqneg.s%#<V_sz_elem> %q0, %q1"
1015   [(set_attr "type" "mve_move")
1016 ])
1017
1018 ;;
1019 ;; [vqabsq_s])
1020 ;;
1021 (define_insn "mve_vqabsq_s<mode>"
1022   [
1023    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1024         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1025          VQABSQ_S))
1026   ]
1027   "TARGET_HAVE_MVE"
1028   "vqabs.s%#<V_sz_elem> %q0, %q1"
1029   [(set_attr "type" "mve_move")
1030 ])
1031
1032 ;;
1033 ;; [vnegq_s])
1034 ;;
1035 (define_insn "mve_vnegq_s<mode>"
1036   [
1037    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1038         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1039          VNEGQ_S))
1040   ]
1041   "TARGET_HAVE_MVE"
1042   "vneg.s%#<V_sz_elem>  %q0, %q1"
1043   [(set_attr "type" "mve_move")
1044 ])
1045
1046 ;;
1047 ;; [vmvnq_u, vmvnq_s])
1048 ;;
1049 (define_insn "mve_vmvnq_<supf><mode>"
1050   [
1051    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1052         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1053          VMVNQ))
1054   ]
1055   "TARGET_HAVE_MVE"
1056   "vmvn %q0, %q1"
1057   [(set_attr "type" "mve_move")
1058 ])
1059
1060 ;;
1061 ;; [vdupq_n_u, vdupq_n_s])
1062 ;;
1063 (define_insn "mve_vdupq_n_<supf><mode>"
1064   [
1065    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1066         (unspec:MVE_2 [(match_operand:<V_elem> 1 "s_register_operand" "r")]
1067          VDUPQ_N))
1068   ]
1069   "TARGET_HAVE_MVE"
1070   "vdup.%#<V_sz_elem>   %q0, %1"
1071   [(set_attr "type" "mve_move")
1072 ])
1073
1074 ;;
1075 ;; [vclzq_u, vclzq_s])
1076 ;;
1077 (define_insn "mve_vclzq_<supf><mode>"
1078   [
1079    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1080         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1081          VCLZQ))
1082   ]
1083   "TARGET_HAVE_MVE"
1084   "vclz.i%#<V_sz_elem>  %q0, %q1"
1085   [(set_attr "type" "mve_move")
1086 ])
1087
1088 ;;
1089 ;; [vclsq_s])
1090 ;;
1091 (define_insn "mve_vclsq_s<mode>"
1092   [
1093    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1094         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1095          VCLSQ_S))
1096   ]
1097   "TARGET_HAVE_MVE"
1098   "vcls.s%#<V_sz_elem>  %q0, %q1"
1099   [(set_attr "type" "mve_move")
1100 ])
1101
1102 ;;
1103 ;; [vaddvq_s, vaddvq_u])
1104 ;;
1105 (define_insn "mve_vaddvq_<supf><mode>"
1106   [
1107    (set (match_operand:SI 0 "s_register_operand" "=e")
1108         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")]
1109          VADDVQ))
1110   ]
1111   "TARGET_HAVE_MVE"
1112   "vaddv.<supf>%#<V_sz_elem>\t%0, %q1"
1113   [(set_attr "type" "mve_move")
1114 ])
1115
1116 ;;
1117 ;; [vabsq_s])
1118 ;;
1119 (define_insn "mve_vabsq_s<mode>"
1120   [
1121    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1122         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")]
1123          VABSQ_S))
1124   ]
1125   "TARGET_HAVE_MVE"
1126   "vabs.s%#<V_sz_elem>\t%q0, %q1"
1127   [(set_attr "type" "mve_move")
1128 ])
1129
1130 ;;
1131 ;; [vrev32q_u, vrev32q_s])
1132 ;;
1133 (define_insn "mve_vrev32q_<supf><mode>"
1134   [
1135    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
1136         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "w")]
1137          VREV32Q))
1138   ]
1139   "TARGET_HAVE_MVE"
1140   "vrev32.%#<V_sz_elem>\t%q0, %q1"
1141   [(set_attr "type" "mve_move")
1142 ])
1143
1144 ;;
1145 ;; [vmovltq_u, vmovltq_s])
1146 ;;
1147 (define_insn "mve_vmovltq_<supf><mode>"
1148   [
1149    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1150         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1151          VMOVLTQ))
1152   ]
1153   "TARGET_HAVE_MVE"
1154   "vmovlt.<supf>%#<V_sz_elem>   %q0, %q1"
1155   [(set_attr "type" "mve_move")
1156 ])
1157
1158 ;;
1159 ;; [vmovlbq_s, vmovlbq_u])
1160 ;;
1161 (define_insn "mve_vmovlbq_<supf><mode>"
1162   [
1163    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
1164         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")]
1165          VMOVLBQ))
1166   ]
1167   "TARGET_HAVE_MVE"
1168   "vmovlb.<supf>%#<V_sz_elem>   %q0, %q1"
1169   [(set_attr "type" "mve_move")
1170 ])
1171
1172 ;;
1173 ;; [vcvtpq_s, vcvtpq_u])
1174 ;;
1175 (define_insn "mve_vcvtpq_<supf><mode>"
1176   [
1177    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1178         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1179          VCVTPQ))
1180   ]
1181   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1182   "vcvtp.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1183   [(set_attr "type" "mve_move")
1184 ])
1185
1186 ;;
1187 ;; [vcvtnq_s, vcvtnq_u])
1188 ;;
1189 (define_insn "mve_vcvtnq_<supf><mode>"
1190   [
1191    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1192         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1193          VCVTNQ))
1194   ]
1195   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1196   "vcvtn.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1197   [(set_attr "type" "mve_move")
1198 ])
1199
1200 ;;
1201 ;; [vcvtmq_s, vcvtmq_u])
1202 ;;
1203 (define_insn "mve_vcvtmq_<supf><mode>"
1204   [
1205    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1206         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1207          VCVTMQ))
1208   ]
1209   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1210   "vcvtm.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1211   [(set_attr "type" "mve_move")
1212 ])
1213
1214 ;;
1215 ;; [vcvtaq_u, vcvtaq_s])
1216 ;;
1217 (define_insn "mve_vcvtaq_<supf><mode>"
1218   [
1219    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1220         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")]
1221          VCVTAQ))
1222   ]
1223   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1224   "vcvta.<supf>%#<V_sz_elem>.f%#<V_sz_elem>      %q0, %q1"
1225   [(set_attr "type" "mve_move")
1226 ])
1227
1228 ;;
1229 ;; [vmvnq_n_u, vmvnq_n_s])
1230 ;;
1231 (define_insn "mve_vmvnq_n_<supf><mode>"
1232   [
1233    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1234         (unspec:MVE_5 [(match_operand:HI 1 "immediate_operand" "i")]
1235          VMVNQ_N))
1236   ]
1237   "TARGET_HAVE_MVE"
1238   "vmvn.i%#<V_sz_elem>  %q0, %1"
1239   [(set_attr "type" "mve_move")
1240 ])
1241
1242 ;;
1243 ;; [vrev16q_u, vrev16q_s])
1244 ;;
1245 (define_insn "mve_vrev16q_<supf>v16qi"
1246   [
1247    (set (match_operand:V16QI 0 "s_register_operand" "=w")
1248         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")]
1249          VREV16Q))
1250   ]
1251   "TARGET_HAVE_MVE"
1252   "vrev16.8 %q0, %q1"
1253   [(set_attr "type" "mve_move")
1254 ])
1255
1256 ;;
1257 ;; [vaddlvq_s vaddlvq_u])
1258 ;;
1259 (define_insn "mve_vaddlvq_<supf>v4si"
1260   [
1261    (set (match_operand:DI 0 "s_register_operand" "=r")
1262         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")]
1263          VADDLVQ))
1264   ]
1265   "TARGET_HAVE_MVE"
1266   "vaddlv.<supf>32 %Q0, %R0, %q1"
1267   [(set_attr "type" "mve_move")
1268 ])
1269
1270 ;;
1271 ;; [vctp8q vctp16q vctp32q vctp64q])
1272 ;;
1273 (define_insn "mve_vctp<mode1>qhi"
1274   [
1275    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1276         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")]
1277         VCTPQ))
1278   ]
1279   "TARGET_HAVE_MVE"
1280   "vctp.<mode1> %1"
1281   [(set_attr "type" "mve_move")
1282 ])
1283
1284 ;;
1285 ;; [vpnot])
1286 ;;
1287 (define_insn "mve_vpnothi"
1288   [
1289    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1290         (unspec:HI [(match_operand:HI 1 "vpr_register_operand" "0")]
1291          VPNOT))
1292   ]
1293   "TARGET_HAVE_MVE"
1294   "vpnot"
1295   [(set_attr "type" "mve_move")
1296 ])
1297
1298 ;;
1299 ;; [vsubq_n_f])
1300 ;;
1301 (define_insn "mve_vsubq_n_f<mode>"
1302   [
1303    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1304         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1305                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1306          VSUBQ_N_F))
1307   ]
1308   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1309   "vsub.f<V_sz_elem>  %q0, %q1, %2"
1310   [(set_attr "type" "mve_move")
1311 ])
1312
1313 ;;
1314 ;; [vbrsrq_n_f])
1315 ;;
1316 (define_insn "mve_vbrsrq_n_f<mode>"
1317   [
1318    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1319         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
1320                        (match_operand:SI 2 "s_register_operand" "r")]
1321          VBRSRQ_N_F))
1322   ]
1323   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1324   "vbrsr.<V_sz_elem>  %q0, %q1, %2"
1325   [(set_attr "type" "mve_move")
1326 ])
1327
1328 ;;
1329 ;; [vcvtq_n_to_f_s, vcvtq_n_to_f_u])
1330 ;;
1331 (define_insn "mve_vcvtq_n_to_f_<supf><mode>"
1332   [
1333    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1334         (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1335                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1336          VCVTQ_N_TO_F))
1337   ]
1338   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1339   "vcvt.f<V_sz_elem>.<supf><V_sz_elem>\t%q0, %q1, %2"
1340   [(set_attr "type" "mve_move")
1341 ])
1342
1343 ;; [vcreateq_f])
1344 ;;
1345 (define_insn "mve_vcreateq_f<mode>"
1346   [
1347    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
1348         (unspec:MVE_0 [(match_operand:DI 1 "s_register_operand" "r")
1349                        (match_operand:DI 2 "s_register_operand" "r")]
1350          VCREATEQ_F))
1351   ]
1352   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1353   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1354   [(set_attr "type" "mve_move")
1355    (set_attr "length""8")])
1356
1357 ;;
1358 ;; [vcreateq_u, vcreateq_s])
1359 ;;
1360 (define_insn "mve_vcreateq_<supf><mode>"
1361   [
1362    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
1363         (unspec:MVE_1 [(match_operand:DI 1 "s_register_operand" "r")
1364                        (match_operand:DI 2 "s_register_operand" "r")]
1365          VCREATEQ))
1366   ]
1367   "TARGET_HAVE_MVE"
1368   "vmov %q0[2], %q0[0], %Q2, %Q1\;vmov %q0[3], %q0[1], %R2, %R1"
1369   [(set_attr "type" "mve_move")
1370    (set_attr "length""8")])
1371
1372 ;;
1373 ;; [vshrq_n_s, vshrq_n_u])
1374 ;;
1375 (define_insn "mve_vshrq_n_<supf><mode>"
1376   [
1377    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1378         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1379                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1380          VSHRQ_N))
1381   ]
1382   "TARGET_HAVE_MVE"
1383   "vshr.<supf><V_sz_elem>\t%q0, %q1, %2"
1384   [(set_attr "type" "mve_move")
1385 ])
1386
1387 ;;
1388 ;; [vcvtq_n_from_f_s, vcvtq_n_from_f_u])
1389 ;;
1390 (define_insn "mve_vcvtq_n_from_f_<supf><mode>"
1391   [
1392    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
1393         (unspec:MVE_5 [(match_operand:<MVE_CNVT> 1 "s_register_operand" "w")
1394                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
1395          VCVTQ_N_FROM_F))
1396   ]
1397   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
1398   "vcvt.<supf><V_sz_elem>.f<V_sz_elem>\t%q0, %q1, %2"
1399   [(set_attr "type" "mve_move")
1400 ])
1401
1402 ;;
1403 ;; [vaddlvq_p_s])
1404 ;;
1405 (define_insn "mve_vaddlvq_p_<supf>v4si"
1406   [
1407    (set (match_operand:DI 0 "s_register_operand" "=r")
1408         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
1409                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1410          VADDLVQ_P))
1411   ]
1412   "TARGET_HAVE_MVE"
1413   "vpst\;vaddlvt.<supf>32 %Q0, %R0, %q1"
1414   [(set_attr "type" "mve_move")
1415    (set_attr "length""8")])
1416
1417 ;;
1418 ;; [vcmpneq_u, vcmpneq_s])
1419 ;;
1420 (define_insn "mve_vcmpneq_<supf><mode>"
1421   [
1422    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1423         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1424                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1425          VCMPNEQ))
1426   ]
1427   "TARGET_HAVE_MVE"
1428   "vcmp.i%#<V_sz_elem>  ne, %q1, %q2"
1429   [(set_attr "type" "mve_move")
1430 ])
1431
1432 ;;
1433 ;; [vshlq_s, vshlq_u])
1434 ;;
1435 (define_insn "mve_vshlq_<supf><mode>"
1436   [
1437    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1438         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1439                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1440          VSHLQ))
1441   ]
1442   "TARGET_HAVE_MVE"
1443   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1444   [(set_attr "type" "mve_move")
1445 ])
1446
1447 ;;
1448 ;; [vabdq_s, vabdq_u])
1449 ;;
1450 (define_insn "mve_vabdq_<supf><mode>"
1451   [
1452    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1453         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1454                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1455          VABDQ))
1456   ]
1457   "TARGET_HAVE_MVE"
1458   "vabd.<supf>%#<V_sz_elem>     %q0, %q1, %q2"
1459   [(set_attr "type" "mve_move")
1460 ])
1461
1462 ;;
1463 ;; [vaddq_n_s, vaddq_n_u])
1464 ;;
1465 (define_insn "mve_vaddq_n_<supf><mode>"
1466   [
1467    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1468         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1469                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1470          VADDQ_N))
1471   ]
1472   "TARGET_HAVE_MVE"
1473   "vadd.i%#<V_sz_elem>  %q0, %q1, %2"
1474   [(set_attr "type" "mve_move")
1475 ])
1476
1477 ;;
1478 ;; [vaddvaq_s, vaddvaq_u])
1479 ;;
1480 (define_insn "mve_vaddvaq_<supf><mode>"
1481   [
1482    (set (match_operand:SI 0 "s_register_operand" "=e")
1483         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
1484                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1485          VADDVAQ))
1486   ]
1487   "TARGET_HAVE_MVE"
1488   "vaddva.<supf>%#<V_sz_elem>   %0, %q2"
1489   [(set_attr "type" "mve_move")
1490 ])
1491
1492 ;;
1493 ;; [vaddvq_p_u, vaddvq_p_s])
1494 ;;
1495 (define_insn "mve_vaddvq_p_<supf><mode>"
1496   [
1497    (set (match_operand:SI 0 "s_register_operand" "=e")
1498         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
1499                     (match_operand:HI 2 "vpr_register_operand" "Up")]
1500          VADDVQ_P))
1501   ]
1502   "TARGET_HAVE_MVE"
1503   "vpst\;vaddvt.<supf>%#<V_sz_elem>     %0, %q1"
1504   [(set_attr "type" "mve_move")
1505    (set_attr "length""8")])
1506
1507 ;;
1508 ;; [vandq_u, vandq_s])
1509 ;;
1510 (define_insn "mve_vandq_<supf><mode>"
1511   [
1512    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1513         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1514                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1515          VANDQ))
1516   ]
1517   "TARGET_HAVE_MVE"
1518   "vand %q0, %q1, %q2"
1519   [(set_attr "type" "mve_move")
1520 ])
1521
1522 ;;
1523 ;; [vbicq_s, vbicq_u])
1524 ;;
1525 (define_insn "mve_vbicq_<supf><mode>"
1526   [
1527    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1528         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1529                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1530          VBICQ))
1531   ]
1532   "TARGET_HAVE_MVE"
1533   "vbic %q0, %q1, %q2"
1534   [(set_attr "type" "mve_move")
1535 ])
1536
1537 ;;
1538 ;; [vbrsrq_n_u, vbrsrq_n_s])
1539 ;;
1540 (define_insn "mve_vbrsrq_n_<supf><mode>"
1541   [
1542    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1543         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1544                        (match_operand:SI 2 "s_register_operand" "r")]
1545          VBRSRQ_N))
1546   ]
1547   "TARGET_HAVE_MVE"
1548   "vbrsr.%#<V_sz_elem>  %q0, %q1, %2"
1549   [(set_attr "type" "mve_move")
1550 ])
1551
1552 ;;
1553 ;; [vcaddq_rot270_s, vcaddq_rot270_u])
1554 ;;
1555 (define_insn "mve_vcaddq_rot270_<supf><mode>"
1556   [
1557    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1558         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1559                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1560          VCADDQ_ROT270))
1561   ]
1562   "TARGET_HAVE_MVE"
1563   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #270"
1564   [(set_attr "type" "mve_move")
1565 ])
1566
1567 ;;
1568 ;; [vcaddq_rot90_u, vcaddq_rot90_s])
1569 ;;
1570 (define_insn "mve_vcaddq_rot90_<supf><mode>"
1571   [
1572    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1573         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1574                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1575          VCADDQ_ROT90))
1576   ]
1577   "TARGET_HAVE_MVE"
1578   "vcadd.i%#<V_sz_elem> %q0, %q1, %q2, #90"
1579   [(set_attr "type" "mve_move")
1580 ])
1581
1582 ;;
1583 ;; [vcmpcsq_n_u])
1584 ;;
1585 (define_insn "mve_vcmpcsq_n_u<mode>"
1586   [
1587    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1588         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1589                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1590          VCMPCSQ_N_U))
1591   ]
1592   "TARGET_HAVE_MVE"
1593   "vcmp.u%#<V_sz_elem>  cs, %q1, %2"
1594   [(set_attr "type" "mve_move")
1595 ])
1596
1597 ;;
1598 ;; [vcmpcsq_u])
1599 ;;
1600 (define_insn "mve_vcmpcsq_u<mode>"
1601   [
1602    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1603         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1604                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1605          VCMPCSQ_U))
1606   ]
1607   "TARGET_HAVE_MVE"
1608   "vcmp.u%#<V_sz_elem>  cs, %q1, %q2"
1609   [(set_attr "type" "mve_move")
1610 ])
1611
1612 ;;
1613 ;; [vcmpeqq_n_s, vcmpeqq_n_u])
1614 ;;
1615 (define_insn "mve_vcmpeqq_n_<supf><mode>"
1616   [
1617    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1618         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1619                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1620          VCMPEQQ_N))
1621   ]
1622   "TARGET_HAVE_MVE"
1623   "vcmp.i%#<V_sz_elem>  eq, %q1, %2"
1624   [(set_attr "type" "mve_move")
1625 ])
1626
1627 ;;
1628 ;; [vcmpeqq_u, vcmpeqq_s])
1629 ;;
1630 (define_insn "mve_vcmpeqq_<supf><mode>"
1631   [
1632    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1633         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1634                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1635          VCMPEQQ))
1636   ]
1637   "TARGET_HAVE_MVE"
1638   "vcmp.i%#<V_sz_elem>  eq, %q1, %q2"
1639   [(set_attr "type" "mve_move")
1640 ])
1641
1642 ;;
1643 ;; [vcmpgeq_n_s])
1644 ;;
1645 (define_insn "mve_vcmpgeq_n_s<mode>"
1646   [
1647    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1648         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1649                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1650          VCMPGEQ_N_S))
1651   ]
1652   "TARGET_HAVE_MVE"
1653   "vcmp.s%#<V_sz_elem>  ge, %q1, %2"
1654   [(set_attr "type" "mve_move")
1655 ])
1656
1657 ;;
1658 ;; [vcmpgeq_s])
1659 ;;
1660 (define_insn "mve_vcmpgeq_s<mode>"
1661   [
1662    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1663         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1664                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1665          VCMPGEQ_S))
1666   ]
1667   "TARGET_HAVE_MVE"
1668   "vcmp.s%#<V_sz_elem>  ge, %q1, %q2"
1669   [(set_attr "type" "mve_move")
1670 ])
1671
1672 ;;
1673 ;; [vcmpgtq_n_s])
1674 ;;
1675 (define_insn "mve_vcmpgtq_n_s<mode>"
1676   [
1677    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1678         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1679                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1680          VCMPGTQ_N_S))
1681   ]
1682   "TARGET_HAVE_MVE"
1683   "vcmp.s%#<V_sz_elem>  gt, %q1, %2"
1684   [(set_attr "type" "mve_move")
1685 ])
1686
1687 ;;
1688 ;; [vcmpgtq_s])
1689 ;;
1690 (define_insn "mve_vcmpgtq_s<mode>"
1691   [
1692    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1693         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1694                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1695          VCMPGTQ_S))
1696   ]
1697   "TARGET_HAVE_MVE"
1698   "vcmp.s%#<V_sz_elem>  gt, %q1, %q2"
1699   [(set_attr "type" "mve_move")
1700 ])
1701
1702 ;;
1703 ;; [vcmphiq_n_u])
1704 ;;
1705 (define_insn "mve_vcmphiq_n_u<mode>"
1706   [
1707    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1708         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1709                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1710          VCMPHIQ_N_U))
1711   ]
1712   "TARGET_HAVE_MVE"
1713   "vcmp.u%#<V_sz_elem>  hi, %q1, %2"
1714   [(set_attr "type" "mve_move")
1715 ])
1716
1717 ;;
1718 ;; [vcmphiq_u])
1719 ;;
1720 (define_insn "mve_vcmphiq_u<mode>"
1721   [
1722    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1723         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1724                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1725          VCMPHIQ_U))
1726   ]
1727   "TARGET_HAVE_MVE"
1728   "vcmp.u%#<V_sz_elem>  hi, %q1, %q2"
1729   [(set_attr "type" "mve_move")
1730 ])
1731
1732 ;;
1733 ;; [vcmpleq_n_s])
1734 ;;
1735 (define_insn "mve_vcmpleq_n_s<mode>"
1736   [
1737    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1738         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1739                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1740          VCMPLEQ_N_S))
1741   ]
1742   "TARGET_HAVE_MVE"
1743   "vcmp.s%#<V_sz_elem>  le, %q1, %2"
1744   [(set_attr "type" "mve_move")
1745 ])
1746
1747 ;;
1748 ;; [vcmpleq_s])
1749 ;;
1750 (define_insn "mve_vcmpleq_s<mode>"
1751   [
1752    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1753         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1754                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1755          VCMPLEQ_S))
1756   ]
1757   "TARGET_HAVE_MVE"
1758   "vcmp.s%#<V_sz_elem>  le, %q1, %q2"
1759   [(set_attr "type" "mve_move")
1760 ])
1761
1762 ;;
1763 ;; [vcmpltq_n_s])
1764 ;;
1765 (define_insn "mve_vcmpltq_n_s<mode>"
1766   [
1767    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1768         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1769                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1770          VCMPLTQ_N_S))
1771   ]
1772   "TARGET_HAVE_MVE"
1773   "vcmp.s%#<V_sz_elem>  lt, %q1, %2"
1774   [(set_attr "type" "mve_move")
1775 ])
1776
1777 ;;
1778 ;; [vcmpltq_s])
1779 ;;
1780 (define_insn "mve_vcmpltq_s<mode>"
1781   [
1782    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1783         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1784                     (match_operand:MVE_2 2 "s_register_operand" "w")]
1785          VCMPLTQ_S))
1786   ]
1787   "TARGET_HAVE_MVE"
1788   "vcmp.s%#<V_sz_elem>  lt, %q1, %q2"
1789   [(set_attr "type" "mve_move")
1790 ])
1791
1792 ;;
1793 ;; [vcmpneq_n_u, vcmpneq_n_s])
1794 ;;
1795 (define_insn "mve_vcmpneq_n_<supf><mode>"
1796   [
1797    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
1798         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
1799                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
1800          VCMPNEQ_N))
1801   ]
1802   "TARGET_HAVE_MVE"
1803   "vcmp.i%#<V_sz_elem>  ne, %q1, %2"
1804   [(set_attr "type" "mve_move")
1805 ])
1806
1807 ;;
1808 ;; [veorq_u, veorq_s])
1809 ;;
1810 (define_insn "mve_veorq_<supf><mode>"
1811   [
1812    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1813         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1814                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1815          VEORQ))
1816   ]
1817   "TARGET_HAVE_MVE"
1818   "veor %q0, %q1, %q2"
1819   [(set_attr "type" "mve_move")
1820 ])
1821
1822 ;;
1823 ;; [vhaddq_n_u, vhaddq_n_s])
1824 ;;
1825 (define_insn "mve_vhaddq_n_<supf><mode>"
1826   [
1827    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1828         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1829                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1830          VHADDQ_N))
1831   ]
1832   "TARGET_HAVE_MVE"
1833   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1834   [(set_attr "type" "mve_move")
1835 ])
1836
1837 ;;
1838 ;; [vhaddq_s, vhaddq_u])
1839 ;;
1840 (define_insn "mve_vhaddq_<supf><mode>"
1841   [
1842    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1843         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1844                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1845          VHADDQ))
1846   ]
1847   "TARGET_HAVE_MVE"
1848   "vhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1849   [(set_attr "type" "mve_move")
1850 ])
1851
1852 ;;
1853 ;; [vhcaddq_rot270_s])
1854 ;;
1855 (define_insn "mve_vhcaddq_rot270_s<mode>"
1856   [
1857    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1858         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1859                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1860          VHCADDQ_ROT270_S))
1861   ]
1862   "TARGET_HAVE_MVE"
1863   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #270"
1864   [(set_attr "type" "mve_move")
1865 ])
1866
1867 ;;
1868 ;; [vhcaddq_rot90_s])
1869 ;;
1870 (define_insn "mve_vhcaddq_rot90_s<mode>"
1871   [
1872    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
1873         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1874                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1875          VHCADDQ_ROT90_S))
1876   ]
1877   "TARGET_HAVE_MVE"
1878   "vhcadd.s%#<V_sz_elem>\t%q0, %q1, %q2, #90"
1879   [(set_attr "type" "mve_move")
1880 ])
1881
1882 ;;
1883 ;; [vhsubq_n_u, vhsubq_n_s])
1884 ;;
1885 (define_insn "mve_vhsubq_n_<supf><mode>"
1886   [
1887    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1888         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1889                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
1890          VHSUBQ_N))
1891   ]
1892   "TARGET_HAVE_MVE"
1893   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
1894   [(set_attr "type" "mve_move")
1895 ])
1896
1897 ;;
1898 ;; [vhsubq_s, vhsubq_u])
1899 ;;
1900 (define_insn "mve_vhsubq_<supf><mode>"
1901   [
1902    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1903         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1904                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1905          VHSUBQ))
1906   ]
1907   "TARGET_HAVE_MVE"
1908   "vhsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1909   [(set_attr "type" "mve_move")
1910 ])
1911
1912 ;;
1913 ;; [vmaxaq_s])
1914 ;;
1915 (define_insn "mve_vmaxaq_s<mode>"
1916   [
1917    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1918         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1919                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1920          VMAXAQ_S))
1921   ]
1922   "TARGET_HAVE_MVE"
1923   "vmaxa.s%#<V_sz_elem> %q0, %q2"
1924   [(set_attr "type" "mve_move")
1925 ])
1926
1927 ;;
1928 ;; [vmaxavq_s])
1929 ;;
1930 (define_insn "mve_vmaxavq_s<mode>"
1931   [
1932    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1933         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1934                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1935          VMAXAVQ_S))
1936   ]
1937   "TARGET_HAVE_MVE"
1938   "vmaxav.s%#<V_sz_elem>\t%0, %q2"
1939   [(set_attr "type" "mve_move")
1940 ])
1941
1942 ;;
1943 ;; [vmaxq_u, vmaxq_s])
1944 ;;
1945 (define_insn "mve_vmaxq_<supf><mode>"
1946   [
1947    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1948         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
1949                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1950          VMAXQ))
1951   ]
1952   "TARGET_HAVE_MVE"
1953   "vmax.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
1954   [(set_attr "type" "mve_move")
1955 ])
1956
1957 ;;
1958 ;; [vmaxvq_u, vmaxvq_s])
1959 ;;
1960 (define_insn "mve_vmaxvq_<supf><mode>"
1961   [
1962    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1963         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1964                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1965          VMAXVQ))
1966   ]
1967   "TARGET_HAVE_MVE"
1968   "vmaxv.<supf>%#<V_sz_elem>\t%0, %q2"
1969   [(set_attr "type" "mve_move")
1970 ])
1971
1972 ;;
1973 ;; [vminaq_s])
1974 ;;
1975 (define_insn "mve_vminaq_s<mode>"
1976   [
1977    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
1978         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
1979                        (match_operand:MVE_2 2 "s_register_operand" "w")]
1980          VMINAQ_S))
1981   ]
1982   "TARGET_HAVE_MVE"
1983   "vmina.s%#<V_sz_elem>\t%q0, %q2"
1984   [(set_attr "type" "mve_move")
1985 ])
1986
1987 ;;
1988 ;; [vminavq_s])
1989 ;;
1990 (define_insn "mve_vminavq_s<mode>"
1991   [
1992    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
1993         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
1994                           (match_operand:MVE_2 2 "s_register_operand" "w")]
1995          VMINAVQ_S))
1996   ]
1997   "TARGET_HAVE_MVE"
1998   "vminav.s%#<V_sz_elem>\t%0, %q2"
1999   [(set_attr "type" "mve_move")
2000 ])
2001
2002 ;;
2003 ;; [vminq_s, vminq_u])
2004 ;;
2005 (define_insn "mve_vminq_<supf><mode>"
2006   [
2007    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2008         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2009                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2010          VMINQ))
2011   ]
2012   "TARGET_HAVE_MVE"
2013   "vmin.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2014   [(set_attr "type" "mve_move")
2015 ])
2016
2017 ;;
2018 ;; [vminvq_u, vminvq_s])
2019 ;;
2020 (define_insn "mve_vminvq_<supf><mode>"
2021   [
2022    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2023         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2024                           (match_operand:MVE_2 2 "s_register_operand" "w")]
2025          VMINVQ))
2026   ]
2027   "TARGET_HAVE_MVE"
2028   "vminv.<supf>%#<V_sz_elem>\t%0, %q2"
2029   [(set_attr "type" "mve_move")
2030 ])
2031
2032 ;;
2033 ;; [vmladavq_u, vmladavq_s])
2034 ;;
2035 (define_insn "mve_vmladavq_<supf><mode>"
2036   [
2037    (set (match_operand:SI 0 "s_register_operand" "=e")
2038         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2039                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2040          VMLADAVQ))
2041   ]
2042   "TARGET_HAVE_MVE"
2043   "vmladav.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
2044   [(set_attr "type" "mve_move")
2045 ])
2046
2047 ;;
2048 ;; [vmladavxq_s])
2049 ;;
2050 (define_insn "mve_vmladavxq_s<mode>"
2051   [
2052    (set (match_operand:SI 0 "s_register_operand" "=e")
2053         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2054                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2055          VMLADAVXQ_S))
2056   ]
2057   "TARGET_HAVE_MVE"
2058   "vmladavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2059   [(set_attr "type" "mve_move")
2060 ])
2061
2062 ;;
2063 ;; [vmlsdavq_s])
2064 ;;
2065 (define_insn "mve_vmlsdavq_s<mode>"
2066   [
2067    (set (match_operand:SI 0 "s_register_operand" "=e")
2068         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2069                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2070          VMLSDAVQ_S))
2071   ]
2072   "TARGET_HAVE_MVE"
2073   "vmlsdav.s%#<V_sz_elem>\t%0, %q1, %q2"
2074   [(set_attr "type" "mve_move")
2075 ])
2076
2077 ;;
2078 ;; [vmlsdavxq_s])
2079 ;;
2080 (define_insn "mve_vmlsdavxq_s<mode>"
2081   [
2082    (set (match_operand:SI 0 "s_register_operand" "=e")
2083         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
2084                     (match_operand:MVE_2 2 "s_register_operand" "w")]
2085          VMLSDAVXQ_S))
2086   ]
2087   "TARGET_HAVE_MVE"
2088   "vmlsdavx.s%#<V_sz_elem>\t%0, %q1, %q2"
2089   [(set_attr "type" "mve_move")
2090 ])
2091
2092 ;;
2093 ;; [vmulhq_s, vmulhq_u])
2094 ;;
2095 (define_insn "mve_vmulhq_<supf><mode>"
2096   [
2097    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2098         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2099                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2100          VMULHQ))
2101   ]
2102   "TARGET_HAVE_MVE"
2103   "vmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2104   [(set_attr "type" "mve_move")
2105 ])
2106
2107 ;;
2108 ;; [vmullbq_int_u, vmullbq_int_s])
2109 ;;
2110 (define_insn "mve_vmullbq_int_<supf><mode>"
2111   [
2112    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2113         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2114                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2115          VMULLBQ_INT))
2116   ]
2117   "TARGET_HAVE_MVE"
2118   "vmullb.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2119   [(set_attr "type" "mve_move")
2120 ])
2121
2122 ;;
2123 ;; [vmulltq_int_u, vmulltq_int_s])
2124 ;;
2125 (define_insn "mve_vmulltq_int_<supf><mode>"
2126   [
2127    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
2128         (unspec:<V_double_width> [(match_operand:MVE_2 1 "s_register_operand" "w")
2129                                   (match_operand:MVE_2 2 "s_register_operand" "w")]
2130          VMULLTQ_INT))
2131   ]
2132   "TARGET_HAVE_MVE"
2133   "vmullt.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2134   [(set_attr "type" "mve_move")
2135 ])
2136
2137 ;;
2138 ;; [vmulq_n_u, vmulq_n_s])
2139 ;;
2140 (define_insn "mve_vmulq_n_<supf><mode>"
2141   [
2142    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2143         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2144                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2145          VMULQ_N))
2146   ]
2147   "TARGET_HAVE_MVE"
2148   "vmul.i%#<V_sz_elem>\t%q0, %q1, %2"
2149   [(set_attr "type" "mve_move")
2150 ])
2151
2152 ;;
2153 ;; [vmulq_u, vmulq_s])
2154 ;;
2155 (define_insn "mve_vmulq_<supf><mode>"
2156   [
2157    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2158         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2159                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2160          VMULQ))
2161   ]
2162   "TARGET_HAVE_MVE"
2163   "vmul.i%#<V_sz_elem>\t%q0, %q1, %q2"
2164   [(set_attr "type" "mve_move")
2165 ])
2166
2167 ;;
2168 ;; [vornq_u, vornq_s])
2169 ;;
2170 (define_insn "mve_vornq_<supf><mode>"
2171   [
2172    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2173         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2174                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2175          VORNQ))
2176   ]
2177   "TARGET_HAVE_MVE"
2178   "vorn %q0, %q1, %q2"
2179   [(set_attr "type" "mve_move")
2180 ])
2181
2182 ;;
2183 ;; [vorrq_s, vorrq_u])
2184 ;;
2185 (define_insn "mve_vorrq_<supf><mode>"
2186   [
2187    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2188         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2189                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2190          VORRQ))
2191   ]
2192   "TARGET_HAVE_MVE"
2193   "vorr %q0, %q1, %q2"
2194   [(set_attr "type" "mve_move")
2195 ])
2196
2197 ;;
2198 ;; [vqaddq_n_s, vqaddq_n_u])
2199 ;;
2200 (define_insn "mve_vqaddq_n_<supf><mode>"
2201   [
2202    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2203         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2204                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2205          VQADDQ_N))
2206   ]
2207   "TARGET_HAVE_MVE"
2208   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2209   [(set_attr "type" "mve_move")
2210 ])
2211
2212 ;;
2213 ;; [vqaddq_u, vqaddq_s])
2214 ;;
2215 (define_insn "mve_vqaddq_<supf><mode>"
2216   [
2217    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2218         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2219                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2220          VQADDQ))
2221   ]
2222   "TARGET_HAVE_MVE"
2223   "vqadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2224   [(set_attr "type" "mve_move")
2225 ])
2226
2227 ;;
2228 ;; [vqdmulhq_n_s])
2229 ;;
2230 (define_insn "mve_vqdmulhq_n_s<mode>"
2231   [
2232    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2233         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2234                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2235          VQDMULHQ_N_S))
2236   ]
2237   "TARGET_HAVE_MVE"
2238   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2239   [(set_attr "type" "mve_move")
2240 ])
2241
2242 ;;
2243 ;; [vqdmulhq_s])
2244 ;;
2245 (define_insn "mve_vqdmulhq_s<mode>"
2246   [
2247    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2248         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2249                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2250          VQDMULHQ_S))
2251   ]
2252   "TARGET_HAVE_MVE"
2253   "vqdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2254   [(set_attr "type" "mve_move")
2255 ])
2256
2257 ;;
2258 ;; [vqrdmulhq_n_s])
2259 ;;
2260 (define_insn "mve_vqrdmulhq_n_s<mode>"
2261   [
2262    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2263         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2264                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2265          VQRDMULHQ_N_S))
2266   ]
2267   "TARGET_HAVE_MVE"
2268   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %2"
2269   [(set_attr "type" "mve_move")
2270 ])
2271
2272 ;;
2273 ;; [vqrdmulhq_s])
2274 ;;
2275 (define_insn "mve_vqrdmulhq_s<mode>"
2276   [
2277    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2278         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2279                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2280          VQRDMULHQ_S))
2281   ]
2282   "TARGET_HAVE_MVE"
2283   "vqrdmulh.s%#<V_sz_elem>\t%q0, %q1, %q2"
2284   [(set_attr "type" "mve_move")
2285 ])
2286
2287 ;;
2288 ;; [vqrshlq_n_s, vqrshlq_n_u])
2289 ;;
2290 (define_insn "mve_vqrshlq_n_<supf><mode>"
2291   [
2292    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2293         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2294                        (match_operand:SI 2 "s_register_operand" "r")]
2295          VQRSHLQ_N))
2296   ]
2297   "TARGET_HAVE_MVE"
2298   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2299   [(set_attr "type" "mve_move")
2300 ])
2301
2302 ;;
2303 ;; [vqrshlq_s, vqrshlq_u])
2304 ;;
2305 (define_insn "mve_vqrshlq_<supf><mode>"
2306   [
2307    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2308         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2309                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2310          VQRSHLQ))
2311   ]
2312   "TARGET_HAVE_MVE"
2313   "vqrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2314   [(set_attr "type" "mve_move")
2315 ])
2316
2317 ;;
2318 ;; [vqshlq_n_s, vqshlq_n_u])
2319 ;;
2320 (define_insn "mve_vqshlq_n_<supf><mode>"
2321   [
2322    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2323         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2324                        (match_operand:SI 2 "immediate_operand" "i")]
2325          VQSHLQ_N))
2326   ]
2327   "TARGET_HAVE_MVE"
2328   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2329   [(set_attr "type" "mve_move")
2330 ])
2331
2332 ;;
2333 ;; [vqshlq_r_u, vqshlq_r_s])
2334 ;;
2335 (define_insn "mve_vqshlq_r_<supf><mode>"
2336   [
2337    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2338         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2339                        (match_operand:SI 2 "s_register_operand" "r")]
2340          VQSHLQ_R))
2341   ]
2342   "TARGET_HAVE_MVE"
2343   "vqshl.<supf>%#<V_sz_elem>\t%q0, %2"
2344   [(set_attr "type" "mve_move")
2345 ])
2346
2347 ;;
2348 ;; [vqshlq_s, vqshlq_u])
2349 ;;
2350 (define_insn "mve_vqshlq_<supf><mode>"
2351   [
2352    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2353         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2354                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2355          VQSHLQ))
2356   ]
2357   "TARGET_HAVE_MVE"
2358   "vqshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2359   [(set_attr "type" "mve_move")
2360 ])
2361
2362 ;;
2363 ;; [vqshluq_n_s])
2364 ;;
2365 (define_insn "mve_vqshluq_n_s<mode>"
2366   [
2367    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2368         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2369                        (match_operand:SI 2 "mve_imm_7" "Ra")]
2370          VQSHLUQ_N_S))
2371   ]
2372   "TARGET_HAVE_MVE"
2373   "vqshlu.s%#<V_sz_elem>\t%q0, %q1, %2"
2374   [(set_attr "type" "mve_move")
2375 ])
2376
2377 ;;
2378 ;; [vqsubq_n_s, vqsubq_n_u])
2379 ;;
2380 (define_insn "mve_vqsubq_n_<supf><mode>"
2381   [
2382    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2383         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2384                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2385          VQSUBQ_N))
2386   ]
2387   "TARGET_HAVE_MVE"
2388   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2389   [(set_attr "type" "mve_move")
2390 ])
2391
2392 ;;
2393 ;; [vqsubq_u, vqsubq_s])
2394 ;;
2395 (define_insn "mve_vqsubq_<supf><mode>"
2396   [
2397    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2398         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2399                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2400          VQSUBQ))
2401   ]
2402   "TARGET_HAVE_MVE"
2403   "vqsub.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2404   [(set_attr "type" "mve_move")
2405 ])
2406
2407 ;;
2408 ;; [vrhaddq_s, vrhaddq_u])
2409 ;;
2410 (define_insn "mve_vrhaddq_<supf><mode>"
2411   [
2412    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2413         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2414                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2415          VRHADDQ))
2416   ]
2417   "TARGET_HAVE_MVE"
2418   "vrhadd.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2419   [(set_attr "type" "mve_move")
2420 ])
2421
2422 ;;
2423 ;; [vrmulhq_s, vrmulhq_u])
2424 ;;
2425 (define_insn "mve_vrmulhq_<supf><mode>"
2426   [
2427    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2428         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2429                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2430          VRMULHQ))
2431   ]
2432   "TARGET_HAVE_MVE"
2433   "vrmulh.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2434   [(set_attr "type" "mve_move")
2435 ])
2436
2437 ;;
2438 ;; [vrshlq_n_u, vrshlq_n_s])
2439 ;;
2440 (define_insn "mve_vrshlq_n_<supf><mode>"
2441   [
2442    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2443         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2444                        (match_operand:SI 2 "s_register_operand" "r")]
2445          VRSHLQ_N))
2446   ]
2447   "TARGET_HAVE_MVE"
2448   "vrshl.<supf>%#<V_sz_elem>\t%q0, %2"
2449   [(set_attr "type" "mve_move")
2450 ])
2451
2452 ;;
2453 ;; [vrshlq_s, vrshlq_u])
2454 ;;
2455 (define_insn "mve_vrshlq_<supf><mode>"
2456   [
2457    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2458         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2459                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2460          VRSHLQ))
2461   ]
2462   "TARGET_HAVE_MVE"
2463   "vrshl.<supf>%#<V_sz_elem>\t%q0, %q1, %q2"
2464   [(set_attr "type" "mve_move")
2465 ])
2466
2467 ;;
2468 ;; [vrshrq_n_s, vrshrq_n_u])
2469 ;;
2470 (define_insn "mve_vrshrq_n_<supf><mode>"
2471   [
2472    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2473         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2474                        (match_operand:SI 2 "<MVE_pred2>" "<MVE_constraint2>")]
2475          VRSHRQ_N))
2476   ]
2477   "TARGET_HAVE_MVE"
2478   "vrshr.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2479   [(set_attr "type" "mve_move")
2480 ])
2481
2482 ;;
2483 ;; [vshlq_n_u, vshlq_n_s])
2484 ;;
2485 (define_insn "mve_vshlq_n_<supf><mode>"
2486   [
2487    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2488         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2489                        (match_operand:SI 2 "immediate_operand" "i")]
2490          VSHLQ_N))
2491   ]
2492   "TARGET_HAVE_MVE"
2493   "vshl.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
2494   [(set_attr "type" "mve_move")
2495 ])
2496
2497 ;;
2498 ;; [vshlq_r_s, vshlq_r_u])
2499 ;;
2500 (define_insn "mve_vshlq_r_<supf><mode>"
2501   [
2502    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2503         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
2504                        (match_operand:SI 2 "s_register_operand" "r")]
2505          VSHLQ_R))
2506   ]
2507   "TARGET_HAVE_MVE"
2508   "vshl.<supf>%#<V_sz_elem>\t%q0, %2"
2509   [(set_attr "type" "mve_move")
2510 ])
2511
2512 ;;
2513 ;; [vsubq_n_s, vsubq_n_u])
2514 ;;
2515 (define_insn "mve_vsubq_n_<supf><mode>"
2516   [
2517    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2518         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2519                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2520          VSUBQ_N))
2521   ]
2522   "TARGET_HAVE_MVE"
2523   "vsub.i%#<V_sz_elem>\t%q0, %q1, %2"
2524   [(set_attr "type" "mve_move")
2525 ])
2526
2527 ;;
2528 ;; [vsubq_s, vsubq_u])
2529 ;;
2530 (define_insn "mve_vsubq_<supf><mode>"
2531   [
2532    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
2533         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "w")
2534                        (match_operand:MVE_2 2 "s_register_operand" "w")]
2535          VSUBQ))
2536   ]
2537   "TARGET_HAVE_MVE"
2538   "vsub.i%#<V_sz_elem>\t%q0, %q1, %q2"
2539   [(set_attr "type" "mve_move")
2540 ])
2541
2542 ;;
2543 ;; [vabdq_f])
2544 ;;
2545 (define_insn "mve_vabdq_f<mode>"
2546   [
2547    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2548         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2549                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2550          VABDQ_F))
2551   ]
2552   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2553   "vabd.f%#<V_sz_elem>  %q0, %q1, %q2"
2554   [(set_attr "type" "mve_move")
2555 ])
2556
2557 ;;
2558 ;; [vaddlvaq_s vaddlvaq_u])
2559 ;;
2560 (define_insn "mve_vaddlvaq_<supf>v4si"
2561   [
2562    (set (match_operand:DI 0 "s_register_operand" "=r")
2563         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
2564                     (match_operand:V4SI 2 "s_register_operand" "w")]
2565          VADDLVAQ))
2566   ]
2567   "TARGET_HAVE_MVE"
2568   "vaddlva.<supf>32 %Q0, %R0, %q2"
2569   [(set_attr "type" "mve_move")
2570 ])
2571
2572 ;;
2573 ;; [vaddq_n_f])
2574 ;;
2575 (define_insn "mve_vaddq_n_f<mode>"
2576   [
2577    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2578         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2579                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
2580          VADDQ_N_F))
2581   ]
2582   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2583   "vadd.f%#<V_sz_elem>  %q0, %q1, %2"
2584   [(set_attr "type" "mve_move")
2585 ])
2586
2587 ;;
2588 ;; [vandq_f])
2589 ;;
2590 (define_insn "mve_vandq_f<mode>"
2591   [
2592    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2593         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2594                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2595          VANDQ_F))
2596   ]
2597   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2598   "vand %q0, %q1, %q2"
2599   [(set_attr "type" "mve_move")
2600 ])
2601
2602 ;;
2603 ;; [vbicq_f])
2604 ;;
2605 (define_insn "mve_vbicq_f<mode>"
2606   [
2607    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2608         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2609                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2610          VBICQ_F))
2611   ]
2612   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2613   "vbic %q0, %q1, %q2"
2614   [(set_attr "type" "mve_move")
2615 ])
2616
2617 ;;
2618 ;; [vbicq_n_s, vbicq_n_u])
2619 ;;
2620 (define_insn "mve_vbicq_n_<supf><mode>"
2621   [
2622    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
2623         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
2624                        (match_operand:SI 2 "immediate_operand" "i")]
2625          VBICQ_N))
2626   ]
2627   "TARGET_HAVE_MVE"
2628   "vbic.i%#<V_sz_elem>  %q0, %2"
2629   [(set_attr "type" "mve_move")
2630 ])
2631
2632 ;;
2633 ;; [vcaddq_rot270_f])
2634 ;;
2635 (define_insn "mve_vcaddq_rot270_f<mode>"
2636   [
2637    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2638         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2639                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2640          VCADDQ_ROT270_F))
2641   ]
2642   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2643   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2644   [(set_attr "type" "mve_move")
2645 ])
2646
2647 ;;
2648 ;; [vcaddq_rot90_f])
2649 ;;
2650 (define_insn "mve_vcaddq_rot90_f<mode>"
2651   [
2652    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2653         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2654                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2655          VCADDQ_ROT90_F))
2656   ]
2657   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2658   "vcadd.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2659   [(set_attr "type" "mve_move")
2660 ])
2661
2662 ;;
2663 ;; [vcmpeqq_f])
2664 ;;
2665 (define_insn "mve_vcmpeqq_f<mode>"
2666   [
2667    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2668         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2669                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2670          VCMPEQQ_F))
2671   ]
2672   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2673   "vcmp.f%#<V_sz_elem>  eq, %q1, %q2"
2674   [(set_attr "type" "mve_move")
2675 ])
2676
2677 ;;
2678 ;; [vcmpeqq_n_f])
2679 ;;
2680 (define_insn "mve_vcmpeqq_n_f<mode>"
2681   [
2682    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2683         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2684                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2685          VCMPEQQ_N_F))
2686   ]
2687   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2688   "vcmp.f%#<V_sz_elem>  eq, %q1, %2"
2689   [(set_attr "type" "mve_move")
2690 ])
2691
2692 ;;
2693 ;; [vcmpgeq_f])
2694 ;;
2695 (define_insn "mve_vcmpgeq_f<mode>"
2696   [
2697    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2698         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2699                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2700          VCMPGEQ_F))
2701   ]
2702   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2703   "vcmp.f%#<V_sz_elem>  ge, %q1, %q2"
2704   [(set_attr "type" "mve_move")
2705 ])
2706
2707 ;;
2708 ;; [vcmpgeq_n_f])
2709 ;;
2710 (define_insn "mve_vcmpgeq_n_f<mode>"
2711   [
2712    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2713         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2714                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2715          VCMPGEQ_N_F))
2716   ]
2717   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2718   "vcmp.f%#<V_sz_elem>  ge, %q1, %2"
2719   [(set_attr "type" "mve_move")
2720 ])
2721
2722 ;;
2723 ;; [vcmpgtq_f])
2724 ;;
2725 (define_insn "mve_vcmpgtq_f<mode>"
2726   [
2727    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2728         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2729                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2730          VCMPGTQ_F))
2731   ]
2732   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2733   "vcmp.f%#<V_sz_elem>  gt, %q1, %q2"
2734   [(set_attr "type" "mve_move")
2735 ])
2736
2737 ;;
2738 ;; [vcmpgtq_n_f])
2739 ;;
2740 (define_insn "mve_vcmpgtq_n_f<mode>"
2741   [
2742    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2743         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2744                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2745          VCMPGTQ_N_F))
2746   ]
2747   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2748   "vcmp.f%#<V_sz_elem>  gt, %q1, %2"
2749   [(set_attr "type" "mve_move")
2750 ])
2751
2752 ;;
2753 ;; [vcmpleq_f])
2754 ;;
2755 (define_insn "mve_vcmpleq_f<mode>"
2756   [
2757    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2758         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2759                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2760          VCMPLEQ_F))
2761   ]
2762   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2763   "vcmp.f%#<V_sz_elem>  le, %q1, %q2"
2764   [(set_attr "type" "mve_move")
2765 ])
2766
2767 ;;
2768 ;; [vcmpleq_n_f])
2769 ;;
2770 (define_insn "mve_vcmpleq_n_f<mode>"
2771   [
2772    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2773         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2774                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2775          VCMPLEQ_N_F))
2776   ]
2777   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2778   "vcmp.f%#<V_sz_elem>  le, %q1, %2"
2779   [(set_attr "type" "mve_move")
2780 ])
2781
2782 ;;
2783 ;; [vcmpltq_f])
2784 ;;
2785 (define_insn "mve_vcmpltq_f<mode>"
2786   [
2787    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2788         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2789                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2790          VCMPLTQ_F))
2791   ]
2792   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2793   "vcmp.f%#<V_sz_elem>  lt, %q1, %q2"
2794   [(set_attr "type" "mve_move")
2795 ])
2796
2797 ;;
2798 ;; [vcmpltq_n_f])
2799 ;;
2800 (define_insn "mve_vcmpltq_n_f<mode>"
2801   [
2802    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2803         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2804                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2805          VCMPLTQ_N_F))
2806   ]
2807   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2808   "vcmp.f%#<V_sz_elem>  lt, %q1, %2"
2809   [(set_attr "type" "mve_move")
2810 ])
2811
2812 ;;
2813 ;; [vcmpneq_f])
2814 ;;
2815 (define_insn "mve_vcmpneq_f<mode>"
2816   [
2817    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2818         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2819                     (match_operand:MVE_0 2 "s_register_operand" "w")]
2820          VCMPNEQ_F))
2821   ]
2822   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2823   "vcmp.f%#<V_sz_elem>  ne, %q1, %q2"
2824   [(set_attr "type" "mve_move")
2825 ])
2826
2827 ;;
2828 ;; [vcmpneq_n_f])
2829 ;;
2830 (define_insn "mve_vcmpneq_n_f<mode>"
2831   [
2832    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2833         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
2834                     (match_operand:<V_elem> 2 "s_register_operand" "r")]
2835          VCMPNEQ_N_F))
2836   ]
2837   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2838   "vcmp.f%#<V_sz_elem>  ne, %q1, %2"
2839   [(set_attr "type" "mve_move")
2840 ])
2841
2842 ;;
2843 ;; [vcmulq_f])
2844 ;;
2845 (define_insn "mve_vcmulq_f<mode>"
2846   [
2847    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2848         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2849                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2850          VCMULQ_F))
2851   ]
2852   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2853   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #0"
2854   [(set_attr "type" "mve_move")
2855 ])
2856
2857 ;;
2858 ;; [vcmulq_rot180_f])
2859 ;;
2860 (define_insn "mve_vcmulq_rot180_f<mode>"
2861   [
2862    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2863         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2864                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2865          VCMULQ_ROT180_F))
2866   ]
2867   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2868   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #180"
2869   [(set_attr "type" "mve_move")
2870 ])
2871
2872 ;;
2873 ;; [vcmulq_rot270_f])
2874 ;;
2875 (define_insn "mve_vcmulq_rot270_f<mode>"
2876   [
2877    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2878         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2879                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2880          VCMULQ_ROT270_F))
2881   ]
2882   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2883   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #270"
2884   [(set_attr "type" "mve_move")
2885 ])
2886
2887 ;;
2888 ;; [vcmulq_rot90_f])
2889 ;;
2890 (define_insn "mve_vcmulq_rot90_f<mode>"
2891   [
2892    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
2893         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2894                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2895          VCMULQ_ROT90_F))
2896   ]
2897   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2898   "vcmul.f%#<V_sz_elem> %q0, %q1, %q2, #90"
2899   [(set_attr "type" "mve_move")
2900 ])
2901
2902 ;;
2903 ;; [vctp8q_m vctp16q_m vctp32q_m vctp64q_m])
2904 ;;
2905 (define_insn "mve_vctp<mode1>q_mhi"
2906   [
2907    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
2908         (unspec:HI [(match_operand:SI 1 "s_register_operand" "r")
2909                     (match_operand:HI 2 "vpr_register_operand" "Up")]
2910          VCTPQ_M))
2911   ]
2912   "TARGET_HAVE_MVE"
2913   "vpst\;vctpt.<mode1> %1"
2914   [(set_attr "type" "mve_move")
2915    (set_attr "length""8")])
2916
2917 ;;
2918 ;; [vcvtbq_f16_f32])
2919 ;;
2920 (define_insn "mve_vcvtbq_f16_f32v8hf"
2921   [
2922    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2923         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2924                       (match_operand:V4SF 2 "s_register_operand" "w")]
2925          VCVTBQ_F16_F32))
2926   ]
2927   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2928   "vcvtb.f16.f32 %q0, %q2"
2929   [(set_attr "type" "mve_move")
2930 ])
2931
2932 ;;
2933 ;; [vcvttq_f16_f32])
2934 ;;
2935 (define_insn "mve_vcvttq_f16_f32v8hf"
2936   [
2937    (set (match_operand:V8HF 0 "s_register_operand" "=w")
2938         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
2939                       (match_operand:V4SF 2 "s_register_operand" "w")]
2940          VCVTTQ_F16_F32))
2941   ]
2942   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2943   "vcvtt.f16.f32 %q0, %q2"
2944   [(set_attr "type" "mve_move")
2945 ])
2946
2947 ;;
2948 ;; [veorq_f])
2949 ;;
2950 (define_insn "mve_veorq_f<mode>"
2951   [
2952    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2953         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2954                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2955          VEORQ_F))
2956   ]
2957   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2958   "veor %q0, %q1, %q2"
2959   [(set_attr "type" "mve_move")
2960 ])
2961
2962 ;;
2963 ;; [vmaxnmaq_f])
2964 ;;
2965 (define_insn "mve_vmaxnmaq_f<mode>"
2966   [
2967    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2968         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
2969                        (match_operand:MVE_0 2 "s_register_operand" "w")]
2970          VMAXNMAQ_F))
2971   ]
2972   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2973   "vmaxnma.f%#<V_sz_elem>       %q0, %q2"
2974   [(set_attr "type" "mve_move")
2975 ])
2976
2977 ;;
2978 ;; [vmaxnmavq_f])
2979 ;;
2980 (define_insn "mve_vmaxnmavq_f<mode>"
2981   [
2982    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
2983         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
2984                           (match_operand:MVE_0 2 "s_register_operand" "w")]
2985          VMAXNMAVQ_F))
2986   ]
2987   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
2988   "vmaxnmav.f%#<V_sz_elem>      %0, %q2"
2989   [(set_attr "type" "mve_move")
2990 ])
2991
2992 ;;
2993 ;; [vmaxnmq_f])
2994 ;;
2995 (define_insn "mve_vmaxnmq_f<mode>"
2996   [
2997    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
2998         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
2999                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3000          VMAXNMQ_F))
3001   ]
3002   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3003   "vmaxnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3004   [(set_attr "type" "mve_move")
3005 ])
3006
3007 ;;
3008 ;; [vmaxnmvq_f])
3009 ;;
3010 (define_insn "mve_vmaxnmvq_f<mode>"
3011   [
3012    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3013         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3014                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3015          VMAXNMVQ_F))
3016   ]
3017   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3018   "vmaxnmv.f%#<V_sz_elem>       %0, %q2"
3019   [(set_attr "type" "mve_move")
3020 ])
3021
3022 ;;
3023 ;; [vminnmaq_f])
3024 ;;
3025 (define_insn "mve_vminnmaq_f<mode>"
3026   [
3027    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3028         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3029                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3030          VMINNMAQ_F))
3031   ]
3032   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3033   "vminnma.f%#<V_sz_elem>       %q0, %q2"
3034   [(set_attr "type" "mve_move")
3035 ])
3036
3037 ;;
3038 ;; [vminnmavq_f])
3039 ;;
3040 (define_insn "mve_vminnmavq_f<mode>"
3041   [
3042    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3043         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3044                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3045          VMINNMAVQ_F))
3046   ]
3047   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3048   "vminnmav.f%#<V_sz_elem>      %0, %q2"
3049   [(set_attr "type" "mve_move")
3050 ])
3051
3052 ;;
3053 ;; [vminnmq_f])
3054 ;;
3055 (define_insn "mve_vminnmq_f<mode>"
3056   [
3057    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3058         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3059                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3060          VMINNMQ_F))
3061   ]
3062   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3063   "vminnm.f%#<V_sz_elem>        %q0, %q1, %q2"
3064   [(set_attr "type" "mve_move")
3065 ])
3066
3067 ;;
3068 ;; [vminnmvq_f])
3069 ;;
3070 (define_insn "mve_vminnmvq_f<mode>"
3071   [
3072    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
3073         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
3074                           (match_operand:MVE_0 2 "s_register_operand" "w")]
3075          VMINNMVQ_F))
3076   ]
3077   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3078   "vminnmv.f%#<V_sz_elem>       %0, %q2"
3079   [(set_attr "type" "mve_move")
3080 ])
3081
3082 ;;
3083 ;; [vmlaldavq_u, vmlaldavq_s])
3084 ;;
3085 (define_insn "mve_vmlaldavq_<supf><mode>"
3086   [
3087    (set (match_operand:DI 0 "s_register_operand" "=r")
3088         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3089                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3090          VMLALDAVQ))
3091   ]
3092   "TARGET_HAVE_MVE"
3093   "vmlaldav.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3094   [(set_attr "type" "mve_move")
3095 ])
3096
3097 ;;
3098 ;; [vmlaldavxq_s])
3099 ;;
3100 (define_insn "mve_vmlaldavxq_s<mode>"
3101   [
3102    (set (match_operand:DI 0 "s_register_operand" "=r")
3103         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3104                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3105          VMLALDAVXQ_S))
3106   ]
3107   "TARGET_HAVE_MVE"
3108   "vmlaldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3109   [(set_attr "type" "mve_move")
3110 ])
3111
3112 ;;
3113 ;; [vmlsldavq_s])
3114 ;;
3115 (define_insn "mve_vmlsldavq_s<mode>"
3116   [
3117    (set (match_operand:DI 0 "s_register_operand" "=r")
3118         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3119                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3120          VMLSLDAVQ_S))
3121   ]
3122   "TARGET_HAVE_MVE"
3123   "vmlsldav.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3124   [(set_attr "type" "mve_move")
3125 ])
3126
3127 ;;
3128 ;; [vmlsldavxq_s])
3129 ;;
3130 (define_insn "mve_vmlsldavxq_s<mode>"
3131   [
3132    (set (match_operand:DI 0 "s_register_operand" "=r")
3133         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
3134                     (match_operand:MVE_5 2 "s_register_operand" "w")]
3135          VMLSLDAVXQ_S))
3136   ]
3137   "TARGET_HAVE_MVE"
3138   "vmlsldavx.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
3139   [(set_attr "type" "mve_move")
3140 ])
3141
3142 ;;
3143 ;; [vmovnbq_u, vmovnbq_s])
3144 ;;
3145 (define_insn "mve_vmovnbq_<supf><mode>"
3146   [
3147    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3148         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3149                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3150          VMOVNBQ))
3151   ]
3152   "TARGET_HAVE_MVE"
3153   "vmovnb.i%#<V_sz_elem>        %q0, %q2"
3154   [(set_attr "type" "mve_move")
3155 ])
3156
3157 ;;
3158 ;; [vmovntq_s, vmovntq_u])
3159 ;;
3160 (define_insn "mve_vmovntq_<supf><mode>"
3161   [
3162    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3163         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3164                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3165          VMOVNTQ))
3166   ]
3167   "TARGET_HAVE_MVE"
3168   "vmovnt.i%#<V_sz_elem>        %q0, %q2"
3169   [(set_attr "type" "mve_move")
3170 ])
3171
3172 ;;
3173 ;; [vmulq_f])
3174 ;;
3175 (define_insn "mve_vmulq_f<mode>"
3176   [
3177    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3178         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3179                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3180          VMULQ_F))
3181   ]
3182   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3183   "vmul.f%#<V_sz_elem>  %q0, %q1, %q2"
3184   [(set_attr "type" "mve_move")
3185 ])
3186
3187 ;;
3188 ;; [vmulq_n_f])
3189 ;;
3190 (define_insn "mve_vmulq_n_f<mode>"
3191   [
3192    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3193         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3194                        (match_operand:<V_elem> 2 "s_register_operand" "r")]
3195          VMULQ_N_F))
3196   ]
3197   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3198   "vmul.f%#<V_sz_elem>  %q0, %q1, %2"
3199   [(set_attr "type" "mve_move")
3200 ])
3201
3202 ;;
3203 ;; [vornq_f])
3204 ;;
3205 (define_insn "mve_vornq_f<mode>"
3206   [
3207    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3208         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3209                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3210          VORNQ_F))
3211   ]
3212   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3213   "vorn %q0, %q1, %q2"
3214   [(set_attr "type" "mve_move")
3215 ])
3216
3217 ;;
3218 ;; [vorrq_f])
3219 ;;
3220 (define_insn "mve_vorrq_f<mode>"
3221   [
3222    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3223         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3224                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3225          VORRQ_F))
3226   ]
3227   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3228   "vorr %q0, %q1, %q2"
3229   [(set_attr "type" "mve_move")
3230 ])
3231
3232 ;;
3233 ;; [vorrq_n_u, vorrq_n_s])
3234 ;;
3235 (define_insn "mve_vorrq_n_<supf><mode>"
3236   [
3237    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3238         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3239                        (match_operand:SI 2 "immediate_operand" "i")]
3240          VORRQ_N))
3241   ]
3242   "TARGET_HAVE_MVE"
3243   "vorr.i%#<V_sz_elem>  %q0, %2"
3244   [(set_attr "type" "mve_move")
3245 ])
3246
3247 ;;
3248 ;; [vqdmullbq_n_s])
3249 ;;
3250 (define_insn "mve_vqdmullbq_n_s<mode>"
3251   [
3252    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3253         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3254                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3255          VQDMULLBQ_N_S))
3256   ]
3257   "TARGET_HAVE_MVE"
3258   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %2"
3259   [(set_attr "type" "mve_move")
3260 ])
3261
3262 ;;
3263 ;; [vqdmullbq_s])
3264 ;;
3265 (define_insn "mve_vqdmullbq_s<mode>"
3266   [
3267    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3268         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3269                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3270          VQDMULLBQ_S))
3271   ]
3272   "TARGET_HAVE_MVE"
3273   "vqdmullb.s%#<V_sz_elem>      %q0, %q1, %q2"
3274   [(set_attr "type" "mve_move")
3275 ])
3276
3277 ;;
3278 ;; [vqdmulltq_n_s])
3279 ;;
3280 (define_insn "mve_vqdmulltq_n_s<mode>"
3281   [
3282    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3283         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3284                                   (match_operand:<V_elem> 2 "s_register_operand" "r")]
3285          VQDMULLTQ_N_S))
3286   ]
3287   "TARGET_HAVE_MVE"
3288   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %2"
3289   [(set_attr "type" "mve_move")
3290 ])
3291
3292 ;;
3293 ;; [vqdmulltq_s])
3294 ;;
3295 (define_insn "mve_vqdmulltq_s<mode>"
3296   [
3297    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
3298         (unspec:<V_double_width> [(match_operand:MVE_5 1 "s_register_operand" "w")
3299                                   (match_operand:MVE_5 2 "s_register_operand" "w")]
3300          VQDMULLTQ_S))
3301   ]
3302   "TARGET_HAVE_MVE"
3303   "vqdmullt.s%#<V_sz_elem>      %q0, %q1, %q2"
3304   [(set_attr "type" "mve_move")
3305 ])
3306
3307 ;;
3308 ;; [vqmovnbq_u, vqmovnbq_s])
3309 ;;
3310 (define_insn "mve_vqmovnbq_<supf><mode>"
3311   [
3312    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3313         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3314                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3315          VQMOVNBQ))
3316   ]
3317   "TARGET_HAVE_MVE"
3318   "vqmovnb.<supf>%#<V_sz_elem>  %q0, %q2"
3319   [(set_attr "type" "mve_move")
3320 ])
3321
3322 ;;
3323 ;; [vqmovntq_u, vqmovntq_s])
3324 ;;
3325 (define_insn "mve_vqmovntq_<supf><mode>"
3326   [
3327    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3328         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3329                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3330          VQMOVNTQ))
3331   ]
3332   "TARGET_HAVE_MVE"
3333   "vqmovnt.<supf>%#<V_sz_elem>  %q0, %q2"
3334   [(set_attr "type" "mve_move")
3335 ])
3336
3337 ;;
3338 ;; [vqmovunbq_s])
3339 ;;
3340 (define_insn "mve_vqmovunbq_s<mode>"
3341   [
3342    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3343         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3344                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3345          VQMOVUNBQ_S))
3346   ]
3347   "TARGET_HAVE_MVE"
3348   "vqmovunb.s%#<V_sz_elem>      %q0, %q2"
3349   [(set_attr "type" "mve_move")
3350 ])
3351
3352 ;;
3353 ;; [vqmovuntq_s])
3354 ;;
3355 (define_insn "mve_vqmovuntq_s<mode>"
3356   [
3357    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3358         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3359                                  (match_operand:MVE_5 2 "s_register_operand" "w")]
3360          VQMOVUNTQ_S))
3361   ]
3362   "TARGET_HAVE_MVE"
3363   "vqmovunt.s%#<V_sz_elem>      %q0, %q2"
3364   [(set_attr "type" "mve_move")
3365 ])
3366
3367 ;;
3368 ;; [vrmlaldavhxq_s])
3369 ;;
3370 (define_insn "mve_vrmlaldavhxq_sv4si"
3371   [
3372    (set (match_operand:DI 0 "s_register_operand" "=r")
3373         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3374                     (match_operand:V4SI 2 "s_register_operand" "w")]
3375          VRMLALDAVHXQ_S))
3376   ]
3377   "TARGET_HAVE_MVE"
3378   "vrmlaldavhx.s32 %Q0, %R0, %q1, %q2"
3379   [(set_attr "type" "mve_move")
3380 ])
3381
3382 ;;
3383 ;; [vrmlsldavhq_s])
3384 ;;
3385 (define_insn "mve_vrmlsldavhq_sv4si"
3386   [
3387    (set (match_operand:DI 0 "s_register_operand" "=r")
3388         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3389                     (match_operand:V4SI 2 "s_register_operand" "w")]
3390          VRMLSLDAVHQ_S))
3391   ]
3392   "TARGET_HAVE_MVE"
3393   "vrmlsldavh.s32\t%Q0, %R0, %q1, %q2"
3394   [(set_attr "type" "mve_move")
3395 ])
3396
3397 ;;
3398 ;; [vrmlsldavhxq_s])
3399 ;;
3400 (define_insn "mve_vrmlsldavhxq_sv4si"
3401   [
3402    (set (match_operand:DI 0 "s_register_operand" "=r")
3403         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3404                     (match_operand:V4SI 2 "s_register_operand" "w")]
3405          VRMLSLDAVHXQ_S))
3406   ]
3407   "TARGET_HAVE_MVE"
3408   "vrmlsldavhx.s32\t%Q0, %R0, %q1, %q2"
3409   [(set_attr "type" "mve_move")
3410 ])
3411
3412 ;;
3413 ;; [vshllbq_n_s, vshllbq_n_u])
3414 ;;
3415 (define_insn "mve_vshllbq_n_<supf><mode>"
3416   [
3417    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3418         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3419                                   (match_operand:SI 2 "immediate_operand" "i")]
3420          VSHLLBQ_N))
3421   ]
3422   "TARGET_HAVE_MVE"
3423   "vshllb.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3424   [(set_attr "type" "mve_move")
3425 ])
3426
3427 ;;
3428 ;; [vshlltq_n_u, vshlltq_n_s])
3429 ;;
3430 (define_insn "mve_vshlltq_n_<supf><mode>"
3431   [
3432    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3433         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3434                                   (match_operand:SI 2 "immediate_operand" "i")]
3435          VSHLLTQ_N))
3436   ]
3437   "TARGET_HAVE_MVE"
3438   "vshllt.<supf>%#<V_sz_elem>\t%q0, %q1, %2"
3439   [(set_attr "type" "mve_move")
3440 ])
3441
3442 ;;
3443 ;; [vsubq_f])
3444 ;;
3445 (define_insn "mve_vsubq_f<mode>"
3446   [
3447    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3448         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
3449                        (match_operand:MVE_0 2 "s_register_operand" "w")]
3450          VSUBQ_F))
3451   ]
3452   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3453   "vsub.f%#<V_sz_elem>\t%q0, %q1, %q2"
3454   [(set_attr "type" "mve_move")
3455 ])
3456
3457 ;;
3458 ;; [vmulltq_poly_p])
3459 ;;
3460 (define_insn "mve_vmulltq_poly_p<mode>"
3461   [
3462    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3463         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3464                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3465          VMULLTQ_POLY_P))
3466   ]
3467   "TARGET_HAVE_MVE"
3468   "vmullt.p%#<V_sz_elem>\t%q0, %q1, %q2"
3469   [(set_attr "type" "mve_move")
3470 ])
3471
3472 ;;
3473 ;; [vmullbq_poly_p])
3474 ;;
3475 (define_insn "mve_vmullbq_poly_p<mode>"
3476   [
3477    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
3478         (unspec:<V_double_width> [(match_operand:MVE_3 1 "s_register_operand" "w")
3479                                   (match_operand:MVE_3 2 "s_register_operand" "w")]
3480          VMULLBQ_POLY_P))
3481   ]
3482   "TARGET_HAVE_MVE"
3483   "vmullb.p%#<V_sz_elem>\t%q0, %q1, %q2"
3484   [(set_attr "type" "mve_move")
3485 ])
3486
3487 ;;
3488 ;; [vrmlaldavhq_u vrmlaldavhq_s])
3489 ;;
3490 (define_insn "mve_vrmlaldavhq_<supf>v4si"
3491   [
3492    (set (match_operand:DI 0 "s_register_operand" "=r")
3493         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
3494                     (match_operand:V4SI 2 "s_register_operand" "w")]
3495          VRMLALDAVHQ))
3496   ]
3497   "TARGET_HAVE_MVE"
3498   "vrmlaldavh.<supf>32 %Q0, %R0, %q1, %q2"
3499   [(set_attr "type" "mve_move")
3500 ])
3501
3502 ;;
3503 ;; [vbicq_m_n_s, vbicq_m_n_u])
3504 ;;
3505 (define_insn "mve_vbicq_m_n_<supf><mode>"
3506   [
3507    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3508         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3509                        (match_operand:SI 2 "immediate_operand" "i")
3510                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3511          VBICQ_M_N))
3512   ]
3513   "TARGET_HAVE_MVE"
3514   "vpst\;vbict.i%#<V_sz_elem>   %q0, %2"
3515   [(set_attr "type" "mve_move")
3516    (set_attr "length""8")])
3517 ;;
3518 ;; [vcmpeqq_m_f])
3519 ;;
3520 (define_insn "mve_vcmpeqq_m_f<mode>"
3521   [
3522    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3523         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
3524                     (match_operand:MVE_0 2 "s_register_operand" "w")
3525                     (match_operand:HI 3 "vpr_register_operand" "Up")]
3526          VCMPEQQ_M_F))
3527   ]
3528   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3529   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %q2"
3530   [(set_attr "type" "mve_move")
3531    (set_attr "length""8")])
3532 ;;
3533 ;; [vcvtaq_m_u, vcvtaq_m_s])
3534 ;;
3535 (define_insn "mve_vcvtaq_m_<supf><mode>"
3536   [
3537    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
3538         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
3539                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3540                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3541          VCVTAQ_M))
3542   ]
3543   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3544   "vpst\;vcvtat.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
3545   [(set_attr "type" "mve_move")
3546    (set_attr "length""8")])
3547 ;;
3548 ;; [vcvtq_m_to_f_s, vcvtq_m_to_f_u])
3549 ;;
3550 (define_insn "mve_vcvtq_m_to_f_<supf><mode>"
3551   [
3552    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
3553         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
3554                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
3555                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3556          VCVTQ_M_TO_F))
3557   ]
3558   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
3559   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>        %q0, %q2"
3560   [(set_attr "type" "mve_move")
3561    (set_attr "length""8")])
3562 ;;
3563 ;; [vqrshrnbq_n_u, vqrshrnbq_n_s])
3564 ;;
3565 (define_insn "mve_vqrshrnbq_n_<supf><mode>"
3566   [
3567    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3568         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3569                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3570                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3571          VQRSHRNBQ_N))
3572   ]
3573   "TARGET_HAVE_MVE"
3574   "vqrshrnb.<supf>%#<V_sz_elem> %q0, %q2, %3"
3575   [(set_attr "type" "mve_move")
3576 ])
3577 ;;
3578 ;; [vqrshrunbq_n_s])
3579 ;;
3580 (define_insn "mve_vqrshrunbq_n_s<mode>"
3581   [
3582    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
3583         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
3584                                  (match_operand:MVE_5 2 "s_register_operand" "w")
3585                                  (match_operand:SI 3 "mve_imm_8" "Rb")]
3586          VQRSHRUNBQ_N_S))
3587   ]
3588   "TARGET_HAVE_MVE"
3589   "vqrshrunb.s%#<V_sz_elem>\t%q0, %q2, %3"
3590   [(set_attr "type" "mve_move")
3591 ])
3592 ;;
3593 ;; [vrmlaldavhaq_s vrmlaldavhaq_u])
3594 ;;
3595 (define_insn "mve_vrmlaldavhaq_<supf>v4si"
3596   [
3597    (set (match_operand:DI 0 "s_register_operand" "=r")
3598         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
3599                     (match_operand:V4SI 2 "s_register_operand" "w")
3600                     (match_operand:V4SI 3 "s_register_operand" "w")]
3601          VRMLALDAVHAQ))
3602   ]
3603   "TARGET_HAVE_MVE"
3604   "vrmlaldavha.<supf>32 %Q0, %R0, %q2, %q3"
3605   [(set_attr "type" "mve_move")
3606 ])
3607
3608 ;;
3609 ;; [vabavq_s, vabavq_u])
3610 ;;
3611 (define_insn "mve_vabavq_<supf><mode>"
3612   [
3613    (set (match_operand:SI 0 "s_register_operand" "=r")
3614         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3615                     (match_operand:MVE_2 2 "s_register_operand" "w")
3616                     (match_operand:MVE_2 3 "s_register_operand" "w")]
3617          VABAVQ))
3618   ]
3619   "TARGET_HAVE_MVE"
3620   "vabav.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
3621   [(set_attr "type" "mve_move")
3622 ])
3623
3624 ;;
3625 ;; [vshlcq_u vshlcq_s]
3626 ;;
3627 (define_expand "mve_vshlcq_vec_<supf><mode>"
3628  [(match_operand:MVE_2 0 "s_register_operand")
3629   (match_operand:MVE_2 1 "s_register_operand")
3630   (match_operand:SI 2 "s_register_operand")
3631   (match_operand:SI 3 "mve_imm_32")
3632   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3633  "TARGET_HAVE_MVE"
3634 {
3635   rtx ignore_wb = gen_reg_rtx (SImode);
3636   emit_insn(gen_mve_vshlcq_<supf><mode>(operands[0], ignore_wb, operands[1],
3637                                       operands[2], operands[3]));
3638   DONE;
3639 })
3640
3641 (define_expand "mve_vshlcq_carry_<supf><mode>"
3642  [(match_operand:SI 0 "s_register_operand")
3643   (match_operand:MVE_2 1 "s_register_operand")
3644   (match_operand:SI 2 "s_register_operand")
3645   (match_operand:SI 3 "mve_imm_32")
3646   (unspec:MVE_2 [(const_int 0)] VSHLCQ)]
3647  "TARGET_HAVE_MVE"
3648 {
3649   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
3650   emit_insn(gen_mve_vshlcq_<supf><mode>(ignore_vec, operands[0], operands[1],
3651                                       operands[2], operands[3]));
3652   DONE;
3653 })
3654
3655 (define_insn "mve_vshlcq_<supf><mode>"
3656  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
3657        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
3658                       (match_operand:SI 3 "s_register_operand" "1")
3659                       (match_operand:SI 4 "mve_imm_32" "Rf")]
3660         VSHLCQ))
3661   (set (match_operand:SI  1 "s_register_operand" "=r")
3662        (unspec:SI [(match_dup 2)
3663                    (match_dup 3)
3664                    (match_dup 4)]
3665         VSHLCQ))]
3666  "TARGET_HAVE_MVE"
3667  "vshlc %q0, %1, %4")
3668
3669 ;;
3670 ;; [vabsq_m_s])
3671 ;;
3672 (define_insn "mve_vabsq_m_s<mode>"
3673   [
3674    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3675         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3676                        (match_operand:MVE_2 2 "s_register_operand" "w")
3677                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3678          VABSQ_M_S))
3679   ]
3680   "TARGET_HAVE_MVE"
3681   "vpst\;vabst.s%#<V_sz_elem>   %q0, %q2"
3682   [(set_attr "type" "mve_move")
3683    (set_attr "length""8")])
3684
3685 ;;
3686 ;; [vaddvaq_p_u, vaddvaq_p_s])
3687 ;;
3688 (define_insn "mve_vaddvaq_p_<supf><mode>"
3689   [
3690    (set (match_operand:SI 0 "s_register_operand" "=e")
3691         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
3692                        (match_operand:MVE_2 2 "s_register_operand" "w")
3693                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3694          VADDVAQ_P))
3695   ]
3696   "TARGET_HAVE_MVE"
3697   "vpst\;vaddvat.<supf>%#<V_sz_elem>    %0, %q2"
3698   [(set_attr "type" "mve_move")
3699    (set_attr "length""8")])
3700
3701 ;;
3702 ;; [vclsq_m_s])
3703 ;;
3704 (define_insn "mve_vclsq_m_s<mode>"
3705   [
3706    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3707         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3708                        (match_operand:MVE_2 2 "s_register_operand" "w")
3709                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3710          VCLSQ_M_S))
3711   ]
3712   "TARGET_HAVE_MVE"
3713   "vpst\;vclst.s%#<V_sz_elem>   %q0, %q2"
3714   [(set_attr "type" "mve_move")
3715    (set_attr "length""8")])
3716
3717 ;;
3718 ;; [vclzq_m_s, vclzq_m_u])
3719 ;;
3720 (define_insn "mve_vclzq_m_<supf><mode>"
3721   [
3722    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3723         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3724                        (match_operand:MVE_2 2 "s_register_operand" "w")
3725                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3726          VCLZQ_M))
3727   ]
3728   "TARGET_HAVE_MVE"
3729   "vpst\;vclzt.i%#<V_sz_elem>   %q0, %q2"
3730   [(set_attr "type" "mve_move")
3731    (set_attr "length""8")])
3732
3733 ;;
3734 ;; [vcmpcsq_m_n_u])
3735 ;;
3736 (define_insn "mve_vcmpcsq_m_n_u<mode>"
3737   [
3738    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3739         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3740                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3741                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3742          VCMPCSQ_M_N_U))
3743   ]
3744   "TARGET_HAVE_MVE"
3745   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %2"
3746   [(set_attr "type" "mve_move")
3747    (set_attr "length""8")])
3748
3749 ;;
3750 ;; [vcmpcsq_m_u])
3751 ;;
3752 (define_insn "mve_vcmpcsq_m_u<mode>"
3753   [
3754    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3755         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3756                        (match_operand:MVE_2 2 "s_register_operand" "w")
3757                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3758          VCMPCSQ_M_U))
3759   ]
3760   "TARGET_HAVE_MVE"
3761   "vpst\;vcmpt.u%#<V_sz_elem>   cs, %q1, %q2"
3762   [(set_attr "type" "mve_move")
3763    (set_attr "length""8")])
3764
3765 ;;
3766 ;; [vcmpeqq_m_n_u, vcmpeqq_m_n_s])
3767 ;;
3768 (define_insn "mve_vcmpeqq_m_n_<supf><mode>"
3769   [
3770    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3771         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3772                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3773                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3774          VCMPEQQ_M_N))
3775   ]
3776   "TARGET_HAVE_MVE"
3777   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %2"
3778   [(set_attr "type" "mve_move")
3779    (set_attr "length""8")])
3780
3781 ;;
3782 ;; [vcmpeqq_m_u, vcmpeqq_m_s])
3783 ;;
3784 (define_insn "mve_vcmpeqq_m_<supf><mode>"
3785   [
3786    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3787         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3788                        (match_operand:MVE_2 2 "s_register_operand" "w")
3789                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3790          VCMPEQQ_M))
3791   ]
3792   "TARGET_HAVE_MVE"
3793   "vpst\;vcmpt.i%#<V_sz_elem>   eq, %q1, %q2"
3794   [(set_attr "type" "mve_move")
3795    (set_attr "length""8")])
3796
3797 ;;
3798 ;; [vcmpgeq_m_n_s])
3799 ;;
3800 (define_insn "mve_vcmpgeq_m_n_s<mode>"
3801   [
3802    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3803         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3804                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3805                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3806          VCMPGEQ_M_N_S))
3807   ]
3808   "TARGET_HAVE_MVE"
3809   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %2"
3810   [(set_attr "type" "mve_move")
3811    (set_attr "length""8")])
3812
3813 ;;
3814 ;; [vcmpgeq_m_s])
3815 ;;
3816 (define_insn "mve_vcmpgeq_m_s<mode>"
3817   [
3818    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3819         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3820                        (match_operand:MVE_2 2 "s_register_operand" "w")
3821                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3822          VCMPGEQ_M_S))
3823   ]
3824   "TARGET_HAVE_MVE"
3825   "vpst\;vcmpt.s%#<V_sz_elem>   ge, %q1, %q2"
3826   [(set_attr "type" "mve_move")
3827    (set_attr "length""8")])
3828
3829 ;;
3830 ;; [vcmpgtq_m_n_s])
3831 ;;
3832 (define_insn "mve_vcmpgtq_m_n_s<mode>"
3833   [
3834    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3835         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3836                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3837                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3838          VCMPGTQ_M_N_S))
3839   ]
3840   "TARGET_HAVE_MVE"
3841   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %2"
3842   [(set_attr "type" "mve_move")
3843    (set_attr "length""8")])
3844
3845 ;;
3846 ;; [vcmpgtq_m_s])
3847 ;;
3848 (define_insn "mve_vcmpgtq_m_s<mode>"
3849   [
3850    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3851         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3852                        (match_operand:MVE_2 2 "s_register_operand" "w")
3853                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3854          VCMPGTQ_M_S))
3855   ]
3856   "TARGET_HAVE_MVE"
3857   "vpst\;vcmpt.s%#<V_sz_elem>   gt, %q1, %q2"
3858   [(set_attr "type" "mve_move")
3859    (set_attr "length""8")])
3860
3861 ;;
3862 ;; [vcmphiq_m_n_u])
3863 ;;
3864 (define_insn "mve_vcmphiq_m_n_u<mode>"
3865   [
3866    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3867         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3868                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3869                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3870          VCMPHIQ_M_N_U))
3871   ]
3872   "TARGET_HAVE_MVE"
3873   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %2"
3874   [(set_attr "type" "mve_move")
3875    (set_attr "length""8")])
3876
3877 ;;
3878 ;; [vcmphiq_m_u])
3879 ;;
3880 (define_insn "mve_vcmphiq_m_u<mode>"
3881   [
3882    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3883         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3884                        (match_operand:MVE_2 2 "s_register_operand" "w")
3885                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3886          VCMPHIQ_M_U))
3887   ]
3888   "TARGET_HAVE_MVE"
3889   "vpst\;vcmpt.u%#<V_sz_elem>   hi, %q1, %q2"
3890   [(set_attr "type" "mve_move")
3891    (set_attr "length""8")])
3892
3893 ;;
3894 ;; [vcmpleq_m_n_s])
3895 ;;
3896 (define_insn "mve_vcmpleq_m_n_s<mode>"
3897   [
3898    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3899         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3900                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3901                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3902          VCMPLEQ_M_N_S))
3903   ]
3904   "TARGET_HAVE_MVE"
3905   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %2"
3906   [(set_attr "type" "mve_move")
3907    (set_attr "length""8")])
3908
3909 ;;
3910 ;; [vcmpleq_m_s])
3911 ;;
3912 (define_insn "mve_vcmpleq_m_s<mode>"
3913   [
3914    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3915         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3916                        (match_operand:MVE_2 2 "s_register_operand" "w")
3917                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3918          VCMPLEQ_M_S))
3919   ]
3920   "TARGET_HAVE_MVE"
3921   "vpst\;vcmpt.s%#<V_sz_elem>   le, %q1, %q2"
3922   [(set_attr "type" "mve_move")
3923    (set_attr "length""8")])
3924
3925 ;;
3926 ;; [vcmpltq_m_n_s])
3927 ;;
3928 (define_insn "mve_vcmpltq_m_n_s<mode>"
3929   [
3930    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3931         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3932                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3933                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3934          VCMPLTQ_M_N_S))
3935   ]
3936   "TARGET_HAVE_MVE"
3937   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %2"
3938   [(set_attr "type" "mve_move")
3939    (set_attr "length""8")])
3940
3941 ;;
3942 ;; [vcmpltq_m_s])
3943 ;;
3944 (define_insn "mve_vcmpltq_m_s<mode>"
3945   [
3946    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3947         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3948                        (match_operand:MVE_2 2 "s_register_operand" "w")
3949                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3950          VCMPLTQ_M_S))
3951   ]
3952   "TARGET_HAVE_MVE"
3953   "vpst\;vcmpt.s%#<V_sz_elem>   lt, %q1, %q2"
3954   [(set_attr "type" "mve_move")
3955    (set_attr "length""8")])
3956
3957 ;;
3958 ;; [vcmpneq_m_n_u, vcmpneq_m_n_s])
3959 ;;
3960 (define_insn "mve_vcmpneq_m_n_<supf><mode>"
3961   [
3962    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3963         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3964                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3965                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3966          VCMPNEQ_M_N))
3967   ]
3968   "TARGET_HAVE_MVE"
3969   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %2"
3970   [(set_attr "type" "mve_move")
3971    (set_attr "length""8")])
3972
3973 ;;
3974 ;; [vcmpneq_m_s, vcmpneq_m_u])
3975 ;;
3976 (define_insn "mve_vcmpneq_m_<supf><mode>"
3977   [
3978    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
3979         (unspec:HI [(match_operand:MVE_2 1 "s_register_operand" "w")
3980                        (match_operand:MVE_2 2 "s_register_operand" "w")
3981                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3982          VCMPNEQ_M))
3983   ]
3984   "TARGET_HAVE_MVE"
3985   "vpst\;vcmpt.i%#<V_sz_elem>   ne, %q1, %q2"
3986   [(set_attr "type" "mve_move")
3987    (set_attr "length""8")])
3988
3989 ;;
3990 ;; [vdupq_m_n_s, vdupq_m_n_u])
3991 ;;
3992 (define_insn "mve_vdupq_m_n_<supf><mode>"
3993   [
3994    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
3995         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
3996                        (match_operand:<V_elem> 2 "s_register_operand" "r")
3997                        (match_operand:HI 3 "vpr_register_operand" "Up")]
3998          VDUPQ_M_N))
3999   ]
4000   "TARGET_HAVE_MVE"
4001   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4002   [(set_attr "type" "mve_move")
4003    (set_attr "length""8")])
4004
4005 ;;
4006 ;; [vmaxaq_m_s])
4007 ;;
4008 (define_insn "mve_vmaxaq_m_s<mode>"
4009   [
4010    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4011         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4012                        (match_operand:MVE_2 2 "s_register_operand" "w")
4013                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4014          VMAXAQ_M_S))
4015   ]
4016   "TARGET_HAVE_MVE"
4017   "vpst\;vmaxat.s%#<V_sz_elem>  %q0, %q2"
4018   [(set_attr "type" "mve_move")
4019    (set_attr "length""8")])
4020
4021 ;;
4022 ;; [vmaxavq_p_s])
4023 ;;
4024 (define_insn "mve_vmaxavq_p_s<mode>"
4025   [
4026    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4027         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4028                        (match_operand:MVE_2 2 "s_register_operand" "w")
4029                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4030          VMAXAVQ_P_S))
4031   ]
4032   "TARGET_HAVE_MVE"
4033   "vpst\;vmaxavt.s%#<V_sz_elem> %0, %q2"
4034   [(set_attr "type" "mve_move")
4035    (set_attr "length""8")])
4036
4037 ;;
4038 ;; [vmaxvq_p_u, vmaxvq_p_s])
4039 ;;
4040 (define_insn "mve_vmaxvq_p_<supf><mode>"
4041   [
4042    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4043         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4044                        (match_operand:MVE_2 2 "s_register_operand" "w")
4045                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4046          VMAXVQ_P))
4047   ]
4048   "TARGET_HAVE_MVE"
4049   "vpst\;vmaxvt.<supf>%#<V_sz_elem>     %0, %q2"
4050   [(set_attr "type" "mve_move")
4051    (set_attr "length""8")])
4052
4053 ;;
4054 ;; [vminaq_m_s])
4055 ;;
4056 (define_insn "mve_vminaq_m_s<mode>"
4057   [
4058    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4059         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4060                        (match_operand:MVE_2 2 "s_register_operand" "w")
4061                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4062          VMINAQ_M_S))
4063   ]
4064   "TARGET_HAVE_MVE"
4065   "vpst\;vminat.s%#<V_sz_elem>  %q0, %q2"
4066   [(set_attr "type" "mve_move")
4067    (set_attr "length""8")])
4068
4069 ;;
4070 ;; [vminavq_p_s])
4071 ;;
4072 (define_insn "mve_vminavq_p_s<mode>"
4073   [
4074    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4075         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4076                        (match_operand:MVE_2 2 "s_register_operand" "w")
4077                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4078          VMINAVQ_P_S))
4079   ]
4080   "TARGET_HAVE_MVE"
4081   "vpst\;vminavt.s%#<V_sz_elem> %0, %q2"
4082   [(set_attr "type" "mve_move")
4083    (set_attr "length""8")])
4084
4085 ;;
4086 ;; [vminvq_p_s, vminvq_p_u])
4087 ;;
4088 (define_insn "mve_vminvq_p_<supf><mode>"
4089   [
4090    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
4091         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
4092                        (match_operand:MVE_2 2 "s_register_operand" "w")
4093                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4094          VMINVQ_P))
4095   ]
4096   "TARGET_HAVE_MVE"
4097   "vpst\;vminvt.<supf>%#<V_sz_elem>\t%0, %q2"
4098   [(set_attr "type" "mve_move")
4099    (set_attr "length""8")])
4100
4101 ;;
4102 ;; [vmladavaq_u, vmladavaq_s])
4103 ;;
4104 (define_insn "mve_vmladavaq_<supf><mode>"
4105   [
4106    (set (match_operand:SI 0 "s_register_operand" "=e")
4107         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4108                        (match_operand:MVE_2 2 "s_register_operand" "w")
4109                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4110          VMLADAVAQ))
4111   ]
4112   "TARGET_HAVE_MVE"
4113   "vmladava.<supf>%#<V_sz_elem> %0, %q2, %q3"
4114   [(set_attr "type" "mve_move")
4115 ])
4116
4117 ;;
4118 ;; [vmladavq_p_u, vmladavq_p_s])
4119 ;;
4120 (define_insn "mve_vmladavq_p_<supf><mode>"
4121   [
4122    (set (match_operand:SI 0 "s_register_operand" "=e")
4123         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4124                        (match_operand:MVE_2 2 "s_register_operand" "w")
4125                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4126          VMLADAVQ_P))
4127   ]
4128   "TARGET_HAVE_MVE"
4129   "vpst\;vmladavt.<supf>%#<V_sz_elem>\t%0, %q1, %q2"
4130   [(set_attr "type" "mve_move")
4131    (set_attr "length""8")])
4132
4133 ;;
4134 ;; [vmladavxq_p_s])
4135 ;;
4136 (define_insn "mve_vmladavxq_p_s<mode>"
4137   [
4138    (set (match_operand:SI 0 "s_register_operand" "=e")
4139         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4140                        (match_operand:MVE_2 2 "s_register_operand" "w")
4141                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4142          VMLADAVXQ_P_S))
4143   ]
4144   "TARGET_HAVE_MVE"
4145   "vpst\;vmladavxt.s%#<V_sz_elem>\t%0, %q1, %q2"
4146   [(set_attr "type" "mve_move")
4147    (set_attr "length""8")])
4148
4149 ;;
4150 ;; [vmlaq_n_u, vmlaq_n_s])
4151 ;;
4152 (define_insn "mve_vmlaq_n_<supf><mode>"
4153   [
4154    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4155         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4156                        (match_operand:MVE_2 2 "s_register_operand" "w")
4157                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4158          VMLAQ_N))
4159   ]
4160   "TARGET_HAVE_MVE"
4161   "vmla.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
4162   [(set_attr "type" "mve_move")
4163 ])
4164
4165 ;;
4166 ;; [vmlasq_n_u, vmlasq_n_s])
4167 ;;
4168 (define_insn "mve_vmlasq_n_<supf><mode>"
4169   [
4170    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4171         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4172                        (match_operand:MVE_2 2 "s_register_operand" "w")
4173                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4174          VMLASQ_N))
4175   ]
4176   "TARGET_HAVE_MVE"
4177   "vmlas.<supf>%#<V_sz_elem>    %q0, %q2, %3"
4178   [(set_attr "type" "mve_move")
4179 ])
4180
4181 ;;
4182 ;; [vmlsdavq_p_s])
4183 ;;
4184 (define_insn "mve_vmlsdavq_p_s<mode>"
4185   [
4186    (set (match_operand:SI 0 "s_register_operand" "=e")
4187         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4188                        (match_operand:MVE_2 2 "s_register_operand" "w")
4189                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4190          VMLSDAVQ_P_S))
4191   ]
4192   "TARGET_HAVE_MVE"
4193   "vpst\;vmlsdavt.s%#<V_sz_elem>        %0, %q1, %q2"
4194   [(set_attr "type" "mve_move")
4195    (set_attr "length""8")])
4196
4197 ;;
4198 ;; [vmlsdavxq_p_s])
4199 ;;
4200 (define_insn "mve_vmlsdavxq_p_s<mode>"
4201   [
4202    (set (match_operand:SI 0 "s_register_operand" "=e")
4203         (unspec:SI [(match_operand:MVE_2 1 "s_register_operand" "w")
4204                        (match_operand:MVE_2 2 "s_register_operand" "w")
4205                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4206          VMLSDAVXQ_P_S))
4207   ]
4208   "TARGET_HAVE_MVE"
4209   "vpst\;vmlsdavxt.s%#<V_sz_elem>       %0, %q1, %q2"
4210   [(set_attr "type" "mve_move")
4211    (set_attr "length""8")])
4212
4213 ;;
4214 ;; [vmvnq_m_s, vmvnq_m_u])
4215 ;;
4216 (define_insn "mve_vmvnq_m_<supf><mode>"
4217   [
4218    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4219         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4220                        (match_operand:MVE_2 2 "s_register_operand" "w")
4221                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4222          VMVNQ_M))
4223   ]
4224   "TARGET_HAVE_MVE"
4225   "vpst\;vmvnt %q0, %q2"
4226   [(set_attr "type" "mve_move")
4227    (set_attr "length""8")])
4228
4229 ;;
4230 ;; [vnegq_m_s])
4231 ;;
4232 (define_insn "mve_vnegq_m_s<mode>"
4233   [
4234    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4235         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4236                        (match_operand:MVE_2 2 "s_register_operand" "w")
4237                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4238          VNEGQ_M_S))
4239   ]
4240   "TARGET_HAVE_MVE"
4241   "vpst\;vnegt.s%#<V_sz_elem>\t%q0, %q2"
4242   [(set_attr "type" "mve_move")
4243    (set_attr "length""8")])
4244
4245 ;;
4246 ;; [vpselq_u, vpselq_s])
4247 ;;
4248 (define_insn "mve_vpselq_<supf><mode>"
4249   [
4250    (set (match_operand:MVE_1 0 "s_register_operand" "=w")
4251         (unspec:MVE_1 [(match_operand:MVE_1 1 "s_register_operand" "w")
4252                        (match_operand:MVE_1 2 "s_register_operand" "w")
4253                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4254          VPSELQ))
4255   ]
4256   "TARGET_HAVE_MVE"
4257   "vpsel %q0, %q1, %q2"
4258   [(set_attr "type" "mve_move")
4259 ])
4260
4261 ;;
4262 ;; [vqabsq_m_s])
4263 ;;
4264 (define_insn "mve_vqabsq_m_s<mode>"
4265   [
4266    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4267         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4268                        (match_operand:MVE_2 2 "s_register_operand" "w")
4269                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4270          VQABSQ_M_S))
4271   ]
4272   "TARGET_HAVE_MVE"
4273   "vpst\;vqabst.s%#<V_sz_elem>\t%q0, %q2"
4274   [(set_attr "type" "mve_move")
4275    (set_attr "length""8")])
4276
4277 ;;
4278 ;; [vqdmlahq_n_s, vqdmlahq_n_u])
4279 ;;
4280 (define_insn "mve_vqdmlahq_n_<supf><mode>"
4281   [
4282    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4283         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4284                        (match_operand:MVE_2 2 "s_register_operand" "w")
4285                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4286          VQDMLAHQ_N))
4287   ]
4288   "TARGET_HAVE_MVE"
4289   "vqdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4290   [(set_attr "type" "mve_move")
4291 ])
4292
4293 ;;
4294 ;; [vqnegq_m_s])
4295 ;;
4296 (define_insn "mve_vqnegq_m_s<mode>"
4297   [
4298    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4299         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4300                        (match_operand:MVE_2 2 "s_register_operand" "w")
4301                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4302          VQNEGQ_M_S))
4303   ]
4304   "TARGET_HAVE_MVE"
4305   "vpst\;vqnegt.s%#<V_sz_elem>  %q0, %q2"
4306   [(set_attr "type" "mve_move")
4307    (set_attr "length""8")])
4308
4309 ;;
4310 ;; [vqrdmladhq_s])
4311 ;;
4312 (define_insn "mve_vqrdmladhq_s<mode>"
4313   [
4314    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4315         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4316                        (match_operand:MVE_2 2 "s_register_operand" "w")
4317                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4318          VQRDMLADHQ_S))
4319   ]
4320   "TARGET_HAVE_MVE"
4321   "vqrdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4322   [(set_attr "type" "mve_move")
4323 ])
4324
4325 ;;
4326 ;; [vqrdmladhxq_s])
4327 ;;
4328 (define_insn "mve_vqrdmladhxq_s<mode>"
4329   [
4330    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4331         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4332                        (match_operand:MVE_2 2 "s_register_operand" "w")
4333                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4334          VQRDMLADHXQ_S))
4335   ]
4336   "TARGET_HAVE_MVE"
4337   "vqrdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4338   [(set_attr "type" "mve_move")
4339 ])
4340
4341 ;;
4342 ;; [vqrdmlahq_n_s, vqrdmlahq_n_u])
4343 ;;
4344 (define_insn "mve_vqrdmlahq_n_<supf><mode>"
4345   [
4346    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4347         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4348                        (match_operand:MVE_2 2 "s_register_operand" "w")
4349                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4350          VQRDMLAHQ_N))
4351   ]
4352   "TARGET_HAVE_MVE"
4353   "vqrdmlah.s%#<V_sz_elem>\t%q0, %q2, %3"
4354   [(set_attr "type" "mve_move")
4355 ])
4356
4357 ;;
4358 ;; [vqrdmlashq_n_s, vqrdmlashq_n_u])
4359 ;;
4360 (define_insn "mve_vqrdmlashq_n_<supf><mode>"
4361   [
4362    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4363         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4364                        (match_operand:MVE_2 2 "s_register_operand" "w")
4365                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
4366          VQRDMLASHQ_N))
4367   ]
4368   "TARGET_HAVE_MVE"
4369   "vqrdmlash.s%#<V_sz_elem>\t%q0, %q2, %3"
4370   [(set_attr "type" "mve_move")
4371 ])
4372
4373 ;;
4374 ;; [vqrdmlsdhq_s])
4375 ;;
4376 (define_insn "mve_vqrdmlsdhq_s<mode>"
4377   [
4378    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4379         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4380                        (match_operand:MVE_2 2 "s_register_operand" "w")
4381                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4382          VQRDMLSDHQ_S))
4383   ]
4384   "TARGET_HAVE_MVE"
4385   "vqrdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4386   [(set_attr "type" "mve_move")
4387 ])
4388
4389 ;;
4390 ;; [vqrdmlsdhxq_s])
4391 ;;
4392 (define_insn "mve_vqrdmlsdhxq_s<mode>"
4393   [
4394    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4395         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4396                        (match_operand:MVE_2 2 "s_register_operand" "w")
4397                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4398          VQRDMLSDHXQ_S))
4399   ]
4400   "TARGET_HAVE_MVE"
4401   "vqrdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4402   [(set_attr "type" "mve_move")
4403 ])
4404
4405 ;;
4406 ;; [vqrshlq_m_n_s, vqrshlq_m_n_u])
4407 ;;
4408 (define_insn "mve_vqrshlq_m_n_<supf><mode>"
4409   [
4410    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4411         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4412                        (match_operand:SI 2 "s_register_operand" "r")
4413                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4414          VQRSHLQ_M_N))
4415   ]
4416   "TARGET_HAVE_MVE"
4417   "vpst\;vqrshlt.<supf>%#<V_sz_elem>    %q0, %2"
4418   [(set_attr "type" "mve_move")
4419    (set_attr "length""8")])
4420
4421 ;;
4422 ;; [vqshlq_m_r_u, vqshlq_m_r_s])
4423 ;;
4424 (define_insn "mve_vqshlq_m_r_<supf><mode>"
4425   [
4426    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4427         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4428                        (match_operand:SI 2 "s_register_operand" "r")
4429                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4430          VQSHLQ_M_R))
4431   ]
4432   "TARGET_HAVE_MVE"
4433   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4434   [(set_attr "type" "mve_move")
4435    (set_attr "length""8")])
4436
4437 ;;
4438 ;; [vrev64q_m_u, vrev64q_m_s])
4439 ;;
4440 (define_insn "mve_vrev64q_m_<supf><mode>"
4441   [
4442    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4443         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4444                        (match_operand:MVE_2 2 "s_register_operand" "w")
4445                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4446          VREV64Q_M))
4447   ]
4448   "TARGET_HAVE_MVE"
4449   "vpst\;vrev64t.%#<V_sz_elem>\t%q0, %q2"
4450   [(set_attr "type" "mve_move")
4451    (set_attr "length""8")])
4452
4453 ;;
4454 ;; [vrshlq_m_n_s, vrshlq_m_n_u])
4455 ;;
4456 (define_insn "mve_vrshlq_m_n_<supf><mode>"
4457   [
4458    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4459         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4460                        (match_operand:SI 2 "s_register_operand" "r")
4461                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4462          VRSHLQ_M_N))
4463   ]
4464   "TARGET_HAVE_MVE"
4465   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4466   [(set_attr "type" "mve_move")
4467    (set_attr "length""8")])
4468
4469 ;;
4470 ;; [vshlq_m_r_u, vshlq_m_r_s])
4471 ;;
4472 (define_insn "mve_vshlq_m_r_<supf><mode>"
4473   [
4474    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4475         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4476                        (match_operand:SI 2 "s_register_operand" "r")
4477                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4478          VSHLQ_M_R))
4479   ]
4480   "TARGET_HAVE_MVE"
4481   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %2"
4482   [(set_attr "type" "mve_move")
4483    (set_attr "length""8")])
4484
4485 ;;
4486 ;; [vsliq_n_u, vsliq_n_s])
4487 ;;
4488 (define_insn "mve_vsliq_n_<supf><mode>"
4489   [
4490    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4491         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4492                        (match_operand:MVE_2 2 "s_register_operand" "w")
4493                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")]
4494          VSLIQ_N))
4495   ]
4496   "TARGET_HAVE_MVE"
4497   "vsli.%#<V_sz_elem>\t%q0, %q2, %3"
4498   [(set_attr "type" "mve_move")
4499 ])
4500
4501 ;;
4502 ;; [vsriq_n_u, vsriq_n_s])
4503 ;;
4504 (define_insn "mve_vsriq_n_<supf><mode>"
4505   [
4506    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4507         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4508                        (match_operand:MVE_2 2 "s_register_operand" "w")
4509                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
4510          VSRIQ_N))
4511   ]
4512   "TARGET_HAVE_MVE"
4513   "vsri.%#<V_sz_elem>\t%q0, %q2, %3"
4514   [(set_attr "type" "mve_move")
4515 ])
4516
4517 ;;
4518 ;; [vqdmlsdhxq_s])
4519 ;;
4520 (define_insn "mve_vqdmlsdhxq_s<mode>"
4521   [
4522    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4523         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4524                        (match_operand:MVE_2 2 "s_register_operand" "w")
4525                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4526          VQDMLSDHXQ_S))
4527   ]
4528   "TARGET_HAVE_MVE"
4529   "vqdmlsdhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4530   [(set_attr "type" "mve_move")
4531 ])
4532
4533 ;;
4534 ;; [vqdmlsdhq_s])
4535 ;;
4536 (define_insn "mve_vqdmlsdhq_s<mode>"
4537   [
4538    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4539         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4540                        (match_operand:MVE_2 2 "s_register_operand" "w")
4541                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4542          VQDMLSDHQ_S))
4543   ]
4544   "TARGET_HAVE_MVE"
4545   "vqdmlsdh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4546   [(set_attr "type" "mve_move")
4547 ])
4548
4549 ;;
4550 ;; [vqdmladhxq_s])
4551 ;;
4552 (define_insn "mve_vqdmladhxq_s<mode>"
4553   [
4554    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4555         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4556                        (match_operand:MVE_2 2 "s_register_operand" "w")
4557                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4558          VQDMLADHXQ_S))
4559   ]
4560   "TARGET_HAVE_MVE"
4561   "vqdmladhx.s%#<V_sz_elem>\t%q0, %q2, %q3"
4562   [(set_attr "type" "mve_move")
4563 ])
4564
4565 ;;
4566 ;; [vqdmladhq_s])
4567 ;;
4568 (define_insn "mve_vqdmladhq_s<mode>"
4569   [
4570    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
4571         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
4572                        (match_operand:MVE_2 2 "s_register_operand" "w")
4573                        (match_operand:MVE_2 3 "s_register_operand" "w")]
4574          VQDMLADHQ_S))
4575   ]
4576   "TARGET_HAVE_MVE"
4577   "vqdmladh.s%#<V_sz_elem>\t%q0, %q2, %q3"
4578   [(set_attr "type" "mve_move")
4579 ])
4580
4581 ;;
4582 ;; [vmlsdavaxq_s])
4583 ;;
4584 (define_insn "mve_vmlsdavaxq_s<mode>"
4585   [
4586    (set (match_operand:SI 0 "s_register_operand" "=e")
4587         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4588                     (match_operand:MVE_2 2 "s_register_operand" "w")
4589                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4590          VMLSDAVAXQ_S))
4591   ]
4592   "TARGET_HAVE_MVE"
4593   "vmlsdavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4594   [(set_attr "type" "mve_move")
4595 ])
4596
4597 ;;
4598 ;; [vmlsdavaq_s])
4599 ;;
4600 (define_insn "mve_vmlsdavaq_s<mode>"
4601   [
4602    (set (match_operand:SI 0 "s_register_operand" "=e")
4603         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4604                     (match_operand:MVE_2 2 "s_register_operand" "w")
4605                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4606          VMLSDAVAQ_S))
4607   ]
4608   "TARGET_HAVE_MVE"
4609   "vmlsdava.s%#<V_sz_elem>\t%0, %q2, %q3"
4610   [(set_attr "type" "mve_move")
4611 ])
4612
4613 ;;
4614 ;; [vmladavaxq_s])
4615 ;;
4616 (define_insn "mve_vmladavaxq_s<mode>"
4617   [
4618    (set (match_operand:SI 0 "s_register_operand" "=e")
4619         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
4620                     (match_operand:MVE_2 2 "s_register_operand" "w")
4621                     (match_operand:MVE_2 3 "s_register_operand" "w")]
4622          VMLADAVAXQ_S))
4623   ]
4624   "TARGET_HAVE_MVE"
4625   "vmladavax.s%#<V_sz_elem>\t%0, %q2, %q3"
4626   [(set_attr "type" "mve_move")
4627 ])
4628 ;;
4629 ;; [vabsq_m_f])
4630 ;;
4631 (define_insn "mve_vabsq_m_f<mode>"
4632   [
4633    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4634         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4635                        (match_operand:MVE_0 2 "s_register_operand" "w")
4636                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4637          VABSQ_M_F))
4638   ]
4639   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4640   "vpst\;vabst.f%#<V_sz_elem>   %q0, %q2"
4641   [(set_attr "type" "mve_move")
4642    (set_attr "length""8")])
4643
4644 ;;
4645 ;; [vaddlvaq_p_s vaddlvaq_p_u])
4646 ;;
4647 (define_insn "mve_vaddlvaq_p_<supf>v4si"
4648   [
4649    (set (match_operand:DI 0 "s_register_operand" "=r")
4650         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
4651                        (match_operand:V4SI 2 "s_register_operand" "w")
4652                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4653          VADDLVAQ_P))
4654   ]
4655   "TARGET_HAVE_MVE"
4656   "vpst\;vaddlvat.<supf>32 %Q0, %R0, %q2"
4657   [(set_attr "type" "mve_move")
4658    (set_attr "length""8")])
4659 ;;
4660 ;; [vcmlaq_f])
4661 ;;
4662 (define_insn "mve_vcmlaq_f<mode>"
4663   [
4664    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4665         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4666                        (match_operand:MVE_0 2 "s_register_operand" "w")
4667                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4668          VCMLAQ_F))
4669   ]
4670   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4671   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #0"
4672   [(set_attr "type" "mve_move")
4673 ])
4674
4675 ;;
4676 ;; [vcmlaq_rot180_f])
4677 ;;
4678 (define_insn "mve_vcmlaq_rot180_f<mode>"
4679   [
4680    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4681         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4682                        (match_operand:MVE_0 2 "s_register_operand" "w")
4683                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4684          VCMLAQ_ROT180_F))
4685   ]
4686   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4687   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #180"
4688   [(set_attr "type" "mve_move")
4689 ])
4690
4691 ;;
4692 ;; [vcmlaq_rot270_f])
4693 ;;
4694 (define_insn "mve_vcmlaq_rot270_f<mode>"
4695   [
4696    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4697         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4698                        (match_operand:MVE_0 2 "s_register_operand" "w")
4699                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4700          VCMLAQ_ROT270_F))
4701   ]
4702   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4703   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #270"
4704   [(set_attr "type" "mve_move")
4705 ])
4706
4707 ;;
4708 ;; [vcmlaq_rot90_f])
4709 ;;
4710 (define_insn "mve_vcmlaq_rot90_f<mode>"
4711   [
4712    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4713         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4714                        (match_operand:MVE_0 2 "s_register_operand" "w")
4715                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4716          VCMLAQ_ROT90_F))
4717   ]
4718   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4719   "vcmla.f%#<V_sz_elem> %q0, %q2, %q3, #90"
4720   [(set_attr "type" "mve_move")
4721 ])
4722
4723 ;;
4724 ;; [vcmpeqq_m_n_f])
4725 ;;
4726 (define_insn "mve_vcmpeqq_m_n_f<mode>"
4727   [
4728    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4729         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4730                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4731                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4732          VCMPEQQ_M_N_F))
4733   ]
4734   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4735   "vpst\;vcmpt.f%#<V_sz_elem>   eq, %q1, %2"
4736   [(set_attr "type" "mve_move")
4737    (set_attr "length""8")])
4738
4739 ;;
4740 ;; [vcmpgeq_m_f])
4741 ;;
4742 (define_insn "mve_vcmpgeq_m_f<mode>"
4743   [
4744    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4745         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4746                        (match_operand:MVE_0 2 "s_register_operand" "w")
4747                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4748          VCMPGEQ_M_F))
4749   ]
4750   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4751   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %q2"
4752   [(set_attr "type" "mve_move")
4753    (set_attr "length""8")])
4754
4755 ;;
4756 ;; [vcmpgeq_m_n_f])
4757 ;;
4758 (define_insn "mve_vcmpgeq_m_n_f<mode>"
4759   [
4760    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4761         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4762                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4763                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4764          VCMPGEQ_M_N_F))
4765   ]
4766   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4767   "vpst\;vcmpt.f%#<V_sz_elem>   ge, %q1, %2"
4768   [(set_attr "type" "mve_move")
4769    (set_attr "length""8")])
4770
4771 ;;
4772 ;; [vcmpgtq_m_f])
4773 ;;
4774 (define_insn "mve_vcmpgtq_m_f<mode>"
4775   [
4776    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4777         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4778                        (match_operand:MVE_0 2 "s_register_operand" "w")
4779                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4780          VCMPGTQ_M_F))
4781   ]
4782   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4783   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %q2"
4784   [(set_attr "type" "mve_move")
4785    (set_attr "length""8")])
4786
4787 ;;
4788 ;; [vcmpgtq_m_n_f])
4789 ;;
4790 (define_insn "mve_vcmpgtq_m_n_f<mode>"
4791   [
4792    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4793         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4794                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4795                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4796          VCMPGTQ_M_N_F))
4797   ]
4798   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4799   "vpst\;vcmpt.f%#<V_sz_elem>   gt, %q1, %2"
4800   [(set_attr "type" "mve_move")
4801    (set_attr "length""8")])
4802
4803 ;;
4804 ;; [vcmpleq_m_f])
4805 ;;
4806 (define_insn "mve_vcmpleq_m_f<mode>"
4807   [
4808    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4809         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4810                        (match_operand:MVE_0 2 "s_register_operand" "w")
4811                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4812          VCMPLEQ_M_F))
4813   ]
4814   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4815   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %q2"
4816   [(set_attr "type" "mve_move")
4817    (set_attr "length""8")])
4818
4819 ;;
4820 ;; [vcmpleq_m_n_f])
4821 ;;
4822 (define_insn "mve_vcmpleq_m_n_f<mode>"
4823   [
4824    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4825         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4826                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4827                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4828          VCMPLEQ_M_N_F))
4829   ]
4830   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4831   "vpst\;vcmpt.f%#<V_sz_elem>   le, %q1, %2"
4832   [(set_attr "type" "mve_move")
4833    (set_attr "length""8")])
4834
4835 ;;
4836 ;; [vcmpltq_m_f])
4837 ;;
4838 (define_insn "mve_vcmpltq_m_f<mode>"
4839   [
4840    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4841         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4842                        (match_operand:MVE_0 2 "s_register_operand" "w")
4843                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4844          VCMPLTQ_M_F))
4845   ]
4846   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4847   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %q2"
4848   [(set_attr "type" "mve_move")
4849    (set_attr "length""8")])
4850
4851 ;;
4852 ;; [vcmpltq_m_n_f])
4853 ;;
4854 (define_insn "mve_vcmpltq_m_n_f<mode>"
4855   [
4856    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4857         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4858                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4859                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4860          VCMPLTQ_M_N_F))
4861   ]
4862   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4863   "vpst\;vcmpt.f%#<V_sz_elem>   lt, %q1, %2"
4864   [(set_attr "type" "mve_move")
4865    (set_attr "length""8")])
4866
4867 ;;
4868 ;; [vcmpneq_m_f])
4869 ;;
4870 (define_insn "mve_vcmpneq_m_f<mode>"
4871   [
4872    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4873         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4874                        (match_operand:MVE_0 2 "s_register_operand" "w")
4875                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4876          VCMPNEQ_M_F))
4877   ]
4878   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4879   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %q2"
4880   [(set_attr "type" "mve_move")
4881    (set_attr "length""8")])
4882
4883 ;;
4884 ;; [vcmpneq_m_n_f])
4885 ;;
4886 (define_insn "mve_vcmpneq_m_n_f<mode>"
4887   [
4888    (set (match_operand:HI 0 "vpr_register_operand" "=Up")
4889         (unspec:HI [(match_operand:MVE_0 1 "s_register_operand" "w")
4890                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4891                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4892          VCMPNEQ_M_N_F))
4893   ]
4894   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4895   "vpst\;vcmpt.f%#<V_sz_elem>   ne, %q1, %2"
4896   [(set_attr "type" "mve_move")
4897    (set_attr "length""8")])
4898
4899 ;;
4900 ;; [vcvtbq_m_f16_f32])
4901 ;;
4902 (define_insn "mve_vcvtbq_m_f16_f32v8hf"
4903   [
4904    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4905         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4906                        (match_operand:V4SF 2 "s_register_operand" "w")
4907                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4908          VCVTBQ_M_F16_F32))
4909   ]
4910   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4911   "vpst\;vcvtbt.f16.f32 %q0, %q2"
4912   [(set_attr "type" "mve_move")
4913    (set_attr "length""8")])
4914
4915 ;;
4916 ;; [vcvtbq_m_f32_f16])
4917 ;;
4918 (define_insn "mve_vcvtbq_m_f32_f16v4sf"
4919   [
4920    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4921         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4922                        (match_operand:V8HF 2 "s_register_operand" "w")
4923                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4924          VCVTBQ_M_F32_F16))
4925   ]
4926   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4927   "vpst\;vcvtbt.f32.f16 %q0, %q2"
4928   [(set_attr "type" "mve_move")
4929    (set_attr "length""8")])
4930
4931 ;;
4932 ;; [vcvttq_m_f16_f32])
4933 ;;
4934 (define_insn "mve_vcvttq_m_f16_f32v8hf"
4935   [
4936    (set (match_operand:V8HF 0 "s_register_operand" "=w")
4937         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
4938                        (match_operand:V4SF 2 "s_register_operand" "w")
4939                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4940          VCVTTQ_M_F16_F32))
4941   ]
4942   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4943   "vpst\;vcvttt.f16.f32 %q0, %q2"
4944   [(set_attr "type" "mve_move")
4945    (set_attr "length""8")])
4946
4947 ;;
4948 ;; [vcvttq_m_f32_f16])
4949 ;;
4950 (define_insn "mve_vcvttq_m_f32_f16v4sf"
4951   [
4952    (set (match_operand:V4SF 0 "s_register_operand" "=w")
4953         (unspec:V4SF [(match_operand:V4SF 1 "s_register_operand" "0")
4954                        (match_operand:V8HF 2 "s_register_operand" "w")
4955                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4956          VCVTTQ_M_F32_F16))
4957   ]
4958   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4959   "vpst\;vcvttt.f32.f16 %q0, %q2"
4960   [(set_attr "type" "mve_move")
4961    (set_attr "length""8")])
4962
4963 ;;
4964 ;; [vdupq_m_n_f])
4965 ;;
4966 (define_insn "mve_vdupq_m_n_f<mode>"
4967   [
4968    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4969         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4970                        (match_operand:<V_elem> 2 "s_register_operand" "r")
4971                        (match_operand:HI 3 "vpr_register_operand" "Up")]
4972          VDUPQ_M_N_F))
4973   ]
4974   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4975   "vpst\;vdupt.%#<V_sz_elem>    %q0, %2"
4976   [(set_attr "type" "mve_move")
4977    (set_attr "length""8")])
4978
4979 ;;
4980 ;; [vfmaq_f])
4981 ;;
4982 (define_insn "mve_vfmaq_f<mode>"
4983   [
4984    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
4985         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
4986                        (match_operand:MVE_0 2 "s_register_operand" "w")
4987                        (match_operand:MVE_0 3 "s_register_operand" "w")]
4988          VFMAQ_F))
4989   ]
4990   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
4991   "vfma.f%#<V_sz_elem>  %q0, %q2, %q3"
4992   [(set_attr "type" "mve_move")
4993 ])
4994
4995 ;;
4996 ;; [vfmaq_n_f])
4997 ;;
4998 (define_insn "mve_vfmaq_n_f<mode>"
4999   [
5000    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5001         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5002                        (match_operand:MVE_0 2 "s_register_operand" "w")
5003                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5004          VFMAQ_N_F))
5005   ]
5006   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5007   "vfma.f%#<V_sz_elem>  %q0, %q2, %3"
5008   [(set_attr "type" "mve_move")
5009 ])
5010
5011 ;;
5012 ;; [vfmasq_n_f])
5013 ;;
5014 (define_insn "mve_vfmasq_n_f<mode>"
5015   [
5016    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5017         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5018                        (match_operand:MVE_0 2 "s_register_operand" "w")
5019                        (match_operand:<V_elem> 3 "s_register_operand" "r")]
5020          VFMASQ_N_F))
5021   ]
5022   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5023   "vfmas.f%#<V_sz_elem> %q0, %q2, %3"
5024   [(set_attr "type" "mve_move")
5025 ])
5026 ;;
5027 ;; [vfmsq_f])
5028 ;;
5029 (define_insn "mve_vfmsq_f<mode>"
5030   [
5031    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5032         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5033                        (match_operand:MVE_0 2 "s_register_operand" "w")
5034                        (match_operand:MVE_0 3 "s_register_operand" "w")]
5035          VFMSQ_F))
5036   ]
5037   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5038   "vfms.f%#<V_sz_elem>  %q0, %q2, %q3"
5039   [(set_attr "type" "mve_move")
5040 ])
5041
5042 ;;
5043 ;; [vmaxnmaq_m_f])
5044 ;;
5045 (define_insn "mve_vmaxnmaq_m_f<mode>"
5046   [
5047    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5048         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5049                        (match_operand:MVE_0 2 "s_register_operand" "w")
5050                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5051          VMAXNMAQ_M_F))
5052   ]
5053   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5054   "vpst\;vmaxnmat.f%#<V_sz_elem>        %q0, %q2"
5055   [(set_attr "type" "mve_move")
5056    (set_attr "length""8")])
5057 ;;
5058 ;; [vmaxnmavq_p_f])
5059 ;;
5060 (define_insn "mve_vmaxnmavq_p_f<mode>"
5061   [
5062    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5063         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5064                        (match_operand:MVE_0 2 "s_register_operand" "w")
5065                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5066          VMAXNMAVQ_P_F))
5067   ]
5068   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5069   "vpst\;vmaxnmavt.f%#<V_sz_elem>       %0, %q2"
5070   [(set_attr "type" "mve_move")
5071    (set_attr "length""8")])
5072
5073 ;;
5074 ;; [vmaxnmvq_p_f])
5075 ;;
5076 (define_insn "mve_vmaxnmvq_p_f<mode>"
5077   [
5078    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5079         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5080                        (match_operand:MVE_0 2 "s_register_operand" "w")
5081                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5082          VMAXNMVQ_P_F))
5083   ]
5084   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5085   "vpst\;vmaxnmvt.f%#<V_sz_elem>        %0, %q2"
5086   [(set_attr "type" "mve_move")
5087    (set_attr "length""8")])
5088 ;;
5089 ;; [vminnmaq_m_f])
5090 ;;
5091 (define_insn "mve_vminnmaq_m_f<mode>"
5092   [
5093    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5094         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5095                        (match_operand:MVE_0 2 "s_register_operand" "w")
5096                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5097          VMINNMAQ_M_F))
5098   ]
5099   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5100   "vpst\;vminnmat.f%#<V_sz_elem>        %q0, %q2"
5101   [(set_attr "type" "mve_move")
5102    (set_attr "length""8")])
5103
5104 ;;
5105 ;; [vminnmavq_p_f])
5106 ;;
5107 (define_insn "mve_vminnmavq_p_f<mode>"
5108   [
5109    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5110         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5111                        (match_operand:MVE_0 2 "s_register_operand" "w")
5112                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5113          VMINNMAVQ_P_F))
5114   ]
5115   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5116   "vpst\;vminnmavt.f%#<V_sz_elem>       %0, %q2"
5117   [(set_attr "type" "mve_move")
5118    (set_attr "length""8")])
5119 ;;
5120 ;; [vminnmvq_p_f])
5121 ;;
5122 (define_insn "mve_vminnmvq_p_f<mode>"
5123   [
5124    (set (match_operand:<V_elem> 0 "s_register_operand" "=r")
5125         (unspec:<V_elem> [(match_operand:<V_elem> 1 "s_register_operand" "0")
5126                        (match_operand:MVE_0 2 "s_register_operand" "w")
5127                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5128          VMINNMVQ_P_F))
5129   ]
5130   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5131   "vpst\;vminnmvt.f%#<V_sz_elem>        %0, %q2"
5132   [(set_attr "type" "mve_move")
5133    (set_attr "length""8")])
5134
5135 ;;
5136 ;; [vmlaldavaq_s, vmlaldavaq_u])
5137 ;;
5138 (define_insn "mve_vmlaldavaq_<supf><mode>"
5139   [
5140    (set (match_operand:DI 0 "s_register_operand" "=r")
5141         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5142                        (match_operand:MVE_5 2 "s_register_operand" "w")
5143                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5144          VMLALDAVAQ))
5145   ]
5146   "TARGET_HAVE_MVE"
5147   "vmlaldava.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5148   [(set_attr "type" "mve_move")
5149 ])
5150
5151 ;;
5152 ;; [vmlaldavaxq_s])
5153 ;;
5154 (define_insn "mve_vmlaldavaxq_s<mode>"
5155   [
5156    (set (match_operand:DI 0 "s_register_operand" "=r")
5157         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5158                        (match_operand:MVE_5 2 "s_register_operand" "w")
5159                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5160          VMLALDAVAXQ_S))
5161   ]
5162   "TARGET_HAVE_MVE"
5163   "vmlaldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5164   [(set_attr "type" "mve_move")
5165 ])
5166
5167 ;;
5168 ;; [vmlaldavq_p_u, vmlaldavq_p_s])
5169 ;;
5170 (define_insn "mve_vmlaldavq_p_<supf><mode>"
5171   [
5172    (set (match_operand:DI 0 "s_register_operand" "=r")
5173         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5174                        (match_operand:MVE_5 2 "s_register_operand" "w")
5175                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5176          VMLALDAVQ_P))
5177   ]
5178   "TARGET_HAVE_MVE"
5179   "vpst\;vmlaldavt.<supf>%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5180   [(set_attr "type" "mve_move")
5181    (set_attr "length""8")])
5182
5183 ;;
5184 ;; [vmlaldavxq_p_s])
5185 ;;
5186 (define_insn "mve_vmlaldavxq_p_s<mode>"
5187   [
5188    (set (match_operand:DI 0 "s_register_operand" "=r")
5189         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5190                        (match_operand:MVE_5 2 "s_register_operand" "w")
5191                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5192          VMLALDAVXQ_P_S))
5193   ]
5194   "TARGET_HAVE_MVE"
5195   "vpst\;vmlaldavxt.s%#<V_sz_elem>\t%Q0, %R0, %q1, %q2"
5196   [(set_attr "type" "mve_move")
5197    (set_attr "length""8")])
5198 ;;
5199 ;; [vmlsldavaq_s])
5200 ;;
5201 (define_insn "mve_vmlsldavaq_s<mode>"
5202   [
5203    (set (match_operand:DI 0 "s_register_operand" "=r")
5204         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5205                        (match_operand:MVE_5 2 "s_register_operand" "w")
5206                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5207          VMLSLDAVAQ_S))
5208   ]
5209   "TARGET_HAVE_MVE"
5210   "vmlsldava.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5211   [(set_attr "type" "mve_move")
5212 ])
5213
5214 ;;
5215 ;; [vmlsldavaxq_s])
5216 ;;
5217 (define_insn "mve_vmlsldavaxq_s<mode>"
5218   [
5219    (set (match_operand:DI 0 "s_register_operand" "=r")
5220         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5221                        (match_operand:MVE_5 2 "s_register_operand" "w")
5222                        (match_operand:MVE_5 3 "s_register_operand" "w")]
5223          VMLSLDAVAXQ_S))
5224   ]
5225   "TARGET_HAVE_MVE"
5226   "vmlsldavax.s%#<V_sz_elem> %Q0, %R0, %q2, %q3"
5227   [(set_attr "type" "mve_move")
5228 ])
5229
5230 ;;
5231 ;; [vmlsldavq_p_s])
5232 ;;
5233 (define_insn "mve_vmlsldavq_p_s<mode>"
5234   [
5235    (set (match_operand:DI 0 "s_register_operand" "=r")
5236         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5237                        (match_operand:MVE_5 2 "s_register_operand" "w")
5238                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5239          VMLSLDAVQ_P_S))
5240   ]
5241   "TARGET_HAVE_MVE"
5242   "vpst\;vmlsldavt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5243   [(set_attr "type" "mve_move")
5244    (set_attr "length""8")])
5245
5246 ;;
5247 ;; [vmlsldavxq_p_s])
5248 ;;
5249 (define_insn "mve_vmlsldavxq_p_s<mode>"
5250   [
5251    (set (match_operand:DI 0 "s_register_operand" "=r")
5252         (unspec:DI [(match_operand:MVE_5 1 "s_register_operand" "w")
5253                        (match_operand:MVE_5 2 "s_register_operand" "w")
5254                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5255          VMLSLDAVXQ_P_S))
5256   ]
5257   "TARGET_HAVE_MVE"
5258   "vpst\;vmlsldavxt.s%#<V_sz_elem> %Q0, %R0, %q1, %q2"
5259   [(set_attr "type" "mve_move")
5260    (set_attr "length""8")])
5261 ;;
5262 ;; [vmovlbq_m_u, vmovlbq_m_s])
5263 ;;
5264 (define_insn "mve_vmovlbq_m_<supf><mode>"
5265   [
5266    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5267         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5268                        (match_operand:MVE_3 2 "s_register_operand" "w")
5269                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5270          VMOVLBQ_M))
5271   ]
5272   "TARGET_HAVE_MVE"
5273   "vpst\;vmovlbt.<supf>%#<V_sz_elem>    %q0, %q2"
5274   [(set_attr "type" "mve_move")
5275    (set_attr "length""8")])
5276 ;;
5277 ;; [vmovltq_m_u, vmovltq_m_s])
5278 ;;
5279 (define_insn "mve_vmovltq_m_<supf><mode>"
5280   [
5281    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
5282         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
5283                        (match_operand:MVE_3 2 "s_register_operand" "w")
5284                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5285          VMOVLTQ_M))
5286   ]
5287   "TARGET_HAVE_MVE"
5288   "vpst\;vmovltt.<supf>%#<V_sz_elem>    %q0, %q2"
5289   [(set_attr "type" "mve_move")
5290    (set_attr "length""8")])
5291 ;;
5292 ;; [vmovnbq_m_u, vmovnbq_m_s])
5293 ;;
5294 (define_insn "mve_vmovnbq_m_<supf><mode>"
5295   [
5296    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5297         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5298                        (match_operand:MVE_5 2 "s_register_operand" "w")
5299                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5300          VMOVNBQ_M))
5301   ]
5302   "TARGET_HAVE_MVE"
5303   "vpst\;vmovnbt.i%#<V_sz_elem> %q0, %q2"
5304   [(set_attr "type" "mve_move")
5305    (set_attr "length""8")])
5306
5307 ;;
5308 ;; [vmovntq_m_u, vmovntq_m_s])
5309 ;;
5310 (define_insn "mve_vmovntq_m_<supf><mode>"
5311   [
5312    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5313         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5314                        (match_operand:MVE_5 2 "s_register_operand" "w")
5315                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5316          VMOVNTQ_M))
5317   ]
5318   "TARGET_HAVE_MVE"
5319   "vpst\;vmovntt.i%#<V_sz_elem> %q0, %q2"
5320   [(set_attr "type" "mve_move")
5321    (set_attr "length""8")])
5322
5323 ;;
5324 ;; [vmvnq_m_n_u, vmvnq_m_n_s])
5325 ;;
5326 (define_insn "mve_vmvnq_m_n_<supf><mode>"
5327   [
5328    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5329         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5330                        (match_operand:SI 2 "immediate_operand" "i")
5331                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5332          VMVNQ_M_N))
5333   ]
5334   "TARGET_HAVE_MVE"
5335   "vpst\;vmvnt.i%#<V_sz_elem>   %q0, %2"
5336   [(set_attr "type" "mve_move")
5337    (set_attr "length""8")])
5338 ;;
5339 ;; [vnegq_m_f])
5340 ;;
5341 (define_insn "mve_vnegq_m_f<mode>"
5342   [
5343    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5344         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5345                        (match_operand:MVE_0 2 "s_register_operand" "w")
5346                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5347          VNEGQ_M_F))
5348   ]
5349   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5350   "vpst\;vnegt.f%#<V_sz_elem>   %q0, %q2"
5351   [(set_attr "type" "mve_move")
5352    (set_attr "length""8")])
5353
5354 ;;
5355 ;; [vorrq_m_n_s, vorrq_m_n_u])
5356 ;;
5357 (define_insn "mve_vorrq_m_n_<supf><mode>"
5358   [
5359    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5360         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5361                        (match_operand:SI 2 "immediate_operand" "i")
5362                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5363          VORRQ_M_N))
5364   ]
5365   "TARGET_HAVE_MVE"
5366   "vpst\;vorrt.i%#<V_sz_elem>   %q0, %2"
5367   [(set_attr "type" "mve_move")
5368    (set_attr "length""8")])
5369 ;;
5370 ;; [vpselq_f])
5371 ;;
5372 (define_insn "mve_vpselq_f<mode>"
5373   [
5374    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5375         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "w")
5376                        (match_operand:MVE_0 2 "s_register_operand" "w")
5377                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5378          VPSELQ_F))
5379   ]
5380   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5381   "vpsel %q0, %q1, %q2"
5382   [(set_attr "type" "mve_move")
5383 ])
5384
5385 ;;
5386 ;; [vqmovnbq_m_s, vqmovnbq_m_u])
5387 ;;
5388 (define_insn "mve_vqmovnbq_m_<supf><mode>"
5389   [
5390    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5391         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5392                        (match_operand:MVE_5 2 "s_register_operand" "w")
5393                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5394          VQMOVNBQ_M))
5395   ]
5396   "TARGET_HAVE_MVE"
5397   "vpst\;vqmovnbt.<supf>%#<V_sz_elem>   %q0, %q2"
5398   [(set_attr "type" "mve_move")
5399    (set_attr "length""8")])
5400
5401 ;;
5402 ;; [vqmovntq_m_u, vqmovntq_m_s])
5403 ;;
5404 (define_insn "mve_vqmovntq_m_<supf><mode>"
5405   [
5406    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5407         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5408                        (match_operand:MVE_5 2 "s_register_operand" "w")
5409                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5410          VQMOVNTQ_M))
5411   ]
5412   "TARGET_HAVE_MVE"
5413   "vpst\;vqmovntt.<supf>%#<V_sz_elem>   %q0, %q2"
5414   [(set_attr "type" "mve_move")
5415    (set_attr "length""8")])
5416
5417 ;;
5418 ;; [vqmovunbq_m_s])
5419 ;;
5420 (define_insn "mve_vqmovunbq_m_s<mode>"
5421   [
5422    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5423         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5424                        (match_operand:MVE_5 2 "s_register_operand" "w")
5425                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5426          VQMOVUNBQ_M_S))
5427   ]
5428   "TARGET_HAVE_MVE"
5429   "vpst\;vqmovunbt.s%#<V_sz_elem>       %q0, %q2"
5430   [(set_attr "type" "mve_move")
5431    (set_attr "length""8")])
5432
5433 ;;
5434 ;; [vqmovuntq_m_s])
5435 ;;
5436 (define_insn "mve_vqmovuntq_m_s<mode>"
5437   [
5438    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5439         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5440                        (match_operand:MVE_5 2 "s_register_operand" "w")
5441                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5442          VQMOVUNTQ_M_S))
5443   ]
5444   "TARGET_HAVE_MVE"
5445   "vpst\;vqmovuntt.s%#<V_sz_elem>       %q0, %q2"
5446   [(set_attr "type" "mve_move")
5447    (set_attr "length""8")])
5448
5449 ;;
5450 ;; [vqrshrntq_n_u, vqrshrntq_n_s])
5451 ;;
5452 (define_insn "mve_vqrshrntq_n_<supf><mode>"
5453   [
5454    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5455         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5456                        (match_operand:MVE_5 2 "s_register_operand" "w")
5457                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5458          VQRSHRNTQ_N))
5459   ]
5460   "TARGET_HAVE_MVE"
5461   "vqrshrnt.<supf>%#<V_sz_elem> %q0, %q2, %3"
5462   [(set_attr "type" "mve_move")
5463 ])
5464
5465 ;;
5466 ;; [vqrshruntq_n_s])
5467 ;;
5468 (define_insn "mve_vqrshruntq_n_s<mode>"
5469   [
5470    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5471         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5472                        (match_operand:MVE_5 2 "s_register_operand" "w")
5473                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5474          VQRSHRUNTQ_N_S))
5475   ]
5476   "TARGET_HAVE_MVE"
5477   "vqrshrunt.s%#<V_sz_elem>     %q0, %q2, %3"
5478   [(set_attr "type" "mve_move")
5479 ])
5480
5481 ;;
5482 ;; [vqshrnbq_n_u, vqshrnbq_n_s])
5483 ;;
5484 (define_insn "mve_vqshrnbq_n_<supf><mode>"
5485   [
5486    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5487         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5488                        (match_operand:MVE_5 2 "s_register_operand" "w")
5489                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5490          VQSHRNBQ_N))
5491   ]
5492   "TARGET_HAVE_MVE"
5493   "vqshrnb.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
5494   [(set_attr "type" "mve_move")
5495 ])
5496
5497 ;;
5498 ;; [vqshrntq_n_u, vqshrntq_n_s])
5499 ;;
5500 (define_insn "mve_vqshrntq_n_<supf><mode>"
5501   [
5502    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5503         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5504                        (match_operand:MVE_5 2 "s_register_operand" "w")
5505                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5506          VQSHRNTQ_N))
5507   ]
5508   "TARGET_HAVE_MVE"
5509   "vqshrnt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
5510   [(set_attr "type" "mve_move")
5511 ])
5512
5513 ;;
5514 ;; [vqshrunbq_n_s])
5515 ;;
5516 (define_insn "mve_vqshrunbq_n_s<mode>"
5517   [
5518    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5519         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5520                        (match_operand:MVE_5 2 "s_register_operand" "w")
5521                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5522          VQSHRUNBQ_N_S))
5523   ]
5524   "TARGET_HAVE_MVE"
5525   "vqshrunb.s%#<V_sz_elem>      %q0, %q2, %3"
5526   [(set_attr "type" "mve_move")
5527 ])
5528
5529 ;;
5530 ;; [vqshruntq_n_s])
5531 ;;
5532 (define_insn "mve_vqshruntq_n_s<mode>"
5533   [
5534    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5535         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5536                        (match_operand:MVE_5 2 "s_register_operand" "w")
5537                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5538          VQSHRUNTQ_N_S))
5539   ]
5540   "TARGET_HAVE_MVE"
5541   "vqshrunt.s%#<V_sz_elem>      %q0, %q2, %3"
5542   [(set_attr "type" "mve_move")
5543 ])
5544
5545 ;;
5546 ;; [vrev32q_m_f])
5547 ;;
5548 (define_insn "mve_vrev32q_m_fv8hf"
5549   [
5550    (set (match_operand:V8HF 0 "s_register_operand" "=w")
5551         (unspec:V8HF [(match_operand:V8HF 1 "s_register_operand" "0")
5552                        (match_operand:V8HF 2 "s_register_operand" "w")
5553                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5554          VREV32Q_M_F))
5555   ]
5556   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5557   "vpst\;vrev32t.16 %q0, %q2"
5558   [(set_attr "type" "mve_move")
5559    (set_attr "length""8")])
5560
5561 ;;
5562 ;; [vrev32q_m_s, vrev32q_m_u])
5563 ;;
5564 (define_insn "mve_vrev32q_m_<supf><mode>"
5565   [
5566    (set (match_operand:MVE_3 0 "s_register_operand" "=w")
5567         (unspec:MVE_3 [(match_operand:MVE_3 1 "s_register_operand" "0")
5568                        (match_operand:MVE_3 2 "s_register_operand" "w")
5569                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5570          VREV32Q_M))
5571   ]
5572   "TARGET_HAVE_MVE"
5573   "vpst\;vrev32t.%#<V_sz_elem>  %q0, %q2"
5574   [(set_attr "type" "mve_move")
5575    (set_attr "length""8")])
5576
5577 ;;
5578 ;; [vrev64q_m_f])
5579 ;;
5580 (define_insn "mve_vrev64q_m_f<mode>"
5581   [
5582    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5583         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5584                        (match_operand:MVE_0 2 "s_register_operand" "w")
5585                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5586          VREV64Q_M_F))
5587   ]
5588   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5589   "vpst\;vrev64t.%#<V_sz_elem>  %q0, %q2"
5590   [(set_attr "type" "mve_move")
5591    (set_attr "length""8")])
5592
5593 ;;
5594 ;; [vrmlaldavhaxq_s])
5595 ;;
5596 (define_insn "mve_vrmlaldavhaxq_sv4si"
5597   [
5598    (set (match_operand:DI 0 "s_register_operand" "=r")
5599         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5600                        (match_operand:V4SI 2 "s_register_operand" "w")
5601                        (match_operand:V4SI 3 "s_register_operand" "w")]
5602          VRMLALDAVHAXQ_S))
5603   ]
5604   "TARGET_HAVE_MVE"
5605   "vrmlaldavhax.s32 %Q0, %R0, %q2, %q3"
5606   [(set_attr "type" "mve_move")
5607 ])
5608
5609 ;;
5610 ;; [vrmlaldavhxq_p_s])
5611 ;;
5612 (define_insn "mve_vrmlaldavhxq_p_sv4si"
5613   [
5614    (set (match_operand:DI 0 "s_register_operand" "=r")
5615         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5616                        (match_operand:V4SI 2 "s_register_operand" "w")
5617                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5618          VRMLALDAVHXQ_P_S))
5619   ]
5620   "TARGET_HAVE_MVE"
5621   "vpst\;vrmlaldavhxt.s32 %Q0, %R0, %q1, %q2"
5622   [(set_attr "type" "mve_move")
5623    (set_attr "length""8")])
5624
5625 ;;
5626 ;; [vrmlsldavhaxq_s])
5627 ;;
5628 (define_insn "mve_vrmlsldavhaxq_sv4si"
5629   [
5630    (set (match_operand:DI 0 "s_register_operand" "=r")
5631         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5632                        (match_operand:V4SI 2 "s_register_operand" "w")
5633                        (match_operand:V4SI 3 "s_register_operand" "w")]
5634          VRMLSLDAVHAXQ_S))
5635   ]
5636   "TARGET_HAVE_MVE"
5637   "vrmlsldavhax.s32 %Q0, %R0, %q2, %q3"
5638   [(set_attr "type" "mve_move")
5639 ])
5640
5641 ;;
5642 ;; [vrmlsldavhq_p_s])
5643 ;;
5644 (define_insn "mve_vrmlsldavhq_p_sv4si"
5645   [
5646    (set (match_operand:DI 0 "s_register_operand" "=r")
5647         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5648                        (match_operand:V4SI 2 "s_register_operand" "w")
5649                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5650          VRMLSLDAVHQ_P_S))
5651   ]
5652   "TARGET_HAVE_MVE"
5653   "vpst\;vrmlsldavht.s32 %Q0, %R0, %q1, %q2"
5654   [(set_attr "type" "mve_move")
5655    (set_attr "length""8")])
5656
5657 ;;
5658 ;; [vrmlsldavhxq_p_s])
5659 ;;
5660 (define_insn "mve_vrmlsldavhxq_p_sv4si"
5661   [
5662    (set (match_operand:DI 0 "s_register_operand" "=r")
5663         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5664                        (match_operand:V4SI 2 "s_register_operand" "w")
5665                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5666          VRMLSLDAVHXQ_P_S))
5667   ]
5668   "TARGET_HAVE_MVE"
5669   "vpst\;vrmlsldavhxt.s32 %Q0, %R0, %q1, %q2"
5670   [(set_attr "type" "mve_move")
5671    (set_attr "length""8")])
5672
5673 ;;
5674 ;; [vrndaq_m_f])
5675 ;;
5676 (define_insn "mve_vrndaq_m_f<mode>"
5677   [
5678    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5679         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5680                        (match_operand:MVE_0 2 "s_register_operand" "w")
5681                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5682          VRNDAQ_M_F))
5683   ]
5684   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5685   "vpst\;vrintat.f%#<V_sz_elem> %q0, %q2"
5686   [(set_attr "type" "mve_move")
5687    (set_attr "length""8")])
5688
5689 ;;
5690 ;; [vrndmq_m_f])
5691 ;;
5692 (define_insn "mve_vrndmq_m_f<mode>"
5693   [
5694    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5695         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5696                        (match_operand:MVE_0 2 "s_register_operand" "w")
5697                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5698          VRNDMQ_M_F))
5699   ]
5700   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5701   "vpst\;vrintmt.f%#<V_sz_elem> %q0, %q2"
5702   [(set_attr "type" "mve_move")
5703    (set_attr "length""8")])
5704
5705 ;;
5706 ;; [vrndnq_m_f])
5707 ;;
5708 (define_insn "mve_vrndnq_m_f<mode>"
5709   [
5710    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5711         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5712                        (match_operand:MVE_0 2 "s_register_operand" "w")
5713                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5714          VRNDNQ_M_F))
5715   ]
5716   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5717   "vpst\;vrintnt.f%#<V_sz_elem> %q0, %q2"
5718   [(set_attr "type" "mve_move")
5719    (set_attr "length""8")])
5720
5721 ;;
5722 ;; [vrndpq_m_f])
5723 ;;
5724 (define_insn "mve_vrndpq_m_f<mode>"
5725   [
5726    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5727         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5728                        (match_operand:MVE_0 2 "s_register_operand" "w")
5729                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5730          VRNDPQ_M_F))
5731   ]
5732   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5733   "vpst\;vrintpt.f%#<V_sz_elem> %q0, %q2"
5734   [(set_attr "type" "mve_move")
5735    (set_attr "length""8")])
5736
5737 ;;
5738 ;; [vrndxq_m_f])
5739 ;;
5740 (define_insn "mve_vrndxq_m_f<mode>"
5741   [
5742    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
5743         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
5744                        (match_operand:MVE_0 2 "s_register_operand" "w")
5745                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5746          VRNDXQ_M_F))
5747   ]
5748   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5749   "vpst\;vrintxt.f%#<V_sz_elem> %q0, %q2"
5750   [(set_attr "type" "mve_move")
5751    (set_attr "length""8")])
5752
5753 ;;
5754 ;; [vrshrnbq_n_s, vrshrnbq_n_u])
5755 ;;
5756 (define_insn "mve_vrshrnbq_n_<supf><mode>"
5757   [
5758    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5759         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5760                        (match_operand:MVE_5 2 "s_register_operand" "w")
5761                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5762          VRSHRNBQ_N))
5763   ]
5764   "TARGET_HAVE_MVE"
5765   "vrshrnb.i%#<V_sz_elem>       %q0, %q2, %3"
5766   [(set_attr "type" "mve_move")
5767 ])
5768
5769 ;;
5770 ;; [vrshrntq_n_u, vrshrntq_n_s])
5771 ;;
5772 (define_insn "mve_vrshrntq_n_<supf><mode>"
5773   [
5774    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5775         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5776                        (match_operand:MVE_5 2 "s_register_operand" "w")
5777                        (match_operand:SI 3 "mve_imm_8" "Rb")]
5778          VRSHRNTQ_N))
5779   ]
5780   "TARGET_HAVE_MVE"
5781   "vrshrnt.i%#<V_sz_elem>       %q0, %q2, %3"
5782   [(set_attr "type" "mve_move")
5783 ])
5784
5785 ;;
5786 ;; [vshrnbq_n_u, vshrnbq_n_s])
5787 ;;
5788 (define_insn "mve_vshrnbq_n_<supf><mode>"
5789   [
5790    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5791         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5792                        (match_operand:MVE_5 2 "s_register_operand" "w")
5793                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5794          VSHRNBQ_N))
5795   ]
5796   "TARGET_HAVE_MVE"
5797   "vshrnb.i%#<V_sz_elem>        %q0, %q2, %3"
5798   [(set_attr "type" "mve_move")
5799 ])
5800
5801 ;;
5802 ;; [vshrntq_n_s, vshrntq_n_u])
5803 ;;
5804 (define_insn "mve_vshrntq_n_<supf><mode>"
5805   [
5806    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
5807         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
5808                                  (match_operand:MVE_5 2 "s_register_operand" "w")
5809                                  (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")]
5810          VSHRNTQ_N))
5811   ]
5812   "TARGET_HAVE_MVE"
5813   "vshrnt.i%#<V_sz_elem>\t%q0, %q2, %3"
5814   [(set_attr "type" "mve_move")
5815 ])
5816
5817 ;;
5818 ;; [vcvtmq_m_s, vcvtmq_m_u])
5819 ;;
5820 (define_insn "mve_vcvtmq_m_<supf><mode>"
5821   [
5822    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5823         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5824                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5825                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5826          VCVTMQ_M))
5827   ]
5828   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5829   "vpst\;vcvtmt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5830   [(set_attr "type" "mve_move")
5831    (set_attr "length""8")])
5832
5833 ;;
5834 ;; [vcvtpq_m_u, vcvtpq_m_s])
5835 ;;
5836 (define_insn "mve_vcvtpq_m_<supf><mode>"
5837   [
5838    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5839         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5840                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5841                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5842          VCVTPQ_M))
5843   ]
5844   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5845   "vpst\;vcvtpt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5846   [(set_attr "type" "mve_move")
5847    (set_attr "length""8")])
5848
5849 ;;
5850 ;; [vcvtnq_m_s, vcvtnq_m_u])
5851 ;;
5852 (define_insn "mve_vcvtnq_m_<supf><mode>"
5853   [
5854    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5855         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5856                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5857                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5858          VCVTNQ_M))
5859   ]
5860   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5861   "vpst\;vcvtnt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5862   [(set_attr "type" "mve_move")
5863    (set_attr "length""8")])
5864
5865 ;;
5866 ;; [vcvtq_m_n_from_f_s, vcvtq_m_n_from_f_u])
5867 ;;
5868 (define_insn "mve_vcvtq_m_n_from_f_<supf><mode>"
5869   [
5870    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5871         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5872                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5873                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
5874                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5875          VCVTQ_M_N_FROM_F))
5876   ]
5877   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5878   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2, %3"
5879   [(set_attr "type" "mve_move")
5880    (set_attr "length""8")])
5881
5882 ;;
5883 ;; [vrev16q_m_u, vrev16q_m_s])
5884 ;;
5885 (define_insn "mve_vrev16q_m_<supf>v16qi"
5886   [
5887    (set (match_operand:V16QI 0 "s_register_operand" "=w")
5888         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "0")
5889                        (match_operand:V16QI 2 "s_register_operand" "w")
5890                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5891          VREV16Q_M))
5892   ]
5893   "TARGET_HAVE_MVE"
5894   "vpst\;vrev16t.8 %q0, %q2"
5895   [(set_attr "type" "mve_move")
5896    (set_attr "length""8")])
5897
5898 ;;
5899 ;; [vcvtq_m_from_f_u, vcvtq_m_from_f_s])
5900 ;;
5901 (define_insn "mve_vcvtq_m_from_f_<supf><mode>"
5902   [
5903    (set (match_operand:MVE_5 0 "s_register_operand" "=w")
5904         (unspec:MVE_5 [(match_operand:MVE_5 1 "s_register_operand" "0")
5905                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
5906                        (match_operand:HI 3 "vpr_register_operand" "Up")]
5907          VCVTQ_M_FROM_F))
5908   ]
5909   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
5910   "vpst\;vcvtt.<supf>%#<V_sz_elem>.f%#<V_sz_elem>\t%q0, %q2"
5911   [(set_attr "type" "mve_move")
5912    (set_attr "length""8")])
5913
5914 ;;
5915 ;; [vrmlaldavhq_p_u vrmlaldavhq_p_s])
5916 ;;
5917 (define_insn "mve_vrmlaldavhq_p_<supf>v4si"
5918   [
5919    (set (match_operand:DI 0 "s_register_operand" "=r")
5920         (unspec:DI [(match_operand:V4SI 1 "s_register_operand" "w")
5921                     (match_operand:V4SI 2 "s_register_operand" "w")
5922                     (match_operand:HI 3 "vpr_register_operand" "Up")]
5923          VRMLALDAVHQ_P))
5924   ]
5925   "TARGET_HAVE_MVE"
5926   "vpst\;vrmlaldavht.<supf>32 %Q0, %R0, %q1, %q2"
5927   [(set_attr "type" "mve_move")
5928    (set_attr "length""8")])
5929
5930 ;;
5931 ;; [vrmlsldavhaq_s])
5932 ;;
5933 (define_insn "mve_vrmlsldavhaq_sv4si"
5934   [
5935    (set (match_operand:DI 0 "s_register_operand" "=r")
5936         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
5937                     (match_operand:V4SI 2 "s_register_operand" "w")
5938                     (match_operand:V4SI 3 "s_register_operand" "w")]
5939          VRMLSLDAVHAQ_S))
5940   ]
5941   "TARGET_HAVE_MVE"
5942   "vrmlsldavha.s32 %Q0, %R0, %q2, %q3"
5943   [(set_attr "type" "mve_move")
5944 ])
5945
5946 ;;
5947 ;; [vabavq_p_s, vabavq_p_u])
5948 ;;
5949 (define_insn "mve_vabavq_p_<supf><mode>"
5950   [
5951    (set (match_operand:SI 0 "s_register_operand" "=r")
5952         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
5953                     (match_operand:MVE_2 2 "s_register_operand" "w")
5954                     (match_operand:MVE_2 3 "s_register_operand" "w")
5955                     (match_operand:HI 4 "vpr_register_operand" "Up")]
5956          VABAVQ_P))
5957   ]
5958   "TARGET_HAVE_MVE"
5959   "vpst\;vabavt.<supf>%#<V_sz_elem>\t%0, %q2, %q3"
5960   [(set_attr "type" "mve_move")
5961 ])
5962
5963 ;;
5964 ;; [vqshluq_m_n_s])
5965 ;;
5966 (define_insn "mve_vqshluq_m_n_s<mode>"
5967   [
5968    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5969         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5970                        (match_operand:MVE_2 2 "s_register_operand" "w")
5971                        (match_operand:SI 3 "mve_imm_7" "Ra")
5972                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5973          VQSHLUQ_M_N_S))
5974   ]
5975   "TARGET_HAVE_MVE"
5976   "vpst\n\tvqshlut.s%#<V_sz_elem>\t%q0, %q2, %3"
5977   [(set_attr "type" "mve_move")])
5978
5979 ;;
5980 ;; [vshlq_m_s, vshlq_m_u])
5981 ;;
5982 (define_insn "mve_vshlq_m_<supf><mode>"
5983   [
5984    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
5985         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
5986                        (match_operand:MVE_2 2 "s_register_operand" "w")
5987                        (match_operand:MVE_2 3 "s_register_operand" "w")
5988                        (match_operand:HI 4 "vpr_register_operand" "Up")]
5989          VSHLQ_M))
5990   ]
5991   "TARGET_HAVE_MVE"
5992   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
5993   [(set_attr "type" "mve_move")])
5994
5995 ;;
5996 ;; [vsriq_m_n_s, vsriq_m_n_u])
5997 ;;
5998 (define_insn "mve_vsriq_m_n_<supf><mode>"
5999   [
6000    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6001         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6002                        (match_operand:MVE_2 2 "s_register_operand" "w")
6003                        (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")
6004                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6005          VSRIQ_M_N))
6006   ]
6007   "TARGET_HAVE_MVE"
6008   "vpst\;vsrit.%#<V_sz_elem>\t%q0, %q2, %3"
6009   [(set_attr "type" "mve_move")])
6010
6011 ;;
6012 ;; [vsubq_m_u, vsubq_m_s])
6013 ;;
6014 (define_insn "mve_vsubq_m_<supf><mode>"
6015   [
6016    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6017         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6018                        (match_operand:MVE_2 2 "s_register_operand" "w")
6019                        (match_operand:MVE_2 3 "s_register_operand" "w")
6020                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6021          VSUBQ_M))
6022   ]
6023   "TARGET_HAVE_MVE"
6024   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %q3"
6025   [(set_attr "type" "mve_move")])
6026
6027 ;;
6028 ;; [vcvtq_m_n_to_f_u, vcvtq_m_n_to_f_s])
6029 ;;
6030 (define_insn "mve_vcvtq_m_n_to_f_<supf><mode>"
6031   [
6032    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
6033         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
6034                        (match_operand:<MVE_CNVT> 2 "s_register_operand" "w")
6035                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6036                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6037          VCVTQ_M_N_TO_F))
6038   ]
6039   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
6040   "vpst\;vcvtt.f%#<V_sz_elem>.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6041   [(set_attr "type" "mve_move")
6042    (set_attr "length""8")])
6043 ;;
6044 ;; [vabdq_m_s, vabdq_m_u])
6045 ;;
6046 (define_insn "mve_vabdq_m_<supf><mode>"
6047   [
6048    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6049         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6050                        (match_operand:MVE_2 2 "s_register_operand" "w")
6051                        (match_operand:MVE_2 3 "s_register_operand" "w")
6052                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6053          VABDQ_M))
6054   ]
6055   "TARGET_HAVE_MVE"
6056   "vpst\;vabdt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6057   [(set_attr "type" "mve_move")
6058    (set_attr "length""8")])
6059
6060 ;;
6061 ;; [vaddq_m_n_s, vaddq_m_n_u])
6062 ;;
6063 (define_insn "mve_vaddq_m_n_<supf><mode>"
6064   [
6065    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6066         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6067                        (match_operand:MVE_2 2 "s_register_operand" "w")
6068                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6069                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6070          VADDQ_M_N))
6071   ]
6072   "TARGET_HAVE_MVE"
6073   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %3"
6074   [(set_attr "type" "mve_move")
6075    (set_attr "length""8")])
6076
6077 ;;
6078 ;; [vaddq_m_u, vaddq_m_s])
6079 ;;
6080 (define_insn "mve_vaddq_m_<supf><mode>"
6081   [
6082    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6083         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6084                        (match_operand:MVE_2 2 "s_register_operand" "w")
6085                        (match_operand:MVE_2 3 "s_register_operand" "w")
6086                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6087          VADDQ_M))
6088   ]
6089   "TARGET_HAVE_MVE"
6090   "vpst\;vaddt.i%#<V_sz_elem>   %q0, %q2, %q3"
6091   [(set_attr "type" "mve_move")
6092    (set_attr "length""8")])
6093
6094 ;;
6095 ;; [vandq_m_u, vandq_m_s])
6096 ;;
6097 (define_insn "mve_vandq_m_<supf><mode>"
6098   [
6099    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6100         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6101                        (match_operand:MVE_2 2 "s_register_operand" "w")
6102                        (match_operand:MVE_2 3 "s_register_operand" "w")
6103                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6104          VANDQ_M))
6105   ]
6106   "TARGET_HAVE_MVE"
6107   "vpst\;vandt %q0, %q2, %q3"
6108   [(set_attr "type" "mve_move")
6109    (set_attr "length""8")])
6110
6111 ;;
6112 ;; [vbicq_m_u, vbicq_m_s])
6113 ;;
6114 (define_insn "mve_vbicq_m_<supf><mode>"
6115   [
6116    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6117         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6118                        (match_operand:MVE_2 2 "s_register_operand" "w")
6119                        (match_operand:MVE_2 3 "s_register_operand" "w")
6120                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6121          VBICQ_M))
6122   ]
6123   "TARGET_HAVE_MVE"
6124   "vpst\;vbict %q0, %q2, %q3"
6125   [(set_attr "type" "mve_move")
6126    (set_attr "length""8")])
6127
6128 ;;
6129 ;; [vbrsrq_m_n_u, vbrsrq_m_n_s])
6130 ;;
6131 (define_insn "mve_vbrsrq_m_n_<supf><mode>"
6132   [
6133    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6134         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6135                        (match_operand:MVE_2 2 "s_register_operand" "w")
6136                        (match_operand:SI 3 "s_register_operand" "r")
6137                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6138          VBRSRQ_M_N))
6139   ]
6140   "TARGET_HAVE_MVE"
6141   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
6142   [(set_attr "type" "mve_move")
6143    (set_attr "length""8")])
6144
6145 ;;
6146 ;; [vcaddq_rot270_m_u, vcaddq_rot270_m_s])
6147 ;;
6148 (define_insn "mve_vcaddq_rot270_m_<supf><mode>"
6149   [
6150    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6151         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6152                        (match_operand:MVE_2 2 "s_register_operand" "w")
6153                        (match_operand:MVE_2 3 "s_register_operand" "w")
6154                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6155          VCADDQ_ROT270_M))
6156   ]
6157   "TARGET_HAVE_MVE"
6158   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #270"
6159   [(set_attr "type" "mve_move")
6160    (set_attr "length""8")])
6161
6162 ;;
6163 ;; [vcaddq_rot90_m_u, vcaddq_rot90_m_s])
6164 ;;
6165 (define_insn "mve_vcaddq_rot90_m_<supf><mode>"
6166   [
6167    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6168         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6169                        (match_operand:MVE_2 2 "s_register_operand" "w")
6170                        (match_operand:MVE_2 3 "s_register_operand" "w")
6171                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6172          VCADDQ_ROT90_M))
6173   ]
6174   "TARGET_HAVE_MVE"
6175   "vpst\;vcaddt.i%#<V_sz_elem>  %q0, %q2, %q3, #90"
6176   [(set_attr "type" "mve_move")
6177    (set_attr "length""8")])
6178
6179 ;;
6180 ;; [veorq_m_s, veorq_m_u])
6181 ;;
6182 (define_insn "mve_veorq_m_<supf><mode>"
6183   [
6184    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6185         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6186                        (match_operand:MVE_2 2 "s_register_operand" "w")
6187                        (match_operand:MVE_2 3 "s_register_operand" "w")
6188                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6189          VEORQ_M))
6190   ]
6191   "TARGET_HAVE_MVE"
6192   "vpst\;veort %q0, %q2, %q3"
6193   [(set_attr "type" "mve_move")
6194    (set_attr "length""8")])
6195
6196 ;;
6197 ;; [vhaddq_m_n_s, vhaddq_m_n_u])
6198 ;;
6199 (define_insn "mve_vhaddq_m_n_<supf><mode>"
6200   [
6201    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6202         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6203                        (match_operand:MVE_2 2 "s_register_operand" "w")
6204                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6205                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6206          VHADDQ_M_N))
6207   ]
6208   "TARGET_HAVE_MVE"
6209   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6210   [(set_attr "type" "mve_move")
6211    (set_attr "length""8")])
6212
6213 ;;
6214 ;; [vhaddq_m_s, vhaddq_m_u])
6215 ;;
6216 (define_insn "mve_vhaddq_m_<supf><mode>"
6217   [
6218    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6219         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6220                        (match_operand:MVE_2 2 "s_register_operand" "w")
6221                        (match_operand:MVE_2 3 "s_register_operand" "w")
6222                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6223          VHADDQ_M))
6224   ]
6225   "TARGET_HAVE_MVE"
6226   "vpst\;vhaddt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6227   [(set_attr "type" "mve_move")
6228    (set_attr "length""8")])
6229
6230 ;;
6231 ;; [vhsubq_m_n_s, vhsubq_m_n_u])
6232 ;;
6233 (define_insn "mve_vhsubq_m_n_<supf><mode>"
6234   [
6235    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6236         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6237                        (match_operand:MVE_2 2 "s_register_operand" "w")
6238                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6239                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6240          VHSUBQ_M_N))
6241   ]
6242   "TARGET_HAVE_MVE"
6243   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6244   [(set_attr "type" "mve_move")
6245    (set_attr "length""8")])
6246
6247 ;;
6248 ;; [vhsubq_m_s, vhsubq_m_u])
6249 ;;
6250 (define_insn "mve_vhsubq_m_<supf><mode>"
6251   [
6252    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6253         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6254                        (match_operand:MVE_2 2 "s_register_operand" "w")
6255                        (match_operand:MVE_2 3 "s_register_operand" "w")
6256                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6257          VHSUBQ_M))
6258   ]
6259   "TARGET_HAVE_MVE"
6260   "vpst\;vhsubt.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6261   [(set_attr "type" "mve_move")
6262    (set_attr "length""8")])
6263
6264 ;;
6265 ;; [vmaxq_m_s, vmaxq_m_u])
6266 ;;
6267 (define_insn "mve_vmaxq_m_<supf><mode>"
6268   [
6269    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6270         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6271                        (match_operand:MVE_2 2 "s_register_operand" "w")
6272                        (match_operand:MVE_2 3 "s_register_operand" "w")
6273                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6274          VMAXQ_M))
6275   ]
6276   "TARGET_HAVE_MVE"
6277   "vpst\;vmaxt.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6278   [(set_attr "type" "mve_move")
6279    (set_attr "length""8")])
6280
6281 ;;
6282 ;; [vminq_m_s, vminq_m_u])
6283 ;;
6284 (define_insn "mve_vminq_m_<supf><mode>"
6285   [
6286    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6287         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6288                        (match_operand:MVE_2 2 "s_register_operand" "w")
6289                        (match_operand:MVE_2 3 "s_register_operand" "w")
6290                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6291          VMINQ_M))
6292   ]
6293   "TARGET_HAVE_MVE"
6294   "vpst\;vmint.<supf>%#<V_sz_elem>      %q0, %q2, %q3"
6295   [(set_attr "type" "mve_move")
6296    (set_attr "length""8")])
6297
6298 ;;
6299 ;; [vmladavaq_p_u, vmladavaq_p_s])
6300 ;;
6301 (define_insn "mve_vmladavaq_p_<supf><mode>"
6302   [
6303    (set (match_operand:SI 0 "s_register_operand" "=e")
6304         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6305                     (match_operand:MVE_2 2 "s_register_operand" "w")
6306                     (match_operand:MVE_2 3 "s_register_operand" "w")
6307                     (match_operand:HI 4 "vpr_register_operand" "Up")]
6308          VMLADAVAQ_P))
6309   ]
6310   "TARGET_HAVE_MVE"
6311   "vpst\;vmladavat.<supf>%#<V_sz_elem>  %0, %q2, %q3"
6312   [(set_attr "type" "mve_move")
6313    (set_attr "length""8")])
6314
6315 ;;
6316 ;; [vmlaq_m_n_s, vmlaq_m_n_u])
6317 ;;
6318 (define_insn "mve_vmlaq_m_n_<supf><mode>"
6319   [
6320    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6321         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6322                        (match_operand:MVE_2 2 "s_register_operand" "w")
6323                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6324                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6325          VMLAQ_M_N))
6326   ]
6327   "TARGET_HAVE_MVE"
6328   "vpst\;vmlat.<supf>%#<V_sz_elem>      %q0, %q2, %3"
6329   [(set_attr "type" "mve_move")
6330    (set_attr "length""8")])
6331
6332 ;;
6333 ;; [vmlasq_m_n_u, vmlasq_m_n_s])
6334 ;;
6335 (define_insn "mve_vmlasq_m_n_<supf><mode>"
6336   [
6337    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6338         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6339                        (match_operand:MVE_2 2 "s_register_operand" "w")
6340                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6341                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6342          VMLASQ_M_N))
6343   ]
6344   "TARGET_HAVE_MVE"
6345   "vpst\;vmlast.<supf>%#<V_sz_elem>     %q0, %q2, %3"
6346   [(set_attr "type" "mve_move")
6347    (set_attr "length""8")])
6348
6349 ;;
6350 ;; [vmulhq_m_s, vmulhq_m_u])
6351 ;;
6352 (define_insn "mve_vmulhq_m_<supf><mode>"
6353   [
6354    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6355         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6356                        (match_operand:MVE_2 2 "s_register_operand" "w")
6357                        (match_operand:MVE_2 3 "s_register_operand" "w")
6358                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6359          VMULHQ_M))
6360   ]
6361   "TARGET_HAVE_MVE"
6362   "vpst\;vmulht.<supf>%#<V_sz_elem>     %q0, %q2, %q3"
6363   [(set_attr "type" "mve_move")
6364    (set_attr "length""8")])
6365
6366 ;;
6367 ;; [vmullbq_int_m_u, vmullbq_int_m_s])
6368 ;;
6369 (define_insn "mve_vmullbq_int_m_<supf><mode>"
6370   [
6371    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6372         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6373                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6374                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6375                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6376          VMULLBQ_INT_M))
6377   ]
6378   "TARGET_HAVE_MVE"
6379   "vpst\;vmullbt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6380   [(set_attr "type" "mve_move")
6381    (set_attr "length""8")])
6382
6383 ;;
6384 ;; [vmulltq_int_m_s, vmulltq_int_m_u])
6385 ;;
6386 (define_insn "mve_vmulltq_int_m_<supf><mode>"
6387   [
6388    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
6389         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
6390                                   (match_operand:MVE_2 2 "s_register_operand" "w")
6391                                   (match_operand:MVE_2 3 "s_register_operand" "w")
6392                                   (match_operand:HI 4 "vpr_register_operand" "Up")]
6393          VMULLTQ_INT_M))
6394   ]
6395   "TARGET_HAVE_MVE"
6396   "vpst\;vmulltt.<supf>%#<V_sz_elem>    %q0, %q2, %q3"
6397   [(set_attr "type" "mve_move")
6398    (set_attr "length""8")])
6399
6400 ;;
6401 ;; [vmulq_m_n_u, vmulq_m_n_s])
6402 ;;
6403 (define_insn "mve_vmulq_m_n_<supf><mode>"
6404   [
6405    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6406         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6407                        (match_operand:MVE_2 2 "s_register_operand" "w")
6408                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6409                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6410          VMULQ_M_N))
6411   ]
6412   "TARGET_HAVE_MVE"
6413   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %3"
6414   [(set_attr "type" "mve_move")
6415    (set_attr "length""8")])
6416
6417 ;;
6418 ;; [vmulq_m_s, vmulq_m_u])
6419 ;;
6420 (define_insn "mve_vmulq_m_<supf><mode>"
6421   [
6422    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6423         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6424                        (match_operand:MVE_2 2 "s_register_operand" "w")
6425                        (match_operand:MVE_2 3 "s_register_operand" "w")
6426                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6427          VMULQ_M))
6428   ]
6429   "TARGET_HAVE_MVE"
6430   "vpst\;vmult.i%#<V_sz_elem>   %q0, %q2, %q3"
6431   [(set_attr "type" "mve_move")
6432    (set_attr "length""8")])
6433
6434 ;;
6435 ;; [vornq_m_u, vornq_m_s])
6436 ;;
6437 (define_insn "mve_vornq_m_<supf><mode>"
6438   [
6439    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6440         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6441                        (match_operand:MVE_2 2 "s_register_operand" "w")
6442                        (match_operand:MVE_2 3 "s_register_operand" "w")
6443                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6444          VORNQ_M))
6445   ]
6446   "TARGET_HAVE_MVE"
6447   "vpst\;vornt %q0, %q2, %q3"
6448   [(set_attr "type" "mve_move")
6449    (set_attr "length""8")])
6450
6451 ;;
6452 ;; [vorrq_m_s, vorrq_m_u])
6453 ;;
6454 (define_insn "mve_vorrq_m_<supf><mode>"
6455   [
6456    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6457         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6458                        (match_operand:MVE_2 2 "s_register_operand" "w")
6459                        (match_operand:MVE_2 3 "s_register_operand" "w")
6460                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6461          VORRQ_M))
6462   ]
6463   "TARGET_HAVE_MVE"
6464   "vpst\;vorrt %q0, %q2, %q3"
6465   [(set_attr "type" "mve_move")
6466    (set_attr "length""8")])
6467
6468 ;;
6469 ;; [vqaddq_m_n_u, vqaddq_m_n_s])
6470 ;;
6471 (define_insn "mve_vqaddq_m_n_<supf><mode>"
6472   [
6473    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6474         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6475                        (match_operand:MVE_2 2 "s_register_operand" "w")
6476                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6477                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6478          VQADDQ_M_N))
6479   ]
6480   "TARGET_HAVE_MVE"
6481   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6482   [(set_attr "type" "mve_move")
6483    (set_attr "length""8")])
6484
6485 ;;
6486 ;; [vqaddq_m_u, vqaddq_m_s])
6487 ;;
6488 (define_insn "mve_vqaddq_m_<supf><mode>"
6489   [
6490    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6491         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6492                        (match_operand:MVE_2 2 "s_register_operand" "w")
6493                        (match_operand:MVE_2 3 "s_register_operand" "w")
6494                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6495          VQADDQ_M))
6496   ]
6497   "TARGET_HAVE_MVE"
6498   "vpst\;vqaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6499   [(set_attr "type" "mve_move")
6500    (set_attr "length""8")])
6501
6502 ;;
6503 ;; [vqdmlahq_m_n_s])
6504 ;;
6505 (define_insn "mve_vqdmlahq_m_n_s<mode>"
6506   [
6507    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6508         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6509                        (match_operand:MVE_2 2 "s_register_operand" "w")
6510                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6511                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6512          VQDMLAHQ_M_N_S))
6513   ]
6514   "TARGET_HAVE_MVE"
6515   "vpst\;vqdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6516   [(set_attr "type" "mve_move")
6517    (set_attr "length""8")])
6518
6519 ;;
6520 ;; [vqrdmlahq_m_n_s])
6521 ;;
6522 (define_insn "mve_vqrdmlahq_m_n_s<mode>"
6523   [
6524    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6525         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6526                        (match_operand:MVE_2 2 "s_register_operand" "w")
6527                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6528                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6529          VQRDMLAHQ_M_N_S))
6530   ]
6531   "TARGET_HAVE_MVE"
6532   "vpst\;vqrdmlaht.s%#<V_sz_elem>\t%q0, %q2, %3"
6533   [(set_attr "type" "mve_move")
6534    (set_attr "length""8")])
6535
6536 ;;
6537 ;; [vqrdmlashq_m_n_s])
6538 ;;
6539 (define_insn "mve_vqrdmlashq_m_n_s<mode>"
6540   [
6541    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6542         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6543                        (match_operand:MVE_2 2 "s_register_operand" "w")
6544                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6545                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6546          VQRDMLASHQ_M_N_S))
6547   ]
6548   "TARGET_HAVE_MVE"
6549   "vpst\;vqrdmlasht.s%#<V_sz_elem>\t%q0, %q2, %3"
6550   [(set_attr "type" "mve_move")
6551    (set_attr "length""8")])
6552
6553 ;;
6554 ;; [vqrshlq_m_u, vqrshlq_m_s])
6555 ;;
6556 (define_insn "mve_vqrshlq_m_<supf><mode>"
6557   [
6558    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6559         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6560                        (match_operand:MVE_2 2 "s_register_operand" "w")
6561                        (match_operand:MVE_2 3 "s_register_operand" "w")
6562                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6563          VQRSHLQ_M))
6564   ]
6565   "TARGET_HAVE_MVE"
6566   "vpst\;vqrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6567   [(set_attr "type" "mve_move")
6568    (set_attr "length""8")])
6569
6570 ;;
6571 ;; [vqshlq_m_n_s, vqshlq_m_n_u])
6572 ;;
6573 (define_insn "mve_vqshlq_m_n_<supf><mode>"
6574   [
6575    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6576         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6577                        (match_operand:MVE_2 2 "s_register_operand" "w")
6578                        (match_operand:SI 3 "immediate_operand" "i")
6579                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6580          VQSHLQ_M_N))
6581   ]
6582   "TARGET_HAVE_MVE"
6583   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6584   [(set_attr "type" "mve_move")
6585    (set_attr "length""8")])
6586
6587 ;;
6588 ;; [vqshlq_m_u, vqshlq_m_s])
6589 ;;
6590 (define_insn "mve_vqshlq_m_<supf><mode>"
6591   [
6592    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6593         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6594                        (match_operand:MVE_2 2 "s_register_operand" "w")
6595                        (match_operand:MVE_2 3 "s_register_operand" "w")
6596                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6597          VQSHLQ_M))
6598   ]
6599   "TARGET_HAVE_MVE"
6600   "vpst\;vqshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6601   [(set_attr "type" "mve_move")
6602    (set_attr "length""8")])
6603
6604 ;;
6605 ;; [vqsubq_m_n_u, vqsubq_m_n_s])
6606 ;;
6607 (define_insn "mve_vqsubq_m_n_<supf><mode>"
6608   [
6609    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6610         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6611                        (match_operand:MVE_2 2 "s_register_operand" "w")
6612                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6613                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6614          VQSUBQ_M_N))
6615   ]
6616   "TARGET_HAVE_MVE"
6617   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6618   [(set_attr "type" "mve_move")
6619    (set_attr "length""8")])
6620
6621 ;;
6622 ;; [vqsubq_m_u, vqsubq_m_s])
6623 ;;
6624 (define_insn "mve_vqsubq_m_<supf><mode>"
6625   [
6626    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6627         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6628                        (match_operand:MVE_2 2 "s_register_operand" "w")
6629                        (match_operand:MVE_2 3 "s_register_operand" "w")
6630                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6631          VQSUBQ_M))
6632   ]
6633   "TARGET_HAVE_MVE"
6634   "vpst\;vqsubt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6635   [(set_attr "type" "mve_move")
6636    (set_attr "length""8")])
6637
6638 ;;
6639 ;; [vrhaddq_m_u, vrhaddq_m_s])
6640 ;;
6641 (define_insn "mve_vrhaddq_m_<supf><mode>"
6642   [
6643    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6644         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6645                        (match_operand:MVE_2 2 "s_register_operand" "w")
6646                        (match_operand:MVE_2 3 "s_register_operand" "w")
6647                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6648          VRHADDQ_M))
6649   ]
6650   "TARGET_HAVE_MVE"
6651   "vpst\;vrhaddt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6652   [(set_attr "type" "mve_move")
6653    (set_attr "length""8")])
6654
6655 ;;
6656 ;; [vrmulhq_m_u, vrmulhq_m_s])
6657 ;;
6658 (define_insn "mve_vrmulhq_m_<supf><mode>"
6659   [
6660    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6661         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6662                        (match_operand:MVE_2 2 "s_register_operand" "w")
6663                        (match_operand:MVE_2 3 "s_register_operand" "w")
6664                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6665          VRMULHQ_M))
6666   ]
6667   "TARGET_HAVE_MVE"
6668   "vpst\;vrmulht.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6669   [(set_attr "type" "mve_move")
6670    (set_attr "length""8")])
6671
6672 ;;
6673 ;; [vrshlq_m_s, vrshlq_m_u])
6674 ;;
6675 (define_insn "mve_vrshlq_m_<supf><mode>"
6676   [
6677    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6678         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6679                        (match_operand:MVE_2 2 "s_register_operand" "w")
6680                        (match_operand:MVE_2 3 "s_register_operand" "w")
6681                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6682          VRSHLQ_M))
6683   ]
6684   "TARGET_HAVE_MVE"
6685   "vpst\;vrshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %q3"
6686   [(set_attr "type" "mve_move")
6687    (set_attr "length""8")])
6688
6689 ;;
6690 ;; [vrshrq_m_n_s, vrshrq_m_n_u])
6691 ;;
6692 (define_insn "mve_vrshrq_m_n_<supf><mode>"
6693   [
6694    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6695         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6696                        (match_operand:MVE_2 2 "s_register_operand" "w")
6697                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6698                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6699          VRSHRQ_M_N))
6700   ]
6701   "TARGET_HAVE_MVE"
6702   "vpst\;vrshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6703   [(set_attr "type" "mve_move")
6704    (set_attr "length""8")])
6705
6706 ;;
6707 ;; [vshlq_m_n_s, vshlq_m_n_u])
6708 ;;
6709 (define_insn "mve_vshlq_m_n_<supf><mode>"
6710   [
6711    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6712         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6713                        (match_operand:MVE_2 2 "s_register_operand" "w")
6714                        (match_operand:SI 3 "immediate_operand" "i")
6715                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6716          VSHLQ_M_N))
6717   ]
6718   "TARGET_HAVE_MVE"
6719   "vpst\;vshlt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6720   [(set_attr "type" "mve_move")
6721    (set_attr "length""8")])
6722
6723 ;;
6724 ;; [vshrq_m_n_s, vshrq_m_n_u])
6725 ;;
6726 (define_insn "mve_vshrq_m_n_<supf><mode>"
6727   [
6728    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6729         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6730                        (match_operand:MVE_2 2 "s_register_operand" "w")
6731                        (match_operand:SI 3 "<MVE_pred2>" "<MVE_constraint2>")
6732                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6733          VSHRQ_M_N))
6734   ]
6735   "TARGET_HAVE_MVE"
6736   "vpst\;vshrt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
6737   [(set_attr "type" "mve_move")
6738    (set_attr "length""8")])
6739
6740 ;;
6741 ;; [vsliq_m_n_u, vsliq_m_n_s])
6742 ;;
6743 (define_insn "mve_vsliq_m_n_<supf><mode>"
6744    [
6745    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6746        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6747                        (match_operand:MVE_2 2 "s_register_operand" "w")
6748                        (match_operand:SI 3 "<MVE_pred>" "<MVE_constraint>")
6749                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6750          VSLIQ_M_N))
6751   ]
6752   "TARGET_HAVE_MVE"
6753   "vpst\;vslit.%#<V_sz_elem>\t%q0, %q2, %3"
6754   [(set_attr "type" "mve_move")
6755    (set_attr "length""8")])
6756
6757 ;;
6758 ;; [vsubq_m_n_s, vsubq_m_n_u])
6759 ;;
6760 (define_insn "mve_vsubq_m_n_<supf><mode>"
6761   [
6762    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6763         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6764                        (match_operand:MVE_2 2 "s_register_operand" "w")
6765                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6766                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6767          VSUBQ_M_N))
6768   ]
6769   "TARGET_HAVE_MVE"
6770   "vpst\;vsubt.i%#<V_sz_elem>\t%q0, %q2, %3"
6771   [(set_attr "type" "mve_move")
6772    (set_attr "length""8")])
6773
6774 ;;
6775 ;; [vhcaddq_rot270_m_s])
6776 ;;
6777 (define_insn "mve_vhcaddq_rot270_m_s<mode>"
6778   [
6779    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6780         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6781                        (match_operand:MVE_2 2 "s_register_operand" "w")
6782                        (match_operand:MVE_2 3 "s_register_operand" "w")
6783                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6784          VHCADDQ_ROT270_M_S))
6785   ]
6786   "TARGET_HAVE_MVE"
6787   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #270"
6788   [(set_attr "type" "mve_move")
6789    (set_attr "length""8")])
6790
6791 ;;
6792 ;; [vhcaddq_rot90_m_s])
6793 ;;
6794 (define_insn "mve_vhcaddq_rot90_m_s<mode>"
6795   [
6796    (set (match_operand:MVE_2 0 "s_register_operand" "<earlyclobber_32>")
6797         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6798                        (match_operand:MVE_2 2 "s_register_operand" "w")
6799                        (match_operand:MVE_2 3 "s_register_operand" "w")
6800                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6801          VHCADDQ_ROT90_M_S))
6802   ]
6803   "TARGET_HAVE_MVE"
6804   "vpst\;vhcaddt.s%#<V_sz_elem>\t%q0, %q2, %q3, #90"
6805   [(set_attr "type" "mve_move")
6806    (set_attr "length""8")])
6807
6808 ;;
6809 ;; [vmladavaxq_p_s])
6810 ;;
6811 (define_insn "mve_vmladavaxq_p_s<mode>"
6812   [
6813    (set (match_operand:SI 0 "s_register_operand" "=e")
6814         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6815                        (match_operand:MVE_2 2 "s_register_operand" "w")
6816                        (match_operand:MVE_2 3 "s_register_operand" "w")
6817                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6818          VMLADAVAXQ_P_S))
6819   ]
6820   "TARGET_HAVE_MVE"
6821   "vpst\;vmladavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6822   [(set_attr "type" "mve_move")
6823    (set_attr "length""8")])
6824
6825 ;;
6826 ;; [vmlsdavaq_p_s])
6827 ;;
6828 (define_insn "mve_vmlsdavaq_p_s<mode>"
6829   [
6830    (set (match_operand:SI 0 "s_register_operand" "=e")
6831         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6832                        (match_operand:MVE_2 2 "s_register_operand" "w")
6833                        (match_operand:MVE_2 3 "s_register_operand" "w")
6834                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6835          VMLSDAVAQ_P_S))
6836   ]
6837   "TARGET_HAVE_MVE"
6838   "vpst\;vmlsdavat.s%#<V_sz_elem>\t%0, %q2, %q3"
6839   [(set_attr "type" "mve_move")
6840    (set_attr "length""8")])
6841
6842 ;;
6843 ;; [vmlsdavaxq_p_s])
6844 ;;
6845 (define_insn "mve_vmlsdavaxq_p_s<mode>"
6846   [
6847    (set (match_operand:SI 0 "s_register_operand" "=e")
6848         (unspec:SI [(match_operand:SI 1 "s_register_operand" "0")
6849                        (match_operand:MVE_2 2 "s_register_operand" "w")
6850                        (match_operand:MVE_2 3 "s_register_operand" "w")
6851                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6852          VMLSDAVAXQ_P_S))
6853   ]
6854   "TARGET_HAVE_MVE"
6855   "vpst\;vmlsdavaxt.s%#<V_sz_elem>\t%0, %q2, %q3"
6856   [(set_attr "type" "mve_move")
6857    (set_attr "length""8")])
6858
6859 ;;
6860 ;; [vqdmladhq_m_s])
6861 ;;
6862 (define_insn "mve_vqdmladhq_m_s<mode>"
6863   [
6864    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6865         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6866                        (match_operand:MVE_2 2 "s_register_operand" "w")
6867                        (match_operand:MVE_2 3 "s_register_operand" "w")
6868                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6869          VQDMLADHQ_M_S))
6870   ]
6871   "TARGET_HAVE_MVE"
6872   "vpst\;vqdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6873   [(set_attr "type" "mve_move")
6874    (set_attr "length""8")])
6875
6876 ;;
6877 ;; [vqdmladhxq_m_s])
6878 ;;
6879 (define_insn "mve_vqdmladhxq_m_s<mode>"
6880   [
6881    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6882         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6883                        (match_operand:MVE_2 2 "s_register_operand" "w")
6884                        (match_operand:MVE_2 3 "s_register_operand" "w")
6885                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6886          VQDMLADHXQ_M_S))
6887   ]
6888   "TARGET_HAVE_MVE"
6889   "vpst\;vqdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6890   [(set_attr "type" "mve_move")
6891    (set_attr "length""8")])
6892
6893 ;;
6894 ;; [vqdmlsdhq_m_s])
6895 ;;
6896 (define_insn "mve_vqdmlsdhq_m_s<mode>"
6897   [
6898    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6899         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6900                        (match_operand:MVE_2 2 "s_register_operand" "w")
6901                        (match_operand:MVE_2 3 "s_register_operand" "w")
6902                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6903          VQDMLSDHQ_M_S))
6904   ]
6905   "TARGET_HAVE_MVE"
6906   "vpst\;vqdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6907   [(set_attr "type" "mve_move")
6908    (set_attr "length""8")])
6909
6910 ;;
6911 ;; [vqdmlsdhxq_m_s])
6912 ;;
6913 (define_insn "mve_vqdmlsdhxq_m_s<mode>"
6914   [
6915    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6916         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6917                        (match_operand:MVE_2 2 "s_register_operand" "w")
6918                        (match_operand:MVE_2 3 "s_register_operand" "w")
6919                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6920          VQDMLSDHXQ_M_S))
6921   ]
6922   "TARGET_HAVE_MVE"
6923   "vpst\;vqdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6924   [(set_attr "type" "mve_move")
6925    (set_attr "length""8")])
6926
6927 ;;
6928 ;; [vqdmulhq_m_n_s])
6929 ;;
6930 (define_insn "mve_vqdmulhq_m_n_s<mode>"
6931   [
6932    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6933         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6934                        (match_operand:MVE_2 2 "s_register_operand" "w")
6935                        (match_operand:<V_elem> 3 "s_register_operand" "r")
6936                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6937          VQDMULHQ_M_N_S))
6938   ]
6939   "TARGET_HAVE_MVE"
6940   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
6941   [(set_attr "type" "mve_move")
6942    (set_attr "length""8")])
6943
6944 ;;
6945 ;; [vqdmulhq_m_s])
6946 ;;
6947 (define_insn "mve_vqdmulhq_m_s<mode>"
6948   [
6949    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6950         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6951                        (match_operand:MVE_2 2 "s_register_operand" "w")
6952                        (match_operand:MVE_2 3 "s_register_operand" "w")
6953                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6954          VQDMULHQ_M_S))
6955   ]
6956   "TARGET_HAVE_MVE"
6957   "vpst\;vqdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6958   [(set_attr "type" "mve_move")
6959    (set_attr "length""8")])
6960
6961 ;;
6962 ;; [vqrdmladhq_m_s])
6963 ;;
6964 (define_insn "mve_vqrdmladhq_m_s<mode>"
6965   [
6966    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6967         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6968                        (match_operand:MVE_2 2 "s_register_operand" "w")
6969                        (match_operand:MVE_2 3 "s_register_operand" "w")
6970                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6971          VQRDMLADHQ_M_S))
6972   ]
6973   "TARGET_HAVE_MVE"
6974   "vpst\;vqrdmladht.s%#<V_sz_elem>\t%q0, %q2, %q3"
6975   [(set_attr "type" "mve_move")
6976    (set_attr "length""8")])
6977
6978 ;;
6979 ;; [vqrdmladhxq_m_s])
6980 ;;
6981 (define_insn "mve_vqrdmladhxq_m_s<mode>"
6982   [
6983    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
6984         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
6985                        (match_operand:MVE_2 2 "s_register_operand" "w")
6986                        (match_operand:MVE_2 3 "s_register_operand" "w")
6987                        (match_operand:HI 4 "vpr_register_operand" "Up")]
6988          VQRDMLADHXQ_M_S))
6989   ]
6990   "TARGET_HAVE_MVE"
6991   "vpst\;vqrdmladhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
6992   [(set_attr "type" "mve_move")
6993    (set_attr "length""8")])
6994
6995 ;;
6996 ;; [vqrdmlsdhq_m_s])
6997 ;;
6998 (define_insn "mve_vqrdmlsdhq_m_s<mode>"
6999   [
7000    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7001         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7002                        (match_operand:MVE_2 2 "s_register_operand" "w")
7003                        (match_operand:MVE_2 3 "s_register_operand" "w")
7004                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7005          VQRDMLSDHQ_M_S))
7006   ]
7007   "TARGET_HAVE_MVE"
7008   "vpst\;vqrdmlsdht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7009   [(set_attr "type" "mve_move")
7010    (set_attr "length""8")])
7011
7012 ;;
7013 ;; [vqrdmlsdhxq_m_s])
7014 ;;
7015 (define_insn "mve_vqrdmlsdhxq_m_s<mode>"
7016   [
7017    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7018         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7019                        (match_operand:MVE_2 2 "s_register_operand" "w")
7020                        (match_operand:MVE_2 3 "s_register_operand" "w")
7021                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7022          VQRDMLSDHXQ_M_S))
7023   ]
7024   "TARGET_HAVE_MVE"
7025   "vpst\;vqrdmlsdhxt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7026   [(set_attr "type" "mve_move")
7027    (set_attr "length""8")])
7028
7029 ;;
7030 ;; [vqrdmulhq_m_n_s])
7031 ;;
7032 (define_insn "mve_vqrdmulhq_m_n_s<mode>"
7033   [
7034    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7035         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7036                        (match_operand:MVE_2 2 "s_register_operand" "w")
7037                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7038                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7039          VQRDMULHQ_M_N_S))
7040   ]
7041   "TARGET_HAVE_MVE"
7042   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %3"
7043   [(set_attr "type" "mve_move")
7044    (set_attr "length""8")])
7045
7046 ;;
7047 ;; [vqrdmulhq_m_s])
7048 ;;
7049 (define_insn "mve_vqrdmulhq_m_s<mode>"
7050   [
7051    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
7052         (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
7053                        (match_operand:MVE_2 2 "s_register_operand" "w")
7054                        (match_operand:MVE_2 3 "s_register_operand" "w")
7055                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7056          VQRDMULHQ_M_S))
7057   ]
7058   "TARGET_HAVE_MVE"
7059   "vpst\;vqrdmulht.s%#<V_sz_elem>\t%q0, %q2, %q3"
7060   [(set_attr "type" "mve_move")
7061    (set_attr "length""8")])
7062
7063 ;;
7064 ;; [vmlaldavaq_p_u, vmlaldavaq_p_s])
7065 ;;
7066 (define_insn "mve_vmlaldavaq_p_<supf><mode>"
7067   [
7068    (set (match_operand:DI 0 "s_register_operand" "=r")
7069         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7070                        (match_operand:MVE_5 2 "s_register_operand" "w")
7071                        (match_operand:MVE_5 3 "s_register_operand" "w")
7072                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7073          VMLALDAVAQ_P))
7074   ]
7075   "TARGET_HAVE_MVE"
7076   "vpst\;vmlaldavat.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7077   [(set_attr "type" "mve_move")
7078    (set_attr "length""8")])
7079
7080 ;;
7081 ;; [vmlaldavaxq_p_u, vmlaldavaxq_p_s])
7082 ;;
7083 (define_insn "mve_vmlaldavaxq_p_<supf><mode>"
7084   [
7085    (set (match_operand:DI 0 "s_register_operand" "=r")
7086         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7087                        (match_operand:MVE_5 2 "s_register_operand" "w")
7088                        (match_operand:MVE_5 3 "s_register_operand" "w")
7089                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7090          VMLALDAVAXQ_P))
7091   ]
7092   "TARGET_HAVE_MVE"
7093   "vpst\;vmlaldavaxt.<supf>%#<V_sz_elem> %Q0, %R0, %q2, %q3"
7094   [(set_attr "type" "mve_move")
7095    (set_attr "length""8")])
7096
7097 ;;
7098 ;; [vqrshrnbq_m_n_u, vqrshrnbq_m_n_s])
7099 ;;
7100 (define_insn "mve_vqrshrnbq_m_n_<supf><mode>"
7101   [
7102    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7103         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7104                        (match_operand:MVE_5 2 "s_register_operand" "w")
7105                        (match_operand:SI 3 "mve_imm_8" "Rb")
7106                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7107          VQRSHRNBQ_M_N))
7108   ]
7109   "TARGET_HAVE_MVE"
7110   "vpst\;vqrshrnbt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7111   [(set_attr "type" "mve_move")
7112    (set_attr "length""8")])
7113
7114 ;;
7115 ;; [vqrshrntq_m_n_s, vqrshrntq_m_n_u])
7116 ;;
7117 (define_insn "mve_vqrshrntq_m_n_<supf><mode>"
7118   [
7119    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7120         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7121                        (match_operand:MVE_5 2 "s_register_operand" "w")
7122                        (match_operand:SI 3 "mve_imm_8" "Rb")
7123                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7124          VQRSHRNTQ_M_N))
7125   ]
7126   "TARGET_HAVE_MVE"
7127   "vpst\;vqrshrntt.<supf>%#<V_sz_elem>  %q0, %q2, %3"
7128   [(set_attr "type" "mve_move")
7129    (set_attr "length""8")])
7130
7131 ;;
7132 ;; [vqshrnbq_m_n_u, vqshrnbq_m_n_s])
7133 ;;
7134 (define_insn "mve_vqshrnbq_m_n_<supf><mode>"
7135   [
7136    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7137         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7138                        (match_operand:MVE_5 2 "s_register_operand" "w")
7139                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7140                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7141          VQSHRNBQ_M_N))
7142   ]
7143   "TARGET_HAVE_MVE"
7144   "vpst\n\tvqshrnbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7145   [(set_attr "type" "mve_move")
7146    (set_attr "length""8")])
7147
7148 ;;
7149 ;; [vqshrntq_m_n_s, vqshrntq_m_n_u])
7150 ;;
7151 (define_insn "mve_vqshrntq_m_n_<supf><mode>"
7152   [
7153    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7154         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7155                        (match_operand:MVE_5 2 "s_register_operand" "w")
7156                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7157                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7158          VQSHRNTQ_M_N))
7159   ]
7160   "TARGET_HAVE_MVE"
7161   "vpst\;vqshrntt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7162   [(set_attr "type" "mve_move")
7163    (set_attr "length""8")])
7164
7165 ;;
7166 ;; [vrmlaldavhaq_p_s])
7167 ;;
7168 (define_insn "mve_vrmlaldavhaq_p_sv4si"
7169   [
7170    (set (match_operand:DI 0 "s_register_operand" "=r")
7171         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7172                        (match_operand:V4SI 2 "s_register_operand" "w")
7173                        (match_operand:V4SI 3 "s_register_operand" "w")
7174                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7175          VRMLALDAVHAQ_P_S))
7176   ]
7177   "TARGET_HAVE_MVE"
7178   "vpst\;vrmlaldavhat.s32\t%Q0, %R0, %q2, %q3"
7179   [(set_attr "type" "mve_move")
7180    (set_attr "length""8")])
7181
7182 ;;
7183 ;; [vrshrnbq_m_n_u, vrshrnbq_m_n_s])
7184 ;;
7185 (define_insn "mve_vrshrnbq_m_n_<supf><mode>"
7186   [
7187    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7188         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7189                        (match_operand:MVE_5 2 "s_register_operand" "w")
7190                        (match_operand:SI 3 "mve_imm_8" "Rb")
7191                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7192          VRSHRNBQ_M_N))
7193   ]
7194   "TARGET_HAVE_MVE"
7195   "vpst\;vrshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7196   [(set_attr "type" "mve_move")
7197    (set_attr "length""8")])
7198
7199 ;;
7200 ;; [vrshrntq_m_n_u, vrshrntq_m_n_s])
7201 ;;
7202 (define_insn "mve_vrshrntq_m_n_<supf><mode>"
7203   [
7204    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7205         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7206                        (match_operand:MVE_5 2 "s_register_operand" "w")
7207                        (match_operand:SI 3 "mve_imm_8" "Rb")
7208                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7209          VRSHRNTQ_M_N))
7210   ]
7211   "TARGET_HAVE_MVE"
7212   "vpst\;vrshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7213   [(set_attr "type" "mve_move")
7214    (set_attr "length""8")])
7215
7216 ;;
7217 ;; [vshllbq_m_n_u, vshllbq_m_n_s])
7218 ;;
7219 (define_insn "mve_vshllbq_m_n_<supf><mode>"
7220   [
7221    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7222         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7223                        (match_operand:MVE_3 2 "s_register_operand" "w")
7224                        (match_operand:SI 3 "immediate_operand" "i")
7225                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7226          VSHLLBQ_M_N))
7227   ]
7228   "TARGET_HAVE_MVE"
7229   "vpst\;vshllbt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7230   [(set_attr "type" "mve_move")
7231    (set_attr "length""8")])
7232
7233 ;;
7234 ;; [vshlltq_m_n_u, vshlltq_m_n_s])
7235 ;;
7236 (define_insn "mve_vshlltq_m_n_<supf><mode>"
7237   [
7238    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7239         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7240                        (match_operand:MVE_3 2 "s_register_operand" "w")
7241                        (match_operand:SI 3 "immediate_operand" "i")
7242                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7243          VSHLLTQ_M_N))
7244   ]
7245   "TARGET_HAVE_MVE"
7246   "vpst\;vshlltt.<supf>%#<V_sz_elem>\t%q0, %q2, %3"
7247   [(set_attr "type" "mve_move")
7248    (set_attr "length""8")])
7249
7250 ;;
7251 ;; [vshrnbq_m_n_s, vshrnbq_m_n_u])
7252 ;;
7253 (define_insn "mve_vshrnbq_m_n_<supf><mode>"
7254   [
7255    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7256         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7257                        (match_operand:MVE_5 2 "s_register_operand" "w")
7258                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7259                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7260          VSHRNBQ_M_N))
7261   ]
7262   "TARGET_HAVE_MVE"
7263   "vpst\;vshrnbt.i%#<V_sz_elem>\t%q0, %q2, %3"
7264   [(set_attr "type" "mve_move")
7265    (set_attr "length""8")])
7266
7267 ;;
7268 ;; [vshrntq_m_n_s, vshrntq_m_n_u])
7269 ;;
7270 (define_insn "mve_vshrntq_m_n_<supf><mode>"
7271   [
7272    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7273         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7274                        (match_operand:MVE_5 2 "s_register_operand" "w")
7275                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7276                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7277          VSHRNTQ_M_N))
7278   ]
7279   "TARGET_HAVE_MVE"
7280   "vpst\;vshrntt.i%#<V_sz_elem>\t%q0, %q2, %3"
7281   [(set_attr "type" "mve_move")
7282    (set_attr "length""8")])
7283
7284 ;;
7285 ;; [vmlsldavaq_p_s])
7286 ;;
7287 (define_insn "mve_vmlsldavaq_p_s<mode>"
7288   [
7289    (set (match_operand:DI 0 "s_register_operand" "=r")
7290         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7291                        (match_operand:MVE_5 2 "s_register_operand" "w")
7292                        (match_operand:MVE_5 3 "s_register_operand" "w")
7293                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7294          VMLSLDAVAQ_P_S))
7295   ]
7296   "TARGET_HAVE_MVE"
7297   "vpst\;vmlsldavat.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7298   [(set_attr "type" "mve_move")
7299    (set_attr "length""8")])
7300
7301 ;;
7302 ;; [vmlsldavaxq_p_s])
7303 ;;
7304 (define_insn "mve_vmlsldavaxq_p_s<mode>"
7305   [
7306    (set (match_operand:DI 0 "s_register_operand" "=r")
7307         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7308                        (match_operand:MVE_5 2 "s_register_operand" "w")
7309                        (match_operand:MVE_5 3 "s_register_operand" "w")
7310                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7311          VMLSLDAVAXQ_P_S))
7312   ]
7313   "TARGET_HAVE_MVE"
7314   "vpst\;vmlsldavaxt.s%#<V_sz_elem>\t%Q0, %R0, %q2, %q3"
7315   [(set_attr "type" "mve_move")
7316    (set_attr "length""8")])
7317
7318 ;;
7319 ;; [vmullbq_poly_m_p])
7320 ;;
7321 (define_insn "mve_vmullbq_poly_m_p<mode>"
7322   [
7323    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7324         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7325                        (match_operand:MVE_3 2 "s_register_operand" "w")
7326                        (match_operand:MVE_3 3 "s_register_operand" "w")
7327                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7328          VMULLBQ_POLY_M_P))
7329   ]
7330   "TARGET_HAVE_MVE"
7331   "vpst\;vmullbt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7332   [(set_attr "type" "mve_move")
7333    (set_attr "length""8")])
7334
7335 ;;
7336 ;; [vmulltq_poly_m_p])
7337 ;;
7338 (define_insn "mve_vmulltq_poly_m_p<mode>"
7339   [
7340    (set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
7341         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7342                        (match_operand:MVE_3 2 "s_register_operand" "w")
7343                        (match_operand:MVE_3 3 "s_register_operand" "w")
7344                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7345          VMULLTQ_POLY_M_P))
7346   ]
7347   "TARGET_HAVE_MVE"
7348   "vpst\;vmulltt.p%#<V_sz_elem>\t%q0, %q2, %q3"
7349   [(set_attr "type" "mve_move")
7350    (set_attr "length""8")])
7351
7352 ;;
7353 ;; [vqdmullbq_m_n_s])
7354 ;;
7355 (define_insn "mve_vqdmullbq_m_n_s<mode>"
7356   [
7357    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7358         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7359                        (match_operand:MVE_5 2 "s_register_operand" "w")
7360                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7361                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7362          VQDMULLBQ_M_N_S))
7363   ]
7364   "TARGET_HAVE_MVE"
7365   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7366   [(set_attr "type" "mve_move")
7367    (set_attr "length""8")])
7368
7369 ;;
7370 ;; [vqdmullbq_m_s])
7371 ;;
7372 (define_insn "mve_vqdmullbq_m_s<mode>"
7373   [
7374    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7375         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7376                        (match_operand:MVE_5 2 "s_register_operand" "w")
7377                        (match_operand:MVE_5 3 "s_register_operand" "w")
7378                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7379          VQDMULLBQ_M_S))
7380   ]
7381   "TARGET_HAVE_MVE"
7382   "vpst\;vqdmullbt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7383   [(set_attr "type" "mve_move")
7384    (set_attr "length""8")])
7385
7386 ;;
7387 ;; [vqdmulltq_m_n_s])
7388 ;;
7389 (define_insn "mve_vqdmulltq_m_n_s<mode>"
7390   [
7391    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7392         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7393                        (match_operand:MVE_5 2 "s_register_operand" "w")
7394                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7395                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7396          VQDMULLTQ_M_N_S))
7397   ]
7398   "TARGET_HAVE_MVE"
7399   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %3"
7400   [(set_attr "type" "mve_move")
7401    (set_attr "length""8")])
7402
7403 ;;
7404 ;; [vqdmulltq_m_s])
7405 ;;
7406 (define_insn "mve_vqdmulltq_m_s<mode>"
7407   [
7408    (set (match_operand:<V_double_width> 0 "s_register_operand" "<earlyclobber_32>")
7409         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
7410                        (match_operand:MVE_5 2 "s_register_operand" "w")
7411                        (match_operand:MVE_5 3 "s_register_operand" "w")
7412                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7413          VQDMULLTQ_M_S))
7414   ]
7415   "TARGET_HAVE_MVE"
7416   "vpst\;vqdmulltt.s%#<V_sz_elem>\t%q0, %q2, %q3"
7417   [(set_attr "type" "mve_move")
7418    (set_attr "length""8")])
7419
7420 ;;
7421 ;; [vqrshrunbq_m_n_s])
7422 ;;
7423 (define_insn "mve_vqrshrunbq_m_n_s<mode>"
7424   [
7425    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7426         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7427                        (match_operand:MVE_5 2 "s_register_operand" "w")
7428                        (match_operand:SI 3 "mve_imm_8" "Rb")
7429                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7430          VQRSHRUNBQ_M_N_S))
7431   ]
7432   "TARGET_HAVE_MVE"
7433   "vpst\;vqrshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7434   [(set_attr "type" "mve_move")
7435    (set_attr "length""8")])
7436
7437 ;;
7438 ;; [vqrshruntq_m_n_s])
7439 ;;
7440 (define_insn "mve_vqrshruntq_m_n_s<mode>"
7441   [
7442    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7443         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7444                        (match_operand:MVE_5 2 "s_register_operand" "w")
7445                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7446                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7447          VQRSHRUNTQ_M_N_S))
7448   ]
7449   "TARGET_HAVE_MVE"
7450   "vpst\;vqrshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7451   [(set_attr "type" "mve_move")
7452    (set_attr "length""8")])
7453
7454 ;;
7455 ;; [vqshrunbq_m_n_s])
7456 ;;
7457 (define_insn "mve_vqshrunbq_m_n_s<mode>"
7458   [
7459    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7460         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7461                        (match_operand:MVE_5 2 "s_register_operand" "w")
7462                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7463                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7464          VQSHRUNBQ_M_N_S))
7465   ]
7466   "TARGET_HAVE_MVE"
7467   "vpst\;vqshrunbt.s%#<V_sz_elem>\t%q0, %q2, %3"
7468   [(set_attr "type" "mve_move")
7469    (set_attr "length""8")])
7470
7471 ;;
7472 ;; [vqshruntq_m_n_s])
7473 ;;
7474 (define_insn "mve_vqshruntq_m_n_s<mode>"
7475   [
7476    (set (match_operand:<V_narrow_pack> 0 "s_register_operand" "=w")
7477         (unspec:<V_narrow_pack> [(match_operand:<V_narrow_pack> 1 "s_register_operand" "0")
7478                        (match_operand:MVE_5 2 "s_register_operand" "w")
7479                        (match_operand:SI 3 "<MVE_pred3>" "<MVE_constraint3>")
7480                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7481          VQSHRUNTQ_M_N_S))
7482   ]
7483   "TARGET_HAVE_MVE"
7484   "vpst\;vqshruntt.s%#<V_sz_elem>\t%q0, %q2, %3"
7485   [(set_attr "type" "mve_move")
7486    (set_attr "length""8")])
7487
7488 ;;
7489 ;; [vrmlaldavhaq_p_u])
7490 ;;
7491 (define_insn "mve_vrmlaldavhaq_p_uv4si"
7492   [
7493    (set (match_operand:DI 0 "s_register_operand" "=r")
7494         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7495                        (match_operand:V4SI 2 "s_register_operand" "w")
7496                        (match_operand:V4SI 3 "s_register_operand" "w")
7497                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7498          VRMLALDAVHAQ_P_U))
7499   ]
7500   "TARGET_HAVE_MVE"
7501   "vpst\;vrmlaldavhat.u32\t%Q0, %R0, %q2, %q3"
7502   [(set_attr "type" "mve_move")
7503    (set_attr "length""8")])
7504
7505 ;;
7506 ;; [vrmlaldavhaxq_p_s])
7507 ;;
7508 (define_insn "mve_vrmlaldavhaxq_p_sv4si"
7509   [
7510    (set (match_operand:DI 0 "s_register_operand" "=r")
7511         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7512                        (match_operand:V4SI 2 "s_register_operand" "w")
7513                        (match_operand:V4SI 3 "s_register_operand" "w")
7514                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7515          VRMLALDAVHAXQ_P_S))
7516   ]
7517   "TARGET_HAVE_MVE"
7518   "vpst\;vrmlaldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7519   [(set_attr "type" "mve_move")
7520    (set_attr "length""8")])
7521
7522 ;;
7523 ;; [vrmlsldavhaq_p_s])
7524 ;;
7525 (define_insn "mve_vrmlsldavhaq_p_sv4si"
7526   [
7527    (set (match_operand:DI 0 "s_register_operand" "=r")
7528         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7529                        (match_operand:V4SI 2 "s_register_operand" "w")
7530                        (match_operand:V4SI 3 "s_register_operand" "w")
7531                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7532          VRMLSLDAVHAQ_P_S))
7533   ]
7534   "TARGET_HAVE_MVE"
7535   "vpst\;vrmlsldavhat.s32\t%Q0, %R0, %q2, %q3"
7536   [(set_attr "type" "mve_move")
7537    (set_attr "length""8")])
7538
7539 ;;
7540 ;; [vrmlsldavhaxq_p_s])
7541 ;;
7542 (define_insn "mve_vrmlsldavhaxq_p_sv4si"
7543   [
7544    (set (match_operand:DI 0 "s_register_operand" "=r")
7545         (unspec:DI [(match_operand:DI 1 "s_register_operand" "0")
7546                        (match_operand:V4SI 2 "s_register_operand" "w")
7547                        (match_operand:V4SI 3 "s_register_operand" "w")
7548                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7549          VRMLSLDAVHAXQ_P_S))
7550   ]
7551   "TARGET_HAVE_MVE"
7552   "vpst\;vrmlsldavhaxt.s32\t%Q0, %R0, %q2, %q3"
7553   [(set_attr "type" "mve_move")
7554    (set_attr "length""8")])
7555 ;;
7556 ;; [vabdq_m_f])
7557 ;;
7558 (define_insn "mve_vabdq_m_f<mode>"
7559   [
7560    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7561         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7562                        (match_operand:MVE_0 2 "s_register_operand" "w")
7563                        (match_operand:MVE_0 3 "s_register_operand" "w")
7564                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7565          VABDQ_M_F))
7566   ]
7567   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7568   "vpst\;vabdt.f%#<V_sz_elem>   %q0, %q2, %q3"
7569   [(set_attr "type" "mve_move")
7570    (set_attr "length""8")])
7571
7572 ;;
7573 ;; [vaddq_m_f])
7574 ;;
7575 (define_insn "mve_vaddq_m_f<mode>"
7576   [
7577    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7578         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7579                        (match_operand:MVE_0 2 "s_register_operand" "w")
7580                        (match_operand:MVE_0 3 "s_register_operand" "w")
7581                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7582          VADDQ_M_F))
7583   ]
7584   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7585   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %q3"
7586   [(set_attr "type" "mve_move")
7587    (set_attr "length""8")])
7588
7589 ;;
7590 ;; [vaddq_m_n_f])
7591 ;;
7592 (define_insn "mve_vaddq_m_n_f<mode>"
7593   [
7594    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7595         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7596                        (match_operand:MVE_0 2 "s_register_operand" "w")
7597                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7598                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7599          VADDQ_M_N_F))
7600   ]
7601   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7602   "vpst\;vaddt.f%#<V_sz_elem>   %q0, %q2, %3"
7603   [(set_attr "type" "mve_move")
7604    (set_attr "length""8")])
7605
7606 ;;
7607 ;; [vandq_m_f])
7608 ;;
7609 (define_insn "mve_vandq_m_f<mode>"
7610   [
7611    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7612         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7613                        (match_operand:MVE_0 2 "s_register_operand" "w")
7614                        (match_operand:MVE_0 3 "s_register_operand" "w")
7615                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7616          VANDQ_M_F))
7617   ]
7618   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7619   "vpst\;vandt %q0, %q2, %q3"
7620   [(set_attr "type" "mve_move")
7621    (set_attr "length""8")])
7622
7623 ;;
7624 ;; [vbicq_m_f])
7625 ;;
7626 (define_insn "mve_vbicq_m_f<mode>"
7627   [
7628    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7629         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7630                        (match_operand:MVE_0 2 "s_register_operand" "w")
7631                        (match_operand:MVE_0 3 "s_register_operand" "w")
7632                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7633          VBICQ_M_F))
7634   ]
7635   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7636   "vpst\;vbict %q0, %q2, %q3"
7637   [(set_attr "type" "mve_move")
7638    (set_attr "length""8")])
7639
7640 ;;
7641 ;; [vbrsrq_m_n_f])
7642 ;;
7643 (define_insn "mve_vbrsrq_m_n_f<mode>"
7644   [
7645    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7646         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7647                        (match_operand:MVE_0 2 "s_register_operand" "w")
7648                        (match_operand:SI 3 "s_register_operand" "r")
7649                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7650          VBRSRQ_M_N_F))
7651   ]
7652   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7653   "vpst\;vbrsrt.%#<V_sz_elem>   %q0, %q2, %3"
7654   [(set_attr "type" "mve_move")
7655    (set_attr "length""8")])
7656
7657 ;;
7658 ;; [vcaddq_rot270_m_f])
7659 ;;
7660 (define_insn "mve_vcaddq_rot270_m_f<mode>"
7661   [
7662    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7663         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7664                        (match_operand:MVE_0 2 "s_register_operand" "w")
7665                        (match_operand:MVE_0 3 "s_register_operand" "w")
7666                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7667          VCADDQ_ROT270_M_F))
7668   ]
7669   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7670   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7671   [(set_attr "type" "mve_move")
7672    (set_attr "length""8")])
7673
7674 ;;
7675 ;; [vcaddq_rot90_m_f])
7676 ;;
7677 (define_insn "mve_vcaddq_rot90_m_f<mode>"
7678   [
7679    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7680         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7681                        (match_operand:MVE_0 2 "s_register_operand" "w")
7682                        (match_operand:MVE_0 3 "s_register_operand" "w")
7683                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7684          VCADDQ_ROT90_M_F))
7685   ]
7686   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7687   "vpst\;vcaddt.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7688   [(set_attr "type" "mve_move")
7689    (set_attr "length""8")])
7690
7691 ;;
7692 ;; [vcmlaq_m_f])
7693 ;;
7694 (define_insn "mve_vcmlaq_m_f<mode>"
7695   [
7696    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7697         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7698                        (match_operand:MVE_0 2 "s_register_operand" "w")
7699                        (match_operand:MVE_0 3 "s_register_operand" "w")
7700                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7701          VCMLAQ_M_F))
7702   ]
7703   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7704   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7705   [(set_attr "type" "mve_move")
7706    (set_attr "length""8")])
7707
7708 ;;
7709 ;; [vcmlaq_rot180_m_f])
7710 ;;
7711 (define_insn "mve_vcmlaq_rot180_m_f<mode>"
7712   [
7713    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7714         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7715                        (match_operand:MVE_0 2 "s_register_operand" "w")
7716                        (match_operand:MVE_0 3 "s_register_operand" "w")
7717                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7718          VCMLAQ_ROT180_M_F))
7719   ]
7720   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7721   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7722   [(set_attr "type" "mve_move")
7723    (set_attr "length""8")])
7724
7725 ;;
7726 ;; [vcmlaq_rot270_m_f])
7727 ;;
7728 (define_insn "mve_vcmlaq_rot270_m_f<mode>"
7729   [
7730    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7731         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7732                        (match_operand:MVE_0 2 "s_register_operand" "w")
7733                        (match_operand:MVE_0 3 "s_register_operand" "w")
7734                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7735          VCMLAQ_ROT270_M_F))
7736   ]
7737   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7738   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7739   [(set_attr "type" "mve_move")
7740    (set_attr "length""8")])
7741
7742 ;;
7743 ;; [vcmlaq_rot90_m_f])
7744 ;;
7745 (define_insn "mve_vcmlaq_rot90_m_f<mode>"
7746   [
7747    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7748         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7749                        (match_operand:MVE_0 2 "s_register_operand" "w")
7750                        (match_operand:MVE_0 3 "s_register_operand" "w")
7751                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7752          VCMLAQ_ROT90_M_F))
7753   ]
7754   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7755   "vpst\;vcmlat.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7756   [(set_attr "type" "mve_move")
7757    (set_attr "length""8")])
7758
7759 ;;
7760 ;; [vcmulq_m_f])
7761 ;;
7762 (define_insn "mve_vcmulq_m_f<mode>"
7763   [
7764    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7765         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7766                        (match_operand:MVE_0 2 "s_register_operand" "w")
7767                        (match_operand:MVE_0 3 "s_register_operand" "w")
7768                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7769          VCMULQ_M_F))
7770   ]
7771   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7772   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #0"
7773   [(set_attr "type" "mve_move")
7774    (set_attr "length""8")])
7775
7776 ;;
7777 ;; [vcmulq_rot180_m_f])
7778 ;;
7779 (define_insn "mve_vcmulq_rot180_m_f<mode>"
7780   [
7781    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7782         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7783                        (match_operand:MVE_0 2 "s_register_operand" "w")
7784                        (match_operand:MVE_0 3 "s_register_operand" "w")
7785                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7786          VCMULQ_ROT180_M_F))
7787   ]
7788   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7789   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #180"
7790   [(set_attr "type" "mve_move")
7791    (set_attr "length""8")])
7792
7793 ;;
7794 ;; [vcmulq_rot270_m_f])
7795 ;;
7796 (define_insn "mve_vcmulq_rot270_m_f<mode>"
7797   [
7798    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7799         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7800                        (match_operand:MVE_0 2 "s_register_operand" "w")
7801                        (match_operand:MVE_0 3 "s_register_operand" "w")
7802                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7803          VCMULQ_ROT270_M_F))
7804   ]
7805   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7806   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #270"
7807   [(set_attr "type" "mve_move")
7808    (set_attr "length""8")])
7809
7810 ;;
7811 ;; [vcmulq_rot90_m_f])
7812 ;;
7813 (define_insn "mve_vcmulq_rot90_m_f<mode>"
7814   [
7815    (set (match_operand:MVE_0 0 "s_register_operand" "<earlyclobber_32>")
7816         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7817                        (match_operand:MVE_0 2 "s_register_operand" "w")
7818                        (match_operand:MVE_0 3 "s_register_operand" "w")
7819                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7820          VCMULQ_ROT90_M_F))
7821   ]
7822   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7823   "vpst\;vcmult.f%#<V_sz_elem>  %q0, %q2, %q3, #90"
7824   [(set_attr "type" "mve_move")
7825    (set_attr "length""8")])
7826
7827 ;;
7828 ;; [veorq_m_f])
7829 ;;
7830 (define_insn "mve_veorq_m_f<mode>"
7831   [
7832    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7833         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7834                        (match_operand:MVE_0 2 "s_register_operand" "w")
7835                        (match_operand:MVE_0 3 "s_register_operand" "w")
7836                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7837          VEORQ_M_F))
7838   ]
7839   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7840   "vpst\;veort %q0, %q2, %q3"
7841   [(set_attr "type" "mve_move")
7842    (set_attr "length""8")])
7843
7844 ;;
7845 ;; [vfmaq_m_f])
7846 ;;
7847 (define_insn "mve_vfmaq_m_f<mode>"
7848   [
7849    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7850         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7851                        (match_operand:MVE_0 2 "s_register_operand" "w")
7852                        (match_operand:MVE_0 3 "s_register_operand" "w")
7853                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7854          VFMAQ_M_F))
7855   ]
7856   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7857   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %q3"
7858   [(set_attr "type" "mve_move")
7859    (set_attr "length""8")])
7860
7861 ;;
7862 ;; [vfmaq_m_n_f])
7863 ;;
7864 (define_insn "mve_vfmaq_m_n_f<mode>"
7865   [
7866    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7867         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7868                        (match_operand:MVE_0 2 "s_register_operand" "w")
7869                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7870                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7871          VFMAQ_M_N_F))
7872   ]
7873   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7874   "vpst\;vfmat.f%#<V_sz_elem>   %q0, %q2, %3"
7875   [(set_attr "type" "mve_move")
7876    (set_attr "length""8")])
7877
7878 ;;
7879 ;; [vfmasq_m_n_f])
7880 ;;
7881 (define_insn "mve_vfmasq_m_n_f<mode>"
7882   [
7883    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7884         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7885                        (match_operand:MVE_0 2 "s_register_operand" "w")
7886                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7887                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7888          VFMASQ_M_N_F))
7889   ]
7890   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7891   "vpst\;vfmast.f%#<V_sz_elem>  %q0, %q2, %3"
7892   [(set_attr "type" "mve_move")
7893    (set_attr "length""8")])
7894
7895 ;;
7896 ;; [vfmsq_m_f])
7897 ;;
7898 (define_insn "mve_vfmsq_m_f<mode>"
7899   [
7900    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7901         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7902                        (match_operand:MVE_0 2 "s_register_operand" "w")
7903                        (match_operand:MVE_0 3 "s_register_operand" "w")
7904                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7905          VFMSQ_M_F))
7906   ]
7907   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7908   "vpst\;vfmst.f%#<V_sz_elem>   %q0, %q2, %q3"
7909   [(set_attr "type" "mve_move")
7910    (set_attr "length""8")])
7911
7912 ;;
7913 ;; [vmaxnmq_m_f])
7914 ;;
7915 (define_insn "mve_vmaxnmq_m_f<mode>"
7916   [
7917    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7918         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7919                        (match_operand:MVE_0 2 "s_register_operand" "w")
7920                        (match_operand:MVE_0 3 "s_register_operand" "w")
7921                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7922          VMAXNMQ_M_F))
7923   ]
7924   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7925   "vpst\;vmaxnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7926   [(set_attr "type" "mve_move")
7927    (set_attr "length""8")])
7928
7929 ;;
7930 ;; [vminnmq_m_f])
7931 ;;
7932 (define_insn "mve_vminnmq_m_f<mode>"
7933   [
7934    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7935         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7936                        (match_operand:MVE_0 2 "s_register_operand" "w")
7937                        (match_operand:MVE_0 3 "s_register_operand" "w")
7938                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7939          VMINNMQ_M_F))
7940   ]
7941   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7942   "vpst\;vminnmt.f%#<V_sz_elem> %q0, %q2, %q3"
7943   [(set_attr "type" "mve_move")
7944    (set_attr "length""8")])
7945
7946 ;;
7947 ;; [vmulq_m_f])
7948 ;;
7949 (define_insn "mve_vmulq_m_f<mode>"
7950   [
7951    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7952         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7953                        (match_operand:MVE_0 2 "s_register_operand" "w")
7954                        (match_operand:MVE_0 3 "s_register_operand" "w")
7955                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7956          VMULQ_M_F))
7957   ]
7958   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7959   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %q3"
7960   [(set_attr "type" "mve_move")
7961    (set_attr "length""8")])
7962
7963 ;;
7964 ;; [vmulq_m_n_f])
7965 ;;
7966 (define_insn "mve_vmulq_m_n_f<mode>"
7967   [
7968    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7969         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7970                        (match_operand:MVE_0 2 "s_register_operand" "w")
7971                        (match_operand:<V_elem> 3 "s_register_operand" "r")
7972                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7973          VMULQ_M_N_F))
7974   ]
7975   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7976   "vpst\;vmult.f%#<V_sz_elem>   %q0, %q2, %3"
7977   [(set_attr "type" "mve_move")
7978    (set_attr "length""8")])
7979
7980 ;;
7981 ;; [vornq_m_f])
7982 ;;
7983 (define_insn "mve_vornq_m_f<mode>"
7984   [
7985    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
7986         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
7987                        (match_operand:MVE_0 2 "s_register_operand" "w")
7988                        (match_operand:MVE_0 3 "s_register_operand" "w")
7989                        (match_operand:HI 4 "vpr_register_operand" "Up")]
7990          VORNQ_M_F))
7991   ]
7992   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
7993   "vpst\;vornt %q0, %q2, %q3"
7994   [(set_attr "type" "mve_move")
7995    (set_attr "length""8")])
7996
7997 ;;
7998 ;; [vorrq_m_f])
7999 ;;
8000 (define_insn "mve_vorrq_m_f<mode>"
8001   [
8002    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8003         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8004                        (match_operand:MVE_0 2 "s_register_operand" "w")
8005                        (match_operand:MVE_0 3 "s_register_operand" "w")
8006                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8007          VORRQ_M_F))
8008   ]
8009   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8010   "vpst\;vorrt %q0, %q2, %q3"
8011   [(set_attr "type" "mve_move")
8012    (set_attr "length""8")])
8013
8014 ;;
8015 ;; [vsubq_m_f])
8016 ;;
8017 (define_insn "mve_vsubq_m_f<mode>"
8018   [
8019    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8020         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8021                        (match_operand:MVE_0 2 "s_register_operand" "w")
8022                        (match_operand:MVE_0 3 "s_register_operand" "w")
8023                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8024          VSUBQ_M_F))
8025   ]
8026   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8027   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %q3"
8028   [(set_attr "type" "mve_move")
8029    (set_attr "length""8")])
8030
8031 ;;
8032 ;; [vsubq_m_n_f])
8033 ;;
8034 (define_insn "mve_vsubq_m_n_f<mode>"
8035   [
8036    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
8037         (unspec:MVE_0 [(match_operand:MVE_0 1 "s_register_operand" "0")
8038                        (match_operand:MVE_0 2 "s_register_operand" "w")
8039                        (match_operand:<V_elem> 3 "s_register_operand" "r")
8040                        (match_operand:HI 4 "vpr_register_operand" "Up")]
8041          VSUBQ_M_N_F))
8042   ]
8043   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8044   "vpst\;vsubt.f%#<V_sz_elem>\t%q0, %q2, %3"
8045   [(set_attr "type" "mve_move")
8046    (set_attr "length""8")])
8047
8048 ;;
8049 ;; [vstrbq_s vstrbq_u]
8050 ;;
8051 (define_insn "mve_vstrbq_<supf><mode>"
8052   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8053         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")]
8054          VSTRBQ))
8055   ]
8056   "TARGET_HAVE_MVE"
8057 {
8058    rtx ops[2];
8059    int regno = REGNO (operands[1]);
8060    ops[1] = gen_rtx_REG (TImode, regno);
8061    ops[0]  = operands[0];
8062    output_asm_insn("vstrb.<V_sz_elem>\t%q1, %E0",ops);
8063    return "";
8064 }
8065   [(set_attr "length" "4")])
8066
8067 ;;
8068 ;; [vstrbq_scatter_offset_s vstrbq_scatter_offset_u]
8069 ;;
8070 (define_insn "mve_vstrbq_scatter_offset_<supf><mode>"
8071   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8072         (unspec:<MVE_B_ELEM>
8073                 [(match_operand:MVE_2 1 "s_register_operand" "w")
8074                  (match_operand:MVE_2 2 "s_register_operand" "w")]
8075          VSTRBSOQ))
8076   ]
8077   "TARGET_HAVE_MVE"
8078 {
8079    rtx ops[3];
8080    ops[0] = operands[0];
8081    ops[1] = operands[1];
8082    ops[2] = operands[2];
8083    output_asm_insn("vstrb.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
8084    return "";
8085 }
8086   [(set_attr "length" "4")])
8087
8088 ;;
8089 ;; [vstrwq_scatter_base_s vstrwq_scatter_base_u]
8090 ;;
8091 (define_insn "mve_vstrwq_scatter_base_<supf>v4si"
8092   [(set (mem:BLK (scratch))
8093         (unspec:BLK
8094                 [(match_operand:V4SI 0 "s_register_operand" "w")
8095                  (match_operand:SI 1 "immediate_operand" "i")
8096                  (match_operand:V4SI 2 "s_register_operand" "w")]
8097          VSTRWSBQ))
8098   ]
8099   "TARGET_HAVE_MVE"
8100 {
8101    rtx ops[3];
8102    ops[0] = operands[0];
8103    ops[1] = operands[1];
8104    ops[2] = operands[2];
8105    output_asm_insn("vstrw.u32\t%q2, [%q0, %1]",ops);
8106    return "";
8107 }
8108   [(set_attr "length" "4")])
8109
8110 ;;
8111 ;; [vldrbq_gather_offset_s vldrbq_gather_offset_u]
8112 ;;
8113 (define_insn "mve_vldrbq_gather_offset_<supf><mode>"
8114   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8115         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8116                        (match_operand:MVE_2 2 "s_register_operand" "w")]
8117          VLDRBGOQ))
8118   ]
8119   "TARGET_HAVE_MVE"
8120 {
8121    rtx ops[3];
8122    ops[0] = operands[0];
8123    ops[1] = operands[1];
8124    ops[2] = operands[2];
8125    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8126      output_asm_insn ("vldrb.u8\t%q0, [%m1, %q2]",ops);
8127    else
8128      output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8129    return "";
8130 }
8131   [(set_attr "length" "4")])
8132
8133 ;;
8134 ;; [vldrbq_s vldrbq_u]
8135 ;;
8136 (define_insn "mve_vldrbq_<supf><mode>"
8137   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8138         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")]
8139          VLDRBQ))
8140   ]
8141   "TARGET_HAVE_MVE"
8142 {
8143    rtx ops[2];
8144    int regno = REGNO (operands[0]);
8145    ops[0] = gen_rtx_REG (TImode, regno);
8146    ops[1]  = operands[1];
8147    output_asm_insn ("vldrb.<supf><V_sz_elem>\t%q0, %E1",ops);
8148    return "";
8149 }
8150   [(set_attr "length" "4")])
8151
8152 ;;
8153 ;; [vldrwq_gather_base_s vldrwq_gather_base_u]
8154 ;;
8155 (define_insn "mve_vldrwq_gather_base_<supf>v4si"
8156   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8157         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8158                       (match_operand:SI 2 "immediate_operand" "i")]
8159          VLDRWGBQ))
8160   ]
8161   "TARGET_HAVE_MVE"
8162 {
8163    rtx ops[3];
8164    ops[0] = operands[0];
8165    ops[1] = operands[1];
8166    ops[2] = operands[2];
8167    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8168    return "";
8169 }
8170   [(set_attr "length" "4")])
8171
8172 ;;
8173 ;; [vstrbq_scatter_offset_p_s vstrbq_scatter_offset_p_u]
8174 ;;
8175 (define_insn "mve_vstrbq_scatter_offset_p_<supf><mode>"
8176   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8177         (unspec:<MVE_B_ELEM>
8178                 [(match_operand:MVE_2 1 "s_register_operand" "w")
8179                  (match_operand:MVE_2 2 "s_register_operand" "w")
8180                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8181          VSTRBSOQ))
8182   ]
8183   "TARGET_HAVE_MVE"
8184 {
8185    rtx ops[3];
8186    ops[0] = operands[0];
8187    ops[1] = operands[1];
8188    ops[2] = operands[2];
8189    output_asm_insn ("vpst\n\tvstrbt.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
8190    return "";
8191 }
8192   [(set_attr "length" "8")])
8193
8194 ;;
8195 ;; [vstrwq_scatter_base_p_s vstrwq_scatter_base_p_u]
8196 ;;
8197 (define_insn "mve_vstrwq_scatter_base_p_<supf>v4si"
8198   [(set (mem:BLK (scratch))
8199         (unspec:BLK
8200                 [(match_operand:V4SI 0 "s_register_operand" "w")
8201                  (match_operand:SI 1 "immediate_operand" "i")
8202                  (match_operand:V4SI 2 "s_register_operand" "w")
8203                  (match_operand:HI 3 "vpr_register_operand" "Up")]
8204          VSTRWSBQ))
8205   ]
8206   "TARGET_HAVE_MVE"
8207 {
8208    rtx ops[3];
8209    ops[0] = operands[0];
8210    ops[1] = operands[1];
8211    ops[2] = operands[2];
8212    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
8213    return "";
8214 }
8215   [(set_attr "length" "8")])
8216
8217 ;;
8218 ;; [vstrbq_p_s vstrbq_p_u]
8219 ;;
8220 (define_insn "mve_vstrbq_p_<supf><mode>"
8221   [(set (match_operand:<MVE_B_ELEM> 0 "memory_operand" "=Us")
8222         (unspec:<MVE_B_ELEM> [(match_operand:MVE_2 1 "s_register_operand" "w")
8223                               (match_operand:HI 2 "vpr_register_operand" "Up")]
8224          VSTRBQ))
8225   ]
8226   "TARGET_HAVE_MVE"
8227 {
8228    rtx ops[2];
8229    int regno = REGNO (operands[1]);
8230    ops[1] = gen_rtx_REG (TImode, regno);
8231    ops[0]  = operands[0];
8232    output_asm_insn ("vpst\n\tvstrbt.<V_sz_elem>\t%q1, %E0",ops);
8233    return "";
8234 }
8235   [(set_attr "length" "8")])
8236
8237 ;;
8238 ;; [vldrbq_gather_offset_z_s vldrbq_gather_offset_z_u]
8239 ;;
8240 (define_insn "mve_vldrbq_gather_offset_z_<supf><mode>"
8241   [(set (match_operand:MVE_2 0 "s_register_operand" "=&w")
8242         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8243                        (match_operand:MVE_2 2 "s_register_operand" "w")
8244                        (match_operand:HI 3 "vpr_register_operand" "Up")]
8245          VLDRBGOQ))
8246   ]
8247   "TARGET_HAVE_MVE"
8248 {
8249    rtx ops[4];
8250    ops[0] = operands[0];
8251    ops[1] = operands[1];
8252    ops[2] = operands[2];
8253    ops[3] = operands[3];
8254    if (!strcmp ("<supf>","s") && <V_sz_elem> == 8)
8255      output_asm_insn ("vpst\n\tvldrbt.u8\t%q0, [%m1, %q2]",ops);
8256    else
8257      output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8258    return "";
8259 }
8260   [(set_attr "length" "8")])
8261
8262 ;;
8263 ;; [vldrbq_z_s vldrbq_z_u]
8264 ;;
8265 (define_insn "mve_vldrbq_z_<supf><mode>"
8266   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
8267         (unspec:MVE_2 [(match_operand:<MVE_B_ELEM> 1 "memory_operand" "Us")
8268                        (match_operand:HI 2 "vpr_register_operand" "Up")]
8269          VLDRBQ))
8270   ]
8271   "TARGET_HAVE_MVE"
8272 {
8273    rtx ops[2];
8274    int regno = REGNO (operands[0]);
8275    ops[0] = gen_rtx_REG (TImode, regno);
8276    ops[1]  = operands[1];
8277    output_asm_insn ("vpst\n\tvldrbt.<supf><V_sz_elem>\t%q0, %E1",ops);
8278    return "";
8279 }
8280   [(set_attr "length" "8")])
8281
8282 ;;
8283 ;; [vldrwq_gather_base_z_s vldrwq_gather_base_z_u]
8284 ;;
8285 (define_insn "mve_vldrwq_gather_base_z_<supf>v4si"
8286   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8287         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
8288                       (match_operand:SI 2 "immediate_operand" "i")
8289                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8290          VLDRWGBQ))
8291   ]
8292   "TARGET_HAVE_MVE"
8293 {
8294    rtx ops[3];
8295    ops[0] = operands[0];
8296    ops[1] = operands[1];
8297    ops[2] = operands[2];
8298    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8299    return "";
8300 }
8301   [(set_attr "length" "8")])
8302
8303 ;;
8304 ;; [vldrhq_f]
8305 ;;
8306 (define_insn "mve_vldrhq_fv8hf"
8307   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8308         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")]
8309          VLDRHQ_F))
8310   ]
8311   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8312 {
8313    rtx ops[2];
8314    int regno = REGNO (operands[0]);
8315    ops[0] = gen_rtx_REG (TImode, regno);
8316    ops[1]  = operands[1];
8317    output_asm_insn ("vldrh.f16\t%q0, %E1",ops);
8318    return "";
8319 }
8320   [(set_attr "length" "4")])
8321
8322 ;;
8323 ;; [vldrhq_gather_offset_s vldrhq_gather_offset_u]
8324 ;;
8325 (define_insn "mve_vldrhq_gather_offset_<supf><mode>"
8326   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8327         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8328                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8329         VLDRHGOQ))
8330   ]
8331   "TARGET_HAVE_MVE"
8332 {
8333    rtx ops[3];
8334    ops[0] = operands[0];
8335    ops[1] = operands[1];
8336    ops[2] = operands[2];
8337    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8338      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2]",ops);
8339    else
8340      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8341    return "";
8342 }
8343   [(set_attr "length" "4")])
8344
8345 ;;
8346 ;; [vldrhq_gather_offset_z_s vldrhq_gather_offset_z_u]
8347 ;;
8348 (define_insn "mve_vldrhq_gather_offset_z_<supf><mode>"
8349   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8350         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8351                        (match_operand:MVE_6 2 "s_register_operand" "w")
8352                        (match_operand:HI 3 "vpr_register_operand" "Up")
8353         ]VLDRHGOQ))
8354   ]
8355   "TARGET_HAVE_MVE"
8356 {
8357    rtx ops[4];
8358    ops[0] = operands[0];
8359    ops[1] = operands[1];
8360    ops[2] = operands[2];
8361    ops[3] = operands[3];
8362    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8363      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2]",ops);
8364    else
8365      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2]",ops);
8366    return "";
8367 }
8368  [(set_attr "length" "8")])
8369
8370 ;;
8371 ;; [vldrhq_gather_shifted_offset_s vldrhq_gather_shifted_offset_u]
8372 ;;
8373 (define_insn "mve_vldrhq_gather_shifted_offset_<supf><mode>"
8374   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8375         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8376                        (match_operand:MVE_6 2 "s_register_operand" "w")]
8377         VLDRHGSOQ))
8378   ]
8379   "TARGET_HAVE_MVE"
8380 {
8381    rtx ops[3];
8382    ops[0] = operands[0];
8383    ops[1] = operands[1];
8384    ops[2] = operands[2];
8385       if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8386      output_asm_insn ("vldrh.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8387    else
8388      output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8389    return "";
8390 }
8391   [(set_attr "length" "4")])
8392
8393 ;;
8394 ;; [vldrhq_gather_shifted_offset_z_s vldrhq_gather_shited_offset_z_u]
8395 ;;
8396 (define_insn "mve_vldrhq_gather_shifted_offset_z_<supf><mode>"
8397   [(set (match_operand:MVE_6 0 "s_register_operand" "=&w")
8398         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8399                        (match_operand:MVE_6 2 "s_register_operand" "w")
8400                        (match_operand:HI 3 "vpr_register_operand" "Up")
8401         ]VLDRHGSOQ))
8402   ]
8403   "TARGET_HAVE_MVE"
8404 {
8405    rtx ops[4];
8406    ops[0] = operands[0];
8407    ops[1] = operands[1];
8408    ops[2] = operands[2];
8409    ops[3] = operands[3];
8410    if (!strcmp ("<supf>","s") && <V_sz_elem> == 16)
8411      output_asm_insn ("vpst\n\tvldrht.u16\t%q0, [%m1, %q2, uxtw #1]",ops);
8412    else
8413      output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, [%m1, %q2, uxtw #1]",ops);
8414    return "";
8415 }
8416   [(set_attr "length" "8")])
8417
8418 ;;
8419 ;;
8420 ;; [vldrhq_s, vldrhq_u]
8421 ;;
8422 (define_insn "mve_vldrhq_<supf><mode>"
8423   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8424         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")]
8425          VLDRHQ))
8426   ]
8427   "TARGET_HAVE_MVE"
8428 {
8429    rtx ops[2];
8430    int regno = REGNO (operands[0]);
8431    ops[0] = gen_rtx_REG (TImode, regno);
8432    ops[1]  = operands[1];
8433    output_asm_insn ("vldrh.<supf><V_sz_elem>\t%q0, %E1",ops);
8434    return "";
8435 }
8436   [(set_attr "length" "4")])
8437
8438 ;;
8439 ;; [vldrhq_z_f]
8440 ;;
8441 (define_insn "mve_vldrhq_z_fv8hf"
8442   [(set (match_operand:V8HF 0 "s_register_operand" "=w")
8443         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8444         (match_operand:HI 2 "vpr_register_operand" "Up")]
8445          VLDRHQ_F))
8446   ]
8447   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8448 {
8449    rtx ops[2];
8450    int regno = REGNO (operands[0]);
8451    ops[0] = gen_rtx_REG (TImode, regno);
8452    ops[1]  = operands[1];
8453    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, %E1",ops);
8454    return "";
8455 }
8456   [(set_attr "length" "8")])
8457
8458 ;;
8459 ;; [vldrhq_z_s vldrhq_z_u]
8460 ;;
8461 (define_insn "mve_vldrhq_z_<supf><mode>"
8462   [(set (match_operand:MVE_6 0 "s_register_operand" "=w")
8463         (unspec:MVE_6 [(match_operand:<MVE_H_ELEM> 1 "memory_operand" "Us")
8464         (match_operand:HI 2 "vpr_register_operand" "Up")]
8465          VLDRHQ))
8466   ]
8467   "TARGET_HAVE_MVE"
8468 {
8469    rtx ops[2];
8470    int regno = REGNO (operands[0]);
8471    ops[0] = gen_rtx_REG (TImode, regno);
8472    ops[1]  = operands[1];
8473    output_asm_insn ("vpst\n\tvldrht.<supf><V_sz_elem>\t%q0, %E1",ops);
8474    return "";
8475 }
8476   [(set_attr "length" "8")])
8477
8478 ;;
8479 ;; [vldrwq_f]
8480 ;;
8481 (define_insn "mve_vldrwq_fv4sf"
8482   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8483         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")]
8484          VLDRWQ_F))
8485   ]
8486   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8487 {
8488    rtx ops[2];
8489    int regno = REGNO (operands[0]);
8490    ops[0] = gen_rtx_REG (TImode, regno);
8491    ops[1]  = operands[1];
8492    output_asm_insn ("vldrw.f32\t%q0, %E1",ops);
8493    return "";
8494 }
8495   [(set_attr "length" "4")])
8496
8497 ;;
8498 ;; [vldrwq_s vldrwq_u]
8499 ;;
8500 (define_insn "mve_vldrwq_<supf>v4si"
8501   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8502         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")]
8503          VLDRWQ))
8504   ]
8505   "TARGET_HAVE_MVE"
8506 {
8507    rtx ops[2];
8508    int regno = REGNO (operands[0]);
8509    ops[0] = gen_rtx_REG (TImode, regno);
8510    ops[1]  = operands[1];
8511    output_asm_insn ("vldrw.<supf>32\t%q0, %E1",ops);
8512    return "";
8513 }
8514   [(set_attr "length" "4")])
8515
8516 ;;
8517 ;; [vldrwq_z_f]
8518 ;;
8519 (define_insn "mve_vldrwq_z_fv4sf"
8520   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
8521         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8522         (match_operand:HI 2 "vpr_register_operand" "Up")]
8523          VLDRWQ_F))
8524   ]
8525   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8526 {
8527    rtx ops[2];
8528    int regno = REGNO (operands[0]);
8529    ops[0] = gen_rtx_REG (TImode, regno);
8530    ops[1]  = operands[1];
8531    output_asm_insn ("vpst\n\tvldrwt.f32\t%q0, %E1",ops);
8532    return "";
8533 }
8534   [(set_attr "length" "8")])
8535
8536 ;;
8537 ;; [vldrwq_z_s vldrwq_z_u]
8538 ;;
8539 (define_insn "mve_vldrwq_z_<supf>v4si"
8540   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
8541         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8542         (match_operand:HI 2 "vpr_register_operand" "Up")]
8543          VLDRWQ))
8544   ]
8545   "TARGET_HAVE_MVE"
8546 {
8547    rtx ops[2];
8548    int regno = REGNO (operands[0]);
8549    ops[0] = gen_rtx_REG (TImode, regno);
8550    ops[1]  = operands[1];
8551    output_asm_insn ("vpst\n\tvldrwt.<supf>32\t%q0, %E1",ops);
8552    return "";
8553 }
8554   [(set_attr "length" "8")])
8555
8556 (define_expand "mve_vld1q_f<mode>"
8557   [(match_operand:MVE_0 0 "s_register_operand")
8558    (unspec:MVE_0 [(match_operand:<MVE_CNVT> 1 "memory_operand")] VLD1Q_F)
8559   ]
8560   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
8561 {
8562   emit_insn (gen_mve_vldr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
8563   DONE;
8564 })
8565
8566 (define_expand "mve_vld1q_<supf><mode>"
8567   [(match_operand:MVE_2 0 "s_register_operand")
8568    (unspec:MVE_2 [(match_operand:MVE_2 1 "memory_operand")] VLD1Q)
8569   ]
8570   "TARGET_HAVE_MVE"
8571 {
8572   emit_insn (gen_mve_vldr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
8573   DONE;
8574 })
8575
8576 ;;
8577 ;; [vldrdq_gather_base_s vldrdq_gather_base_u]
8578 ;;
8579 (define_insn "mve_vldrdq_gather_base_<supf>v2di"
8580   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8581         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8582                       (match_operand:SI 2 "immediate_operand" "i")]
8583          VLDRDGBQ))
8584   ]
8585   "TARGET_HAVE_MVE"
8586 {
8587    rtx ops[3];
8588    ops[0] = operands[0];
8589    ops[1] = operands[1];
8590    ops[2] = operands[2];
8591    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]",ops);
8592    return "";
8593 }
8594   [(set_attr "length" "4")])
8595
8596 ;;
8597 ;; [vldrdq_gather_base_z_s vldrdq_gather_base_z_u]
8598 ;;
8599 (define_insn "mve_vldrdq_gather_base_z_<supf>v2di"
8600   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8601         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")
8602                       (match_operand:SI 2 "immediate_operand" "i")
8603                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8604          VLDRDGBQ))
8605   ]
8606   "TARGET_HAVE_MVE"
8607 {
8608    rtx ops[3];
8609    ops[0] = operands[0];
8610    ops[1] = operands[1];
8611    ops[2] = operands[2];
8612    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%q1, %2]",ops);
8613    return "";
8614 }
8615   [(set_attr "length" "8")])
8616
8617 ;;
8618 ;; [vldrdq_gather_offset_s vldrdq_gather_offset_u]
8619 ;;
8620 (define_insn "mve_vldrdq_gather_offset_<supf>v2di"
8621  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8622        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8623                      (match_operand:V2DI 2 "s_register_operand" "w")]
8624         VLDRDGOQ))
8625  ]
8626  "TARGET_HAVE_MVE"
8627 {
8628   rtx ops[3];
8629   ops[0] = operands[0];
8630   ops[1] = operands[1];
8631   ops[2] = operands[2];
8632   output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2]",ops);
8633   return "";
8634 }
8635  [(set_attr "length" "4")])
8636
8637 ;;
8638 ;; [vldrdq_gather_offset_z_s vldrdq_gather_offset_z_u]
8639 ;;
8640 (define_insn "mve_vldrdq_gather_offset_z_<supf>v2di"
8641  [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8642        (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8643                      (match_operand:V2DI 2 "s_register_operand" "w")
8644                      (match_operand:HI 3 "vpr_register_operand" "Up")]
8645         VLDRDGOQ))
8646  ]
8647  "TARGET_HAVE_MVE"
8648 {
8649   rtx ops[3];
8650   ops[0] = operands[0];
8651   ops[1] = operands[1];
8652   ops[2] = operands[2];
8653   output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2]",ops);
8654   return "";
8655 }
8656  [(set_attr "length" "8")])
8657
8658 ;;
8659 ;; [vldrdq_gather_shifted_offset_s vldrdq_gather_shifted_offset_u]
8660 ;;
8661 (define_insn "mve_vldrdq_gather_shifted_offset_<supf>v2di"
8662   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8663         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8664                       (match_operand:V2DI 2 "s_register_operand" "w")]
8665          VLDRDGSOQ))
8666   ]
8667   "TARGET_HAVE_MVE"
8668 {
8669    rtx ops[3];
8670    ops[0] = operands[0];
8671    ops[1] = operands[1];
8672    ops[2] = operands[2];
8673    output_asm_insn ("vldrd.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8674    return "";
8675 }
8676   [(set_attr "length" "4")])
8677
8678 ;;
8679 ;; [vldrdq_gather_shifted_offset_z_s vldrdq_gather_shifted_offset_z_u]
8680 ;;
8681 (define_insn "mve_vldrdq_gather_shifted_offset_z_<supf>v2di"
8682   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
8683         (unspec:V2DI [(match_operand:V2DI 1 "memory_operand" "Us")
8684                       (match_operand:V2DI 2 "s_register_operand" "w")
8685                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8686          VLDRDGSOQ))
8687   ]
8688   "TARGET_HAVE_MVE"
8689 {
8690    rtx ops[3];
8691    ops[0] = operands[0];
8692    ops[1] = operands[1];
8693    ops[2] = operands[2];
8694    output_asm_insn ("vpst\n\tvldrdt.u64\t%q0, [%m1, %q2, uxtw #3]",ops);
8695    return "";
8696 }
8697   [(set_attr "length" "8")])
8698
8699 ;;
8700 ;; [vldrhq_gather_offset_f]
8701 ;;
8702 (define_insn "mve_vldrhq_gather_offset_fv8hf"
8703   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8704         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8705                       (match_operand:V8HI 2 "s_register_operand" "w")]
8706          VLDRHQGO_F))
8707   ]
8708   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8709 {
8710    rtx ops[3];
8711    ops[0] = operands[0];
8712    ops[1] = operands[1];
8713    ops[2] = operands[2];
8714    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2]",ops);
8715    return "";
8716 }
8717   [(set_attr "length" "4")])
8718
8719 ;;
8720 ;; [vldrhq_gather_offset_z_f]
8721 ;;
8722 (define_insn "mve_vldrhq_gather_offset_z_fv8hf"
8723   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8724         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8725                       (match_operand:V8HI 2 "s_register_operand" "w")
8726                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8727          VLDRHQGO_F))
8728   ]
8729   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8730 {
8731    rtx ops[4];
8732    ops[0] = operands[0];
8733    ops[1] = operands[1];
8734    ops[2] = operands[2];
8735    ops[3] = operands[3];
8736    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2]",ops);
8737    return "";
8738 }
8739   [(set_attr "length" "8")])
8740
8741 ;;
8742 ;; [vldrhq_gather_shifted_offset_f]
8743 ;;
8744 (define_insn "mve_vldrhq_gather_shifted_offset_fv8hf"
8745   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8746         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8747                       (match_operand:V8HI 2 "s_register_operand" "w")]
8748          VLDRHQGSO_F))
8749   ]
8750   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8751 {
8752    rtx ops[3];
8753    ops[0] = operands[0];
8754    ops[1] = operands[1];
8755    ops[2] = operands[2];
8756    output_asm_insn ("vldrh.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8757    return "";
8758 }
8759   [(set_attr "length" "4")])
8760
8761 ;;
8762 ;; [vldrhq_gather_shifted_offset_z_f]
8763 ;;
8764 (define_insn "mve_vldrhq_gather_shifted_offset_z_fv8hf"
8765   [(set (match_operand:V8HF 0 "s_register_operand" "=&w")
8766         (unspec:V8HF [(match_operand:V8HI 1 "memory_operand" "Us")
8767                       (match_operand:V8HI 2 "s_register_operand" "w")
8768                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8769          VLDRHQGSO_F))
8770   ]
8771   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8772 {
8773    rtx ops[4];
8774    ops[0] = operands[0];
8775    ops[1] = operands[1];
8776    ops[2] = operands[2];
8777    ops[3] = operands[3];
8778    output_asm_insn ("vpst\n\tvldrht.f16\t%q0, [%m1, %q2, uxtw #1]",ops);
8779    return "";
8780 }
8781   [(set_attr "length" "8")])
8782
8783 ;;
8784 ;; [vldrwq_gather_base_f]
8785 ;;
8786 (define_insn "mve_vldrwq_gather_base_fv4sf"
8787   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8788         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8789                       (match_operand:SI 2 "immediate_operand" "i")]
8790          VLDRWQGB_F))
8791   ]
8792   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8793 {
8794    rtx ops[3];
8795    ops[0] = operands[0];
8796    ops[1] = operands[1];
8797    ops[2] = operands[2];
8798    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]",ops);
8799    return "";
8800 }
8801   [(set_attr "length" "4")])
8802
8803 ;;
8804 ;; [vldrwq_gather_base_z_f]
8805 ;;
8806 (define_insn "mve_vldrwq_gather_base_z_fv4sf"
8807   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8808         (unspec:V4SF [(match_operand:V4SI 1 "s_register_operand" "w")
8809                       (match_operand:SI 2 "immediate_operand" "i")
8810                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8811          VLDRWQGB_F))
8812   ]
8813   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8814 {
8815    rtx ops[3];
8816    ops[0] = operands[0];
8817    ops[1] = operands[1];
8818    ops[2] = operands[2];
8819    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%q1, %2]",ops);
8820    return "";
8821 }
8822   [(set_attr "length" "8")])
8823
8824 ;;
8825 ;; [vldrwq_gather_offset_f]
8826 ;;
8827 (define_insn "mve_vldrwq_gather_offset_fv4sf"
8828   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8829         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8830                        (match_operand:V4SI 2 "s_register_operand" "w")]
8831          VLDRWQGO_F))
8832   ]
8833   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8834 {
8835    rtx ops[3];
8836    ops[0] = operands[0];
8837    ops[1] = operands[1];
8838    ops[2] = operands[2];
8839    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8840    return "";
8841 }
8842   [(set_attr "length" "4")])
8843
8844 ;;
8845 ;; [vldrwq_gather_offset_s vldrwq_gather_offset_u]
8846 ;;
8847 (define_insn "mve_vldrwq_gather_offset_<supf>v4si"
8848   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8849         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8850                        (match_operand:V4SI 2 "s_register_operand" "w")]
8851          VLDRWGOQ))
8852   ]
8853   "TARGET_HAVE_MVE"
8854 {
8855    rtx ops[3];
8856    ops[0] = operands[0];
8857    ops[1] = operands[1];
8858    ops[2] = operands[2];
8859    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2]",ops);
8860    return "";
8861 }
8862   [(set_attr "length" "4")])
8863
8864 ;;
8865 ;; [vldrwq_gather_offset_z_f]
8866 ;;
8867 (define_insn "mve_vldrwq_gather_offset_z_fv4sf"
8868   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8869         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8870                       (match_operand:V4SI 2 "s_register_operand" "w")
8871                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8872          VLDRWQGO_F))
8873   ]
8874   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8875 {
8876    rtx ops[4];
8877    ops[0] = operands[0];
8878    ops[1] = operands[1];
8879    ops[2] = operands[2];
8880    ops[3] = operands[3];
8881    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8882    return "";
8883 }
8884   [(set_attr "length" "8")])
8885
8886 ;;
8887 ;; [vldrwq_gather_offset_z_s vldrwq_gather_offset_z_u]
8888 ;;
8889 (define_insn "mve_vldrwq_gather_offset_z_<supf>v4si"
8890   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8891         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8892                       (match_operand:V4SI 2 "s_register_operand" "w")
8893                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8894          VLDRWGOQ))
8895   ]
8896   "TARGET_HAVE_MVE"
8897 {
8898    rtx ops[4];
8899    ops[0] = operands[0];
8900    ops[1] = operands[1];
8901    ops[2] = operands[2];
8902    ops[3] = operands[3];
8903    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2]",ops);
8904    return "";
8905 }
8906   [(set_attr "length" "8")])
8907
8908 ;;
8909 ;; [vldrwq_gather_shifted_offset_f]
8910 ;;
8911 (define_insn "mve_vldrwq_gather_shifted_offset_fv4sf"
8912   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8913         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8914                       (match_operand:V4SI 2 "s_register_operand" "w")]
8915          VLDRWQGSO_F))
8916   ]
8917   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8918 {
8919    rtx ops[3];
8920    ops[0] = operands[0];
8921    ops[1] = operands[1];
8922    ops[2] = operands[2];
8923    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8924    return "";
8925 }
8926   [(set_attr "length" "4")])
8927
8928 ;;
8929 ;; [vldrwq_gather_shifted_offset_s vldrwq_gather_shifted_offset_u]
8930 ;;
8931 (define_insn "mve_vldrwq_gather_shifted_offset_<supf>v4si"
8932   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8933         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8934                       (match_operand:V4SI 2 "s_register_operand" "w")]
8935          VLDRWGSOQ))
8936   ]
8937   "TARGET_HAVE_MVE"
8938 {
8939    rtx ops[3];
8940    ops[0] = operands[0];
8941    ops[1] = operands[1];
8942    ops[2] = operands[2];
8943    output_asm_insn ("vldrw.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8944    return "";
8945 }
8946   [(set_attr "length" "4")])
8947
8948 ;;
8949 ;; [vldrwq_gather_shifted_offset_z_f]
8950 ;;
8951 (define_insn "mve_vldrwq_gather_shifted_offset_z_fv4sf"
8952   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
8953         (unspec:V4SF [(match_operand:V4SI 1 "memory_operand" "Us")
8954                       (match_operand:V4SI 2 "s_register_operand" "w")
8955                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8956          VLDRWQGSO_F))
8957   ]
8958   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
8959 {
8960    rtx ops[4];
8961    ops[0] = operands[0];
8962    ops[1] = operands[1];
8963    ops[2] = operands[2];
8964    ops[3] = operands[3];
8965    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8966    return "";
8967 }
8968   [(set_attr "length" "8")])
8969
8970 ;;
8971 ;; [vldrwq_gather_shifted_offset_z_s vldrwq_gather_shifted_offset_z_u]
8972 ;;
8973 (define_insn "mve_vldrwq_gather_shifted_offset_z_<supf>v4si"
8974   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
8975         (unspec:V4SI [(match_operand:V4SI 1 "memory_operand" "Us")
8976                       (match_operand:V4SI 2 "s_register_operand" "w")
8977                       (match_operand:HI 3 "vpr_register_operand" "Up")]
8978          VLDRWGSOQ))
8979   ]
8980   "TARGET_HAVE_MVE"
8981 {
8982    rtx ops[4];
8983    ops[0] = operands[0];
8984    ops[1] = operands[1];
8985    ops[2] = operands[2];
8986    ops[3] = operands[3];
8987    output_asm_insn ("vpst\n\tvldrwt.u32\t%q0, [%m1, %q2, uxtw #2]",ops);
8988    return "";
8989 }
8990   [(set_attr "length" "8")])
8991
8992 ;;
8993 ;; [vstrhq_f]
8994 ;;
8995 (define_insn "mve_vstrhq_fv8hf"
8996   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
8997         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")]
8998          VSTRHQ_F))
8999   ]
9000   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9001 {
9002    rtx ops[2];
9003    int regno = REGNO (operands[1]);
9004    ops[1] = gen_rtx_REG (TImode, regno);
9005    ops[0]  = operands[0];
9006    output_asm_insn ("vstrh.16\t%q1, %E0",ops);
9007    return "";
9008 }
9009   [(set_attr "length" "4")])
9010
9011 ;;
9012 ;; [vstrhq_p_f]
9013 ;;
9014 (define_insn "mve_vstrhq_p_fv8hf"
9015   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9016         (unspec:V8HI [(match_operand:V8HF 1 "s_register_operand" "w")
9017                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9018          VSTRHQ_F))
9019   ]
9020   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9021 {
9022    rtx ops[2];
9023    int regno = REGNO (operands[1]);
9024    ops[1] = gen_rtx_REG (TImode, regno);
9025    ops[0]  = operands[0];
9026    output_asm_insn ("vpst\n\tvstrht.16\t%q1, %E0",ops);
9027    return "";
9028 }
9029   [(set_attr "length" "8")])
9030
9031 ;;
9032 ;; [vstrhq_p_s vstrhq_p_u]
9033 ;;
9034 (define_insn "mve_vstrhq_p_<supf><mode>"
9035   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9036         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")
9037                               (match_operand:HI 2 "vpr_register_operand" "Up")]
9038          VSTRHQ))
9039   ]
9040   "TARGET_HAVE_MVE"
9041 {
9042    rtx ops[2];
9043    int regno = REGNO (operands[1]);
9044    ops[1] = gen_rtx_REG (TImode, regno);
9045    ops[0]  = operands[0];
9046    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q1, %E0",ops);
9047    return "";
9048 }
9049   [(set_attr "length" "8")])
9050
9051 ;;
9052 ;; [vstrhq_scatter_offset_p_s vstrhq_scatter_offset_p_u]
9053 ;;
9054 (define_insn "mve_vstrhq_scatter_offset_p_<supf><mode>"
9055   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9056         (unspec:<MVE_H_ELEM>
9057                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9058                  (match_operand:MVE_6 2 "s_register_operand" "w")
9059                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9060          VSTRHSOQ))
9061   ]
9062   "TARGET_HAVE_MVE"
9063 {
9064    rtx ops[3];
9065    ops[0] = operands[0];
9066    ops[1] = operands[1];
9067    ops[2] = operands[2];
9068    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
9069    return "";
9070 }
9071   [(set_attr "length" "8")])
9072
9073 ;;
9074 ;; [vstrhq_scatter_offset_s vstrhq_scatter_offset_u]
9075 ;;
9076 (define_insn "mve_vstrhq_scatter_offset_<supf><mode>"
9077   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9078         (unspec:<MVE_H_ELEM>
9079                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9080                  (match_operand:MVE_6 2 "s_register_operand" "w")]
9081          VSTRHSOQ))
9082   ]
9083   "TARGET_HAVE_MVE"
9084 {
9085    rtx ops[3];
9086    ops[0] = operands[0];
9087    ops[1] = operands[1];
9088    ops[2] = operands[2];
9089    output_asm_insn ("vstrh.<V_sz_elem>\t%q2, [%m0, %q1]",ops);
9090    return "";
9091 }
9092   [(set_attr "length" "4")])
9093
9094 ;;
9095 ;; [vstrhq_scatter_shifted_offset_p_s vstrhq_scatter_shifted_offset_p_u]
9096 ;;
9097 (define_insn "mve_vstrhq_scatter_shifted_offset_p_<supf><mode>"
9098   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9099         (unspec:<MVE_H_ELEM>
9100                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9101                  (match_operand:MVE_6 2 "s_register_operand" "w")
9102                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9103          VSTRHSSOQ))
9104   ]
9105   "TARGET_HAVE_MVE"
9106 {
9107    rtx ops[3];
9108    ops[0] = operands[0];
9109    ops[1] = operands[1];
9110    ops[2] = operands[2];
9111    output_asm_insn ("vpst\n\tvstrht.<V_sz_elem>\t%q2, [%m0, %q1, uxtw #1]",ops);
9112    return "";
9113 }
9114   [(set_attr "length" "8")])
9115
9116 ;;
9117 ;; [vstrhq_scatter_shifted_offset_s vstrhq_scatter_shifted_offset_u]
9118 ;;
9119 (define_insn "mve_vstrhq_scatter_shifted_offset_<supf><mode>"
9120   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9121         (unspec:<MVE_H_ELEM>
9122                 [(match_operand:MVE_6 1 "s_register_operand" "w")
9123                  (match_operand:MVE_6 2 "s_register_operand" "w")]
9124          VSTRHSSOQ))
9125   ]
9126   "TARGET_HAVE_MVE"
9127 {
9128    rtx ops[3];
9129    ops[0] = operands[0];
9130    ops[1] = operands[1];
9131    ops[2] = operands[2];
9132    output_asm_insn ("vstrh.<V_sz_elem>\t%q2, [%m0, %q1, uxtw #1]",ops);
9133    return "";
9134 }
9135   [(set_attr "length" "4")])
9136
9137 ;;
9138 ;; [vstrhq_s, vstrhq_u]
9139 ;;
9140 (define_insn "mve_vstrhq_<supf><mode>"
9141   [(set (match_operand:<MVE_H_ELEM> 0 "memory_operand" "=Us")
9142         (unspec:<MVE_H_ELEM> [(match_operand:MVE_6 1 "s_register_operand" "w")]
9143          VSTRHQ))
9144   ]
9145   "TARGET_HAVE_MVE"
9146 {
9147    rtx ops[2];
9148    int regno = REGNO (operands[1]);
9149    ops[1] = gen_rtx_REG (TImode, regno);
9150    ops[0]  = operands[0];
9151    output_asm_insn ("vstrh.<V_sz_elem>\t%q1, %E0",ops);
9152    return "";
9153 }
9154   [(set_attr "length" "4")])
9155
9156 ;;
9157 ;; [vstrwq_f]
9158 ;;
9159 (define_insn "mve_vstrwq_fv4sf"
9160   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9161         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")]
9162          VSTRWQ_F))
9163   ]
9164   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9165 {
9166    rtx ops[2];
9167    int regno = REGNO (operands[1]);
9168    ops[1] = gen_rtx_REG (TImode, regno);
9169    ops[0]  = operands[0];
9170    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9171    return "";
9172 }
9173   [(set_attr "length" "4")])
9174
9175 ;;
9176 ;; [vstrwq_p_f]
9177 ;;
9178 (define_insn "mve_vstrwq_p_fv4sf"
9179   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9180         (unspec:V4SI [(match_operand:V4SF 1 "s_register_operand" "w")
9181                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9182          VSTRWQ_F))
9183   ]
9184   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9185 {
9186    rtx ops[2];
9187    int regno = REGNO (operands[1]);
9188    ops[1] = gen_rtx_REG (TImode, regno);
9189    ops[0]  = operands[0];
9190    output_asm_insn ("vpst\n\tvstrwt.32\t%q1, %E0",ops);
9191    return "";
9192 }
9193   [(set_attr "length" "8")])
9194
9195 ;;
9196 ;; [vstrwq_p_s vstrwq_p_u]
9197 ;;
9198 (define_insn "mve_vstrwq_p_<supf>v4si"
9199   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9200         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
9201                       (match_operand:HI 2 "vpr_register_operand" "Up")]
9202          VSTRWQ))
9203   ]
9204   "TARGET_HAVE_MVE"
9205 {
9206    rtx ops[2];
9207    int regno = REGNO (operands[1]);
9208    ops[1] = gen_rtx_REG (TImode, regno);
9209    ops[0]  = operands[0];
9210    output_asm_insn ("vpst\n\tvstrwt.32\t%q1, %E0",ops);
9211    return "";
9212 }
9213   [(set_attr "length" "8")])
9214
9215 ;;
9216 ;; [vstrwq_s vstrwq_u]
9217 ;;
9218 (define_insn "mve_vstrwq_<supf>v4si"
9219   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9220         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")]
9221          VSTRWQ))
9222   ]
9223   "TARGET_HAVE_MVE"
9224 {
9225    rtx ops[2];
9226    int regno = REGNO (operands[1]);
9227    ops[1] = gen_rtx_REG (TImode, regno);
9228    ops[0]  = operands[0];
9229    output_asm_insn ("vstrw.32\t%q1, %E0",ops);
9230    return "";
9231 }
9232   [(set_attr "length" "4")])
9233
9234 (define_expand "mve_vst1q_f<mode>"
9235   [(match_operand:<MVE_CNVT> 0 "memory_operand")
9236    (unspec:<MVE_CNVT> [(match_operand:MVE_0 1 "s_register_operand")] VST1Q_F)
9237   ]
9238   "TARGET_HAVE_MVE || TARGET_HAVE_MVE_FLOAT"
9239 {
9240   emit_insn (gen_mve_vstr<V_sz_elem1>q_f<mode>(operands[0],operands[1]));
9241   DONE;
9242 })
9243
9244 (define_expand "mve_vst1q_<supf><mode>"
9245   [(match_operand:MVE_2 0 "memory_operand")
9246    (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand")] VST1Q)
9247   ]
9248   "TARGET_HAVE_MVE"
9249 {
9250   emit_insn (gen_mve_vstr<V_sz_elem1>q_<supf><mode>(operands[0],operands[1]));
9251   DONE;
9252 })
9253
9254 ;;
9255 ;; [vstrdq_scatter_base_p_s vstrdq_scatter_base_p_u]
9256 ;;
9257 (define_insn "mve_vstrdq_scatter_base_p_<supf>v2di"
9258   [(set (mem:BLK (scratch))
9259         (unspec:BLK
9260                 [(match_operand:V2DI 0 "s_register_operand" "w")
9261                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9262                  (match_operand:V2DI 2 "s_register_operand" "w")
9263                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9264          VSTRDSBQ))
9265   ]
9266   "TARGET_HAVE_MVE"
9267 {
9268    rtx ops[3];
9269    ops[0] = operands[0];
9270    ops[1] = operands[1];
9271    ops[2] = operands[2];
9272    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]",ops);
9273    return "";
9274 }
9275   [(set_attr "length" "8")])
9276
9277 ;;
9278 ;; [vstrdq_scatter_base_s vstrdq_scatter_base_u]
9279 ;;
9280 (define_insn "mve_vstrdq_scatter_base_<supf>v2di"
9281   [(set (mem:BLK (scratch))
9282         (unspec:BLK
9283                 [(match_operand:V2DI 0 "s_register_operand" "=w")
9284                  (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
9285                  (match_operand:V2DI 2 "s_register_operand" "w")]
9286          VSTRDSBQ))
9287   ]
9288   "TARGET_HAVE_MVE"
9289 {
9290    rtx ops[3];
9291    ops[0] = operands[0];
9292    ops[1] = operands[1];
9293    ops[2] = operands[2];
9294    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]",ops);
9295    return "";
9296 }
9297   [(set_attr "length" "4")])
9298
9299 ;;
9300 ;; [vstrdq_scatter_offset_p_s vstrdq_scatter_offset_p_u]
9301 ;;
9302 (define_insn "mve_vstrdq_scatter_offset_p_<supf>v2di"
9303   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9304         (unspec:V2DI
9305                 [(match_operand:V2DI 1 "s_register_operand" "w")
9306                  (match_operand:V2DI 2 "s_register_operand" "w")
9307                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9308          VSTRDSOQ))
9309   ]
9310   "TARGET_HAVE_MVE"
9311 {
9312    rtx ops[3];
9313    ops[0] = operands[0];
9314    ops[1] = operands[1];
9315    ops[2] = operands[2];
9316    output_asm_insn ("vpst\;\tvstrdt.64\t%q2, [%m0, %q1]",ops);
9317    return "";
9318 }
9319   [(set_attr "length" "8")])
9320
9321 ;;
9322 ;; [vstrdq_scatter_offset_s vstrdq_scatter_offset_u]
9323 ;;
9324 (define_insn "mve_vstrdq_scatter_offset_<supf>v2di"
9325   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9326         (unspec:V2DI
9327                 [(match_operand:V2DI 1 "s_register_operand" "w")
9328                  (match_operand:V2DI 2 "s_register_operand" "w")]
9329          VSTRDSOQ))
9330   ]
9331   "TARGET_HAVE_MVE"
9332 {
9333    rtx ops[3];
9334    ops[0] = operands[0];
9335    ops[1] = operands[1];
9336    ops[2] = operands[2];
9337    output_asm_insn ("vstrd.64\t%q2, [%m0, %q1]",ops);
9338    return "";
9339 }
9340   [(set_attr "length" "4")])
9341
9342 ;;
9343 ;; [vstrdq_scatter_shifted_offset_p_s vstrdq_scatter_shifted_offset_p_u]
9344 ;;
9345 (define_insn "mve_vstrdq_scatter_shifted_offset_p_<supf>v2di"
9346   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9347         (unspec:V2DI
9348                 [(match_operand:V2DI 1 "s_register_operand" "w")
9349                  (match_operand:V2DI 2 "s_register_operand" "w")
9350                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9351          VSTRDSSOQ))
9352   ]
9353   "TARGET_HAVE_MVE"
9354 {
9355    rtx ops[3];
9356    ops[0] = operands[0];
9357    ops[1] = operands[1];
9358    ops[2] = operands[2];
9359    output_asm_insn ("vpst\;\tvstrdt.64\t%q2, [%m0, %q1, UXTW #3]",ops);
9360    return "";
9361 }
9362   [(set_attr "length" "8")])
9363
9364 ;;
9365 ;; [vstrdq_scatter_shifted_offset_s vstrdq_scatter_shifted_offset_u]
9366 ;;
9367 (define_insn "mve_vstrdq_scatter_shifted_offset_<supf>v2di"
9368   [(set (match_operand:V2DI 0 "memory_operand" "=Us")
9369         (unspec:V2DI
9370                 [(match_operand:V2DI 1 "s_register_operand" "w")
9371                  (match_operand:V2DI 2 "s_register_operand" "w")]
9372          VSTRDSSOQ))
9373   ]
9374   "TARGET_HAVE_MVE"
9375 {
9376    rtx ops[3];
9377    ops[0] = operands[0];
9378    ops[1] = operands[1];
9379    ops[2] = operands[2];
9380    output_asm_insn ("vstrd.64\t%q2, [%m0, %q1, UXTW #3]",ops);
9381    return "";
9382 }
9383   [(set_attr "length" "4")])
9384
9385 ;;
9386 ;; [vstrhq_scatter_offset_f]
9387 ;;
9388 (define_insn "mve_vstrhq_scatter_offset_fv8hf"
9389   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9390         (unspec:V8HI
9391                 [(match_operand:V8HI 1 "s_register_operand" "w")
9392                  (match_operand:V8HF 2 "s_register_operand" "w")]
9393          VSTRHQSO_F))
9394   ]
9395   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9396 {
9397    rtx ops[3];
9398    ops[0] = operands[0];
9399    ops[1] = operands[1];
9400    ops[2] = operands[2];
9401    output_asm_insn ("vstrh.16\t%q2, [%m0, %q1]",ops);
9402    return "";
9403 }
9404   [(set_attr "length" "4")])
9405
9406 ;;
9407 ;; [vstrhq_scatter_offset_p_f]
9408 ;;
9409 (define_insn "mve_vstrhq_scatter_offset_p_fv8hf"
9410   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9411         (unspec:V8HI
9412                 [(match_operand:V8HI 1 "s_register_operand" "w")
9413                  (match_operand:V8HF 2 "s_register_operand" "w")
9414                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9415          VSTRHQSO_F))
9416   ]
9417   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9418 {
9419    rtx ops[3];
9420    ops[0] = operands[0];
9421    ops[1] = operands[1];
9422    ops[2] = operands[2];
9423    output_asm_insn ("vpst\n\tvstrht.16\t%q2, [%m0, %q1]",ops);
9424    return "";
9425 }
9426   [(set_attr "length" "8")])
9427
9428 ;;
9429 ;; [vstrhq_scatter_shifted_offset_f]
9430 ;;
9431 (define_insn "mve_vstrhq_scatter_shifted_offset_fv8hf"
9432   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9433         (unspec:V8HI
9434                 [(match_operand:V8HI 1 "s_register_operand" "w")
9435                  (match_operand:V8HF 2 "s_register_operand" "w")]
9436          VSTRHQSSO_F))
9437   ]
9438   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9439 {
9440    rtx ops[3];
9441    ops[0] = operands[0];
9442    ops[1] = operands[1];
9443    ops[2] = operands[2];
9444    output_asm_insn ("vstrh.16\t%q2, [%m0, %q1, uxtw #1]",ops);
9445    return "";
9446 }
9447   [(set_attr "length" "4")])
9448
9449 ;;
9450 ;; [vstrhq_scatter_shifted_offset_p_f]
9451 ;;
9452 (define_insn "mve_vstrhq_scatter_shifted_offset_p_fv8hf"
9453   [(set (match_operand:V8HI 0 "memory_operand" "=Us")
9454         (unspec:V8HI
9455                 [(match_operand:V8HI 1 "s_register_operand" "w")
9456                  (match_operand:V8HF 2 "s_register_operand" "w")
9457                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9458          VSTRHQSSO_F))
9459   ]
9460   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9461 {
9462    rtx ops[3];
9463    ops[0] = operands[0];
9464    ops[1] = operands[1];
9465    ops[2] = operands[2];
9466    output_asm_insn ("vpst\n\tvstrht.16\t%q2, [%m0, %q1, uxtw #1]",ops);
9467    return "";
9468 }
9469   [(set_attr "length" "8")])
9470
9471 ;;
9472 ;; [vstrwq_scatter_base_f]
9473 ;;
9474 (define_insn "mve_vstrwq_scatter_base_fv4sf"
9475   [(set (mem:BLK (scratch))
9476         (unspec:BLK
9477                 [(match_operand:V4SI 0 "s_register_operand" "w")
9478                  (match_operand:SI 1 "immediate_operand" "i")
9479                  (match_operand:V4SF 2 "s_register_operand" "w")]
9480          VSTRWQSB_F))
9481   ]
9482   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9483 {
9484    rtx ops[3];
9485    ops[0] = operands[0];
9486    ops[1] = operands[1];
9487    ops[2] = operands[2];
9488    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]",ops);
9489    return "";
9490 }
9491   [(set_attr "length" "4")])
9492
9493 ;;
9494 ;; [vstrwq_scatter_base_p_f]
9495 ;;
9496 (define_insn "mve_vstrwq_scatter_base_p_fv4sf"
9497   [(set (mem:BLK (scratch))
9498         (unspec:BLK
9499                 [(match_operand:V4SI 0 "s_register_operand" "w")
9500                  (match_operand:SI 1 "immediate_operand" "i")
9501                  (match_operand:V4SF 2 "s_register_operand" "w")
9502                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9503          VSTRWQSB_F))
9504   ]
9505   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9506 {
9507    rtx ops[3];
9508    ops[0] = operands[0];
9509    ops[1] = operands[1];
9510    ops[2] = operands[2];
9511    output_asm_insn ("vpst\n\tvstrwt.u32\t%q2, [%q0, %1]",ops);
9512    return "";
9513 }
9514   [(set_attr "length" "8")])
9515
9516 ;;
9517 ;; [vstrwq_scatter_offset_f]
9518 ;;
9519 (define_insn "mve_vstrwq_scatter_offset_fv4sf"
9520   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9521         (unspec:V4SI
9522                 [(match_operand:V4SI 1 "s_register_operand" "w")
9523                  (match_operand:V4SF 2 "s_register_operand" "w")]
9524          VSTRWQSO_F))
9525   ]
9526   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9527 {
9528    rtx ops[3];
9529    ops[0] = operands[0];
9530    ops[1] = operands[1];
9531    ops[2] = operands[2];
9532    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1]",ops);
9533    return "";
9534 }
9535   [(set_attr "length" "4")])
9536
9537 ;;
9538 ;; [vstrwq_scatter_offset_p_f]
9539 ;;
9540 (define_insn "mve_vstrwq_scatter_offset_p_fv4sf"
9541   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9542         (unspec:V4SI
9543                 [(match_operand:V4SI 1 "s_register_operand" "w")
9544                  (match_operand:V4SF 2 "s_register_operand" "w")
9545                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9546          VSTRWQSO_F))
9547   ]
9548   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9549 {
9550    rtx ops[3];
9551    ops[0] = operands[0];
9552    ops[1] = operands[1];
9553    ops[2] = operands[2];
9554    output_asm_insn ("vpst\n\tvstrwt.32\t%q2, [%m0, %q1]",ops);
9555    return "";
9556 }
9557   [(set_attr "length" "8")])
9558
9559 ;;
9560 ;; [vstrwq_scatter_offset_p_s vstrwq_scatter_offset_p_u]
9561 ;;
9562 (define_insn "mve_vstrwq_scatter_offset_p_<supf>v4si"
9563   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9564         (unspec:V4SI
9565                 [(match_operand:V4SI 1 "s_register_operand" "w")
9566                  (match_operand:V4SI 2 "s_register_operand" "w")
9567                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9568          VSTRWSOQ))
9569   ]
9570   "TARGET_HAVE_MVE"
9571 {
9572    rtx ops[3];
9573    ops[0] = operands[0];
9574    ops[1] = operands[1];
9575    ops[2] = operands[2];
9576    output_asm_insn ("vpst\n\tvstrwt.32\t%q2, [%m0, %q1]",ops);
9577    return "";
9578 }
9579   [(set_attr "length" "8")])
9580
9581 ;;
9582 ;; [vstrwq_scatter_offset_s vstrwq_scatter_offset_u]
9583 ;;
9584 (define_insn "mve_vstrwq_scatter_offset_<supf>v4si"
9585   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9586         (unspec:V4SI
9587                 [(match_operand:V4SI 1 "s_register_operand" "w")
9588                  (match_operand:V4SI 2 "s_register_operand" "w")]
9589          VSTRWSOQ))
9590   ]
9591   "TARGET_HAVE_MVE"
9592 {
9593    rtx ops[3];
9594    ops[0] = operands[0];
9595    ops[1] = operands[1];
9596    ops[2] = operands[2];
9597    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1]",ops);
9598    return "";
9599 }
9600   [(set_attr "length" "4")])
9601
9602 ;;
9603 ;; [vstrwq_scatter_shifted_offset_f]
9604 ;;
9605 (define_insn "mve_vstrwq_scatter_shifted_offset_fv4sf"
9606   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9607         (unspec:V4SI
9608                 [(match_operand:V4SI 1 "s_register_operand" "w")
9609                  (match_operand:V4SF 2 "s_register_operand" "w")]
9610          VSTRWQSSO_F))
9611   ]
9612   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9613 {
9614    rtx ops[3];
9615    ops[0] = operands[0];
9616    ops[1] = operands[1];
9617    ops[2] = operands[2];
9618    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9619    return "";
9620 }
9621   [(set_attr "length" "4")])
9622
9623 ;;
9624 ;; [vstrwq_scatter_shifted_offset_p_f]
9625 ;;
9626 (define_insn "mve_vstrwq_scatter_shifted_offset_p_fv4sf"
9627   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9628         (unspec:V4SI
9629                 [(match_operand:V4SI 1 "s_register_operand" "w")
9630                  (match_operand:V4SF 2 "s_register_operand" "w")
9631                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9632          VSTRWQSSO_F))
9633   ]
9634   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9635 {
9636    rtx ops[3];
9637    ops[0] = operands[0];
9638    ops[1] = operands[1];
9639    ops[2] = operands[2];
9640    output_asm_insn ("vpst\;\tvstrwt.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9641    return "";
9642 }
9643   [(set_attr "length" "8")])
9644
9645 ;;
9646 ;; [vstrwq_scatter_shifted_offset_p_s vstrwq_scatter_shifted_offset_p_u]
9647 ;;
9648 (define_insn "mve_vstrwq_scatter_shifted_offset_p_<supf>v4si"
9649   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9650         (unspec:V4SI
9651                 [(match_operand:V4SI 1 "s_register_operand" "w")
9652                  (match_operand:V4SI 2 "s_register_operand" "w")
9653                  (match_operand:HI 3 "vpr_register_operand" "Up")]
9654          VSTRWSSOQ))
9655   ]
9656   "TARGET_HAVE_MVE"
9657 {
9658    rtx ops[3];
9659    ops[0] = operands[0];
9660    ops[1] = operands[1];
9661    ops[2] = operands[2];
9662    output_asm_insn ("vpst\;\tvstrwt.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9663    return "";
9664 }
9665   [(set_attr "length" "8")])
9666
9667 ;;
9668 ;; [vstrwq_scatter_shifted_offset_s vstrwq_scatter_shifted_offset_u]
9669 ;;
9670 (define_insn "mve_vstrwq_scatter_shifted_offset_<supf>v4si"
9671   [(set (match_operand:V4SI 0 "memory_operand" "=Us")
9672         (unspec:V4SI
9673                 [(match_operand:V4SI 1 "s_register_operand" "w")
9674                  (match_operand:V4SI 2 "s_register_operand" "w")]
9675          VSTRWSSOQ))
9676   ]
9677   "TARGET_HAVE_MVE"
9678 {
9679    rtx ops[3];
9680    ops[0] = operands[0];
9681    ops[1] = operands[1];
9682    ops[2] = operands[2];
9683    output_asm_insn ("vstrw.32\t%q2, [%m0, %q1, uxtw #2]",ops);
9684    return "";
9685 }
9686   [(set_attr "length" "4")])
9687
9688 ;;
9689 ;; [vaddq_s, vaddq_u])
9690 ;;
9691 (define_insn "mve_vaddq<mode>"
9692   [
9693    (set (match_operand:MVE_2 0 "s_register_operand" "=w")
9694         (plus:MVE_2 (match_operand:MVE_2 1 "s_register_operand" "w")
9695                     (match_operand:MVE_2 2 "s_register_operand" "w")))
9696   ]
9697   "TARGET_HAVE_MVE"
9698   "vadd.i%#<V_sz_elem>  %q0, %q1, %q2"
9699   [(set_attr "type" "mve_move")
9700 ])
9701
9702 ;;
9703 ;; [vaddq_f])
9704 ;;
9705 (define_insn "mve_vaddq_f<mode>"
9706   [
9707    (set (match_operand:MVE_0 0 "s_register_operand" "=w")
9708         (plus:MVE_0 (match_operand:MVE_0 1 "s_register_operand" "w")
9709                     (match_operand:MVE_0 2 "s_register_operand" "w")))
9710   ]
9711   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
9712   "vadd.f%#<V_sz_elem> %q0, %q1, %q2"
9713   [(set_attr "type" "mve_move")
9714 ])
9715
9716 ;;
9717 ;; [vidupq_n_u])
9718 ;;
9719 (define_expand "mve_vidupq_n_u<mode>"
9720  [(match_operand:MVE_2 0 "s_register_operand")
9721   (match_operand:SI 1 "s_register_operand")
9722   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9723  "TARGET_HAVE_MVE"
9724 {
9725   rtx temp = gen_reg_rtx (SImode);
9726   emit_move_insn (temp, operands[1]);
9727   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9728   emit_insn (gen_mve_vidupq_u<mode>_insn (operands[0], temp, operands[1],
9729                                           operands[2], inc));
9730   DONE;
9731 })
9732
9733 ;;
9734 ;; [vidupq_u_insn])
9735 ;;
9736 (define_insn "mve_vidupq_u<mode>_insn"
9737  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9738        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9739                       (match_operand:SI 3 "mve_imm_selective_upto_8" "Rg")]
9740          VIDUPQ))
9741   (set (match_operand:SI 1 "s_register_operand" "=e")
9742        (plus:SI (match_dup 2)
9743                 (match_operand:SI 4 "immediate_operand" "i")))]
9744  "TARGET_HAVE_MVE"
9745  "vidup.u%#<V_sz_elem>\t%q0, %1, %3")
9746
9747 ;;
9748 ;; [vidupq_m_n_u])
9749 ;;
9750 (define_expand "mve_vidupq_m_n_u<mode>"
9751   [(match_operand:MVE_2 0 "s_register_operand")
9752    (match_operand:MVE_2 1 "s_register_operand")
9753    (match_operand:SI 2 "s_register_operand")
9754    (match_operand:SI 3 "mve_imm_selective_upto_8")
9755    (match_operand:HI 4 "vpr_register_operand")]
9756   "TARGET_HAVE_MVE"
9757 {
9758   rtx temp = gen_reg_rtx (SImode);
9759   emit_move_insn (temp, operands[2]);
9760   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9761   emit_insn (gen_mve_vidupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9762                                              operands[2], operands[3],
9763                                              operands[4], inc));
9764   DONE;
9765 })
9766
9767 ;;
9768 ;; [vidupq_m_wb_u_insn])
9769 ;;
9770 (define_insn "mve_vidupq_m_wb_u<mode>_insn"
9771  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9772        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9773                       (match_operand:SI 3 "s_register_operand" "2")
9774                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9775                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9776         VIDUPQ_M))
9777   (set (match_operand:SI 2 "s_register_operand" "=e")
9778        (plus:SI (match_dup 3)
9779                 (match_operand:SI 6 "immediate_operand" "i")))]
9780  "TARGET_HAVE_MVE"
9781  "vpst\;\tvidupt.u%#<V_sz_elem>\t%q0, %2, %4"
9782  [(set_attr "length""8")])
9783
9784 ;;
9785 ;; [vddupq_n_u])
9786 ;;
9787 (define_expand "mve_vddupq_n_u<mode>"
9788  [(match_operand:MVE_2 0 "s_register_operand")
9789   (match_operand:SI 1 "s_register_operand")
9790   (match_operand:SI 2 "mve_imm_selective_upto_8")]
9791  "TARGET_HAVE_MVE"
9792 {
9793   rtx temp = gen_reg_rtx (SImode);
9794   emit_move_insn (temp, operands[1]);
9795   rtx inc = gen_int_mode (INTVAL(operands[2]) * <MVE_LANES>, SImode);
9796   emit_insn (gen_mve_vddupq_u<mode>_insn (operands[0], temp, operands[1],
9797                                           operands[2], inc));
9798   DONE;
9799 })
9800
9801 ;;
9802 ;; [vddupq_u_insn])
9803 ;;
9804 (define_insn "mve_vddupq_u<mode>_insn"
9805  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9806        (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9807                       (match_operand:SI 3 "immediate_operand" "i")]
9808         VDDUPQ))
9809   (set (match_operand:SI 1 "s_register_operand" "=e")
9810        (minus:SI (match_dup 2)
9811                  (match_operand:SI 4 "immediate_operand" "i")))]
9812  "TARGET_HAVE_MVE"
9813  "vddup.u%#<V_sz_elem>  %q0, %1, %3")
9814
9815 ;;
9816 ;; [vddupq_m_n_u])
9817 ;;
9818 (define_expand "mve_vddupq_m_n_u<mode>"
9819   [(match_operand:MVE_2 0 "s_register_operand")
9820    (match_operand:MVE_2 1 "s_register_operand")
9821    (match_operand:SI 2 "s_register_operand")
9822    (match_operand:SI 3 "mve_imm_selective_upto_8")
9823    (match_operand:HI 4 "vpr_register_operand")]
9824   "TARGET_HAVE_MVE"
9825 {
9826   rtx temp = gen_reg_rtx (SImode);
9827   emit_move_insn (temp, operands[2]);
9828   rtx inc = gen_int_mode (INTVAL(operands[3]) * <MVE_LANES>, SImode);
9829   emit_insn (gen_mve_vddupq_m_wb_u<mode>_insn(operands[0], operands[1], temp,
9830                                              operands[2], operands[3],
9831                                              operands[4], inc));
9832   DONE;
9833 })
9834
9835 ;;
9836 ;; [vddupq_m_wb_u_insn])
9837 ;;
9838 (define_insn "mve_vddupq_m_wb_u<mode>_insn"
9839  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9840        (unspec:MVE_2 [(match_operand:MVE_2 1 "s_register_operand" "0")
9841                       (match_operand:SI 3 "s_register_operand" "2")
9842                       (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")
9843                       (match_operand:HI 5 "vpr_register_operand" "Up")]
9844         VDDUPQ_M))
9845   (set (match_operand:SI 2 "s_register_operand" "=e")
9846        (minus:SI (match_dup 3)
9847                  (match_operand:SI 6 "immediate_operand" "i")))]
9848  "TARGET_HAVE_MVE"
9849  "vpst\;\tvddupt.u%#<V_sz_elem>\t%q0, %2, %4"
9850  [(set_attr "length""8")])
9851
9852 ;;
9853 ;; [vdwdupq_n_u])
9854 ;;
9855 (define_expand "mve_vdwdupq_n_u<mode>"
9856  [(match_operand:MVE_2 0 "s_register_operand")
9857   (match_operand:SI 1 "s_register_operand")
9858   (match_operand:DI 2 "s_register_operand")
9859   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9860  "TARGET_HAVE_MVE"
9861 {
9862   rtx ignore_wb = gen_reg_rtx (SImode);
9863   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9864                                               operands[1], operands[2],
9865                                               operands[3]));
9866   DONE;
9867 })
9868
9869 ;;
9870 ;; [vdwdupq_wb_u])
9871 ;;
9872 (define_expand "mve_vdwdupq_wb_u<mode>"
9873  [(match_operand:SI 0 "s_register_operand")
9874   (match_operand:SI 1 "s_register_operand")
9875   (match_operand:DI 2 "s_register_operand")
9876   (match_operand:SI 3 "mve_imm_selective_upto_8")
9877   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9878  "TARGET_HAVE_MVE"
9879 {
9880   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9881   emit_insn (gen_mve_vdwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9882                                               operands[1], operands[2],
9883                                               operands[3]));
9884   DONE;
9885 })
9886
9887 ;;
9888 ;; [vdwdupq_wb_u_insn])
9889 ;;
9890 (define_insn "mve_vdwdupq_wb_u<mode>_insn"
9891   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9892         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
9893                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
9894                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
9895          VDWDUPQ))
9896    (set (match_operand:SI 1 "s_register_operand" "=e")
9897         (unspec:SI [(match_dup 2)
9898                     (subreg:SI (match_dup 3) 4)
9899                     (match_dup 4)]
9900          VDWDUPQ))]
9901   "TARGET_HAVE_MVE"
9902   "vdwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
9903 )
9904
9905 ;;
9906 ;; [vdwdupq_m_n_u])
9907 ;;
9908 (define_expand "mve_vdwdupq_m_n_u<mode>"
9909  [(match_operand:MVE_2 0 "s_register_operand")
9910   (match_operand:MVE_2 1 "s_register_operand")
9911   (match_operand:SI 2 "s_register_operand")
9912   (match_operand:DI 3 "s_register_operand")
9913   (match_operand:SI 4 "mve_imm_selective_upto_8")
9914   (match_operand:HI 5 "vpr_register_operand")]
9915  "TARGET_HAVE_MVE"
9916 {
9917   rtx ignore_wb = gen_reg_rtx (SImode);
9918   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
9919                                                 operands[1], operands[2],
9920                                                 operands[3], operands[4],
9921                                                 operands[5]));
9922   DONE;
9923 })
9924
9925 ;;
9926 ;; [vdwdupq_m_wb_u])
9927 ;;
9928 (define_expand "mve_vdwdupq_m_wb_u<mode>"
9929  [(match_operand:SI 0 "s_register_operand")
9930   (match_operand:MVE_2 1 "s_register_operand")
9931   (match_operand:SI 2 "s_register_operand")
9932   (match_operand:DI 3 "s_register_operand")
9933   (match_operand:SI 4 "mve_imm_selective_upto_8")
9934   (match_operand:HI 5 "vpr_register_operand")]
9935  "TARGET_HAVE_MVE"
9936 {
9937   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9938   emit_insn (gen_mve_vdwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
9939                                                 operands[1], operands[2],
9940                                                 operands[3], operands[4],
9941                                                 operands[5]));
9942   DONE;
9943 })
9944
9945 ;;
9946 ;; [vdwdupq_m_wb_u_insn])
9947 ;;
9948 (define_insn "mve_vdwdupq_m_wb_u<mode>_insn"
9949   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
9950         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
9951                        (match_operand:SI 3 "s_register_operand" "1")
9952                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
9953                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
9954                        (match_operand:HI 6 "vpr_register_operand" "Up")]
9955          VDWDUPQ_M))
9956    (set (match_operand:SI 1 "s_register_operand" "=e")
9957         (unspec:SI [(match_dup 2)
9958                     (match_dup 3)
9959                     (subreg:SI (match_dup 4) 4)
9960                     (match_dup 5)
9961                     (match_dup 6)]
9962          VDWDUPQ_M))
9963   ]
9964   "TARGET_HAVE_MVE"
9965   "vpst\;\tvdwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
9966   [(set_attr "type" "mve_move")
9967    (set_attr "length""8")])
9968
9969 ;;
9970 ;; [viwdupq_n_u])
9971 ;;
9972 (define_expand "mve_viwdupq_n_u<mode>"
9973  [(match_operand:MVE_2 0 "s_register_operand")
9974   (match_operand:SI 1 "s_register_operand")
9975   (match_operand:DI 2 "s_register_operand")
9976   (match_operand:SI 3 "mve_imm_selective_upto_8")]
9977  "TARGET_HAVE_MVE"
9978 {
9979   rtx ignore_wb = gen_reg_rtx (SImode);
9980   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (operands[0], ignore_wb,
9981                                               operands[1], operands[2],
9982                                               operands[3]));
9983   DONE;
9984 })
9985
9986 ;;
9987 ;; [viwdupq_wb_u])
9988 ;;
9989 (define_expand "mve_viwdupq_wb_u<mode>"
9990  [(match_operand:SI 0 "s_register_operand")
9991   (match_operand:SI 1 "s_register_operand")
9992   (match_operand:DI 2 "s_register_operand")
9993   (match_operand:SI 3 "mve_imm_selective_upto_8")
9994   (unspec:MVE_2 [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
9995  "TARGET_HAVE_MVE"
9996 {
9997   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
9998   emit_insn (gen_mve_viwdupq_wb_u<mode>_insn (ignore_vec, operands[0],
9999                                               operands[1], operands[2],
10000                                               operands[3]));
10001   DONE;
10002 })
10003
10004 ;;
10005 ;; [viwdupq_wb_u_insn])
10006 ;;
10007 (define_insn "mve_viwdupq_wb_u<mode>_insn"
10008   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10009         (unspec:MVE_2 [(match_operand:SI 2 "s_register_operand" "1")
10010                        (subreg:SI (match_operand:DI 3 "s_register_operand" "r") 4)
10011                        (match_operand:SI 4 "mve_imm_selective_upto_8" "Rg")]
10012          VIWDUPQ))
10013    (set (match_operand:SI 1 "s_register_operand" "=e")
10014         (unspec:SI [(match_dup 2)
10015                     (subreg:SI (match_dup 3) 4)
10016                     (match_dup 4)]
10017          VIWDUPQ))]
10018   "TARGET_HAVE_MVE"
10019   "viwdup.u%#<V_sz_elem>\t%q0, %2, %R3, %4"
10020 )
10021
10022 ;;
10023 ;; [viwdupq_m_n_u])
10024 ;;
10025 (define_expand "mve_viwdupq_m_n_u<mode>"
10026  [(match_operand:MVE_2 0 "s_register_operand")
10027   (match_operand:MVE_2 1 "s_register_operand")
10028   (match_operand:SI 2 "s_register_operand")
10029   (match_operand:DI 3 "s_register_operand")
10030   (match_operand:SI 4 "mve_imm_selective_upto_8")
10031   (match_operand:HI 5 "vpr_register_operand")]
10032  "TARGET_HAVE_MVE"
10033 {
10034   rtx ignore_wb = gen_reg_rtx (SImode);
10035   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (operands[0], ignore_wb,
10036                                                 operands[1], operands[2],
10037                                                 operands[3], operands[4],
10038                                                 operands[5]));
10039   DONE;
10040 })
10041
10042 ;;
10043 ;; [viwdupq_m_wb_u])
10044 ;;
10045 (define_expand "mve_viwdupq_m_wb_u<mode>"
10046  [(match_operand:SI 0 "s_register_operand")
10047   (match_operand:MVE_2 1 "s_register_operand")
10048   (match_operand:SI 2 "s_register_operand")
10049   (match_operand:DI 3 "s_register_operand")
10050   (match_operand:SI 4 "mve_imm_selective_upto_8")
10051   (match_operand:HI 5 "vpr_register_operand")]
10052  "TARGET_HAVE_MVE"
10053 {
10054   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
10055   emit_insn (gen_mve_viwdupq_m_wb_u<mode>_insn (ignore_vec, operands[0],
10056                                                 operands[1], operands[2],
10057                                                 operands[3], operands[4],
10058                                                 operands[5]));
10059   DONE;
10060 })
10061
10062 ;;
10063 ;; [viwdupq_m_wb_u_insn])
10064 ;;
10065 (define_insn "mve_viwdupq_m_wb_u<mode>_insn"
10066   [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
10067         (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
10068                        (match_operand:SI 3 "s_register_operand" "1")
10069                        (subreg:SI (match_operand:DI 4 "s_register_operand" "r") 4)
10070                        (match_operand:SI 5 "mve_imm_selective_upto_8" "Rg")
10071                        (match_operand:HI 6 "vpr_register_operand" "Up")]
10072          VIWDUPQ_M))
10073    (set (match_operand:SI 1 "s_register_operand" "=e")
10074         (unspec:SI [(match_dup 2)
10075                     (match_dup 3)
10076                     (subreg:SI (match_dup 4) 4)
10077                     (match_dup 5)
10078                     (match_dup 6)]
10079          VIWDUPQ_M))
10080   ]
10081   "TARGET_HAVE_MVE"
10082   "vpst\;\tviwdupt.u%#<V_sz_elem>\t%q2, %3, %R4, %5"
10083   [(set_attr "type" "mve_move")
10084    (set_attr "length""8")])
10085
10086 (define_expand "mve_vstrwq_scatter_base_wb_<supf>v4si"
10087   [(match_operand:V4SI 0 "s_register_operand" "=w")
10088    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10089    (match_operand:V4SI 2 "s_register_operand" "w")
10090    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10091   "TARGET_HAVE_MVE"
10092 {
10093   rtx ignore_wb = gen_reg_rtx (V4SImode);
10094   emit_insn (
10095   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (ignore_wb, operands[0],
10096                                                   operands[1], operands[2]));
10097   DONE;
10098 })
10099
10100 (define_expand "mve_vstrwq_scatter_base_wb_add_<supf>v4si"
10101   [(match_operand:V4SI 0 "s_register_operand" "=w")
10102    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10103    (match_operand:V4SI 2 "s_register_operand" "0")
10104    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10105   "TARGET_HAVE_MVE"
10106 {
10107   rtx ignore_vec = gen_reg_rtx (V4SImode);
10108   emit_insn (
10109   gen_mve_vstrwq_scatter_base_wb_<supf>v4si_insn (operands[0], operands[2],
10110                                                   operands[1], ignore_vec));
10111   DONE;
10112 })
10113
10114 ;;
10115 ;; [vstrwq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10116 ;;
10117 (define_insn "mve_vstrwq_scatter_base_wb_<supf>v4si_insn"
10118   [(set (mem:BLK (scratch))
10119         (unspec:BLK
10120                 [(match_operand:V4SI 1 "s_register_operand" "0")
10121                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10122                  (match_operand:V4SI 3 "s_register_operand" "w")]
10123          VSTRWSBWBQ))
10124    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10125         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10126          VSTRWSBWBQ))
10127   ]
10128   "TARGET_HAVE_MVE"
10129 {
10130    rtx ops[3];
10131    ops[0] = operands[1];
10132    ops[1] = operands[2];
10133    ops[2] = operands[3];
10134    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10135    return "";
10136 }
10137   [(set_attr "length" "4")])
10138
10139 (define_expand "mve_vstrwq_scatter_base_wb_p_<supf>v4si"
10140   [(match_operand:V4SI 0 "s_register_operand" "=w")
10141    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10142    (match_operand:V4SI 2 "s_register_operand" "w")
10143    (match_operand:HI 3 "vpr_register_operand")
10144    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10145   "TARGET_HAVE_MVE"
10146 {
10147   rtx ignore_wb = gen_reg_rtx (V4SImode);
10148   emit_insn (
10149   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (ignore_wb, operands[0],
10150                                                     operands[1], operands[2],
10151                                                     operands[3]));
10152   DONE;
10153 })
10154
10155 (define_expand "mve_vstrwq_scatter_base_wb_p_add_<supf>v4si"
10156   [(match_operand:V4SI 0 "s_register_operand" "=w")
10157    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10158    (match_operand:V4SI 2 "s_register_operand" "0")
10159    (match_operand:HI 3 "vpr_register_operand")
10160    (unspec:V4SI [(const_int 0)] VSTRWSBWBQ)]
10161   "TARGET_HAVE_MVE"
10162 {
10163   rtx ignore_vec = gen_reg_rtx (V4SImode);
10164   emit_insn (
10165   gen_mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn (operands[0], operands[2],
10166                                                     operands[1], ignore_vec,
10167                                                     operands[3]));
10168   DONE;
10169 })
10170
10171 ;;
10172 ;; [vstrwq_scatter_base_wb_p_s vstrwq_scatter_base_wb_p_u]
10173 ;;
10174 (define_insn "mve_vstrwq_scatter_base_wb_p_<supf>v4si_insn"
10175  [(set (mem:BLK (scratch))
10176        (unspec:BLK
10177                 [(match_operand:V4SI 1 "s_register_operand" "0")
10178                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10179                  (match_operand:V4SI 3 "s_register_operand" "w")
10180                  (match_operand:HI 4 "vpr_register_operand")]
10181         VSTRWSBWBQ))
10182    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10183         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10184          VSTRWSBWBQ))
10185   ]
10186   "TARGET_HAVE_MVE"
10187 {
10188    rtx ops[3];
10189    ops[0] = operands[1];
10190    ops[1] = operands[2];
10191    ops[2] = operands[3];
10192    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10193    return "";
10194 }
10195   [(set_attr "length" "8")])
10196
10197 (define_expand "mve_vstrwq_scatter_base_wb_fv4sf"
10198   [(match_operand:V4SI 0 "s_register_operand" "=w")
10199    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10200    (match_operand:V4SF 2 "s_register_operand" "w")
10201    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10202   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10203 {
10204   rtx ignore_wb = gen_reg_rtx (V4SImode);
10205   emit_insn (
10206   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (ignore_wb,operands[0],
10207                                              operands[1], operands[2]));
10208   DONE;
10209 })
10210
10211 (define_expand "mve_vstrwq_scatter_base_wb_add_fv4sf"
10212   [(match_operand:V4SI 0 "s_register_operand" "=w")
10213    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10214    (match_operand:V4SI 2 "s_register_operand" "0")
10215    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10216   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10217 {
10218   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10219   emit_insn (
10220   gen_mve_vstrwq_scatter_base_wb_fv4sf_insn (operands[0], operands[2],
10221                                              operands[1], ignore_vec));
10222   DONE;
10223 })
10224
10225 ;;
10226 ;; [vstrwq_scatter_base_wb_f]
10227 ;;
10228 (define_insn "mve_vstrwq_scatter_base_wb_fv4sf_insn"
10229  [(set (mem:BLK (scratch))
10230        (unspec:BLK
10231                 [(match_operand:V4SI 1 "s_register_operand" "0")
10232                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10233                  (match_operand:V4SF 3 "s_register_operand" "w")]
10234          VSTRWQSBWB_F))
10235    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10236         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10237          VSTRWQSBWB_F))
10238   ]
10239   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10240 {
10241    rtx ops[3];
10242    ops[0] = operands[1];
10243    ops[1] = operands[2];
10244    ops[2] = operands[3];
10245    output_asm_insn ("vstrw.u32\t%q2, [%q0, %1]!",ops);
10246    return "";
10247 }
10248   [(set_attr "length" "4")])
10249
10250 (define_expand "mve_vstrwq_scatter_base_wb_p_fv4sf"
10251   [(match_operand:V4SI 0 "s_register_operand" "=w")
10252    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10253    (match_operand:V4SF 2 "s_register_operand" "w")
10254    (match_operand:HI 3 "vpr_register_operand")
10255    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10256   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10257 {
10258   rtx ignore_wb = gen_reg_rtx (V4SImode);
10259   emit_insn (
10260   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (ignore_wb, operands[0],
10261                                                operands[1], operands[2],
10262                                                operands[3]));
10263   DONE;
10264 })
10265
10266 (define_expand "mve_vstrwq_scatter_base_wb_p_add_fv4sf"
10267   [(match_operand:V4SI 0 "s_register_operand" "=w")
10268    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10269    (match_operand:V4SI 2 "s_register_operand" "0")
10270    (match_operand:HI 3 "vpr_register_operand")
10271    (unspec:V4SI [(const_int 0)] VSTRWQSBWB_F)]
10272   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10273 {
10274   rtx ignore_vec = gen_reg_rtx (V4SFmode);
10275   emit_insn (
10276   gen_mve_vstrwq_scatter_base_wb_p_fv4sf_insn (operands[0], operands[2],
10277                                                operands[1], ignore_vec,
10278                                                operands[3]));
10279   DONE;
10280 })
10281
10282 ;;
10283 ;; [vstrwq_scatter_base_wb_p_f]
10284 ;;
10285 (define_insn "mve_vstrwq_scatter_base_wb_p_fv4sf_insn"
10286  [(set (mem:BLK (scratch))
10287        (unspec:BLK
10288                 [(match_operand:V4SI 1 "s_register_operand" "0")
10289                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10290                  (match_operand:V4SF 3 "s_register_operand" "w")
10291                  (match_operand:HI 4 "vpr_register_operand")]
10292         VSTRWQSBWB_F))
10293    (set (match_operand:V4SI 0 "s_register_operand" "=w")
10294         (unspec:V4SI [(match_dup 1) (match_dup 2)]
10295          VSTRWQSBWB_F))
10296   ]
10297   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10298 {
10299    rtx ops[3];
10300    ops[0] = operands[1];
10301    ops[1] = operands[2];
10302    ops[2] = operands[3];
10303    output_asm_insn ("vpst\;\tvstrwt.u32\t%q2, [%q0, %1]!",ops);
10304    return "";
10305 }
10306   [(set_attr "length" "8")])
10307
10308 (define_expand "mve_vstrdq_scatter_base_wb_<supf>v2di"
10309   [(match_operand:V2DI 0 "s_register_operand" "=w")
10310    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10311    (match_operand:V2DI 2 "s_register_operand" "w")
10312    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10313   "TARGET_HAVE_MVE"
10314 {
10315   rtx ignore_wb = gen_reg_rtx (V2DImode);
10316   emit_insn (
10317   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (ignore_wb, operands[0],
10318                                                   operands[1], operands[2]));
10319   DONE;
10320 })
10321
10322 (define_expand "mve_vstrdq_scatter_base_wb_add_<supf>v2di"
10323   [(match_operand:V2DI 0 "s_register_operand" "=w")
10324    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10325    (match_operand:V2DI 2 "s_register_operand" "0")
10326    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10327   "TARGET_HAVE_MVE"
10328 {
10329   rtx ignore_vec = gen_reg_rtx (V2DImode);
10330   emit_insn (
10331   gen_mve_vstrdq_scatter_base_wb_<supf>v2di_insn (operands[0], operands[2],
10332                                                   operands[1], ignore_vec));
10333   DONE;
10334 })
10335
10336 ;;
10337 ;; [vstrdq_scatter_base_wb_s vstrdq_scatter_base_wb_u]
10338 ;;
10339 (define_insn "mve_vstrdq_scatter_base_wb_<supf>v2di_insn"
10340   [(set (mem:BLK (scratch))
10341         (unspec:BLK
10342                 [(match_operand:V2DI 1 "s_register_operand" "0")
10343                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10344                  (match_operand:V2DI 3 "s_register_operand" "w")]
10345          VSTRDSBWBQ))
10346    (set (match_operand:V2DI 0 "s_register_operand" "=&w")
10347         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10348          VSTRDSBWBQ))
10349   ]
10350   "TARGET_HAVE_MVE"
10351 {
10352    rtx ops[3];
10353    ops[0] = operands[1];
10354    ops[1] = operands[2];
10355    ops[2] = operands[3];
10356    output_asm_insn ("vstrd.u64\t%q2, [%q0, %1]!",ops);
10357    return "";
10358 }
10359   [(set_attr "length" "4")])
10360
10361 (define_expand "mve_vstrdq_scatter_base_wb_p_<supf>v2di"
10362   [(match_operand:V2DI 0 "s_register_operand" "=w")
10363    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10364    (match_operand:V2DI 2 "s_register_operand" "w")
10365    (match_operand:HI 3 "vpr_register_operand")
10366    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10367   "TARGET_HAVE_MVE"
10368 {
10369   rtx ignore_wb = gen_reg_rtx (V2DImode);
10370   emit_insn (
10371   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (ignore_wb, operands[0],
10372                                                     operands[1], operands[2],
10373                                                     operands[3]));
10374   DONE;
10375 })
10376
10377 (define_expand "mve_vstrdq_scatter_base_wb_p_add_<supf>v2di"
10378   [(match_operand:V2DI 0 "s_register_operand" "=w")
10379    (match_operand:SI 1 "mve_vldrd_immediate" "Ri")
10380    (match_operand:V2DI 2 "s_register_operand" "0")
10381    (match_operand:HI 3 "vpr_register_operand")
10382    (unspec:V2DI [(const_int 0)] VSTRDSBWBQ)]
10383   "TARGET_HAVE_MVE"
10384 {
10385   rtx ignore_vec = gen_reg_rtx (V2DImode);
10386   emit_insn (
10387   gen_mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn (operands[0], operands[2],
10388                                                     operands[1], ignore_vec,
10389                                                     operands[3]));
10390   DONE;
10391 })
10392
10393 ;;
10394 ;; [vstrdq_scatter_base_wb_p_s vstrdq_scatter_base_wb_p_u]
10395 ;;
10396 (define_insn "mve_vstrdq_scatter_base_wb_p_<supf>v2di_insn"
10397   [(set (mem:BLK (scratch))
10398         (unspec:BLK
10399                 [(match_operand:V2DI 1 "s_register_operand" "0")
10400                  (match_operand:SI 2 "mve_vldrd_immediate" "Ri")
10401                  (match_operand:V2DI 3 "s_register_operand" "w")
10402                  (match_operand:HI 4 "vpr_register_operand")]
10403          VSTRDSBWBQ))
10404    (set (match_operand:V2DI 0 "s_register_operand" "=w")
10405         (unspec:V2DI [(match_dup 1) (match_dup 2)]
10406          VSTRDSBWBQ))
10407   ]
10408   "TARGET_HAVE_MVE"
10409 {
10410    rtx ops[3];
10411    ops[0] = operands[1];
10412    ops[1] = operands[2];
10413    ops[2] = operands[3];
10414    output_asm_insn ("vpst\;\tvstrdt.u64\t%q2, [%q0, %1]!",ops);
10415    return "";
10416 }
10417   [(set_attr "length" "8")])
10418
10419 (define_expand "mve_vldrwq_gather_base_wb_<supf>v4si"
10420   [(match_operand:V4SI 0 "s_register_operand")
10421    (match_operand:V4SI 1 "s_register_operand")
10422    (match_operand:SI 2 "mve_vldrd_immediate")
10423    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10424   "TARGET_HAVE_MVE"
10425 {
10426   rtx ignore_result = gen_reg_rtx (V4SImode);
10427   emit_insn (
10428   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (ignore_result, operands[0],
10429                                                  operands[1], operands[2]));
10430   DONE;
10431 })
10432
10433 (define_expand "mve_vldrwq_gather_base_nowb_<supf>v4si"
10434   [(match_operand:V4SI 0 "s_register_operand")
10435    (match_operand:V4SI 1 "s_register_operand")
10436    (match_operand:SI 2 "mve_vldrd_immediate")
10437    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10438   "TARGET_HAVE_MVE"
10439 {
10440   rtx ignore_wb = gen_reg_rtx (V4SImode);
10441   emit_insn (
10442   gen_mve_vldrwq_gather_base_wb_<supf>v4si_insn (operands[0], ignore_wb,
10443                                                  operands[1], operands[2]));
10444   DONE;
10445 })
10446
10447 ;;
10448 ;; [vldrwq_gather_base_wb_s vldrwq_gather_base_wb_u]
10449 ;;
10450 (define_insn "mve_vldrwq_gather_base_wb_<supf>v4si_insn"
10451   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10452         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10453                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10454                       (mem:BLK (scratch))]
10455          VLDRWGBWBQ))
10456    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10457         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10458          VLDRWGBWBQ))
10459   ]
10460   "TARGET_HAVE_MVE"
10461 {
10462    rtx ops[3];
10463    ops[0] = operands[0];
10464    ops[1] = operands[2];
10465    ops[2] = operands[3];
10466    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10467    return "";
10468 }
10469   [(set_attr "length" "4")])
10470
10471 (define_expand "mve_vldrwq_gather_base_wb_z_<supf>v4si"
10472   [(match_operand:V4SI 0 "s_register_operand")
10473    (match_operand:V4SI 1 "s_register_operand")
10474    (match_operand:SI 2 "mve_vldrd_immediate")
10475    (match_operand:HI 3 "vpr_register_operand")
10476    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10477   "TARGET_HAVE_MVE"
10478 {
10479   rtx ignore_result = gen_reg_rtx (V4SImode);
10480   emit_insn (
10481   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (ignore_result, operands[0],
10482                                                    operands[1], operands[2],
10483                                                    operands[3]));
10484   DONE;
10485 })
10486 (define_expand "mve_vldrwq_gather_base_nowb_z_<supf>v4si"
10487   [(match_operand:V4SI 0 "s_register_operand")
10488    (match_operand:V4SI 1 "s_register_operand")
10489    (match_operand:SI 2 "mve_vldrd_immediate")
10490    (match_operand:HI 3 "vpr_register_operand")
10491    (unspec:V4SI [(const_int 0)] VLDRWGBWBQ)]
10492   "TARGET_HAVE_MVE"
10493 {
10494   rtx ignore_wb = gen_reg_rtx (V4SImode);
10495   emit_insn (
10496   gen_mve_vldrwq_gather_base_wb_z_<supf>v4si_insn (operands[0], ignore_wb,
10497                                                    operands[1], operands[2],
10498                                                    operands[3]));
10499   DONE;
10500 })
10501
10502 ;;
10503 ;; [vldrwq_gather_base_wb_z_s vldrwq_gather_base_wb_z_u]
10504 ;;
10505 (define_insn "mve_vldrwq_gather_base_wb_z_<supf>v4si_insn"
10506   [(set (match_operand:V4SI 0 "s_register_operand" "=&w")
10507         (unspec:V4SI [(match_operand:V4SI 2 "s_register_operand" "1")
10508                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10509                       (match_operand:HI 4 "vpr_register_operand" "Up")
10510                       (mem:BLK (scratch))]
10511          VLDRWGBWBQ))
10512    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10513         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10514          VLDRWGBWBQ))
10515   ]
10516   "TARGET_HAVE_MVE"
10517 {
10518    rtx ops[3];
10519    ops[0] = operands[0];
10520    ops[1] = operands[2];
10521    ops[2] = operands[3];
10522    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10523    return "";
10524 }
10525   [(set_attr "length" "8")])
10526
10527 (define_expand "mve_vldrwq_gather_base_wb_fv4sf"
10528   [(match_operand:V4SI 0 "s_register_operand")
10529    (match_operand:V4SI 1 "s_register_operand")
10530    (match_operand:SI 2 "mve_vldrd_immediate")
10531    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10532   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10533 {
10534   rtx ignore_result = gen_reg_rtx (V4SFmode);
10535   emit_insn (
10536   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (ignore_result, operands[0],
10537                                             operands[1], operands[2]));
10538   DONE;
10539 })
10540
10541 (define_expand "mve_vldrwq_gather_base_nowb_fv4sf"
10542   [(match_operand:V4SF 0 "s_register_operand")
10543    (match_operand:V4SI 1 "s_register_operand")
10544    (match_operand:SI 2 "mve_vldrd_immediate")
10545    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10546   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10547 {
10548   rtx ignore_wb = gen_reg_rtx (V4SImode);
10549   emit_insn (
10550   gen_mve_vldrwq_gather_base_wb_fv4sf_insn (operands[0], ignore_wb,
10551                                             operands[1], operands[2]));
10552   DONE;
10553 })
10554
10555 ;;
10556 ;; [vldrwq_gather_base_wb_f]
10557 ;;
10558 (define_insn "mve_vldrwq_gather_base_wb_fv4sf_insn"
10559   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10560         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10561                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10562                       (mem:BLK (scratch))]
10563          VLDRWQGBWB_F))
10564    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10565         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10566          VLDRWQGBWB_F))
10567   ]
10568   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10569 {
10570    rtx ops[3];
10571    ops[0] = operands[0];
10572    ops[1] = operands[2];
10573    ops[2] = operands[3];
10574    output_asm_insn ("vldrw.u32\t%q0, [%q1, %2]!",ops);
10575    return "";
10576 }
10577   [(set_attr "length" "4")])
10578
10579 (define_expand "mve_vldrwq_gather_base_wb_z_fv4sf"
10580   [(match_operand:V4SI 0 "s_register_operand")
10581    (match_operand:V4SI 1 "s_register_operand")
10582    (match_operand:SI 2 "mve_vldrd_immediate")
10583    (match_operand:HI 3 "vpr_register_operand")
10584    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10585   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10586 {
10587   rtx ignore_result = gen_reg_rtx (V4SFmode);
10588   emit_insn (
10589   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (ignore_result, operands[0],
10590                                               operands[1], operands[2],
10591                                               operands[3]));
10592   DONE;
10593 })
10594
10595 (define_expand "mve_vldrwq_gather_base_nowb_z_fv4sf"
10596   [(match_operand:V4SF 0 "s_register_operand")
10597    (match_operand:V4SI 1 "s_register_operand")
10598    (match_operand:SI 2 "mve_vldrd_immediate")
10599    (match_operand:HI 3 "vpr_register_operand")
10600    (unspec:V4SI [(const_int 0)] VLDRWQGBWB_F)]
10601   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10602 {
10603   rtx ignore_wb = gen_reg_rtx (V4SImode);
10604   emit_insn (
10605   gen_mve_vldrwq_gather_base_wb_z_fv4sf_insn (operands[0], ignore_wb,
10606                                               operands[1], operands[2],
10607                                               operands[3]));
10608   DONE;
10609 })
10610
10611 ;;
10612 ;; [vldrwq_gather_base_wb_z_f]
10613 ;;
10614 (define_insn "mve_vldrwq_gather_base_wb_z_fv4sf_insn"
10615   [(set (match_operand:V4SF 0 "s_register_operand" "=&w")
10616         (unspec:V4SF [(match_operand:V4SI 2 "s_register_operand" "1")
10617                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10618                       (match_operand:HI 4 "vpr_register_operand" "Up")
10619                       (mem:BLK (scratch))]
10620          VLDRWQGBWB_F))
10621    (set (match_operand:V4SI 1 "s_register_operand" "=&w")
10622         (unspec:V4SI [(match_dup 2) (match_dup 3)]
10623          VLDRWQGBWB_F))
10624   ]
10625   "TARGET_HAVE_MVE && TARGET_HAVE_MVE_FLOAT"
10626 {
10627    rtx ops[3];
10628    ops[0] = operands[0];
10629    ops[1] = operands[2];
10630    ops[2] = operands[3];
10631    output_asm_insn ("vpst\;vldrwt.u32\t%q0, [%q1, %2]!",ops);
10632    return "";
10633 }
10634   [(set_attr "length" "8")])
10635
10636 (define_expand "mve_vldrdq_gather_base_wb_<supf>v2di"
10637   [(match_operand:V2DI 0 "s_register_operand")
10638    (match_operand:V2DI 1 "s_register_operand")
10639    (match_operand:SI 2 "mve_vldrd_immediate")
10640    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10641   "TARGET_HAVE_MVE"
10642 {
10643   rtx ignore_result = gen_reg_rtx (V2DImode);
10644   emit_insn (
10645   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (ignore_result, operands[0],
10646                                                  operands[1], operands[2]));
10647   DONE;
10648 })
10649
10650 (define_expand "mve_vldrdq_gather_base_nowb_<supf>v2di"
10651   [(match_operand:V2DI 0 "s_register_operand")
10652    (match_operand:V2DI 1 "s_register_operand")
10653    (match_operand:SI 2 "mve_vldrd_immediate")
10654    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10655   "TARGET_HAVE_MVE"
10656 {
10657   rtx ignore_wb = gen_reg_rtx (V2DImode);
10658   emit_insn (
10659   gen_mve_vldrdq_gather_base_wb_<supf>v2di_insn (operands[0], ignore_wb,
10660                                                  operands[1], operands[2]));
10661   DONE;
10662 })
10663
10664
10665 ;;
10666 ;; [vldrdq_gather_base_wb_s vldrdq_gather_base_wb_u]
10667 ;;
10668 (define_insn "mve_vldrdq_gather_base_wb_<supf>v2di_insn"
10669   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10670         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10671                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10672                       (mem:BLK (scratch))]
10673          VLDRDGBWBQ))
10674    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10675         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10676          VLDRDGBWBQ))
10677   ]
10678   "TARGET_HAVE_MVE"
10679 {
10680    rtx ops[3];
10681    ops[0] = operands[0];
10682    ops[1] = operands[2];
10683    ops[2] = operands[3];
10684    output_asm_insn ("vldrd.64\t%q0, [%q1, %2]!",ops);
10685    return "";
10686 }
10687   [(set_attr "length" "4")])
10688
10689 (define_expand "mve_vldrdq_gather_base_wb_z_<supf>v2di"
10690   [(match_operand:V2DI 0 "s_register_operand")
10691    (match_operand:V2DI 1 "s_register_operand")
10692    (match_operand:SI 2 "mve_vldrd_immediate")
10693    (match_operand:HI 3 "vpr_register_operand")
10694    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10695   "TARGET_HAVE_MVE"
10696 {
10697   rtx ignore_result = gen_reg_rtx (V2DImode);
10698   emit_insn (
10699   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (ignore_result, operands[0],
10700                                                    operands[1], operands[2],
10701                                                    operands[3]));
10702   DONE;
10703 })
10704
10705 (define_expand "mve_vldrdq_gather_base_nowb_z_<supf>v2di"
10706   [(match_operand:V2DI 0 "s_register_operand")
10707    (match_operand:V2DI 1 "s_register_operand")
10708    (match_operand:SI 2 "mve_vldrd_immediate")
10709    (match_operand:HI 3 "vpr_register_operand")
10710    (unspec:V2DI [(const_int 0)] VLDRDGBWBQ)]
10711   "TARGET_HAVE_MVE"
10712 {
10713   rtx ignore_wb = gen_reg_rtx (V2DImode);
10714   emit_insn (
10715   gen_mve_vldrdq_gather_base_wb_z_<supf>v2di_insn (operands[0], ignore_wb,
10716                                                    operands[1], operands[2],
10717                                                    operands[3]));
10718   DONE;
10719 })
10720
10721 (define_insn "get_fpscr_nzcvqc"
10722  [(set (match_operand:SI 0 "register_operand" "=r")
10723    (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))]
10724  "TARGET_HAVE_MVE"
10725  "vmrs\\t%0, FPSCR_nzcvqc"
10726   [(set_attr "type" "mve_move")])
10727
10728 (define_insn "set_fpscr_nzcvqc"
10729  [(set (reg:SI VFPCC_REGNUM)
10730    (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")]
10731     VUNSPEC_SET_FPSCR_NZCVQC))]
10732  "TARGET_HAVE_MVE"
10733  "vmsr\\tFPSCR_nzcvqc, %0"
10734   [(set_attr "type" "mve_move")])
10735
10736 ;;
10737 ;; [vldrdq_gather_base_wb_z_s vldrdq_gather_base_wb_z_u]
10738 ;;
10739 (define_insn "mve_vldrdq_gather_base_wb_z_<supf>v2di_insn"
10740   [(set (match_operand:V2DI 0 "s_register_operand" "=&w")
10741         (unspec:V2DI [(match_operand:V2DI 2 "s_register_operand" "1")
10742                       (match_operand:SI 3 "mve_vldrd_immediate" "Ri")
10743                       (match_operand:HI 4 "vpr_register_operand" "Up")
10744                       (mem:BLK (scratch))]
10745          VLDRDGBWBQ))
10746    (set (match_operand:V2DI 1 "s_register_operand" "=&w")
10747         (unspec:V2DI [(match_dup 2) (match_dup 3)]
10748          VLDRDGBWBQ))
10749   ]
10750   "TARGET_HAVE_MVE"
10751 {
10752    rtx ops[3];
10753    ops[0] = operands[0];
10754    ops[1] = operands[2];
10755    ops[2] = operands[3];
10756    output_asm_insn ("vpst\;vldrdt.u64\t%q0, [%q1, %2]!",ops);
10757    return "";
10758 }
10759   [(set_attr "length" "8")])
10760 ;;
10761 ;; [vadciq_m_s, vadciq_m_u])
10762 ;;
10763 (define_insn "mve_vadciq_m_<supf>v4si"
10764   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10765         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10766                       (match_operand:V4SI 2 "s_register_operand" "w")
10767                       (match_operand:V4SI 3 "s_register_operand" "w")
10768                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10769          VADCIQ_M))
10770    (set (reg:SI VFPCC_REGNUM)
10771         (unspec:SI [(const_int 0)]
10772          VADCIQ_M))
10773   ]
10774   "TARGET_HAVE_MVE"
10775   "vpst\;vadcit.i32\t%q0, %q2, %q3"
10776   [(set_attr "type" "mve_move")
10777    (set_attr "length" "8")])
10778
10779 ;;
10780 ;; [vadciq_u, vadciq_s])
10781 ;;
10782 (define_insn "mve_vadciq_<supf>v4si"
10783   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10784         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10785                       (match_operand:V4SI 2 "s_register_operand" "w")]
10786          VADCIQ))
10787    (set (reg:SI VFPCC_REGNUM)
10788         (unspec:SI [(const_int 0)]
10789          VADCIQ))
10790   ]
10791   "TARGET_HAVE_MVE"
10792   "vadci.i32\t%q0, %q1, %q2"
10793   [(set_attr "type" "mve_move")
10794    (set_attr "length" "4")])
10795
10796 ;;
10797 ;; [vadcq_m_s, vadcq_m_u])
10798 ;;
10799 (define_insn "mve_vadcq_m_<supf>v4si"
10800   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10801         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0")
10802                       (match_operand:V4SI 2 "s_register_operand" "w")
10803                       (match_operand:V4SI 3 "s_register_operand" "w")
10804                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10805          VADCQ_M))
10806    (set (reg:SI VFPCC_REGNUM)
10807         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10808          VADCQ_M))
10809   ]
10810   "TARGET_HAVE_MVE"
10811   "vpst\;vadct.i32\t%q0, %q2, %q3"
10812   [(set_attr "type" "mve_move")
10813    (set_attr "length" "8")])
10814
10815 ;;
10816 ;; [vadcq_u, vadcq_s])
10817 ;;
10818 (define_insn "mve_vadcq_<supf>v4si"
10819   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10820         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10821                        (match_operand:V4SI 2 "s_register_operand" "w")]
10822          VADCQ))
10823    (set (reg:SI VFPCC_REGNUM)
10824         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10825          VADCQ))
10826   ]
10827   "TARGET_HAVE_MVE"
10828   "vadc.i32\t%q0, %q1, %q2"
10829   [(set_attr "type" "mve_move")
10830    (set_attr "length" "4")
10831    (set_attr "conds" "set")])
10832
10833 ;;
10834 ;; [vsbciq_m_u, vsbciq_m_s])
10835 ;;
10836 (define_insn "mve_vsbciq_m_<supf>v4si"
10837   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10838         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10839                       (match_operand:V4SI 2 "s_register_operand" "w")
10840                       (match_operand:V4SI 3 "s_register_operand" "w")
10841                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10842          VSBCIQ_M))
10843    (set (reg:SI VFPCC_REGNUM)
10844         (unspec:SI [(const_int 0)]
10845          VSBCIQ_M))
10846   ]
10847   "TARGET_HAVE_MVE"
10848   "vpst\;vsbcit.i32\t%q0, %q2, %q3"
10849   [(set_attr "type" "mve_move")
10850    (set_attr "length" "8")])
10851
10852 ;;
10853 ;; [vsbciq_s, vsbciq_u])
10854 ;;
10855 (define_insn "mve_vsbciq_<supf>v4si"
10856   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10857         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10858                       (match_operand:V4SI 2 "s_register_operand" "w")]
10859          VSBCIQ))
10860    (set (reg:SI VFPCC_REGNUM)
10861         (unspec:SI [(const_int 0)]
10862          VSBCIQ))
10863   ]
10864   "TARGET_HAVE_MVE"
10865   "vsbci.i32\t%q0, %q1, %q2"
10866   [(set_attr "type" "mve_move")
10867    (set_attr "length" "4")])
10868
10869 ;;
10870 ;; [vsbcq_m_u, vsbcq_m_s])
10871 ;;
10872 (define_insn "mve_vsbcq_m_<supf>v4si"
10873   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10874         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10875                       (match_operand:V4SI 2 "s_register_operand" "w")
10876                       (match_operand:V4SI 3 "s_register_operand" "w")
10877                       (match_operand:HI 4 "vpr_register_operand" "Up")]
10878          VSBCQ_M))
10879    (set (reg:SI VFPCC_REGNUM)
10880         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10881          VSBCQ_M))
10882   ]
10883   "TARGET_HAVE_MVE"
10884   "vpst\;vsbct.i32\t%q0, %q2, %q3"
10885   [(set_attr "type" "mve_move")
10886    (set_attr "length" "8")])
10887
10888 ;;
10889 ;; [vsbcq_s, vsbcq_u])
10890 ;;
10891 (define_insn "mve_vsbcq_<supf>v4si"
10892   [(set (match_operand:V4SI 0 "s_register_operand" "=w")
10893         (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w")
10894                       (match_operand:V4SI 2 "s_register_operand" "w")]
10895          VSBCQ))
10896    (set (reg:SI VFPCC_REGNUM)
10897         (unspec:SI [(reg:SI VFPCC_REGNUM)]
10898          VSBCQ))
10899   ]
10900   "TARGET_HAVE_MVE"
10901   "vsbc.i32\t%q0, %q1, %q2"
10902   [(set_attr "type" "mve_move")
10903    (set_attr "length" "4")])
10904
10905 ;;
10906 ;; [vst2q])
10907 ;;
10908 (define_insn "mve_vst2q<mode>"
10909   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
10910         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
10911                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10912          VST2Q))
10913   ]
10914   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10915    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10916 {
10917    rtx ops[4];
10918    int regno = REGNO (operands[1]);
10919    ops[0] = gen_rtx_REG (TImode, regno);
10920    ops[1] = gen_rtx_REG (TImode, regno + 4);
10921    rtx reg  = operands[0];
10922    while (reg && !REG_P (reg))
10923     reg = XEXP (reg, 0);
10924    gcc_assert (REG_P (reg));
10925    ops[2] = reg;
10926    ops[3] = operands[0];
10927    output_asm_insn ("vst20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10928                     "vst21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10929    return "";
10930 }
10931   [(set_attr "length" "8")])
10932
10933 ;;
10934 ;; [vld2q])
10935 ;;
10936 (define_insn "mve_vld2q<mode>"
10937   [(set (match_operand:OI 0 "s_register_operand" "=w")
10938         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
10939                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10940          VLD2Q))
10941   ]
10942   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10943    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10944 {
10945    rtx ops[4];
10946    int regno = REGNO (operands[0]);
10947    ops[0] = gen_rtx_REG (TImode, regno);
10948    ops[1] = gen_rtx_REG (TImode, regno + 4);
10949    rtx reg  = operands[1];
10950    while (reg && !REG_P (reg))
10951     reg = XEXP (reg, 0);
10952    gcc_assert (REG_P (reg));
10953    ops[2] = reg;
10954    ops[3] = operands[1];
10955    output_asm_insn ("vld20.<V_sz_elem>\t{%q0, %q1}, [%2]\n\t"
10956                     "vld21.<V_sz_elem>\t{%q0, %q1}, %3", ops);
10957    return "";
10958 }
10959   [(set_attr "length" "8")])
10960
10961 ;;
10962 ;; [vld4q])
10963 ;;
10964 (define_insn "mve_vld4q<mode>"
10965   [(set (match_operand:XI 0 "s_register_operand" "=w")
10966         (unspec:XI [(match_operand:XI 1 "neon_struct_operand" "Um")
10967                     (unspec:MVE_VLD_ST [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
10968          VLD4Q))
10969   ]
10970   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
10971    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
10972 {
10973    rtx ops[6];
10974    int regno = REGNO (operands[0]);
10975    ops[0] = gen_rtx_REG (TImode, regno);
10976    ops[1] = gen_rtx_REG (TImode, regno+4);
10977    ops[2] = gen_rtx_REG (TImode, regno+8);
10978    ops[3] = gen_rtx_REG (TImode, regno + 12);
10979    rtx reg  = operands[1];
10980    while (reg && !REG_P (reg))
10981     reg = XEXP (reg, 0);
10982    gcc_assert (REG_P (reg));
10983    ops[4] = reg;
10984    ops[5] = operands[1];
10985    output_asm_insn ("vld40.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10986                     "vld41.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10987                     "vld42.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, [%4]\n\t"
10988                     "vld43.<V_sz_elem>\t{%q0, %q1, %q2, %q3}, %5", ops);
10989    return "";
10990 }
10991   [(set_attr "length" "16")])
10992 ;;
10993 ;; [vgetq_lane_u, vgetq_lane_s, vgetq_lane_f])
10994 ;;
10995 (define_insn "mve_vec_extract<mode><V_elem_l>"
10996  [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
10997    (vec_select:<V_elem>
10998     (match_operand:MVE_VLD_ST 1 "s_register_operand" "w")
10999     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
11000   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11001    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11002 {
11003   if (BYTES_BIG_ENDIAN)
11004     {
11005       int elt = INTVAL (operands[2]);
11006       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11007       operands[2] = GEN_INT (elt);
11008     }
11009   return "vmov.<V_extr_elem>\t%0, %q1[%c2]";
11010 }
11011  [(set_attr "type" "mve_move")])
11012
11013 (define_insn "mve_vec_extractv2didi"
11014  [(set (match_operand:DI 0 "s_register_operand" "=r")
11015    (vec_select:DI
11016     (match_operand:V2DI 1 "s_register_operand" "w")
11017     (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
11018   "TARGET_HAVE_MVE"
11019 {
11020   int elt = INTVAL (operands[2]);
11021   if (BYTES_BIG_ENDIAN)
11022     elt = 1 - elt;
11023
11024   if (elt == 0)
11025    return "vmov\t%Q0, %R0, %e1";
11026   else
11027    return "vmov\t%J0, %K0, %f1";
11028 }
11029  [(set_attr "type" "mve_move")])
11030
11031 (define_insn "*mve_vec_extract_sext_internal<mode>"
11032  [(set (match_operand:SI 0 "s_register_operand" "=r")
11033    (sign_extend:SI
11034     (vec_select:<V_elem>
11035      (match_operand:MVE_2 1 "s_register_operand" "w")
11036      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
11037   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11038    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11039 {
11040   if (BYTES_BIG_ENDIAN)
11041     {
11042       int elt = INTVAL (operands[2]);
11043       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11044       operands[2] = GEN_INT (elt);
11045     }
11046   return "vmov.s<V_sz_elem>\t%0, %q1[%c2]";
11047 }
11048  [(set_attr "type" "mve_move")])
11049
11050 (define_insn "*mve_vec_extract_zext_internal<mode>"
11051  [(set (match_operand:SI 0 "s_register_operand" "=r")
11052    (zero_extend:SI
11053     (vec_select:<V_elem>
11054      (match_operand:MVE_2 1 "s_register_operand" "w")
11055      (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
11056   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11057    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11058 {
11059   if (BYTES_BIG_ENDIAN)
11060     {
11061       int elt = INTVAL (operands[2]);
11062       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11063       operands[2] = GEN_INT (elt);
11064     }
11065   return "vmov.u<V_sz_elem>\t%0, %q1[%c2]";
11066 }
11067  [(set_attr "type" "mve_move")])
11068
11069 ;;
11070 ;; [vsetq_lane_u, vsetq_lane_s, vsetq_lane_f])
11071 ;;
11072 (define_insn "mve_vec_set<mode>_internal"
11073  [(set (match_operand:VQ2 0 "s_register_operand" "=w")
11074        (vec_merge:VQ2
11075         (vec_duplicate:VQ2
11076           (match_operand:<V_elem> 1 "nonimmediate_operand" "r"))
11077         (match_operand:VQ2 3 "s_register_operand" "0")
11078         (match_operand:SI 2 "immediate_operand" "i")))]
11079   "(TARGET_HAVE_MVE && VALID_MVE_SI_MODE (<MODE>mode))
11080    || (TARGET_HAVE_MVE_FLOAT && VALID_MVE_SF_MODE (<MODE>mode))"
11081 {
11082   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11083   if (BYTES_BIG_ENDIAN)
11084     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
11085   operands[2] = GEN_INT (elt);
11086
11087   return "vmov.<V_sz_elem>\t%q0[%c2], %1";
11088 }
11089  [(set_attr "type" "mve_move")])
11090
11091 (define_insn "mve_vec_setv2di_internal"
11092  [(set (match_operand:V2DI 0 "s_register_operand" "=w")
11093        (vec_merge:V2DI
11094         (vec_duplicate:V2DI
11095           (match_operand:DI 1 "nonimmediate_operand" "r"))
11096         (match_operand:V2DI 3 "s_register_operand" "0")
11097         (match_operand:SI 2 "immediate_operand" "i")))]
11098  "TARGET_HAVE_MVE"
11099 {
11100   int elt = ffs ((int) INTVAL (operands[2])) - 1;
11101   if (BYTES_BIG_ENDIAN)
11102     elt = 1 - elt;
11103
11104   if (elt == 0)
11105    return "vmov\t%e0, %Q1, %R1";
11106   else
11107    return "vmov\t%f0, %J1, %K1";
11108 }
11109  [(set_attr "type" "mve_move")])
11110
11111 ;;
11112 ;; [uqrshll_di]
11113 ;;
11114 (define_insn "mve_uqrshll_sat<supf>_di"
11115   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11116         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11117                     (match_operand:SI 2 "s_register_operand" "r")]
11118          UQRSHLLQ))]
11119   "TARGET_HAVE_MVE"
11120   "uqrshll%?\\t%Q1, %R1, #<supf>, %2"
11121   [(set_attr "predicable" "yes")])
11122
11123 ;;
11124 ;; [sqrshrl_di]
11125 ;;
11126 (define_insn "mve_sqrshrl_sat<supf>_di"
11127   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11128         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11129                     (match_operand:SI 2 "s_register_operand" "r")]
11130          SQRSHRLQ))]
11131   "TARGET_HAVE_MVE"
11132   "sqrshrl%?\\t%Q1, %R1, #<supf>, %2"
11133   [(set_attr "predicable" "yes")])
11134
11135 ;;
11136 ;; [uqrshl_si]
11137 ;;
11138 (define_insn "mve_uqrshl_si"
11139   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11140         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11141                     (match_operand:SI 2 "s_register_operand" "r")]
11142          UQRSHL))]
11143   "TARGET_HAVE_MVE"
11144   "uqrshl%?\\t%1, %2"
11145   [(set_attr "predicable" "yes")])
11146
11147 ;;
11148 ;; [sqrshr_si]
11149 ;;
11150 (define_insn "mve_sqrshr_si"
11151   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11152         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11153                     (match_operand:SI 2 "s_register_operand" "r")]
11154          SQRSHR))]
11155   "TARGET_HAVE_MVE"
11156   "sqrshr%?\\t%1, %2"
11157   [(set_attr "predicable" "yes")])
11158
11159 ;;
11160 ;; [uqshll_di]
11161 ;;
11162 (define_insn "mve_uqshll_di"
11163   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11164         (us_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11165                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11166   "TARGET_HAVE_MVE"
11167   "uqshll%?\\t%Q1, %R1, %2"
11168   [(set_attr "predicable" "yes")])
11169
11170 ;;
11171 ;; [urshrl_di]
11172 ;;
11173 (define_insn "mve_urshrl_di"
11174   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11175         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11176                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11177          URSHRL))]
11178   "TARGET_HAVE_MVE"
11179   "urshrl%?\\t%Q1, %R1, %2"
11180   [(set_attr "predicable" "yes")])
11181
11182 ;;
11183 ;; [uqshl_si]
11184 ;;
11185 (define_insn "mve_uqshl_si"
11186   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11187         (us_ashift:SI (match_operand:SI 1 "arm_general_register_operand" "r")
11188                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11189   "TARGET_HAVE_MVE"
11190   "uqshl%?\\t%1, %2"
11191   [(set_attr "predicable" "yes")])
11192
11193 ;;
11194 ;; [urshr_si]
11195 ;;
11196 (define_insn "mve_urshr_si"
11197   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11198         (unspec:SI [(match_operand:SI 1 "arm_general_register_operand" "r")
11199                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11200          URSHR))]
11201   "TARGET_HAVE_MVE"
11202   "urshr%?\\t%1, %2"
11203   [(set_attr "predicable" "yes")])
11204
11205 ;;
11206 ;; [sqshl_si]
11207 ;;
11208 (define_insn "mve_sqshl_si"
11209   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11210         (ss_ashift:SI (match_operand:DI 1 "arm_general_register_operand" "r")
11211                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11212   "TARGET_HAVE_MVE"
11213   "sqshl%?\\t%1, %2"
11214   [(set_attr "predicable" "yes")])
11215
11216 ;;
11217 ;; [srshr_si]
11218 ;;
11219 (define_insn "mve_srshr_si"
11220   [(set (match_operand:SI 0 "arm_general_register_operand" "+r")
11221         (unspec:SI [(match_operand:DI 1 "arm_general_register_operand" "r")
11222                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11223          SRSHR))]
11224   "TARGET_HAVE_MVE"
11225   "srshr%?\\t%1, %2"
11226   [(set_attr "predicable" "yes")])
11227
11228 ;;
11229 ;; [srshrl_di]
11230 ;;
11231 (define_insn "mve_srshrl_di"
11232   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11233         (unspec:DI [(match_operand:DI 1 "arm_general_register_operand" "r")
11234                     (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")]
11235          SRSHRL))]
11236   "TARGET_HAVE_MVE"
11237   "srshrl%?\\t%Q1, %R1, %2"
11238   [(set_attr "predicable" "yes")])
11239
11240 ;;
11241 ;; [sqshll_di]
11242 ;;
11243 (define_insn "mve_sqshll_di"
11244   [(set (match_operand:DI 0 "arm_general_register_operand" "+r")
11245         (ss_ashift:DI (match_operand:DI 1 "arm_general_register_operand" "r")
11246                       (match_operand:SI 2 "arm_reg_or_long_shift_imm" "rPg")))]
11247   "TARGET_HAVE_MVE"
11248   "sqshll%?\\t%Q1, %R1, %2"
11249   [(set_attr "predicable" "yes")])
11250
11251 ;;
11252 ;; [vshlcq_m_u vshlcq_m_s]
11253 ;;
11254 (define_expand "mve_vshlcq_m_vec_<supf><mode>"
11255  [(match_operand:MVE_2 0 "s_register_operand")
11256   (match_operand:MVE_2 1 "s_register_operand")
11257   (match_operand:SI 2 "s_register_operand")
11258   (match_operand:SI 3 "mve_imm_32")
11259   (match_operand:HI 4 "vpr_register_operand")
11260   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
11261  "TARGET_HAVE_MVE"
11262 {
11263   rtx ignore_wb = gen_reg_rtx (SImode);
11264   emit_insn (gen_mve_vshlcq_m_<supf><mode> (operands[0], ignore_wb, operands[1],
11265                                             operands[2], operands[3],
11266                                             operands[4]));
11267   DONE;
11268 })
11269
11270 (define_expand "mve_vshlcq_m_carry_<supf><mode>"
11271  [(match_operand:SI 0 "s_register_operand")
11272   (match_operand:MVE_2 1 "s_register_operand")
11273   (match_operand:SI 2 "s_register_operand")
11274   (match_operand:SI 3 "mve_imm_32")
11275   (match_operand:HI 4 "vpr_register_operand")
11276   (unspec:MVE_2 [(const_int 0)] VSHLCQ_M)]
11277  "TARGET_HAVE_MVE"
11278 {
11279   rtx ignore_vec = gen_reg_rtx (<MODE>mode);
11280   emit_insn (gen_mve_vshlcq_m_<supf><mode> (ignore_vec, operands[0],
11281                                             operands[1], operands[2],
11282                                             operands[3], operands[4]));
11283   DONE;
11284 })
11285
11286 (define_insn "mve_vshlcq_m_<supf><mode>"
11287  [(set (match_operand:MVE_2 0 "s_register_operand" "=w")
11288        (unspec:MVE_2 [(match_operand:MVE_2 2 "s_register_operand" "0")
11289                       (match_operand:SI 3 "s_register_operand" "1")
11290                       (match_operand:SI 4 "mve_imm_32" "Rf")
11291                       (match_operand:HI 5 "vpr_register_operand" "Up")]
11292         VSHLCQ_M))
11293   (set (match_operand:SI  1 "s_register_operand" "=r")
11294        (unspec:SI [(match_dup 2)
11295                    (match_dup 3)
11296                    (match_dup 4)
11297                    (match_dup 5)]
11298         VSHLCQ_M))
11299  ]
11300  "TARGET_HAVE_MVE"
11301  "vpst\;vshlct\t%q0, %1, %4"
11302  [(set_attr "type" "mve_move")
11303   (set_attr "length" "8")])