gcc/config/rs6000/vsx.md

   1 ;; VSX patterns.
   2 ;; Copyright (C) 2009-2013 Free Software Foundation, Inc.
   3 ;; Contributed by Michael Meissner <meissner@linux.vnet.ibm.com>
   4
   5 ;; This file is part of GCC.
   6
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published
   9 ;; by the Free Software Foundation; either version 3, or (at your
  10 ;; option) any later version.
  11
  12 ;; GCC is distributed in the hope that it will be useful, but WITHOUT
  13 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  14 ;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  15 ;; License for more details.
  16
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Iterator for both scalar and vector floating point types supported by VSX
  22 (define_mode_iterator VSX_B [DF V4SF V2DF])
  23
  24 ;; Iterator for the 2 64-bit vector types
  25 (define_mode_iterator VSX_D [V2DF V2DI])
  26
  27 ;; Iterator for the 2 32-bit vector types
  28 (define_mode_iterator VSX_W [V4SF V4SI])
  29
  30 ;; Iterator for the DF types
  31 (define_mode_iterator VSX_DF [V2DF DF])
  32
  33 ;; Iterator for vector floating point types supported by VSX
  34 (define_mode_iterator VSX_F [V4SF V2DF])
  35
  36 ;; Iterator for logical types supported by VSX
  37 (define_mode_iterator VSX_L [V16QI V8HI V4SI V2DI V4SF V2DF TI])
  38
  39 ;; Like VSX_L, but don't support TImode for doing logical instructions in
  40 ;; 32-bit
  41 (define_mode_iterator VSX_L2 [V16QI V8HI V4SI V2DI V4SF V2DF])
  42
  43 ;; Iterator for memory move.  Handle TImode specially to allow
  44 ;; it to use gprs as well as vsx registers.
  45 (define_mode_iterator VSX_M [V16QI V8HI V4SI V2DI V4SF V2DF])
  46
  47 ;; Map into the appropriate load/store name based on the type
  48 (define_mode_attr VSm  [(V16QI "vw4")
  49                         (V8HI  "vw4")
  50                         (V4SI  "vw4")
  51                         (V4SF  "vw4")
  52                         (V2DF  "vd2")
  53                         (V2DI  "vd2")
  54                         (DF    "d")
  55                         (TI    "vd2")])
  56
  57 ;; Map into the appropriate suffix based on the type
  58 (define_mode_attr VSs   [(V16QI "sp")
  59                          (V8HI  "sp")
  60                          (V4SI  "sp")
  61                          (V4SF  "sp")
  62                          (V2DF  "dp")
  63                          (V2DI  "dp")
  64                          (DF    "dp")
  65                          (SF    "sp")
  66                          (TI    "dp")])
  67
  68 ;; Map the register class used
  69 (define_mode_attr VSr   [(V16QI "v")
  70                          (V8HI  "v")
  71                          (V4SI  "v")
  72                          (V4SF  "wf")
  73                          (V2DI  "wd")
  74                          (V2DF  "wd")
  75                          (DF    "ws")
  76                          (SF    "d")
  77                          (TI    "wt")])
  78
  79 ;; Map the register class used for float<->int conversions
  80 (define_mode_attr VSr2  [(V2DF  "wd")
  81                          (V4SF  "wf")
  82                          (DF    "ws")])
  83
  84 (define_mode_attr VSr3  [(V2DF  "wa")
  85                          (V4SF  "wa")
  86                          (DF    "ws")])
  87
  88 ;; Map the register class for sp<->dp float conversions, destination
  89 (define_mode_attr VSr4  [(SF    "ws")
  90                          (DF    "f")
  91                          (V2DF  "wd")
  92                          (V4SF  "v")])
  93
  94 ;; Map the register class for sp<->dp float conversions, destination
  95 (define_mode_attr VSr5  [(SF    "ws")
  96                          (DF    "f")
  97                          (V2DF  "v")
  98                          (V4SF  "wd")])
  99
 100 ;; Same size integer type for floating point data
 101 (define_mode_attr VSi [(V4SF  "v4si")
 102                        (V2DF  "v2di")
 103                        (DF    "di")])
 104
 105 (define_mode_attr VSI [(V4SF  "V4SI")
 106                        (V2DF  "V2DI")
 107                        (DF    "DI")])
 108
 109 ;; Word size for same size conversion
 110 (define_mode_attr VSc [(V4SF "w")
 111                        (V2DF "d")
 112                        (DF   "d")])
 113
 114 ;; Map into either s or v, depending on whether this is a scalar or vector
 115 ;; operation
 116 (define_mode_attr VSv   [(V16QI "v")
 117                          (V8HI  "v")
 118                          (V4SI  "v")
 119                          (V4SF  "v")
 120                          (V2DI  "v")
 121                          (V2DF  "v")
 122                          (DF    "s")])
 123
 124 ;; Appropriate type for add ops (and other simple FP ops)
 125 (define_mode_attr VStype_simple [(V2DF "vecdouble")
 126                                  (V4SF "vecfloat")
 127                                  (DF   "fp")])
 128
 129 (define_mode_attr VSfptype_simple [(V2DF "fp_addsub_d")
 130                                    (V4SF "fp_addsub_s")
 131                                    (DF   "fp_addsub_d")])
 132
 133 ;; Appropriate type for multiply ops
 134 (define_mode_attr VStype_mul    [(V2DF "vecdouble")
 135                                  (V4SF "vecfloat")
 136                                  (DF   "dmul")])
 137
 138 (define_mode_attr VSfptype_mul  [(V2DF "fp_mul_d")
 139                                  (V4SF "fp_mul_s")
 140                                  (DF   "fp_mul_d")])
 141
 142 ;; Appropriate type for divide ops.
 143 (define_mode_attr VStype_div    [(V2DF "vecdiv")
 144                                  (V4SF "vecfdiv")
 145                                  (DF   "ddiv")])
 146
 147 (define_mode_attr VSfptype_div  [(V2DF "fp_div_d")
 148                                  (V4SF "fp_div_s")
 149                                  (DF   "fp_div_d")])
 150
 151 ;; Appropriate type for sqrt ops.  For now, just lump the vector sqrt with
 152 ;; the scalar sqrt
 153 (define_mode_attr VStype_sqrt   [(V2DF "dsqrt")
 154                                  (V4SF "ssqrt")
 155                                  (DF   "dsqrt")])
 156
 157 (define_mode_attr VSfptype_sqrt [(V2DF "fp_sqrt_d")
 158                                  (V4SF "fp_sqrt_s")
 159                                  (DF   "fp_sqrt_d")])
 160
 161 ;; Iterator and modes for sp<->dp conversions
 162 ;; Because scalar SF values are represented internally as double, use the
 163 ;; V4SF type to represent this than SF.
 164 (define_mode_iterator VSX_SPDP [DF V4SF V2DF])
 165
 166 (define_mode_attr VS_spdp_res [(DF      "V4SF")
 167                                (V4SF    "V2DF")
 168                                (V2DF    "V4SF")])
 169
 170 (define_mode_attr VS_spdp_insn [(DF     "xscvdpsp")
 171                                 (V4SF   "xvcvspdp")
 172                                 (V2DF   "xvcvdpsp")])
 173
 174 (define_mode_attr VS_spdp_type [(DF     "fp")
 175                                 (V4SF   "vecdouble")
 176                                 (V2DF   "vecdouble")])
 177
 178 ;; Map the scalar mode for a vector type
 179 (define_mode_attr VS_scalar [(V2DF      "DF")
 180                              (V2DI      "DI")
 181                              (V4SF      "SF")
 182                              (V4SI      "SI")
 183                              (V8HI      "HI")
 184                              (V16QI     "QI")])
 185
 186 ;; Map to a double-sized vector mode
 187 (define_mode_attr VS_double [(V4SI      "V8SI")
 188                              (V4SF      "V8SF")
 189                              (V2DI      "V4DI")
 190                              (V2DF      "V4DF")])
 191
 192 ;; Constants for creating unspecs
 193 (define_c_enum "unspec"
 194   [UNSPEC_VSX_CONCAT
 195    UNSPEC_VSX_CVDPSXWS
 196    UNSPEC_VSX_CVDPUXWS
 197    UNSPEC_VSX_CVSPDP
 198    UNSPEC_VSX_CVSPDPN
 199    UNSPEC_VSX_CVDPSPN
 200    UNSPEC_VSX_CVSXWDP
 201    UNSPEC_VSX_CVUXWDP
 202    UNSPEC_VSX_CVSXDSP
 203    UNSPEC_VSX_CVUXDSP
 204    UNSPEC_VSX_CVSPSXDS
 205    UNSPEC_VSX_CVSPUXDS
 206    UNSPEC_VSX_TDIV
 207    UNSPEC_VSX_TSQRT
 208    UNSPEC_VSX_SET
 209    UNSPEC_VSX_ROUND_I
 210    UNSPEC_VSX_ROUND_IC
 211    UNSPEC_VSX_SLDWI
 212   ])
 213
 214 ;; VSX moves
 215 (define_insn "*vsx_mov<mode>"
 216   [(set (match_operand:VSX_M 0 "nonimmediate_operand" "=Z,<VSr>,<VSr>,?Z,?wa,?wa,wQ,?&r,??Y,??r,??r,<VSr>,?wa,*r,v,wZ, v")
 217         (match_operand:VSX_M 1 "input_operand" "<VSr>,Z,<VSr>,wa,Z,wa,r,wQ,r,Y,r,j,j,j,W,v,wZ"))]
 218   "VECTOR_MEM_VSX_P (<MODE>mode)
 219    && (register_operand (operands[0], <MODE>mode)
 220        || register_operand (operands[1], <MODE>mode))"
 221 {
 222   return rs6000_output_move_128bit (operands);
 223 }
 224   [(set_attr "type" "vecstore,vecload,vecsimple,vecstore,vecload,vecsimple,load,store,store,load, *,vecsimple,vecsimple,*, *,vecstore,vecload")
 225    (set_attr "length" "4,4,4,4,4,4,12,12,12,12,16,4,4,*,16,4,4")])
 226
 227 ;; Unlike other VSX moves, allow the GPRs even for reloading, since a normal
 228 ;; use of TImode is for unions.  However for plain data movement, slightly
 229 ;; favor the vector loads
 230 (define_insn "*vsx_movti_64bit"
 231   [(set (match_operand:TI 0 "nonimmediate_operand" "=Z,wa,wa,wa,v,v,wZ,wQ,&r,Y,r,r,?r")
 232         (match_operand:TI 1 "input_operand" "wa,Z,wa,O,W,wZ,v,r,wQ,r,Y,r,n"))]
 233   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (TImode)
 234    && (register_operand (operands[0], TImode)
 235        || register_operand (operands[1], TImode))"
 236 {
 237   return rs6000_output_move_128bit (operands);
 238 }
 239   [(set_attr "type" "vecstore,vecload,vecsimple,vecsimple,vecsimple,vecstore,vecload,store,load,store,load,*,*")
 240    (set_attr "length" "4,4,4,4,16,4,4,8,8,8,8,8,8")])
 241
 242 (define_insn "*vsx_movti_32bit"
 243   [(set (match_operand:TI 0 "nonimmediate_operand" "=Z,wa,wa,wa,v, v,wZ,Q,Y,????r,????r,????r,r")
 244         (match_operand:TI 1 "input_operand"        "wa, Z,wa, O,W,wZ, v,r,r,    Q,    Y,    r,n"))]
 245   "! TARGET_POWERPC64 && VECTOR_MEM_VSX_P (TImode)
 246    && (register_operand (operands[0], TImode)
 247        || register_operand (operands[1], TImode))"
 248 {
 249   switch (which_alternative)
 250     {
 251     case 0:
 252       return "stxvd2x %x1,%y0";
 253
 254     case 1:
 255       return "lxvd2x %x0,%y1";
 256
 257     case 2:
 258       return "xxlor %x0,%x1,%x1";
 259
 260     case 3:
 261       return "xxlxor %x0,%x0,%x0";
 262
 263     case 4:
 264       return output_vec_const_move (operands);
 265
 266     case 5:
 267       return "stvx %1,%y0";
 268
 269     case 6:
 270       return "lvx %0,%y1";
 271
 272     case 7:
 273       if (TARGET_STRING)
 274         return \"stswi %1,%P0,16\";
 275
 276     case 8:
 277       return \"#\";
 278
 279     case 9:
 280       /* If the address is not used in the output, we can use lsi.  Otherwise,
 281          fall through to generating four loads.  */
 282       if (TARGET_STRING
 283           && ! reg_overlap_mentioned_p (operands[0], operands[1]))
 284         return \"lswi %0,%P1,16\";
 285       /* ... fall through ...  */
 286
 287     case 10:
 288     case 11:
 289     case 12:
 290       return \"#\";
 291     default:
 292       gcc_unreachable ();
 293     }
 294 }
 295   [(set_attr "type" "vecstore,vecload,vecsimple,vecsimple,vecsimple,vecstore,vecload,store_ux,store_ux,load_ux,load_ux, *, *")
 296    (set_attr "length" "     4,      4,        4,       4,         8,       4,      4,      16,      16,     16,     16,16,16")
 297    (set (attr "cell_micro") (if_then_else (match_test "TARGET_STRING")
 298                                           (const_string "always")
 299                                           (const_string "conditional")))])
 300
 301 ;; Explicit  load/store expanders for the builtin functions
 302 (define_expand "vsx_load_<mode>"
 303   [(set (match_operand:VSX_M 0 "vsx_register_operand" "")
 304         (match_operand:VSX_M 1 "memory_operand" ""))]
 305   "VECTOR_MEM_VSX_P (<MODE>mode)"
 306   "")
 307
 308 (define_expand "vsx_store_<mode>"
 309   [(set (match_operand:VSX_M 0 "memory_operand" "")
 310         (match_operand:VSX_M 1 "vsx_register_operand" ""))]
 311   "VECTOR_MEM_VSX_P (<MODE>mode)"
 312   "")
 313
 314 \f
 315 ;; VSX scalar and vector floating point arithmetic instructions
 316 (define_insn "*vsx_add<mode>3"
 317   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 318         (plus:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 319                     (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")))]
 320   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 321   "x<VSv>add<VSs> %x0,%x1,%x2"
 322   [(set_attr "type" "<VStype_simple>")
 323    (set_attr "fp_type" "<VSfptype_simple>")])
 324
 325 (define_insn "*vsx_sub<mode>3"
 326   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 327         (minus:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 328                      (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")))]
 329   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 330   "x<VSv>sub<VSs> %x0,%x1,%x2"
 331   [(set_attr "type" "<VStype_simple>")
 332    (set_attr "fp_type" "<VSfptype_simple>")])
 333
 334 (define_insn "*vsx_mul<mode>3"
 335   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 336         (mult:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 337                     (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")))]
 338   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 339   "x<VSv>mul<VSs> %x0,%x1,%x2"
 340   [(set_attr "type" "<VStype_mul>")
 341    (set_attr "fp_type" "<VSfptype_mul>")])
 342
 343 (define_insn "*vsx_div<mode>3"
 344   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 345         (div:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 346                    (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")))]
 347   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 348   "x<VSv>div<VSs> %x0,%x1,%x2"
 349   [(set_attr "type" "<VStype_div>")
 350    (set_attr "fp_type" "<VSfptype_div>")])
 351
 352 ;; *tdiv* instruction returning the FG flag
 353 (define_expand "vsx_tdiv<mode>3_fg"
 354   [(set (match_dup 3)
 355         (unspec:CCFP [(match_operand:VSX_B 1 "vsx_register_operand" "")
 356                       (match_operand:VSX_B 2 "vsx_register_operand" "")]
 357                      UNSPEC_VSX_TDIV))
 358    (set (match_operand:SI 0 "gpc_reg_operand" "")
 359         (gt:SI (match_dup 3)
 360                (const_int 0)))]
 361   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 362 {
 363   operands[3] = gen_reg_rtx (CCFPmode);
 364 })
 365
 366 ;; *tdiv* instruction returning the FE flag
 367 (define_expand "vsx_tdiv<mode>3_fe"
 368   [(set (match_dup 3)
 369         (unspec:CCFP [(match_operand:VSX_B 1 "vsx_register_operand" "")
 370                       (match_operand:VSX_B 2 "vsx_register_operand" "")]
 371                      UNSPEC_VSX_TDIV))
 372    (set (match_operand:SI 0 "gpc_reg_operand" "")
 373         (eq:SI (match_dup 3)
 374                (const_int 0)))]
 375   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 376 {
 377   operands[3] = gen_reg_rtx (CCFPmode);
 378 })
 379
 380 (define_insn "*vsx_tdiv<mode>3_internal"
 381   [(set (match_operand:CCFP 0 "cc_reg_operand" "=x,x")
 382         (unspec:CCFP [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 383                       (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")]
 384                    UNSPEC_VSX_TDIV))]
 385   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 386   "x<VSv>tdiv<VSs> %0,%x1,%x2"
 387   [(set_attr "type" "<VStype_simple>")
 388    (set_attr "fp_type" "<VSfptype_simple>")])
 389
 390 (define_insn "vsx_fre<mode>2"
 391   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 392         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 393                       UNSPEC_FRES))]
 394   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 395   "x<VSv>re<VSs> %x0,%x1"
 396   [(set_attr "type" "<VStype_simple>")
 397    (set_attr "fp_type" "<VSfptype_simple>")])
 398
 399 (define_insn "*vsx_neg<mode>2"
 400   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 401         (neg:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")))]
 402   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 403   "x<VSv>neg<VSs> %x0,%x1"
 404   [(set_attr "type" "<VStype_simple>")
 405    (set_attr "fp_type" "<VSfptype_simple>")])
 406
 407 (define_insn "*vsx_abs<mode>2"
 408   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 409         (abs:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")))]
 410   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 411   "x<VSv>abs<VSs> %x0,%x1"
 412   [(set_attr "type" "<VStype_simple>")
 413    (set_attr "fp_type" "<VSfptype_simple>")])
 414
 415 (define_insn "vsx_nabs<mode>2"
 416   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 417         (neg:VSX_B
 418          (abs:VSX_B
 419           (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa"))))]
 420   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 421   "x<VSv>nabs<VSs> %x0,%x1"
 422   [(set_attr "type" "<VStype_simple>")
 423    (set_attr "fp_type" "<VSfptype_simple>")])
 424
 425 (define_insn "vsx_smax<mode>3"
 426   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 427         (smax:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 428                     (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")))]
 429   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 430   "x<VSv>max<VSs> %x0,%x1,%x2"
 431   [(set_attr "type" "<VStype_simple>")
 432    (set_attr "fp_type" "<VSfptype_simple>")])
 433
 434 (define_insn "*vsx_smin<mode>3"
 435   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 436         (smin:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 437                     (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")))]
 438   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 439   "x<VSv>min<VSs> %x0,%x1,%x2"
 440   [(set_attr "type" "<VStype_simple>")
 441    (set_attr "fp_type" "<VSfptype_simple>")])
 442
 443 ;; Special VSX version of smin/smax for single precision floating point.  Since
 444 ;; both numbers are rounded to single precision, we can just use the DP version
 445 ;; of the instruction.
 446
 447 (define_insn "*vsx_smaxsf3"
 448   [(set (match_operand:SF 0 "vsx_register_operand" "=f")
 449         (smax:SF (match_operand:SF 1 "vsx_register_operand" "f")
 450                  (match_operand:SF 2 "vsx_register_operand" "f")))]
 451   "VECTOR_UNIT_VSX_P (DFmode)"
 452   "xsmaxdp %x0,%x1,%x2"
 453   [(set_attr "type" "fp")
 454    (set_attr "fp_type" "fp_addsub_d")])
 455
 456 (define_insn "*vsx_sminsf3"
 457   [(set (match_operand:SF 0 "vsx_register_operand" "=f")
 458         (smin:SF (match_operand:SF 1 "vsx_register_operand" "f")
 459                  (match_operand:SF 2 "vsx_register_operand" "f")))]
 460   "VECTOR_UNIT_VSX_P (DFmode)"
 461   "xsmindp %x0,%x1,%x2"
 462   [(set_attr "type" "fp")
 463    (set_attr "fp_type" "fp_addsub_d")])
 464
 465 (define_insn "*vsx_sqrt<mode>2"
 466   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 467         (sqrt:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")))]
 468   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 469   "x<VSv>sqrt<VSs> %x0,%x1"
 470   [(set_attr "type" "<VStype_sqrt>")
 471    (set_attr "fp_type" "<VSfptype_sqrt>")])
 472
 473 (define_insn "*vsx_rsqrte<mode>2"
 474   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 475         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 476                       UNSPEC_RSQRT))]
 477   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 478   "x<VSv>rsqrte<VSs> %x0,%x1"
 479   [(set_attr "type" "<VStype_simple>")
 480    (set_attr "fp_type" "<VSfptype_simple>")])
 481
 482 ;; *tsqrt* returning the fg flag
 483 (define_expand "vsx_tsqrt<mode>2_fg"
 484   [(set (match_dup 3)
 485         (unspec:CCFP [(match_operand:VSX_B 1 "vsx_register_operand" "")]
 486                      UNSPEC_VSX_TSQRT))
 487    (set (match_operand:SI 0 "gpc_reg_operand" "")
 488         (gt:SI (match_dup 3)
 489                (const_int 0)))]
 490   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 491 {
 492   operands[3] = gen_reg_rtx (CCFPmode);
 493 })
 494
 495 ;; *tsqrt* returning the fe flag
 496 (define_expand "vsx_tsqrt<mode>2_fe"
 497   [(set (match_dup 3)
 498         (unspec:CCFP [(match_operand:VSX_B 1 "vsx_register_operand" "")]
 499                      UNSPEC_VSX_TSQRT))
 500    (set (match_operand:SI 0 "gpc_reg_operand" "")
 501         (eq:SI (match_dup 3)
 502                (const_int 0)))]
 503   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 504 {
 505   operands[3] = gen_reg_rtx (CCFPmode);
 506 })
 507
 508 (define_insn "*vsx_tsqrt<mode>2_internal"
 509   [(set (match_operand:CCFP 0 "cc_reg_operand" "=x,x")
 510         (unspec:CCFP [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 511                      UNSPEC_VSX_TSQRT))]
 512   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 513   "x<VSv>tsqrt<VSs> %0,%x1"
 514   [(set_attr "type" "<VStype_simple>")
 515    (set_attr "fp_type" "<VSfptype_simple>")])
 516
 517 ;; Fused vector multiply/add instructions Support the classical DF versions of
 518 ;; fma, which allows the target to be a separate register from the 3 inputs.
 519 ;; Under VSX, the target must be either the addend or the first multiply.
 520 ;; Where we can, also do the same for the Altivec V4SF fmas.
 521
 522 (define_insn "*vsx_fmadf4"
 523   [(set (match_operand:DF 0 "vsx_register_operand" "=ws,ws,?wa,?wa,d")
 524         (fma:DF
 525           (match_operand:DF 1 "vsx_register_operand" "%ws,ws,wa,wa,d")
 526           (match_operand:DF 2 "vsx_register_operand" "ws,0,wa,0,d")
 527           (match_operand:DF 3 "vsx_register_operand" "0,ws,0,wa,d")))]
 528   "VECTOR_UNIT_VSX_P (DFmode)"
 529   "@
 530    xsmaddadp %x0,%x1,%x2
 531    xsmaddmdp %x0,%x1,%x3
 532    xsmaddadp %x0,%x1,%x2
 533    xsmaddmdp %x0,%x1,%x3
 534    fmadd %0,%1,%2,%3"
 535   [(set_attr "type" "fp")
 536    (set_attr "fp_type" "fp_maddsub_d")])
 537
 538 (define_insn "*vsx_fmav4sf4"
 539   [(set (match_operand:V4SF 0 "vsx_register_operand" "=ws,ws,?wa,?wa,v")
 540         (fma:V4SF
 541           (match_operand:V4SF 1 "vsx_register_operand" "%ws,ws,wa,wa,v")
 542           (match_operand:V4SF 2 "vsx_register_operand" "ws,0,wa,0,v")
 543           (match_operand:V4SF 3 "vsx_register_operand" "0,ws,0,wa,v")))]
 544   "VECTOR_UNIT_VSX_P (V4SFmode)"
 545   "@
 546    xvmaddasp %x0,%x1,%x2
 547    xvmaddmsp %x0,%x1,%x3
 548    xvmaddasp %x0,%x1,%x2
 549    xvmaddmsp %x0,%x1,%x3
 550    vmaddfp %0,%1,%2,%3"
 551   [(set_attr "type" "vecfloat")])
 552
 553 (define_insn "*vsx_fmav2df4"
 554   [(set (match_operand:V2DF 0 "vsx_register_operand" "=ws,ws,?wa,?wa")
 555         (fma:V2DF
 556           (match_operand:V2DF 1 "vsx_register_operand" "%ws,ws,wa,wa")
 557           (match_operand:V2DF 2 "vsx_register_operand" "ws,0,wa,0")
 558           (match_operand:V2DF 3 "vsx_register_operand" "0,ws,0,wa")))]
 559   "VECTOR_UNIT_VSX_P (V2DFmode)"
 560   "@
 561    xvmaddadp %x0,%x1,%x2
 562    xvmaddmdp %x0,%x1,%x3
 563    xvmaddadp %x0,%x1,%x2
 564    xvmaddmdp %x0,%x1,%x3"
 565   [(set_attr "type" "vecdouble")])
 566
 567 (define_insn "*vsx_fmsdf4"
 568   [(set (match_operand:DF 0 "vsx_register_operand" "=ws,ws,?wa,?wa,d")
 569         (fma:DF
 570           (match_operand:DF 1 "vsx_register_operand" "%ws,ws,wa,wa,d")
 571           (match_operand:DF 2 "vsx_register_operand" "ws,0,wa,0,d")
 572           (neg:DF
 573             (match_operand:DF 3 "vsx_register_operand" "0,ws,0,wa,d"))))]
 574   "VECTOR_UNIT_VSX_P (DFmode)"
 575   "@
 576    xsmsubadp %x0,%x1,%x2
 577    xsmsubmdp %x0,%x1,%x3
 578    xsmsubadp %x0,%x1,%x2
 579    xsmsubmdp %x0,%x1,%x3
 580    fmsub %0,%1,%2,%3"
 581   [(set_attr "type" "fp")
 582    (set_attr "fp_type" "fp_maddsub_d")])
 583
 584 (define_insn "*vsx_fms<mode>4"
 585   [(set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,<VSr>,?wa,?wa")
 586         (fma:VSX_F
 587           (match_operand:VSX_F 1 "vsx_register_operand" "%<VSr>,<VSr>,wa,wa")
 588           (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,0,wa,0")
 589           (neg:VSX_F
 590             (match_operand:VSX_F 3 "vsx_register_operand" "0,<VSr>,0,wa"))))]
 591   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 592   "@
 593    x<VSv>msuba<VSs> %x0,%x1,%x2
 594    x<VSv>msubm<VSs> %x0,%x1,%x3
 595    x<VSv>msuba<VSs> %x0,%x1,%x2
 596    x<VSv>msubm<VSs> %x0,%x1,%x3"
 597   [(set_attr "type" "<VStype_mul>")])
 598
 599 (define_insn "*vsx_nfmadf4"
 600   [(set (match_operand:DF 0 "vsx_register_operand" "=ws,ws,?wa,?wa,d")
 601         (neg:DF
 602          (fma:DF
 603           (match_operand:DF 1 "vsx_register_operand" "ws,ws,wa,wa,d")
 604           (match_operand:DF 2 "vsx_register_operand" "ws,0,wa,0,d")
 605           (match_operand:DF 3 "vsx_register_operand" "0,ws,0,wa,d"))))]
 606   "VECTOR_UNIT_VSX_P (DFmode)"
 607   "@
 608    xsnmaddadp %x0,%x1,%x2
 609    xsnmaddmdp %x0,%x1,%x3
 610    xsnmaddadp %x0,%x1,%x2
 611    xsnmaddmdp %x0,%x1,%x3
 612    fnmadd %0,%1,%2,%3"
 613   [(set_attr "type" "fp")
 614    (set_attr "fp_type" "fp_maddsub_d")])
 615
 616 (define_insn "*vsx_nfma<mode>4"
 617   [(set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,<VSr>,?wa,?wa")
 618         (neg:VSX_F
 619          (fma:VSX_F
 620           (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,<VSr>,wa,wa")
 621           (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,0,wa,0")
 622           (match_operand:VSX_F 3 "vsx_register_operand" "0,<VSr>,0,wa"))))]
 623   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 624   "@
 625    x<VSv>nmadda<VSs> %x0,%x1,%x2
 626    x<VSv>nmaddm<VSs> %x0,%x1,%x3
 627    x<VSv>nmadda<VSs> %x0,%x1,%x2
 628    x<VSv>nmaddm<VSs> %x0,%x1,%x3"
 629   [(set_attr "type" "<VStype_mul>")
 630    (set_attr "fp_type" "<VSfptype_mul>")])
 631
 632 (define_insn "*vsx_nfmsdf4"
 633   [(set (match_operand:DF 0 "vsx_register_operand" "=ws,ws,?wa,?wa,d")
 634         (neg:DF
 635          (fma:DF
 636            (match_operand:DF 1 "vsx_register_operand" "%ws,ws,wa,wa,d")
 637            (match_operand:DF 2 "vsx_register_operand" "ws,0,wa,0,d")
 638            (neg:DF
 639              (match_operand:DF 3 "vsx_register_operand" "0,ws,0,wa,d")))))]
 640   "VECTOR_UNIT_VSX_P (DFmode)"
 641   "@
 642    xsnmsubadp %x0,%x1,%x2
 643    xsnmsubmdp %x0,%x1,%x3
 644    xsnmsubadp %x0,%x1,%x2
 645    xsnmsubmdp %x0,%x1,%x3
 646    fnmsub %0,%1,%2,%3"
 647   [(set_attr "type" "fp")
 648    (set_attr "fp_type" "fp_maddsub_d")])
 649
 650 (define_insn "*vsx_nfmsv4sf4"
 651   [(set (match_operand:V4SF 0 "vsx_register_operand" "=wf,wf,?wa,?wa,v")
 652         (neg:V4SF
 653          (fma:V4SF
 654            (match_operand:V4SF 1 "vsx_register_operand" "%wf,wf,wa,wa,v")
 655            (match_operand:V4SF 2 "vsx_register_operand" "wf,0,wa,0,v")
 656            (neg:V4SF
 657              (match_operand:V4SF 3 "vsx_register_operand" "0,wf,0,wa,v")))))]
 658   "VECTOR_UNIT_VSX_P (V4SFmode)"
 659   "@
 660    xvnmsubasp %x0,%x1,%x2
 661    xvnmsubmsp %x0,%x1,%x3
 662    xvnmsubasp %x0,%x1,%x2
 663    xvnmsubmsp %x0,%x1,%x3
 664    vnmsubfp %0,%1,%2,%3"
 665   [(set_attr "type" "vecfloat")])
 666
 667 (define_insn "*vsx_nfmsv2df4"
 668   [(set (match_operand:V2DF 0 "vsx_register_operand" "=wd,wd,?wa,?wa")
 669         (neg:V2DF
 670          (fma:V2DF
 671            (match_operand:V2DF 1 "vsx_register_operand" "%wd,wd,wa,wa")
 672            (match_operand:V2DF 2 "vsx_register_operand" "wd,0,wa,0")
 673            (neg:V2DF
 674              (match_operand:V2DF 3 "vsx_register_operand" "0,wd,0,wa")))))]
 675   "VECTOR_UNIT_VSX_P (V2DFmode)"
 676   "@
 677    xvnmsubadp %x0,%x1,%x2
 678    xvnmsubmdp %x0,%x1,%x3
 679    xvnmsubadp %x0,%x1,%x2
 680    xvnmsubmdp %x0,%x1,%x3"
 681   [(set_attr "type" "vecdouble")])
 682
 683 ;; Vector conditional expressions (no scalar version for these instructions)
 684 (define_insn "vsx_eq<mode>"
 685   [(set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,?wa")
 686         (eq:VSX_F (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,wa")
 687                   (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,wa")))]
 688   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 689   "xvcmpeq<VSs> %x0,%x1,%x2"
 690   [(set_attr "type" "<VStype_simple>")
 691    (set_attr "fp_type" "<VSfptype_simple>")])
 692
 693 (define_insn "vsx_gt<mode>"
 694   [(set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,?wa")
 695         (gt:VSX_F (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,wa")
 696                   (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,wa")))]
 697   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 698   "xvcmpgt<VSs> %x0,%x1,%x2"
 699   [(set_attr "type" "<VStype_simple>")
 700    (set_attr "fp_type" "<VSfptype_simple>")])
 701
 702 (define_insn "*vsx_ge<mode>"
 703   [(set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,?wa")
 704         (ge:VSX_F (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,wa")
 705                   (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,wa")))]
 706   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 707   "xvcmpge<VSs> %x0,%x1,%x2"
 708   [(set_attr "type" "<VStype_simple>")
 709    (set_attr "fp_type" "<VSfptype_simple>")])
 710
 711 ;; Floating point scalar compare
 712 (define_insn "*vsx_cmpdf_internal1"
 713   [(set (match_operand:CCFP 0 "cc_reg_operand" "=y,?y")
 714         (compare:CCFP (match_operand:DF 1 "gpc_reg_operand" "ws,wa")
 715                       (match_operand:DF 2 "gpc_reg_operand" "ws,wa")))]
 716   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
 717    && VECTOR_UNIT_VSX_P (DFmode)"
 718   "xscmpudp %0,%x1,%x2"
 719   [(set_attr "type" "fpcompare")])
 720
 721 ;; Compare vectors producing a vector result and a predicate, setting CR6 to
 722 ;; indicate a combined status
 723 (define_insn "*vsx_eq_<mode>_p"
 724   [(set (reg:CC 74)
 725         (unspec:CC
 726          [(eq:CC (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,?wa")
 727                  (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,?wa"))]
 728          UNSPEC_PREDICATE))
 729    (set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,?wa")
 730         (eq:VSX_F (match_dup 1)
 731                   (match_dup 2)))]
 732   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 733   "xvcmpeq<VSs>. %x0,%x1,%x2"
 734   [(set_attr "type" "<VStype_simple>")])
 735
 736 (define_insn "*vsx_gt_<mode>_p"
 737   [(set (reg:CC 74)
 738         (unspec:CC
 739          [(gt:CC (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,?wa")
 740                  (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,?wa"))]
 741          UNSPEC_PREDICATE))
 742    (set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,?wa")
 743         (gt:VSX_F (match_dup 1)
 744                   (match_dup 2)))]
 745   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 746   "xvcmpgt<VSs>. %x0,%x1,%x2"
 747   [(set_attr "type" "<VStype_simple>")])
 748
 749 (define_insn "*vsx_ge_<mode>_p"
 750   [(set (reg:CC 74)
 751         (unspec:CC
 752          [(ge:CC (match_operand:VSX_F 1 "vsx_register_operand" "<VSr>,?wa")
 753                  (match_operand:VSX_F 2 "vsx_register_operand" "<VSr>,?wa"))]
 754          UNSPEC_PREDICATE))
 755    (set (match_operand:VSX_F 0 "vsx_register_operand" "=<VSr>,?wa")
 756         (ge:VSX_F (match_dup 1)
 757                   (match_dup 2)))]
 758   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 759   "xvcmpge<VSs>. %x0,%x1,%x2"
 760   [(set_attr "type" "<VStype_simple>")])
 761
 762 ;; Vector select
 763 (define_insn "*vsx_xxsel<mode>"
 764   [(set (match_operand:VSX_L 0 "vsx_register_operand" "=<VSr>,?wa")
 765         (if_then_else:VSX_L
 766          (ne:CC (match_operand:VSX_L 1 "vsx_register_operand" "<VSr>,wa")
 767                 (match_operand:VSX_L 4 "zero_constant" ""))
 768          (match_operand:VSX_L 2 "vsx_register_operand" "<VSr>,wa")
 769          (match_operand:VSX_L 3 "vsx_register_operand" "<VSr>,wa")))]
 770   "VECTOR_MEM_VSX_P (<MODE>mode)"
 771   "xxsel %x0,%x3,%x2,%x1"
 772   [(set_attr "type" "vecperm")])
 773
 774 (define_insn "*vsx_xxsel<mode>_uns"
 775   [(set (match_operand:VSX_L 0 "vsx_register_operand" "=<VSr>,?wa")
 776         (if_then_else:VSX_L
 777          (ne:CCUNS (match_operand:VSX_L 1 "vsx_register_operand" "<VSr>,wa")
 778                    (match_operand:VSX_L 4 "zero_constant" ""))
 779          (match_operand:VSX_L 2 "vsx_register_operand" "<VSr>,wa")
 780          (match_operand:VSX_L 3 "vsx_register_operand" "<VSr>,wa")))]
 781   "VECTOR_MEM_VSX_P (<MODE>mode)"
 782   "xxsel %x0,%x3,%x2,%x1"
 783   [(set_attr "type" "vecperm")])
 784
 785 ;; Copy sign
 786 (define_insn "vsx_copysign<mode>3"
 787   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 788         (unspec:VSX_B
 789          [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")
 790           (match_operand:VSX_B 2 "vsx_register_operand" "<VSr>,wa")]
 791          UNSPEC_COPYSIGN))]
 792   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 793   "x<VSv>cpsgn<VSs> %x0,%x2,%x1"
 794   [(set_attr "type" "<VStype_simple>")
 795    (set_attr "fp_type" "<VSfptype_simple>")])
 796
 797 ;; For the conversions, limit the register class for the integer value to be
 798 ;; the fprs because we don't want to add the altivec registers to movdi/movsi.
 799 ;; For the unsigned tests, there isn't a generic double -> unsigned conversion
 800 ;; in rs6000.md so don't test VECTOR_UNIT_VSX_P, just test against VSX.
 801 ;; Don't use vsx_register_operand here, use gpc_reg_operand to match rs6000.md.
 802 (define_insn "vsx_float<VSi><mode>2"
 803   [(set (match_operand:VSX_B 0 "gpc_reg_operand" "=<VSr>,?wa")
 804         (float:VSX_B (match_operand:<VSI> 1 "gpc_reg_operand" "<VSr2>,<VSr3>")))]
 805   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 806   "x<VSv>cvsx<VSc><VSs> %x0,%x1"
 807   [(set_attr "type" "<VStype_simple>")
 808    (set_attr "fp_type" "<VSfptype_simple>")])
 809
 810 (define_insn "vsx_floatuns<VSi><mode>2"
 811   [(set (match_operand:VSX_B 0 "gpc_reg_operand" "=<VSr>,?wa")
 812         (unsigned_float:VSX_B (match_operand:<VSI> 1 "gpc_reg_operand" "<VSr2>,<VSr3>")))]
 813   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 814   "x<VSv>cvux<VSc><VSs> %x0,%x1"
 815   [(set_attr "type" "<VStype_simple>")
 816    (set_attr "fp_type" "<VSfptype_simple>")])
 817
 818 (define_insn "vsx_fix_trunc<mode><VSi>2"
 819   [(set (match_operand:<VSI> 0 "gpc_reg_operand" "=<VSr2>,?<VSr3>")
 820         (fix:<VSI> (match_operand:VSX_B 1 "gpc_reg_operand" "<VSr>,wa")))]
 821   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 822   "x<VSv>cv<VSs>sx<VSc>s %x0,%x1"
 823   [(set_attr "type" "<VStype_simple>")
 824    (set_attr "fp_type" "<VSfptype_simple>")])
 825
 826 (define_insn "vsx_fixuns_trunc<mode><VSi>2"
 827   [(set (match_operand:<VSI> 0 "gpc_reg_operand" "=<VSr2>,?<VSr3>")
 828         (unsigned_fix:<VSI> (match_operand:VSX_B 1 "gpc_reg_operand" "<VSr>,wa")))]
 829   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 830   "x<VSv>cv<VSs>ux<VSc>s %x0,%x1"
 831   [(set_attr "type" "<VStype_simple>")
 832    (set_attr "fp_type" "<VSfptype_simple>")])
 833
 834 ;; Math rounding functions
 835 (define_insn "vsx_x<VSv>r<VSs>i"
 836   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 837         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 838                       UNSPEC_VSX_ROUND_I))]
 839   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 840   "x<VSv>r<VSs>i %x0,%x1"
 841   [(set_attr "type" "<VStype_simple>")
 842    (set_attr "fp_type" "<VSfptype_simple>")])
 843
 844 (define_insn "vsx_x<VSv>r<VSs>ic"
 845   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 846         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 847                       UNSPEC_VSX_ROUND_IC))]
 848   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 849   "x<VSv>r<VSs>ic %x0,%x1"
 850   [(set_attr "type" "<VStype_simple>")
 851    (set_attr "fp_type" "<VSfptype_simple>")])
 852
 853 (define_insn "vsx_btrunc<mode>2"
 854   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 855         (fix:VSX_B (match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")))]
 856   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 857   "x<VSv>r<VSs>iz %x0,%x1"
 858   [(set_attr "type" "<VStype_simple>")
 859    (set_attr "fp_type" "<VSfptype_simple>")])
 860
 861 (define_insn "*vsx_b2trunc<mode>2"
 862   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 863         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 864                       UNSPEC_FRIZ))]
 865   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 866   "x<VSv>r<VSs>iz %x0,%x1"
 867   [(set_attr "type" "<VStype_simple>")
 868    (set_attr "fp_type" "<VSfptype_simple>")])
 869
 870 (define_insn "vsx_floor<mode>2"
 871   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 872         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 873                       UNSPEC_FRIM))]
 874   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 875   "x<VSv>r<VSs>im %x0,%x1"
 876   [(set_attr "type" "<VStype_simple>")
 877    (set_attr "fp_type" "<VSfptype_simple>")])
 878
 879 (define_insn "vsx_ceil<mode>2"
 880   [(set (match_operand:VSX_B 0 "vsx_register_operand" "=<VSr>,?wa")
 881         (unspec:VSX_B [(match_operand:VSX_B 1 "vsx_register_operand" "<VSr>,wa")]
 882                       UNSPEC_FRIP))]
 883   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 884   "x<VSv>r<VSs>ip %x0,%x1"
 885   [(set_attr "type" "<VStype_simple>")
 886    (set_attr "fp_type" "<VSfptype_simple>")])
 887
 888 \f
 889 ;; VSX convert to/from double vector
 890
 891 ;; Convert between single and double precision
 892 ;; Don't use xscvspdp and xscvdpsp for scalar conversions, since the normal
 893 ;; scalar single precision instructions internally use the double format.
 894 ;; Prefer the altivec registers, since we likely will need to do a vperm
 895 (define_insn "vsx_<VS_spdp_insn>"
 896   [(set (match_operand:<VS_spdp_res> 0 "vsx_register_operand" "=<VSr4>,?wa")
 897         (unspec:<VS_spdp_res> [(match_operand:VSX_SPDP 1 "vsx_register_operand" "<VSr5>,wa")]
 898                               UNSPEC_VSX_CVSPDP))]
 899   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 900   "<VS_spdp_insn> %x0,%x1"
 901   [(set_attr "type" "<VS_spdp_type>")])
 902
 903 ;; xscvspdp, represent the scalar SF type as V4SF
 904 (define_insn "vsx_xscvspdp"
 905   [(set (match_operand:DF 0 "vsx_register_operand" "=ws,?wa")
 906         (unspec:DF [(match_operand:V4SF 1 "vsx_register_operand" "wa,wa")]
 907                    UNSPEC_VSX_CVSPDP))]
 908   "VECTOR_UNIT_VSX_P (V4SFmode)"
 909   "xscvspdp %x0,%x1"
 910   [(set_attr "type" "fp")])
 911
 912 ;; xscvdpsp used for splat'ing a scalar to V4SF, knowing that the internal SF
 913 ;; format of scalars is actually DF.
 914 (define_insn "vsx_xscvdpsp_scalar"
 915   [(set (match_operand:V4SF 0 "vsx_register_operand" "=wa")
 916         (unspec:V4SF [(match_operand:SF 1 "vsx_register_operand" "f")]
 917                      UNSPEC_VSX_CVSPDP))]
 918   "VECTOR_UNIT_VSX_P (V4SFmode)"
 919   "xscvdpsp %x0,%x1"
 920   [(set_attr "type" "fp")])
 921
 922 ;; Same as vsx_xscvspdp, but use SF as the type
 923 (define_insn "vsx_xscvspdp_scalar2"
 924   [(set (match_operand:SF 0 "vsx_register_operand" "=f")
 925         (unspec:SF [(match_operand:V4SF 1 "vsx_register_operand" "wa")]
 926                    UNSPEC_VSX_CVSPDP))]
 927   "VECTOR_UNIT_VSX_P (V4SFmode)"
 928   "xscvspdp %x0,%x1"
 929   [(set_attr "type" "fp")])
 930
 931 ;; ISA 2.07 xscvdpspn/xscvspdpn that does not raise an error on signalling NaNs
 932 (define_insn "vsx_xscvdpspn"
 933   [(set (match_operand:V4SF 0 "vsx_register_operand" "=ws,?wa")
 934         (unspec:V4SF [(match_operand:DF 1 "vsx_register_operand" "wd,wa")]
 935                      UNSPEC_VSX_CVDPSPN))]
 936   "TARGET_XSCVDPSPN"
 937   "xscvdpspn %x0,%x1"
 938   [(set_attr "type" "fp")])
 939
 940 (define_insn "vsx_xscvspdpn"
 941   [(set (match_operand:DF 0 "vsx_register_operand" "=ws,?wa")
 942         (unspec:DF [(match_operand:V4SF 1 "vsx_register_operand" "wa,wa")]
 943                    UNSPEC_VSX_CVSPDPN))]
 944   "TARGET_XSCVSPDPN"
 945   "xscvspdpn %x0,%x1"
 946   [(set_attr "type" "fp")])
 947
 948 (define_insn "vsx_xscvdpspn_scalar"
 949   [(set (match_operand:V4SF 0 "vsx_register_operand" "=wa")
 950         (unspec:V4SF [(match_operand:SF 1 "vsx_register_operand" "f")]
 951                      UNSPEC_VSX_CVDPSPN))]
 952   "TARGET_XSCVDPSPN"
 953   "xscvdpspn %x0,%x1"
 954   [(set_attr "type" "fp")])
 955
 956 ;; Used by direct move to move a SFmode value from GPR to VSX register
 957 (define_insn "vsx_xscvspdpn_directmove"
 958   [(set (match_operand:SF 0 "vsx_register_operand" "=wa")
 959         (unspec:SF [(match_operand:SF 1 "vsx_register_operand" "wa")]
 960                    UNSPEC_VSX_CVSPDPN))]
 961   "TARGET_XSCVSPDPN"
 962   "xscvspdpn %x0,%x1"
 963   [(set_attr "type" "fp")])
 964
 965 ;; Convert from 64-bit to 32-bit types
 966 ;; Note, favor the Altivec registers since the usual use of these instructions
 967 ;; is in vector converts and we need to use the Altivec vperm instruction.
 968
 969 (define_insn "vsx_xvcvdpsxws"
 970   [(set (match_operand:V4SI 0 "vsx_register_operand" "=v,?wa")
 971         (unspec:V4SI [(match_operand:V2DF 1 "vsx_register_operand" "wd,wa")]
 972                      UNSPEC_VSX_CVDPSXWS))]
 973   "VECTOR_UNIT_VSX_P (V2DFmode)"
 974   "xvcvdpsxws %x0,%x1"
 975   [(set_attr "type" "vecdouble")])
 976
 977 (define_insn "vsx_xvcvdpuxws"
 978   [(set (match_operand:V4SI 0 "vsx_register_operand" "=v,?wa")
 979         (unspec:V4SI [(match_operand:V2DF 1 "vsx_register_operand" "wd,wa")]
 980                      UNSPEC_VSX_CVDPUXWS))]
 981   "VECTOR_UNIT_VSX_P (V2DFmode)"
 982   "xvcvdpuxws %x0,%x1"
 983   [(set_attr "type" "vecdouble")])
 984
 985 (define_insn "vsx_xvcvsxdsp"
 986   [(set (match_operand:V4SI 0 "vsx_register_operand" "=wd,?wa")
 987         (unspec:V4SI [(match_operand:V2DF 1 "vsx_register_operand" "wf,wa")]
 988                      UNSPEC_VSX_CVSXDSP))]
 989   "VECTOR_UNIT_VSX_P (V2DFmode)"
 990   "xvcvsxdsp %x0,%x1"
 991   [(set_attr "type" "vecfloat")])
 992
 993 (define_insn "vsx_xvcvuxdsp"
 994   [(set (match_operand:V4SI 0 "vsx_register_operand" "=wd,?wa")
 995         (unspec:V4SI [(match_operand:V2DF 1 "vsx_register_operand" "wf,wa")]
 996                      UNSPEC_VSX_CVUXDSP))]
 997   "VECTOR_UNIT_VSX_P (V2DFmode)"
 998   "xvcvuxwdp %x0,%x1"
 999   [(set_attr "type" "vecdouble")])
1000
1001 ;; Convert from 32-bit to 64-bit types
1002 (define_insn "vsx_xvcvsxwdp"
1003   [(set (match_operand:V2DF 0 "vsx_register_operand" "=wd,?wa")
1004         (unspec:V2DF [(match_operand:V4SI 1 "vsx_register_operand" "wf,wa")]
1005                      UNSPEC_VSX_CVSXWDP))]
1006   "VECTOR_UNIT_VSX_P (V2DFmode)"
1007   "xvcvsxwdp %x0,%x1"
1008   [(set_attr "type" "vecdouble")])
1009
1010 (define_insn "vsx_xvcvuxwdp"
1011   [(set (match_operand:V2DF 0 "vsx_register_operand" "=wd,?wa")
1012         (unspec:V2DF [(match_operand:V4SI 1 "vsx_register_operand" "wf,wa")]
1013                      UNSPEC_VSX_CVUXWDP))]
1014   "VECTOR_UNIT_VSX_P (V2DFmode)"
1015   "xvcvuxwdp %x0,%x1"
1016   [(set_attr "type" "vecdouble")])
1017
1018 (define_insn "vsx_xvcvspsxds"
1019   [(set (match_operand:V2DI 0 "vsx_register_operand" "=v,?wa")
1020         (unspec:V2DI [(match_operand:V4SF 1 "vsx_register_operand" "wd,wa")]
1021                      UNSPEC_VSX_CVSPSXDS))]
1022   "VECTOR_UNIT_VSX_P (V2DFmode)"
1023   "xvcvspsxds %x0,%x1"
1024   [(set_attr "type" "vecdouble")])
1025
1026 (define_insn "vsx_xvcvspuxds"
1027   [(set (match_operand:V2DI 0 "vsx_register_operand" "=v,?wa")
1028         (unspec:V2DI [(match_operand:V4SF 1 "vsx_register_operand" "wd,wa")]
1029                      UNSPEC_VSX_CVSPUXDS))]
1030   "VECTOR_UNIT_VSX_P (V2DFmode)"
1031   "xvcvspuxds %x0,%x1"
1032   [(set_attr "type" "vecdouble")])
1033
1034 ;; Only optimize (float (fix x)) -> frz if we are in fast-math mode, since
1035 ;; since the xsrdpiz instruction does not truncate the value if the floating
1036 ;; point value is < LONG_MIN or > LONG_MAX.
1037 (define_insn "*vsx_float_fix_<mode>2"
1038   [(set (match_operand:VSX_DF 0 "vsx_register_operand" "=<VSr>,?wa")
1039         (float:VSX_DF
1040          (fix:<VSI>
1041           (match_operand:VSX_DF 1 "vsx_register_operand" "<VSr>,?wa"))))]
1042   "TARGET_HARD_FLOAT && TARGET_FPRS && TARGET_DOUBLE_FLOAT
1043    && VECTOR_UNIT_VSX_P (<MODE>mode) && flag_unsafe_math_optimizations
1044    && !flag_trapping_math && TARGET_FRIZ"
1045   "x<VSv>r<VSs>iz %x0,%x1"
1046   [(set_attr "type" "<VStype_simple>")
1047    (set_attr "fp_type" "<VSfptype_simple>")])
1048
1049 \f
1050 ;; Logical operations.  Do not support TImode logical instructions on 32-bit at
1051 ;; present, because the compiler will see that we have a TImode and when it
1052 ;; wanted DImode, and convert the DImode to TImode, store it on the stack, and
1053 ;; load it in a VSX register or generate extra logical instructions in GPR
1054 ;; registers.
1055
1056 ;; When we are splitting the operations to GPRs, we use three alternatives, two
1057 ;; where the first/second inputs and output are in the same register, and the
1058 ;; third where the output specifies an early clobber so that we don't have to
1059 ;; worry about overlapping registers.
1060
1061 (define_insn "*vsx_and<mode>3_32bit"
1062   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1063         (and:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "%wa")
1064                     (match_operand:VSX_L2 2 "vlogical_operand" "wa")))
1065    (clobber (match_scratch:CC 3 "X"))]
1066   "!TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1067   "xxland %x0,%x1,%x2"
1068   [(set_attr "type" "vecsimple")
1069    (set_attr "length" "4")])
1070
1071 (define_insn_and_split "*vsx_and<mode>3_64bit"
1072   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,&?r")
1073         (and:VSX_L
1074          (match_operand:VSX_L 1 "vlogical_operand" "%wa,0,r,r")
1075          (match_operand:VSX_L 2 "vlogical_operand" "wa,r,0,r")))
1076    (clobber (match_scratch:CC 3 "X,X,X,X"))]
1077   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1078   "@
1079    xxland %x0,%x1,%x2
1080    #
1081    #
1082    #"
1083   "reload_completed && TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)
1084    && int_reg_operand (operands[0], <MODE>mode)"
1085   [(parallel [(set (match_dup 4) (and:DI (match_dup 5) (match_dup 6)))
1086               (clobber (match_dup 3))])
1087    (parallel [(set (match_dup 7) (and:DI (match_dup 8) (match_dup 9)))
1088               (clobber (match_dup 3))])]
1089 {
1090   operands[4] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1091   operands[5] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1092   operands[6] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1093   operands[7] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1094   operands[8] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1095   operands[9] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1096 }
1097   [(set_attr "type" "vecsimple,two,two,two")
1098    (set_attr "length" "4,8,8,8")])
1099
1100 (define_insn "*vsx_ior<mode>3_32bit"
1101   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1102         (ior:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "%wa")
1103                     (match_operand:VSX_L2 2 "vlogical_operand" "wa")))]
1104   "!TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1105   "xxlor %x0,%x1,%x2"
1106   [(set_attr "type" "vecsimple")
1107    (set_attr "length" "4")])
1108
1109 (define_insn_and_split "*vsx_ior<mode>3_64bit"
1110   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,&?r,?r,&?r")
1111         (ior:VSX_L
1112          (match_operand:VSX_L 1 "vlogical_operand" "%wa,0,r,r,0,r")
1113          (match_operand:VSX_L 2 "vsx_reg_or_cint_operand" "wa,r,0,r,n,n")))]
1114   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1115   "@
1116    xxlor %x0,%x1,%x2
1117    #
1118    #
1119    #
1120    #
1121    #"
1122   "reload_completed && TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)
1123    && int_reg_operand (operands[0], <MODE>mode)"
1124   [(const_int 0)]
1125 {
1126   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1127   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1128   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1129   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1130   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1131   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1132
1133   if (operands[5] == constm1_rtx)
1134     emit_move_insn (operands[3], constm1_rtx);
1135
1136   else if (operands[5] == const0_rtx)
1137     {
1138       if (!rtx_equal_p (operands[3], operands[4]))
1139         emit_move_insn (operands[3], operands[4]);
1140     }
1141   else
1142     emit_insn (gen_iordi3 (operands[3], operands[4], operands[5]));
1143
1144   if (operands[8] == constm1_rtx)
1145     emit_move_insn (operands[8], constm1_rtx);
1146
1147   else if (operands[8] == const0_rtx)
1148     {
1149       if (!rtx_equal_p (operands[6], operands[7]))
1150         emit_move_insn (operands[6], operands[7]);
1151     }
1152   else
1153     emit_insn (gen_iordi3 (operands[6], operands[7], operands[8]));
1154   DONE;
1155 }
1156   [(set_attr "type" "vecsimple,two,two,two,three,three")
1157    (set_attr "length" "4,8,8,8,16,16")])
1158
1159 (define_insn "*vsx_xor<mode>3_32bit"
1160   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1161         (xor:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "%wa")
1162                     (match_operand:VSX_L2 2 "vlogical_operand" "wa")))]
1163   "VECTOR_MEM_VSX_P (<MODE>mode) && !TARGET_POWERPC64"
1164   "xxlxor %x0,%x1,%x2"
1165   [(set_attr "type" "vecsimple")
1166    (set_attr "length" "4")])
1167
1168 (define_insn_and_split "*vsx_xor<mode>3_64bit"
1169   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,&?r,?r,&?r")
1170         (xor:VSX_L
1171          (match_operand:VSX_L 1 "vlogical_operand" "%wa,0,r,r,0,r")
1172          (match_operand:VSX_L 2 "vsx_reg_or_cint_operand" "wa,r,0,r,n,n")))]
1173   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1174   "@
1175    xxlxor %x0,%x1,%x2
1176    #
1177    #
1178    #
1179    #
1180    #"
1181   "reload_completed && TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)
1182    && int_reg_operand (operands[0], <MODE>mode)"
1183   [(set (match_dup 3) (xor:DI (match_dup 4) (match_dup 5)))
1184    (set (match_dup 6) (xor:DI (match_dup 7) (match_dup 8)))]
1185 {
1186   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1187   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1188   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1189   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1190   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1191   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1192 }
1193   [(set_attr "type" "vecsimple,two,two,two,three,three")
1194    (set_attr "length" "4,8,8,8,16,16")])
1195
1196 (define_insn "*vsx_one_cmpl<mode>2_32bit"
1197   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1198         (not:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "wa")))]
1199   "!TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1200   "xxlnor %x0,%x1,%x1"
1201   [(set_attr "type" "vecsimple")
1202    (set_attr "length" "4")])
1203
1204 (define_insn_and_split "*vsx_one_cmpl<mode>2_64bit"
1205   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,&?r")
1206         (not:VSX_L (match_operand:VSX_L 1 "vlogical_operand" "wa,0,r")))]
1207   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1208   "@
1209    xxlnor %x0,%x1,%x1
1210    #
1211    #"
1212   "reload_completed && TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)
1213    && int_reg_operand (operands[0], <MODE>mode)"
1214   [(set (match_dup 2) (not:DI (match_dup 3)))
1215    (set (match_dup 4) (not:DI (match_dup 5)))]
1216 {
1217   operands[2] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1218   operands[3] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1219   operands[4] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1220   operands[5] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1221 }
1222   [(set_attr "type" "vecsimple,two,two")
1223    (set_attr "length" "4,8,8")])
1224
1225 (define_insn "*vsx_nor<mode>3_32bit"
1226   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1227         (and:VSX_L2
1228          (not:VSX_L2 (match_operand:VSX_L 1 "vlogical_operand" "%wa"))
1229          (not:VSX_L2 (match_operand:VSX_L 2 "vlogical_operand" "wa"))))]
1230   "!TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1231   "xxlnor %x0,%x1,%x2"
1232   [(set_attr "type" "vecsimple")
1233    (set_attr "length" "4")])
1234
1235 (define_insn_and_split "*vsx_nor<mode>3_64bit"
1236   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,&?r")
1237         (and:VSX_L
1238          (not:VSX_L (match_operand:VSX_L 1 "vlogical_operand" "%wa,0,r,r"))
1239          (not:VSX_L (match_operand:VSX_L 2 "vlogical_operand" "wa,r,0,r"))))]
1240   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1241   "@
1242    xxlnor %x0,%x1,%x2
1243    #
1244    #
1245    #"
1246   "reload_completed && TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)
1247    && int_reg_operand (operands[0], <MODE>mode)"
1248   [(set (match_dup 3) (and:DI (not:DI (match_dup 4)) (not:DI (match_dup 5))))
1249    (set (match_dup 6) (and:DI (not:DI (match_dup 7)) (not:DI (match_dup 8))))]
1250 {
1251   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1252   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1253   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1254   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1255   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1256   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1257 }
1258   [(set_attr "type" "vecsimple,two,two,two")
1259    (set_attr "length" "4,8,8,8")])
1260
1261 (define_insn "*vsx_andc<mode>3_32bit"
1262   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1263         (and:VSX_L2
1264          (not:VSX_L2
1265           (match_operand:VSX_L2 2 "vlogical_operand" "wa"))
1266          (match_operand:VSX_L2 1 "vlogical_operand" "wa")))]
1267   "!TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1268   "xxlandc %x0,%x1,%x2"
1269   [(set_attr "type" "vecsimple")
1270    (set_attr "length" "4")])
1271
1272 (define_insn_and_split "*vsx_andc<mode>3_64bit"
1273   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,?r")
1274         (and:VSX_L
1275          (not:VSX_L
1276           (match_operand:VSX_L 2 "vlogical_operand" "wa,0,r,r"))
1277          (match_operand:VSX_L 1 "vlogical_operand" "wa,r,0,r")))]
1278   "TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)"
1279   "@
1280    xxlandc %x0,%x1,%x2
1281    #
1282    #
1283    #"
1284   "reload_completed && TARGET_POWERPC64 && VECTOR_MEM_VSX_P (<MODE>mode)
1285    && int_reg_operand (operands[0], <MODE>mode)"
1286   [(set (match_dup 3) (and:DI (not:DI (match_dup 4)) (match_dup 5)))
1287    (set (match_dup 6) (and:DI (not:DI (match_dup 7)) (match_dup 8)))]
1288 {
1289   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1290   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1291   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1292   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1293   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1294   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1295 }
1296   [(set_attr "type" "vecsimple,two,two,two")
1297    (set_attr "length" "4,8,8,8")])
1298
1299 ;; Power8 vector logical instructions.
1300 (define_insn "*vsx_eqv<mode>3_32bit"
1301   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1302         (not:VSX_L2
1303          (xor:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "wa")
1304                      (match_operand:VSX_L2 2 "vlogical_operand" "wa"))))]
1305   "!TARGET_POWERPC64 && TARGET_P8_VECTOR && VECTOR_MEM_VSX_P (<MODE>mode)"
1306   "xxleqv %x0,%x1,%x2"
1307   [(set_attr "type" "vecsimple")
1308    (set_attr "length" "4")])
1309
1310 (define_insn_and_split "*vsx_eqv<mode>3_64bit"
1311   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,?r")
1312         (not:VSX_L
1313          (xor:VSX_L (match_operand:VSX_L 1 "vlogical_operand" "wa,0,r,r")
1314                     (match_operand:VSX_L 2 "vlogical_operand" "wa,r,0,r"))))]
1315   "TARGET_POWERPC64 && TARGET_P8_VECTOR && VECTOR_MEM_VSX_P (<MODE>mode)"
1316   "@
1317    xxleqv %x0,%x1,%x2
1318    #
1319    #
1320    #"
1321   "reload_completed && TARGET_POWERPC64 && TARGET_P8_VECTOR
1322    && VECTOR_MEM_VSX_P (<MODE>mode)
1323    && int_reg_operand (operands[0], <MODE>mode)"
1324   [(set (match_dup 3) (not:DI (xor:DI (match_dup 4) (match_dup 5))))
1325    (set (match_dup 6) (not:DI (xor:DI (match_dup 7) (match_dup 8))))]
1326 {
1327   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1328   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1329   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1330   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1331   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1332   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1333 }
1334   [(set_attr "type" "vecsimple,two,two,two")
1335    (set_attr "length" "4,8,8,8")])
1336
1337 ;; Rewrite nand into canonical form
1338 (define_insn "*vsx_nand<mode>3_32bit"
1339   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1340         (ior:VSX_L2
1341          (not:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "wa"))
1342          (not:VSX_L2 (match_operand:VSX_L2 2 "vlogical_operand" "wa"))))]
1343   "!TARGET_POWERPC64 && TARGET_P8_VECTOR && VECTOR_MEM_VSX_P (<MODE>mode)"
1344   "xxlnand %x0,%x1,%x2"
1345   [(set_attr "type" "vecsimple")
1346    (set_attr "length" "4")])
1347
1348 (define_insn_and_split "*vsx_nand<mode>3_64bit"
1349   [(set (match_operand:VSX_L 0 "register_operand" "=wa,?r,?r,?r")
1350         (ior:VSX_L
1351          (not:VSX_L (match_operand:VSX_L 1 "register_operand" "wa,0,r,r"))
1352          (not:VSX_L (match_operand:VSX_L 2 "register_operand" "wa,r,0,r"))))]
1353   "TARGET_POWERPC64 && TARGET_P8_VECTOR && VECTOR_MEM_VSX_P (<MODE>mode)"
1354   "@
1355    xxlnand %x0,%x1,%x2
1356    #
1357    #
1358    #"
1359   "reload_completed && TARGET_POWERPC64 && TARGET_P8_VECTOR
1360    && VECTOR_MEM_VSX_P (<MODE>mode)
1361    && int_reg_operand (operands[0], <MODE>mode)"
1362   [(set (match_dup 3) (ior:DI (not:DI (match_dup 4)) (not:DI (match_dup 5))))
1363    (set (match_dup 6) (ior:DI (not:DI (match_dup 7)) (not:DI (match_dup 8))))]
1364 {
1365   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1366   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1367   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1368   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1369   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1370   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1371 }
1372   [(set_attr "type" "vecsimple,two,two,two")
1373    (set_attr "length" "4,8,8,8")])
1374
1375 ;; Rewrite or complement into canonical form, by reversing the arguments
1376 (define_insn "*vsx_orc<mode>3_32bit"
1377   [(set (match_operand:VSX_L2 0 "vlogical_operand" "=wa")
1378         (ior:VSX_L2
1379          (not:VSX_L2 (match_operand:VSX_L2 1 "vlogical_operand" "wa"))
1380          (match_operand:VSX_L2 2 "vlogical_operand" "wa")))]
1381   "!TARGET_POWERPC64 && TARGET_P8_VECTOR && VECTOR_MEM_VSX_P (<MODE>mode)"
1382   "xxlorc %x0,%x2,%x1"
1383   [(set_attr "type" "vecsimple")
1384    (set_attr "length" "4")])
1385
1386 (define_insn_and_split "*vsx_orc<mode>3_64bit"
1387   [(set (match_operand:VSX_L 0 "vlogical_operand" "=wa,?r,?r,?r")
1388         (ior:VSX_L
1389          (not:VSX_L (match_operand:VSX_L 1 "vlogical_operand" "wa,0,r,r"))
1390          (match_operand:VSX_L 2 "vlogical_operand" "wa,r,0,r")))]
1391   "TARGET_POWERPC64 && TARGET_P8_VECTOR && VECTOR_MEM_VSX_P (<MODE>mode)"
1392   "@
1393    xxlorc %x0,%x2,%x1
1394    #
1395    #
1396    #"
1397   "reload_completed && TARGET_POWERPC64 && TARGET_P8_VECTOR
1398    && VECTOR_MEM_VSX_P (<MODE>mode)
1399    && int_reg_operand (operands[0], <MODE>mode)"
1400   [(set (match_dup 3) (ior:DI (not:DI (match_dup 4)) (match_dup 5)))
1401    (set (match_dup 6) (ior:DI (not:DI (match_dup 7)) (match_dup 8)))]
1402 {
1403   operands[3] = simplify_subreg (DImode, operands[0], <MODE>mode, 0);
1404   operands[4] = simplify_subreg (DImode, operands[1], <MODE>mode, 0);
1405   operands[5] = simplify_subreg (DImode, operands[2], <MODE>mode, 0);
1406   operands[6] = simplify_subreg (DImode, operands[0], <MODE>mode, 8);
1407   operands[7] = simplify_subreg (DImode, operands[1], <MODE>mode, 8);
1408   operands[8] = simplify_subreg (DImode, operands[2], <MODE>mode, 8);
1409 }
1410   [(set_attr "type" "vecsimple,two,two,two")
1411    (set_attr "length" "4,8,8,8")])
1412
1413 \f
1414 ;; Permute operations
1415
1416 ;; Build a V2DF/V2DI vector from two scalars
1417 (define_insn "vsx_concat_<mode>"
1418   [(set (match_operand:VSX_D 0 "vsx_register_operand" "=<VSr>,?wa")
1419         (vec_concat:VSX_D
1420          (match_operand:<VS_scalar> 1 "vsx_register_operand" "ws,wa")
1421          (match_operand:<VS_scalar> 2 "vsx_register_operand" "ws,wa")))]
1422   "VECTOR_MEM_VSX_P (<MODE>mode)"
1423   "xxpermdi %x0,%x1,%x2,0"
1424   [(set_attr "type" "vecperm")])
1425
1426 ;; Special purpose concat using xxpermdi to glue two single precision values
1427 ;; together, relying on the fact that internally scalar floats are represented
1428 ;; as doubles.  This is used to initialize a V4SF vector with 4 floats
1429 (define_insn "vsx_concat_v2sf"
1430   [(set (match_operand:V2DF 0 "vsx_register_operand" "=wd,?wa")
1431         (unspec:V2DF
1432          [(match_operand:SF 1 "vsx_register_operand" "f,f")
1433           (match_operand:SF 2 "vsx_register_operand" "f,f")]
1434          UNSPEC_VSX_CONCAT))]
1435   "VECTOR_MEM_VSX_P (V2DFmode)"
1436   "xxpermdi %x0,%x1,%x2,0"
1437   [(set_attr "type" "vecperm")])
1438
1439 ;; Set the element of a V2DI/VD2F mode
1440 (define_insn "vsx_set_<mode>"
1441   [(set (match_operand:VSX_D 0 "vsx_register_operand" "=wd,?wa")
1442         (unspec:VSX_D [(match_operand:VSX_D 1 "vsx_register_operand" "wd,wa")
1443                        (match_operand:<VS_scalar> 2 "vsx_register_operand" "ws,wa")
1444                        (match_operand:QI 3 "u5bit_cint_operand" "i,i")]
1445                       UNSPEC_VSX_SET))]
1446   "VECTOR_MEM_VSX_P (<MODE>mode)"
1447 {
1448   if (INTVAL (operands[3]) == 0)
1449     return \"xxpermdi %x0,%x2,%x1,1\";
1450   else if (INTVAL (operands[3]) == 1)
1451     return \"xxpermdi %x0,%x1,%x2,0\";
1452   else
1453     gcc_unreachable ();
1454 }
1455   [(set_attr "type" "vecperm")])
1456
1457 ;; Extract a DF/DI element from V2DF/V2DI
1458 (define_insn "vsx_extract_<mode>"
1459   [(set (match_operand:<VS_scalar> 0 "vsx_register_operand" "=ws,d,?wa")
1460         (vec_select:<VS_scalar> (match_operand:VSX_D 1 "vsx_register_operand" "wd,wd,wa")
1461                        (parallel
1462                         [(match_operand:QI 2 "u5bit_cint_operand" "i,i,i")])))]
1463   "VECTOR_MEM_VSX_P (<MODE>mode)"
1464 {
1465   gcc_assert (UINTVAL (operands[2]) <= 1);
1466   operands[3] = GEN_INT (INTVAL (operands[2]) << 1);
1467   return \"xxpermdi %x0,%x1,%x1,%3\";
1468 }
1469   [(set_attr "type" "vecperm")])
1470
1471 ;; Optimize extracting element 0 from memory
1472 (define_insn "*vsx_extract_<mode>_zero"
1473   [(set (match_operand:<VS_scalar> 0 "vsx_register_operand" "=ws,d,?wa")
1474         (vec_select:<VS_scalar>
1475          (match_operand:VSX_D 1 "indexed_or_indirect_operand" "Z,Z,Z")
1476          (parallel [(const_int 0)])))]
1477   "VECTOR_MEM_VSX_P (<MODE>mode) && WORDS_BIG_ENDIAN"
1478   "lxsd%U1x %x0,%y1"
1479   [(set (attr "type")
1480       (if_then_else
1481         (match_test "update_indexed_address_mem (operands[1], VOIDmode)")
1482         (const_string "fpload_ux")
1483         (const_string "fpload")))
1484    (set_attr "length" "4")])
1485
1486 ;; Extract a SF element from V4SF
1487 (define_insn_and_split "vsx_extract_v4sf"
1488   [(set (match_operand:SF 0 "vsx_register_operand" "=f,f")
1489         (vec_select:SF
1490          (match_operand:V4SF 1 "vsx_register_operand" "wa,wa")
1491          (parallel [(match_operand:QI 2 "u5bit_cint_operand" "O,i")])))
1492    (clobber (match_scratch:V4SF 3 "=X,0"))]
1493   "VECTOR_UNIT_VSX_P (V4SFmode)"
1494   "@
1495    xscvspdp %x0,%x1
1496    #"
1497   ""
1498   [(const_int 0)]
1499   "
1500 {
1501   rtx op0 = operands[0];
1502   rtx op1 = operands[1];
1503   rtx op2 = operands[2];
1504   rtx op3 = operands[3];
1505   rtx tmp;
1506   HOST_WIDE_INT ele = INTVAL (op2);
1507
1508   if (ele == 0)
1509     tmp = op1;
1510   else
1511     {
1512       if (GET_CODE (op3) == SCRATCH)
1513         op3 = gen_reg_rtx (V4SFmode);
1514       emit_insn (gen_vsx_xxsldwi_v4sf (op3, op1, op1, op2));
1515       tmp = op3;
1516     }
1517   emit_insn (gen_vsx_xscvspdp_scalar2 (op0, tmp));
1518   DONE;
1519 }"
1520   [(set_attr "length" "4,8")
1521    (set_attr "type" "fp")])
1522
1523 ;; Expand the builtin form of xxpermdi to canonical rtl.
1524 (define_expand "vsx_xxpermdi_<mode>"
1525   [(match_operand:VSX_L 0 "vsx_register_operand" "")
1526    (match_operand:VSX_L 1 "vsx_register_operand" "")
1527    (match_operand:VSX_L 2 "vsx_register_operand" "")
1528    (match_operand:QI 3 "u5bit_cint_operand" "")]
1529   "VECTOR_MEM_VSX_P (<MODE>mode)"
1530 {
1531   rtx target = operands[0];
1532   rtx op0 = operands[1];
1533   rtx op1 = operands[2];
1534   int mask = INTVAL (operands[3]);
1535   rtx perm0 = GEN_INT ((mask >> 1) & 1);
1536   rtx perm1 = GEN_INT ((mask & 1) + 2);
1537   rtx (*gen) (rtx, rtx, rtx, rtx, rtx);
1538
1539   if (<MODE>mode == V2DFmode)
1540     gen = gen_vsx_xxpermdi2_v2df_1;
1541   else
1542     {
1543       gen = gen_vsx_xxpermdi2_v2di_1;
1544       if (<MODE>mode != V2DImode)
1545         {
1546           target = gen_lowpart (V2DImode, target);
1547           op0 = gen_lowpart (V2DImode, op0);
1548           op1 = gen_lowpart (V2DImode, op1);
1549         }
1550     }
1551   emit_insn (gen (target, op0, op1, perm0, perm1));
1552   DONE;
1553 })
1554
1555 (define_insn "vsx_xxpermdi2_<mode>_1"
1556   [(set (match_operand:VSX_D 0 "vsx_register_operand" "=wd")
1557         (vec_select:VSX_D
1558           (vec_concat:<VS_double>
1559             (match_operand:VSX_D 1 "vsx_register_operand" "wd")
1560             (match_operand:VSX_D 2 "vsx_register_operand" "wd"))
1561           (parallel [(match_operand 3 "const_0_to_1_operand" "")
1562                      (match_operand 4 "const_2_to_3_operand" "")])))]
1563   "VECTOR_MEM_VSX_P (<MODE>mode)"
1564 {
1565   int mask = (INTVAL (operands[3]) << 1) | (INTVAL (operands[4]) - 2);
1566   operands[3] = GEN_INT (mask);
1567   return "xxpermdi %x0,%x1,%x2,%3";
1568 }
1569   [(set_attr "type" "vecperm")])
1570
1571 (define_expand "vec_perm_const<mode>"
1572   [(match_operand:VSX_D 0 "vsx_register_operand" "")
1573    (match_operand:VSX_D 1 "vsx_register_operand" "")
1574    (match_operand:VSX_D 2 "vsx_register_operand" "")
1575    (match_operand:V2DI  3 "" "")]
1576   "VECTOR_MEM_VSX_P (<MODE>mode)"
1577 {
1578   if (rs6000_expand_vec_perm_const (operands))
1579     DONE;
1580   else
1581     FAIL;
1582 })
1583
1584 ;; Expanders for builtins
1585 (define_expand "vsx_mergel_<mode>"
1586   [(set (match_operand:VSX_D 0 "vsx_register_operand" "")
1587         (vec_select:VSX_D
1588           (vec_concat:<VS_double>
1589             (match_operand:VSX_D 1 "vsx_register_operand" "")
1590             (match_operand:VSX_D 2 "vsx_register_operand" ""))
1591           (parallel [(const_int 1) (const_int 3)])))]
1592   "VECTOR_MEM_VSX_P (<MODE>mode)"
1593   "")
1594
1595 (define_expand "vsx_mergeh_<mode>"
1596   [(set (match_operand:VSX_D 0 "vsx_register_operand" "")
1597         (vec_select:VSX_D
1598           (vec_concat:<VS_double>
1599             (match_operand:VSX_D 1 "vsx_register_operand" "")
1600             (match_operand:VSX_D 2 "vsx_register_operand" ""))
1601           (parallel [(const_int 0) (const_int 2)])))]
1602   "VECTOR_MEM_VSX_P (<MODE>mode)"
1603   "")
1604
1605 ;; V2DF/V2DI splat
1606 (define_insn "vsx_splat_<mode>"
1607   [(set (match_operand:VSX_D 0 "vsx_register_operand" "=wd,wd,wd,?wa,?wa,?wa")
1608         (vec_duplicate:VSX_D
1609          (match_operand:<VS_scalar> 1 "splat_input_operand" "ws,f,Z,wa,wa,Z")))]
1610   "VECTOR_MEM_VSX_P (<MODE>mode)"
1611   "@
1612    xxpermdi %x0,%x1,%x1,0
1613    xxpermdi %x0,%x1,%x1,0
1614    lxvdsx %x0,%y1
1615    xxpermdi %x0,%x1,%x1,0
1616    xxpermdi %x0,%x1,%x1,0
1617    lxvdsx %x0,%y1"
1618   [(set_attr "type" "vecperm,vecperm,vecload,vecperm,vecperm,vecload")])
1619
1620 ;; V4SF/V4SI splat
1621 (define_insn "vsx_xxspltw_<mode>"
1622   [(set (match_operand:VSX_W 0 "vsx_register_operand" "=wf,?wa")
1623         (vec_duplicate:VSX_W
1624          (vec_select:<VS_scalar>
1625           (match_operand:VSX_W 1 "vsx_register_operand" "wf,wa")
1626           (parallel
1627            [(match_operand:QI 2 "u5bit_cint_operand" "i,i")]))))]
1628   "VECTOR_MEM_VSX_P (<MODE>mode)"
1629   "xxspltw %x0,%x1,%2"
1630   [(set_attr "type" "vecperm")])
1631
1632 ;; V4SF/V4SI interleave
1633 (define_insn "vsx_xxmrghw_<mode>"
1634   [(set (match_operand:VSX_W 0 "vsx_register_operand" "=wf,?wa")
1635         (vec_select:VSX_W
1636           (vec_concat:<VS_double>
1637             (match_operand:VSX_W 1 "vsx_register_operand" "wf,wa")
1638             (match_operand:VSX_W 2 "vsx_register_operand" "wf,wa"))
1639           (parallel [(const_int 0) (const_int 4)
1640                      (const_int 1) (const_int 5)])))]
1641   "VECTOR_MEM_VSX_P (<MODE>mode)"
1642   "xxmrghw %x0,%x1,%x2"
1643   [(set_attr "type" "vecperm")])
1644
1645 (define_insn "vsx_xxmrglw_<mode>"
1646   [(set (match_operand:VSX_W 0 "vsx_register_operand" "=wf,?wa")
1647         (vec_select:VSX_W
1648           (vec_concat:<VS_double>
1649             (match_operand:VSX_W 1 "vsx_register_operand" "wf,wa")
1650             (match_operand:VSX_W 2 "vsx_register_operand" "wf,?wa"))
1651           (parallel [(const_int 2) (const_int 6)
1652                      (const_int 3) (const_int 7)])))]
1653   "VECTOR_MEM_VSX_P (<MODE>mode)"
1654   "xxmrglw %x0,%x1,%x2"
1655   [(set_attr "type" "vecperm")])
1656
1657 ;; Shift left double by word immediate
1658 (define_insn "vsx_xxsldwi_<mode>"
1659   [(set (match_operand:VSX_L 0 "vsx_register_operand" "=wa")
1660         (unspec:VSX_L [(match_operand:VSX_L 1 "vsx_register_operand" "wa")
1661                        (match_operand:VSX_L 2 "vsx_register_operand" "wa")
1662                        (match_operand:QI 3 "u5bit_cint_operand" "i")]
1663                       UNSPEC_VSX_SLDWI))]
1664   "VECTOR_MEM_VSX_P (<MODE>mode)"
1665   "xxsldwi %x0,%x1,%x2,%3"
1666   [(set_attr "type" "vecperm")])
1667
1668 \f
1669 ;; Vector reduction insns and splitters
1670
1671 (define_insn_and_split "*vsx_reduc_<VEC_reduc_name>_v2df"
1672   [(set (match_operand:V2DF 0 "vfloat_operand" "=&wd,&?wa,wd,?wa")
1673         (VEC_reduc:V2DF
1674          (vec_concat:V2DF
1675           (vec_select:DF
1676            (match_operand:V2DF 1 "vfloat_operand" "wd,wa,wd,wa")
1677            (parallel [(const_int 1)]))
1678           (vec_select:DF
1679            (match_dup 1)
1680            (parallel [(const_int 0)])))
1681          (match_dup 1)))
1682    (clobber (match_scratch:V2DF 2 "=0,0,&wd,&wa"))]
1683   "VECTOR_UNIT_VSX_P (V2DFmode)"
1684   "#"
1685   ""
1686   [(const_int 0)]
1687   "
1688 {
1689   rtx tmp = (GET_CODE (operands[2]) == SCRATCH)
1690              ? gen_reg_rtx (V2DFmode)
1691              : operands[2];
1692   emit_insn (gen_vsx_xxsldwi_v2df (tmp, operands[1], operands[1], const2_rtx));
1693   emit_insn (gen_<VEC_reduc_rtx>v2df3 (operands[0], tmp, operands[1]));
1694   DONE;
1695 }"
1696   [(set_attr "length" "8")
1697    (set_attr "type" "veccomplex")])
1698
1699 (define_insn_and_split "*vsx_reduc_<VEC_reduc_name>_v4sf"
1700   [(set (match_operand:V4SF 0 "vfloat_operand" "=wf,?wa")
1701         (VEC_reduc:V4SF
1702          (unspec:V4SF [(const_int 0)] UNSPEC_REDUC)
1703          (match_operand:V4SF 1 "vfloat_operand" "wf,wa")))
1704    (clobber (match_scratch:V4SF 2 "=&wf,&wa"))
1705    (clobber (match_scratch:V4SF 3 "=&wf,&wa"))]
1706   "VECTOR_UNIT_VSX_P (V4SFmode)"
1707   "#"
1708   ""
1709   [(const_int 0)]
1710   "
1711 {
1712   rtx op0 = operands[0];
1713   rtx op1 = operands[1];
1714   rtx tmp2, tmp3, tmp4;
1715
1716   if (can_create_pseudo_p ())
1717     {
1718       tmp2 = gen_reg_rtx (V4SFmode);
1719       tmp3 = gen_reg_rtx (V4SFmode);
1720       tmp4 = gen_reg_rtx (V4SFmode);
1721     }
1722   else
1723     {
1724       tmp2 = operands[2];
1725       tmp3 = operands[3];
1726       tmp4 = tmp2;
1727     }
1728
1729   emit_insn (gen_vsx_xxsldwi_v4sf (tmp2, op1, op1, const2_rtx));
1730   emit_insn (gen_<VEC_reduc_rtx>v4sf3 (tmp3, tmp2, op1));
1731   emit_insn (gen_vsx_xxsldwi_v4sf (tmp4, tmp3, tmp3, GEN_INT (3)));
1732   emit_insn (gen_<VEC_reduc_rtx>v4sf3 (op0, tmp4, tmp3));
1733   DONE;
1734 }"
1735   [(set_attr "length" "16")
1736    (set_attr "type" "veccomplex")])
1737
1738 ;; Combiner patterns with the vector reduction patterns that knows we can get
1739 ;; to the top element of the V2DF array without doing an extract.
1740
1741 (define_insn_and_split "*vsx_reduc_<VEC_reduc_name>_v2df_scalar"
1742   [(set (match_operand:DF 0 "vfloat_operand" "=&ws,&?wa,ws,?wa")
1743         (vec_select:DF
1744          (VEC_reduc:V2DF
1745           (vec_concat:V2DF
1746            (vec_select:DF
1747             (match_operand:V2DF 1 "vfloat_operand" "wd,wa,wd,wa")
1748             (parallel [(const_int 1)]))
1749            (vec_select:DF
1750             (match_dup 1)
1751             (parallel [(const_int 0)])))
1752           (match_dup 1))
1753          (parallel [(const_int 1)])))
1754    (clobber (match_scratch:DF 2 "=0,0,&wd,&wa"))]
1755   "VECTOR_UNIT_VSX_P (V2DFmode)"
1756   "#"
1757   ""
1758   [(const_int 0)]
1759   "
1760 {
1761   rtx hi = gen_highpart (DFmode, operands[1]);
1762   rtx lo = (GET_CODE (operands[2]) == SCRATCH)
1763             ? gen_reg_rtx (DFmode)
1764             : operands[2];
1765
1766   emit_insn (gen_vsx_extract_v2df (lo, operands[1], const1_rtx));
1767   emit_insn (gen_<VEC_reduc_rtx>df3 (operands[0], hi, lo));
1768   DONE;
1769 }"
1770   [(set_attr "length" "8")
1771    (set_attr "type" "veccomplex")])
1772
1773 (define_insn_and_split "*vsx_reduc_<VEC_reduc_name>_v4sf_scalar"
1774   [(set (match_operand:SF 0 "vfloat_operand" "=f,?f")
1775         (vec_select:SF
1776          (VEC_reduc:V4SF
1777           (unspec:V4SF [(const_int 0)] UNSPEC_REDUC)
1778           (match_operand:V4SF 1 "vfloat_operand" "wf,wa"))
1779          (parallel [(const_int 3)])))
1780    (clobber (match_scratch:V4SF 2 "=&wf,&wa"))
1781    (clobber (match_scratch:V4SF 3 "=&wf,&wa"))
1782    (clobber (match_scratch:V4SF 4 "=0,0"))]
1783   "VECTOR_UNIT_VSX_P (V4SFmode)"
1784   "#"
1785   ""
1786   [(const_int 0)]
1787   "
1788 {
1789   rtx op0 = operands[0];
1790   rtx op1 = operands[1];
1791   rtx tmp2, tmp3, tmp4, tmp5;
1792
1793   if (can_create_pseudo_p ())
1794     {
1795       tmp2 = gen_reg_rtx (V4SFmode);
1796       tmp3 = gen_reg_rtx (V4SFmode);
1797       tmp4 = gen_reg_rtx (V4SFmode);
1798       tmp5 = gen_reg_rtx (V4SFmode);
1799     }
1800   else
1801     {
1802       tmp2 = operands[2];
1803       tmp3 = operands[3];
1804       tmp4 = tmp2;
1805       tmp5 = operands[4];
1806     }
1807
1808   emit_insn (gen_vsx_xxsldwi_v4sf (tmp2, op1, op1, const2_rtx));
1809   emit_insn (gen_<VEC_reduc_rtx>v4sf3 (tmp3, tmp2, op1));
1810   emit_insn (gen_vsx_xxsldwi_v4sf (tmp4, tmp3, tmp3, GEN_INT (3)));
1811   emit_insn (gen_<VEC_reduc_rtx>v4sf3 (tmp5, tmp4, tmp3));
1812   emit_insn (gen_vsx_xscvspdp_scalar2 (op0, tmp5));
1813   DONE;
1814 }"
1815   [(set_attr "length" "20")
1816    (set_attr "type" "veccomplex")])