src/compiler/spirv/vtn_alu.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <math.h>
  25 #include "vtn_private.h"
  26 #include "spirv_info.h"
  27
  28 /*
  29  * Normally, column vectors in SPIR-V correspond to a single NIR SSA
  30  * definition. But for matrix multiplies, we want to do one routine for
  31  * multiplying a matrix by a matrix and then pretend that vectors are matrices
  32  * with one column. So we "wrap" these things, and unwrap the result before we
  33  * send it off.
  34  */
  35
  36 static struct vtn_ssa_value *
  37 wrap_matrix(struct vtn_builder *b, struct vtn_ssa_value *val)
  38 {
  39    if (val == NULL)
  40       return NULL;
  41
  42    if (glsl_type_is_matrix(val->type))
  43       return val;
  44
  45    struct vtn_ssa_value *dest = rzalloc(b, struct vtn_ssa_value);
  46    dest->type = val->type;
  47    dest->elems = ralloc_array(b, struct vtn_ssa_value *, 1);
  48    dest->elems[0] = val;
  49
  50    return dest;
  51 }
  52
  53 static struct vtn_ssa_value *
  54 unwrap_matrix(struct vtn_ssa_value *val)
  55 {
  56    if (glsl_type_is_matrix(val->type))
  57          return val;
  58
  59    return val->elems[0];
  60 }
  61
  62 static struct vtn_ssa_value *
  63 matrix_multiply(struct vtn_builder *b,
  64                 struct vtn_ssa_value *_src0, struct vtn_ssa_value *_src1)
  65 {
  66
  67    struct vtn_ssa_value *src0 = wrap_matrix(b, _src0);
  68    struct vtn_ssa_value *src1 = wrap_matrix(b, _src1);
  69    struct vtn_ssa_value *src0_transpose = wrap_matrix(b, _src0->transposed);
  70    struct vtn_ssa_value *src1_transpose = wrap_matrix(b, _src1->transposed);
  71
  72    unsigned src0_rows = glsl_get_vector_elements(src0->type);
  73    unsigned src0_columns = glsl_get_matrix_columns(src0->type);
  74    unsigned src1_columns = glsl_get_matrix_columns(src1->type);
  75
  76    const struct glsl_type *dest_type;
  77    if (src1_columns > 1) {
  78       dest_type = glsl_matrix_type(glsl_get_base_type(src0->type),
  79                                    src0_rows, src1_columns);
  80    } else {
  81       dest_type = glsl_vector_type(glsl_get_base_type(src0->type), src0_rows);
  82    }
  83    struct vtn_ssa_value *dest = vtn_create_ssa_value(b, dest_type);
  84
  85    dest = wrap_matrix(b, dest);
  86
  87    bool transpose_result = false;
  88    if (src0_transpose && src1_transpose) {
  89       /* transpose(A) * transpose(B) = transpose(B * A) */
  90       src1 = src0_transpose;
  91       src0 = src1_transpose;
  92       src0_transpose = NULL;
  93       src1_transpose = NULL;
  94       transpose_result = true;
  95    }
  96
  97    if (src0_transpose && !src1_transpose &&
  98        glsl_get_base_type(src0->type) == GLSL_TYPE_FLOAT) {
  99       /* We already have the rows of src0 and the columns of src1 available,
 100        * so we can just take the dot product of each row with each column to
 101        * get the result.
 102        */
 103
 104       for (unsigned i = 0; i < src1_columns; i++) {
 105          nir_ssa_def *vec_src[4];
 106          for (unsigned j = 0; j < src0_rows; j++) {
 107             vec_src[j] = nir_fdot(&b->nb, src0_transpose->elems[j]->def,
 108                                           src1->elems[i]->def);
 109          }
 110          dest->elems[i]->def = nir_vec(&b->nb, vec_src, src0_rows);
 111       }
 112    } else {
 113       /* We don't handle the case where src1 is transposed but not src0, since
 114        * the general case only uses individual components of src1 so the
 115        * optimizer should chew through the transpose we emitted for src1.
 116        */
 117
 118       for (unsigned i = 0; i < src1_columns; i++) {
 119          /* dest[i] = sum(src0[j] * src1[i][j] for all j) */
 120          dest->elems[i]->def =
 121             nir_fmul(&b->nb, src0->elems[0]->def,
 122                      nir_channel(&b->nb, src1->elems[i]->def, 0));
 123          for (unsigned j = 1; j < src0_columns; j++) {
 124             dest->elems[i]->def =
 125                nir_fadd(&b->nb, dest->elems[i]->def,
 126                         nir_fmul(&b->nb, src0->elems[j]->def,
 127                                  nir_channel(&b->nb, src1->elems[i]->def, j)));
 128          }
 129       }
 130    }
 131
 132    dest = unwrap_matrix(dest);
 133
 134    if (transpose_result)
 135       dest = vtn_ssa_transpose(b, dest);
 136
 137    return dest;
 138 }
 139
 140 static struct vtn_ssa_value *
 141 mat_times_scalar(struct vtn_builder *b,
 142                  struct vtn_ssa_value *mat,
 143                  nir_ssa_def *scalar)
 144 {
 145    struct vtn_ssa_value *dest = vtn_create_ssa_value(b, mat->type);
 146    for (unsigned i = 0; i < glsl_get_matrix_columns(mat->type); i++) {
 147       if (glsl_base_type_is_integer(glsl_get_base_type(mat->type)))
 148          dest->elems[i]->def = nir_imul(&b->nb, mat->elems[i]->def, scalar);
 149       else
 150          dest->elems[i]->def = nir_fmul(&b->nb, mat->elems[i]->def, scalar);
 151    }
 152
 153    return dest;
 154 }
 155
 156 static void
 157 vtn_handle_matrix_alu(struct vtn_builder *b, SpvOp opcode,
 158                       struct vtn_value *dest,
 159                       struct vtn_ssa_value *src0, struct vtn_ssa_value *src1)
 160 {
 161    switch (opcode) {
 162    case SpvOpFNegate: {
 163       dest->ssa = vtn_create_ssa_value(b, src0->type);
 164       unsigned cols = glsl_get_matrix_columns(src0->type);
 165       for (unsigned i = 0; i < cols; i++)
 166          dest->ssa->elems[i]->def = nir_fneg(&b->nb, src0->elems[i]->def);
 167       break;
 168    }
 169
 170    case SpvOpFAdd: {
 171       dest->ssa = vtn_create_ssa_value(b, src0->type);
 172       unsigned cols = glsl_get_matrix_columns(src0->type);
 173       for (unsigned i = 0; i < cols; i++)
 174          dest->ssa->elems[i]->def =
 175             nir_fadd(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
 176       break;
 177    }
 178
 179    case SpvOpFSub: {
 180       dest->ssa = vtn_create_ssa_value(b, src0->type);
 181       unsigned cols = glsl_get_matrix_columns(src0->type);
 182       for (unsigned i = 0; i < cols; i++)
 183          dest->ssa->elems[i]->def =
 184             nir_fsub(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
 185       break;
 186    }
 187
 188    case SpvOpTranspose:
 189       dest->ssa = vtn_ssa_transpose(b, src0);
 190       break;
 191
 192    case SpvOpMatrixTimesScalar:
 193       if (src0->transposed) {
 194          dest->ssa = vtn_ssa_transpose(b, mat_times_scalar(b, src0->transposed,
 195                                                            src1->def));
 196       } else {
 197          dest->ssa = mat_times_scalar(b, src0, src1->def);
 198       }
 199       break;
 200
 201    case SpvOpVectorTimesMatrix:
 202    case SpvOpMatrixTimesVector:
 203    case SpvOpMatrixTimesMatrix:
 204       if (opcode == SpvOpVectorTimesMatrix) {
 205          dest->ssa = matrix_multiply(b, vtn_ssa_transpose(b, src1), src0);
 206       } else {
 207          dest->ssa = matrix_multiply(b, src0, src1);
 208       }
 209       break;
 210
 211    default: vtn_fail_with_opcode("unknown matrix opcode", opcode);
 212    }
 213 }
 214
 215 nir_op
 216 vtn_nir_alu_op_for_spirv_opcode(struct vtn_builder *b,
 217                                 SpvOp opcode, bool *swap,
 218                                 unsigned src_bit_size, unsigned dst_bit_size)
 219 {
 220    /* Indicates that the first two arguments should be swapped.  This is
 221     * used for implementing greater-than and less-than-or-equal.
 222     */
 223    *swap = false;
 224
 225    switch (opcode) {
 226    case SpvOpSNegate:            return nir_op_ineg;
 227    case SpvOpFNegate:            return nir_op_fneg;
 228    case SpvOpNot:                return nir_op_inot;
 229    case SpvOpIAdd:               return nir_op_iadd;
 230    case SpvOpFAdd:               return nir_op_fadd;
 231    case SpvOpISub:               return nir_op_isub;
 232    case SpvOpFSub:               return nir_op_fsub;
 233    case SpvOpIMul:               return nir_op_imul;
 234    case SpvOpFMul:               return nir_op_fmul;
 235    case SpvOpUDiv:               return nir_op_udiv;
 236    case SpvOpSDiv:               return nir_op_idiv;
 237    case SpvOpFDiv:               return nir_op_fdiv;
 238    case SpvOpUMod:               return nir_op_umod;
 239    case SpvOpSMod:               return nir_op_imod;
 240    case SpvOpFMod:               return nir_op_fmod;
 241    case SpvOpSRem:               return nir_op_irem;
 242    case SpvOpFRem:               return nir_op_frem;
 243
 244    case SpvOpShiftRightLogical:     return nir_op_ushr;
 245    case SpvOpShiftRightArithmetic:  return nir_op_ishr;
 246    case SpvOpShiftLeftLogical:      return nir_op_ishl;
 247    case SpvOpLogicalOr:             return nir_op_ior;
 248    case SpvOpLogicalEqual:          return nir_op_ieq;
 249    case SpvOpLogicalNotEqual:       return nir_op_ine;
 250    case SpvOpLogicalAnd:            return nir_op_iand;
 251    case SpvOpLogicalNot:            return nir_op_inot;
 252    case SpvOpBitwiseOr:             return nir_op_ior;
 253    case SpvOpBitwiseXor:            return nir_op_ixor;
 254    case SpvOpBitwiseAnd:            return nir_op_iand;
 255    case SpvOpSelect:                return nir_op_bcsel;
 256    case SpvOpIEqual:                return nir_op_ieq;
 257
 258    case SpvOpBitFieldInsert:        return nir_op_bitfield_insert;
 259    case SpvOpBitFieldSExtract:      return nir_op_ibitfield_extract;
 260    case SpvOpBitFieldUExtract:      return nir_op_ubitfield_extract;
 261    case SpvOpBitReverse:            return nir_op_bitfield_reverse;
 262    case SpvOpBitCount:              return nir_op_bit_count;
 263
 264    case SpvOpUCountLeadingZerosINTEL: return nir_op_uclz;
 265    /* SpvOpUCountTrailingZerosINTEL is handled elsewhere. */
 266    case SpvOpAbsISubINTEL:          return nir_op_uabs_isub;
 267    case SpvOpAbsUSubINTEL:          return nir_op_uabs_usub;
 268    case SpvOpIAddSatINTEL:          return nir_op_iadd_sat;
 269    case SpvOpUAddSatINTEL:          return nir_op_uadd_sat;
 270    case SpvOpIAverageINTEL:         return nir_op_ihadd;
 271    case SpvOpUAverageINTEL:         return nir_op_uhadd;
 272    case SpvOpIAverageRoundedINTEL:  return nir_op_irhadd;
 273    case SpvOpUAverageRoundedINTEL:  return nir_op_urhadd;
 274    case SpvOpISubSatINTEL:          return nir_op_isub_sat;
 275    case SpvOpUSubSatINTEL:          return nir_op_usub_sat;
 276    case SpvOpIMul32x16INTEL:        return nir_op_imul_32x16;
 277    case SpvOpUMul32x16INTEL:        return nir_op_umul_32x16;
 278
 279    /* The ordered / unordered operators need special implementation besides
 280     * the logical operator to use since they also need to check if operands are
 281     * ordered.
 282     */
 283    case SpvOpFOrdEqual:                            return nir_op_feq;
 284    case SpvOpFUnordEqual:                          return nir_op_feq;
 285    case SpvOpINotEqual:                            return nir_op_ine;
 286    case SpvOpFOrdNotEqual:                         return nir_op_fne;
 287    case SpvOpFUnordNotEqual:                       return nir_op_fne;
 288    case SpvOpULessThan:                            return nir_op_ult;
 289    case SpvOpSLessThan:                            return nir_op_ilt;
 290    case SpvOpFOrdLessThan:                         return nir_op_flt;
 291    case SpvOpFUnordLessThan:                       return nir_op_flt;
 292    case SpvOpUGreaterThan:          *swap = true;  return nir_op_ult;
 293    case SpvOpSGreaterThan:          *swap = true;  return nir_op_ilt;
 294    case SpvOpFOrdGreaterThan:       *swap = true;  return nir_op_flt;
 295    case SpvOpFUnordGreaterThan:     *swap = true;  return nir_op_flt;
 296    case SpvOpULessThanEqual:        *swap = true;  return nir_op_uge;
 297    case SpvOpSLessThanEqual:        *swap = true;  return nir_op_ige;
 298    case SpvOpFOrdLessThanEqual:     *swap = true;  return nir_op_fge;
 299    case SpvOpFUnordLessThanEqual:   *swap = true;  return nir_op_fge;
 300    case SpvOpUGreaterThanEqual:                    return nir_op_uge;
 301    case SpvOpSGreaterThanEqual:                    return nir_op_ige;
 302    case SpvOpFOrdGreaterThanEqual:                 return nir_op_fge;
 303    case SpvOpFUnordGreaterThanEqual:               return nir_op_fge;
 304
 305    /* Conversions: */
 306    case SpvOpQuantizeToF16:         return nir_op_fquantize2f16;
 307    case SpvOpUConvert:
 308    case SpvOpConvertFToU:
 309    case SpvOpConvertFToS:
 310    case SpvOpConvertSToF:
 311    case SpvOpConvertUToF:
 312    case SpvOpSConvert:
 313    case SpvOpFConvert: {
 314       nir_alu_type src_type;
 315       nir_alu_type dst_type;
 316
 317       switch (opcode) {
 318       case SpvOpConvertFToS:
 319          src_type = nir_type_float;
 320          dst_type = nir_type_int;
 321          break;
 322       case SpvOpConvertFToU:
 323          src_type = nir_type_float;
 324          dst_type = nir_type_uint;
 325          break;
 326       case SpvOpFConvert:
 327          src_type = dst_type = nir_type_float;
 328          break;
 329       case SpvOpConvertSToF:
 330          src_type = nir_type_int;
 331          dst_type = nir_type_float;
 332          break;
 333       case SpvOpSConvert:
 334          src_type = dst_type = nir_type_int;
 335          break;
 336       case SpvOpConvertUToF:
 337          src_type = nir_type_uint;
 338          dst_type = nir_type_float;
 339          break;
 340       case SpvOpUConvert:
 341          src_type = dst_type = nir_type_uint;
 342          break;
 343       default:
 344          unreachable("Invalid opcode");
 345       }
 346       src_type |= src_bit_size;
 347       dst_type |= dst_bit_size;
 348       return nir_type_conversion_op(src_type, dst_type, nir_rounding_mode_undef);
 349    }
 350    /* Derivatives: */
 351    case SpvOpDPdx:         return nir_op_fddx;
 352    case SpvOpDPdy:         return nir_op_fddy;
 353    case SpvOpDPdxFine:     return nir_op_fddx_fine;
 354    case SpvOpDPdyFine:     return nir_op_fddy_fine;
 355    case SpvOpDPdxCoarse:   return nir_op_fddx_coarse;
 356    case SpvOpDPdyCoarse:   return nir_op_fddy_coarse;
 357
 358    default:
 359       vtn_fail("No NIR equivalent: %u", opcode);
 360    }
 361 }
 362
 363 static void
 364 handle_no_contraction(struct vtn_builder *b, struct vtn_value *val, int member,
 365                       const struct vtn_decoration *dec, void *_void)
 366 {
 367    vtn_assert(dec->scope == VTN_DEC_DECORATION);
 368    if (dec->decoration != SpvDecorationNoContraction)
 369       return;
 370
 371    b->nb.exact = true;
 372 }
 373
 374 static void
 375 handle_rounding_mode(struct vtn_builder *b, struct vtn_value *val, int member,
 376                      const struct vtn_decoration *dec, void *_out_rounding_mode)
 377 {
 378    nir_rounding_mode *out_rounding_mode = _out_rounding_mode;
 379    assert(dec->scope == VTN_DEC_DECORATION);
 380    if (dec->decoration != SpvDecorationFPRoundingMode)
 381       return;
 382    switch (dec->operands[0]) {
 383    case SpvFPRoundingModeRTE:
 384       *out_rounding_mode = nir_rounding_mode_rtne;
 385       break;
 386    case SpvFPRoundingModeRTZ:
 387       *out_rounding_mode = nir_rounding_mode_rtz;
 388       break;
 389    default:
 390       unreachable("Not supported rounding mode");
 391       break;
 392    }
 393 }
 394
 395 static void
 396 handle_no_wrap(struct vtn_builder *b, struct vtn_value *val, int member,
 397                const struct vtn_decoration *dec, void *_alu)
 398 {
 399    nir_alu_instr *alu = _alu;
 400    switch (dec->decoration) {
 401    case SpvDecorationNoSignedWrap:
 402       alu->no_signed_wrap = true;
 403       break;
 404    case SpvDecorationNoUnsignedWrap:
 405       alu->no_unsigned_wrap = true;
 406       break;
 407    default:
 408       /* Do nothing. */
 409       break;
 410    }
 411 }
 412
 413 void
 414 vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
 415                const uint32_t *w, unsigned count)
 416 {
 417    struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
 418    const struct glsl_type *type =
 419       vtn_value(b, w[1], vtn_value_type_type)->type->type;
 420
 421    vtn_foreach_decoration(b, val, handle_no_contraction, NULL);
 422
 423    /* Collect the various SSA sources */
 424    const unsigned num_inputs = count - 3;
 425    struct vtn_ssa_value *vtn_src[4] = { NULL, };
 426    for (unsigned i = 0; i < num_inputs; i++)
 427       vtn_src[i] = vtn_ssa_value(b, w[i + 3]);
 428
 429    if (glsl_type_is_matrix(vtn_src[0]->type) ||
 430        (num_inputs >= 2 && glsl_type_is_matrix(vtn_src[1]->type))) {
 431       vtn_handle_matrix_alu(b, opcode, val, vtn_src[0], vtn_src[1]);
 432       b->nb.exact = b->exact;
 433       return;
 434    }
 435
 436    val->ssa = vtn_create_ssa_value(b, type);
 437    nir_ssa_def *src[4] = { NULL, };
 438    for (unsigned i = 0; i < num_inputs; i++) {
 439       vtn_assert(glsl_type_is_vector_or_scalar(vtn_src[i]->type));
 440       src[i] = vtn_src[i]->def;
 441    }
 442
 443    switch (opcode) {
 444    case SpvOpAny:
 445       if (src[0]->num_components == 1) {
 446          val->ssa->def = nir_mov(&b->nb, src[0]);
 447       } else {
 448          nir_op op;
 449          switch (src[0]->num_components) {
 450          case 2:  op = nir_op_bany_inequal2; break;
 451          case 3:  op = nir_op_bany_inequal3; break;
 452          case 4:  op = nir_op_bany_inequal4; break;
 453          default: vtn_fail("invalid number of components");
 454          }
 455          val->ssa->def = nir_build_alu(&b->nb, op, src[0],
 456                                        nir_imm_false(&b->nb),
 457                                        NULL, NULL);
 458       }
 459       break;
 460
 461    case SpvOpAll:
 462       if (src[0]->num_components == 1) {
 463          val->ssa->def = nir_mov(&b->nb, src[0]);
 464       } else {
 465          nir_op op;
 466          switch (src[0]->num_components) {
 467          case 2:  op = nir_op_ball_iequal2;  break;
 468          case 3:  op = nir_op_ball_iequal3;  break;
 469          case 4:  op = nir_op_ball_iequal4;  break;
 470          default: vtn_fail("invalid number of components");
 471          }
 472          val->ssa->def = nir_build_alu(&b->nb, op, src[0],
 473                                        nir_imm_true(&b->nb),
 474                                        NULL, NULL);
 475       }
 476       break;
 477
 478    case SpvOpOuterProduct: {
 479       for (unsigned i = 0; i < src[1]->num_components; i++) {
 480          val->ssa->elems[i]->def =
 481             nir_fmul(&b->nb, src[0], nir_channel(&b->nb, src[1], i));
 482       }
 483       break;
 484    }
 485
 486    case SpvOpDot:
 487       val->ssa->def = nir_fdot(&b->nb, src[0], src[1]);
 488       break;
 489
 490    case SpvOpIAddCarry:
 491       vtn_assert(glsl_type_is_struct_or_ifc(val->ssa->type));
 492       val->ssa->elems[0]->def = nir_iadd(&b->nb, src[0], src[1]);
 493       val->ssa->elems[1]->def = nir_uadd_carry(&b->nb, src[0], src[1]);
 494       break;
 495
 496    case SpvOpISubBorrow:
 497       vtn_assert(glsl_type_is_struct_or_ifc(val->ssa->type));
 498       val->ssa->elems[0]->def = nir_isub(&b->nb, src[0], src[1]);
 499       val->ssa->elems[1]->def = nir_usub_borrow(&b->nb, src[0], src[1]);
 500       break;
 501
 502    case SpvOpUMulExtended: {
 503       vtn_assert(glsl_type_is_struct_or_ifc(val->ssa->type));
 504       nir_ssa_def *umul = nir_umul_2x32_64(&b->nb, src[0], src[1]);
 505       val->ssa->elems[0]->def = nir_unpack_64_2x32_split_x(&b->nb, umul);
 506       val->ssa->elems[1]->def = nir_unpack_64_2x32_split_y(&b->nb, umul);
 507       break;
 508    }
 509
 510    case SpvOpSMulExtended: {
 511       vtn_assert(glsl_type_is_struct_or_ifc(val->ssa->type));
 512       nir_ssa_def *smul = nir_imul_2x32_64(&b->nb, src[0], src[1]);
 513       val->ssa->elems[0]->def = nir_unpack_64_2x32_split_x(&b->nb, smul);
 514       val->ssa->elems[1]->def = nir_unpack_64_2x32_split_y(&b->nb, smul);
 515       break;
 516    }
 517
 518    case SpvOpFwidth:
 519       val->ssa->def = nir_fadd(&b->nb,
 520                                nir_fabs(&b->nb, nir_fddx(&b->nb, src[0])),
 521                                nir_fabs(&b->nb, nir_fddy(&b->nb, src[0])));
 522       break;
 523    case SpvOpFwidthFine:
 524       val->ssa->def = nir_fadd(&b->nb,
 525                                nir_fabs(&b->nb, nir_fddx_fine(&b->nb, src[0])),
 526                                nir_fabs(&b->nb, nir_fddy_fine(&b->nb, src[0])));
 527       break;
 528    case SpvOpFwidthCoarse:
 529       val->ssa->def = nir_fadd(&b->nb,
 530                                nir_fabs(&b->nb, nir_fddx_coarse(&b->nb, src[0])),
 531                                nir_fabs(&b->nb, nir_fddy_coarse(&b->nb, src[0])));
 532       break;
 533
 534    case SpvOpVectorTimesScalar:
 535       /* The builder will take care of splatting for us. */
 536       val->ssa->def = nir_fmul(&b->nb, src[0], src[1]);
 537       break;
 538
 539    case SpvOpIsNan:
 540       val->ssa->def = nir_fne(&b->nb, src[0], src[0]);
 541       break;
 542
 543    case SpvOpIsInf: {
 544       nir_ssa_def *inf = nir_imm_floatN_t(&b->nb, INFINITY, src[0]->bit_size);
 545       val->ssa->def = nir_ieq(&b->nb, nir_fabs(&b->nb, src[0]), inf);
 546       break;
 547    }
 548
 549    case SpvOpFUnordEqual:
 550    case SpvOpFUnordNotEqual:
 551    case SpvOpFUnordLessThan:
 552    case SpvOpFUnordGreaterThan:
 553    case SpvOpFUnordLessThanEqual:
 554    case SpvOpFUnordGreaterThanEqual: {
 555       bool swap;
 556       unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
 557       unsigned dst_bit_size = glsl_get_bit_size(type);
 558       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 559                                                   src_bit_size, dst_bit_size);
 560
 561       if (swap) {
 562          nir_ssa_def *tmp = src[0];
 563          src[0] = src[1];
 564          src[1] = tmp;
 565       }
 566
 567       val->ssa->def =
 568          nir_ior(&b->nb,
 569                  nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
 570                  nir_ior(&b->nb,
 571                          nir_fne(&b->nb, src[0], src[0]),
 572                          nir_fne(&b->nb, src[1], src[1])));
 573       break;
 574    }
 575
 576    case SpvOpFOrdNotEqual: {
 577       /* For all the SpvOpFOrd* comparisons apart from NotEqual, the value
 578        * from the ALU will probably already be false if the operands are not
 579        * ordered so we don’t need to handle it specially.
 580        */
 581       bool swap;
 582       unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
 583       unsigned dst_bit_size = glsl_get_bit_size(type);
 584       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 585                                                   src_bit_size, dst_bit_size);
 586
 587       assert(!swap);
 588
 589       val->ssa->def =
 590          nir_iand(&b->nb,
 591                   nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
 592                   nir_iand(&b->nb,
 593                           nir_feq(&b->nb, src[0], src[0]),
 594                           nir_feq(&b->nb, src[1], src[1])));
 595       break;
 596    }
 597
 598    case SpvOpFConvert: {
 599       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 600       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 601       nir_rounding_mode rounding_mode = nir_rounding_mode_undef;
 602
 603       vtn_foreach_decoration(b, val, handle_rounding_mode, &rounding_mode);
 604       nir_op op = nir_type_conversion_op(src_alu_type, dst_alu_type, rounding_mode);
 605
 606       val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL);
 607       break;
 608    }
 609
 610    case SpvOpBitFieldInsert:
 611    case SpvOpBitFieldSExtract:
 612    case SpvOpBitFieldUExtract:
 613    case SpvOpShiftLeftLogical:
 614    case SpvOpShiftRightArithmetic:
 615    case SpvOpShiftRightLogical: {
 616       bool swap;
 617       unsigned src0_bit_size = glsl_get_bit_size(vtn_src[0]->type);
 618       unsigned dst_bit_size = glsl_get_bit_size(type);
 619       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 620                                                   src0_bit_size, dst_bit_size);
 621
 622       assert (op == nir_op_ushr || op == nir_op_ishr || op == nir_op_ishl ||
 623               op == nir_op_bitfield_insert || op == nir_op_ubitfield_extract ||
 624               op == nir_op_ibitfield_extract);
 625
 626       for (unsigned i = 0; i < nir_op_infos[op].num_inputs; i++) {
 627          unsigned src_bit_size =
 628             nir_alu_type_get_type_size(nir_op_infos[op].input_types[i]);
 629          if (src_bit_size == 0)
 630             continue;
 631          if (src_bit_size != src[i]->bit_size) {
 632             assert(src_bit_size == 32);
 633             /* Convert the Shift, Offset and Count  operands to 32 bits, which is the bitsize
 634              * supported by the NIR instructions. See discussion here:
 635              *
 636              * https://lists.freedesktop.org/archives/mesa-dev/2018-April/193026.html
 637              */
 638             src[i] = nir_u2u32(&b->nb, src[i]);
 639          }
 640       }
 641       val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
 642       break;
 643    }
 644
 645    case SpvOpSignBitSet: {
 646       unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
 647       if (src[0]->num_components == 1)
 648          val->ssa->def =
 649             nir_ushr(&b->nb, src[0], nir_imm_int(&b->nb, src_bit_size - 1));
 650       else
 651          val->ssa->def =
 652             nir_ishr(&b->nb, src[0], nir_imm_int(&b->nb, src_bit_size - 1));
 653
 654       val->ssa->def = nir_i2b(&b->nb, val->ssa->def);
 655       break;
 656    }
 657
 658    case SpvOpUCountTrailingZerosINTEL:
 659       val->ssa->def = nir_umin(&b->nb,
 660                                nir_find_lsb(&b->nb, src[0]),
 661                                nir_imm_int(&b->nb, 32u));
 662       break;
 663
 664    default: {
 665       bool swap;
 666       unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
 667       unsigned dst_bit_size = glsl_get_bit_size(type);
 668       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 669                                                   src_bit_size, dst_bit_size);
 670
 671       if (swap) {
 672          nir_ssa_def *tmp = src[0];
 673          src[0] = src[1];
 674          src[1] = tmp;
 675       }
 676
 677       switch (op) {
 678       case nir_op_ishl:
 679       case nir_op_ishr:
 680       case nir_op_ushr:
 681          if (src[1]->bit_size != 32)
 682             src[1] = nir_u2u32(&b->nb, src[1]);
 683          break;
 684       default:
 685          break;
 686       }
 687
 688       val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
 689       break;
 690    } /* default */
 691    }
 692
 693    switch (opcode) {
 694    case SpvOpIAdd:
 695    case SpvOpIMul:
 696    case SpvOpISub:
 697    case SpvOpShiftLeftLogical:
 698    case SpvOpSNegate: {
 699       nir_alu_instr *alu = nir_instr_as_alu(val->ssa->def->parent_instr);
 700       vtn_foreach_decoration(b, val, handle_no_wrap, alu);
 701       break;
 702    }
 703    default:
 704       /* Do nothing. */
 705       break;
 706    }
 707
 708    b->nb.exact = b->exact;
 709 }
 710
 711 void
 712 vtn_handle_bitcast(struct vtn_builder *b, const uint32_t *w, unsigned count)
 713 {
 714    vtn_assert(count == 4);
 715    /* From the definition of OpBitcast in the SPIR-V 1.2 spec:
 716     *
 717     *    "If Result Type has the same number of components as Operand, they
 718     *    must also have the same component width, and results are computed per
 719     *    component.
 720     *
 721     *    If Result Type has a different number of components than Operand, the
 722     *    total number of bits in Result Type must equal the total number of
 723     *    bits in Operand. Let L be the type, either Result Type or Operand’s
 724     *    type, that has the larger number of components. Let S be the other
 725     *    type, with the smaller number of components. The number of components
 726     *    in L must be an integer multiple of the number of components in S.
 727     *    The first component (that is, the only or lowest-numbered component)
 728     *    of S maps to the first components of L, and so on, up to the last
 729     *    component of S mapping to the last components of L. Within this
 730     *    mapping, any single component of S (mapping to multiple components of
 731     *    L) maps its lower-ordered bits to the lower-numbered components of L."
 732     */
 733
 734    struct vtn_type *type = vtn_value(b, w[1], vtn_value_type_type)->type;
 735    struct vtn_ssa_value *vtn_src = vtn_ssa_value(b, w[3]);
 736    struct nir_ssa_def *src = vtn_src->def;
 737    struct vtn_ssa_value *val = vtn_create_ssa_value(b, type->type);
 738
 739    vtn_assert(glsl_type_is_vector_or_scalar(vtn_src->type));
 740
 741    vtn_fail_if(src->num_components * src->bit_size !=
 742                glsl_get_vector_elements(type->type) * glsl_get_bit_size(type->type),
 743                "Source and destination of OpBitcast must have the same "
 744                "total number of bits");
 745    val->def = nir_bitcast_vector(&b->nb, src, glsl_get_bit_size(type->type));
 746    vtn_push_ssa(b, w[2], type, val);
 747 }