src/compiler/spirv/vtn_alu.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "vtn_private.h"
  25
  26 /*
  27  * Normally, column vectors in SPIR-V correspond to a single NIR SSA
  28  * definition. But for matrix multiplies, we want to do one routine for
  29  * multiplying a matrix by a matrix and then pretend that vectors are matrices
  30  * with one column. So we "wrap" these things, and unwrap the result before we
  31  * send it off.
  32  */
  33
  34 static struct vtn_ssa_value *
  35 wrap_matrix(struct vtn_builder *b, struct vtn_ssa_value *val)
  36 {
  37    if (val == NULL)
  38       return NULL;
  39
  40    if (glsl_type_is_matrix(val->type))
  41       return val;
  42
  43    struct vtn_ssa_value *dest = rzalloc(b, struct vtn_ssa_value);
  44    dest->type = val->type;
  45    dest->elems = ralloc_array(b, struct vtn_ssa_value *, 1);
  46    dest->elems[0] = val;
  47
  48    return dest;
  49 }
  50
  51 static struct vtn_ssa_value *
  52 unwrap_matrix(struct vtn_ssa_value *val)
  53 {
  54    if (glsl_type_is_matrix(val->type))
  55          return val;
  56
  57    return val->elems[0];
  58 }
  59
  60 static struct vtn_ssa_value *
  61 matrix_multiply(struct vtn_builder *b,
  62                 struct vtn_ssa_value *_src0, struct vtn_ssa_value *_src1)
  63 {
  64
  65    struct vtn_ssa_value *src0 = wrap_matrix(b, _src0);
  66    struct vtn_ssa_value *src1 = wrap_matrix(b, _src1);
  67    struct vtn_ssa_value *src0_transpose = wrap_matrix(b, _src0->transposed);
  68    struct vtn_ssa_value *src1_transpose = wrap_matrix(b, _src1->transposed);
  69
  70    unsigned src0_rows = glsl_get_vector_elements(src0->type);
  71    unsigned src0_columns = glsl_get_matrix_columns(src0->type);
  72    unsigned src1_columns = glsl_get_matrix_columns(src1->type);
  73
  74    const struct glsl_type *dest_type;
  75    if (src1_columns > 1) {
  76       dest_type = glsl_matrix_type(glsl_get_base_type(src0->type),
  77                                    src0_rows, src1_columns);
  78    } else {
  79       dest_type = glsl_vector_type(glsl_get_base_type(src0->type), src0_rows);
  80    }
  81    struct vtn_ssa_value *dest = vtn_create_ssa_value(b, dest_type);
  82
  83    dest = wrap_matrix(b, dest);
  84
  85    bool transpose_result = false;
  86    if (src0_transpose && src1_transpose) {
  87       /* transpose(A) * transpose(B) = transpose(B * A) */
  88       src1 = src0_transpose;
  89       src0 = src1_transpose;
  90       src0_transpose = NULL;
  91       src1_transpose = NULL;
  92       transpose_result = true;
  93    }
  94
  95    if (src0_transpose && !src1_transpose &&
  96        glsl_get_base_type(src0->type) == GLSL_TYPE_FLOAT) {
  97       /* We already have the rows of src0 and the columns of src1 available,
  98        * so we can just take the dot product of each row with each column to
  99        * get the result.
 100        */
 101
 102       for (unsigned i = 0; i < src1_columns; i++) {
 103          nir_ssa_def *vec_src[4];
 104          for (unsigned j = 0; j < src0_rows; j++) {
 105             vec_src[j] = nir_fdot(&b->nb, src0_transpose->elems[j]->def,
 106                                           src1->elems[i]->def);
 107          }
 108          dest->elems[i]->def = nir_vec(&b->nb, vec_src, src0_rows);
 109       }
 110    } else {
 111       /* We don't handle the case where src1 is transposed but not src0, since
 112        * the general case only uses individual components of src1 so the
 113        * optimizer should chew through the transpose we emitted for src1.
 114        */
 115
 116       for (unsigned i = 0; i < src1_columns; i++) {
 117          /* dest[i] = sum(src0[j] * src1[i][j] for all j) */
 118          dest->elems[i]->def =
 119             nir_fmul(&b->nb, src0->elems[0]->def,
 120                      nir_channel(&b->nb, src1->elems[i]->def, 0));
 121          for (unsigned j = 1; j < src0_columns; j++) {
 122             dest->elems[i]->def =
 123                nir_fadd(&b->nb, dest->elems[i]->def,
 124                         nir_fmul(&b->nb, src0->elems[j]->def,
 125                                  nir_channel(&b->nb, src1->elems[i]->def, j)));
 126          }
 127       }
 128    }
 129
 130    dest = unwrap_matrix(dest);
 131
 132    if (transpose_result)
 133       dest = vtn_ssa_transpose(b, dest);
 134
 135    return dest;
 136 }
 137
 138 static struct vtn_ssa_value *
 139 mat_times_scalar(struct vtn_builder *b,
 140                  struct vtn_ssa_value *mat,
 141                  nir_ssa_def *scalar)
 142 {
 143    struct vtn_ssa_value *dest = vtn_create_ssa_value(b, mat->type);
 144    for (unsigned i = 0; i < glsl_get_matrix_columns(mat->type); i++) {
 145       if (glsl_get_base_type(mat->type) == GLSL_TYPE_FLOAT)
 146          dest->elems[i]->def = nir_fmul(&b->nb, mat->elems[i]->def, scalar);
 147       else
 148          dest->elems[i]->def = nir_imul(&b->nb, mat->elems[i]->def, scalar);
 149    }
 150
 151    return dest;
 152 }
 153
 154 static void
 155 vtn_handle_matrix_alu(struct vtn_builder *b, SpvOp opcode,
 156                       struct vtn_value *dest,
 157                       struct vtn_ssa_value *src0, struct vtn_ssa_value *src1)
 158 {
 159    switch (opcode) {
 160    case SpvOpFNegate: {
 161       dest->ssa = vtn_create_ssa_value(b, src0->type);
 162       unsigned cols = glsl_get_matrix_columns(src0->type);
 163       for (unsigned i = 0; i < cols; i++)
 164          dest->ssa->elems[i]->def = nir_fneg(&b->nb, src0->elems[i]->def);
 165       break;
 166    }
 167
 168    case SpvOpFAdd: {
 169       dest->ssa = vtn_create_ssa_value(b, src0->type);
 170       unsigned cols = glsl_get_matrix_columns(src0->type);
 171       for (unsigned i = 0; i < cols; i++)
 172          dest->ssa->elems[i]->def =
 173             nir_fadd(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
 174       break;
 175    }
 176
 177    case SpvOpFSub: {
 178       dest->ssa = vtn_create_ssa_value(b, src0->type);
 179       unsigned cols = glsl_get_matrix_columns(src0->type);
 180       for (unsigned i = 0; i < cols; i++)
 181          dest->ssa->elems[i]->def =
 182             nir_fsub(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
 183       break;
 184    }
 185
 186    case SpvOpTranspose:
 187       dest->ssa = vtn_ssa_transpose(b, src0);
 188       break;
 189
 190    case SpvOpMatrixTimesScalar:
 191       if (src0->transposed) {
 192          dest->ssa = vtn_ssa_transpose(b, mat_times_scalar(b, src0->transposed,
 193                                                            src1->def));
 194       } else {
 195          dest->ssa = mat_times_scalar(b, src0, src1->def);
 196       }
 197       break;
 198
 199    case SpvOpVectorTimesMatrix:
 200    case SpvOpMatrixTimesVector:
 201    case SpvOpMatrixTimesMatrix:
 202       if (opcode == SpvOpVectorTimesMatrix) {
 203          dest->ssa = matrix_multiply(b, vtn_ssa_transpose(b, src1), src0);
 204       } else {
 205          dest->ssa = matrix_multiply(b, src0, src1);
 206       }
 207       break;
 208
 209    default: vtn_fail("unknown matrix opcode");
 210    }
 211 }
 212
 213 static void
 214 vtn_handle_bitcast(struct vtn_builder *b, struct vtn_ssa_value *dest,
 215                    struct nir_ssa_def *src)
 216 {
 217    if (glsl_get_vector_elements(dest->type) == src->num_components) {
 218       /* From the definition of OpBitcast in the SPIR-V 1.2 spec:
 219        *
 220        * "If Result Type has the same number of components as Operand, they
 221        * must also have the same component width, and results are computed per
 222        * component."
 223        */
 224       dest->def = nir_imov(&b->nb, src);
 225       return;
 226    }
 227
 228    /* From the definition of OpBitcast in the SPIR-V 1.2 spec:
 229     *
 230     * "If Result Type has a different number of components than Operand, the
 231     * total number of bits in Result Type must equal the total number of bits
 232     * in Operand. Let L be the type, either Result Type or Operand’s type, that
 233     * has the larger number of components. Let S be the other type, with the
 234     * smaller number of components. The number of components in L must be an
 235     * integer multiple of the number of components in S. The first component
 236     * (that is, the only or lowest-numbered component) of S maps to the first
 237     * components of L, and so on, up to the last component of S mapping to the
 238     * last components of L. Within this mapping, any single component of S
 239     * (mapping to multiple components of L) maps its lower-ordered bits to the
 240     * lower-numbered components of L."
 241     */
 242    unsigned src_bit_size = src->bit_size;
 243    unsigned dest_bit_size = glsl_get_bit_size(dest->type);
 244    unsigned src_components = src->num_components;
 245    unsigned dest_components = glsl_get_vector_elements(dest->type);
 246    vtn_assert(src_bit_size * src_components == dest_bit_size * dest_components);
 247
 248    nir_ssa_def *dest_chan[4];
 249    if (src_bit_size > dest_bit_size) {
 250       vtn_assert(src_bit_size % dest_bit_size == 0);
 251       unsigned divisor = src_bit_size / dest_bit_size;
 252       for (unsigned comp = 0; comp < src_components; comp++) {
 253          vtn_assert(src_bit_size == 64);
 254          vtn_assert(dest_bit_size == 32);
 255          nir_ssa_def *split =
 256             nir_unpack_64_2x32(&b->nb, nir_channel(&b->nb, src, comp));
 257          for (unsigned i = 0; i < divisor; i++)
 258             dest_chan[divisor * comp + i] = nir_channel(&b->nb, split, i);
 259       }
 260    } else {
 261       vtn_assert(dest_bit_size % src_bit_size == 0);
 262       unsigned divisor = dest_bit_size / src_bit_size;
 263       for (unsigned comp = 0; comp < dest_components; comp++) {
 264          unsigned channels = ((1 << divisor) - 1) << (comp * divisor);
 265          nir_ssa_def *src_chan =
 266             nir_channels(&b->nb, src, channels);
 267          vtn_assert(dest_bit_size == 64);
 268          vtn_assert(src_bit_size == 32);
 269          dest_chan[comp] = nir_pack_64_2x32(&b->nb, src_chan);
 270       }
 271    }
 272    dest->def = nir_vec(&b->nb, dest_chan, dest_components);
 273 }
 274
 275 nir_op
 276 vtn_nir_alu_op_for_spirv_opcode(struct vtn_builder *b,
 277                                 SpvOp opcode, bool *swap,
 278                                 nir_alu_type src, nir_alu_type dst)
 279 {
 280    /* Indicates that the first two arguments should be swapped.  This is
 281     * used for implementing greater-than and less-than-or-equal.
 282     */
 283    *swap = false;
 284
 285    switch (opcode) {
 286    case SpvOpSNegate:            return nir_op_ineg;
 287    case SpvOpFNegate:            return nir_op_fneg;
 288    case SpvOpNot:                return nir_op_inot;
 289    case SpvOpIAdd:               return nir_op_iadd;
 290    case SpvOpFAdd:               return nir_op_fadd;
 291    case SpvOpISub:               return nir_op_isub;
 292    case SpvOpFSub:               return nir_op_fsub;
 293    case SpvOpIMul:               return nir_op_imul;
 294    case SpvOpFMul:               return nir_op_fmul;
 295    case SpvOpUDiv:               return nir_op_udiv;
 296    case SpvOpSDiv:               return nir_op_idiv;
 297    case SpvOpFDiv:               return nir_op_fdiv;
 298    case SpvOpUMod:               return nir_op_umod;
 299    case SpvOpSMod:               return nir_op_imod;
 300    case SpvOpFMod:               return nir_op_fmod;
 301    case SpvOpSRem:               return nir_op_irem;
 302    case SpvOpFRem:               return nir_op_frem;
 303
 304    case SpvOpShiftRightLogical:     return nir_op_ushr;
 305    case SpvOpShiftRightArithmetic:  return nir_op_ishr;
 306    case SpvOpShiftLeftLogical:      return nir_op_ishl;
 307    case SpvOpLogicalOr:             return nir_op_ior;
 308    case SpvOpLogicalEqual:          return nir_op_ieq;
 309    case SpvOpLogicalNotEqual:       return nir_op_ine;
 310    case SpvOpLogicalAnd:            return nir_op_iand;
 311    case SpvOpLogicalNot:            return nir_op_inot;
 312    case SpvOpBitwiseOr:             return nir_op_ior;
 313    case SpvOpBitwiseXor:            return nir_op_ixor;
 314    case SpvOpBitwiseAnd:            return nir_op_iand;
 315    case SpvOpSelect:                return nir_op_bcsel;
 316    case SpvOpIEqual:                return nir_op_ieq;
 317
 318    case SpvOpBitFieldInsert:        return nir_op_bitfield_insert;
 319    case SpvOpBitFieldSExtract:      return nir_op_ibitfield_extract;
 320    case SpvOpBitFieldUExtract:      return nir_op_ubitfield_extract;
 321    case SpvOpBitReverse:            return nir_op_bitfield_reverse;
 322    case SpvOpBitCount:              return nir_op_bit_count;
 323
 324    /* The ordered / unordered operators need special implementation besides
 325     * the logical operator to use since they also need to check if operands are
 326     * ordered.
 327     */
 328    case SpvOpFOrdEqual:                            return nir_op_feq;
 329    case SpvOpFUnordEqual:                          return nir_op_feq;
 330    case SpvOpINotEqual:                            return nir_op_ine;
 331    case SpvOpFOrdNotEqual:                         return nir_op_fne;
 332    case SpvOpFUnordNotEqual:                       return nir_op_fne;
 333    case SpvOpULessThan:                            return nir_op_ult;
 334    case SpvOpSLessThan:                            return nir_op_ilt;
 335    case SpvOpFOrdLessThan:                         return nir_op_flt;
 336    case SpvOpFUnordLessThan:                       return nir_op_flt;
 337    case SpvOpUGreaterThan:          *swap = true;  return nir_op_ult;
 338    case SpvOpSGreaterThan:          *swap = true;  return nir_op_ilt;
 339    case SpvOpFOrdGreaterThan:       *swap = true;  return nir_op_flt;
 340    case SpvOpFUnordGreaterThan:     *swap = true;  return nir_op_flt;
 341    case SpvOpULessThanEqual:        *swap = true;  return nir_op_uge;
 342    case SpvOpSLessThanEqual:        *swap = true;  return nir_op_ige;
 343    case SpvOpFOrdLessThanEqual:     *swap = true;  return nir_op_fge;
 344    case SpvOpFUnordLessThanEqual:   *swap = true;  return nir_op_fge;
 345    case SpvOpUGreaterThanEqual:                    return nir_op_uge;
 346    case SpvOpSGreaterThanEqual:                    return nir_op_ige;
 347    case SpvOpFOrdGreaterThanEqual:                 return nir_op_fge;
 348    case SpvOpFUnordGreaterThanEqual:               return nir_op_fge;
 349
 350    /* Conversions: */
 351    case SpvOpQuantizeToF16:         return nir_op_fquantize2f16;
 352    case SpvOpUConvert:
 353    case SpvOpConvertFToU:
 354    case SpvOpConvertFToS:
 355    case SpvOpConvertSToF:
 356    case SpvOpConvertUToF:
 357    case SpvOpSConvert:
 358    case SpvOpFConvert:
 359       return nir_type_conversion_op(src, dst);
 360
 361    /* Derivatives: */
 362    case SpvOpDPdx:         return nir_op_fddx;
 363    case SpvOpDPdy:         return nir_op_fddy;
 364    case SpvOpDPdxFine:     return nir_op_fddx_fine;
 365    case SpvOpDPdyFine:     return nir_op_fddy_fine;
 366    case SpvOpDPdxCoarse:   return nir_op_fddx_coarse;
 367    case SpvOpDPdyCoarse:   return nir_op_fddy_coarse;
 368
 369    default:
 370       vtn_fail("No NIR equivalent");
 371    }
 372 }
 373
 374 static void
 375 handle_no_contraction(struct vtn_builder *b, struct vtn_value *val, int member,
 376                       const struct vtn_decoration *dec, void *_void)
 377 {
 378    vtn_assert(dec->scope == VTN_DEC_DECORATION);
 379    if (dec->decoration != SpvDecorationNoContraction)
 380       return;
 381
 382    b->nb.exact = true;
 383 }
 384
 385 void
 386 vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
 387                const uint32_t *w, unsigned count)
 388 {
 389    struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
 390    const struct glsl_type *type =
 391       vtn_value(b, w[1], vtn_value_type_type)->type->type;
 392
 393    vtn_foreach_decoration(b, val, handle_no_contraction, NULL);
 394
 395    /* Collect the various SSA sources */
 396    const unsigned num_inputs = count - 3;
 397    struct vtn_ssa_value *vtn_src[4] = { NULL, };
 398    for (unsigned i = 0; i < num_inputs; i++)
 399       vtn_src[i] = vtn_ssa_value(b, w[i + 3]);
 400
 401    if (glsl_type_is_matrix(vtn_src[0]->type) ||
 402        (num_inputs >= 2 && glsl_type_is_matrix(vtn_src[1]->type))) {
 403       vtn_handle_matrix_alu(b, opcode, val, vtn_src[0], vtn_src[1]);
 404       b->nb.exact = false;
 405       return;
 406    }
 407
 408    val->ssa = vtn_create_ssa_value(b, type);
 409    nir_ssa_def *src[4] = { NULL, };
 410    for (unsigned i = 0; i < num_inputs; i++) {
 411       vtn_assert(glsl_type_is_vector_or_scalar(vtn_src[i]->type));
 412       src[i] = vtn_src[i]->def;
 413    }
 414
 415    switch (opcode) {
 416    case SpvOpAny:
 417       if (src[0]->num_components == 1) {
 418          val->ssa->def = nir_imov(&b->nb, src[0]);
 419       } else {
 420          nir_op op;
 421          switch (src[0]->num_components) {
 422          case 2:  op = nir_op_bany_inequal2; break;
 423          case 3:  op = nir_op_bany_inequal3; break;
 424          case 4:  op = nir_op_bany_inequal4; break;
 425          default: vtn_fail("invalid number of components");
 426          }
 427          val->ssa->def = nir_build_alu(&b->nb, op, src[0],
 428                                        nir_imm_int(&b->nb, NIR_FALSE),
 429                                        NULL, NULL);
 430       }
 431       break;
 432
 433    case SpvOpAll:
 434       if (src[0]->num_components == 1) {
 435          val->ssa->def = nir_imov(&b->nb, src[0]);
 436       } else {
 437          nir_op op;
 438          switch (src[0]->num_components) {
 439          case 2:  op = nir_op_ball_iequal2;  break;
 440          case 3:  op = nir_op_ball_iequal3;  break;
 441          case 4:  op = nir_op_ball_iequal4;  break;
 442          default: vtn_fail("invalid number of components");
 443          }
 444          val->ssa->def = nir_build_alu(&b->nb, op, src[0],
 445                                        nir_imm_int(&b->nb, NIR_TRUE),
 446                                        NULL, NULL);
 447       }
 448       break;
 449
 450    case SpvOpOuterProduct: {
 451       for (unsigned i = 0; i < src[1]->num_components; i++) {
 452          val->ssa->elems[i]->def =
 453             nir_fmul(&b->nb, src[0], nir_channel(&b->nb, src[1], i));
 454       }
 455       break;
 456    }
 457
 458    case SpvOpDot:
 459       val->ssa->def = nir_fdot(&b->nb, src[0], src[1]);
 460       break;
 461
 462    case SpvOpIAddCarry:
 463       vtn_assert(glsl_type_is_struct(val->ssa->type));
 464       val->ssa->elems[0]->def = nir_iadd(&b->nb, src[0], src[1]);
 465       val->ssa->elems[1]->def = nir_uadd_carry(&b->nb, src[0], src[1]);
 466       break;
 467
 468    case SpvOpISubBorrow:
 469       vtn_assert(glsl_type_is_struct(val->ssa->type));
 470       val->ssa->elems[0]->def = nir_isub(&b->nb, src[0], src[1]);
 471       val->ssa->elems[1]->def = nir_usub_borrow(&b->nb, src[0], src[1]);
 472       break;
 473
 474    case SpvOpUMulExtended:
 475       vtn_assert(glsl_type_is_struct(val->ssa->type));
 476       val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
 477       val->ssa->elems[1]->def = nir_umul_high(&b->nb, src[0], src[1]);
 478       break;
 479
 480    case SpvOpSMulExtended:
 481       vtn_assert(glsl_type_is_struct(val->ssa->type));
 482       val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
 483       val->ssa->elems[1]->def = nir_imul_high(&b->nb, src[0], src[1]);
 484       break;
 485
 486    case SpvOpFwidth:
 487       val->ssa->def = nir_fadd(&b->nb,
 488                                nir_fabs(&b->nb, nir_fddx(&b->nb, src[0])),
 489                                nir_fabs(&b->nb, nir_fddy(&b->nb, src[0])));
 490       break;
 491    case SpvOpFwidthFine:
 492       val->ssa->def = nir_fadd(&b->nb,
 493                                nir_fabs(&b->nb, nir_fddx_fine(&b->nb, src[0])),
 494                                nir_fabs(&b->nb, nir_fddy_fine(&b->nb, src[0])));
 495       break;
 496    case SpvOpFwidthCoarse:
 497       val->ssa->def = nir_fadd(&b->nb,
 498                                nir_fabs(&b->nb, nir_fddx_coarse(&b->nb, src[0])),
 499                                nir_fabs(&b->nb, nir_fddy_coarse(&b->nb, src[0])));
 500       break;
 501
 502    case SpvOpVectorTimesScalar:
 503       /* The builder will take care of splatting for us. */
 504       val->ssa->def = nir_fmul(&b->nb, src[0], src[1]);
 505       break;
 506
 507    case SpvOpIsNan:
 508       val->ssa->def = nir_fne(&b->nb, src[0], src[0]);
 509       break;
 510
 511    case SpvOpIsInf:
 512       val->ssa->def = nir_ieq(&b->nb, nir_fabs(&b->nb, src[0]),
 513                                       nir_imm_float(&b->nb, INFINITY));
 514       break;
 515
 516    case SpvOpFUnordEqual:
 517    case SpvOpFUnordNotEqual:
 518    case SpvOpFUnordLessThan:
 519    case SpvOpFUnordGreaterThan:
 520    case SpvOpFUnordLessThanEqual:
 521    case SpvOpFUnordGreaterThanEqual: {
 522       bool swap;
 523       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 524       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 525       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 526                                                   src_alu_type, dst_alu_type);
 527
 528       if (swap) {
 529          nir_ssa_def *tmp = src[0];
 530          src[0] = src[1];
 531          src[1] = tmp;
 532       }
 533
 534       val->ssa->def =
 535          nir_ior(&b->nb,
 536                  nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
 537                  nir_ior(&b->nb,
 538                          nir_fne(&b->nb, src[0], src[0]),
 539                          nir_fne(&b->nb, src[1], src[1])));
 540       break;
 541    }
 542
 543    case SpvOpFOrdEqual:
 544    case SpvOpFOrdNotEqual:
 545    case SpvOpFOrdLessThan:
 546    case SpvOpFOrdGreaterThan:
 547    case SpvOpFOrdLessThanEqual:
 548    case SpvOpFOrdGreaterThanEqual: {
 549       bool swap;
 550       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 551       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 552       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 553                                                   src_alu_type, dst_alu_type);
 554
 555       if (swap) {
 556          nir_ssa_def *tmp = src[0];
 557          src[0] = src[1];
 558          src[1] = tmp;
 559       }
 560
 561       val->ssa->def =
 562          nir_iand(&b->nb,
 563                   nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
 564                   nir_iand(&b->nb,
 565                           nir_feq(&b->nb, src[0], src[0]),
 566                           nir_feq(&b->nb, src[1], src[1])));
 567       break;
 568    }
 569
 570    case SpvOpBitcast:
 571       vtn_handle_bitcast(b, val->ssa, src[0]);
 572       break;
 573
 574    default: {
 575       bool swap;
 576       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 577       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 578       nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
 579                                                   src_alu_type, dst_alu_type);
 580
 581       if (swap) {
 582          nir_ssa_def *tmp = src[0];
 583          src[0] = src[1];
 584          src[1] = tmp;
 585       }
 586
 587       val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
 588       break;
 589    } /* default */
 590    }
 591
 592    b->nb.exact = false;
 593 }