src/compiler/spirv/vtn_alu.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "vtn_private.h"
  25
  26 /*
  27  * Normally, column vectors in SPIR-V correspond to a single NIR SSA
  28  * definition. But for matrix multiplies, we want to do one routine for
  29  * multiplying a matrix by a matrix and then pretend that vectors are matrices
  30  * with one column. So we "wrap" these things, and unwrap the result before we
  31  * send it off.
  32  */
  33
  34 static struct vtn_ssa_value *
  35 wrap_matrix(struct vtn_builder *b, struct vtn_ssa_value *val)
  36 {
  37    if (val == NULL)
  38       return NULL;
  39
  40    if (glsl_type_is_matrix(val->type))
  41       return val;
  42
  43    struct vtn_ssa_value *dest = rzalloc(b, struct vtn_ssa_value);
  44    dest->type = val->type;
  45    dest->elems = ralloc_array(b, struct vtn_ssa_value *, 1);
  46    dest->elems[0] = val;
  47
  48    return dest;
  49 }
  50
  51 static struct vtn_ssa_value *
  52 unwrap_matrix(struct vtn_ssa_value *val)
  53 {
  54    if (glsl_type_is_matrix(val->type))
  55          return val;
  56
  57    return val->elems[0];
  58 }
  59
  60 static struct vtn_ssa_value *
  61 matrix_multiply(struct vtn_builder *b,
  62                 struct vtn_ssa_value *_src0, struct vtn_ssa_value *_src1)
  63 {
  64
  65    struct vtn_ssa_value *src0 = wrap_matrix(b, _src0);
  66    struct vtn_ssa_value *src1 = wrap_matrix(b, _src1);
  67    struct vtn_ssa_value *src0_transpose = wrap_matrix(b, _src0->transposed);
  68    struct vtn_ssa_value *src1_transpose = wrap_matrix(b, _src1->transposed);
  69
  70    unsigned src0_rows = glsl_get_vector_elements(src0->type);
  71    unsigned src0_columns = glsl_get_matrix_columns(src0->type);
  72    unsigned src1_columns = glsl_get_matrix_columns(src1->type);
  73
  74    const struct glsl_type *dest_type;
  75    if (src1_columns > 1) {
  76       dest_type = glsl_matrix_type(glsl_get_base_type(src0->type),
  77                                    src0_rows, src1_columns);
  78    } else {
  79       dest_type = glsl_vector_type(glsl_get_base_type(src0->type), src0_rows);
  80    }
  81    struct vtn_ssa_value *dest = vtn_create_ssa_value(b, dest_type);
  82
  83    dest = wrap_matrix(b, dest);
  84
  85    bool transpose_result = false;
  86    if (src0_transpose && src1_transpose) {
  87       /* transpose(A) * transpose(B) = transpose(B * A) */
  88       src1 = src0_transpose;
  89       src0 = src1_transpose;
  90       src0_transpose = NULL;
  91       src1_transpose = NULL;
  92       transpose_result = true;
  93    }
  94
  95    if (src0_transpose && !src1_transpose &&
  96        glsl_get_base_type(src0->type) == GLSL_TYPE_FLOAT) {
  97       /* We already have the rows of src0 and the columns of src1 available,
  98        * so we can just take the dot product of each row with each column to
  99        * get the result.
 100        */
 101
 102       for (unsigned i = 0; i < src1_columns; i++) {
 103          nir_ssa_def *vec_src[4];
 104          for (unsigned j = 0; j < src0_rows; j++) {
 105             vec_src[j] = nir_fdot(&b->nb, src0_transpose->elems[j]->def,
 106                                           src1->elems[i]->def);
 107          }
 108          dest->elems[i]->def = nir_vec(&b->nb, vec_src, src0_rows);
 109       }
 110    } else {
 111       /* We don't handle the case where src1 is transposed but not src0, since
 112        * the general case only uses individual components of src1 so the
 113        * optimizer should chew through the transpose we emitted for src1.
 114        */
 115
 116       for (unsigned i = 0; i < src1_columns; i++) {
 117          /* dest[i] = sum(src0[j] * src1[i][j] for all j) */
 118          dest->elems[i]->def =
 119             nir_fmul(&b->nb, src0->elems[0]->def,
 120                      nir_channel(&b->nb, src1->elems[i]->def, 0));
 121          for (unsigned j = 1; j < src0_columns; j++) {
 122             dest->elems[i]->def =
 123                nir_fadd(&b->nb, dest->elems[i]->def,
 124                         nir_fmul(&b->nb, src0->elems[j]->def,
 125                                  nir_channel(&b->nb, src1->elems[i]->def, j)));
 126          }
 127       }
 128    }
 129
 130    dest = unwrap_matrix(dest);
 131
 132    if (transpose_result)
 133       dest = vtn_ssa_transpose(b, dest);
 134
 135    return dest;
 136 }
 137
 138 static struct vtn_ssa_value *
 139 mat_times_scalar(struct vtn_builder *b,
 140                  struct vtn_ssa_value *mat,
 141                  nir_ssa_def *scalar)
 142 {
 143    struct vtn_ssa_value *dest = vtn_create_ssa_value(b, mat->type);
 144    for (unsigned i = 0; i < glsl_get_matrix_columns(mat->type); i++) {
 145       if (glsl_get_base_type(mat->type) == GLSL_TYPE_FLOAT)
 146          dest->elems[i]->def = nir_fmul(&b->nb, mat->elems[i]->def, scalar);
 147       else
 148          dest->elems[i]->def = nir_imul(&b->nb, mat->elems[i]->def, scalar);
 149    }
 150
 151    return dest;
 152 }
 153
 154 static void
 155 vtn_handle_matrix_alu(struct vtn_builder *b, SpvOp opcode,
 156                       struct vtn_value *dest,
 157                       struct vtn_ssa_value *src0, struct vtn_ssa_value *src1)
 158 {
 159    switch (opcode) {
 160    case SpvOpFNegate: {
 161       dest->ssa = vtn_create_ssa_value(b, src0->type);
 162       unsigned cols = glsl_get_matrix_columns(src0->type);
 163       for (unsigned i = 0; i < cols; i++)
 164          dest->ssa->elems[i]->def = nir_fneg(&b->nb, src0->elems[i]->def);
 165       break;
 166    }
 167
 168    case SpvOpFAdd: {
 169       dest->ssa = vtn_create_ssa_value(b, src0->type);
 170       unsigned cols = glsl_get_matrix_columns(src0->type);
 171       for (unsigned i = 0; i < cols; i++)
 172          dest->ssa->elems[i]->def =
 173             nir_fadd(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
 174       break;
 175    }
 176
 177    case SpvOpFSub: {
 178       dest->ssa = vtn_create_ssa_value(b, src0->type);
 179       unsigned cols = glsl_get_matrix_columns(src0->type);
 180       for (unsigned i = 0; i < cols; i++)
 181          dest->ssa->elems[i]->def =
 182             nir_fsub(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
 183       break;
 184    }
 185
 186    case SpvOpTranspose:
 187       dest->ssa = vtn_ssa_transpose(b, src0);
 188       break;
 189
 190    case SpvOpMatrixTimesScalar:
 191       if (src0->transposed) {
 192          dest->ssa = vtn_ssa_transpose(b, mat_times_scalar(b, src0->transposed,
 193                                                            src1->def));
 194       } else {
 195          dest->ssa = mat_times_scalar(b, src0, src1->def);
 196       }
 197       break;
 198
 199    case SpvOpVectorTimesMatrix:
 200    case SpvOpMatrixTimesVector:
 201    case SpvOpMatrixTimesMatrix:
 202       if (opcode == SpvOpVectorTimesMatrix) {
 203          dest->ssa = matrix_multiply(b, vtn_ssa_transpose(b, src1), src0);
 204       } else {
 205          dest->ssa = matrix_multiply(b, src0, src1);
 206       }
 207       break;
 208
 209    default: unreachable("unknown matrix opcode");
 210    }
 211 }
 212
 213 static void
 214 vtn_handle_bitcast(struct vtn_builder *b, struct vtn_ssa_value *dest,
 215                    struct nir_ssa_def *src)
 216 {
 217    if (glsl_get_vector_elements(dest->type) == src->num_components) {
 218       /* From the definition of OpBitcast in the SPIR-V 1.2 spec:
 219        *
 220        * "If Result Type has the same number of components as Operand, they
 221        * must also have the same component width, and results are computed per
 222        * component."
 223        */
 224       dest->def = nir_imov(&b->nb, src);
 225       return;
 226    }
 227
 228    /* From the definition of OpBitcast in the SPIR-V 1.2 spec:
 229     *
 230     * "If Result Type has a different number of components than Operand, the
 231     * total number of bits in Result Type must equal the total number of bits
 232     * in Operand. Let L be the type, either Result Type or Operand’s type, that
 233     * has the larger number of components. Let S be the other type, with the
 234     * smaller number of components. The number of components in L must be an
 235     * integer multiple of the number of components in S. The first component
 236     * (that is, the only or lowest-numbered component) of S maps to the first
 237     * components of L, and so on, up to the last component of S mapping to the
 238     * last components of L. Within this mapping, any single component of S
 239     * (mapping to multiple components of L) maps its lower-ordered bits to the
 240     * lower-numbered components of L."
 241     */
 242    unsigned src_bit_size = src->bit_size;
 243    unsigned dest_bit_size = glsl_get_bit_size(dest->type);
 244    unsigned src_components = src->num_components;
 245    unsigned dest_components = glsl_get_vector_elements(dest->type);
 246    vtn_assert(src_bit_size * src_components == dest_bit_size * dest_components);
 247
 248    nir_ssa_def *dest_chan[4];
 249    if (src_bit_size > dest_bit_size) {
 250       vtn_assert(src_bit_size % dest_bit_size == 0);
 251       unsigned divisor = src_bit_size / dest_bit_size;
 252       for (unsigned comp = 0; comp < src_components; comp++) {
 253          vtn_assert(src_bit_size == 64);
 254          vtn_assert(dest_bit_size == 32);
 255          nir_ssa_def *split =
 256             nir_unpack_64_2x32(&b->nb, nir_channel(&b->nb, src, comp));
 257          for (unsigned i = 0; i < divisor; i++)
 258             dest_chan[divisor * comp + i] = nir_channel(&b->nb, split, i);
 259       }
 260    } else {
 261       vtn_assert(dest_bit_size % src_bit_size == 0);
 262       unsigned divisor = dest_bit_size / src_bit_size;
 263       for (unsigned comp = 0; comp < dest_components; comp++) {
 264          unsigned channels = ((1 << divisor) - 1) << (comp * divisor);
 265          nir_ssa_def *src_chan =
 266             nir_channels(&b->nb, src, channels);
 267          vtn_assert(dest_bit_size == 64);
 268          vtn_assert(src_bit_size == 32);
 269          dest_chan[comp] = nir_pack_64_2x32(&b->nb, src_chan);
 270       }
 271    }
 272    dest->def = nir_vec(&b->nb, dest_chan, dest_components);
 273 }
 274
 275 nir_op
 276 vtn_nir_alu_op_for_spirv_opcode(SpvOp opcode, bool *swap,
 277                                 nir_alu_type src, nir_alu_type dst)
 278 {
 279    /* Indicates that the first two arguments should be swapped.  This is
 280     * used for implementing greater-than and less-than-or-equal.
 281     */
 282    *swap = false;
 283
 284    switch (opcode) {
 285    case SpvOpSNegate:            return nir_op_ineg;
 286    case SpvOpFNegate:            return nir_op_fneg;
 287    case SpvOpNot:                return nir_op_inot;
 288    case SpvOpIAdd:               return nir_op_iadd;
 289    case SpvOpFAdd:               return nir_op_fadd;
 290    case SpvOpISub:               return nir_op_isub;
 291    case SpvOpFSub:               return nir_op_fsub;
 292    case SpvOpIMul:               return nir_op_imul;
 293    case SpvOpFMul:               return nir_op_fmul;
 294    case SpvOpUDiv:               return nir_op_udiv;
 295    case SpvOpSDiv:               return nir_op_idiv;
 296    case SpvOpFDiv:               return nir_op_fdiv;
 297    case SpvOpUMod:               return nir_op_umod;
 298    case SpvOpSMod:               return nir_op_imod;
 299    case SpvOpFMod:               return nir_op_fmod;
 300    case SpvOpSRem:               return nir_op_irem;
 301    case SpvOpFRem:               return nir_op_frem;
 302
 303    case SpvOpShiftRightLogical:     return nir_op_ushr;
 304    case SpvOpShiftRightArithmetic:  return nir_op_ishr;
 305    case SpvOpShiftLeftLogical:      return nir_op_ishl;
 306    case SpvOpLogicalOr:             return nir_op_ior;
 307    case SpvOpLogicalEqual:          return nir_op_ieq;
 308    case SpvOpLogicalNotEqual:       return nir_op_ine;
 309    case SpvOpLogicalAnd:            return nir_op_iand;
 310    case SpvOpLogicalNot:            return nir_op_inot;
 311    case SpvOpBitwiseOr:             return nir_op_ior;
 312    case SpvOpBitwiseXor:            return nir_op_ixor;
 313    case SpvOpBitwiseAnd:            return nir_op_iand;
 314    case SpvOpSelect:                return nir_op_bcsel;
 315    case SpvOpIEqual:                return nir_op_ieq;
 316
 317    case SpvOpBitFieldInsert:        return nir_op_bitfield_insert;
 318    case SpvOpBitFieldSExtract:      return nir_op_ibitfield_extract;
 319    case SpvOpBitFieldUExtract:      return nir_op_ubitfield_extract;
 320    case SpvOpBitReverse:            return nir_op_bitfield_reverse;
 321    case SpvOpBitCount:              return nir_op_bit_count;
 322
 323    /* The ordered / unordered operators need special implementation besides
 324     * the logical operator to use since they also need to check if operands are
 325     * ordered.
 326     */
 327    case SpvOpFOrdEqual:                            return nir_op_feq;
 328    case SpvOpFUnordEqual:                          return nir_op_feq;
 329    case SpvOpINotEqual:                            return nir_op_ine;
 330    case SpvOpFOrdNotEqual:                         return nir_op_fne;
 331    case SpvOpFUnordNotEqual:                       return nir_op_fne;
 332    case SpvOpULessThan:                            return nir_op_ult;
 333    case SpvOpSLessThan:                            return nir_op_ilt;
 334    case SpvOpFOrdLessThan:                         return nir_op_flt;
 335    case SpvOpFUnordLessThan:                       return nir_op_flt;
 336    case SpvOpUGreaterThan:          *swap = true;  return nir_op_ult;
 337    case SpvOpSGreaterThan:          *swap = true;  return nir_op_ilt;
 338    case SpvOpFOrdGreaterThan:       *swap = true;  return nir_op_flt;
 339    case SpvOpFUnordGreaterThan:     *swap = true;  return nir_op_flt;
 340    case SpvOpULessThanEqual:        *swap = true;  return nir_op_uge;
 341    case SpvOpSLessThanEqual:        *swap = true;  return nir_op_ige;
 342    case SpvOpFOrdLessThanEqual:     *swap = true;  return nir_op_fge;
 343    case SpvOpFUnordLessThanEqual:   *swap = true;  return nir_op_fge;
 344    case SpvOpUGreaterThanEqual:                    return nir_op_uge;
 345    case SpvOpSGreaterThanEqual:                    return nir_op_ige;
 346    case SpvOpFOrdGreaterThanEqual:                 return nir_op_fge;
 347    case SpvOpFUnordGreaterThanEqual:               return nir_op_fge;
 348
 349    /* Conversions: */
 350    case SpvOpQuantizeToF16:         return nir_op_fquantize2f16;
 351    case SpvOpUConvert:
 352    case SpvOpConvertFToU:
 353    case SpvOpConvertFToS:
 354    case SpvOpConvertSToF:
 355    case SpvOpConvertUToF:
 356    case SpvOpSConvert:
 357    case SpvOpFConvert:
 358       return nir_type_conversion_op(src, dst);
 359
 360    /* Derivatives: */
 361    case SpvOpDPdx:         return nir_op_fddx;
 362    case SpvOpDPdy:         return nir_op_fddy;
 363    case SpvOpDPdxFine:     return nir_op_fddx_fine;
 364    case SpvOpDPdyFine:     return nir_op_fddy_fine;
 365    case SpvOpDPdxCoarse:   return nir_op_fddx_coarse;
 366    case SpvOpDPdyCoarse:   return nir_op_fddy_coarse;
 367
 368    default:
 369       unreachable("No NIR equivalent");
 370    }
 371 }
 372
 373 static void
 374 handle_no_contraction(struct vtn_builder *b, struct vtn_value *val, int member,
 375                       const struct vtn_decoration *dec, void *_void)
 376 {
 377    vtn_assert(dec->scope == VTN_DEC_DECORATION);
 378    if (dec->decoration != SpvDecorationNoContraction)
 379       return;
 380
 381    b->nb.exact = true;
 382 }
 383
 384 void
 385 vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
 386                const uint32_t *w, unsigned count)
 387 {
 388    struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
 389    const struct glsl_type *type =
 390       vtn_value(b, w[1], vtn_value_type_type)->type->type;
 391
 392    vtn_foreach_decoration(b, val, handle_no_contraction, NULL);
 393
 394    /* Collect the various SSA sources */
 395    const unsigned num_inputs = count - 3;
 396    struct vtn_ssa_value *vtn_src[4] = { NULL, };
 397    for (unsigned i = 0; i < num_inputs; i++)
 398       vtn_src[i] = vtn_ssa_value(b, w[i + 3]);
 399
 400    if (glsl_type_is_matrix(vtn_src[0]->type) ||
 401        (num_inputs >= 2 && glsl_type_is_matrix(vtn_src[1]->type))) {
 402       vtn_handle_matrix_alu(b, opcode, val, vtn_src[0], vtn_src[1]);
 403       b->nb.exact = false;
 404       return;
 405    }
 406
 407    val->ssa = vtn_create_ssa_value(b, type);
 408    nir_ssa_def *src[4] = { NULL, };
 409    for (unsigned i = 0; i < num_inputs; i++) {
 410       vtn_assert(glsl_type_is_vector_or_scalar(vtn_src[i]->type));
 411       src[i] = vtn_src[i]->def;
 412    }
 413
 414    switch (opcode) {
 415    case SpvOpAny:
 416       if (src[0]->num_components == 1) {
 417          val->ssa->def = nir_imov(&b->nb, src[0]);
 418       } else {
 419          nir_op op;
 420          switch (src[0]->num_components) {
 421          case 2:  op = nir_op_bany_inequal2; break;
 422          case 3:  op = nir_op_bany_inequal3; break;
 423          case 4:  op = nir_op_bany_inequal4; break;
 424          default: unreachable("invalid number of components");
 425          }
 426          val->ssa->def = nir_build_alu(&b->nb, op, src[0],
 427                                        nir_imm_int(&b->nb, NIR_FALSE),
 428                                        NULL, NULL);
 429       }
 430       break;
 431
 432    case SpvOpAll:
 433       if (src[0]->num_components == 1) {
 434          val->ssa->def = nir_imov(&b->nb, src[0]);
 435       } else {
 436          nir_op op;
 437          switch (src[0]->num_components) {
 438          case 2:  op = nir_op_ball_iequal2;  break;
 439          case 3:  op = nir_op_ball_iequal3;  break;
 440          case 4:  op = nir_op_ball_iequal4;  break;
 441          default: unreachable("invalid number of components");
 442          }
 443          val->ssa->def = nir_build_alu(&b->nb, op, src[0],
 444                                        nir_imm_int(&b->nb, NIR_TRUE),
 445                                        NULL, NULL);
 446       }
 447       break;
 448
 449    case SpvOpOuterProduct: {
 450       for (unsigned i = 0; i < src[1]->num_components; i++) {
 451          val->ssa->elems[i]->def =
 452             nir_fmul(&b->nb, src[0], nir_channel(&b->nb, src[1], i));
 453       }
 454       break;
 455    }
 456
 457    case SpvOpDot:
 458       val->ssa->def = nir_fdot(&b->nb, src[0], src[1]);
 459       break;
 460
 461    case SpvOpIAddCarry:
 462       vtn_assert(glsl_type_is_struct(val->ssa->type));
 463       val->ssa->elems[0]->def = nir_iadd(&b->nb, src[0], src[1]);
 464       val->ssa->elems[1]->def = nir_uadd_carry(&b->nb, src[0], src[1]);
 465       break;
 466
 467    case SpvOpISubBorrow:
 468       vtn_assert(glsl_type_is_struct(val->ssa->type));
 469       val->ssa->elems[0]->def = nir_isub(&b->nb, src[0], src[1]);
 470       val->ssa->elems[1]->def = nir_usub_borrow(&b->nb, src[0], src[1]);
 471       break;
 472
 473    case SpvOpUMulExtended:
 474       vtn_assert(glsl_type_is_struct(val->ssa->type));
 475       val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
 476       val->ssa->elems[1]->def = nir_umul_high(&b->nb, src[0], src[1]);
 477       break;
 478
 479    case SpvOpSMulExtended:
 480       vtn_assert(glsl_type_is_struct(val->ssa->type));
 481       val->ssa->elems[0]->def = nir_imul(&b->nb, src[0], src[1]);
 482       val->ssa->elems[1]->def = nir_imul_high(&b->nb, src[0], src[1]);
 483       break;
 484
 485    case SpvOpFwidth:
 486       val->ssa->def = nir_fadd(&b->nb,
 487                                nir_fabs(&b->nb, nir_fddx(&b->nb, src[0])),
 488                                nir_fabs(&b->nb, nir_fddy(&b->nb, src[0])));
 489       break;
 490    case SpvOpFwidthFine:
 491       val->ssa->def = nir_fadd(&b->nb,
 492                                nir_fabs(&b->nb, nir_fddx_fine(&b->nb, src[0])),
 493                                nir_fabs(&b->nb, nir_fddy_fine(&b->nb, src[0])));
 494       break;
 495    case SpvOpFwidthCoarse:
 496       val->ssa->def = nir_fadd(&b->nb,
 497                                nir_fabs(&b->nb, nir_fddx_coarse(&b->nb, src[0])),
 498                                nir_fabs(&b->nb, nir_fddy_coarse(&b->nb, src[0])));
 499       break;
 500
 501    case SpvOpVectorTimesScalar:
 502       /* The builder will take care of splatting for us. */
 503       val->ssa->def = nir_fmul(&b->nb, src[0], src[1]);
 504       break;
 505
 506    case SpvOpIsNan:
 507       val->ssa->def = nir_fne(&b->nb, src[0], src[0]);
 508       break;
 509
 510    case SpvOpIsInf:
 511       val->ssa->def = nir_ieq(&b->nb, nir_fabs(&b->nb, src[0]),
 512                                       nir_imm_float(&b->nb, INFINITY));
 513       break;
 514
 515    case SpvOpFUnordEqual:
 516    case SpvOpFUnordNotEqual:
 517    case SpvOpFUnordLessThan:
 518    case SpvOpFUnordGreaterThan:
 519    case SpvOpFUnordLessThanEqual:
 520    case SpvOpFUnordGreaterThanEqual: {
 521       bool swap;
 522       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 523       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 524       nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
 525
 526       if (swap) {
 527          nir_ssa_def *tmp = src[0];
 528          src[0] = src[1];
 529          src[1] = tmp;
 530       }
 531
 532       val->ssa->def =
 533          nir_ior(&b->nb,
 534                  nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
 535                  nir_ior(&b->nb,
 536                          nir_fne(&b->nb, src[0], src[0]),
 537                          nir_fne(&b->nb, src[1], src[1])));
 538       break;
 539    }
 540
 541    case SpvOpFOrdEqual:
 542    case SpvOpFOrdNotEqual:
 543    case SpvOpFOrdLessThan:
 544    case SpvOpFOrdGreaterThan:
 545    case SpvOpFOrdLessThanEqual:
 546    case SpvOpFOrdGreaterThanEqual: {
 547       bool swap;
 548       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 549       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 550       nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
 551
 552       if (swap) {
 553          nir_ssa_def *tmp = src[0];
 554          src[0] = src[1];
 555          src[1] = tmp;
 556       }
 557
 558       val->ssa->def =
 559          nir_iand(&b->nb,
 560                   nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
 561                   nir_iand(&b->nb,
 562                           nir_feq(&b->nb, src[0], src[0]),
 563                           nir_feq(&b->nb, src[1], src[1])));
 564       break;
 565    }
 566
 567    case SpvOpBitcast:
 568       vtn_handle_bitcast(b, val->ssa, src[0]);
 569       break;
 570
 571    default: {
 572       bool swap;
 573       nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
 574       nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(type);
 575       nir_op op = vtn_nir_alu_op_for_spirv_opcode(opcode, &swap, src_alu_type, dst_alu_type);
 576
 577       if (swap) {
 578          nir_ssa_def *tmp = src[0];
 579          src[0] = src[1];
 580          src[1] = tmp;
 581       }
 582
 583       val->ssa->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
 584       break;
 585    } /* default */
 586    }
 587
 588    b->nb.exact = false;
 589 }