src/compiler/nir/nir_opcodes.py

   1 #
   2 # Copyright (C) 2014 Connor Abbott
   3 #
   4 # Permission is hereby granted, free of charge, to any person obtaining a
   5 # copy of this software and associated documentation files (the "Software"),
   6 # to deal in the Software without restriction, including without limitation
   7 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 # and/or sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following conditions:
  10 #
  11 # The above copyright notice and this permission notice (including the next
  12 # paragraph) shall be included in all copies or substantial portions of the
  13 # Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22 #
  23 # Authors:
  24 #    Connor Abbott (cwabbott0@gmail.com)
  25
  26
  27 # Class that represents all the information we have about the opcode
  28 # NOTE: this must be kept in sync with nir_op_info
  29
  30 class Opcode(object):
  31    """Class that represents all the information we have about the opcode
  32    NOTE: this must be kept in sync with nir_op_info
  33    """
  34    def __init__(self, name, output_size, output_type, input_sizes,
  35                 input_types, algebraic_properties, const_expr):
  36       """Parameters:
  37
  38       - name is the name of the opcode (prepend nir_op_ for the enum name)
  39       - all types are strings that get nir_type_ prepended to them
  40       - input_types is a list of types
  41       - algebraic_properties is a space-seperated string, where nir_op_is_ is
  42         prepended before each entry
  43       - const_expr is an expression or series of statements that computes the
  44         constant value of the opcode given the constant values of its inputs.
  45
  46       Constant expressions are formed from the variables src0, src1, ...,
  47       src(N-1), where N is the number of arguments.  The output of the
  48       expression should be stored in the dst variable.  Per-component input
  49       and output variables will be scalars and non-per-component input and
  50       output variables will be a struct with fields named x, y, z, and w
  51       all of the correct type.  Input and output variables can be assumed
  52       to already be of the correct type and need no conversion.  In
  53       particular, the conversion from the C bool type to/from  NIR_TRUE and
  54       NIR_FALSE happens automatically.
  55
  56       For per-component instructions, the entire expression will be
  57       executed once for each component.  For non-per-component
  58       instructions, the expression is expected to store the correct values
  59       in dst.x, dst.y, etc.  If "dst" does not exist anywhere in the
  60       constant expression, an assignment to dst will happen automatically
  61       and the result will be equivalent to "dst = <expression>" for
  62       per-component instructions and "dst.x = dst.y = ... = <expression>"
  63       for non-per-component instructions.
  64       """
  65       assert isinstance(name, str)
  66       assert isinstance(output_size, int)
  67       assert isinstance(output_type, str)
  68       assert isinstance(input_sizes, list)
  69       assert isinstance(input_sizes[0], int)
  70       assert isinstance(input_types, list)
  71       assert isinstance(input_types[0], str)
  72       assert isinstance(algebraic_properties, str)
  73       assert isinstance(const_expr, str)
  74       assert len(input_sizes) == len(input_types)
  75       assert 0 <= output_size <= 4
  76       for size in input_sizes:
  77          assert 0 <= size <= 4
  78          if output_size != 0:
  79             assert size != 0
  80       self.name = name
  81       self.num_inputs = len(input_sizes)
  82       self.output_size = output_size
  83       self.output_type = output_type
  84       self.input_sizes = input_sizes
  85       self.input_types = input_types
  86       self.algebraic_properties = algebraic_properties
  87       self.const_expr = const_expr
  88
  89 # helper variables for strings
  90 tfloat = "float"
  91 tint = "int"
  92 tbool = "bool32"
  93 tuint = "uint"
  94 tfloat32 = "float32"
  95 tint32 = "int32"
  96 tuint32 = "uint32"
  97 tint64 = "int64"
  98 tuint64 = "uint64"
  99 tfloat64 = "float64"
 100
 101 commutative = "commutative "
 102 associative = "associative "
 103
 104 # global dictionary of opcodes
 105 opcodes = {}
 106
 107 def opcode(name, output_size, output_type, input_sizes, input_types,
 108            algebraic_properties, const_expr):
 109    assert name not in opcodes
 110    opcodes[name] = Opcode(name, output_size, output_type, input_sizes,
 111                           input_types, algebraic_properties, const_expr)
 112
 113 def unop_convert(name, out_type, in_type, const_expr):
 114    opcode(name, 0, out_type, [0], [in_type], "", const_expr)
 115
 116 def unop(name, ty, const_expr):
 117    opcode(name, 0, ty, [0], [ty], "", const_expr)
 118
 119 def unop_horiz(name, output_size, output_type, input_size, input_type,
 120                const_expr):
 121    opcode(name, output_size, output_type, [input_size], [input_type], "",
 122           const_expr)
 123
 124 def unop_reduce(name, output_size, output_type, input_type, prereduce_expr,
 125                 reduce_expr, final_expr):
 126    def prereduce(src):
 127       return "(" + prereduce_expr.format(src=src) + ")"
 128    def final(src):
 129       return final_expr.format(src="(" + src + ")")
 130    def reduce_(src0, src1):
 131       return reduce_expr.format(src0=src0, src1=src1)
 132    src0 = prereduce("src0.x")
 133    src1 = prereduce("src0.y")
 134    src2 = prereduce("src0.z")
 135    src3 = prereduce("src0.w")
 136    unop_horiz(name + "2", output_size, output_type, 2, input_type,
 137               final(reduce_(src0, src1)))
 138    unop_horiz(name + "3", output_size, output_type, 3, input_type,
 139               final(reduce_(reduce_(src0, src1), src2)))
 140    unop_horiz(name + "4", output_size, output_type, 4, input_type,
 141               final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 142
 143
 144 # These two move instructions differ in what modifiers they support and what
 145 # the negate modifier means. Otherwise, they are identical.
 146 unop("fmov", tfloat, "src0")
 147 unop("imov", tint, "src0")
 148
 149 unop("ineg", tint, "-src0")
 150 unop("fneg", tfloat, "-src0")
 151 unop("inot", tint, "~src0") # invert every bit of the integer
 152 unop("fnot", tfloat, ("bit_size == 64 ? ((src0 == 0.0) ? 1.0 : 0.0f) : " +
 153                       "((src0 == 0.0f) ? 1.0f : 0.0f)"))
 154 unop("fsign", tfloat, ("bit_size == 64 ? " +
 155                        "((src0 == 0.0) ? 0.0 : ((src0 > 0.0) ? 1.0 : -1.0)) : " +
 156                        "((src0 == 0.0f) ? 0.0f : ((src0 > 0.0f) ? 1.0f : -1.0f))"))
 157 unop("isign", tint, "(src0 == 0) ? 0 : ((src0 > 0) ? 1 : -1)")
 158 unop("iabs", tint, "(src0 < 0) ? -src0 : src0")
 159 unop("fabs", tfloat, "fabs(src0)")
 160 unop("fsat", tfloat, ("bit_size == 64 ? " +
 161                       "((src0 > 1.0) ? 1.0 : ((src0 <= 0.0) ? 0.0 : src0)) : " +
 162                       "((src0 > 1.0f) ? 1.0f : ((src0 <= 0.0f) ? 0.0f : src0))"))
 163 unop("frcp", tfloat, "bit_size == 64 ? 1.0 / src0 : 1.0f / src0")
 164 unop("frsq", tfloat, "bit_size == 64 ? 1.0 / sqrt(src0) : 1.0f / sqrtf(src0)")
 165 unop("fsqrt", tfloat, "bit_size == 64 ? sqrt(src0) : sqrtf(src0)")
 166 unop("fexp2", tfloat, "exp2f(src0)")
 167 unop("flog2", tfloat, "log2f(src0)")
 168
 169 # Generate all of the numeric conversion opcodes
 170 for src_t in [tint, tuint, tfloat]:
 171    if src_t in (tint, tuint):
 172       dst_types = [tfloat, src_t]
 173    elif src_t == tfloat:
 174       dst_types = [tint, tuint, tfloat]
 175
 176    for dst_t in dst_types:
 177       if dst_t == tfloat:
 178          bit_sizes = [16, 32, 64]
 179       else:
 180          bit_sizes = [8, 16, 32, 64]
 181       for bit_size in bit_sizes:
 182           if bit_size == 16 and dst_t == tfloat and src_t == tfloat:
 183               rnd_modes = ['rtne', 'rtz', 'undef']
 184               for rnd_mode in rnd_modes:
 185                   unop_convert("{0}2{1}{2}_{3}".format(src_t[0], dst_t[0],
 186                                                        bit_size, rnd_mode),
 187                                dst_t + str(bit_size), src_t, "src0")
 188           else:
 189               unop_convert("{0}2{1}{2}".format(src_t[0], dst_t[0], bit_size),
 190                            dst_t + str(bit_size), src_t, "src0")
 191
 192 # We'll hand-code the to/from bool conversion opcodes.  Because bool doesn't
 193 # have multiple bit-sizes, we can always infer the size from the other type.
 194 unop_convert("f2b", tbool, tfloat, "src0 != 0.0")
 195 unop_convert("i2b", tbool, tint, "src0 != 0")
 196 unop_convert("b2f", tfloat, tbool, "src0 ? 1.0 : 0.0")
 197 unop_convert("b2i", tint, tbool, "src0 ? 1 : 0")
 198
 199
 200 # Unary floating-point rounding operations.
 201
 202
 203 unop("ftrunc", tfloat, "bit_size == 64 ? trunc(src0) : truncf(src0)")
 204 unop("fceil", tfloat, "bit_size == 64 ? ceil(src0) : ceilf(src0)")
 205 unop("ffloor", tfloat, "bit_size == 64 ? floor(src0) : floorf(src0)")
 206 unop("ffract", tfloat, "src0 - (bit_size == 64 ? floor(src0) : floorf(src0))")
 207 unop("fround_even", tfloat, "bit_size == 64 ? _mesa_roundeven(src0) : _mesa_roundevenf(src0)")
 208
 209 unop("fquantize2f16", tfloat, "(fabs(src0) < ldexpf(1.0, -14)) ? copysignf(0.0f, src0) : _mesa_half_to_float(_mesa_float_to_half(src0))")
 210
 211 # Trigonometric operations.
 212
 213
 214 unop("fsin", tfloat, "bit_size == 64 ? sin(src0) : sinf(src0)")
 215 unop("fcos", tfloat, "bit_size == 64 ? cos(src0) : cosf(src0)")
 216
 217
 218 # Partial derivatives.
 219
 220
 221 unop("fddx", tfloat, "0.0") # the derivative of a constant is 0.
 222 unop("fddy", tfloat, "0.0")
 223 unop("fddx_fine", tfloat, "0.0")
 224 unop("fddy_fine", tfloat, "0.0")
 225 unop("fddx_coarse", tfloat, "0.0")
 226 unop("fddy_coarse", tfloat, "0.0")
 227
 228
 229 # Floating point pack and unpack operations.
 230
 231 def pack_2x16(fmt):
 232    unop_horiz("pack_" + fmt + "_2x16", 1, tuint32, 2, tfloat32, """
 233 dst.x = (uint32_t) pack_fmt_1x16(src0.x);
 234 dst.x |= ((uint32_t) pack_fmt_1x16(src0.y)) << 16;
 235 """.replace("fmt", fmt))
 236
 237 def pack_4x8(fmt):
 238    unop_horiz("pack_" + fmt + "_4x8", 1, tuint32, 4, tfloat32, """
 239 dst.x = (uint32_t) pack_fmt_1x8(src0.x);
 240 dst.x |= ((uint32_t) pack_fmt_1x8(src0.y)) << 8;
 241 dst.x |= ((uint32_t) pack_fmt_1x8(src0.z)) << 16;
 242 dst.x |= ((uint32_t) pack_fmt_1x8(src0.w)) << 24;
 243 """.replace("fmt", fmt))
 244
 245 def unpack_2x16(fmt):
 246    unop_horiz("unpack_" + fmt + "_2x16", 2, tfloat32, 1, tuint32, """
 247 dst.x = unpack_fmt_1x16((uint16_t)(src0.x & 0xffff));
 248 dst.y = unpack_fmt_1x16((uint16_t)(src0.x << 16));
 249 """.replace("fmt", fmt))
 250
 251 def unpack_4x8(fmt):
 252    unop_horiz("unpack_" + fmt + "_4x8", 4, tfloat32, 1, tuint32, """
 253 dst.x = unpack_fmt_1x8((uint8_t)(src0.x & 0xff));
 254 dst.y = unpack_fmt_1x8((uint8_t)((src0.x >> 8) & 0xff));
 255 dst.z = unpack_fmt_1x8((uint8_t)((src0.x >> 16) & 0xff));
 256 dst.w = unpack_fmt_1x8((uint8_t)(src0.x >> 24));
 257 """.replace("fmt", fmt))
 258
 259
 260 pack_2x16("snorm")
 261 pack_4x8("snorm")
 262 pack_2x16("unorm")
 263 pack_4x8("unorm")
 264 pack_2x16("half")
 265 unpack_2x16("snorm")
 266 unpack_4x8("snorm")
 267 unpack_2x16("unorm")
 268 unpack_4x8("unorm")
 269 unpack_2x16("half")
 270
 271 unop_horiz("pack_uvec2_to_uint", 1, tuint32, 2, tuint32, """
 272 dst.x = (src0.x & 0xffff) | (src0.y << 16);
 273 """)
 274
 275 unop_horiz("pack_uvec4_to_uint", 1, tuint32, 4, tuint32, """
 276 dst.x = (src0.x <<  0) |
 277         (src0.y <<  8) |
 278         (src0.z << 16) |
 279         (src0.w << 24);
 280 """)
 281
 282 unop_horiz("pack_64_2x32", 1, tuint64, 2, tuint32,
 283            "dst.x = src0.x | ((uint64_t)src0.y << 32);")
 284
 285 unop_horiz("unpack_64_2x32", 2, tuint32, 1, tuint64,
 286            "dst.x = src0.x; dst.y = src0.x >> 32;")
 287
 288 # Lowered floating point unpacking operations.
 289
 290
 291 unop_horiz("unpack_half_2x16_split_x", 1, tfloat32, 1, tuint32,
 292            "unpack_half_1x16((uint16_t)(src0.x & 0xffff))")
 293 unop_horiz("unpack_half_2x16_split_y", 1, tfloat32, 1, tuint32,
 294            "unpack_half_1x16((uint16_t)(src0.x >> 16))")
 295
 296 unop_convert("unpack_64_2x32_split_x", tuint32, tuint64, "src0")
 297 unop_convert("unpack_64_2x32_split_y", tuint32, tuint64, "src0 >> 32")
 298
 299 # Bit operations, part of ARB_gpu_shader5.
 300
 301
 302 unop("bitfield_reverse", tuint32, """
 303 /* we're not winning any awards for speed here, but that's ok */
 304 dst = 0;
 305 for (unsigned bit = 0; bit < 32; bit++)
 306    dst |= ((src0 >> bit) & 1) << (31 - bit);
 307 """)
 308 unop("bit_count", tuint32, """
 309 dst = 0;
 310 for (unsigned bit = 0; bit < 32; bit++) {
 311    if ((src0 >> bit) & 1)
 312       dst++;
 313 }
 314 """)
 315
 316 unop_convert("ufind_msb", tint32, tuint32, """
 317 dst = -1;
 318 for (int bit = 31; bit >= 0; bit--) {
 319    if ((src0 >> bit) & 1) {
 320       dst = bit;
 321       break;
 322    }
 323 }
 324 """)
 325
 326 unop("ifind_msb", tint32, """
 327 dst = -1;
 328 for (int bit = 31; bit >= 0; bit--) {
 329    /* If src0 < 0, we're looking for the first 0 bit.
 330     * if src0 >= 0, we're looking for the first 1 bit.
 331     */
 332    if ((((src0 >> bit) & 1) && (src0 >= 0)) ||
 333       (!((src0 >> bit) & 1) && (src0 < 0))) {
 334       dst = bit;
 335       break;
 336    }
 337 }
 338 """)
 339
 340 unop("find_lsb", tint32, """
 341 dst = -1;
 342 for (unsigned bit = 0; bit < 32; bit++) {
 343    if ((src0 >> bit) & 1) {
 344       dst = bit;
 345       break;
 346    }
 347 }
 348 """)
 349
 350
 351 for i in xrange(1, 5):
 352    for j in xrange(1, 5):
 353       unop_horiz("fnoise{0}_{1}".format(i, j), i, tfloat, j, tfloat, "0.0f")
 354
 355
 356 # AMD_gcn_shader extended instructions
 357 unop_horiz("cube_face_coord", 2, tfloat32, 3, tfloat32, """
 358 dst.x = dst.y = 0.0;
 359 float absX = fabs(src0.x);
 360 float absY = fabs(src0.y);
 361 float absZ = fabs(src0.z);
 362 if (src0.x >= 0 && absX >= absY && absX >= absZ) { dst.x = -src0.y; dst.y = -src0.z; }
 363 if (src0.x < 0 && absX >= absY && absX >= absZ) { dst.x = -src0.y; dst.y = src0.z; }
 364 if (src0.y >= 0 && absY >= absX && absY >= absZ) { dst.x = src0.z; dst.y = src0.x; }
 365 if (src0.y < 0 && absY >= absX && absY >= absZ) { dst.x = -src0.z; dst.y = src0.x; }
 366 if (src0.z >= 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.y; dst.y = src0.x; }
 367 if (src0.z < 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.y; dst.y = -src0.x; }
 368 """)
 369
 370 unop_horiz("cube_face_index", 1, tfloat32, 3, tfloat32, """
 371 float absX = fabs(src0.x);
 372 float absY = fabs(src0.y);
 373 float absZ = fabs(src0.z);
 374 if (src0.x >= 0 && absX >= absY && absX >= absZ) dst.x = 0;
 375 if (src0.x < 0 && absX >= absY && absX >= absZ) dst.x = 1;
 376 if (src0.y >= 0 && absY >= absX && absY >= absZ) dst.x = 2;
 377 if (src0.y < 0 && absY >= absX && absY >= absZ) dst.x = 3;
 378 if (src0.z >= 0 && absZ >= absX && absZ >= absY) dst.x = 4;
 379 if (src0.z < 0 && absZ >= absX && absZ >= absY) dst.x = 5;
 380 """)
 381
 382
 383 def binop_convert(name, out_type, in_type, alg_props, const_expr):
 384    opcode(name, 0, out_type, [0, 0], [in_type, in_type], alg_props, const_expr)
 385
 386 def binop(name, ty, alg_props, const_expr):
 387    binop_convert(name, ty, ty, alg_props, const_expr)
 388
 389 def binop_compare(name, ty, alg_props, const_expr):
 390    binop_convert(name, tbool, ty, alg_props, const_expr)
 391
 392 def binop_horiz(name, out_size, out_type, src1_size, src1_type, src2_size,
 393                 src2_type, const_expr):
 394    opcode(name, out_size, out_type, [src1_size, src2_size], [src1_type, src2_type],
 395           "", const_expr)
 396
 397 def binop_reduce(name, output_size, output_type, src_type, prereduce_expr,
 398                  reduce_expr, final_expr):
 399    def final(src):
 400       return final_expr.format(src= "(" + src + ")")
 401    def reduce_(src0, src1):
 402       return reduce_expr.format(src0=src0, src1=src1)
 403    def prereduce(src0, src1):
 404       return "(" + prereduce_expr.format(src0=src0, src1=src1) + ")"
 405    src0 = prereduce("src0.x", "src1.x")
 406    src1 = prereduce("src0.y", "src1.y")
 407    src2 = prereduce("src0.z", "src1.z")
 408    src3 = prereduce("src0.w", "src1.w")
 409    opcode(name + "2", output_size, output_type,
 410           [2, 2], [src_type, src_type], commutative,
 411           final(reduce_(src0, src1)))
 412    opcode(name + "3", output_size, output_type,
 413           [3, 3], [src_type, src_type], commutative,
 414           final(reduce_(reduce_(src0, src1), src2)))
 415    opcode(name + "4", output_size, output_type,
 416           [4, 4], [src_type, src_type], commutative,
 417           final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 418
 419 binop("fadd", tfloat, commutative + associative, "src0 + src1")
 420 binop("iadd", tint, commutative + associative, "src0 + src1")
 421 binop("fsub", tfloat, "", "src0 - src1")
 422 binop("isub", tint, "", "src0 - src1")
 423
 424 binop("fmul", tfloat, commutative + associative, "src0 * src1")
 425 # low 32-bits of signed/unsigned integer multiply
 426 binop("imul", tint, commutative + associative, "src0 * src1")
 427 # high 32-bits of signed integer multiply
 428 binop("imul_high", tint32, commutative,
 429       "(int32_t)(((int64_t) src0 * (int64_t) src1) >> 32)")
 430 # high 32-bits of unsigned integer multiply
 431 binop("umul_high", tuint32, commutative,
 432       "(uint32_t)(((uint64_t) src0 * (uint64_t) src1) >> 32)")
 433
 434 binop("fdiv", tfloat, "", "src0 / src1")
 435 binop("idiv", tint, "", "src1 == 0 ? 0 : (src0 / src1)")
 436 binop("udiv", tuint, "", "src1 == 0 ? 0 : (src0 / src1)")
 437
 438 # returns a boolean representing the carry resulting from the addition of
 439 # the two unsigned arguments.
 440
 441 binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0")
 442
 443 # returns a boolean representing the borrow resulting from the subtraction
 444 # of the two unsigned arguments.
 445
 446 binop_convert("usub_borrow", tuint, tuint, "", "src0 < src1")
 447
 448 binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1")
 449
 450 # For signed integers, there are several different possible definitions of
 451 # "modulus" or "remainder".  We follow the conventions used by LLVM and
 452 # SPIR-V.  The irem opcode implements the standard C/C++ signed "%"
 453 # operation while the imod opcode implements the more mathematical
 454 # "modulus" operation.  For details on the difference, see
 455 #
 456 # http://mathforum.org/library/drmath/view/52343.html
 457
 458 binop("irem", tint, "", "src1 == 0 ? 0 : src0 % src1")
 459 binop("imod", tint, "",
 460       "src1 == 0 ? 0 : ((src0 % src1 == 0 || (src0 >= 0) == (src1 >= 0)) ?"
 461       "                 src0 % src1 : src0 % src1 + src1)")
 462 binop("fmod", tfloat, "", "src0 - src1 * floorf(src0 / src1)")
 463 binop("frem", tfloat, "", "src0 - src1 * truncf(src0 / src1)")
 464
 465 #
 466 # Comparisons
 467 #
 468
 469
 470 # these integer-aware comparisons return a boolean (0 or ~0)
 471
 472 binop_compare("flt", tfloat, "", "src0 < src1")
 473 binop_compare("fge", tfloat, "", "src0 >= src1")
 474 binop_compare("feq", tfloat, commutative, "src0 == src1")
 475 binop_compare("fne", tfloat, commutative, "src0 != src1")
 476 binop_compare("ilt", tint, "", "src0 < src1")
 477 binop_compare("ige", tint, "", "src0 >= src1")
 478 binop_compare("ieq", tint, commutative, "src0 == src1")
 479 binop_compare("ine", tint, commutative, "src0 != src1")
 480 binop_compare("ult", tuint, "", "src0 < src1")
 481 binop_compare("uge", tuint, "", "src0 >= src1")
 482
 483 # integer-aware GLSL-style comparisons that compare floats and ints
 484
 485 binop_reduce("ball_fequal",  1, tbool, tfloat, "{src0} == {src1}",
 486              "{src0} && {src1}", "{src}")
 487 binop_reduce("bany_fnequal", 1, tbool, tfloat, "{src0} != {src1}",
 488              "{src0} || {src1}", "{src}")
 489 binop_reduce("ball_iequal",  1, tbool, tint, "{src0} == {src1}",
 490              "{src0} && {src1}", "{src}")
 491 binop_reduce("bany_inequal", 1, tbool, tint, "{src0} != {src1}",
 492              "{src0} || {src1}", "{src}")
 493
 494 # non-integer-aware GLSL-style comparisons that return 0.0 or 1.0
 495
 496 binop_reduce("fall_equal",  1, tfloat32, tfloat32, "{src0} == {src1}",
 497              "{src0} && {src1}", "{src} ? 1.0f : 0.0f")
 498 binop_reduce("fany_nequal", 1, tfloat32, tfloat32, "{src0} != {src1}",
 499              "{src0} || {src1}", "{src} ? 1.0f : 0.0f")
 500
 501 # These comparisons for integer-less hardware return 1.0 and 0.0 for true
 502 # and false respectively
 503
 504 binop("slt", tfloat32, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than
 505 binop("sge", tfloat, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal
 506 binop("seq", tfloat32, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
 507 binop("sne", tfloat32, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
 508
 509
 510 opcode("ishl", 0, tint, [0, 0], [tint, tuint32], "", "src0 << src1")
 511 opcode("ishr", 0, tint, [0, 0], [tint, tuint32], "", "src0 >> src1")
 512 opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], "", "src0 >> src1")
 513
 514 # bitwise logic operators
 515 #
 516 # These are also used as boolean and, or, xor for hardware supporting
 517 # integers.
 518
 519
 520 binop("iand", tuint, commutative + associative, "src0 & src1")
 521 binop("ior", tuint, commutative + associative, "src0 | src1")
 522 binop("ixor", tuint, commutative + associative, "src0 ^ src1")
 523
 524
 525 # floating point logic operators
 526 #
 527 # These use (src != 0.0) for testing the truth of the input, and output 1.0
 528 # for true and 0.0 for false
 529
 530 binop("fand", tfloat32, commutative,
 531       "((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f")
 532 binop("for", tfloat32, commutative,
 533       "((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f")
 534 binop("fxor", tfloat32, commutative,
 535       "(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f")
 536
 537 binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}",
 538              "{src}")
 539
 540 binop_reduce("fdot_replicated", 4, tfloat, tfloat,
 541              "{src0} * {src1}", "{src0} + {src1}", "{src}")
 542
 543 opcode("fdph", 1, tfloat, [3, 4], [tfloat, tfloat], "",
 544        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 545 opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], "",
 546        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 547
 548 binop("fmin", tfloat, "", "fminf(src0, src1)")
 549 binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1")
 550 binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1")
 551 binop("fmax", tfloat, "", "fmaxf(src0, src1)")
 552 binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0")
 553 binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0")
 554
 555 # Saturated vector add for 4 8bit ints.
 556 binop("usadd_4x8", tint32, commutative + associative, """
 557 dst = 0;
 558 for (int i = 0; i < 32; i += 8) {
 559    dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i;
 560 }
 561 """)
 562
 563 # Saturated vector subtract for 4 8bit ints.
 564 binop("ussub_4x8", tint32, "", """
 565 dst = 0;
 566 for (int i = 0; i < 32; i += 8) {
 567    int src0_chan = (src0 >> i) & 0xff;
 568    int src1_chan = (src1 >> i) & 0xff;
 569    if (src0_chan > src1_chan)
 570       dst |= (src0_chan - src1_chan) << i;
 571 }
 572 """)
 573
 574 # vector min for 4 8bit ints.
 575 binop("umin_4x8", tint32, commutative + associative, """
 576 dst = 0;
 577 for (int i = 0; i < 32; i += 8) {
 578    dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 579 }
 580 """)
 581
 582 # vector max for 4 8bit ints.
 583 binop("umax_4x8", tint32, commutative + associative, """
 584 dst = 0;
 585 for (int i = 0; i < 32; i += 8) {
 586    dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 587 }
 588 """)
 589
 590 # unorm multiply: (a * b) / 255.
 591 binop("umul_unorm_4x8", tint32, commutative + associative, """
 592 dst = 0;
 593 for (int i = 0; i < 32; i += 8) {
 594    int src0_chan = (src0 >> i) & 0xff;
 595    int src1_chan = (src1 >> i) & 0xff;
 596    dst |= ((src0_chan * src1_chan) / 255) << i;
 597 }
 598 """)
 599
 600 binop("fpow", tfloat, "", "bit_size == 64 ? powf(src0, src1) : pow(src0, src1)")
 601
 602 binop_horiz("pack_half_2x16_split", 1, tuint32, 1, tfloat32, 1, tfloat32,
 603             "pack_half_1x16(src0.x) | (pack_half_1x16(src1.x) << 16)")
 604
 605 binop_convert("pack_64_2x32_split", tuint64, tuint32, "",
 606               "src0 | ((uint64_t)src1 << 32)")
 607
 608 # bfm implements the behavior of the first operation of the SM5 "bfi" assembly
 609 # and that of the "bfi1" i965 instruction. That is, it has undefined behavior
 610 # if either of its arguments are 32.
 611 binop_convert("bfm", tuint32, tint32, "", """
 612 int bits = src0, offset = src1;
 613 if (offset < 0 || bits < 0 || offset > 31 || bits > 31 || offset + bits > 32)
 614    dst = 0; /* undefined */
 615 else
 616    dst = ((1u << bits) - 1) << offset;
 617 """)
 618
 619 opcode("ldexp", 0, tfloat, [0, 0], [tfloat, tint32], "", """
 620 dst = (bit_size == 64) ? ldexp(src0, src1) : ldexpf(src0, src1);
 621 /* flush denormals to zero. */
 622 if (!isnormal(dst))
 623    dst = copysignf(0.0f, src0);
 624 """)
 625
 626 # Combines the first component of each input to make a 2-component vector.
 627
 628 binop_horiz("vec2", 2, tuint, 1, tuint, 1, tuint, """
 629 dst.x = src0.x;
 630 dst.y = src1.x;
 631 """)
 632
 633 # Byte extraction
 634 binop("extract_u8", tuint, "", "(uint8_t)(src0 >> (src1 * 8))")
 635 binop("extract_i8", tint, "", "(int8_t)(src0 >> (src1 * 8))")
 636
 637 # Word extraction
 638 binop("extract_u16", tuint, "", "(uint16_t)(src0 >> (src1 * 16))")
 639 binop("extract_i16", tint, "", "(int16_t)(src0 >> (src1 * 16))")
 640
 641
 642 def triop(name, ty, const_expr):
 643    opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], "", const_expr)
 644 def triop_horiz(name, output_size, src1_size, src2_size, src3_size, const_expr):
 645    opcode(name, output_size, tuint,
 646    [src1_size, src2_size, src3_size],
 647    [tuint, tuint, tuint], "", const_expr)
 648
 649 triop("ffma", tfloat, "src0 * src1 + src2")
 650
 651 triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2")
 652
 653 # Conditional Select
 654 #
 655 # A vector conditional select instruction (like ?:, but operating per-
 656 # component on vectors). There are two versions, one for floating point
 657 # bools (0.0 vs 1.0) and one for integer bools (0 vs ~0).
 658
 659
 660 triop("fcsel", tfloat32, "(src0 != 0.0f) ? src1 : src2")
 661 opcode("bcsel", 0, tuint, [0, 0, 0],
 662       [tbool, tuint, tuint], "", "src0 ? src1 : src2")
 663
 664 # SM5 bfi assembly
 665 triop("bfi", tuint32, """
 666 unsigned mask = src0, insert = src1, base = src2;
 667 if (mask == 0) {
 668    dst = base;
 669 } else {
 670    unsigned tmp = mask;
 671    while (!(tmp & 1)) {
 672       tmp >>= 1;
 673       insert <<= 1;
 674    }
 675    dst = (base & ~mask) | (insert & mask);
 676 }
 677 """)
 678
 679 # SM5 ubfe/ibfe assembly
 680 opcode("ubfe", 0, tuint32,
 681        [0, 0, 0], [tuint32, tint32, tint32], "", """
 682 unsigned base = src0;
 683 int offset = src1, bits = src2;
 684 if (bits == 0) {
 685    dst = 0;
 686 } else if (bits < 0 || offset < 0) {
 687    dst = 0; /* undefined */
 688 } else if (offset + bits < 32) {
 689    dst = (base << (32 - bits - offset)) >> (32 - bits);
 690 } else {
 691    dst = base >> offset;
 692 }
 693 """)
 694 opcode("ibfe", 0, tint32,
 695        [0, 0, 0], [tint32, tint32, tint32], "", """
 696 int base = src0;
 697 int offset = src1, bits = src2;
 698 if (bits == 0) {
 699    dst = 0;
 700 } else if (bits < 0 || offset < 0) {
 701    dst = 0; /* undefined */
 702 } else if (offset + bits < 32) {
 703    dst = (base << (32 - bits - offset)) >> (32 - bits);
 704 } else {
 705    dst = base >> offset;
 706 }
 707 """)
 708
 709 # GLSL bitfieldExtract()
 710 opcode("ubitfield_extract", 0, tuint32,
 711        [0, 0, 0], [tuint32, tint32, tint32], "", """
 712 unsigned base = src0;
 713 int offset = src1, bits = src2;
 714 if (bits == 0) {
 715    dst = 0;
 716 } else if (bits < 0 || offset < 0 || offset + bits > 32) {
 717    dst = 0; /* undefined per the spec */
 718 } else {
 719    dst = (base >> offset) & ((1ull << bits) - 1);
 720 }
 721 """)
 722 opcode("ibitfield_extract", 0, tint32,
 723        [0, 0, 0], [tint32, tint32, tint32], "", """
 724 int base = src0;
 725 int offset = src1, bits = src2;
 726 if (bits == 0) {
 727    dst = 0;
 728 } else if (offset < 0 || bits < 0 || offset + bits > 32) {
 729    dst = 0;
 730 } else {
 731    dst = (base << (32 - offset - bits)) >> offset; /* use sign-extending shift */
 732 }
 733 """)
 734
 735 # Combines the first component of each input to make a 3-component vector.
 736
 737 triop_horiz("vec3", 3, 1, 1, 1, """
 738 dst.x = src0.x;
 739 dst.y = src1.x;
 740 dst.z = src2.x;
 741 """)
 742
 743 def quadop_horiz(name, output_size, src1_size, src2_size, src3_size,
 744                  src4_size, const_expr):
 745    opcode(name, output_size, tuint,
 746           [src1_size, src2_size, src3_size, src4_size],
 747           [tuint, tuint, tuint, tuint],
 748           "", const_expr)
 749
 750 opcode("bitfield_insert", 0, tuint32, [0, 0, 0, 0],
 751        [tuint32, tuint32, tint32, tint32], "", """
 752 unsigned base = src0, insert = src1;
 753 int offset = src2, bits = src3;
 754 if (bits == 0) {
 755    dst = base;
 756 } else if (offset < 0 || bits < 0 || bits + offset > 32) {
 757    dst = 0;
 758 } else {
 759    unsigned mask = ((1ull << bits) - 1) << offset;
 760    dst = (base & ~mask) | ((insert << offset) & mask);
 761 }
 762 """)
 763
 764 quadop_horiz("vec4", 4, 1, 1, 1, 1, """
 765 dst.x = src0.x;
 766 dst.y = src1.x;
 767 dst.z = src2.x;
 768 dst.w = src3.x;
 769 """)
 770
 771