src/compiler/nir/nir_opcodes.py

   1 #
   2 # Copyright (C) 2014 Connor Abbott
   3 #
   4 # Permission is hereby granted, free of charge, to any person obtaining a
   5 # copy of this software and associated documentation files (the "Software"),
   6 # to deal in the Software without restriction, including without limitation
   7 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 # and/or sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following conditions:
  10 #
  11 # The above copyright notice and this permission notice (including the next
  12 # paragraph) shall be included in all copies or substantial portions of the
  13 # Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22 #
  23 # Authors:
  24 #    Connor Abbott (cwabbott0@gmail.com)
  25
  26
  27 # Class that represents all the information we have about the opcode
  28 # NOTE: this must be kept in sync with nir_op_info
  29
  30 class Opcode(object):
  31    """Class that represents all the information we have about the opcode
  32    NOTE: this must be kept in sync with nir_op_info
  33    """
  34    def __init__(self, name, output_size, output_type, input_sizes,
  35                 input_types, algebraic_properties, const_expr):
  36       """Parameters:
  37
  38       - name is the name of the opcode (prepend nir_op_ for the enum name)
  39       - all types are strings that get nir_type_ prepended to them
  40       - input_types is a list of types
  41       - algebraic_properties is a space-seperated string, where nir_op_is_ is
  42         prepended before each entry
  43       - const_expr is an expression or series of statements that computes the
  44         constant value of the opcode given the constant values of its inputs.
  45
  46       Constant expressions are formed from the variables src0, src1, ...,
  47       src(N-1), where N is the number of arguments.  The output of the
  48       expression should be stored in the dst variable.  Per-component input
  49       and output variables will be scalars and non-per-component input and
  50       output variables will be a struct with fields named x, y, z, and w
  51       all of the correct type.  Input and output variables can be assumed
  52       to already be of the correct type and need no conversion.  In
  53       particular, the conversion from the C bool type to/from  NIR_TRUE and
  54       NIR_FALSE happens automatically.
  55
  56       For per-component instructions, the entire expression will be
  57       executed once for each component.  For non-per-component
  58       instructions, the expression is expected to store the correct values
  59       in dst.x, dst.y, etc.  If "dst" does not exist anywhere in the
  60       constant expression, an assignment to dst will happen automatically
  61       and the result will be equivalent to "dst = <expression>" for
  62       per-component instructions and "dst.x = dst.y = ... = <expression>"
  63       for non-per-component instructions.
  64       """
  65       assert isinstance(name, str)
  66       assert isinstance(output_size, int)
  67       assert isinstance(output_type, str)
  68       assert isinstance(input_sizes, list)
  69       assert isinstance(input_sizes[0], int)
  70       assert isinstance(input_types, list)
  71       assert isinstance(input_types[0], str)
  72       assert isinstance(algebraic_properties, str)
  73       assert isinstance(const_expr, str)
  74       assert len(input_sizes) == len(input_types)
  75       assert 0 <= output_size <= 4
  76       for size in input_sizes:
  77          assert 0 <= size <= 4
  78          if output_size != 0:
  79             assert size != 0
  80       self.name = name
  81       self.num_inputs = len(input_sizes)
  82       self.output_size = output_size
  83       self.output_type = output_type
  84       self.input_sizes = input_sizes
  85       self.input_types = input_types
  86       self.algebraic_properties = algebraic_properties
  87       self.const_expr = const_expr
  88
  89 # helper variables for strings
  90 tfloat = "float"
  91 tint = "int"
  92 tbool = "bool32"
  93 tuint = "uint"
  94 tfloat32 = "float32"
  95 tint32 = "int32"
  96 tuint32 = "uint32"
  97 tint64 = "int64"
  98 tuint64 = "uint64"
  99 tfloat64 = "float64"
 100
 101 commutative = "commutative "
 102 associative = "associative "
 103
 104 # global dictionary of opcodes
 105 opcodes = {}
 106
 107 def opcode(name, output_size, output_type, input_sizes, input_types,
 108            algebraic_properties, const_expr):
 109    assert name not in opcodes
 110    opcodes[name] = Opcode(name, output_size, output_type, input_sizes,
 111                           input_types, algebraic_properties, const_expr)
 112
 113 def unop_convert(name, out_type, in_type, const_expr):
 114    opcode(name, 0, out_type, [0], [in_type], "", const_expr)
 115
 116 def unop(name, ty, const_expr):
 117    opcode(name, 0, ty, [0], [ty], "", const_expr)
 118
 119 def unop_horiz(name, output_size, output_type, input_size, input_type,
 120                const_expr):
 121    opcode(name, output_size, output_type, [input_size], [input_type], "",
 122           const_expr)
 123
 124 def unop_reduce(name, output_size, output_type, input_type, prereduce_expr,
 125                 reduce_expr, final_expr):
 126    def prereduce(src):
 127       return "(" + prereduce_expr.format(src=src) + ")"
 128    def final(src):
 129       return final_expr.format(src="(" + src + ")")
 130    def reduce_(src0, src1):
 131       return reduce_expr.format(src0=src0, src1=src1)
 132    src0 = prereduce("src0.x")
 133    src1 = prereduce("src0.y")
 134    src2 = prereduce("src0.z")
 135    src3 = prereduce("src0.w")
 136    unop_horiz(name + "2", output_size, output_type, 2, input_type,
 137               final(reduce_(src0, src1)))
 138    unop_horiz(name + "3", output_size, output_type, 3, input_type,
 139               final(reduce_(reduce_(src0, src1), src2)))
 140    unop_horiz(name + "4", output_size, output_type, 4, input_type,
 141               final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 142
 143
 144 # These two move instructions differ in what modifiers they support and what
 145 # the negate modifier means. Otherwise, they are identical.
 146 unop("fmov", tfloat, "src0")
 147 unop("imov", tint, "src0")
 148
 149 unop("ineg", tint, "-src0")
 150 unop("fneg", tfloat, "-src0")
 151 unop("inot", tint, "~src0") # invert every bit of the integer
 152 unop("fnot", tfloat, ("bit_size == 64 ? ((src0 == 0.0) ? 1.0 : 0.0f) : " +
 153                       "((src0 == 0.0f) ? 1.0f : 0.0f)"))
 154 unop("fsign", tfloat, ("bit_size == 64 ? " +
 155                        "((src0 == 0.0) ? 0.0 : ((src0 > 0.0) ? 1.0 : -1.0)) : " +
 156                        "((src0 == 0.0f) ? 0.0f : ((src0 > 0.0f) ? 1.0f : -1.0f))"))
 157 unop("isign", tint, "(src0 == 0) ? 0 : ((src0 > 0) ? 1 : -1)")
 158 unop("iabs", tint, "(src0 < 0) ? -src0 : src0")
 159 unop("fabs", tfloat, "bit_size == 64 ? fabs(src0) : fabsf(src0)")
 160 unop("fsat", tfloat, ("bit_size == 64 ? " +
 161                       "((src0 > 1.0) ? 1.0 : ((src0 <= 0.0) ? 0.0 : src0)) : " +
 162                       "((src0 > 1.0f) ? 1.0f : ((src0 <= 0.0f) ? 0.0f : src0))"))
 163 unop("frcp", tfloat, "bit_size == 64 ? 1.0 / src0 : 1.0f / src0")
 164 unop("frsq", tfloat, "bit_size == 64 ? 1.0 / sqrt(src0) : 1.0f / sqrtf(src0)")
 165 unop("fsqrt", tfloat, "bit_size == 64 ? sqrt(src0) : sqrtf(src0)")
 166 unop("fexp2", tfloat, "exp2f(src0)")
 167 unop("flog2", tfloat, "log2f(src0)")
 168
 169 # Generate all of the numeric conversion opcodes
 170 for src_t in [tint, tuint, tfloat]:
 171    if src_t in (tint, tuint):
 172       dst_types = [tfloat, src_t]
 173    elif src_t == tfloat:
 174       dst_types = [tint, tuint, tfloat]
 175
 176    for dst_t in dst_types:
 177       for bit_size in [32, 64]:
 178          unop_convert("{0}2{1}{2}".format(src_t[0], dst_t[0], bit_size),
 179                       dst_t + str(bit_size), src_t, "src0")
 180
 181 # We'll hand-code the to/from bool conversion opcodes.  Because bool doesn't
 182 # have multiple bit-sizes, we can always infer the size from the other type.
 183 unop_convert("f2b", tbool, tfloat, "src0 != 0.0")
 184 unop_convert("i2b", tbool, tint, "src0 != 0")
 185 unop_convert("b2f", tfloat, tbool, "src0 ? 1.0 : 0.0")
 186 unop_convert("b2i", tint, tbool, "src0 ? 1 : 0")
 187
 188
 189 # Unary floating-point rounding operations.
 190
 191
 192 unop("ftrunc", tfloat, "bit_size == 64 ? trunc(src0) : truncf(src0)")
 193 unop("fceil", tfloat, "bit_size == 64 ? ceil(src0) : ceilf(src0)")
 194 unop("ffloor", tfloat, "bit_size == 64 ? floor(src0) : floorf(src0)")
 195 unop("ffract", tfloat, "src0 - (bit_size == 64 ? floor(src0) : floorf(src0))")
 196 unop("fround_even", tfloat, "bit_size == 64 ? _mesa_roundeven(src0) : _mesa_roundevenf(src0)")
 197
 198 unop("fquantize2f16", tfloat, "(fabs(src0) < ldexpf(1.0, -14)) ? copysignf(0.0f, src0) : _mesa_half_to_float(_mesa_float_to_half(src0))")
 199
 200 # Trigonometric operations.
 201
 202
 203 unop("fsin", tfloat, "bit_size == 64 ? sin(src0) : sinf(src0)")
 204 unop("fcos", tfloat, "bit_size == 64 ? cos(src0) : cosf(src0)")
 205
 206
 207 # Partial derivatives.
 208
 209
 210 unop("fddx", tfloat, "0.0") # the derivative of a constant is 0.
 211 unop("fddy", tfloat, "0.0")
 212 unop("fddx_fine", tfloat, "0.0")
 213 unop("fddy_fine", tfloat, "0.0")
 214 unop("fddx_coarse", tfloat, "0.0")
 215 unop("fddy_coarse", tfloat, "0.0")
 216
 217
 218 # Floating point pack and unpack operations.
 219
 220 def pack_2x16(fmt):
 221    unop_horiz("pack_" + fmt + "_2x16", 1, tuint32, 2, tfloat32, """
 222 dst.x = (uint32_t) pack_fmt_1x16(src0.x);
 223 dst.x |= ((uint32_t) pack_fmt_1x16(src0.y)) << 16;
 224 """.replace("fmt", fmt))
 225
 226 def pack_4x8(fmt):
 227    unop_horiz("pack_" + fmt + "_4x8", 1, tuint32, 4, tfloat32, """
 228 dst.x = (uint32_t) pack_fmt_1x8(src0.x);
 229 dst.x |= ((uint32_t) pack_fmt_1x8(src0.y)) << 8;
 230 dst.x |= ((uint32_t) pack_fmt_1x8(src0.z)) << 16;
 231 dst.x |= ((uint32_t) pack_fmt_1x8(src0.w)) << 24;
 232 """.replace("fmt", fmt))
 233
 234 def unpack_2x16(fmt):
 235    unop_horiz("unpack_" + fmt + "_2x16", 2, tfloat32, 1, tuint32, """
 236 dst.x = unpack_fmt_1x16((uint16_t)(src0.x & 0xffff));
 237 dst.y = unpack_fmt_1x16((uint16_t)(src0.x << 16));
 238 """.replace("fmt", fmt))
 239
 240 def unpack_4x8(fmt):
 241    unop_horiz("unpack_" + fmt + "_4x8", 4, tfloat32, 1, tuint32, """
 242 dst.x = unpack_fmt_1x8((uint8_t)(src0.x & 0xff));
 243 dst.y = unpack_fmt_1x8((uint8_t)((src0.x >> 8) & 0xff));
 244 dst.z = unpack_fmt_1x8((uint8_t)((src0.x >> 16) & 0xff));
 245 dst.w = unpack_fmt_1x8((uint8_t)(src0.x >> 24));
 246 """.replace("fmt", fmt))
 247
 248
 249 pack_2x16("snorm")
 250 pack_4x8("snorm")
 251 pack_2x16("unorm")
 252 pack_4x8("unorm")
 253 pack_2x16("half")
 254 unpack_2x16("snorm")
 255 unpack_4x8("snorm")
 256 unpack_2x16("unorm")
 257 unpack_4x8("unorm")
 258 unpack_2x16("half")
 259
 260 unop_horiz("pack_uvec2_to_uint", 1, tuint32, 2, tuint32, """
 261 dst.x = (src0.x & 0xffff) | (src0.y << 16);
 262 """)
 263
 264 unop_horiz("pack_uvec4_to_uint", 1, tuint32, 4, tuint32, """
 265 dst.x = (src0.x <<  0) |
 266         (src0.y <<  8) |
 267         (src0.z << 16) |
 268         (src0.w << 24);
 269 """)
 270
 271 unop_horiz("pack_64_2x32", 1, tuint64, 2, tuint32,
 272            "dst.x = src0.x | ((uint64_t)src0.y << 32);")
 273
 274 unop_horiz("unpack_64_2x32", 2, tuint32, 1, tuint64,
 275            "dst.x = src0.x; dst.y = src0.x >> 32;")
 276
 277 # Lowered floating point unpacking operations.
 278
 279
 280 unop_horiz("unpack_half_2x16_split_x", 1, tfloat32, 1, tuint32,
 281            "unpack_half_1x16((uint16_t)(src0.x & 0xffff))")
 282 unop_horiz("unpack_half_2x16_split_y", 1, tfloat32, 1, tuint32,
 283            "unpack_half_1x16((uint16_t)(src0.x >> 16))")
 284
 285 unop_convert("unpack_64_2x32_split_x", tuint32, tuint64, "src0")
 286 unop_convert("unpack_64_2x32_split_y", tuint32, tuint64, "src0 >> 32")
 287
 288 # Bit operations, part of ARB_gpu_shader5.
 289
 290
 291 unop("bitfield_reverse", tuint32, """
 292 /* we're not winning any awards for speed here, but that's ok */
 293 dst = 0;
 294 for (unsigned bit = 0; bit < 32; bit++)
 295    dst |= ((src0 >> bit) & 1) << (31 - bit);
 296 """)
 297 unop("bit_count", tuint32, """
 298 dst = 0;
 299 for (unsigned bit = 0; bit < 32; bit++) {
 300    if ((src0 >> bit) & 1)
 301       dst++;
 302 }
 303 """)
 304
 305 unop_convert("ufind_msb", tint32, tuint32, """
 306 dst = -1;
 307 for (int bit = 31; bit > 0; bit--) {
 308    if ((src0 >> bit) & 1) {
 309       dst = bit;
 310       break;
 311    }
 312 }
 313 """)
 314
 315 unop("ifind_msb", tint32, """
 316 dst = -1;
 317 for (int bit = 31; bit >= 0; bit--) {
 318    /* If src0 < 0, we're looking for the first 0 bit.
 319     * if src0 >= 0, we're looking for the first 1 bit.
 320     */
 321    if ((((src0 >> bit) & 1) && (src0 >= 0)) ||
 322       (!((src0 >> bit) & 1) && (src0 < 0))) {
 323       dst = bit;
 324       break;
 325    }
 326 }
 327 """)
 328
 329 unop("find_lsb", tint32, """
 330 dst = -1;
 331 for (unsigned bit = 0; bit < 32; bit++) {
 332    if ((src0 >> bit) & 1) {
 333       dst = bit;
 334       break;
 335    }
 336 }
 337 """)
 338
 339
 340 for i in xrange(1, 5):
 341    for j in xrange(1, 5):
 342       unop_horiz("fnoise{0}_{1}".format(i, j), i, tfloat, j, tfloat, "0.0f")
 343
 344 def binop_convert(name, out_type, in_type, alg_props, const_expr):
 345    opcode(name, 0, out_type, [0, 0], [in_type, in_type], alg_props, const_expr)
 346
 347 def binop(name, ty, alg_props, const_expr):
 348    binop_convert(name, ty, ty, alg_props, const_expr)
 349
 350 def binop_compare(name, ty, alg_props, const_expr):
 351    binop_convert(name, tbool, ty, alg_props, const_expr)
 352
 353 def binop_horiz(name, out_size, out_type, src1_size, src1_type, src2_size,
 354                 src2_type, const_expr):
 355    opcode(name, out_size, out_type, [src1_size, src2_size], [src1_type, src2_type],
 356           "", const_expr)
 357
 358 def binop_reduce(name, output_size, output_type, src_type, prereduce_expr,
 359                  reduce_expr, final_expr):
 360    def final(src):
 361       return final_expr.format(src= "(" + src + ")")
 362    def reduce_(src0, src1):
 363       return reduce_expr.format(src0=src0, src1=src1)
 364    def prereduce(src0, src1):
 365       return "(" + prereduce_expr.format(src0=src0, src1=src1) + ")"
 366    src0 = prereduce("src0.x", "src1.x")
 367    src1 = prereduce("src0.y", "src1.y")
 368    src2 = prereduce("src0.z", "src1.z")
 369    src3 = prereduce("src0.w", "src1.w")
 370    opcode(name + "2", output_size, output_type,
 371           [2, 2], [src_type, src_type], commutative,
 372           final(reduce_(src0, src1)))
 373    opcode(name + "3", output_size, output_type,
 374           [3, 3], [src_type, src_type], commutative,
 375           final(reduce_(reduce_(src0, src1), src2)))
 376    opcode(name + "4", output_size, output_type,
 377           [4, 4], [src_type, src_type], commutative,
 378           final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 379
 380 binop("fadd", tfloat, commutative + associative, "src0 + src1")
 381 binop("iadd", tint, commutative + associative, "src0 + src1")
 382 binop("fsub", tfloat, "", "src0 - src1")
 383 binop("isub", tint, "", "src0 - src1")
 384
 385 binop("fmul", tfloat, commutative + associative, "src0 * src1")
 386 # low 32-bits of signed/unsigned integer multiply
 387 binop("imul", tint, commutative + associative, "src0 * src1")
 388 # high 32-bits of signed integer multiply
 389 binop("imul_high", tint32, commutative,
 390       "(int32_t)(((int64_t) src0 * (int64_t) src1) >> 32)")
 391 # high 32-bits of unsigned integer multiply
 392 binop("umul_high", tuint32, commutative,
 393       "(uint32_t)(((uint64_t) src0 * (uint64_t) src1) >> 32)")
 394
 395 binop("fdiv", tfloat, "", "src0 / src1")
 396 binop("idiv", tint, "", "src0 / src1")
 397 binop("udiv", tuint, "", "src0 / src1")
 398
 399 # returns a boolean representing the carry resulting from the addition of
 400 # the two unsigned arguments.
 401
 402 binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0")
 403
 404 # returns a boolean representing the borrow resulting from the subtraction
 405 # of the two unsigned arguments.
 406
 407 binop_convert("usub_borrow", tuint, tuint, "", "src0 < src1")
 408
 409 binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1")
 410
 411 # For signed integers, there are several different possible definitions of
 412 # "modulus" or "remainder".  We follow the conventions used by LLVM and
 413 # SPIR-V.  The irem opcode implements the standard C/C++ signed "%"
 414 # operation while the imod opcode implements the more mathematical
 415 # "modulus" operation.  For details on the difference, see
 416 #
 417 # http://mathforum.org/library/drmath/view/52343.html
 418
 419 binop("irem", tint, "", "src1 == 0 ? 0 : src0 % src1")
 420 binop("imod", tint, "",
 421       "src1 == 0 ? 0 : ((src0 % src1 == 0 || (src0 >= 0) == (src1 >= 0)) ?"
 422       "                 src0 % src1 : src0 % src1 + src1)")
 423 binop("fmod", tfloat, "", "src0 - src1 * floorf(src0 / src1)")
 424 binop("frem", tfloat, "", "src0 - src1 * truncf(src0 / src1)")
 425
 426 #
 427 # Comparisons
 428 #
 429
 430
 431 # these integer-aware comparisons return a boolean (0 or ~0)
 432
 433 binop_compare("flt", tfloat, "", "src0 < src1")
 434 binop_compare("fge", tfloat, "", "src0 >= src1")
 435 binop_compare("feq", tfloat, commutative, "src0 == src1")
 436 binop_compare("fne", tfloat, commutative, "src0 != src1")
 437 binop_compare("ilt", tint, "", "src0 < src1")
 438 binop_compare("ige", tint, "", "src0 >= src1")
 439 binop_compare("ieq", tint, commutative, "src0 == src1")
 440 binop_compare("ine", tint, commutative, "src0 != src1")
 441 binop_compare("ult", tuint, "", "src0 < src1")
 442 binop_compare("uge", tuint, "", "src0 >= src1")
 443
 444 # integer-aware GLSL-style comparisons that compare floats and ints
 445
 446 binop_reduce("ball_fequal",  1, tbool, tfloat, "{src0} == {src1}",
 447              "{src0} && {src1}", "{src}")
 448 binop_reduce("bany_fnequal", 1, tbool, tfloat, "{src0} != {src1}",
 449              "{src0} || {src1}", "{src}")
 450 binop_reduce("ball_iequal",  1, tbool, tint, "{src0} == {src1}",
 451              "{src0} && {src1}", "{src}")
 452 binop_reduce("bany_inequal", 1, tbool, tint, "{src0} != {src1}",
 453              "{src0} || {src1}", "{src}")
 454
 455 # non-integer-aware GLSL-style comparisons that return 0.0 or 1.0
 456
 457 binop_reduce("fall_equal",  1, tfloat32, tfloat32, "{src0} == {src1}",
 458              "{src0} && {src1}", "{src} ? 1.0f : 0.0f")
 459 binop_reduce("fany_nequal", 1, tfloat32, tfloat32, "{src0} != {src1}",
 460              "{src0} || {src1}", "{src} ? 1.0f : 0.0f")
 461
 462 # These comparisons for integer-less hardware return 1.0 and 0.0 for true
 463 # and false respectively
 464
 465 binop("slt", tfloat32, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than
 466 binop("sge", tfloat32, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal
 467 binop("seq", tfloat32, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
 468 binop("sne", tfloat32, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
 469
 470
 471 opcode("ishl", 0, tint, [0, 0], [tint, tuint32], "", "src0 << src1")
 472 opcode("ishr", 0, tint, [0, 0], [tint, tuint32], "", "src0 >> src1")
 473 opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], "", "src0 >> src1")
 474
 475 # bitwise logic operators
 476 #
 477 # These are also used as boolean and, or, xor for hardware supporting
 478 # integers.
 479
 480
 481 binop("iand", tuint, commutative + associative, "src0 & src1")
 482 binop("ior", tuint, commutative + associative, "src0 | src1")
 483 binop("ixor", tuint, commutative + associative, "src0 ^ src1")
 484
 485
 486 # floating point logic operators
 487 #
 488 # These use (src != 0.0) for testing the truth of the input, and output 1.0
 489 # for true and 0.0 for false
 490
 491 binop("fand", tfloat32, commutative,
 492       "((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f")
 493 binop("for", tfloat32, commutative,
 494       "((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f")
 495 binop("fxor", tfloat32, commutative,
 496       "(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f")
 497
 498 binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}",
 499              "{src}")
 500
 501 binop_reduce("fdot_replicated", 4, tfloat, tfloat,
 502              "{src0} * {src1}", "{src0} + {src1}", "{src}")
 503
 504 opcode("fdph", 1, tfloat, [3, 4], [tfloat, tfloat], "",
 505        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 506 opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], "",
 507        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 508
 509 binop("fmin", tfloat, "", "fminf(src0, src1)")
 510 binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1")
 511 binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1")
 512 binop("fmax", tfloat, "", "fmaxf(src0, src1)")
 513 binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0")
 514 binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0")
 515
 516 # Saturated vector add for 4 8bit ints.
 517 binop("usadd_4x8", tint32, commutative + associative, """
 518 dst = 0;
 519 for (int i = 0; i < 32; i += 8) {
 520    dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i;
 521 }
 522 """)
 523
 524 # Saturated vector subtract for 4 8bit ints.
 525 binop("ussub_4x8", tint32, "", """
 526 dst = 0;
 527 for (int i = 0; i < 32; i += 8) {
 528    int src0_chan = (src0 >> i) & 0xff;
 529    int src1_chan = (src1 >> i) & 0xff;
 530    if (src0_chan > src1_chan)
 531       dst |= (src0_chan - src1_chan) << i;
 532 }
 533 """)
 534
 535 # vector min for 4 8bit ints.
 536 binop("umin_4x8", tint32, commutative + associative, """
 537 dst = 0;
 538 for (int i = 0; i < 32; i += 8) {
 539    dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 540 }
 541 """)
 542
 543 # vector max for 4 8bit ints.
 544 binop("umax_4x8", tint32, commutative + associative, """
 545 dst = 0;
 546 for (int i = 0; i < 32; i += 8) {
 547    dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 548 }
 549 """)
 550
 551 # unorm multiply: (a * b) / 255.
 552 binop("umul_unorm_4x8", tint32, commutative + associative, """
 553 dst = 0;
 554 for (int i = 0; i < 32; i += 8) {
 555    int src0_chan = (src0 >> i) & 0xff;
 556    int src1_chan = (src1 >> i) & 0xff;
 557    dst |= ((src0_chan * src1_chan) / 255) << i;
 558 }
 559 """)
 560
 561 binop("fpow", tfloat, "", "bit_size == 64 ? powf(src0, src1) : pow(src0, src1)")
 562
 563 binop_horiz("pack_half_2x16_split", 1, tuint32, 1, tfloat32, 1, tfloat32,
 564             "pack_half_1x16(src0.x) | (pack_half_1x16(src1.x) << 16)")
 565
 566 binop_convert("pack_64_2x32_split", tuint64, tuint32, "",
 567               "src0 | ((uint64_t)src1 << 32)")
 568
 569 # bfm implements the behavior of the first operation of the SM5 "bfi" assembly
 570 # and that of the "bfi1" i965 instruction. That is, it has undefined behavior
 571 # if either of its arguments are 32.
 572 binop_convert("bfm", tuint32, tint32, "", """
 573 int bits = src0, offset = src1;
 574 if (offset < 0 || bits < 0 || offset > 31 || bits > 31 || offset + bits > 32)
 575    dst = 0; /* undefined */
 576 else
 577    dst = ((1u << bits) - 1) << offset;
 578 """)
 579
 580 opcode("ldexp", 0, tfloat, [0, 0], [tfloat, tint32], "", """
 581 dst = (bit_size == 64) ? ldexp(src0, src1) : ldexpf(src0, src1);
 582 /* flush denormals to zero. */
 583 if (!isnormal(dst))
 584    dst = copysignf(0.0f, src0);
 585 """)
 586
 587 # Combines the first component of each input to make a 2-component vector.
 588
 589 binop_horiz("vec2", 2, tuint, 1, tuint, 1, tuint, """
 590 dst.x = src0.x;
 591 dst.y = src1.x;
 592 """)
 593
 594 # Byte extraction
 595 binop("extract_u8", tuint, "", "(uint8_t)(src0 >> (src1 * 8))")
 596 binop("extract_i8", tint, "", "(int8_t)(src0 >> (src1 * 8))")
 597
 598 # Word extraction
 599 binop("extract_u16", tuint, "", "(uint16_t)(src0 >> (src1 * 16))")
 600 binop("extract_i16", tint, "", "(int16_t)(src0 >> (src1 * 16))")
 601
 602
 603 def triop(name, ty, const_expr):
 604    opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], "", const_expr)
 605 def triop_horiz(name, output_size, src1_size, src2_size, src3_size, const_expr):
 606    opcode(name, output_size, tuint,
 607    [src1_size, src2_size, src3_size],
 608    [tuint, tuint, tuint], "", const_expr)
 609
 610 triop("ffma", tfloat, "src0 * src1 + src2")
 611
 612 triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2")
 613
 614 # Conditional Select
 615 #
 616 # A vector conditional select instruction (like ?:, but operating per-
 617 # component on vectors). There are two versions, one for floating point
 618 # bools (0.0 vs 1.0) and one for integer bools (0 vs ~0).
 619
 620
 621 triop("fcsel", tfloat32, "(src0 != 0.0f) ? src1 : src2")
 622 opcode("bcsel", 0, tuint, [0, 0, 0],
 623       [tbool, tuint, tuint], "", "src0 ? src1 : src2")
 624
 625 # SM5 bfi assembly
 626 triop("bfi", tuint32, """
 627 unsigned mask = src0, insert = src1, base = src2;
 628 if (mask == 0) {
 629    dst = base;
 630 } else {
 631    unsigned tmp = mask;
 632    while (!(tmp & 1)) {
 633       tmp >>= 1;
 634       insert <<= 1;
 635    }
 636    dst = (base & ~mask) | (insert & mask);
 637 }
 638 """)
 639
 640 # SM5 ubfe/ibfe assembly
 641 opcode("ubfe", 0, tuint32,
 642        [0, 0, 0], [tuint32, tint32, tint32], "", """
 643 unsigned base = src0;
 644 int offset = src1, bits = src2;
 645 if (bits == 0) {
 646    dst = 0;
 647 } else if (bits < 0 || offset < 0) {
 648    dst = 0; /* undefined */
 649 } else if (offset + bits < 32) {
 650    dst = (base << (32 - bits - offset)) >> (32 - bits);
 651 } else {
 652    dst = base >> offset;
 653 }
 654 """)
 655 opcode("ibfe", 0, tint32,
 656        [0, 0, 0], [tint32, tint32, tint32], "", """
 657 int base = src0;
 658 int offset = src1, bits = src2;
 659 if (bits == 0) {
 660    dst = 0;
 661 } else if (bits < 0 || offset < 0) {
 662    dst = 0; /* undefined */
 663 } else if (offset + bits < 32) {
 664    dst = (base << (32 - bits - offset)) >> (32 - bits);
 665 } else {
 666    dst = base >> offset;
 667 }
 668 """)
 669
 670 # GLSL bitfieldExtract()
 671 opcode("ubitfield_extract", 0, tuint32,
 672        [0, 0, 0], [tuint32, tint32, tint32], "", """
 673 unsigned base = src0;
 674 int offset = src1, bits = src2;
 675 if (bits == 0) {
 676    dst = 0;
 677 } else if (bits < 0 || offset < 0 || offset + bits > 32) {
 678    dst = 0; /* undefined per the spec */
 679 } else {
 680    dst = (base >> offset) & ((1ull << bits) - 1);
 681 }
 682 """)
 683 opcode("ibitfield_extract", 0, tint32,
 684        [0, 0, 0], [tint32, tint32, tint32], "", """
 685 int base = src0;
 686 int offset = src1, bits = src2;
 687 if (bits == 0) {
 688    dst = 0;
 689 } else if (offset < 0 || bits < 0 || offset + bits > 32) {
 690    dst = 0;
 691 } else {
 692    dst = (base << (32 - offset - bits)) >> offset; /* use sign-extending shift */
 693 }
 694 """)
 695
 696 # Combines the first component of each input to make a 3-component vector.
 697
 698 triop_horiz("vec3", 3, 1, 1, 1, """
 699 dst.x = src0.x;
 700 dst.y = src1.x;
 701 dst.z = src2.x;
 702 """)
 703
 704 def quadop_horiz(name, output_size, src1_size, src2_size, src3_size,
 705                  src4_size, const_expr):
 706    opcode(name, output_size, tuint,
 707           [src1_size, src2_size, src3_size, src4_size],
 708           [tuint, tuint, tuint, tuint],
 709           "", const_expr)
 710
 711 opcode("bitfield_insert", 0, tuint32, [0, 0, 0, 0],
 712        [tuint32, tuint32, tint32, tint32], "", """
 713 unsigned base = src0, insert = src1;
 714 int offset = src2, bits = src3;
 715 if (bits == 0) {
 716    dst = 0;
 717 } else if (offset < 0 || bits < 0 || bits + offset > 32) {
 718    dst = 0;
 719 } else {
 720    unsigned mask = ((1ull << bits) - 1) << offset;
 721    dst = (base & ~mask) | ((insert << bits) & mask);
 722 }
 723 """)
 724
 725 quadop_horiz("vec4", 4, 1, 1, 1, 1, """
 726 dst.x = src0.x;
 727 dst.y = src1.x;
 728 dst.z = src2.x;
 729 dst.w = src3.x;
 730 """)
 731
 732