src/compiler/nir/nir_opcodes.py

   1 #! /usr/bin/env python
   2 #
   3 # Copyright (C) 2014 Connor Abbott
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a
   6 # copy of this software and associated documentation files (the "Software"),
   7 # to deal in the Software without restriction, including without limitation
   8 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 # and/or sell copies of the Software, and to permit persons to whom the
  10 # Software is furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice (including the next
  13 # paragraph) shall be included in all copies or substantial portions of the
  14 # Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22 # IN THE SOFTWARE.
  23 #
  24 # Authors:
  25 #    Connor Abbott (cwabbott0@gmail.com)
  26
  27
  28 # Class that represents all the information we have about the opcode
  29 # NOTE: this must be kept in sync with nir_op_info
  30
  31 class Opcode(object):
  32    """Class that represents all the information we have about the opcode
  33    NOTE: this must be kept in sync with nir_op_info
  34    """
  35    def __init__(self, name, output_size, output_type, input_sizes,
  36                 input_types, algebraic_properties, const_expr):
  37       """Parameters:
  38
  39       - name is the name of the opcode (prepend nir_op_ for the enum name)
  40       - all types are strings that get nir_type_ prepended to them
  41       - input_types is a list of types
  42       - algebraic_properties is a space-seperated string, where nir_op_is_ is
  43         prepended before each entry
  44       - const_expr is an expression or series of statements that computes the
  45         constant value of the opcode given the constant values of its inputs.
  46
  47       Constant expressions are formed from the variables src0, src1, ...,
  48       src(N-1), where N is the number of arguments.  The output of the
  49       expression should be stored in the dst variable.  Per-component input
  50       and output variables will be scalars and non-per-component input and
  51       output variables will be a struct with fields named x, y, z, and w
  52       all of the correct type.  Input and output variables can be assumed
  53       to already be of the correct type and need no conversion.  In
  54       particular, the conversion from the C bool type to/from  NIR_TRUE and
  55       NIR_FALSE happens automatically.
  56
  57       For per-component instructions, the entire expression will be
  58       executed once for each component.  For non-per-component
  59       instructions, the expression is expected to store the correct values
  60       in dst.x, dst.y, etc.  If "dst" does not exist anywhere in the
  61       constant expression, an assignment to dst will happen automatically
  62       and the result will be equivalent to "dst = <expression>" for
  63       per-component instructions and "dst.x = dst.y = ... = <expression>"
  64       for non-per-component instructions.
  65       """
  66       assert isinstance(name, str)
  67       assert isinstance(output_size, int)
  68       assert isinstance(output_type, str)
  69       assert isinstance(input_sizes, list)
  70       assert isinstance(input_sizes[0], int)
  71       assert isinstance(input_types, list)
  72       assert isinstance(input_types[0], str)
  73       assert isinstance(algebraic_properties, str)
  74       assert isinstance(const_expr, str)
  75       assert len(input_sizes) == len(input_types)
  76       assert 0 <= output_size <= 4
  77       for size in input_sizes:
  78          assert 0 <= size <= 4
  79          if output_size != 0:
  80             assert size != 0
  81       self.name = name
  82       self.num_inputs = len(input_sizes)
  83       self.output_size = output_size
  84       self.output_type = output_type
  85       self.input_sizes = input_sizes
  86       self.input_types = input_types
  87       self.algebraic_properties = algebraic_properties
  88       self.const_expr = const_expr
  89
  90 # helper variables for strings
  91 tfloat = "float"
  92 tint = "int"
  93 tbool = "bool"
  94 tuint = "uint"
  95
  96 commutative = "commutative "
  97 associative = "associative "
  98
  99 # global dictionary of opcodes
 100 opcodes = {}
 101
 102 def opcode(name, output_size, output_type, input_sizes, input_types,
 103            algebraic_properties, const_expr):
 104    assert name not in opcodes
 105    opcodes[name] = Opcode(name, output_size, output_type, input_sizes,
 106                           input_types, algebraic_properties, const_expr)
 107
 108 def unop_convert(name, out_type, in_type, const_expr):
 109    opcode(name, 0, out_type, [0], [in_type], "", const_expr)
 110
 111 def unop(name, ty, const_expr):
 112    opcode(name, 0, ty, [0], [ty], "", const_expr)
 113
 114 def unop_horiz(name, output_size, output_type, input_size, input_type,
 115                const_expr):
 116    opcode(name, output_size, output_type, [input_size], [input_type], "",
 117           const_expr)
 118
 119 def unop_reduce(name, output_size, output_type, input_type, prereduce_expr,
 120                 reduce_expr, final_expr):
 121    def prereduce(src):
 122       return "(" + prereduce_expr.format(src=src) + ")"
 123    def final(src):
 124       return final_expr.format(src="(" + src + ")")
 125    def reduce_(src0, src1):
 126       return reduce_expr.format(src0=src0, src1=src1)
 127    src0 = prereduce("src0.x")
 128    src1 = prereduce("src0.y")
 129    src2 = prereduce("src0.z")
 130    src3 = prereduce("src0.w")
 131    unop_horiz(name + "2", output_size, output_type, 2, input_type,
 132               final(reduce_(src0, src1)))
 133    unop_horiz(name + "3", output_size, output_type, 3, input_type,
 134               final(reduce_(reduce_(src0, src1), src2)))
 135    unop_horiz(name + "4", output_size, output_type, 4, input_type,
 136               final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 137
 138
 139 # These two move instructions differ in what modifiers they support and what
 140 # the negate modifier means. Otherwise, they are identical.
 141 unop("fmov", tfloat, "src0")
 142 unop("imov", tint, "src0")
 143
 144 unop("ineg", tint, "-src0")
 145 unop("fneg", tfloat, "-src0")
 146 unop("inot", tint, "~src0") # invert every bit of the integer
 147 unop("fnot", tfloat, "(src0 == 0.0f) ? 1.0f : 0.0f")
 148 unop("fsign", tfloat, "(src0 == 0.0f) ? 0.0f : ((src0 > 0.0f) ? 1.0f : -1.0f)")
 149 unop("isign", tint, "(src0 == 0) ? 0 : ((src0 > 0) ? 1 : -1)")
 150 unop("iabs", tint, "(src0 < 0) ? -src0 : src0")
 151 unop("fabs", tfloat, "fabsf(src0)")
 152 unop("fsat", tfloat, "(src0 > 1.0f) ? 1.0f : ((src0 <= 0.0f) ? 0.0f : src0)")
 153 unop("frcp", tfloat, "1.0f / src0")
 154 unop("frsq", tfloat, "1.0f / sqrtf(src0)")
 155 unop("fsqrt", tfloat, "sqrtf(src0)")
 156 unop("fexp2", tfloat, "exp2f(src0)")
 157 unop("flog2", tfloat, "log2f(src0)")
 158 unop_convert("f2i", tint, tfloat, "src0") # Float-to-integer conversion.
 159 unop_convert("f2u", tuint, tfloat, "src0") # Float-to-unsigned conversion
 160 unop_convert("i2f", tfloat, tint, "src0") # Integer-to-float conversion.
 161 # Float-to-boolean conversion
 162 unop_convert("f2b", tbool, tfloat, "src0 != 0.0f")
 163 # Boolean-to-float conversion
 164 unop_convert("b2f", tfloat, tbool, "src0 ? 1.0f : 0.0f")
 165 # Int-to-boolean conversion
 166 unop_convert("i2b", tbool, tint, "src0 != 0")
 167 unop_convert("b2i", tint, tbool, "src0 ? 1 : 0") # Boolean-to-int conversion
 168 unop_convert("u2f", tfloat, tuint, "src0") # Unsigned-to-float conversion.
 169
 170 # Unary floating-point rounding operations.
 171
 172
 173 unop("ftrunc", tfloat, "truncf(src0)")
 174 unop("fceil", tfloat, "ceilf(src0)")
 175 unop("ffloor", tfloat, "floorf(src0)")
 176 unop("ffract", tfloat, "src0 - floorf(src0)")
 177 unop("fround_even", tfloat, "_mesa_roundevenf(src0)")
 178
 179 unop("fquantize2f16", tfloat, "(fabs(src0) < ldexpf(1.0, -14)) ? copysignf(0.0f, src0) : _mesa_half_to_float(_mesa_float_to_half(src0))")
 180
 181 # Trigonometric operations.
 182
 183
 184 unop("fsin", tfloat, "sinf(src0)")
 185 unop("fcos", tfloat, "cosf(src0)")
 186
 187
 188 # Partial derivatives.
 189
 190
 191 unop("fddx", tfloat, "0.0f") # the derivative of a constant is 0.
 192 unop("fddy", tfloat, "0.0f")
 193 unop("fddx_fine", tfloat, "0.0f")
 194 unop("fddy_fine", tfloat, "0.0f")
 195 unop("fddx_coarse", tfloat, "0.0f")
 196 unop("fddy_coarse", tfloat, "0.0f")
 197
 198
 199 # Floating point pack and unpack operations.
 200
 201 def pack_2x16(fmt):
 202    unop_horiz("pack_" + fmt + "_2x16", 1, tuint, 2, tfloat, """
 203 dst.x = (uint32_t) pack_fmt_1x16(src0.x);
 204 dst.x |= ((uint32_t) pack_fmt_1x16(src0.y)) << 16;
 205 """.replace("fmt", fmt))
 206
 207 def pack_4x8(fmt):
 208    unop_horiz("pack_" + fmt + "_4x8", 1, tuint, 4, tfloat, """
 209 dst.x = (uint32_t) pack_fmt_1x8(src0.x);
 210 dst.x |= ((uint32_t) pack_fmt_1x8(src0.y)) << 8;
 211 dst.x |= ((uint32_t) pack_fmt_1x8(src0.z)) << 16;
 212 dst.x |= ((uint32_t) pack_fmt_1x8(src0.w)) << 24;
 213 """.replace("fmt", fmt))
 214
 215 def unpack_2x16(fmt):
 216    unop_horiz("unpack_" + fmt + "_2x16", 2, tfloat, 1, tuint, """
 217 dst.x = unpack_fmt_1x16((uint16_t)(src0.x & 0xffff));
 218 dst.y = unpack_fmt_1x16((uint16_t)(src0.x << 16));
 219 """.replace("fmt", fmt))
 220
 221 def unpack_4x8(fmt):
 222    unop_horiz("unpack_" + fmt + "_4x8", 4, tfloat, 1, tuint, """
 223 dst.x = unpack_fmt_1x8((uint8_t)(src0.x & 0xff));
 224 dst.y = unpack_fmt_1x8((uint8_t)((src0.x >> 8) & 0xff));
 225 dst.z = unpack_fmt_1x8((uint8_t)((src0.x >> 16) & 0xff));
 226 dst.w = unpack_fmt_1x8((uint8_t)(src0.x >> 24));
 227 """.replace("fmt", fmt))
 228
 229
 230 pack_2x16("snorm")
 231 pack_4x8("snorm")
 232 pack_2x16("unorm")
 233 pack_4x8("unorm")
 234 pack_2x16("half")
 235 unpack_2x16("snorm")
 236 unpack_4x8("snorm")
 237 unpack_2x16("unorm")
 238 unpack_4x8("unorm")
 239 unpack_2x16("half")
 240
 241 unop_horiz("pack_uvec2_to_uint", 0, tuint, 2, tuint, """
 242 dst = (src0.x & 0xffff) | (src0.y >> 16);
 243 """)
 244
 245 unop_horiz("pack_uvec4_to_uint", 0, tuint, 4, tuint, """
 246 dst = (src0.x <<  0) |
 247       (src0.y <<  8) |
 248       (src0.z << 16) |
 249       (src0.w << 24);
 250 """)
 251
 252 # Lowered floating point unpacking operations.
 253
 254
 255 unop_horiz("unpack_half_2x16_split_x", 1, tfloat, 1, tuint,
 256            "unpack_half_1x16((uint16_t)(src0.x & 0xffff))")
 257 unop_horiz("unpack_half_2x16_split_y", 1, tfloat, 1, tuint,
 258            "unpack_half_1x16((uint16_t)(src0.x >> 16))")
 259
 260
 261 # Bit operations, part of ARB_gpu_shader5.
 262
 263
 264 unop("bitfield_reverse", tuint, """
 265 /* we're not winning any awards for speed here, but that's ok */
 266 dst = 0;
 267 for (unsigned bit = 0; bit < 32; bit++)
 268    dst |= ((src0 >> bit) & 1) << (31 - bit);
 269 """)
 270 unop("bit_count", tuint, """
 271 dst = 0;
 272 for (unsigned bit = 0; bit < 32; bit++) {
 273    if ((src0 >> bit) & 1)
 274       dst++;
 275 }
 276 """)
 277
 278 unop_convert("ufind_msb", tint, tuint, """
 279 dst = -1;
 280 for (int bit = 31; bit > 0; bit--) {
 281    if ((src0 >> bit) & 1) {
 282       dst = bit;
 283       break;
 284    }
 285 }
 286 """)
 287
 288 unop("ifind_msb", tint, """
 289 dst = -1;
 290 for (int bit = 31; bit >= 0; bit--) {
 291    /* If src0 < 0, we're looking for the first 0 bit.
 292     * if src0 >= 0, we're looking for the first 1 bit.
 293     */
 294    if ((((src0 >> bit) & 1) && (src0 >= 0)) ||
 295       (!((src0 >> bit) & 1) && (src0 < 0))) {
 296       dst = bit;
 297       break;
 298    }
 299 }
 300 """)
 301
 302 unop("find_lsb", tint, """
 303 dst = -1;
 304 for (unsigned bit = 0; bit < 32; bit++) {
 305    if ((src0 >> bit) & 1) {
 306       dst = bit;
 307       break;
 308    }
 309 }
 310 """)
 311
 312
 313 for i in xrange(1, 5):
 314    for j in xrange(1, 5):
 315       unop_horiz("fnoise{0}_{1}".format(i, j), i, tfloat, j, tfloat, "0.0f")
 316
 317 def binop_convert(name, out_type, in_type, alg_props, const_expr):
 318    opcode(name, 0, out_type, [0, 0], [in_type, in_type], alg_props, const_expr)
 319
 320 def binop(name, ty, alg_props, const_expr):
 321    binop_convert(name, ty, ty, alg_props, const_expr)
 322
 323 def binop_compare(name, ty, alg_props, const_expr):
 324    binop_convert(name, tbool, ty, alg_props, const_expr)
 325
 326 def binop_horiz(name, out_size, out_type, src1_size, src1_type, src2_size,
 327                 src2_type, const_expr):
 328    opcode(name, out_size, out_type, [src1_size, src2_size], [src1_type, src2_type],
 329           "", const_expr)
 330
 331 def binop_reduce(name, output_size, output_type, src_type, prereduce_expr,
 332                  reduce_expr, final_expr):
 333    def final(src):
 334       return final_expr.format(src= "(" + src + ")")
 335    def reduce_(src0, src1):
 336       return reduce_expr.format(src0=src0, src1=src1)
 337    def prereduce(src0, src1):
 338       return "(" + prereduce_expr.format(src0=src0, src1=src1) + ")"
 339    src0 = prereduce("src0.x", "src1.x")
 340    src1 = prereduce("src0.y", "src1.y")
 341    src2 = prereduce("src0.z", "src1.z")
 342    src3 = prereduce("src0.w", "src1.w")
 343    opcode(name + "2", output_size, output_type,
 344           [2, 2], [src_type, src_type], commutative,
 345           final(reduce_(src0, src1)))
 346    opcode(name + "3", output_size, output_type,
 347           [3, 3], [src_type, src_type], commutative,
 348           final(reduce_(reduce_(src0, src1), src2)))
 349    opcode(name + "4", output_size, output_type,
 350           [4, 4], [src_type, src_type], commutative,
 351           final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 352
 353 binop("fadd", tfloat, commutative + associative, "src0 + src1")
 354 binop("iadd", tint, commutative + associative, "src0 + src1")
 355 binop("fsub", tfloat, "", "src0 - src1")
 356 binop("isub", tint, "", "src0 - src1")
 357
 358 binop("fmul", tfloat, commutative + associative, "src0 * src1")
 359 # low 32-bits of signed/unsigned integer multiply
 360 binop("imul", tint, commutative + associative, "src0 * src1")
 361 # high 32-bits of signed integer multiply
 362 binop("imul_high", tint, commutative,
 363       "(int32_t)(((int64_t) src0 * (int64_t) src1) >> 32)")
 364 # high 32-bits of unsigned integer multiply
 365 binop("umul_high", tuint, commutative,
 366       "(uint32_t)(((uint64_t) src0 * (uint64_t) src1) >> 32)")
 367
 368 binop("fdiv", tfloat, "", "src0 / src1")
 369 binop("idiv", tint, "", "src0 / src1")
 370 binop("udiv", tuint, "", "src0 / src1")
 371
 372 # returns a boolean representing the carry resulting from the addition of
 373 # the two unsigned arguments.
 374
 375 binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0")
 376
 377 # returns a boolean representing the borrow resulting from the subtraction
 378 # of the two unsigned arguments.
 379
 380 binop_convert("usub_borrow", tuint, tuint, "", "src0 < src1")
 381
 382 binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1")
 383
 384 # For signed integers, there are several different possible definitions of
 385 # "modulus" or "remainder".  We follow the conventions used by LLVM and
 386 # SPIR-V.  The irem opcode implements the standard C/C++ signed "%"
 387 # operation while the imod opcode implements the more mathematical
 388 # "modulus" operation.  For details on the difference, see
 389 #
 390 # http://mathforum.org/library/drmath/view/52343.html
 391
 392 binop("irem", tint, "", "src1 == 0 ? 0 : src0 % src1")
 393 binop("imod", tint, "",
 394       "src1 == 0 ? 0 : ((src0 % src1 == 0 || (src0 >= 0) == (src1 >= 0)) ?"
 395       "                 src0 % src1 : src0 % src1 + src1)")
 396 binop("fmod", tfloat, "", "src0 - src1 * floorf(src0 / src1)")
 397 binop("frem", tfloat, "", "src0 - src1 * truncf(src0 / src1)")
 398
 399 #
 400 # Comparisons
 401 #
 402
 403
 404 # these integer-aware comparisons return a boolean (0 or ~0)
 405
 406 binop_compare("flt", tfloat, "", "src0 < src1")
 407 binop_compare("fge", tfloat, "", "src0 >= src1")
 408 binop_compare("feq", tfloat, commutative, "src0 == src1")
 409 binop_compare("fne", tfloat, commutative, "src0 != src1")
 410 binop_compare("ilt", tint, "", "src0 < src1")
 411 binop_compare("ige", tint, "", "src0 >= src1")
 412 binop_compare("ieq", tint, commutative, "src0 == src1")
 413 binop_compare("ine", tint, commutative, "src0 != src1")
 414 binop_compare("ult", tuint, "", "src0 < src1")
 415 binop_compare("uge", tuint, "", "src0 >= src1")
 416
 417 # integer-aware GLSL-style comparisons that compare floats and ints
 418
 419 binop_reduce("ball_fequal",  1, tbool, tfloat, "{src0} == {src1}",
 420              "{src0} && {src1}", "{src}")
 421 binop_reduce("bany_fnequal", 1, tbool, tfloat, "{src0} != {src1}",
 422              "{src0} || {src1}", "{src}")
 423 binop_reduce("ball_iequal",  1, tbool, tint, "{src0} == {src1}",
 424              "{src0} && {src1}", "{src}")
 425 binop_reduce("bany_inequal", 1, tbool, tint, "{src0} != {src1}",
 426              "{src0} || {src1}", "{src}")
 427
 428 # non-integer-aware GLSL-style comparisons that return 0.0 or 1.0
 429
 430 binop_reduce("fall_equal",  1, tfloat, tfloat, "{src0} == {src1}",
 431              "{src0} && {src1}", "{src} ? 1.0f : 0.0f")
 432 binop_reduce("fany_nequal", 1, tfloat, tfloat, "{src0} != {src1}",
 433              "{src0} || {src1}", "{src} ? 1.0f : 0.0f")
 434
 435 # These comparisons for integer-less hardware return 1.0 and 0.0 for true
 436 # and false respectively
 437
 438 binop("slt", tfloat, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than
 439 binop("sge", tfloat, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal
 440 binop("seq", tfloat, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
 441 binop("sne", tfloat, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
 442
 443
 444 binop("ishl", tint, "", "src0 << src1")
 445 binop("ishr", tint, "", "src0 >> src1")
 446 binop("ushr", tuint, "", "src0 >> src1")
 447
 448 # bitwise logic operators
 449 #
 450 # These are also used as boolean and, or, xor for hardware supporting
 451 # integers.
 452
 453
 454 binop("iand", tuint, commutative + associative, "src0 & src1")
 455 binop("ior", tuint, commutative + associative, "src0 | src1")
 456 binop("ixor", tuint, commutative + associative, "src0 ^ src1")
 457
 458
 459 # floating point logic operators
 460 #
 461 # These use (src != 0.0) for testing the truth of the input, and output 1.0
 462 # for true and 0.0 for false
 463
 464 binop("fand", tfloat, commutative,
 465       "((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f")
 466 binop("for", tfloat, commutative,
 467       "((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f")
 468 binop("fxor", tfloat, commutative,
 469       "(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f")
 470
 471 binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}",
 472              "{src}")
 473
 474 binop_reduce("fdot_replicated", 4, tfloat, tfloat,
 475              "{src0} * {src1}", "{src0} + {src1}", "{src}")
 476
 477 opcode("fdph", 1, tfloat, [3, 4], [tfloat, tfloat], "",
 478        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 479 opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], "",
 480        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 481
 482 binop("fmin", tfloat, "", "fminf(src0, src1)")
 483 binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1")
 484 binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1")
 485 binop("fmax", tfloat, "", "fmaxf(src0, src1)")
 486 binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0")
 487 binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0")
 488
 489 # Saturated vector add for 4 8bit ints.
 490 binop("usadd_4x8", tint, commutative + associative, """
 491 dst = 0;
 492 for (int i = 0; i < 32; i += 8) {
 493    dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i;
 494 }
 495 """)
 496
 497 # Saturated vector subtract for 4 8bit ints.
 498 binop("ussub_4x8", tint, "", """
 499 dst = 0;
 500 for (int i = 0; i < 32; i += 8) {
 501    int src0_chan = (src0 >> i) & 0xff;
 502    int src1_chan = (src1 >> i) & 0xff;
 503    if (src0_chan > src1_chan)
 504       dst |= (src0_chan - src1_chan) << i;
 505 }
 506 """)
 507
 508 # vector min for 4 8bit ints.
 509 binop("umin_4x8", tint, commutative + associative, """
 510 dst = 0;
 511 for (int i = 0; i < 32; i += 8) {
 512    dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 513 }
 514 """)
 515
 516 # vector max for 4 8bit ints.
 517 binop("umax_4x8", tint, commutative + associative, """
 518 dst = 0;
 519 for (int i = 0; i < 32; i += 8) {
 520    dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 521 }
 522 """)
 523
 524 # unorm multiply: (a * b) / 255.
 525 binop("umul_unorm_4x8", tint, commutative + associative, """
 526 dst = 0;
 527 for (int i = 0; i < 32; i += 8) {
 528    int src0_chan = (src0 >> i) & 0xff;
 529    int src1_chan = (src1 >> i) & 0xff;
 530    dst |= ((src0_chan * src1_chan) / 255) << i;
 531 }
 532 """)
 533
 534 binop("fpow", tfloat, "", "powf(src0, src1)")
 535
 536 binop_horiz("pack_half_2x16_split", 1, tuint, 1, tfloat, 1, tfloat,
 537             "pack_half_1x16(src0.x) | (pack_half_1x16(src1.x) << 16)")
 538
 539 # bfm implements the behavior of the first operation of the SM5 "bfi" assembly
 540 # and that of the "bfi1" i965 instruction. That is, it has undefined behavior
 541 # if either of its arguments are 32.
 542 binop_convert("bfm", tuint, tint, "", """
 543 int bits = src0, offset = src1;
 544 if (offset < 0 || bits < 0 || offset > 31 || bits > 31 || offset + bits > 32)
 545    dst = 0; /* undefined */
 546 else
 547    dst = ((1u << bits) - 1) << offset;
 548 """)
 549
 550 opcode("ldexp", 0, tfloat, [0, 0], [tfloat, tint], "", """
 551 dst = ldexpf(src0, src1);
 552 /* flush denormals to zero. */
 553 if (!isnormal(dst))
 554    dst = copysignf(0.0f, src0);
 555 """)
 556
 557 # Combines the first component of each input to make a 2-component vector.
 558
 559 binop_horiz("vec2", 2, tuint, 1, tuint, 1, tuint, """
 560 dst.x = src0.x;
 561 dst.y = src1.x;
 562 """)
 563
 564 # Byte extraction
 565 binop("extract_ubyte", tuint, "", "(uint8_t)(src0 >> (src1 * 8))")
 566 binop("extract_ibyte", tint, "", "(int8_t)(src0 >> (src1 * 8))")
 567
 568 # Word extraction
 569 binop("extract_uword", tuint, "", "(uint16_t)(src0 >> (src1 * 16))")
 570 binop("extract_iword", tint, "", "(int16_t)(src0 >> (src1 * 16))")
 571
 572
 573 def triop(name, ty, const_expr):
 574    opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], "", const_expr)
 575 def triop_horiz(name, output_size, src1_size, src2_size, src3_size, const_expr):
 576    opcode(name, output_size, tuint,
 577    [src1_size, src2_size, src3_size],
 578    [tuint, tuint, tuint], "", const_expr)
 579
 580 triop("ffma", tfloat, "src0 * src1 + src2")
 581
 582 triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2")
 583
 584 # Conditional Select
 585 #
 586 # A vector conditional select instruction (like ?:, but operating per-
 587 # component on vectors). There are two versions, one for floating point
 588 # bools (0.0 vs 1.0) and one for integer bools (0 vs ~0).
 589
 590
 591 triop("fcsel", tfloat, "(src0 != 0.0f) ? src1 : src2")
 592 opcode("bcsel", 0, tuint, [0, 0, 0],
 593       [tbool, tuint, tuint], "", "src0 ? src1 : src2")
 594
 595 # SM5 bfi assembly
 596 triop("bfi", tuint, """
 597 unsigned mask = src0, insert = src1, base = src2;
 598 if (mask == 0) {
 599    dst = base;
 600 } else {
 601    unsigned tmp = mask;
 602    while (!(tmp & 1)) {
 603       tmp >>= 1;
 604       insert <<= 1;
 605    }
 606    dst = (base & ~mask) | (insert & mask);
 607 }
 608 """)
 609
 610 # SM5 ubfe/ibfe assembly
 611 opcode("ubfe", 0, tuint,
 612        [0, 0, 0], [tuint, tint, tint], "", """
 613 unsigned base = src0;
 614 int offset = src1, bits = src2;
 615 if (bits == 0) {
 616    dst = 0;
 617 } else if (bits < 0 || offset < 0) {
 618    dst = 0; /* undefined */
 619 } else if (offset + bits < 32) {
 620    dst = (base << (32 - bits - offset)) >> (32 - bits);
 621 } else {
 622    dst = base >> offset;
 623 }
 624 """)
 625 opcode("ibfe", 0, tint,
 626        [0, 0, 0], [tint, tint, tint], "", """
 627 int base = src0;
 628 int offset = src1, bits = src2;
 629 if (bits == 0) {
 630    dst = 0;
 631 } else if (bits < 0 || offset < 0) {
 632    dst = 0; /* undefined */
 633 } else if (offset + bits < 32) {
 634    dst = (base << (32 - bits - offset)) >> (32 - bits);
 635 } else {
 636    dst = base >> offset;
 637 }
 638 """)
 639
 640 # GLSL bitfieldExtract()
 641 opcode("ubitfield_extract", 0, tuint,
 642        [0, 0, 0], [tuint, tint, tint], "", """
 643 unsigned base = src0;
 644 int offset = src1, bits = src2;
 645 if (bits == 0) {
 646    dst = 0;
 647 } else if (bits < 0 || offset < 0 || offset + bits > 32) {
 648    dst = 0; /* undefined per the spec */
 649 } else {
 650    dst = (base >> offset) & ((1ull << bits) - 1);
 651 }
 652 """)
 653 opcode("ibitfield_extract", 0, tint,
 654        [0, 0, 0], [tint, tint, tint], "", """
 655 int base = src0;
 656 int offset = src1, bits = src2;
 657 if (bits == 0) {
 658    dst = 0;
 659 } else if (offset < 0 || bits < 0 || offset + bits > 32) {
 660    dst = 0;
 661 } else {
 662    dst = (base << (32 - offset - bits)) >> offset; /* use sign-extending shift */
 663 }
 664 """)
 665
 666 # Combines the first component of each input to make a 3-component vector.
 667
 668 triop_horiz("vec3", 3, 1, 1, 1, """
 669 dst.x = src0.x;
 670 dst.y = src1.x;
 671 dst.z = src2.x;
 672 """)
 673
 674 def quadop_horiz(name, output_size, src1_size, src2_size, src3_size,
 675                  src4_size, const_expr):
 676    opcode(name, output_size, tuint,
 677           [src1_size, src2_size, src3_size, src4_size],
 678           [tuint, tuint, tuint, tuint],
 679           "", const_expr)
 680
 681 opcode("bitfield_insert", 0, tuint, [0, 0, 0, 0],
 682        [tuint, tuint, tint, tint], "", """
 683 unsigned base = src0, insert = src1;
 684 int offset = src2, bits = src3;
 685 if (bits == 0) {
 686    dst = 0;
 687 } else if (offset < 0 || bits < 0 || bits + offset > 32) {
 688    dst = 0;
 689 } else {
 690    unsigned mask = ((1ull << bits) - 1) << offset;
 691    dst = (base & ~mask) | ((insert << bits) & mask);
 692 }
 693 """)
 694
 695 quadop_horiz("vec4", 4, 1, 1, 1, 1, """
 696 dst.x = src0.x;
 697 dst.y = src1.x;
 698 dst.z = src2.x;
 699 dst.w = src3.x;
 700 """)
 701
 702