src/compiler/nir/nir_opcodes.py

   1 #
   2 # Copyright (C) 2014 Connor Abbott
   3 #
   4 # Permission is hereby granted, free of charge, to any person obtaining a
   5 # copy of this software and associated documentation files (the "Software"),
   6 # to deal in the Software without restriction, including without limitation
   7 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 # and/or sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following conditions:
  10 #
  11 # The above copyright notice and this permission notice (including the next
  12 # paragraph) shall be included in all copies or substantial portions of the
  13 # Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22 #
  23 # Authors:
  24 #    Connor Abbott (cwabbott0@gmail.com)
  25
  26 import re
  27
  28 # Class that represents all the information we have about the opcode
  29 # NOTE: this must be kept in sync with nir_op_info
  30
  31 class Opcode(object):
  32    """Class that represents all the information we have about the opcode
  33    NOTE: this must be kept in sync with nir_op_info
  34    """
  35    def __init__(self, name, output_size, output_type, input_sizes,
  36                 input_types, algebraic_properties, const_expr):
  37       """Parameters:
  38
  39       - name is the name of the opcode (prepend nir_op_ for the enum name)
  40       - all types are strings that get nir_type_ prepended to them
  41       - input_types is a list of types
  42       - algebraic_properties is a space-seperated string, where nir_op_is_ is
  43         prepended before each entry
  44       - const_expr is an expression or series of statements that computes the
  45         constant value of the opcode given the constant values of its inputs.
  46
  47       Constant expressions are formed from the variables src0, src1, ...,
  48       src(N-1), where N is the number of arguments.  The output of the
  49       expression should be stored in the dst variable.  Per-component input
  50       and output variables will be scalars and non-per-component input and
  51       output variables will be a struct with fields named x, y, z, and w
  52       all of the correct type.  Input and output variables can be assumed
  53       to already be of the correct type and need no conversion.  In
  54       particular, the conversion from the C bool type to/from  NIR_TRUE and
  55       NIR_FALSE happens automatically.
  56
  57       For per-component instructions, the entire expression will be
  58       executed once for each component.  For non-per-component
  59       instructions, the expression is expected to store the correct values
  60       in dst.x, dst.y, etc.  If "dst" does not exist anywhere in the
  61       constant expression, an assignment to dst will happen automatically
  62       and the result will be equivalent to "dst = <expression>" for
  63       per-component instructions and "dst.x = dst.y = ... = <expression>"
  64       for non-per-component instructions.
  65       """
  66       assert isinstance(name, str)
  67       assert isinstance(output_size, int)
  68       assert isinstance(output_type, str)
  69       assert isinstance(input_sizes, list)
  70       assert isinstance(input_sizes[0], int)
  71       assert isinstance(input_types, list)
  72       assert isinstance(input_types[0], str)
  73       assert isinstance(algebraic_properties, str)
  74       assert isinstance(const_expr, str)
  75       assert len(input_sizes) == len(input_types)
  76       assert 0 <= output_size <= 4
  77       for size in input_sizes:
  78          assert 0 <= size <= 4
  79          if output_size != 0:
  80             assert size != 0
  81       self.name = name
  82       self.num_inputs = len(input_sizes)
  83       self.output_size = output_size
  84       self.output_type = output_type
  85       self.input_sizes = input_sizes
  86       self.input_types = input_types
  87       self.algebraic_properties = algebraic_properties
  88       self.const_expr = const_expr
  89
  90 # helper variables for strings
  91 tfloat = "float"
  92 tint = "int"
  93 tbool = "bool"
  94 tbool32 = "bool32"
  95 tuint = "uint"
  96 tuint16 = "uint16"
  97 tfloat32 = "float32"
  98 tint32 = "int32"
  99 tuint32 = "uint32"
 100 tint64 = "int64"
 101 tuint64 = "uint64"
 102 tfloat64 = "float64"
 103
 104 _TYPE_SPLIT_RE = re.compile(r'(?P<type>int|uint|float|bool)(?P<bits>\d+)?')
 105
 106 def type_has_size(type_):
 107     m = _TYPE_SPLIT_RE.match(type_)
 108     assert m is not None, 'Invalid NIR type string: "{}"'.format(type_)
 109     return m.group('bits') is not None
 110
 111 def type_size(type_):
 112     m = _TYPE_SPLIT_RE.match(type_)
 113     assert m is not None, 'Invalid NIR type string: "{}"'.format(type_)
 114     assert m.group('bits') is not None, \
 115            'NIR type string has no bit size: "{}"'.format(type_)
 116     return int(m.group('bits'))
 117
 118 def type_sizes(type_):
 119     if type_has_size(type_):
 120         return [type_size(type_)]
 121     elif type_ == 'bool':
 122         return [32]
 123     elif type_ == 'float':
 124         return [16, 32, 64]
 125     else:
 126         return [8, 16, 32, 64]
 127
 128 def type_base_type(type_):
 129     m = _TYPE_SPLIT_RE.match(type_)
 130     assert m is not None, 'Invalid NIR type string: "{}"'.format(type_)
 131     return m.group('type')
 132
 133 commutative = "commutative "
 134 associative = "associative "
 135
 136 # global dictionary of opcodes
 137 opcodes = {}
 138
 139 def opcode(name, output_size, output_type, input_sizes, input_types,
 140            algebraic_properties, const_expr):
 141    assert name not in opcodes
 142    opcodes[name] = Opcode(name, output_size, output_type, input_sizes,
 143                           input_types, algebraic_properties, const_expr)
 144
 145 def unop_convert(name, out_type, in_type, const_expr):
 146    opcode(name, 0, out_type, [0], [in_type], "", const_expr)
 147
 148 def unop(name, ty, const_expr):
 149    opcode(name, 0, ty, [0], [ty], "", const_expr)
 150
 151 def unop_horiz(name, output_size, output_type, input_size, input_type,
 152                const_expr):
 153    opcode(name, output_size, output_type, [input_size], [input_type], "",
 154           const_expr)
 155
 156 def unop_reduce(name, output_size, output_type, input_type, prereduce_expr,
 157                 reduce_expr, final_expr):
 158    def prereduce(src):
 159       return "(" + prereduce_expr.format(src=src) + ")"
 160    def final(src):
 161       return final_expr.format(src="(" + src + ")")
 162    def reduce_(src0, src1):
 163       return reduce_expr.format(src0=src0, src1=src1)
 164    src0 = prereduce("src0.x")
 165    src1 = prereduce("src0.y")
 166    src2 = prereduce("src0.z")
 167    src3 = prereduce("src0.w")
 168    unop_horiz(name + "2", output_size, output_type, 2, input_type,
 169               final(reduce_(src0, src1)))
 170    unop_horiz(name + "3", output_size, output_type, 3, input_type,
 171               final(reduce_(reduce_(src0, src1), src2)))
 172    unop_horiz(name + "4", output_size, output_type, 4, input_type,
 173               final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 174
 175
 176 # These two move instructions differ in what modifiers they support and what
 177 # the negate modifier means. Otherwise, they are identical.
 178 unop("fmov", tfloat, "src0")
 179 unop("imov", tint, "src0")
 180
 181 unop("ineg", tint, "-src0")
 182 unop("fneg", tfloat, "-src0")
 183 unop("inot", tint, "~src0") # invert every bit of the integer
 184 unop("fnot", tfloat, ("bit_size == 64 ? ((src0 == 0.0) ? 1.0 : 0.0f) : " +
 185                       "((src0 == 0.0f) ? 1.0f : 0.0f)"))
 186 unop("fsign", tfloat, ("bit_size == 64 ? " +
 187                        "((src0 == 0.0) ? 0.0 : ((src0 > 0.0) ? 1.0 : -1.0)) : " +
 188                        "((src0 == 0.0f) ? 0.0f : ((src0 > 0.0f) ? 1.0f : -1.0f))"))
 189 unop("isign", tint, "(src0 == 0) ? 0 : ((src0 > 0) ? 1 : -1)")
 190 unop("iabs", tint, "(src0 < 0) ? -src0 : src0")
 191 unop("fabs", tfloat, "fabs(src0)")
 192 unop("fsat", tfloat, ("bit_size == 64 ? " +
 193                       "((src0 > 1.0) ? 1.0 : ((src0 <= 0.0) ? 0.0 : src0)) : " +
 194                       "((src0 > 1.0f) ? 1.0f : ((src0 <= 0.0f) ? 0.0f : src0))"))
 195 unop("frcp", tfloat, "bit_size == 64 ? 1.0 / src0 : 1.0f / src0")
 196 unop("frsq", tfloat, "bit_size == 64 ? 1.0 / sqrt(src0) : 1.0f / sqrtf(src0)")
 197 unop("fsqrt", tfloat, "bit_size == 64 ? sqrt(src0) : sqrtf(src0)")
 198 unop("fexp2", tfloat, "exp2f(src0)")
 199 unop("flog2", tfloat, "log2f(src0)")
 200
 201 # Generate all of the numeric conversion opcodes
 202 for src_t in [tint, tuint, tfloat, tbool]:
 203    if src_t == tbool:
 204       dst_types = [tfloat, tint]
 205    elif src_t == tint:
 206       dst_types = [tfloat, tint, tbool]
 207    elif src_t == tuint:
 208       dst_types = [tfloat, tuint]
 209    elif src_t == tfloat:
 210       dst_types = [tint, tuint, tfloat, tbool]
 211
 212    for dst_t in dst_types:
 213       for bit_size in type_sizes(dst_t):
 214           if bit_size == 16 and dst_t == tfloat and src_t == tfloat:
 215               rnd_modes = ['_rtne', '_rtz', '']
 216               for rnd_mode in rnd_modes:
 217                   unop_convert("{0}2{1}{2}{3}".format(src_t[0], dst_t[0],
 218                                                        bit_size, rnd_mode),
 219                                dst_t + str(bit_size), src_t, "src0")
 220           else:
 221               conv_expr = "src0 != 0" if dst_t == tbool else "src0"
 222               unop_convert("{0}2{1}{2}".format(src_t[0], dst_t[0], bit_size),
 223                            dst_t + str(bit_size), src_t, conv_expr)
 224
 225
 226 # Unary floating-point rounding operations.
 227
 228
 229 unop("ftrunc", tfloat, "bit_size == 64 ? trunc(src0) : truncf(src0)")
 230 unop("fceil", tfloat, "bit_size == 64 ? ceil(src0) : ceilf(src0)")
 231 unop("ffloor", tfloat, "bit_size == 64 ? floor(src0) : floorf(src0)")
 232 unop("ffract", tfloat, "src0 - (bit_size == 64 ? floor(src0) : floorf(src0))")
 233 unop("fround_even", tfloat, "bit_size == 64 ? _mesa_roundeven(src0) : _mesa_roundevenf(src0)")
 234
 235 unop("fquantize2f16", tfloat, "(fabs(src0) < ldexpf(1.0, -14)) ? copysignf(0.0f, src0) : _mesa_half_to_float(_mesa_float_to_half(src0))")
 236
 237 # Trigonometric operations.
 238
 239
 240 unop("fsin", tfloat, "bit_size == 64 ? sin(src0) : sinf(src0)")
 241 unop("fcos", tfloat, "bit_size == 64 ? cos(src0) : cosf(src0)")
 242
 243 # dfrexp
 244 unop_convert("frexp_exp", tint32, tfloat64, "frexp(src0, &dst);")
 245 unop_convert("frexp_sig", tfloat64, tfloat64, "int n; dst = frexp(src0, &n);")
 246
 247 # Partial derivatives.
 248
 249
 250 unop("fddx", tfloat, "0.0") # the derivative of a constant is 0.
 251 unop("fddy", tfloat, "0.0")
 252 unop("fddx_fine", tfloat, "0.0")
 253 unop("fddy_fine", tfloat, "0.0")
 254 unop("fddx_coarse", tfloat, "0.0")
 255 unop("fddy_coarse", tfloat, "0.0")
 256
 257
 258 # Floating point pack and unpack operations.
 259
 260 def pack_2x16(fmt):
 261    unop_horiz("pack_" + fmt + "_2x16", 1, tuint32, 2, tfloat32, """
 262 dst.x = (uint32_t) pack_fmt_1x16(src0.x);
 263 dst.x |= ((uint32_t) pack_fmt_1x16(src0.y)) << 16;
 264 """.replace("fmt", fmt))
 265
 266 def pack_4x8(fmt):
 267    unop_horiz("pack_" + fmt + "_4x8", 1, tuint32, 4, tfloat32, """
 268 dst.x = (uint32_t) pack_fmt_1x8(src0.x);
 269 dst.x |= ((uint32_t) pack_fmt_1x8(src0.y)) << 8;
 270 dst.x |= ((uint32_t) pack_fmt_1x8(src0.z)) << 16;
 271 dst.x |= ((uint32_t) pack_fmt_1x8(src0.w)) << 24;
 272 """.replace("fmt", fmt))
 273
 274 def unpack_2x16(fmt):
 275    unop_horiz("unpack_" + fmt + "_2x16", 2, tfloat32, 1, tuint32, """
 276 dst.x = unpack_fmt_1x16((uint16_t)(src0.x & 0xffff));
 277 dst.y = unpack_fmt_1x16((uint16_t)(src0.x << 16));
 278 """.replace("fmt", fmt))
 279
 280 def unpack_4x8(fmt):
 281    unop_horiz("unpack_" + fmt + "_4x8", 4, tfloat32, 1, tuint32, """
 282 dst.x = unpack_fmt_1x8((uint8_t)(src0.x & 0xff));
 283 dst.y = unpack_fmt_1x8((uint8_t)((src0.x >> 8) & 0xff));
 284 dst.z = unpack_fmt_1x8((uint8_t)((src0.x >> 16) & 0xff));
 285 dst.w = unpack_fmt_1x8((uint8_t)(src0.x >> 24));
 286 """.replace("fmt", fmt))
 287
 288
 289 pack_2x16("snorm")
 290 pack_4x8("snorm")
 291 pack_2x16("unorm")
 292 pack_4x8("unorm")
 293 pack_2x16("half")
 294 unpack_2x16("snorm")
 295 unpack_4x8("snorm")
 296 unpack_2x16("unorm")
 297 unpack_4x8("unorm")
 298 unpack_2x16("half")
 299
 300 unop_horiz("pack_uvec2_to_uint", 1, tuint32, 2, tuint32, """
 301 dst.x = (src0.x & 0xffff) | (src0.y << 16);
 302 """)
 303
 304 unop_horiz("pack_uvec4_to_uint", 1, tuint32, 4, tuint32, """
 305 dst.x = (src0.x <<  0) |
 306         (src0.y <<  8) |
 307         (src0.z << 16) |
 308         (src0.w << 24);
 309 """)
 310
 311 unop_horiz("pack_32_2x16", 1, tuint32, 2, tuint16,
 312            "dst.x = src0.x | ((uint32_t)src0.y << 16);")
 313
 314 unop_horiz("pack_64_2x32", 1, tuint64, 2, tuint32,
 315            "dst.x = src0.x | ((uint64_t)src0.y << 32);")
 316
 317 unop_horiz("pack_64_4x16", 1, tuint64, 4, tuint16,
 318            "dst.x = src0.x | ((uint64_t)src0.y << 16) | ((uint64_t)src0.z << 32) | ((uint64_t)src0.w << 48);")
 319
 320 unop_horiz("unpack_64_2x32", 2, tuint32, 1, tuint64,
 321            "dst.x = src0.x; dst.y = src0.x >> 32;")
 322
 323 unop_horiz("unpack_64_4x16", 4, tuint16, 1, tuint64,
 324            "dst.x = src0.x; dst.y = src0.x >> 16; dst.z = src0.x >> 32; dst.w = src0.w >> 48;")
 325
 326 unop_horiz("unpack_32_2x16", 2, tuint16, 1, tuint32,
 327            "dst.x = src0.x; dst.y = src0.x >> 16;")
 328
 329 # Lowered floating point unpacking operations.
 330
 331
 332 unop_convert("unpack_half_2x16_split_x", tfloat32, tuint32,
 333              "unpack_half_1x16((uint16_t)(src0 & 0xffff))")
 334 unop_convert("unpack_half_2x16_split_y", tfloat32, tuint32,
 335              "unpack_half_1x16((uint16_t)(src0 >> 16))")
 336
 337 unop_convert("unpack_32_2x16_split_x", tuint16, tuint32, "src0")
 338 unop_convert("unpack_32_2x16_split_y", tuint16, tuint32, "src0 >> 16")
 339
 340 unop_convert("unpack_64_2x32_split_x", tuint32, tuint64, "src0")
 341 unop_convert("unpack_64_2x32_split_y", tuint32, tuint64, "src0 >> 32")
 342
 343 # Bit operations, part of ARB_gpu_shader5.
 344
 345
 346 unop("bitfield_reverse", tuint32, """
 347 /* we're not winning any awards for speed here, but that's ok */
 348 dst = 0;
 349 for (unsigned bit = 0; bit < 32; bit++)
 350    dst |= ((src0 >> bit) & 1) << (31 - bit);
 351 """)
 352 unop_convert("bit_count", tuint32, tuint, """
 353 dst = 0;
 354 for (unsigned bit = 0; bit < bit_size; bit++) {
 355    if ((src0 >> bit) & 1)
 356       dst++;
 357 }
 358 """)
 359
 360 unop_convert("ufind_msb", tint32, tuint, """
 361 dst = -1;
 362 for (int bit = bit_size - 1; bit >= 0; bit--) {
 363    if ((src0 >> bit) & 1) {
 364       dst = bit;
 365       break;
 366    }
 367 }
 368 """)
 369
 370 unop("ifind_msb", tint32, """
 371 dst = -1;
 372 for (int bit = 31; bit >= 0; bit--) {
 373    /* If src0 < 0, we're looking for the first 0 bit.
 374     * if src0 >= 0, we're looking for the first 1 bit.
 375     */
 376    if ((((src0 >> bit) & 1) && (src0 >= 0)) ||
 377       (!((src0 >> bit) & 1) && (src0 < 0))) {
 378       dst = bit;
 379       break;
 380    }
 381 }
 382 """)
 383
 384 unop_convert("find_lsb", tint32, tint, """
 385 dst = -1;
 386 for (unsigned bit = 0; bit < bit_size; bit++) {
 387    if ((src0 >> bit) & 1) {
 388       dst = bit;
 389       break;
 390    }
 391 }
 392 """)
 393
 394
 395 for i in range(1, 5):
 396    for j in range(1, 5):
 397       unop_horiz("fnoise{0}_{1}".format(i, j), i, tfloat, j, tfloat, "0.0f")
 398
 399
 400 # AMD_gcn_shader extended instructions
 401 unop_horiz("cube_face_coord", 2, tfloat32, 3, tfloat32, """
 402 dst.x = dst.y = 0.0;
 403 float absX = fabs(src0.x);
 404 float absY = fabs(src0.y);
 405 float absZ = fabs(src0.z);
 406 if (src0.x >= 0 && absX >= absY && absX >= absZ) { dst.x = -src0.y; dst.y = -src0.z; }
 407 if (src0.x < 0 && absX >= absY && absX >= absZ) { dst.x = -src0.y; dst.y = src0.z; }
 408 if (src0.y >= 0 && absY >= absX && absY >= absZ) { dst.x = src0.z; dst.y = src0.x; }
 409 if (src0.y < 0 && absY >= absX && absY >= absZ) { dst.x = -src0.z; dst.y = src0.x; }
 410 if (src0.z >= 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.y; dst.y = src0.x; }
 411 if (src0.z < 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.y; dst.y = -src0.x; }
 412 """)
 413
 414 unop_horiz("cube_face_index", 1, tfloat32, 3, tfloat32, """
 415 float absX = fabs(src0.x);
 416 float absY = fabs(src0.y);
 417 float absZ = fabs(src0.z);
 418 if (src0.x >= 0 && absX >= absY && absX >= absZ) dst.x = 0;
 419 if (src0.x < 0 && absX >= absY && absX >= absZ) dst.x = 1;
 420 if (src0.y >= 0 && absY >= absX && absY >= absZ) dst.x = 2;
 421 if (src0.y < 0 && absY >= absX && absY >= absZ) dst.x = 3;
 422 if (src0.z >= 0 && absZ >= absX && absZ >= absY) dst.x = 4;
 423 if (src0.z < 0 && absZ >= absX && absZ >= absY) dst.x = 5;
 424 """)
 425
 426
 427 def binop_convert(name, out_type, in_type, alg_props, const_expr):
 428    opcode(name, 0, out_type, [0, 0], [in_type, in_type], alg_props, const_expr)
 429
 430 def binop(name, ty, alg_props, const_expr):
 431    binop_convert(name, ty, ty, alg_props, const_expr)
 432
 433 def binop_compare(name, ty, alg_props, const_expr):
 434    binop_convert(name, tbool32, ty, alg_props, const_expr)
 435
 436 def binop_horiz(name, out_size, out_type, src1_size, src1_type, src2_size,
 437                 src2_type, const_expr):
 438    opcode(name, out_size, out_type, [src1_size, src2_size], [src1_type, src2_type],
 439           "", const_expr)
 440
 441 def binop_reduce(name, output_size, output_type, src_type, prereduce_expr,
 442                  reduce_expr, final_expr):
 443    def final(src):
 444       return final_expr.format(src= "(" + src + ")")
 445    def reduce_(src0, src1):
 446       return reduce_expr.format(src0=src0, src1=src1)
 447    def prereduce(src0, src1):
 448       return "(" + prereduce_expr.format(src0=src0, src1=src1) + ")"
 449    src0 = prereduce("src0.x", "src1.x")
 450    src1 = prereduce("src0.y", "src1.y")
 451    src2 = prereduce("src0.z", "src1.z")
 452    src3 = prereduce("src0.w", "src1.w")
 453    opcode(name + "2", output_size, output_type,
 454           [2, 2], [src_type, src_type], commutative,
 455           final(reduce_(src0, src1)))
 456    opcode(name + "3", output_size, output_type,
 457           [3, 3], [src_type, src_type], commutative,
 458           final(reduce_(reduce_(src0, src1), src2)))
 459    opcode(name + "4", output_size, output_type,
 460           [4, 4], [src_type, src_type], commutative,
 461           final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
 462
 463 binop("fadd", tfloat, commutative + associative, "src0 + src1")
 464 binop("iadd", tint, commutative + associative, "src0 + src1")
 465 binop("fsub", tfloat, "", "src0 - src1")
 466 binop("isub", tint, "", "src0 - src1")
 467
 468 binop("fmul", tfloat, commutative + associative, "src0 * src1")
 469 # low 32-bits of signed/unsigned integer multiply
 470 binop("imul", tint, commutative + associative, "src0 * src1")
 471 # high 32-bits of signed integer multiply
 472 binop("imul_high", tint32, commutative,
 473       "(int32_t)(((int64_t) src0 * (int64_t) src1) >> 32)")
 474 # high 32-bits of unsigned integer multiply
 475 binop("umul_high", tuint32, commutative,
 476       "(uint32_t)(((uint64_t) src0 * (uint64_t) src1) >> 32)")
 477
 478 binop("fdiv", tfloat, "", "src0 / src1")
 479 binop("idiv", tint, "", "src1 == 0 ? 0 : (src0 / src1)")
 480 binop("udiv", tuint, "", "src1 == 0 ? 0 : (src0 / src1)")
 481
 482 # returns a boolean representing the carry resulting from the addition of
 483 # the two unsigned arguments.
 484
 485 binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0")
 486
 487 # returns a boolean representing the borrow resulting from the subtraction
 488 # of the two unsigned arguments.
 489
 490 binop_convert("usub_borrow", tuint, tuint, "", "src0 < src1")
 491
 492 binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1")
 493
 494 # For signed integers, there are several different possible definitions of
 495 # "modulus" or "remainder".  We follow the conventions used by LLVM and
 496 # SPIR-V.  The irem opcode implements the standard C/C++ signed "%"
 497 # operation while the imod opcode implements the more mathematical
 498 # "modulus" operation.  For details on the difference, see
 499 #
 500 # http://mathforum.org/library/drmath/view/52343.html
 501
 502 binop("irem", tint, "", "src1 == 0 ? 0 : src0 % src1")
 503 binop("imod", tint, "",
 504       "src1 == 0 ? 0 : ((src0 % src1 == 0 || (src0 >= 0) == (src1 >= 0)) ?"
 505       "                 src0 % src1 : src0 % src1 + src1)")
 506 binop("fmod", tfloat, "", "src0 - src1 * floorf(src0 / src1)")
 507 binop("frem", tfloat, "", "src0 - src1 * truncf(src0 / src1)")
 508
 509 #
 510 # Comparisons
 511 #
 512
 513
 514 # these integer-aware comparisons return a boolean (0 or ~0)
 515
 516 binop_compare("flt", tfloat, "", "src0 < src1")
 517 binop_compare("fge", tfloat, "", "src0 >= src1")
 518 binop_compare("feq", tfloat, commutative, "src0 == src1")
 519 binop_compare("fne", tfloat, commutative, "src0 != src1")
 520 binop_compare("ilt", tint, "", "src0 < src1")
 521 binop_compare("ige", tint, "", "src0 >= src1")
 522 binop_compare("ieq", tint, commutative, "src0 == src1")
 523 binop_compare("ine", tint, commutative, "src0 != src1")
 524 binop_compare("ult", tuint, "", "src0 < src1")
 525 binop_compare("uge", tuint, "", "src0 >= src1")
 526
 527 # integer-aware GLSL-style comparisons that compare floats and ints
 528
 529 binop_reduce("ball_fequal",  1, tbool32, tfloat, "{src0} == {src1}",
 530              "{src0} && {src1}", "{src}")
 531 binop_reduce("bany_fnequal", 1, tbool32, tfloat, "{src0} != {src1}",
 532              "{src0} || {src1}", "{src}")
 533 binop_reduce("ball_iequal",  1, tbool32, tint, "{src0} == {src1}",
 534              "{src0} && {src1}", "{src}")
 535 binop_reduce("bany_inequal", 1, tbool32, tint, "{src0} != {src1}",
 536              "{src0} || {src1}", "{src}")
 537
 538 # non-integer-aware GLSL-style comparisons that return 0.0 or 1.0
 539
 540 binop_reduce("fall_equal",  1, tfloat32, tfloat32, "{src0} == {src1}",
 541              "{src0} && {src1}", "{src} ? 1.0f : 0.0f")
 542 binop_reduce("fany_nequal", 1, tfloat32, tfloat32, "{src0} != {src1}",
 543              "{src0} || {src1}", "{src} ? 1.0f : 0.0f")
 544
 545 # These comparisons for integer-less hardware return 1.0 and 0.0 for true
 546 # and false respectively
 547
 548 binop("slt", tfloat32, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than
 549 binop("sge", tfloat, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal
 550 binop("seq", tfloat32, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
 551 binop("sne", tfloat32, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
 552
 553
 554 opcode("ishl", 0, tint, [0, 0], [tint, tuint32], "", "src0 << src1")
 555 opcode("ishr", 0, tint, [0, 0], [tint, tuint32], "", "src0 >> src1")
 556 opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], "", "src0 >> src1")
 557
 558 # bitwise logic operators
 559 #
 560 # These are also used as boolean and, or, xor for hardware supporting
 561 # integers.
 562
 563
 564 binop("iand", tuint, commutative + associative, "src0 & src1")
 565 binop("ior", tuint, commutative + associative, "src0 | src1")
 566 binop("ixor", tuint, commutative + associative, "src0 ^ src1")
 567
 568
 569 # floating point logic operators
 570 #
 571 # These use (src != 0.0) for testing the truth of the input, and output 1.0
 572 # for true and 0.0 for false
 573
 574 binop("fand", tfloat32, commutative,
 575       "((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f")
 576 binop("for", tfloat32, commutative,
 577       "((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f")
 578 binop("fxor", tfloat32, commutative,
 579       "(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f")
 580
 581 binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}",
 582              "{src}")
 583
 584 binop_reduce("fdot_replicated", 4, tfloat, tfloat,
 585              "{src0} * {src1}", "{src0} + {src1}", "{src}")
 586
 587 opcode("fdph", 1, tfloat, [3, 4], [tfloat, tfloat], "",
 588        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 589 opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], "",
 590        "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
 591
 592 binop("fmin", tfloat, "", "fminf(src0, src1)")
 593 binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1")
 594 binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1")
 595 binop("fmax", tfloat, "", "fmaxf(src0, src1)")
 596 binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0")
 597 binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0")
 598
 599 # Saturated vector add for 4 8bit ints.
 600 binop("usadd_4x8", tint32, commutative + associative, """
 601 dst = 0;
 602 for (int i = 0; i < 32; i += 8) {
 603    dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i;
 604 }
 605 """)
 606
 607 # Saturated vector subtract for 4 8bit ints.
 608 binop("ussub_4x8", tint32, "", """
 609 dst = 0;
 610 for (int i = 0; i < 32; i += 8) {
 611    int src0_chan = (src0 >> i) & 0xff;
 612    int src1_chan = (src1 >> i) & 0xff;
 613    if (src0_chan > src1_chan)
 614       dst |= (src0_chan - src1_chan) << i;
 615 }
 616 """)
 617
 618 # vector min for 4 8bit ints.
 619 binop("umin_4x8", tint32, commutative + associative, """
 620 dst = 0;
 621 for (int i = 0; i < 32; i += 8) {
 622    dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 623 }
 624 """)
 625
 626 # vector max for 4 8bit ints.
 627 binop("umax_4x8", tint32, commutative + associative, """
 628 dst = 0;
 629 for (int i = 0; i < 32; i += 8) {
 630    dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
 631 }
 632 """)
 633
 634 # unorm multiply: (a * b) / 255.
 635 binop("umul_unorm_4x8", tint32, commutative + associative, """
 636 dst = 0;
 637 for (int i = 0; i < 32; i += 8) {
 638    int src0_chan = (src0 >> i) & 0xff;
 639    int src1_chan = (src1 >> i) & 0xff;
 640    dst |= ((src0_chan * src1_chan) / 255) << i;
 641 }
 642 """)
 643
 644 binop("fpow", tfloat, "", "bit_size == 64 ? powf(src0, src1) : pow(src0, src1)")
 645
 646 binop_horiz("pack_half_2x16_split", 1, tuint32, 1, tfloat32, 1, tfloat32,
 647             "pack_half_1x16(src0.x) | (pack_half_1x16(src1.x) << 16)")
 648
 649 binop_convert("pack_64_2x32_split", tuint64, tuint32, "",
 650               "src0 | ((uint64_t)src1 << 32)")
 651
 652 binop_convert("pack_32_2x16_split", tuint32, tuint16, "",
 653               "src0 | ((uint32_t)src1 << 16)")
 654
 655 # bfm implements the behavior of the first operation of the SM5 "bfi" assembly
 656 # and that of the "bfi1" i965 instruction. That is, it has undefined behavior
 657 # if either of its arguments are 32.
 658 binop_convert("bfm", tuint32, tint32, "", """
 659 int bits = src0, offset = src1;
 660 if (offset < 0 || bits < 0 || offset > 31 || bits > 31 || offset + bits > 32)
 661    dst = 0; /* undefined */
 662 else
 663    dst = ((1u << bits) - 1) << offset;
 664 """)
 665
 666 opcode("ldexp", 0, tfloat, [0, 0], [tfloat, tint32], "", """
 667 dst = (bit_size == 64) ? ldexp(src0, src1) : ldexpf(src0, src1);
 668 /* flush denormals to zero. */
 669 if (!isnormal(dst))
 670    dst = copysignf(0.0f, src0);
 671 """)
 672
 673 # Combines the first component of each input to make a 2-component vector.
 674
 675 binop_horiz("vec2", 2, tuint, 1, tuint, 1, tuint, """
 676 dst.x = src0.x;
 677 dst.y = src1.x;
 678 """)
 679
 680 # Byte extraction
 681 binop("extract_u8", tuint, "", "(uint8_t)(src0 >> (src1 * 8))")
 682 binop("extract_i8", tint, "", "(int8_t)(src0 >> (src1 * 8))")
 683
 684 # Word extraction
 685 binop("extract_u16", tuint, "", "(uint16_t)(src0 >> (src1 * 16))")
 686 binop("extract_i16", tint, "", "(int16_t)(src0 >> (src1 * 16))")
 687
 688
 689 def triop(name, ty, const_expr):
 690    opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], "", const_expr)
 691 def triop_horiz(name, output_size, src1_size, src2_size, src3_size, const_expr):
 692    opcode(name, output_size, tuint,
 693    [src1_size, src2_size, src3_size],
 694    [tuint, tuint, tuint], "", const_expr)
 695
 696 triop("ffma", tfloat, "src0 * src1 + src2")
 697
 698 triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2")
 699
 700 # Conditional Select
 701 #
 702 # A vector conditional select instruction (like ?:, but operating per-
 703 # component on vectors). There are two versions, one for floating point
 704 # bools (0.0 vs 1.0) and one for integer bools (0 vs ~0).
 705
 706
 707 triop("fcsel", tfloat32, "(src0 != 0.0f) ? src1 : src2")
 708
 709 # 3 way min/max/med
 710 triop("fmin3", tfloat, "fminf(src0, fminf(src1, src2))")
 711 triop("imin3", tint, "MIN2(src0, MIN2(src1, src2))")
 712 triop("umin3", tuint, "MIN2(src0, MIN2(src1, src2))")
 713
 714 triop("fmax3", tfloat, "fmaxf(src0, fmaxf(src1, src2))")
 715 triop("imax3", tint, "MAX2(src0, MAX2(src1, src2))")
 716 triop("umax3", tuint, "MAX2(src0, MAX2(src1, src2))")
 717
 718 triop("fmed3", tfloat, "fmaxf(fminf(fmaxf(src0, src1), src2), fminf(src0, src1))")
 719 triop("imed3", tint, "MAX2(MIN2(MAX2(src0, src1), src2), MIN2(src0, src1))")
 720 triop("umed3", tuint, "MAX2(MIN2(MAX2(src0, src1), src2), MIN2(src0, src1))")
 721
 722 opcode("bcsel", 0, tuint, [0, 0, 0],
 723       [tbool32, tuint, tuint], "", "src0 ? src1 : src2")
 724
 725 # SM5 bfi assembly
 726 triop("bfi", tuint32, """
 727 unsigned mask = src0, insert = src1, base = src2;
 728 if (mask == 0) {
 729    dst = base;
 730 } else {
 731    unsigned tmp = mask;
 732    while (!(tmp & 1)) {
 733       tmp >>= 1;
 734       insert <<= 1;
 735    }
 736    dst = (base & ~mask) | (insert & mask);
 737 }
 738 """)
 739
 740 # SM5 ubfe/ibfe assembly
 741 opcode("ubfe", 0, tuint32,
 742        [0, 0, 0], [tuint32, tint32, tint32], "", """
 743 unsigned base = src0;
 744 int offset = src1, bits = src2;
 745 if (bits == 0) {
 746    dst = 0;
 747 } else if (bits < 0 || offset < 0) {
 748    dst = 0; /* undefined */
 749 } else if (offset + bits < 32) {
 750    dst = (base << (32 - bits - offset)) >> (32 - bits);
 751 } else {
 752    dst = base >> offset;
 753 }
 754 """)
 755 opcode("ibfe", 0, tint32,
 756        [0, 0, 0], [tint32, tint32, tint32], "", """
 757 int base = src0;
 758 int offset = src1, bits = src2;
 759 if (bits == 0) {
 760    dst = 0;
 761 } else if (bits < 0 || offset < 0) {
 762    dst = 0; /* undefined */
 763 } else if (offset + bits < 32) {
 764    dst = (base << (32 - bits - offset)) >> (32 - bits);
 765 } else {
 766    dst = base >> offset;
 767 }
 768 """)
 769
 770 # GLSL bitfieldExtract()
 771 opcode("ubitfield_extract", 0, tuint32,
 772        [0, 0, 0], [tuint32, tint32, tint32], "", """
 773 unsigned base = src0;
 774 int offset = src1, bits = src2;
 775 if (bits == 0) {
 776    dst = 0;
 777 } else if (bits < 0 || offset < 0 || offset + bits > 32) {
 778    dst = 0; /* undefined per the spec */
 779 } else {
 780    dst = (base >> offset) & ((1ull << bits) - 1);
 781 }
 782 """)
 783 opcode("ibitfield_extract", 0, tint32,
 784        [0, 0, 0], [tint32, tint32, tint32], "", """
 785 int base = src0;
 786 int offset = src1, bits = src2;
 787 if (bits == 0) {
 788    dst = 0;
 789 } else if (offset < 0 || bits < 0 || offset + bits > 32) {
 790    dst = 0;
 791 } else {
 792    dst = (base << (32 - offset - bits)) >> offset; /* use sign-extending shift */
 793 }
 794 """)
 795
 796 # Combines the first component of each input to make a 3-component vector.
 797
 798 triop_horiz("vec3", 3, 1, 1, 1, """
 799 dst.x = src0.x;
 800 dst.y = src1.x;
 801 dst.z = src2.x;
 802 """)
 803
 804 def quadop_horiz(name, output_size, src1_size, src2_size, src3_size,
 805                  src4_size, const_expr):
 806    opcode(name, output_size, tuint,
 807           [src1_size, src2_size, src3_size, src4_size],
 808           [tuint, tuint, tuint, tuint],
 809           "", const_expr)
 810
 811 opcode("bitfield_insert", 0, tuint32, [0, 0, 0, 0],
 812        [tuint32, tuint32, tint32, tint32], "", """
 813 unsigned base = src0, insert = src1;
 814 int offset = src2, bits = src3;
 815 if (bits == 0) {
 816    dst = base;
 817 } else if (offset < 0 || bits < 0 || bits + offset > 32) {
 818    dst = 0;
 819 } else {
 820    unsigned mask = ((1ull << bits) - 1) << offset;
 821    dst = (base & ~mask) | ((insert << offset) & mask);
 822 }
 823 """)
 824
 825 quadop_horiz("vec4", 4, 1, 1, 1, 1, """
 826 dst.x = src0.x;
 827 dst.y = src1.x;
 828 dst.z = src2.x;
 829 dst.w = src3.x;
 830 """)
 831
 832