X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fnir%2Fnir_opcodes.py;h=e19d7b00a7d3e956a57334ad32b674dfbc9f28ad;hb=e5d4bbd840622d43c25cf125aad2440abda3ccdc;hp=ece673cda36d00c4a21e8ca5b000b77123e852ee;hpb=824e1bb078ad0caeb0b6a7292aebddd06ed84291;p=mesa.git diff --git a/src/compiler/nir/nir_opcodes.py b/src/compiler/nir/nir_opcodes.py index ece673cda36..e19d7b00a7d 100644 --- a/src/compiler/nir/nir_opcodes.py +++ b/src/compiler/nir/nir_opcodes.py @@ -1,4 +1,3 @@ -#! /usr/bin/env python # # Copyright (C) 2014 Connor Abbott # @@ -24,6 +23,7 @@ # Authors: # Connor Abbott (cwabbott0@gmail.com) +import re # Class that represents all the information we have about the opcode # NOTE: this must be kept in sync with nir_op_info @@ -33,12 +33,13 @@ class Opcode(object): NOTE: this must be kept in sync with nir_op_info """ def __init__(self, name, output_size, output_type, input_sizes, - input_types, algebraic_properties, const_expr): + input_types, is_conversion, algebraic_properties, const_expr): """Parameters: - name is the name of the opcode (prepend nir_op_ for the enum name) - all types are strings that get nir_type_ prepended to them - input_types is a list of types + - is_conversion is true if this opcode represents a type conversion - algebraic_properties is a space-seperated string, where nir_op_is_ is prepended before each entry - const_expr is an expression or series of statements that computes the @@ -70,12 +71,13 @@ class Opcode(object): assert isinstance(input_sizes[0], int) assert isinstance(input_types, list) assert isinstance(input_types[0], str) + assert isinstance(is_conversion, bool) assert isinstance(algebraic_properties, str) assert isinstance(const_expr, str) assert len(input_sizes) == len(input_types) - assert 0 <= output_size <= 4 + assert 0 <= output_size <= 4 or (output_size == 8) or (output_size == 16) for size in input_sizes: - assert 0 <= size <= 4 + assert 0 <= size <= 4 or (size == 8) or (size == 16) if output_size != 0: assert size != 0 self.name = name @@ -84,14 +86,23 @@ class Opcode(object): self.output_type = output_type self.input_sizes = input_sizes self.input_types = input_types + self.is_conversion = is_conversion self.algebraic_properties = algebraic_properties self.const_expr = const_expr # helper variables for strings tfloat = "float" tint = "int" -tbool = "bool32" +tbool = "bool" +tbool1 = "bool1" +tbool8 = "bool8" +tbool16 = "bool16" +tbool32 = "bool32" tuint = "uint" +tuint8 = "uint8" +tint16 = "int16" +tuint16 = "uint16" +tfloat16 = "float16" tfloat32 = "float32" tint32 = "int32" tuint32 = "uint32" @@ -99,28 +110,63 @@ tint64 = "int64" tuint64 = "uint64" tfloat64 = "float64" -commutative = "commutative " +_TYPE_SPLIT_RE = re.compile(r'(?Pint|uint|float|bool)(?P\d+)?') + +def type_has_size(type_): + m = _TYPE_SPLIT_RE.match(type_) + assert m is not None, 'Invalid NIR type string: "{}"'.format(type_) + return m.group('bits') is not None + +def type_size(type_): + m = _TYPE_SPLIT_RE.match(type_) + assert m is not None, 'Invalid NIR type string: "{}"'.format(type_) + assert m.group('bits') is not None, \ + 'NIR type string has no bit size: "{}"'.format(type_) + return int(m.group('bits')) + +def type_sizes(type_): + if type_has_size(type_): + return [type_size(type_)] + elif type_ == 'bool': + return [1, 8, 16, 32] + elif type_ == 'float': + return [16, 32, 64] + else: + return [1, 8, 16, 32, 64] + +def type_base_type(type_): + m = _TYPE_SPLIT_RE.match(type_) + assert m is not None, 'Invalid NIR type string: "{}"'.format(type_) + return m.group('type') + +# Operation where the first two sources are commutative. +# +# For 2-source operations, this just mathematical commutativity. Some +# 3-source operations, like ffma, are only commutative in the first two +# sources. +_2src_commutative = "2src_commutative " associative = "associative " # global dictionary of opcodes opcodes = {} def opcode(name, output_size, output_type, input_sizes, input_types, - algebraic_properties, const_expr): + is_conversion, algebraic_properties, const_expr): assert name not in opcodes opcodes[name] = Opcode(name, output_size, output_type, input_sizes, - input_types, algebraic_properties, const_expr) + input_types, is_conversion, algebraic_properties, + const_expr) def unop_convert(name, out_type, in_type, const_expr): - opcode(name, 0, out_type, [0], [in_type], "", const_expr) + opcode(name, 0, out_type, [0], [in_type], False, "", const_expr) def unop(name, ty, const_expr): - opcode(name, 0, ty, [0], [ty], "", const_expr) + opcode(name, 0, ty, [0], [ty], False, "", const_expr) def unop_horiz(name, output_size, output_type, input_size, input_type, const_expr): - opcode(name, output_size, output_type, [input_size], [input_type], "", - const_expr) + opcode(name, output_size, output_type, [input_size], [input_type], + False, "", const_expr) def unop_reduce(name, output_size, output_type, input_type, prereduce_expr, reduce_expr, final_expr): @@ -141,67 +187,94 @@ def unop_reduce(name, output_size, output_type, input_type, prereduce_expr, unop_horiz(name + "4", output_size, output_type, 4, input_type, final(reduce_(reduce_(src0, src1), reduce_(src2, src3)))) +def unop_numeric_convert(name, out_type, in_type, const_expr): + opcode(name, 0, out_type, [0], [in_type], True, "", const_expr) -# These two move instructions differ in what modifiers they support and what -# the negate modifier means. Otherwise, they are identical. -unop("fmov", tfloat, "src0") -unop("imov", tint, "src0") +unop("mov", tuint, "src0") unop("ineg", tint, "-src0") unop("fneg", tfloat, "-src0") unop("inot", tint, "~src0") # invert every bit of the integer -unop("fnot", tfloat, ("bit_size == 64 ? ((src0 == 0.0) ? 1.0 : 0.0f) : " + - "((src0 == 0.0f) ? 1.0f : 0.0f)")) unop("fsign", tfloat, ("bit_size == 64 ? " + "((src0 == 0.0) ? 0.0 : ((src0 > 0.0) ? 1.0 : -1.0)) : " + "((src0 == 0.0f) ? 0.0f : ((src0 > 0.0f) ? 1.0f : -1.0f))")) unop("isign", tint, "(src0 == 0) ? 0 : ((src0 > 0) ? 1 : -1)") unop("iabs", tint, "(src0 < 0) ? -src0 : src0") -unop("fabs", tfloat, "bit_size == 64 ? fabs(src0) : fabsf(src0)") -unop("fsat", tfloat, ("bit_size == 64 ? " + - "((src0 > 1.0) ? 1.0 : ((src0 <= 0.0) ? 0.0 : src0)) : " + - "((src0 > 1.0f) ? 1.0f : ((src0 <= 0.0f) ? 0.0f : src0))")) +unop("fabs", tfloat, "fabs(src0)") +unop("fsat", tfloat, ("fmin(fmax(src0, 0.0), 1.0)")) +unop("fsat_signed", tfloat, ("fmin(fmax(src0, -1.0), 1.0)")) +unop("fclamp_pos", tfloat, ("fmax(src0, 0.0)")) unop("frcp", tfloat, "bit_size == 64 ? 1.0 / src0 : 1.0f / src0") unop("frsq", tfloat, "bit_size == 64 ? 1.0 / sqrt(src0) : 1.0f / sqrtf(src0)") unop("fsqrt", tfloat, "bit_size == 64 ? sqrt(src0) : sqrtf(src0)") unop("fexp2", tfloat, "exp2f(src0)") unop("flog2", tfloat, "log2f(src0)") -unop_convert("f2i", tint32, tfloat32, "src0") # Float-to-integer conversion. -unop_convert("f2u", tuint32, tfloat32, "src0") # Float-to-unsigned conversion -unop_convert("d2i", tint32, tfloat64, "src0") # Double-to-integer conversion. -unop_convert("d2u", tuint32, tfloat64, "src0") # Double-to-unsigned conversion. -unop_convert("i2f", tfloat32, tint32, "src0") # Integer-to-float conversion. -unop_convert("i2d", tfloat64, tint32, "src0") # Integer-to-double conversion. -unop_convert("i2i32", tint32, tint, "src0") # General int (int8_t, int64_t, etc.) to int32_t conversion -unop_convert("u2i32", tint32, tuint, "src0") # General uint (uint8_t, uint64_t, etc.) to int32_t conversion -unop_convert("i2u32", tuint32, tint, "src0") # General int (int8_t, int64_t, etc.) to uint32_t conversion -unop_convert("u2u32", tuint32, tuint, "src0") # General uint (uint8_t, uint64_t, etc.) to uint32_t conversion -unop_convert("i2i64", tint64, tint, "src0") # General int (int8_t, int32_t, etc.) to int64_t conversion -unop_convert("u2i64", tint64, tuint, "src0") # General uint (uint8_t, uint64_t, etc.) to int64_t conversion -unop_convert("f2i64", tint64, tfloat, "src0") # General float (float or double) to int64_t conversion -unop_convert("i2u64", tuint64, tint, "src0") # General int (int8_t, int64_t, etc.) to uint64_t conversion -unop_convert("u2u64", tuint64, tuint, "src0") # General uint (uint8_t, uint32_t, etc.) to uint64_t conversion -unop_convert("f2u64", tuint64, tfloat, "src0") # General float (float or double) to uint64_t conversion -unop_convert("i642f", tfloat32, tint64, "src0") # int64_t-to-float conversion. -unop_convert("i642b", tbool, tint64, "src0") # int64_t-to-bool conversion. -unop_convert("i642d", tfloat64, tint64, "src0") # int64_t-to-double conversion. -unop_convert("u642f", tfloat32, tuint64, "src0") # uint64_t-to-float conversion. -unop_convert("u642d", tfloat64, tuint64, "src0") # uint64_t-to-double conversion. - -# Float-to-boolean conversion -unop_convert("f2b", tbool, tfloat32, "src0 != 0.0f") -unop_convert("d2b", tbool, tfloat64, "src0 != 0.0") -# Boolean-to-float conversion -unop_convert("b2f", tfloat32, tbool, "src0 ? 1.0f : 0.0f") -# Int-to-boolean conversion -unop_convert("i2b", tbool, tint, "src0 != 0") -unop_convert("b2i", tint32, tbool, "src0 ? 1 : 0") # Boolean-to-int conversion -unop_convert("b2i64", tint64, tbool, "src0 ? 1 : 0") # Boolean-to-int64_t conversion. -unop_convert("u2f", tfloat32, tuint32, "src0") # Unsigned-to-float conversion. -unop_convert("u2d", tfloat64, tuint32, "src0") # Unsigned-to-double conversion. -# double-to-float conversion -unop_convert("d2f", tfloat32, tfloat64, "src0") # Double to single precision -unop_convert("f2d", tfloat64, tfloat32, "src0") # Single to double precision + +# Generate all of the numeric conversion opcodes +for src_t in [tint, tuint, tfloat, tbool]: + if src_t == tbool: + dst_types = [tfloat, tint, tbool] + elif src_t == tint: + dst_types = [tfloat, tint, tbool] + elif src_t == tuint: + dst_types = [tfloat, tuint] + elif src_t == tfloat: + dst_types = [tint, tuint, tfloat, tbool] + + for dst_t in dst_types: + for dst_bit_size in type_sizes(dst_t): + if dst_bit_size == 16 and dst_t == tfloat and src_t == tfloat: + rnd_modes = ['_rtne', '_rtz', ''] + for rnd_mode in rnd_modes: + if rnd_mode == '_rtne': + conv_expr = """ + if (bit_size > 16) { + dst = _mesa_half_to_float(_mesa_float_to_float16_rtne(src0)); + } else { + dst = src0; + } + """ + elif rnd_mode == '_rtz': + conv_expr = """ + if (bit_size > 16) { + dst = _mesa_half_to_float(_mesa_float_to_float16_rtz(src0)); + } else { + dst = src0; + } + """ + else: + conv_expr = "src0" + + unop_numeric_convert("{0}2{1}{2}{3}".format(src_t[0], + dst_t[0], + dst_bit_size, + rnd_mode), + dst_t + str(dst_bit_size), + src_t, conv_expr) + elif dst_bit_size == 32 and dst_t == tfloat and src_t == tfloat: + conv_expr = """ + if (bit_size > 32 && nir_is_rounding_mode_rtz(execution_mode, 32)) { + dst = _mesa_double_to_float_rtz(src0); + } else { + dst = src0; + } + """ + unop_numeric_convert("{0}2{1}{2}".format(src_t[0], dst_t[0], + dst_bit_size), + dst_t + str(dst_bit_size), src_t, conv_expr) + else: + conv_expr = "src0 != 0" if dst_t == tbool else "src0" + unop_numeric_convert("{0}2{1}{2}".format(src_t[0], dst_t[0], + dst_bit_size), + dst_t + str(dst_bit_size), src_t, conv_expr) + +# Special opcode that is the same as f2f16, i2i16, u2u16 except that it is safe +# to remove it if the result is immediately converted back to 32 bits again. +# This is generated as part of the precision lowering pass. mp stands for medium +# precision. +unop_numeric_convert("f2fmp", tfloat16, tfloat, opcodes["f2f16"].const_expr) +unop_numeric_convert("i2imp", tint16, tint, opcodes["i2i16"].const_expr) +unop_numeric_convert("u2ump", tuint16, tuint, opcodes["u2u16"].const_expr) # Unary floating-point rounding operations. @@ -220,6 +293,9 @@ unop("fquantize2f16", tfloat, "(fabs(src0) < ldexpf(1.0, -14)) ? copysignf(0.0f, unop("fsin", tfloat, "bit_size == 64 ? sin(src0) : sinf(src0)") unop("fcos", tfloat, "bit_size == 64 ? cos(src0) : cosf(src0)") +# dfrexp +unop_convert("frexp_exp", tint32, tfloat, "frexp(src0, &dst);") +unop_convert("frexp_sig", tfloat, tfloat, "int n; dst = frexp(src0, &n);") # Partial derivatives. @@ -285,30 +361,52 @@ dst.x = (src0.x << 0) | (src0.w << 24); """) -unop_horiz("pack_double_2x32", 1, tuint64, 2, tuint32, +unop_horiz("pack_32_4x8", 1, tuint32, 4, tuint8, + "dst.x = src0.x | ((uint32_t)src0.y << 8) | ((uint32_t)src0.z << 16) | ((uint32_t)src0.w << 24);") + +unop_horiz("pack_32_2x16", 1, tuint32, 2, tuint16, + "dst.x = src0.x | ((uint32_t)src0.y << 16);") + +unop_horiz("pack_64_2x32", 1, tuint64, 2, tuint32, "dst.x = src0.x | ((uint64_t)src0.y << 32);") -unop_horiz("pack_int_2x32", 1, tint64, 2, tint32, - "dst.x = src0.x | ((int64_t)src0.y << 32);") +unop_horiz("pack_64_4x16", 1, tuint64, 4, tuint16, + "dst.x = src0.x | ((uint64_t)src0.y << 16) | ((uint64_t)src0.z << 32) | ((uint64_t)src0.w << 48);") -unop_horiz("unpack_double_2x32", 2, tuint32, 1, tuint64, +unop_horiz("unpack_64_2x32", 2, tuint32, 1, tuint64, "dst.x = src0.x; dst.y = src0.x >> 32;") -unop_horiz("unpack_int_2x32", 2, tint32, 1, tint64, - "dst.x = src0.x; dst.y = src0.x >> 32;") +unop_horiz("unpack_64_4x16", 4, tuint16, 1, tuint64, + "dst.x = src0.x; dst.y = src0.x >> 16; dst.z = src0.x >> 32; dst.w = src0.w >> 48;") + +unop_horiz("unpack_32_2x16", 2, tuint16, 1, tuint32, + "dst.x = src0.x; dst.y = src0.x >> 16;") + +unop_horiz("unpack_32_4x8", 4, tuint8, 1, tuint32, + "dst.x = src0.x; dst.y = src0.x >> 8; dst.z = src0.x >> 16; dst.w = src0.x >> 24;") + +unop_horiz("unpack_half_2x16_flush_to_zero", 2, tfloat32, 1, tuint32, """ +dst.x = unpack_half_1x16_flush_to_zero((uint16_t)(src0.x & 0xffff)); +dst.y = unpack_half_1x16_flush_to_zero((uint16_t)(src0.x << 16)); +""") # Lowered floating point unpacking operations. +unop_convert("unpack_half_2x16_split_x", tfloat32, tuint32, + "unpack_half_1x16((uint16_t)(src0 & 0xffff))") +unop_convert("unpack_half_2x16_split_y", tfloat32, tuint32, + "unpack_half_1x16((uint16_t)(src0 >> 16))") -unop_horiz("unpack_half_2x16_split_x", 1, tfloat32, 1, tuint32, - "unpack_half_1x16((uint16_t)(src0.x & 0xffff))") -unop_horiz("unpack_half_2x16_split_y", 1, tfloat32, 1, tuint32, - "unpack_half_1x16((uint16_t)(src0.x >> 16))") +unop_convert("unpack_half_2x16_split_x_flush_to_zero", tfloat32, tuint32, + "unpack_half_1x16_flush_to_zero((uint16_t)(src0 & 0xffff))") +unop_convert("unpack_half_2x16_split_y_flush_to_zero", tfloat32, tuint32, + "unpack_half_1x16_flush_to_zero((uint16_t)(src0 >> 16))") -unop_convert("unpack_double_2x32_split_x", tuint32, tuint64, "src0") -unop_convert("unpack_double_2x32_split_y", tuint32, tuint64, "src0 >> 32") -unop_convert("unpack_int_2x32_split_x", tuint32, tuint64, "src0") -unop_convert("unpack_int_2x32_split_y", tuint32, tuint64, "src0 >> 32") +unop_convert("unpack_32_2x16_split_x", tuint16, tuint32, "src0") +unop_convert("unpack_32_2x16_split_y", tuint16, tuint32, "src0 >> 16") + +unop_convert("unpack_64_2x32_split_x", tuint32, tuint64, "src0") +unop_convert("unpack_64_2x32_split_y", tuint32, tuint64, "src0 >> 32") # Bit operations, part of ARB_gpu_shader5. @@ -319,17 +417,17 @@ dst = 0; for (unsigned bit = 0; bit < 32; bit++) dst |= ((src0 >> bit) & 1) << (31 - bit); """) -unop("bit_count", tuint32, """ +unop_convert("bit_count", tuint32, tuint, """ dst = 0; -for (unsigned bit = 0; bit < 32; bit++) { +for (unsigned bit = 0; bit < bit_size; bit++) { if ((src0 >> bit) & 1) dst++; } """) -unop_convert("ufind_msb", tint32, tuint32, """ +unop_convert("ufind_msb", tint32, tuint, """ dst = -1; -for (int bit = 31; bit > 0; bit--) { +for (int bit = bit_size - 1; bit >= 0; bit--) { if ((src0 >> bit) & 1) { dst = bit; break; @@ -337,6 +435,15 @@ for (int bit = 31; bit > 0; bit--) { } """) +unop("uclz", tuint32, """ +int bit; +for (bit = bit_size - 1; bit >= 0; bit--) { + if ((src0 & (1u << bit)) != 0) + break; +} +dst = (unsigned)(31 - bit); +""") + unop("ifind_msb", tint32, """ dst = -1; for (int bit = 31; bit >= 0; bit--) { @@ -351,9 +458,9 @@ for (int bit = 31; bit >= 0; bit--) { } """) -unop("find_lsb", tint32, """ +unop_convert("find_lsb", tint32, tint, """ dst = -1; -for (unsigned bit = 0; bit < 32; bit++) { +for (unsigned bit = 0; bit < bit_size; bit++) { if ((src0 >> bit) & 1) { dst = bit; break; @@ -361,24 +468,73 @@ for (unsigned bit = 0; bit < 32; bit++) { } """) +# AMD_gcn_shader extended instructions +unop_horiz("cube_face_coord", 2, tfloat32, 3, tfloat32, """ +dst.x = dst.y = 0.0; +float absX = fabsf(src0.x); +float absY = fabsf(src0.y); +float absZ = fabsf(src0.z); + +float ma = 0.0; +if (absX >= absY && absX >= absZ) { ma = 2 * src0.x; } +if (absY >= absX && absY >= absZ) { ma = 2 * src0.y; } +if (absZ >= absX && absZ >= absY) { ma = 2 * src0.z; } + +if (src0.x >= 0 && absX >= absY && absX >= absZ) { dst.x = -src0.z; dst.y = -src0.y; } +if (src0.x < 0 && absX >= absY && absX >= absZ) { dst.x = src0.z; dst.y = -src0.y; } +if (src0.y >= 0 && absY >= absX && absY >= absZ) { dst.x = src0.x; dst.y = src0.z; } +if (src0.y < 0 && absY >= absX && absY >= absZ) { dst.x = src0.x; dst.y = -src0.z; } +if (src0.z >= 0 && absZ >= absX && absZ >= absY) { dst.x = src0.x; dst.y = -src0.y; } +if (src0.z < 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.x; dst.y = -src0.y; } + +dst.x = dst.x * (1.0f / ma) + 0.5f; +dst.y = dst.y * (1.0f / ma) + 0.5f; +""") + +unop_horiz("cube_face_index", 1, tfloat32, 3, tfloat32, """ +float absX = fabsf(src0.x); +float absY = fabsf(src0.y); +float absZ = fabsf(src0.z); +if (src0.x >= 0 && absX >= absY && absX >= absZ) dst.x = 0; +if (src0.x < 0 && absX >= absY && absX >= absZ) dst.x = 1; +if (src0.y >= 0 && absY >= absX && absY >= absZ) dst.x = 2; +if (src0.y < 0 && absY >= absX && absY >= absZ) dst.x = 3; +if (src0.z >= 0 && absZ >= absX && absZ >= absY) dst.x = 4; +if (src0.z < 0 && absZ >= absX && absZ >= absY) dst.x = 5; +""") -for i in xrange(1, 5): - for j in xrange(1, 5): - unop_horiz("fnoise{0}_{1}".format(i, j), i, tfloat, j, tfloat, "0.0f") +# Sum of vector components +unop_reduce("fsum", 1, tfloat, tfloat, "{src}", "{src0} + {src1}", "{src}") def binop_convert(name, out_type, in_type, alg_props, const_expr): - opcode(name, 0, out_type, [0, 0], [in_type, in_type], alg_props, const_expr) + opcode(name, 0, out_type, [0, 0], [in_type, in_type], + False, alg_props, const_expr) def binop(name, ty, alg_props, const_expr): binop_convert(name, ty, ty, alg_props, const_expr) def binop_compare(name, ty, alg_props, const_expr): - binop_convert(name, tbool, ty, alg_props, const_expr) + binop_convert(name, tbool1, ty, alg_props, const_expr) + +def binop_compare8(name, ty, alg_props, const_expr): + binop_convert(name, tbool8, ty, alg_props, const_expr) + +def binop_compare16(name, ty, alg_props, const_expr): + binop_convert(name, tbool16, ty, alg_props, const_expr) + +def binop_compare32(name, ty, alg_props, const_expr): + binop_convert(name, tbool32, ty, alg_props, const_expr) + +def binop_compare_all_sizes(name, ty, alg_props, const_expr): + binop_compare(name, ty, alg_props, const_expr) + binop_compare8(name + "8", ty, alg_props, const_expr) + binop_compare16(name + "16", ty, alg_props, const_expr) + binop_compare32(name + "32", ty, alg_props, const_expr) def binop_horiz(name, out_size, out_type, src1_size, src1_type, src2_size, src2_type, const_expr): opcode(name, out_size, out_type, [src1_size, src2_size], [src1_type, src2_type], - "", const_expr) + False, "", const_expr) def binop_reduce(name, output_size, output_type, src_type, prereduce_expr, reduce_expr, final_expr): @@ -388,49 +544,182 @@ def binop_reduce(name, output_size, output_type, src_type, prereduce_expr, return reduce_expr.format(src0=src0, src1=src1) def prereduce(src0, src1): return "(" + prereduce_expr.format(src0=src0, src1=src1) + ")" - src0 = prereduce("src0.x", "src1.x") - src1 = prereduce("src0.y", "src1.y") - src2 = prereduce("src0.z", "src1.z") - src3 = prereduce("src0.w", "src1.w") - opcode(name + "2", output_size, output_type, - [2, 2], [src_type, src_type], commutative, - final(reduce_(src0, src1))) + srcs = [prereduce("src0." + letter, "src1." + letter) for letter in "xyzwefghijklmnop"] + def pairwise_reduce(start, size): + if (size == 1): + return srcs[start] + return reduce_(pairwise_reduce(start, size // 2), pairwise_reduce(start + size // 2, size // 2)) + for size in [2, 4, 8, 16]: + opcode(name + str(size), output_size, output_type, + [size, size], [src_type, src_type], False, _2src_commutative, + final(pairwise_reduce(0, size))) opcode(name + "3", output_size, output_type, - [3, 3], [src_type, src_type], commutative, - final(reduce_(reduce_(src0, src1), src2))) - opcode(name + "4", output_size, output_type, - [4, 4], [src_type, src_type], commutative, - final(reduce_(reduce_(src0, src1), reduce_(src2, src3)))) - -binop("fadd", tfloat, commutative + associative, "src0 + src1") -binop("iadd", tint, commutative + associative, "src0 + src1") -binop("fsub", tfloat, "", "src0 - src1") + [3, 3], [src_type, src_type], False, _2src_commutative, + final(reduce_(reduce_(srcs[0], srcs[1]), srcs[2]))) + +def binop_reduce_all_sizes(name, output_size, src_type, prereduce_expr, + reduce_expr, final_expr): + binop_reduce(name, output_size, tbool1, src_type, + prereduce_expr, reduce_expr, final_expr) + binop_reduce("b8" + name[1:], output_size, tbool8, src_type, + prereduce_expr, reduce_expr, final_expr) + binop_reduce("b16" + name[1:], output_size, tbool16, src_type, + prereduce_expr, reduce_expr, final_expr) + binop_reduce("b32" + name[1:], output_size, tbool32, src_type, + prereduce_expr, reduce_expr, final_expr) + +binop("fadd", tfloat, _2src_commutative + associative,""" +if (nir_is_rounding_mode_rtz(execution_mode, bit_size)) { + if (bit_size == 64) + dst = _mesa_double_add_rtz(src0, src1); + else + dst = _mesa_double_to_float_rtz((double)src0 + (double)src1); +} else { + dst = src0 + src1; +} +""") +binop("iadd", tint, _2src_commutative + associative, "src0 + src1") +binop("iadd_sat", tint, _2src_commutative, """ + src1 > 0 ? + (src0 + src1 < src0 ? (1ull << (bit_size - 1)) - 1 : src0 + src1) : + (src0 < src0 + src1 ? (1ull << (bit_size - 1)) : src0 + src1) +""") +binop("uadd_sat", tuint, _2src_commutative, + "(src0 + src1) < src0 ? MAX_UINT_FOR_SIZE(sizeof(src0) * 8) : (src0 + src1)") +binop("isub_sat", tint, "", """ + src1 < 0 ? + (src0 - src1 < src0 ? (1ull << (bit_size - 1)) - 1 : src0 - src1) : + (src0 < src0 - src1 ? (1ull << (bit_size - 1)) : src0 - src1) +""") +binop("usub_sat", tuint, "", "src0 < src1 ? 0 : src0 - src1") + +binop("fsub", tfloat, "", """ +if (nir_is_rounding_mode_rtz(execution_mode, bit_size)) { + if (bit_size == 64) + dst = _mesa_double_sub_rtz(src0, src1); + else + dst = _mesa_double_to_float_rtz((double)src0 - (double)src1); +} else { + dst = src0 - src1; +} +""") binop("isub", tint, "", "src0 - src1") - -binop("fmul", tfloat, commutative + associative, "src0 * src1") +binop_convert("uabs_isub", tuint, tint, "", """ + src1 > src0 ? (uint64_t) src1 - (uint64_t) src0 + : (uint64_t) src0 - (uint64_t) src1 +""") +binop("uabs_usub", tuint, "", "(src1 > src0) ? (src1 - src0) : (src0 - src1)") + +binop("fmul", tfloat, _2src_commutative + associative, """ +if (nir_is_rounding_mode_rtz(execution_mode, bit_size)) { + if (bit_size == 64) + dst = _mesa_double_mul_rtz(src0, src1); + else + dst = _mesa_double_to_float_rtz((double)src0 * (double)src1); +} else { + dst = src0 * src1; +} +""") # low 32-bits of signed/unsigned integer multiply -binop("imul", tint, commutative + associative, "src0 * src1") +binop("imul", tint, _2src_commutative + associative, "src0 * src1") + +# Generate 64 bit result from 2 32 bits quantity +binop_convert("imul_2x32_64", tint64, tint32, _2src_commutative, + "(int64_t)src0 * (int64_t)src1") +binop_convert("umul_2x32_64", tuint64, tuint32, _2src_commutative, + "(uint64_t)src0 * (uint64_t)src1") + # high 32-bits of signed integer multiply -binop("imul_high", tint32, commutative, - "(int32_t)(((int64_t) src0 * (int64_t) src1) >> 32)") +binop("imul_high", tint, _2src_commutative, """ +if (bit_size == 64) { + /* We need to do a full 128-bit x 128-bit multiply in order for the sign + * extension to work properly. The casts are kind-of annoying but needed + * to prevent compiler warnings. + */ + uint32_t src0_u32[4] = { + src0, + (int64_t)src0 >> 32, + (int64_t)src0 >> 63, + (int64_t)src0 >> 63, + }; + uint32_t src1_u32[4] = { + src1, + (int64_t)src1 >> 32, + (int64_t)src1 >> 63, + (int64_t)src1 >> 63, + }; + uint32_t prod_u32[4]; + ubm_mul_u32arr(prod_u32, src0_u32, src1_u32); + dst = (uint64_t)prod_u32[2] | ((uint64_t)prod_u32[3] << 32); +} else { + dst = ((int64_t)src0 * (int64_t)src1) >> bit_size; +} +""") + # high 32-bits of unsigned integer multiply -binop("umul_high", tuint32, commutative, - "(uint32_t)(((uint64_t) src0 * (uint64_t) src1) >> 32)") +binop("umul_high", tuint, _2src_commutative, """ +if (bit_size == 64) { + /* The casts are kind-of annoying but needed to prevent compiler warnings. */ + uint32_t src0_u32[2] = { src0, (uint64_t)src0 >> 32 }; + uint32_t src1_u32[2] = { src1, (uint64_t)src1 >> 32 }; + uint32_t prod_u32[4]; + ubm_mul_u32arr(prod_u32, src0_u32, src1_u32); + dst = (uint64_t)prod_u32[2] | ((uint64_t)prod_u32[3] << 32); +} else { + dst = ((uint64_t)src0 * (uint64_t)src1) >> bit_size; +} +""") + +# low 32-bits of unsigned integer multiply +binop("umul_low", tuint32, _2src_commutative, """ +uint64_t mask = (1 << (bit_size / 2)) - 1; +dst = ((uint64_t)src0 & mask) * ((uint64_t)src1 & mask); +""") + +# Multiply 32-bits with low 16-bits. +binop("imul_32x16", tint32, "", "src0 * (int16_t) src1") +binop("umul_32x16", tuint32, "", "src0 * (uint16_t) src1") binop("fdiv", tfloat, "", "src0 / src1") -binop("idiv", tint, "", "src0 / src1") -binop("udiv", tuint, "", "src0 / src1") +binop("idiv", tint, "", "src1 == 0 ? 0 : (src0 / src1)") +binop("udiv", tuint, "", "src1 == 0 ? 0 : (src0 / src1)") # returns a boolean representing the carry resulting from the addition of # the two unsigned arguments. -binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0") +binop_convert("uadd_carry", tuint, tuint, _2src_commutative, "src0 + src1 < src0") # returns a boolean representing the borrow resulting from the subtraction # of the two unsigned arguments. binop_convert("usub_borrow", tuint, tuint, "", "src0 < src1") +# hadd: (a + b) >> 1 (without overflow) +# x + y = x - (x & ~y) + (x & ~y) + y - (~x & y) + (~x & y) +# = (x & y) + (x & ~y) + (x & y) + (~x & y) +# = 2 * (x & y) + (x & ~y) + (~x & y) +# = ((x & y) << 1) + (x ^ y) +# +# Since we know that the bottom bit of (x & y) << 1 is zero, +# +# (x + y) >> 1 = (((x & y) << 1) + (x ^ y)) >> 1 +# = (x & y) + ((x ^ y) >> 1) +binop("ihadd", tint, _2src_commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)") +binop("uhadd", tuint, _2src_commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)") + +# rhadd: (a + b + 1) >> 1 (without overflow) +# x + y + 1 = x + (~x & y) - (~x & y) + y + (x & ~y) - (x & ~y) + 1 +# = (x | y) - (~x & y) + (x | y) - (x & ~y) + 1 +# = 2 * (x | y) - ((~x & y) + (x & ~y)) + 1 +# = ((x | y) << 1) - (x ^ y) + 1 +# +# Since we know that the bottom bit of (x & y) << 1 is zero, +# +# (x + y + 1) >> 1 = (x | y) + (-(x ^ y) + 1) >> 1) +# = (x | y) - ((x ^ y) >> 1) +binop("irhadd", tint, _2src_commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)") +binop("urhadd", tuint, _2src_commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)") + binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1") # For signed integers, there are several different possible definitions of @@ -455,27 +744,27 @@ binop("frem", tfloat, "", "src0 - src1 * truncf(src0 / src1)") # these integer-aware comparisons return a boolean (0 or ~0) -binop_compare("flt", tfloat, "", "src0 < src1") -binop_compare("fge", tfloat, "", "src0 >= src1") -binop_compare("feq", tfloat, commutative, "src0 == src1") -binop_compare("fne", tfloat, commutative, "src0 != src1") -binop_compare("ilt", tint, "", "src0 < src1") -binop_compare("ige", tint, "", "src0 >= src1") -binop_compare("ieq", tint, commutative, "src0 == src1") -binop_compare("ine", tint, commutative, "src0 != src1") -binop_compare("ult", tuint, "", "src0 < src1") -binop_compare("uge", tuint, "", "src0 >= src1") +binop_compare_all_sizes("flt", tfloat, "", "src0 < src1") +binop_compare_all_sizes("fge", tfloat, "", "src0 >= src1") +binop_compare_all_sizes("feq", tfloat, _2src_commutative, "src0 == src1") +binop_compare_all_sizes("fneu", tfloat, _2src_commutative, "src0 != src1") +binop_compare_all_sizes("ilt", tint, "", "src0 < src1") +binop_compare_all_sizes("ige", tint, "", "src0 >= src1") +binop_compare_all_sizes("ieq", tint, _2src_commutative, "src0 == src1") +binop_compare_all_sizes("ine", tint, _2src_commutative, "src0 != src1") +binop_compare_all_sizes("ult", tuint, "", "src0 < src1") +binop_compare_all_sizes("uge", tuint, "", "src0 >= src1") # integer-aware GLSL-style comparisons that compare floats and ints -binop_reduce("ball_fequal", 1, tbool, tfloat, "{src0} == {src1}", - "{src0} && {src1}", "{src}") -binop_reduce("bany_fnequal", 1, tbool, tfloat, "{src0} != {src1}", - "{src0} || {src1}", "{src}") -binop_reduce("ball_iequal", 1, tbool, tint, "{src0} == {src1}", - "{src0} && {src1}", "{src}") -binop_reduce("bany_inequal", 1, tbool, tint, "{src0} != {src1}", - "{src0} || {src1}", "{src}") +binop_reduce_all_sizes("ball_fequal", 1, tfloat, "{src0} == {src1}", + "{src0} && {src1}", "{src}") +binop_reduce_all_sizes("bany_fnequal", 1, tfloat, "{src0} != {src1}", + "{src0} || {src1}", "{src}") +binop_reduce_all_sizes("ball_iequal", 1, tint, "{src0} == {src1}", + "{src0} && {src1}", "{src}") +binop_reduce_all_sizes("bany_inequal", 1, tint, "{src0} != {src1}", + "{src0} || {src1}", "{src}") # non-integer-aware GLSL-style comparisons that return 0.0 or 1.0 @@ -488,14 +777,30 @@ binop_reduce("fany_nequal", 1, tfloat32, tfloat32, "{src0} != {src1}", # and false respectively binop("slt", tfloat32, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than -binop("sge", tfloat32, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal -binop("seq", tfloat32, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal -binop("sne", tfloat32, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal - - -opcode("ishl", 0, tint, [0, 0], [tint, tuint32], "", "src0 << src1") -opcode("ishr", 0, tint, [0, 0], [tint, tuint32], "", "src0 >> src1") -opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], "", "src0 >> src1") +binop("sge", tfloat, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal +binop("seq", tfloat32, _2src_commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal +binop("sne", tfloat32, _2src_commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal + +# SPIRV shifts are undefined for shift-operands >= bitsize, +# but SM5 shifts are defined to use the least significant bits, only +# The NIR definition is according to the SM5 specification. +opcode("ishl", 0, tint, [0, 0], [tint, tuint32], False, "", + "(uint64_t)src0 << (src1 & (sizeof(src0) * 8 - 1))") +opcode("ishr", 0, tint, [0, 0], [tint, tuint32], False, "", + "src0 >> (src1 & (sizeof(src0) * 8 - 1))") +opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], False, "", + "src0 >> (src1 & (sizeof(src0) * 8 - 1))") + +opcode("urol", 0, tuint, [0, 0], [tuint, tuint32], False, "", """ + uint32_t rotate_mask = sizeof(src0) * 8 - 1; + dst = (src0 << (src1 & rotate_mask)) | + (src0 >> (-src1 & rotate_mask)); +""") +opcode("uror", 0, tuint, [0, 0], [tuint, tuint32], False, "", """ + uint32_t rotate_mask = sizeof(src0) * 8 - 1; + dst = (src0 >> (src1 & rotate_mask)) | + (src0 << (-src1 & rotate_mask)); +""") # bitwise logic operators # @@ -503,43 +808,31 @@ opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], "", "src0 >> src1") # integers. -binop("iand", tuint, commutative + associative, "src0 & src1") -binop("ior", tuint, commutative + associative, "src0 | src1") -binop("ixor", tuint, commutative + associative, "src0 ^ src1") +binop("iand", tuint, _2src_commutative + associative, "src0 & src1") +binop("ior", tuint, _2src_commutative + associative, "src0 | src1") +binop("ixor", tuint, _2src_commutative + associative, "src0 ^ src1") -# floating point logic operators -# -# These use (src != 0.0) for testing the truth of the input, and output 1.0 -# for true and 0.0 for false - -binop("fand", tfloat32, commutative, - "((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f") -binop("for", tfloat32, commutative, - "((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f") -binop("fxor", tfloat32, commutative, - "(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f") - binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}", "{src}") binop_reduce("fdot_replicated", 4, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}", "{src}") -opcode("fdph", 1, tfloat, [3, 4], [tfloat, tfloat], "", +opcode("fdph", 1, tfloat, [3, 4], [tfloat, tfloat], False, "", "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w") -opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], "", +opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], False, "", "src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w") -binop("fmin", tfloat, "", "fminf(src0, src1)") -binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1") -binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1") -binop("fmax", tfloat, "", "fmaxf(src0, src1)") -binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0") -binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0") +binop("fmin", tfloat, _2src_commutative + associative, "fmin(src0, src1)") +binop("imin", tint, _2src_commutative + associative, "src1 > src0 ? src0 : src1") +binop("umin", tuint, _2src_commutative + associative, "src1 > src0 ? src0 : src1") +binop("fmax", tfloat, _2src_commutative + associative, "fmax(src0, src1)") +binop("imax", tint, _2src_commutative + associative, "src1 > src0 ? src1 : src0") +binop("umax", tuint, _2src_commutative + associative, "src1 > src0 ? src1 : src0") # Saturated vector add for 4 8bit ints. -binop("usadd_4x8", tint32, commutative + associative, """ +binop("usadd_4x8", tint32, _2src_commutative + associative, """ dst = 0; for (int i = 0; i < 32; i += 8) { dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i; @@ -558,7 +851,7 @@ for (int i = 0; i < 32; i += 8) { """) # vector min for 4 8bit ints. -binop("umin_4x8", tint32, commutative + associative, """ +binop("umin_4x8", tint32, _2src_commutative + associative, """ dst = 0; for (int i = 0; i < 32; i += 8) { dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i; @@ -566,7 +859,7 @@ for (int i = 0; i < 32; i += 8) { """) # vector max for 4 8bit ints. -binop("umax_4x8", tint32, commutative + associative, """ +binop("umax_4x8", tint32, _2src_commutative + associative, """ dst = 0; for (int i = 0; i < 32; i += 8) { dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i; @@ -574,7 +867,7 @@ for (int i = 0; i < 32; i += 8) { """) # unorm multiply: (a * b) / 255. -binop("umul_unorm_4x8", tint32, commutative + associative, """ +binop("umul_unorm_4x8", tint32, _2src_commutative + associative, """ dst = 0; for (int i = 0; i < 32; i += 8) { int src0_chan = (src0 >> i) & 0xff; @@ -588,24 +881,22 @@ binop("fpow", tfloat, "", "bit_size == 64 ? powf(src0, src1) : pow(src0, src1)") binop_horiz("pack_half_2x16_split", 1, tuint32, 1, tfloat32, 1, tfloat32, "pack_half_1x16(src0.x) | (pack_half_1x16(src1.x) << 16)") -binop_convert("pack_double_2x32_split", tuint64, tuint32, "", +binop_convert("pack_64_2x32_split", tuint64, tuint32, "", "src0 | ((uint64_t)src1 << 32)") -binop_convert("pack_int_2x32_split", tuint64, tuint32, "", - "src0 | ((uint64_t)src1 << 32)") +binop_convert("pack_32_2x16_split", tuint32, tuint16, "", + "src0 | ((uint32_t)src1 << 16)") # bfm implements the behavior of the first operation of the SM5 "bfi" assembly -# and that of the "bfi1" i965 instruction. That is, it has undefined behavior -# if either of its arguments are 32. +# and that of the "bfi1" i965 instruction. That is, the bits and offset values +# are from the low five bits of src0 and src1, respectively. binop_convert("bfm", tuint32, tint32, "", """ -int bits = src0, offset = src1; -if (offset < 0 || bits < 0 || offset > 31 || bits > 31 || offset + bits > 32) - dst = 0; /* undefined */ -else - dst = ((1u << bits) - 1) << offset; +int bits = src0 & 0x1F; +int offset = src1 & 0x1F; +dst = ((1u << bits) - 1) << offset; """) -opcode("ldexp", 0, tfloat, [0, 0], [tfloat, tint32], "", """ +opcode("ldexp", 0, tfloat, [0, 0], [tfloat, tint32], False, "", """ dst = (bit_size == 64) ? ldexp(src0, src1) : ldexpf(src0, src1); /* flush denormals to zero. */ if (!isnormal(dst)) @@ -628,16 +919,30 @@ binop("extract_u16", tuint, "", "(uint16_t)(src0 >> (src1 * 16))") binop("extract_i16", tint, "", "(int16_t)(src0 >> (src1 * 16))") -def triop(name, ty, const_expr): - opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], "", const_expr) +def triop(name, ty, alg_props, const_expr): + opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], False, alg_props, const_expr) def triop_horiz(name, output_size, src1_size, src2_size, src3_size, const_expr): opcode(name, output_size, tuint, [src1_size, src2_size, src3_size], - [tuint, tuint, tuint], "", const_expr) - -triop("ffma", tfloat, "src0 * src1 + src2") + [tuint, tuint, tuint], False, "", const_expr) + +triop("ffma", tfloat, _2src_commutative, """ +if (nir_is_rounding_mode_rtz(execution_mode, bit_size)) { + if (bit_size == 64) + dst = _mesa_double_fma_rtz(src0, src1, src2); + else if (bit_size == 32) + dst = _mesa_float_fma_rtz(src0, src1, src2); + else + dst = _mesa_double_to_float_rtz(_mesa_double_fma_rtz(src0, src1, src2)); +} else { + if (bit_size == 32) + dst = fmaf(src0, src1, src2); + else + dst = fma(src0, src1, src2); +} +""") -triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2") +triop("flrp", tfloat, "", "src0 * (1 - src2) + src1 * src2") # Conditional Select # @@ -645,13 +950,19 @@ triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2") # component on vectors). There are two versions, one for floating point # bools (0.0 vs 1.0) and one for integer bools (0 vs ~0). +triop("fcsel", tfloat32, "", "(src0 != 0.0f) ? src1 : src2") -triop("fcsel", tfloat32, "(src0 != 0.0f) ? src1 : src2") opcode("bcsel", 0, tuint, [0, 0, 0], - [tbool, tuint, tuint], "", "src0 ? src1 : src2") + [tbool1, tuint, tuint], False, "", "src0 ? src1 : src2") +opcode("b8csel", 0, tuint, [0, 0, 0], + [tbool8, tuint, tuint], False, "", "src0 ? src1 : src2") +opcode("b16csel", 0, tuint, [0, 0, 0], + [tbool16, tuint, tuint], False, "", "src0 ? src1 : src2") +opcode("b32csel", 0, tuint, [0, 0, 0], + [tbool32, tuint, tuint], False, "", "src0 ? src1 : src2") # SM5 bfi assembly -triop("bfi", tuint32, """ +triop("bfi", tuint32, "", """ unsigned mask = src0, insert = src1, base = src2; if (mask == 0) { dst = base; @@ -665,15 +976,17 @@ if (mask == 0) { } """) -# SM5 ubfe/ibfe assembly + +triop("bitfield_select", tuint, "", "(src0 & src1) | (~src0 & src2)") + +# SM5 ubfe/ibfe assembly: only the 5 least significant bits of offset and bits are used. opcode("ubfe", 0, tuint32, - [0, 0, 0], [tuint32, tint32, tint32], "", """ + [0, 0, 0], [tuint32, tuint32, tuint32], False, "", """ unsigned base = src0; -int offset = src1, bits = src2; +unsigned offset = src1 & 0x1F; +unsigned bits = src2 & 0x1F; if (bits == 0) { dst = 0; -} else if (bits < 0 || offset < 0) { - dst = 0; /* undefined */ } else if (offset + bits < 32) { dst = (base << (32 - bits - offset)) >> (32 - bits); } else { @@ -681,13 +994,12 @@ if (bits == 0) { } """) opcode("ibfe", 0, tint32, - [0, 0, 0], [tint32, tint32, tint32], "", """ + [0, 0, 0], [tint32, tuint32, tuint32], False, "", """ int base = src0; -int offset = src1, bits = src2; +unsigned offset = src1 & 0x1F; +unsigned bits = src2 & 0x1F; if (bits == 0) { dst = 0; -} else if (bits < 0 || offset < 0) { - dst = 0; /* undefined */ } else if (offset + bits < 32) { dst = (base << (32 - bits - offset)) >> (32 - bits); } else { @@ -697,7 +1009,7 @@ if (bits == 0) { # GLSL bitfieldExtract() opcode("ubitfield_extract", 0, tuint32, - [0, 0, 0], [tuint32, tint32, tint32], "", """ + [0, 0, 0], [tuint32, tint32, tint32], False, "", """ unsigned base = src0; int offset = src1, bits = src2; if (bits == 0) { @@ -709,7 +1021,7 @@ if (bits == 0) { } """) opcode("ibitfield_extract", 0, tint32, - [0, 0, 0], [tint32, tint32, tint32], "", """ + [0, 0, 0], [tint32, tint32, tint32], False, "", """ int base = src0; int offset = src1, bits = src2; if (bits == 0) { @@ -734,19 +1046,19 @@ def quadop_horiz(name, output_size, src1_size, src2_size, src3_size, opcode(name, output_size, tuint, [src1_size, src2_size, src3_size, src4_size], [tuint, tuint, tuint, tuint], - "", const_expr) + False, "", const_expr) opcode("bitfield_insert", 0, tuint32, [0, 0, 0, 0], - [tuint32, tuint32, tint32, tint32], "", """ + [tuint32, tuint32, tint32, tint32], False, "", """ unsigned base = src0, insert = src1; int offset = src2, bits = src3; if (bits == 0) { - dst = 0; + dst = base; } else if (offset < 0 || bits < 0 || bits + offset > 32) { dst = 0; } else { unsigned mask = ((1ull << bits) - 1) << offset; - dst = (base & ~mask) | ((insert << bits) & mask); + dst = (base & ~mask) | ((insert << offset) & mask); } """) @@ -757,4 +1069,77 @@ dst.z = src2.x; dst.w = src3.x; """) +opcode("vec8", 8, tuint, + [1] * 8, [tuint] * 8, + False, "", """ +dst.x = src0.x; +dst.y = src1.x; +dst.z = src2.x; +dst.w = src3.x; +dst.e = src4.x; +dst.f = src5.x; +dst.g = src6.x; +dst.h = src7.x; +""") + +opcode("vec16", 16, tuint, + [1] * 16, [tuint] * 16, + False, "", """ +dst.x = src0.x; +dst.y = src1.x; +dst.z = src2.x; +dst.w = src3.x; +dst.e = src4.x; +dst.f = src5.x; +dst.g = src6.x; +dst.h = src7.x; +dst.i = src8.x; +dst.j = src9.x; +dst.k = src10.x; +dst.l = src11.x; +dst.m = src12.x; +dst.n = src13.x; +dst.o = src14.x; +dst.p = src15.x; +""") + +# An integer multiply instruction for address calculation. This is +# similar to imul, except that the results are undefined in case of +# overflow. Overflow is defined according to the size of the variable +# being dereferenced. +# +# This relaxed definition, compared to imul, allows an optimization +# pass to propagate bounds (ie, from an load/store intrinsic) to the +# sources, such that lower precision integer multiplies can be used. +# This is useful on hw that has 24b or perhaps 16b integer multiply +# instructions. +binop("amul", tint, _2src_commutative + associative, "src0 * src1") + +# ir3-specific instruction that maps directly to mul-add shift high mix, +# (IMADSH_MIX16 i.e. ah * bl << 16 + c). It is used for lowering integer +# multiplication (imul) on Freedreno backend.. +opcode("imadsh_mix16", 0, tint32, + [0, 0, 0], [tint32, tint32, tint32], False, "", """ +dst = ((((src0 & 0xffff0000) >> 16) * (src1 & 0x0000ffff)) << 16) + src2; +""") + +# ir3-specific instruction that maps directly to ir3 mad.s24. +# +# 24b multiply into 32b result (with sign extension) plus 32b int +triop("imad24_ir3", tint32, _2src_commutative, + "(((int32_t)src0 << 8) >> 8) * (((int32_t)src1 << 8) >> 8) + src2") + +# 24b multiply into 32b result (with sign extension) +binop("imul24", tint32, _2src_commutative + associative, + "(((int32_t)src0 << 8) >> 8) * (((int32_t)src1 << 8) >> 8)") + +# unsigned 24b multiply into 32b result plus 32b int +triop("umad24", tuint32, _2src_commutative, + "(((uint32_t)src0 << 8) >> 8) * (((uint32_t)src1 << 8) >> 8) + src2") + +# unsigned 24b multiply into 32b result uint +binop("umul24", tint32, _2src_commutative + associative, + "(((uint32_t)src0 << 8) >> 8) * (((uint32_t)src1 << 8) >> 8)") +unop_convert("fisnormal", tbool1, tfloat, "isnormal(src0)") +unop_convert("fisfinite", tbool1, tfloat, "isfinite(src0)")