assert m is not None, 'Invalid NIR type string: "{}"'.format(type_)
return m.group('type')
-commutative = "commutative "
+# Operation where the first two sources are commutative.
+#
+# For 2-source operations, this just mathematical commutativity. Some
+# 3-source operations, like ffma, are only commutative in the first two
+# sources.
+_2src_commutative = "2src_commutative "
associative = "associative "
# global dictionary of opcodes
def unop_numeric_convert(name, out_type, in_type, const_expr):
opcode(name, 0, out_type, [0], [in_type], True, "", const_expr)
-# These two move instructions differ in what modifiers they support and what
-# the negate modifier means. Otherwise, they are identical.
-unop("fmov", tfloat, "src0")
-unop("imov", tint, "src0")
+unop("mov", tuint, "src0")
unop("ineg", tint, "-src0")
unop("fneg", tfloat, "-src0")
float absX = fabs(src0.x);
float absY = fabs(src0.y);
float absZ = fabs(src0.z);
-if (src0.x >= 0 && absX >= absY && absX >= absZ) { dst.x = -src0.y; dst.y = -src0.z; }
-if (src0.x < 0 && absX >= absY && absX >= absZ) { dst.x = -src0.y; dst.y = src0.z; }
-if (src0.y >= 0 && absY >= absX && absY >= absZ) { dst.x = src0.z; dst.y = src0.x; }
-if (src0.y < 0 && absY >= absX && absY >= absZ) { dst.x = -src0.z; dst.y = src0.x; }
-if (src0.z >= 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.y; dst.y = src0.x; }
-if (src0.z < 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.y; dst.y = -src0.x; }
+
+float ma = 0.0;
+if (absX >= absY && absX >= absZ) { ma = 2 * src0.x; }
+if (absY >= absX && absY >= absZ) { ma = 2 * src0.y; }
+if (absZ >= absX && absZ >= absY) { ma = 2 * src0.z; }
+
+if (src0.x >= 0 && absX >= absY && absX >= absZ) { dst.x = -src0.z; dst.y = -src0.y; }
+if (src0.x < 0 && absX >= absY && absX >= absZ) { dst.x = src0.z; dst.y = -src0.y; }
+if (src0.y >= 0 && absY >= absX && absY >= absZ) { dst.x = src0.x; dst.y = src0.z; }
+if (src0.y < 0 && absY >= absX && absY >= absZ) { dst.x = src0.x; dst.y = -src0.z; }
+if (src0.z >= 0 && absZ >= absX && absZ >= absY) { dst.x = src0.x; dst.y = -src0.y; }
+if (src0.z < 0 && absZ >= absX && absZ >= absY) { dst.x = -src0.x; dst.y = -src0.y; }
+
+dst.x = dst.x / ma + 0.5;
+dst.y = dst.y / ma + 0.5;
""")
unop_horiz("cube_face_index", 1, tfloat32, 3, tfloat32, """
src2 = prereduce("src0.z", "src1.z")
src3 = prereduce("src0.w", "src1.w")
opcode(name + "2", output_size, output_type,
- [2, 2], [src_type, src_type], False, commutative,
+ [2, 2], [src_type, src_type], False, _2src_commutative,
final(reduce_(src0, src1)))
opcode(name + "3", output_size, output_type,
- [3, 3], [src_type, src_type], False, commutative,
+ [3, 3], [src_type, src_type], False, _2src_commutative,
final(reduce_(reduce_(src0, src1), src2)))
opcode(name + "4", output_size, output_type,
- [4, 4], [src_type, src_type], False, commutative,
+ [4, 4], [src_type, src_type], False, _2src_commutative,
final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
-binop("fadd", tfloat, commutative + associative, "src0 + src1")
-binop("iadd", tint, commutative + associative, "src0 + src1")
-binop("iadd_sat", tint, commutative + associative, """
+binop("fadd", tfloat, _2src_commutative + associative, "src0 + src1")
+binop("iadd", tint, _2src_commutative + associative, "src0 + src1")
+binop("iadd_sat", tint, _2src_commutative, """
src1 > 0 ?
(src0 + src1 < src0 ? (1ull << (bit_size - 1)) - 1 : src0 + src1) :
(src0 < src0 + src1 ? (1ull << (bit_size - 1)) : src0 + src1)
""")
-binop("uadd_sat", tuint, commutative,
- "(src0 + src1) < src0 ? UINT64_MAX : (src0 + src1)")
+binop("uadd_sat", tuint, _2src_commutative,
+ "(src0 + src1) < src0 ? MAX_UINT_FOR_SIZE(sizeof(src0) * 8) : (src0 + src1)")
binop("isub_sat", tint, "", """
src1 < 0 ?
(src0 - src1 < src0 ? (1ull << (bit_size - 1)) - 1 : src0 - src1) :
binop("fsub", tfloat, "", "src0 - src1")
binop("isub", tint, "", "src0 - src1")
-binop("fmul", tfloat, commutative + associative, "src0 * src1")
+binop("fmul", tfloat, _2src_commutative + associative, "src0 * src1")
# low 32-bits of signed/unsigned integer multiply
-binop("imul", tint, commutative + associative, "src0 * src1")
+binop("imul", tint, _2src_commutative + associative, "src0 * src1")
# Generate 64 bit result from 2 32 bits quantity
-binop_convert("imul_2x32_64", tint64, tint32, commutative,
+binop_convert("imul_2x32_64", tint64, tint32, _2src_commutative,
"(int64_t)src0 * (int64_t)src1")
-binop_convert("umul_2x32_64", tuint64, tuint32, commutative,
+binop_convert("umul_2x32_64", tuint64, tuint32, _2src_commutative,
"(uint64_t)src0 * (uint64_t)src1")
# high 32-bits of signed integer multiply
-binop("imul_high", tint, commutative, """
+binop("imul_high", tint, _2src_commutative, """
if (bit_size == 64) {
/* We need to do a full 128-bit x 128-bit multiply in order for the sign
* extension to work properly. The casts are kind-of annoying but needed
""")
# high 32-bits of unsigned integer multiply
-binop("umul_high", tuint, commutative, """
+binop("umul_high", tuint, _2src_commutative, """
if (bit_size == 64) {
/* The casts are kind-of annoying but needed to prevent compiler warnings. */
uint32_t src0_u32[2] = { src0, (uint64_t)src0 >> 32 };
}
""")
+# low 32-bits of unsigned integer multiply
+binop("umul_low", tuint32, _2src_commutative, """
+uint64_t mask = (1 << (bit_size / 2)) - 1;
+dst = ((uint64_t)src0 & mask) * ((uint64_t)src1 & mask);
+""")
+
+
binop("fdiv", tfloat, "", "src0 / src1")
binop("idiv", tint, "", "src1 == 0 ? 0 : (src0 / src1)")
binop("udiv", tuint, "", "src1 == 0 ? 0 : (src0 / src1)")
# returns a boolean representing the carry resulting from the addition of
# the two unsigned arguments.
-binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0")
+binop_convert("uadd_carry", tuint, tuint, _2src_commutative, "src0 + src1 < src0")
# returns a boolean representing the borrow resulting from the subtraction
# of the two unsigned arguments.
#
# (x + y) >> 1 = (((x & y) << 1) + (x ^ y)) >> 1
# = (x & y) + ((x ^ y) >> 1)
-binop("ihadd", tint, commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
-binop("uhadd", tuint, commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
+binop("ihadd", tint, _2src_commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
+binop("uhadd", tuint, _2src_commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
# rhadd: (a + b + 1) >> 1 (without overflow)
# x + y + 1 = x + (~x & y) - (~x & y) + y + (x & ~y) - (x & ~y) + 1
#
# (x + y + 1) >> 1 = (x | y) + (-(x ^ y) + 1) >> 1)
# = (x | y) - ((x ^ y) >> 1)
-binop("irhadd", tint, commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
-binop("urhadd", tuint, commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
+binop("irhadd", tint, _2src_commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
+binop("urhadd", tuint, _2src_commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1")
binop_compare("flt", tfloat, "", "src0 < src1")
binop_compare("fge", tfloat, "", "src0 >= src1")
-binop_compare("feq", tfloat, commutative, "src0 == src1")
-binop_compare("fne", tfloat, commutative, "src0 != src1")
+binop_compare("feq", tfloat, _2src_commutative, "src0 == src1")
+binop_compare("fne", tfloat, _2src_commutative, "src0 != src1")
binop_compare("ilt", tint, "", "src0 < src1")
binop_compare("ige", tint, "", "src0 >= src1")
-binop_compare("ieq", tint, commutative, "src0 == src1")
-binop_compare("ine", tint, commutative, "src0 != src1")
+binop_compare("ieq", tint, _2src_commutative, "src0 == src1")
+binop_compare("ine", tint, _2src_commutative, "src0 != src1")
binop_compare("ult", tuint, "", "src0 < src1")
binop_compare("uge", tuint, "", "src0 >= src1")
binop_compare32("flt32", tfloat, "", "src0 < src1")
binop_compare32("fge32", tfloat, "", "src0 >= src1")
-binop_compare32("feq32", tfloat, commutative, "src0 == src1")
-binop_compare32("fne32", tfloat, commutative, "src0 != src1")
+binop_compare32("feq32", tfloat, _2src_commutative, "src0 == src1")
+binop_compare32("fne32", tfloat, _2src_commutative, "src0 != src1")
binop_compare32("ilt32", tint, "", "src0 < src1")
binop_compare32("ige32", tint, "", "src0 >= src1")
-binop_compare32("ieq32", tint, commutative, "src0 == src1")
-binop_compare32("ine32", tint, commutative, "src0 != src1")
+binop_compare32("ieq32", tint, _2src_commutative, "src0 == src1")
+binop_compare32("ine32", tint, _2src_commutative, "src0 != src1")
binop_compare32("ult32", tuint, "", "src0 < src1")
binop_compare32("uge32", tuint, "", "src0 >= src1")
binop("slt", tfloat32, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than
binop("sge", tfloat, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal
-binop("seq", tfloat32, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
-binop("sne", tfloat32, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
+binop("seq", tfloat32, _2src_commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
+binop("sne", tfloat32, _2src_commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
# SPIRV shifts are undefined for shift-operands >= bitsize,
# but SM5 shifts are defined to use the least significant bits, only
# integers.
-binop("iand", tuint, commutative + associative, "src0 & src1")
-binop("ior", tuint, commutative + associative, "src0 | src1")
-binop("ixor", tuint, commutative + associative, "src0 ^ src1")
+binop("iand", tuint, _2src_commutative + associative, "src0 & src1")
+binop("ior", tuint, _2src_commutative + associative, "src0 | src1")
+binop("ixor", tuint, _2src_commutative + associative, "src0 ^ src1")
# floating point logic operators
# These use (src != 0.0) for testing the truth of the input, and output 1.0
# for true and 0.0 for false
-binop("fand", tfloat32, commutative,
+binop("fand", tfloat32, _2src_commutative,
"((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f")
-binop("for", tfloat32, commutative,
+binop("for", tfloat32, _2src_commutative,
"((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f")
-binop("fxor", tfloat32, commutative,
+binop("fxor", tfloat32, _2src_commutative,
"(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f")
binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}",
"src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
binop("fmin", tfloat, "", "fminf(src0, src1)")
-binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1")
-binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1")
+binop("imin", tint, _2src_commutative + associative, "src1 > src0 ? src0 : src1")
+binop("umin", tuint, _2src_commutative + associative, "src1 > src0 ? src0 : src1")
binop("fmax", tfloat, "", "fmaxf(src0, src1)")
-binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0")
-binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0")
+binop("imax", tint, _2src_commutative + associative, "src1 > src0 ? src1 : src0")
+binop("umax", tuint, _2src_commutative + associative, "src1 > src0 ? src1 : src0")
# Saturated vector add for 4 8bit ints.
-binop("usadd_4x8", tint32, commutative + associative, """
+binop("usadd_4x8", tint32, _2src_commutative + associative, """
dst = 0;
for (int i = 0; i < 32; i += 8) {
dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i;
""")
# vector min for 4 8bit ints.
-binop("umin_4x8", tint32, commutative + associative, """
+binop("umin_4x8", tint32, _2src_commutative + associative, """
dst = 0;
for (int i = 0; i < 32; i += 8) {
dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
""")
# vector max for 4 8bit ints.
-binop("umax_4x8", tint32, commutative + associative, """
+binop("umax_4x8", tint32, _2src_commutative + associative, """
dst = 0;
for (int i = 0; i < 32; i += 8) {
dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
""")
# unorm multiply: (a * b) / 255.
-binop("umul_unorm_4x8", tint32, commutative + associative, """
+binop("umul_unorm_4x8", tint32, _2src_commutative + associative, """
dst = 0;
for (int i = 0; i < 32; i += 8) {
int src0_chan = (src0 >> i) & 0xff;
binop("extract_i16", tint, "", "(int16_t)(src0 >> (src1 * 16))")
-def triop(name, ty, const_expr):
- opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], False, "", const_expr)
+def triop(name, ty, alg_props, const_expr):
+ opcode(name, 0, ty, [0, 0, 0], [ty, ty, ty], False, alg_props, const_expr)
def triop_horiz(name, output_size, src1_size, src2_size, src3_size, const_expr):
opcode(name, output_size, tuint,
[src1_size, src2_size, src3_size],
[tuint, tuint, tuint], False, "", const_expr)
-triop("ffma", tfloat, "src0 * src1 + src2")
+triop("ffma", tfloat, _2src_commutative, "src0 * src1 + src2")
-triop("flrp", tfloat, "src0 * (1 - src2) + src1 * src2")
+triop("flrp", tfloat, "", "src0 * (1 - src2) + src1 * src2")
# Conditional Select
#
# bools (0.0 vs 1.0) and one for integer bools (0 vs ~0).
-triop("fcsel", tfloat32, "(src0 != 0.0f) ? src1 : src2")
+triop("fcsel", tfloat32, "", "(src0 != 0.0f) ? src1 : src2")
# 3 way min/max/med
-triop("fmin3", tfloat, "fminf(src0, fminf(src1, src2))")
-triop("imin3", tint, "MIN2(src0, MIN2(src1, src2))")
-triop("umin3", tuint, "MIN2(src0, MIN2(src1, src2))")
+triop("fmin3", tfloat, "", "fminf(src0, fminf(src1, src2))")
+triop("imin3", tint, "", "MIN2(src0, MIN2(src1, src2))")
+triop("umin3", tuint, "", "MIN2(src0, MIN2(src1, src2))")
-triop("fmax3", tfloat, "fmaxf(src0, fmaxf(src1, src2))")
-triop("imax3", tint, "MAX2(src0, MAX2(src1, src2))")
-triop("umax3", tuint, "MAX2(src0, MAX2(src1, src2))")
+triop("fmax3", tfloat, "", "fmaxf(src0, fmaxf(src1, src2))")
+triop("imax3", tint, "", "MAX2(src0, MAX2(src1, src2))")
+triop("umax3", tuint, "", "MAX2(src0, MAX2(src1, src2))")
-triop("fmed3", tfloat, "fmaxf(fminf(fmaxf(src0, src1), src2), fminf(src0, src1))")
-triop("imed3", tint, "MAX2(MIN2(MAX2(src0, src1), src2), MIN2(src0, src1))")
-triop("umed3", tuint, "MAX2(MIN2(MAX2(src0, src1), src2), MIN2(src0, src1))")
+triop("fmed3", tfloat, "", "fmaxf(fminf(fmaxf(src0, src1), src2), fminf(src0, src1))")
+triop("imed3", tint, "", "MAX2(MIN2(MAX2(src0, src1), src2), MIN2(src0, src1))")
+triop("umed3", tuint, "", "MAX2(MIN2(MAX2(src0, src1), src2), MIN2(src0, src1))")
opcode("bcsel", 0, tuint, [0, 0, 0],
[tbool1, tuint, tuint], False, "", "src0 ? src1 : src2")
[tbool32, tuint, tuint], False, "", "src0 ? src1 : src2")
# SM5 bfi assembly
-triop("bfi", tuint32, """
+triop("bfi", tuint32, "", """
unsigned mask = src0, insert = src1, base = src2;
if (mask == 0) {
dst = base;
dst.w = src3.x;
""")
-
+# ir3-specific instruction that maps directly to mul-add shift high mix,
+# (IMADSH_MIX16 i.e. ah * bl << 16 + c). It is used for lowering integer
+# multiplication (imul) on Freedreno backend..
+opcode("imadsh_mix16", 1, tint32,
+ [1, 1, 1], [tint32, tint32, tint32], False, "", """
+dst.x = ((((src0.x & 0xffff0000) >> 16) * (src1.x & 0x0000ffff)) << 16) + src2.x;
+""")
projects / mesa.git / blobdiff