*/
-#include "util/u_cpu_detect.h"
#include "util/u_format.h"
+#include "util/u_memory.h"
+#include "util/u_math.h"
+#include "util/u_string.h"
+#include "lp_bld_arit.h"
+#include "lp_bld_init.h"
#include "lp_bld_type.h"
+#include "lp_bld_flow.h"
#include "lp_bld_const.h"
+#include "lp_bld_conv.h"
#include "lp_bld_swizzle.h"
+#include "lp_bld_gather.h"
#include "lp_bld_format.h"
+/**
+ * Basic swizzling. Rearrange the order of the unswizzled array elements
+ * according to the format description. PIPE_SWIZZLE_ZERO/ONE are supported
+ * too.
+ * Ex: if unswizzled[4] = {B, G, R, x}, then swizzled_out[4] = {R, G, B, 1}.
+ */
+LLVMValueRef
+lp_build_format_swizzle_aos(const struct util_format_description *desc,
+ struct lp_build_context *bld,
+ LLVMValueRef unswizzled)
+{
+ unsigned char swizzles[4];
+ unsigned chan;
+
+ assert(bld->type.length % 4 == 0);
+
+ for (chan = 0; chan < 4; ++chan) {
+ enum util_format_swizzle swizzle;
+
+ if (desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) {
+ /*
+ * For ZS formats do RGBA = ZZZ1
+ */
+ if (chan == 3) {
+ swizzle = UTIL_FORMAT_SWIZZLE_1;
+ } else if (desc->swizzle[0] == UTIL_FORMAT_SWIZZLE_NONE) {
+ swizzle = UTIL_FORMAT_SWIZZLE_0;
+ } else {
+ swizzle = desc->swizzle[0];
+ }
+ } else {
+ swizzle = desc->swizzle[chan];
+ }
+ swizzles[chan] = swizzle;
+ }
+
+ return lp_build_swizzle_aos(bld, unswizzled, swizzles);
+}
+
+
+/**
+ * Whether the format matches the vector type, apart of swizzles.
+ */
+static INLINE boolean
+format_matches_type(const struct util_format_description *desc,
+ struct lp_type type)
+{
+ enum util_format_type chan_type;
+ unsigned chan;
+
+ assert(type.length % 4 == 0);
+
+ if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN ||
+ desc->colorspace != UTIL_FORMAT_COLORSPACE_RGB ||
+ desc->block.width != 1 ||
+ desc->block.height != 1) {
+ return FALSE;
+ }
+
+ if (type.floating) {
+ chan_type = UTIL_FORMAT_TYPE_FLOAT;
+ } else if (type.fixed) {
+ chan_type = UTIL_FORMAT_TYPE_FIXED;
+ } else if (type.sign) {
+ chan_type = UTIL_FORMAT_TYPE_SIGNED;
+ } else {
+ chan_type = UTIL_FORMAT_TYPE_UNSIGNED;
+ }
+
+ for (chan = 0; chan < desc->nr_channels; ++chan) {
+ if (desc->channel[chan].size != type.width) {
+ return FALSE;
+ }
+
+ if (desc->channel[chan].type != UTIL_FORMAT_TYPE_VOID) {
+ if (desc->channel[chan].type != chan_type ||
+ desc->channel[chan].normalized != type.norm) {
+ return FALSE;
+ }
+ }
+ }
+
+ return TRUE;
+}
+
+
/**
* Unpack a single pixel into its RGBA components.
*
- * @param packed integer.
+ * @param desc the pixel format for the packed pixel value
+ * @param packed integer pixel in a format such as PIPE_FORMAT_B8G8R8A8_UNORM
*
- * @return RGBA in a 4 floats vector.
- *
- * XXX: This is mostly for reference and testing -- operating a single pixel at
- * a time is rarely if ever needed.
+ * @return RGBA in a float[4] or ubyte[4] or ushort[4] vector.
*/
-LLVMValueRef
-lp_build_unpack_rgba_aos(LLVMBuilderRef builder,
- const struct util_format_description *desc,
- LLVMValueRef packed)
+static INLINE LLVMValueRef
+lp_build_unpack_arith_rgba_aos(LLVMBuilderRef builder,
+ const struct util_format_description *desc,
+ LLVMValueRef packed)
{
- LLVMTypeRef type;
LLVMValueRef shifted, casted, scaled, masked;
LLVMValueRef shifts[4];
LLVMValueRef masks[4];
LLVMValueRef scales[4];
- LLVMValueRef swizzles[4];
- LLVMValueRef aux[4];
- bool normalized;
- int empty_channel;
+
+ boolean normalized;
+ boolean needs_uitofp;
unsigned shift;
unsigned i;
- /* FIXME: Support more formats */
+ /* TODO: Support more formats */
assert(desc->layout == UTIL_FORMAT_LAYOUT_PLAIN);
assert(desc->block.width == 1);
assert(desc->block.height == 1);
assert(desc->block.bits <= 32);
- type = LLVMIntType(desc->block.bits);
-
/* Do the intermediate integer computations with 32bit integers since it
* matches floating point size */
- if (desc->block.bits < 32)
- packed = LLVMBuildZExt(builder, packed, LLVMInt32Type(), "");
+ assert (LLVMTypeOf(packed) == LLVMInt32Type());
- /* Broadcast the packed value to all four channels */
+ /* Broadcast the packed value to all four channels
+ * before: packed = BGRA
+ * after: packed = {BGRA, BGRA, BGRA, BGRA}
+ */
packed = LLVMBuildInsertElement(builder,
LLVMGetUndef(LLVMVectorType(LLVMInt32Type(), 4)),
packed,
/* Initialize vector constants */
normalized = FALSE;
- empty_channel = -1;
+ needs_uitofp = FALSE;
shift = 0;
+
+ /* Loop over 4 color components */
for (i = 0; i < 4; ++i) {
unsigned bits = desc->channel[i].size;
shifts[i] = LLVMGetUndef(LLVMInt32Type());
masks[i] = LLVMConstNull(LLVMInt32Type());
scales[i] = LLVMConstNull(LLVMFloatType());
- empty_channel = i;
}
else {
- unsigned mask = (1 << bits) - 1;
+ unsigned long long mask = (1ULL << bits) - 1;
assert(desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED);
- assert(bits < 32);
+
+ if (bits == 32) {
+ needs_uitofp = TRUE;
+ }
shifts[i] = LLVMConstInt(LLVMInt32Type(), shift, 0);
masks[i] = LLVMConstInt(LLVMInt32Type(), mask, 0);
shift += bits;
}
+ /* Ex: convert packed = {BGRA, BGRA, BGRA, BGRA}
+ * into masked = {B, G, R, A}
+ */
shifted = LLVMBuildLShr(builder, packed, LLVMConstVector(shifts, 4), "");
masked = LLVMBuildAnd(builder, shifted, LLVMConstVector(masks, 4), "");
- /* UIToFP can't be expressed in SSE2 */
- casted = LLVMBuildSIToFP(builder, masked, LLVMVectorType(LLVMFloatType(), 4), "");
- if (normalized)
- scaled = LLVMBuildMul(builder, casted, LLVMConstVector(scales, 4), "");
- else
- scaled = casted;
- for (i = 0; i < 4; ++i)
- aux[i] = LLVMGetUndef(LLVMFloatType());
-
- for (i = 0; i < 4; ++i) {
- enum util_format_swizzle swizzle = desc->swizzle[i];
-
- switch (swizzle) {
- case UTIL_FORMAT_SWIZZLE_X:
- case UTIL_FORMAT_SWIZZLE_Y:
- case UTIL_FORMAT_SWIZZLE_Z:
- case UTIL_FORMAT_SWIZZLE_W:
- swizzles[i] = LLVMConstInt(LLVMInt32Type(), swizzle, 0);
- break;
- case UTIL_FORMAT_SWIZZLE_0:
- assert(empty_channel >= 0);
- swizzles[i] = LLVMConstInt(LLVMInt32Type(), empty_channel, 0);
- break;
- case UTIL_FORMAT_SWIZZLE_1:
- swizzles[i] = LLVMConstInt(LLVMInt32Type(), 4, 0);
- aux[0] = LLVMConstReal(LLVMFloatType(), 1.0);
- break;
- case UTIL_FORMAT_SWIZZLE_NONE:
- swizzles[i] = LLVMGetUndef(LLVMFloatType());
- assert(0);
- break;
- }
+ if (!needs_uitofp) {
+ /* UIToFP can't be expressed in SSE2 */
+ casted = LLVMBuildSIToFP(builder, masked, LLVMVectorType(LLVMFloatType(), 4), "");
+ } else {
+ casted = LLVMBuildUIToFP(builder, masked, LLVMVectorType(LLVMFloatType(), 4), "");
}
- return LLVMBuildShuffleVector(builder, scaled, LLVMConstVector(aux, 4), LLVMConstVector(swizzles, 4), "");
-}
-
-
-/**
- * Take a vector with packed pixels and unpack into a rgba8 vector.
- *
- * Formats with bit depth smaller than 32bits are accepted, but they must be
- * padded to 32bits.
- */
-LLVMValueRef
-lp_build_unpack_rgba8_aos(LLVMBuilderRef builder,
- const struct util_format_description *desc,
- struct lp_type type,
- LLVMValueRef packed)
-{
- struct lp_build_context bld;
- bool rgba8;
- LLVMValueRef res;
- unsigned i;
-
- lp_build_context_init(&bld, builder, type);
-
- /* FIXME: Support more formats */
- assert(desc->layout == UTIL_FORMAT_LAYOUT_PLAIN);
- assert(desc->block.width == 1);
- assert(desc->block.height == 1);
- assert(desc->block.bits <= 32);
+ /* At this point 'casted' may be a vector of floats such as
+ * {255.0, 255.0, 255.0, 255.0}. Next, if the pixel values are normalized
+ * we'll scale this to {1.0, 1.0, 1.0, 1.0}.
+ */
- assert(!type.floating);
- assert(!type.fixed);
- assert(type.norm);
- assert(type.width == 8);
- assert(type.length % 4 == 0);
-
- rgba8 = TRUE;
- for(i = 0; i < 4; ++i) {
- assert(desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED ||
- desc->channel[i].type == UTIL_FORMAT_TYPE_VOID);
- if(desc->channel[0].size != 8)
- rgba8 = FALSE;
- }
-
- if(rgba8) {
- /*
- * The pixel is already in a rgba8 format variant. All it is necessary
- * is to swizzle the channels.
- */
-
- unsigned char swizzles[4];
- boolean zeros[4]; /* bitwise AND mask */
- boolean ones[4]; /* bitwise OR mask */
- boolean swizzles_needed = FALSE;
- boolean zeros_needed = FALSE;
- boolean ones_needed = FALSE;
-
- for(i = 0; i < 4; ++i) {
- enum util_format_swizzle swizzle = desc->swizzle[i];
-
- /* Initialize with the no-op case */
- swizzles[i] = util_cpu_caps.little_endian ? 3 - i : i;
- zeros[i] = TRUE;
- ones[i] = FALSE;
-
- switch (swizzle) {
- case UTIL_FORMAT_SWIZZLE_X:
- case UTIL_FORMAT_SWIZZLE_Y:
- case UTIL_FORMAT_SWIZZLE_Z:
- case UTIL_FORMAT_SWIZZLE_W:
- if(swizzle != swizzles[i]) {
- swizzles[i] = swizzle;
- swizzles_needed = TRUE;
- }
- break;
- case UTIL_FORMAT_SWIZZLE_0:
- zeros[i] = FALSE;
- zeros_needed = TRUE;
- break;
- case UTIL_FORMAT_SWIZZLE_1:
- ones[i] = TRUE;
- ones_needed = TRUE;
- break;
- case UTIL_FORMAT_SWIZZLE_NONE:
- assert(0);
- break;
- }
- }
-
- res = packed;
-
- if(swizzles_needed)
- res = lp_build_swizzle1_aos(&bld, res, swizzles);
-
- if(zeros_needed) {
- /* Mask out zero channels */
- LLVMValueRef mask = lp_build_const_mask_aos(type, zeros);
- res = LLVMBuildAnd(builder, res, mask, "");
- }
-
- if(ones_needed) {
- /* Or one channels */
- LLVMValueRef mask = lp_build_const_mask_aos(type, ones);
- res = LLVMBuildOr(builder, res, mask, "");
- }
- }
- else {
- /* FIXME */
- assert(0);
- res = lp_build_undef(type);
- }
+ if (normalized)
+ scaled = LLVMBuildMul(builder, casted, LLVMConstVector(scales, 4), "");
+ else
+ scaled = casted;
- return res;
+ return scaled;
}
LLVMValueRef shifted, casted, scaled, unswizzled;
LLVMValueRef shifts[4];
LLVMValueRef scales[4];
- bool normalized;
+ boolean normalized;
unsigned shift;
unsigned i, j;
return packed;
}
+
+
+
+
+/**
+ * Fetch a pixel into a 4 float AoS.
+ *
+ * \param format_desc describes format of the image we're fetching from
+ * \param ptr address of the pixel block (or the texel if uncompressed)
+ * \param i, j the sub-block pixel coordinates. For non-compressed formats
+ * these will always be (0, 0).
+ * \return a 4 element vector with the pixel's RGBA values.
+ */
+LLVMValueRef
+lp_build_fetch_rgba_aos(LLVMBuilderRef builder,
+ const struct util_format_description *format_desc,
+ struct lp_type type,
+ LLVMValueRef base_ptr,
+ LLVMValueRef offset,
+ LLVMValueRef i,
+ LLVMValueRef j)
+{
+ unsigned num_pixels = type.length / 4;
+ struct lp_build_context bld;
+
+ assert(type.length <= LP_MAX_VECTOR_LENGTH);
+ assert(type.length % 4 == 0);
+
+ lp_build_context_init(&bld, builder, type);
+
+ /*
+ * Trivial case
+ *
+ * The format matches the type (apart of a swizzle) so no need for
+ * scaling or converting.
+ */
+
+ if (format_matches_type(format_desc, type) &&
+ format_desc->block.bits <= type.width * 4 &&
+ util_is_pot(format_desc->block.bits)) {
+ LLVMValueRef packed;
+
+ /*
+ * The format matches the type (apart of a swizzle) so no need for
+ * scaling or converting.
+ */
+
+ packed = lp_build_gather(builder, type.length/4,
+ format_desc->block.bits, type.width*4,
+ base_ptr, offset);
+
+ assert(format_desc->block.bits <= type.width * type.length);
+
+ packed = LLVMBuildBitCast(builder, packed, lp_build_vec_type(type), "");
+
+ return lp_build_format_swizzle_aos(format_desc, &bld, packed);
+ }
+
+ /*
+ * Bit arithmetic
+ */
+
+ if (format_desc->layout == UTIL_FORMAT_LAYOUT_PLAIN &&
+ (format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
+ format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS) &&
+ format_desc->block.width == 1 &&
+ format_desc->block.height == 1 &&
+ util_is_pot(format_desc->block.bits) &&
+ format_desc->block.bits <= 32 &&
+ format_desc->is_bitmask &&
+ !format_desc->is_mixed &&
+ (format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED ||
+ format_desc->channel[1].type == UTIL_FORMAT_TYPE_UNSIGNED)) {
+
+ LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
+ LLVMValueRef res;
+ unsigned k;
+
+ /*
+ * Unpack a pixel at a time into a <4 x float> RGBA vector
+ */
+
+ for (k = 0; k < num_pixels; ++k) {
+ LLVMValueRef packed;
+
+ packed = lp_build_gather_elem(builder, num_pixels,
+ format_desc->block.bits, 32,
+ base_ptr, offset, k);
+
+ tmps[k] = lp_build_unpack_arith_rgba_aos(builder, format_desc,
+ packed);
+ }
+
+ /*
+ * Type conversion.
+ *
+ * TODO: We could avoid floating conversion for integer to
+ * integer conversions.
+ */
+
+ lp_build_conv(builder,
+ lp_float32_vec4_type(),
+ type,
+ tmps, num_pixels, &res, 1);
+
+ return lp_build_format_swizzle_aos(format_desc, &bld, res);
+ }
+
+ /*
+ * YUV / subsampled formats
+ */
+
+ if (format_desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED) {
+ struct lp_type tmp_type;
+ LLVMValueRef tmp;
+
+ memset(&tmp_type, 0, sizeof tmp_type);
+ tmp_type.width = 8;
+ tmp_type.length = num_pixels * 4;
+ tmp_type.norm = TRUE;
+
+ tmp = lp_build_fetch_subsampled_rgba_aos(builder,
+ format_desc,
+ num_pixels,
+ base_ptr,
+ offset,
+ i, j);
+
+ lp_build_conv(builder,
+ tmp_type, type,
+ &tmp, 1, &tmp, 1);
+
+ return tmp;
+ }
+
+ /*
+ * Fallback to util_format_description::fetch_rgba_8unorm().
+ */
+
+ if (format_desc->fetch_rgba_8unorm &&
+ !type.floating && type.width == 8 && !type.sign && type.norm) {
+ /*
+ * Fallback to calling util_format_description::fetch_rgba_8unorm.
+ *
+ * This is definitely not the most efficient way of fetching pixels, as
+ * we miss the opportunity to do vectorization, but this it is a
+ * convenient for formats or scenarios for which there was no opportunity
+ * or incentive to optimize.
+ */
+
+ LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
+ char name[256];
+ LLVMTypeRef i8t = LLVMInt8Type();
+ LLVMTypeRef pi8t = LLVMPointerType(i8t, 0);
+ LLVMTypeRef i32t = LLVMInt32Type();
+ LLVMValueRef function;
+ LLVMValueRef tmp_ptr;
+ LLVMValueRef tmp;
+ LLVMValueRef res;
+ unsigned k;
+
+ util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_8unorm",
+ format_desc->short_name);
+
+ /*
+ * Declare and bind format_desc->fetch_rgba_8unorm().
+ */
+
+ function = LLVMGetNamedFunction(module, name);
+ if (!function) {
+ LLVMTypeRef ret_type;
+ LLVMTypeRef arg_types[4];
+ LLVMTypeRef function_type;
+
+ ret_type = LLVMVoidType();
+ arg_types[0] = pi8t;
+ arg_types[1] = pi8t;
+ arg_types[3] = arg_types[2] = LLVMIntType(sizeof(unsigned) * 8);
+ function_type = LLVMFunctionType(ret_type, arg_types, Elements(arg_types), 0);
+ function = LLVMAddFunction(module, name, function_type);
+
+ LLVMSetFunctionCallConv(function, LLVMCCallConv);
+ LLVMSetLinkage(function, LLVMExternalLinkage);
+
+ assert(LLVMIsDeclaration(function));
+
+ LLVMAddGlobalMapping(lp_build_engine, function,
+ func_to_pointer((func_pointer)format_desc->fetch_rgba_8unorm));
+ }
+
+ tmp_ptr = lp_build_alloca(builder, i32t, "");
+
+ res = LLVMGetUndef(LLVMVectorType(i32t, num_pixels));
+
+ /*
+ * Invoke format_desc->fetch_rgba_8unorm() for each pixel and insert the result
+ * in the SoA vectors.
+ */
+
+ for (k = 0; k < num_pixels; ++k) {
+ LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), k, 0);
+ LLVMValueRef args[4];
+
+ args[0] = LLVMBuildBitCast(builder, tmp_ptr, pi8t, "");
+ args[1] = lp_build_gather_elem_ptr(builder, num_pixels,
+ base_ptr, offset, k);
+
+ if (num_pixels == 1) {
+ args[2] = i;
+ args[3] = j;
+ }
+ else {
+ args[2] = LLVMBuildExtractElement(builder, i, index, "");
+ args[3] = LLVMBuildExtractElement(builder, j, index, "");
+ }
+
+ LLVMBuildCall(builder, function, args, Elements(args), "");
+
+ tmp = LLVMBuildLoad(builder, tmp_ptr, "");
+
+ if (num_pixels == 1) {
+ res = tmp;
+ }
+ else {
+ res = LLVMBuildInsertElement(builder, res, tmp, index, "");
+ }
+ }
+
+ /* Bitcast from <n x i32> to <4n x i8> */
+ res = LLVMBuildBitCast(builder, res, bld.vec_type, "");
+
+ return res;
+ }
+
+
+ /*
+ * Fallback to util_format_description::fetch_rgba_float().
+ */
+
+ if (format_desc->fetch_rgba_float) {
+ /*
+ * Fallback to calling util_format_description::fetch_rgba_float.
+ *
+ * This is definitely not the most efficient way of fetching pixels, as
+ * we miss the opportunity to do vectorization, but this it is a
+ * convenient for formats or scenarios for which there was no opportunity
+ * or incentive to optimize.
+ */
+
+ LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
+ char name[256];
+ LLVMTypeRef f32t = LLVMFloatType();
+ LLVMTypeRef f32x4t = LLVMVectorType(f32t, 4);
+ LLVMTypeRef pf32t = LLVMPointerType(f32t, 0);
+ LLVMValueRef function;
+ LLVMValueRef tmp_ptr;
+ LLVMValueRef tmps[LP_MAX_VECTOR_LENGTH/4];
+ LLVMValueRef res;
+ unsigned k;
+
+ util_snprintf(name, sizeof name, "util_format_%s_fetch_rgba_float",
+ format_desc->short_name);
+
+ /*
+ * Declare and bind format_desc->fetch_rgba_float().
+ */
+
+ function = LLVMGetNamedFunction(module, name);
+ if (!function) {
+ LLVMTypeRef ret_type;
+ LLVMTypeRef arg_types[4];
+ LLVMTypeRef function_type;
+
+ ret_type = LLVMVoidType();
+ arg_types[0] = pf32t;
+ arg_types[1] = LLVMPointerType(LLVMInt8Type(), 0);
+ arg_types[3] = arg_types[2] = LLVMIntType(sizeof(unsigned) * 8);
+ function_type = LLVMFunctionType(ret_type, arg_types, Elements(arg_types), 0);
+ function = LLVMAddFunction(module, name, function_type);
+
+ LLVMSetFunctionCallConv(function, LLVMCCallConv);
+ LLVMSetLinkage(function, LLVMExternalLinkage);
+
+ assert(LLVMIsDeclaration(function));
+
+ LLVMAddGlobalMapping(lp_build_engine, function,
+ func_to_pointer((func_pointer)format_desc->fetch_rgba_float));
+ }
+
+ tmp_ptr = lp_build_alloca(builder, f32x4t, "");
+
+ /*
+ * Invoke format_desc->fetch_rgba_float() for each pixel and insert the result
+ * in the SoA vectors.
+ */
+
+ for (k = 0; k < num_pixels; ++k) {
+ LLVMValueRef args[4];
+
+ args[0] = LLVMBuildBitCast(builder, tmp_ptr, pf32t, "");
+ args[1] = lp_build_gather_elem_ptr(builder, num_pixels,
+ base_ptr, offset, k);
+
+ if (num_pixels == 1) {
+ args[2] = i;
+ args[3] = j;
+ }
+ else {
+ LLVMValueRef index = LLVMConstInt(LLVMInt32Type(), k, 0);
+ args[2] = LLVMBuildExtractElement(builder, i, index, "");
+ args[3] = LLVMBuildExtractElement(builder, j, index, "");
+ }
+
+ LLVMBuildCall(builder, function, args, Elements(args), "");
+
+ tmps[k] = LLVMBuildLoad(builder, tmp_ptr, "");
+ }
+
+ lp_build_conv(builder,
+ lp_float32_vec4_type(),
+ type,
+ tmps, num_pixels, &res, 1);
+
+ return res;
+ }
+
+ assert(0);
+ return lp_build_undef(type);
+}
projects / mesa.git / blobdiff