* USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
+#include <llvm/Config/llvm-config.h>
+
#include "si_shader_internal.h"
#include "si_pipe.h"
#include "sid.h"
switch (target) {
case TGSI_TEXTURE_1D:
case TGSI_TEXTURE_SHADOW1D:
- if (screen->info.chip_class >= GFX9)
+ if (screen->info.chip_class == GFX9)
return ac_image_2d;
return ac_image_1d;
case TGSI_TEXTURE_2D:
return ac_image_cube;
case TGSI_TEXTURE_1D_ARRAY:
case TGSI_TEXTURE_SHADOW1D_ARRAY:
- if (screen->info.chip_class >= GFX9)
+ if (screen->info.chip_class == GFX9)
return ac_image_2darray;
return ac_image_1darray;
case TGSI_TEXTURE_2D_ARRAY:
}
}
+/* AC_DESC_FMASK is handled exactly like AC_DESC_IMAGE. The caller should
+ * adjust "index" to point to FMASK. */
LLVMValueRef si_load_image_desc(struct si_shader_context *ctx,
LLVMValueRef list, LLVMValueRef index,
enum ac_descriptor_type desc_type,
list = LLVMBuildPointerCast(builder, list,
ac_array_in_const32_addr_space(ctx->v4i32), "");
} else {
- assert(desc_type == AC_DESC_IMAGE);
+ assert(desc_type == AC_DESC_IMAGE ||
+ desc_type == AC_DESC_FMASK);
}
if (bindless)
else
rsrc = ac_build_load_to_sgpr(&ctx->ac, list, index);
- if (ctx->ac.chip_class <= GFX9 &&
- desc_type == AC_DESC_IMAGE && uses_store)
+ if (desc_type == AC_DESC_IMAGE && uses_store)
rsrc = force_dcc_off(ctx, rsrc);
return rsrc;
}
image_fetch_rsrc(
struct lp_build_tgsi_context *bld_base,
const struct tgsi_full_src_register *image,
- bool is_store, unsigned target,
+ bool fmask, bool is_store, unsigned target,
LLVMValueRef *rsrc)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef rsrc_ptr = LLVMGetParam(ctx->main_fn,
- ctx->param_samplers_and_images);
- LLVMValueRef index;
-
- if (!image->Register.Indirect) {
- index = LLVMConstInt(ctx->i32,
- si_get_image_slot(image->Register.Index), 0);
- } else {
- /* From the GL_ARB_shader_image_load_store extension spec:
- *
- * If a shader performs an image load, store, or atomic
- * operation using an image variable declared as an array,
- * and if the index used to select an individual element is
- * negative or greater than or equal to the size of the
- * array, the results of the operation are undefined but may
- * not lead to termination.
- */
- index = si_get_bounded_indirect_index(ctx, &image->Indirect,
- image->Register.Index,
- ctx->num_images);
- index = LLVMBuildSub(ctx->ac.builder,
- LLVMConstInt(ctx->i32, SI_NUM_IMAGES - 1, 0),
- index, "");
- }
+ bool bindless = image->Register.File != TGSI_FILE_IMAGE;
+ LLVMValueRef rsrc_ptr, index;
- bool bindless = false;
-
- if (image->Register.File != TGSI_FILE_IMAGE) {
+ if (bindless) {
/* Bindless descriptors are accessible from a different pair of
* user SGPR indices.
*/
- rsrc_ptr = LLVMGetParam(ctx->main_fn,
- ctx->param_bindless_samplers_and_images);
- index = lp_build_emit_fetch_src(bld_base, image,
- TGSI_TYPE_UNSIGNED, 0);
+ rsrc_ptr = ac_get_arg(&ctx->ac,
+ ctx->bindless_samplers_and_images);
+ index = lp_build_emit_fetch_src(bld_base, image, TGSI_TYPE_UNSIGNED, 0);
- /* For simplicity, bindless image descriptors use fixed
- * 16-dword slots for now.
- */
+ /* Bindless image descriptors use 16-dword slots. */
index = LLVMBuildMul(ctx->ac.builder, index,
LLVMConstInt(ctx->i32, 2, 0), "");
- bindless = true;
+ /* FMASK is right after the image. */
+ if (fmask)
+ index = LLVMBuildAdd(ctx->ac.builder, index, ctx->i32_1, "");
+ } else {
+ rsrc_ptr = ac_get_arg(&ctx->ac, ctx->samplers_and_images);
+
+ if (!image->Register.Indirect) {
+ index = LLVMConstInt(ctx->i32, image->Register.Index, 0);
+ } else {
+ /* From the GL_ARB_shader_image_load_store extension spec:
+ *
+ * If a shader performs an image load, store, or atomic
+ * operation using an image variable declared as an array,
+ * and if the index used to select an individual element is
+ * negative or greater than or equal to the size of the
+ * array, the results of the operation are undefined but may
+ * not lead to termination.
+ */
+ index = si_get_bounded_indirect_index(ctx, &image->Indirect,
+ image->Register.Index,
+ ctx->num_images);
+ }
+ /* FMASKs are separate from images. */
+ if (fmask) {
+ index = LLVMBuildAdd(ctx->ac.builder, index,
+ LLVMConstInt(ctx->i32, SI_NUM_IMAGES, 0), "");
+ }
+ index = LLVMBuildSub(ctx->ac.builder,
+ LLVMConstInt(ctx->i32, SI_NUM_IMAGE_SLOTS - 1, 0),
+ index, "");
}
*rsrc = si_load_image_desc(ctx, rsrc_ptr, index,
+ fmask ? AC_DESC_FMASK :
target == TGSI_TEXTURE_BUFFER ? AC_DESC_BUFFER : AC_DESC_IMAGE,
is_store, bindless);
}
LLVMValueRef tmp;
int chan;
- if (target == TGSI_TEXTURE_2D_MSAA ||
- target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
- /* Need the sample index as well. */
- num_coords++;
- }
-
for (chan = 0; chan < num_coords; ++chan) {
tmp = lp_build_emit_fetch(bld_base, inst, src, chan);
tmp = ac_to_integer(&ctx->ac, tmp);
coords[chan] = tmp;
}
- if (ctx->screen->info.chip_class >= GFX9) {
+ if (target == TGSI_TEXTURE_2D_MSAA ||
+ target == TGSI_TEXTURE_2D_ARRAY_MSAA) {
+ /* Need the sample index as well. */
+ tmp = lp_build_emit_fetch(bld_base, inst, src, TGSI_SWIZZLE_W);
+ coords[chan] = ac_to_integer(&ctx->ac, tmp);
+ }
+
+ if (ctx->screen->info.chip_class == GFX9) {
/* 1D textures are allocated and used as 2D on GFX9. */
if (target == TGSI_TEXTURE_1D) {
coords[1] = ctx->i32_0;
} else if (target == TGSI_TEXTURE_1D_ARRAY) {
coords[2] = coords[1];
coords[1] = ctx->i32_0;
- } else if (ctx->screen->info.chip_class == GFX9 &&
- target == TGSI_TEXTURE_2D) {
+ } else if (target == TGSI_TEXTURE_2D) {
/* The hw can't bind a slice of a 3D image as a 2D
* image, because it ignores BASE_ARRAY if the target
* is 3D. The workaround is to read BASE_ARRAY and set
} else {
unsigned target = inst->Memory.Texture;
- image_fetch_rsrc(bld_base, &inst->Src[0], false, target, &args.resource);
+ image_fetch_rsrc(bld_base, &inst->Src[0], false, false, target, &args.resource);
image_fetch_coords(bld_base, inst, 1, args.resource, args.coords);
+
+ if ((inst->Memory.Texture == TGSI_TEXTURE_2D_MSAA ||
+ inst->Memory.Texture == TGSI_TEXTURE_2D_ARRAY_MSAA) &&
+ !(ctx->screen->debug_flags & DBG(NO_FMASK))) {
+ LLVMValueRef fmask;
+
+ image_fetch_rsrc(bld_base, &inst->Src[0], true, false, target, &fmask);
+ ac_apply_fmask_to_sample(&ctx->ac, fmask, args.coords,
+ inst->Memory.Texture == TGSI_TEXTURE_2D_ARRAY_MSAA);
+ }
vindex = args.coords[0]; /* for buffers only */
}
args.resource = shader_buffer_fetch_rsrc(ctx, &resource_reg, false);
voffset = ac_to_integer(&ctx->ac, lp_build_emit_fetch(bld_base, inst, 0, 0));
} else {
- image_fetch_rsrc(bld_base, &resource_reg, true, target, &args.resource);
+ image_fetch_rsrc(bld_base, &resource_reg, false, true, target, &args.resource);
image_fetch_coords(bld_base, inst, 0, args.resource, args.coords);
vindex = args.coords[0]; /* for buffers only */
}
LLVMBuilderRef builder = ctx->ac.builder;
const struct tgsi_full_instruction * inst = emit_data->inst;
LLVMValueRef ptr, result, arg;
- const char *sync_scope = HAVE_LLVM >= 0x0900 ? "workgroup-one-as" : "workgroup";
+ const char *sync_scope = LLVM_VERSION_MAJOR >= 9 ? "workgroup-one-as" : "workgroup";
ptr = get_memory_ptr(ctx, inst, ctx->i32, 1);
args.data[num_data++] =
ac_to_integer(&ctx->ac, lp_build_emit_fetch(bld_base, inst, 2, 0));
+
args.cache_policy = get_cache_policy(ctx, inst, true, false, false);
if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
args.resource = shader_buffer_fetch_rsrc(ctx, &inst->Src[0], false);
voffset = ac_to_integer(&ctx->ac, lp_build_emit_fetch(bld_base, inst, 1, 0));
} else {
- image_fetch_rsrc(bld_base, &inst->Src[0], true,
+ image_fetch_rsrc(bld_base, &inst->Src[0], false, true,
inst->Memory.Texture, &args.resource);
image_fetch_coords(bld_base, inst, 1, args.resource, args.coords);
vindex = args.coords[0]; /* for buffers only */
}
- if (HAVE_LLVM >= 0x0800 &&
- inst->Src[0].Register.File != TGSI_FILE_BUFFER &&
+ if (inst->Src[0].Register.File != TGSI_FILE_BUFFER &&
inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
LLVMValueRef buf_args[7];
unsigned num_args = 0;
return;
}
- if (inst->Src[0].Register.File == TGSI_FILE_BUFFER ||
- (HAVE_LLVM < 0x0800 &&
- inst->Memory.Texture == TGSI_TEXTURE_BUFFER)) {
+ if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) {
LLVMValueRef buf_args[7];
unsigned num_args = 0;
case TGSI_OPCODE_ATOMUMAX: args.atomic = ac_atomic_umax; break;
case TGSI_OPCODE_ATOMIMIN: args.atomic = ac_atomic_smin; break;
case TGSI_OPCODE_ATOMIMAX: args.atomic = ac_atomic_smax; break;
+ case TGSI_OPCODE_ATOMINC_WRAP:
+ args.atomic = ac_atomic_inc_wrap;
+ break;
+ case TGSI_OPCODE_ATOMDEC_WRAP:
+ args.atomic = ac_atomic_dec_wrap;
+ break;
default: unreachable("unhandled image atomic");
}
}
LLVMBuilderRef builder = ctx->ac.builder;
/* 1D textures are allocated and used as 2D on GFX9. */
- if (ctx->screen->info.chip_class >= GFX9 &&
+ if (ctx->screen->info.chip_class == GFX9 &&
(target == TGSI_TEXTURE_1D_ARRAY ||
target == TGSI_TEXTURE_SHADOW1D_ARRAY)) {
LLVMValueRef layers =
inst->Memory.Texture == TGSI_TEXTURE_BUFFER) {
LLVMValueRef rsrc;
- image_fetch_rsrc(bld_base, reg, false, inst->Memory.Texture, &rsrc);
+ image_fetch_rsrc(bld_base, reg, false, false, inst->Memory.Texture, &rsrc);
emit_data->output[emit_data->chan] =
get_buffer_size(bld_base, rsrc);
return;
tex_fetch_ptrs(bld_base, emit_data, &args.resource, NULL, NULL);
args.lod = lp_build_emit_fetch(bld_base, inst, 0, TGSI_CHAN_X);
} else {
- image_fetch_rsrc(bld_base, reg, false, target, &args.resource);
+ image_fetch_rsrc(bld_base, reg, false, false, target, &args.resource);
args.lod = ctx->i32_0;
}
emit_data->output[emit_data->chan] =
fix_resinfo(ctx, target, ac_build_image_opcode(&ctx->ac, &args));
+
+ if (inst->Instruction.Opcode == TGSI_OPCODE_RESQ &&
+ (target == TGSI_TEXTURE_2D_MSAA ||
+ target == TGSI_TEXTURE_2D_ARRAY_MSAA)) {
+ LLVMValueRef samples =
+ ac_build_image_get_sample_count(&ctx->ac, args.resource);
+
+ emit_data->output[emit_data->chan] =
+ LLVMBuildInsertElement(ctx->ac.builder,
+ emit_data->output[emit_data->chan],
+ samples,
+ LLVMConstInt(ctx->i32, 3, 0), "");
+ }
}
/**
LLVMValueRef *fmask_ptr)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef list = LLVMGetParam(ctx->main_fn, ctx->param_samplers_and_images);
+ LLVMValueRef list = ac_get_arg(&ctx->ac, ctx->samplers_and_images);
const struct tgsi_full_instruction *inst = emit_data->inst;
const struct tgsi_full_src_register *reg;
unsigned target = inst->Texture.Texture;
reg->Register.Index,
ctx->num_samplers);
index = LLVMBuildAdd(ctx->ac.builder, index,
- LLVMConstInt(ctx->i32, SI_NUM_IMAGES / 2, 0), "");
+ LLVMConstInt(ctx->i32, SI_NUM_IMAGE_SLOTS / 2, 0), "");
} else {
index = LLVMConstInt(ctx->i32,
si_get_sampler_slot(reg->Register.Index), 0);
/* Bindless descriptors are accessible from a different pair of
* user SGPR indices.
*/
- list = LLVMGetParam(ctx->main_fn,
- ctx->param_bindless_samplers_and_images);
+ list = ac_get_arg(&ctx->ac, ctx->bindless_samplers_and_images);
index = lp_build_emit_fetch_src(bld_base, reg,
TGSI_TYPE_UNSIGNED, 0);
num_src_deriv_channels = 1;
/* 1D textures are allocated and used as 2D on GFX9. */
- if (ctx->screen->info.chip_class >= GFX9) {
+ if (ctx->screen->info.chip_class == GFX9) {
num_dst_deriv_channels = 2;
} else {
num_dst_deriv_channels = 1;
}
/* 1D textures are allocated and used as 2D on GFX9. */
- if (ctx->screen->info.chip_class >= GFX9) {
+ if (ctx->screen->info.chip_class == GFX9) {
LLVMValueRef filler;
/* Use 0.5, so that we don't sample the border color. */
struct lp_build_emit_data *emit_data)
{
struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef res, samples;
- LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL;
-
- tex_fetch_ptrs(bld_base, emit_data, &res_ptr, &samp_ptr, &fmask_ptr);
-
- /* Read the samples from the descriptor directly. */
- res = LLVMBuildBitCast(ctx->ac.builder, res_ptr, ctx->v8i32, "");
- samples = LLVMBuildExtractElement(ctx->ac.builder, res,
- LLVMConstInt(ctx->i32, 3, 0), "");
- samples = LLVMBuildLShr(ctx->ac.builder, samples,
- LLVMConstInt(ctx->i32, 16, 0), "");
- samples = LLVMBuildAnd(ctx->ac.builder, samples,
- LLVMConstInt(ctx->i32, 0xf, 0), "");
- samples = LLVMBuildShl(ctx->ac.builder, ctx->i32_1,
- samples, "");
-
- emit_data->output[emit_data->chan] = samples;
+ LLVMValueRef rsrc;
+
+ tex_fetch_ptrs(bld_base, emit_data, &rsrc, NULL, NULL);
+
+ rsrc = LLVMBuildBitCast(ctx->ac.builder, rsrc, ctx->v8i32, "");
+ emit_data->output[emit_data->chan] =
+ ac_build_image_get_sample_count(&ctx->ac, rsrc);
}
-static void si_llvm_emit_fbfetch(const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
+static LLVMValueRef si_llvm_emit_fbfetch(struct si_shader_context *ctx)
{
- struct si_shader_context *ctx = si_shader_context(bld_base);
struct ac_image_args args = {};
LLVMValueRef ptr, image, fmask;
/* Load the image descriptor. */
STATIC_ASSERT(SI_PS_IMAGE_COLORBUF0 % 2 == 0);
- ptr = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
+ ptr = ac_get_arg(&ctx->ac, ctx->rw_buffers);
ptr = LLVMBuildPointerCast(ctx->ac.builder, ptr,
ac_array_in_const32_addr_space(ctx->v8i32), "");
image = ac_build_load_to_sgpr(&ctx->ac, ptr,
unsigned chan = 0;
- args.coords[chan++] = si_unpack_param(ctx, SI_PARAM_POS_FIXED_PT, 0, 16);
+ args.coords[chan++] = si_unpack_param(ctx, ctx->pos_fixed_pt, 0, 16);
if (!ctx->shader->key.mono.u.ps.fbfetch_is_1D)
- args.coords[chan++] = si_unpack_param(ctx, SI_PARAM_POS_FIXED_PT, 16, 16);
+ args.coords[chan++] = si_unpack_param(ctx, ctx->pos_fixed_pt, 16, 16);
/* Get the current render target layer index. */
if (ctx->shader->key.mono.u.ps.fbfetch_layered)
- args.coords[chan++] = si_unpack_param(ctx, SI_PARAM_ANCILLARY, 16, 11);
+ args.coords[chan++] = si_unpack_param(ctx, ctx->args.ancillary, 16, 11);
if (ctx->shader->key.mono.u.ps.fbfetch_msaa)
args.coords[chan++] = si_get_sample_id(ctx);
args.dim = ctx->shader->key.mono.u.ps.fbfetch_layered ?
ac_image_2darray : ac_image_2d;
- emit_data->output[emit_data->chan] =
- ac_build_image_opcode(&ctx->ac, &args);
+ return ac_build_image_opcode(&ctx->ac, &args);
+}
+
+static void si_tgsi_emit_fbfetch(const struct lp_build_tgsi_action *action,
+ struct lp_build_tgsi_context *bld_base,
+ struct lp_build_emit_data *emit_data)
+{
+ struct si_shader_context *ctx = si_shader_context(bld_base);
+
+ emit_data->output[emit_data->chan] = si_llvm_emit_fbfetch(ctx);
+}
+
+LLVMValueRef si_nir_emit_fbfetch(struct ac_shader_abi *abi)
+{
+ struct si_shader_context *ctx = si_shader_context_from_abi(abi);
+
+ return si_llvm_emit_fbfetch(ctx);
}
/**
bld_base->op_actions[TGSI_OPCODE_LODQ].emit = build_tex_intrinsic;
bld_base->op_actions[TGSI_OPCODE_TXQS].emit = si_llvm_emit_txqs;
- bld_base->op_actions[TGSI_OPCODE_FBFETCH].emit = si_llvm_emit_fbfetch;
+ bld_base->op_actions[TGSI_OPCODE_FBFETCH].emit = si_tgsi_emit_fbfetch;
bld_base->op_actions[TGSI_OPCODE_LOAD].emit = load_emit;
bld_base->op_actions[TGSI_OPCODE_STORE].emit = store_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMIMIN].intr_name = "smin";
bld_base->op_actions[TGSI_OPCODE_ATOMIMAX].emit = atomic_emit;
bld_base->op_actions[TGSI_OPCODE_ATOMIMAX].intr_name = "smax";
+ bld_base->op_actions[TGSI_OPCODE_ATOMINC_WRAP].emit = atomic_emit;
+ bld_base->op_actions[TGSI_OPCODE_ATOMINC_WRAP].intr_name = "inc";
+ bld_base->op_actions[TGSI_OPCODE_ATOMDEC_WRAP].emit = atomic_emit;
+ bld_base->op_actions[TGSI_OPCODE_ATOMDEC_WRAP].intr_name = "dec";
}
projects / mesa.git / blobdiff