* USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
-#include <llvm/Config/llvm-config.h>
-
#include "util/u_memory.h"
#include "tgsi/tgsi_strings.h"
#include "tgsi/tgsi_from_mesa.h"
#include "ac_exp_param.h"
-#include "ac_shader_util.h"
#include "ac_rtld.h"
-#include "ac_llvm_util.h"
#include "si_shader_internal.h"
#include "si_pipe.h"
#include "sid.h"
static const char scratch_rsrc_dword1_symbol[] =
"SCRATCH_RSRC_DWORD1";
-static void si_llvm_emit_barrier(struct si_shader_context *ctx);
-
static void si_dump_shader_key(const struct si_shader *shader, FILE *f);
static void si_build_vs_prolog_function(struct si_shader_context *ctx,
union si_shader_part_key *key);
-static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
- union si_shader_part_key *key);
-static void si_fix_resource_usage(struct si_screen *sscreen,
- struct si_shader *shader);
-
-static bool llvm_type_is_64bit(struct si_shader_context *ctx,
- LLVMTypeRef type)
-{
- if (type == ctx->ac.i64 || type == ctx->ac.f64)
- return true;
-
- return false;
-}
/** Whether the shader runs as a combination of multiple API shaders */
static bool is_multi_part_shader(struct si_shader_context *ctx)
return unpack_llvm_param(ctx, value, rshift, bitwidth);
}
-static LLVMValueRef get_rel_patch_id(struct si_shader_context *ctx)
-{
- switch (ctx->type) {
- case PIPE_SHADER_TESS_CTRL:
- return si_unpack_param(ctx, ctx->args.tcs_rel_ids, 0, 8);
-
- case PIPE_SHADER_TESS_EVAL:
- return ac_get_arg(&ctx->ac, ctx->tes_rel_patch_id);
-
- default:
- assert(0);
- return NULL;
- }
-}
-
-/* Tessellation shaders pass outputs to the next shader using LDS.
- *
- * LS outputs = TCS inputs
- * TCS outputs = TES inputs
- *
- * The LDS layout is:
- * - TCS inputs for patch 0
- * - TCS inputs for patch 1
- * - TCS inputs for patch 2 = get_tcs_in_current_patch_offset (if RelPatchID==2)
- * - ...
- * - TCS outputs for patch 0 = get_tcs_out_patch0_offset
- * - Per-patch TCS outputs for patch 0 = get_tcs_out_patch0_patch_data_offset
- * - TCS outputs for patch 1
- * - Per-patch TCS outputs for patch 1
- * - TCS outputs for patch 2 = get_tcs_out_current_patch_offset (if RelPatchID==2)
- * - Per-patch TCS outputs for patch 2 = get_tcs_out_current_patch_data_offset (if RelPatchID==2)
- * - ...
- *
- * All three shaders VS(LS), TCS, TES share the same LDS space.
- */
-
-static LLVMValueRef
-get_tcs_in_patch_stride(struct si_shader_context *ctx)
-{
- return si_unpack_param(ctx, ctx->vs_state_bits, 11, 13);
-}
-
-static unsigned get_tcs_out_vertex_dw_stride_constant(struct si_shader_context *ctx)
-{
- assert(ctx->type == PIPE_SHADER_TESS_CTRL);
-
- if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
- return util_last_bit64(ctx->shader->key.mono.u.ff_tcs_inputs_to_copy) * 4;
-
- return util_last_bit64(ctx->shader->selector->outputs_written) * 4;
-}
-
-static LLVMValueRef get_tcs_out_vertex_dw_stride(struct si_shader_context *ctx)
-{
- unsigned stride = get_tcs_out_vertex_dw_stride_constant(ctx);
-
- return LLVMConstInt(ctx->i32, stride, 0);
-}
-
-static LLVMValueRef get_tcs_out_patch_stride(struct si_shader_context *ctx)
-{
- if (ctx->shader->key.mono.u.ff_tcs_inputs_to_copy)
- return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 0, 13);
-
- const struct si_shader_info *info = &ctx->shader->selector->info;
- unsigned tcs_out_vertices = info->properties[TGSI_PROPERTY_TCS_VERTICES_OUT];
- unsigned vertex_dw_stride = get_tcs_out_vertex_dw_stride_constant(ctx);
- unsigned num_patch_outputs = util_last_bit64(ctx->shader->selector->patch_outputs_written);
- unsigned patch_dw_stride = tcs_out_vertices * vertex_dw_stride +
- num_patch_outputs * 4;
- return LLVMConstInt(ctx->i32, patch_dw_stride, 0);
-}
-
-static LLVMValueRef
-get_tcs_out_patch0_offset(struct si_shader_context *ctx)
-{
- return LLVMBuildMul(ctx->ac.builder,
- si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 0, 16),
- LLVMConstInt(ctx->i32, 4, 0), "");
-}
-
-static LLVMValueRef
-get_tcs_out_patch0_patch_data_offset(struct si_shader_context *ctx)
-{
- return LLVMBuildMul(ctx->ac.builder,
- si_unpack_param(ctx, ctx->tcs_out_lds_offsets, 16, 16),
- LLVMConstInt(ctx->i32, 4, 0), "");
-}
-
-static LLVMValueRef
-get_tcs_in_current_patch_offset(struct si_shader_context *ctx)
-{
- LLVMValueRef patch_stride = get_tcs_in_patch_stride(ctx);
- LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
-
- return LLVMBuildMul(ctx->ac.builder, patch_stride, rel_patch_id, "");
-}
-
-static LLVMValueRef
-get_tcs_out_current_patch_offset(struct si_shader_context *ctx)
-{
- LLVMValueRef patch0_offset = get_tcs_out_patch0_offset(ctx);
- LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
- LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
-
- return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_offset);
-}
-
-static LLVMValueRef
-get_tcs_out_current_patch_data_offset(struct si_shader_context *ctx)
-{
- LLVMValueRef patch0_patch_data_offset =
- get_tcs_out_patch0_patch_data_offset(ctx);
- LLVMValueRef patch_stride = get_tcs_out_patch_stride(ctx);
- LLVMValueRef rel_patch_id = get_rel_patch_id(ctx);
-
- return ac_build_imad(&ctx->ac, patch_stride, rel_patch_id, patch0_patch_data_offset);
-}
-
-static LLVMValueRef get_num_tcs_out_vertices(struct si_shader_context *ctx)
-{
- unsigned tcs_out_vertices =
- ctx->shader->selector ?
- ctx->shader->selector->info.properties[TGSI_PROPERTY_TCS_VERTICES_OUT] : 0;
-
- /* If !tcs_out_vertices, it's either the fixed-func TCS or the TCS epilog. */
- if (ctx->type == PIPE_SHADER_TESS_CTRL && tcs_out_vertices)
- return LLVMConstInt(ctx->i32, tcs_out_vertices, 0);
-
- return si_unpack_param(ctx, ctx->tcs_offchip_layout, 6, 6);
-}
-
-static LLVMValueRef get_tcs_in_vertex_dw_stride(struct si_shader_context *ctx)
-{
- unsigned stride;
-
- switch (ctx->type) {
- case PIPE_SHADER_VERTEX:
- stride = ctx->shader->selector->lshs_vertex_stride / 4;
- return LLVMConstInt(ctx->i32, stride, 0);
-
- case PIPE_SHADER_TESS_CTRL:
- if (ctx->screen->info.chip_class >= GFX9 &&
- ctx->shader->is_monolithic) {
- stride = ctx->shader->key.part.tcs.ls->lshs_vertex_stride / 4;
- return LLVMConstInt(ctx->i32, stride, 0);
- }
- return si_unpack_param(ctx, ctx->vs_state_bits, 24, 8);
-
- default:
- assert(0);
- return NULL;
- }
-}
-
static LLVMValueRef unpack_sint16(struct si_shader_context *ctx,
LLVMValueRef i32, unsigned index)
{
}
}
-static LLVMValueRef get_dw_address_from_generic_indices(struct si_shader_context *ctx,
- LLVMValueRef vertex_dw_stride,
- LLVMValueRef base_addr,
- LLVMValueRef vertex_index,
- LLVMValueRef param_index,
- ubyte name, ubyte index)
-{
- if (vertex_dw_stride) {
- base_addr = ac_build_imad(&ctx->ac, vertex_index,
- vertex_dw_stride, base_addr);
- }
-
- if (param_index) {
- base_addr = ac_build_imad(&ctx->ac, param_index,
- LLVMConstInt(ctx->i32, 4, 0), base_addr);
- }
-
- int param = name == TGSI_SEMANTIC_PATCH ||
- name == TGSI_SEMANTIC_TESSINNER ||
- name == TGSI_SEMANTIC_TESSOUTER ?
- si_shader_io_get_unique_index_patch(name, index) :
- si_shader_io_get_unique_index(name, index, false);
-
- /* Add the base address of the element. */
- return LLVMBuildAdd(ctx->ac.builder, base_addr,
- LLVMConstInt(ctx->i32, param * 4, 0), "");
-}
-
-/* The offchip buffer layout for TCS->TES is
- *
- * - attribute 0 of patch 0 vertex 0
- * - attribute 0 of patch 0 vertex 1
- * - attribute 0 of patch 0 vertex 2
- * ...
- * - attribute 0 of patch 1 vertex 0
- * - attribute 0 of patch 1 vertex 1
- * ...
- * - attribute 1 of patch 0 vertex 0
- * - attribute 1 of patch 0 vertex 1
- * ...
- * - per patch attribute 0 of patch 0
- * - per patch attribute 0 of patch 1
- * ...
- *
- * Note that every attribute has 4 components.
- */
-static LLVMValueRef get_tcs_tes_buffer_address(struct si_shader_context *ctx,
- LLVMValueRef rel_patch_id,
- LLVMValueRef vertex_index,
- LLVMValueRef param_index)
-{
- LLVMValueRef base_addr, vertices_per_patch, num_patches, total_vertices;
- LLVMValueRef param_stride, constant16;
-
- vertices_per_patch = get_num_tcs_out_vertices(ctx);
- num_patches = si_unpack_param(ctx, ctx->tcs_offchip_layout, 0, 6);
- total_vertices = LLVMBuildMul(ctx->ac.builder, vertices_per_patch,
- num_patches, "");
-
- constant16 = LLVMConstInt(ctx->i32, 16, 0);
- if (vertex_index) {
- base_addr = ac_build_imad(&ctx->ac, rel_patch_id,
- vertices_per_patch, vertex_index);
- param_stride = total_vertices;
- } else {
- base_addr = rel_patch_id;
- param_stride = num_patches;
- }
-
- base_addr = ac_build_imad(&ctx->ac, param_index, param_stride, base_addr);
- base_addr = LLVMBuildMul(ctx->ac.builder, base_addr, constant16, "");
-
- if (!vertex_index) {
- LLVMValueRef patch_data_offset =
- si_unpack_param(ctx, ctx->tcs_offchip_layout, 12, 20);
-
- base_addr = LLVMBuildAdd(ctx->ac.builder, base_addr,
- patch_data_offset, "");
- }
- return base_addr;
-}
-
-static LLVMValueRef get_tcs_tes_buffer_address_from_generic_indices(
- struct si_shader_context *ctx,
- LLVMValueRef vertex_index,
- LLVMValueRef param_index,
- ubyte name, ubyte index)
-{
- unsigned param_index_base;
-
- param_index_base = name == TGSI_SEMANTIC_PATCH ||
- name == TGSI_SEMANTIC_TESSINNER ||
- name == TGSI_SEMANTIC_TESSOUTER ?
- si_shader_io_get_unique_index_patch(name, index) :
- si_shader_io_get_unique_index(name, index, false);
-
- if (param_index) {
- param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
- LLVMConstInt(ctx->i32, param_index_base, 0),
- "");
- } else {
- param_index = LLVMConstInt(ctx->i32, param_index_base, 0);
- }
-
- return get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx),
- vertex_index, param_index);
-}
-
-static LLVMValueRef si_build_gather_64bit(struct si_shader_context *ctx,
- LLVMTypeRef type,
- LLVMValueRef val1,
- LLVMValueRef val2)
-{
- LLVMValueRef values[2] = {
- ac_to_integer(&ctx->ac, val1),
- ac_to_integer(&ctx->ac, val2),
- };
- LLVMValueRef result = ac_build_gather_values(&ctx->ac, values, 2);
- return LLVMBuildBitCast(ctx->ac.builder, result, type, "");
-}
-
-static LLVMValueRef buffer_load(struct si_shader_context *ctx,
- LLVMTypeRef type, unsigned swizzle,
- LLVMValueRef buffer, LLVMValueRef offset,
- LLVMValueRef base, bool can_speculate)
-{
- LLVMValueRef value, value2;
- LLVMTypeRef vec_type = LLVMVectorType(type, 4);
-
- if (swizzle == ~0) {
- value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
- 0, ac_glc, can_speculate, false);
-
- return LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
- }
-
- if (!llvm_type_is_64bit(ctx, type)) {
- value = ac_build_buffer_load(&ctx->ac, buffer, 4, NULL, base, offset,
- 0, ac_glc, can_speculate, false);
-
- value = LLVMBuildBitCast(ctx->ac.builder, value, vec_type, "");
- return LLVMBuildExtractElement(ctx->ac.builder, value,
- LLVMConstInt(ctx->i32, swizzle, 0), "");
- }
-
- value = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
- swizzle * 4, ac_glc, can_speculate, false);
-
- value2 = ac_build_buffer_load(&ctx->ac, buffer, 1, NULL, base, offset,
- swizzle * 4 + 4, ac_glc, can_speculate, false);
-
- return si_build_gather_64bit(ctx, type, value, value2);
-}
-
-/**
- * Load from LSHS LDS storage.
- *
- * \param type output value type
- * \param swizzle offset (typically 0..3); it can be ~0, which loads a vec4
- * \param dw_addr address in dwords
- */
-static LLVMValueRef lshs_lds_load(struct si_shader_context *ctx,
- LLVMTypeRef type, unsigned swizzle,
- LLVMValueRef dw_addr)
-{
- LLVMValueRef value;
-
- if (swizzle == ~0) {
- LLVMValueRef values[4];
-
- for (unsigned chan = 0; chan < 4; chan++)
- values[chan] = lshs_lds_load(ctx, type, chan, dw_addr);
-
- return ac_build_gather_values(&ctx->ac, values, 4);
- }
-
- /* Split 64-bit loads. */
- if (llvm_type_is_64bit(ctx, type)) {
- LLVMValueRef lo, hi;
-
- lo = lshs_lds_load(ctx, ctx->i32, swizzle, dw_addr);
- hi = lshs_lds_load(ctx, ctx->i32, swizzle + 1, dw_addr);
- return si_build_gather_64bit(ctx, type, lo, hi);
- }
-
- dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr,
- LLVMConstInt(ctx->i32, swizzle, 0), "");
-
- value = ac_lds_load(&ctx->ac, dw_addr);
-
- return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
-}
-
-/**
- * Store to LSHS LDS storage.
- *
- * \param swizzle offset (typically 0..3)
- * \param dw_addr address in dwords
- * \param value value to store
- */
-static void lshs_lds_store(struct si_shader_context *ctx,
- unsigned dw_offset_imm, LLVMValueRef dw_addr,
- LLVMValueRef value)
-{
- dw_addr = LLVMBuildAdd(ctx->ac.builder, dw_addr,
- LLVMConstInt(ctx->i32, dw_offset_imm, 0), "");
-
- ac_lds_store(&ctx->ac, dw_addr, value);
-}
-
-enum si_tess_ring {
- TCS_FACTOR_RING,
- TESS_OFFCHIP_RING_TCS,
- TESS_OFFCHIP_RING_TES,
-};
-
-static LLVMValueRef get_tess_ring_descriptor(struct si_shader_context *ctx,
- enum si_tess_ring ring)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef addr = ac_get_arg(&ctx->ac,
- ring == TESS_OFFCHIP_RING_TES ?
- ctx->tes_offchip_addr :
- ctx->tcs_out_lds_layout);
-
- /* TCS only receives high 13 bits of the address. */
- if (ring == TESS_OFFCHIP_RING_TCS || ring == TCS_FACTOR_RING) {
- addr = LLVMBuildAnd(builder, addr,
- LLVMConstInt(ctx->i32, 0xfff80000, 0), "");
- }
-
- if (ring == TCS_FACTOR_RING) {
- unsigned tf_offset = ctx->screen->tess_offchip_ring_size;
- addr = LLVMBuildAdd(builder, addr,
- LLVMConstInt(ctx->i32, tf_offset, 0), "");
- }
-
- uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
- S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
- S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
- S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
- if (ctx->screen->info.chip_class >= GFX10)
- rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
- S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
- S_008F0C_RESOURCE_LEVEL(1);
- else
- rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
- S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-
- LLVMValueRef desc[4];
- desc[0] = addr;
- desc[1] = LLVMConstInt(ctx->i32,
- S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
- desc[2] = LLVMConstInt(ctx->i32, 0xffffffff, 0);
- desc[3] = LLVMConstInt(ctx->i32, rsrc3, false);
-
- return ac_build_gather_values(&ctx->ac, desc, 4);
-}
-
-static LLVMValueRef si_nir_load_tcs_varyings(struct ac_shader_abi *abi,
- LLVMTypeRef type,
- LLVMValueRef vertex_index,
- LLVMValueRef param_index,
- unsigned const_index,
- unsigned location,
- unsigned driver_location,
- unsigned component,
- unsigned num_components,
- bool is_patch,
- bool is_compact,
- bool load_input)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader_info *info = &ctx->shader->selector->info;
- LLVMValueRef dw_addr, stride;
- ubyte name, index;
-
- driver_location = driver_location / 4;
-
- if (load_input) {
- name = info->input_semantic_name[driver_location];
- index = info->input_semantic_index[driver_location];
- } else {
- name = info->output_semantic_name[driver_location];
- index = info->output_semantic_index[driver_location];
- }
-
- assert((name == TGSI_SEMANTIC_PATCH ||
- name == TGSI_SEMANTIC_TESSINNER ||
- name == TGSI_SEMANTIC_TESSOUTER) == is_patch);
-
- if (load_input) {
- stride = get_tcs_in_vertex_dw_stride(ctx);
- dw_addr = get_tcs_in_current_patch_offset(ctx);
- } else {
- if (is_patch) {
- stride = NULL;
- dw_addr = get_tcs_out_current_patch_data_offset(ctx);
- } else {
- stride = get_tcs_out_vertex_dw_stride(ctx);
- dw_addr = get_tcs_out_current_patch_offset(ctx);
- }
- }
-
- if (!param_index) {
- param_index = LLVMConstInt(ctx->i32, const_index, 0);
- }
-
- dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
- vertex_index, param_index,
- name, index);
-
- LLVMValueRef value[4];
- for (unsigned i = 0; i < num_components; i++) {
- unsigned offset = i;
- if (llvm_type_is_64bit(ctx, type))
- offset *= 2;
-
- offset += component;
- value[i + component] = lshs_lds_load(ctx, type, offset, dw_addr);
- }
-
- return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
-}
-
-LLVMValueRef si_nir_load_input_tes(struct ac_shader_abi *abi,
- LLVMTypeRef type,
- LLVMValueRef vertex_index,
- LLVMValueRef param_index,
- unsigned const_index,
- unsigned location,
- unsigned driver_location,
- unsigned component,
- unsigned num_components,
- bool is_patch,
- bool is_compact,
- bool load_input)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader_info *info = &ctx->shader->selector->info;
- LLVMValueRef base, addr;
-
- driver_location = driver_location / 4;
- ubyte name = info->input_semantic_name[driver_location];
- ubyte index = info->input_semantic_index[driver_location];
-
- assert((name == TGSI_SEMANTIC_PATCH ||
- name == TGSI_SEMANTIC_TESSINNER ||
- name == TGSI_SEMANTIC_TESSOUTER) == is_patch);
-
- base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
-
- if (!param_index) {
- param_index = LLVMConstInt(ctx->i32, const_index, 0);
- }
-
- addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
- param_index,
- name, index);
-
- /* TODO: This will generate rather ordinary llvm code, although it
- * should be easy for the optimiser to fix up. In future we might want
- * to refactor buffer_load().
- */
- LLVMValueRef value[4];
- for (unsigned i = 0; i < num_components; i++) {
- unsigned offset = i;
- if (llvm_type_is_64bit(ctx, type)) {
- offset *= 2;
- if (offset == 4) {
- ubyte name = info->input_semantic_name[driver_location + 1];
- ubyte index = info->input_semantic_index[driver_location + 1];
- addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
- vertex_index,
- param_index,
- name, index);
- }
-
- offset = offset % 4;
- }
-
- offset += component;
- value[i + component] = buffer_load(ctx, type, offset,
- ctx->tess_offchip_ring, base, addr, true);
- }
-
- return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
-}
-
-static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
- const struct nir_variable *var,
- LLVMValueRef vertex_index,
- LLVMValueRef param_index,
- unsigned const_index,
- LLVMValueRef src,
- unsigned writemask)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader_info *info = &ctx->shader->selector->info;
- const unsigned component = var->data.location_frac;
- unsigned driver_location = var->data.driver_location;
- LLVMValueRef dw_addr, stride;
- LLVMValueRef buffer, base, addr;
- LLVMValueRef values[8];
- bool skip_lds_store;
- bool is_tess_factor = false, is_tess_inner = false;
-
- driver_location = driver_location / 4;
- ubyte name = info->output_semantic_name[driver_location];
- ubyte index = info->output_semantic_index[driver_location];
-
- bool is_const = !param_index;
- if (!param_index)
- param_index = LLVMConstInt(ctx->i32, const_index, 0);
-
- const bool is_patch = var->data.patch ||
- var->data.location == VARYING_SLOT_TESS_LEVEL_INNER ||
- var->data.location == VARYING_SLOT_TESS_LEVEL_OUTER;
-
- assert((name == TGSI_SEMANTIC_PATCH ||
- name == TGSI_SEMANTIC_TESSINNER ||
- name == TGSI_SEMANTIC_TESSOUTER) == is_patch);
-
- if (!is_patch) {
- stride = get_tcs_out_vertex_dw_stride(ctx);
- dw_addr = get_tcs_out_current_patch_offset(ctx);
- dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
- vertex_index, param_index,
- name, index);
-
- skip_lds_store = !info->reads_pervertex_outputs;
- } else {
- dw_addr = get_tcs_out_current_patch_data_offset(ctx);
- dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr,
- vertex_index, param_index,
- name, index);
-
- skip_lds_store = !info->reads_perpatch_outputs;
-
- if (is_const && const_index == 0) {
- int name = info->output_semantic_name[driver_location];
-
- /* Always write tess factors into LDS for the TCS epilog. */
- if (name == TGSI_SEMANTIC_TESSINNER ||
- name == TGSI_SEMANTIC_TESSOUTER) {
- /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
- skip_lds_store = !info->reads_tessfactor_outputs &&
- ctx->shader->selector->info.tessfactors_are_def_in_all_invocs;
- is_tess_factor = true;
- is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
- }
- }
- }
-
- buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
-
- base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
-
- addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
- param_index, name, index);
-
- for (unsigned chan = component; chan < 8; chan++) {
- if (!(writemask & (1 << chan)))
- continue;
- LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
-
- unsigned buffer_store_offset = chan % 4;
- if (chan == 4) {
- ubyte name = info->output_semantic_name[driver_location + 1];
- ubyte index = info->output_semantic_index[driver_location + 1];
- addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
- vertex_index,
- param_index,
- name, index);
- }
-
- /* Skip LDS stores if there is no LDS read of this output. */
- if (!skip_lds_store)
- lshs_lds_store(ctx, chan, dw_addr, value);
-
- value = ac_to_integer(&ctx->ac, value);
- values[chan] = value;
-
- if (writemask != 0xF && !is_tess_factor) {
- ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
- addr, base,
- 4 * buffer_store_offset,
- ac_glc);
- }
-
- /* Write tess factors into VGPRs for the epilog. */
- if (is_tess_factor &&
- ctx->shader->selector->info.tessfactors_are_def_in_all_invocs) {
- if (!is_tess_inner) {
- LLVMBuildStore(ctx->ac.builder, value, /* outer */
- ctx->invoc0_tess_factors[chan]);
- } else if (chan < 2) {
- LLVMBuildStore(ctx->ac.builder, value, /* inner */
- ctx->invoc0_tess_factors[4 + chan]);
- }
- }
- }
-
- if (writemask == 0xF && !is_tess_factor) {
- LLVMValueRef value = ac_build_gather_values(&ctx->ac,
- values, 4);
- ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
- base, 0, ac_glc);
- }
-}
-
-static LLVMValueRef si_llvm_load_input_gs(struct ac_shader_abi *abi,
- unsigned input_index,
- unsigned vtx_offset_param,
- LLVMTypeRef type,
- unsigned swizzle)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader *shader = ctx->shader;
- LLVMValueRef vtx_offset, soffset;
- struct si_shader_info *info = &shader->selector->info;
- unsigned semantic_name = info->input_semantic_name[input_index];
- unsigned semantic_index = info->input_semantic_index[input_index];
- unsigned param;
- LLVMValueRef value;
-
- param = si_shader_io_get_unique_index(semantic_name, semantic_index, false);
-
- /* GFX9 has the ESGS ring in LDS. */
- if (ctx->screen->info.chip_class >= GFX9) {
- unsigned index = vtx_offset_param;
-
- switch (index / 2) {
- case 0:
- vtx_offset = si_unpack_param(ctx, ctx->gs_vtx01_offset,
- index % 2 ? 16 : 0, 16);
- break;
- case 1:
- vtx_offset = si_unpack_param(ctx, ctx->gs_vtx23_offset,
- index % 2 ? 16 : 0, 16);
- break;
- case 2:
- vtx_offset = si_unpack_param(ctx, ctx->gs_vtx45_offset,
- index % 2 ? 16 : 0, 16);
- break;
- default:
- assert(0);
- return NULL;
- }
-
- unsigned offset = param * 4 + swizzle;
- vtx_offset = LLVMBuildAdd(ctx->ac.builder, vtx_offset,
- LLVMConstInt(ctx->i32, offset, false), "");
-
- LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->esgs_ring, vtx_offset);
- LLVMValueRef value = LLVMBuildLoad(ctx->ac.builder, ptr, "");
- if (llvm_type_is_64bit(ctx, type)) {
- ptr = LLVMBuildGEP(ctx->ac.builder, ptr,
- &ctx->ac.i32_1, 1, "");
- LLVMValueRef values[2] = {
- value,
- LLVMBuildLoad(ctx->ac.builder, ptr, "")
- };
- value = ac_build_gather_values(&ctx->ac, values, 2);
- }
- return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
- }
-
- /* GFX6: input load from the ESGS ring in memory. */
- if (swizzle == ~0) {
- LLVMValueRef values[4];
- unsigned chan;
- for (chan = 0; chan < 4; chan++) {
- values[chan] = si_llvm_load_input_gs(abi, input_index, vtx_offset_param,
- type, chan);
- }
- return ac_build_gather_values(&ctx->ac, values, 4);
- }
-
- /* Get the vertex offset parameter on GFX6. */
- LLVMValueRef gs_vtx_offset = ac_get_arg(&ctx->ac,
- ctx->gs_vtx_offset[vtx_offset_param]);
-
- vtx_offset = LLVMBuildMul(ctx->ac.builder, gs_vtx_offset,
- LLVMConstInt(ctx->i32, 4, 0), "");
-
- soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
-
- value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
- vtx_offset, soffset, 0, ac_glc, true, false);
- if (llvm_type_is_64bit(ctx, type)) {
- LLVMValueRef value2;
- soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
-
- value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
- ctx->i32_0, vtx_offset, soffset,
- 0, ac_glc, true, false);
- return si_build_gather_64bit(ctx, type, value, value2);
- }
- return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
-}
-
-static LLVMValueRef si_nir_load_input_gs(struct ac_shader_abi *abi,
- unsigned location,
- unsigned driver_location,
- unsigned component,
- unsigned num_components,
- unsigned vertex_index,
- unsigned const_index,
- LLVMTypeRef type)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
-
- LLVMValueRef value[4];
- for (unsigned i = 0; i < num_components; i++) {
- unsigned offset = i;
- if (llvm_type_is_64bit(ctx, type))
- offset *= 2;
-
- offset += component;
- value[i + component] = si_llvm_load_input_gs(&ctx->abi, driver_location / 4 + const_index,
- vertex_index, type, offset);
- }
-
- return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
-}
-
static LLVMValueRef get_base_vertex(struct ac_shader_abi *abi)
{
struct si_shader_context *ctx = si_shader_context_from_abi(abi);
return result;
}
-static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- LLVMValueRef coord[4] = {
- ac_get_arg(&ctx->ac, ctx->tes_u),
- ac_get_arg(&ctx->ac, ctx->tes_v),
- ctx->ac.f32_0,
- ctx->ac.f32_0
- };
-
- /* For triangles, the vector should be (u, v, 1-u-v). */
- if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
- PIPE_PRIM_TRIANGLES) {
- coord[2] = LLVMBuildFSub(ctx->ac.builder, ctx->ac.f32_1,
- LLVMBuildFAdd(ctx->ac.builder,
- coord[0], coord[1], ""), "");
- }
- return ac_build_gather_values(&ctx->ac, coord, 4);
-}
-
-static LLVMValueRef load_tess_level(struct si_shader_context *ctx,
- unsigned semantic_name)
-{
- LLVMValueRef base, addr;
-
- int param = si_shader_io_get_unique_index_patch(semantic_name, 0);
-
- base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
- addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
- LLVMConstInt(ctx->i32, param, 0));
-
- return buffer_load(ctx, ctx->f32,
- ~0, ctx->tess_offchip_ring, base, addr, true);
-
-}
-
-static LLVMValueRef load_tess_level_default(struct si_shader_context *ctx,
- unsigned semantic_name)
-{
- LLVMValueRef buf, slot, val[4];
- int i, offset;
-
- slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
- buf = ac_get_arg(&ctx->ac, ctx->rw_buffers);
- buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
- offset = semantic_name == TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL ? 4 : 0;
-
- for (i = 0; i < 4; i++)
- val[i] = si_buffer_load_const(ctx, buf,
- LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
- return ac_build_gather_values(&ctx->ac, val, 4);
-}
-
-static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi,
- unsigned varying_id,
- bool load_default_state)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- unsigned semantic_name;
-
- if (load_default_state) {
- switch (varying_id) {
- case VARYING_SLOT_TESS_LEVEL_INNER:
- semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL;
- break;
- case VARYING_SLOT_TESS_LEVEL_OUTER:
- semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL;
- break;
- default:
- unreachable("unknown tess level");
- }
- return load_tess_level_default(ctx, semantic_name);
- }
-
- switch (varying_id) {
- case VARYING_SLOT_TESS_LEVEL_INNER:
- semantic_name = TGSI_SEMANTIC_TESSINNER;
- break;
- case VARYING_SLOT_TESS_LEVEL_OUTER:
- semantic_name = TGSI_SEMANTIC_TESSOUTER;
- break;
- default:
- unreachable("unknown tess level");
- }
-
- return load_tess_level(ctx, semantic_name);
-
-}
-
-static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- if (ctx->type == PIPE_SHADER_TESS_CTRL)
- return si_unpack_param(ctx, ctx->tcs_out_lds_layout, 13, 6);
- else if (ctx->type == PIPE_SHADER_TESS_EVAL)
- return get_num_tcs_out_vertices(ctx);
- else
- unreachable("invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
-}
-
void si_declare_compute_memory(struct si_shader_context *ctx)
{
struct si_shader_selector *sel = ctx->shader->selector;
ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, "");
}
-static LLVMValueRef load_const_buffer_desc_fast_path(struct si_shader_context *ctx)
-{
- LLVMValueRef ptr =
- ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
- struct si_shader_selector *sel = ctx->shader->selector;
-
- /* Do the bounds checking with a descriptor, because
- * doing computation and manual bounds checking of 64-bit
- * addresses generates horrible VALU code with very high
- * VGPR usage and very low SIMD occupancy.
- */
- ptr = LLVMBuildPtrToInt(ctx->ac.builder, ptr, ctx->ac.intptr, "");
-
- LLVMValueRef desc0, desc1;
- desc0 = ptr;
- desc1 = LLVMConstInt(ctx->i32,
- S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
-
- uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
- S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
- S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
- S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
- if (ctx->screen->info.chip_class >= GFX10)
- rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
- S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
- S_008F0C_RESOURCE_LEVEL(1);
- else
- rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
- S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-
- LLVMValueRef desc_elems[] = {
- desc0,
- desc1,
- LLVMConstInt(ctx->i32, sel->info.constbuf0_num_slots * 16, 0),
- LLVMConstInt(ctx->i32, rsrc3, false)
- };
-
- return ac_build_gather_values(&ctx->ac, desc_elems, 4);
-}
-
-static LLVMValueRef load_ubo(struct ac_shader_abi *abi, LLVMValueRef index)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader_selector *sel = ctx->shader->selector;
-
- LLVMValueRef ptr = ac_get_arg(&ctx->ac, ctx->const_and_shader_buffers);
-
- if (sel->info.const_buffers_declared == 1 &&
- sel->info.shader_buffers_declared == 0) {
- return load_const_buffer_desc_fast_path(ctx);
- }
-
- index = si_llvm_bound_index(ctx, index, ctx->num_const_buffers);
- index = LLVMBuildAdd(ctx->ac.builder, index,
- LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
-
- return ac_build_load_to_sgpr(&ctx->ac, ptr, index);
-}
-
-static LLVMValueRef
-load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- LLVMValueRef rsrc_ptr = ac_get_arg(&ctx->ac,
- ctx->const_and_shader_buffers);
-
- index = si_llvm_bound_index(ctx, index, ctx->num_shader_buffers);
- index = LLVMBuildSub(ctx->ac.builder,
- LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS - 1, 0),
- index, "");
-
- return ac_build_load_to_sgpr(&ctx->ac, rsrc_ptr, index);
-}
-
/* Initialize arguments for the shader export intrinsic */
static void si_llvm_init_vs_export_args(struct si_shader_context *ctx,
LLVMValueRef *values,
* Write streamout data to buffers for vertex stream @p stream (different
* vertex streams can occur for GS copy shaders).
*/
-static void si_llvm_emit_streamout(struct si_shader_context *ctx,
- struct si_shader_output_values *outputs,
- unsigned noutput, unsigned stream)
+void si_llvm_emit_streamout(struct si_shader_context *ctx,
+ struct si_shader_output_values *outputs,
+ unsigned noutput, unsigned stream)
{
struct si_shader_selector *sel = ctx->shader->selector;
struct pipe_stream_output_info *so = &sel->so;
pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
if (pos_idx == shader->info.nr_pos_exports)
- /* Specify that this is the last export */
- pos_args[i].done = 1;
-
- ac_build_export(&ctx->ac, &pos_args[i]);
- }
-
- /* Build parameter exports. */
- si_build_param_exports(ctx, outputs, noutput);
-}
-
-/**
- * Forward all outputs from the vertex shader to the TES. This is only used
- * for the fixed function TCS.
- */
-static void si_copy_tcs_inputs(struct si_shader_context *ctx)
-{
- LLVMValueRef invocation_id, buffer, buffer_offset;
- LLVMValueRef lds_vertex_stride, lds_base;
- uint64_t inputs;
-
- invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
- buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
- buffer_offset = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
-
- lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
- lds_base = get_tcs_in_current_patch_offset(ctx);
- lds_base = ac_build_imad(&ctx->ac, invocation_id, lds_vertex_stride,
- lds_base);
-
- inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
- while (inputs) {
- unsigned i = u_bit_scan64(&inputs);
-
- LLVMValueRef lds_ptr = LLVMBuildAdd(ctx->ac.builder, lds_base,
- LLVMConstInt(ctx->i32, 4 * i, 0),
- "");
-
- LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
- get_rel_patch_id(ctx),
- invocation_id,
- LLVMConstInt(ctx->i32, i, 0));
-
- LLVMValueRef value = lshs_lds_load(ctx, ctx->ac.i32, ~0, lds_ptr);
-
- ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
- buffer_offset, 0, ac_glc);
- }
-}
-
-static void si_write_tess_factors(struct si_shader_context *ctx,
- LLVMValueRef rel_patch_id,
- LLVMValueRef invocation_id,
- LLVMValueRef tcs_out_current_patch_data_offset,
- LLVMValueRef invoc0_tf_outer[4],
- LLVMValueRef invoc0_tf_inner[2])
-{
- struct si_shader *shader = ctx->shader;
- unsigned tess_inner_index, tess_outer_index;
- LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
- LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
- unsigned stride, outer_comps, inner_comps, i, offset;
-
- /* Add a barrier before loading tess factors from LDS. */
- if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
- si_llvm_emit_barrier(ctx);
-
- /* Do this only for invocation 0, because the tess levels are per-patch,
- * not per-vertex.
- *
- * This can't jump, because invocation 0 executes this. It should
- * at least mask out the loads and stores for other invocations.
- */
- ac_build_ifcc(&ctx->ac,
- LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
- invocation_id, ctx->i32_0, ""), 6503);
-
- /* Determine the layout of one tess factor element in the buffer. */
- switch (shader->key.part.tcs.epilog.prim_mode) {
- case PIPE_PRIM_LINES:
- stride = 2; /* 2 dwords, 1 vec2 store */
- outer_comps = 2;
- inner_comps = 0;
- break;
- case PIPE_PRIM_TRIANGLES:
- stride = 4; /* 4 dwords, 1 vec4 store */
- outer_comps = 3;
- inner_comps = 1;
- break;
- case PIPE_PRIM_QUADS:
- stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
- outer_comps = 4;
- inner_comps = 2;
- break;
- default:
- assert(0);
- return;
- }
-
- for (i = 0; i < 4; i++) {
- inner[i] = LLVMGetUndef(ctx->i32);
- outer[i] = LLVMGetUndef(ctx->i32);
- }
-
- if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
- /* Tess factors are in VGPRs. */
- for (i = 0; i < outer_comps; i++)
- outer[i] = out[i] = invoc0_tf_outer[i];
- for (i = 0; i < inner_comps; i++)
- inner[i] = out[outer_comps+i] = invoc0_tf_inner[i];
- } else {
- /* Load tess_inner and tess_outer from LDS.
- * Any invocation can write them, so we can't get them from a temporary.
- */
- tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
- tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
-
- lds_base = tcs_out_current_patch_data_offset;
- lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
- LLVMConstInt(ctx->i32,
- tess_inner_index * 4, 0), "");
- lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
- LLVMConstInt(ctx->i32,
- tess_outer_index * 4, 0), "");
-
- for (i = 0; i < outer_comps; i++) {
- outer[i] = out[i] =
- lshs_lds_load(ctx, ctx->ac.i32, i, lds_outer);
- }
- for (i = 0; i < inner_comps; i++) {
- inner[i] = out[outer_comps+i] =
- lshs_lds_load(ctx, ctx->ac.i32, i, lds_inner);
- }
- }
-
- if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
- /* For isolines, the hardware expects tess factors in the
- * reverse order from what NIR specifies.
- */
- LLVMValueRef tmp = out[0];
- out[0] = out[1];
- out[1] = tmp;
- }
-
- /* Convert the outputs to vectors for stores. */
- vec0 = ac_build_gather_values(&ctx->ac, out, MIN2(stride, 4));
- vec1 = NULL;
-
- if (stride > 4)
- vec1 = ac_build_gather_values(&ctx->ac, out+4, stride - 4);
-
- /* Get the buffer. */
- buffer = get_tess_ring_descriptor(ctx, TCS_FACTOR_RING);
-
- /* Get the offset. */
- tf_base = ac_get_arg(&ctx->ac,
- ctx->tcs_factor_offset);
- byteoffset = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
- LLVMConstInt(ctx->i32, 4 * stride, 0), "");
-
- ac_build_ifcc(&ctx->ac,
- LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
- rel_patch_id, ctx->i32_0, ""), 6504);
-
- /* Store the dynamic HS control word. */
- offset = 0;
- if (ctx->screen->info.chip_class <= GFX8) {
- ac_build_buffer_store_dword(&ctx->ac, buffer,
- LLVMConstInt(ctx->i32, 0x80000000, 0),
- 1, ctx->i32_0, tf_base,
- offset, ac_glc);
- offset += 4;
- }
-
- ac_build_endif(&ctx->ac, 6504);
-
- /* Store the tessellation factors. */
- ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
- MIN2(stride, 4), byteoffset, tf_base,
- offset, ac_glc);
- offset += 16;
- if (vec1)
- ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
- stride - 4, byteoffset, tf_base,
- offset, ac_glc);
-
- /* Store the tess factors into the offchip buffer if TES reads them. */
- if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
- LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
- LLVMValueRef tf_inner_offset;
- unsigned param_outer, param_inner;
-
- buf = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
- base = ac_get_arg(&ctx->ac, ctx->tcs_offchip_offset);
-
- param_outer = si_shader_io_get_unique_index_patch(
- TGSI_SEMANTIC_TESSOUTER, 0);
- tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
- LLVMConstInt(ctx->i32, param_outer, 0));
-
- unsigned outer_vec_size =
- ac_has_vec3_support(ctx->screen->info.chip_class, false) ?
- outer_comps : util_next_power_of_two(outer_comps);
- outer_vec = ac_build_gather_values(&ctx->ac, outer, outer_vec_size);
-
- ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
- outer_comps, tf_outer_offset,
- base, 0, ac_glc);
- if (inner_comps) {
- param_inner = si_shader_io_get_unique_index_patch(
- TGSI_SEMANTIC_TESSINNER, 0);
- tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
- LLVMConstInt(ctx->i32, param_inner, 0));
-
- inner_vec = inner_comps == 1 ? inner[0] :
- ac_build_gather_values(&ctx->ac, inner, inner_comps);
- ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
- inner_comps, tf_inner_offset,
- base, 0, ac_glc);
- }
- }
-
- ac_build_endif(&ctx->ac, 6503);
-}
-
-LLVMValueRef si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
- struct ac_arg param, unsigned return_index)
-{
- return LLVMBuildInsertValue(ctx->ac.builder, ret,
- ac_get_arg(&ctx->ac, param),
- return_index, "");
-}
-
-LLVMValueRef si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
- struct ac_arg param, unsigned return_index)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef p = ac_get_arg(&ctx->ac, param);
-
- return LLVMBuildInsertValue(builder, ret,
- ac_to_float(&ctx->ac, p),
- return_index, "");
-}
-
-LLVMValueRef si_insert_input_ptr(struct si_shader_context *ctx, LLVMValueRef ret,
- struct ac_arg param, unsigned return_index)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef ptr = ac_get_arg(&ctx->ac, param);
- ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i32, "");
- return LLVMBuildInsertValue(builder, ret, ptr, return_index, "");
-}
-
-/* This only writes the tessellation factor levels. */
-static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
-
- si_copy_tcs_inputs(ctx);
-
- rel_patch_id = get_rel_patch_id(ctx);
- invocation_id = si_unpack_param(ctx, ctx->args.tcs_rel_ids, 8, 5);
- tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
-
- if (ctx->screen->info.chip_class >= GFX9) {
- LLVMBasicBlockRef blocks[2] = {
- LLVMGetInsertBlock(builder),
- ctx->merged_wrap_if_entry_block
- };
- LLVMValueRef values[2];
-
- ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
-
- values[0] = rel_patch_id;
- values[1] = LLVMGetUndef(ctx->i32);
- rel_patch_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
-
- values[0] = tf_lds_offset;
- values[1] = LLVMGetUndef(ctx->i32);
- tf_lds_offset = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
-
- values[0] = invocation_id;
- values[1] = ctx->i32_1; /* cause the epilog to skip threads */
- invocation_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
- }
-
- /* Return epilog parameters from this function. */
- LLVMValueRef ret = ctx->return_value;
- unsigned vgpr;
-
- if (ctx->screen->info.chip_class >= GFX9) {
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout,
- 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout,
- 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
- /* Tess offchip and tess factor offsets are at the beginning. */
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, 2);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, 4);
- vgpr = 8 + GFX9_SGPR_TCS_OUT_LAYOUT + 1;
- } else {
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout,
- GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout,
- GFX6_SGPR_TCS_OUT_LAYOUT);
- /* Tess offchip and tess factor offsets are after user SGPRs. */
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset,
- GFX6_TCS_NUM_USER_SGPR);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset,
- GFX6_TCS_NUM_USER_SGPR + 1);
- vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
- }
-
- /* VGPRs */
- rel_patch_id = ac_to_float(&ctx->ac, rel_patch_id);
- invocation_id = ac_to_float(&ctx->ac, invocation_id);
- tf_lds_offset = ac_to_float(&ctx->ac, tf_lds_offset);
-
- /* Leave a hole corresponding to the two input VGPRs. This ensures that
- * the invocation_id output does not alias the tcs_rel_ids input,
- * which saves a V_MOV on gfx9.
- */
- vgpr += 2;
-
- ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
- ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
-
- if (ctx->shader->selector->info.tessfactors_are_def_in_all_invocs) {
- vgpr++; /* skip the tess factor LDS offset */
- for (unsigned i = 0; i < 6; i++) {
- LLVMValueRef value =
- LLVMBuildLoad(builder, ctx->invoc0_tess_factors[i], "");
- value = ac_to_float(&ctx->ac, value);
- ret = LLVMBuildInsertValue(builder, ret, value, vgpr++, "");
- }
- } else {
- ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
- }
- ctx->return_value = ret;
-}
-
-/* Pass TCS inputs from LS to TCS on GFX9. */
-static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
-{
- LLVMValueRef ret = ctx->return_value;
-
- ret = si_insert_input_ptr(ctx, ret, ctx->other_const_and_shader_buffers, 0);
- ret = si_insert_input_ptr(ctx, ret, ctx->other_samplers_and_images, 1);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_offset, 2);
- ret = si_insert_input_ret(ctx, ret, ctx->merged_wave_info, 3);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_factor_offset, 4);
- ret = si_insert_input_ret(ctx, ret, ctx->merged_scratch_offset, 5);
-
- ret = si_insert_input_ptr(ctx, ret, ctx->rw_buffers,
- 8 + SI_SGPR_RW_BUFFERS);
- ret = si_insert_input_ptr(ctx, ret,
- ctx->bindless_samplers_and_images,
- 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
-
- ret = si_insert_input_ret(ctx, ret, ctx->vs_state_bits,
- 8 + SI_SGPR_VS_STATE_BITS);
-
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_offchip_layout,
- 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_offsets,
- 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
- ret = si_insert_input_ret(ctx, ret, ctx->tcs_out_lds_layout,
- 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
-
- unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
- ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
- ac_to_float(&ctx->ac,
- ac_get_arg(&ctx->ac, ctx->args.tcs_patch_id)),
- vgpr++, "");
- ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
- ac_to_float(&ctx->ac,
- ac_get_arg(&ctx->ac, ctx->args.tcs_rel_ids)),
- vgpr++, "");
- ctx->return_value = ret;
-}
-
-/* Pass GS inputs from ES to GS on GFX9. */
-static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
-{
- LLVMValueRef ret = ctx->return_value;
-
- ret = si_insert_input_ptr(ctx, ret, ctx->other_const_and_shader_buffers, 0);
- ret = si_insert_input_ptr(ctx, ret, ctx->other_samplers_and_images, 1);
- if (ctx->shader->key.as_ngg)
- ret = si_insert_input_ptr(ctx, ret, ctx->gs_tg_info, 2);
- else
- ret = si_insert_input_ret(ctx, ret, ctx->gs2vs_offset, 2);
- ret = si_insert_input_ret(ctx, ret, ctx->merged_wave_info, 3);
- ret = si_insert_input_ret(ctx, ret, ctx->merged_scratch_offset, 5);
-
- ret = si_insert_input_ptr(ctx, ret, ctx->rw_buffers,
- 8 + SI_SGPR_RW_BUFFERS);
- ret = si_insert_input_ptr(ctx, ret,
- ctx->bindless_samplers_and_images,
- 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
- if (ctx->screen->use_ngg) {
- ret = si_insert_input_ptr(ctx, ret, ctx->vs_state_bits,
- 8 + SI_SGPR_VS_STATE_BITS);
- }
-
- unsigned vgpr;
- if (ctx->type == PIPE_SHADER_VERTEX)
- vgpr = 8 + GFX9_VSGS_NUM_USER_SGPR;
- else
- vgpr = 8 + GFX9_TESGS_NUM_USER_SGPR;
-
- ret = si_insert_input_ret_float(ctx, ret, ctx->gs_vtx01_offset, vgpr++);
- ret = si_insert_input_ret_float(ctx, ret, ctx->gs_vtx23_offset, vgpr++);
- ret = si_insert_input_ret_float(ctx, ret, ctx->args.gs_prim_id, vgpr++);
- ret = si_insert_input_ret_float(ctx, ret, ctx->args.gs_invocation_id, vgpr++);
- ret = si_insert_input_ret_float(ctx, ret, ctx->gs_vtx45_offset, vgpr++);
- ctx->return_value = ret;
-}
-
-static void si_llvm_emit_ls_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader *shader = ctx->shader;
- struct si_shader_info *info = &shader->selector->info;
- unsigned i, chan;
- LLVMValueRef vertex_id = ac_get_arg(&ctx->ac, ctx->rel_auto_id);
- LLVMValueRef vertex_dw_stride = get_tcs_in_vertex_dw_stride(ctx);
- LLVMValueRef base_dw_addr = LLVMBuildMul(ctx->ac.builder, vertex_id,
- vertex_dw_stride, "");
-
- /* Write outputs to LDS. The next shader (TCS aka HS) will read
- * its inputs from it. */
- for (i = 0; i < info->num_outputs; i++) {
- unsigned name = info->output_semantic_name[i];
- unsigned index = info->output_semantic_index[i];
-
- /* The ARB_shader_viewport_layer_array spec contains the
- * following issue:
- *
- * 2) What happens if gl_ViewportIndex or gl_Layer is
- * written in the vertex shader and a geometry shader is
- * present?
- *
- * RESOLVED: The value written by the last vertex processing
- * stage is used. If the last vertex processing stage
- * (vertex, tessellation evaluation or geometry) does not
- * statically assign to gl_ViewportIndex or gl_Layer, index
- * or layer zero is assumed.
- *
- * So writes to those outputs in VS-as-LS are simply ignored.
- */
- if (name == TGSI_SEMANTIC_LAYER ||
- name == TGSI_SEMANTIC_VIEWPORT_INDEX)
- continue;
-
- int param = si_shader_io_get_unique_index(name, index, false);
- LLVMValueRef dw_addr = LLVMBuildAdd(ctx->ac.builder, base_dw_addr,
- LLVMConstInt(ctx->i32, param * 4, 0), "");
-
- for (chan = 0; chan < 4; chan++) {
- if (!(info->output_usagemask[i] & (1 << chan)))
- continue;
-
- lshs_lds_store(ctx, chan, dw_addr,
- LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], ""));
- }
- }
-
- if (ctx->screen->info.chip_class >= GFX9)
- si_set_ls_return_value_for_tcs(ctx);
-}
-
-static void si_llvm_emit_es_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader *es = ctx->shader;
- struct si_shader_info *info = &es->selector->info;
- LLVMValueRef lds_base = NULL;
- unsigned chan;
- int i;
-
- if (ctx->screen->info.chip_class >= GFX9 && info->num_outputs) {
- unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
- LLVMValueRef vertex_idx = ac_get_thread_id(&ctx->ac);
- LLVMValueRef wave_idx = si_unpack_param(ctx, ctx->merged_wave_info, 24, 4);
- vertex_idx = LLVMBuildOr(ctx->ac.builder, vertex_idx,
- LLVMBuildMul(ctx->ac.builder, wave_idx,
- LLVMConstInt(ctx->i32, ctx->ac.wave_size, false), ""), "");
- lds_base = LLVMBuildMul(ctx->ac.builder, vertex_idx,
- LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
- }
-
- for (i = 0; i < info->num_outputs; i++) {
- int param;
-
- if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
- info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
- continue;
-
- param = si_shader_io_get_unique_index(info->output_semantic_name[i],
- info->output_semantic_index[i], false);
-
- for (chan = 0; chan < 4; chan++) {
- if (!(info->output_usagemask[i] & (1 << chan)))
- continue;
-
- LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
- out_val = ac_to_integer(&ctx->ac, out_val);
-
- /* GFX9 has the ESGS ring in LDS. */
- if (ctx->screen->info.chip_class >= GFX9) {
- LLVMValueRef idx = LLVMConstInt(ctx->i32, param * 4 + chan, false);
- idx = LLVMBuildAdd(ctx->ac.builder, lds_base, idx, "");
- ac_build_indexed_store(&ctx->ac, ctx->esgs_ring, idx, out_val);
- continue;
- }
-
- ac_build_buffer_store_dword(&ctx->ac,
- ctx->esgs_ring,
- out_val, 1, NULL,
- ac_get_arg(&ctx->ac, ctx->es2gs_offset),
- (4 * param + chan) * 4,
- ac_glc | ac_slc | ac_swizzled);
- }
- }
-
- if (ctx->screen->info.chip_class >= GFX9)
- si_set_es_return_value_for_gs(ctx);
-}
-
-static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
-{
- if (ctx->screen->info.chip_class >= GFX9)
- return si_unpack_param(ctx, ctx->merged_wave_info, 16, 8);
- else
- return ac_get_arg(&ctx->ac, ctx->gs_wave_id);
-}
-
-static void emit_gs_epilogue(struct si_shader_context *ctx)
-{
- if (ctx->shader->key.as_ngg) {
- gfx10_ngg_gs_emit_epilogue(ctx);
- return;
- }
-
- if (ctx->screen->info.chip_class >= GFX10)
- LLVMBuildFence(ctx->ac.builder, LLVMAtomicOrderingRelease, false, "");
-
- ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
- si_get_gs_wave_id(ctx));
-
- if (ctx->screen->info.chip_class >= GFX9)
- ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
-}
-
-static void si_llvm_emit_gs_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader_info UNUSED *info = &ctx->shader->selector->info;
+ /* Specify that this is the last export */
+ pos_args[i].done = 1;
- assert(info->num_outputs <= max_outputs);
+ ac_build_export(&ctx->ac, &pos_args[i]);
+ }
- emit_gs_epilogue(ctx);
+ /* Build parameter exports. */
+ si_build_param_exports(ctx, outputs, noutput);
}
static void si_llvm_emit_vs_epilogue(struct ac_shader_abi *abi,
ctx->return_value = ret;
}
-/* Emit one vertex from the geometry shader */
-static void si_llvm_emit_vertex(struct ac_shader_abi *abi,
- unsigned stream,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
-
- if (ctx->shader->key.as_ngg) {
- gfx10_ngg_gs_emit_vertex(ctx, stream, addrs);
- return;
- }
-
- struct si_shader_info *info = &ctx->shader->selector->info;
- struct si_shader *shader = ctx->shader;
- LLVMValueRef soffset = ac_get_arg(&ctx->ac, ctx->gs2vs_offset);
- LLVMValueRef gs_next_vertex;
- LLVMValueRef can_emit;
- unsigned chan, offset;
- int i;
-
- /* Write vertex attribute values to GSVS ring */
- gs_next_vertex = LLVMBuildLoad(ctx->ac.builder,
- ctx->gs_next_vertex[stream],
- "");
-
- /* If this thread has already emitted the declared maximum number of
- * vertices, skip the write: excessive vertex emissions are not
- * supposed to have any effect.
- *
- * If the shader has no writes to memory, kill it instead. This skips
- * further memory loads and may allow LLVM to skip to the end
- * altogether.
- */
- can_emit = LLVMBuildICmp(ctx->ac.builder, LLVMIntULT, gs_next_vertex,
- LLVMConstInt(ctx->i32,
- shader->selector->gs_max_out_vertices, 0), "");
-
- bool use_kill = !info->writes_memory;
- if (use_kill) {
- ac_build_kill_if_false(&ctx->ac, can_emit);
- } else {
- ac_build_ifcc(&ctx->ac, can_emit, 6505);
- }
-
- offset = 0;
- for (i = 0; i < info->num_outputs; i++) {
- for (chan = 0; chan < 4; chan++) {
- if (!(info->output_usagemask[i] & (1 << chan)) ||
- ((info->output_streams[i] >> (2 * chan)) & 3) != stream)
- continue;
-
- LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
- LLVMValueRef voffset =
- LLVMConstInt(ctx->i32, offset *
- shader->selector->gs_max_out_vertices, 0);
- offset++;
-
- voffset = LLVMBuildAdd(ctx->ac.builder, voffset, gs_next_vertex, "");
- voffset = LLVMBuildMul(ctx->ac.builder, voffset,
- LLVMConstInt(ctx->i32, 4, 0), "");
-
- out_val = ac_to_integer(&ctx->ac, out_val);
-
- ac_build_buffer_store_dword(&ctx->ac,
- ctx->gsvs_ring[stream],
- out_val, 1,
- voffset, soffset, 0,
- ac_glc | ac_slc | ac_swizzled);
- }
- }
-
- gs_next_vertex = LLVMBuildAdd(ctx->ac.builder, gs_next_vertex, ctx->i32_1, "");
- LLVMBuildStore(ctx->ac.builder, gs_next_vertex, ctx->gs_next_vertex[stream]);
-
- /* Signal vertex emission if vertex data was written. */
- if (offset) {
- ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_EMIT | AC_SENDMSG_GS | (stream << 8),
- si_get_gs_wave_id(ctx));
- }
-
- if (!use_kill)
- ac_build_endif(&ctx->ac, 6505);
-}
-
-/* Cut one primitive from the geometry shader */
-static void si_llvm_emit_primitive(struct ac_shader_abi *abi,
- unsigned stream)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
-
- if (ctx->shader->key.as_ngg) {
- LLVMBuildStore(ctx->ac.builder, ctx->ac.i32_0, ctx->gs_curprim_verts[stream]);
- return;
- }
-
- /* Signal primitive cut */
- ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_CUT | AC_SENDMSG_GS | (stream << 8),
- si_get_gs_wave_id(ctx));
-}
-
-static void si_llvm_emit_barrier(struct si_shader_context *ctx)
-{
- /* GFX6 only (thanks to a hw bug workaround):
- * The real barrier instruction isn’t needed, because an entire patch
- * always fits into a single wave.
- */
- if (ctx->screen->info.chip_class == GFX6 &&
- ctx->type == PIPE_SHADER_TESS_CTRL) {
- ac_build_waitcnt(&ctx->ac, AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE);
- return;
- }
-
- ac_build_s_barrier(&ctx->ac);
-}
-
static void declare_streamout_params(struct si_shader_context *ctx,
struct pipe_stream_output_info *so)
{
ac_add_arg(args, file, registers, type, arg);
}
-static void create_function(struct si_shader_context *ctx)
+void si_create_function(struct si_shader_context *ctx)
{
struct si_shader *shader = ctx->shader;
LLVMTypeRef returns[AC_MAX_ARGS];
}
}
-/* Ensure that the esgs ring is declared.
- *
- * We declare it with 64KB alignment as a hint that the
- * pointer value will always be 0.
- */
-static void declare_esgs_ring(struct si_shader_context *ctx)
-{
- if (ctx->esgs_ring)
- return;
-
- assert(!LLVMGetNamedGlobal(ctx->ac.module, "esgs_ring"));
-
- ctx->esgs_ring = LLVMAddGlobalInAddressSpace(
- ctx->ac.module, LLVMArrayType(ctx->i32, 0),
- "esgs_ring",
- AC_ADDR_SPACE_LDS);
- LLVMSetLinkage(ctx->esgs_ring, LLVMExternalLinkage);
- LLVMSetAlignment(ctx->esgs_ring, 64 * 1024);
-}
-
-/**
- * Load ESGS and GSVS ring buffer resource descriptors and save the variables
- * for later use.
- */
-static void preload_ring_buffers(struct si_shader_context *ctx)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
-
- LLVMValueRef buf_ptr = ac_get_arg(&ctx->ac, ctx->rw_buffers);
-
- if (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY) {
- if (ctx->screen->info.chip_class <= GFX8) {
- unsigned ring =
- ctx->type == PIPE_SHADER_GEOMETRY ? SI_GS_RING_ESGS
- : SI_ES_RING_ESGS;
- LLVMValueRef offset = LLVMConstInt(ctx->i32, ring, 0);
-
- ctx->esgs_ring =
- ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
- } else {
- if (USE_LDS_SYMBOLS && LLVM_VERSION_MAJOR >= 9) {
- /* Declare the ESGS ring as an explicit LDS symbol. */
- declare_esgs_ring(ctx);
- } else {
- ac_declare_lds_as_pointer(&ctx->ac);
- ctx->esgs_ring = ctx->ac.lds;
- }
- }
- }
-
- if (ctx->shader->is_gs_copy_shader) {
- LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
-
- ctx->gsvs_ring[0] =
- ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
- } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
- const struct si_shader_selector *sel = ctx->shader->selector;
- LLVMValueRef offset = LLVMConstInt(ctx->i32, SI_RING_GSVS, 0);
- LLVMValueRef base_ring;
-
- base_ring = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
-
- /* The conceptual layout of the GSVS ring is
- * v0c0 .. vLv0 v0c1 .. vLc1 ..
- * but the real memory layout is swizzled across
- * threads:
- * t0v0c0 .. t15v0c0 t0v1c0 .. t15v1c0 ... t15vLcL
- * t16v0c0 ..
- * Override the buffer descriptor accordingly.
- */
- LLVMTypeRef v2i64 = LLVMVectorType(ctx->i64, 2);
- uint64_t stream_offset = 0;
-
- for (unsigned stream = 0; stream < 4; ++stream) {
- unsigned num_components;
- unsigned stride;
- unsigned num_records;
- LLVMValueRef ring, tmp;
-
- num_components = sel->info.num_stream_output_components[stream];
- if (!num_components)
- continue;
-
- stride = 4 * num_components * sel->gs_max_out_vertices;
-
- /* Limit on the stride field for <= GFX7. */
- assert(stride < (1 << 14));
-
- num_records = ctx->ac.wave_size;
-
- ring = LLVMBuildBitCast(builder, base_ring, v2i64, "");
- tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_0, "");
- tmp = LLVMBuildAdd(builder, tmp,
- LLVMConstInt(ctx->i64,
- stream_offset, 0), "");
- stream_offset += stride * ctx->ac.wave_size;
-
- ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_0, "");
- ring = LLVMBuildBitCast(builder, ring, ctx->v4i32, "");
- tmp = LLVMBuildExtractElement(builder, ring, ctx->i32_1, "");
- tmp = LLVMBuildOr(builder, tmp,
- LLVMConstInt(ctx->i32,
- S_008F04_STRIDE(stride) |
- S_008F04_SWIZZLE_ENABLE(1), 0), "");
- ring = LLVMBuildInsertElement(builder, ring, tmp, ctx->i32_1, "");
- ring = LLVMBuildInsertElement(builder, ring,
- LLVMConstInt(ctx->i32, num_records, 0),
- LLVMConstInt(ctx->i32, 2, 0), "");
-
- uint32_t rsrc3 =
- S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
- S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
- S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
- S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
- S_008F0C_INDEX_STRIDE(1) | /* index_stride = 16 (elements) */
- S_008F0C_ADD_TID_ENABLE(1);
-
- if (ctx->ac.chip_class >= GFX10) {
- rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
- S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
- S_008F0C_RESOURCE_LEVEL(1);
- } else {
- rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
- S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
- S_008F0C_ELEMENT_SIZE(1); /* element_size = 4 (bytes) */
- }
-
- ring = LLVMBuildInsertElement(builder, ring,
- LLVMConstInt(ctx->i32, rsrc3, false),
- LLVMConstInt(ctx->i32, 3, 0), "");
-
- ctx->gsvs_ring[stream] = ring;
- }
- } else if (ctx->type == PIPE_SHADER_TESS_EVAL) {
- ctx->tess_offchip_ring = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TES);
- }
-}
-
/* For the UMR disassembler. */
#define DEBUGGER_END_OF_CODE_MARKER 0xbf9f0000 /* invalid instruction */
#define DEBUGGER_NUM_MARKERS 5
si_shader_dump_stats(sscreen, shader, file, check_debug_option);
}
-static int si_compile_llvm(struct si_screen *sscreen,
- struct si_shader_binary *binary,
- struct ac_shader_config *conf,
- struct ac_llvm_compiler *compiler,
- LLVMModuleRef mod,
- struct pipe_debug_callback *debug,
- enum pipe_shader_type shader_type,
- unsigned wave_size,
- const char *name,
- bool less_optimized)
+int si_compile_llvm(struct si_screen *sscreen,
+ struct si_shader_binary *binary,
+ struct ac_shader_config *conf,
+ struct ac_llvm_compiler *compiler,
+ LLVMModuleRef mod,
+ struct pipe_debug_callback *debug,
+ enum pipe_shader_type shader_type,
+ unsigned wave_size,
+ const char *name,
+ bool less_optimized)
{
unsigned count = p_atomic_inc_return(&sscreen->num_compilations);
return 0;
}
-/* Generate code for the hardware VS shader stage to go with a geometry shader */
-struct si_shader *
-si_generate_gs_copy_shader(struct si_screen *sscreen,
- struct ac_llvm_compiler *compiler,
- struct si_shader_selector *gs_selector,
- struct pipe_debug_callback *debug)
-{
- struct si_shader_context ctx;
- struct si_shader *shader;
- LLVMBuilderRef builder;
- struct si_shader_output_values outputs[SI_MAX_VS_OUTPUTS];
- struct si_shader_info *gsinfo = &gs_selector->info;
- int i;
-
-
- shader = CALLOC_STRUCT(si_shader);
- if (!shader)
- return NULL;
-
- /* We can leave the fence as permanently signaled because the GS copy
- * shader only becomes visible globally after it has been compiled. */
- util_queue_fence_init(&shader->ready);
-
- shader->selector = gs_selector;
- shader->is_gs_copy_shader = true;
-
- si_llvm_context_init(&ctx, sscreen, compiler,
- si_get_wave_size(sscreen, PIPE_SHADER_VERTEX, false, false));
- ctx.shader = shader;
- ctx.type = PIPE_SHADER_VERTEX;
-
- builder = ctx.ac.builder;
-
- create_function(&ctx);
- preload_ring_buffers(&ctx);
-
- LLVMValueRef voffset =
- LLVMBuildMul(ctx.ac.builder, ctx.abi.vertex_id,
- LLVMConstInt(ctx.i32, 4, 0), "");
-
- /* Fetch the vertex stream ID.*/
- LLVMValueRef stream_id;
-
- if (!sscreen->use_ngg_streamout && gs_selector->so.num_outputs)
- stream_id = si_unpack_param(&ctx, ctx.streamout_config, 24, 2);
- else
- stream_id = ctx.i32_0;
-
- /* Fill in output information. */
- for (i = 0; i < gsinfo->num_outputs; ++i) {
- outputs[i].semantic_name = gsinfo->output_semantic_name[i];
- outputs[i].semantic_index = gsinfo->output_semantic_index[i];
-
- for (int chan = 0; chan < 4; chan++) {
- outputs[i].vertex_stream[chan] =
- (gsinfo->output_streams[i] >> (2 * chan)) & 3;
- }
- }
-
- LLVMBasicBlockRef end_bb;
- LLVMValueRef switch_inst;
-
- end_bb = LLVMAppendBasicBlockInContext(ctx.ac.context, ctx.main_fn, "end");
- switch_inst = LLVMBuildSwitch(builder, stream_id, end_bb, 4);
-
- for (int stream = 0; stream < 4; stream++) {
- LLVMBasicBlockRef bb;
- unsigned offset;
-
- if (!gsinfo->num_stream_output_components[stream])
- continue;
-
- if (stream > 0 && !gs_selector->so.num_outputs)
- continue;
-
- bb = LLVMInsertBasicBlockInContext(ctx.ac.context, end_bb, "out");
- LLVMAddCase(switch_inst, LLVMConstInt(ctx.i32, stream, 0), bb);
- LLVMPositionBuilderAtEnd(builder, bb);
-
- /* Fetch vertex data from GSVS ring */
- offset = 0;
- for (i = 0; i < gsinfo->num_outputs; ++i) {
- for (unsigned chan = 0; chan < 4; chan++) {
- if (!(gsinfo->output_usagemask[i] & (1 << chan)) ||
- outputs[i].vertex_stream[chan] != stream) {
- outputs[i].values[chan] = LLVMGetUndef(ctx.f32);
- continue;
- }
-
- LLVMValueRef soffset = LLVMConstInt(ctx.i32,
- offset * gs_selector->gs_max_out_vertices * 16 * 4, 0);
- offset++;
-
- outputs[i].values[chan] =
- ac_build_buffer_load(&ctx.ac,
- ctx.gsvs_ring[0], 1,
- ctx.i32_0, voffset,
- soffset, 0, ac_glc | ac_slc,
- true, false);
- }
- }
-
- /* Streamout and exports. */
- if (!sscreen->use_ngg_streamout && gs_selector->so.num_outputs) {
- si_llvm_emit_streamout(&ctx, outputs,
- gsinfo->num_outputs,
- stream);
- }
-
- if (stream == 0)
- si_llvm_export_vs(&ctx, outputs, gsinfo->num_outputs);
-
- LLVMBuildBr(builder, end_bb);
- }
-
- LLVMPositionBuilderAtEnd(builder, end_bb);
-
- LLVMBuildRetVoid(ctx.ac.builder);
-
- ctx.type = PIPE_SHADER_GEOMETRY; /* override for shader dumping */
- si_llvm_optimize_module(&ctx);
-
- bool ok = false;
- if (si_compile_llvm(sscreen, &ctx.shader->binary,
- &ctx.shader->config, ctx.compiler,
- ctx.ac.module,
- debug, PIPE_SHADER_GEOMETRY, ctx.ac.wave_size,
- "GS Copy Shader", false) == 0) {
- if (si_can_dump_shader(sscreen, PIPE_SHADER_GEOMETRY))
- fprintf(stderr, "GS Copy Shader:\n");
- si_shader_dump(sscreen, ctx.shader, debug, stderr, true);
-
- if (!ctx.shader->config.scratch_bytes_per_wave)
- ok = si_shader_binary_upload(sscreen, ctx.shader, 0);
- else
- ok = true;
- }
-
- si_llvm_dispose(&ctx);
-
- if (!ok) {
- FREE(shader);
- shader = NULL;
- } else {
- si_fix_resource_usage(sscreen, shader);
- }
- return shader;
-}
-
static void si_dump_shader_key_vs(const struct si_shader_key *key,
const struct si_vs_prolog_bits *prolog,
const char *prefix, FILE *f)
key->unpack_instance_id_from_vertex_id;
}
-LLVMValueRef si_is_es_thread(struct si_shader_context *ctx)
-{
- /* Return true if the current thread should execute an ES thread. */
- return LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
- ac_get_thread_id(&ctx->ac),
- si_unpack_param(ctx, ctx->merged_wave_info, 0, 8), "");
-}
-
-LLVMValueRef si_is_gs_thread(struct si_shader_context *ctx)
-{
- /* Return true if the current thread should execute a GS thread. */
- return LLVMBuildICmp(ctx->ac.builder, LLVMIntULT,
- ac_get_thread_id(&ctx->ac),
- si_unpack_param(ctx, ctx->merged_wave_info, 8, 8), "");
-}
-
static bool si_build_main_function(struct si_shader_context *ctx,
struct nir_shader *nir, bool free_nir)
{
struct si_shader *shader = ctx->shader;
struct si_shader_selector *sel = shader->selector;
+ si_llvm_init_resource_callbacks(ctx);
+
switch (ctx->type) {
case PIPE_SHADER_VERTEX:
if (shader->key.as_ls)
ctx->abi.load_base_vertex = get_base_vertex;
break;
case PIPE_SHADER_TESS_CTRL:
- ctx->abi.load_tess_varyings = si_nir_load_tcs_varyings;
- ctx->abi.load_tess_level = si_load_tess_level;
- ctx->abi.store_tcs_outputs = si_nir_store_output_tcs;
- ctx->abi.emit_outputs = si_llvm_emit_tcs_epilogue;
- ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
+ si_llvm_init_tcs_callbacks(ctx);
break;
case PIPE_SHADER_TESS_EVAL:
- ctx->abi.load_tess_varyings = si_nir_load_input_tes;
- ctx->abi.load_tess_coord = si_load_tess_coord;
- ctx->abi.load_tess_level = si_load_tess_level;
- ctx->abi.load_patch_vertices_in = si_load_patch_vertices_in;
+ si_llvm_init_tes_callbacks(ctx);
+
if (shader->key.as_es)
ctx->abi.emit_outputs = si_llvm_emit_es_epilogue;
else if (shader->key.as_ngg)
ctx->abi.emit_outputs = si_llvm_emit_vs_epilogue;
break;
case PIPE_SHADER_GEOMETRY:
- ctx->abi.load_inputs = si_nir_load_input_gs;
- ctx->abi.emit_vertex = si_llvm_emit_vertex;
- ctx->abi.emit_primitive = si_llvm_emit_primitive;
- ctx->abi.emit_outputs = si_llvm_emit_gs_epilogue;
+ si_llvm_init_gs_callbacks(ctx);
break;
case PIPE_SHADER_FRAGMENT:
si_llvm_init_ps_callbacks(ctx);
return false;
}
- ctx->abi.load_ubo = load_ubo;
- ctx->abi.load_ssbo = load_ssbo;
+ si_create_function(ctx);
+
+ if (ctx->shader->key.as_es || ctx->type == PIPE_SHADER_GEOMETRY)
+ si_preload_esgs_ring(ctx);
- create_function(ctx);
- preload_ring_buffers(ctx);
+ if (ctx->type == PIPE_SHADER_GEOMETRY)
+ si_preload_gs_rings(ctx);
+ else if (ctx->type == PIPE_SHADER_TESS_EVAL)
+ si_llvm_preload_tes_rings(ctx);
if (ctx->type == PIPE_SHADER_TESS_CTRL &&
sel->info.tessfactors_are_def_in_all_invocs) {
* avoids bank conflicts for SoA accesses.
*/
if (!gfx10_is_ngg_passthrough(shader))
- declare_esgs_ring(ctx);
+ si_llvm_declare_esgs_ring(ctx);
/* This is really only needed when streamout and / or vertex
* compaction is enabled.
shader_out->info.uses_instanceid = true;
}
-/**
- * Build the GS prolog function. Rotate the input vertices for triangle strips
- * with adjacency.
- */
-static void si_build_gs_prolog_function(struct si_shader_context *ctx,
- union si_shader_part_key *key)
-{
- unsigned num_sgprs, num_vgprs;
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMTypeRef returns[AC_MAX_ARGS];
- LLVMValueRef func, ret;
-
- memset(&ctx->args, 0, sizeof(ctx->args));
-
- if (ctx->screen->info.chip_class >= GFX9) {
- if (key->gs_prolog.states.gfx9_prev_is_vs)
- num_sgprs = 8 + GFX9_VSGS_NUM_USER_SGPR;
- else
- num_sgprs = 8 + GFX9_TESGS_NUM_USER_SGPR;
- num_vgprs = 5; /* ES inputs are not needed by GS */
- } else {
- num_sgprs = GFX6_GS_NUM_USER_SGPR + 2;
- num_vgprs = 8;
- }
-
- for (unsigned i = 0; i < num_sgprs; ++i) {
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- returns[i] = ctx->i32;
- }
-
- for (unsigned i = 0; i < num_vgprs; ++i) {
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL);
- returns[num_sgprs + i] = ctx->f32;
- }
-
- /* Create the function. */
- si_llvm_create_func(ctx, "gs_prolog", returns, num_sgprs + num_vgprs, 0);
- func = ctx->main_fn;
-
- /* Set the full EXEC mask for the prolog, because we are only fiddling
- * with registers here. The main shader part will set the correct EXEC
- * mask.
- */
- if (ctx->screen->info.chip_class >= GFX9 && !key->gs_prolog.is_monolithic)
- ac_init_exec_full_mask(&ctx->ac);
-
- /* Copy inputs to outputs. This should be no-op, as the registers match,
- * but it will prevent the compiler from overwriting them unintentionally.
- */
- ret = ctx->return_value;
- for (unsigned i = 0; i < num_sgprs; i++) {
- LLVMValueRef p = LLVMGetParam(func, i);
- ret = LLVMBuildInsertValue(builder, ret, p, i, "");
- }
- for (unsigned i = 0; i < num_vgprs; i++) {
- LLVMValueRef p = LLVMGetParam(func, num_sgprs + i);
- p = ac_to_float(&ctx->ac, p);
- ret = LLVMBuildInsertValue(builder, ret, p, num_sgprs + i, "");
- }
-
- if (key->gs_prolog.states.tri_strip_adj_fix) {
- /* Remap the input vertices for every other primitive. */
- const struct ac_arg gfx6_vtx_params[6] = {
- { .used = true, .arg_index = num_sgprs },
- { .used = true, .arg_index = num_sgprs + 1 },
- { .used = true, .arg_index = num_sgprs + 3 },
- { .used = true, .arg_index = num_sgprs + 4 },
- { .used = true, .arg_index = num_sgprs + 5 },
- { .used = true, .arg_index = num_sgprs + 6 },
- };
- const struct ac_arg gfx9_vtx_params[3] = {
- { .used = true, .arg_index = num_sgprs },
- { .used = true, .arg_index = num_sgprs + 1 },
- { .used = true, .arg_index = num_sgprs + 4 },
- };
- LLVMValueRef vtx_in[6], vtx_out[6];
- LLVMValueRef prim_id, rotate;
-
- if (ctx->screen->info.chip_class >= GFX9) {
- for (unsigned i = 0; i < 3; i++) {
- vtx_in[i*2] = si_unpack_param(ctx, gfx9_vtx_params[i], 0, 16);
- vtx_in[i*2+1] = si_unpack_param(ctx, gfx9_vtx_params[i], 16, 16);
- }
- } else {
- for (unsigned i = 0; i < 6; i++)
- vtx_in[i] = ac_get_arg(&ctx->ac, gfx6_vtx_params[i]);
- }
-
- prim_id = LLVMGetParam(func, num_sgprs + 2);
- rotate = LLVMBuildTrunc(builder, prim_id, ctx->i1, "");
-
- for (unsigned i = 0; i < 6; ++i) {
- LLVMValueRef base, rotated;
- base = vtx_in[i];
- rotated = vtx_in[(i + 4) % 6];
- vtx_out[i] = LLVMBuildSelect(builder, rotate, rotated, base, "");
- }
-
- if (ctx->screen->info.chip_class >= GFX9) {
- for (unsigned i = 0; i < 3; i++) {
- LLVMValueRef hi, out;
-
- hi = LLVMBuildShl(builder, vtx_out[i*2+1],
- LLVMConstInt(ctx->i32, 16, 0), "");
- out = LLVMBuildOr(builder, vtx_out[i*2], hi, "");
- out = ac_to_float(&ctx->ac, out);
- ret = LLVMBuildInsertValue(builder, ret, out,
- gfx9_vtx_params[i].arg_index, "");
- }
- } else {
- for (unsigned i = 0; i < 6; i++) {
- LLVMValueRef out;
-
- out = ac_to_float(&ctx->ac, vtx_out[i]);
- ret = LLVMBuildInsertValue(builder, ret, out,
- gfx6_vtx_params[i].arg_index, "");
- }
- }
- }
-
- LLVMBuildRet(builder, ret);
-}
-
/**
* Given a list of shader part functions, build a wrapper function that
* runs them in sequence to form a monolithic shader.
*/
enum ac_arg_type arg_type = AC_ARG_INT;
if (LLVMGetTypeKind(type) == LLVMPointerTypeKind) {
- arg_type = AC_ARG_CONST_PTR;
+ type = LLVMGetElementType(type);
+
+ if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
+ if (LLVMGetVectorSize(type) == 4)
+ arg_type = AC_ARG_CONST_DESC_PTR;
+ else if (LLVMGetVectorSize(type) == 8)
+ arg_type = AC_ARG_CONST_IMAGE_PTR;
+ else
+ assert(0);
+ } else if (type == ctx->f32) {
+ arg_type = AC_ARG_CONST_FLOAT_PTR;
+ } else {
+ assert(0);
+ }
}
ac_add_arg(&ctx->args, gprs < num_sgprs ? AC_ARG_SGPR : AC_ARG_VGPR,
union si_shader_part_key tcs_epilog_key;
memset(&tcs_epilog_key, 0, sizeof(tcs_epilog_key));
tcs_epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
- si_build_tcs_epilog_function(&ctx, &tcs_epilog_key);
+ si_llvm_build_tcs_epilog(&ctx, &tcs_epilog_key);
parts[3] = ctx.main_fn;
/* VS as LS main part */
memset(&epilog_key, 0, sizeof(epilog_key));
epilog_key.tcs_epilog.states = shader->key.part.tcs.epilog;
- si_build_tcs_epilog_function(&ctx, &epilog_key);
+ si_llvm_build_tcs_epilog(&ctx, &epilog_key);
parts[1] = ctx.main_fn;
si_build_wrapper_function(&ctx, parts, 2, 0, 0);
gs_prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
gs_prolog_key.gs_prolog.is_monolithic = true;
gs_prolog_key.gs_prolog.as_ngg = shader->key.as_ngg;
- si_build_gs_prolog_function(&ctx, &gs_prolog_key);
+ si_llvm_build_gs_prolog(&ctx, &gs_prolog_key);
gs_prolog = ctx.main_fn;
/* ES main part */
memset(&prolog_key, 0, sizeof(prolog_key));
prolog_key.gs_prolog.states = shader->key.part.gs.prolog;
- si_build_gs_prolog_function(&ctx, &prolog_key);
+ si_llvm_build_gs_prolog(&ctx, &prolog_key);
parts[0] = ctx.main_fn;
si_build_wrapper_function(&ctx, parts, 2, 1, 0);
&shader->key.part.vs.prolog);
}
-/**
- * Compile the TCS epilog function. This writes tesselation factors to memory
- * based on the output primitive type of the tesselator (determined by TES).
- */
-static void si_build_tcs_epilog_function(struct si_shader_context *ctx,
- union si_shader_part_key *key)
-{
- memset(&ctx->args, 0, sizeof(ctx->args));
-
- if (ctx->screen->info.chip_class >= GFX9) {
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_offchip_offset);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); /* wave info */
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_factor_offset);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_offchip_layout);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_out_lds_layout);
- } else {
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_offchip_layout);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_out_lds_layout);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_offchip_offset);
- ac_add_arg(&ctx->args, AC_ARG_SGPR, 1, AC_ARG_INT,
- &ctx->tcs_factor_offset);
- }
-
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* VGPR gap */
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* VGPR gap */
- struct ac_arg rel_patch_id; /* patch index within the wave (REL_PATCH_ID) */
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &rel_patch_id);
- struct ac_arg invocation_id; /* invocation ID within the patch */
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &invocation_id);
- struct ac_arg tcs_out_current_patch_data_offset; /* LDS offset where tess factors should be loaded from */
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT,
- &tcs_out_current_patch_data_offset);
-
- struct ac_arg tess_factors[6];
- for (unsigned i = 0; i < 6; i++)
- ac_add_arg(&ctx->args, AC_ARG_VGPR, 1, AC_ARG_INT, &tess_factors[i]);
-
- /* Create the function. */
- si_llvm_create_func(ctx, "tcs_epilog", NULL, 0,
- ctx->screen->info.chip_class >= GFX7 ? 128 : 0);
- ac_declare_lds_as_pointer(&ctx->ac);
-
- LLVMValueRef invoc0_tess_factors[6];
- for (unsigned i = 0; i < 6; i++)
- invoc0_tess_factors[i] = ac_get_arg(&ctx->ac, tess_factors[i]);
-
- si_write_tess_factors(ctx,
- ac_get_arg(&ctx->ac, rel_patch_id),
- ac_get_arg(&ctx->ac, invocation_id),
- ac_get_arg(&ctx->ac, tcs_out_current_patch_data_offset),
- invoc0_tess_factors, invoc0_tess_factors + 4);
-
- LLVMBuildRetVoid(ctx->ac.builder);
-}
-
/**
* Select and compile (or reuse) TCS parts (epilog).
*/
shader->epilog = si_get_shader_part(sscreen, &sscreen->tcs_epilogs,
PIPE_SHADER_TESS_CTRL, false,
&epilog_key, compiler, debug,
- si_build_tcs_epilog_function,
+ si_llvm_build_tcs_epilog,
"Tessellation Control Shader Epilog");
return shader->epilog != NULL;
}
shader->prolog2 = si_get_shader_part(sscreen, &sscreen->gs_prologs,
PIPE_SHADER_GEOMETRY, true,
&prolog_key, compiler, debug,
- si_build_gs_prolog_function,
+ si_llvm_build_gs_prolog,
"Geometry Shader Prolog");
return shader->prolog2 != NULL;
}
*lds_size = MAX2(*lds_size, 8);
}
-static void si_fix_resource_usage(struct si_screen *sscreen,
- struct si_shader *shader)
+void si_fix_resource_usage(struct si_screen *sscreen, struct si_shader *shader)
{
unsigned min_sgprs = shader->info.num_input_sgprs + 2; /* VCC */
projects / mesa.git / blobdiff