- switch (index / 2) {
- case 0:
- vtx_offset = si_unpack_param(ctx, ctx->param_gs_vtx01_offset,
- index % 2 ? 16 : 0, 16);
- break;
- case 1:
- vtx_offset = si_unpack_param(ctx, ctx->param_gs_vtx23_offset,
- index % 2 ? 16 : 0, 16);
- break;
- case 2:
- vtx_offset = si_unpack_param(ctx, ctx->param_gs_vtx45_offset,
- index % 2 ? 16 : 0, 16);
- break;
- default:
- assert(0);
- return NULL;
- }
-
- unsigned offset = param * 4 + swizzle;
- vtx_offset = LLVMBuildAdd(ctx->ac.builder, vtx_offset,
- LLVMConstInt(ctx->i32, offset, false), "");
-
- LLVMValueRef ptr = ac_build_gep0(&ctx->ac, ctx->esgs_ring, vtx_offset);
- LLVMValueRef value = LLVMBuildLoad(ctx->ac.builder, ptr, "");
- if (llvm_type_is_64bit(ctx, type)) {
- ptr = LLVMBuildGEP(ctx->ac.builder, ptr,
- &ctx->ac.i32_1, 1, "");
- LLVMValueRef values[2] = {
- value,
- LLVMBuildLoad(ctx->ac.builder, ptr, "")
- };
- value = ac_build_gather_values(&ctx->ac, values, 2);
- }
- return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
- }
-
- /* GFX6: input load from the ESGS ring in memory. */
- if (swizzle == ~0) {
- LLVMValueRef values[TGSI_NUM_CHANNELS];
- unsigned chan;
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- values[chan] = si_llvm_load_input_gs(abi, input_index, vtx_offset_param,
- type, chan);
- }
- return ac_build_gather_values(&ctx->ac, values,
- TGSI_NUM_CHANNELS);
- }
-
- /* Get the vertex offset parameter on GFX6. */
- LLVMValueRef gs_vtx_offset = ctx->gs_vtx_offset[vtx_offset_param];
-
- vtx_offset = LLVMBuildMul(ctx->ac.builder, gs_vtx_offset,
- LLVMConstInt(ctx->i32, 4, 0), "");
-
- soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle) * 256, 0);
-
- value = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1, ctx->i32_0,
- vtx_offset, soffset, 0, ac_glc, true, false);
- if (llvm_type_is_64bit(ctx, type)) {
- LLVMValueRef value2;
- soffset = LLVMConstInt(ctx->i32, (param * 4 + swizzle + 1) * 256, 0);
-
- value2 = ac_build_buffer_load(&ctx->ac, ctx->esgs_ring, 1,
- ctx->i32_0, vtx_offset, soffset,
- 0, ac_glc, true, false);
- return si_llvm_emit_fetch_64bit(bld_base, type, value, value2);
- }
- return LLVMBuildBitCast(ctx->ac.builder, value, type, "");
-}
-
-static LLVMValueRef si_nir_load_input_gs(struct ac_shader_abi *abi,
- unsigned location,
- unsigned driver_location,
- unsigned component,
- unsigned num_components,
- unsigned vertex_index,
- unsigned const_index,
- LLVMTypeRef type)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
-
- LLVMValueRef value[4];
- for (unsigned i = 0; i < num_components; i++) {
- unsigned offset = i;
- if (llvm_type_is_64bit(ctx, type))
- offset *= 2;
-
- offset += component;
- value[i + component] = si_llvm_load_input_gs(&ctx->abi, driver_location / 4 + const_index,
- vertex_index, type, offset);
- }
-
- return ac_build_varying_gather_values(&ctx->ac, value, num_components, component);
-}
-
-static LLVMValueRef fetch_input_gs(
- struct lp_build_tgsi_context *bld_base,
- const struct tgsi_full_src_register *reg,
- enum tgsi_opcode_type type,
- unsigned swizzle_in)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- struct tgsi_shader_info *info = &ctx->shader->selector->info;
- unsigned swizzle = swizzle_in & 0xffff;
-
- unsigned semantic_name = info->input_semantic_name[reg->Register.Index];
- if (swizzle != ~0 && semantic_name == TGSI_SEMANTIC_PRIMID)
- return si_get_primitive_id(ctx, swizzle);
-
- if (!reg->Register.Dimension)
- return NULL;
-
- return si_llvm_load_input_gs(&ctx->abi, reg->Register.Index,
- reg->Dimension.Index,
- tgsi2llvmtype(bld_base, type),
- swizzle);
-}
-
-static int lookup_interp_param_index(unsigned interpolate, unsigned location)
-{
- switch (interpolate) {
- case TGSI_INTERPOLATE_CONSTANT:
- return 0;
-
- case TGSI_INTERPOLATE_LINEAR:
- if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
- return SI_PARAM_LINEAR_SAMPLE;
- else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
- return SI_PARAM_LINEAR_CENTROID;
- else
- return SI_PARAM_LINEAR_CENTER;
- break;
- case TGSI_INTERPOLATE_COLOR:
- case TGSI_INTERPOLATE_PERSPECTIVE:
- if (location == TGSI_INTERPOLATE_LOC_SAMPLE)
- return SI_PARAM_PERSP_SAMPLE;
- else if (location == TGSI_INTERPOLATE_LOC_CENTROID)
- return SI_PARAM_PERSP_CENTROID;
- else
- return SI_PARAM_PERSP_CENTER;
- break;
- default:
- fprintf(stderr, "Warning: Unhandled interpolation mode.\n");
- return -1;
- }
-}
-
-static LLVMValueRef si_build_fs_interp(struct si_shader_context *ctx,
- unsigned attr_index, unsigned chan,
- LLVMValueRef prim_mask,
- LLVMValueRef i, LLVMValueRef j)
-{
- if (i || j) {
- return ac_build_fs_interp(&ctx->ac,
- LLVMConstInt(ctx->i32, chan, 0),
- LLVMConstInt(ctx->i32, attr_index, 0),
- prim_mask, i, j);
- }
- return ac_build_fs_interp_mov(&ctx->ac,
- LLVMConstInt(ctx->i32, 2, 0), /* P0 */
- LLVMConstInt(ctx->i32, chan, 0),
- LLVMConstInt(ctx->i32, attr_index, 0),
- prim_mask);
-}
-
-/**
- * Interpolate a fragment shader input.
- *
- * @param ctx context
- * @param input_index index of the input in hardware
- * @param semantic_name TGSI_SEMANTIC_*
- * @param semantic_index semantic index
- * @param num_interp_inputs number of all interpolated inputs (= BCOLOR offset)
- * @param colors_read_mask color components read (4 bits for each color, 8 bits in total)
- * @param interp_param interpolation weights (i,j)
- * @param prim_mask SI_PARAM_PRIM_MASK
- * @param face SI_PARAM_FRONT_FACE
- * @param result the return value (4 components)
- */
-static void interp_fs_input(struct si_shader_context *ctx,
- unsigned input_index,
- unsigned semantic_name,
- unsigned semantic_index,
- unsigned num_interp_inputs,
- unsigned colors_read_mask,
- LLVMValueRef interp_param,
- LLVMValueRef prim_mask,
- LLVMValueRef face,
- LLVMValueRef result[4])
-{
- LLVMValueRef i = NULL, j = NULL;
- unsigned chan;
-
- /* fs.constant returns the param from the middle vertex, so it's not
- * really useful for flat shading. It's meant to be used for custom
- * interpolation (but the intrinsic can't fetch from the other two
- * vertices).
- *
- * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
- * to do the right thing. The only reason we use fs.constant is that
- * fs.interp cannot be used on integers, because they can be equal
- * to NaN.
- *
- * When interp is false we will use fs.constant or for newer llvm,
- * amdgcn.interp.mov.
- */
- bool interp = interp_param != NULL;
-
- if (interp) {
- interp_param = LLVMBuildBitCast(ctx->ac.builder, interp_param,
- LLVMVectorType(ctx->f32, 2), "");
-
- i = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
- ctx->i32_0, "");
- j = LLVMBuildExtractElement(ctx->ac.builder, interp_param,
- ctx->i32_1, "");
- }
-
- if (semantic_name == TGSI_SEMANTIC_COLOR &&
- ctx->shader->key.part.ps.prolog.color_two_side) {
- LLVMValueRef is_face_positive;
-
- /* If BCOLOR0 is used, BCOLOR1 is at offset "num_inputs + 1",
- * otherwise it's at offset "num_inputs".
- */
- unsigned back_attr_offset = num_interp_inputs;
- if (semantic_index == 1 && colors_read_mask & 0xf)
- back_attr_offset += 1;
-
- is_face_positive = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE,
- face, ctx->i32_0, "");
-
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- LLVMValueRef front, back;
-
- front = si_build_fs_interp(ctx,
- input_index, chan,
- prim_mask, i, j);
- back = si_build_fs_interp(ctx,
- back_attr_offset, chan,
- prim_mask, i, j);
-
- result[chan] = LLVMBuildSelect(ctx->ac.builder,
- is_face_positive,
- front,
- back,
- "");
- }
- } else if (semantic_name == TGSI_SEMANTIC_FOG) {
- result[0] = si_build_fs_interp(ctx, input_index,
- 0, prim_mask, i, j);
- result[1] =
- result[2] = LLVMConstReal(ctx->f32, 0.0f);
- result[3] = LLVMConstReal(ctx->f32, 1.0f);
- } else {
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- result[chan] = si_build_fs_interp(ctx,
- input_index, chan,
- prim_mask, i, j);
- }
- }
-}
-
-void si_llvm_load_input_fs(
- struct si_shader_context *ctx,
- unsigned input_index,
- LLVMValueRef out[4])
-{
- struct si_shader *shader = ctx->shader;
- struct tgsi_shader_info *info = &shader->selector->info;
- LLVMValueRef main_fn = ctx->main_fn;
- LLVMValueRef interp_param = NULL;
- int interp_param_idx;
- enum tgsi_semantic semantic_name = info->input_semantic_name[input_index];
- unsigned semantic_index = info->input_semantic_index[input_index];
- enum tgsi_interpolate_mode interp_mode = info->input_interpolate[input_index];
- enum tgsi_interpolate_loc interp_loc = info->input_interpolate_loc[input_index];
-
- /* Get colors from input VGPRs (set by the prolog). */
- if (semantic_name == TGSI_SEMANTIC_COLOR) {
- unsigned colors_read = shader->selector->info.colors_read;
- unsigned mask = colors_read >> (semantic_index * 4);
- unsigned offset = SI_PARAM_POS_FIXED_PT + 1 +
- (semantic_index ? util_bitcount(colors_read & 0xf) : 0);
- LLVMValueRef undef = LLVMGetUndef(ctx->f32);
-
- out[0] = mask & 0x1 ? LLVMGetParam(main_fn, offset++) : undef;
- out[1] = mask & 0x2 ? LLVMGetParam(main_fn, offset++) : undef;
- out[2] = mask & 0x4 ? LLVMGetParam(main_fn, offset++) : undef;
- out[3] = mask & 0x8 ? LLVMGetParam(main_fn, offset++) : undef;
- return;
- }
-
- interp_param_idx = lookup_interp_param_index(interp_mode, interp_loc);
- if (interp_param_idx == -1)
- return;
- else if (interp_param_idx) {
- interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
- }
-
- interp_fs_input(ctx, input_index, semantic_name,
- semantic_index, 0, /* this param is unused */
- shader->selector->info.colors_read, interp_param,
- ctx->abi.prim_mask,
- LLVMGetParam(main_fn, SI_PARAM_FRONT_FACE),
- &out[0]);
-}
-
-static void declare_input_fs(
- struct si_shader_context *ctx,
- unsigned input_index,
- const struct tgsi_full_declaration *decl,
- LLVMValueRef out[4])
-{
- si_llvm_load_input_fs(ctx, input_index, out);
-}
-
-LLVMValueRef si_get_sample_id(struct si_shader_context *ctx)
-{
- return si_unpack_param(ctx, SI_PARAM_ANCILLARY, 8, 4);
-}
-
-static LLVMValueRef get_base_vertex(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
-
- /* For non-indexed draws, the base vertex set by the driver
- * (for direct draws) or the CP (for indirect draws) is the
- * first vertex ID, but GLSL expects 0 to be returned.
- */
- LLVMValueRef vs_state = LLVMGetParam(ctx->main_fn,
- ctx->param_vs_state_bits);
- LLVMValueRef indexed;
-
- indexed = LLVMBuildLShr(ctx->ac.builder, vs_state, ctx->i32_1, "");
- indexed = LLVMBuildTrunc(ctx->ac.builder, indexed, ctx->i1, "");
-
- return LLVMBuildSelect(ctx->ac.builder, indexed, ctx->abi.base_vertex,
- ctx->i32_0, "");
-}
-
-static LLVMValueRef get_block_size(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
-
- LLVMValueRef values[3];
- LLVMValueRef result;
- unsigned i;
- unsigned *properties = ctx->shader->selector->info.properties;
-
- if (properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] != 0) {
- unsigned sizes[3] = {
- properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH],
- properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT],
- properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH]
- };
-
- for (i = 0; i < 3; ++i)
- values[i] = LLVMConstInt(ctx->i32, sizes[i], 0);
-
- result = ac_build_gather_values(&ctx->ac, values, 3);
- } else {
- result = LLVMGetParam(ctx->main_fn, ctx->param_block_size);
- }
-
- return result;
-}
-
-/**
- * Load a dword from a constant buffer.
- */
-static LLVMValueRef buffer_load_const(struct si_shader_context *ctx,
- LLVMValueRef resource,
- LLVMValueRef offset)
-{
- return ac_build_buffer_load(&ctx->ac, resource, 1, NULL, offset, NULL,
- 0, 0, true, true);
-}
-
-static LLVMValueRef load_sample_position(struct ac_shader_abi *abi, LLVMValueRef sample_id)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- LLVMValueRef desc = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
- LLVMValueRef buf_index = LLVMConstInt(ctx->i32, SI_PS_CONST_SAMPLE_POSITIONS, 0);
- LLVMValueRef resource = ac_build_load_to_sgpr(&ctx->ac, desc, buf_index);
-
- /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
- LLVMValueRef offset0 = LLVMBuildMul(ctx->ac.builder, sample_id, LLVMConstInt(ctx->i32, 8, 0), "");
- LLVMValueRef offset1 = LLVMBuildAdd(ctx->ac.builder, offset0, LLVMConstInt(ctx->i32, 4, 0), "");
-
- LLVMValueRef pos[4] = {
- buffer_load_const(ctx, resource, offset0),
- buffer_load_const(ctx, resource, offset1),
- LLVMConstReal(ctx->f32, 0),
- LLVMConstReal(ctx->f32, 0)
- };
-
- return ac_build_gather_values(&ctx->ac, pos, 4);
-}
-
-static LLVMValueRef load_sample_mask_in(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- return ac_to_integer(&ctx->ac, abi->sample_coverage);
-}
-
-static LLVMValueRef si_load_tess_coord(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- LLVMValueRef coord[4] = {
- LLVMGetParam(ctx->main_fn, ctx->param_tes_u),
- LLVMGetParam(ctx->main_fn, ctx->param_tes_v),
- ctx->ac.f32_0,
- ctx->ac.f32_0
- };
-
- /* For triangles, the vector should be (u, v, 1-u-v). */
- if (ctx->shader->selector->info.properties[TGSI_PROPERTY_TES_PRIM_MODE] ==
- PIPE_PRIM_TRIANGLES) {
- coord[2] = LLVMBuildFSub(ctx->ac.builder, ctx->ac.f32_1,
- LLVMBuildFAdd(ctx->ac.builder,
- coord[0], coord[1], ""), "");
- }
- return ac_build_gather_values(&ctx->ac, coord, 4);
-}
-
-static LLVMValueRef load_tess_level(struct si_shader_context *ctx,
- unsigned semantic_name)
-{
- LLVMValueRef base, addr;
-
- int param = si_shader_io_get_unique_index_patch(semantic_name, 0);
-
- base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
- addr = get_tcs_tes_buffer_address(ctx, get_rel_patch_id(ctx), NULL,
- LLVMConstInt(ctx->i32, param, 0));
-
- return buffer_load(&ctx->bld_base, ctx->f32,
- ~0, ctx->tess_offchip_ring, base, addr, true);
-
-}
-
-static LLVMValueRef load_tess_level_default(struct si_shader_context *ctx,
- unsigned semantic_name)
-{
- LLVMValueRef buf, slot, val[4];
- int i, offset;
-
- slot = LLVMConstInt(ctx->i32, SI_HS_CONST_DEFAULT_TESS_LEVELS, 0);
- buf = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
- buf = ac_build_load_to_sgpr(&ctx->ac, buf, slot);
- offset = semantic_name == TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL ? 4 : 0;
-
- for (i = 0; i < 4; i++)
- val[i] = buffer_load_const(ctx, buf,
- LLVMConstInt(ctx->i32, (offset + i) * 4, 0));
- return ac_build_gather_values(&ctx->ac, val, 4);
-}
-
-static LLVMValueRef si_load_tess_level(struct ac_shader_abi *abi,
- unsigned varying_id,
- bool load_default_state)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- unsigned semantic_name;
-
- if (load_default_state) {
- switch (varying_id) {
- case VARYING_SLOT_TESS_LEVEL_INNER:
- semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL;
- break;
- case VARYING_SLOT_TESS_LEVEL_OUTER:
- semantic_name = TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL;
- break;
- default:
- unreachable("unknown tess level");
- }
- return load_tess_level_default(ctx, semantic_name);
- }
-
- switch (varying_id) {
- case VARYING_SLOT_TESS_LEVEL_INNER:
- semantic_name = TGSI_SEMANTIC_TESSINNER;
- break;
- case VARYING_SLOT_TESS_LEVEL_OUTER:
- semantic_name = TGSI_SEMANTIC_TESSOUTER;
- break;
- default:
- unreachable("unknown tess level");
- }
-
- return load_tess_level(ctx, semantic_name);
-
-}
-
-static LLVMValueRef si_load_patch_vertices_in(struct ac_shader_abi *abi)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- if (ctx->type == PIPE_SHADER_TESS_CTRL)
- return si_unpack_param(ctx, ctx->param_tcs_out_lds_layout, 13, 6);
- else if (ctx->type == PIPE_SHADER_TESS_EVAL)
- return get_num_tcs_out_vertices(ctx);
- else
- unreachable("invalid shader stage for TGSI_SEMANTIC_VERTICESIN");
-}
-
-void si_load_system_value(struct si_shader_context *ctx,
- unsigned index,
- const struct tgsi_full_declaration *decl)
-{
- LLVMValueRef value = 0;
-
- assert(index < RADEON_LLVM_MAX_SYSTEM_VALUES);
-
- switch (decl->Semantic.Name) {
- case TGSI_SEMANTIC_INSTANCEID:
- value = ctx->abi.instance_id;
- break;
-
- case TGSI_SEMANTIC_VERTEXID:
- value = LLVMBuildAdd(ctx->ac.builder,
- ctx->abi.vertex_id,
- ctx->abi.base_vertex, "");
- break;
-
- case TGSI_SEMANTIC_VERTEXID_NOBASE:
- /* Unused. Clarify the meaning in indexed vs. non-indexed
- * draws if this is ever used again. */
- assert(false);
- break;
-
- case TGSI_SEMANTIC_BASEVERTEX:
- value = get_base_vertex(&ctx->abi);
- break;
-
- case TGSI_SEMANTIC_BASEINSTANCE:
- value = ctx->abi.start_instance;
- break;
-
- case TGSI_SEMANTIC_DRAWID:
- value = ctx->abi.draw_id;
- break;
-
- case TGSI_SEMANTIC_INVOCATIONID:
- if (ctx->type == PIPE_SHADER_TESS_CTRL) {
- value = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
- } else if (ctx->type == PIPE_SHADER_GEOMETRY) {
- if (ctx->screen->info.chip_class >= GFX10) {
- value = LLVMBuildAnd(ctx->ac.builder,
- ctx->abi.gs_invocation_id,
- LLVMConstInt(ctx->i32, 127, 0), "");
- } else {
- value = ctx->abi.gs_invocation_id;
- }
- } else {
- assert(!"INVOCATIONID not implemented");
- }
- break;
-
- case TGSI_SEMANTIC_POSITION:
- {
- LLVMValueRef pos[4] = {
- LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
- LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
- LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Z_FLOAT),
- ac_build_fdiv(&ctx->ac, ctx->ac.f32_1,
- LLVMGetParam(ctx->main_fn, SI_PARAM_POS_W_FLOAT)),
- };
- value = ac_build_gather_values(&ctx->ac, pos, 4);
- break;
- }
-
- case TGSI_SEMANTIC_FACE:
- value = ctx->abi.front_face;
- break;
-
- case TGSI_SEMANTIC_SAMPLEID:
- value = si_get_sample_id(ctx);
- break;
-
- case TGSI_SEMANTIC_SAMPLEPOS: {
- LLVMValueRef pos[4] = {
- LLVMGetParam(ctx->main_fn, SI_PARAM_POS_X_FLOAT),
- LLVMGetParam(ctx->main_fn, SI_PARAM_POS_Y_FLOAT),
- LLVMConstReal(ctx->f32, 0),
- LLVMConstReal(ctx->f32, 0)
- };
- pos[0] = ac_build_fract(&ctx->ac, pos[0], 32);
- pos[1] = ac_build_fract(&ctx->ac, pos[1], 32);
- value = ac_build_gather_values(&ctx->ac, pos, 4);
- break;
- }
-
- case TGSI_SEMANTIC_SAMPLEMASK:
- /* This can only occur with the OpenGL Core profile, which
- * doesn't support smoothing.
- */
- value = LLVMGetParam(ctx->main_fn, SI_PARAM_SAMPLE_COVERAGE);
- break;
-
- case TGSI_SEMANTIC_TESSCOORD:
- value = si_load_tess_coord(&ctx->abi);
- break;
-
- case TGSI_SEMANTIC_VERTICESIN:
- value = si_load_patch_vertices_in(&ctx->abi);
- break;
-
- case TGSI_SEMANTIC_TESSINNER:
- case TGSI_SEMANTIC_TESSOUTER:
- value = load_tess_level(ctx, decl->Semantic.Name);
- break;
-
- case TGSI_SEMANTIC_TESS_DEFAULT_OUTER_LEVEL:
- case TGSI_SEMANTIC_TESS_DEFAULT_INNER_LEVEL:
- value = load_tess_level_default(ctx, decl->Semantic.Name);
- break;
-
- case TGSI_SEMANTIC_PRIMID:
- value = si_get_primitive_id(ctx, 0);
- break;
-
- case TGSI_SEMANTIC_GRID_SIZE:
- value = ctx->abi.num_work_groups;
- break;
-
- case TGSI_SEMANTIC_BLOCK_SIZE:
- value = get_block_size(&ctx->abi);
- break;
-
- case TGSI_SEMANTIC_BLOCK_ID:
- {
- LLVMValueRef values[3];
-
- for (int i = 0; i < 3; i++) {
- values[i] = ctx->i32_0;
- if (ctx->abi.workgroup_ids[i]) {
- values[i] = ctx->abi.workgroup_ids[i];
- }
- }
- value = ac_build_gather_values(&ctx->ac, values, 3);
- break;
- }
-
- case TGSI_SEMANTIC_THREAD_ID:
- value = ctx->abi.local_invocation_ids;
- break;
-
- case TGSI_SEMANTIC_HELPER_INVOCATION:
- value = ac_build_load_helper_invocation(&ctx->ac);
- break;
-
- case TGSI_SEMANTIC_SUBGROUP_SIZE:
- value = LLVMConstInt(ctx->i32, ctx->ac.wave_size, 0);
- break;
-
- case TGSI_SEMANTIC_SUBGROUP_INVOCATION:
- value = ac_get_thread_id(&ctx->ac);
- break;
-
- case TGSI_SEMANTIC_SUBGROUP_EQ_MASK:
- {
- LLVMValueRef id = ac_get_thread_id(&ctx->ac);
- if (ctx->ac.wave_size == 64)
- id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
- value = LLVMBuildShl(ctx->ac.builder,
- LLVMConstInt(ctx->ac.iN_wavemask, 1, 0), id, "");
- if (ctx->ac.wave_size == 32)
- value = LLVMBuildZExt(ctx->ac.builder, value, ctx->i64, "");
- value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
- break;
- }
-
- case TGSI_SEMANTIC_SUBGROUP_GE_MASK:
- case TGSI_SEMANTIC_SUBGROUP_GT_MASK:
- case TGSI_SEMANTIC_SUBGROUP_LE_MASK:
- case TGSI_SEMANTIC_SUBGROUP_LT_MASK:
- {
- LLVMValueRef id = ac_get_thread_id(&ctx->ac);
- if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_GT_MASK ||
- decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK) {
- /* All bits set except LSB */
- value = LLVMConstInt(ctx->ac.iN_wavemask, -2, 0);
- } else {
- /* All bits set */
- value = LLVMConstInt(ctx->ac.iN_wavemask, -1, 0);
- }
- if (ctx->ac.wave_size == 64)
- id = LLVMBuildZExt(ctx->ac.builder, id, ctx->i64, "");
- value = LLVMBuildShl(ctx->ac.builder, value, id, "");
- if (decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LE_MASK ||
- decl->Semantic.Name == TGSI_SEMANTIC_SUBGROUP_LT_MASK)
- value = LLVMBuildNot(ctx->ac.builder, value, "");
- if (ctx->ac.wave_size == 32)
- value = LLVMBuildZExt(ctx->ac.builder, value, ctx->i64, "");
- value = LLVMBuildBitCast(ctx->ac.builder, value, ctx->v2i32, "");
- break;
- }
-
- case TGSI_SEMANTIC_CS_USER_DATA_AMD:
- value = LLVMGetParam(ctx->main_fn, ctx->param_cs_user_data);
- break;
-
- default:
- assert(!"unknown system value");
- return;
- }
-
- ctx->system_values[index] = value;
-}
-
-void si_declare_compute_memory(struct si_shader_context *ctx)
-{
- struct si_shader_selector *sel = ctx->shader->selector;
- unsigned lds_size = sel->info.properties[TGSI_PROPERTY_CS_LOCAL_SIZE];
-
- LLVMTypeRef i8p = LLVMPointerType(ctx->i8, AC_ADDR_SPACE_LDS);
- LLVMValueRef var;
-
- assert(!ctx->ac.lds);
-
- var = LLVMAddGlobalInAddressSpace(ctx->ac.module,
- LLVMArrayType(ctx->i8, lds_size),
- "compute_lds",
- AC_ADDR_SPACE_LDS);
- LLVMSetAlignment(var, 64 * 1024);
-
- ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, "");
-}
-
-void si_tgsi_declare_compute_memory(struct si_shader_context *ctx,
- const struct tgsi_full_declaration *decl)
-{
- assert(decl->Declaration.MemType == TGSI_MEMORY_TYPE_SHARED);
- assert(decl->Range.First == decl->Range.Last);
-
- si_declare_compute_memory(ctx);
-}
-
-static LLVMValueRef load_const_buffer_desc_fast_path(struct si_shader_context *ctx)
-{
- LLVMValueRef ptr =
- LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
- struct si_shader_selector *sel = ctx->shader->selector;
-
- /* Do the bounds checking with a descriptor, because
- * doing computation and manual bounds checking of 64-bit
- * addresses generates horrible VALU code with very high
- * VGPR usage and very low SIMD occupancy.
- */
- ptr = LLVMBuildPtrToInt(ctx->ac.builder, ptr, ctx->ac.intptr, "");
-
- LLVMValueRef desc0, desc1;
- desc0 = ptr;
- desc1 = LLVMConstInt(ctx->i32,
- S_008F04_BASE_ADDRESS_HI(ctx->screen->info.address32_hi), 0);
-
- uint32_t rsrc3 = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
- S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
- S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
- S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
-
- if (ctx->screen->info.chip_class >= GFX10)
- rsrc3 |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
- S_008F0C_OOB_SELECT(3) |
- S_008F0C_RESOURCE_LEVEL(1);
- else
- rsrc3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
- S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
-
- LLVMValueRef desc_elems[] = {
- desc0,
- desc1,
- LLVMConstInt(ctx->i32, (sel->info.const_file_max[0] + 1) * 16, 0),
- LLVMConstInt(ctx->i32, rsrc3, false)
- };
-
- return ac_build_gather_values(&ctx->ac, desc_elems, 4);
-}
-
-static LLVMValueRef load_const_buffer_desc(struct si_shader_context *ctx, int i)
-{
- LLVMValueRef list_ptr = LLVMGetParam(ctx->main_fn,
- ctx->param_const_and_shader_buffers);
-
- return ac_build_load_to_sgpr(&ctx->ac, list_ptr,
- LLVMConstInt(ctx->i32, si_get_constbuf_slot(i), 0));
-}
-
-static LLVMValueRef load_ubo(struct ac_shader_abi *abi, LLVMValueRef index)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader_selector *sel = ctx->shader->selector;
-
- LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
-
- if (sel->info.const_buffers_declared == 1 &&
- sel->info.shader_buffers_declared == 0) {
- return load_const_buffer_desc_fast_path(ctx);
- }
-
- index = si_llvm_bound_index(ctx, index, ctx->num_const_buffers);
- index = LLVMBuildAdd(ctx->ac.builder, index,
- LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
-
- return ac_build_load_to_sgpr(&ctx->ac, ptr, index);
-}
-
-static LLVMValueRef
-load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- LLVMValueRef rsrc_ptr = LLVMGetParam(ctx->main_fn,
- ctx->param_const_and_shader_buffers);
-
- index = si_llvm_bound_index(ctx, index, ctx->num_shader_buffers);
- index = LLVMBuildSub(ctx->ac.builder,
- LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS - 1, 0),
- index, "");
-
- return ac_build_load_to_sgpr(&ctx->ac, rsrc_ptr, index);
-}
-
-static LLVMValueRef fetch_constant(
- struct lp_build_tgsi_context *bld_base,
- const struct tgsi_full_src_register *reg,
- enum tgsi_opcode_type type,
- unsigned swizzle_in)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- struct si_shader_selector *sel = ctx->shader->selector;
- const struct tgsi_ind_register *ireg = ®->Indirect;
- unsigned buf, idx;
- unsigned swizzle = swizzle_in & 0xffff;
-
- LLVMValueRef addr, bufp;
-
- if (swizzle_in == LP_CHAN_ALL) {
- unsigned chan;
- LLVMValueRef values[4];
- for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan)
- values[chan] = fetch_constant(bld_base, reg, type, chan);
-
- return ac_build_gather_values(&ctx->ac, values, 4);
- }
-
- /* Split 64-bit loads. */
- if (tgsi_type_is_64bit(type)) {
- LLVMValueRef lo, hi;
-
- lo = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, swizzle);
- hi = fetch_constant(bld_base, reg, TGSI_TYPE_UNSIGNED, (swizzle_in >> 16));
- return si_llvm_emit_fetch_64bit(bld_base, tgsi2llvmtype(bld_base, type),
- lo, hi);
- }
-
- idx = reg->Register.Index * 4 + swizzle;
- if (reg->Register.Indirect) {
- addr = si_get_indirect_index(ctx, ireg, 16, idx * 4);
- } else {
- addr = LLVMConstInt(ctx->i32, idx * 4, 0);
- }
-
- /* Fast path when user data SGPRs point to constant buffer 0 directly. */
- if (sel->info.const_buffers_declared == 1 &&
- sel->info.shader_buffers_declared == 0) {
- LLVMValueRef desc = load_const_buffer_desc_fast_path(ctx);
- LLVMValueRef result = buffer_load_const(ctx, desc, addr);
- return bitcast(bld_base, type, result);
- }
-
- assert(reg->Register.Dimension);
- buf = reg->Dimension.Index;
-
- if (reg->Dimension.Indirect) {
- LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_const_and_shader_buffers);
- LLVMValueRef index;
- index = si_get_bounded_indirect_index(ctx, ®->DimIndirect,
- reg->Dimension.Index,
- ctx->num_const_buffers);
- index = LLVMBuildAdd(ctx->ac.builder, index,
- LLVMConstInt(ctx->i32, SI_NUM_SHADER_BUFFERS, 0), "");
- bufp = ac_build_load_to_sgpr(&ctx->ac, ptr, index);
- } else
- bufp = load_const_buffer_desc(ctx, buf);
-
- return bitcast(bld_base, type, buffer_load_const(ctx, bufp, addr));
-}
-
-/* Initialize arguments for the shader export intrinsic */
-static void si_llvm_init_export_args(struct si_shader_context *ctx,
- LLVMValueRef *values,
- unsigned target,
- struct ac_export_args *args)
-{
- LLVMValueRef f32undef = LLVMGetUndef(ctx->ac.f32);
- unsigned spi_shader_col_format = V_028714_SPI_SHADER_32_ABGR;
- unsigned chan;
- bool is_int8, is_int10;
-
- /* Default is 0xf. Adjusted below depending on the format. */
- args->enabled_channels = 0xf; /* writemask */
-
- /* Specify whether the EXEC mask represents the valid mask */
- args->valid_mask = 0;
-
- /* Specify whether this is the last export */
- args->done = 0;
-
- /* Specify the target we are exporting */
- args->target = target;
-
- if (ctx->type == PIPE_SHADER_FRAGMENT) {
- const struct si_shader_key *key = &ctx->shader->key;
- unsigned col_formats = key->part.ps.epilog.spi_shader_col_format;
- int cbuf = target - V_008DFC_SQ_EXP_MRT;
-
- assert(cbuf >= 0 && cbuf < 8);
- spi_shader_col_format = (col_formats >> (cbuf * 4)) & 0xf;
- is_int8 = (key->part.ps.epilog.color_is_int8 >> cbuf) & 0x1;
- is_int10 = (key->part.ps.epilog.color_is_int10 >> cbuf) & 0x1;
- }
-
- args->compr = false;
- args->out[0] = f32undef;
- args->out[1] = f32undef;
- args->out[2] = f32undef;
- args->out[3] = f32undef;
-
- LLVMValueRef (*packf)(struct ac_llvm_context *ctx, LLVMValueRef args[2]) = NULL;
- LLVMValueRef (*packi)(struct ac_llvm_context *ctx, LLVMValueRef args[2],
- unsigned bits, bool hi) = NULL;
-
- switch (spi_shader_col_format) {
- case V_028714_SPI_SHADER_ZERO:
- args->enabled_channels = 0; /* writemask */
- args->target = V_008DFC_SQ_EXP_NULL;
- break;
-
- case V_028714_SPI_SHADER_32_R:
- args->enabled_channels = 1; /* writemask */
- args->out[0] = values[0];
- break;
-
- case V_028714_SPI_SHADER_32_GR:
- args->enabled_channels = 0x3; /* writemask */
- args->out[0] = values[0];
- args->out[1] = values[1];
- break;
-
- case V_028714_SPI_SHADER_32_AR:
- if (ctx->screen->info.chip_class >= GFX10) {
- args->enabled_channels = 0x3; /* writemask */
- args->out[0] = values[0];
- args->out[1] = values[3];
- } else {
- args->enabled_channels = 0x9; /* writemask */
- args->out[0] = values[0];
- args->out[3] = values[3];
- }
- break;
-
- case V_028714_SPI_SHADER_FP16_ABGR:
- packf = ac_build_cvt_pkrtz_f16;
- break;
-
- case V_028714_SPI_SHADER_UNORM16_ABGR:
- packf = ac_build_cvt_pknorm_u16;
- break;
-
- case V_028714_SPI_SHADER_SNORM16_ABGR:
- packf = ac_build_cvt_pknorm_i16;
- break;
-
- case V_028714_SPI_SHADER_UINT16_ABGR:
- packi = ac_build_cvt_pk_u16;
- break;
-
- case V_028714_SPI_SHADER_SINT16_ABGR:
- packi = ac_build_cvt_pk_i16;
- break;
-
- case V_028714_SPI_SHADER_32_ABGR:
- memcpy(&args->out[0], values, sizeof(values[0]) * 4);
- break;
- }
-
- /* Pack f16 or norm_i16/u16. */
- if (packf) {
- for (chan = 0; chan < 2; chan++) {
- LLVMValueRef pack_args[2] = {
- values[2 * chan],
- values[2 * chan + 1]
- };
- LLVMValueRef packed;
-
- packed = packf(&ctx->ac, pack_args);
- args->out[chan] = ac_to_float(&ctx->ac, packed);
- }
- args->compr = 1; /* COMPR flag */
- }
- /* Pack i16/u16. */
- if (packi) {
- for (chan = 0; chan < 2; chan++) {
- LLVMValueRef pack_args[2] = {
- ac_to_integer(&ctx->ac, values[2 * chan]),
- ac_to_integer(&ctx->ac, values[2 * chan + 1])
- };
- LLVMValueRef packed;
-
- packed = packi(&ctx->ac, pack_args,
- is_int8 ? 8 : is_int10 ? 10 : 16,
- chan == 1);
- args->out[chan] = ac_to_float(&ctx->ac, packed);
- }
- args->compr = 1; /* COMPR flag */
- }
-}
-
-static void si_alpha_test(struct lp_build_tgsi_context *bld_base,
- LLVMValueRef alpha)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
-
- if (ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_NEVER) {
- static LLVMRealPredicate cond_map[PIPE_FUNC_ALWAYS + 1] = {
- [PIPE_FUNC_LESS] = LLVMRealOLT,
- [PIPE_FUNC_EQUAL] = LLVMRealOEQ,
- [PIPE_FUNC_LEQUAL] = LLVMRealOLE,
- [PIPE_FUNC_GREATER] = LLVMRealOGT,
- [PIPE_FUNC_NOTEQUAL] = LLVMRealONE,
- [PIPE_FUNC_GEQUAL] = LLVMRealOGE,
- };
- LLVMRealPredicate cond = cond_map[ctx->shader->key.part.ps.epilog.alpha_func];
- assert(cond);
-
- LLVMValueRef alpha_ref = LLVMGetParam(ctx->main_fn,
- SI_PARAM_ALPHA_REF);
- LLVMValueRef alpha_pass =
- LLVMBuildFCmp(ctx->ac.builder, cond, alpha, alpha_ref, "");
- ac_build_kill_if_false(&ctx->ac, alpha_pass);
- } else {
- ac_build_kill_if_false(&ctx->ac, ctx->i1false);
- }
-}
-
-static LLVMValueRef si_scale_alpha_by_sample_mask(struct lp_build_tgsi_context *bld_base,
- LLVMValueRef alpha,
- unsigned samplemask_param)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef coverage;
-
- /* alpha = alpha * popcount(coverage) / SI_NUM_SMOOTH_AA_SAMPLES */
- coverage = LLVMGetParam(ctx->main_fn,
- samplemask_param);
- coverage = ac_to_integer(&ctx->ac, coverage);
-
- coverage = ac_build_intrinsic(&ctx->ac, "llvm.ctpop.i32",
- ctx->i32,
- &coverage, 1, AC_FUNC_ATTR_READNONE);
-
- coverage = LLVMBuildUIToFP(ctx->ac.builder, coverage,
- ctx->f32, "");
-
- coverage = LLVMBuildFMul(ctx->ac.builder, coverage,
- LLVMConstReal(ctx->f32,
- 1.0 / SI_NUM_SMOOTH_AA_SAMPLES), "");
-
- return LLVMBuildFMul(ctx->ac.builder, alpha, coverage, "");
-}
-
-static void si_llvm_emit_clipvertex(struct si_shader_context *ctx,
- struct ac_export_args *pos, LLVMValueRef *out_elts)
-{
- unsigned reg_index;
- unsigned chan;
- unsigned const_chan;
- LLVMValueRef base_elt;
- LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, ctx->param_rw_buffers);
- LLVMValueRef constbuf_index = LLVMConstInt(ctx->i32,
- SI_VS_CONST_CLIP_PLANES, 0);
- LLVMValueRef const_resource = ac_build_load_to_sgpr(&ctx->ac, ptr, constbuf_index);
-
- for (reg_index = 0; reg_index < 2; reg_index ++) {
- struct ac_export_args *args = &pos[2 + reg_index];
-
- args->out[0] =
- args->out[1] =
- args->out[2] =
- args->out[3] = LLVMConstReal(ctx->f32, 0.0f);
-
- /* Compute dot products of position and user clip plane vectors */
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- for (const_chan = 0; const_chan < TGSI_NUM_CHANNELS; const_chan++) {
- LLVMValueRef addr =
- LLVMConstInt(ctx->i32, ((reg_index * 4 + chan) * 4 +
- const_chan) * 4, 0);
- base_elt = buffer_load_const(ctx, const_resource,
- addr);
- args->out[chan] = ac_build_fmad(&ctx->ac, base_elt,
- out_elts[const_chan], args->out[chan]);
- }
- }
-
- args->enabled_channels = 0xf;
- args->valid_mask = 0;
- args->done = 0;
- args->target = V_008DFC_SQ_EXP_POS + 2 + reg_index;
- args->compr = 0;
- }
-}
-
-static void si_dump_streamout(struct pipe_stream_output_info *so)
-{
- unsigned i;
-
- if (so->num_outputs)
- fprintf(stderr, "STREAMOUT\n");
-
- for (i = 0; i < so->num_outputs; i++) {
- unsigned mask = ((1 << so->output[i].num_components) - 1) <<
- so->output[i].start_component;
- fprintf(stderr, " %i: BUF%i[%i..%i] <- OUT[%i].%s%s%s%s\n",
- i, so->output[i].output_buffer,
- so->output[i].dst_offset, so->output[i].dst_offset + so->output[i].num_components - 1,
- so->output[i].register_index,
- mask & 1 ? "x" : "",
- mask & 2 ? "y" : "",
- mask & 4 ? "z" : "",
- mask & 8 ? "w" : "");
- }
-}
-
-void si_emit_streamout_output(struct si_shader_context *ctx,
- LLVMValueRef const *so_buffers,
- LLVMValueRef const *so_write_offsets,
- struct pipe_stream_output *stream_out,
- struct si_shader_output_values *shader_out)
-{
- unsigned buf_idx = stream_out->output_buffer;
- unsigned start = stream_out->start_component;
- unsigned num_comps = stream_out->num_components;
- LLVMValueRef out[4];
-
- assert(num_comps && num_comps <= 4);
- if (!num_comps || num_comps > 4)
- return;
-
- /* Load the output as int. */
- for (int j = 0; j < num_comps; j++) {
- assert(stream_out->stream == shader_out->vertex_stream[start + j]);
-
- out[j] = ac_to_integer(&ctx->ac, shader_out->values[start + j]);
- }
-
- /* Pack the output. */
- LLVMValueRef vdata = NULL;
-
- switch (num_comps) {
- case 1: /* as i32 */
- vdata = out[0];
- break;
- case 2: /* as v2i32 */
- case 3: /* as v3i32 */
- if (ac_has_vec3_support(ctx->screen->info.chip_class, false)) {
- vdata = ac_build_gather_values(&ctx->ac, out, num_comps);
- break;
- }
- /* as v4i32 (aligned to 4) */
- out[3] = LLVMGetUndef(ctx->i32);
- /* fall through */
- case 4: /* as v4i32 */
- vdata = ac_build_gather_values(&ctx->ac, out, util_next_power_of_two(num_comps));
- break;
- }
-
- ac_build_buffer_store_dword(&ctx->ac, so_buffers[buf_idx],
- vdata, num_comps,
- so_write_offsets[buf_idx],
- ctx->i32_0,
- stream_out->dst_offset * 4, ac_glc | ac_slc, false);
-}
-
-/**
- * Write streamout data to buffers for vertex stream @p stream (different
- * vertex streams can occur for GS copy shaders).
- */
-static void si_llvm_emit_streamout(struct si_shader_context *ctx,
- struct si_shader_output_values *outputs,
- unsigned noutput, unsigned stream)
-{
- struct si_shader_selector *sel = ctx->shader->selector;
- struct pipe_stream_output_info *so = &sel->so;
- LLVMBuilderRef builder = ctx->ac.builder;
- int i;
-
- /* Get bits [22:16], i.e. (so_param >> 16) & 127; */
- LLVMValueRef so_vtx_count =
- si_unpack_param(ctx, ctx->param_streamout_config, 16, 7);
-
- LLVMValueRef tid = ac_get_thread_id(&ctx->ac);
-
- /* can_emit = tid < so_vtx_count; */
- LLVMValueRef can_emit =
- LLVMBuildICmp(builder, LLVMIntULT, tid, so_vtx_count, "");
-
- /* Emit the streamout code conditionally. This actually avoids
- * out-of-bounds buffer access. The hw tells us via the SGPR
- * (so_vtx_count) which threads are allowed to emit streamout data. */
- ac_build_ifcc(&ctx->ac, can_emit, 6501);
- {
- /* The buffer offset is computed as follows:
- * ByteOffset = streamout_offset[buffer_id]*4 +
- * (streamout_write_index + thread_id)*stride[buffer_id] +
- * attrib_offset
- */
-
- LLVMValueRef so_write_index =
- LLVMGetParam(ctx->main_fn,
- ctx->param_streamout_write_index);
-
- /* Compute (streamout_write_index + thread_id). */
- so_write_index = LLVMBuildAdd(builder, so_write_index, tid, "");
-
- /* Load the descriptor and compute the write offset for each
- * enabled buffer. */
- LLVMValueRef so_write_offset[4] = {};
- LLVMValueRef so_buffers[4];
- LLVMValueRef buf_ptr = LLVMGetParam(ctx->main_fn,
- ctx->param_rw_buffers);
-
- for (i = 0; i < 4; i++) {
- if (!so->stride[i])
- continue;
-
- LLVMValueRef offset = LLVMConstInt(ctx->i32,
- SI_VS_STREAMOUT_BUF0 + i, 0);
-
- so_buffers[i] = ac_build_load_to_sgpr(&ctx->ac, buf_ptr, offset);
-
- LLVMValueRef so_offset = LLVMGetParam(ctx->main_fn,
- ctx->param_streamout_offset[i]);
- so_offset = LLVMBuildMul(builder, so_offset, LLVMConstInt(ctx->i32, 4, 0), "");
-
- so_write_offset[i] = ac_build_imad(&ctx->ac, so_write_index,
- LLVMConstInt(ctx->i32, so->stride[i]*4, 0),
- so_offset);
- }
-
- /* Write streamout data. */
- for (i = 0; i < so->num_outputs; i++) {
- unsigned reg = so->output[i].register_index;
-
- if (reg >= noutput)
- continue;
-
- if (stream != so->output[i].stream)
- continue;
-
- si_emit_streamout_output(ctx, so_buffers, so_write_offset,
- &so->output[i], &outputs[reg]);
- }
- }
- ac_build_endif(&ctx->ac, 6501);
-}
-
-static void si_export_param(struct si_shader_context *ctx, unsigned index,
- LLVMValueRef *values)
-{
- struct ac_export_args args;
-
- si_llvm_init_export_args(ctx, values,
- V_008DFC_SQ_EXP_PARAM + index, &args);
- ac_build_export(&ctx->ac, &args);
-}
-
-static void si_build_param_exports(struct si_shader_context *ctx,
- struct si_shader_output_values *outputs,
- unsigned noutput)
-{
- struct si_shader *shader = ctx->shader;
- unsigned param_count = 0;
-
- for (unsigned i = 0; i < noutput; i++) {
- unsigned semantic_name = outputs[i].semantic_name;
- unsigned semantic_index = outputs[i].semantic_index;
-
- if (outputs[i].vertex_stream[0] != 0 &&
- outputs[i].vertex_stream[1] != 0 &&
- outputs[i].vertex_stream[2] != 0 &&
- outputs[i].vertex_stream[3] != 0)
- continue;
-
- switch (semantic_name) {
- case TGSI_SEMANTIC_LAYER:
- case TGSI_SEMANTIC_VIEWPORT_INDEX:
- case TGSI_SEMANTIC_CLIPDIST:
- case TGSI_SEMANTIC_COLOR:
- case TGSI_SEMANTIC_BCOLOR:
- case TGSI_SEMANTIC_PRIMID:
- case TGSI_SEMANTIC_FOG:
- case TGSI_SEMANTIC_TEXCOORD:
- case TGSI_SEMANTIC_GENERIC:
- break;
- default:
- continue;
- }
-
- if ((semantic_name != TGSI_SEMANTIC_GENERIC ||
- semantic_index < SI_MAX_IO_GENERIC) &&
- shader->key.opt.kill_outputs &
- (1ull << si_shader_io_get_unique_index(semantic_name,
- semantic_index, true)))
- continue;
-
- si_export_param(ctx, param_count, outputs[i].values);
-
- assert(i < ARRAY_SIZE(shader->info.vs_output_param_offset));
- shader->info.vs_output_param_offset[i] = param_count++;
- }
-
- shader->info.nr_param_exports = param_count;
-}
-
-/**
- * Vertex color clamping.
- *
- * This uses a state constant loaded in a user data SGPR and
- * an IF statement is added that clamps all colors if the constant
- * is true.
- */
-static void si_vertex_color_clamping(struct si_shader_context *ctx,
- struct si_shader_output_values *outputs,
- unsigned noutput)
-{
- LLVMValueRef addr[SI_MAX_VS_OUTPUTS][4];
- bool has_colors = false;
-
- /* Store original colors to alloca variables. */
- for (unsigned i = 0; i < noutput; i++) {
- if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
- outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
- continue;
-
- for (unsigned j = 0; j < 4; j++) {
- addr[i][j] = ac_build_alloca_undef(&ctx->ac, ctx->f32, "");
- LLVMBuildStore(ctx->ac.builder, outputs[i].values[j], addr[i][j]);
- }
- has_colors = true;
- }
-
- if (!has_colors)
- return;
-
- /* The state is in the first bit of the user SGPR. */
- LLVMValueRef cond = LLVMGetParam(ctx->main_fn, ctx->param_vs_state_bits);
- cond = LLVMBuildTrunc(ctx->ac.builder, cond, ctx->i1, "");
-
- ac_build_ifcc(&ctx->ac, cond, 6502);
-
- /* Store clamped colors to alloca variables within the conditional block. */
- for (unsigned i = 0; i < noutput; i++) {
- if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
- outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
- continue;
-
- for (unsigned j = 0; j < 4; j++) {
- LLVMBuildStore(ctx->ac.builder,
- ac_build_clamp(&ctx->ac, outputs[i].values[j]),
- addr[i][j]);
- }
- }
- ac_build_endif(&ctx->ac, 6502);
-
- /* Load clamped colors */
- for (unsigned i = 0; i < noutput; i++) {
- if (outputs[i].semantic_name != TGSI_SEMANTIC_COLOR &&
- outputs[i].semantic_name != TGSI_SEMANTIC_BCOLOR)
- continue;
-
- for (unsigned j = 0; j < 4; j++) {
- outputs[i].values[j] =
- LLVMBuildLoad(ctx->ac.builder, addr[i][j], "");
- }
- }
-}
-
-/* Generate export instructions for hardware VS shader stage or NGG GS stage
- * (position and parameter data only).
- */
-void si_llvm_export_vs(struct si_shader_context *ctx,
- struct si_shader_output_values *outputs,
- unsigned noutput)
-{
- struct si_shader *shader = ctx->shader;
- struct ac_export_args pos_args[4] = {};
- LLVMValueRef psize_value = NULL, edgeflag_value = NULL, layer_value = NULL, viewport_index_value = NULL;
- unsigned pos_idx;
- int i;
-
- si_vertex_color_clamping(ctx, outputs, noutput);
-
- /* Build position exports. */
- for (i = 0; i < noutput; i++) {
- switch (outputs[i].semantic_name) {
- case TGSI_SEMANTIC_POSITION:
- si_llvm_init_export_args(ctx, outputs[i].values,
- V_008DFC_SQ_EXP_POS, &pos_args[0]);
- break;
- case TGSI_SEMANTIC_PSIZE:
- psize_value = outputs[i].values[0];
- break;
- case TGSI_SEMANTIC_LAYER:
- layer_value = outputs[i].values[0];
- break;
- case TGSI_SEMANTIC_VIEWPORT_INDEX:
- viewport_index_value = outputs[i].values[0];
- break;
- case TGSI_SEMANTIC_EDGEFLAG:
- edgeflag_value = outputs[i].values[0];
- break;
- case TGSI_SEMANTIC_CLIPDIST:
- if (!shader->key.opt.clip_disable) {
- unsigned index = 2 + outputs[i].semantic_index;
- si_llvm_init_export_args(ctx, outputs[i].values,
- V_008DFC_SQ_EXP_POS + index,
- &pos_args[index]);
- }
- break;
- case TGSI_SEMANTIC_CLIPVERTEX:
- if (!shader->key.opt.clip_disable) {
- si_llvm_emit_clipvertex(ctx, pos_args,
- outputs[i].values);
- }
- break;
- }
- }
-
- /* We need to add the position output manually if it's missing. */
- if (!pos_args[0].out[0]) {
- pos_args[0].enabled_channels = 0xf; /* writemask */
- pos_args[0].valid_mask = 0; /* EXEC mask */
- pos_args[0].done = 0; /* last export? */
- pos_args[0].target = V_008DFC_SQ_EXP_POS;
- pos_args[0].compr = 0; /* COMPR flag */
- pos_args[0].out[0] = ctx->ac.f32_0; /* X */
- pos_args[0].out[1] = ctx->ac.f32_0; /* Y */
- pos_args[0].out[2] = ctx->ac.f32_0; /* Z */
- pos_args[0].out[3] = ctx->ac.f32_1; /* W */
- }
-
- /* Write the misc vector (point size, edgeflag, layer, viewport). */
- if (shader->selector->info.writes_psize ||
- shader->selector->pos_writes_edgeflag ||
- shader->selector->info.writes_viewport_index ||
- shader->selector->info.writes_layer) {
- pos_args[1].enabled_channels = shader->selector->info.writes_psize |
- (shader->selector->pos_writes_edgeflag << 1) |
- (shader->selector->info.writes_layer << 2);
-
- pos_args[1].valid_mask = 0; /* EXEC mask */
- pos_args[1].done = 0; /* last export? */
- pos_args[1].target = V_008DFC_SQ_EXP_POS + 1;
- pos_args[1].compr = 0; /* COMPR flag */
- pos_args[1].out[0] = ctx->ac.f32_0; /* X */
- pos_args[1].out[1] = ctx->ac.f32_0; /* Y */
- pos_args[1].out[2] = ctx->ac.f32_0; /* Z */
- pos_args[1].out[3] = ctx->ac.f32_0; /* W */
-
- if (shader->selector->info.writes_psize)
- pos_args[1].out[0] = psize_value;
-
- if (shader->selector->pos_writes_edgeflag) {
- /* The output is a float, but the hw expects an integer
- * with the first bit containing the edge flag. */
- edgeflag_value = LLVMBuildFPToUI(ctx->ac.builder,
- edgeflag_value,
- ctx->i32, "");
- edgeflag_value = ac_build_umin(&ctx->ac,
- edgeflag_value,
- ctx->i32_1);
-
- /* The LLVM intrinsic expects a float. */
- pos_args[1].out[1] = ac_to_float(&ctx->ac, edgeflag_value);
- }
-
- if (ctx->screen->info.chip_class >= GFX9) {
- /* GFX9 has the layer in out.z[10:0] and the viewport
- * index in out.z[19:16].
- */
- if (shader->selector->info.writes_layer)
- pos_args[1].out[2] = layer_value;
-
- if (shader->selector->info.writes_viewport_index) {
- LLVMValueRef v = viewport_index_value;
-
- v = ac_to_integer(&ctx->ac, v);
- v = LLVMBuildShl(ctx->ac.builder, v,
- LLVMConstInt(ctx->i32, 16, 0), "");
- v = LLVMBuildOr(ctx->ac.builder, v,
- ac_to_integer(&ctx->ac, pos_args[1].out[2]), "");
- pos_args[1].out[2] = ac_to_float(&ctx->ac, v);
- pos_args[1].enabled_channels |= 1 << 2;
- }
- } else {
- if (shader->selector->info.writes_layer)
- pos_args[1].out[2] = layer_value;
-
- if (shader->selector->info.writes_viewport_index) {
- pos_args[1].out[3] = viewport_index_value;
- pos_args[1].enabled_channels |= 1 << 3;
- }
- }
- }
-
- for (i = 0; i < 4; i++)
- if (pos_args[i].out[0])
- shader->info.nr_pos_exports++;
-
- /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
- * Setting valid_mask=1 prevents it and has no other effect.
- */
- if (ctx->screen->info.family == CHIP_NAVI10 ||
- ctx->screen->info.family == CHIP_NAVI12 ||
- ctx->screen->info.family == CHIP_NAVI14)
- pos_args[0].valid_mask = 1;
-
- pos_idx = 0;
- for (i = 0; i < 4; i++) {
- if (!pos_args[i].out[0])
- continue;
-
- /* Specify the target we are exporting */
- pos_args[i].target = V_008DFC_SQ_EXP_POS + pos_idx++;
-
- if (pos_idx == shader->info.nr_pos_exports)
- /* Specify that this is the last export */
- pos_args[i].done = 1;
-
- ac_build_export(&ctx->ac, &pos_args[i]);
- }
-
- /* Build parameter exports. */
- si_build_param_exports(ctx, outputs, noutput);
-}
-
-/**
- * Forward all outputs from the vertex shader to the TES. This is only used
- * for the fixed function TCS.
- */
-static void si_copy_tcs_inputs(struct lp_build_tgsi_context *bld_base)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef invocation_id, buffer, buffer_offset;
- LLVMValueRef lds_vertex_stride, lds_base;
- uint64_t inputs;
-
- invocation_id = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
- buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
- buffer_offset = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
-
- lds_vertex_stride = get_tcs_in_vertex_dw_stride(ctx);
- lds_base = get_tcs_in_current_patch_offset(ctx);
- lds_base = ac_build_imad(&ctx->ac, invocation_id, lds_vertex_stride,
- lds_base);
-
- inputs = ctx->shader->key.mono.u.ff_tcs_inputs_to_copy;
- while (inputs) {
- unsigned i = u_bit_scan64(&inputs);
-
- LLVMValueRef lds_ptr = LLVMBuildAdd(ctx->ac.builder, lds_base,
- LLVMConstInt(ctx->i32, 4 * i, 0),
- "");
-
- LLVMValueRef buffer_addr = get_tcs_tes_buffer_address(ctx,
- get_rel_patch_id(ctx),
- invocation_id,
- LLVMConstInt(ctx->i32, i, 0));
-
- LLVMValueRef value = lshs_lds_load(bld_base, ctx->ac.i32, ~0, lds_ptr);
-
- ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, buffer_addr,
- buffer_offset, 0, ac_glc, false);
- }
-}
-
-static void si_write_tess_factors(struct lp_build_tgsi_context *bld_base,
- LLVMValueRef rel_patch_id,
- LLVMValueRef invocation_id,
- LLVMValueRef tcs_out_current_patch_data_offset,
- LLVMValueRef invoc0_tf_outer[4],
- LLVMValueRef invoc0_tf_inner[2])
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- struct si_shader *shader = ctx->shader;
- unsigned tess_inner_index, tess_outer_index;
- LLVMValueRef lds_base, lds_inner, lds_outer, byteoffset, buffer;
- LLVMValueRef out[6], vec0, vec1, tf_base, inner[4], outer[4];
- unsigned stride, outer_comps, inner_comps, i, offset;
-
- /* Add a barrier before loading tess factors from LDS. */
- if (!shader->key.part.tcs.epilog.invoc0_tess_factors_are_def)
- si_llvm_emit_barrier(NULL, bld_base, NULL);
-
- /* Do this only for invocation 0, because the tess levels are per-patch,
- * not per-vertex.
- *
- * This can't jump, because invocation 0 executes this. It should
- * at least mask out the loads and stores for other invocations.
- */
- ac_build_ifcc(&ctx->ac,
- LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
- invocation_id, ctx->i32_0, ""), 6503);
-
- /* Determine the layout of one tess factor element in the buffer. */
- switch (shader->key.part.tcs.epilog.prim_mode) {
- case PIPE_PRIM_LINES:
- stride = 2; /* 2 dwords, 1 vec2 store */
- outer_comps = 2;
- inner_comps = 0;
- break;
- case PIPE_PRIM_TRIANGLES:
- stride = 4; /* 4 dwords, 1 vec4 store */
- outer_comps = 3;
- inner_comps = 1;
- break;
- case PIPE_PRIM_QUADS:
- stride = 6; /* 6 dwords, 2 stores (vec4 + vec2) */
- outer_comps = 4;
- inner_comps = 2;
- break;
- default:
- assert(0);
- return;
- }
-
- for (i = 0; i < 4; i++) {
- inner[i] = LLVMGetUndef(ctx->i32);
- outer[i] = LLVMGetUndef(ctx->i32);
- }
-
- if (shader->key.part.tcs.epilog.invoc0_tess_factors_are_def) {
- /* Tess factors are in VGPRs. */
- for (i = 0; i < outer_comps; i++)
- outer[i] = out[i] = invoc0_tf_outer[i];
- for (i = 0; i < inner_comps; i++)
- inner[i] = out[outer_comps+i] = invoc0_tf_inner[i];
- } else {
- /* Load tess_inner and tess_outer from LDS.
- * Any invocation can write them, so we can't get them from a temporary.
- */
- tess_inner_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSINNER, 0);
- tess_outer_index = si_shader_io_get_unique_index_patch(TGSI_SEMANTIC_TESSOUTER, 0);
-
- lds_base = tcs_out_current_patch_data_offset;
- lds_inner = LLVMBuildAdd(ctx->ac.builder, lds_base,
- LLVMConstInt(ctx->i32,
- tess_inner_index * 4, 0), "");
- lds_outer = LLVMBuildAdd(ctx->ac.builder, lds_base,
- LLVMConstInt(ctx->i32,
- tess_outer_index * 4, 0), "");
-
- for (i = 0; i < outer_comps; i++) {
- outer[i] = out[i] =
- lshs_lds_load(bld_base, ctx->ac.i32, i, lds_outer);
- }
- for (i = 0; i < inner_comps; i++) {
- inner[i] = out[outer_comps+i] =
- lshs_lds_load(bld_base, ctx->ac.i32, i, lds_inner);
- }
- }
-
- if (shader->key.part.tcs.epilog.prim_mode == PIPE_PRIM_LINES) {
- /* For isolines, the hardware expects tess factors in the
- * reverse order from what GLSL / TGSI specify.
- */
- LLVMValueRef tmp = out[0];
- out[0] = out[1];
- out[1] = tmp;
- }
-
- /* Convert the outputs to vectors for stores. */
- vec0 = ac_build_gather_values(&ctx->ac, out, MIN2(stride, 4));
- vec1 = NULL;
-
- if (stride > 4)
- vec1 = ac_build_gather_values(&ctx->ac, out+4, stride - 4);
-
- /* Get the buffer. */
- buffer = get_tess_ring_descriptor(ctx, TCS_FACTOR_RING);
-
- /* Get the offset. */
- tf_base = LLVMGetParam(ctx->main_fn,
- ctx->param_tcs_factor_offset);
- byteoffset = LLVMBuildMul(ctx->ac.builder, rel_patch_id,
- LLVMConstInt(ctx->i32, 4 * stride, 0), "");
-
- ac_build_ifcc(&ctx->ac,
- LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ,
- rel_patch_id, ctx->i32_0, ""), 6504);
-
- /* Store the dynamic HS control word. */
- offset = 0;
- if (ctx->screen->info.chip_class <= GFX8) {
- ac_build_buffer_store_dword(&ctx->ac, buffer,
- LLVMConstInt(ctx->i32, 0x80000000, 0),
- 1, ctx->i32_0, tf_base,
- offset, ac_glc, false);
- offset += 4;
- }
-
- ac_build_endif(&ctx->ac, 6504);
-
- /* Store the tessellation factors. */
- ac_build_buffer_store_dword(&ctx->ac, buffer, vec0,
- MIN2(stride, 4), byteoffset, tf_base,
- offset, ac_glc, false);
- offset += 16;
- if (vec1)
- ac_build_buffer_store_dword(&ctx->ac, buffer, vec1,
- stride - 4, byteoffset, tf_base,
- offset, ac_glc, false);
-
- /* Store the tess factors into the offchip buffer if TES reads them. */
- if (shader->key.part.tcs.epilog.tes_reads_tess_factors) {
- LLVMValueRef buf, base, inner_vec, outer_vec, tf_outer_offset;
- LLVMValueRef tf_inner_offset;
- unsigned param_outer, param_inner;
-
- buf = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
- base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
-
- param_outer = si_shader_io_get_unique_index_patch(
- TGSI_SEMANTIC_TESSOUTER, 0);
- tf_outer_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
- LLVMConstInt(ctx->i32, param_outer, 0));
-
- unsigned outer_vec_size =
- ac_has_vec3_support(ctx->screen->info.chip_class, false) ?
- outer_comps : util_next_power_of_two(outer_comps);
- outer_vec = ac_build_gather_values(&ctx->ac, outer, outer_vec_size);
-
- ac_build_buffer_store_dword(&ctx->ac, buf, outer_vec,
- outer_comps, tf_outer_offset,
- base, 0, ac_glc, false);
- if (inner_comps) {
- param_inner = si_shader_io_get_unique_index_patch(
- TGSI_SEMANTIC_TESSINNER, 0);
- tf_inner_offset = get_tcs_tes_buffer_address(ctx, rel_patch_id, NULL,
- LLVMConstInt(ctx->i32, param_inner, 0));
-
- inner_vec = inner_comps == 1 ? inner[0] :
- ac_build_gather_values(&ctx->ac, inner, inner_comps);
- ac_build_buffer_store_dword(&ctx->ac, buf, inner_vec,
- inner_comps, tf_inner_offset,
- base, 0, ac_glc, false);
- }
- }
-
- ac_build_endif(&ctx->ac, 6503);
-}
-
-static LLVMValueRef
-si_insert_input_ret(struct si_shader_context *ctx, LLVMValueRef ret,
- unsigned param, unsigned return_index)
-{
- return LLVMBuildInsertValue(ctx->ac.builder, ret,
- LLVMGetParam(ctx->main_fn, param),
- return_index, "");
-}
-
-static LLVMValueRef
-si_insert_input_ret_float(struct si_shader_context *ctx, LLVMValueRef ret,
- unsigned param, unsigned return_index)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef p = LLVMGetParam(ctx->main_fn, param);
-
- return LLVMBuildInsertValue(builder, ret,
- ac_to_float(&ctx->ac, p),
- return_index, "");
-}
-
-static LLVMValueRef
-si_insert_input_ptr(struct si_shader_context *ctx, LLVMValueRef ret,
- unsigned param, unsigned return_index)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef ptr = LLVMGetParam(ctx->main_fn, param);
- ptr = LLVMBuildPtrToInt(builder, ptr, ctx->i32, "");
- return LLVMBuildInsertValue(builder, ret, ptr, return_index, "");
-}
-
-/* This only writes the tessellation factor levels. */
-static void si_llvm_emit_tcs_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct lp_build_tgsi_context *bld_base = &ctx->bld_base;
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef rel_patch_id, invocation_id, tf_lds_offset;
-
- si_copy_tcs_inputs(bld_base);
-
- rel_patch_id = get_rel_patch_id(ctx);
- invocation_id = unpack_llvm_param(ctx, ctx->abi.tcs_rel_ids, 8, 5);
- tf_lds_offset = get_tcs_out_current_patch_data_offset(ctx);
-
- if (ctx->screen->info.chip_class >= GFX9) {
- LLVMBasicBlockRef blocks[2] = {
- LLVMGetInsertBlock(builder),
- ctx->merged_wrap_if_entry_block
- };
- LLVMValueRef values[2];
-
- ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
-
- values[0] = rel_patch_id;
- values[1] = LLVMGetUndef(ctx->i32);
- rel_patch_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
-
- values[0] = tf_lds_offset;
- values[1] = LLVMGetUndef(ctx->i32);
- tf_lds_offset = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
-
- values[0] = invocation_id;
- values[1] = ctx->i32_1; /* cause the epilog to skip threads */
- invocation_id = ac_build_phi(&ctx->ac, ctx->i32, 2, values, blocks);
- }
-
- /* Return epilog parameters from this function. */
- LLVMValueRef ret = ctx->return_value;
- unsigned vgpr;
-
- if (ctx->screen->info.chip_class >= GFX9) {
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
- 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
- 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
- /* Tess offchip and tess factor offsets are at the beginning. */
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
- vgpr = 8 + GFX9_SGPR_TCS_OUT_LAYOUT + 1;
- } else {
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
- GFX6_SGPR_TCS_OFFCHIP_LAYOUT);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
- GFX6_SGPR_TCS_OUT_LAYOUT);
- /* Tess offchip and tess factor offsets are after user SGPRs. */
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset,
- GFX6_TCS_NUM_USER_SGPR);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset,
- GFX6_TCS_NUM_USER_SGPR + 1);
- vgpr = GFX6_TCS_NUM_USER_SGPR + 2;
- }
-
- /* VGPRs */
- rel_patch_id = ac_to_float(&ctx->ac, rel_patch_id);
- invocation_id = ac_to_float(&ctx->ac, invocation_id);
- tf_lds_offset = ac_to_float(&ctx->ac, tf_lds_offset);
-
- /* Leave a hole corresponding to the two input VGPRs. This ensures that
- * the invocation_id output does not alias the tcs_rel_ids input,
- * which saves a V_MOV on gfx9.
- */
- vgpr += 2;
-
- ret = LLVMBuildInsertValue(builder, ret, rel_patch_id, vgpr++, "");
- ret = LLVMBuildInsertValue(builder, ret, invocation_id, vgpr++, "");
-
- if (ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
- vgpr++; /* skip the tess factor LDS offset */
- for (unsigned i = 0; i < 6; i++) {
- LLVMValueRef value =
- LLVMBuildLoad(builder, ctx->invoc0_tess_factors[i], "");
- value = ac_to_float(&ctx->ac, value);
- ret = LLVMBuildInsertValue(builder, ret, value, vgpr++, "");
- }
- } else {
- ret = LLVMBuildInsertValue(builder, ret, tf_lds_offset, vgpr++, "");
- }
- ctx->return_value = ret;
-}
-
-/* Pass TCS inputs from LS to TCS on GFX9. */
-static void si_set_ls_return_value_for_tcs(struct si_shader_context *ctx)
-{
- LLVMValueRef ret = ctx->return_value;
-
- ret = si_insert_input_ptr(ctx, ret, 0, 0);
- ret = si_insert_input_ptr(ctx, ret, 1, 1);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_offset, 2);
- ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_factor_offset, 4);
- ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
-
- ret = si_insert_input_ptr(ctx, ret, ctx->param_rw_buffers,
- 8 + SI_SGPR_RW_BUFFERS);
- ret = si_insert_input_ptr(ctx, ret,
- ctx->param_bindless_samplers_and_images,
- 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
-
- ret = si_insert_input_ret(ctx, ret, ctx->param_vs_state_bits,
- 8 + SI_SGPR_VS_STATE_BITS);
-
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_offchip_layout,
- 8 + GFX9_SGPR_TCS_OFFCHIP_LAYOUT);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_offsets,
- 8 + GFX9_SGPR_TCS_OUT_OFFSETS);
- ret = si_insert_input_ret(ctx, ret, ctx->param_tcs_out_lds_layout,
- 8 + GFX9_SGPR_TCS_OUT_LAYOUT);
-
- unsigned vgpr = 8 + GFX9_TCS_NUM_USER_SGPR;
- ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
- ac_to_float(&ctx->ac, ctx->abi.tcs_patch_id),
- vgpr++, "");
- ret = LLVMBuildInsertValue(ctx->ac.builder, ret,
- ac_to_float(&ctx->ac, ctx->abi.tcs_rel_ids),
- vgpr++, "");
- ctx->return_value = ret;
-}
-
-/* Pass GS inputs from ES to GS on GFX9. */
-static void si_set_es_return_value_for_gs(struct si_shader_context *ctx)
-{
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef ret = ctx->return_value;
-
- ret = si_insert_input_ptr(ctx, ret, 0, 0);
- ret = si_insert_input_ptr(ctx, ret, 1, 1);
- if (ctx->shader->key.as_ngg)
- ret = LLVMBuildInsertValue(builder, ret, ctx->gs_tg_info, 2, "");
- else
- ret = si_insert_input_ret(ctx, ret, ctx->param_gs2vs_offset, 2);
- ret = si_insert_input_ret(ctx, ret, ctx->param_merged_wave_info, 3);
- ret = si_insert_input_ret(ctx, ret, ctx->param_merged_scratch_offset, 5);
-
- ret = si_insert_input_ptr(ctx, ret, ctx->param_rw_buffers,
- 8 + SI_SGPR_RW_BUFFERS);
- ret = si_insert_input_ptr(ctx, ret,
- ctx->param_bindless_samplers_and_images,
- 8 + SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES);
- if (ctx->screen->use_ngg) {
- ret = si_insert_input_ptr(ctx, ret, ctx->param_vs_state_bits,
- 8 + SI_SGPR_VS_STATE_BITS);
- }
-
- unsigned vgpr;
- if (ctx->type == PIPE_SHADER_VERTEX)
- vgpr = 8 + GFX9_VSGS_NUM_USER_SGPR;
- else
- vgpr = 8 + GFX9_TESGS_NUM_USER_SGPR;
-
- for (unsigned i = 0; i < 5; i++) {
- unsigned param = ctx->param_gs_vtx01_offset + i;
- ret = si_insert_input_ret_float(ctx, ret, param, vgpr++);
- }
- ctx->return_value = ret;
-}
-
-static void si_llvm_emit_ls_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader *shader = ctx->shader;
- struct tgsi_shader_info *info = &shader->selector->info;
- unsigned i, chan;
- LLVMValueRef vertex_id = LLVMGetParam(ctx->main_fn,
- ctx->param_rel_auto_id);
- LLVMValueRef vertex_dw_stride = get_tcs_in_vertex_dw_stride(ctx);
- LLVMValueRef base_dw_addr = LLVMBuildMul(ctx->ac.builder, vertex_id,
- vertex_dw_stride, "");
-
- /* Write outputs to LDS. The next shader (TCS aka HS) will read
- * its inputs from it. */
- for (i = 0; i < info->num_outputs; i++) {
- unsigned name = info->output_semantic_name[i];
- unsigned index = info->output_semantic_index[i];
-
- /* The ARB_shader_viewport_layer_array spec contains the
- * following issue:
- *
- * 2) What happens if gl_ViewportIndex or gl_Layer is
- * written in the vertex shader and a geometry shader is
- * present?
- *
- * RESOLVED: The value written by the last vertex processing
- * stage is used. If the last vertex processing stage
- * (vertex, tessellation evaluation or geometry) does not
- * statically assign to gl_ViewportIndex or gl_Layer, index
- * or layer zero is assumed.
- *
- * So writes to those outputs in VS-as-LS are simply ignored.
- */
- if (name == TGSI_SEMANTIC_LAYER ||
- name == TGSI_SEMANTIC_VIEWPORT_INDEX)
- continue;
-
- int param = si_shader_io_get_unique_index(name, index, false);
- LLVMValueRef dw_addr = LLVMBuildAdd(ctx->ac.builder, base_dw_addr,
- LLVMConstInt(ctx->i32, param * 4, 0), "");
-
- for (chan = 0; chan < 4; chan++) {
- if (!(info->output_usagemask[i] & (1 << chan)))
- continue;
-
- lshs_lds_store(ctx, chan, dw_addr,
- LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], ""));
- }
- }
-
- if (ctx->screen->info.chip_class >= GFX9)
- si_set_ls_return_value_for_tcs(ctx);
-}
-
-static void si_llvm_emit_es_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader *es = ctx->shader;
- struct tgsi_shader_info *info = &es->selector->info;
- LLVMValueRef soffset = LLVMGetParam(ctx->main_fn,
- ctx->param_es2gs_offset);
- LLVMValueRef lds_base = NULL;
- unsigned chan;
- int i;
-
- if (ctx->screen->info.chip_class >= GFX9 && info->num_outputs) {
- unsigned itemsize_dw = es->selector->esgs_itemsize / 4;
- LLVMValueRef vertex_idx = ac_get_thread_id(&ctx->ac);
- LLVMValueRef wave_idx = si_unpack_param(ctx, ctx->param_merged_wave_info, 24, 4);
- vertex_idx = LLVMBuildOr(ctx->ac.builder, vertex_idx,
- LLVMBuildMul(ctx->ac.builder, wave_idx,
- LLVMConstInt(ctx->i32, ctx->ac.wave_size, false), ""), "");
- lds_base = LLVMBuildMul(ctx->ac.builder, vertex_idx,
- LLVMConstInt(ctx->i32, itemsize_dw, 0), "");
- }
-
- for (i = 0; i < info->num_outputs; i++) {
- int param;
-
- if (info->output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX ||
- info->output_semantic_name[i] == TGSI_SEMANTIC_LAYER)
- continue;
-
- param = si_shader_io_get_unique_index(info->output_semantic_name[i],
- info->output_semantic_index[i], false);
-
- for (chan = 0; chan < 4; chan++) {
- if (!(info->output_usagemask[i] & (1 << chan)))
- continue;
-
- LLVMValueRef out_val = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
- out_val = ac_to_integer(&ctx->ac, out_val);
-
- /* GFX9 has the ESGS ring in LDS. */
- if (ctx->screen->info.chip_class >= GFX9) {
- LLVMValueRef idx = LLVMConstInt(ctx->i32, param * 4 + chan, false);
- idx = LLVMBuildAdd(ctx->ac.builder, lds_base, idx, "");
- ac_build_indexed_store(&ctx->ac, ctx->esgs_ring, idx, out_val);
- continue;
- }
-
- ac_build_buffer_store_dword(&ctx->ac,
- ctx->esgs_ring,
- out_val, 1, NULL, soffset,
- (4 * param + chan) * 4,
- ac_glc | ac_slc, true);
- }
- }
-
- if (ctx->screen->info.chip_class >= GFX9)
- si_set_es_return_value_for_gs(ctx);
-}
-
-static LLVMValueRef si_get_gs_wave_id(struct si_shader_context *ctx)
-{
- if (ctx->screen->info.chip_class >= GFX9)
- return si_unpack_param(ctx, ctx->param_merged_wave_info, 16, 8);
- else
- return LLVMGetParam(ctx->main_fn, ctx->param_gs_wave_id);
-}
-
-static void emit_gs_epilogue(struct si_shader_context *ctx)
-{
- if (ctx->shader->key.as_ngg) {
- gfx10_ngg_gs_emit_epilogue(ctx);
- return;
- }
-
- if (ctx->screen->info.chip_class >= GFX10)
- LLVMBuildFence(ctx->ac.builder, LLVMAtomicOrderingRelease, false, "");
-
- ac_build_sendmsg(&ctx->ac, AC_SENDMSG_GS_OP_NOP | AC_SENDMSG_GS_DONE,
- si_get_gs_wave_id(ctx));
-
- if (ctx->screen->info.chip_class >= GFX9)
- ac_build_endif(&ctx->ac, ctx->merged_wrap_if_label);
-}
-
-static void si_llvm_emit_gs_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct tgsi_shader_info UNUSED *info = &ctx->shader->selector->info;
-
- assert(info->num_outputs <= max_outputs);
-
- emit_gs_epilogue(ctx);
-}
-
-static void si_tgsi_emit_gs_epilogue(struct lp_build_tgsi_context *bld_base)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- emit_gs_epilogue(ctx);
-}
-
-static void si_llvm_emit_vs_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct tgsi_shader_info *info = &ctx->shader->selector->info;
- struct si_shader_output_values *outputs = NULL;
- int i,j;
-
- assert(!ctx->shader->is_gs_copy_shader);
- assert(info->num_outputs <= max_outputs);
-
- outputs = MALLOC((info->num_outputs + 1) * sizeof(outputs[0]));
-
- for (i = 0; i < info->num_outputs; i++) {
- outputs[i].semantic_name = info->output_semantic_name[i];
- outputs[i].semantic_index = info->output_semantic_index[i];
-
- for (j = 0; j < 4; j++) {
- outputs[i].values[j] =
- LLVMBuildLoad(ctx->ac.builder,
- addrs[4 * i + j],
- "");
- outputs[i].vertex_stream[j] =
- (info->output_streams[i] >> (2 * j)) & 3;
- }
- }
-
- if (!ctx->screen->use_ngg_streamout &&
- ctx->shader->selector->so.num_outputs)
- si_llvm_emit_streamout(ctx, outputs, i, 0);
-
- /* Export PrimitiveID. */
- if (ctx->shader->key.mono.u.vs_export_prim_id) {
- outputs[i].semantic_name = TGSI_SEMANTIC_PRIMID;
- outputs[i].semantic_index = 0;
- outputs[i].values[0] = ac_to_float(&ctx->ac, si_get_primitive_id(ctx, 0));
- for (j = 1; j < 4; j++)
- outputs[i].values[j] = LLVMConstReal(ctx->f32, 0);
-
- memset(outputs[i].vertex_stream, 0,
- sizeof(outputs[i].vertex_stream));
- i++;
- }
-
- si_llvm_export_vs(ctx, outputs, i);
- FREE(outputs);
-}
-
-static void si_llvm_emit_prim_discard_cs_epilogue(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct tgsi_shader_info *info = &ctx->shader->selector->info;
- LLVMValueRef pos[4] = {};
-
- assert(info->num_outputs <= max_outputs);
-
- for (unsigned i = 0; i < info->num_outputs; i++) {
- if (info->output_semantic_name[i] != TGSI_SEMANTIC_POSITION)
- continue;
-
- for (unsigned chan = 0; chan < 4; chan++)
- pos[chan] = LLVMBuildLoad(ctx->ac.builder, addrs[4 * i + chan], "");
- break;
- }
- assert(pos[0] != NULL);
-
- /* Return the position output. */
- LLVMValueRef ret = ctx->return_value;
- for (unsigned chan = 0; chan < 4; chan++)
- ret = LLVMBuildInsertValue(ctx->ac.builder, ret, pos[chan], chan, "");
- ctx->return_value = ret;
-}
-
-static void si_tgsi_emit_epilogue(struct lp_build_tgsi_context *bld_base)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
-
- ctx->abi.emit_outputs(&ctx->abi, RADEON_LLVM_MAX_OUTPUTS,
- &ctx->outputs[0][0]);
-}
-
-struct si_ps_exports {
- unsigned num;
- struct ac_export_args args[10];
-};
-
-static void si_export_mrt_z(struct lp_build_tgsi_context *bld_base,
- LLVMValueRef depth, LLVMValueRef stencil,
- LLVMValueRef samplemask, struct si_ps_exports *exp)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- struct ac_export_args args;
-
- ac_export_mrt_z(&ctx->ac, depth, stencil, samplemask, &args);
-
- memcpy(&exp->args[exp->num++], &args, sizeof(args));
-}
-
-static void si_export_mrt_color(struct lp_build_tgsi_context *bld_base,
- LLVMValueRef *color, unsigned index,
- unsigned samplemask_param,
- bool is_last, struct si_ps_exports *exp)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- int i;
-
- /* Clamp color */
- if (ctx->shader->key.part.ps.epilog.clamp_color)
- for (i = 0; i < 4; i++)
- color[i] = ac_build_clamp(&ctx->ac, color[i]);
-
- /* Alpha to one */
- if (ctx->shader->key.part.ps.epilog.alpha_to_one)
- color[3] = ctx->ac.f32_1;
-
- /* Alpha test */
- if (index == 0 &&
- ctx->shader->key.part.ps.epilog.alpha_func != PIPE_FUNC_ALWAYS)
- si_alpha_test(bld_base, color[3]);
-
- /* Line & polygon smoothing */
- if (ctx->shader->key.part.ps.epilog.poly_line_smoothing)
- color[3] = si_scale_alpha_by_sample_mask(bld_base, color[3],
- samplemask_param);
-
- /* If last_cbuf > 0, FS_COLOR0_WRITES_ALL_CBUFS is true. */
- if (ctx->shader->key.part.ps.epilog.last_cbuf > 0) {
- struct ac_export_args args[8];
- int c, last = -1;
-
- /* Get the export arguments, also find out what the last one is. */
- for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
- si_llvm_init_export_args(ctx, color,
- V_008DFC_SQ_EXP_MRT + c, &args[c]);
- if (args[c].enabled_channels)
- last = c;
- }
-
- /* Emit all exports. */
- for (c = 0; c <= ctx->shader->key.part.ps.epilog.last_cbuf; c++) {
- if (is_last && last == c) {
- args[c].valid_mask = 1; /* whether the EXEC mask is valid */
- args[c].done = 1; /* DONE bit */
- } else if (!args[c].enabled_channels)
- continue; /* unnecessary NULL export */
-
- memcpy(&exp->args[exp->num++], &args[c], sizeof(args[c]));
- }
- } else {
- struct ac_export_args args;
-
- /* Export */
- si_llvm_init_export_args(ctx, color, V_008DFC_SQ_EXP_MRT + index,
- &args);
- if (is_last) {
- args.valid_mask = 1; /* whether the EXEC mask is valid */
- args.done = 1; /* DONE bit */
- } else if (!args.enabled_channels)
- return; /* unnecessary NULL export */
-
- memcpy(&exp->args[exp->num++], &args, sizeof(args));
- }
-}
-
-static void si_emit_ps_exports(struct si_shader_context *ctx,
- struct si_ps_exports *exp)
-{
- for (unsigned i = 0; i < exp->num; i++)
- ac_build_export(&ctx->ac, &exp->args[i]);
-}
-
-/**
- * Return PS outputs in this order:
- *
- * v[0:3] = color0.xyzw
- * v[4:7] = color1.xyzw
- * ...
- * vN+0 = Depth
- * vN+1 = Stencil
- * vN+2 = SampleMask
- * vN+3 = SampleMaskIn (used for OpenGL smoothing)
- *
- * The alpha-ref SGPR is returned via its original location.
- */
-static void si_llvm_return_fs_outputs(struct ac_shader_abi *abi,
- unsigned max_outputs,
- LLVMValueRef *addrs)
-{
- struct si_shader_context *ctx = si_shader_context_from_abi(abi);
- struct si_shader *shader = ctx->shader;
- struct tgsi_shader_info *info = &shader->selector->info;
- LLVMBuilderRef builder = ctx->ac.builder;
- unsigned i, j, first_vgpr, vgpr;
-
- LLVMValueRef color[8][4] = {};
- LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
- LLVMValueRef ret;
-
- if (ctx->postponed_kill)
- ac_build_kill_if_false(&ctx->ac, LLVMBuildLoad(builder, ctx->postponed_kill, ""));
-
- /* Read the output values. */
- for (i = 0; i < info->num_outputs; i++) {
- unsigned semantic_name = info->output_semantic_name[i];
- unsigned semantic_index = info->output_semantic_index[i];
-
- switch (semantic_name) {
- case TGSI_SEMANTIC_COLOR:
- assert(semantic_index < 8);
- for (j = 0; j < 4; j++) {
- LLVMValueRef ptr = addrs[4 * i + j];
- LLVMValueRef result = LLVMBuildLoad(builder, ptr, "");
- color[semantic_index][j] = result;
- }
- break;