+ base, 0, ac_glc, false);
+ }
+}
+
+static void si_nir_store_output_tcs(struct ac_shader_abi *abi,
+ const struct nir_variable *var,
+ LLVMValueRef vertex_index,
+ LLVMValueRef param_index,
+ unsigned const_index,
+ LLVMValueRef src,
+ unsigned writemask)
+{
+ struct si_shader_context *ctx = si_shader_context_from_abi(abi);
+ struct tgsi_shader_info *info = &ctx->shader->selector->info;
+ const unsigned component = var->data.location_frac;
+ const bool is_patch = var->data.patch;
+ unsigned driver_location = var->data.driver_location;
+ LLVMValueRef dw_addr, stride;
+ LLVMValueRef buffer, base, addr;
+ LLVMValueRef values[8];
+ bool skip_lds_store;
+ bool is_tess_factor = false, is_tess_inner = false;
+
+ driver_location = driver_location / 4;
+
+ if (param_index) {
+ /* Add the constant index to the indirect index */
+ param_index = LLVMBuildAdd(ctx->ac.builder, param_index,
+ LLVMConstInt(ctx->i32, const_index, 0), "");
+ } else {
+ if (const_index != 0)
+ param_index = LLVMConstInt(ctx->i32, const_index, 0);
+ }
+
+ if (!is_patch) {
+ stride = get_tcs_out_vertex_dw_stride(ctx);
+ dw_addr = get_tcs_out_current_patch_offset(ctx);
+ dw_addr = get_dw_address_from_generic_indices(ctx, stride, dw_addr,
+ vertex_index, param_index,
+ driver_location,
+ info->output_semantic_name,
+ info->output_semantic_index,
+ is_patch);
+
+ skip_lds_store = !info->reads_pervertex_outputs;
+ } else {
+ dw_addr = get_tcs_out_current_patch_data_offset(ctx);
+ dw_addr = get_dw_address_from_generic_indices(ctx, NULL, dw_addr,
+ vertex_index, param_index,
+ driver_location,
+ info->output_semantic_name,
+ info->output_semantic_index,
+ is_patch);
+
+ skip_lds_store = !info->reads_perpatch_outputs;
+
+ if (!param_index) {
+ int name = info->output_semantic_name[driver_location];
+
+ /* Always write tess factors into LDS for the TCS epilog. */
+ if (name == TGSI_SEMANTIC_TESSINNER ||
+ name == TGSI_SEMANTIC_TESSOUTER) {
+ /* The epilog doesn't read LDS if invocation 0 defines tess factors. */
+ skip_lds_store = !info->reads_tessfactor_outputs &&
+ ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs;
+ is_tess_factor = true;
+ is_tess_inner = name == TGSI_SEMANTIC_TESSINNER;
+ }
+ }
+ }
+
+ buffer = get_tess_ring_descriptor(ctx, TESS_OFFCHIP_RING_TCS);
+
+ base = LLVMGetParam(ctx->main_fn, ctx->param_tcs_offchip_offset);
+
+ addr = get_tcs_tes_buffer_address_from_generic_indices(ctx, vertex_index,
+ param_index, driver_location,
+ info->output_semantic_name,
+ info->output_semantic_index,
+ is_patch);
+
+ for (unsigned chan = 0; chan < 8; chan++) {
+ if (!(writemask & (1 << chan)))
+ continue;
+ LLVMValueRef value = ac_llvm_extract_elem(&ctx->ac, src, chan - component);
+
+ unsigned buffer_store_offset = chan % 4;
+ if (chan == 4) {
+ addr = get_tcs_tes_buffer_address_from_generic_indices(ctx,
+ vertex_index,
+ param_index,
+ driver_location + 1,
+ info->output_semantic_name,
+ info->output_semantic_index,
+ is_patch);
+ }
+
+ /* Skip LDS stores if there is no LDS read of this output. */
+ if (!skip_lds_store)
+ lshs_lds_store(ctx, chan, dw_addr, value);
+
+ value = ac_to_integer(&ctx->ac, value);
+ values[chan] = value;
+
+ if (writemask != 0xF && !is_tess_factor) {
+ ac_build_buffer_store_dword(&ctx->ac, buffer, value, 1,
+ addr, base,
+ 4 * buffer_store_offset,
+ ac_glc, false);
+ }
+
+ /* Write tess factors into VGPRs for the epilog. */
+ if (is_tess_factor &&
+ ctx->shader->selector->tcs_info.tessfactors_are_def_in_all_invocs) {
+ if (!is_tess_inner) {
+ LLVMBuildStore(ctx->ac.builder, value, /* outer */
+ ctx->invoc0_tess_factors[chan]);
+ } else if (chan < 2) {
+ LLVMBuildStore(ctx->ac.builder, value, /* inner */
+ ctx->invoc0_tess_factors[4 + chan]);
+ }
+ }
+ }
+
+ if (writemask == 0xF && !is_tess_factor) {
+ LLVMValueRef value = ac_build_gather_values(&ctx->ac,
+ values, 4);
+ ac_build_buffer_store_dword(&ctx->ac, buffer, value, 4, addr,
+ base, 0, ac_glc, false);