+static void
+calculate_gs_ring_sizes(struct radv_pipeline *pipeline)
+{
+ struct radv_device *device = pipeline->device;
+ unsigned num_se = device->physical_device->rad_info.max_se;
+ unsigned wave_size = 64;
+ unsigned max_gs_waves = 32 * num_se; /* max 32 per SE on GCN */
+ unsigned gs_vertex_reuse = 16 * num_se; /* GS_VERTEX_REUSE register (per SE) */
+ unsigned alignment = 256 * num_se;
+ /* The maximum size is 63.999 MB per SE. */
+ unsigned max_size = ((unsigned)(63.999 * 1024 * 1024) & ~255) * num_se;
+ struct ac_shader_variant_info *gs_info = &pipeline->shaders[MESA_SHADER_GEOMETRY]->info;
+ struct ac_es_output_info *es_info = radv_pipeline_has_tess(pipeline) ?
+ &pipeline->shaders[MESA_SHADER_TESS_EVAL]->info.tes.es_info :
+ &pipeline->shaders[MESA_SHADER_VERTEX]->info.vs.es_info;
+
+ /* Calculate the minimum size. */
+ unsigned min_esgs_ring_size = align(es_info->esgs_itemsize * gs_vertex_reuse *
+ wave_size, alignment);
+ /* These are recommended sizes, not minimum sizes. */
+ unsigned esgs_ring_size = max_gs_waves * 2 * wave_size *
+ es_info->esgs_itemsize * gs_info->gs.vertices_in;
+ unsigned gsvs_ring_size = max_gs_waves * 2 * wave_size *
+ gs_info->gs.max_gsvs_emit_size * 1; // no streams in VK (gs->max_gs_stream + 1);
+
+ min_esgs_ring_size = align(min_esgs_ring_size, alignment);
+ esgs_ring_size = align(esgs_ring_size, alignment);
+ gsvs_ring_size = align(gsvs_ring_size, alignment);
+
+ pipeline->graphics.esgs_ring_size = CLAMP(esgs_ring_size, min_esgs_ring_size, max_size);
+ pipeline->graphics.gsvs_ring_size = MIN2(gsvs_ring_size, max_size);
+}
+
+static void si_multiwave_lds_size_workaround(struct radv_device *device,
+ unsigned *lds_size)
+{
+ /* SPI barrier management bug:
+ * Make sure we have at least 4k of LDS in use to avoid the bug.
+ * It applies to workgroup sizes of more than one wavefront.
+ */
+ if (device->physical_device->rad_info.family == CHIP_BONAIRE ||
+ device->physical_device->rad_info.family == CHIP_KABINI ||
+ device->physical_device->rad_info.family == CHIP_MULLINS)
+ *lds_size = MAX2(*lds_size, 8);
+}
+
+static void
+calculate_tess_state(struct radv_pipeline *pipeline,
+ const VkGraphicsPipelineCreateInfo *pCreateInfo)
+{
+ unsigned num_tcs_input_cp = pCreateInfo->pTessellationState->patchControlPoints;
+ unsigned num_tcs_output_cp, num_tcs_inputs, num_tcs_outputs;
+ unsigned num_tcs_patch_outputs;
+ unsigned input_vertex_size, output_vertex_size, pervertex_output_patch_size;
+ unsigned input_patch_size, output_patch_size, output_patch0_offset;
+ unsigned lds_size, hardware_lds_size;
+ unsigned perpatch_output_offset;
+ unsigned num_patches;
+ struct radv_tessellation_state *tess = &pipeline->graphics.tess;
+
+ /* This calculates how shader inputs and outputs among VS, TCS, and TES
+ * are laid out in LDS. */
+ num_tcs_inputs = util_last_bit64(pipeline->shaders[MESA_SHADER_VERTEX]->info.vs.outputs_written);
+
+ num_tcs_outputs = util_last_bit64(pipeline->shaders[MESA_SHADER_TESS_CTRL]->info.tcs.outputs_written); //tcs->outputs_written
+ num_tcs_output_cp = pipeline->shaders[MESA_SHADER_TESS_CTRL]->info.tcs.tcs_vertices_out; //TCS VERTICES OUT
+ num_tcs_patch_outputs = util_last_bit64(pipeline->shaders[MESA_SHADER_TESS_CTRL]->info.tcs.patch_outputs_written);
+
+ /* Ensure that we only need one wave per SIMD so we don't need to check
+ * resource usage. Also ensures that the number of tcs in and out
+ * vertices per threadgroup are at most 256.
+ */
+ input_vertex_size = num_tcs_inputs * 16;
+ output_vertex_size = num_tcs_outputs * 16;
+
+ input_patch_size = num_tcs_input_cp * input_vertex_size;
+
+ pervertex_output_patch_size = num_tcs_output_cp * output_vertex_size;
+ output_patch_size = pervertex_output_patch_size + num_tcs_patch_outputs * 16;
+ /* Ensure that we only need one wave per SIMD so we don't need to check
+ * resource usage. Also ensures that the number of tcs in and out
+ * vertices per threadgroup are at most 256.
+ */
+ num_patches = 64 / MAX2(num_tcs_input_cp, num_tcs_output_cp) * 4;
+
+ /* Make sure that the data fits in LDS. This assumes the shaders only
+ * use LDS for the inputs and outputs.
+ */
+ hardware_lds_size = pipeline->device->physical_device->rad_info.chip_class >= CIK ? 65536 : 32768;
+ num_patches = MIN2(num_patches, hardware_lds_size / (input_patch_size + output_patch_size));
+
+ /* Make sure the output data fits in the offchip buffer */
+ num_patches = MIN2(num_patches,
+ (pipeline->device->tess_offchip_block_dw_size * 4) /
+ output_patch_size);
+
+ /* Not necessary for correctness, but improves performance. The
+ * specific value is taken from the proprietary driver.
+ */
+ num_patches = MIN2(num_patches, 40);
+
+ /* SI bug workaround - limit LS-HS threadgroups to only one wave. */
+ if (pipeline->device->physical_device->rad_info.chip_class == SI) {
+ unsigned one_wave = 64 / MAX2(num_tcs_input_cp, num_tcs_output_cp);
+ num_patches = MIN2(num_patches, one_wave);
+ }
+
+ output_patch0_offset = input_patch_size * num_patches;
+ perpatch_output_offset = output_patch0_offset + pervertex_output_patch_size;
+
+ lds_size = output_patch0_offset + output_patch_size * num_patches;
+
+ if (pipeline->device->physical_device->rad_info.chip_class >= CIK) {
+ assert(lds_size <= 65536);
+ lds_size = align(lds_size, 512) / 512;
+ } else {
+ assert(lds_size <= 32768);
+ lds_size = align(lds_size, 256) / 256;
+ }
+ si_multiwave_lds_size_workaround(pipeline->device, &lds_size);
+
+ tess->lds_size = lds_size;
+
+ tess->tcs_in_layout = (input_patch_size / 4) |
+ ((input_vertex_size / 4) << 13);
+ tess->tcs_out_layout = (output_patch_size / 4) |
+ ((output_vertex_size / 4) << 13);
+ tess->tcs_out_offsets = (output_patch0_offset / 16) |
+ ((perpatch_output_offset / 16) << 16);
+ tess->offchip_layout = (pervertex_output_patch_size * num_patches << 16) |
+ (num_tcs_output_cp << 9) | num_patches;
+
+ tess->ls_hs_config = S_028B58_NUM_PATCHES(num_patches) |
+ S_028B58_HS_NUM_INPUT_CP(num_tcs_input_cp) |
+ S_028B58_HS_NUM_OUTPUT_CP(num_tcs_output_cp);
+ tess->num_patches = num_patches;
+ tess->num_tcs_input_cp = num_tcs_input_cp;
+
+ struct radv_shader_variant *tes = pipeline->shaders[MESA_SHADER_TESS_EVAL];
+ unsigned type = 0, partitioning = 0, topology = 0, distribution_mode = 0;
+
+ switch (tes->info.tes.primitive_mode) {
+ case GL_TRIANGLES:
+ type = V_028B6C_TESS_TRIANGLE;
+ break;
+ case GL_QUADS:
+ type = V_028B6C_TESS_QUAD;
+ break;
+ case GL_ISOLINES:
+ type = V_028B6C_TESS_ISOLINE;
+ break;
+ }
+
+ switch (tes->info.tes.spacing) {
+ case TESS_SPACING_EQUAL:
+ partitioning = V_028B6C_PART_INTEGER;
+ break;
+ case TESS_SPACING_FRACTIONAL_ODD:
+ partitioning = V_028B6C_PART_FRAC_ODD;
+ break;
+ case TESS_SPACING_FRACTIONAL_EVEN:
+ partitioning = V_028B6C_PART_FRAC_EVEN;
+ break;
+ default:
+ break;
+ }
+
+ bool ccw = tes->info.tes.ccw;
+ const VkPipelineTessellationDomainOriginStateCreateInfoKHR *domain_origin_state =
+ vk_find_struct_const(pCreateInfo->pTessellationState,
+ PIPELINE_TESSELLATION_DOMAIN_ORIGIN_STATE_CREATE_INFO_KHR);
+
+ if (domain_origin_state && domain_origin_state->domainOrigin != VK_TESSELLATION_DOMAIN_ORIGIN_UPPER_LEFT_KHR)
+ ccw = !ccw;
+
+ if (tes->info.tes.point_mode)
+ topology = V_028B6C_OUTPUT_POINT;
+ else if (tes->info.tes.primitive_mode == GL_ISOLINES)
+ topology = V_028B6C_OUTPUT_LINE;
+ else if (ccw)
+ topology = V_028B6C_OUTPUT_TRIANGLE_CCW;
+ else
+ topology = V_028B6C_OUTPUT_TRIANGLE_CW;
+
+ if (pipeline->device->has_distributed_tess) {
+ if (pipeline->device->physical_device->rad_info.family == CHIP_FIJI ||
+ pipeline->device->physical_device->rad_info.family >= CHIP_POLARIS10)
+ distribution_mode = V_028B6C_DISTRIBUTION_MODE_TRAPEZOIDS;
+ else
+ distribution_mode = V_028B6C_DISTRIBUTION_MODE_DONUTS;
+ } else
+ distribution_mode = V_028B6C_DISTRIBUTION_MODE_NO_DIST;
+
+ tess->tf_param = S_028B6C_TYPE(type) |
+ S_028B6C_PARTITIONING(partitioning) |
+ S_028B6C_TOPOLOGY(topology) |
+ S_028B6C_DISTRIBUTION_MODE(distribution_mode);
+}
+
+static const struct radv_prim_vertex_count prim_size_table[] = {
+ [V_008958_DI_PT_NONE] = {0, 0},
+ [V_008958_DI_PT_POINTLIST] = {1, 1},
+ [V_008958_DI_PT_LINELIST] = {2, 2},
+ [V_008958_DI_PT_LINESTRIP] = {2, 1},
+ [V_008958_DI_PT_TRILIST] = {3, 3},
+ [V_008958_DI_PT_TRIFAN] = {3, 1},
+ [V_008958_DI_PT_TRISTRIP] = {3, 1},
+ [V_008958_DI_PT_LINELIST_ADJ] = {4, 4},
+ [V_008958_DI_PT_LINESTRIP_ADJ] = {4, 1},
+ [V_008958_DI_PT_TRILIST_ADJ] = {6, 6},
+ [V_008958_DI_PT_TRISTRIP_ADJ] = {6, 2},
+ [V_008958_DI_PT_RECTLIST] = {3, 3},
+ [V_008958_DI_PT_LINELOOP] = {2, 1},
+ [V_008958_DI_PT_POLYGON] = {3, 1},
+ [V_008958_DI_PT_2D_TRI_STRIP] = {0, 0},
+};
+
+static uint32_t si_vgt_gs_mode(struct radv_shader_variant *gs)
+{
+ unsigned gs_max_vert_out = gs->info.gs.vertices_out;
+ unsigned cut_mode;
+
+ if (gs_max_vert_out <= 128) {
+ cut_mode = V_028A40_GS_CUT_128;
+ } else if (gs_max_vert_out <= 256) {
+ cut_mode = V_028A40_GS_CUT_256;
+ } else if (gs_max_vert_out <= 512) {
+ cut_mode = V_028A40_GS_CUT_512;
+ } else {
+ assert(gs_max_vert_out <= 1024);
+ cut_mode = V_028A40_GS_CUT_1024;
+ }
+
+ return S_028A40_MODE(V_028A40_GS_SCENARIO_G) |
+ S_028A40_CUT_MODE(cut_mode)|
+ S_028A40_ES_WRITE_OPTIMIZE(1) |
+ S_028A40_GS_WRITE_OPTIMIZE(1);
+}
+
+static void calculate_vgt_gs_mode(struct radv_pipeline *pipeline)
+{
+ struct radv_shader_variant *vs;
+ vs = radv_pipeline_has_gs(pipeline) ? pipeline->gs_copy_shader : (radv_pipeline_has_tess(pipeline) ? pipeline->shaders[MESA_SHADER_TESS_EVAL] : pipeline->shaders[MESA_SHADER_VERTEX]);
+
+ struct ac_vs_output_info *outinfo = &vs->info.vs.outinfo;
+
+ pipeline->graphics.vgt_primitiveid_en = false;
+ pipeline->graphics.vgt_gs_mode = 0;
+
+ if (radv_pipeline_has_gs(pipeline)) {
+ pipeline->graphics.vgt_gs_mode = si_vgt_gs_mode(pipeline->shaders[MESA_SHADER_GEOMETRY]);
+ } else if (outinfo->export_prim_id) {
+ pipeline->graphics.vgt_gs_mode = S_028A40_MODE(V_028A40_GS_SCENARIO_A);
+ pipeline->graphics.vgt_primitiveid_en = true;
+ }
+}
+
+static void calculate_pa_cl_vs_out_cntl(struct radv_pipeline *pipeline)
+{
+ struct radv_shader_variant *vs;
+ vs = radv_pipeline_has_gs(pipeline) ? pipeline->gs_copy_shader : (radv_pipeline_has_tess(pipeline) ? pipeline->shaders[MESA_SHADER_TESS_EVAL] : pipeline->shaders[MESA_SHADER_VERTEX]);
+
+ struct ac_vs_output_info *outinfo = &vs->info.vs.outinfo;
+
+ unsigned clip_dist_mask, cull_dist_mask, total_mask;
+ clip_dist_mask = outinfo->clip_dist_mask;
+ cull_dist_mask = outinfo->cull_dist_mask;
+ total_mask = clip_dist_mask | cull_dist_mask;
+
+ bool misc_vec_ena = outinfo->writes_pointsize ||
+ outinfo->writes_layer ||
+ outinfo->writes_viewport_index;
+ pipeline->graphics.pa_cl_vs_out_cntl =
+ S_02881C_USE_VTX_POINT_SIZE(outinfo->writes_pointsize) |
+ S_02881C_USE_VTX_RENDER_TARGET_INDX(outinfo->writes_layer) |
+ S_02881C_USE_VTX_VIEWPORT_INDX(outinfo->writes_viewport_index) |
+ S_02881C_VS_OUT_MISC_VEC_ENA(misc_vec_ena) |
+ S_02881C_VS_OUT_MISC_SIDE_BUS_ENA(misc_vec_ena) |
+ S_02881C_VS_OUT_CCDIST0_VEC_ENA((total_mask & 0x0f) != 0) |
+ S_02881C_VS_OUT_CCDIST1_VEC_ENA((total_mask & 0xf0) != 0) |
+ cull_dist_mask << 8 |
+ clip_dist_mask;
+
+}
+
+static uint32_t offset_to_ps_input(uint32_t offset, bool flat_shade)
+{
+ uint32_t ps_input_cntl;
+ if (offset <= AC_EXP_PARAM_OFFSET_31) {
+ ps_input_cntl = S_028644_OFFSET(offset);
+ if (flat_shade)
+ ps_input_cntl |= S_028644_FLAT_SHADE(1);
+ } else {
+ /* The input is a DEFAULT_VAL constant. */
+ assert(offset >= AC_EXP_PARAM_DEFAULT_VAL_0000 &&
+ offset <= AC_EXP_PARAM_DEFAULT_VAL_1111);
+ offset -= AC_EXP_PARAM_DEFAULT_VAL_0000;
+ ps_input_cntl = S_028644_OFFSET(0x20) |
+ S_028644_DEFAULT_VAL(offset);
+ }
+ return ps_input_cntl;
+}
+
+static void calculate_ps_inputs(struct radv_pipeline *pipeline)
+{
+ struct radv_shader_variant *ps, *vs;
+ struct ac_vs_output_info *outinfo;
+
+ ps = pipeline->shaders[MESA_SHADER_FRAGMENT];
+ vs = radv_pipeline_has_gs(pipeline) ? pipeline->gs_copy_shader : (radv_pipeline_has_tess(pipeline) ? pipeline->shaders[MESA_SHADER_TESS_EVAL] : pipeline->shaders[MESA_SHADER_VERTEX]);
+
+ outinfo = &vs->info.vs.outinfo;
+
+ unsigned ps_offset = 0;
+
+ if (ps->info.fs.prim_id_input) {
+ unsigned vs_offset = outinfo->vs_output_param_offset[VARYING_SLOT_PRIMITIVE_ID];
+ if (vs_offset != AC_EXP_PARAM_UNDEFINED) {
+ pipeline->graphics.ps_input_cntl[ps_offset] = offset_to_ps_input(vs_offset, true);
+ ++ps_offset;
+ }
+ }
+
+ if (ps->info.fs.layer_input) {
+ unsigned vs_offset = outinfo->vs_output_param_offset[VARYING_SLOT_LAYER];
+ if (vs_offset != AC_EXP_PARAM_UNDEFINED)
+ pipeline->graphics.ps_input_cntl[ps_offset] = offset_to_ps_input(vs_offset, true);
+ else
+ pipeline->graphics.ps_input_cntl[ps_offset] = offset_to_ps_input(AC_EXP_PARAM_DEFAULT_VAL_0000, true);
+ ++ps_offset;
+ }
+
+ if (ps->info.fs.has_pcoord) {
+ unsigned val;
+ val = S_028644_PT_SPRITE_TEX(1) | S_028644_OFFSET(0x20);
+ pipeline->graphics.ps_input_cntl[ps_offset] = val;
+ ps_offset++;
+ }
+
+ for (unsigned i = 0; i < 32 && (1u << i) <= ps->info.fs.input_mask; ++i) {
+ unsigned vs_offset;
+ bool flat_shade;
+ if (!(ps->info.fs.input_mask & (1u << i)))
+ continue;
+
+ vs_offset = outinfo->vs_output_param_offset[VARYING_SLOT_VAR0 + i];
+ if (vs_offset == AC_EXP_PARAM_UNDEFINED) {
+ pipeline->graphics.ps_input_cntl[ps_offset] = S_028644_OFFSET(0x20);
+ ++ps_offset;
+ continue;
+ }
+
+ flat_shade = !!(ps->info.fs.flat_shaded_mask & (1u << ps_offset));
+
+ pipeline->graphics.ps_input_cntl[ps_offset] = offset_to_ps_input(vs_offset, flat_shade);
+ ++ps_offset;
+ }
+
+ pipeline->graphics.ps_input_cntl_num = ps_offset;
+}
+
+static void
+radv_link_shaders(struct radv_pipeline *pipeline, nir_shader **shaders)
+{
+ nir_shader* ordered_shaders[MESA_SHADER_STAGES];
+ int shader_count = 0;
+
+ if(shaders[MESA_SHADER_FRAGMENT]) {
+ ordered_shaders[shader_count++] = shaders[MESA_SHADER_FRAGMENT];
+ }
+ if(shaders[MESA_SHADER_GEOMETRY]) {
+ ordered_shaders[shader_count++] = shaders[MESA_SHADER_GEOMETRY];
+ }
+ if(shaders[MESA_SHADER_TESS_EVAL]) {
+ ordered_shaders[shader_count++] = shaders[MESA_SHADER_TESS_EVAL];
+ }
+ if(shaders[MESA_SHADER_TESS_CTRL]) {
+ ordered_shaders[shader_count++] = shaders[MESA_SHADER_TESS_CTRL];
+ }
+ if(shaders[MESA_SHADER_VERTEX]) {
+ ordered_shaders[shader_count++] = shaders[MESA_SHADER_VERTEX];
+ }
+
+ for (int i = 1; i < shader_count; ++i) {
+ nir_remove_dead_variables(ordered_shaders[i],
+ nir_var_shader_out);
+ nir_remove_dead_variables(ordered_shaders[i - 1],
+ nir_var_shader_in);
+
+ bool progress = nir_remove_unused_varyings(ordered_shaders[i],
+ ordered_shaders[i - 1]);
+
+ if (progress) {
+ nir_lower_global_vars_to_local(ordered_shaders[i]);
+ radv_optimize_nir(ordered_shaders[i]);
+ nir_lower_global_vars_to_local(ordered_shaders[i - 1]);
+ radv_optimize_nir(ordered_shaders[i - 1]);
+ }
+ }
+}
+
+static
+void radv_create_shaders(struct radv_pipeline *pipeline,
+ struct radv_device *device,
+ struct radv_pipeline_cache *cache,
+ struct ac_shader_variant_key *keys,
+ const VkPipelineShaderStageCreateInfo **pStages)
+{
+ struct radv_shader_module fs_m = {0};
+ struct radv_shader_module *modules[MESA_SHADER_STAGES] = { 0, };
+ nir_shader *nir[MESA_SHADER_STAGES] = {0};
+ void *codes[MESA_SHADER_STAGES] = {0};
+ unsigned code_sizes[MESA_SHADER_STAGES] = {0};
+ unsigned char hash[20], gs_copy_hash[20];
+
+ for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) {
+ if (pStages[i]) {
+ modules[i] = radv_shader_module_from_handle(pStages[i]->module);
+ if (modules[i]->nir)
+ _mesa_sha1_compute(modules[i]->nir->info.name,
+ strlen(modules[i]->nir->info.name),
+ modules[i]->sha1);
+ }
+ }
+
+ radv_hash_shaders(hash, pStages, pipeline->layout, keys, get_hash_flags(device));
+ memcpy(gs_copy_hash, hash, 20);
+ gs_copy_hash[0] ^= 1;
+
+ if (modules[MESA_SHADER_GEOMETRY]) {
+ struct radv_shader_variant *variants[MESA_SHADER_STAGES] = {0};
+ radv_create_shader_variants_from_pipeline_cache(device, cache, gs_copy_hash, variants);
+ pipeline->gs_copy_shader = variants[MESA_SHADER_GEOMETRY];
+ }
+
+ if (radv_create_shader_variants_from_pipeline_cache(device, cache, hash, pipeline->shaders) &&
+ (!modules[MESA_SHADER_GEOMETRY] || pipeline->gs_copy_shader))
+ return;
+
+ if (!modules[MESA_SHADER_FRAGMENT] && !modules[MESA_SHADER_COMPUTE]) {
+ nir_builder fs_b;
+ nir_builder_init_simple_shader(&fs_b, NULL, MESA_SHADER_FRAGMENT, NULL);
+ fs_b.shader->info.name = ralloc_strdup(fs_b.shader, "noop_fs");
+ fs_m.nir = fs_b.shader;
+ modules[MESA_SHADER_FRAGMENT] = &fs_m;
+ }
+
+ for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) {
+ const VkPipelineShaderStageCreateInfo *stage = pStages[i];
+
+ if (!modules[i])
+ continue;
+
+ nir[i] = radv_shader_compile_to_nir(device, modules[i],
+ stage ? stage->pName : "main", i,
+ stage ? stage->pSpecializationInfo : NULL);
+ pipeline->active_stages |= mesa_to_vk_shader_stage(i);
+ }
+
+ if (nir[MESA_SHADER_TESS_CTRL]) {
+ /* TODO: This is no longer used as a key we should refactor this */
+ if (keys)
+ keys[MESA_SHADER_TESS_CTRL].tcs.primitive_mode = nir[MESA_SHADER_TESS_EVAL]->info.tess.primitive_mode;
+
+ nir_lower_tes_patch_vertices(nir[MESA_SHADER_TESS_EVAL], nir[MESA_SHADER_TESS_CTRL]->info.tess.tcs_vertices_out);
+ }
+
+ radv_link_shaders(pipeline, nir);
+
+ if (nir[MESA_SHADER_FRAGMENT]) {
+ pipeline->shaders[MESA_SHADER_FRAGMENT] =
+ radv_shader_variant_create(device, modules[MESA_SHADER_FRAGMENT], nir[MESA_SHADER_FRAGMENT],
+ pipeline->layout, keys ? keys + MESA_SHADER_FRAGMENT : 0,
+ &codes[MESA_SHADER_FRAGMENT], &code_sizes[MESA_SHADER_FRAGMENT]);
+
+ /* TODO: These are no longer used as keys we should refactor this */
+ if (keys) {
+ keys[MESA_SHADER_VERTEX].vs.export_prim_id =
+ pipeline->shaders[MESA_SHADER_FRAGMENT]->info.fs.prim_id_input;
+ keys[MESA_SHADER_TESS_EVAL].tes.export_prim_id =
+ pipeline->shaders[MESA_SHADER_FRAGMENT]->info.fs.prim_id_input;
+ }
+
+ pipeline->active_stages |= mesa_to_vk_shader_stage(MESA_SHADER_FRAGMENT);
+ }
+
+ for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
+ if(modules[i] && !pipeline->shaders[i]) {
+ pipeline->shaders[i] = radv_shader_variant_create(device, modules[i], nir[i],
+ pipeline->layout,
+ keys ? keys + i : 0, &codes[i],
+ &code_sizes[i]);
+
+ pipeline->active_stages |= mesa_to_vk_shader_stage(i);
+ }
+ }
+
+ if(modules[MESA_SHADER_GEOMETRY]) {
+ void *gs_copy_code = NULL;
+ unsigned gs_copy_code_size = 0;
+ if (!pipeline->gs_copy_shader) {
+ pipeline->gs_copy_shader = radv_create_gs_copy_shader(
+ device, nir[MESA_SHADER_GEOMETRY], &gs_copy_code,
+ &gs_copy_code_size,
+ keys[MESA_SHADER_GEOMETRY].has_multiview_view_index);
+ }
+
+ if (pipeline->gs_copy_shader) {
+ void *code[MESA_SHADER_STAGES] = {0};
+ unsigned code_size[MESA_SHADER_STAGES] = {0};
+ struct radv_shader_variant *variants[MESA_SHADER_STAGES] = {0};
+
+ code[MESA_SHADER_GEOMETRY] = gs_copy_code;
+ code_size[MESA_SHADER_GEOMETRY] = gs_copy_code_size;
+ variants[MESA_SHADER_GEOMETRY] = pipeline->gs_copy_shader;
+
+ radv_pipeline_cache_insert_shaders(device, cache,
+ gs_copy_hash,
+ variants,
+ (const void**)code,
+ code_size);
+ }
+ free(gs_copy_code);
+ }
+
+ radv_pipeline_cache_insert_shaders(device, cache, hash, pipeline->shaders,
+ (const void**)codes, code_sizes);
+
+ for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
+ free(codes[i]);
+ if (modules[i] && !modules[i]->nir && !pipeline->device->trace_bo)
+ ralloc_free(nir[i]);
+ }
+
+ if (fs_m.nir)
+ ralloc_free(fs_m.nir);
+}
+
+static VkResult