anv: Pre-compute push ranges for graphics pipelines
[mesa.git] / src / intel / vulkan / anv_pipeline.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <fcntl.h>
29
30 #include "util/mesa-sha1.h"
31 #include "util/os_time.h"
32 #include "common/gen_l3_config.h"
33 #include "common/gen_disasm.h"
34 #include "anv_private.h"
35 #include "compiler/brw_nir.h"
36 #include "anv_nir.h"
37 #include "nir/nir_xfb_info.h"
38 #include "spirv/nir_spirv.h"
39 #include "vk_util.h"
40
41 /* Needed for SWIZZLE macros */
42 #include "program/prog_instruction.h"
43
44 // Shader functions
45
46 VkResult anv_CreateShaderModule(
47 VkDevice _device,
48 const VkShaderModuleCreateInfo* pCreateInfo,
49 const VkAllocationCallbacks* pAllocator,
50 VkShaderModule* pShaderModule)
51 {
52 ANV_FROM_HANDLE(anv_device, device, _device);
53 struct anv_shader_module *module;
54
55 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO);
56 assert(pCreateInfo->flags == 0);
57
58 module = vk_alloc2(&device->alloc, pAllocator,
59 sizeof(*module) + pCreateInfo->codeSize, 8,
60 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
61 if (module == NULL)
62 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
63
64 module->size = pCreateInfo->codeSize;
65 memcpy(module->data, pCreateInfo->pCode, module->size);
66
67 _mesa_sha1_compute(module->data, module->size, module->sha1);
68
69 *pShaderModule = anv_shader_module_to_handle(module);
70
71 return VK_SUCCESS;
72 }
73
74 void anv_DestroyShaderModule(
75 VkDevice _device,
76 VkShaderModule _module,
77 const VkAllocationCallbacks* pAllocator)
78 {
79 ANV_FROM_HANDLE(anv_device, device, _device);
80 ANV_FROM_HANDLE(anv_shader_module, module, _module);
81
82 if (!module)
83 return;
84
85 vk_free2(&device->alloc, pAllocator, module);
86 }
87
88 #define SPIR_V_MAGIC_NUMBER 0x07230203
89
90 static const uint64_t stage_to_debug[] = {
91 [MESA_SHADER_VERTEX] = DEBUG_VS,
92 [MESA_SHADER_TESS_CTRL] = DEBUG_TCS,
93 [MESA_SHADER_TESS_EVAL] = DEBUG_TES,
94 [MESA_SHADER_GEOMETRY] = DEBUG_GS,
95 [MESA_SHADER_FRAGMENT] = DEBUG_WM,
96 [MESA_SHADER_COMPUTE] = DEBUG_CS,
97 };
98
99 struct anv_spirv_debug_data {
100 struct anv_device *device;
101 const struct anv_shader_module *module;
102 };
103
104 static void anv_spirv_nir_debug(void *private_data,
105 enum nir_spirv_debug_level level,
106 size_t spirv_offset,
107 const char *message)
108 {
109 struct anv_spirv_debug_data *debug_data = private_data;
110 static const VkDebugReportFlagsEXT vk_flags[] = {
111 [NIR_SPIRV_DEBUG_LEVEL_INFO] = VK_DEBUG_REPORT_INFORMATION_BIT_EXT,
112 [NIR_SPIRV_DEBUG_LEVEL_WARNING] = VK_DEBUG_REPORT_WARNING_BIT_EXT,
113 [NIR_SPIRV_DEBUG_LEVEL_ERROR] = VK_DEBUG_REPORT_ERROR_BIT_EXT,
114 };
115 char buffer[256];
116
117 snprintf(buffer, sizeof(buffer), "SPIR-V offset %lu: %s", (unsigned long) spirv_offset, message);
118
119 vk_debug_report(&debug_data->device->instance->debug_report_callbacks,
120 vk_flags[level],
121 VK_DEBUG_REPORT_OBJECT_TYPE_SHADER_MODULE_EXT,
122 (uint64_t) (uintptr_t) debug_data->module,
123 0, 0, "anv", buffer);
124 }
125
126 /* Eventually, this will become part of anv_CreateShader. Unfortunately,
127 * we can't do that yet because we don't have the ability to copy nir.
128 */
129 static nir_shader *
130 anv_shader_compile_to_nir(struct anv_device *device,
131 void *mem_ctx,
132 const struct anv_shader_module *module,
133 const char *entrypoint_name,
134 gl_shader_stage stage,
135 const VkSpecializationInfo *spec_info)
136 {
137 const struct anv_physical_device *pdevice =
138 &device->instance->physicalDevice;
139 const struct brw_compiler *compiler = pdevice->compiler;
140 const nir_shader_compiler_options *nir_options =
141 compiler->glsl_compiler_options[stage].NirOptions;
142
143 uint32_t *spirv = (uint32_t *) module->data;
144 assert(spirv[0] == SPIR_V_MAGIC_NUMBER);
145 assert(module->size % 4 == 0);
146
147 uint32_t num_spec_entries = 0;
148 struct nir_spirv_specialization *spec_entries = NULL;
149 if (spec_info && spec_info->mapEntryCount > 0) {
150 num_spec_entries = spec_info->mapEntryCount;
151 spec_entries = malloc(num_spec_entries * sizeof(*spec_entries));
152 for (uint32_t i = 0; i < num_spec_entries; i++) {
153 VkSpecializationMapEntry entry = spec_info->pMapEntries[i];
154 const void *data = spec_info->pData + entry.offset;
155 assert(data + entry.size <= spec_info->pData + spec_info->dataSize);
156
157 spec_entries[i].id = spec_info->pMapEntries[i].constantID;
158 if (spec_info->dataSize == 8)
159 spec_entries[i].data64 = *(const uint64_t *)data;
160 else
161 spec_entries[i].data32 = *(const uint32_t *)data;
162 }
163 }
164
165 struct anv_spirv_debug_data spirv_debug_data = {
166 .device = device,
167 .module = module,
168 };
169 struct spirv_to_nir_options spirv_options = {
170 .frag_coord_is_sysval = true,
171 .use_scoped_memory_barrier = true,
172 .caps = {
173 .demote_to_helper_invocation = true,
174 .derivative_group = true,
175 .descriptor_array_dynamic_indexing = true,
176 .descriptor_array_non_uniform_indexing = true,
177 .descriptor_indexing = true,
178 .device_group = true,
179 .draw_parameters = true,
180 .float16 = pdevice->info.gen >= 8,
181 .float64 = pdevice->info.gen >= 8,
182 .fragment_shader_sample_interlock = pdevice->info.gen >= 9,
183 .fragment_shader_pixel_interlock = pdevice->info.gen >= 9,
184 .geometry_streams = true,
185 .image_write_without_format = true,
186 .int8 = pdevice->info.gen >= 8,
187 .int16 = pdevice->info.gen >= 8,
188 .int64 = pdevice->info.gen >= 8,
189 .int64_atomics = pdevice->info.gen >= 9 && pdevice->use_softpin,
190 .min_lod = true,
191 .multiview = true,
192 .physical_storage_buffer_address = pdevice->has_a64_buffer_access,
193 .post_depth_coverage = pdevice->info.gen >= 9,
194 .runtime_descriptor_array = true,
195 .float_controls = pdevice->info.gen >= 8,
196 .shader_clock = true,
197 .shader_viewport_index_layer = true,
198 .stencil_export = pdevice->info.gen >= 9,
199 .storage_8bit = pdevice->info.gen >= 8,
200 .storage_16bit = pdevice->info.gen >= 8,
201 .subgroup_arithmetic = true,
202 .subgroup_basic = true,
203 .subgroup_ballot = true,
204 .subgroup_quad = true,
205 .subgroup_shuffle = true,
206 .subgroup_vote = true,
207 .tessellation = true,
208 .transform_feedback = pdevice->info.gen >= 8,
209 .variable_pointers = true,
210 .vk_memory_model = true,
211 .vk_memory_model_device_scope = true,
212 },
213 .ubo_addr_format = nir_address_format_32bit_index_offset,
214 .ssbo_addr_format =
215 anv_nir_ssbo_addr_format(pdevice, device->robust_buffer_access),
216 .phys_ssbo_addr_format = nir_address_format_64bit_global,
217 .push_const_addr_format = nir_address_format_logical,
218
219 /* TODO: Consider changing this to an address format that has the NULL
220 * pointer equals to 0. That might be a better format to play nice
221 * with certain code / code generators.
222 */
223 .shared_addr_format = nir_address_format_32bit_offset,
224 .debug = {
225 .func = anv_spirv_nir_debug,
226 .private_data = &spirv_debug_data,
227 },
228 };
229
230
231 nir_shader *nir =
232 spirv_to_nir(spirv, module->size / 4,
233 spec_entries, num_spec_entries,
234 stage, entrypoint_name, &spirv_options, nir_options);
235 assert(nir->info.stage == stage);
236 nir_validate_shader(nir, "after spirv_to_nir");
237 ralloc_steal(mem_ctx, nir);
238
239 free(spec_entries);
240
241 if (unlikely(INTEL_DEBUG & stage_to_debug[stage])) {
242 fprintf(stderr, "NIR (from SPIR-V) for %s shader:\n",
243 gl_shader_stage_name(stage));
244 nir_print_shader(nir, stderr);
245 }
246
247 /* We have to lower away local constant initializers right before we
248 * inline functions. That way they get properly initialized at the top
249 * of the function and not at the top of its caller.
250 */
251 NIR_PASS_V(nir, nir_lower_constant_initializers, nir_var_function_temp);
252 NIR_PASS_V(nir, nir_lower_returns);
253 NIR_PASS_V(nir, nir_inline_functions);
254 NIR_PASS_V(nir, nir_opt_deref);
255
256 /* Pick off the single entrypoint that we want */
257 foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
258 if (!func->is_entrypoint)
259 exec_node_remove(&func->node);
260 }
261 assert(exec_list_length(&nir->functions) == 1);
262
263 /* Now that we've deleted all but the main function, we can go ahead and
264 * lower the rest of the constant initializers. We do this here so that
265 * nir_remove_dead_variables and split_per_member_structs below see the
266 * corresponding stores.
267 */
268 NIR_PASS_V(nir, nir_lower_constant_initializers, ~0);
269
270 /* Split member structs. We do this before lower_io_to_temporaries so that
271 * it doesn't lower system values to temporaries by accident.
272 */
273 NIR_PASS_V(nir, nir_split_var_copies);
274 NIR_PASS_V(nir, nir_split_per_member_structs);
275
276 NIR_PASS_V(nir, nir_remove_dead_variables,
277 nir_var_shader_in | nir_var_shader_out | nir_var_system_value);
278
279 NIR_PASS_V(nir, nir_propagate_invariant);
280 NIR_PASS_V(nir, nir_lower_io_to_temporaries,
281 nir_shader_get_entrypoint(nir), true, false);
282
283 NIR_PASS_V(nir, nir_lower_frexp);
284
285 /* Vulkan uses the separate-shader linking model */
286 nir->info.separate_shader = true;
287
288 brw_preprocess_nir(compiler, nir, NULL);
289
290 return nir;
291 }
292
293 void anv_DestroyPipeline(
294 VkDevice _device,
295 VkPipeline _pipeline,
296 const VkAllocationCallbacks* pAllocator)
297 {
298 ANV_FROM_HANDLE(anv_device, device, _device);
299 ANV_FROM_HANDLE(anv_pipeline, pipeline, _pipeline);
300
301 if (!pipeline)
302 return;
303
304 anv_reloc_list_finish(&pipeline->batch_relocs,
305 pAllocator ? pAllocator : &device->alloc);
306
307 ralloc_free(pipeline->mem_ctx);
308
309 if (pipeline->blend_state.map)
310 anv_state_pool_free(&device->dynamic_state_pool, pipeline->blend_state);
311
312 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
313 if (pipeline->shaders[s])
314 anv_shader_bin_unref(device, pipeline->shaders[s]);
315 }
316
317 vk_free2(&device->alloc, pAllocator, pipeline);
318 }
319
320 static const uint32_t vk_to_gen_primitive_type[] = {
321 [VK_PRIMITIVE_TOPOLOGY_POINT_LIST] = _3DPRIM_POINTLIST,
322 [VK_PRIMITIVE_TOPOLOGY_LINE_LIST] = _3DPRIM_LINELIST,
323 [VK_PRIMITIVE_TOPOLOGY_LINE_STRIP] = _3DPRIM_LINESTRIP,
324 [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST] = _3DPRIM_TRILIST,
325 [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP] = _3DPRIM_TRISTRIP,
326 [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN] = _3DPRIM_TRIFAN,
327 [VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY] = _3DPRIM_LINELIST_ADJ,
328 [VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY] = _3DPRIM_LINESTRIP_ADJ,
329 [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY] = _3DPRIM_TRILIST_ADJ,
330 [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY] = _3DPRIM_TRISTRIP_ADJ,
331 };
332
333 static void
334 populate_sampler_prog_key(const struct gen_device_info *devinfo,
335 struct brw_sampler_prog_key_data *key)
336 {
337 /* Almost all multisampled textures are compressed. The only time when we
338 * don't compress a multisampled texture is for 16x MSAA with a surface
339 * width greater than 8k which is a bit of an edge case. Since the sampler
340 * just ignores the MCS parameter to ld2ms when MCS is disabled, it's safe
341 * to tell the compiler to always assume compression.
342 */
343 key->compressed_multisample_layout_mask = ~0;
344
345 /* SkyLake added support for 16x MSAA. With this came a new message for
346 * reading from a 16x MSAA surface with compression. The new message was
347 * needed because now the MCS data is 64 bits instead of 32 or lower as is
348 * the case for 8x, 4x, and 2x. The key->msaa_16 bit-field controls which
349 * message we use. Fortunately, the 16x message works for 8x, 4x, and 2x
350 * so we can just use it unconditionally. This may not be quite as
351 * efficient but it saves us from recompiling.
352 */
353 if (devinfo->gen >= 9)
354 key->msaa_16 = ~0;
355
356 /* XXX: Handle texture swizzle on HSW- */
357 for (int i = 0; i < MAX_SAMPLERS; i++) {
358 /* Assume color sampler, no swizzling. (Works for BDW+) */
359 key->swizzles[i] = SWIZZLE_XYZW;
360 }
361 }
362
363 static void
364 populate_base_prog_key(const struct gen_device_info *devinfo,
365 VkPipelineShaderStageCreateFlags flags,
366 struct brw_base_prog_key *key)
367 {
368 if (flags & VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT_EXT)
369 key->subgroup_size_type = BRW_SUBGROUP_SIZE_VARYING;
370 else
371 key->subgroup_size_type = BRW_SUBGROUP_SIZE_API_CONSTANT;
372
373 populate_sampler_prog_key(devinfo, &key->tex);
374 }
375
376 static void
377 populate_vs_prog_key(const struct gen_device_info *devinfo,
378 VkPipelineShaderStageCreateFlags flags,
379 struct brw_vs_prog_key *key)
380 {
381 memset(key, 0, sizeof(*key));
382
383 populate_base_prog_key(devinfo, flags, &key->base);
384
385 /* XXX: Handle vertex input work-arounds */
386
387 /* XXX: Handle sampler_prog_key */
388 }
389
390 static void
391 populate_tcs_prog_key(const struct gen_device_info *devinfo,
392 VkPipelineShaderStageCreateFlags flags,
393 unsigned input_vertices,
394 struct brw_tcs_prog_key *key)
395 {
396 memset(key, 0, sizeof(*key));
397
398 populate_base_prog_key(devinfo, flags, &key->base);
399
400 key->input_vertices = input_vertices;
401 }
402
403 static void
404 populate_tes_prog_key(const struct gen_device_info *devinfo,
405 VkPipelineShaderStageCreateFlags flags,
406 struct brw_tes_prog_key *key)
407 {
408 memset(key, 0, sizeof(*key));
409
410 populate_base_prog_key(devinfo, flags, &key->base);
411 }
412
413 static void
414 populate_gs_prog_key(const struct gen_device_info *devinfo,
415 VkPipelineShaderStageCreateFlags flags,
416 struct brw_gs_prog_key *key)
417 {
418 memset(key, 0, sizeof(*key));
419
420 populate_base_prog_key(devinfo, flags, &key->base);
421 }
422
423 static void
424 populate_wm_prog_key(const struct gen_device_info *devinfo,
425 VkPipelineShaderStageCreateFlags flags,
426 const struct anv_subpass *subpass,
427 const VkPipelineMultisampleStateCreateInfo *ms_info,
428 struct brw_wm_prog_key *key)
429 {
430 memset(key, 0, sizeof(*key));
431
432 populate_base_prog_key(devinfo, flags, &key->base);
433
434 /* We set this to 0 here and set to the actual value before we call
435 * brw_compile_fs.
436 */
437 key->input_slots_valid = 0;
438
439 /* Vulkan doesn't specify a default */
440 key->high_quality_derivatives = false;
441
442 /* XXX Vulkan doesn't appear to specify */
443 key->clamp_fragment_color = false;
444
445 assert(subpass->color_count <= MAX_RTS);
446 for (uint32_t i = 0; i < subpass->color_count; i++) {
447 if (subpass->color_attachments[i].attachment != VK_ATTACHMENT_UNUSED)
448 key->color_outputs_valid |= (1 << i);
449 }
450
451 key->nr_color_regions = subpass->color_count;
452
453 /* To reduce possible shader recompilations we would need to know if
454 * there is a SampleMask output variable to compute if we should emit
455 * code to workaround the issue that hardware disables alpha to coverage
456 * when there is SampleMask output.
457 */
458 key->alpha_to_coverage = ms_info && ms_info->alphaToCoverageEnable;
459
460 /* Vulkan doesn't support fixed-function alpha test */
461 key->alpha_test_replicate_alpha = false;
462
463 if (ms_info) {
464 /* We should probably pull this out of the shader, but it's fairly
465 * harmless to compute it and then let dead-code take care of it.
466 */
467 if (ms_info->rasterizationSamples > 1) {
468 key->persample_interp = ms_info->sampleShadingEnable &&
469 (ms_info->minSampleShading * ms_info->rasterizationSamples) > 1;
470 key->multisample_fbo = true;
471 }
472
473 key->frag_coord_adds_sample_pos = key->persample_interp;
474 }
475 }
476
477 static void
478 populate_cs_prog_key(const struct gen_device_info *devinfo,
479 VkPipelineShaderStageCreateFlags flags,
480 const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT *rss_info,
481 struct brw_cs_prog_key *key)
482 {
483 memset(key, 0, sizeof(*key));
484
485 populate_base_prog_key(devinfo, flags, &key->base);
486
487 if (rss_info) {
488 assert(key->base.subgroup_size_type != BRW_SUBGROUP_SIZE_VARYING);
489
490 /* These enum values are expressly chosen to be equal to the subgroup
491 * size that they require.
492 */
493 assert(rss_info->requiredSubgroupSize == 8 ||
494 rss_info->requiredSubgroupSize == 16 ||
495 rss_info->requiredSubgroupSize == 32);
496 key->base.subgroup_size_type = rss_info->requiredSubgroupSize;
497 } else if (flags & VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT) {
498 /* If the client expressly requests full subgroups and they don't
499 * specify a subgroup size, we need to pick one. If they're requested
500 * varying subgroup sizes, we set it to UNIFORM and let the back-end
501 * compiler pick. Otherwise, we specify the API value of 32.
502 * Performance will likely be terrible in this case but there's nothing
503 * we can do about that. The client should have chosen a size.
504 */
505 if (flags & VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT_EXT)
506 key->base.subgroup_size_type = BRW_SUBGROUP_SIZE_UNIFORM;
507 else
508 key->base.subgroup_size_type = BRW_SUBGROUP_SIZE_REQUIRE_32;
509 }
510 }
511
512 struct anv_pipeline_stage {
513 gl_shader_stage stage;
514
515 const struct anv_shader_module *module;
516 const char *entrypoint;
517 const VkSpecializationInfo *spec_info;
518
519 unsigned char shader_sha1[20];
520
521 union brw_any_prog_key key;
522
523 struct {
524 gl_shader_stage stage;
525 unsigned char sha1[20];
526 } cache_key;
527
528 nir_shader *nir;
529
530 struct anv_pipeline_binding surface_to_descriptor[256];
531 struct anv_pipeline_binding sampler_to_descriptor[256];
532 struct anv_pipeline_bind_map bind_map;
533
534 union brw_any_prog_data prog_data;
535
536 uint32_t num_stats;
537 struct brw_compile_stats stats[3];
538 char *disasm[3];
539
540 VkPipelineCreationFeedbackEXT feedback;
541
542 const unsigned *code;
543 };
544
545 static void
546 anv_pipeline_hash_shader(const struct anv_shader_module *module,
547 const char *entrypoint,
548 gl_shader_stage stage,
549 const VkSpecializationInfo *spec_info,
550 unsigned char *sha1_out)
551 {
552 struct mesa_sha1 ctx;
553 _mesa_sha1_init(&ctx);
554
555 _mesa_sha1_update(&ctx, module->sha1, sizeof(module->sha1));
556 _mesa_sha1_update(&ctx, entrypoint, strlen(entrypoint));
557 _mesa_sha1_update(&ctx, &stage, sizeof(stage));
558 if (spec_info) {
559 _mesa_sha1_update(&ctx, spec_info->pMapEntries,
560 spec_info->mapEntryCount *
561 sizeof(*spec_info->pMapEntries));
562 _mesa_sha1_update(&ctx, spec_info->pData,
563 spec_info->dataSize);
564 }
565
566 _mesa_sha1_final(&ctx, sha1_out);
567 }
568
569 static void
570 anv_pipeline_hash_graphics(struct anv_pipeline *pipeline,
571 struct anv_pipeline_layout *layout,
572 struct anv_pipeline_stage *stages,
573 unsigned char *sha1_out)
574 {
575 struct mesa_sha1 ctx;
576 _mesa_sha1_init(&ctx);
577
578 _mesa_sha1_update(&ctx, &pipeline->subpass->view_mask,
579 sizeof(pipeline->subpass->view_mask));
580
581 if (layout)
582 _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
583
584 const bool rba = pipeline->device->robust_buffer_access;
585 _mesa_sha1_update(&ctx, &rba, sizeof(rba));
586
587 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
588 if (stages[s].entrypoint) {
589 _mesa_sha1_update(&ctx, stages[s].shader_sha1,
590 sizeof(stages[s].shader_sha1));
591 _mesa_sha1_update(&ctx, &stages[s].key, brw_prog_key_size(s));
592 }
593 }
594
595 _mesa_sha1_final(&ctx, sha1_out);
596 }
597
598 static void
599 anv_pipeline_hash_compute(struct anv_pipeline *pipeline,
600 struct anv_pipeline_layout *layout,
601 struct anv_pipeline_stage *stage,
602 unsigned char *sha1_out)
603 {
604 struct mesa_sha1 ctx;
605 _mesa_sha1_init(&ctx);
606
607 if (layout)
608 _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
609
610 const bool rba = pipeline->device->robust_buffer_access;
611 _mesa_sha1_update(&ctx, &rba, sizeof(rba));
612
613 _mesa_sha1_update(&ctx, stage->shader_sha1,
614 sizeof(stage->shader_sha1));
615 _mesa_sha1_update(&ctx, &stage->key.cs, sizeof(stage->key.cs));
616
617 _mesa_sha1_final(&ctx, sha1_out);
618 }
619
620 static nir_shader *
621 anv_pipeline_stage_get_nir(struct anv_pipeline *pipeline,
622 struct anv_pipeline_cache *cache,
623 void *mem_ctx,
624 struct anv_pipeline_stage *stage)
625 {
626 const struct brw_compiler *compiler =
627 pipeline->device->instance->physicalDevice.compiler;
628 const nir_shader_compiler_options *nir_options =
629 compiler->glsl_compiler_options[stage->stage].NirOptions;
630 nir_shader *nir;
631
632 nir = anv_device_search_for_nir(pipeline->device, cache,
633 nir_options,
634 stage->shader_sha1,
635 mem_ctx);
636 if (nir) {
637 assert(nir->info.stage == stage->stage);
638 return nir;
639 }
640
641 nir = anv_shader_compile_to_nir(pipeline->device,
642 mem_ctx,
643 stage->module,
644 stage->entrypoint,
645 stage->stage,
646 stage->spec_info);
647 if (nir) {
648 anv_device_upload_nir(pipeline->device, cache, nir, stage->shader_sha1);
649 return nir;
650 }
651
652 return NULL;
653 }
654
655 static void
656 anv_pipeline_lower_nir(struct anv_pipeline *pipeline,
657 void *mem_ctx,
658 struct anv_pipeline_stage *stage,
659 struct anv_pipeline_layout *layout)
660 {
661 const struct anv_physical_device *pdevice =
662 &pipeline->device->instance->physicalDevice;
663 const struct brw_compiler *compiler = pdevice->compiler;
664
665 struct brw_stage_prog_data *prog_data = &stage->prog_data.base;
666 nir_shader *nir = stage->nir;
667
668 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
669 NIR_PASS_V(nir, nir_lower_wpos_center, pipeline->sample_shading_enable);
670 NIR_PASS_V(nir, nir_lower_input_attachments, true);
671 }
672
673 NIR_PASS_V(nir, anv_nir_lower_ycbcr_textures, layout);
674
675 NIR_PASS_V(nir, anv_nir_lower_push_constants);
676
677 if (nir->info.stage != MESA_SHADER_COMPUTE)
678 NIR_PASS_V(nir, anv_nir_lower_multiview, pipeline->subpass->view_mask);
679
680 nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
681
682 if (nir->num_uniforms > 0) {
683 assert(prog_data->nr_params == 0);
684
685 /* If the shader uses any push constants at all, we'll just give
686 * them the maximum possible number
687 */
688 assert(nir->num_uniforms <= MAX_PUSH_CONSTANTS_SIZE);
689 nir->num_uniforms = MAX_PUSH_CONSTANTS_SIZE;
690 prog_data->nr_params += MAX_PUSH_CONSTANTS_SIZE / sizeof(float);
691 prog_data->param = ralloc_array(mem_ctx, uint32_t, prog_data->nr_params);
692
693 /* We now set the param values to be offsets into a
694 * anv_push_constant_data structure. Since the compiler doesn't
695 * actually dereference any of the gl_constant_value pointers in the
696 * params array, it doesn't really matter what we put here.
697 */
698 struct anv_push_constants *null_data = NULL;
699 /* Fill out the push constants section of the param array */
700 for (unsigned i = 0; i < MAX_PUSH_CONSTANTS_SIZE / sizeof(float); i++) {
701 prog_data->param[i] = ANV_PARAM_PUSH(
702 (uintptr_t)&null_data->client_data[i * sizeof(float)]);
703 }
704 }
705
706 if (nir->info.num_ssbos > 0 || nir->info.num_images > 0)
707 pipeline->needs_data_cache = true;
708
709 NIR_PASS_V(nir, brw_nir_lower_image_load_store, compiler->devinfo);
710
711 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_global,
712 nir_address_format_64bit_global);
713
714 /* Apply the actual pipeline layout to UBOs, SSBOs, and textures */
715 anv_nir_apply_pipeline_layout(pdevice,
716 pipeline->device->robust_buffer_access,
717 layout, nir, prog_data,
718 &stage->bind_map);
719
720 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_ubo,
721 nir_address_format_32bit_index_offset);
722 NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_ssbo,
723 anv_nir_ssbo_addr_format(pdevice,
724 pipeline->device->robust_buffer_access));
725
726 NIR_PASS_V(nir, nir_opt_constant_folding);
727
728 /* We don't support non-uniform UBOs and non-uniform SSBO access is
729 * handled naturally by falling back to A64 messages.
730 */
731 NIR_PASS_V(nir, nir_lower_non_uniform_access,
732 nir_lower_non_uniform_texture_access |
733 nir_lower_non_uniform_image_access);
734
735 if (nir->info.stage != MESA_SHADER_COMPUTE)
736 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
737
738 assert(nir->num_uniforms == prog_data->nr_params * 4);
739
740 stage->nir = nir;
741 }
742
743 static void
744 anv_pipeline_link_vs(const struct brw_compiler *compiler,
745 struct anv_pipeline_stage *vs_stage,
746 struct anv_pipeline_stage *next_stage)
747 {
748 if (next_stage)
749 brw_nir_link_shaders(compiler, vs_stage->nir, next_stage->nir);
750 }
751
752 static void
753 anv_pipeline_compile_vs(const struct brw_compiler *compiler,
754 void *mem_ctx,
755 struct anv_device *device,
756 struct anv_pipeline_stage *vs_stage)
757 {
758 brw_compute_vue_map(compiler->devinfo,
759 &vs_stage->prog_data.vs.base.vue_map,
760 vs_stage->nir->info.outputs_written,
761 vs_stage->nir->info.separate_shader);
762
763 vs_stage->num_stats = 1;
764 vs_stage->code = brw_compile_vs(compiler, device, mem_ctx,
765 &vs_stage->key.vs,
766 &vs_stage->prog_data.vs,
767 vs_stage->nir, -1,
768 vs_stage->stats, NULL);
769 }
770
771 static void
772 merge_tess_info(struct shader_info *tes_info,
773 const struct shader_info *tcs_info)
774 {
775 /* The Vulkan 1.0.38 spec, section 21.1 Tessellator says:
776 *
777 * "PointMode. Controls generation of points rather than triangles
778 * or lines. This functionality defaults to disabled, and is
779 * enabled if either shader stage includes the execution mode.
780 *
781 * and about Triangles, Quads, IsoLines, VertexOrderCw, VertexOrderCcw,
782 * PointMode, SpacingEqual, SpacingFractionalEven, SpacingFractionalOdd,
783 * and OutputVertices, it says:
784 *
785 * "One mode must be set in at least one of the tessellation
786 * shader stages."
787 *
788 * So, the fields can be set in either the TCS or TES, but they must
789 * agree if set in both. Our backend looks at TES, so bitwise-or in
790 * the values from the TCS.
791 */
792 assert(tcs_info->tess.tcs_vertices_out == 0 ||
793 tes_info->tess.tcs_vertices_out == 0 ||
794 tcs_info->tess.tcs_vertices_out == tes_info->tess.tcs_vertices_out);
795 tes_info->tess.tcs_vertices_out |= tcs_info->tess.tcs_vertices_out;
796
797 assert(tcs_info->tess.spacing == TESS_SPACING_UNSPECIFIED ||
798 tes_info->tess.spacing == TESS_SPACING_UNSPECIFIED ||
799 tcs_info->tess.spacing == tes_info->tess.spacing);
800 tes_info->tess.spacing |= tcs_info->tess.spacing;
801
802 assert(tcs_info->tess.primitive_mode == 0 ||
803 tes_info->tess.primitive_mode == 0 ||
804 tcs_info->tess.primitive_mode == tes_info->tess.primitive_mode);
805 tes_info->tess.primitive_mode |= tcs_info->tess.primitive_mode;
806 tes_info->tess.ccw |= tcs_info->tess.ccw;
807 tes_info->tess.point_mode |= tcs_info->tess.point_mode;
808 }
809
810 static void
811 anv_pipeline_link_tcs(const struct brw_compiler *compiler,
812 struct anv_pipeline_stage *tcs_stage,
813 struct anv_pipeline_stage *tes_stage)
814 {
815 assert(tes_stage && tes_stage->stage == MESA_SHADER_TESS_EVAL);
816
817 brw_nir_link_shaders(compiler, tcs_stage->nir, tes_stage->nir);
818
819 nir_lower_patch_vertices(tes_stage->nir,
820 tcs_stage->nir->info.tess.tcs_vertices_out,
821 NULL);
822
823 /* Copy TCS info into the TES info */
824 merge_tess_info(&tes_stage->nir->info, &tcs_stage->nir->info);
825
826 /* Whacking the key after cache lookup is a bit sketchy, but all of
827 * this comes from the SPIR-V, which is part of the hash used for the
828 * pipeline cache. So it should be safe.
829 */
830 tcs_stage->key.tcs.tes_primitive_mode =
831 tes_stage->nir->info.tess.primitive_mode;
832 tcs_stage->key.tcs.quads_workaround =
833 compiler->devinfo->gen < 9 &&
834 tes_stage->nir->info.tess.primitive_mode == 7 /* GL_QUADS */ &&
835 tes_stage->nir->info.tess.spacing == TESS_SPACING_EQUAL;
836 }
837
838 static void
839 anv_pipeline_compile_tcs(const struct brw_compiler *compiler,
840 void *mem_ctx,
841 struct anv_device *device,
842 struct anv_pipeline_stage *tcs_stage,
843 struct anv_pipeline_stage *prev_stage)
844 {
845 tcs_stage->key.tcs.outputs_written =
846 tcs_stage->nir->info.outputs_written;
847 tcs_stage->key.tcs.patch_outputs_written =
848 tcs_stage->nir->info.patch_outputs_written;
849
850 tcs_stage->num_stats = 1;
851 tcs_stage->code = brw_compile_tcs(compiler, device, mem_ctx,
852 &tcs_stage->key.tcs,
853 &tcs_stage->prog_data.tcs,
854 tcs_stage->nir, -1,
855 tcs_stage->stats, NULL);
856 }
857
858 static void
859 anv_pipeline_link_tes(const struct brw_compiler *compiler,
860 struct anv_pipeline_stage *tes_stage,
861 struct anv_pipeline_stage *next_stage)
862 {
863 if (next_stage)
864 brw_nir_link_shaders(compiler, tes_stage->nir, next_stage->nir);
865 }
866
867 static void
868 anv_pipeline_compile_tes(const struct brw_compiler *compiler,
869 void *mem_ctx,
870 struct anv_device *device,
871 struct anv_pipeline_stage *tes_stage,
872 struct anv_pipeline_stage *tcs_stage)
873 {
874 tes_stage->key.tes.inputs_read =
875 tcs_stage->nir->info.outputs_written;
876 tes_stage->key.tes.patch_inputs_read =
877 tcs_stage->nir->info.patch_outputs_written;
878
879 tes_stage->num_stats = 1;
880 tes_stage->code = brw_compile_tes(compiler, device, mem_ctx,
881 &tes_stage->key.tes,
882 &tcs_stage->prog_data.tcs.base.vue_map,
883 &tes_stage->prog_data.tes,
884 tes_stage->nir, -1,
885 tes_stage->stats, NULL);
886 }
887
888 static void
889 anv_pipeline_link_gs(const struct brw_compiler *compiler,
890 struct anv_pipeline_stage *gs_stage,
891 struct anv_pipeline_stage *next_stage)
892 {
893 if (next_stage)
894 brw_nir_link_shaders(compiler, gs_stage->nir, next_stage->nir);
895 }
896
897 static void
898 anv_pipeline_compile_gs(const struct brw_compiler *compiler,
899 void *mem_ctx,
900 struct anv_device *device,
901 struct anv_pipeline_stage *gs_stage,
902 struct anv_pipeline_stage *prev_stage)
903 {
904 brw_compute_vue_map(compiler->devinfo,
905 &gs_stage->prog_data.gs.base.vue_map,
906 gs_stage->nir->info.outputs_written,
907 gs_stage->nir->info.separate_shader);
908
909 gs_stage->num_stats = 1;
910 gs_stage->code = brw_compile_gs(compiler, device, mem_ctx,
911 &gs_stage->key.gs,
912 &gs_stage->prog_data.gs,
913 gs_stage->nir, NULL, -1,
914 gs_stage->stats, NULL);
915 }
916
917 static void
918 anv_pipeline_link_fs(const struct brw_compiler *compiler,
919 struct anv_pipeline_stage *stage)
920 {
921 unsigned num_rt_bindings;
922 struct anv_pipeline_binding rt_bindings[MAX_RTS];
923 if (stage->key.wm.nr_color_regions > 0) {
924 assert(stage->key.wm.nr_color_regions <= MAX_RTS);
925 for (unsigned rt = 0; rt < stage->key.wm.nr_color_regions; rt++) {
926 if (stage->key.wm.color_outputs_valid & BITFIELD_BIT(rt)) {
927 rt_bindings[rt] = (struct anv_pipeline_binding) {
928 .set = ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS,
929 .index = rt,
930 };
931 } else {
932 /* Setup a null render target */
933 rt_bindings[rt] = (struct anv_pipeline_binding) {
934 .set = ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS,
935 .index = UINT32_MAX,
936 };
937 }
938 }
939 num_rt_bindings = stage->key.wm.nr_color_regions;
940 } else {
941 /* Setup a null render target */
942 rt_bindings[0] = (struct anv_pipeline_binding) {
943 .set = ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS,
944 .index = UINT32_MAX,
945 };
946 num_rt_bindings = 1;
947 }
948
949 assert(num_rt_bindings <= MAX_RTS);
950 assert(stage->bind_map.surface_count == 0);
951 typed_memcpy(stage->bind_map.surface_to_descriptor,
952 rt_bindings, num_rt_bindings);
953 stage->bind_map.surface_count += num_rt_bindings;
954
955 /* Now that we've set up the color attachments, we can go through and
956 * eliminate any shader outputs that map to VK_ATTACHMENT_UNUSED in the
957 * hopes that dead code can clean them up in this and any earlier shader
958 * stages.
959 */
960 nir_function_impl *impl = nir_shader_get_entrypoint(stage->nir);
961 bool deleted_output = false;
962 nir_foreach_variable_safe(var, &stage->nir->outputs) {
963 /* TODO: We don't delete depth/stencil writes. We probably could if the
964 * subpass doesn't have a depth/stencil attachment.
965 */
966 if (var->data.location < FRAG_RESULT_DATA0)
967 continue;
968
969 const unsigned rt = var->data.location - FRAG_RESULT_DATA0;
970
971 /* If this is the RT at location 0 and we have alpha to coverage
972 * enabled we still need that write because it will affect the coverage
973 * mask even if it's never written to a color target.
974 */
975 if (rt == 0 && stage->key.wm.alpha_to_coverage)
976 continue;
977
978 const unsigned array_len =
979 glsl_type_is_array(var->type) ? glsl_get_length(var->type) : 1;
980 assert(rt + array_len <= MAX_RTS);
981
982 if (rt >= MAX_RTS || !(stage->key.wm.color_outputs_valid &
983 BITFIELD_RANGE(rt, array_len))) {
984 deleted_output = true;
985 var->data.mode = nir_var_function_temp;
986 exec_node_remove(&var->node);
987 exec_list_push_tail(&impl->locals, &var->node);
988 }
989 }
990
991 if (deleted_output)
992 nir_fixup_deref_modes(stage->nir);
993
994 /* We stored the number of subpass color attachments in nr_color_regions
995 * when calculating the key for caching. Now that we've computed the bind
996 * map, we can reduce this to the actual max before we go into the back-end
997 * compiler.
998 */
999 stage->key.wm.nr_color_regions =
1000 util_last_bit(stage->key.wm.color_outputs_valid);
1001 }
1002
1003 static void
1004 anv_pipeline_compile_fs(const struct brw_compiler *compiler,
1005 void *mem_ctx,
1006 struct anv_device *device,
1007 struct anv_pipeline_stage *fs_stage,
1008 struct anv_pipeline_stage *prev_stage)
1009 {
1010 /* TODO: we could set this to 0 based on the information in nir_shader, but
1011 * we need this before we call spirv_to_nir.
1012 */
1013 assert(prev_stage);
1014 fs_stage->key.wm.input_slots_valid =
1015 prev_stage->prog_data.vue.vue_map.slots_valid;
1016
1017 fs_stage->code = brw_compile_fs(compiler, device, mem_ctx,
1018 &fs_stage->key.wm,
1019 &fs_stage->prog_data.wm,
1020 fs_stage->nir, -1, -1, -1,
1021 true, false, NULL,
1022 fs_stage->stats, NULL);
1023
1024 fs_stage->num_stats = (uint32_t)fs_stage->prog_data.wm.dispatch_8 +
1025 (uint32_t)fs_stage->prog_data.wm.dispatch_16 +
1026 (uint32_t)fs_stage->prog_data.wm.dispatch_32;
1027
1028 if (fs_stage->key.wm.color_outputs_valid == 0 &&
1029 !fs_stage->prog_data.wm.has_side_effects &&
1030 !fs_stage->prog_data.wm.uses_omask &&
1031 !fs_stage->key.wm.alpha_to_coverage &&
1032 !fs_stage->prog_data.wm.uses_kill &&
1033 fs_stage->prog_data.wm.computed_depth_mode == BRW_PSCDEPTH_OFF &&
1034 !fs_stage->prog_data.wm.computed_stencil) {
1035 /* This fragment shader has no outputs and no side effects. Go ahead
1036 * and return the code pointer so we don't accidentally think the
1037 * compile failed but zero out prog_data which will set program_size to
1038 * zero and disable the stage.
1039 */
1040 memset(&fs_stage->prog_data, 0, sizeof(fs_stage->prog_data));
1041 }
1042 }
1043
1044 static void
1045 anv_pipeline_add_executable(struct anv_pipeline *pipeline,
1046 struct anv_pipeline_stage *stage,
1047 struct brw_compile_stats *stats,
1048 uint32_t code_offset)
1049 {
1050 char *nir = NULL;
1051 if (stage->nir &&
1052 (pipeline->flags &
1053 VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR)) {
1054 char *stream_data = NULL;
1055 size_t stream_size = 0;
1056 FILE *stream = open_memstream(&stream_data, &stream_size);
1057
1058 nir_print_shader(stage->nir, stream);
1059
1060 fclose(stream);
1061
1062 /* Copy it to a ralloc'd thing */
1063 nir = ralloc_size(pipeline->mem_ctx, stream_size + 1);
1064 memcpy(nir, stream_data, stream_size);
1065 nir[stream_size] = 0;
1066
1067 free(stream_data);
1068 }
1069
1070 char *disasm = NULL;
1071 if (stage->code &&
1072 (pipeline->flags &
1073 VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR)) {
1074 char *stream_data = NULL;
1075 size_t stream_size = 0;
1076 FILE *stream = open_memstream(&stream_data, &stream_size);
1077
1078 /* Creating this is far cheaper than it looks. It's perfectly fine to
1079 * do it for every binary.
1080 */
1081 struct gen_disasm *d = gen_disasm_create(&pipeline->device->info);
1082 gen_disasm_disassemble(d, stage->code, code_offset, stream);
1083 gen_disasm_destroy(d);
1084
1085 fclose(stream);
1086
1087 /* Copy it to a ralloc'd thing */
1088 disasm = ralloc_size(pipeline->mem_ctx, stream_size + 1);
1089 memcpy(disasm, stream_data, stream_size);
1090 disasm[stream_size] = 0;
1091
1092 free(stream_data);
1093 }
1094
1095 pipeline->executables[pipeline->num_executables++] =
1096 (struct anv_pipeline_executable) {
1097 .stage = stage->stage,
1098 .stats = *stats,
1099 .nir = nir,
1100 .disasm = disasm,
1101 };
1102 }
1103
1104 static void
1105 anv_pipeline_add_executables(struct anv_pipeline *pipeline,
1106 struct anv_pipeline_stage *stage,
1107 struct anv_shader_bin *bin)
1108 {
1109 if (stage->stage == MESA_SHADER_FRAGMENT) {
1110 /* We pull the prog data and stats out of the anv_shader_bin because
1111 * the anv_pipeline_stage may not be fully populated if we successfully
1112 * looked up the shader in a cache.
1113 */
1114 const struct brw_wm_prog_data *wm_prog_data =
1115 (const struct brw_wm_prog_data *)bin->prog_data;
1116 struct brw_compile_stats *stats = bin->stats;
1117
1118 if (wm_prog_data->dispatch_8) {
1119 anv_pipeline_add_executable(pipeline, stage, stats++, 0);
1120 }
1121
1122 if (wm_prog_data->dispatch_16) {
1123 anv_pipeline_add_executable(pipeline, stage, stats++,
1124 wm_prog_data->prog_offset_16);
1125 }
1126
1127 if (wm_prog_data->dispatch_32) {
1128 anv_pipeline_add_executable(pipeline, stage, stats++,
1129 wm_prog_data->prog_offset_32);
1130 }
1131 } else {
1132 anv_pipeline_add_executable(pipeline, stage, bin->stats, 0);
1133 }
1134 }
1135
1136 static VkResult
1137 anv_pipeline_compile_graphics(struct anv_pipeline *pipeline,
1138 struct anv_pipeline_cache *cache,
1139 const VkGraphicsPipelineCreateInfo *info)
1140 {
1141 VkPipelineCreationFeedbackEXT pipeline_feedback = {
1142 .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT_EXT,
1143 };
1144 int64_t pipeline_start = os_time_get_nano();
1145
1146 const struct brw_compiler *compiler =
1147 pipeline->device->instance->physicalDevice.compiler;
1148 struct anv_pipeline_stage stages[MESA_SHADER_STAGES] = {};
1149
1150 pipeline->active_stages = 0;
1151
1152 VkResult result;
1153 for (uint32_t i = 0; i < info->stageCount; i++) {
1154 const VkPipelineShaderStageCreateInfo *sinfo = &info->pStages[i];
1155 gl_shader_stage stage = vk_to_mesa_shader_stage(sinfo->stage);
1156
1157 pipeline->active_stages |= sinfo->stage;
1158
1159 int64_t stage_start = os_time_get_nano();
1160
1161 stages[stage].stage = stage;
1162 stages[stage].module = anv_shader_module_from_handle(sinfo->module);
1163 stages[stage].entrypoint = sinfo->pName;
1164 stages[stage].spec_info = sinfo->pSpecializationInfo;
1165 anv_pipeline_hash_shader(stages[stage].module,
1166 stages[stage].entrypoint,
1167 stage,
1168 stages[stage].spec_info,
1169 stages[stage].shader_sha1);
1170
1171 const struct gen_device_info *devinfo = &pipeline->device->info;
1172 switch (stage) {
1173 case MESA_SHADER_VERTEX:
1174 populate_vs_prog_key(devinfo, sinfo->flags, &stages[stage].key.vs);
1175 break;
1176 case MESA_SHADER_TESS_CTRL:
1177 populate_tcs_prog_key(devinfo, sinfo->flags,
1178 info->pTessellationState->patchControlPoints,
1179 &stages[stage].key.tcs);
1180 break;
1181 case MESA_SHADER_TESS_EVAL:
1182 populate_tes_prog_key(devinfo, sinfo->flags, &stages[stage].key.tes);
1183 break;
1184 case MESA_SHADER_GEOMETRY:
1185 populate_gs_prog_key(devinfo, sinfo->flags, &stages[stage].key.gs);
1186 break;
1187 case MESA_SHADER_FRAGMENT:
1188 populate_wm_prog_key(devinfo, sinfo->flags,
1189 pipeline->subpass,
1190 info->pMultisampleState,
1191 &stages[stage].key.wm);
1192 break;
1193 default:
1194 unreachable("Invalid graphics shader stage");
1195 }
1196
1197 stages[stage].feedback.duration += os_time_get_nano() - stage_start;
1198 stages[stage].feedback.flags |= VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT_EXT;
1199 }
1200
1201 if (pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT)
1202 pipeline->active_stages |= VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT;
1203
1204 assert(pipeline->active_stages & VK_SHADER_STAGE_VERTEX_BIT);
1205
1206 ANV_FROM_HANDLE(anv_pipeline_layout, layout, info->layout);
1207
1208 unsigned char sha1[20];
1209 anv_pipeline_hash_graphics(pipeline, layout, stages, sha1);
1210
1211 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1212 if (!stages[s].entrypoint)
1213 continue;
1214
1215 stages[s].cache_key.stage = s;
1216 memcpy(stages[s].cache_key.sha1, sha1, sizeof(sha1));
1217 }
1218
1219 const bool skip_cache_lookup =
1220 (pipeline->flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR);
1221
1222 if (!skip_cache_lookup) {
1223 unsigned found = 0;
1224 unsigned cache_hits = 0;
1225 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1226 if (!stages[s].entrypoint)
1227 continue;
1228
1229 int64_t stage_start = os_time_get_nano();
1230
1231 bool cache_hit;
1232 struct anv_shader_bin *bin =
1233 anv_device_search_for_kernel(pipeline->device, cache,
1234 &stages[s].cache_key,
1235 sizeof(stages[s].cache_key), &cache_hit);
1236 if (bin) {
1237 found++;
1238 pipeline->shaders[s] = bin;
1239 }
1240
1241 if (cache_hit) {
1242 cache_hits++;
1243 stages[s].feedback.flags |=
1244 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT_EXT;
1245 }
1246 stages[s].feedback.duration += os_time_get_nano() - stage_start;
1247 }
1248
1249 if (found == __builtin_popcount(pipeline->active_stages)) {
1250 if (cache_hits == found) {
1251 pipeline_feedback.flags |=
1252 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT_EXT;
1253 }
1254 /* We found all our shaders in the cache. We're done. */
1255 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1256 if (!stages[s].entrypoint)
1257 continue;
1258
1259 anv_pipeline_add_executables(pipeline, &stages[s],
1260 pipeline->shaders[s]);
1261 }
1262 goto done;
1263 } else if (found > 0) {
1264 /* We found some but not all of our shaders. This shouldn't happen
1265 * most of the time but it can if we have a partially populated
1266 * pipeline cache.
1267 */
1268 assert(found < __builtin_popcount(pipeline->active_stages));
1269
1270 vk_debug_report(&pipeline->device->instance->debug_report_callbacks,
1271 VK_DEBUG_REPORT_WARNING_BIT_EXT |
1272 VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT,
1273 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_CACHE_EXT,
1274 (uint64_t)(uintptr_t)cache,
1275 0, 0, "anv",
1276 "Found a partial pipeline in the cache. This is "
1277 "most likely caused by an incomplete pipeline cache "
1278 "import or export");
1279
1280 /* We're going to have to recompile anyway, so just throw away our
1281 * references to the shaders in the cache. We'll get them out of the
1282 * cache again as part of the compilation process.
1283 */
1284 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1285 stages[s].feedback.flags = 0;
1286 if (pipeline->shaders[s]) {
1287 anv_shader_bin_unref(pipeline->device, pipeline->shaders[s]);
1288 pipeline->shaders[s] = NULL;
1289 }
1290 }
1291 }
1292 }
1293
1294 void *pipeline_ctx = ralloc_context(NULL);
1295
1296 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1297 if (!stages[s].entrypoint)
1298 continue;
1299
1300 int64_t stage_start = os_time_get_nano();
1301
1302 assert(stages[s].stage == s);
1303 assert(pipeline->shaders[s] == NULL);
1304
1305 stages[s].bind_map = (struct anv_pipeline_bind_map) {
1306 .surface_to_descriptor = stages[s].surface_to_descriptor,
1307 .sampler_to_descriptor = stages[s].sampler_to_descriptor
1308 };
1309
1310 stages[s].nir = anv_pipeline_stage_get_nir(pipeline, cache,
1311 pipeline_ctx,
1312 &stages[s]);
1313 if (stages[s].nir == NULL) {
1314 result = vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1315 goto fail;
1316 }
1317
1318 stages[s].feedback.duration += os_time_get_nano() - stage_start;
1319 }
1320
1321 /* Walk backwards to link */
1322 struct anv_pipeline_stage *next_stage = NULL;
1323 for (int s = MESA_SHADER_STAGES - 1; s >= 0; s--) {
1324 if (!stages[s].entrypoint)
1325 continue;
1326
1327 switch (s) {
1328 case MESA_SHADER_VERTEX:
1329 anv_pipeline_link_vs(compiler, &stages[s], next_stage);
1330 break;
1331 case MESA_SHADER_TESS_CTRL:
1332 anv_pipeline_link_tcs(compiler, &stages[s], next_stage);
1333 break;
1334 case MESA_SHADER_TESS_EVAL:
1335 anv_pipeline_link_tes(compiler, &stages[s], next_stage);
1336 break;
1337 case MESA_SHADER_GEOMETRY:
1338 anv_pipeline_link_gs(compiler, &stages[s], next_stage);
1339 break;
1340 case MESA_SHADER_FRAGMENT:
1341 anv_pipeline_link_fs(compiler, &stages[s]);
1342 break;
1343 default:
1344 unreachable("Invalid graphics shader stage");
1345 }
1346
1347 next_stage = &stages[s];
1348 }
1349
1350 struct anv_pipeline_stage *prev_stage = NULL;
1351 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1352 if (!stages[s].entrypoint)
1353 continue;
1354
1355 int64_t stage_start = os_time_get_nano();
1356
1357 void *stage_ctx = ralloc_context(NULL);
1358
1359 nir_xfb_info *xfb_info = NULL;
1360 if (s == MESA_SHADER_VERTEX ||
1361 s == MESA_SHADER_TESS_EVAL ||
1362 s == MESA_SHADER_GEOMETRY)
1363 xfb_info = nir_gather_xfb_info(stages[s].nir, stage_ctx);
1364
1365 anv_pipeline_lower_nir(pipeline, stage_ctx, &stages[s], layout);
1366
1367 switch (s) {
1368 case MESA_SHADER_VERTEX:
1369 anv_pipeline_compile_vs(compiler, stage_ctx, pipeline->device,
1370 &stages[s]);
1371 break;
1372 case MESA_SHADER_TESS_CTRL:
1373 anv_pipeline_compile_tcs(compiler, stage_ctx, pipeline->device,
1374 &stages[s], prev_stage);
1375 break;
1376 case MESA_SHADER_TESS_EVAL:
1377 anv_pipeline_compile_tes(compiler, stage_ctx, pipeline->device,
1378 &stages[s], prev_stage);
1379 break;
1380 case MESA_SHADER_GEOMETRY:
1381 anv_pipeline_compile_gs(compiler, stage_ctx, pipeline->device,
1382 &stages[s], prev_stage);
1383 break;
1384 case MESA_SHADER_FRAGMENT:
1385 anv_pipeline_compile_fs(compiler, stage_ctx, pipeline->device,
1386 &stages[s], prev_stage);
1387 break;
1388 default:
1389 unreachable("Invalid graphics shader stage");
1390 }
1391 if (stages[s].code == NULL) {
1392 ralloc_free(stage_ctx);
1393 result = vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1394 goto fail;
1395 }
1396
1397 anv_compute_push_layout(&pipeline->device->instance->physicalDevice,
1398 &stages[s].prog_data.base,
1399 &stages[s].bind_map);
1400
1401 struct anv_shader_bin *bin =
1402 anv_device_upload_kernel(pipeline->device, cache,
1403 &stages[s].cache_key,
1404 sizeof(stages[s].cache_key),
1405 stages[s].code,
1406 stages[s].prog_data.base.program_size,
1407 stages[s].nir->constant_data,
1408 stages[s].nir->constant_data_size,
1409 &stages[s].prog_data.base,
1410 brw_prog_data_size(s),
1411 stages[s].stats, stages[s].num_stats,
1412 xfb_info, &stages[s].bind_map);
1413 if (!bin) {
1414 ralloc_free(stage_ctx);
1415 result = vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1416 goto fail;
1417 }
1418
1419 anv_pipeline_add_executables(pipeline, &stages[s], bin);
1420
1421 pipeline->shaders[s] = bin;
1422 ralloc_free(stage_ctx);
1423
1424 stages[s].feedback.duration += os_time_get_nano() - stage_start;
1425
1426 prev_stage = &stages[s];
1427 }
1428
1429 ralloc_free(pipeline_ctx);
1430
1431 done:
1432
1433 if (pipeline->shaders[MESA_SHADER_FRAGMENT] &&
1434 pipeline->shaders[MESA_SHADER_FRAGMENT]->prog_data->program_size == 0) {
1435 /* This can happen if we decided to implicitly disable the fragment
1436 * shader. See anv_pipeline_compile_fs().
1437 */
1438 anv_shader_bin_unref(pipeline->device,
1439 pipeline->shaders[MESA_SHADER_FRAGMENT]);
1440 pipeline->shaders[MESA_SHADER_FRAGMENT] = NULL;
1441 pipeline->active_stages &= ~VK_SHADER_STAGE_FRAGMENT_BIT;
1442 }
1443
1444 pipeline_feedback.duration = os_time_get_nano() - pipeline_start;
1445
1446 const VkPipelineCreationFeedbackCreateInfoEXT *create_feedback =
1447 vk_find_struct_const(info->pNext, PIPELINE_CREATION_FEEDBACK_CREATE_INFO_EXT);
1448 if (create_feedback) {
1449 *create_feedback->pPipelineCreationFeedback = pipeline_feedback;
1450
1451 assert(info->stageCount == create_feedback->pipelineStageCreationFeedbackCount);
1452 for (uint32_t i = 0; i < info->stageCount; i++) {
1453 gl_shader_stage s = vk_to_mesa_shader_stage(info->pStages[i].stage);
1454 create_feedback->pPipelineStageCreationFeedbacks[i] = stages[s].feedback;
1455 }
1456 }
1457
1458 return VK_SUCCESS;
1459
1460 fail:
1461 ralloc_free(pipeline_ctx);
1462
1463 for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1464 if (pipeline->shaders[s])
1465 anv_shader_bin_unref(pipeline->device, pipeline->shaders[s]);
1466 }
1467
1468 return result;
1469 }
1470
1471 static void
1472 shared_type_info(const struct glsl_type *type, unsigned *size, unsigned *align)
1473 {
1474 assert(glsl_type_is_vector_or_scalar(type));
1475
1476 uint32_t comp_size = glsl_type_is_boolean(type)
1477 ? 4 : glsl_get_bit_size(type) / 8;
1478 unsigned length = glsl_get_vector_elements(type);
1479 *size = comp_size * length,
1480 *align = comp_size * (length == 3 ? 4 : length);
1481 }
1482
1483 VkResult
1484 anv_pipeline_compile_cs(struct anv_pipeline *pipeline,
1485 struct anv_pipeline_cache *cache,
1486 const VkComputePipelineCreateInfo *info,
1487 const struct anv_shader_module *module,
1488 const char *entrypoint,
1489 const VkSpecializationInfo *spec_info)
1490 {
1491 VkPipelineCreationFeedbackEXT pipeline_feedback = {
1492 .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT_EXT,
1493 };
1494 int64_t pipeline_start = os_time_get_nano();
1495
1496 const struct brw_compiler *compiler =
1497 pipeline->device->instance->physicalDevice.compiler;
1498
1499 struct anv_pipeline_stage stage = {
1500 .stage = MESA_SHADER_COMPUTE,
1501 .module = module,
1502 .entrypoint = entrypoint,
1503 .spec_info = spec_info,
1504 .cache_key = {
1505 .stage = MESA_SHADER_COMPUTE,
1506 },
1507 .feedback = {
1508 .flags = VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT_EXT,
1509 },
1510 };
1511 anv_pipeline_hash_shader(stage.module,
1512 stage.entrypoint,
1513 MESA_SHADER_COMPUTE,
1514 stage.spec_info,
1515 stage.shader_sha1);
1516
1517 struct anv_shader_bin *bin = NULL;
1518
1519 const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT *rss_info =
1520 vk_find_struct_const(info->stage.pNext,
1521 PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT);
1522
1523 populate_cs_prog_key(&pipeline->device->info, info->stage.flags,
1524 rss_info, &stage.key.cs);
1525
1526 ANV_FROM_HANDLE(anv_pipeline_layout, layout, info->layout);
1527
1528 const bool skip_cache_lookup =
1529 (pipeline->flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR);
1530
1531 anv_pipeline_hash_compute(pipeline, layout, &stage, stage.cache_key.sha1);
1532
1533 bool cache_hit = false;
1534 if (!skip_cache_lookup) {
1535 bin = anv_device_search_for_kernel(pipeline->device, cache,
1536 &stage.cache_key,
1537 sizeof(stage.cache_key),
1538 &cache_hit);
1539 }
1540
1541 void *mem_ctx = ralloc_context(NULL);
1542 if (bin == NULL) {
1543 int64_t stage_start = os_time_get_nano();
1544
1545 stage.bind_map = (struct anv_pipeline_bind_map) {
1546 .surface_to_descriptor = stage.surface_to_descriptor,
1547 .sampler_to_descriptor = stage.sampler_to_descriptor
1548 };
1549
1550 /* Set up a binding for the gl_NumWorkGroups */
1551 stage.bind_map.surface_count = 1;
1552 stage.bind_map.surface_to_descriptor[0] = (struct anv_pipeline_binding) {
1553 .set = ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS,
1554 };
1555
1556 stage.nir = anv_pipeline_stage_get_nir(pipeline, cache, mem_ctx, &stage);
1557 if (stage.nir == NULL) {
1558 ralloc_free(mem_ctx);
1559 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1560 }
1561
1562 anv_pipeline_lower_nir(pipeline, mem_ctx, &stage, layout);
1563
1564 NIR_PASS_V(stage.nir, anv_nir_add_base_work_group_id,
1565 &stage.prog_data.cs);
1566
1567 NIR_PASS_V(stage.nir, nir_lower_vars_to_explicit_types,
1568 nir_var_mem_shared, shared_type_info);
1569 NIR_PASS_V(stage.nir, nir_lower_explicit_io,
1570 nir_var_mem_shared, nir_address_format_32bit_offset);
1571
1572 stage.num_stats = 1;
1573 stage.code = brw_compile_cs(compiler, pipeline->device, mem_ctx,
1574 &stage.key.cs, &stage.prog_data.cs,
1575 stage.nir, -1, stage.stats, NULL);
1576 if (stage.code == NULL) {
1577 ralloc_free(mem_ctx);
1578 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1579 }
1580
1581 const unsigned code_size = stage.prog_data.base.program_size;
1582 bin = anv_device_upload_kernel(pipeline->device, cache,
1583 &stage.cache_key, sizeof(stage.cache_key),
1584 stage.code, code_size,
1585 stage.nir->constant_data,
1586 stage.nir->constant_data_size,
1587 &stage.prog_data.base,
1588 sizeof(stage.prog_data.cs),
1589 stage.stats, stage.num_stats,
1590 NULL, &stage.bind_map);
1591 if (!bin) {
1592 ralloc_free(mem_ctx);
1593 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1594 }
1595
1596 stage.feedback.duration = os_time_get_nano() - stage_start;
1597 }
1598
1599 anv_pipeline_add_executables(pipeline, &stage, bin);
1600
1601 ralloc_free(mem_ctx);
1602
1603 if (cache_hit) {
1604 stage.feedback.flags |=
1605 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT_EXT;
1606 pipeline_feedback.flags |=
1607 VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT_EXT;
1608 }
1609 pipeline_feedback.duration = os_time_get_nano() - pipeline_start;
1610
1611 const VkPipelineCreationFeedbackCreateInfoEXT *create_feedback =
1612 vk_find_struct_const(info->pNext, PIPELINE_CREATION_FEEDBACK_CREATE_INFO_EXT);
1613 if (create_feedback) {
1614 *create_feedback->pPipelineCreationFeedback = pipeline_feedback;
1615
1616 assert(create_feedback->pipelineStageCreationFeedbackCount == 1);
1617 create_feedback->pPipelineStageCreationFeedbacks[0] = stage.feedback;
1618 }
1619
1620 pipeline->active_stages = VK_SHADER_STAGE_COMPUTE_BIT;
1621 pipeline->shaders[MESA_SHADER_COMPUTE] = bin;
1622
1623 return VK_SUCCESS;
1624 }
1625
1626 /**
1627 * Copy pipeline state not marked as dynamic.
1628 * Dynamic state is pipeline state which hasn't been provided at pipeline
1629 * creation time, but is dynamically provided afterwards using various
1630 * vkCmdSet* functions.
1631 *
1632 * The set of state considered "non_dynamic" is determined by the pieces of
1633 * state that have their corresponding VkDynamicState enums omitted from
1634 * VkPipelineDynamicStateCreateInfo::pDynamicStates.
1635 *
1636 * @param[out] pipeline Destination non_dynamic state.
1637 * @param[in] pCreateInfo Source of non_dynamic state to be copied.
1638 */
1639 static void
1640 copy_non_dynamic_state(struct anv_pipeline *pipeline,
1641 const VkGraphicsPipelineCreateInfo *pCreateInfo)
1642 {
1643 anv_cmd_dirty_mask_t states = ANV_CMD_DIRTY_DYNAMIC_ALL;
1644 struct anv_subpass *subpass = pipeline->subpass;
1645
1646 pipeline->dynamic_state = default_dynamic_state;
1647
1648 if (pCreateInfo->pDynamicState) {
1649 /* Remove all of the states that are marked as dynamic */
1650 uint32_t count = pCreateInfo->pDynamicState->dynamicStateCount;
1651 for (uint32_t s = 0; s < count; s++) {
1652 states &= ~anv_cmd_dirty_bit_for_vk_dynamic_state(
1653 pCreateInfo->pDynamicState->pDynamicStates[s]);
1654 }
1655 }
1656
1657 struct anv_dynamic_state *dynamic = &pipeline->dynamic_state;
1658
1659 /* Section 9.2 of the Vulkan 1.0.15 spec says:
1660 *
1661 * pViewportState is [...] NULL if the pipeline
1662 * has rasterization disabled.
1663 */
1664 if (!pCreateInfo->pRasterizationState->rasterizerDiscardEnable) {
1665 assert(pCreateInfo->pViewportState);
1666
1667 dynamic->viewport.count = pCreateInfo->pViewportState->viewportCount;
1668 if (states & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT) {
1669 typed_memcpy(dynamic->viewport.viewports,
1670 pCreateInfo->pViewportState->pViewports,
1671 pCreateInfo->pViewportState->viewportCount);
1672 }
1673
1674 dynamic->scissor.count = pCreateInfo->pViewportState->scissorCount;
1675 if (states & ANV_CMD_DIRTY_DYNAMIC_SCISSOR) {
1676 typed_memcpy(dynamic->scissor.scissors,
1677 pCreateInfo->pViewportState->pScissors,
1678 pCreateInfo->pViewportState->scissorCount);
1679 }
1680 }
1681
1682 if (states & ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH) {
1683 assert(pCreateInfo->pRasterizationState);
1684 dynamic->line_width = pCreateInfo->pRasterizationState->lineWidth;
1685 }
1686
1687 if (states & ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS) {
1688 assert(pCreateInfo->pRasterizationState);
1689 dynamic->depth_bias.bias =
1690 pCreateInfo->pRasterizationState->depthBiasConstantFactor;
1691 dynamic->depth_bias.clamp =
1692 pCreateInfo->pRasterizationState->depthBiasClamp;
1693 dynamic->depth_bias.slope =
1694 pCreateInfo->pRasterizationState->depthBiasSlopeFactor;
1695 }
1696
1697 /* Section 9.2 of the Vulkan 1.0.15 spec says:
1698 *
1699 * pColorBlendState is [...] NULL if the pipeline has rasterization
1700 * disabled or if the subpass of the render pass the pipeline is
1701 * created against does not use any color attachments.
1702 */
1703 bool uses_color_att = false;
1704 for (unsigned i = 0; i < subpass->color_count; ++i) {
1705 if (subpass->color_attachments[i].attachment != VK_ATTACHMENT_UNUSED) {
1706 uses_color_att = true;
1707 break;
1708 }
1709 }
1710
1711 if (uses_color_att &&
1712 !pCreateInfo->pRasterizationState->rasterizerDiscardEnable) {
1713 assert(pCreateInfo->pColorBlendState);
1714
1715 if (states & ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS)
1716 typed_memcpy(dynamic->blend_constants,
1717 pCreateInfo->pColorBlendState->blendConstants, 4);
1718 }
1719
1720 /* If there is no depthstencil attachment, then don't read
1721 * pDepthStencilState. The Vulkan spec states that pDepthStencilState may
1722 * be NULL in this case. Even if pDepthStencilState is non-NULL, there is
1723 * no need to override the depthstencil defaults in
1724 * anv_pipeline::dynamic_state when there is no depthstencil attachment.
1725 *
1726 * Section 9.2 of the Vulkan 1.0.15 spec says:
1727 *
1728 * pDepthStencilState is [...] NULL if the pipeline has rasterization
1729 * disabled or if the subpass of the render pass the pipeline is created
1730 * against does not use a depth/stencil attachment.
1731 */
1732 if (!pCreateInfo->pRasterizationState->rasterizerDiscardEnable &&
1733 subpass->depth_stencil_attachment) {
1734 assert(pCreateInfo->pDepthStencilState);
1735
1736 if (states & ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS) {
1737 dynamic->depth_bounds.min =
1738 pCreateInfo->pDepthStencilState->minDepthBounds;
1739 dynamic->depth_bounds.max =
1740 pCreateInfo->pDepthStencilState->maxDepthBounds;
1741 }
1742
1743 if (states & ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK) {
1744 dynamic->stencil_compare_mask.front =
1745 pCreateInfo->pDepthStencilState->front.compareMask;
1746 dynamic->stencil_compare_mask.back =
1747 pCreateInfo->pDepthStencilState->back.compareMask;
1748 }
1749
1750 if (states & ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK) {
1751 dynamic->stencil_write_mask.front =
1752 pCreateInfo->pDepthStencilState->front.writeMask;
1753 dynamic->stencil_write_mask.back =
1754 pCreateInfo->pDepthStencilState->back.writeMask;
1755 }
1756
1757 if (states & ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE) {
1758 dynamic->stencil_reference.front =
1759 pCreateInfo->pDepthStencilState->front.reference;
1760 dynamic->stencil_reference.back =
1761 pCreateInfo->pDepthStencilState->back.reference;
1762 }
1763 }
1764
1765 const VkPipelineRasterizationLineStateCreateInfoEXT *line_state =
1766 vk_find_struct_const(pCreateInfo->pRasterizationState->pNext,
1767 PIPELINE_RASTERIZATION_LINE_STATE_CREATE_INFO_EXT);
1768 if (line_state) {
1769 if (states & ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE) {
1770 dynamic->line_stipple.factor = line_state->lineStippleFactor;
1771 dynamic->line_stipple.pattern = line_state->lineStipplePattern;
1772 }
1773 }
1774
1775 pipeline->dynamic_state_mask = states;
1776 }
1777
1778 static void
1779 anv_pipeline_validate_create_info(const VkGraphicsPipelineCreateInfo *info)
1780 {
1781 #ifdef DEBUG
1782 struct anv_render_pass *renderpass = NULL;
1783 struct anv_subpass *subpass = NULL;
1784
1785 /* Assert that all required members of VkGraphicsPipelineCreateInfo are
1786 * present. See the Vulkan 1.0.28 spec, Section 9.2 Graphics Pipelines.
1787 */
1788 assert(info->sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO);
1789
1790 renderpass = anv_render_pass_from_handle(info->renderPass);
1791 assert(renderpass);
1792
1793 assert(info->subpass < renderpass->subpass_count);
1794 subpass = &renderpass->subpasses[info->subpass];
1795
1796 assert(info->stageCount >= 1);
1797 assert(info->pVertexInputState);
1798 assert(info->pInputAssemblyState);
1799 assert(info->pRasterizationState);
1800 if (!info->pRasterizationState->rasterizerDiscardEnable) {
1801 assert(info->pViewportState);
1802 assert(info->pMultisampleState);
1803
1804 if (subpass && subpass->depth_stencil_attachment)
1805 assert(info->pDepthStencilState);
1806
1807 if (subpass && subpass->color_count > 0) {
1808 bool all_color_unused = true;
1809 for (int i = 0; i < subpass->color_count; i++) {
1810 if (subpass->color_attachments[i].attachment != VK_ATTACHMENT_UNUSED)
1811 all_color_unused = false;
1812 }
1813 /* pColorBlendState is ignored if the pipeline has rasterization
1814 * disabled or if the subpass of the render pass the pipeline is
1815 * created against does not use any color attachments.
1816 */
1817 assert(info->pColorBlendState || all_color_unused);
1818 }
1819 }
1820
1821 for (uint32_t i = 0; i < info->stageCount; ++i) {
1822 switch (info->pStages[i].stage) {
1823 case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
1824 case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
1825 assert(info->pTessellationState);
1826 break;
1827 default:
1828 break;
1829 }
1830 }
1831 #endif
1832 }
1833
1834 /**
1835 * Calculate the desired L3 partitioning based on the current state of the
1836 * pipeline. For now this simply returns the conservative defaults calculated
1837 * by get_default_l3_weights(), but we could probably do better by gathering
1838 * more statistics from the pipeline state (e.g. guess of expected URB usage
1839 * and bound surfaces), or by using feed-back from performance counters.
1840 */
1841 void
1842 anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm)
1843 {
1844 const struct gen_device_info *devinfo = &pipeline->device->info;
1845
1846 const struct gen_l3_weights w =
1847 gen_get_default_l3_weights(devinfo, pipeline->needs_data_cache, needs_slm);
1848
1849 pipeline->urb.l3_config = gen_get_l3_config(devinfo, w);
1850 pipeline->urb.total_size =
1851 gen_get_l3_config_urb_size(devinfo, pipeline->urb.l3_config);
1852 }
1853
1854 VkResult
1855 anv_pipeline_init(struct anv_pipeline *pipeline,
1856 struct anv_device *device,
1857 struct anv_pipeline_cache *cache,
1858 const VkGraphicsPipelineCreateInfo *pCreateInfo,
1859 const VkAllocationCallbacks *alloc)
1860 {
1861 VkResult result;
1862
1863 anv_pipeline_validate_create_info(pCreateInfo);
1864
1865 if (alloc == NULL)
1866 alloc = &device->alloc;
1867
1868 pipeline->device = device;
1869
1870 ANV_FROM_HANDLE(anv_render_pass, render_pass, pCreateInfo->renderPass);
1871 assert(pCreateInfo->subpass < render_pass->subpass_count);
1872 pipeline->subpass = &render_pass->subpasses[pCreateInfo->subpass];
1873
1874 result = anv_reloc_list_init(&pipeline->batch_relocs, alloc);
1875 if (result != VK_SUCCESS)
1876 return result;
1877
1878 pipeline->batch.alloc = alloc;
1879 pipeline->batch.next = pipeline->batch.start = pipeline->batch_data;
1880 pipeline->batch.end = pipeline->batch.start + sizeof(pipeline->batch_data);
1881 pipeline->batch.relocs = &pipeline->batch_relocs;
1882 pipeline->batch.status = VK_SUCCESS;
1883
1884 pipeline->mem_ctx = ralloc_context(NULL);
1885 pipeline->flags = pCreateInfo->flags;
1886
1887 copy_non_dynamic_state(pipeline, pCreateInfo);
1888 pipeline->depth_clamp_enable = pCreateInfo->pRasterizationState &&
1889 pCreateInfo->pRasterizationState->depthClampEnable;
1890
1891 /* Previously we enabled depth clipping when !depthClampEnable.
1892 * DepthClipStateCreateInfo now makes depth clipping explicit so if the
1893 * clipping info is available, use its enable value to determine clipping,
1894 * otherwise fallback to the previous !depthClampEnable logic.
1895 */
1896 const VkPipelineRasterizationDepthClipStateCreateInfoEXT *clip_info =
1897 vk_find_struct_const(pCreateInfo->pRasterizationState->pNext,
1898 PIPELINE_RASTERIZATION_DEPTH_CLIP_STATE_CREATE_INFO_EXT);
1899 pipeline->depth_clip_enable = clip_info ? clip_info->depthClipEnable : !pipeline->depth_clamp_enable;
1900
1901 pipeline->sample_shading_enable = pCreateInfo->pMultisampleState &&
1902 pCreateInfo->pMultisampleState->sampleShadingEnable;
1903
1904 pipeline->needs_data_cache = false;
1905
1906 /* When we free the pipeline, we detect stages based on the NULL status
1907 * of various prog_data pointers. Make them NULL by default.
1908 */
1909 memset(pipeline->shaders, 0, sizeof(pipeline->shaders));
1910 pipeline->num_executables = 0;
1911
1912 result = anv_pipeline_compile_graphics(pipeline, cache, pCreateInfo);
1913 if (result != VK_SUCCESS) {
1914 ralloc_free(pipeline->mem_ctx);
1915 anv_reloc_list_finish(&pipeline->batch_relocs, alloc);
1916 return result;
1917 }
1918
1919 assert(pipeline->shaders[MESA_SHADER_VERTEX]);
1920
1921 anv_pipeline_setup_l3_config(pipeline, false);
1922
1923 const VkPipelineVertexInputStateCreateInfo *vi_info =
1924 pCreateInfo->pVertexInputState;
1925
1926 const uint64_t inputs_read = get_vs_prog_data(pipeline)->inputs_read;
1927
1928 pipeline->vb_used = 0;
1929 for (uint32_t i = 0; i < vi_info->vertexAttributeDescriptionCount; i++) {
1930 const VkVertexInputAttributeDescription *desc =
1931 &vi_info->pVertexAttributeDescriptions[i];
1932
1933 if (inputs_read & (1ull << (VERT_ATTRIB_GENERIC0 + desc->location)))
1934 pipeline->vb_used |= 1 << desc->binding;
1935 }
1936
1937 for (uint32_t i = 0; i < vi_info->vertexBindingDescriptionCount; i++) {
1938 const VkVertexInputBindingDescription *desc =
1939 &vi_info->pVertexBindingDescriptions[i];
1940
1941 pipeline->vb[desc->binding].stride = desc->stride;
1942
1943 /* Step rate is programmed per vertex element (attribute), not
1944 * binding. Set up a map of which bindings step per instance, for
1945 * reference by vertex element setup. */
1946 switch (desc->inputRate) {
1947 default:
1948 case VK_VERTEX_INPUT_RATE_VERTEX:
1949 pipeline->vb[desc->binding].instanced = false;
1950 break;
1951 case VK_VERTEX_INPUT_RATE_INSTANCE:
1952 pipeline->vb[desc->binding].instanced = true;
1953 break;
1954 }
1955
1956 pipeline->vb[desc->binding].instance_divisor = 1;
1957 }
1958
1959 const VkPipelineVertexInputDivisorStateCreateInfoEXT *vi_div_state =
1960 vk_find_struct_const(vi_info->pNext,
1961 PIPELINE_VERTEX_INPUT_DIVISOR_STATE_CREATE_INFO_EXT);
1962 if (vi_div_state) {
1963 for (uint32_t i = 0; i < vi_div_state->vertexBindingDivisorCount; i++) {
1964 const VkVertexInputBindingDivisorDescriptionEXT *desc =
1965 &vi_div_state->pVertexBindingDivisors[i];
1966
1967 pipeline->vb[desc->binding].instance_divisor = desc->divisor;
1968 }
1969 }
1970
1971 /* Our implementation of VK_KHR_multiview uses instancing to draw the
1972 * different views. If the client asks for instancing, we need to multiply
1973 * the instance divisor by the number of views ensure that we repeat the
1974 * client's per-instance data once for each view.
1975 */
1976 if (pipeline->subpass->view_mask) {
1977 const uint32_t view_count = anv_subpass_view_count(pipeline->subpass);
1978 for (uint32_t vb = 0; vb < MAX_VBS; vb++) {
1979 if (pipeline->vb[vb].instanced)
1980 pipeline->vb[vb].instance_divisor *= view_count;
1981 }
1982 }
1983
1984 const VkPipelineInputAssemblyStateCreateInfo *ia_info =
1985 pCreateInfo->pInputAssemblyState;
1986 const VkPipelineTessellationStateCreateInfo *tess_info =
1987 pCreateInfo->pTessellationState;
1988 pipeline->primitive_restart = ia_info->primitiveRestartEnable;
1989
1990 if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL))
1991 pipeline->topology = _3DPRIM_PATCHLIST(tess_info->patchControlPoints);
1992 else
1993 pipeline->topology = vk_to_gen_primitive_type[ia_info->topology];
1994
1995 return VK_SUCCESS;
1996 }
1997
1998 #define WRITE_STR(field, ...) ({ \
1999 memset(field, 0, sizeof(field)); \
2000 UNUSED int i = snprintf(field, sizeof(field), __VA_ARGS__); \
2001 assert(i > 0 && i < sizeof(field)); \
2002 })
2003
2004 VkResult anv_GetPipelineExecutablePropertiesKHR(
2005 VkDevice device,
2006 const VkPipelineInfoKHR* pPipelineInfo,
2007 uint32_t* pExecutableCount,
2008 VkPipelineExecutablePropertiesKHR* pProperties)
2009 {
2010 ANV_FROM_HANDLE(anv_pipeline, pipeline, pPipelineInfo->pipeline);
2011 VK_OUTARRAY_MAKE(out, pProperties, pExecutableCount);
2012
2013 for (uint32_t i = 0; i < pipeline->num_executables; i++) {
2014 vk_outarray_append(&out, props) {
2015 gl_shader_stage stage = pipeline->executables[i].stage;
2016 props->stages = mesa_to_vk_shader_stage(stage);
2017
2018 unsigned simd_width = pipeline->executables[i].stats.dispatch_width;
2019 if (stage == MESA_SHADER_FRAGMENT) {
2020 WRITE_STR(props->name, "%s%d %s",
2021 simd_width ? "SIMD" : "vec",
2022 simd_width ? simd_width : 4,
2023 _mesa_shader_stage_to_string(stage));
2024 } else {
2025 WRITE_STR(props->name, "%s", _mesa_shader_stage_to_string(stage));
2026 }
2027 WRITE_STR(props->description, "%s%d %s shader",
2028 simd_width ? "SIMD" : "vec",
2029 simd_width ? simd_width : 4,
2030 _mesa_shader_stage_to_string(stage));
2031
2032 /* The compiler gives us a dispatch width of 0 for vec4 but Vulkan
2033 * wants a subgroup size of 1.
2034 */
2035 props->subgroupSize = MAX2(simd_width, 1);
2036 }
2037 }
2038
2039 return vk_outarray_status(&out);
2040 }
2041
2042 VkResult anv_GetPipelineExecutableStatisticsKHR(
2043 VkDevice device,
2044 const VkPipelineExecutableInfoKHR* pExecutableInfo,
2045 uint32_t* pStatisticCount,
2046 VkPipelineExecutableStatisticKHR* pStatistics)
2047 {
2048 ANV_FROM_HANDLE(anv_pipeline, pipeline, pExecutableInfo->pipeline);
2049 VK_OUTARRAY_MAKE(out, pStatistics, pStatisticCount);
2050
2051 assert(pExecutableInfo->executableIndex < pipeline->num_executables);
2052 const struct anv_pipeline_executable *exe =
2053 &pipeline->executables[pExecutableInfo->executableIndex];
2054 const struct brw_stage_prog_data *prog_data =
2055 pipeline->shaders[exe->stage]->prog_data;
2056
2057 vk_outarray_append(&out, stat) {
2058 WRITE_STR(stat->name, "Instruction Count");
2059 WRITE_STR(stat->description,
2060 "Number of GEN instructions in the final generated "
2061 "shader executable.");
2062 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2063 stat->value.u64 = exe->stats.instructions;
2064 }
2065
2066 vk_outarray_append(&out, stat) {
2067 WRITE_STR(stat->name, "Loop Count");
2068 WRITE_STR(stat->description,
2069 "Number of loops (not unrolled) in the final generated "
2070 "shader executable.");
2071 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2072 stat->value.u64 = exe->stats.loops;
2073 }
2074
2075 vk_outarray_append(&out, stat) {
2076 WRITE_STR(stat->name, "Cycle Count");
2077 WRITE_STR(stat->description,
2078 "Estimate of the number of EU cycles required to execute "
2079 "the final generated executable. This is an estimate only "
2080 "and may vary greatly from actual run-time performance.");
2081 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2082 stat->value.u64 = exe->stats.cycles;
2083 }
2084
2085 vk_outarray_append(&out, stat) {
2086 WRITE_STR(stat->name, "Spill Count");
2087 WRITE_STR(stat->description,
2088 "Number of scratch spill operations. This gives a rough "
2089 "estimate of the cost incurred due to spilling temporary "
2090 "values to memory. If this is non-zero, you may want to "
2091 "adjust your shader to reduce register pressure.");
2092 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2093 stat->value.u64 = exe->stats.spills;
2094 }
2095
2096 vk_outarray_append(&out, stat) {
2097 WRITE_STR(stat->name, "Fill Count");
2098 WRITE_STR(stat->description,
2099 "Number of scratch fill operations. This gives a rough "
2100 "estimate of the cost incurred due to spilling temporary "
2101 "values to memory. If this is non-zero, you may want to "
2102 "adjust your shader to reduce register pressure.");
2103 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2104 stat->value.u64 = exe->stats.fills;
2105 }
2106
2107 vk_outarray_append(&out, stat) {
2108 WRITE_STR(stat->name, "Scratch Memory Size");
2109 WRITE_STR(stat->description,
2110 "Number of bytes of scratch memory required by the "
2111 "generated shader executable. If this is non-zero, you "
2112 "may want to adjust your shader to reduce register "
2113 "pressure.");
2114 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2115 stat->value.u64 = prog_data->total_scratch;
2116 }
2117
2118 if (exe->stage == MESA_SHADER_COMPUTE) {
2119 vk_outarray_append(&out, stat) {
2120 WRITE_STR(stat->name, "Workgroup Memory Size");
2121 WRITE_STR(stat->description,
2122 "Number of bytes of workgroup shared memory used by this "
2123 "compute shader including any padding.");
2124 stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR;
2125 stat->value.u64 = prog_data->total_scratch;
2126 }
2127 }
2128
2129 return vk_outarray_status(&out);
2130 }
2131
2132 static bool
2133 write_ir_text(VkPipelineExecutableInternalRepresentationKHR* ir,
2134 const char *data)
2135 {
2136 ir->isText = VK_TRUE;
2137
2138 size_t data_len = strlen(data) + 1;
2139
2140 if (ir->pData == NULL) {
2141 ir->dataSize = data_len;
2142 return true;
2143 }
2144
2145 strncpy(ir->pData, data, ir->dataSize);
2146 if (ir->dataSize < data_len)
2147 return false;
2148
2149 ir->dataSize = data_len;
2150 return true;
2151 }
2152
2153 VkResult anv_GetPipelineExecutableInternalRepresentationsKHR(
2154 VkDevice device,
2155 const VkPipelineExecutableInfoKHR* pExecutableInfo,
2156 uint32_t* pInternalRepresentationCount,
2157 VkPipelineExecutableInternalRepresentationKHR* pInternalRepresentations)
2158 {
2159 ANV_FROM_HANDLE(anv_pipeline, pipeline, pExecutableInfo->pipeline);
2160 VK_OUTARRAY_MAKE(out, pInternalRepresentations,
2161 pInternalRepresentationCount);
2162 bool incomplete_text = false;
2163
2164 assert(pExecutableInfo->executableIndex < pipeline->num_executables);
2165 const struct anv_pipeline_executable *exe =
2166 &pipeline->executables[pExecutableInfo->executableIndex];
2167
2168 if (exe->nir) {
2169 vk_outarray_append(&out, ir) {
2170 WRITE_STR(ir->name, "Final NIR");
2171 WRITE_STR(ir->description,
2172 "Final NIR before going into the back-end compiler");
2173
2174 if (!write_ir_text(ir, exe->nir))
2175 incomplete_text = true;
2176 }
2177 }
2178
2179 if (exe->disasm) {
2180 vk_outarray_append(&out, ir) {
2181 WRITE_STR(ir->name, "GEN Assembly");
2182 WRITE_STR(ir->description,
2183 "Final GEN assembly for the generated shader binary");
2184
2185 if (!write_ir_text(ir, exe->disasm))
2186 incomplete_text = true;
2187 }
2188 }
2189
2190 return incomplete_text ? VK_INCOMPLETE : vk_outarray_status(&out);
2191 }