X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fintel%2Fvulkan%2Fanv_pipeline.c;h=cafe5668f1f45440d7123cc67337889679a55376;hb=d243bf10326b4eb5163e4f1ae35ca8692a0f6839;hp=1161e6fa9e03a5a23a216da3027ec48715e545c7;hpb=88df3bf79ae96b0f03332828125c9e2dc653494d;p=mesa.git diff --git a/src/intel/vulkan/anv_pipeline.c b/src/intel/vulkan/anv_pipeline.c index 1161e6fa9e0..cafe5668f1f 100644 --- a/src/intel/vulkan/anv_pipeline.c +++ b/src/intel/vulkan/anv_pipeline.c @@ -55,12 +55,14 @@ VkResult anv_CreateShaderModule( assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO); assert(pCreateInfo->flags == 0); - module = vk_alloc2(&device->alloc, pAllocator, + module = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*module) + pCreateInfo->codeSize, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (module == NULL) return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &module->base, + VK_OBJECT_TYPE_SHADER_MODULE); module->size = pCreateInfo->codeSize; memcpy(module->data, pCreateInfo->pCode, module->size); @@ -82,7 +84,8 @@ void anv_DestroyShaderModule( if (!module) return; - vk_free2(&device->alloc, pAllocator, module); + vk_object_base_finish(&module->base); + vk_free2(&device->vk.alloc, pAllocator, module); } #define SPIR_V_MAGIC_NUMBER 0x07230203 @@ -140,17 +143,30 @@ anv_shader_compile_to_nir(struct anv_device *device, struct nir_spirv_specialization *spec_entries = NULL; if (spec_info && spec_info->mapEntryCount > 0) { num_spec_entries = spec_info->mapEntryCount; - spec_entries = malloc(num_spec_entries * sizeof(*spec_entries)); + spec_entries = calloc(num_spec_entries, sizeof(*spec_entries)); for (uint32_t i = 0; i < num_spec_entries; i++) { VkSpecializationMapEntry entry = spec_info->pMapEntries[i]; const void *data = spec_info->pData + entry.offset; assert(data + entry.size <= spec_info->pData + spec_info->dataSize); spec_entries[i].id = spec_info->pMapEntries[i].constantID; - if (spec_info->dataSize == 8) - spec_entries[i].data64 = *(const uint64_t *)data; - else - spec_entries[i].data32 = *(const uint32_t *)data; + switch (entry.size) { + case 8: + spec_entries[i].value.u64 = *(const uint64_t *)data; + break; + case 4: + spec_entries[i].value.u32 = *(const uint32_t *)data; + break; + case 2: + spec_entries[i].value.u16 = *(const uint16_t *)data; + break; + case 1: + spec_entries[i].value.u8 = *(const uint8_t *)data; + break; + default: + assert(!"Invalid spec constant size"); + break; + } } } @@ -243,6 +259,7 @@ anv_shader_compile_to_nir(struct anv_device *device, NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp); NIR_PASS_V(nir, nir_lower_returns); NIR_PASS_V(nir, nir_inline_functions); + NIR_PASS_V(nir, nir_copy_prop); NIR_PASS_V(nir, nir_opt_deref); /* Pick off the single entrypoint that we want */ @@ -266,7 +283,8 @@ anv_shader_compile_to_nir(struct anv_device *device, NIR_PASS_V(nir, nir_split_per_member_structs); NIR_PASS_V(nir, nir_remove_dead_variables, - nir_var_shader_in | nir_var_shader_out | nir_var_system_value); + nir_var_shader_in | nir_var_shader_out | nir_var_system_value, + NULL); NIR_PASS_V(nir, nir_propagate_invariant); NIR_PASS_V(nir, nir_lower_io_to_temporaries, @@ -282,6 +300,54 @@ anv_shader_compile_to_nir(struct anv_device *device, return nir; } +VkResult +anv_pipeline_init(struct anv_pipeline *pipeline, + struct anv_device *device, + enum anv_pipeline_type type, + VkPipelineCreateFlags flags, + const VkAllocationCallbacks *pAllocator) +{ + VkResult result; + + memset(pipeline, 0, sizeof(*pipeline)); + + vk_object_base_init(&device->vk, &pipeline->base, + VK_OBJECT_TYPE_PIPELINE); + pipeline->device = device; + + /* It's the job of the child class to provide actual backing storage for + * the batch by setting batch.start, batch.next, and batch.end. + */ + pipeline->batch.alloc = pAllocator ? pAllocator : &device->vk.alloc; + pipeline->batch.relocs = &pipeline->batch_relocs; + pipeline->batch.status = VK_SUCCESS; + + result = anv_reloc_list_init(&pipeline->batch_relocs, + pipeline->batch.alloc); + if (result != VK_SUCCESS) + return result; + + pipeline->mem_ctx = ralloc_context(NULL); + + pipeline->type = type; + pipeline->flags = flags; + + util_dynarray_init(&pipeline->executables, pipeline->mem_ctx); + + return VK_SUCCESS; +} + +void +anv_pipeline_finish(struct anv_pipeline *pipeline, + struct anv_device *device, + const VkAllocationCallbacks *pAllocator) +{ + anv_reloc_list_finish(&pipeline->batch_relocs, + pAllocator ? pAllocator : &device->vk.alloc); + ralloc_free(pipeline->mem_ctx); + vk_object_base_finish(&pipeline->base); +} + void anv_DestroyPipeline( VkDevice _device, VkPipeline _pipeline, @@ -293,20 +359,37 @@ void anv_DestroyPipeline( if (!pipeline) return; - anv_reloc_list_finish(&pipeline->batch_relocs, - pAllocator ? pAllocator : &device->alloc); + switch (pipeline->type) { + case ANV_PIPELINE_GRAPHICS: { + struct anv_graphics_pipeline *gfx_pipeline = + anv_pipeline_to_graphics(pipeline); - ralloc_free(pipeline->mem_ctx); + if (gfx_pipeline->blend_state.map) + anv_state_pool_free(&device->dynamic_state_pool, gfx_pipeline->blend_state); + + for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { + if (gfx_pipeline->shaders[s]) + anv_shader_bin_unref(device, gfx_pipeline->shaders[s]); + } + break; + } - if (pipeline->blend_state.map) - anv_state_pool_free(&device->dynamic_state_pool, pipeline->blend_state); + case ANV_PIPELINE_COMPUTE: { + struct anv_compute_pipeline *compute_pipeline = + anv_pipeline_to_compute(pipeline); - for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { - if (pipeline->shaders[s]) - anv_shader_bin_unref(device, pipeline->shaders[s]); + if (compute_pipeline->cs) + anv_shader_bin_unref(device, compute_pipeline->cs); + + break; } - vk_free2(&device->alloc, pAllocator, pipeline); + default: + unreachable("invalid pipeline type"); + } + + anv_pipeline_finish(pipeline, device, pAllocator); + vk_free2(&device->vk.alloc, pAllocator, pipeline); } static const uint32_t vk_to_gen_primitive_type[] = { @@ -434,6 +517,8 @@ populate_wm_prog_key(const struct gen_device_info *devinfo, /* XXX Vulkan doesn't appear to specify */ key->clamp_fragment_color = false; + key->ignore_sample_mask_out = false; + assert(subpass->color_count <= MAX_RTS); for (uint32_t i = 0; i < subpass->color_count; i++) { if (subpass->color_attachments[i].attachment != VK_ATTACHMENT_UNUSED) @@ -559,7 +644,7 @@ anv_pipeline_hash_shader(const struct anv_shader_module *module, } static void -anv_pipeline_hash_graphics(struct anv_pipeline *pipeline, +anv_pipeline_hash_graphics(struct anv_graphics_pipeline *pipeline, struct anv_pipeline_layout *layout, struct anv_pipeline_stage *stages, unsigned char *sha1_out) @@ -573,7 +658,7 @@ anv_pipeline_hash_graphics(struct anv_pipeline *pipeline, if (layout) _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1)); - const bool rba = pipeline->device->robust_buffer_access; + const bool rba = pipeline->base.device->robust_buffer_access; _mesa_sha1_update(&ctx, &rba, sizeof(rba)); for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { @@ -588,7 +673,7 @@ anv_pipeline_hash_graphics(struct anv_pipeline *pipeline, } static void -anv_pipeline_hash_compute(struct anv_pipeline *pipeline, +anv_pipeline_hash_compute(struct anv_compute_pipeline *pipeline, struct anv_pipeline_layout *layout, struct anv_pipeline_stage *stage, unsigned char *sha1_out) @@ -599,7 +684,7 @@ anv_pipeline_hash_compute(struct anv_pipeline *pipeline, if (layout) _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1)); - const bool rba = pipeline->device->robust_buffer_access; + const bool rba = pipeline->base.device->robust_buffer_access; _mesa_sha1_update(&ctx, &rba, sizeof(rba)); _mesa_sha1_update(&ctx, stage->shader_sha1, @@ -657,18 +742,25 @@ anv_pipeline_lower_nir(struct anv_pipeline *pipeline, nir_shader *nir = stage->nir; if (nir->info.stage == MESA_SHADER_FRAGMENT) { - NIR_PASS_V(nir, nir_lower_wpos_center, pipeline->sample_shading_enable); - NIR_PASS_V(nir, nir_lower_input_attachments, true); + NIR_PASS_V(nir, nir_lower_wpos_center, + anv_pipeline_to_graphics(pipeline)->sample_shading_enable); + NIR_PASS_V(nir, nir_lower_input_attachments, + &(nir_input_attachment_options) { + .use_fragcoord_sysval = true, + .use_layer_id_sysval = true, + }); } NIR_PASS_V(nir, anv_nir_lower_ycbcr_textures, layout); - if (pipeline->type == ANV_PIPELINE_GRAPHICS) - NIR_PASS_V(nir, anv_nir_lower_multiview, pipeline->subpass->view_mask); + if (pipeline->type == ANV_PIPELINE_GRAPHICS) { + NIR_PASS_V(nir, anv_nir_lower_multiview, + anv_pipeline_to_graphics(pipeline)); + } nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir)); - NIR_PASS_V(nir, brw_nir_lower_image_load_store, compiler->devinfo); + NIR_PASS_V(nir, brw_nir_lower_image_load_store, compiler->devinfo, NULL); NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_global, nir_address_format_64bit_global); @@ -711,16 +803,23 @@ anv_pipeline_link_vs(const struct brw_compiler *compiler, static void anv_pipeline_compile_vs(const struct brw_compiler *compiler, void *mem_ctx, - struct anv_device *device, + struct anv_graphics_pipeline *pipeline, struct anv_pipeline_stage *vs_stage) { + /* When using Primitive Replication for multiview, each view gets its own + * position slot. + */ + uint32_t pos_slots = pipeline->use_primitive_replication ? + anv_subpass_view_count(pipeline->subpass) : 1; + brw_compute_vue_map(compiler->devinfo, &vs_stage->prog_data.vs.base.vue_map, vs_stage->nir->info.outputs_written, - vs_stage->nir->info.separate_shader); + vs_stage->nir->info.separate_shader, + pos_slots); vs_stage->num_stats = 1; - vs_stage->code = brw_compile_vs(compiler, device, mem_ctx, + vs_stage->code = brw_compile_vs(compiler, pipeline->base.device, mem_ctx, &vs_stage->key.vs, &vs_stage->prog_data.vs, vs_stage->nir, -1, @@ -863,7 +962,7 @@ anv_pipeline_compile_gs(const struct brw_compiler *compiler, brw_compute_vue_map(compiler->devinfo, &gs_stage->prog_data.gs.base.vue_map, gs_stage->nir->info.outputs_written, - gs_stage->nir->info.separate_shader); + gs_stage->nir->info.separate_shader, 1); gs_stage->num_stats = 1; gs_stage->code = brw_compile_gs(compiler, device, mem_ctx, @@ -918,7 +1017,7 @@ anv_pipeline_link_fs(const struct brw_compiler *compiler, */ nir_function_impl *impl = nir_shader_get_entrypoint(stage->nir); bool deleted_output = false; - nir_foreach_variable_safe(var, &stage->nir->outputs) { + nir_foreach_shader_out_variable_safe(var, stage->nir) { /* TODO: We don't delete depth/stencil writes. We probably could if the * subpass doesn't have a depth/stencil attachment. */ @@ -1034,6 +1133,56 @@ anv_pipeline_add_executable(struct anv_pipeline *pipeline, size_t stream_size = 0; FILE *stream = open_memstream(&stream_data, &stream_size); + uint32_t push_size = 0; + for (unsigned i = 0; i < 4; i++) + push_size += stage->bind_map.push_ranges[i].length; + if (push_size > 0) { + fprintf(stream, "Push constant ranges:\n"); + for (unsigned i = 0; i < 4; i++) { + if (stage->bind_map.push_ranges[i].length == 0) + continue; + + fprintf(stream, " RANGE%d (%dB): ", i, + stage->bind_map.push_ranges[i].length * 32); + + switch (stage->bind_map.push_ranges[i].set) { + case ANV_DESCRIPTOR_SET_NULL: + fprintf(stream, "NULL"); + break; + + case ANV_DESCRIPTOR_SET_PUSH_CONSTANTS: + fprintf(stream, "Vulkan push constants and API params"); + break; + + case ANV_DESCRIPTOR_SET_DESCRIPTORS: + fprintf(stream, "Descriptor buffer for set %d (start=%dB)", + stage->bind_map.push_ranges[i].index, + stage->bind_map.push_ranges[i].start * 32); + break; + + case ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS: + unreachable("gl_NumWorkgroups is never pushed"); + + case ANV_DESCRIPTOR_SET_SHADER_CONSTANTS: + fprintf(stream, "Inline shader constant data (start=%dB)", + stage->bind_map.push_ranges[i].start * 32); + break; + + case ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS: + unreachable("Color attachments can't be pushed"); + + default: + fprintf(stream, "UBO (set=%d binding=%d start=%dB)", + stage->bind_map.push_ranges[i].set, + stage->bind_map.push_ranges[i].index, + stage->bind_map.push_ranges[i].start * 32); + break; + } + fprintf(stream, "\n"); + } + fprintf(stream, "\n"); + } + /* Creating this is far cheaper than it looks. It's perfectly fine to * do it for every binary. */ @@ -1093,8 +1242,29 @@ anv_pipeline_add_executables(struct anv_pipeline *pipeline, } } +static void +anv_pipeline_init_from_cached_graphics(struct anv_graphics_pipeline *pipeline) +{ + /* TODO: Cache this pipeline-wide information. */ + + /* Primitive replication depends on information from all the shaders. + * Recover this bit from the fact that we have more than one position slot + * in the vertex shader when using it. + */ + assert(pipeline->active_stages & VK_SHADER_STAGE_VERTEX_BIT); + int pos_slots = 0; + const struct brw_vue_prog_data *vue_prog_data = + (const void *) pipeline->shaders[MESA_SHADER_VERTEX]->prog_data; + const struct brw_vue_map *vue_map = &vue_prog_data->vue_map; + for (int i = 0; i < vue_map->num_slots; i++) { + if (vue_map->slot_to_varying[i] == VARYING_SLOT_POS) + pos_slots++; + } + pipeline->use_primitive_replication = pos_slots > 1; +} + static VkResult -anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, +anv_pipeline_compile_graphics(struct anv_graphics_pipeline *pipeline, struct anv_pipeline_cache *cache, const VkGraphicsPipelineCreateInfo *info) { @@ -1103,7 +1273,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, }; int64_t pipeline_start = os_time_get_nano(); - const struct brw_compiler *compiler = pipeline->device->physical->compiler; + const struct brw_compiler *compiler = pipeline->base.device->physical->compiler; struct anv_pipeline_stage stages[MESA_SHADER_STAGES] = {}; pipeline->active_stages = 0; @@ -1127,7 +1297,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, stages[stage].spec_info, stages[stage].shader_sha1); - const struct gen_device_info *devinfo = &pipeline->device->info; + const struct gen_device_info *devinfo = &pipeline->base.device->info; switch (stage) { case MESA_SHADER_VERTEX: populate_vs_prog_key(devinfo, sinfo->flags, &stages[stage].key.vs); @@ -1179,7 +1349,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, } const bool skip_cache_lookup = - (pipeline->flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR); + (pipeline->base.flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR); if (!skip_cache_lookup) { unsigned found = 0; @@ -1192,7 +1362,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, bool cache_hit; struct anv_shader_bin *bin = - anv_device_search_for_kernel(pipeline->device, cache, + anv_device_search_for_kernel(pipeline->base.device, cache, &stages[s].cache_key, sizeof(stages[s].cache_key), &cache_hit); if (bin) { @@ -1218,9 +1388,10 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, if (!stages[s].entrypoint) continue; - anv_pipeline_add_executables(pipeline, &stages[s], + anv_pipeline_add_executables(&pipeline->base, &stages[s], pipeline->shaders[s]); } + anv_pipeline_init_from_cached_graphics(pipeline); goto done; } else if (found > 0) { /* We found some but not all of our shaders. This shouldn't happen @@ -1229,7 +1400,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, */ assert(found < __builtin_popcount(pipeline->active_stages)); - vk_debug_report(&pipeline->device->physical->instance->debug_report_callbacks, + vk_debug_report(&pipeline->base.device->physical->instance->debug_report_callbacks, VK_DEBUG_REPORT_WARNING_BIT_EXT | VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT, VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_CACHE_EXT, @@ -1246,13 +1417,16 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { stages[s].feedback.flags = 0; if (pipeline->shaders[s]) { - anv_shader_bin_unref(pipeline->device, pipeline->shaders[s]); + anv_shader_bin_unref(pipeline->base.device, pipeline->shaders[s]); pipeline->shaders[s] = NULL; } } } } + if (info->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_EXT) + return VK_PIPELINE_COMPILE_REQUIRED_EXT; + void *pipeline_ctx = ralloc_context(NULL); for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { @@ -1269,7 +1443,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, .sampler_to_descriptor = stages[s].sampler_to_descriptor }; - stages[s].nir = anv_pipeline_stage_get_nir(pipeline, cache, + stages[s].nir = anv_pipeline_stage_get_nir(&pipeline->base, cache, pipeline_ctx, &stages[s]); if (stages[s].nir == NULL) { @@ -1309,6 +1483,23 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, next_stage = &stages[s]; } + if (pipeline->base.device->info.gen >= 12 && + pipeline->subpass->view_mask != 0) { + /* For some pipelines HW Primitive Replication can be used instead of + * instancing to implement Multiview. This depend on how viewIndex is + * used in all the active shaders, so this check can't be done per + * individual shaders. + */ + nir_shader *shaders[MESA_SHADER_STAGES] = {}; + for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) + shaders[s] = stages[s].nir; + + pipeline->use_primitive_replication = + anv_check_for_primitive_replication(shaders, pipeline); + } else { + pipeline->use_primitive_replication = false; + } + struct anv_pipeline_stage *prev_stage = NULL; for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { if (!stages[s].entrypoint) @@ -1318,33 +1509,58 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, void *stage_ctx = ralloc_context(NULL); + anv_pipeline_lower_nir(&pipeline->base, stage_ctx, &stages[s], layout); + + if (prev_stage && compiler->glsl_compiler_options[s].NirOptions->unify_interfaces) { + prev_stage->nir->info.outputs_written |= stages[s].nir->info.inputs_read & + ~(VARYING_BIT_TESS_LEVEL_INNER | VARYING_BIT_TESS_LEVEL_OUTER); + stages[s].nir->info.inputs_read |= prev_stage->nir->info.outputs_written & + ~(VARYING_BIT_TESS_LEVEL_INNER | VARYING_BIT_TESS_LEVEL_OUTER); + prev_stage->nir->info.patch_outputs_written |= stages[s].nir->info.patch_inputs_read; + stages[s].nir->info.patch_inputs_read |= prev_stage->nir->info.patch_outputs_written; + } + + ralloc_free(stage_ctx); + + stages[s].feedback.duration += os_time_get_nano() - stage_start; + + prev_stage = &stages[s]; + } + + prev_stage = NULL; + for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { + if (!stages[s].entrypoint) + continue; + + int64_t stage_start = os_time_get_nano(); + + void *stage_ctx = ralloc_context(NULL); + nir_xfb_info *xfb_info = NULL; if (s == MESA_SHADER_VERTEX || s == MESA_SHADER_TESS_EVAL || s == MESA_SHADER_GEOMETRY) xfb_info = nir_gather_xfb_info(stages[s].nir, stage_ctx); - anv_pipeline_lower_nir(pipeline, stage_ctx, &stages[s], layout); - switch (s) { case MESA_SHADER_VERTEX: - anv_pipeline_compile_vs(compiler, stage_ctx, pipeline->device, + anv_pipeline_compile_vs(compiler, stage_ctx, pipeline, &stages[s]); break; case MESA_SHADER_TESS_CTRL: - anv_pipeline_compile_tcs(compiler, stage_ctx, pipeline->device, + anv_pipeline_compile_tcs(compiler, stage_ctx, pipeline->base.device, &stages[s], prev_stage); break; case MESA_SHADER_TESS_EVAL: - anv_pipeline_compile_tes(compiler, stage_ctx, pipeline->device, + anv_pipeline_compile_tes(compiler, stage_ctx, pipeline->base.device, &stages[s], prev_stage); break; case MESA_SHADER_GEOMETRY: - anv_pipeline_compile_gs(compiler, stage_ctx, pipeline->device, + anv_pipeline_compile_gs(compiler, stage_ctx, pipeline->base.device, &stages[s], prev_stage); break; case MESA_SHADER_FRAGMENT: - anv_pipeline_compile_fs(compiler, stage_ctx, pipeline->device, + anv_pipeline_compile_fs(compiler, stage_ctx, pipeline->base.device, &stages[s], prev_stage); break; default: @@ -1360,7 +1576,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, &stages[s].bind_map); struct anv_shader_bin *bin = - anv_device_upload_kernel(pipeline->device, cache, s, + anv_device_upload_kernel(pipeline->base.device, cache, s, &stages[s].cache_key, sizeof(stages[s].cache_key), stages[s].code, @@ -1377,7 +1593,7 @@ anv_pipeline_compile_graphics(struct anv_pipeline *pipeline, goto fail; } - anv_pipeline_add_executables(pipeline, &stages[s], bin); + anv_pipeline_add_executables(&pipeline->base, &stages[s], bin); pipeline->shaders[s] = bin; ralloc_free(stage_ctx); @@ -1396,7 +1612,7 @@ done: /* This can happen if we decided to implicitly disable the fragment * shader. See anv_pipeline_compile_fs(). */ - anv_shader_bin_unref(pipeline->device, + anv_shader_bin_unref(pipeline->base.device, pipeline->shaders[MESA_SHADER_FRAGMENT]); pipeline->shaders[MESA_SHADER_FRAGMENT] = NULL; pipeline->active_stages &= ~VK_SHADER_STAGE_FRAGMENT_BIT; @@ -1423,7 +1639,7 @@ fail: for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { if (pipeline->shaders[s]) - anv_shader_bin_unref(pipeline->device, pipeline->shaders[s]); + anv_shader_bin_unref(pipeline->base.device, pipeline->shaders[s]); } return result; @@ -1442,7 +1658,7 @@ shared_type_info(const struct glsl_type *type, unsigned *size, unsigned *align) } VkResult -anv_pipeline_compile_cs(struct anv_pipeline *pipeline, +anv_pipeline_compile_cs(struct anv_compute_pipeline *pipeline, struct anv_pipeline_cache *cache, const VkComputePipelineCreateInfo *info, const struct anv_shader_module *module, @@ -1454,7 +1670,7 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, }; int64_t pipeline_start = os_time_get_nano(); - const struct brw_compiler *compiler = pipeline->device->physical->compiler; + const struct brw_compiler *compiler = pipeline->base.device->physical->compiler; struct anv_pipeline_stage stage = { .stage = MESA_SHADER_COMPUTE, @@ -1480,24 +1696,28 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, vk_find_struct_const(info->stage.pNext, PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT); - populate_cs_prog_key(&pipeline->device->info, info->stage.flags, + populate_cs_prog_key(&pipeline->base.device->info, info->stage.flags, rss_info, &stage.key.cs); ANV_FROM_HANDLE(anv_pipeline_layout, layout, info->layout); const bool skip_cache_lookup = - (pipeline->flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR); + (pipeline->base.flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR); anv_pipeline_hash_compute(pipeline, layout, &stage, stage.cache_key.sha1); bool cache_hit = false; if (!skip_cache_lookup) { - bin = anv_device_search_for_kernel(pipeline->device, cache, + bin = anv_device_search_for_kernel(pipeline->base.device, cache, &stage.cache_key, sizeof(stage.cache_key), &cache_hit); } + if (bin == NULL && + (info->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_EXT)) + return VK_PIPELINE_COMPILE_REQUIRED_EXT; + void *mem_ctx = ralloc_context(NULL); if (bin == NULL) { int64_t stage_start = os_time_get_nano(); @@ -1513,7 +1733,7 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, .set = ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS, }; - stage.nir = anv_pipeline_stage_get_nir(pipeline, cache, mem_ctx, &stage); + stage.nir = anv_pipeline_stage_get_nir(&pipeline->base, cache, mem_ctx, &stage); if (stage.nir == NULL) { ralloc_free(mem_ctx); return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); @@ -1521,15 +1741,16 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, NIR_PASS_V(stage.nir, anv_nir_add_base_work_group_id); - anv_pipeline_lower_nir(pipeline, mem_ctx, &stage, layout); + anv_pipeline_lower_nir(&pipeline->base, mem_ctx, &stage, layout); NIR_PASS_V(stage.nir, nir_lower_vars_to_explicit_types, nir_var_mem_shared, shared_type_info); NIR_PASS_V(stage.nir, nir_lower_explicit_io, nir_var_mem_shared, nir_address_format_32bit_offset); + NIR_PASS_V(stage.nir, brw_nir_lower_cs_intrinsics); stage.num_stats = 1; - stage.code = brw_compile_cs(compiler, pipeline->device, mem_ctx, + stage.code = brw_compile_cs(compiler, pipeline->base.device, mem_ctx, &stage.key.cs, &stage.prog_data.cs, stage.nir, -1, stage.stats, NULL); if (stage.code == NULL) { @@ -1546,7 +1767,7 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, } const unsigned code_size = stage.prog_data.base.program_size; - bin = anv_device_upload_kernel(pipeline->device, cache, + bin = anv_device_upload_kernel(pipeline->base.device, cache, MESA_SHADER_COMPUTE, &stage.cache_key, sizeof(stage.cache_key), stage.code, code_size, @@ -1564,7 +1785,7 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, stage.feedback.duration = os_time_get_nano() - stage_start; } - anv_pipeline_add_executables(pipeline, &stage, bin); + anv_pipeline_add_executables(&pipeline->base, &stage, bin); ralloc_free(mem_ctx); @@ -1585,12 +1806,29 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, create_feedback->pPipelineStageCreationFeedbacks[0] = stage.feedback; } - pipeline->active_stages = VK_SHADER_STAGE_COMPUTE_BIT; - pipeline->shaders[MESA_SHADER_COMPUTE] = bin; + pipeline->cs = bin; return VK_SUCCESS; } +struct anv_cs_parameters +anv_cs_parameters(const struct anv_compute_pipeline *pipeline) +{ + const struct brw_cs_prog_data *cs_prog_data = get_cs_prog_data(pipeline); + + struct anv_cs_parameters cs_params = {}; + + cs_params.group_size = cs_prog_data->local_size[0] * + cs_prog_data->local_size[1] * + cs_prog_data->local_size[2]; + cs_params.simd_size = + brw_cs_simd_size_for_group_size(&pipeline->base.device->info, + cs_prog_data, cs_params.group_size); + cs_params.threads = DIV_ROUND_UP(cs_params.group_size, cs_params.simd_size); + + return cs_params; +} + /** * Copy pipeline state not marked as dynamic. * Dynamic state is pipeline state which hasn't been provided at pipeline @@ -1605,7 +1843,7 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline, * @param[in] pCreateInfo Source of non_dynamic state to be copied. */ static void -copy_non_dynamic_state(struct anv_pipeline *pipeline, +copy_non_dynamic_state(struct anv_graphics_pipeline *pipeline, const VkGraphicsPipelineCreateInfo *pCreateInfo) { anv_cmd_dirty_mask_t states = ANV_CMD_DIRTY_DYNAMIC_ALL; @@ -1662,6 +1900,36 @@ copy_non_dynamic_state(struct anv_pipeline *pipeline, pCreateInfo->pRasterizationState->depthBiasSlopeFactor; } + if (states & ANV_CMD_DIRTY_DYNAMIC_CULL_MODE) { + assert(pCreateInfo->pRasterizationState); + dynamic->cull_mode = + pCreateInfo->pRasterizationState->cullMode; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_FRONT_FACE) { + assert(pCreateInfo->pRasterizationState); + dynamic->front_face = + pCreateInfo->pRasterizationState->frontFace; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_PRIMITIVE_TOPOLOGY) { + assert(pCreateInfo->pInputAssemblyState); + bool has_tess = false; + for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) { + const VkPipelineShaderStageCreateInfo *sinfo = &pCreateInfo->pStages[i]; + gl_shader_stage stage = vk_to_mesa_shader_stage(sinfo->stage); + if (stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_TESS_EVAL) + has_tess = true; + } + if (has_tess) { + const VkPipelineTessellationStateCreateInfo *tess_info = + pCreateInfo->pTessellationState; + dynamic->primitive_topology = _3DPRIM_PATCHLIST(tess_info->patchControlPoints); + } else { + dynamic->primitive_topology = pCreateInfo->pInputAssemblyState->topology; + } + } + /* Section 9.2 of the Vulkan 1.0.15 spec says: * * pColorBlendState is [...] NULL if the pipeline has rasterization @@ -1728,6 +1996,40 @@ copy_non_dynamic_state(struct anv_pipeline *pipeline, dynamic->stencil_reference.back = pCreateInfo->pDepthStencilState->back.reference; } + + if (states & ANV_CMD_DIRTY_DYNAMIC_DEPTH_TEST_ENABLE) { + dynamic->depth_test_enable = + pCreateInfo->pDepthStencilState->depthTestEnable; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_DEPTH_WRITE_ENABLE) { + dynamic->depth_write_enable = + pCreateInfo->pDepthStencilState->depthWriteEnable; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_DEPTH_COMPARE_OP) { + dynamic->depth_compare_op = + pCreateInfo->pDepthStencilState->depthCompareOp; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE) { + dynamic->depth_bounds_test_enable = + pCreateInfo->pDepthStencilState->depthBoundsTestEnable; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_STENCIL_TEST_ENABLE) { + dynamic->stencil_test_enable = + pCreateInfo->pDepthStencilState->stencilTestEnable; + } + + if (states & ANV_CMD_DIRTY_DYNAMIC_STENCIL_OP) { + const VkPipelineDepthStencilStateCreateInfo *info = + pCreateInfo->pDepthStencilState; + memcpy(&dynamic->stencil_op.front, &info->front, + sizeof(dynamic->stencil_op.front)); + memcpy(&dynamic->stencil_op.back, &info->back, + sizeof(dynamic->stencil_op.back)); + } } const VkPipelineRasterizationLineStateCreateInfoEXT *line_state = @@ -1818,39 +2120,29 @@ anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm) } VkResult -anv_pipeline_init(struct anv_pipeline *pipeline, - struct anv_device *device, - struct anv_pipeline_cache *cache, - const VkGraphicsPipelineCreateInfo *pCreateInfo, - const VkAllocationCallbacks *alloc) +anv_graphics_pipeline_init(struct anv_graphics_pipeline *pipeline, + struct anv_device *device, + struct anv_pipeline_cache *cache, + const VkGraphicsPipelineCreateInfo *pCreateInfo, + const VkAllocationCallbacks *alloc) { VkResult result; anv_pipeline_validate_create_info(pCreateInfo); - if (alloc == NULL) - alloc = &device->alloc; + result = anv_pipeline_init(&pipeline->base, device, + ANV_PIPELINE_GRAPHICS, pCreateInfo->flags, + alloc); + if (result != VK_SUCCESS) + return result; - pipeline->device = device; - pipeline->type = ANV_PIPELINE_GRAPHICS; + anv_batch_set_storage(&pipeline->base.batch, ANV_NULL_ADDRESS, + pipeline->batch_data, sizeof(pipeline->batch_data)); ANV_FROM_HANDLE(anv_render_pass, render_pass, pCreateInfo->renderPass); assert(pCreateInfo->subpass < render_pass->subpass_count); pipeline->subpass = &render_pass->subpasses[pCreateInfo->subpass]; - result = anv_reloc_list_init(&pipeline->batch_relocs, alloc); - if (result != VK_SUCCESS) - return result; - - pipeline->batch.alloc = alloc; - pipeline->batch.next = pipeline->batch.start = pipeline->batch_data; - pipeline->batch.end = pipeline->batch.start + sizeof(pipeline->batch_data); - pipeline->batch.relocs = &pipeline->batch_relocs; - pipeline->batch.status = VK_SUCCESS; - - pipeline->mem_ctx = ralloc_context(NULL); - pipeline->flags = pCreateInfo->flags; - assert(pCreateInfo->pRasterizationState); copy_non_dynamic_state(pipeline, pCreateInfo); @@ -1876,18 +2168,15 @@ anv_pipeline_init(struct anv_pipeline *pipeline, */ memset(pipeline->shaders, 0, sizeof(pipeline->shaders)); - util_dynarray_init(&pipeline->executables, pipeline->mem_ctx); - result = anv_pipeline_compile_graphics(pipeline, cache, pCreateInfo); if (result != VK_SUCCESS) { - ralloc_free(pipeline->mem_ctx); - anv_reloc_list_finish(&pipeline->batch_relocs, alloc); + anv_pipeline_finish(&pipeline->base, device, alloc); return result; } assert(pipeline->shaders[MESA_SHADER_VERTEX]); - anv_pipeline_setup_l3_config(pipeline, false); + anv_pipeline_setup_l3_config(&pipeline->base, false); const VkPipelineVertexInputStateCreateInfo *vi_info = pCreateInfo->pVertexInputState; @@ -1942,7 +2231,7 @@ anv_pipeline_init(struct anv_pipeline *pipeline, * the instance divisor by the number of views ensure that we repeat the * client's per-instance data once for each view. */ - if (pipeline->subpass->view_mask) { + if (pipeline->subpass->view_mask && !pipeline->use_primitive_replication) { const uint32_t view_count = anv_subpass_view_count(pipeline->subpass); for (uint32_t vb = 0; vb < MAX_VBS; vb++) { if (pipeline->vb[vb].instanced) @@ -2028,8 +2317,20 @@ VkResult anv_GetPipelineExecutableStatisticsKHR( const struct anv_pipeline_executable *exe = anv_pipeline_get_executable(pipeline, pExecutableInfo->executableIndex); - const struct brw_stage_prog_data *prog_data = - pipeline->shaders[exe->stage]->prog_data; + + const struct brw_stage_prog_data *prog_data; + switch (pipeline->type) { + case ANV_PIPELINE_GRAPHICS: { + prog_data = anv_pipeline_to_graphics(pipeline)->shaders[exe->stage]->prog_data; + break; + } + case ANV_PIPELINE_COMPUTE: { + prog_data = anv_pipeline_to_compute(pipeline)->cs->prog_data; + break; + } + default: + unreachable("invalid pipeline type"); + } vk_outarray_append(&out, stat) { WRITE_STR(stat->name, "Instruction Count"); @@ -2040,6 +2341,16 @@ VkResult anv_GetPipelineExecutableStatisticsKHR( stat->value.u64 = exe->stats.instructions; } + vk_outarray_append(&out, stat) { + WRITE_STR(stat->name, "SEND Count"); + WRITE_STR(stat->description, + "Number of instructions in the final generated shader " + "executable which access external units such as the " + "constant cache or the sampler."); + stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR; + stat->value.u64 = exe->stats.sends; + } + vk_outarray_append(&out, stat) { WRITE_STR(stat->name, "Loop Count"); WRITE_STR(stat->description, @@ -2099,7 +2410,7 @@ VkResult anv_GetPipelineExecutableStatisticsKHR( "Number of bytes of workgroup shared memory used by this " "compute shader including any padding."); stat->format = VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR; - stat->value.u64 = prog_data->total_scratch; + stat->value.u64 = brw_cs_prog_data_const(prog_data)->slm_size; } }