X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fvulkan%2Fradv_pipeline_cache.c;h=a85634963259658003582dfa96e92e463068f16e;hb=b50ae770144ef6622591c7cc23aa96e45933cf37;hp=fa760f31eeec147f698a4c11a59e1a40a61172b4;hpb=45ea90ef1f1018d51f7c58e1de593f8534c14e00;p=mesa.git diff --git a/src/amd/vulkan/radv_pipeline_cache.c b/src/amd/vulkan/radv_pipeline_cache.c index fa760f31eee..a8563496325 100644 --- a/src/amd/vulkan/radv_pipeline_cache.c +++ b/src/amd/vulkan/radv_pipeline_cache.c @@ -21,12 +21,15 @@ * IN THE SOFTWARE. */ +#include "util/macros.h" #include "util/mesa-sha1.h" #include "util/debug.h" +#include "util/disk_cache.h" #include "util/u_atomic.h" #include "radv_debug.h" #include "radv_private.h" #include "radv_shader.h" +#include "vulkan/util/vk_util.h" #include "ac_nir_to_llvm.h" @@ -35,20 +38,36 @@ struct cache_entry { unsigned char sha1[20]; uint32_t sha1_dw[5]; }; - uint32_t code_size; - struct ac_shader_variant_info variant_info; - struct ac_shader_config config; - uint32_t rsrc1, rsrc2; - struct radv_shader_variant *variant; - uint32_t code[0]; + uint32_t binary_sizes[MESA_SHADER_STAGES]; + struct radv_shader_variant *variants[MESA_SHADER_STAGES]; + char code[0]; }; +static void +radv_pipeline_cache_lock(struct radv_pipeline_cache *cache) +{ + if (cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT) + return; + + pthread_mutex_lock(&cache->mutex); +} + +static void +radv_pipeline_cache_unlock(struct radv_pipeline_cache *cache) +{ + if (cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT) + return; + + pthread_mutex_unlock(&cache->mutex); +} + void radv_pipeline_cache_init(struct radv_pipeline_cache *cache, struct radv_device *device) { cache->device = device; pthread_mutex_init(&cache->mutex, NULL); + cache->flags = 0; cache->modified = false; cache->kernel_count = 0; @@ -58,9 +77,10 @@ radv_pipeline_cache_init(struct radv_pipeline_cache *cache, cache->hash_table = malloc(byte_size); /* We don't consider allocation failure fatal, we just start with a 0-sized - * cache. */ + * cache. Disable caching when we want to keep shader debug info, since + * we don't get the debug info on cached shaders. */ if (cache->hash_table == NULL || - (device->debug_flags & RADV_DEBUG_NO_CACHE)) + (device->instance->debug_flags & RADV_DEBUG_NO_CACHE)) cache->table_size = 0; else memset(cache->hash_table, 0, byte_size); @@ -71,9 +91,11 @@ radv_pipeline_cache_finish(struct radv_pipeline_cache *cache) { for (unsigned i = 0; i < cache->table_size; ++i) if (cache->hash_table[i]) { - if (cache->hash_table[i]->variant) - radv_shader_variant_destroy(cache->device, - cache->hash_table[i]->variant); + for(int j = 0; j < MESA_SHADER_STAGES; ++j) { + if (cache->hash_table[i]->variants[j]) + radv_shader_variant_destroy(cache->device, + cache->hash_table[i]->variants[j]); + } vk_free(&cache->alloc, cache->hash_table[i]); } pthread_mutex_destroy(&cache->mutex); @@ -83,32 +105,44 @@ radv_pipeline_cache_finish(struct radv_pipeline_cache *cache) static uint32_t entry_size(struct cache_entry *entry) { - return sizeof(*entry) + entry->code_size; + size_t ret = sizeof(*entry); + for (int i = 0; i < MESA_SHADER_STAGES; ++i) + if (entry->binary_sizes[i]) + ret += entry->binary_sizes[i]; + ret = align(ret, alignof(struct cache_entry)); + return ret; } void -radv_hash_shader(unsigned char *hash, struct radv_shader_module *module, - const char *entrypoint, - const VkSpecializationInfo *spec_info, - const struct radv_pipeline_layout *layout, - const struct ac_shader_variant_key *key, - uint32_t is_geom_copy_shader) +radv_hash_shaders(unsigned char *hash, + const VkPipelineShaderStageCreateInfo **stages, + const struct radv_pipeline_layout *layout, + const struct radv_pipeline_key *key, + uint32_t flags) { struct mesa_sha1 ctx; _mesa_sha1_init(&ctx); if (key) _mesa_sha1_update(&ctx, key, sizeof(*key)); - _mesa_sha1_update(&ctx, module->sha1, sizeof(module->sha1)); - _mesa_sha1_update(&ctx, entrypoint, strlen(entrypoint)); if (layout) _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1)); - if (spec_info) { - _mesa_sha1_update(&ctx, spec_info->pMapEntries, - spec_info->mapEntryCount * sizeof spec_info->pMapEntries[0]); - _mesa_sha1_update(&ctx, spec_info->pData, spec_info->dataSize); + + for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + if (stages[i]) { + RADV_FROM_HANDLE(radv_shader_module, module, stages[i]->module); + const VkSpecializationInfo *spec_info = stages[i]->pSpecializationInfo; + + _mesa_sha1_update(&ctx, module->sha1, sizeof(module->sha1)); + _mesa_sha1_update(&ctx, stages[i]->pName, strlen(stages[i]->pName)); + if (spec_info && spec_info->mapEntryCount) { + _mesa_sha1_update(&ctx, spec_info->pMapEntries, + spec_info->mapEntryCount * sizeof spec_info->pMapEntries[0]); + _mesa_sha1_update(&ctx, spec_info->pData, spec_info->dataSize); + } + } } - _mesa_sha1_update(&ctx, &is_geom_copy_shader, 4); + _mesa_sha1_update(&ctx, &flags, 4); _mesa_sha1_final(&ctx, hash); } @@ -144,54 +178,15 @@ radv_pipeline_cache_search(struct radv_pipeline_cache *cache, { struct cache_entry *entry; - pthread_mutex_lock(&cache->mutex); + radv_pipeline_cache_lock(cache); entry = radv_pipeline_cache_search_unlocked(cache, sha1); - pthread_mutex_unlock(&cache->mutex); + radv_pipeline_cache_unlock(cache); return entry; } -struct radv_shader_variant * -radv_create_shader_variant_from_pipeline_cache(struct radv_device *device, - struct radv_pipeline_cache *cache, - const unsigned char *sha1) -{ - struct cache_entry *entry = NULL; - - if (cache) - entry = radv_pipeline_cache_search(cache, sha1); - - if (!entry) - return NULL; - - if (!entry->variant) { - struct radv_shader_variant *variant; - - variant = calloc(1, sizeof(struct radv_shader_variant)); - if (!variant) - return NULL; - - variant->code_size = entry->code_size; - variant->config = entry->config; - variant->info = entry->variant_info; - variant->rsrc1 = entry->rsrc1; - variant->rsrc2 = entry->rsrc2; - variant->code_size = entry->code_size; - variant->ref_count = 1; - - void *ptr = radv_alloc_shader_memory(device, variant); - memcpy(ptr, entry->code, entry->code_size); - - entry->variant = variant; - } - - p_atomic_inc(&entry->variant->ref_count); - return entry->variant; -} - - static void radv_pipeline_cache_set_entry(struct radv_pipeline_cache *cache, struct cache_entry *entry) @@ -226,7 +221,7 @@ radv_pipeline_cache_grow(struct radv_pipeline_cache *cache) table = malloc(byte_size); if (table == NULL) - return VK_ERROR_OUT_OF_HOST_MEMORY; + return vk_error(cache->device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); cache->hash_table = table; cache->table_size = table_size; @@ -261,81 +256,226 @@ radv_pipeline_cache_add_entry(struct radv_pipeline_cache *cache, radv_pipeline_cache_set_entry(cache, entry); } -struct radv_shader_variant * -radv_pipeline_cache_insert_shader(struct radv_pipeline_cache *cache, - const unsigned char *sha1, - struct radv_shader_variant *variant, - const void *code, unsigned code_size) +static bool +radv_is_cache_disabled(struct radv_device *device) +{ + /* Pipeline caches can be disabled with RADV_DEBUG=nocache, with + * MESA_GLSL_CACHE_DISABLE=1, and when VK_AMD_shader_info is requested. + */ + return (device->instance->debug_flags & RADV_DEBUG_NO_CACHE); +} + +bool +radv_create_shader_variants_from_pipeline_cache(struct radv_device *device, + struct radv_pipeline_cache *cache, + const unsigned char *sha1, + struct radv_shader_variant **variants, + bool *found_in_application_cache) +{ + struct cache_entry *entry; + + if (!cache) { + cache = device->mem_cache; + *found_in_application_cache = false; + } + + radv_pipeline_cache_lock(cache); + + entry = radv_pipeline_cache_search_unlocked(cache, sha1); + + if (!entry) { + *found_in_application_cache = false; + + /* Don't cache when we want debug info, since this isn't + * present in the cache. + */ + if (radv_is_cache_disabled(device) || !device->physical_device->disk_cache) { + radv_pipeline_cache_unlock(cache); + return false; + } + + uint8_t disk_sha1[20]; + disk_cache_compute_key(device->physical_device->disk_cache, + sha1, 20, disk_sha1); + + entry = (struct cache_entry *) + disk_cache_get(device->physical_device->disk_cache, + disk_sha1, NULL); + if (!entry) { + radv_pipeline_cache_unlock(cache); + return false; + } else { + size_t size = entry_size(entry); + struct cache_entry *new_entry = vk_alloc(&cache->alloc, size, 8, + VK_SYSTEM_ALLOCATION_SCOPE_CACHE); + if (!new_entry) { + free(entry); + radv_pipeline_cache_unlock(cache); + return false; + } + + memcpy(new_entry, entry, entry_size(entry)); + free(entry); + entry = new_entry; + + if (!(device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE) || + cache != device->mem_cache) + radv_pipeline_cache_add_entry(cache, new_entry); + } + } + + char *p = entry->code; + for(int i = 0; i < MESA_SHADER_STAGES; ++i) { + if (!entry->variants[i] && entry->binary_sizes[i]) { + struct radv_shader_binary *binary = calloc(1, entry->binary_sizes[i]); + memcpy(binary, p, entry->binary_sizes[i]); + p += entry->binary_sizes[i]; + + entry->variants[i] = radv_shader_variant_create(device, binary, false); + free(binary); + } else if (entry->binary_sizes[i]) { + p += entry->binary_sizes[i]; + } + + } + + memcpy(variants, entry->variants, sizeof(entry->variants)); + + if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE && + cache == device->mem_cache) + vk_free(&cache->alloc, entry); + else { + for (int i = 0; i < MESA_SHADER_STAGES; ++i) + if (entry->variants[i]) + p_atomic_inc(&entry->variants[i]->ref_count); + } + + radv_pipeline_cache_unlock(cache); + return true; +} + +void +radv_pipeline_cache_insert_shaders(struct radv_device *device, + struct radv_pipeline_cache *cache, + const unsigned char *sha1, + struct radv_shader_variant **variants, + struct radv_shader_binary *const *binaries) { if (!cache) - return variant; + cache = device->mem_cache; - pthread_mutex_lock(&cache->mutex); + radv_pipeline_cache_lock(cache); struct cache_entry *entry = radv_pipeline_cache_search_unlocked(cache, sha1); if (entry) { - if (entry->variant) { - radv_shader_variant_destroy(cache->device, variant); - variant = entry->variant; - } else { - entry->variant = variant; + for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + if (entry->variants[i]) { + radv_shader_variant_destroy(cache->device, variants[i]); + variants[i] = entry->variants[i]; + } else { + entry->variants[i] = variants[i]; + } + if (variants[i]) + p_atomic_inc(&variants[i]->ref_count); } - p_atomic_inc(&variant->ref_count); - pthread_mutex_unlock(&cache->mutex); - return variant; + radv_pipeline_cache_unlock(cache); + return; } - entry = vk_alloc(&cache->alloc, sizeof(*entry) + code_size, 8, + /* Don't cache when we want debug info, since this isn't + * present in the cache. + */ + if (radv_is_cache_disabled(device)) { + radv_pipeline_cache_unlock(cache); + return; + } + + size_t size = sizeof(*entry); + for (int i = 0; i < MESA_SHADER_STAGES; ++i) + if (variants[i]) + size += binaries[i]->total_size; + size = align(size, alignof(struct cache_entry)); + + + entry = vk_alloc(&cache->alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE); if (!entry) { - pthread_mutex_unlock(&cache->mutex); - return variant; + radv_pipeline_cache_unlock(cache); + return; } + memset(entry, 0, sizeof(*entry)); memcpy(entry->sha1, sha1, 20); - memcpy(entry->code, code, code_size); - entry->config = variant->config; - entry->variant_info = variant->info; - entry->rsrc1 = variant->rsrc1; - entry->rsrc2 = variant->rsrc2; - entry->code_size = code_size; - entry->variant = variant; - p_atomic_inc(&variant->ref_count); + + char* p = entry->code; + + for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + if (!variants[i]) + continue; + + entry->binary_sizes[i] = binaries[i]->total_size; + + memcpy(p, binaries[i], binaries[i]->total_size); + p += binaries[i]->total_size; + } + + /* Always add cache items to disk. This will allow collection of + * compiled shaders by third parties such as steam, even if the app + * implements its own pipeline cache. + */ + if (device->physical_device->disk_cache) { + uint8_t disk_sha1[20]; + disk_cache_compute_key(device->physical_device->disk_cache, sha1, 20, + disk_sha1); + + disk_cache_put(device->physical_device->disk_cache, disk_sha1, + entry, entry_size(entry), NULL); + } + + if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE && + cache == device->mem_cache) { + vk_free2(&cache->alloc, NULL, entry); + radv_pipeline_cache_unlock(cache); + return; + } + + /* We delay setting the variant so we have reproducible disk cache + * items. + */ + for (int i = 0; i < MESA_SHADER_STAGES; ++i) { + if (!variants[i]) + continue; + + entry->variants[i] = variants[i]; + p_atomic_inc(&variants[i]->ref_count); + } radv_pipeline_cache_add_entry(cache, entry); cache->modified = true; - pthread_mutex_unlock(&cache->mutex); - return variant; + radv_pipeline_cache_unlock(cache); + return; } -struct cache_header { - uint32_t header_size; - uint32_t header_version; - uint32_t vendor_id; - uint32_t device_id; - uint8_t uuid[VK_UUID_SIZE]; -}; - -void +bool radv_pipeline_cache_load(struct radv_pipeline_cache *cache, const void *data, size_t size) { struct radv_device *device = cache->device; - struct cache_header header; + struct vk_pipeline_cache_header header; if (size < sizeof(header)) - return; + return false; memcpy(&header, data, sizeof(header)); if (header.header_size < sizeof(header)) - return; + return false; if (header.header_version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE) - return; + return false; if (header.vendor_id != ATI_VENDOR_ID) - return; + return false; if (header.device_id != device->physical_device->rad_info.pci_id) - return; + return false; if (memcmp(header.uuid, device->physical_device->cache_uuid, VK_UUID_SIZE) != 0) - return; + return false; char *end = (void *) data + size; char *p = (void *) data + header.header_size; @@ -343,18 +483,22 @@ radv_pipeline_cache_load(struct radv_pipeline_cache *cache, while (end - p >= sizeof(struct cache_entry)) { struct cache_entry *entry = (struct cache_entry*)p; struct cache_entry *dest_entry; - if(end - p < sizeof(*entry) + entry->code_size) + size_t size = entry_size(entry); + if(end - p < size) break; - dest_entry = vk_alloc(&cache->alloc, sizeof(*entry) + entry->code_size, + dest_entry = vk_alloc(&cache->alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE); if (dest_entry) { - memcpy(dest_entry, entry, sizeof(*entry) + entry->code_size); - dest_entry->variant = NULL; + memcpy(dest_entry, entry, size); + for (int i = 0; i < MESA_SHADER_STAGES; ++i) + dest_entry->variants[i] = NULL; radv_pipeline_cache_add_entry(cache, dest_entry); } - p += sizeof (*entry) + entry->code_size; + p += size; } + + return true; } VkResult radv_CreatePipelineCache( @@ -369,18 +513,22 @@ VkResult radv_CreatePipelineCache( assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO); assert(pCreateInfo->flags == 0); - cache = vk_alloc2(&device->alloc, pAllocator, + cache = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*cache), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (cache == NULL) - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + + vk_object_base_init(&device->vk, &cache->base, + VK_OBJECT_TYPE_PIPELINE_CACHE); if (pAllocator) cache->alloc = *pAllocator; else - cache->alloc = device->alloc; + cache->alloc = device->vk.alloc; radv_pipeline_cache_init(cache, device); + cache->flags = pCreateInfo->flags; if (pCreateInfo->initialDataSize > 0) { radv_pipeline_cache_load(cache, @@ -405,7 +553,8 @@ void radv_DestroyPipelineCache( return; radv_pipeline_cache_finish(cache); - vk_free2(&device->alloc, pAllocator, cache); + vk_object_base_finish(&cache->base); + vk_free2(&device->vk.alloc, pAllocator, cache); } VkResult radv_GetPipelineCacheData( @@ -416,20 +565,25 @@ VkResult radv_GetPipelineCacheData( { RADV_FROM_HANDLE(radv_device, device, _device); RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache); - struct cache_header *header; + struct vk_pipeline_cache_header *header; VkResult result = VK_SUCCESS; + + radv_pipeline_cache_lock(cache); + const size_t size = sizeof(*header) + cache->total_size; if (pData == NULL) { + radv_pipeline_cache_unlock(cache); *pDataSize = size; return VK_SUCCESS; } if (*pDataSize < sizeof(*header)) { + radv_pipeline_cache_unlock(cache); *pDataSize = 0; return VK_INCOMPLETE; } void *p = pData, *end = pData + *pDataSize; header = p; - header->header_size = sizeof(*header); + header->header_size = align(sizeof(*header), alignof(struct cache_entry)); header->header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE; header->vendor_id = ATI_VENDOR_ID; header->device_id = device->physical_device->rad_info.pci_id; @@ -448,11 +602,13 @@ VkResult radv_GetPipelineCacheData( } memcpy(p, entry, size); - ((struct cache_entry*)p)->variant = NULL; + for(int j = 0; j < MESA_SHADER_STAGES; ++j) + ((struct cache_entry*)p)->variants[j] = NULL; p += size; } *pDataSize = p - pData; + radv_pipeline_cache_unlock(cache); return result; }