radv: don't pass null to _mesa_sha1_update
[mesa.git] / src / amd / vulkan / radv_pipeline_cache.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "util/mesa-sha1.h"
25 #include "util/debug.h"
26 #include "util/disk_cache.h"
27 #include "util/u_atomic.h"
28 #include "radv_debug.h"
29 #include "radv_private.h"
30 #include "radv_shader.h"
31 #include "vulkan/util/vk_util.h"
32
33 #include "ac_nir_to_llvm.h"
34
35 struct cache_entry {
36 union {
37 unsigned char sha1[20];
38 uint32_t sha1_dw[5];
39 };
40 uint32_t binary_sizes[MESA_SHADER_STAGES];
41 struct radv_shader_variant *variants[MESA_SHADER_STAGES];
42 char code[0];
43 };
44
45 static void
46 radv_pipeline_cache_lock(struct radv_pipeline_cache *cache)
47 {
48 if (cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT)
49 return;
50
51 pthread_mutex_lock(&cache->mutex);
52 }
53
54 static void
55 radv_pipeline_cache_unlock(struct radv_pipeline_cache *cache)
56 {
57 if (cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT)
58 return;
59
60 pthread_mutex_unlock(&cache->mutex);
61 }
62
63 void
64 radv_pipeline_cache_init(struct radv_pipeline_cache *cache,
65 struct radv_device *device)
66 {
67 cache->device = device;
68 pthread_mutex_init(&cache->mutex, NULL);
69 cache->flags = 0;
70
71 cache->modified = false;
72 cache->kernel_count = 0;
73 cache->total_size = 0;
74 cache->table_size = 1024;
75 const size_t byte_size = cache->table_size * sizeof(cache->hash_table[0]);
76 cache->hash_table = malloc(byte_size);
77
78 /* We don't consider allocation failure fatal, we just start with a 0-sized
79 * cache. Disable caching when we want to keep shader debug info, since
80 * we don't get the debug info on cached shaders. */
81 if (cache->hash_table == NULL ||
82 (device->instance->debug_flags & RADV_DEBUG_NO_CACHE))
83 cache->table_size = 0;
84 else
85 memset(cache->hash_table, 0, byte_size);
86 }
87
88 void
89 radv_pipeline_cache_finish(struct radv_pipeline_cache *cache)
90 {
91 for (unsigned i = 0; i < cache->table_size; ++i)
92 if (cache->hash_table[i]) {
93 for(int j = 0; j < MESA_SHADER_STAGES; ++j) {
94 if (cache->hash_table[i]->variants[j])
95 radv_shader_variant_destroy(cache->device,
96 cache->hash_table[i]->variants[j]);
97 }
98 vk_free(&cache->alloc, cache->hash_table[i]);
99 }
100 pthread_mutex_destroy(&cache->mutex);
101 free(cache->hash_table);
102 }
103
104 static uint32_t
105 entry_size(struct cache_entry *entry)
106 {
107 size_t ret = sizeof(*entry);
108 for (int i = 0; i < MESA_SHADER_STAGES; ++i)
109 if (entry->binary_sizes[i])
110 ret += entry->binary_sizes[i];
111 return ret;
112 }
113
114 void
115 radv_hash_shaders(unsigned char *hash,
116 const VkPipelineShaderStageCreateInfo **stages,
117 const struct radv_pipeline_layout *layout,
118 const struct radv_pipeline_key *key,
119 uint32_t flags)
120 {
121 struct mesa_sha1 ctx;
122
123 _mesa_sha1_init(&ctx);
124 if (key)
125 _mesa_sha1_update(&ctx, key, sizeof(*key));
126 if (layout)
127 _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
128
129 for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
130 if (stages[i]) {
131 RADV_FROM_HANDLE(radv_shader_module, module, stages[i]->module);
132 const VkSpecializationInfo *spec_info = stages[i]->pSpecializationInfo;
133
134 _mesa_sha1_update(&ctx, module->sha1, sizeof(module->sha1));
135 _mesa_sha1_update(&ctx, stages[i]->pName, strlen(stages[i]->pName));
136 if (spec_info && spec_info->mapEntryCount) {
137 _mesa_sha1_update(&ctx, spec_info->pMapEntries,
138 spec_info->mapEntryCount * sizeof spec_info->pMapEntries[0]);
139 _mesa_sha1_update(&ctx, spec_info->pData, spec_info->dataSize);
140 }
141 }
142 }
143 _mesa_sha1_update(&ctx, &flags, 4);
144 _mesa_sha1_final(&ctx, hash);
145 }
146
147
148 static struct cache_entry *
149 radv_pipeline_cache_search_unlocked(struct radv_pipeline_cache *cache,
150 const unsigned char *sha1)
151 {
152 const uint32_t mask = cache->table_size - 1;
153 const uint32_t start = (*(uint32_t *) sha1);
154
155 if (cache->table_size == 0)
156 return NULL;
157
158 for (uint32_t i = 0; i < cache->table_size; i++) {
159 const uint32_t index = (start + i) & mask;
160 struct cache_entry *entry = cache->hash_table[index];
161
162 if (!entry)
163 return NULL;
164
165 if (memcmp(entry->sha1, sha1, sizeof(entry->sha1)) == 0) {
166 return entry;
167 }
168 }
169
170 unreachable("hash table should never be full");
171 }
172
173 static struct cache_entry *
174 radv_pipeline_cache_search(struct radv_pipeline_cache *cache,
175 const unsigned char *sha1)
176 {
177 struct cache_entry *entry;
178
179 radv_pipeline_cache_lock(cache);
180
181 entry = radv_pipeline_cache_search_unlocked(cache, sha1);
182
183 radv_pipeline_cache_unlock(cache);
184
185 return entry;
186 }
187
188 static void
189 radv_pipeline_cache_set_entry(struct radv_pipeline_cache *cache,
190 struct cache_entry *entry)
191 {
192 const uint32_t mask = cache->table_size - 1;
193 const uint32_t start = entry->sha1_dw[0];
194
195 /* We'll always be able to insert when we get here. */
196 assert(cache->kernel_count < cache->table_size / 2);
197
198 for (uint32_t i = 0; i < cache->table_size; i++) {
199 const uint32_t index = (start + i) & mask;
200 if (!cache->hash_table[index]) {
201 cache->hash_table[index] = entry;
202 break;
203 }
204 }
205
206 cache->total_size += entry_size(entry);
207 cache->kernel_count++;
208 }
209
210
211 static VkResult
212 radv_pipeline_cache_grow(struct radv_pipeline_cache *cache)
213 {
214 const uint32_t table_size = cache->table_size * 2;
215 const uint32_t old_table_size = cache->table_size;
216 const size_t byte_size = table_size * sizeof(cache->hash_table[0]);
217 struct cache_entry **table;
218 struct cache_entry **old_table = cache->hash_table;
219
220 table = malloc(byte_size);
221 if (table == NULL)
222 return vk_error(cache->device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
223
224 cache->hash_table = table;
225 cache->table_size = table_size;
226 cache->kernel_count = 0;
227 cache->total_size = 0;
228
229 memset(cache->hash_table, 0, byte_size);
230 for (uint32_t i = 0; i < old_table_size; i++) {
231 struct cache_entry *entry = old_table[i];
232 if (!entry)
233 continue;
234
235 radv_pipeline_cache_set_entry(cache, entry);
236 }
237
238 free(old_table);
239
240 return VK_SUCCESS;
241 }
242
243 static void
244 radv_pipeline_cache_add_entry(struct radv_pipeline_cache *cache,
245 struct cache_entry *entry)
246 {
247 if (cache->kernel_count == cache->table_size / 2)
248 radv_pipeline_cache_grow(cache);
249
250 /* Failing to grow that hash table isn't fatal, but may mean we don't
251 * have enough space to add this new kernel. Only add it if there's room.
252 */
253 if (cache->kernel_count < cache->table_size / 2)
254 radv_pipeline_cache_set_entry(cache, entry);
255 }
256
257 static bool
258 radv_is_cache_disabled(struct radv_device *device)
259 {
260 /* Pipeline caches can be disabled with RADV_DEBUG=nocache, with
261 * MESA_GLSL_CACHE_DISABLE=1, and when VK_AMD_shader_info is requested.
262 */
263 return (device->instance->debug_flags & RADV_DEBUG_NO_CACHE);
264 }
265
266 bool
267 radv_create_shader_variants_from_pipeline_cache(struct radv_device *device,
268 struct radv_pipeline_cache *cache,
269 const unsigned char *sha1,
270 struct radv_shader_variant **variants,
271 bool *found_in_application_cache)
272 {
273 struct cache_entry *entry;
274
275 if (!cache) {
276 cache = device->mem_cache;
277 *found_in_application_cache = false;
278 }
279
280 radv_pipeline_cache_lock(cache);
281
282 entry = radv_pipeline_cache_search_unlocked(cache, sha1);
283
284 if (!entry) {
285 *found_in_application_cache = false;
286
287 /* Don't cache when we want debug info, since this isn't
288 * present in the cache.
289 */
290 if (radv_is_cache_disabled(device) || !device->physical_device->disk_cache) {
291 radv_pipeline_cache_unlock(cache);
292 return false;
293 }
294
295 uint8_t disk_sha1[20];
296 disk_cache_compute_key(device->physical_device->disk_cache,
297 sha1, 20, disk_sha1);
298
299 entry = (struct cache_entry *)
300 disk_cache_get(device->physical_device->disk_cache,
301 disk_sha1, NULL);
302 if (!entry) {
303 radv_pipeline_cache_unlock(cache);
304 return false;
305 } else {
306 size_t size = entry_size(entry);
307 struct cache_entry *new_entry = vk_alloc(&cache->alloc, size, 8,
308 VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
309 if (!new_entry) {
310 free(entry);
311 radv_pipeline_cache_unlock(cache);
312 return false;
313 }
314
315 memcpy(new_entry, entry, entry_size(entry));
316 free(entry);
317 entry = new_entry;
318
319 if (!(device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE) ||
320 cache != device->mem_cache)
321 radv_pipeline_cache_add_entry(cache, new_entry);
322 }
323 }
324
325 char *p = entry->code;
326 for(int i = 0; i < MESA_SHADER_STAGES; ++i) {
327 if (!entry->variants[i] && entry->binary_sizes[i]) {
328 struct radv_shader_binary *binary = calloc(1, entry->binary_sizes[i]);
329 memcpy(binary, p, entry->binary_sizes[i]);
330 p += entry->binary_sizes[i];
331
332 entry->variants[i] = radv_shader_variant_create(device, binary, false);
333 free(binary);
334 } else if (entry->binary_sizes[i]) {
335 p += entry->binary_sizes[i];
336 }
337
338 }
339
340 memcpy(variants, entry->variants, sizeof(entry->variants));
341
342 if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE &&
343 cache == device->mem_cache)
344 vk_free(&cache->alloc, entry);
345 else {
346 for (int i = 0; i < MESA_SHADER_STAGES; ++i)
347 if (entry->variants[i])
348 p_atomic_inc(&entry->variants[i]->ref_count);
349 }
350
351 radv_pipeline_cache_unlock(cache);
352 return true;
353 }
354
355 void
356 radv_pipeline_cache_insert_shaders(struct radv_device *device,
357 struct radv_pipeline_cache *cache,
358 const unsigned char *sha1,
359 struct radv_shader_variant **variants,
360 struct radv_shader_binary *const *binaries)
361 {
362 if (!cache)
363 cache = device->mem_cache;
364
365 radv_pipeline_cache_lock(cache);
366 struct cache_entry *entry = radv_pipeline_cache_search_unlocked(cache, sha1);
367 if (entry) {
368 for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
369 if (entry->variants[i]) {
370 radv_shader_variant_destroy(cache->device, variants[i]);
371 variants[i] = entry->variants[i];
372 } else {
373 entry->variants[i] = variants[i];
374 }
375 if (variants[i])
376 p_atomic_inc(&variants[i]->ref_count);
377 }
378 radv_pipeline_cache_unlock(cache);
379 return;
380 }
381
382 /* Don't cache when we want debug info, since this isn't
383 * present in the cache.
384 */
385 if (radv_is_cache_disabled(device)) {
386 radv_pipeline_cache_unlock(cache);
387 return;
388 }
389
390 size_t size = sizeof(*entry);
391 for (int i = 0; i < MESA_SHADER_STAGES; ++i)
392 if (variants[i])
393 size += binaries[i]->total_size;
394
395
396 entry = vk_alloc(&cache->alloc, size, 8,
397 VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
398 if (!entry) {
399 radv_pipeline_cache_unlock(cache);
400 return;
401 }
402
403 memset(entry, 0, sizeof(*entry));
404 memcpy(entry->sha1, sha1, 20);
405
406 char* p = entry->code;
407
408 for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
409 if (!variants[i])
410 continue;
411
412 entry->binary_sizes[i] = binaries[i]->total_size;
413
414 memcpy(p, binaries[i], binaries[i]->total_size);
415 p += binaries[i]->total_size;
416 }
417
418 /* Always add cache items to disk. This will allow collection of
419 * compiled shaders by third parties such as steam, even if the app
420 * implements its own pipeline cache.
421 */
422 if (device->physical_device->disk_cache) {
423 uint8_t disk_sha1[20];
424 disk_cache_compute_key(device->physical_device->disk_cache, sha1, 20,
425 disk_sha1);
426
427 disk_cache_put(device->physical_device->disk_cache, disk_sha1,
428 entry, entry_size(entry), NULL);
429 }
430
431 if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE &&
432 cache == device->mem_cache) {
433 vk_free2(&cache->alloc, NULL, entry);
434 radv_pipeline_cache_unlock(cache);
435 return;
436 }
437
438 /* We delay setting the variant so we have reproducible disk cache
439 * items.
440 */
441 for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
442 if (!variants[i])
443 continue;
444
445 entry->variants[i] = variants[i];
446 p_atomic_inc(&variants[i]->ref_count);
447 }
448
449 radv_pipeline_cache_add_entry(cache, entry);
450
451 cache->modified = true;
452 radv_pipeline_cache_unlock(cache);
453 return;
454 }
455
456 bool
457 radv_pipeline_cache_load(struct radv_pipeline_cache *cache,
458 const void *data, size_t size)
459 {
460 struct radv_device *device = cache->device;
461 struct vk_pipeline_cache_header header;
462
463 if (size < sizeof(header))
464 return false;
465 memcpy(&header, data, sizeof(header));
466 if (header.header_size < sizeof(header))
467 return false;
468 if (header.header_version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE)
469 return false;
470 if (header.vendor_id != ATI_VENDOR_ID)
471 return false;
472 if (header.device_id != device->physical_device->rad_info.pci_id)
473 return false;
474 if (memcmp(header.uuid, device->physical_device->cache_uuid, VK_UUID_SIZE) != 0)
475 return false;
476
477 char *end = (void *) data + size;
478 char *p = (void *) data + header.header_size;
479
480 while (end - p >= sizeof(struct cache_entry)) {
481 struct cache_entry *entry = (struct cache_entry*)p;
482 struct cache_entry *dest_entry;
483 size_t size = entry_size(entry);
484 if(end - p < size)
485 break;
486
487 dest_entry = vk_alloc(&cache->alloc, size,
488 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
489 if (dest_entry) {
490 memcpy(dest_entry, entry, size);
491 for (int i = 0; i < MESA_SHADER_STAGES; ++i)
492 dest_entry->variants[i] = NULL;
493 radv_pipeline_cache_add_entry(cache, dest_entry);
494 }
495 p += size;
496 }
497
498 return true;
499 }
500
501 VkResult radv_CreatePipelineCache(
502 VkDevice _device,
503 const VkPipelineCacheCreateInfo* pCreateInfo,
504 const VkAllocationCallbacks* pAllocator,
505 VkPipelineCache* pPipelineCache)
506 {
507 RADV_FROM_HANDLE(radv_device, device, _device);
508 struct radv_pipeline_cache *cache;
509
510 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
511 assert(pCreateInfo->flags == 0);
512
513 cache = vk_alloc2(&device->vk.alloc, pAllocator,
514 sizeof(*cache), 8,
515 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
516 if (cache == NULL)
517 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
518
519 vk_object_base_init(&device->vk, &cache->base,
520 VK_OBJECT_TYPE_PIPELINE_CACHE);
521
522 if (pAllocator)
523 cache->alloc = *pAllocator;
524 else
525 cache->alloc = device->vk.alloc;
526
527 radv_pipeline_cache_init(cache, device);
528 cache->flags = pCreateInfo->flags;
529
530 if (pCreateInfo->initialDataSize > 0) {
531 radv_pipeline_cache_load(cache,
532 pCreateInfo->pInitialData,
533 pCreateInfo->initialDataSize);
534 }
535
536 *pPipelineCache = radv_pipeline_cache_to_handle(cache);
537
538 return VK_SUCCESS;
539 }
540
541 void radv_DestroyPipelineCache(
542 VkDevice _device,
543 VkPipelineCache _cache,
544 const VkAllocationCallbacks* pAllocator)
545 {
546 RADV_FROM_HANDLE(radv_device, device, _device);
547 RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
548
549 if (!cache)
550 return;
551 radv_pipeline_cache_finish(cache);
552
553 vk_object_base_finish(&cache->base);
554 vk_free2(&device->vk.alloc, pAllocator, cache);
555 }
556
557 VkResult radv_GetPipelineCacheData(
558 VkDevice _device,
559 VkPipelineCache _cache,
560 size_t* pDataSize,
561 void* pData)
562 {
563 RADV_FROM_HANDLE(radv_device, device, _device);
564 RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
565 struct vk_pipeline_cache_header *header;
566 VkResult result = VK_SUCCESS;
567
568 radv_pipeline_cache_lock(cache);
569
570 const size_t size = sizeof(*header) + cache->total_size;
571 if (pData == NULL) {
572 radv_pipeline_cache_unlock(cache);
573 *pDataSize = size;
574 return VK_SUCCESS;
575 }
576 if (*pDataSize < sizeof(*header)) {
577 radv_pipeline_cache_unlock(cache);
578 *pDataSize = 0;
579 return VK_INCOMPLETE;
580 }
581 void *p = pData, *end = pData + *pDataSize;
582 header = p;
583 header->header_size = sizeof(*header);
584 header->header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE;
585 header->vendor_id = ATI_VENDOR_ID;
586 header->device_id = device->physical_device->rad_info.pci_id;
587 memcpy(header->uuid, device->physical_device->cache_uuid, VK_UUID_SIZE);
588 p += header->header_size;
589
590 struct cache_entry *entry;
591 for (uint32_t i = 0; i < cache->table_size; i++) {
592 if (!cache->hash_table[i])
593 continue;
594 entry = cache->hash_table[i];
595 const uint32_t size = entry_size(entry);
596 if (end < p + size) {
597 result = VK_INCOMPLETE;
598 break;
599 }
600
601 memcpy(p, entry, size);
602 for(int j = 0; j < MESA_SHADER_STAGES; ++j)
603 ((struct cache_entry*)p)->variants[j] = NULL;
604 p += size;
605 }
606 *pDataSize = p - pData;
607
608 radv_pipeline_cache_unlock(cache);
609 return result;
610 }
611
612 static void
613 radv_pipeline_cache_merge(struct radv_pipeline_cache *dst,
614 struct radv_pipeline_cache *src)
615 {
616 for (uint32_t i = 0; i < src->table_size; i++) {
617 struct cache_entry *entry = src->hash_table[i];
618 if (!entry || radv_pipeline_cache_search(dst, entry->sha1))
619 continue;
620
621 radv_pipeline_cache_add_entry(dst, entry);
622
623 src->hash_table[i] = NULL;
624 }
625 }
626
627 VkResult radv_MergePipelineCaches(
628 VkDevice _device,
629 VkPipelineCache destCache,
630 uint32_t srcCacheCount,
631 const VkPipelineCache* pSrcCaches)
632 {
633 RADV_FROM_HANDLE(radv_pipeline_cache, dst, destCache);
634
635 for (uint32_t i = 0; i < srcCacheCount; i++) {
636 RADV_FROM_HANDLE(radv_pipeline_cache, src, pSrcCaches[i]);
637
638 radv_pipeline_cache_merge(dst, src);
639 }
640
641 return VK_SUCCESS;
642 }