src/amd/vulkan/radv_pipeline_cache.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "util/mesa-sha1.h"
  25 #include "util/debug.h"
  26 #include "util/disk_cache.h"
  27 #include "util/u_atomic.h"
  28 #include "radv_debug.h"
  29 #include "radv_private.h"
  30 #include "radv_shader.h"
  31
  32 #include "ac_nir_to_llvm.h"
  33
  34 struct cache_entry_variant_info {
  35         struct ac_shader_variant_info variant_info;
  36         struct ac_shader_config config;
  37         uint32_t rsrc1, rsrc2;
  38 };
  39
  40 struct cache_entry {
  41         union {
  42                 unsigned char sha1[20];
  43                 uint32_t sha1_dw[5];
  44         };
  45         uint32_t code_sizes[MESA_SHADER_STAGES];
  46         struct radv_shader_variant *variants[MESA_SHADER_STAGES];
  47         char code[0];
  48 };
  49
  50 void
  51 radv_pipeline_cache_init(struct radv_pipeline_cache *cache,
  52                          struct radv_device *device)
  53 {
  54         cache->device = device;
  55         pthread_mutex_init(&cache->mutex, NULL);
  56
  57         cache->modified = false;
  58         cache->kernel_count = 0;
  59         cache->total_size = 0;
  60         cache->table_size = 1024;
  61         const size_t byte_size = cache->table_size * sizeof(cache->hash_table[0]);
  62         cache->hash_table = malloc(byte_size);
  63
  64         /* We don't consider allocation failure fatal, we just start with a 0-sized
  65          * cache. */
  66         if (cache->hash_table == NULL ||
  67             (device->instance->debug_flags & RADV_DEBUG_NO_CACHE))
  68                 cache->table_size = 0;
  69         else
  70                 memset(cache->hash_table, 0, byte_size);
  71 }
  72
  73 void
  74 radv_pipeline_cache_finish(struct radv_pipeline_cache *cache)
  75 {
  76         for (unsigned i = 0; i < cache->table_size; ++i)
  77                 if (cache->hash_table[i]) {
  78                         for(int j = 0; j < MESA_SHADER_STAGES; ++j)  {
  79                                 if (cache->hash_table[i]->variants[j])
  80                                         radv_shader_variant_destroy(cache->device,
  81                                                                     cache->hash_table[i]->variants[j]);
  82                         }
  83                         vk_free(&cache->alloc, cache->hash_table[i]);
  84                 }
  85         pthread_mutex_destroy(&cache->mutex);
  86         free(cache->hash_table);
  87 }
  88
  89 static uint32_t
  90 entry_size(struct cache_entry *entry)
  91 {
  92         size_t ret = sizeof(*entry);
  93         for (int i = 0; i < MESA_SHADER_STAGES; ++i)
  94                 if (entry->code_sizes[i])
  95                         ret += sizeof(struct cache_entry_variant_info) + entry->code_sizes[i];
  96         return ret;
  97 }
  98
  99 void
 100 radv_hash_shader(unsigned char *hash, struct radv_shader_module *module,
 101                  const char *entrypoint,
 102                  const VkSpecializationInfo *spec_info,
 103                  const struct radv_pipeline_layout *layout,
 104                  const struct ac_shader_variant_key *key,
 105                  uint32_t flags)
 106 {
 107         struct mesa_sha1 ctx;
 108
 109         _mesa_sha1_init(&ctx);
 110         if (key)
 111                 _mesa_sha1_update(&ctx, key, sizeof(*key));
 112         _mesa_sha1_update(&ctx, module->sha1, sizeof(module->sha1));
 113         _mesa_sha1_update(&ctx, entrypoint, strlen(entrypoint));
 114         if (layout)
 115                 _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
 116         if (spec_info) {
 117                 _mesa_sha1_update(&ctx, spec_info->pMapEntries,
 118                                   spec_info->mapEntryCount * sizeof spec_info->pMapEntries[0]);
 119                 _mesa_sha1_update(&ctx, spec_info->pData, spec_info->dataSize);
 120         }
 121         _mesa_sha1_update(&ctx, &flags, 4);
 122         _mesa_sha1_final(&ctx, hash);
 123 }
 124
 125 void
 126 radv_hash_shaders(unsigned char *hash,
 127                   const VkPipelineShaderStageCreateInfo **stages,
 128                   const struct radv_pipeline_layout *layout,
 129                   const struct ac_shader_variant_key *keys,
 130                   uint32_t flags)
 131 {
 132         struct mesa_sha1 ctx;
 133
 134         _mesa_sha1_init(&ctx);
 135         if (keys)
 136                 _mesa_sha1_update(&ctx, keys, sizeof(*keys) * MESA_SHADER_STAGES);
 137         if (layout)
 138                 _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
 139
 140         for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
 141                 if (stages[i]) {
 142                         RADV_FROM_HANDLE(radv_shader_module, module, stages[i]->module);
 143                         const VkSpecializationInfo *spec_info = stages[i]->pSpecializationInfo;
 144
 145                         _mesa_sha1_update(&ctx, module->sha1, sizeof(module->sha1));
 146                         _mesa_sha1_update(&ctx, stages[i]->pName, strlen(stages[i]->pName));
 147                         if (spec_info) {
 148                                 _mesa_sha1_update(&ctx, spec_info->pMapEntries,
 149                                                   spec_info->mapEntryCount * sizeof spec_info->pMapEntries[0]);
 150                                 _mesa_sha1_update(&ctx, spec_info->pData, spec_info->dataSize);
 151                         }
 152                 }
 153         }
 154         _mesa_sha1_update(&ctx, &flags, 4);
 155         _mesa_sha1_final(&ctx, hash);
 156 }
 157
 158
 159 static struct cache_entry *
 160 radv_pipeline_cache_search_unlocked(struct radv_pipeline_cache *cache,
 161                                     const unsigned char *sha1)
 162 {
 163         const uint32_t mask = cache->table_size - 1;
 164         const uint32_t start = (*(uint32_t *) sha1);
 165
 166         if (cache->table_size == 0)
 167                 return NULL;
 168
 169         for (uint32_t i = 0; i < cache->table_size; i++) {
 170                 const uint32_t index = (start + i) & mask;
 171                 struct cache_entry *entry = cache->hash_table[index];
 172
 173                 if (!entry)
 174                         return NULL;
 175
 176                 if (memcmp(entry->sha1, sha1, sizeof(entry->sha1)) == 0) {
 177                         return entry;
 178                 }
 179         }
 180
 181         unreachable("hash table should never be full");
 182 }
 183
 184 static struct cache_entry *
 185 radv_pipeline_cache_search(struct radv_pipeline_cache *cache,
 186                            const unsigned char *sha1)
 187 {
 188         struct cache_entry *entry;
 189
 190         pthread_mutex_lock(&cache->mutex);
 191
 192         entry = radv_pipeline_cache_search_unlocked(cache, sha1);
 193
 194         pthread_mutex_unlock(&cache->mutex);
 195
 196         return entry;
 197 }
 198
 199 struct radv_shader_variant *
 200 radv_create_shader_variant_from_pipeline_cache(struct radv_device *device,
 201                                                struct radv_pipeline_cache *cache,
 202                                                const unsigned char *sha1)
 203 {
 204         struct cache_entry *entry = NULL;
 205
 206         if (cache)
 207                 entry = radv_pipeline_cache_search(cache, sha1);
 208         else
 209                 entry = radv_pipeline_cache_search(device->mem_cache, sha1);
 210
 211         if (!entry) {
 212                 if (!device->physical_device->disk_cache)
 213                         return NULL;
 214                 uint8_t disk_sha1[20];
 215                 disk_cache_compute_key(device->physical_device->disk_cache,
 216                                        sha1, 20, disk_sha1);
 217                 entry = (struct cache_entry *)
 218                         disk_cache_get(device->physical_device->disk_cache,
 219                                        disk_sha1, NULL);
 220                 if (!entry)
 221                         return NULL;
 222         }
 223
 224         if (!entry->variants[0]) {
 225                 struct radv_shader_variant *variant;
 226                 char *p = entry->code;
 227                 struct cache_entry_variant_info info;
 228
 229                 variant = calloc(1, sizeof(struct radv_shader_variant));
 230                 if (!variant)
 231                         return NULL;
 232
 233                 memcpy(&info, p, sizeof(struct cache_entry_variant_info));
 234                 p += sizeof(struct cache_entry_variant_info);
 235
 236                 variant->code_size = entry->code_sizes[0];
 237                 variant->config = info.config;
 238                 variant->info = info.variant_info;
 239                 variant->rsrc1 = info.rsrc1;
 240                 variant->rsrc2 = info.rsrc2;
 241                 variant->ref_count = 1;
 242
 243                 void *ptr = radv_alloc_shader_memory(device, variant);
 244                 memcpy(ptr, p, entry->code_sizes[0]);
 245
 246                 entry->variants[0] = variant;
 247         }
 248
 249         p_atomic_inc(&entry->variants[0]->ref_count);
 250         return entry->variants[0];
 251 }
 252
 253 bool
 254 radv_create_shader_variants_from_pipeline_cache(struct radv_device *device,
 255                                                 struct radv_pipeline_cache *cache,
 256                                                 const unsigned char *sha1,
 257                                                 struct radv_shader_variant **variants)
 258 {
 259         struct cache_entry *entry;
 260         if (cache)
 261                 entry = radv_pipeline_cache_search(cache, sha1);
 262         else
 263                 entry = radv_pipeline_cache_search(device->mem_cache, sha1);
 264
 265         if (!entry) {
 266                 if (!device->physical_device->disk_cache)
 267                         return false;
 268
 269                 uint8_t disk_sha1[20];
 270                 disk_cache_compute_key(device->physical_device->disk_cache,
 271                                        sha1, 20, disk_sha1);
 272                 entry = (struct cache_entry *)
 273                         disk_cache_get(device->physical_device->disk_cache,
 274                                        disk_sha1, NULL);
 275                 if (!entry)
 276                         return false;
 277         }
 278
 279         char *p = entry->code;
 280         for(int i = 0; i < MESA_SHADER_STAGES; ++i) {
 281                 if (!entry->variants[i] && entry->code_sizes[i]) {
 282                         struct radv_shader_variant *variant;
 283                         struct cache_entry_variant_info info;
 284
 285                         variant = calloc(1, sizeof(struct radv_shader_variant));
 286                         if (!variant)
 287                                 return false;
 288
 289                         memcpy(&info, p, sizeof(struct cache_entry_variant_info));
 290                         p += sizeof(struct cache_entry_variant_info);
 291
 292                         variant->config = info.config;
 293                         variant->info = info.variant_info;
 294                         variant->rsrc1 = info.rsrc1;
 295                         variant->rsrc2 = info.rsrc2;
 296                         variant->code_size = entry->code_sizes[i];
 297                         variant->ref_count = 1;
 298
 299                         void *ptr = radv_alloc_shader_memory(device, variant);
 300                         memcpy(ptr, p, entry->code_sizes[i]);
 301                         p += entry->code_sizes[i];
 302
 303                         entry->variants[i] = variant;
 304                 }
 305
 306         }
 307
 308         for (int i = 0; i < MESA_SHADER_STAGES; ++i)
 309                 if (entry->variants[i])
 310                         p_atomic_inc(&entry->variants[i]->ref_count);
 311
 312         memcpy(variants, entry->variants, sizeof(entry->variants));
 313         return true;
 314 }
 315
 316
 317 static void
 318 radv_pipeline_cache_set_entry(struct radv_pipeline_cache *cache,
 319                               struct cache_entry *entry)
 320 {
 321         const uint32_t mask = cache->table_size - 1;
 322         const uint32_t start = entry->sha1_dw[0];
 323
 324         /* We'll always be able to insert when we get here. */
 325         assert(cache->kernel_count < cache->table_size / 2);
 326
 327         for (uint32_t i = 0; i < cache->table_size; i++) {
 328                 const uint32_t index = (start + i) & mask;
 329                 if (!cache->hash_table[index]) {
 330                         cache->hash_table[index] = entry;
 331                         break;
 332                 }
 333         }
 334
 335         cache->total_size += entry_size(entry);
 336         cache->kernel_count++;
 337 }
 338
 339
 340 static VkResult
 341 radv_pipeline_cache_grow(struct radv_pipeline_cache *cache)
 342 {
 343         const uint32_t table_size = cache->table_size * 2;
 344         const uint32_t old_table_size = cache->table_size;
 345         const size_t byte_size = table_size * sizeof(cache->hash_table[0]);
 346         struct cache_entry **table;
 347         struct cache_entry **old_table = cache->hash_table;
 348
 349         table = malloc(byte_size);
 350         if (table == NULL)
 351                 return VK_ERROR_OUT_OF_HOST_MEMORY;
 352
 353         cache->hash_table = table;
 354         cache->table_size = table_size;
 355         cache->kernel_count = 0;
 356         cache->total_size = 0;
 357
 358         memset(cache->hash_table, 0, byte_size);
 359         for (uint32_t i = 0; i < old_table_size; i++) {
 360                 struct cache_entry *entry = old_table[i];
 361                 if (!entry)
 362                         continue;
 363
 364                 radv_pipeline_cache_set_entry(cache, entry);
 365         }
 366
 367         free(old_table);
 368
 369         return VK_SUCCESS;
 370 }
 371
 372 static void
 373 radv_pipeline_cache_add_entry(struct radv_pipeline_cache *cache,
 374                               struct cache_entry *entry)
 375 {
 376         if (cache->kernel_count == cache->table_size / 2)
 377                 radv_pipeline_cache_grow(cache);
 378
 379         /* Failing to grow that hash table isn't fatal, but may mean we don't
 380          * have enough space to add this new kernel. Only add it if there's room.
 381          */
 382         if (cache->kernel_count < cache->table_size / 2)
 383                 radv_pipeline_cache_set_entry(cache, entry);
 384 }
 385
 386 struct radv_shader_variant *
 387 radv_pipeline_cache_insert_shader(struct radv_device *device,
 388                                   struct radv_pipeline_cache *cache,
 389                                   const unsigned char *sha1,
 390                                   struct radv_shader_variant *variant,
 391                                   const void *code, unsigned code_size)
 392 {
 393         if (!cache)
 394                 cache = device->mem_cache;
 395
 396         pthread_mutex_lock(&cache->mutex);
 397         struct cache_entry *entry = radv_pipeline_cache_search_unlocked(cache, sha1);
 398         if (entry) {
 399                 if (entry->variants[0]) {
 400                         radv_shader_variant_destroy(cache->device, variant);
 401                         variant = entry->variants[0];
 402                 } else {
 403                         entry->variants[0] = variant;
 404                 }
 405                 p_atomic_inc(&variant->ref_count);
 406                 pthread_mutex_unlock(&cache->mutex);
 407                 return variant;
 408         }
 409
 410         entry = vk_alloc(&cache->alloc, sizeof(*entry) + sizeof(struct cache_entry_variant_info) + code_size, 8,
 411                            VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
 412         if (!entry) {
 413                 pthread_mutex_unlock(&cache->mutex);
 414                 return variant;
 415         }
 416
 417         memset(entry, 0, sizeof(*entry));
 418
 419         char* p = entry->code;
 420         struct cache_entry_variant_info info;
 421
 422         info.config = variant->config;
 423         info.variant_info = variant->info;
 424         info.rsrc1 = variant->rsrc1;
 425         info.rsrc2 = variant->rsrc2;
 426         memcpy(p, &info, sizeof(struct cache_entry_variant_info));
 427         p += sizeof(struct cache_entry_variant_info);
 428
 429         memcpy(entry->sha1, sha1, 20);
 430         memcpy(p, code, code_size);
 431
 432         entry->code_sizes[0] = code_size;
 433
 434         /* Set variant to NULL so we have reproducible cache items */
 435         entry->variants[0] = NULL;
 436
 437         /* Always add cache items to disk. This will allow collection of
 438          * compiled shaders by third parties such as steam, even if the app
 439          * implements its own pipeline cache.
 440          */
 441         if (device->physical_device->disk_cache) {
 442                 uint8_t disk_sha1[20];
 443                 disk_cache_compute_key(device->physical_device->disk_cache, sha1, 20,
 444                                        disk_sha1);
 445                 disk_cache_put(device->physical_device->disk_cache,
 446                                disk_sha1, entry, entry_size(entry), NULL);
 447         }
 448
 449         entry->variants[0] = variant;
 450         p_atomic_inc(&variant->ref_count);
 451
 452         radv_pipeline_cache_add_entry(cache, entry);
 453
 454         cache->modified = true;
 455         pthread_mutex_unlock(&cache->mutex);
 456         return variant;
 457 }
 458
 459 void
 460 radv_pipeline_cache_insert_shaders(struct radv_device *device,
 461                                    struct radv_pipeline_cache *cache,
 462                                    const unsigned char *sha1,
 463                                    struct radv_shader_variant **variants,
 464                                    const void *const *codes,
 465                                    const unsigned *code_sizes)
 466 {
 467         if (!cache)
 468                 cache = device->mem_cache;
 469
 470         pthread_mutex_lock(&cache->mutex);
 471         struct cache_entry *entry = radv_pipeline_cache_search_unlocked(cache, sha1);
 472         if (entry) {
 473                 for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
 474                         if (entry->variants[i]) {
 475                                 radv_shader_variant_destroy(cache->device, variants[i]);
 476                                 variants[i] = entry->variants[i];
 477                         } else {
 478                                 entry->variants[i] = variants[i];
 479                         }
 480                         if (variants[i])
 481                                 p_atomic_inc(&variants[i]->ref_count);
 482                 }
 483                 pthread_mutex_unlock(&cache->mutex);
 484                 return;
 485         }
 486         size_t size = sizeof(*entry);
 487         for (int i = 0; i < MESA_SHADER_STAGES; ++i)
 488                 if (variants[i])
 489                         size += sizeof(struct cache_entry_variant_info) + code_sizes[i];
 490
 491
 492         entry = vk_alloc(&cache->alloc, size, 8,
 493                            VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
 494         if (!entry) {
 495                 pthread_mutex_unlock(&cache->mutex);
 496                 return;
 497         }
 498
 499         memset(entry, 0, sizeof(*entry));
 500         memcpy(entry->sha1, sha1, 20);
 501
 502         char* p = entry->code;
 503         struct cache_entry_variant_info info;
 504
 505         for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
 506                 if (!variants[i])
 507                         continue;
 508
 509                 entry->code_sizes[i] = code_sizes[i];
 510
 511                 info.config = variants[i]->config;
 512                 info.variant_info = variants[i]->info;
 513                 info.rsrc1 = variants[i]->rsrc1;
 514                 info.rsrc2 = variants[i]->rsrc2;
 515                 memcpy(p, &info, sizeof(struct cache_entry_variant_info));
 516                 p += sizeof(struct cache_entry_variant_info);
 517
 518                 memcpy(p, codes[i], code_sizes[i]);
 519                 p += code_sizes[i];
 520         }
 521
 522         /* Always add cache items to disk. This will allow collection of
 523          * compiled shaders by third parties such as steam, even if the app
 524          * implements its own pipeline cache.
 525          */
 526         if (device->physical_device->disk_cache) {
 527                 uint8_t disk_sha1[20];
 528                 disk_cache_compute_key(device->physical_device->disk_cache, sha1, 20,
 529                                disk_sha1);
 530                 disk_cache_put(device->physical_device->disk_cache,
 531                                disk_sha1, entry, entry_size(entry), NULL);
 532         }
 533
 534         /* We delay setting the variant so we have reproducible disk cache
 535          * items.
 536          */
 537         for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
 538                 if (!variants[i])
 539                         continue;
 540
 541                 entry->variants[i] = variants[i];
 542                 p_atomic_inc(&variants[i]->ref_count);
 543         }
 544
 545         radv_pipeline_cache_add_entry(cache, entry);
 546
 547         cache->modified = true;
 548         pthread_mutex_unlock(&cache->mutex);
 549         return;
 550 }
 551
 552 struct cache_header {
 553         uint32_t header_size;
 554         uint32_t header_version;
 555         uint32_t vendor_id;
 556         uint32_t device_id;
 557         uint8_t  uuid[VK_UUID_SIZE];
 558 };
 559
 560 void
 561 radv_pipeline_cache_load(struct radv_pipeline_cache *cache,
 562                          const void *data, size_t size)
 563 {
 564         struct radv_device *device = cache->device;
 565         struct cache_header header;
 566
 567         if (size < sizeof(header))
 568                 return;
 569         memcpy(&header, data, sizeof(header));
 570         if (header.header_size < sizeof(header))
 571                 return;
 572         if (header.header_version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE)
 573                 return;
 574         if (header.vendor_id != ATI_VENDOR_ID)
 575                 return;
 576         if (header.device_id != device->physical_device->rad_info.pci_id)
 577                 return;
 578         if (memcmp(header.uuid, device->physical_device->cache_uuid, VK_UUID_SIZE) != 0)
 579                 return;
 580
 581         char *end = (void *) data + size;
 582         char *p = (void *) data + header.header_size;
 583
 584         while (end - p >= sizeof(struct cache_entry)) {
 585                 struct cache_entry *entry = (struct cache_entry*)p;
 586                 struct cache_entry *dest_entry;
 587                 size_t size = entry_size(entry);
 588                 if(end - p < size)
 589                         break;
 590
 591                 dest_entry = vk_alloc(&cache->alloc, size,
 592                                         8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
 593                 if (dest_entry) {
 594                         memcpy(dest_entry, entry, size);
 595                         for (int i = 0; i < MESA_SHADER_STAGES; ++i)
 596                                 dest_entry->variants[i] = NULL;
 597                         radv_pipeline_cache_add_entry(cache, dest_entry);
 598                 }
 599                 p += size;
 600         }
 601 }
 602
 603 VkResult radv_CreatePipelineCache(
 604         VkDevice                                    _device,
 605         const VkPipelineCacheCreateInfo*            pCreateInfo,
 606         const VkAllocationCallbacks*                pAllocator,
 607         VkPipelineCache*                            pPipelineCache)
 608 {
 609         RADV_FROM_HANDLE(radv_device, device, _device);
 610         struct radv_pipeline_cache *cache;
 611
 612         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
 613         assert(pCreateInfo->flags == 0);
 614
 615         cache = vk_alloc2(&device->alloc, pAllocator,
 616                             sizeof(*cache), 8,
 617                             VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 618         if (cache == NULL)
 619                 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 620
 621         if (pAllocator)
 622                 cache->alloc = *pAllocator;
 623         else
 624                 cache->alloc = device->alloc;
 625
 626         radv_pipeline_cache_init(cache, device);
 627
 628         if (pCreateInfo->initialDataSize > 0) {
 629                 radv_pipeline_cache_load(cache,
 630                                          pCreateInfo->pInitialData,
 631                                          pCreateInfo->initialDataSize);
 632         }
 633
 634         *pPipelineCache = radv_pipeline_cache_to_handle(cache);
 635
 636         return VK_SUCCESS;
 637 }
 638
 639 void radv_DestroyPipelineCache(
 640         VkDevice                                    _device,
 641         VkPipelineCache                             _cache,
 642         const VkAllocationCallbacks*                pAllocator)
 643 {
 644         RADV_FROM_HANDLE(radv_device, device, _device);
 645         RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
 646
 647         if (!cache)
 648                 return;
 649         radv_pipeline_cache_finish(cache);
 650
 651         vk_free2(&device->alloc, pAllocator, cache);
 652 }
 653
 654 VkResult radv_GetPipelineCacheData(
 655         VkDevice                                    _device,
 656         VkPipelineCache                             _cache,
 657         size_t*                                     pDataSize,
 658         void*                                       pData)
 659 {
 660         RADV_FROM_HANDLE(radv_device, device, _device);
 661         RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
 662         struct cache_header *header;
 663         VkResult result = VK_SUCCESS;
 664         const size_t size = sizeof(*header) + cache->total_size;
 665         if (pData == NULL) {
 666                 *pDataSize = size;
 667                 return VK_SUCCESS;
 668         }
 669         if (*pDataSize < sizeof(*header)) {
 670                 *pDataSize = 0;
 671                 return VK_INCOMPLETE;
 672         }
 673         void *p = pData, *end = pData + *pDataSize;
 674         header = p;
 675         header->header_size = sizeof(*header);
 676         header->header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE;
 677         header->vendor_id = ATI_VENDOR_ID;
 678         header->device_id = device->physical_device->rad_info.pci_id;
 679         memcpy(header->uuid, device->physical_device->cache_uuid, VK_UUID_SIZE);
 680         p += header->header_size;
 681
 682         struct cache_entry *entry;
 683         for (uint32_t i = 0; i < cache->table_size; i++) {
 684                 if (!cache->hash_table[i])
 685                         continue;
 686                 entry = cache->hash_table[i];
 687                 const uint32_t size = entry_size(entry);
 688                 if (end < p + size) {
 689                         result = VK_INCOMPLETE;
 690                         break;
 691                 }
 692
 693                 memcpy(p, entry, size);
 694                 for(int j = 0; j < MESA_SHADER_STAGES; ++j)
 695                         ((struct cache_entry*)p)->variants[j] = NULL;
 696                 p += size;
 697         }
 698         *pDataSize = p - pData;
 699
 700         return result;
 701 }
 702
 703 static void
 704 radv_pipeline_cache_merge(struct radv_pipeline_cache *dst,
 705                           struct radv_pipeline_cache *src)
 706 {
 707         for (uint32_t i = 0; i < src->table_size; i++) {
 708                 struct cache_entry *entry = src->hash_table[i];
 709                 if (!entry || radv_pipeline_cache_search(dst, entry->sha1))
 710                         continue;
 711
 712                 radv_pipeline_cache_add_entry(dst, entry);
 713
 714                 src->hash_table[i] = NULL;
 715         }
 716 }
 717
 718 VkResult radv_MergePipelineCaches(
 719         VkDevice                                    _device,
 720         VkPipelineCache                             destCache,
 721         uint32_t                                    srcCacheCount,
 722         const VkPipelineCache*                      pSrcCaches)
 723 {
 724         RADV_FROM_HANDLE(radv_pipeline_cache, dst, destCache);
 725
 726         for (uint32_t i = 0; i < srcCacheCount; i++) {
 727                 RADV_FROM_HANDLE(radv_pipeline_cache, src, pSrcCaches[i]);
 728
 729                 radv_pipeline_cache_merge(dst, src);
 730         }
 731
 732         return VK_SUCCESS;
 733 }