2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "compiler/blob.h"
25 #include "util/hash_table.h"
26 #include "util/debug.h"
27 #include "util/disk_cache.h"
28 #include "util/mesa-sha1.h"
29 #include "anv_private.h"
31 struct anv_shader_bin
*
32 anv_shader_bin_create(struct anv_device
*device
,
33 const void *key_data
, uint32_t key_size
,
34 const void *kernel_data
, uint32_t kernel_size
,
35 const void *constant_data
, uint32_t constant_data_size
,
36 const struct brw_stage_prog_data
*prog_data_in
,
37 uint32_t prog_data_size
, const void *prog_data_param_in
,
38 const struct anv_pipeline_bind_map
*bind_map
)
40 struct anv_shader_bin
*shader
;
41 struct anv_shader_bin_key
*key
;
42 struct brw_stage_prog_data
*prog_data
;
43 uint32_t *prog_data_param
;
44 struct anv_pipeline_binding
*surface_to_descriptor
, *sampler_to_descriptor
;
47 anv_multialloc_add(&ma
, &shader
, 1);
48 anv_multialloc_add_size(&ma
, &key
, sizeof(*key
) + key_size
);
49 anv_multialloc_add_size(&ma
, &prog_data
, prog_data_size
);
50 anv_multialloc_add(&ma
, &prog_data_param
, prog_data_in
->nr_params
);
51 anv_multialloc_add(&ma
, &surface_to_descriptor
,
52 bind_map
->surface_count
);
53 anv_multialloc_add(&ma
, &sampler_to_descriptor
,
54 bind_map
->sampler_count
);
56 if (!anv_multialloc_alloc(&ma
, &device
->alloc
,
57 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
))
63 memcpy(key
->data
, key_data
, key_size
);
67 anv_state_pool_alloc(&device
->instruction_state_pool
, kernel_size
, 64);
68 memcpy(shader
->kernel
.map
, kernel_data
, kernel_size
);
69 shader
->kernel_size
= kernel_size
;
71 if (constant_data_size
) {
72 shader
->constant_data
=
73 anv_state_pool_alloc(&device
->dynamic_state_pool
,
74 constant_data_size
, 32);
75 memcpy(shader
->constant_data
.map
, constant_data
, constant_data_size
);
77 shader
->constant_data
= ANV_STATE_NULL
;
79 shader
->constant_data_size
= constant_data_size
;
81 memcpy(prog_data
, prog_data_in
, prog_data_size
);
82 memcpy(prog_data_param
, prog_data_param_in
,
83 prog_data
->nr_params
* sizeof(*prog_data_param
));
84 prog_data
->param
= prog_data_param
;
85 shader
->prog_data
= prog_data
;
86 shader
->prog_data_size
= prog_data_size
;
88 shader
->bind_map
= *bind_map
;
89 typed_memcpy(surface_to_descriptor
, bind_map
->surface_to_descriptor
,
90 bind_map
->surface_count
);
91 shader
->bind_map
.surface_to_descriptor
= surface_to_descriptor
;
92 typed_memcpy(sampler_to_descriptor
, bind_map
->sampler_to_descriptor
,
93 bind_map
->sampler_count
);
94 shader
->bind_map
.sampler_to_descriptor
= sampler_to_descriptor
;
100 anv_shader_bin_destroy(struct anv_device
*device
,
101 struct anv_shader_bin
*shader
)
103 assert(shader
->ref_cnt
== 0);
104 anv_state_pool_free(&device
->instruction_state_pool
, shader
->kernel
);
105 anv_state_pool_free(&device
->dynamic_state_pool
, shader
->constant_data
);
106 vk_free(&device
->alloc
, shader
);
110 anv_shader_bin_write_to_blob(const struct anv_shader_bin
*shader
,
115 ok
= blob_write_uint32(blob
, shader
->key
->size
);
116 ok
= blob_write_bytes(blob
, shader
->key
->data
, shader
->key
->size
);
118 ok
= blob_write_uint32(blob
, shader
->kernel_size
);
119 ok
= blob_write_bytes(blob
, shader
->kernel
.map
, shader
->kernel_size
);
121 ok
= blob_write_uint32(blob
, shader
->constant_data_size
);
122 ok
= blob_write_bytes(blob
, shader
->constant_data
.map
,
123 shader
->constant_data_size
);
125 ok
= blob_write_uint32(blob
, shader
->prog_data_size
);
126 ok
= blob_write_bytes(blob
, shader
->prog_data
, shader
->prog_data_size
);
127 ok
= blob_write_bytes(blob
, shader
->prog_data
->param
,
128 shader
->prog_data
->nr_params
*
129 sizeof(*shader
->prog_data
->param
));
131 ok
= blob_write_uint32(blob
, shader
->bind_map
.surface_count
);
132 ok
= blob_write_uint32(blob
, shader
->bind_map
.sampler_count
);
133 ok
= blob_write_uint32(blob
, shader
->bind_map
.image_count
);
134 ok
= blob_write_bytes(blob
, shader
->bind_map
.surface_to_descriptor
,
135 shader
->bind_map
.surface_count
*
136 sizeof(*shader
->bind_map
.surface_to_descriptor
));
137 ok
= blob_write_bytes(blob
, shader
->bind_map
.sampler_to_descriptor
,
138 shader
->bind_map
.sampler_count
*
139 sizeof(*shader
->bind_map
.sampler_to_descriptor
));
144 static struct anv_shader_bin
*
145 anv_shader_bin_create_from_blob(struct anv_device
*device
,
146 struct blob_reader
*blob
)
148 uint32_t key_size
= blob_read_uint32(blob
);
149 const void *key_data
= blob_read_bytes(blob
, key_size
);
151 uint32_t kernel_size
= blob_read_uint32(blob
);
152 const void *kernel_data
= blob_read_bytes(blob
, kernel_size
);
154 uint32_t constant_data_size
= blob_read_uint32(blob
);
155 const void *constant_data
= blob_read_bytes(blob
, constant_data_size
);
157 uint32_t prog_data_size
= blob_read_uint32(blob
);
158 const struct brw_stage_prog_data
*prog_data
=
159 blob_read_bytes(blob
, prog_data_size
);
162 const void *prog_data_param
=
163 blob_read_bytes(blob
, prog_data
->nr_params
* sizeof(*prog_data
->param
));
165 struct anv_pipeline_bind_map bind_map
;
166 bind_map
.surface_count
= blob_read_uint32(blob
);
167 bind_map
.sampler_count
= blob_read_uint32(blob
);
168 bind_map
.image_count
= blob_read_uint32(blob
);
169 bind_map
.surface_to_descriptor
= (void *)
170 blob_read_bytes(blob
, bind_map
.surface_count
*
171 sizeof(*bind_map
.surface_to_descriptor
));
172 bind_map
.sampler_to_descriptor
= (void *)
173 blob_read_bytes(blob
, bind_map
.sampler_count
*
174 sizeof(*bind_map
.sampler_to_descriptor
));
179 return anv_shader_bin_create(device
,
181 kernel_data
, kernel_size
,
182 constant_data
, constant_data_size
,
183 prog_data
, prog_data_size
, prog_data_param
,
189 * - Compact binding table layout so it's tight and not dependent on
190 * descriptor set layout.
192 * - Review prog_data struct for size and cacheability: struct
193 * brw_stage_prog_data has binding_table which uses a lot of uint32_t for 8
194 * bit quantities etc; use bit fields for all bools, eg dual_src_blend.
198 shader_bin_key_hash_func(const void *void_key
)
200 const struct anv_shader_bin_key
*key
= void_key
;
201 return _mesa_hash_data(key
->data
, key
->size
);
205 shader_bin_key_compare_func(const void *void_a
, const void *void_b
)
207 const struct anv_shader_bin_key
*a
= void_a
, *b
= void_b
;
208 if (a
->size
!= b
->size
)
211 return memcmp(a
->data
, b
->data
, a
->size
) == 0;
215 anv_pipeline_cache_init(struct anv_pipeline_cache
*cache
,
216 struct anv_device
*device
,
219 cache
->device
= device
;
220 pthread_mutex_init(&cache
->mutex
, NULL
);
223 cache
->cache
= _mesa_hash_table_create(NULL
, shader_bin_key_hash_func
,
224 shader_bin_key_compare_func
);
231 anv_pipeline_cache_finish(struct anv_pipeline_cache
*cache
)
233 pthread_mutex_destroy(&cache
->mutex
);
236 /* This is a bit unfortunate. In order to keep things from randomly
237 * going away, the shader cache has to hold a reference to all shader
238 * binaries it contains. We unref them when we destroy the cache.
240 struct hash_entry
*entry
;
241 hash_table_foreach(cache
->cache
, entry
)
242 anv_shader_bin_unref(cache
->device
, entry
->data
);
244 _mesa_hash_table_destroy(cache
->cache
, NULL
);
248 static struct anv_shader_bin
*
249 anv_pipeline_cache_search_locked(struct anv_pipeline_cache
*cache
,
250 const void *key_data
, uint32_t key_size
)
252 uint32_t vla
[1 + DIV_ROUND_UP(key_size
, sizeof(uint32_t))];
253 struct anv_shader_bin_key
*key
= (void *)vla
;
254 key
->size
= key_size
;
255 memcpy(key
->data
, key_data
, key_size
);
257 struct hash_entry
*entry
= _mesa_hash_table_search(cache
->cache
, key
);
264 struct anv_shader_bin
*
265 anv_pipeline_cache_search(struct anv_pipeline_cache
*cache
,
266 const void *key_data
, uint32_t key_size
)
271 pthread_mutex_lock(&cache
->mutex
);
273 struct anv_shader_bin
*shader
=
274 anv_pipeline_cache_search_locked(cache
, key_data
, key_size
);
276 pthread_mutex_unlock(&cache
->mutex
);
278 /* We increment refcount before handing it to the caller */
280 anv_shader_bin_ref(shader
);
286 anv_pipeline_cache_add_shader_bin(struct anv_pipeline_cache
*cache
,
287 struct anv_shader_bin
*bin
)
292 pthread_mutex_lock(&cache
->mutex
);
294 struct hash_entry
*entry
= _mesa_hash_table_search(cache
->cache
, bin
->key
);
296 /* Take a reference for the cache */
297 anv_shader_bin_ref(bin
);
298 _mesa_hash_table_insert(cache
->cache
, bin
->key
, bin
);
301 pthread_mutex_unlock(&cache
->mutex
);
304 static struct anv_shader_bin
*
305 anv_pipeline_cache_add_shader_locked(struct anv_pipeline_cache
*cache
,
306 const void *key_data
, uint32_t key_size
,
307 const void *kernel_data
,
308 uint32_t kernel_size
,
309 const void *constant_data
,
310 uint32_t constant_data_size
,
311 const struct brw_stage_prog_data
*prog_data
,
312 uint32_t prog_data_size
,
313 const void *prog_data_param
,
314 const struct anv_pipeline_bind_map
*bind_map
)
316 struct anv_shader_bin
*shader
=
317 anv_pipeline_cache_search_locked(cache
, key_data
, key_size
);
321 struct anv_shader_bin
*bin
=
322 anv_shader_bin_create(cache
->device
, key_data
, key_size
,
323 kernel_data
, kernel_size
,
324 constant_data
, constant_data_size
,
325 prog_data
, prog_data_size
, prog_data_param
,
330 _mesa_hash_table_insert(cache
->cache
, bin
->key
, bin
);
335 struct anv_shader_bin
*
336 anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache
*cache
,
337 const void *key_data
, uint32_t key_size
,
338 const void *kernel_data
, uint32_t kernel_size
,
339 const void *constant_data
,
340 uint32_t constant_data_size
,
341 const struct brw_stage_prog_data
*prog_data
,
342 uint32_t prog_data_size
,
343 const struct anv_pipeline_bind_map
*bind_map
)
346 pthread_mutex_lock(&cache
->mutex
);
348 struct anv_shader_bin
*bin
=
349 anv_pipeline_cache_add_shader_locked(cache
, key_data
, key_size
,
350 kernel_data
, kernel_size
,
351 constant_data
, constant_data_size
,
352 prog_data
, prog_data_size
,
353 prog_data
->param
, bind_map
);
355 pthread_mutex_unlock(&cache
->mutex
);
357 /* We increment refcount before handing it to the caller */
359 anv_shader_bin_ref(bin
);
363 /* In this case, we're not caching it so the caller owns it entirely */
364 return anv_shader_bin_create(cache
->device
, key_data
, key_size
,
365 kernel_data
, kernel_size
,
366 constant_data
, constant_data_size
,
367 prog_data
, prog_data_size
,
368 prog_data
->param
, bind_map
);
372 struct cache_header
{
373 uint32_t header_size
;
374 uint32_t header_version
;
377 uint8_t uuid
[VK_UUID_SIZE
];
381 anv_pipeline_cache_load(struct anv_pipeline_cache
*cache
,
382 const void *data
, size_t size
)
384 struct anv_device
*device
= cache
->device
;
385 struct anv_physical_device
*pdevice
= &device
->instance
->physicalDevice
;
387 if (cache
->cache
== NULL
)
390 struct blob_reader blob
;
391 blob_reader_init(&blob
, data
, size
);
393 struct cache_header header
;
394 blob_copy_bytes(&blob
, &header
, sizeof(header
));
395 uint32_t count
= blob_read_uint32(&blob
);
399 if (header
.header_size
< sizeof(header
))
401 if (header
.header_version
!= VK_PIPELINE_CACHE_HEADER_VERSION_ONE
)
403 if (header
.vendor_id
!= 0x8086)
405 if (header
.device_id
!= device
->chipset_id
)
407 if (memcmp(header
.uuid
, pdevice
->pipeline_cache_uuid
, VK_UUID_SIZE
) != 0)
410 for (uint32_t i
= 0; i
< count
; i
++) {
411 struct anv_shader_bin
*bin
=
412 anv_shader_bin_create_from_blob(device
, &blob
);
415 _mesa_hash_table_insert(cache
->cache
, bin
->key
, bin
);
419 VkResult
anv_CreatePipelineCache(
421 const VkPipelineCacheCreateInfo
* pCreateInfo
,
422 const VkAllocationCallbacks
* pAllocator
,
423 VkPipelineCache
* pPipelineCache
)
425 ANV_FROM_HANDLE(anv_device
, device
, _device
);
426 struct anv_pipeline_cache
*cache
;
428 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO
);
429 assert(pCreateInfo
->flags
== 0);
431 cache
= vk_alloc2(&device
->alloc
, pAllocator
,
433 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
435 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
437 anv_pipeline_cache_init(cache
, device
,
438 device
->instance
->pipeline_cache_enabled
);
440 if (pCreateInfo
->initialDataSize
> 0)
441 anv_pipeline_cache_load(cache
,
442 pCreateInfo
->pInitialData
,
443 pCreateInfo
->initialDataSize
);
445 *pPipelineCache
= anv_pipeline_cache_to_handle(cache
);
450 void anv_DestroyPipelineCache(
452 VkPipelineCache _cache
,
453 const VkAllocationCallbacks
* pAllocator
)
455 ANV_FROM_HANDLE(anv_device
, device
, _device
);
456 ANV_FROM_HANDLE(anv_pipeline_cache
, cache
, _cache
);
461 anv_pipeline_cache_finish(cache
);
463 vk_free2(&device
->alloc
, pAllocator
, cache
);
466 VkResult
anv_GetPipelineCacheData(
468 VkPipelineCache _cache
,
472 ANV_FROM_HANDLE(anv_device
, device
, _device
);
473 ANV_FROM_HANDLE(anv_pipeline_cache
, cache
, _cache
);
474 struct anv_physical_device
*pdevice
= &device
->instance
->physicalDevice
;
478 blob_init_fixed(&blob
, pData
, *pDataSize
);
480 blob_init_fixed(&blob
, NULL
, SIZE_MAX
);
483 struct cache_header header
= {
484 .header_size
= sizeof(struct cache_header
),
485 .header_version
= VK_PIPELINE_CACHE_HEADER_VERSION_ONE
,
487 .device_id
= device
->chipset_id
,
489 memcpy(header
.uuid
, pdevice
->pipeline_cache_uuid
, VK_UUID_SIZE
);
490 blob_write_bytes(&blob
, &header
, sizeof(header
));
493 intptr_t count_offset
= blob_reserve_uint32(&blob
);
494 if (count_offset
< 0) {
497 return VK_INCOMPLETE
;
500 VkResult result
= VK_SUCCESS
;
502 struct hash_entry
*entry
;
503 hash_table_foreach(cache
->cache
, entry
) {
504 struct anv_shader_bin
*shader
= entry
->data
;
506 size_t save_size
= blob
.size
;
507 if (!anv_shader_bin_write_to_blob(shader
, &blob
)) {
508 /* If it fails reset to the previous size and bail */
509 blob
.size
= save_size
;
510 result
= VK_INCOMPLETE
;
518 blob_overwrite_uint32(&blob
, count_offset
, count
);
520 *pDataSize
= blob
.size
;
527 VkResult
anv_MergePipelineCaches(
529 VkPipelineCache destCache
,
530 uint32_t srcCacheCount
,
531 const VkPipelineCache
* pSrcCaches
)
533 ANV_FROM_HANDLE(anv_pipeline_cache
, dst
, destCache
);
538 for (uint32_t i
= 0; i
< srcCacheCount
; i
++) {
539 ANV_FROM_HANDLE(anv_pipeline_cache
, src
, pSrcCaches
[i
]);
543 struct hash_entry
*entry
;
544 hash_table_foreach(src
->cache
, entry
) {
545 struct anv_shader_bin
*bin
= entry
->data
;
548 if (_mesa_hash_table_search(dst
->cache
, bin
->key
))
551 anv_shader_bin_ref(bin
);
552 _mesa_hash_table_insert(dst
->cache
, bin
->key
, bin
);
559 struct anv_shader_bin
*
560 anv_device_search_for_kernel(struct anv_device
*device
,
561 struct anv_pipeline_cache
*cache
,
562 const void *key_data
, uint32_t key_size
)
564 struct anv_shader_bin
*bin
;
567 bin
= anv_pipeline_cache_search(cache
, key_data
, key_size
);
572 #ifdef ENABLE_SHADER_CACHE
573 struct disk_cache
*disk_cache
= device
->instance
->physicalDevice
.disk_cache
;
574 if (disk_cache
&& device
->instance
->pipeline_cache_enabled
) {
576 disk_cache_compute_key(disk_cache
, key_data
, key_size
, cache_key
);
579 uint8_t *buffer
= disk_cache_get(disk_cache
, cache_key
, &buffer_size
);
581 struct blob_reader blob
;
582 blob_reader_init(&blob
, buffer
, buffer_size
);
583 bin
= anv_shader_bin_create_from_blob(device
, &blob
);
588 anv_pipeline_cache_add_shader_bin(cache
, bin
);
598 struct anv_shader_bin
*
599 anv_device_upload_kernel(struct anv_device
*device
,
600 struct anv_pipeline_cache
*cache
,
601 const void *key_data
, uint32_t key_size
,
602 const void *kernel_data
, uint32_t kernel_size
,
603 const void *constant_data
,
604 uint32_t constant_data_size
,
605 const struct brw_stage_prog_data
*prog_data
,
606 uint32_t prog_data_size
,
607 const struct anv_pipeline_bind_map
*bind_map
)
609 struct anv_shader_bin
*bin
;
611 bin
= anv_pipeline_cache_upload_kernel(cache
, key_data
, key_size
,
612 kernel_data
, kernel_size
,
613 constant_data
, constant_data_size
,
614 prog_data
, prog_data_size
,
617 bin
= anv_shader_bin_create(device
, key_data
, key_size
,
618 kernel_data
, kernel_size
,
619 constant_data
, constant_data_size
,
620 prog_data
, prog_data_size
,
621 prog_data
->param
, bind_map
);
627 #ifdef ENABLE_SHADER_CACHE
628 struct disk_cache
*disk_cache
= device
->instance
->physicalDevice
.disk_cache
;
632 anv_shader_bin_write_to_blob(bin
, &binary
);
634 if (!binary
.out_of_memory
) {
636 disk_cache_compute_key(disk_cache
, key_data
, key_size
, cache_key
);
638 disk_cache_put(disk_cache
, cache_key
, binary
.data
, binary
.size
, NULL
);
641 blob_finish(&binary
);