2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "util/mesa-sha1.h"
25 #include "util/hash_table.h"
26 #include "util/debug.h"
27 #include "anv_private.h"
30 anv_shader_bin_size(uint32_t prog_data_size
, uint32_t nr_params
,
32 uint32_t surface_count
, uint32_t sampler_count
)
34 const uint32_t binding_data_size
=
35 (surface_count
+ sampler_count
) * sizeof(struct anv_pipeline_binding
);
37 return align_u32(sizeof(struct anv_shader_bin
), 8) +
38 align_u32(prog_data_size
, 8) +
39 align_u32(nr_params
* sizeof(void *), 8) +
40 align_u32(sizeof(uint32_t) + key_size
, 8) +
41 align_u32(binding_data_size
, 8);
44 struct anv_shader_bin
*
45 anv_shader_bin_create(struct anv_device
*device
,
46 const void *key_data
, uint32_t key_size
,
47 const void *kernel_data
, uint32_t kernel_size
,
48 const struct brw_stage_prog_data
*prog_data
,
49 uint32_t prog_data_size
, const void *prog_data_param
,
50 const struct anv_pipeline_bind_map
*bind_map
)
53 anv_shader_bin_size(prog_data_size
, prog_data
->nr_params
, key_size
,
54 bind_map
->surface_count
, bind_map
->sampler_count
);
56 struct anv_shader_bin
*shader
=
57 vk_alloc(&device
->alloc
, size
, 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
64 anv_state_pool_alloc(&device
->instruction_state_pool
, kernel_size
, 64);
65 memcpy(shader
->kernel
.map
, kernel_data
, kernel_size
);
66 shader
->kernel_size
= kernel_size
;
67 shader
->bind_map
= *bind_map
;
68 shader
->prog_data_size
= prog_data_size
;
70 /* Now we fill out the floating data at the end */
72 data
+= align_u32(sizeof(struct anv_shader_bin
), 8);
74 shader
->prog_data
= data
;
75 struct brw_stage_prog_data
*new_prog_data
= data
;
76 memcpy(data
, prog_data
, prog_data_size
);
77 data
+= align_u32(prog_data_size
, 8);
79 assert(prog_data
->nr_pull_params
== 0);
80 assert(prog_data
->nr_image_params
== 0);
81 new_prog_data
->param
= data
;
82 uint32_t param_size
= prog_data
->nr_params
* sizeof(void *);
83 memcpy(data
, prog_data_param
, param_size
);
84 data
+= align_u32(param_size
, 8);
87 struct anv_shader_bin_key
*key
= data
;
89 memcpy(key
->data
, key_data
, key_size
);
90 data
+= align_u32(sizeof(*key
) + key_size
, 8);
92 shader
->bind_map
.surface_to_descriptor
= data
;
93 memcpy(data
, bind_map
->surface_to_descriptor
,
94 bind_map
->surface_count
* sizeof(struct anv_pipeline_binding
));
95 data
+= bind_map
->surface_count
* sizeof(struct anv_pipeline_binding
);
97 shader
->bind_map
.sampler_to_descriptor
= data
;
98 memcpy(data
, bind_map
->sampler_to_descriptor
,
99 bind_map
->sampler_count
* sizeof(struct anv_pipeline_binding
));
105 anv_shader_bin_destroy(struct anv_device
*device
,
106 struct anv_shader_bin
*shader
)
108 assert(shader
->ref_cnt
== 0);
109 anv_state_pool_free(&device
->instruction_state_pool
, shader
->kernel
);
110 vk_free(&device
->alloc
, shader
);
114 anv_shader_bin_data_size(const struct anv_shader_bin
*shader
)
116 return anv_shader_bin_size(shader
->prog_data_size
,
117 shader
->prog_data
->nr_params
, shader
->key
->size
,
118 shader
->bind_map
.surface_count
,
119 shader
->bind_map
.sampler_count
) +
120 align_u32(shader
->kernel_size
, 8);
124 anv_shader_bin_write_data(const struct anv_shader_bin
*shader
, void *data
)
127 anv_shader_bin_size(shader
->prog_data_size
,
128 shader
->prog_data
->nr_params
, shader
->key
->size
,
129 shader
->bind_map
.surface_count
,
130 shader
->bind_map
.sampler_count
);
132 memcpy(data
, shader
, struct_size
);
135 memcpy(data
, shader
->kernel
.map
, shader
->kernel_size
);
140 * - Compact binding table layout so it's tight and not dependent on
141 * descriptor set layout.
143 * - Review prog_data struct for size and cacheability: struct
144 * brw_stage_prog_data has binding_table which uses a lot of uint32_t for 8
145 * bit quantities etc; param, pull_param, and image_params are pointers, we
146 * just need the compation map. use bit fields for all bools, eg
151 shader_bin_key_hash_func(const void *void_key
)
153 const struct anv_shader_bin_key
*key
= void_key
;
154 return _mesa_hash_data(key
->data
, key
->size
);
158 shader_bin_key_compare_func(const void *void_a
, const void *void_b
)
160 const struct anv_shader_bin_key
*a
= void_a
, *b
= void_b
;
161 if (a
->size
!= b
->size
)
164 return memcmp(a
->data
, b
->data
, a
->size
) == 0;
168 anv_pipeline_cache_init(struct anv_pipeline_cache
*cache
,
169 struct anv_device
*device
,
172 cache
->device
= device
;
173 pthread_mutex_init(&cache
->mutex
, NULL
);
176 cache
->cache
= _mesa_hash_table_create(NULL
, shader_bin_key_hash_func
,
177 shader_bin_key_compare_func
);
184 anv_pipeline_cache_finish(struct anv_pipeline_cache
*cache
)
186 pthread_mutex_destroy(&cache
->mutex
);
189 /* This is a bit unfortunate. In order to keep things from randomly
190 * going away, the shader cache has to hold a reference to all shader
191 * binaries it contains. We unref them when we destroy the cache.
193 struct hash_entry
*entry
;
194 hash_table_foreach(cache
->cache
, entry
)
195 anv_shader_bin_unref(cache
->device
, entry
->data
);
197 _mesa_hash_table_destroy(cache
->cache
, NULL
);
202 anv_hash_shader(unsigned char *hash
, const void *key
, size_t key_size
,
203 struct anv_shader_module
*module
,
204 const char *entrypoint
,
205 const struct anv_pipeline_layout
*pipeline_layout
,
206 const VkSpecializationInfo
*spec_info
)
208 struct mesa_sha1 ctx
;
210 _mesa_sha1_init(&ctx
);
211 _mesa_sha1_update(&ctx
, key
, key_size
);
212 _mesa_sha1_update(&ctx
, module
->sha1
, sizeof(module
->sha1
));
213 _mesa_sha1_update(&ctx
, entrypoint
, strlen(entrypoint
));
214 if (pipeline_layout
) {
215 _mesa_sha1_update(&ctx
, pipeline_layout
->sha1
,
216 sizeof(pipeline_layout
->sha1
));
218 /* hash in shader stage, pipeline layout? */
220 _mesa_sha1_update(&ctx
, spec_info
->pMapEntries
,
221 spec_info
->mapEntryCount
* sizeof spec_info
->pMapEntries
[0]);
222 _mesa_sha1_update(&ctx
, spec_info
->pData
, spec_info
->dataSize
);
224 _mesa_sha1_final(&ctx
, hash
);
227 static struct anv_shader_bin
*
228 anv_pipeline_cache_search_locked(struct anv_pipeline_cache
*cache
,
229 const void *key_data
, uint32_t key_size
)
231 uint32_t vla
[1 + DIV_ROUND_UP(key_size
, sizeof(uint32_t))];
232 struct anv_shader_bin_key
*key
= (void *)vla
;
233 key
->size
= key_size
;
234 memcpy(key
->data
, key_data
, key_size
);
236 struct hash_entry
*entry
= _mesa_hash_table_search(cache
->cache
, key
);
243 struct anv_shader_bin
*
244 anv_pipeline_cache_search(struct anv_pipeline_cache
*cache
,
245 const void *key_data
, uint32_t key_size
)
250 pthread_mutex_lock(&cache
->mutex
);
252 struct anv_shader_bin
*shader
=
253 anv_pipeline_cache_search_locked(cache
, key_data
, key_size
);
255 pthread_mutex_unlock(&cache
->mutex
);
257 /* We increment refcount before handing it to the caller */
259 anv_shader_bin_ref(shader
);
264 static struct anv_shader_bin
*
265 anv_pipeline_cache_add_shader(struct anv_pipeline_cache
*cache
,
266 const void *key_data
, uint32_t key_size
,
267 const void *kernel_data
, uint32_t kernel_size
,
268 const struct brw_stage_prog_data
*prog_data
,
269 uint32_t prog_data_size
,
270 const void *prog_data_param
,
271 const struct anv_pipeline_bind_map
*bind_map
)
273 struct anv_shader_bin
*shader
=
274 anv_pipeline_cache_search_locked(cache
, key_data
, key_size
);
278 struct anv_shader_bin
*bin
=
279 anv_shader_bin_create(cache
->device
, key_data
, key_size
,
280 kernel_data
, kernel_size
,
281 prog_data
, prog_data_size
, prog_data_param
,
286 _mesa_hash_table_insert(cache
->cache
, bin
->key
, bin
);
291 struct anv_shader_bin
*
292 anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache
*cache
,
293 const void *key_data
, uint32_t key_size
,
294 const void *kernel_data
, uint32_t kernel_size
,
295 const struct brw_stage_prog_data
*prog_data
,
296 uint32_t prog_data_size
,
297 const struct anv_pipeline_bind_map
*bind_map
)
300 pthread_mutex_lock(&cache
->mutex
);
302 struct anv_shader_bin
*bin
=
303 anv_pipeline_cache_add_shader(cache
, key_data
, key_size
,
304 kernel_data
, kernel_size
,
305 prog_data
, prog_data_size
,
306 prog_data
->param
, bind_map
);
308 pthread_mutex_unlock(&cache
->mutex
);
310 /* We increment refcount before handing it to the caller */
312 anv_shader_bin_ref(bin
);
316 /* In this case, we're not caching it so the caller owns it entirely */
317 return anv_shader_bin_create(cache
->device
, key_data
, key_size
,
318 kernel_data
, kernel_size
,
319 prog_data
, prog_data_size
,
320 prog_data
->param
, bind_map
);
324 struct cache_header
{
325 uint32_t header_size
;
326 uint32_t header_version
;
329 uint8_t uuid
[VK_UUID_SIZE
];
333 anv_pipeline_cache_load(struct anv_pipeline_cache
*cache
,
334 const void *data
, size_t size
)
336 struct anv_device
*device
= cache
->device
;
337 struct anv_physical_device
*pdevice
= &device
->instance
->physicalDevice
;
338 struct cache_header header
;
340 if (cache
->cache
== NULL
)
343 if (size
< sizeof(header
))
345 memcpy(&header
, data
, sizeof(header
));
346 if (header
.header_size
< sizeof(header
))
348 if (header
.header_version
!= VK_PIPELINE_CACHE_HEADER_VERSION_ONE
)
350 if (header
.vendor_id
!= 0x8086)
352 if (header
.device_id
!= device
->chipset_id
)
354 if (memcmp(header
.uuid
, pdevice
->pipeline_cache_uuid
, VK_UUID_SIZE
) != 0)
357 const void *end
= data
+ size
;
358 const void *p
= data
+ header
.header_size
;
360 /* Count is the total number of valid entries */
362 if (p
+ sizeof(count
) >= end
)
364 memcpy(&count
, p
, sizeof(count
));
365 p
+= align_u32(sizeof(count
), 8);
367 for (uint32_t i
= 0; i
< count
; i
++) {
368 struct anv_shader_bin bin
;
369 if (p
+ sizeof(bin
) > end
)
371 memcpy(&bin
, p
, sizeof(bin
));
372 p
+= align_u32(sizeof(struct anv_shader_bin
), 8);
374 const struct brw_stage_prog_data
*prog_data
= p
;
375 p
+= align_u32(bin
.prog_data_size
, 8);
379 uint32_t param_size
= prog_data
->nr_params
* sizeof(void *);
380 const void *prog_data_param
= p
;
381 p
+= align_u32(param_size
, 8);
383 struct anv_shader_bin_key key
;
384 if (p
+ sizeof(key
) > end
)
386 memcpy(&key
, p
, sizeof(key
));
387 const void *key_data
= p
+ sizeof(key
);
388 p
+= align_u32(sizeof(key
) + key
.size
, 8);
390 /* We're going to memcpy this so getting rid of const is fine */
391 struct anv_pipeline_binding
*bindings
= (void *)p
;
392 p
+= align_u32((bin
.bind_map
.surface_count
+ bin
.bind_map
.sampler_count
) *
393 sizeof(struct anv_pipeline_binding
), 8);
394 bin
.bind_map
.surface_to_descriptor
= bindings
;
395 bin
.bind_map
.sampler_to_descriptor
= bindings
+ bin
.bind_map
.surface_count
;
397 const void *kernel_data
= p
;
398 p
+= align_u32(bin
.kernel_size
, 8);
403 anv_pipeline_cache_add_shader(cache
, key_data
, key
.size
,
404 kernel_data
, bin
.kernel_size
,
405 prog_data
, bin
.prog_data_size
,
406 prog_data_param
, &bin
.bind_map
);
411 pipeline_cache_enabled()
413 static int enabled
= -1;
415 enabled
= env_var_as_boolean("ANV_ENABLE_PIPELINE_CACHE", true);
419 VkResult
anv_CreatePipelineCache(
421 const VkPipelineCacheCreateInfo
* pCreateInfo
,
422 const VkAllocationCallbacks
* pAllocator
,
423 VkPipelineCache
* pPipelineCache
)
425 ANV_FROM_HANDLE(anv_device
, device
, _device
);
426 struct anv_pipeline_cache
*cache
;
428 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO
);
429 assert(pCreateInfo
->flags
== 0);
431 cache
= vk_alloc2(&device
->alloc
, pAllocator
,
433 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
435 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
437 anv_pipeline_cache_init(cache
, device
, pipeline_cache_enabled());
439 if (pCreateInfo
->initialDataSize
> 0)
440 anv_pipeline_cache_load(cache
,
441 pCreateInfo
->pInitialData
,
442 pCreateInfo
->initialDataSize
);
444 *pPipelineCache
= anv_pipeline_cache_to_handle(cache
);
449 void anv_DestroyPipelineCache(
451 VkPipelineCache _cache
,
452 const VkAllocationCallbacks
* pAllocator
)
454 ANV_FROM_HANDLE(anv_device
, device
, _device
);
455 ANV_FROM_HANDLE(anv_pipeline_cache
, cache
, _cache
);
460 anv_pipeline_cache_finish(cache
);
462 vk_free2(&device
->alloc
, pAllocator
, cache
);
465 VkResult
anv_GetPipelineCacheData(
467 VkPipelineCache _cache
,
471 ANV_FROM_HANDLE(anv_device
, device
, _device
);
472 ANV_FROM_HANDLE(anv_pipeline_cache
, cache
, _cache
);
473 struct anv_physical_device
*pdevice
= &device
->instance
->physicalDevice
;
474 struct cache_header
*header
;
477 size_t size
= align_u32(sizeof(*header
), 8) +
478 align_u32(sizeof(uint32_t), 8);
481 struct hash_entry
*entry
;
482 hash_table_foreach(cache
->cache
, entry
)
483 size
+= anv_shader_bin_data_size(entry
->data
);
490 if (*pDataSize
< sizeof(*header
)) {
492 return VK_INCOMPLETE
;
495 void *p
= pData
, *end
= pData
+ *pDataSize
;
497 header
->header_size
= sizeof(*header
);
498 header
->header_version
= VK_PIPELINE_CACHE_HEADER_VERSION_ONE
;
499 header
->vendor_id
= 0x8086;
500 header
->device_id
= device
->chipset_id
;
501 memcpy(header
->uuid
, pdevice
->pipeline_cache_uuid
, VK_UUID_SIZE
);
502 p
+= align_u32(header
->header_size
, 8);
505 p
+= align_u32(sizeof(*count
), 8);
508 VkResult result
= VK_SUCCESS
;
510 struct hash_entry
*entry
;
511 hash_table_foreach(cache
->cache
, entry
) {
512 struct anv_shader_bin
*shader
= entry
->data
;
513 size_t data_size
= anv_shader_bin_data_size(entry
->data
);
514 if (p
+ data_size
> end
) {
515 result
= VK_INCOMPLETE
;
519 anv_shader_bin_write_data(shader
, p
);
526 *pDataSize
= p
- pData
;
531 VkResult
anv_MergePipelineCaches(
533 VkPipelineCache destCache
,
534 uint32_t srcCacheCount
,
535 const VkPipelineCache
* pSrcCaches
)
537 ANV_FROM_HANDLE(anv_pipeline_cache
, dst
, destCache
);
542 for (uint32_t i
= 0; i
< srcCacheCount
; i
++) {
543 ANV_FROM_HANDLE(anv_pipeline_cache
, src
, pSrcCaches
[i
]);
547 struct hash_entry
*entry
;
548 hash_table_foreach(src
->cache
, entry
) {
549 struct anv_shader_bin
*bin
= entry
->data
;
552 if (_mesa_hash_table_search(dst
->cache
, bin
->key
))
555 anv_shader_bin_ref(bin
);
556 _mesa_hash_table_insert(dst
->cache
, bin
->key
, bin
);