2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "util/mesa-sha1.h"
25 #include "util/hash_table.h"
26 #include "util/debug.h"
27 #include "anv_private.h"
29 struct shader_bin_key
{
35 anv_shader_bin_size(uint32_t prog_data_size
, uint32_t key_size
,
36 uint32_t surface_count
, uint32_t sampler_count
)
38 const uint32_t binding_data_size
=
39 (surface_count
+ sampler_count
) * sizeof(struct anv_pipeline_binding
);
41 return align_u32(sizeof(struct anv_shader_bin
), 8) +
42 align_u32(prog_data_size
, 8) +
43 align_u32(sizeof(uint32_t) + key_size
, 8) +
44 align_u32(binding_data_size
, 8);
47 static inline const struct shader_bin_key
*
48 anv_shader_bin_get_key(const struct anv_shader_bin
*shader
)
50 const void *data
= shader
;
51 data
+= align_u32(sizeof(struct anv_shader_bin
), 8);
52 data
+= align_u32(shader
->prog_data_size
, 8);
56 struct anv_shader_bin
*
57 anv_shader_bin_create(struct anv_device
*device
,
58 const void *key_data
, uint32_t key_size
,
59 const void *kernel_data
, uint32_t kernel_size
,
60 const void *prog_data
, uint32_t prog_data_size
,
61 const struct anv_pipeline_bind_map
*bind_map
)
64 anv_shader_bin_size(prog_data_size
, key_size
,
65 bind_map
->surface_count
, bind_map
->sampler_count
);
67 struct anv_shader_bin
*shader
=
68 vk_alloc(&device
->alloc
, size
, 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
75 anv_state_pool_alloc(&device
->instruction_state_pool
, kernel_size
, 64);
76 memcpy(shader
->kernel
.map
, kernel_data
, kernel_size
);
77 shader
->kernel_size
= kernel_size
;
78 shader
->bind_map
= *bind_map
;
79 shader
->prog_data_size
= prog_data_size
;
81 /* Now we fill out the floating data at the end */
83 data
+= align_u32(sizeof(struct anv_shader_bin
), 8);
85 memcpy(data
, prog_data
, prog_data_size
);
86 data
+= align_u32(prog_data_size
, 8);
88 struct shader_bin_key
*key
= data
;
90 memcpy(key
->data
, key_data
, key_size
);
91 data
+= align_u32(sizeof(*key
) + key_size
, 8);
93 shader
->bind_map
.surface_to_descriptor
= data
;
94 memcpy(data
, bind_map
->surface_to_descriptor
,
95 bind_map
->surface_count
* sizeof(struct anv_pipeline_binding
));
96 data
+= bind_map
->surface_count
* sizeof(struct anv_pipeline_binding
);
98 shader
->bind_map
.sampler_to_descriptor
= data
;
99 memcpy(data
, bind_map
->sampler_to_descriptor
,
100 bind_map
->sampler_count
* sizeof(struct anv_pipeline_binding
));
106 anv_shader_bin_destroy(struct anv_device
*device
,
107 struct anv_shader_bin
*shader
)
109 assert(shader
->ref_cnt
== 0);
110 anv_state_pool_free(&device
->instruction_state_pool
, shader
->kernel
);
111 vk_free(&device
->alloc
, shader
);
115 anv_shader_bin_data_size(const struct anv_shader_bin
*shader
)
117 return anv_shader_bin_size(shader
->prog_data_size
,
118 anv_shader_bin_get_key(shader
)->size
,
119 shader
->bind_map
.surface_count
,
120 shader
->bind_map
.sampler_count
) +
121 align_u32(shader
->kernel_size
, 8);
125 anv_shader_bin_write_data(const struct anv_shader_bin
*shader
, void *data
)
128 anv_shader_bin_size(shader
->prog_data_size
,
129 anv_shader_bin_get_key(shader
)->size
,
130 shader
->bind_map
.surface_count
,
131 shader
->bind_map
.sampler_count
);
133 memcpy(data
, shader
, struct_size
);
136 memcpy(data
, shader
->kernel
.map
, shader
->kernel_size
);
141 * - Compact binding table layout so it's tight and not dependent on
142 * descriptor set layout.
144 * - Review prog_data struct for size and cacheability: struct
145 * brw_stage_prog_data has binding_table which uses a lot of uint32_t for 8
146 * bit quantities etc; param, pull_param, and image_params are pointers, we
147 * just need the compation map. use bit fields for all bools, eg
152 shader_bin_key_hash_func(const void *void_key
)
154 const struct shader_bin_key
*key
= void_key
;
155 return _mesa_hash_data(key
->data
, key
->size
);
159 shader_bin_key_compare_func(const void *void_a
, const void *void_b
)
161 const struct shader_bin_key
*a
= void_a
, *b
= void_b
;
162 if (a
->size
!= b
->size
)
165 return memcmp(a
->data
, b
->data
, a
->size
) == 0;
169 anv_pipeline_cache_init(struct anv_pipeline_cache
*cache
,
170 struct anv_device
*device
,
173 cache
->device
= device
;
174 pthread_mutex_init(&cache
->mutex
, NULL
);
177 cache
->cache
= _mesa_hash_table_create(NULL
, shader_bin_key_hash_func
,
178 shader_bin_key_compare_func
);
185 anv_pipeline_cache_finish(struct anv_pipeline_cache
*cache
)
187 pthread_mutex_destroy(&cache
->mutex
);
190 /* This is a bit unfortunate. In order to keep things from randomly
191 * going away, the shader cache has to hold a reference to all shader
192 * binaries it contains. We unref them when we destroy the cache.
194 struct hash_entry
*entry
;
195 hash_table_foreach(cache
->cache
, entry
)
196 anv_shader_bin_unref(cache
->device
, entry
->data
);
198 _mesa_hash_table_destroy(cache
->cache
, NULL
);
203 anv_hash_shader(unsigned char *hash
, const void *key
, size_t key_size
,
204 struct anv_shader_module
*module
,
205 const char *entrypoint
,
206 const struct anv_pipeline_layout
*pipeline_layout
,
207 const VkSpecializationInfo
*spec_info
)
209 struct mesa_sha1
*ctx
;
211 ctx
= _mesa_sha1_init();
212 _mesa_sha1_update(ctx
, key
, key_size
);
213 _mesa_sha1_update(ctx
, module
->sha1
, sizeof(module
->sha1
));
214 _mesa_sha1_update(ctx
, entrypoint
, strlen(entrypoint
));
215 if (pipeline_layout
) {
216 _mesa_sha1_update(ctx
, pipeline_layout
->sha1
,
217 sizeof(pipeline_layout
->sha1
));
219 /* hash in shader stage, pipeline layout? */
221 _mesa_sha1_update(ctx
, spec_info
->pMapEntries
,
222 spec_info
->mapEntryCount
* sizeof spec_info
->pMapEntries
[0]);
223 _mesa_sha1_update(ctx
, spec_info
->pData
, spec_info
->dataSize
);
225 _mesa_sha1_final(ctx
, hash
);
228 static struct anv_shader_bin
*
229 anv_pipeline_cache_search_locked(struct anv_pipeline_cache
*cache
,
230 const void *key_data
, uint32_t key_size
)
232 uint32_t vla
[1 + DIV_ROUND_UP(key_size
, sizeof(uint32_t))];
233 struct shader_bin_key
*key
= (void *)vla
;
234 key
->size
= key_size
;
235 memcpy(key
->data
, key_data
, key_size
);
237 struct hash_entry
*entry
= _mesa_hash_table_search(cache
->cache
, key
);
244 struct anv_shader_bin
*
245 anv_pipeline_cache_search(struct anv_pipeline_cache
*cache
,
246 const void *key_data
, uint32_t key_size
)
251 pthread_mutex_lock(&cache
->mutex
);
253 struct anv_shader_bin
*shader
=
254 anv_pipeline_cache_search_locked(cache
, key_data
, key_size
);
256 pthread_mutex_unlock(&cache
->mutex
);
258 /* We increment refcount before handing it to the caller */
260 anv_shader_bin_ref(shader
);
265 static struct anv_shader_bin
*
266 anv_pipeline_cache_add_shader(struct anv_pipeline_cache
*cache
,
267 const void *key_data
, uint32_t key_size
,
268 const void *kernel_data
, uint32_t kernel_size
,
269 const void *prog_data
, uint32_t prog_data_size
,
270 const struct anv_pipeline_bind_map
*bind_map
)
272 struct anv_shader_bin
*shader
=
273 anv_pipeline_cache_search_locked(cache
, key_data
, key_size
);
277 struct anv_shader_bin
*bin
=
278 anv_shader_bin_create(cache
->device
, key_data
, key_size
,
279 kernel_data
, kernel_size
,
280 prog_data
, prog_data_size
, bind_map
);
284 _mesa_hash_table_insert(cache
->cache
, anv_shader_bin_get_key(bin
), bin
);
289 struct anv_shader_bin
*
290 anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache
*cache
,
291 const void *key_data
, uint32_t key_size
,
292 const void *kernel_data
, uint32_t kernel_size
,
293 const void *prog_data
, uint32_t prog_data_size
,
294 const struct anv_pipeline_bind_map
*bind_map
)
297 pthread_mutex_lock(&cache
->mutex
);
299 struct anv_shader_bin
*bin
=
300 anv_pipeline_cache_add_shader(cache
, key_data
, key_size
,
301 kernel_data
, kernel_size
,
302 prog_data
, prog_data_size
, bind_map
);
304 pthread_mutex_unlock(&cache
->mutex
);
306 /* We increment refcount before handing it to the caller */
307 anv_shader_bin_ref(bin
);
311 /* In this case, we're not caching it so the caller owns it entirely */
312 return anv_shader_bin_create(cache
->device
, key_data
, key_size
,
313 kernel_data
, kernel_size
,
314 prog_data
, prog_data_size
, bind_map
);
318 struct cache_header
{
319 uint32_t header_size
;
320 uint32_t header_version
;
323 uint8_t uuid
[VK_UUID_SIZE
];
327 anv_pipeline_cache_load(struct anv_pipeline_cache
*cache
,
328 const void *data
, size_t size
)
330 struct anv_device
*device
= cache
->device
;
331 struct cache_header header
;
332 uint8_t uuid
[VK_UUID_SIZE
];
334 if (cache
->cache
== NULL
)
337 if (size
< sizeof(header
))
339 memcpy(&header
, data
, sizeof(header
));
340 if (header
.header_size
< sizeof(header
))
342 if (header
.header_version
!= VK_PIPELINE_CACHE_HEADER_VERSION_ONE
)
344 if (header
.vendor_id
!= 0x8086)
346 if (header
.device_id
!= device
->chipset_id
)
348 anv_device_get_cache_uuid(uuid
);
349 if (memcmp(header
.uuid
, uuid
, VK_UUID_SIZE
) != 0)
352 const void *end
= data
+ size
;
353 const void *p
= data
+ header
.header_size
;
355 /* Count is the total number of valid entries */
357 if (p
+ sizeof(count
) >= end
)
359 memcpy(&count
, p
, sizeof(count
));
360 p
+= align_u32(sizeof(count
), 8);
362 for (uint32_t i
= 0; i
< count
; i
++) {
363 struct anv_shader_bin bin
;
364 if (p
+ sizeof(bin
) > end
)
366 memcpy(&bin
, p
, sizeof(bin
));
367 p
+= align_u32(sizeof(struct anv_shader_bin
), 8);
369 const void *prog_data
= p
;
370 p
+= align_u32(bin
.prog_data_size
, 8);
372 struct shader_bin_key key
;
373 if (p
+ sizeof(key
) > end
)
375 memcpy(&key
, p
, sizeof(key
));
376 const void *key_data
= p
+ sizeof(key
);
377 p
+= align_u32(sizeof(key
) + key
.size
, 8);
379 /* We're going to memcpy this so getting rid of const is fine */
380 struct anv_pipeline_binding
*bindings
= (void *)p
;
381 p
+= align_u32((bin
.bind_map
.surface_count
+ bin
.bind_map
.sampler_count
) *
382 sizeof(struct anv_pipeline_binding
), 8);
383 bin
.bind_map
.surface_to_descriptor
= bindings
;
384 bin
.bind_map
.sampler_to_descriptor
= bindings
+ bin
.bind_map
.surface_count
;
386 const void *kernel_data
= p
;
387 p
+= align_u32(bin
.kernel_size
, 8);
392 anv_pipeline_cache_add_shader(cache
, key_data
, key
.size
,
393 kernel_data
, bin
.kernel_size
,
394 prog_data
, bin
.prog_data_size
,
400 pipeline_cache_enabled()
402 static int enabled
= -1;
404 enabled
= env_var_as_boolean("ANV_ENABLE_PIPELINE_CACHE", true);
408 VkResult
anv_CreatePipelineCache(
410 const VkPipelineCacheCreateInfo
* pCreateInfo
,
411 const VkAllocationCallbacks
* pAllocator
,
412 VkPipelineCache
* pPipelineCache
)
414 ANV_FROM_HANDLE(anv_device
, device
, _device
);
415 struct anv_pipeline_cache
*cache
;
417 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO
);
418 assert(pCreateInfo
->flags
== 0);
420 cache
= vk_alloc2(&device
->alloc
, pAllocator
,
422 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
424 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
426 anv_pipeline_cache_init(cache
, device
, pipeline_cache_enabled());
428 if (pCreateInfo
->initialDataSize
> 0)
429 anv_pipeline_cache_load(cache
,
430 pCreateInfo
->pInitialData
,
431 pCreateInfo
->initialDataSize
);
433 *pPipelineCache
= anv_pipeline_cache_to_handle(cache
);
438 void anv_DestroyPipelineCache(
440 VkPipelineCache _cache
,
441 const VkAllocationCallbacks
* pAllocator
)
443 ANV_FROM_HANDLE(anv_device
, device
, _device
);
444 ANV_FROM_HANDLE(anv_pipeline_cache
, cache
, _cache
);
446 anv_pipeline_cache_finish(cache
);
448 vk_free2(&device
->alloc
, pAllocator
, cache
);
451 VkResult
anv_GetPipelineCacheData(
453 VkPipelineCache _cache
,
457 ANV_FROM_HANDLE(anv_device
, device
, _device
);
458 ANV_FROM_HANDLE(anv_pipeline_cache
, cache
, _cache
);
459 struct cache_header
*header
;
462 size_t size
= align_u32(sizeof(*header
), 8) +
463 align_u32(sizeof(uint32_t), 8);
466 struct hash_entry
*entry
;
467 hash_table_foreach(cache
->cache
, entry
)
468 size
+= anv_shader_bin_data_size(entry
->data
);
475 if (*pDataSize
< sizeof(*header
)) {
477 return VK_INCOMPLETE
;
480 void *p
= pData
, *end
= pData
+ *pDataSize
;
482 header
->header_size
= sizeof(*header
);
483 header
->header_version
= VK_PIPELINE_CACHE_HEADER_VERSION_ONE
;
484 header
->vendor_id
= 0x8086;
485 header
->device_id
= device
->chipset_id
;
486 anv_device_get_cache_uuid(header
->uuid
);
487 p
+= align_u32(header
->header_size
, 8);
490 p
+= align_u32(sizeof(*count
), 8);
493 VkResult result
= VK_SUCCESS
;
495 struct hash_entry
*entry
;
496 hash_table_foreach(cache
->cache
, entry
) {
497 struct anv_shader_bin
*shader
= entry
->data
;
498 size_t data_size
= anv_shader_bin_data_size(entry
->data
);
499 if (p
+ data_size
> end
) {
500 result
= VK_INCOMPLETE
;
504 anv_shader_bin_write_data(shader
, p
);
511 *pDataSize
= p
- pData
;
516 VkResult
anv_MergePipelineCaches(
518 VkPipelineCache destCache
,
519 uint32_t srcCacheCount
,
520 const VkPipelineCache
* pSrcCaches
)
522 ANV_FROM_HANDLE(anv_pipeline_cache
, dst
, destCache
);
527 for (uint32_t i
= 0; i
< srcCacheCount
; i
++) {
528 ANV_FROM_HANDLE(anv_pipeline_cache
, src
, pSrcCaches
[i
]);
532 struct hash_entry
*entry
;
533 hash_table_foreach(src
->cache
, entry
) {
534 struct anv_shader_bin
*bin
= entry
->data
;
535 if (_mesa_hash_table_search(dst
->cache
, anv_shader_bin_get_key(bin
)))
538 anv_shader_bin_ref(bin
);
539 _mesa_hash_table_insert(dst
->cache
, anv_shader_bin_get_key(bin
), bin
);