2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
24 * @file iris_program_cache.c
26 * The in-memory program cache. This is basically a hash table mapping
27 * API-specified shaders and a state key to a compiled variant. It also
28 * takes care of uploading shader assembly into a BO for use on the GPU.
33 #include "pipe/p_defines.h"
34 #include "pipe/p_state.h"
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/u_atomic.h"
38 #include "util/u_upload_mgr.h"
39 #include "compiler/nir/nir.h"
40 #include "compiler/nir/nir_builder.h"
41 #include "intel/common/gen_disasm.h"
42 #include "intel/compiler/brw_compiler.h"
43 #include "intel/compiler/brw_eu.h"
44 #include "intel/compiler/brw_nir.h"
45 #include "iris_context.h"
46 #include "iris_resource.h"
50 enum iris_program_cache_id cache_id
;
54 static struct keybox
*
55 make_keybox(void *mem_ctx
,
56 enum iris_program_cache_id cache_id
,
60 struct keybox
*keybox
=
61 ralloc_size(mem_ctx
, sizeof(struct keybox
) + key_size
);
63 keybox
->cache_id
= cache_id
;
64 keybox
->size
= key_size
;
65 memcpy(keybox
->data
, key
, key_size
);
71 keybox_hash(const void *void_key
)
73 const struct keybox
*key
= void_key
;
74 return _mesa_hash_data(&key
->cache_id
, key
->size
+ sizeof(key
->cache_id
));
78 keybox_equals(const void *void_a
, const void *void_b
)
80 const struct keybox
*a
= void_a
, *b
= void_b
;
81 if (a
->size
!= b
->size
)
84 return memcmp(a
->data
, b
->data
, a
->size
) == 0;
87 struct iris_compiled_shader
*
88 iris_find_cached_shader(struct iris_context
*ice
,
89 enum iris_program_cache_id cache_id
,
93 struct keybox
*keybox
= make_keybox(NULL
, cache_id
, key
, key_size
);
94 struct hash_entry
*entry
=
95 _mesa_hash_table_search(ice
->shaders
.cache
, keybox
);
99 return entry
? entry
->data
: NULL
;
103 iris_find_previous_compile(const struct iris_context
*ice
,
104 enum iris_program_cache_id cache_id
,
105 unsigned program_string_id
)
107 hash_table_foreach(ice
->shaders
.cache
, entry
) {
108 const struct keybox
*keybox
= entry
->key
;
109 const struct brw_base_prog_key
*key
= (const void *)keybox
->data
;
110 if (keybox
->cache_id
== cache_id
&&
111 key
->program_string_id
== program_string_id
) {
120 iris_delete_shader_variants(struct iris_context
*ice
,
121 struct iris_uncompiled_shader
*ish
)
123 struct hash_table
*cache
= ice
->shaders
.cache
;
124 gl_shader_stage stage
= ish
->nir
->info
.stage
;
125 enum iris_program_cache_id cache_id
= stage
;
127 hash_table_foreach(cache
, entry
) {
128 const struct keybox
*keybox
= entry
->key
;
129 const struct brw_base_prog_key
*key
= (const void *)keybox
->data
;
131 if (keybox
->cache_id
== cache_id
&&
132 key
->program_string_id
== ish
->program_id
) {
133 struct iris_compiled_shader
*shader
= entry
->data
;
135 _mesa_hash_table_remove(cache
, entry
);
137 /* Shader variants may still be bound in the context even after
138 * the API-facing shader has been deleted. In particular, a draw
139 * may not have triggered iris_update_compiled_shaders() yet. In
140 * that case, we may be referring to that shader's VUE map, stream
141 * output settings, and so on. We also like to compare the old and
142 * new shader programs when swapping them out to flag dirty state.
144 * So, it's hazardous to delete a bound shader variant. We avoid
145 * doing so, choosing to instead move "deleted" shader variants to
146 * a list, deferring the actual deletion until they're not bound.
148 * For simplicity, we always move deleted variants to the list,
149 * even if we could delete them immediately. We'll then process
150 * the list, catching both these variants and any others.
152 list_addtail(&shader
->link
, &ice
->shaders
.deleted_variants
[stage
]);
156 /* Process any pending deferred variant deletions. */
157 list_for_each_entry_safe(struct iris_compiled_shader
, shader
,
158 &ice
->shaders
.deleted_variants
[stage
], link
) {
159 /* If the shader is still bound, defer deletion. */
160 if (ice
->shaders
.prog
[stage
] == shader
)
163 list_del(&shader
->link
);
165 /* Actually delete the variant. */
166 pipe_resource_reference(&shader
->assembly
.res
, NULL
);
173 * Look for an existing entry in the cache that has identical assembly code.
175 * This is useful for programs generating shaders at runtime, where multiple
176 * distinct shaders (from an API perspective) may compile to the same assembly
177 * in our backend. This saves space in the program cache buffer.
179 static const struct iris_compiled_shader
*
180 find_existing_assembly(struct hash_table
*cache
,
181 const void *assembly
,
182 unsigned assembly_size
)
184 hash_table_foreach(cache
, entry
) {
185 const struct iris_compiled_shader
*existing
= entry
->data
;
186 if (existing
->prog_data
->program_size
== assembly_size
&&
187 memcmp(existing
->map
, assembly
, assembly_size
) == 0)
193 struct iris_compiled_shader
*
194 iris_upload_shader(struct iris_context
*ice
,
195 enum iris_program_cache_id cache_id
,
198 const void *assembly
,
199 struct brw_stage_prog_data
*prog_data
,
201 enum brw_param_builtin
*system_values
,
202 unsigned num_system_values
,
203 unsigned kernel_input_size
,
205 const struct iris_binding_table
*bt
)
207 struct hash_table
*cache
= ice
->shaders
.cache
;
208 struct iris_screen
*screen
= (struct iris_screen
*)ice
->ctx
.screen
;
209 struct iris_compiled_shader
*shader
=
210 rzalloc_size(cache
, sizeof(struct iris_compiled_shader
) +
211 screen
->vtbl
.derived_program_state_size(cache_id
));
212 const struct iris_compiled_shader
*existing
=
213 find_existing_assembly(cache
, assembly
, prog_data
->program_size
);
215 /* If we can find a matching prog in the cache already, then reuse the
216 * existing stuff without creating new copy into the underlying buffer
217 * object. This is notably useful for programs generating shaders at
218 * runtime, where multiple shaders may compile to the same thing in our
222 pipe_resource_reference(&shader
->assembly
.res
, existing
->assembly
.res
);
223 shader
->assembly
.offset
= existing
->assembly
.offset
;
224 shader
->map
= existing
->map
;
226 shader
->assembly
.res
= NULL
;
227 u_upload_alloc(ice
->shaders
.uploader
, 0, prog_data
->program_size
, 64,
228 &shader
->assembly
.offset
, &shader
->assembly
.res
,
230 memcpy(shader
->map
, assembly
, prog_data
->program_size
);
232 uint64_t shader_data_addr
= IRIS_MEMZONE_SHADER_START
+
233 shader
->assembly
.offset
+
234 prog_data
->const_data_offset
;
236 struct brw_shader_reloc_value reloc_values
[] = {
238 .id
= IRIS_SHADER_RELOC_CONST_DATA_ADDR_LOW
,
239 .value
= shader_data_addr
,
242 .id
= IRIS_SHADER_RELOC_CONST_DATA_ADDR_HIGH
,
243 .value
= shader_data_addr
>> 32,
246 brw_write_shader_relocs(&screen
->devinfo
, shader
->map
, prog_data
,
247 reloc_values
, ARRAY_SIZE(reloc_values
));
250 list_inithead(&shader
->link
);
252 shader
->prog_data
= prog_data
;
253 shader
->streamout
= streamout
;
254 shader
->system_values
= system_values
;
255 shader
->num_system_values
= num_system_values
;
256 shader
->kernel_input_size
= kernel_input_size
;
257 shader
->num_cbufs
= num_cbufs
;
260 ralloc_steal(shader
, shader
->prog_data
);
261 ralloc_steal(shader
->prog_data
, (void *)prog_data
->relocs
);
262 ralloc_steal(shader
->prog_data
, prog_data
->param
);
263 ralloc_steal(shader
->prog_data
, prog_data
->pull_param
);
264 ralloc_steal(shader
, shader
->streamout
);
265 ralloc_steal(shader
, shader
->system_values
);
267 /* Store the 3DSTATE shader packets and other derived state. */
268 screen
->vtbl
.store_derived_program_state(ice
, cache_id
, shader
);
270 struct keybox
*keybox
= make_keybox(shader
, cache_id
, key
, key_size
);
271 _mesa_hash_table_insert(ice
->shaders
.cache
, keybox
, shader
);
277 iris_blorp_lookup_shader(struct blorp_batch
*blorp_batch
,
278 const void *key
, uint32_t key_size
,
279 uint32_t *kernel_out
, void *prog_data_out
)
281 struct blorp_context
*blorp
= blorp_batch
->blorp
;
282 struct iris_context
*ice
= blorp
->driver_ctx
;
283 struct iris_batch
*batch
= blorp_batch
->driver_batch
;
284 struct iris_compiled_shader
*shader
=
285 iris_find_cached_shader(ice
, IRIS_CACHE_BLORP
, key_size
, key
);
290 struct iris_bo
*bo
= iris_resource_bo(shader
->assembly
.res
);
292 iris_bo_offset_from_base_address(bo
) + shader
->assembly
.offset
;
293 *((void **) prog_data_out
) = shader
->prog_data
;
295 iris_use_pinned_bo(batch
, bo
, false, IRIS_DOMAIN_NONE
);
301 iris_blorp_upload_shader(struct blorp_batch
*blorp_batch
, uint32_t stage
,
302 const void *key
, uint32_t key_size
,
303 const void *kernel
, UNUSED
uint32_t kernel_size
,
304 const struct brw_stage_prog_data
*prog_data_templ
,
305 UNUSED
uint32_t prog_data_size
,
306 uint32_t *kernel_out
, void *prog_data_out
)
308 struct blorp_context
*blorp
= blorp_batch
->blorp
;
309 struct iris_context
*ice
= blorp
->driver_ctx
;
310 struct iris_batch
*batch
= blorp_batch
->driver_batch
;
312 void *prog_data
= ralloc_size(NULL
, prog_data_size
);
313 memcpy(prog_data
, prog_data_templ
, prog_data_size
);
315 struct iris_binding_table bt
;
316 memset(&bt
, 0, sizeof(bt
));
318 struct iris_compiled_shader
*shader
=
319 iris_upload_shader(ice
, IRIS_CACHE_BLORP
, key_size
, key
, kernel
,
320 prog_data
, NULL
, NULL
, 0, 0, 0, &bt
);
322 struct iris_bo
*bo
= iris_resource_bo(shader
->assembly
.res
);
324 iris_bo_offset_from_base_address(bo
) + shader
->assembly
.offset
;
325 *((void **) prog_data_out
) = shader
->prog_data
;
327 iris_use_pinned_bo(batch
, bo
, false, IRIS_DOMAIN_NONE
);
333 iris_init_program_cache(struct iris_context
*ice
)
336 _mesa_hash_table_create(ice
, keybox_hash
, keybox_equals
);
338 ice
->shaders
.uploader
=
339 u_upload_create(&ice
->ctx
, 16384, PIPE_BIND_CUSTOM
, PIPE_USAGE_IMMUTABLE
,
340 IRIS_RESOURCE_FLAG_SHADER_MEMZONE
);
342 for (int i
= 0; i
< MESA_SHADER_STAGES
; i
++)
343 list_inithead(&ice
->shaders
.deleted_variants
[i
]);
347 iris_destroy_program_cache(struct iris_context
*ice
)
349 for (int i
= 0; i
< MESA_SHADER_STAGES
; i
++) {
350 ice
->shaders
.prog
[i
] = NULL
;
352 list_for_each_entry_safe(struct iris_compiled_shader
, shader
,
353 &ice
->shaders
.deleted_variants
[i
], link
) {
354 pipe_resource_reference(&shader
->assembly
.res
, NULL
);
358 hash_table_foreach(ice
->shaders
.cache
, entry
) {
359 struct iris_compiled_shader
*shader
= entry
->data
;
360 pipe_resource_reference(&shader
->assembly
.res
, NULL
);
363 u_upload_destroy(ice
->shaders
.uploader
);
365 ralloc_free(ice
->shaders
.cache
);
369 cache_name(enum iris_program_cache_id cache_id
)
371 if (cache_id
== IRIS_CACHE_BLORP
)
374 return _mesa_shader_stage_to_string(cache_id
);
378 iris_print_program_cache(struct iris_context
*ice
)
380 struct iris_screen
*screen
= (struct iris_screen
*)ice
->ctx
.screen
;
381 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
383 hash_table_foreach(ice
->shaders
.cache
, entry
) {
384 const struct keybox
*keybox
= entry
->key
;
385 struct iris_compiled_shader
*shader
= entry
->data
;
386 fprintf(stderr
, "%s:\n", cache_name(keybox
->cache_id
));
387 gen_disassemble(devinfo
, shader
->map
, 0, stderr
);