2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 #include "pipe/p_defines.h"
27 #include "pipe/p_state.h"
28 #include "pipe/p_context.h"
29 #include "pipe/p_screen.h"
30 #include "util/u_atomic.h"
31 #include "compiler/nir/nir.h"
32 #include "compiler/nir/nir_builder.h"
33 #include "intel/compiler/brw_compiler.h"
34 #include "intel/compiler/brw_eu.h"
35 #include "intel/compiler/brw_nir.h"
36 #include "iris_context.h"
40 enum iris_program_cache_id cache_id
;
44 // XXX: put packets here, expose this somehow and simplify the interfaces
45 struct iris_program_cache_item
{
46 uint32_t assembly_offset
;
47 uint32_t assembly_size
;
51 static struct keybox
*
52 make_keybox(struct iris_program_cache
*cache
,
53 enum iris_program_cache_id cache_id
,
54 const void *key
, unsigned key_size
)
56 struct keybox
*keybox
=
57 ralloc_size(cache
->table
, sizeof(struct keybox
) + key_size
);
59 keybox
->cache_id
= cache_id
;
60 keybox
->size
= key_size
;
61 memcpy(keybox
->data
, key
, key_size
);
67 keybox_hash(const void *void_key
)
69 const struct keybox
*key
= void_key
;
70 return _mesa_hash_data(&key
->cache_id
, key
->size
+ sizeof(key
->cache_id
));
74 keybox_equals(const void *void_a
, const void *void_b
)
76 const struct keybox
*a
= void_a
, *b
= void_b
;
77 if (a
->size
!= b
->size
)
80 return memcmp(a
->data
, b
->data
, a
->size
) == 0;
84 get_program_string_id(enum iris_program_cache_id cache_id
, const void *key
)
88 return ((struct brw_vs_prog_key
*) key
)->program_string_id
;
90 return ((struct brw_tcs_prog_key
*) key
)->program_string_id
;
92 return ((struct brw_tes_prog_key
*) key
)->program_string_id
;
94 return ((struct brw_gs_prog_key
*) key
)->program_string_id
;
96 return ((struct brw_cs_prog_key
*) key
)->program_string_id
;
98 return ((struct brw_wm_prog_key
*) key
)->program_string_id
;
100 unreachable("no program string id for this kind of program");
105 * Returns the buffer object matching cache_id and key, or NULL.
107 // XXX: rename to iris_bind_cached_shader?
109 iris_search_cache(struct iris_context
*ice
,
110 enum iris_program_cache_id cache_id
,
114 uint32_t *inout_assembly_offset
,
115 void *inout_prog_data
)
117 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
119 struct keybox
*keybox
= make_keybox(cache
, cache_id
, key
, key_size
);
121 struct hash_entry
*entry
= _mesa_hash_table_search(cache
->table
, keybox
);
126 struct iris_program_cache_item
*item
= entry
->data
;
128 if (item
->assembly_offset
!= *inout_assembly_offset
||
129 item
->prog_data
!= *((void **) inout_prog_data
)) {
130 *inout_assembly_offset
= item
->assembly_offset
;
131 *((void **) inout_prog_data
) = item
->prog_data
;
132 ice
->state
.dirty
|= dirty_flag
;
139 recreate_cache_bo(struct iris_context
*ice
, uint32_t size
)
141 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
142 struct iris_screen
*screen
= (struct iris_screen
*)ice
->ctx
.screen
;
143 struct iris_bo
*old_bo
= cache
->bo
;
144 void *old_map
= cache
->map
;
146 cache
->bo
= iris_bo_alloc(screen
->bufmgr
, "program cache", size
, 64);
147 cache
->bo
->kflags
= EXEC_OBJECT_CAPTURE
;
148 cache
->map
= iris_bo_map(&ice
->dbg
, cache
->bo
,
149 MAP_READ
| MAP_WRITE
| MAP_ASYNC
| MAP_PERSISTENT
);
151 /* Copy any existing data that needs to be saved. */
153 perf_debug(&ice
->dbg
,
154 "Copying to larger program cache: %u kB -> %u kB\n",
155 (unsigned) old_bo
->size
/ 1024,
156 (unsigned) cache
->bo
->size
/ 1024);
158 memcpy(cache
->map
, old_map
, cache
->next_offset
);
160 iris_bo_unreference(old_bo
);
161 iris_bo_unmap(old_bo
);
166 iris_find_previous_compile(struct iris_program_cache
*cache
,
167 enum iris_program_cache_id cache_id
,
168 unsigned program_string_id
)
170 hash_table_foreach(cache
->table
, entry
) {
171 const struct keybox
*keybox
= entry
->key
;
172 if (keybox
->cache_id
== cache_id
&&
173 get_program_string_id(cache_id
, keybox
->data
) == program_string_id
) {
182 * Look for an existing entry in the cache that has identical assembly code.
184 * This is useful for programs generating shaders at runtime, where multiple
185 * distinct shaders (from an API perspective) may compile to the same assembly
188 static const struct iris_program_cache_item
*
189 find_existing_assembly(const struct iris_program_cache
*cache
,
190 const void *assembly
,
191 unsigned assembly_size
)
193 hash_table_foreach(cache
->table
, entry
) {
194 const struct iris_program_cache_item
*item
= entry
->data
;
195 if (item
->assembly_size
== assembly_size
&&
196 memcmp(cache
->map
+ item
->assembly_offset
,
197 assembly
, assembly_size
) == 0)
204 upload_new_assembly(struct iris_context
*ice
,
205 const void *assembly
,
208 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
210 /* Allocate space in the cache BO for our new program. */
211 if (cache
->next_offset
+ size
> cache
->bo
->size
) {
212 uint32_t new_size
= cache
->bo
->size
* 2;
214 while (cache
->next_offset
+ size
> new_size
)
217 recreate_cache_bo(ice
, new_size
);
220 uint32_t offset
= cache
->next_offset
;
222 /* Programs are always 64-byte aligned, so set up the next one now */
223 cache
->next_offset
= ALIGN(offset
+ size
, 64);
225 /* Copy data to the buffer */
226 memcpy(cache
->map
+ offset
, assembly
, size
);
232 iris_upload_cache(struct iris_context
*ice
,
233 enum iris_program_cache_id cache_id
,
236 const void *assembly
,
237 unsigned assembly_size
,
238 const void *prog_data
,
239 unsigned prog_data_size
,
240 uint32_t *out_assembly_offset
,
243 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
244 struct iris_program_cache_item
*item
=
245 ralloc(cache
->table
, struct iris_program_cache_item
);
246 const struct iris_program_cache_item
*matching_data
=
247 find_existing_assembly(cache
, assembly
, assembly_size
);
249 /* If we can find a matching prog in the cache already, then reuse the
250 * existing stuff without creating new copy into the underlying buffer
251 * object. This is notably useful for programs generating shaders at
252 * runtime, where multiple shaders may compile to the same thing in our
256 item
->assembly_offset
= matching_data
->assembly_offset
;
258 item
->assembly_offset
=
259 upload_new_assembly(ice
, assembly
, assembly_size
);
262 item
->assembly_size
= assembly_size
;
263 item
->prog_data
= ralloc_size(item
, prog_data_size
);
264 memcpy(item
->prog_data
, prog_data
, prog_data_size
);
266 if (cache_id
!= IRIS_CACHE_BLORP
) {
267 struct brw_stage_prog_data
*stage_prog_data
= prog_data
;
268 ralloc_steal(item
->prog_data
, stage_prog_data
->param
);
269 ralloc_steal(item
->prog_data
, stage_prog_data
->pull_param
);
272 struct keybox
*keybox
= make_keybox(cache
, cache_id
, key
, key_size
);
273 _mesa_hash_table_insert(cache
->table
, keybox
, item
);
275 *out_assembly_offset
= item
->assembly_offset
;
276 *(void **)out_prog_data
= item
->prog_data
;
280 iris_init_program_cache(struct iris_context
*ice
)
282 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
284 cache
->table
= _mesa_hash_table_create(ice
, keybox_hash
, keybox_equals
);
286 recreate_cache_bo(ice
, 16384);
290 iris_destroy_program_cache(struct iris_context
*ice
)
292 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
294 /* This can be NULL if context creation failed early on. */
296 iris_bo_unmap(cache
->bo
);
297 iris_bo_unreference(cache
->bo
);
302 cache
->next_offset
= 0;
304 /* Also, NULL out any stale program pointers. */
305 for (int i
= 0; i
< MESA_SHADER_STAGES
; i
++) {
306 ice
->shaders
.prog_data
[i
] = NULL
;
311 cache_name(enum iris_program_cache_id cache_id
)
313 if (cache_id
== IRIS_CACHE_BLORP
)
316 return _mesa_shader_stage_to_string(cache_id
);
320 iris_print_program_cache(struct iris_context
*ice
)
322 struct iris_program_cache
*cache
= &ice
->shaders
.cache
;
323 struct iris_screen
*screen
= (struct iris_screen
*)ice
->ctx
.screen
;
324 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
326 hash_table_foreach(cache
->table
, entry
) {
327 const struct keybox
*keybox
= entry
->key
;
328 struct iris_program_cache_item
*item
= entry
->data
;
329 fprintf(stderr
, "%s:\n", cache_name(keybox
->cache_id
));
330 brw_disassemble(devinfo
, cache
->map
,
331 item
->assembly_offset
, item
->assembly_size
, stderr
);