2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "compiler/blob.h"
25 #include "compiler/glsl/ir_uniform.h"
26 #include "compiler/glsl/shader_cache.h"
27 #include "main/mtypes.h"
28 #include "util/build_id.h"
29 #include "util/debug.h"
30 #include "util/disk_cache.h"
31 #include "util/macros.h"
32 #include "util/mesa-sha1.h"
34 #include "compiler/brw_eu.h"
35 #include "dev/gen_debug.h"
37 #include "brw_context.h"
38 #include "brw_program.h"
41 #include "brw_state.h"
46 debug_enabled_for_stage(gl_shader_stage stage
)
48 static const uint64_t stage_debug_flags
[] = {
49 DEBUG_VS
, DEBUG_TCS
, DEBUG_TES
, DEBUG_GS
, DEBUG_WM
, DEBUG_CS
,
51 assert((int)stage
>= 0 && stage
< ARRAY_SIZE(stage_debug_flags
));
52 return (INTEL_DEBUG
& stage_debug_flags
[stage
]) != 0;
56 gen_shader_sha1(struct gl_program
*prog
, gl_shader_stage stage
,
57 void *key
, unsigned char *out_sha1
)
60 unsigned char sha1
[20];
64 _mesa_sha1_format(sha1_buf
, prog
->sh
.data
->sha1
);
65 offset
+= snprintf(manifest
, sizeof(manifest
), "program: %s\n", sha1_buf
);
67 _mesa_sha1_compute(key
, brw_prog_key_size(stage
), sha1
);
68 _mesa_sha1_format(sha1_buf
, sha1
);
69 offset
+= snprintf(manifest
+ offset
, sizeof(manifest
) - offset
,
70 "%s_key: %s\n", _mesa_shader_stage_to_abbrev(stage
),
73 _mesa_sha1_compute(manifest
, strlen(manifest
), out_sha1
);
77 read_blob_program_data(struct blob_reader
*binary
, struct gl_program
*prog
,
78 gl_shader_stage stage
, const uint8_t **program
,
79 struct brw_stage_prog_data
*prog_data
)
82 brw_read_blob_program_data(binary
, prog
, stage
, program
, prog_data
) &&
83 (binary
->current
== binary
->end
);
87 read_and_upload(struct brw_context
*brw
, struct disk_cache
*cache
,
88 struct gl_program
*prog
, gl_shader_stage stage
)
90 unsigned char binary_sha1
[20];
92 union brw_any_prog_key prog_key
;
95 case MESA_SHADER_VERTEX
:
96 brw_vs_populate_key(brw
, &prog_key
.vs
);
98 case MESA_SHADER_TESS_CTRL
:
99 brw_tcs_populate_key(brw
, &prog_key
.tcs
);
101 case MESA_SHADER_TESS_EVAL
:
102 brw_tes_populate_key(brw
, &prog_key
.tes
);
104 case MESA_SHADER_GEOMETRY
:
105 brw_gs_populate_key(brw
, &prog_key
.gs
);
107 case MESA_SHADER_FRAGMENT
:
108 brw_wm_populate_key(brw
, &prog_key
.wm
);
110 case MESA_SHADER_COMPUTE
:
111 brw_cs_populate_key(brw
, &prog_key
.cs
);
114 unreachable("Unsupported stage!");
117 /* We don't care what instance of the program it is for the disk cache hash
118 * lookup, so set the id to 0 for the sha1 hashing. program_string_id will
121 prog_key
.base
.program_string_id
= 0;
123 gen_shader_sha1(prog
, stage
, &prog_key
, binary_sha1
);
126 uint8_t *buffer
= disk_cache_get(cache
, binary_sha1
, &buffer_size
);
127 if (buffer
== NULL
) {
128 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_INFO
) {
130 _mesa_sha1_format(sha1_buf
, binary_sha1
);
131 fprintf(stderr
, "No cached %s binary found for: %s\n",
132 _mesa_shader_stage_to_abbrev(stage
), sha1_buf
);
137 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_INFO
) {
139 _mesa_sha1_format(sha1_buf
, binary_sha1
);
140 fprintf(stderr
, "attempting to populate bo cache with binary: %s\n",
144 struct blob_reader binary
;
145 blob_reader_init(&binary
, buffer
, buffer_size
);
147 const uint8_t *program
;
148 struct brw_stage_prog_data
*prog_data
=
149 ralloc_size(NULL
, sizeof(union brw_any_prog_data
));
150 if (!read_blob_program_data(&binary
, prog
, stage
, &program
, prog_data
)) {
151 /* Something very bad has gone wrong discard the item from the cache and
152 * rebuild from source.
154 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_INFO
) {
155 fprintf(stderr
, "Error reading program from cache (invalid i965 "
159 disk_cache_remove(cache
, binary_sha1
);
160 ralloc_free(prog_data
);
165 enum brw_cache_id cache_id
;
166 struct brw_stage_state
*stage_state
;
169 case MESA_SHADER_VERTEX
:
170 cache_id
= BRW_CACHE_VS_PROG
;
171 stage_state
= &brw
->vs
.base
;
173 case MESA_SHADER_TESS_CTRL
:
174 cache_id
= BRW_CACHE_TCS_PROG
;
175 stage_state
= &brw
->tcs
.base
;
177 case MESA_SHADER_TESS_EVAL
:
178 cache_id
= BRW_CACHE_TES_PROG
;
179 stage_state
= &brw
->tes
.base
;
181 case MESA_SHADER_GEOMETRY
:
182 cache_id
= BRW_CACHE_GS_PROG
;
183 stage_state
= &brw
->gs
.base
;
185 case MESA_SHADER_FRAGMENT
:
186 cache_id
= BRW_CACHE_FS_PROG
;
187 stage_state
= &brw
->wm
.base
;
189 case MESA_SHADER_COMPUTE
:
190 cache_id
= BRW_CACHE_CS_PROG
;
191 stage_state
= &brw
->cs
.base
;
194 unreachable("Unsupported stage!");
197 prog_key
.base
.program_string_id
= brw_program(prog
)->id
;
199 brw_alloc_stage_scratch(brw
, stage_state
, prog_data
->total_scratch
);
201 if (unlikely(debug_enabled_for_stage(stage
))) {
202 fprintf(stderr
, "NIR for %s program %d loaded from disk shader cache:\n",
203 _mesa_shader_stage_to_abbrev(stage
), brw_program(prog
)->id
);
204 brw_program_deserialize_driver_blob(&brw
->ctx
, prog
, stage
);
205 nir_shader
*nir
= prog
->nir
;
206 nir_print_shader(nir
, stderr
);
207 fprintf(stderr
, "Native code for %s %s shader %s from disk cache:\n",
208 nir
->info
.label
? nir
->info
.label
: "unnamed",
209 _mesa_shader_stage_to_string(nir
->info
.stage
), nir
->info
.name
);
210 brw_disassemble(&brw
->screen
->devinfo
, program
, 0,
211 prog_data
->program_size
, stderr
);
214 brw_upload_cache(&brw
->cache
, cache_id
, &prog_key
, brw_prog_key_size(stage
),
215 program
, prog_data
->program_size
, prog_data
,
216 brw_prog_data_size(stage
), &stage_state
->prog_offset
,
217 &stage_state
->prog_data
);
219 prog
->program_written_to_cache
= true;
221 ralloc_free(prog_data
);
228 brw_disk_cache_upload_program(struct brw_context
*brw
, gl_shader_stage stage
)
230 struct disk_cache
*cache
= brw
->ctx
.Cache
;
234 struct gl_program
*prog
= brw
->ctx
._Shader
->CurrentProgram
[stage
];
238 if (prog
->sh
.data
->spirv
)
241 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_FALLBACK
)
244 if (!read_and_upload(brw
, cache
, prog
, stage
))
247 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_INFO
) {
248 fprintf(stderr
, "read gen program from cache\n");
254 prog
->program_written_to_cache
= false;
255 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_INFO
) {
256 fprintf(stderr
, "falling back to nir %s.\n",
257 _mesa_shader_stage_to_abbrev(prog
->info
.stage
));
260 brw_program_deserialize_driver_blob(&brw
->ctx
, prog
, stage
);
266 write_program_data(struct brw_context
*brw
, struct gl_program
*prog
,
267 void *key
, struct brw_stage_prog_data
*prog_data
,
268 uint32_t prog_offset
, struct disk_cache
*cache
,
269 gl_shader_stage stage
)
274 const void *program_map
= brw
->cache
.map
+ prog_offset
;
275 /* TODO: Improve perf for non-LLC. It would be best to save it at program
276 * generation time when the program is in normal memory accessible with
277 * cache to the CPU. Another easier change would be to use
278 * _mesa_streaming_load_memcpy to read from the program mapped memory. */
279 brw_write_blob_program_data(&binary
, stage
, program_map
, prog_data
);
281 unsigned char sha1
[20];
283 gen_shader_sha1(prog
, stage
, key
, sha1
);
284 _mesa_sha1_format(buf
, sha1
);
285 if (brw
->ctx
._Shader
->Flags
& GLSL_CACHE_INFO
) {
286 fprintf(stderr
, "putting binary in cache: %s\n", buf
);
289 disk_cache_put(cache
, sha1
, binary
.data
, binary
.size
, NULL
);
291 prog
->program_written_to_cache
= true;
292 blob_finish(&binary
);
296 brw_disk_cache_write_render_programs(struct brw_context
*brw
)
298 struct disk_cache
*cache
= brw
->ctx
.Cache
;
302 struct gl_program
*prog
;
303 gl_shader_stage stage
;
304 for (stage
= MESA_SHADER_VERTEX
; stage
<= MESA_SHADER_FRAGMENT
; stage
++) {
305 prog
= brw
->ctx
._Shader
->CurrentProgram
[stage
];
306 if (prog
&& prog
->sh
.data
->spirv
)
310 prog
= brw
->ctx
._Shader
->CurrentProgram
[MESA_SHADER_VERTEX
];
311 if (prog
&& !prog
->program_written_to_cache
) {
312 struct brw_vs_prog_key vs_key
;
313 brw_vs_populate_key(brw
, &vs_key
);
314 vs_key
.base
.program_string_id
= 0;
316 write_program_data(brw
, prog
, &vs_key
, brw
->vs
.base
.prog_data
,
317 brw
->vs
.base
.prog_offset
, cache
,
321 prog
= brw
->ctx
._Shader
->CurrentProgram
[MESA_SHADER_TESS_CTRL
];
322 if (prog
&& !prog
->program_written_to_cache
) {
323 struct brw_tcs_prog_key tcs_key
;
324 brw_tcs_populate_key(brw
, &tcs_key
);
325 tcs_key
.base
.program_string_id
= 0;
327 write_program_data(brw
, prog
, &tcs_key
, brw
->tcs
.base
.prog_data
,
328 brw
->tcs
.base
.prog_offset
, cache
,
329 MESA_SHADER_TESS_CTRL
);
332 prog
= brw
->ctx
._Shader
->CurrentProgram
[MESA_SHADER_TESS_EVAL
];
333 if (prog
&& !prog
->program_written_to_cache
) {
334 struct brw_tes_prog_key tes_key
;
335 brw_tes_populate_key(brw
, &tes_key
);
336 tes_key
.base
.program_string_id
= 0;
338 write_program_data(brw
, prog
, &tes_key
, brw
->tes
.base
.prog_data
,
339 brw
->tes
.base
.prog_offset
, cache
,
340 MESA_SHADER_TESS_EVAL
);
343 prog
= brw
->ctx
._Shader
->CurrentProgram
[MESA_SHADER_GEOMETRY
];
344 if (prog
&& !prog
->program_written_to_cache
) {
345 struct brw_gs_prog_key gs_key
;
346 brw_gs_populate_key(brw
, &gs_key
);
347 gs_key
.base
.program_string_id
= 0;
349 write_program_data(brw
, prog
, &gs_key
, brw
->gs
.base
.prog_data
,
350 brw
->gs
.base
.prog_offset
, cache
,
351 MESA_SHADER_GEOMETRY
);
354 prog
= brw
->ctx
._Shader
->CurrentProgram
[MESA_SHADER_FRAGMENT
];
355 if (prog
&& !prog
->program_written_to_cache
) {
356 struct brw_wm_prog_key wm_key
;
357 brw_wm_populate_key(brw
, &wm_key
);
358 wm_key
.base
.program_string_id
= 0;
360 write_program_data(brw
, prog
, &wm_key
, brw
->wm
.base
.prog_data
,
361 brw
->wm
.base
.prog_offset
, cache
,
362 MESA_SHADER_FRAGMENT
);
367 brw_disk_cache_write_compute_program(struct brw_context
*brw
)
369 struct disk_cache
*cache
= brw
->ctx
.Cache
;
373 struct gl_program
*prog
=
374 brw
->ctx
._Shader
->CurrentProgram
[MESA_SHADER_COMPUTE
];
376 if (prog
&& prog
->sh
.data
->spirv
)
379 if (prog
&& !prog
->program_written_to_cache
) {
380 struct brw_cs_prog_key cs_key
;
381 brw_cs_populate_key(brw
, &cs_key
);
382 cs_key
.base
.program_string_id
= 0;
384 write_program_data(brw
, prog
, &cs_key
, brw
->cs
.base
.prog_data
,
385 brw
->cs
.base
.prog_offset
, cache
,
386 MESA_SHADER_COMPUTE
);
391 brw_disk_cache_init(struct intel_screen
*screen
)
393 #ifdef ENABLE_SHADER_CACHE
394 if (INTEL_DEBUG
& DEBUG_DISK_CACHE_DISABLE_MASK
)
397 /* array length: print length + null char + 1 extra to verify it is unused */
399 MAYBE_UNUSED
int len
= snprintf(renderer
, sizeof(renderer
), "i965_%04x",
401 assert(len
== sizeof(renderer
) - 2);
403 const struct build_id_note
*note
=
404 build_id_find_nhdr_for_addr(brw_disk_cache_init
);
405 assert(note
&& build_id_length(note
) == 20 /* sha1 */);
407 const uint8_t *id_sha1
= build_id_data(note
);
411 _mesa_sha1_format(timestamp
, id_sha1
);
413 const uint64_t driver_flags
=
414 brw_get_compiler_config_value(screen
->compiler
);
415 screen
->disk_cache
= disk_cache_create(renderer
, timestamp
, driver_flags
);