uint32_t next_offset;
};
+struct iris_compiled_shader {
+ /** Offset to the start of the assembly in the program cache BO */
+ uint32_t prog_offset;
+
+ /** The program data (owned by the program cache hash table) */
+ struct brw_stage_prog_data *prog_data;
+
+ /**
+ * Shader packets and other data derived from prog_data. These must be
+ * completely determined from prog_data.
+ */
+ uint8_t derived_data[0];
+};
+
struct iris_context {
struct pipe_context ctx;
struct pipe_debug_callback dbg;
struct {
- struct iris_uncompiled_shader *progs[MESA_SHADER_STAGES];
- struct brw_stage_prog_data *prog_data[MESA_SHADER_STAGES];
- uint32_t prog_offset[MESA_SHADER_STAGES];
+ struct iris_uncompiled_shader *uncompiled[MESA_SHADER_STAGES];
+ struct iris_compiled_shader *prog[MESA_SHADER_STAGES];
struct brw_vue_map *last_vue_map;
struct iris_program_cache cache;
IRIS_CACHE_GS = MESA_SHADER_GEOMETRY,
IRIS_CACHE_FS = MESA_SHADER_FRAGMENT,
IRIS_CACHE_CS = MESA_SHADER_COMPUTE,
- IRIS_CACHE_BLORP,
+ IRIS_CACHE_BLORP_BLIT,
};
void iris_init_state(struct iris_context *ice);
void iris_init_program_cache(struct iris_context *ice);
void iris_destroy_program_cache(struct iris_context *ice);
void iris_print_program_cache(struct iris_context *ice);
-bool iris_search_cache(struct iris_context *ice,
- enum iris_program_cache_id cache_id,
- const void *key,
- unsigned key_size,
- uint64_t dirty_flag,
- uint32_t *inout_assembly_offset,
- void *inout_prog_data);
-void iris_upload_cache(struct iris_context *ice,
- enum iris_program_cache_id cache_id,
- const void *key,
- unsigned key_size,
- const void *assembly,
- unsigned assembly_size,
- const void *prog_data,
- unsigned prog_data_size,
- uint32_t *out_assembly_offset,
- void *out_prog_data);
+bool iris_bind_cached_shader(struct iris_context *ice,
+ enum iris_program_cache_id cache_id,
+ const void *key);
+void iris_upload_and_bind_shader(struct iris_context *ice,
+ enum iris_program_cache_id cache_id,
+ const void *key,
+ const void *assembly,
+ const struct brw_stage_prog_data *prog_data);
const void *iris_find_previous_compile(struct iris_program_cache *cache,
enum iris_program_cache_id cache_id,
unsigned program_string_id);
{
struct iris_context *ice = (struct iris_context *)ctx;
- ice->shaders.progs[MESA_SHADER_VERTEX] = hwcso;
+ ice->shaders.uncompiled[MESA_SHADER_VERTEX] = hwcso;
ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_VS;
}
{
struct iris_context *ice = (struct iris_context *)ctx;
- ice->shaders.progs[MESA_SHADER_TESS_CTRL] = hwcso;
+ ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL] = hwcso;
ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_TCS;
}
{
struct iris_context *ice = (struct iris_context *)ctx;
- if (!!hwcso != !!ice->shaders.progs[MESA_SHADER_TESS_EVAL])
+ if (!!hwcso != !!ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL])
ice->state.dirty |= IRIS_DIRTY_URB;
- ice->shaders.progs[MESA_SHADER_TESS_EVAL] = hwcso;
+ ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL] = hwcso;
ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_TES;
}
{
struct iris_context *ice = (struct iris_context *)ctx;
- if (!!hwcso != !!ice->shaders.progs[MESA_SHADER_GEOMETRY])
+ if (!!hwcso != !!ice->shaders.uncompiled[MESA_SHADER_GEOMETRY])
ice->state.dirty |= IRIS_DIRTY_URB;
- ice->shaders.progs[MESA_SHADER_GEOMETRY] = hwcso;
+ ice->shaders.uncompiled[MESA_SHADER_GEOMETRY] = hwcso;
ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_GS;
}
{
struct iris_context *ice = (struct iris_context *)ctx;
- ice->shaders.progs[MESA_SHADER_FRAGMENT] = hwcso;
+ ice->shaders.uncompiled[MESA_SHADER_FRAGMENT] = hwcso;
ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_FS;
}
struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
const struct brw_compiler *compiler = screen->compiler;
const struct gen_device_info *devinfo = &screen->devinfo;
- const unsigned *program;
- struct brw_vs_prog_data vs_prog_data;
- struct brw_stage_prog_data *prog_data = &vs_prog_data.base.base;
void *mem_ctx = ralloc_context(NULL);
+ struct brw_vs_prog_data *vs_prog_data =
+ rzalloc(mem_ctx, struct brw_vs_prog_data);
+ struct brw_vue_prog_data *vue_prog_data = &vs_prog_data->base;
+ struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
assert(ish->base.type == PIPE_SHADER_IR_NIR);
nir_shader *nir = ish->base.ir.nir;
- memset(&vs_prog_data, 0, sizeof(vs_prog_data));
-
// XXX: alt mode
assign_common_binding_table_offsets(devinfo, &nir->info, prog_data, 0);
brw_compute_vue_map(devinfo,
- &vs_prog_data.base.vue_map, nir->info.outputs_written,
+ &vue_prog_data->vue_map, nir->info.outputs_written,
nir->info.separate_shader);
char *error_str = NULL;
- program = brw_compile_vs(compiler, &ice->dbg, mem_ctx, key, &vs_prog_data,
- nir, -1, &error_str);
+ const unsigned *program =
+ brw_compile_vs(compiler, &ice->dbg, mem_ctx, key, vs_prog_data,
+ nir, -1, &error_str);
if (program == NULL) {
dbg_printf("Failed to compile vertex shader: %s\n", error_str);
-
ralloc_free(mem_ctx);
return false;
}
- /* The param and pull_param arrays will be freed by the shader cache. */
- ralloc_steal(NULL, prog_data->param);
- ralloc_steal(NULL, prog_data->pull_param);
- iris_upload_cache(ice, IRIS_CACHE_VS, key, sizeof(*key), program,
- prog_data->program_size, prog_data, sizeof(vs_prog_data),
- &ice->shaders.prog_offset[MESA_SHADER_VERTEX],
- &ice->shaders.prog_data[MESA_SHADER_VERTEX]);
- ralloc_free(mem_ctx);
+ iris_upload_and_bind_shader(ice, IRIS_CACHE_VS, key, program, prog_data);
+ ralloc_free(mem_ctx);
return true;
}
struct brw_vs_prog_key key;
iris_populate_vs_key(ice, &key);
- if (iris_search_cache(ice, IRIS_CACHE_VS, &key, sizeof(key), IRIS_DIRTY_VS,
- &ice->shaders.prog_offset[MESA_SHADER_VERTEX],
- &ice->shaders.prog_data[MESA_SHADER_VERTEX]))
+ if (iris_bind_cached_shader(ice, IRIS_CACHE_VS, &key))
return;
UNUSED bool success =
- iris_compile_vs(ice, ice->shaders.progs[MESA_SHADER_VERTEX], &key);
+ iris_compile_vs(ice, ice->shaders.uncompiled[MESA_SHADER_VERTEX], &key);
}
static void
struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
const struct brw_compiler *compiler = screen->compiler;
const struct gen_device_info *devinfo = &screen->devinfo;
- const unsigned *program;
- struct brw_wm_prog_data fs_prog_data;
- struct brw_stage_prog_data *prog_data = &fs_prog_data.base;
void *mem_ctx = ralloc_context(NULL);
+ struct brw_wm_prog_data *fs_prog_data =
+ rzalloc(mem_ctx, struct brw_wm_prog_data);
+ struct brw_stage_prog_data *prog_data = &fs_prog_data->base;
assert(ish->base.type == PIPE_SHADER_IR_NIR);
nir_shader *nir = ish->base.ir.nir;
- memset(&fs_prog_data, 0, sizeof(fs_prog_data));
-
// XXX: alt mode
assign_common_binding_table_offsets(devinfo, &nir->info, prog_data,
MAX2(key->nr_color_regions, 1));
char *error_str = NULL;
- program = brw_compile_fs(compiler, &ice->dbg, mem_ctx, key, &fs_prog_data,
- nir, NULL, -1, -1, -1, true, false, vue_map,
- &error_str);
+ const unsigned *program =
+ brw_compile_fs(compiler, &ice->dbg, mem_ctx, key, fs_prog_data,
+ nir, NULL, -1, -1, -1, true, false, vue_map, &error_str);
if (program == NULL) {
dbg_printf("Failed to compile fragment shader: %s\n", error_str);
-
ralloc_free(mem_ctx);
return false;
}
//brw_alloc_stage_scratch(brw, &brw->wm.base, prog_data.base.total_scratch);
- /* The param and pull_param arrays will be freed by the shader cache. */
- ralloc_steal(NULL, prog_data->param);
- ralloc_steal(NULL, prog_data->pull_param);
- #if 0
- brw_upload_cache(&brw->cache, BRW_CACHE_FS_PROG,
- key, sizeof(struct brw_wm_prog_key),
- program, prog_data.base.program_size,
- &prog_data, sizeof(prog_data),
- &brw->wm.base.prog_offset, &brw->wm.base.prog_data);
- #endif
+ iris_upload_and_bind_shader(ice, IRIS_CACHE_FS, key, program, prog_data);
ralloc_free(mem_ctx);
-
return true;
}
struct brw_wm_prog_key key;
iris_populate_fs_key(ice, &key);
- if (iris_search_cache(ice, IRIS_CACHE_FS, &key, sizeof(key), IRIS_DIRTY_FS,
- &ice->shaders.prog_offset[MESA_SHADER_FRAGMENT],
- &ice->shaders.prog_data[MESA_SHADER_FRAGMENT]))
+ if (iris_bind_cached_shader(ice, IRIS_CACHE_FS, &key))
return;
UNUSED bool success =
- iris_compile_fs(ice, ice->shaders.progs[MESA_SHADER_FRAGMENT], &key,
+ iris_compile_fs(ice, ice->shaders.uncompiled[MESA_SHADER_FRAGMENT], &key,
ice->shaders.last_vue_map);
}
{
struct brw_stage_prog_data *prog_data;
- if (ice->shaders.progs[MESA_SHADER_GEOMETRY])
- prog_data = ice->shaders.prog_data[MESA_SHADER_GEOMETRY];
- else if (ice->shaders.progs[MESA_SHADER_TESS_EVAL])
- prog_data = ice->shaders.prog_data[MESA_SHADER_TESS_EVAL];
+ if (ice->shaders.prog[MESA_SHADER_GEOMETRY])
+ prog_data = ice->shaders.prog[MESA_SHADER_GEOMETRY]->prog_data;
+ else if (ice->shaders.prog[MESA_SHADER_TESS_EVAL])
+ prog_data = ice->shaders.prog[MESA_SHADER_TESS_EVAL]->prog_data;
else
- prog_data = ice->shaders.prog_data[MESA_SHADER_VERTEX];
+ prog_data = ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data;
struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
ice->shaders.last_vue_map = &vue_prog_data->vue_map;
}
+static struct brw_vue_prog_data *
+get_vue_prog_data(struct iris_context *ice, gl_shader_stage stage)
+{
+ if (!ice->shaders.prog[stage])
+ return NULL;
+
+ return (void *) ice->shaders.prog[stage]->prog_data;
+}
+
void
iris_update_compiled_shaders(struct iris_context *ice)
{
struct brw_vue_prog_data *old_prog_datas[4];
if (!(ice->state.dirty & IRIS_DIRTY_URB)) {
for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++)
- old_prog_datas[i] = (void *) ice->shaders.prog_data[i];
+ old_prog_datas[i] = get_vue_prog_data(ice, i);
}
iris_update_compiled_vs(ice);
if (!(ice->state.dirty & IRIS_DIRTY_URB)) {
for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
struct brw_vue_prog_data *old = old_prog_datas[i];
- struct brw_vue_prog_data *new = (void *) ice->shaders.prog_data[i];
+ struct brw_vue_prog_data *new = get_vue_prog_data(ice, i);
if (!!old != !!new ||
(new && new->urb_entry_size != old->urb_entry_size)) {
ice->state.dirty |= IRIS_DIRTY_URB;
uint8_t data[0];
};
-// XXX: put packets here, expose this somehow and simplify the interfaces
-struct iris_program_cache_item {
- uint32_t assembly_offset;
- uint32_t assembly_size;
- void *prog_data;
-};
-
static struct keybox *
make_keybox(struct iris_program_cache *cache,
enum iris_program_cache_id cache_id,
- const void *key, unsigned key_size)
+ const void *key)
{
+ static const unsigned key_sizes[] = {
+ [IRIS_CACHE_VS] = sizeof(struct brw_vs_prog_key),
+ [IRIS_CACHE_TCS] = sizeof(struct brw_tcs_prog_key),
+ [IRIS_CACHE_TES] = sizeof(struct brw_tes_prog_key),
+ [IRIS_CACHE_GS] = sizeof(struct brw_gs_prog_key),
+ [IRIS_CACHE_FS] = sizeof(struct brw_wm_prog_key),
+ [IRIS_CACHE_CS] = sizeof(struct brw_cs_prog_key),
+ //[IRIS_CACHE_BLORP_BLIT] = sizeof(struct brw_blorp_blit_prog_key),
+ };
+
struct keybox *keybox =
- ralloc_size(cache->table, sizeof(struct keybox) + key_size);
+ ralloc_size(cache->table, sizeof(struct keybox) + key_sizes[cache_id]);
keybox->cache_id = cache_id;
- keybox->size = key_size;
- memcpy(keybox->data, key, key_size);
+ keybox->size = key_sizes[cache_id];
+ memcpy(keybox->data, key, key_sizes[cache_id]);
return keybox;
}
return memcmp(a->data, b->data, a->size) == 0;
}
+static uint64_t
+dirty_flag_for_cache(enum iris_program_cache_id cache_id)
+{
+ assert(cache_id <= MESA_SHADER_STAGES);
+ return IRIS_DIRTY_VS << cache_id;
+}
+
static unsigned
get_program_string_id(enum iris_program_cache_id cache_id, const void *key)
{
}
/**
- * Returns the buffer object matching cache_id and key, or NULL.
+ * Looks for a program in the cache and binds it.
+ *
+ * If no program was found, returns false and leaves the binding alone.
*/
-// XXX: rename to iris_bind_cached_shader?
bool
-iris_search_cache(struct iris_context *ice,
- enum iris_program_cache_id cache_id,
- const void *key,
- unsigned key_size,
- uint64_t dirty_flag,
- uint32_t *inout_assembly_offset,
- void *inout_prog_data)
+iris_bind_cached_shader(struct iris_context *ice,
+ enum iris_program_cache_id cache_id,
+ const void *key)
{
struct iris_program_cache *cache = &ice->shaders.cache;
- struct keybox *keybox = make_keybox(cache, cache_id, key, key_size);
+ struct keybox *keybox = make_keybox(cache, cache_id, key);
struct hash_entry *entry = _mesa_hash_table_search(cache->table, keybox);
- if (entry == NULL)
+ if (!entry)
return false;
- struct iris_program_cache_item *item = entry->data;
+ struct iris_compiled_shader *shader = entry->data;
- if (item->assembly_offset != *inout_assembly_offset ||
- item->prog_data != *((void **) inout_prog_data)) {
- *inout_assembly_offset = item->assembly_offset;
- *((void **) inout_prog_data) = item->prog_data;
- ice->state.dirty |= dirty_flag;
+ if (cache_id <= MESA_SHADER_STAGES &&
+ memcmp(shader, ice->shaders.prog[cache_id], sizeof(*shader)) != 0) {
+ ice->shaders.prog[cache_id] = shader;
+ ice->state.dirty |= dirty_flag_for_cache(cache_id);
}
return true;
*
* This is useful for programs generating shaders at runtime, where multiple
* distinct shaders (from an API perspective) may compile to the same assembly
- * in our backend.
+ * in our backend. This saves space in the program cache buffer.
*/
-static const struct iris_program_cache_item *
+static const struct iris_compiled_shader *
find_existing_assembly(const struct iris_program_cache *cache,
const void *assembly,
unsigned assembly_size)
{
hash_table_foreach(cache->table, entry) {
- const struct iris_program_cache_item *item = entry->data;
- if (item->assembly_size == assembly_size &&
- memcmp(cache->map + item->assembly_offset,
+ const struct iris_compiled_shader *existing = entry->data;
+ if (existing->prog_data->program_size == assembly_size &&
+ memcmp(cache->map + existing->prog_offset,
assembly, assembly_size) == 0)
- return item;
+ return existing;
}
return NULL;
}
return offset;
}
+/**
+ * Upload a new shader to the program cache, and bind it for use.
+ *
+ * \param prog_data must be ralloc'd and will be stolen.
+ */
void
-iris_upload_cache(struct iris_context *ice,
- enum iris_program_cache_id cache_id,
- const void *key,
- unsigned key_size,
- const void *assembly,
- unsigned assembly_size,
- const void *prog_data,
- unsigned prog_data_size,
- uint32_t *out_assembly_offset,
- void *out_prog_data)
+iris_upload_and_bind_shader(struct iris_context *ice,
+ enum iris_program_cache_id cache_id,
+ const void *key,
+ const void *assembly,
+ const struct brw_stage_prog_data *prog_data)
{
struct iris_program_cache *cache = &ice->shaders.cache;
- struct iris_program_cache_item *item =
- ralloc(cache->table, struct iris_program_cache_item);
- const struct iris_program_cache_item *matching_data =
- find_existing_assembly(cache, assembly, assembly_size);
+ struct iris_compiled_shader *shader =
+ ralloc(cache->table, struct iris_compiled_shader);
+ const struct iris_compiled_shader *existing =
+ find_existing_assembly(cache, assembly, prog_data->program_size);
/* If we can find a matching prog in the cache already, then reuse the
* existing stuff without creating new copy into the underlying buffer
* runtime, where multiple shaders may compile to the same thing in our
* backend.
*/
- if (matching_data) {
- item->assembly_offset = matching_data->assembly_offset;
+ if (existing) {
+ shader->prog_offset = existing->prog_offset;
} else {
- item->assembly_offset =
- upload_new_assembly(ice, assembly, assembly_size);
+ shader->prog_offset =
+ upload_new_assembly(ice, assembly, prog_data->program_size);
}
- item->assembly_size = assembly_size;
- item->prog_data = ralloc_size(item, prog_data_size);
- memcpy(item->prog_data, prog_data, prog_data_size);
+ shader->prog_data = prog_data;
- if (cache_id != IRIS_CACHE_BLORP) {
- struct brw_stage_prog_data *stage_prog_data = prog_data;
- ralloc_steal(item->prog_data, stage_prog_data->param);
- ralloc_steal(item->prog_data, stage_prog_data->pull_param);
- }
+ ralloc_steal(shader, shader->prog_data);
+ ralloc_steal(shader->prog_data, prog_data->param);
+ ralloc_steal(shader->prog_data, prog_data->pull_param);
- struct keybox *keybox = make_keybox(cache, cache_id, key, key_size);
- _mesa_hash_table_insert(cache->table, keybox, item);
+ struct keybox *keybox = make_keybox(cache, cache_id, key);
+ _mesa_hash_table_insert(cache->table, keybox, shader);
- *out_assembly_offset = item->assembly_offset;
- *(void **)out_prog_data = item->prog_data;
+ if (cache_id <= MESA_SHADER_STAGES) {
+ ice->shaders.prog[cache_id] = shader;
+ ice->state.dirty |= dirty_flag_for_cache(cache_id);
+ }
}
void
cache->next_offset = 0;
- /* Also, NULL out any stale program pointers. */
for (int i = 0; i < MESA_SHADER_STAGES; i++) {
- ice->shaders.prog_data[i] = NULL;
+ ice->shaders.prog[i] = NULL;
}
+
+ ralloc_free(cache->table);
}
static const char *
cache_name(enum iris_program_cache_id cache_id)
{
- if (cache_id == IRIS_CACHE_BLORP)
+ if (cache_id == IRIS_CACHE_BLORP_BLIT)
return "BLORP";
return _mesa_shader_stage_to_string(cache_id);
hash_table_foreach(cache->table, entry) {
const struct keybox *keybox = entry->key;
- struct iris_program_cache_item *item = entry->data;
+ struct iris_compiled_shader *shader = entry->data;
fprintf(stderr, "%s:\n", cache_name(keybox->cache_id));
- brw_disassemble(devinfo, cache->map,
- item->assembly_offset, item->assembly_size, stderr);
+ brw_disassemble(devinfo, cache->map, shader->prog_offset,
+ shader->prog_data->program_size, stderr);
}
}
projects / mesa.git / commitdiff