From 57c1b71418cc811e467144f07193c4df3343975b Mon Sep 17 00:00:00 2001 From: Kenneth Graunke Date: Mon, 22 Jan 2018 11:52:58 -0800 Subject: [PATCH] iris: promote iris_program_cache_item to iris_compiled_shader --- src/gallium/drivers/iris/iris_context.h | 46 +++--- src/gallium/drivers/iris/iris_program.c | 103 ++++++------- src/gallium/drivers/iris/iris_program_cache.c | 145 +++++++++--------- 3 files changed, 144 insertions(+), 150 deletions(-) diff --git a/src/gallium/drivers/iris/iris_context.h b/src/gallium/drivers/iris/iris_context.h index 022a79a1dc4..aec2e30c68c 100644 --- a/src/gallium/drivers/iris/iris_context.h +++ b/src/gallium/drivers/iris/iris_context.h @@ -84,15 +84,28 @@ struct iris_program_cache { uint32_t next_offset; }; +struct iris_compiled_shader { + /** Offset to the start of the assembly in the program cache BO */ + uint32_t prog_offset; + + /** The program data (owned by the program cache hash table) */ + struct brw_stage_prog_data *prog_data; + + /** + * Shader packets and other data derived from prog_data. These must be + * completely determined from prog_data. + */ + uint8_t derived_data[0]; +}; + struct iris_context { struct pipe_context ctx; struct pipe_debug_callback dbg; struct { - struct iris_uncompiled_shader *progs[MESA_SHADER_STAGES]; - struct brw_stage_prog_data *prog_data[MESA_SHADER_STAGES]; - uint32_t prog_offset[MESA_SHADER_STAGES]; + struct iris_uncompiled_shader *uncompiled[MESA_SHADER_STAGES]; + struct iris_compiled_shader *prog[MESA_SHADER_STAGES]; struct brw_vue_map *last_vue_map; struct iris_program_cache cache; @@ -160,30 +173,21 @@ enum iris_program_cache_id { IRIS_CACHE_GS = MESA_SHADER_GEOMETRY, IRIS_CACHE_FS = MESA_SHADER_FRAGMENT, IRIS_CACHE_CS = MESA_SHADER_COMPUTE, - IRIS_CACHE_BLORP, + IRIS_CACHE_BLORP_BLIT, }; void iris_init_state(struct iris_context *ice); void iris_init_program_cache(struct iris_context *ice); void iris_destroy_program_cache(struct iris_context *ice); void iris_print_program_cache(struct iris_context *ice); -bool iris_search_cache(struct iris_context *ice, - enum iris_program_cache_id cache_id, - const void *key, - unsigned key_size, - uint64_t dirty_flag, - uint32_t *inout_assembly_offset, - void *inout_prog_data); -void iris_upload_cache(struct iris_context *ice, - enum iris_program_cache_id cache_id, - const void *key, - unsigned key_size, - const void *assembly, - unsigned assembly_size, - const void *prog_data, - unsigned prog_data_size, - uint32_t *out_assembly_offset, - void *out_prog_data); +bool iris_bind_cached_shader(struct iris_context *ice, + enum iris_program_cache_id cache_id, + const void *key); +void iris_upload_and_bind_shader(struct iris_context *ice, + enum iris_program_cache_id cache_id, + const void *key, + const void *assembly, + const struct brw_stage_prog_data *prog_data); const void *iris_find_previous_compile(struct iris_program_cache *cache, enum iris_program_cache_id cache_id, unsigned program_string_id); diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c index 4772baea45e..a47af6827ac 100644 --- a/src/gallium/drivers/iris/iris_program.c +++ b/src/gallium/drivers/iris/iris_program.c @@ -90,7 +90,7 @@ iris_bind_vs_state(struct pipe_context *ctx, void *hwcso) { struct iris_context *ice = (struct iris_context *)ctx; - ice->shaders.progs[MESA_SHADER_VERTEX] = hwcso; + ice->shaders.uncompiled[MESA_SHADER_VERTEX] = hwcso; ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_VS; } @@ -99,7 +99,7 @@ iris_bind_tcs_state(struct pipe_context *ctx, void *hwcso) { struct iris_context *ice = (struct iris_context *)ctx; - ice->shaders.progs[MESA_SHADER_TESS_CTRL] = hwcso; + ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL] = hwcso; ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_TCS; } @@ -108,10 +108,10 @@ iris_bind_tes_state(struct pipe_context *ctx, void *hwcso) { struct iris_context *ice = (struct iris_context *)ctx; - if (!!hwcso != !!ice->shaders.progs[MESA_SHADER_TESS_EVAL]) + if (!!hwcso != !!ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL]) ice->state.dirty |= IRIS_DIRTY_URB; - ice->shaders.progs[MESA_SHADER_TESS_EVAL] = hwcso; + ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL] = hwcso; ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_TES; } @@ -120,10 +120,10 @@ iris_bind_gs_state(struct pipe_context *ctx, void *hwcso) { struct iris_context *ice = (struct iris_context *)ctx; - if (!!hwcso != !!ice->shaders.progs[MESA_SHADER_GEOMETRY]) + if (!!hwcso != !!ice->shaders.uncompiled[MESA_SHADER_GEOMETRY]) ice->state.dirty |= IRIS_DIRTY_URB; - ice->shaders.progs[MESA_SHADER_GEOMETRY] = hwcso; + ice->shaders.uncompiled[MESA_SHADER_GEOMETRY] = hwcso; ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_GS; } @@ -132,7 +132,7 @@ iris_bind_fs_state(struct pipe_context *ctx, void *hwcso) { struct iris_context *ice = (struct iris_context *)ctx; - ice->shaders.progs[MESA_SHADER_FRAGMENT] = hwcso; + ice->shaders.uncompiled[MESA_SHADER_FRAGMENT] = hwcso; ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_FS; } @@ -207,42 +207,35 @@ iris_compile_vs(struct iris_context *ice, struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen; const struct brw_compiler *compiler = screen->compiler; const struct gen_device_info *devinfo = &screen->devinfo; - const unsigned *program; - struct brw_vs_prog_data vs_prog_data; - struct brw_stage_prog_data *prog_data = &vs_prog_data.base.base; void *mem_ctx = ralloc_context(NULL); + struct brw_vs_prog_data *vs_prog_data = + rzalloc(mem_ctx, struct brw_vs_prog_data); + struct brw_vue_prog_data *vue_prog_data = &vs_prog_data->base; + struct brw_stage_prog_data *prog_data = &vue_prog_data->base; assert(ish->base.type == PIPE_SHADER_IR_NIR); nir_shader *nir = ish->base.ir.nir; - memset(&vs_prog_data, 0, sizeof(vs_prog_data)); - // XXX: alt mode assign_common_binding_table_offsets(devinfo, &nir->info, prog_data, 0); brw_compute_vue_map(devinfo, - &vs_prog_data.base.vue_map, nir->info.outputs_written, + &vue_prog_data->vue_map, nir->info.outputs_written, nir->info.separate_shader); char *error_str = NULL; - program = brw_compile_vs(compiler, &ice->dbg, mem_ctx, key, &vs_prog_data, - nir, -1, &error_str); + const unsigned *program = + brw_compile_vs(compiler, &ice->dbg, mem_ctx, key, vs_prog_data, + nir, -1, &error_str); if (program == NULL) { dbg_printf("Failed to compile vertex shader: %s\n", error_str); - ralloc_free(mem_ctx); return false; } - /* The param and pull_param arrays will be freed by the shader cache. */ - ralloc_steal(NULL, prog_data->param); - ralloc_steal(NULL, prog_data->pull_param); - iris_upload_cache(ice, IRIS_CACHE_VS, key, sizeof(*key), program, - prog_data->program_size, prog_data, sizeof(vs_prog_data), - &ice->shaders.prog_offset[MESA_SHADER_VERTEX], - &ice->shaders.prog_data[MESA_SHADER_VERTEX]); - ralloc_free(mem_ctx); + iris_upload_and_bind_shader(ice, IRIS_CACHE_VS, key, program, prog_data); + ralloc_free(mem_ctx); return true; } @@ -258,13 +251,11 @@ iris_update_compiled_vs(struct iris_context *ice) struct brw_vs_prog_key key; iris_populate_vs_key(ice, &key); - if (iris_search_cache(ice, IRIS_CACHE_VS, &key, sizeof(key), IRIS_DIRTY_VS, - &ice->shaders.prog_offset[MESA_SHADER_VERTEX], - &ice->shaders.prog_data[MESA_SHADER_VERTEX])) + if (iris_bind_cached_shader(ice, IRIS_CACHE_VS, &key)) return; UNUSED bool success = - iris_compile_vs(ice, ice->shaders.progs[MESA_SHADER_VERTEX], &key); + iris_compile_vs(ice, ice->shaders.uncompiled[MESA_SHADER_VERTEX], &key); } static void @@ -294,47 +285,34 @@ iris_compile_fs(struct iris_context *ice, struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen; const struct brw_compiler *compiler = screen->compiler; const struct gen_device_info *devinfo = &screen->devinfo; - const unsigned *program; - struct brw_wm_prog_data fs_prog_data; - struct brw_stage_prog_data *prog_data = &fs_prog_data.base; void *mem_ctx = ralloc_context(NULL); + struct brw_wm_prog_data *fs_prog_data = + rzalloc(mem_ctx, struct brw_wm_prog_data); + struct brw_stage_prog_data *prog_data = &fs_prog_data->base; assert(ish->base.type == PIPE_SHADER_IR_NIR); nir_shader *nir = ish->base.ir.nir; - memset(&fs_prog_data, 0, sizeof(fs_prog_data)); - // XXX: alt mode assign_common_binding_table_offsets(devinfo, &nir->info, prog_data, MAX2(key->nr_color_regions, 1)); char *error_str = NULL; - program = brw_compile_fs(compiler, &ice->dbg, mem_ctx, key, &fs_prog_data, - nir, NULL, -1, -1, -1, true, false, vue_map, - &error_str); + const unsigned *program = + brw_compile_fs(compiler, &ice->dbg, mem_ctx, key, fs_prog_data, + nir, NULL, -1, -1, -1, true, false, vue_map, &error_str); if (program == NULL) { dbg_printf("Failed to compile fragment shader: %s\n", error_str); - ralloc_free(mem_ctx); return false; } //brw_alloc_stage_scratch(brw, &brw->wm.base, prog_data.base.total_scratch); - /* The param and pull_param arrays will be freed by the shader cache. */ - ralloc_steal(NULL, prog_data->param); - ralloc_steal(NULL, prog_data->pull_param); - #if 0 - brw_upload_cache(&brw->cache, BRW_CACHE_FS_PROG, - key, sizeof(struct brw_wm_prog_key), - program, prog_data.base.program_size, - &prog_data, sizeof(prog_data), - &brw->wm.base.prog_offset, &brw->wm.base.prog_data); - #endif + iris_upload_and_bind_shader(ice, IRIS_CACHE_FS, key, program, prog_data); ralloc_free(mem_ctx); - return true; } @@ -374,13 +352,11 @@ iris_update_compiled_fs(struct iris_context *ice) struct brw_wm_prog_key key; iris_populate_fs_key(ice, &key); - if (iris_search_cache(ice, IRIS_CACHE_FS, &key, sizeof(key), IRIS_DIRTY_FS, - &ice->shaders.prog_offset[MESA_SHADER_FRAGMENT], - &ice->shaders.prog_data[MESA_SHADER_FRAGMENT])) + if (iris_bind_cached_shader(ice, IRIS_CACHE_FS, &key)) return; UNUSED bool success = - iris_compile_fs(ice, ice->shaders.progs[MESA_SHADER_FRAGMENT], &key, + iris_compile_fs(ice, ice->shaders.uncompiled[MESA_SHADER_FRAGMENT], &key, ice->shaders.last_vue_map); } @@ -389,24 +365,33 @@ update_last_vue_map(struct iris_context *ice) { struct brw_stage_prog_data *prog_data; - if (ice->shaders.progs[MESA_SHADER_GEOMETRY]) - prog_data = ice->shaders.prog_data[MESA_SHADER_GEOMETRY]; - else if (ice->shaders.progs[MESA_SHADER_TESS_EVAL]) - prog_data = ice->shaders.prog_data[MESA_SHADER_TESS_EVAL]; + if (ice->shaders.prog[MESA_SHADER_GEOMETRY]) + prog_data = ice->shaders.prog[MESA_SHADER_GEOMETRY]->prog_data; + else if (ice->shaders.prog[MESA_SHADER_TESS_EVAL]) + prog_data = ice->shaders.prog[MESA_SHADER_TESS_EVAL]->prog_data; else - prog_data = ice->shaders.prog_data[MESA_SHADER_VERTEX]; + prog_data = ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data; struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; ice->shaders.last_vue_map = &vue_prog_data->vue_map; } +static struct brw_vue_prog_data * +get_vue_prog_data(struct iris_context *ice, gl_shader_stage stage) +{ + if (!ice->shaders.prog[stage]) + return NULL; + + return (void *) ice->shaders.prog[stage]->prog_data; +} + void iris_update_compiled_shaders(struct iris_context *ice) { struct brw_vue_prog_data *old_prog_datas[4]; if (!(ice->state.dirty & IRIS_DIRTY_URB)) { for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) - old_prog_datas[i] = (void *) ice->shaders.prog_data[i]; + old_prog_datas[i] = get_vue_prog_data(ice, i); } iris_update_compiled_vs(ice); @@ -420,7 +405,7 @@ iris_update_compiled_shaders(struct iris_context *ice) if (!(ice->state.dirty & IRIS_DIRTY_URB)) { for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) { struct brw_vue_prog_data *old = old_prog_datas[i]; - struct brw_vue_prog_data *new = (void *) ice->shaders.prog_data[i]; + struct brw_vue_prog_data *new = get_vue_prog_data(ice, i); if (!!old != !!new || (new && new->urb_entry_size != old->urb_entry_size)) { ice->state.dirty |= IRIS_DIRTY_URB; diff --git a/src/gallium/drivers/iris/iris_program_cache.c b/src/gallium/drivers/iris/iris_program_cache.c index 8287b7eda15..eb5e3a644c5 100644 --- a/src/gallium/drivers/iris/iris_program_cache.c +++ b/src/gallium/drivers/iris/iris_program_cache.c @@ -41,24 +41,27 @@ struct keybox { uint8_t data[0]; }; -// XXX: put packets here, expose this somehow and simplify the interfaces -struct iris_program_cache_item { - uint32_t assembly_offset; - uint32_t assembly_size; - void *prog_data; -}; - static struct keybox * make_keybox(struct iris_program_cache *cache, enum iris_program_cache_id cache_id, - const void *key, unsigned key_size) + const void *key) { + static const unsigned key_sizes[] = { + [IRIS_CACHE_VS] = sizeof(struct brw_vs_prog_key), + [IRIS_CACHE_TCS] = sizeof(struct brw_tcs_prog_key), + [IRIS_CACHE_TES] = sizeof(struct brw_tes_prog_key), + [IRIS_CACHE_GS] = sizeof(struct brw_gs_prog_key), + [IRIS_CACHE_FS] = sizeof(struct brw_wm_prog_key), + [IRIS_CACHE_CS] = sizeof(struct brw_cs_prog_key), + //[IRIS_CACHE_BLORP_BLIT] = sizeof(struct brw_blorp_blit_prog_key), + }; + struct keybox *keybox = - ralloc_size(cache->table, sizeof(struct keybox) + key_size); + ralloc_size(cache->table, sizeof(struct keybox) + key_sizes[cache_id]); keybox->cache_id = cache_id; - keybox->size = key_size; - memcpy(keybox->data, key, key_size); + keybox->size = key_sizes[cache_id]; + memcpy(keybox->data, key, key_sizes[cache_id]); return keybox; } @@ -80,6 +83,13 @@ keybox_equals(const void *void_a, const void *void_b) return memcmp(a->data, b->data, a->size) == 0; } +static uint64_t +dirty_flag_for_cache(enum iris_program_cache_id cache_id) +{ + assert(cache_id <= MESA_SHADER_STAGES); + return IRIS_DIRTY_VS << cache_id; +} + static unsigned get_program_string_id(enum iris_program_cache_id cache_id, const void *key) { @@ -102,34 +112,30 @@ get_program_string_id(enum iris_program_cache_id cache_id, const void *key) } /** - * Returns the buffer object matching cache_id and key, or NULL. + * Looks for a program in the cache and binds it. + * + * If no program was found, returns false and leaves the binding alone. */ -// XXX: rename to iris_bind_cached_shader? bool -iris_search_cache(struct iris_context *ice, - enum iris_program_cache_id cache_id, - const void *key, - unsigned key_size, - uint64_t dirty_flag, - uint32_t *inout_assembly_offset, - void *inout_prog_data) +iris_bind_cached_shader(struct iris_context *ice, + enum iris_program_cache_id cache_id, + const void *key) { struct iris_program_cache *cache = &ice->shaders.cache; - struct keybox *keybox = make_keybox(cache, cache_id, key, key_size); + struct keybox *keybox = make_keybox(cache, cache_id, key); struct hash_entry *entry = _mesa_hash_table_search(cache->table, keybox); - if (entry == NULL) + if (!entry) return false; - struct iris_program_cache_item *item = entry->data; + struct iris_compiled_shader *shader = entry->data; - if (item->assembly_offset != *inout_assembly_offset || - item->prog_data != *((void **) inout_prog_data)) { - *inout_assembly_offset = item->assembly_offset; - *((void **) inout_prog_data) = item->prog_data; - ice->state.dirty |= dirty_flag; + if (cache_id <= MESA_SHADER_STAGES && + memcmp(shader, ice->shaders.prog[cache_id], sizeof(*shader)) != 0) { + ice->shaders.prog[cache_id] = shader; + ice->state.dirty |= dirty_flag_for_cache(cache_id); } return true; @@ -185,19 +191,19 @@ iris_find_previous_compile(struct iris_program_cache *cache, * * This is useful for programs generating shaders at runtime, where multiple * distinct shaders (from an API perspective) may compile to the same assembly - * in our backend. + * in our backend. This saves space in the program cache buffer. */ -static const struct iris_program_cache_item * +static const struct iris_compiled_shader * find_existing_assembly(const struct iris_program_cache *cache, const void *assembly, unsigned assembly_size) { hash_table_foreach(cache->table, entry) { - const struct iris_program_cache_item *item = entry->data; - if (item->assembly_size == assembly_size && - memcmp(cache->map + item->assembly_offset, + const struct iris_compiled_shader *existing = entry->data; + if (existing->prog_data->program_size == assembly_size && + memcmp(cache->map + existing->prog_offset, assembly, assembly_size) == 0) - return item; + return existing; } return NULL; } @@ -230,23 +236,23 @@ upload_new_assembly(struct iris_context *ice, return offset; } +/** + * Upload a new shader to the program cache, and bind it for use. + * + * \param prog_data must be ralloc'd and will be stolen. + */ void -iris_upload_cache(struct iris_context *ice, - enum iris_program_cache_id cache_id, - const void *key, - unsigned key_size, - const void *assembly, - unsigned assembly_size, - const void *prog_data, - unsigned prog_data_size, - uint32_t *out_assembly_offset, - void *out_prog_data) +iris_upload_and_bind_shader(struct iris_context *ice, + enum iris_program_cache_id cache_id, + const void *key, + const void *assembly, + const struct brw_stage_prog_data *prog_data) { struct iris_program_cache *cache = &ice->shaders.cache; - struct iris_program_cache_item *item = - ralloc(cache->table, struct iris_program_cache_item); - const struct iris_program_cache_item *matching_data = - find_existing_assembly(cache, assembly, assembly_size); + struct iris_compiled_shader *shader = + ralloc(cache->table, struct iris_compiled_shader); + const struct iris_compiled_shader *existing = + find_existing_assembly(cache, assembly, prog_data->program_size); /* If we can find a matching prog in the cache already, then reuse the * existing stuff without creating new copy into the underlying buffer @@ -254,28 +260,26 @@ iris_upload_cache(struct iris_context *ice, * runtime, where multiple shaders may compile to the same thing in our * backend. */ - if (matching_data) { - item->assembly_offset = matching_data->assembly_offset; + if (existing) { + shader->prog_offset = existing->prog_offset; } else { - item->assembly_offset = - upload_new_assembly(ice, assembly, assembly_size); + shader->prog_offset = + upload_new_assembly(ice, assembly, prog_data->program_size); } - item->assembly_size = assembly_size; - item->prog_data = ralloc_size(item, prog_data_size); - memcpy(item->prog_data, prog_data, prog_data_size); + shader->prog_data = prog_data; - if (cache_id != IRIS_CACHE_BLORP) { - struct brw_stage_prog_data *stage_prog_data = prog_data; - ralloc_steal(item->prog_data, stage_prog_data->param); - ralloc_steal(item->prog_data, stage_prog_data->pull_param); - } + ralloc_steal(shader, shader->prog_data); + ralloc_steal(shader->prog_data, prog_data->param); + ralloc_steal(shader->prog_data, prog_data->pull_param); - struct keybox *keybox = make_keybox(cache, cache_id, key, key_size); - _mesa_hash_table_insert(cache->table, keybox, item); + struct keybox *keybox = make_keybox(cache, cache_id, key); + _mesa_hash_table_insert(cache->table, keybox, shader); - *out_assembly_offset = item->assembly_offset; - *(void **)out_prog_data = item->prog_data; + if (cache_id <= MESA_SHADER_STAGES) { + ice->shaders.prog[cache_id] = shader; + ice->state.dirty |= dirty_flag_for_cache(cache_id); + } } void @@ -303,16 +307,17 @@ iris_destroy_program_cache(struct iris_context *ice) cache->next_offset = 0; - /* Also, NULL out any stale program pointers. */ for (int i = 0; i < MESA_SHADER_STAGES; i++) { - ice->shaders.prog_data[i] = NULL; + ice->shaders.prog[i] = NULL; } + + ralloc_free(cache->table); } static const char * cache_name(enum iris_program_cache_id cache_id) { - if (cache_id == IRIS_CACHE_BLORP) + if (cache_id == IRIS_CACHE_BLORP_BLIT) return "BLORP"; return _mesa_shader_stage_to_string(cache_id); @@ -327,9 +332,9 @@ iris_print_program_cache(struct iris_context *ice) hash_table_foreach(cache->table, entry) { const struct keybox *keybox = entry->key; - struct iris_program_cache_item *item = entry->data; + struct iris_compiled_shader *shader = entry->data; fprintf(stderr, "%s:\n", cache_name(keybox->cache_id)); - brw_disassemble(devinfo, cache->map, - item->assembly_offset, item->assembly_size, stderr); + brw_disassemble(devinfo, cache->map, shader->prog_offset, + shader->prog_data->program_size, stderr); } } -- 2.30.2