src/gallium/drivers/iris/iris_program_cache.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_program_cache.c
  25  *
  26  * The in-memory program cache.  This is basically a hash table mapping
  27  * API-specified shaders and a state key to a compiled variant.  It also
  28  * takes care of uploading shader assembly into a BO for use on the GPU.
  29  */
  30
  31 #include <stdio.h>
  32 #include <errno.h>
  33 #include "pipe/p_defines.h"
  34 #include "pipe/p_state.h"
  35 #include "pipe/p_context.h"
  36 #include "pipe/p_screen.h"
  37 #include "util/u_atomic.h"
  38 #include "util/u_upload_mgr.h"
  39 #include "compiler/nir/nir.h"
  40 #include "compiler/nir/nir_builder.h"
  41 #include "intel/common/gen_disasm.h"
  42 #include "intel/compiler/brw_compiler.h"
  43 #include "intel/compiler/brw_eu.h"
  44 #include "intel/compiler/brw_nir.h"
  45 #include "iris_context.h"
  46 #include "iris_resource.h"
  47
  48 struct keybox {
  49    uint16_t size;
  50    enum iris_program_cache_id cache_id;
  51    uint8_t data[0];
  52 };
  53
  54 static struct keybox *
  55 make_keybox(void *mem_ctx,
  56             enum iris_program_cache_id cache_id,
  57             const void *key,
  58             uint32_t key_size)
  59 {
  60    struct keybox *keybox =
  61       ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
  62
  63    keybox->cache_id = cache_id;
  64    keybox->size = key_size;
  65    memcpy(keybox->data, key, key_size);
  66
  67    return keybox;
  68 }
  69
  70 static uint32_t
  71 keybox_hash(const void *void_key)
  72 {
  73    const struct keybox *key = void_key;
  74    return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
  75 }
  76
  77 static bool
  78 keybox_equals(const void *void_a, const void *void_b)
  79 {
  80    const struct keybox *a = void_a, *b = void_b;
  81    if (a->size != b->size)
  82       return false;
  83
  84    return memcmp(a->data, b->data, a->size) == 0;
  85 }
  86
  87 struct iris_compiled_shader *
  88 iris_find_cached_shader(struct iris_context *ice,
  89                         enum iris_program_cache_id cache_id,
  90                         uint32_t key_size,
  91                         const void *key)
  92 {
  93    struct keybox *keybox = make_keybox(NULL, cache_id, key, key_size);
  94    struct hash_entry *entry =
  95       _mesa_hash_table_search(ice->shaders.cache, keybox);
  96
  97    ralloc_free(keybox);
  98
  99    return entry ? entry->data : NULL;
 100 }
 101
 102 const void *
 103 iris_find_previous_compile(const struct iris_context *ice,
 104                            enum iris_program_cache_id cache_id,
 105                            unsigned program_string_id)
 106 {
 107    hash_table_foreach(ice->shaders.cache, entry) {
 108       const struct keybox *keybox = entry->key;
 109       const struct brw_base_prog_key *key = (const void *)keybox->data;
 110       if (keybox->cache_id == cache_id &&
 111           key->program_string_id == program_string_id) {
 112          return keybox->data;
 113       }
 114    }
 115
 116    return NULL;
 117 }
 118
 119 void
 120 iris_delete_shader_variants(struct iris_context *ice,
 121                             struct iris_uncompiled_shader *ish)
 122 {
 123    struct hash_table *cache = ice->shaders.cache;
 124    gl_shader_stage stage = ish->nir->info.stage;
 125    enum iris_program_cache_id cache_id = stage;
 126
 127    hash_table_foreach(cache, entry) {
 128       const struct keybox *keybox = entry->key;
 129       const struct brw_base_prog_key *key = (const void *)keybox->data;
 130
 131       if (keybox->cache_id == cache_id &&
 132           key->program_string_id == ish->program_id) {
 133          struct iris_compiled_shader *shader = entry->data;
 134
 135          _mesa_hash_table_remove(cache, entry);
 136
 137          /* Shader variants may still be bound in the context even after
 138           * the API-facing shader has been deleted.  In particular, a draw
 139           * may not have triggered iris_update_compiled_shaders() yet.  In
 140           * that case, we may be referring to that shader's VUE map, stream
 141           * output settings, and so on.  We also like to compare the old and
 142           * new shader programs when swapping them out to flag dirty state.
 143           *
 144           * So, it's hazardous to delete a bound shader variant.  We avoid
 145           * doing so, choosing to instead move "deleted" shader variants to
 146           * a list, deferring the actual deletion until they're not bound.
 147           *
 148           * For simplicity, we always move deleted variants to the list,
 149           * even if we could delete them immediately.  We'll then process
 150           * the list, catching both these variants and any others.
 151           */
 152          list_addtail(&shader->link, &ice->shaders.deleted_variants[stage]);
 153       }
 154    }
 155
 156    /* Process any pending deferred variant deletions. */
 157    list_for_each_entry_safe(struct iris_compiled_shader, shader,
 158                             &ice->shaders.deleted_variants[stage], link) {
 159       /* If the shader is still bound, defer deletion. */
 160       if (ice->shaders.prog[stage] == shader)
 161          continue;
 162
 163       list_del(&shader->link);
 164
 165       /* Actually delete the variant. */
 166       pipe_resource_reference(&shader->assembly.res, NULL);
 167       ralloc_free(shader);
 168    }
 169 }
 170
 171
 172 /**
 173  * Look for an existing entry in the cache that has identical assembly code.
 174  *
 175  * This is useful for programs generating shaders at runtime, where multiple
 176  * distinct shaders (from an API perspective) may compile to the same assembly
 177  * in our backend.  This saves space in the program cache buffer.
 178  */
 179 static const struct iris_compiled_shader *
 180 find_existing_assembly(struct hash_table *cache,
 181                        const void *assembly,
 182                        unsigned assembly_size)
 183 {
 184    hash_table_foreach(cache, entry) {
 185       const struct iris_compiled_shader *existing = entry->data;
 186       if (existing->prog_data->program_size == assembly_size &&
 187           memcmp(existing->map, assembly, assembly_size) == 0)
 188          return existing;
 189    }
 190    return NULL;
 191 }
 192
 193 struct iris_compiled_shader *
 194 iris_upload_shader(struct iris_context *ice,
 195                    enum iris_program_cache_id cache_id,
 196                    uint32_t key_size,
 197                    const void *key,
 198                    const void *assembly,
 199                    struct brw_stage_prog_data *prog_data,
 200                    uint32_t *streamout,
 201                    enum brw_param_builtin *system_values,
 202                    unsigned num_system_values,
 203                    unsigned kernel_input_size,
 204                    unsigned num_cbufs,
 205                    const struct iris_binding_table *bt)
 206 {
 207    struct hash_table *cache = ice->shaders.cache;
 208    struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
 209    struct iris_compiled_shader *shader =
 210       rzalloc_size(cache, sizeof(struct iris_compiled_shader) +
 211                    screen->vtbl.derived_program_state_size(cache_id));
 212    const struct iris_compiled_shader *existing =
 213       find_existing_assembly(cache, assembly, prog_data->program_size);
 214
 215    /* If we can find a matching prog in the cache already, then reuse the
 216     * existing stuff without creating new copy into the underlying buffer
 217     * object.  This is notably useful for programs generating shaders at
 218     * runtime, where multiple shaders may compile to the same thing in our
 219     * backend.
 220     */
 221    if (existing) {
 222       pipe_resource_reference(&shader->assembly.res, existing->assembly.res);
 223       shader->assembly.offset = existing->assembly.offset;
 224       shader->map = existing->map;
 225    } else {
 226       shader->assembly.res = NULL;
 227       u_upload_alloc(ice->shaders.uploader, 0, prog_data->program_size, 64,
 228                      &shader->assembly.offset, &shader->assembly.res,
 229                      &shader->map);
 230       memcpy(shader->map, assembly, prog_data->program_size);
 231    }
 232
 233    list_inithead(&shader->link);
 234
 235    shader->prog_data = prog_data;
 236    shader->streamout = streamout;
 237    shader->system_values = system_values;
 238    shader->num_system_values = num_system_values;
 239    shader->kernel_input_size = kernel_input_size;
 240    shader->num_cbufs = num_cbufs;
 241    shader->bt = *bt;
 242
 243    ralloc_steal(shader, shader->prog_data);
 244    ralloc_steal(shader->prog_data, prog_data->param);
 245    ralloc_steal(shader->prog_data, prog_data->pull_param);
 246    ralloc_steal(shader, shader->streamout);
 247    ralloc_steal(shader, shader->system_values);
 248
 249    /* Store the 3DSTATE shader packets and other derived state. */
 250    screen->vtbl.store_derived_program_state(ice, cache_id, shader);
 251
 252    struct keybox *keybox = make_keybox(shader, cache_id, key, key_size);
 253    _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
 254
 255    return shader;
 256 }
 257
 258 bool
 259 iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
 260                          const void *key, uint32_t key_size,
 261                          uint32_t *kernel_out, void *prog_data_out)
 262 {
 263    struct blorp_context *blorp = blorp_batch->blorp;
 264    struct iris_context *ice = blorp->driver_ctx;
 265    struct iris_batch *batch = blorp_batch->driver_batch;
 266    struct iris_compiled_shader *shader =
 267       iris_find_cached_shader(ice, IRIS_CACHE_BLORP, key_size, key);
 268
 269    if (!shader)
 270       return false;
 271
 272    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 273    *kernel_out =
 274       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 275    *((void **) prog_data_out) = shader->prog_data;
 276
 277    iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
 278
 279    return true;
 280 }
 281
 282 bool
 283 iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
 284                          const void *key, uint32_t key_size,
 285                          const void *kernel, UNUSED uint32_t kernel_size,
 286                          const struct brw_stage_prog_data *prog_data_templ,
 287                          UNUSED uint32_t prog_data_size,
 288                          uint32_t *kernel_out, void *prog_data_out)
 289 {
 290    struct blorp_context *blorp = blorp_batch->blorp;
 291    struct iris_context *ice = blorp->driver_ctx;
 292    struct iris_batch *batch = blorp_batch->driver_batch;
 293
 294    void *prog_data = ralloc_size(NULL, prog_data_size);
 295    memcpy(prog_data, prog_data_templ, prog_data_size);
 296
 297    struct iris_binding_table bt;
 298    memset(&bt, 0, sizeof(bt));
 299
 300    struct iris_compiled_shader *shader =
 301       iris_upload_shader(ice, IRIS_CACHE_BLORP, key_size, key, kernel,
 302                          prog_data, NULL, NULL, 0, 0, 0, &bt);
 303
 304    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 305    *kernel_out =
 306       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 307    *((void **) prog_data_out) = shader->prog_data;
 308
 309    iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
 310
 311    return true;
 312 }
 313
 314 void
 315 iris_init_program_cache(struct iris_context *ice)
 316 {
 317    ice->shaders.cache =
 318       _mesa_hash_table_create(ice, keybox_hash, keybox_equals);
 319
 320    ice->shaders.uploader =
 321       u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
 322                       IRIS_RESOURCE_FLAG_SHADER_MEMZONE);
 323
 324    for (int i = 0; i < MESA_SHADER_STAGES; i++)
 325       list_inithead(&ice->shaders.deleted_variants[i]);
 326 }
 327
 328 void
 329 iris_destroy_program_cache(struct iris_context *ice)
 330 {
 331    for (int i = 0; i < MESA_SHADER_STAGES; i++) {
 332       ice->shaders.prog[i] = NULL;
 333
 334       list_for_each_entry_safe(struct iris_compiled_shader, shader,
 335                                &ice->shaders.deleted_variants[i], link) {
 336          pipe_resource_reference(&shader->assembly.res, NULL);
 337       }
 338    }
 339
 340    hash_table_foreach(ice->shaders.cache, entry) {
 341       struct iris_compiled_shader *shader = entry->data;
 342       pipe_resource_reference(&shader->assembly.res, NULL);
 343    }
 344
 345    u_upload_destroy(ice->shaders.uploader);
 346
 347    ralloc_free(ice->shaders.cache);
 348 }
 349
 350 static const char *
 351 cache_name(enum iris_program_cache_id cache_id)
 352 {
 353    if (cache_id == IRIS_CACHE_BLORP)
 354       return "BLORP";
 355
 356    return _mesa_shader_stage_to_string(cache_id);
 357 }
 358
 359 void
 360 iris_print_program_cache(struct iris_context *ice)
 361 {
 362    struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
 363    const struct gen_device_info *devinfo = &screen->devinfo;
 364
 365    hash_table_foreach(ice->shaders.cache, entry) {
 366       const struct keybox *keybox = entry->key;
 367       struct iris_compiled_shader *shader = entry->data;
 368       fprintf(stderr, "%s:\n", cache_name(keybox->cache_id));
 369       gen_disassemble(devinfo, shader->map, 0, stderr);
 370    }
 371 }