src/gallium/drivers/iris/iris_program_cache.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * @file iris_program_cache.c
  26  *
  27  * The in-memory program cache.  This is basically a hash table mapping
  28  * API-specified shaders and a state key to a compiled variant.  It also
  29  * takes care of uploading shader assembly into a BO for use on the GPU.
  30  */
  31
  32 #include <stdio.h>
  33 #include <errno.h>
  34 #include "pipe/p_defines.h"
  35 #include "pipe/p_state.h"
  36 #include "pipe/p_context.h"
  37 #include "pipe/p_screen.h"
  38 #include "util/u_atomic.h"
  39 #include "util/u_upload_mgr.h"
  40 #include "compiler/nir/nir.h"
  41 #include "compiler/nir/nir_builder.h"
  42 #include "intel/compiler/brw_compiler.h"
  43 #include "intel/compiler/brw_eu.h"
  44 #include "intel/compiler/brw_nir.h"
  45 #include "iris_context.h"
  46 #include "iris_resource.h"
  47
  48 struct keybox {
  49    uint8_t size;
  50    enum iris_program_cache_id cache_id;
  51    uint8_t data[0];
  52 };
  53
  54 static uint32_t
  55 key_size_for_cache(enum iris_program_cache_id cache_id)
  56 {
  57    static const unsigned key_sizes[] = {
  58       [IRIS_CACHE_VS]         = sizeof(struct brw_vs_prog_key),
  59       [IRIS_CACHE_TCS]        = sizeof(struct brw_tcs_prog_key),
  60       [IRIS_CACHE_TES]        = sizeof(struct brw_tes_prog_key),
  61       [IRIS_CACHE_GS]         = sizeof(struct brw_gs_prog_key),
  62       [IRIS_CACHE_FS]         = sizeof(struct brw_wm_prog_key),
  63       [IRIS_CACHE_CS]         = sizeof(struct brw_cs_prog_key),
  64    };
  65
  66    /* BLORP keys aren't all the same size. */
  67    assert(cache_id != IRIS_CACHE_BLORP);
  68
  69    return key_sizes[cache_id];
  70 }
  71
  72 static struct keybox *
  73 make_keybox(void *mem_ctx,
  74             enum iris_program_cache_id cache_id,
  75             const void *key,
  76             uint32_t key_size)
  77 {
  78    struct keybox *keybox =
  79       ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
  80
  81    keybox->cache_id = cache_id;
  82    keybox->size = key_size;
  83    memcpy(keybox->data, key, key_size);
  84
  85    return keybox;
  86 }
  87
  88 static uint32_t
  89 keybox_hash(const void *void_key)
  90 {
  91    const struct keybox *key = void_key;
  92    return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
  93 }
  94
  95 static bool
  96 keybox_equals(const void *void_a, const void *void_b)
  97 {
  98    const struct keybox *a = void_a, *b = void_b;
  99    if (a->size != b->size)
 100       return false;
 101
 102    return memcmp(a->data, b->data, a->size) == 0;
 103 }
 104
 105 static uint64_t
 106 dirty_flag_for_cache(enum iris_program_cache_id cache_id)
 107 {
 108    assert(cache_id <= MESA_SHADER_STAGES);
 109
 110    // XXX: ugly...
 111    // XXX: move this flagging out to a higher level, allow comparison of
 112    // XXX: new and old programs to decide what bits to twiddle
 113    // XXX: CLIP: toggle if barycentric modes has any NONPERSPECTIVE or not
 114    if (cache_id == IRIS_CACHE_FS)
 115       return IRIS_DIRTY_WM | IRIS_DIRTY_FS | IRIS_DIRTY_CLIP;
 116    if (cache_id == IRIS_CACHE_VS)
 117       return IRIS_DIRTY_VS | IRIS_DIRTY_VF_SGVS;
 118
 119    return IRIS_DIRTY_VS << cache_id | IRIS_DIRTY_BINDINGS_VS << cache_id;
 120 }
 121
 122 static unsigned
 123 get_program_string_id(enum iris_program_cache_id cache_id, const void *key)
 124 {
 125    switch (cache_id) {
 126    case IRIS_CACHE_VS:
 127       return ((struct brw_vs_prog_key *) key)->program_string_id;
 128    case IRIS_CACHE_TCS:
 129       return ((struct brw_tcs_prog_key *) key)->program_string_id;
 130    case IRIS_CACHE_TES:
 131       return ((struct brw_tes_prog_key *) key)->program_string_id;
 132    case IRIS_CACHE_GS:
 133       return ((struct brw_gs_prog_key *) key)->program_string_id;
 134    case IRIS_CACHE_CS:
 135       return ((struct brw_cs_prog_key *) key)->program_string_id;
 136    case IRIS_CACHE_FS:
 137       return ((struct brw_wm_prog_key *) key)->program_string_id;
 138    default:
 139       unreachable("no program string id for this kind of program");
 140    }
 141 }
 142
 143 static struct iris_compiled_shader *
 144 iris_find_cached_shader(struct iris_context *ice,
 145                         enum iris_program_cache_id cache_id,
 146                         const void *key,
 147                         uint32_t key_size)
 148 {
 149    struct keybox *keybox =
 150       make_keybox(ice->shaders.cache, cache_id, key, key_size);
 151    struct hash_entry *entry =
 152       _mesa_hash_table_search(ice->shaders.cache, keybox);
 153
 154    ralloc_free(keybox);
 155
 156    return entry ? entry->data : NULL;
 157 }
 158
 159 /**
 160  * Looks for a program in the cache and binds it.
 161  *
 162  * If no program was found, returns false and leaves the binding alone.
 163  */
 164 bool
 165 iris_bind_cached_shader(struct iris_context *ice,
 166                         enum iris_program_cache_id cache_id,
 167                         const void *key)
 168 {
 169    unsigned key_size = key_size_for_cache(cache_id);
 170    struct iris_compiled_shader *shader =
 171       iris_find_cached_shader(ice, cache_id, key, key_size);
 172
 173    if (!shader)
 174       return false;
 175
 176    if (memcmp(shader, ice->shaders.prog[cache_id], sizeof(*shader)) != 0) {
 177       ice->shaders.prog[cache_id] = shader;
 178       ice->state.dirty |= dirty_flag_for_cache(cache_id);
 179    }
 180
 181    return true;
 182 }
 183
 184 void
 185 iris_unbind_shader(struct iris_context *ice,
 186                    enum iris_program_cache_id cache_id)
 187 {
 188    if (ice->shaders.prog[cache_id]) {
 189       ice->shaders.prog[cache_id] = NULL;
 190       ice->state.dirty |= dirty_flag_for_cache(cache_id);
 191    }
 192 }
 193
 194 const void *
 195 iris_find_previous_compile(const struct iris_context *ice,
 196                            enum iris_program_cache_id cache_id,
 197                            unsigned program_string_id)
 198 {
 199    hash_table_foreach(ice->shaders.cache, entry) {
 200       const struct keybox *keybox = entry->key;
 201       if (keybox->cache_id == cache_id &&
 202           get_program_string_id(cache_id, keybox->data) == program_string_id) {
 203          return keybox->data;
 204       }
 205    }
 206
 207    return NULL;
 208 }
 209
 210 /**
 211  * Look for an existing entry in the cache that has identical assembly code.
 212  *
 213  * This is useful for programs generating shaders at runtime, where multiple
 214  * distinct shaders (from an API perspective) may compile to the same assembly
 215  * in our backend.  This saves space in the program cache buffer.
 216  */
 217 static const struct iris_compiled_shader *
 218 find_existing_assembly(struct hash_table *cache,
 219                        const void *assembly,
 220                        unsigned assembly_size)
 221 {
 222    hash_table_foreach(cache, entry) {
 223       const struct iris_compiled_shader *existing = entry->data;
 224       if (existing->prog_data->program_size == assembly_size &&
 225           memcmp(existing->map, assembly, assembly_size) == 0)
 226          return existing;
 227    }
 228    return NULL;
 229 }
 230
 231 static struct iris_compiled_shader *
 232 iris_upload_shader(struct iris_context *ice,
 233                    enum iris_program_cache_id cache_id,
 234                    uint32_t key_size,
 235                    const void *key,
 236                    const void *assembly,
 237                    struct brw_stage_prog_data *prog_data,
 238                    uint32_t *streamout)
 239 {
 240    struct iris_screen *screen = (void *) ice->ctx.screen;
 241    struct gen_device_info *devinfo = &screen->devinfo;
 242    struct hash_table *cache = ice->shaders.cache;
 243    struct iris_compiled_shader *shader =
 244       rzalloc_size(cache, sizeof(struct iris_compiled_shader) +
 245                    ice->vtbl.derived_program_state_size(cache_id));
 246    const struct iris_compiled_shader *existing =
 247       find_existing_assembly(cache, assembly, prog_data->program_size);
 248
 249    /* If we can find a matching prog in the cache already, then reuse the
 250     * existing stuff without creating new copy into the underlying buffer
 251     * object.  This is notably useful for programs generating shaders at
 252     * runtime, where multiple shaders may compile to the same thing in our
 253     * backend.
 254     */
 255    if (existing) {
 256       pipe_resource_reference(&shader->assembly.res, existing->assembly.res);
 257       shader->assembly.offset = existing->assembly.offset;
 258       shader->map = existing->map;
 259    } else {
 260       shader->assembly.res = NULL;
 261       u_upload_alloc(ice->shaders.uploader, 0, prog_data->program_size, 64,
 262                      &shader->assembly.offset, &shader->assembly.res,
 263                      &shader->map);
 264       memcpy(shader->map, assembly, prog_data->program_size);
 265    }
 266
 267    shader->prog_data = prog_data;
 268    shader->streamout = streamout;
 269
 270    ralloc_steal(shader, shader->prog_data);
 271    ralloc_steal(shader->prog_data, prog_data->param);
 272    ralloc_steal(shader->prog_data, prog_data->pull_param);
 273    ralloc_steal(shader, shader->streamout);
 274
 275    /* Store the 3DSTATE shader packets and other derived state. */
 276    ice->vtbl.store_derived_program_state(devinfo, cache_id, shader);
 277
 278    struct keybox *keybox = make_keybox(cache, cache_id, key, key_size);
 279    _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
 280
 281    return shader;
 282 }
 283
 284 /**
 285  * Upload a new shader to the program cache, and bind it for use.
 286  *
 287  * \param prog_data must be ralloc'd and will be stolen.
 288  */
 289 void
 290 iris_upload_and_bind_shader(struct iris_context *ice,
 291                             enum iris_program_cache_id cache_id,
 292                             const void *key,
 293                             const void *assembly,
 294                             struct brw_stage_prog_data *prog_data,
 295                             uint32_t *streamout)
 296 {
 297    assert(cache_id != IRIS_CACHE_BLORP);
 298
 299    struct iris_compiled_shader *shader =
 300       iris_upload_shader(ice, cache_id, key_size_for_cache(cache_id), key,
 301                          assembly, prog_data, streamout);
 302
 303    ice->shaders.prog[cache_id] = shader;
 304    ice->state.dirty |= dirty_flag_for_cache(cache_id);
 305 }
 306
 307 bool
 308 iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
 309                          const void *key, uint32_t key_size,
 310                          uint32_t *kernel_out, void *prog_data_out)
 311 {
 312    struct blorp_context *blorp = blorp_batch->blorp;
 313    struct iris_context *ice = blorp->driver_ctx;
 314    struct iris_batch *batch = blorp_batch->driver_batch;
 315    struct iris_compiled_shader *shader =
 316       iris_find_cached_shader(ice, IRIS_CACHE_BLORP, key, key_size);
 317
 318    if (!shader)
 319       return false;
 320
 321    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 322    *kernel_out =
 323       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 324    *((void **) prog_data_out) = shader->prog_data;
 325
 326    iris_use_pinned_bo(batch, bo, false);
 327
 328    return true;
 329 }
 330
 331 bool
 332 iris_blorp_upload_shader(struct blorp_batch *blorp_batch,
 333                          const void *key, uint32_t key_size,
 334                          const void *kernel, UNUSED uint32_t kernel_size,
 335                          const struct brw_stage_prog_data *prog_data_templ,
 336                          UNUSED uint32_t prog_data_size,
 337                          uint32_t *kernel_out, void *prog_data_out)
 338 {
 339    struct blorp_context *blorp = blorp_batch->blorp;
 340    struct iris_context *ice = blorp->driver_ctx;
 341    struct iris_batch *batch = blorp_batch->driver_batch;
 342
 343    void *prog_data = ralloc_size(NULL, prog_data_size);
 344    memcpy(prog_data, prog_data_templ, prog_data_size);
 345
 346    struct iris_compiled_shader *shader =
 347       iris_upload_shader(ice, IRIS_CACHE_BLORP, key_size, key, kernel,
 348                          prog_data, NULL);
 349
 350    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 351    *kernel_out =
 352       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 353    *((void **) prog_data_out) = shader->prog_data;
 354
 355    iris_use_pinned_bo(batch, bo, false);
 356
 357    return true;
 358 }
 359
 360 void
 361 iris_init_program_cache(struct iris_context *ice)
 362 {
 363    ice->shaders.cache =
 364       _mesa_hash_table_create(ice, keybox_hash, keybox_equals);
 365
 366    ice->shaders.uploader =
 367       u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
 368                       IRIS_RESOURCE_FLAG_SHADER_MEMZONE);
 369 }
 370
 371 void
 372 iris_destroy_program_cache(struct iris_context *ice)
 373 {
 374    for (int i = 0; i < MESA_SHADER_STAGES; i++) {
 375       ice->shaders.prog[i] = NULL;
 376    }
 377
 378    hash_table_foreach(ice->shaders.cache, entry) {
 379       struct iris_compiled_shader *shader = entry->data;
 380       pipe_resource_reference(&shader->assembly.res, NULL);
 381    }
 382
 383    u_upload_destroy(ice->shaders.uploader);
 384
 385    ralloc_free(ice->shaders.cache);
 386 }
 387
 388 static const char *
 389 cache_name(enum iris_program_cache_id cache_id)
 390 {
 391    if (cache_id == IRIS_CACHE_BLORP)
 392       return "BLORP";
 393
 394    return _mesa_shader_stage_to_string(cache_id);
 395 }
 396
 397 void
 398 iris_print_program_cache(struct iris_context *ice)
 399 {
 400    struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
 401    const struct gen_device_info *devinfo = &screen->devinfo;
 402
 403    hash_table_foreach(ice->shaders.cache, entry) {
 404       const struct keybox *keybox = entry->key;
 405       struct iris_compiled_shader *shader = entry->data;
 406       fprintf(stderr, "%s:\n", cache_name(keybox->cache_id));
 407       brw_disassemble(devinfo, shader->map, 0,
 408                       shader->prog_data->program_size, stderr);
 409    }
 410 }