src/gallium/drivers/iris/iris_program_cache.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_program_cache.c
  25  *
  26  * The in-memory program cache.  This is basically a hash table mapping
  27  * API-specified shaders and a state key to a compiled variant.  It also
  28  * takes care of uploading shader assembly into a BO for use on the GPU.
  29  */
  30
  31 #include <stdio.h>
  32 #include <errno.h>
  33 #include "pipe/p_defines.h"
  34 #include "pipe/p_state.h"
  35 #include "pipe/p_context.h"
  36 #include "pipe/p_screen.h"
  37 #include "util/u_atomic.h"
  38 #include "util/u_upload_mgr.h"
  39 #include "compiler/nir/nir.h"
  40 #include "compiler/nir/nir_builder.h"
  41 #include "intel/compiler/brw_compiler.h"
  42 #include "intel/compiler/brw_eu.h"
  43 #include "intel/compiler/brw_nir.h"
  44 #include "iris_context.h"
  45 #include "iris_resource.h"
  46
  47 struct keybox {
  48    uint8_t size;
  49    enum iris_program_cache_id cache_id;
  50    uint8_t data[0];
  51 };
  52
  53 static uint32_t
  54 key_size_for_cache(enum iris_program_cache_id cache_id)
  55 {
  56    static const unsigned key_sizes[] = {
  57       [IRIS_CACHE_VS]         = sizeof(struct brw_vs_prog_key),
  58       [IRIS_CACHE_TCS]        = sizeof(struct brw_tcs_prog_key),
  59       [IRIS_CACHE_TES]        = sizeof(struct brw_tes_prog_key),
  60       [IRIS_CACHE_GS]         = sizeof(struct brw_gs_prog_key),
  61       [IRIS_CACHE_FS]         = sizeof(struct brw_wm_prog_key),
  62       [IRIS_CACHE_CS]         = sizeof(struct brw_cs_prog_key),
  63    };
  64
  65    /* BLORP keys aren't all the same size. */
  66    assert(cache_id != IRIS_CACHE_BLORP);
  67
  68    return key_sizes[cache_id];
  69 }
  70
  71 static struct keybox *
  72 make_keybox(void *mem_ctx,
  73             enum iris_program_cache_id cache_id,
  74             const void *key,
  75             uint32_t key_size)
  76 {
  77    struct keybox *keybox =
  78       ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
  79
  80    keybox->cache_id = cache_id;
  81    keybox->size = key_size;
  82    memcpy(keybox->data, key, key_size);
  83
  84    return keybox;
  85 }
  86
  87 static uint32_t
  88 keybox_hash(const void *void_key)
  89 {
  90    const struct keybox *key = void_key;
  91    return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
  92 }
  93
  94 static bool
  95 keybox_equals(const void *void_a, const void *void_b)
  96 {
  97    const struct keybox *a = void_a, *b = void_b;
  98    if (a->size != b->size)
  99       return false;
 100
 101    return memcmp(a->data, b->data, a->size) == 0;
 102 }
 103
 104 static uint64_t
 105 dirty_flag_for_cache(enum iris_program_cache_id cache_id)
 106 {
 107    assert(cache_id <= MESA_SHADER_STAGES);
 108
 109    // XXX: ugly...
 110    // XXX: move this flagging out to a higher level, allow comparison of
 111    // XXX: new and old programs to decide what bits to twiddle
 112    // XXX: CLIP: toggle if barycentric modes has any NONPERSPECTIVE or not
 113    if (cache_id == IRIS_CACHE_FS)
 114       return IRIS_DIRTY_WM | IRIS_DIRTY_FS | IRIS_DIRTY_CLIP | IRIS_DIRTY_SBE;
 115    if (cache_id == IRIS_CACHE_VS)
 116       return IRIS_DIRTY_VS | IRIS_DIRTY_VF_SGVS;
 117
 118    /* For compute, prog_data->threads needs to be uploaded as constants. */
 119    if (cache_id == IRIS_CACHE_CS)
 120       return IRIS_DIRTY_CS | IRIS_DIRTY_CONSTANTS_CS;
 121
 122    return IRIS_DIRTY_VS << cache_id | IRIS_DIRTY_BINDINGS_VS << cache_id;
 123 }
 124
 125 static unsigned
 126 get_program_string_id(enum iris_program_cache_id cache_id, const void *key)
 127 {
 128    switch (cache_id) {
 129    case IRIS_CACHE_VS:
 130       return ((struct brw_vs_prog_key *) key)->program_string_id;
 131    case IRIS_CACHE_TCS:
 132       return ((struct brw_tcs_prog_key *) key)->program_string_id;
 133    case IRIS_CACHE_TES:
 134       return ((struct brw_tes_prog_key *) key)->program_string_id;
 135    case IRIS_CACHE_GS:
 136       return ((struct brw_gs_prog_key *) key)->program_string_id;
 137    case IRIS_CACHE_CS:
 138       return ((struct brw_cs_prog_key *) key)->program_string_id;
 139    case IRIS_CACHE_FS:
 140       return ((struct brw_wm_prog_key *) key)->program_string_id;
 141    default:
 142       unreachable("no program string id for this kind of program");
 143    }
 144 }
 145
 146 static struct iris_compiled_shader *
 147 iris_find_cached_shader(struct iris_context *ice,
 148                         enum iris_program_cache_id cache_id,
 149                         const void *key,
 150                         uint32_t key_size)
 151 {
 152    struct keybox *keybox =
 153       make_keybox(ice->shaders.cache, cache_id, key, key_size);
 154    struct hash_entry *entry =
 155       _mesa_hash_table_search(ice->shaders.cache, keybox);
 156
 157    ralloc_free(keybox);
 158
 159    return entry ? entry->data : NULL;
 160 }
 161
 162 /**
 163  * Looks for a program in the cache and binds it.
 164  *
 165  * If no program was found, returns false and leaves the binding alone.
 166  */
 167 bool
 168 iris_bind_cached_shader(struct iris_context *ice,
 169                         enum iris_program_cache_id cache_id,
 170                         const void *key)
 171 {
 172    unsigned key_size = key_size_for_cache(cache_id);
 173    struct iris_compiled_shader *shader =
 174       iris_find_cached_shader(ice, cache_id, key, key_size);
 175
 176    if (!shader)
 177       return false;
 178
 179    // XXX: why memcmp?
 180    if (!ice->shaders.prog[cache_id] ||
 181        memcmp(shader, ice->shaders.prog[cache_id], sizeof(*shader)) != 0) {
 182       ice->shaders.prog[cache_id] = shader;
 183       ice->state.dirty |= dirty_flag_for_cache(cache_id);
 184    }
 185
 186    return true;
 187 }
 188
 189 void
 190 iris_unbind_shader(struct iris_context *ice,
 191                    enum iris_program_cache_id cache_id)
 192 {
 193    if (ice->shaders.prog[cache_id]) {
 194       ice->shaders.prog[cache_id] = NULL;
 195       ice->state.dirty |= dirty_flag_for_cache(cache_id);
 196    }
 197 }
 198
 199 const void *
 200 iris_find_previous_compile(const struct iris_context *ice,
 201                            enum iris_program_cache_id cache_id,
 202                            unsigned program_string_id)
 203 {
 204    hash_table_foreach(ice->shaders.cache, entry) {
 205       const struct keybox *keybox = entry->key;
 206       if (keybox->cache_id == cache_id &&
 207           get_program_string_id(cache_id, keybox->data) == program_string_id) {
 208          return keybox->data;
 209       }
 210    }
 211
 212    return NULL;
 213 }
 214
 215 /**
 216  * Look for an existing entry in the cache that has identical assembly code.
 217  *
 218  * This is useful for programs generating shaders at runtime, where multiple
 219  * distinct shaders (from an API perspective) may compile to the same assembly
 220  * in our backend.  This saves space in the program cache buffer.
 221  */
 222 static const struct iris_compiled_shader *
 223 find_existing_assembly(struct hash_table *cache,
 224                        const void *assembly,
 225                        unsigned assembly_size)
 226 {
 227    hash_table_foreach(cache, entry) {
 228       const struct iris_compiled_shader *existing = entry->data;
 229       if (existing->prog_data->program_size == assembly_size &&
 230           memcmp(existing->map, assembly, assembly_size) == 0)
 231          return existing;
 232    }
 233    return NULL;
 234 }
 235
 236 static struct iris_compiled_shader *
 237 iris_upload_shader(struct iris_context *ice,
 238                    enum iris_program_cache_id cache_id,
 239                    uint32_t key_size,
 240                    const void *key,
 241                    const void *assembly,
 242                    struct brw_stage_prog_data *prog_data,
 243                    uint32_t *streamout)
 244 {
 245    struct iris_screen *screen = (void *) ice->ctx.screen;
 246    struct gen_device_info *devinfo = &screen->devinfo;
 247    struct hash_table *cache = ice->shaders.cache;
 248    struct iris_compiled_shader *shader =
 249       rzalloc_size(cache, sizeof(struct iris_compiled_shader) +
 250                    ice->vtbl.derived_program_state_size(cache_id));
 251    const struct iris_compiled_shader *existing =
 252       find_existing_assembly(cache, assembly, prog_data->program_size);
 253
 254    /* If we can find a matching prog in the cache already, then reuse the
 255     * existing stuff without creating new copy into the underlying buffer
 256     * object.  This is notably useful for programs generating shaders at
 257     * runtime, where multiple shaders may compile to the same thing in our
 258     * backend.
 259     */
 260    if (existing) {
 261       pipe_resource_reference(&shader->assembly.res, existing->assembly.res);
 262       shader->assembly.offset = existing->assembly.offset;
 263       shader->map = existing->map;
 264    } else {
 265       shader->assembly.res = NULL;
 266       u_upload_alloc(ice->shaders.uploader, 0, prog_data->program_size, 64,
 267                      &shader->assembly.offset, &shader->assembly.res,
 268                      &shader->map);
 269       memcpy(shader->map, assembly, prog_data->program_size);
 270    }
 271
 272    shader->prog_data = prog_data;
 273    shader->streamout = streamout;
 274
 275    ralloc_steal(shader, shader->prog_data);
 276    ralloc_steal(shader->prog_data, prog_data->param);
 277    ralloc_steal(shader->prog_data, prog_data->pull_param);
 278    ralloc_steal(shader, shader->streamout);
 279
 280    /* Store the 3DSTATE shader packets and other derived state. */
 281    ice->vtbl.store_derived_program_state(devinfo, cache_id, shader);
 282
 283    struct keybox *keybox = make_keybox(cache, cache_id, key, key_size);
 284    _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
 285
 286    return shader;
 287 }
 288
 289 /**
 290  * Upload a new shader to the program cache, and bind it for use.
 291  *
 292  * \param prog_data must be ralloc'd and will be stolen.
 293  */
 294 void
 295 iris_upload_and_bind_shader(struct iris_context *ice,
 296                             enum iris_program_cache_id cache_id,
 297                             const void *key,
 298                             const void *assembly,
 299                             struct brw_stage_prog_data *prog_data,
 300                             uint32_t *streamout)
 301 {
 302    assert(cache_id != IRIS_CACHE_BLORP);
 303
 304    struct iris_compiled_shader *shader =
 305       iris_upload_shader(ice, cache_id, key_size_for_cache(cache_id), key,
 306                          assembly, prog_data, streamout);
 307
 308    ice->shaders.prog[cache_id] = shader;
 309    ice->state.dirty |= dirty_flag_for_cache(cache_id);
 310 }
 311
 312 bool
 313 iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
 314                          const void *key, uint32_t key_size,
 315                          uint32_t *kernel_out, void *prog_data_out)
 316 {
 317    struct blorp_context *blorp = blorp_batch->blorp;
 318    struct iris_context *ice = blorp->driver_ctx;
 319    struct iris_batch *batch = blorp_batch->driver_batch;
 320    struct iris_compiled_shader *shader =
 321       iris_find_cached_shader(ice, IRIS_CACHE_BLORP, key, key_size);
 322
 323    if (!shader)
 324       return false;
 325
 326    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 327    *kernel_out =
 328       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 329    *((void **) prog_data_out) = shader->prog_data;
 330
 331    iris_use_pinned_bo(batch, bo, false);
 332
 333    return true;
 334 }
 335
 336 bool
 337 iris_blorp_upload_shader(struct blorp_batch *blorp_batch,
 338                          const void *key, uint32_t key_size,
 339                          const void *kernel, UNUSED uint32_t kernel_size,
 340                          const struct brw_stage_prog_data *prog_data_templ,
 341                          UNUSED uint32_t prog_data_size,
 342                          uint32_t *kernel_out, void *prog_data_out)
 343 {
 344    struct blorp_context *blorp = blorp_batch->blorp;
 345    struct iris_context *ice = blorp->driver_ctx;
 346    struct iris_batch *batch = blorp_batch->driver_batch;
 347
 348    void *prog_data = ralloc_size(NULL, prog_data_size);
 349    memcpy(prog_data, prog_data_templ, prog_data_size);
 350
 351    struct iris_compiled_shader *shader =
 352       iris_upload_shader(ice, IRIS_CACHE_BLORP, key_size, key, kernel,
 353                          prog_data, NULL);
 354
 355    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 356    *kernel_out =
 357       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 358    *((void **) prog_data_out) = shader->prog_data;
 359
 360    iris_use_pinned_bo(batch, bo, false);
 361
 362    return true;
 363 }
 364
 365 void
 366 iris_init_program_cache(struct iris_context *ice)
 367 {
 368    ice->shaders.cache =
 369       _mesa_hash_table_create(ice, keybox_hash, keybox_equals);
 370
 371    ice->shaders.uploader =
 372       u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
 373                       IRIS_RESOURCE_FLAG_SHADER_MEMZONE);
 374 }
 375
 376 void
 377 iris_destroy_program_cache(struct iris_context *ice)
 378 {
 379    for (int i = 0; i < MESA_SHADER_STAGES; i++) {
 380       ice->shaders.prog[i] = NULL;
 381    }
 382
 383    hash_table_foreach(ice->shaders.cache, entry) {
 384       struct iris_compiled_shader *shader = entry->data;
 385       pipe_resource_reference(&shader->assembly.res, NULL);
 386    }
 387
 388    u_upload_destroy(ice->shaders.uploader);
 389
 390    ralloc_free(ice->shaders.cache);
 391 }
 392
 393 static const char *
 394 cache_name(enum iris_program_cache_id cache_id)
 395 {
 396    if (cache_id == IRIS_CACHE_BLORP)
 397       return "BLORP";
 398
 399    return _mesa_shader_stage_to_string(cache_id);
 400 }
 401
 402 void
 403 iris_print_program_cache(struct iris_context *ice)
 404 {
 405    struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
 406    const struct gen_device_info *devinfo = &screen->devinfo;
 407
 408    hash_table_foreach(ice->shaders.cache, entry) {
 409       const struct keybox *keybox = entry->key;
 410       struct iris_compiled_shader *shader = entry->data;
 411       fprintf(stderr, "%s:\n", cache_name(keybox->cache_id));
 412       brw_disassemble(devinfo, shader->map, 0,
 413                       shader->prog_data->program_size, stderr);
 414    }
 415 }