src/gallium/drivers/iris/iris_program_cache.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23 #include <stdio.h>
  24 #include <errno.h>
  25 #include "pipe/p_defines.h"
  26 #include "pipe/p_state.h"
  27 #include "pipe/p_context.h"
  28 #include "pipe/p_screen.h"
  29 #include "util/u_atomic.h"
  30 #include "util/u_upload_mgr.h"
  31 #include "compiler/nir/nir.h"
  32 #include "compiler/nir/nir_builder.h"
  33 #include "intel/compiler/brw_compiler.h"
  34 #include "intel/compiler/brw_eu.h"
  35 #include "intel/compiler/brw_nir.h"
  36 #include "iris_context.h"
  37 #include "iris_resource.h"
  38
  39 struct keybox {
  40    uint8_t size;
  41    enum iris_program_cache_id cache_id;
  42    uint8_t data[0];
  43 };
  44
  45 static uint32_t
  46 key_size_for_cache(enum iris_program_cache_id cache_id)
  47 {
  48    static const unsigned key_sizes[] = {
  49       [IRIS_CACHE_VS]         = sizeof(struct brw_vs_prog_key),
  50       [IRIS_CACHE_TCS]        = sizeof(struct brw_tcs_prog_key),
  51       [IRIS_CACHE_TES]        = sizeof(struct brw_tes_prog_key),
  52       [IRIS_CACHE_GS]         = sizeof(struct brw_gs_prog_key),
  53       [IRIS_CACHE_FS]         = sizeof(struct brw_wm_prog_key),
  54       [IRIS_CACHE_CS]         = sizeof(struct brw_cs_prog_key),
  55    };
  56
  57    /* BLORP keys aren't all the same size. */
  58    assert(cache_id != IRIS_CACHE_BLORP);
  59
  60    return key_sizes[cache_id];
  61 }
  62
  63 static struct keybox *
  64 make_keybox(void *mem_ctx,
  65             enum iris_program_cache_id cache_id,
  66             const void *key,
  67             uint32_t key_size)
  68 {
  69    struct keybox *keybox =
  70       ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
  71
  72    keybox->cache_id = cache_id;
  73    keybox->size = key_size;
  74    memcpy(keybox->data, key, key_size);
  75
  76    return keybox;
  77 }
  78
  79 static uint32_t
  80 keybox_hash(const void *void_key)
  81 {
  82    const struct keybox *key = void_key;
  83    return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
  84 }
  85
  86 static bool
  87 keybox_equals(const void *void_a, const void *void_b)
  88 {
  89    const struct keybox *a = void_a, *b = void_b;
  90    if (a->size != b->size)
  91       return false;
  92
  93    return memcmp(a->data, b->data, a->size) == 0;
  94 }
  95
  96 static uint64_t
  97 dirty_flag_for_cache(enum iris_program_cache_id cache_id)
  98 {
  99    assert(cache_id <= MESA_SHADER_STAGES);
 100
 101    // XXX: ugly...
 102    // XXX: move this flagging out to a higher level, allow comparison of
 103    // XXX: new and old programs to decide what bits to twiddle
 104    // XXX: CLIP: toggle if barycentric modes has any NONPERSPECTIVE or not
 105    if (cache_id == IRIS_CACHE_FS)
 106       return IRIS_DIRTY_WM | IRIS_DIRTY_FS | IRIS_DIRTY_CLIP;
 107    if (cache_id == IRIS_CACHE_VS)
 108       return IRIS_DIRTY_VS | IRIS_DIRTY_VF_SGVS;
 109
 110    return IRIS_DIRTY_VS << cache_id;
 111 }
 112
 113 static unsigned
 114 get_program_string_id(enum iris_program_cache_id cache_id, const void *key)
 115 {
 116    switch (cache_id) {
 117    case IRIS_CACHE_VS:
 118       return ((struct brw_vs_prog_key *) key)->program_string_id;
 119    case IRIS_CACHE_TCS:
 120       return ((struct brw_tcs_prog_key *) key)->program_string_id;
 121    case IRIS_CACHE_TES:
 122       return ((struct brw_tes_prog_key *) key)->program_string_id;
 123    case IRIS_CACHE_GS:
 124       return ((struct brw_gs_prog_key *) key)->program_string_id;
 125    case IRIS_CACHE_CS:
 126       return ((struct brw_cs_prog_key *) key)->program_string_id;
 127    case IRIS_CACHE_FS:
 128       return ((struct brw_wm_prog_key *) key)->program_string_id;
 129    default:
 130       unreachable("no program string id for this kind of program");
 131    }
 132 }
 133
 134 static struct iris_compiled_shader *
 135 iris_find_cached_shader(struct iris_context *ice,
 136                         enum iris_program_cache_id cache_id,
 137                         const void *key,
 138                         uint32_t key_size)
 139 {
 140    struct keybox *keybox =
 141       make_keybox(ice->shaders.cache, cache_id, key, key_size);
 142    struct hash_entry *entry =
 143       _mesa_hash_table_search(ice->shaders.cache, keybox);
 144
 145    ralloc_free(keybox);
 146
 147    return entry ? entry->data : NULL;
 148 }
 149
 150 /**
 151  * Looks for a program in the cache and binds it.
 152  *
 153  * If no program was found, returns false and leaves the binding alone.
 154  */
 155 bool
 156 iris_bind_cached_shader(struct iris_context *ice,
 157                         enum iris_program_cache_id cache_id,
 158                         const void *key)
 159 {
 160    unsigned key_size = key_size_for_cache(cache_id);
 161    struct iris_compiled_shader *shader =
 162       iris_find_cached_shader(ice, cache_id, key, key_size);
 163
 164    if (!shader)
 165       return false;
 166
 167    if (memcmp(shader, ice->shaders.prog[cache_id], sizeof(*shader)) != 0) {
 168       ice->shaders.prog[cache_id] = shader;
 169       ice->state.dirty |= dirty_flag_for_cache(cache_id);
 170    }
 171
 172    return true;
 173 }
 174
 175 void
 176 iris_unbind_shader(struct iris_context *ice,
 177                    enum iris_program_cache_id cache_id)
 178 {
 179    if (ice->shaders.prog[cache_id]) {
 180       ice->shaders.prog[cache_id] = NULL;
 181       ice->state.dirty |= dirty_flag_for_cache(cache_id);
 182    }
 183 }
 184
 185 const void *
 186 iris_find_previous_compile(const struct iris_context *ice,
 187                            enum iris_program_cache_id cache_id,
 188                            unsigned program_string_id)
 189 {
 190    hash_table_foreach(ice->shaders.cache, entry) {
 191       const struct keybox *keybox = entry->key;
 192       if (keybox->cache_id == cache_id &&
 193           get_program_string_id(cache_id, keybox->data) == program_string_id) {
 194          return keybox->data;
 195       }
 196    }
 197
 198    return NULL;
 199 }
 200
 201 /**
 202  * Look for an existing entry in the cache that has identical assembly code.
 203  *
 204  * This is useful for programs generating shaders at runtime, where multiple
 205  * distinct shaders (from an API perspective) may compile to the same assembly
 206  * in our backend.  This saves space in the program cache buffer.
 207  */
 208 static const struct iris_compiled_shader *
 209 find_existing_assembly(struct hash_table *cache,
 210                        const void *assembly,
 211                        unsigned assembly_size)
 212 {
 213    hash_table_foreach(cache, entry) {
 214       const struct iris_compiled_shader *existing = entry->data;
 215       if (existing->prog_data->program_size == assembly_size &&
 216           memcmp(existing->map, assembly, assembly_size) == 0)
 217          return existing;
 218    }
 219    return NULL;
 220 }
 221
 222 static struct iris_compiled_shader *
 223 iris_upload_shader(struct iris_context *ice,
 224                    enum iris_program_cache_id cache_id,
 225                    uint32_t key_size,
 226                    const void *key,
 227                    const void *assembly,
 228                    struct brw_stage_prog_data *prog_data,
 229                    uint32_t *streamout)
 230 {
 231    struct iris_screen *screen = (void *) ice->ctx.screen;
 232    struct gen_device_info *devinfo = &screen->devinfo;
 233    struct hash_table *cache = ice->shaders.cache;
 234    struct iris_compiled_shader *shader =
 235       rzalloc_size(cache, sizeof(struct iris_compiled_shader) +
 236                    ice->vtbl.derived_program_state_size(cache_id));
 237    const struct iris_compiled_shader *existing =
 238       find_existing_assembly(cache, assembly, prog_data->program_size);
 239
 240    /* If we can find a matching prog in the cache already, then reuse the
 241     * existing stuff without creating new copy into the underlying buffer
 242     * object.  This is notably useful for programs generating shaders at
 243     * runtime, where multiple shaders may compile to the same thing in our
 244     * backend.
 245     */
 246    if (existing) {
 247       pipe_resource_reference(&shader->assembly.res, existing->assembly.res);
 248       shader->assembly.offset = existing->assembly.offset;
 249       shader->map = existing->map;
 250    } else {
 251       shader->assembly.res = NULL;
 252       u_upload_alloc(ice->shaders.uploader, 0, prog_data->program_size, 64,
 253                      &shader->assembly.offset, &shader->assembly.res,
 254                      &shader->map);
 255       memcpy(shader->map, assembly, prog_data->program_size);
 256    }
 257
 258    shader->prog_data = prog_data;
 259    shader->streamout = streamout;
 260
 261    ralloc_steal(shader, shader->prog_data);
 262    ralloc_steal(shader->prog_data, prog_data->param);
 263    ralloc_steal(shader->prog_data, prog_data->pull_param);
 264    ralloc_steal(shader, shader->streamout);
 265
 266    /* Store the 3DSTATE shader packets and other derived state. */
 267    ice->vtbl.store_derived_program_state(devinfo, cache_id, shader);
 268
 269    struct keybox *keybox = make_keybox(cache, cache_id, key, key_size);
 270    _mesa_hash_table_insert(ice->shaders.cache, keybox, shader);
 271
 272    return shader;
 273 }
 274
 275 /**
 276  * Upload a new shader to the program cache, and bind it for use.
 277  *
 278  * \param prog_data must be ralloc'd and will be stolen.
 279  */
 280 void
 281 iris_upload_and_bind_shader(struct iris_context *ice,
 282                             enum iris_program_cache_id cache_id,
 283                             const void *key,
 284                             const void *assembly,
 285                             struct brw_stage_prog_data *prog_data,
 286                             uint32_t *streamout)
 287 {
 288    assert(cache_id != IRIS_CACHE_BLORP);
 289
 290    struct iris_compiled_shader *shader =
 291       iris_upload_shader(ice, cache_id, key_size_for_cache(cache_id), key,
 292                          assembly, prog_data, streamout);
 293
 294    ice->shaders.prog[cache_id] = shader;
 295    ice->state.dirty |= dirty_flag_for_cache(cache_id);
 296 }
 297
 298 bool
 299 iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
 300                          const void *key, uint32_t key_size,
 301                          uint32_t *kernel_out, void *prog_data_out)
 302 {
 303    struct blorp_context *blorp = blorp_batch->blorp;
 304    struct iris_context *ice = blorp->driver_ctx;
 305    struct iris_batch *batch = blorp_batch->driver_batch;
 306    struct iris_compiled_shader *shader =
 307       iris_find_cached_shader(ice, IRIS_CACHE_BLORP, key, key_size);
 308
 309    if (!shader)
 310       return false;
 311
 312    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 313    *kernel_out =
 314       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 315    *((void **) prog_data_out) = shader->prog_data;
 316
 317    iris_use_pinned_bo(batch, bo, false);
 318
 319    return true;
 320 }
 321
 322 bool
 323 iris_blorp_upload_shader(struct blorp_batch *blorp_batch,
 324                          const void *key, uint32_t key_size,
 325                          const void *kernel, UNUSED uint32_t kernel_size,
 326                          const struct brw_stage_prog_data *prog_data_templ,
 327                          UNUSED uint32_t prog_data_size,
 328                          uint32_t *kernel_out, void *prog_data_out)
 329 {
 330    struct blorp_context *blorp = blorp_batch->blorp;
 331    struct iris_context *ice = blorp->driver_ctx;
 332    struct iris_batch *batch = blorp_batch->driver_batch;
 333
 334    void *prog_data = ralloc_size(NULL, prog_data_size);
 335    memcpy(prog_data, prog_data_templ, prog_data_size);
 336
 337    struct iris_compiled_shader *shader =
 338       iris_upload_shader(ice, IRIS_CACHE_BLORP, key_size, key, kernel,
 339                          prog_data, NULL);
 340
 341    struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
 342    *kernel_out =
 343       iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
 344    *((void **) prog_data_out) = shader->prog_data;
 345
 346    iris_use_pinned_bo(batch, bo, false);
 347
 348    return true;
 349 }
 350
 351 void
 352 iris_init_program_cache(struct iris_context *ice)
 353 {
 354    ice->shaders.cache =
 355       _mesa_hash_table_create(ice, keybox_hash, keybox_equals);
 356
 357    ice->shaders.uploader =
 358       u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
 359                       IRIS_RESOURCE_FLAG_SHADER_MEMZONE);
 360 }
 361
 362 void
 363 iris_destroy_program_cache(struct iris_context *ice)
 364 {
 365    for (int i = 0; i < MESA_SHADER_STAGES; i++) {
 366       ice->shaders.prog[i] = NULL;
 367    }
 368
 369    hash_table_foreach(ice->shaders.cache, entry) {
 370       struct iris_compiled_shader *shader = entry->data;
 371       pipe_resource_reference(&shader->assembly.res, NULL);
 372    }
 373
 374    u_upload_destroy(ice->shaders.uploader);
 375
 376    ralloc_free(ice->shaders.cache);
 377 }
 378
 379 static const char *
 380 cache_name(enum iris_program_cache_id cache_id)
 381 {
 382    if (cache_id == IRIS_CACHE_BLORP)
 383       return "BLORP";
 384
 385    return _mesa_shader_stage_to_string(cache_id);
 386 }
 387
 388 void
 389 iris_print_program_cache(struct iris_context *ice)
 390 {
 391    struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
 392    const struct gen_device_info *devinfo = &screen->devinfo;
 393
 394    hash_table_foreach(ice->shaders.cache, entry) {
 395       const struct keybox *keybox = entry->key;
 396       struct iris_compiled_shader *shader = entry->data;
 397       fprintf(stderr, "%s:\n", cache_name(keybox->cache_id));
 398       brw_disassemble(devinfo, shader->map, 0,
 399                       shader->prog_data->program_size, stderr);
 400    }
 401 }