src/mesa/drivers/dri/i965/brw_disk_cache.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "compiler/blob.h"
  25 #include "compiler/glsl/ir_uniform.h"
  26 #include "compiler/glsl/shader_cache.h"
  27 #include "main/mtypes.h"
  28 #include "util/build_id.h"
  29 #include "util/debug.h"
  30 #include "util/disk_cache.h"
  31 #include "util/macros.h"
  32 #include "util/mesa-sha1.h"
  33
  34 #include "compiler/brw_eu.h"
  35 #include "common/gen_debug.h"
  36
  37 #include "brw_context.h"
  38 #include "brw_program.h"
  39 #include "brw_cs.h"
  40 #include "brw_gs.h"
  41 #include "brw_state.h"
  42 #include "brw_vs.h"
  43 #include "brw_wm.h"
  44
  45 static bool
  46 debug_enabled_for_stage(gl_shader_stage stage)
  47 {
  48    static const uint64_t stage_debug_flags[] = {
  49       DEBUG_VS, DEBUG_TCS, DEBUG_TES, DEBUG_GS, DEBUG_WM, DEBUG_CS,
  50    };
  51    assert((int)stage >= 0 && stage < ARRAY_SIZE(stage_debug_flags));
  52    return (INTEL_DEBUG & stage_debug_flags[stage]) != 0;
  53 }
  54
  55 static void
  56 gen_shader_sha1(struct brw_context *brw, struct gl_program *prog,
  57                 gl_shader_stage stage, void *key, unsigned char *out_sha1)
  58 {
  59    char sha1_buf[41];
  60    unsigned char sha1[20];
  61    char manifest[256];
  62    int offset = 0;
  63
  64    _mesa_sha1_format(sha1_buf, prog->sh.data->sha1);
  65    offset += snprintf(manifest, sizeof(manifest), "program: %s\n", sha1_buf);
  66
  67    _mesa_sha1_compute(key, brw_prog_key_size(stage), sha1);
  68    _mesa_sha1_format(sha1_buf, sha1);
  69    offset += snprintf(manifest + offset, sizeof(manifest) - offset,
  70                       "%s_key: %s\n", _mesa_shader_stage_to_abbrev(stage),
  71                       sha1_buf);
  72
  73    _mesa_sha1_compute(manifest, strlen(manifest), out_sha1);
  74 }
  75
  76 static void
  77 write_blob_program_data(struct blob *binary, gl_shader_stage stage,
  78                         const void *program,
  79                         struct brw_stage_prog_data *prog_data)
  80 {
  81    /* Write prog_data to blob. */
  82    blob_write_bytes(binary, prog_data, brw_prog_data_size(stage));
  83
  84    /* Write program to blob. */
  85    blob_write_bytes(binary, program, prog_data->program_size);
  86
  87    /* Write push params */
  88    blob_write_bytes(binary, prog_data->param,
  89                     sizeof(uint32_t) * prog_data->nr_params);
  90
  91    /* Write pull params */
  92    blob_write_bytes(binary, prog_data->pull_param,
  93                     sizeof(uint32_t) * prog_data->nr_pull_params);
  94 }
  95
  96 static bool
  97 read_blob_program_data(struct blob_reader *binary, struct gl_program *prog,
  98                        gl_shader_stage stage, const uint8_t **program,
  99                        struct brw_stage_prog_data *prog_data)
 100 {
 101    /* Read shader prog_data from blob. */
 102    blob_copy_bytes(binary, prog_data, brw_prog_data_size(stage));
 103    if (binary->overrun)
 104       return false;
 105
 106    /* Read shader program from blob. */
 107    *program = blob_read_bytes(binary, prog_data->program_size);
 108
 109    /* Read push params */
 110    prog_data->param = rzalloc_array(NULL, uint32_t, prog_data->nr_params);
 111    blob_copy_bytes(binary, prog_data->param,
 112                    sizeof(uint32_t) * prog_data->nr_params);
 113
 114    /* Read pull params */
 115    prog_data->pull_param = rzalloc_array(NULL, uint32_t,
 116                                          prog_data->nr_pull_params);
 117    blob_copy_bytes(binary, prog_data->pull_param,
 118                    sizeof(uint32_t) * prog_data->nr_pull_params);
 119
 120    return (binary->current == binary->end && !binary->overrun);
 121 }
 122
 123 static bool
 124 read_and_upload(struct brw_context *brw, struct disk_cache *cache,
 125                 struct gl_program *prog, gl_shader_stage stage)
 126 {
 127    unsigned char binary_sha1[20];
 128
 129    union brw_any_prog_key prog_key;
 130
 131    switch (stage) {
 132    case MESA_SHADER_VERTEX:
 133       brw_vs_populate_key(brw, &prog_key.vs);
 134       /* We don't care what instance of the program it is for the disk cache
 135        * hash lookup, so set the id to 0 for the sha1 hashing.
 136        * program_string_id will be set below.
 137        */
 138       prog_key.vs.program_string_id = 0;
 139       break;
 140    case MESA_SHADER_TESS_CTRL:
 141       brw_tcs_populate_key(brw, &prog_key.tcs);
 142       prog_key.tcs.program_string_id = 0;
 143       break;
 144    case MESA_SHADER_TESS_EVAL:
 145       brw_tes_populate_key(brw, &prog_key.tes);
 146       prog_key.tes.program_string_id = 0;
 147       break;
 148    case MESA_SHADER_GEOMETRY:
 149       brw_gs_populate_key(brw, &prog_key.gs);
 150       prog_key.gs.program_string_id = 0;
 151       break;
 152    case MESA_SHADER_FRAGMENT:
 153       brw_wm_populate_key(brw, &prog_key.wm);
 154       prog_key.wm.program_string_id = 0;
 155       break;
 156    case MESA_SHADER_COMPUTE:
 157       brw_cs_populate_key(brw, &prog_key.cs);
 158       prog_key.cs.program_string_id = 0;
 159       break;
 160    default:
 161       unreachable("Unsupported stage!");
 162    }
 163
 164    gen_shader_sha1(brw, prog, stage, &prog_key, binary_sha1);
 165
 166    size_t buffer_size;
 167    uint8_t *buffer = disk_cache_get(cache, binary_sha1, &buffer_size);
 168    if (buffer == NULL) {
 169       if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 170          char sha1_buf[41];
 171          _mesa_sha1_format(sha1_buf, binary_sha1);
 172          fprintf(stderr, "No cached %s binary found for: %s\n",
 173                  _mesa_shader_stage_to_abbrev(stage), sha1_buf);
 174       }
 175       return false;
 176    }
 177
 178    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 179       char sha1_buf[41];
 180       _mesa_sha1_format(sha1_buf, binary_sha1);
 181       fprintf(stderr, "attempting to populate bo cache with binary: %s\n",
 182               sha1_buf);
 183    }
 184
 185    struct blob_reader binary;
 186    blob_reader_init(&binary, buffer, buffer_size);
 187
 188    const uint8_t *program;
 189    struct brw_stage_prog_data *prog_data =
 190       ralloc_size(NULL, sizeof(union brw_any_prog_data));
 191    if (!read_blob_program_data(&binary, prog, stage, &program, prog_data)) {
 192       /* Something very bad has gone wrong discard the item from the cache and
 193        * rebuild from source.
 194        */
 195       if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 196          fprintf(stderr, "Error reading program from cache (invalid i965 "
 197                  "cache item)\n");
 198       }
 199
 200       disk_cache_remove(cache, binary_sha1);
 201       ralloc_free(prog_data);
 202       free(buffer);
 203       return false;
 204    }
 205
 206    enum brw_cache_id cache_id;
 207    struct brw_stage_state *stage_state;
 208
 209    switch (stage) {
 210    case MESA_SHADER_VERTEX:
 211       prog_key.vs.program_string_id = brw_program(prog)->id;
 212       cache_id = BRW_CACHE_VS_PROG;
 213       stage_state = &brw->vs.base;
 214       break;
 215    case MESA_SHADER_TESS_CTRL:
 216       prog_key.tcs.program_string_id = brw_program(prog)->id;
 217       cache_id = BRW_CACHE_TCS_PROG;
 218       stage_state = &brw->tcs.base;
 219       break;
 220    case MESA_SHADER_TESS_EVAL:
 221       prog_key.tes.program_string_id = brw_program(prog)->id;
 222       cache_id = BRW_CACHE_TES_PROG;
 223       stage_state = &brw->tes.base;
 224       break;
 225    case MESA_SHADER_GEOMETRY:
 226       prog_key.gs.program_string_id = brw_program(prog)->id;
 227       cache_id = BRW_CACHE_GS_PROG;
 228       stage_state = &brw->gs.base;
 229       break;
 230    case MESA_SHADER_FRAGMENT:
 231       prog_key.wm.program_string_id = brw_program(prog)->id;
 232       cache_id = BRW_CACHE_FS_PROG;
 233       stage_state = &brw->wm.base;
 234       break;
 235    case MESA_SHADER_COMPUTE:
 236       prog_key.cs.program_string_id = brw_program(prog)->id;
 237       cache_id = BRW_CACHE_CS_PROG;
 238       stage_state = &brw->cs.base;
 239       break;
 240    default:
 241       unreachable("Unsupported stage!");
 242    }
 243
 244    brw_alloc_stage_scratch(brw, stage_state, prog_data->total_scratch);
 245
 246    if (unlikely(debug_enabled_for_stage(stage))) {
 247       fprintf(stderr, "NIR for %s program %d loaded from disk shader cache:\n",
 248               _mesa_shader_stage_to_abbrev(stage), brw_program(prog)->id);
 249       brw_program_deserialize_nir(&brw->ctx, prog, stage);
 250       nir_shader *nir = prog->nir;
 251       nir_print_shader(nir, stderr);
 252       fprintf(stderr, "Native code for %s %s shader %s from disk cache:\n",
 253               nir->info.label ? nir->info.label : "unnamed",
 254               _mesa_shader_stage_to_string(nir->info.stage), nir->info.name);
 255       brw_disassemble(&brw->screen->devinfo, program, 0,
 256                       prog_data->program_size, stderr);
 257    }
 258
 259    brw_upload_cache(&brw->cache, cache_id, &prog_key, brw_prog_key_size(stage),
 260                     program, prog_data->program_size, prog_data,
 261                     brw_prog_data_size(stage), &stage_state->prog_offset,
 262                     &stage_state->prog_data);
 263
 264    prog->program_written_to_cache = true;
 265
 266    ralloc_free(prog_data);
 267    free(buffer);
 268
 269    return true;
 270 }
 271
 272 bool
 273 brw_disk_cache_upload_program(struct brw_context *brw, gl_shader_stage stage)
 274 {
 275    struct disk_cache *cache = brw->ctx.Cache;
 276    if (cache == NULL)
 277       return false;
 278
 279    struct gl_program *prog = brw->ctx._Shader->CurrentProgram[stage];
 280    if (prog == NULL)
 281       return false;
 282
 283    if (brw->ctx._Shader->Flags & GLSL_CACHE_FALLBACK)
 284       goto fail;
 285
 286    if (!read_and_upload(brw, cache, prog, stage))
 287       goto fail;
 288
 289    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 290       fprintf(stderr, "read gen program from cache\n");
 291    }
 292
 293    return true;
 294
 295 fail:
 296    prog->program_written_to_cache = false;
 297    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 298       fprintf(stderr, "falling back to nir %s.\n",
 299               _mesa_shader_stage_to_abbrev(prog->info.stage));
 300    }
 301
 302    brw_program_deserialize_nir(&brw->ctx, prog, stage);
 303
 304    return false;
 305 }
 306
 307 static void
 308 write_program_data(struct brw_context *brw, struct gl_program *prog,
 309                    void *key, struct brw_stage_prog_data *prog_data,
 310                    uint32_t prog_offset, struct disk_cache *cache,
 311                    gl_shader_stage stage)
 312 {
 313    struct blob binary;
 314    blob_init(&binary);
 315
 316    const void *program_map = brw->cache.map + prog_offset;
 317    /* TODO: Improve perf for non-LLC. It would be best to save it at program
 318     * generation time when the program is in normal memory accessible with
 319     * cache to the CPU. Another easier change would be to use
 320     * _mesa_streaming_load_memcpy to read from the program mapped memory. */
 321    write_blob_program_data(&binary, stage, program_map, prog_data);
 322
 323    unsigned char sha1[20];
 324    char buf[41];
 325    gen_shader_sha1(brw, prog, stage, key, sha1);
 326    _mesa_sha1_format(buf, sha1);
 327    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 328       fprintf(stderr, "putting binary in cache: %s\n", buf);
 329    }
 330
 331    disk_cache_put(cache, sha1, binary.data, binary.size, NULL);
 332
 333    prog->program_written_to_cache = true;
 334    blob_finish(&binary);
 335 }
 336
 337 void
 338 brw_disk_cache_write_render_programs(struct brw_context *brw)
 339 {
 340    struct disk_cache *cache = brw->ctx.Cache;
 341    if (cache == NULL)
 342       return;
 343
 344    struct gl_program *prog =
 345       brw->ctx._Shader->CurrentProgram[MESA_SHADER_VERTEX];
 346    if (prog && !prog->program_written_to_cache) {
 347       struct brw_vs_prog_key vs_key;
 348       brw_vs_populate_key(brw, &vs_key);
 349       vs_key.program_string_id = 0;
 350
 351       write_program_data(brw, prog, &vs_key, brw->vs.base.prog_data,
 352                          brw->vs.base.prog_offset, cache,
 353                          MESA_SHADER_VERTEX);
 354    }
 355
 356    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_CTRL];
 357    if (prog && !prog->program_written_to_cache) {
 358       struct brw_tcs_prog_key tcs_key;
 359       brw_tcs_populate_key(brw, &tcs_key);
 360       tcs_key.program_string_id = 0;
 361
 362       write_program_data(brw, prog, &tcs_key, brw->tcs.base.prog_data,
 363                          brw->tcs.base.prog_offset, cache,
 364                          MESA_SHADER_TESS_CTRL);
 365    }
 366
 367    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_EVAL];
 368    if (prog && !prog->program_written_to_cache) {
 369       struct brw_tes_prog_key tes_key;
 370       brw_tes_populate_key(brw, &tes_key);
 371       tes_key.program_string_id = 0;
 372
 373       write_program_data(brw, prog, &tes_key, brw->tes.base.prog_data,
 374                          brw->tes.base.prog_offset, cache,
 375                          MESA_SHADER_TESS_EVAL);
 376    }
 377
 378    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_GEOMETRY];
 379    if (prog && !prog->program_written_to_cache) {
 380       struct brw_gs_prog_key gs_key;
 381       brw_gs_populate_key(brw, &gs_key);
 382       gs_key.program_string_id = 0;
 383
 384       write_program_data(brw, prog, &gs_key, brw->gs.base.prog_data,
 385                          brw->gs.base.prog_offset, cache,
 386                          MESA_SHADER_GEOMETRY);
 387    }
 388
 389    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_FRAGMENT];
 390    if (prog && !prog->program_written_to_cache) {
 391       struct brw_wm_prog_key wm_key;
 392       brw_wm_populate_key(brw, &wm_key);
 393       wm_key.program_string_id = 0;
 394
 395       write_program_data(brw, prog, &wm_key, brw->wm.base.prog_data,
 396                          brw->wm.base.prog_offset, cache,
 397                          MESA_SHADER_FRAGMENT);
 398    }
 399 }
 400
 401 void
 402 brw_disk_cache_write_compute_program(struct brw_context *brw)
 403 {
 404    struct disk_cache *cache = brw->ctx.Cache;
 405    if (cache == NULL)
 406       return;
 407
 408    struct gl_program *prog =
 409       brw->ctx._Shader->CurrentProgram[MESA_SHADER_COMPUTE];
 410    if (prog && !prog->program_written_to_cache) {
 411       struct brw_cs_prog_key cs_key;
 412       brw_cs_populate_key(brw, &cs_key);
 413       cs_key.program_string_id = 0;
 414
 415       write_program_data(brw, prog, &cs_key, brw->cs.base.prog_data,
 416                          brw->cs.base.prog_offset, cache,
 417                          MESA_SHADER_COMPUTE);
 418    }
 419 }
 420
 421 void
 422 brw_disk_cache_init(struct intel_screen *screen)
 423 {
 424 #ifdef ENABLE_SHADER_CACHE
 425    char renderer[10];
 426    MAYBE_UNUSED int len = snprintf(renderer, sizeof(renderer), "i965_%04x",
 427                                    screen->deviceID);
 428    assert(len == sizeof(renderer) - 1);
 429
 430    const struct build_id_note *note =
 431       build_id_find_nhdr_for_addr(brw_disk_cache_init);
 432    assert(note && build_id_length(note) == 20 /* sha1 */);
 433
 434    const uint8_t *id_sha1 = build_id_data(note);
 435    assert(id_sha1);
 436
 437    char timestamp[41];
 438    _mesa_sha1_format(timestamp, id_sha1);
 439
 440    screen->disk_cache = disk_cache_create(renderer, timestamp, 0);
 441 #endif
 442 }