src/mesa/drivers/dri/i965/brw_disk_cache.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "compiler/blob.h"
  25 #include "compiler/glsl/ir_uniform.h"
  26 #include "compiler/glsl/shader_cache.h"
  27 #include "main/mtypes.h"
  28 #include "util/build_id.h"
  29 #include "util/debug.h"
  30 #include "util/disk_cache.h"
  31 #include "util/macros.h"
  32 #include "util/mesa-sha1.h"
  33
  34 #include "compiler/brw_eu.h"
  35 #include "dev/gen_debug.h"
  36
  37 #include "brw_context.h"
  38 #include "brw_program.h"
  39 #include "brw_cs.h"
  40 #include "brw_gs.h"
  41 #include "brw_state.h"
  42 #include "brw_vs.h"
  43 #include "brw_wm.h"
  44
  45 static bool
  46 debug_enabled_for_stage(gl_shader_stage stage)
  47 {
  48    static const uint64_t stage_debug_flags[] = {
  49       DEBUG_VS, DEBUG_TCS, DEBUG_TES, DEBUG_GS, DEBUG_WM, DEBUG_CS,
  50    };
  51    assert((int)stage >= 0 && stage < ARRAY_SIZE(stage_debug_flags));
  52    return (INTEL_DEBUG & stage_debug_flags[stage]) != 0;
  53 }
  54
  55 static void
  56 gen_shader_sha1(struct gl_program *prog, gl_shader_stage stage,
  57                 void *key, unsigned char *out_sha1)
  58 {
  59    char sha1_buf[41];
  60    unsigned char sha1[20];
  61    char manifest[256];
  62    int offset = 0;
  63
  64    _mesa_sha1_format(sha1_buf, prog->sh.data->sha1);
  65    offset += snprintf(manifest, sizeof(manifest), "program: %s\n", sha1_buf);
  66
  67    _mesa_sha1_compute(key, brw_prog_key_size(stage), sha1);
  68    _mesa_sha1_format(sha1_buf, sha1);
  69    offset += snprintf(manifest + offset, sizeof(manifest) - offset,
  70                       "%s_key: %s\n", _mesa_shader_stage_to_abbrev(stage),
  71                       sha1_buf);
  72
  73    _mesa_sha1_compute(manifest, strlen(manifest), out_sha1);
  74 }
  75
  76 static bool
  77 read_blob_program_data(struct blob_reader *binary, struct gl_program *prog,
  78                        gl_shader_stage stage, const uint8_t **program,
  79                        struct brw_stage_prog_data *prog_data)
  80 {
  81    return
  82       brw_read_blob_program_data(binary, prog, stage, program, prog_data) &&
  83       (binary->current == binary->end);
  84 }
  85
  86 static bool
  87 read_and_upload(struct brw_context *brw, struct disk_cache *cache,
  88                 struct gl_program *prog, gl_shader_stage stage)
  89 {
  90    unsigned char binary_sha1[20];
  91
  92    union brw_any_prog_key prog_key;
  93
  94    switch (stage) {
  95    case MESA_SHADER_VERTEX:
  96       brw_vs_populate_key(brw, &prog_key.vs);
  97       break;
  98    case MESA_SHADER_TESS_CTRL:
  99       brw_tcs_populate_key(brw, &prog_key.tcs);
 100       break;
 101    case MESA_SHADER_TESS_EVAL:
 102       brw_tes_populate_key(brw, &prog_key.tes);
 103       break;
 104    case MESA_SHADER_GEOMETRY:
 105       brw_gs_populate_key(brw, &prog_key.gs);
 106       break;
 107    case MESA_SHADER_FRAGMENT:
 108       brw_wm_populate_key(brw, &prog_key.wm);
 109       break;
 110    case MESA_SHADER_COMPUTE:
 111       brw_cs_populate_key(brw, &prog_key.cs);
 112       break;
 113    default:
 114       unreachable("Unsupported stage!");
 115    }
 116
 117    /* We don't care what instance of the program it is for the disk cache hash
 118     * lookup, so set the id to 0 for the sha1 hashing. program_string_id will
 119     * be set below.
 120     */
 121    prog_key.base.program_string_id = 0;
 122
 123    gen_shader_sha1(prog, stage, &prog_key, binary_sha1);
 124
 125    size_t buffer_size;
 126    uint8_t *buffer = disk_cache_get(cache, binary_sha1, &buffer_size);
 127    if (buffer == NULL) {
 128       if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 129          char sha1_buf[41];
 130          _mesa_sha1_format(sha1_buf, binary_sha1);
 131          fprintf(stderr, "No cached %s binary found for: %s\n",
 132                  _mesa_shader_stage_to_abbrev(stage), sha1_buf);
 133       }
 134       return false;
 135    }
 136
 137    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 138       char sha1_buf[41];
 139       _mesa_sha1_format(sha1_buf, binary_sha1);
 140       fprintf(stderr, "attempting to populate bo cache with binary: %s\n",
 141               sha1_buf);
 142    }
 143
 144    struct blob_reader binary;
 145    blob_reader_init(&binary, buffer, buffer_size);
 146
 147    const uint8_t *program;
 148    struct brw_stage_prog_data *prog_data =
 149       ralloc_size(NULL, sizeof(union brw_any_prog_data));
 150    if (!read_blob_program_data(&binary, prog, stage, &program, prog_data)) {
 151       /* Something very bad has gone wrong discard the item from the cache and
 152        * rebuild from source.
 153        */
 154       if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 155          fprintf(stderr, "Error reading program from cache (invalid i965 "
 156                  "cache item)\n");
 157       }
 158
 159       disk_cache_remove(cache, binary_sha1);
 160       ralloc_free(prog_data);
 161       free(buffer);
 162       return false;
 163    }
 164
 165    enum brw_cache_id cache_id;
 166    struct brw_stage_state *stage_state;
 167
 168    switch (stage) {
 169    case MESA_SHADER_VERTEX:
 170       cache_id = BRW_CACHE_VS_PROG;
 171       stage_state = &brw->vs.base;
 172       break;
 173    case MESA_SHADER_TESS_CTRL:
 174       cache_id = BRW_CACHE_TCS_PROG;
 175       stage_state = &brw->tcs.base;
 176       break;
 177    case MESA_SHADER_TESS_EVAL:
 178       cache_id = BRW_CACHE_TES_PROG;
 179       stage_state = &brw->tes.base;
 180       break;
 181    case MESA_SHADER_GEOMETRY:
 182       cache_id = BRW_CACHE_GS_PROG;
 183       stage_state = &brw->gs.base;
 184       break;
 185    case MESA_SHADER_FRAGMENT:
 186       cache_id = BRW_CACHE_FS_PROG;
 187       stage_state = &brw->wm.base;
 188       break;
 189    case MESA_SHADER_COMPUTE:
 190       cache_id = BRW_CACHE_CS_PROG;
 191       stage_state = &brw->cs.base;
 192       break;
 193    default:
 194       unreachable("Unsupported stage!");
 195    }
 196
 197    prog_key.base.program_string_id = brw_program(prog)->id;
 198
 199    brw_alloc_stage_scratch(brw, stage_state, prog_data->total_scratch);
 200
 201    if (unlikely(debug_enabled_for_stage(stage))) {
 202       fprintf(stderr, "NIR for %s program %d loaded from disk shader cache:\n",
 203               _mesa_shader_stage_to_abbrev(stage), brw_program(prog)->id);
 204       brw_program_deserialize_driver_blob(&brw->ctx, prog, stage);
 205       nir_shader *nir = prog->nir;
 206       nir_print_shader(nir, stderr);
 207       fprintf(stderr, "Native code for %s %s shader %s from disk cache:\n",
 208               nir->info.label ? nir->info.label : "unnamed",
 209               _mesa_shader_stage_to_string(nir->info.stage), nir->info.name);
 210       brw_disassemble(&brw->screen->devinfo, program, 0,
 211                       prog_data->program_size, stderr);
 212    }
 213
 214    brw_upload_cache(&brw->cache, cache_id, &prog_key, brw_prog_key_size(stage),
 215                     program, prog_data->program_size, prog_data,
 216                     brw_prog_data_size(stage), &stage_state->prog_offset,
 217                     &stage_state->prog_data);
 218
 219    prog->program_written_to_cache = true;
 220
 221    ralloc_free(prog_data);
 222    free(buffer);
 223
 224    return true;
 225 }
 226
 227 bool
 228 brw_disk_cache_upload_program(struct brw_context *brw, gl_shader_stage stage)
 229 {
 230    struct disk_cache *cache = brw->ctx.Cache;
 231    if (cache == NULL)
 232       return false;
 233
 234    struct gl_program *prog = brw->ctx._Shader->CurrentProgram[stage];
 235    if (prog == NULL)
 236       return false;
 237
 238    if (prog->sh.data->spirv)
 239       return false;
 240
 241    if (brw->ctx._Shader->Flags & GLSL_CACHE_FALLBACK)
 242       goto fail;
 243
 244    if (!read_and_upload(brw, cache, prog, stage))
 245       goto fail;
 246
 247    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 248       fprintf(stderr, "read gen program from cache\n");
 249    }
 250
 251    return true;
 252
 253 fail:
 254    prog->program_written_to_cache = false;
 255    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 256       fprintf(stderr, "falling back to nir %s.\n",
 257               _mesa_shader_stage_to_abbrev(prog->info.stage));
 258    }
 259
 260    brw_program_deserialize_driver_blob(&brw->ctx, prog, stage);
 261
 262    return false;
 263 }
 264
 265 static void
 266 write_program_data(struct brw_context *brw, struct gl_program *prog,
 267                    void *key, struct brw_stage_prog_data *prog_data,
 268                    uint32_t prog_offset, struct disk_cache *cache,
 269                    gl_shader_stage stage)
 270 {
 271    struct blob binary;
 272    blob_init(&binary);
 273
 274    const void *program_map = brw->cache.map + prog_offset;
 275    /* TODO: Improve perf for non-LLC. It would be best to save it at program
 276     * generation time when the program is in normal memory accessible with
 277     * cache to the CPU. Another easier change would be to use
 278     * _mesa_streaming_load_memcpy to read from the program mapped memory. */
 279    brw_write_blob_program_data(&binary, stage, program_map, prog_data);
 280
 281    unsigned char sha1[20];
 282    char buf[41];
 283    gen_shader_sha1(prog, stage, key, sha1);
 284    _mesa_sha1_format(buf, sha1);
 285    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 286       fprintf(stderr, "putting binary in cache: %s\n", buf);
 287    }
 288
 289    disk_cache_put(cache, sha1, binary.data, binary.size, NULL);
 290
 291    prog->program_written_to_cache = true;
 292    blob_finish(&binary);
 293 }
 294
 295 void
 296 brw_disk_cache_write_render_programs(struct brw_context *brw)
 297 {
 298    struct disk_cache *cache = brw->ctx.Cache;
 299    if (cache == NULL)
 300       return;
 301
 302    struct gl_program *prog;
 303    gl_shader_stage stage;
 304    for (stage = MESA_SHADER_VERTEX; stage <= MESA_SHADER_FRAGMENT; stage++) {
 305       prog = brw->ctx._Shader->CurrentProgram[stage];
 306       if (prog && prog->sh.data->spirv)
 307          return;
 308    }
 309
 310    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_VERTEX];
 311    if (prog && !prog->program_written_to_cache) {
 312       struct brw_vs_prog_key vs_key;
 313       brw_vs_populate_key(brw, &vs_key);
 314       vs_key.base.program_string_id = 0;
 315
 316       write_program_data(brw, prog, &vs_key, brw->vs.base.prog_data,
 317                          brw->vs.base.prog_offset, cache,
 318                          MESA_SHADER_VERTEX);
 319    }
 320
 321    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_CTRL];
 322    if (prog && !prog->program_written_to_cache) {
 323       struct brw_tcs_prog_key tcs_key;
 324       brw_tcs_populate_key(brw, &tcs_key);
 325       tcs_key.base.program_string_id = 0;
 326
 327       write_program_data(brw, prog, &tcs_key, brw->tcs.base.prog_data,
 328                          brw->tcs.base.prog_offset, cache,
 329                          MESA_SHADER_TESS_CTRL);
 330    }
 331
 332    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_EVAL];
 333    if (prog && !prog->program_written_to_cache) {
 334       struct brw_tes_prog_key tes_key;
 335       brw_tes_populate_key(brw, &tes_key);
 336       tes_key.base.program_string_id = 0;
 337
 338       write_program_data(brw, prog, &tes_key, brw->tes.base.prog_data,
 339                          brw->tes.base.prog_offset, cache,
 340                          MESA_SHADER_TESS_EVAL);
 341    }
 342
 343    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_GEOMETRY];
 344    if (prog && !prog->program_written_to_cache) {
 345       struct brw_gs_prog_key gs_key;
 346       brw_gs_populate_key(brw, &gs_key);
 347       gs_key.base.program_string_id = 0;
 348
 349       write_program_data(brw, prog, &gs_key, brw->gs.base.prog_data,
 350                          brw->gs.base.prog_offset, cache,
 351                          MESA_SHADER_GEOMETRY);
 352    }
 353
 354    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_FRAGMENT];
 355    if (prog && !prog->program_written_to_cache) {
 356       struct brw_wm_prog_key wm_key;
 357       brw_wm_populate_key(brw, &wm_key);
 358       wm_key.base.program_string_id = 0;
 359
 360       write_program_data(brw, prog, &wm_key, brw->wm.base.prog_data,
 361                          brw->wm.base.prog_offset, cache,
 362                          MESA_SHADER_FRAGMENT);
 363    }
 364 }
 365
 366 void
 367 brw_disk_cache_write_compute_program(struct brw_context *brw)
 368 {
 369    struct disk_cache *cache = brw->ctx.Cache;
 370    if (cache == NULL)
 371       return;
 372
 373    struct gl_program *prog =
 374       brw->ctx._Shader->CurrentProgram[MESA_SHADER_COMPUTE];
 375
 376    if (prog && prog->sh.data->spirv)
 377       return;
 378
 379    if (prog && !prog->program_written_to_cache) {
 380       struct brw_cs_prog_key cs_key;
 381       brw_cs_populate_key(brw, &cs_key);
 382       cs_key.base.program_string_id = 0;
 383
 384       write_program_data(brw, prog, &cs_key, brw->cs.base.prog_data,
 385                          brw->cs.base.prog_offset, cache,
 386                          MESA_SHADER_COMPUTE);
 387    }
 388 }
 389
 390 void
 391 brw_disk_cache_init(struct intel_screen *screen)
 392 {
 393 #ifdef ENABLE_SHADER_CACHE
 394    if (INTEL_DEBUG & DEBUG_DISK_CACHE_DISABLE_MASK)
 395       return;
 396
 397    /* array length: print length + null char + 1 extra to verify it is unused */
 398    char renderer[11];
 399    ASSERTED int len = snprintf(renderer, sizeof(renderer), "i965_%04x",
 400                                    screen->deviceID);
 401    assert(len == sizeof(renderer) - 2);
 402
 403    const struct build_id_note *note =
 404       build_id_find_nhdr_for_addr(brw_disk_cache_init);
 405    assert(note && build_id_length(note) == 20 /* sha1 */);
 406
 407    const uint8_t *id_sha1 = build_id_data(note);
 408    assert(id_sha1);
 409
 410    char timestamp[41];
 411    _mesa_sha1_format(timestamp, id_sha1);
 412
 413    const uint64_t driver_flags =
 414       brw_get_compiler_config_value(screen->compiler);
 415    screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
 416 #endif
 417 }