src/mesa/drivers/dri/i965/brw_disk_cache.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "compiler/blob.h"
  25 #include "compiler/glsl/ir_uniform.h"
  26 #include "compiler/glsl/shader_cache.h"
  27 #include "compiler/nir/nir_serialize.h"
  28 #include "main/mtypes.h"
  29 #include "util/build_id.h"
  30 #include "util/debug.h"
  31 #include "util/disk_cache.h"
  32 #include "util/macros.h"
  33 #include "util/mesa-sha1.h"
  34
  35 #include "brw_context.h"
  36 #include "brw_program.h"
  37 #include "brw_cs.h"
  38 #include "brw_gs.h"
  39 #include "brw_state.h"
  40 #include "brw_vs.h"
  41 #include "brw_wm.h"
  42
  43 static void
  44 gen_shader_sha1(struct brw_context *brw, struct gl_program *prog,
  45                 gl_shader_stage stage, void *key, unsigned char *out_sha1)
  46 {
  47    char sha1_buf[41];
  48    unsigned char sha1[20];
  49    char manifest[256];
  50    int offset = 0;
  51
  52    _mesa_sha1_format(sha1_buf, prog->sh.data->sha1);
  53    offset += snprintf(manifest, sizeof(manifest), "program: %s\n", sha1_buf);
  54
  55    _mesa_sha1_compute(key, brw_prog_key_size(stage), sha1);
  56    _mesa_sha1_format(sha1_buf, sha1);
  57    offset += snprintf(manifest + offset, sizeof(manifest) - offset,
  58                       "%s_key: %s\n", _mesa_shader_stage_to_abbrev(stage),
  59                       sha1_buf);
  60
  61    _mesa_sha1_compute(manifest, strlen(manifest), out_sha1);
  62 }
  63
  64 static void
  65 restore_serialized_nir_shader(struct brw_context *brw, struct gl_program *prog,
  66                               gl_shader_stage stage)
  67 {
  68    prog->program_written_to_cache = false;
  69    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
  70       fprintf(stderr, "falling back to nir %s.\n",
  71               _mesa_shader_stage_to_abbrev(prog->info.stage));
  72    }
  73
  74    if (!prog->nir) {
  75       assert(prog->driver_cache_blob && prog->driver_cache_blob_size > 0);
  76       const struct nir_shader_compiler_options *options =
  77          brw->ctx.Const.ShaderCompilerOptions[stage].NirOptions;
  78       struct blob_reader reader;
  79       blob_reader_init(&reader, prog->driver_cache_blob,
  80                        prog->driver_cache_blob_size);
  81       prog->nir = nir_deserialize(NULL, options, &reader);
  82    }
  83 }
  84
  85 static void
  86 write_blob_program_data(struct blob *binary, gl_shader_stage stage,
  87                         const void *program,
  88                         struct brw_stage_prog_data *prog_data)
  89 {
  90    /* Write prog_data to blob. */
  91    blob_write_bytes(binary, prog_data, brw_prog_data_size(stage));
  92
  93    /* Write program to blob. */
  94    blob_write_bytes(binary, program, prog_data->program_size);
  95
  96    /* Write push params */
  97    blob_write_bytes(binary, prog_data->param,
  98                     sizeof(uint32_t) * prog_data->nr_params);
  99
 100    /* Write pull params */
 101    blob_write_bytes(binary, prog_data->pull_param,
 102                     sizeof(uint32_t) * prog_data->nr_pull_params);
 103 }
 104
 105 static bool
 106 read_blob_program_data(struct blob_reader *binary, struct gl_program *prog,
 107                        gl_shader_stage stage, const uint8_t **program,
 108                        struct brw_stage_prog_data *prog_data)
 109 {
 110    /* Read shader prog_data from blob. */
 111    blob_copy_bytes(binary, prog_data, brw_prog_data_size(stage));
 112    if (binary->overrun)
 113       return false;
 114
 115    /* Read shader program from blob. */
 116    *program = blob_read_bytes(binary, prog_data->program_size);
 117
 118    /* Read push params */
 119    prog_data->param = rzalloc_array(NULL, uint32_t, prog_data->nr_params);
 120    blob_copy_bytes(binary, prog_data->param,
 121                    sizeof(uint32_t) * prog_data->nr_params);
 122
 123    /* Read pull params */
 124    prog_data->pull_param = rzalloc_array(NULL, uint32_t,
 125                                          prog_data->nr_pull_params);
 126    blob_copy_bytes(binary, prog_data->pull_param,
 127                    sizeof(uint32_t) * prog_data->nr_pull_params);
 128
 129    return (binary->current == binary->end && !binary->overrun);
 130 }
 131
 132 static bool
 133 read_and_upload(struct brw_context *brw, struct disk_cache *cache,
 134                 struct gl_program *prog, gl_shader_stage stage)
 135 {
 136    unsigned char binary_sha1[20];
 137
 138    union brw_any_prog_key prog_key;
 139
 140    switch (stage) {
 141    case MESA_SHADER_VERTEX:
 142       brw_vs_populate_key(brw, &prog_key.vs);
 143       /* We don't care what instance of the program it is for the disk cache
 144        * hash lookup, so set the id to 0 for the sha1 hashing.
 145        * program_string_id will be set below.
 146        */
 147       prog_key.vs.program_string_id = 0;
 148       break;
 149    case MESA_SHADER_TESS_CTRL:
 150       brw_tcs_populate_key(brw, &prog_key.tcs);
 151       prog_key.tcs.program_string_id = 0;
 152       break;
 153    case MESA_SHADER_TESS_EVAL:
 154       brw_tes_populate_key(brw, &prog_key.tes);
 155       prog_key.tes.program_string_id = 0;
 156       break;
 157    case MESA_SHADER_GEOMETRY:
 158       brw_gs_populate_key(brw, &prog_key.gs);
 159       prog_key.gs.program_string_id = 0;
 160       break;
 161    case MESA_SHADER_FRAGMENT:
 162       brw_wm_populate_key(brw, &prog_key.wm);
 163       prog_key.wm.program_string_id = 0;
 164       break;
 165    case MESA_SHADER_COMPUTE:
 166       brw_cs_populate_key(brw, &prog_key.cs);
 167       prog_key.cs.program_string_id = 0;
 168       break;
 169    default:
 170       unreachable("Unsupported stage!");
 171    }
 172
 173    gen_shader_sha1(brw, prog, stage, &prog_key, binary_sha1);
 174
 175    size_t buffer_size;
 176    uint8_t *buffer = disk_cache_get(cache, binary_sha1, &buffer_size);
 177    if (buffer == NULL) {
 178       if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 179          char sha1_buf[41];
 180          _mesa_sha1_format(sha1_buf, binary_sha1);
 181          fprintf(stderr, "No cached %s binary found for: %s\n",
 182                  _mesa_shader_stage_to_abbrev(stage), sha1_buf);
 183       }
 184       return false;
 185    }
 186
 187    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 188       char sha1_buf[41];
 189       _mesa_sha1_format(sha1_buf, binary_sha1);
 190       fprintf(stderr, "attempting to populate bo cache with binary: %s\n",
 191               sha1_buf);
 192    }
 193
 194    struct blob_reader binary;
 195    blob_reader_init(&binary, buffer, buffer_size);
 196
 197    const uint8_t *program;
 198    struct brw_stage_prog_data *prog_data =
 199       ralloc_size(NULL, sizeof(union brw_any_prog_data));
 200    if (!read_blob_program_data(&binary, prog, stage, &program, prog_data)) {
 201       /* Something very bad has gone wrong discard the item from the cache and
 202        * rebuild from source.
 203        */
 204       if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 205          fprintf(stderr, "Error reading program from cache (invalid i965 "
 206                  "cache item)\n");
 207       }
 208
 209       disk_cache_remove(cache, binary_sha1);
 210       free(buffer);
 211       return false;
 212    }
 213
 214    enum brw_cache_id cache_id;
 215    struct brw_stage_state *stage_state;
 216
 217    switch (stage) {
 218    case MESA_SHADER_VERTEX:
 219       prog_key.vs.program_string_id = brw_program(prog)->id;
 220       cache_id = BRW_CACHE_VS_PROG;
 221       stage_state = &brw->vs.base;
 222       break;
 223    case MESA_SHADER_TESS_CTRL:
 224       prog_key.tcs.program_string_id = brw_program(prog)->id;
 225       cache_id = BRW_CACHE_TCS_PROG;
 226       stage_state = &brw->tcs.base;
 227       break;
 228    case MESA_SHADER_TESS_EVAL:
 229       prog_key.tes.program_string_id = brw_program(prog)->id;
 230       cache_id = BRW_CACHE_TES_PROG;
 231       stage_state = &brw->tes.base;
 232       break;
 233    case MESA_SHADER_GEOMETRY:
 234       prog_key.gs.program_string_id = brw_program(prog)->id;
 235       cache_id = BRW_CACHE_GS_PROG;
 236       stage_state = &brw->gs.base;
 237       break;
 238    case MESA_SHADER_FRAGMENT:
 239       prog_key.wm.program_string_id = brw_program(prog)->id;
 240       cache_id = BRW_CACHE_FS_PROG;
 241       stage_state = &brw->wm.base;
 242       break;
 243    case MESA_SHADER_COMPUTE:
 244       prog_key.cs.program_string_id = brw_program(prog)->id;
 245       cache_id = BRW_CACHE_CS_PROG;
 246       stage_state = &brw->cs.base;
 247       break;
 248    default:
 249       unreachable("Unsupported stage!");
 250    }
 251
 252    brw_alloc_stage_scratch(brw, stage_state, prog_data->total_scratch);
 253
 254    brw_upload_cache(&brw->cache, cache_id, &prog_key, brw_prog_key_size(stage),
 255                     program, prog_data->program_size, prog_data,
 256                     brw_prog_data_size(stage), &stage_state->prog_offset,
 257                     &stage_state->prog_data);
 258
 259    prog->program_written_to_cache = true;
 260
 261    free(buffer);
 262
 263    return true;
 264 }
 265
 266 bool
 267 brw_disk_cache_upload_program(struct brw_context *brw, gl_shader_stage stage)
 268 {
 269    struct disk_cache *cache = brw->ctx.Cache;
 270    if (cache == NULL)
 271       return false;
 272
 273    struct gl_program *prog = brw->ctx._Shader->CurrentProgram[stage];
 274    if (prog == NULL)
 275       return false;
 276
 277    /* FIXME: For now we don't read from the cache if transform feedback is
 278     * enabled via the API. However the shader cache does support transform
 279     * feedback when enabled via in shader xfb qualifiers.
 280     */
 281    if (prog->sh.LinkedTransformFeedback &&
 282        prog->sh.LinkedTransformFeedback->api_enabled)
 283       return false;
 284
 285    if (brw->ctx._Shader->Flags & GLSL_CACHE_FALLBACK)
 286       goto fail;
 287
 288    if (prog->sh.data->LinkStatus != linking_skipped)
 289       goto fail;
 290
 291    if (!read_and_upload(brw, cache, prog, stage))
 292       goto fail;
 293
 294    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 295       fprintf(stderr, "read gen program from cache\n");
 296    }
 297
 298    return true;
 299
 300 fail:
 301    restore_serialized_nir_shader(brw, prog, stage);
 302    return false;
 303 }
 304
 305 static void
 306 write_program_data(struct brw_context *brw, struct gl_program *prog,
 307                    void *key, struct brw_stage_prog_data *prog_data,
 308                    uint32_t prog_offset, struct disk_cache *cache,
 309                    gl_shader_stage stage)
 310 {
 311    struct blob binary;
 312    blob_init(&binary);
 313
 314    const void *program_map = brw->cache.map + prog_offset;
 315    /* TODO: Improve perf for non-LLC. It would be best to save it at program
 316     * generation time when the program is in normal memory accessible with
 317     * cache to the CPU. Another easier change would be to use
 318     * _mesa_streaming_load_memcpy to read from the program mapped memory. */
 319    write_blob_program_data(&binary, stage, program_map, prog_data);
 320
 321    unsigned char sha1[20];
 322    char buf[41];
 323    gen_shader_sha1(brw, prog, stage, key, sha1);
 324    _mesa_sha1_format(buf, sha1);
 325    if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
 326       fprintf(stderr, "putting binary in cache: %s\n", buf);
 327    }
 328
 329    disk_cache_put(cache, sha1, binary.data, binary.size, NULL);
 330
 331    prog->program_written_to_cache = true;
 332    blob_finish(&binary);
 333 }
 334
 335 void
 336 brw_disk_cache_write_render_programs(struct brw_context *brw)
 337 {
 338    struct disk_cache *cache = brw->ctx.Cache;
 339    if (cache == NULL)
 340       return;
 341
 342    struct gl_program *prog =
 343       brw->ctx._Shader->CurrentProgram[MESA_SHADER_VERTEX];
 344    if (prog && !prog->program_written_to_cache) {
 345       struct brw_vs_prog_key vs_key;
 346       brw_vs_populate_key(brw, &vs_key);
 347       vs_key.program_string_id = 0;
 348
 349       write_program_data(brw, prog, &vs_key, brw->vs.base.prog_data,
 350                          brw->vs.base.prog_offset, cache,
 351                          MESA_SHADER_VERTEX);
 352    }
 353
 354    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_CTRL];
 355    if (prog && !prog->program_written_to_cache) {
 356       struct brw_tcs_prog_key tcs_key;
 357       brw_tcs_populate_key(brw, &tcs_key);
 358       tcs_key.program_string_id = 0;
 359
 360       write_program_data(brw, prog, &tcs_key, brw->tcs.base.prog_data,
 361                          brw->tcs.base.prog_offset, cache,
 362                          MESA_SHADER_TESS_CTRL);
 363    }
 364
 365    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_EVAL];
 366    if (prog && !prog->program_written_to_cache) {
 367       struct brw_tes_prog_key tes_key;
 368       brw_tes_populate_key(brw, &tes_key);
 369       tes_key.program_string_id = 0;
 370
 371       write_program_data(brw, prog, &tes_key, brw->tes.base.prog_data,
 372                          brw->tes.base.prog_offset, cache,
 373                          MESA_SHADER_TESS_EVAL);
 374    }
 375
 376    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_GEOMETRY];
 377    if (prog && !prog->program_written_to_cache) {
 378       struct brw_gs_prog_key gs_key;
 379       brw_gs_populate_key(brw, &gs_key);
 380       gs_key.program_string_id = 0;
 381
 382       write_program_data(brw, prog, &gs_key, brw->gs.base.prog_data,
 383                          brw->gs.base.prog_offset, cache,
 384                          MESA_SHADER_GEOMETRY);
 385    }
 386
 387    prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_FRAGMENT];
 388    if (prog && !prog->program_written_to_cache) {
 389       struct brw_wm_prog_key wm_key;
 390       brw_wm_populate_key(brw, &wm_key);
 391       wm_key.program_string_id = 0;
 392
 393       write_program_data(brw, prog, &wm_key, brw->wm.base.prog_data,
 394                          brw->wm.base.prog_offset, cache,
 395                          MESA_SHADER_FRAGMENT);
 396    }
 397 }
 398
 399 void
 400 brw_disk_cache_write_compute_program(struct brw_context *brw)
 401 {
 402    struct disk_cache *cache = brw->ctx.Cache;
 403    if (cache == NULL)
 404       return;
 405
 406    struct gl_program *prog =
 407       brw->ctx._Shader->CurrentProgram[MESA_SHADER_COMPUTE];
 408    if (prog && !prog->program_written_to_cache) {
 409       struct brw_cs_prog_key cs_key;
 410       brw_cs_populate_key(brw, &cs_key);
 411       cs_key.program_string_id = 0;
 412
 413       write_program_data(brw, prog, &cs_key, brw->cs.base.prog_data,
 414                          brw->cs.base.prog_offset, cache,
 415                          MESA_SHADER_COMPUTE);
 416    }
 417 }
 418
 419 void
 420 brw_disk_cache_init(struct brw_context *brw)
 421 {
 422 #ifdef ENABLE_SHADER_CACHE
 423    if (env_var_as_boolean("MESA_GLSL_CACHE_DISABLE", true))
 424       return;
 425
 426    char renderer[10];
 427    MAYBE_UNUSED int len = snprintf(renderer, sizeof(renderer), "i965_%04x",
 428                                    brw->screen->deviceID);
 429    assert(len == sizeof(renderer) - 1);
 430
 431    const struct build_id_note *note =
 432       build_id_find_nhdr_for_addr(brw_disk_cache_init);
 433    assert(note && build_id_length(note) == 20 /* sha1 */);
 434
 435    const uint8_t *id_sha1 = build_id_data(note);
 436    assert(id_sha1);
 437
 438    char timestamp[41];
 439    _mesa_sha1_format(timestamp, id_sha1);
 440
 441    brw->ctx.Cache = disk_cache_create(renderer, timestamp, 0);
 442 #endif
 443 }