src/compiler/glsl/shader_cache.cpp

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * \file shader_cache.cpp
  26  *
  27  * GLSL shader cache implementation
  28  *
  29  * This uses disk_cache.c to write out a serialization of various
  30  * state that's required in order to successfully load and use a
  31  * binary written out by a drivers backend, this state is referred to as
  32  * "metadata" throughout the implementation.
  33  *
  34  * The hash key for glsl metadata is a hash of the hashes of each GLSL
  35  * source string as well as some API settings that change the final program
  36  * such as SSO, attribute bindings, frag data bindings, etc.
  37  *
  38  * In order to avoid caching any actual IR we use the put_key/get_key support
  39  * in the disk_cache to put the SHA-1 hash for each successfully compiled
  40  * shader into the cache, and optimisticly return early from glCompileShader
  41  * (if the identical shader had been successfully compiled in the past),
  42  * in the hope that the final linked shader will be found in the cache.
  43  * If anything goes wrong (shader variant not found, backend cache item is
  44  * corrupt, etc) we will use a fallback path to compile and link the IR.
  45  */
  46
  47 #include "compiler/shader_info.h"
  48 #include "glsl_symbol_table.h"
  49 #include "glsl_parser_extras.h"
  50 #include "ir.h"
  51 #include "ir_optimization.h"
  52 #include "ir_rvalue_visitor.h"
  53 #include "ir_uniform.h"
  54 #include "linker.h"
  55 #include "link_varyings.h"
  56 #include "nir.h"
  57 #include "program.h"
  58 #include "serialize.h"
  59 #include "shader_cache.h"
  60 #include "util/mesa-sha1.h"
  61 #include "string_to_uint_map.h"
  62 #include "main/mtypes.h"
  63
  64 extern "C" {
  65 #include "main/enums.h"
  66 #include "main/shaderobj.h"
  67 #include "program/program.h"
  68 }
  69
  70 static void
  71 compile_shaders(struct gl_context *ctx, struct gl_shader_program *prog) {
  72    for (unsigned i = 0; i < prog->NumShaders; i++) {
  73       _mesa_glsl_compile_shader(ctx, prog->Shaders[i], false, false, true);
  74    }
  75 }
  76
  77 static void
  78 create_binding_str(const char *key, unsigned value, void *closure)
  79 {
  80    char **bindings_str = (char **) closure;
  81    ralloc_asprintf_append(bindings_str, "%s:%u,", key, value);
  82 }
  83
  84 void
  85 shader_cache_write_program_metadata(struct gl_context *ctx,
  86                                     struct gl_shader_program *prog)
  87 {
  88    struct disk_cache *cache = ctx->Cache;
  89    if (!cache)
  90       return;
  91
  92    /* Exit early when we are dealing with a ff shader with no source file to
  93     * generate a source from, or with a SPIR-V shader.
  94     *
  95     * TODO: In future we should use another method to generate a key for ff
  96     * programs, and SPIR-V shaders.
  97     */
  98    static const char zero[sizeof(prog->data->sha1)] = {0};
  99    if (memcmp(prog->data->sha1, zero, sizeof(prog->data->sha1)) == 0)
 100       return;
 101
 102    struct blob metadata;
 103    blob_init(&metadata);
 104
 105    if (ctx->Driver.ShaderCacheSerializeDriverBlob) {
 106       for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
 107          struct gl_linked_shader *sh = prog->_LinkedShaders[i];
 108          if (sh)
 109             ctx->Driver.ShaderCacheSerializeDriverBlob(ctx, sh->Program);
 110       }
 111    }
 112
 113    serialize_glsl_program(&metadata, ctx, prog);
 114
 115    struct cache_item_metadata cache_item_metadata;
 116    cache_item_metadata.type = CACHE_ITEM_TYPE_GLSL;
 117    cache_item_metadata.keys =
 118       (cache_key *) malloc(prog->NumShaders * sizeof(cache_key));
 119    cache_item_metadata.num_keys = prog->NumShaders;
 120
 121    if (!cache_item_metadata.keys)
 122       goto fail;
 123
 124    for (unsigned i = 0; i < prog->NumShaders; i++) {
 125       memcpy(cache_item_metadata.keys[i], prog->Shaders[i]->sha1,
 126              sizeof(cache_key));
 127    }
 128
 129    disk_cache_put(cache, prog->data->sha1, metadata.data, metadata.size,
 130                   &cache_item_metadata);
 131
 132    char sha1_buf[41];
 133    if (ctx->_Shader->Flags & GLSL_CACHE_INFO) {
 134       _mesa_sha1_format(sha1_buf, prog->data->sha1);
 135       fprintf(stderr, "putting program metadata in cache: %s\n", sha1_buf);
 136    }
 137
 138 fail:
 139    free(cache_item_metadata.keys);
 140    blob_finish(&metadata);
 141 }
 142
 143 bool
 144 shader_cache_read_program_metadata(struct gl_context *ctx,
 145                                    struct gl_shader_program *prog)
 146 {
 147    /* Fixed function programs generated by Mesa, or SPIR-V shaders, are not
 148     * cached. So don't try to read metadata for them from the cache.
 149     */
 150    if (prog->Name == 0 || prog->data->spirv)
 151       return false;
 152
 153    struct disk_cache *cache = ctx->Cache;
 154    if (!cache)
 155       return false;
 156
 157    /* Include bindings when creating sha1. These bindings change the resulting
 158     * binary so they are just as important as the shader source.
 159     */
 160    char *buf = ralloc_strdup(NULL, "vb: ");
 161    prog->AttributeBindings->iterate(create_binding_str, &buf);
 162    ralloc_strcat(&buf, "fb: ");
 163    prog->FragDataBindings->iterate(create_binding_str, &buf);
 164    ralloc_strcat(&buf, "fbi: ");
 165    prog->FragDataIndexBindings->iterate(create_binding_str, &buf);
 166    ralloc_asprintf_append(&buf, "tf: %d ", prog->TransformFeedback.BufferMode);
 167    for (unsigned int i = 0; i < prog->TransformFeedback.NumVarying; i++) {
 168       ralloc_asprintf_append(&buf, "%s ",
 169                              prog->TransformFeedback.VaryingNames[i]);
 170    }
 171
 172    /* SSO has an effect on the linked program so include this when generating
 173     * the sha also.
 174     */
 175    ralloc_asprintf_append(&buf, "sso: %s\n",
 176                           prog->SeparateShader ? "T" : "F");
 177
 178    /* A shader might end up producing different output depending on the glsl
 179     * version supported by the compiler. For example a different path might be
 180     * taken by the preprocessor, so add the version to the hash input.
 181     */
 182    ralloc_asprintf_append(&buf, "api: %d glsl: %d fglsl: %d\n",
 183                           ctx->API, ctx->Const.GLSLVersion,
 184                           ctx->Const.ForceGLSLVersion);
 185
 186    /* We run the preprocessor on shaders after hashing them, so we need to
 187     * add any extension override vars to the hash. If we don't do this the
 188     * preprocessor could result in different output and we could load the
 189     * wrong shader.
 190     */
 191    char *ext_override = getenv("MESA_EXTENSION_OVERRIDE");
 192    if (ext_override) {
 193       ralloc_asprintf_append(&buf, "ext:%s", ext_override);
 194    }
 195
 196    /* DRI config options may also change the output from the compiler so
 197     * include them as an input to sha1 creation.
 198     */
 199    char sha1buf[41];
 200    _mesa_sha1_format(sha1buf, ctx->Const.dri_config_options_sha1);
 201    ralloc_strcat(&buf, sha1buf);
 202
 203    for (unsigned i = 0; i < prog->NumShaders; i++) {
 204       struct gl_shader *sh = prog->Shaders[i];
 205       _mesa_sha1_format(sha1buf, sh->sha1);
 206       ralloc_asprintf_append(&buf, "%s: %s\n",
 207                              _mesa_shader_stage_to_abbrev(sh->Stage), sha1buf);
 208    }
 209    disk_cache_compute_key(cache, buf, strlen(buf), prog->data->sha1);
 210    ralloc_free(buf);
 211
 212    size_t size;
 213    uint8_t *buffer = (uint8_t *) disk_cache_get(cache, prog->data->sha1,
 214                                                 &size);
 215    if (buffer == NULL) {
 216       /* Cached program not found. We may have seen the individual shaders
 217        * before and skipped compiling but they may not have been used together
 218        * in this combination before. Fall back to linking shaders but first
 219        * re-compile the shaders.
 220        *
 221        * We could probably only compile the shaders which were skipped here
 222        * but we need to be careful because the source may also have been
 223        * changed since the last compile so for now we just recompile
 224        * everything.
 225        */
 226       compile_shaders(ctx, prog);
 227       return false;
 228    }
 229
 230    if (ctx->_Shader->Flags & GLSL_CACHE_INFO) {
 231       _mesa_sha1_format(sha1buf, prog->data->sha1);
 232       fprintf(stderr, "loading shader program meta data from cache: %s\n",
 233               sha1buf);
 234    }
 235
 236    struct blob_reader metadata;
 237    blob_reader_init(&metadata, buffer, size);
 238
 239    bool deserialized = deserialize_glsl_program(&metadata, ctx, prog);
 240
 241    if (!deserialized || metadata.current != metadata.end || metadata.overrun) {
 242       /* Something has gone wrong discard the item from the cache and rebuild
 243        * from source.
 244        */
 245       assert(!"Invalid GLSL shader disk cache item!");
 246
 247       if (ctx->_Shader->Flags & GLSL_CACHE_INFO) {
 248          fprintf(stderr, "Error reading program from cache (invalid GLSL "
 249                  "cache item)\n");
 250       }
 251
 252       disk_cache_remove(cache, prog->data->sha1);
 253       compile_shaders(ctx, prog);
 254       free(buffer);
 255       return false;
 256    }
 257
 258    /* This is used to flag a shader retrieved from cache */
 259    prog->data->LinkStatus = LINKING_SKIPPED;
 260
 261    free (buffer);
 262
 263    return true;
 264 }