X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Futil%2Fdisk_cache.c;h=a92d621927a857f262e15ea100db65a513b755f2;hb=7fbadfc385c359fd291d58a75fbe6ce3fdc91747;hp=6f467027d8e4a49e922b9104d2c80e495c103403;hpb=28b326238bcc14d372880a0e4fabcece4ffdc871;p=mesa.git

diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c
index 6f467027d8e..a92d621927a 100644
--- a/src/util/disk_cache.c
+++ b/src/util/disk_cache.c
@@ -37,15 +37,21 @@
 #include <pwd.h>
 #include <errno.h>
 #include <dirent.h>
+#include <inttypes.h>
 #include "zlib.h"
 
+#ifdef HAVE_ZSTD
+#include "zstd.h"
+#endif
+
 #include "util/crc32.h"
+#include "util/debug.h"
 #include "util/rand_xor.h"
 #include "util/u_atomic.h"
 #include "util/u_queue.h"
 #include "util/mesa-sha1.h"
 #include "util/ralloc.h"
-#include "main/errors.h"
+#include "util/compiler.h"
 
 #include "disk_cache.h"
 
@@ -72,9 +78,13 @@
  */
 #define CACHE_VERSION 1
 
+/* 3 is the recomended level, with 22 as the absolute maximum */
+#define ZSTD_COMPRESSION_LEVEL 3
+
 struct disk_cache {
    /* The path to the cache directory. */
    char *path;
+   bool path_init_failed;
 
    /* Thread queue for compressing and writing cache entries to disk */
    struct util_queue cache_queue;
@@ -98,6 +108,9 @@ struct disk_cache {
    /* Driver cache keys. */
    uint8_t *driver_keys_blob;
    size_t driver_keys_blob_size;
+
+   disk_cache_put_cb blob_put_cb;
+   disk_cache_get_cb blob_get_cb;
 };
 
 struct disk_cache_put_job {
@@ -112,6 +125,8 @@ struct disk_cache_put_job {
 
    /* Size of data to be compressed and written. */
    size_t size;
+
+   struct cache_item_metadata cache_item_metadata;
 };
 
 /* Create a directory named 'path' if it does not already exist.
@@ -181,7 +196,7 @@ do {                                       \
 } while (0);
 
 struct disk_cache *
-disk_cache_create(const char *gpu_name, const char *timestamp,
+disk_cache_create(const char *gpu_name, const char *driver_id,
                   uint64_t driver_flags)
 {
    void *local;
@@ -192,6 +207,9 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
    struct stat sb;
    size_t size;
 
+   uint8_t cache_version = CACHE_VERSION;
+   size_t cv_size = sizeof(cache_version);
+
    /* If running as a users other than the real user disable cache */
    if (geteuid() != getuid())
       return NULL;
@@ -202,9 +220,16 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
       goto fail;
 
    /* At user request, disable shader cache entirely. */
-   if (getenv("MESA_GLSL_CACHE_DISABLE"))
+   if (env_var_as_boolean("MESA_GLSL_CACHE_DISABLE", false))
       goto fail;
 
+   cache = rzalloc(NULL, struct disk_cache);
+   if (cache == NULL)
+      goto fail;
+
+   /* Assume failure. */
+   cache->path_init_failed = true;
+
    /* Determine path for cache based on the first defined name as follows:
     *
     *   $MESA_GLSL_CACHE_DIR
@@ -214,11 +239,11 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
    path = getenv("MESA_GLSL_CACHE_DIR");
    if (path) {
       if (mkdir_if_needed(path) == -1)
-         goto fail;
+         goto path_fail;
 
       path = concatenate_and_mkdir(local, path, CACHE_DIR_NAME);
       if (path == NULL)
-         goto fail;
+         goto path_fail;
    }
 
    if (path == NULL) {
@@ -226,11 +251,11 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
 
       if (xdg_cache_home) {
          if (mkdir_if_needed(xdg_cache_home) == -1)
-            goto fail;
+            goto path_fail;
 
          path = concatenate_and_mkdir(local, xdg_cache_home, CACHE_DIR_NAME);
          if (path == NULL)
-            goto fail;
+            goto path_fail;
       }
    }
 
@@ -256,43 +281,39 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
             buf = NULL;
             buf_size *= 2;
          } else {
-            goto fail;
+            goto path_fail;
          }
       }
 
       path = concatenate_and_mkdir(local, pwd.pw_dir, ".cache");
       if (path == NULL)
-         goto fail;
+         goto path_fail;
 
       path = concatenate_and_mkdir(local, path, CACHE_DIR_NAME);
       if (path == NULL)
-         goto fail;
+         goto path_fail;
    }
 
-   cache = ralloc(NULL, struct disk_cache);
-   if (cache == NULL)
-      goto fail;
-
    cache->path = ralloc_strdup(cache, path);
    if (cache->path == NULL)
-      goto fail;
+      goto path_fail;
 
    path = ralloc_asprintf(local, "%s/index", cache->path);
    if (path == NULL)
-      goto fail;
+      goto path_fail;
 
    fd = open(path, O_RDWR | O_CREAT | O_CLOEXEC, 0644);
    if (fd == -1)
-      goto fail;
+      goto path_fail;
 
    if (fstat(fd, &sb) == -1)
-      goto fail;
+      goto path_fail;
 
    /* Force the index file to be the expected size. */
    size = sizeof(*cache->size) + CACHE_INDEX_MAX_KEYS * CACHE_KEY_SIZE;
    if (sb.st_size != size) {
       if (ftruncate(fd, size) == -1)
-         goto fail;
+         goto path_fail;
    }
 
    /* We map this shared so that other processes see updates that we
@@ -313,11 +334,9 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
    cache->index_mmap = mmap(NULL, size, PROT_READ | PROT_WRITE,
                             MAP_SHARED, fd, 0);
    if (cache->index_mmap == MAP_FAILED)
-      goto fail;
+      goto path_fail;
    cache->index_mmap_size = size;
 
-   close(fd);
-
    cache->size = (uint64_t *) cache->index_mmap;
    cache->stored_keys = cache->index_mmap + sizeof(uint64_t);
 
@@ -356,22 +375,34 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
 
    cache->max_size = max_size;
 
-   /* A limit of 32 jobs was choosen as observations of Deus Ex start-up times
-    * showed that we reached at most 11 jobs on an Intel i5-6400 CPU@2.70GHz
-    * (a fairly modest desktop CPU). 1 thread was chosen because we don't
-    * really care about getting things to disk quickly just that it's not
-    * blocking other tasks.
+   /* 4 threads were chosen below because just about all modern CPUs currently
+    * available that run Mesa have *at least* 4 cores. For these CPUs allowing
+    * more threads can result in the queue being processed faster, thus
+    * avoiding excessive memory use due to a backlog of cache entrys building
+    * up in the queue. Since we set the UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY
+    * flag this should have little negative impact on low core systems.
+    *
+    * The queue will resize automatically when it's full, so adding new jobs
+    * doesn't stall.
     */
-   util_queue_init(&cache->cache_queue, "disk_cache", 32, 1, 0);
+   util_queue_init(&cache->cache_queue, "disk$", 32, 4,
+                   UTIL_QUEUE_INIT_RESIZE_IF_FULL |
+                   UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY |
+                   UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY);
+
+   cache->path_init_failed = false;
+
+ path_fail:
+
+   if (fd != -1)
+      close(fd);
 
-   uint8_t cache_version = CACHE_VERSION;
-   size_t cv_size = sizeof(cache_version);
    cache->driver_keys_blob_size = cv_size;
 
    /* Create driver id keys */
-   size_t ts_size = strlen(timestamp) + 1;
+   size_t id_size = strlen(driver_id) + 1;
    size_t gpu_name_size = strlen(gpu_name) + 1;
-   cache->driver_keys_blob_size += ts_size;
+   cache->driver_keys_blob_size += id_size;
    cache->driver_keys_blob_size += gpu_name_size;
 
    /* We sometimes store entire structs that contains a pointers in the cache,
@@ -391,7 +422,7 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
 
    uint8_t *drv_key_blob = cache->driver_keys_blob;
    DRV_KEY_CPY(drv_key_blob, &cache_version, cv_size)
-   DRV_KEY_CPY(drv_key_blob, timestamp, ts_size)
+   DRV_KEY_CPY(drv_key_blob, driver_id, id_size)
    DRV_KEY_CPY(drv_key_blob, gpu_name, gpu_name_size)
    DRV_KEY_CPY(drv_key_blob, &ptr_size, ptr_size_size)
    DRV_KEY_CPY(drv_key_blob, &driver_flags, driver_flags_size)
@@ -404,8 +435,6 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
    return cache;
 
  fail:
-   if (fd != -1)
-      close(fd);
    if (cache)
       ralloc_free(cache);
    ralloc_free(local);
@@ -416,7 +445,8 @@ disk_cache_create(const char *gpu_name, const char *timestamp,
 void
 disk_cache_destroy(struct disk_cache *cache)
 {
-   if (cache) {
+   if (cache && !cache->path_init_failed) {
+      util_queue_finish(&cache->cache_queue);
       util_queue_destroy(&cache->cache_queue);
       munmap(cache->index_mmap, cache->index_mmap_size);
    }
@@ -424,6 +454,12 @@ disk_cache_destroy(struct disk_cache *cache)
    ralloc_free(cache);
 }
 
+void
+disk_cache_wait_for_idle(struct disk_cache *cache)
+{
+   util_queue_finish(&cache->cache_queue);
+}
+
 /* Return a filename within the cache's directory corresponding to 'key'. The
  * returned filename is ralloced with 'cache' as the parent context.
  *
@@ -435,6 +471,9 @@ get_cache_file(struct disk_cache *cache, const cache_key key)
    char buf[41];
    char *filename;
 
+   if (cache->path_init_failed)
+      return NULL;
+
    _mesa_sha1_format(buf, key);
    if (asprintf(&filename, "%s/%c%c/%s", cache->path, buf[0],
                 buf[1], buf + 2) == -1)
@@ -711,7 +750,28 @@ static size_t
 deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
                           const char *filename)
 {
-   unsigned char out[BUFSIZE];
+#ifdef HAVE_ZSTD
+   /* from the zstd docs (https://facebook.github.io/zstd/zstd_manual.html):
+    * compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
+    */
+   size_t out_size = ZSTD_compressBound(in_data_size);
+   void * out = malloc(out_size);
+
+   size_t ret = ZSTD_compress(out, out_size, in_data, in_data_size,
+                              ZSTD_COMPRESSION_LEVEL);
+   if (ZSTD_isError(ret)) {
+      free(out);
+      return 0;
+   }
+   ssize_t written = write_all(dest, out, ret);
+   if (written == -1) {
+      free(out);
+      return 0;
+   }
+   free(out);
+   return ret;
+#else
+   unsigned char *out;
 
    /* allocate deflate state */
    z_stream strm;
@@ -728,6 +788,11 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
    /* compress until end of in_data */
    size_t compressed_size = 0;
    int flush;
+
+   out = malloc(BUFSIZE * sizeof(unsigned char));
+   if (out == NULL)
+      return 0;
+
    do {
       int remaining = in_data_size - BUFSIZE;
       flush = remaining > 0 ? Z_NO_FLUSH : Z_FINISH;
@@ -749,6 +814,7 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
          ssize_t written = write_all(dest, out, have);
          if (written == -1) {
             (void)deflateEnd(&strm);
+            free(out);
             return 0;
          }
       } while (strm.avail_out == 0);
@@ -763,12 +829,15 @@ deflate_and_write_to_disk(const void *in_data, size_t in_data_size, int dest,
 
    /* clean up and return */
    (void)deflateEnd(&strm);
+   free(out);
    return compressed_size;
+# endif
 }
 
 static struct disk_cache_put_job *
 create_put_job(struct disk_cache *cache, const cache_key key,
-               const void *data, size_t size)
+               const void *data, size_t size,
+               struct cache_item_metadata *cache_item_metadata)
 {
    struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *)
       malloc(sizeof(struct disk_cache_put_job) + size);
@@ -779,15 +848,44 @@ create_put_job(struct disk_cache *cache, const cache_key key,
       dc_job->data = dc_job + 1;
       memcpy(dc_job->data, data, size);
       dc_job->size = size;
+
+      /* Copy the cache item metadata */
+      if (cache_item_metadata) {
+         dc_job->cache_item_metadata.type = cache_item_metadata->type;
+         if (cache_item_metadata->type == CACHE_ITEM_TYPE_GLSL) {
+            dc_job->cache_item_metadata.num_keys =
+               cache_item_metadata->num_keys;
+            dc_job->cache_item_metadata.keys = (cache_key *)
+               malloc(cache_item_metadata->num_keys * sizeof(cache_key));
+
+            if (!dc_job->cache_item_metadata.keys)
+               goto fail;
+
+            memcpy(dc_job->cache_item_metadata.keys,
+                   cache_item_metadata->keys,
+                   sizeof(cache_key) * cache_item_metadata->num_keys);
+         }
+      } else {
+         dc_job->cache_item_metadata.type = CACHE_ITEM_TYPE_UNKNOWN;
+         dc_job->cache_item_metadata.keys = NULL;
+      }
    }
 
    return dc_job;
+
+fail:
+   free(dc_job);
+
+   return NULL;
 }
 
 static void
 destroy_put_job(void *job, int thread_index)
 {
    if (job) {
+      struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
+      free(dc_job->cache_item_metadata.keys);
+
       free(job);
    }
 }
@@ -844,7 +942,17 @@ cache_put(void *job, int thread_index)
     * open with the flock held. So just let that file be responsible
     * for writing the file.
     */
+#ifdef HAVE_FLOCK
    err = flock(fd, LOCK_EX | LOCK_NB);
+#else
+   struct flock lock = {
+      .l_start = 0,
+      .l_len = 0, /* entire file */
+      .l_type = F_WRLCK,
+      .l_whence = SEEK_SET
+   };
+   err = fcntl(fd, F_SETLK, &lock);
+#endif
    if (err == -1)
       goto done;
 
@@ -876,6 +984,34 @@ cache_put(void *job, int thread_index)
       goto done;
    }
 
+   /* Write the cache item metadata. This data can be used to deal with
+    * hash collisions, as well as providing useful information to 3rd party
+    * tools reading the cache files.
+    */
+   ret = write_all(fd, &dc_job->cache_item_metadata.type,
+                   sizeof(uint32_t));
+   if (ret == -1) {
+      unlink(filename_tmp);
+      goto done;
+   }
+
+   if (dc_job->cache_item_metadata.type == CACHE_ITEM_TYPE_GLSL) {
+      ret = write_all(fd, &dc_job->cache_item_metadata.num_keys,
+                      sizeof(uint32_t));
+      if (ret == -1) {
+         unlink(filename_tmp);
+         goto done;
+      }
+
+      ret = write_all(fd, dc_job->cache_item_metadata.keys[0],
+                      dc_job->cache_item_metadata.num_keys *
+                      sizeof(cache_key));
+      if (ret == -1) {
+         unlink(filename_tmp);
+         goto done;
+      }
+   }
+
    /* Create CRC of the data. We will read this when restoring the cache and
     * use it to check for corruption.
     */
@@ -923,23 +1059,30 @@ cache_put(void *job, int thread_index)
     */
    if (fd != -1)
       close(fd);
-   if (filename_tmp)
-      free(filename_tmp);
-   if (filename)
-      free(filename);
+   free(filename_tmp);
+   free(filename);
 }
 
 void
 disk_cache_put(struct disk_cache *cache, const cache_key key,
-               const void *data, size_t size)
+               const void *data, size_t size,
+               struct cache_item_metadata *cache_item_metadata)
 {
+   if (cache->blob_put_cb) {
+      cache->blob_put_cb(key, CACHE_KEY_SIZE, data, size);
+      return;
+   }
+
+   if (cache->path_init_failed)
+      return;
+
    struct disk_cache_put_job *dc_job =
-      create_put_job(cache, key, data, size);
+      create_put_job(cache, key, data, size, cache_item_metadata);
 
    if (dc_job) {
       util_queue_fence_init(&dc_job->fence);
       util_queue_add_job(&cache->cache_queue, dc_job, &dc_job->fence,
-                         cache_put, destroy_put_job);
+                         cache_put, destroy_put_job, dc_job->size);
    }
 }
 
@@ -950,6 +1093,10 @@ static bool
 inflate_cache_data(uint8_t *in_data, size_t in_data_size,
                    uint8_t *out_data, size_t out_data_size)
 {
+#ifdef HAVE_ZSTD
+   size_t ret = ZSTD_decompress(out_data, out_data_size, in_data, in_data_size);
+   return !ZSTD_isError(ret);
+#else
    z_stream strm;
 
    /* allocate inflate state */
@@ -980,6 +1127,7 @@ inflate_cache_data(uint8_t *in_data, size_t in_data_size,
    /* clean up and return */
    (void)inflateEnd(&strm);
    return true;
+#endif
 }
 
 void *
@@ -990,10 +1138,33 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
    char *filename = NULL;
    uint8_t *data = NULL;
    uint8_t *uncompressed_data = NULL;
+   uint8_t *file_header = NULL;
 
    if (size)
       *size = 0;
 
+   if (cache->blob_get_cb) {
+      /* This is what Android EGL defines as the maxValueSize in egl_cache_t
+       * class implementation.
+       */
+      const signed long max_blob_size = 64 * 1024;
+      void *blob = malloc(max_blob_size);
+      if (!blob)
+         return NULL;
+
+      signed long bytes =
+         cache->blob_get_cb(key, CACHE_KEY_SIZE, blob, max_blob_size);
+
+      if (!bytes) {
+         free(blob);
+         return NULL;
+      }
+
+      if (size)
+         *size = bytes;
+      return blob;
+   }
+
    filename = get_cache_file(cache, key);
    if (filename == NULL)
       goto fail;
@@ -1010,29 +1181,47 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
       goto fail;
 
    size_t ck_size = cache->driver_keys_blob_size;
-#ifndef NDEBUG
-   uint8_t *file_header = malloc(ck_size);
+   file_header = malloc(ck_size);
    if (!file_header)
       goto fail;
 
-   assert(sb.st_size > ck_size);
+   if (sb.st_size < ck_size)
+      goto fail;
+
    ret = read_all(fd, file_header, ck_size);
-   if (ret == -1) {
-      free(file_header);
+   if (ret == -1)
       goto fail;
-   }
 
-   assert(memcmp(cache->driver_keys_blob, file_header, ck_size) == 0);
+   /* Check for extremely unlikely hash collisions */
+   if (memcmp(cache->driver_keys_blob, file_header, ck_size) != 0) {
+      assert(!"Mesa cache keys mismatch!");
+      goto fail;
+   }
 
-   free(file_header);
-#else
-   /* The cache keys are currently just used for distributing precompiled
-    * shaders, they are not used by Mesa so just skip them for now.
-    */
-   ret = lseek(fd, ck_size, SEEK_CUR);
+   size_t cache_item_md_size = sizeof(uint32_t);
+   uint32_t md_type;
+   ret = read_all(fd, &md_type, cache_item_md_size);
    if (ret == -1)
       goto fail;
-#endif
+
+   if (md_type == CACHE_ITEM_TYPE_GLSL) {
+      uint32_t num_keys;
+      cache_item_md_size += sizeof(uint32_t);
+      ret = read_all(fd, &num_keys, sizeof(uint32_t));
+      if (ret == -1)
+         goto fail;
+
+      /* The cache item metadata is currently just used for distributing
+       * precompiled shaders, they are not used by Mesa so just skip them for
+       * now.
+       * TODO: pass the metadata back to the caller and do some basic
+       * validation.
+       */
+      cache_item_md_size += num_keys * sizeof(cache_key);
+      ret = lseek(fd, num_keys * sizeof(cache_key), SEEK_CUR);
+      if (ret == -1)
+         goto fail;
+   }
 
    /* Load the CRC that was created when the file was written. */
    struct cache_entry_file_data cf_data;
@@ -1042,7 +1231,8 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
       goto fail;
 
    /* Load the actual cache data. */
-   size_t cache_data_size = sb.st_size - cf_data_size - ck_size;
+   size_t cache_data_size =
+      sb.st_size - cf_data_size - ck_size - cache_item_md_size;
    ret = read_all(fd, data, cache_data_size);
    if (ret == -1)
       goto fail;
@@ -1060,6 +1250,7 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
 
    free(data);
    free(filename);
+   free(file_header);
    close(fd);
 
    if (size)
@@ -1074,6 +1265,8 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
       free(uncompressed_data);
    if (filename)
       free(filename);
+   if (file_header)
+      free(file_header);
    if (fd != -1)
       close(fd);
 
@@ -1084,9 +1277,17 @@ void
 disk_cache_put_key(struct disk_cache *cache, const cache_key key)
 {
    const uint32_t *key_chunk = (const uint32_t *) key;
-   int i = *key_chunk & CACHE_INDEX_KEY_MASK;
+   int i = CPU_TO_LE32(*key_chunk) & CACHE_INDEX_KEY_MASK;
    unsigned char *entry;
 
+   if (cache->blob_put_cb) {
+      cache->blob_put_cb(key, CACHE_KEY_SIZE, key_chunk, sizeof(uint32_t));
+      return;
+   }
+
+   if (cache->path_init_failed)
+      return;
+
    entry = &cache->stored_keys[i * CACHE_KEY_SIZE];
 
    memcpy(entry, key, CACHE_KEY_SIZE);
@@ -1103,9 +1304,17 @@ bool
 disk_cache_has_key(struct disk_cache *cache, const cache_key key)
 {
    const uint32_t *key_chunk = (const uint32_t *) key;
-   int i = *key_chunk & CACHE_INDEX_KEY_MASK;
+   int i = CPU_TO_LE32(*key_chunk) & CACHE_INDEX_KEY_MASK;
    unsigned char *entry;
 
+   if (cache->blob_get_cb) {
+      uint32_t blob;
+      return cache->blob_get_cb(key, CACHE_KEY_SIZE, &blob, sizeof(uint32_t));
+   }
+
+   if (cache->path_init_failed)
+      return false;
+
    entry = &cache->stored_keys[i * CACHE_KEY_SIZE];
 
    return memcmp(entry, key, CACHE_KEY_SIZE) == 0;
@@ -1124,4 +1333,12 @@ disk_cache_compute_key(struct disk_cache *cache, const void *data, size_t size,
    _mesa_sha1_final(&ctx, key);
 }
 
+void
+disk_cache_set_callbacks(struct disk_cache *cache, disk_cache_put_cb put,
+                         disk_cache_get_cb get)
+{
+   cache->blob_put_cb = put;
+   cache->blob_get_cb = get;
+}
+
 #endif /* ENABLE_SHADER_CACHE */