2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #ifdef ENABLE_SHADER_CACHE
32 #include <sys/types.h>
47 #include "util/crc32.h"
48 #include "util/debug.h"
49 #include "util/rand_xor.h"
50 #include "util/u_atomic.h"
51 #include "util/u_queue.h"
52 #include "util/mesa-sha1.h"
53 #include "util/ralloc.h"
54 #include "main/compiler.h"
55 #include "main/errors.h"
57 #include "disk_cache.h"
59 /* Number of bits to mask off from a cache key to get an index. */
60 #define CACHE_INDEX_KEY_BITS 16
62 /* Mask for computing an index from a key. */
63 #define CACHE_INDEX_KEY_MASK ((1 << CACHE_INDEX_KEY_BITS) - 1)
65 /* The number of keys that can be stored in the index. */
66 #define CACHE_INDEX_MAX_KEYS (1 << CACHE_INDEX_KEY_BITS)
68 /* The cache version should be bumped whenever a change is made to the
69 * structure of cache entries or the index. This will give any 3rd party
70 * applications reading the cache entries a chance to adjust to the changes.
72 * - The cache version is checked internally when reading a cache entry. If we
73 * ever have a mismatch we are in big trouble as this means we had a cache
74 * collision. In case of such an event please check the skys for giant
75 * asteroids and that the entire Mesa team hasn't been eaten by wolves.
77 * - There is no strict requirement that cache versions be backwards
78 * compatible but effort should be taken to limit disruption where possible.
80 #define CACHE_VERSION 1
82 /* 3 is the recomended level, with 22 as the absolute maximum */
83 #define ZSTD_COMPRESSION_LEVEL 3
86 /* The path to the cache directory. */
88 bool path_init_failed
;
90 /* Thread queue for compressing and writing cache entries to disk */
91 struct util_queue cache_queue
;
93 /* Seed for rand, which is used to pick a random directory */
94 uint64_t seed_xorshift128plus
[2];
96 /* A pointer to the mmapped index file within the cache directory. */
98 size_t index_mmap_size
;
100 /* Pointer to total size of all objects in cache (within index_mmap) */
103 /* Pointer to stored keys, (within index_mmap). */
104 uint8_t *stored_keys
;
106 /* Maximum size of all cached objects (in bytes). */
109 /* Driver cache keys. */
110 uint8_t *driver_keys_blob
;
111 size_t driver_keys_blob_size
;
113 disk_cache_put_cb blob_put_cb
;
114 disk_cache_get_cb blob_get_cb
;
117 struct disk_cache_put_job
{
118 struct util_queue_fence fence
;
120 struct disk_cache
*cache
;
124 /* Copy of cache data to be compressed and written. */
127 /* Size of data to be compressed and written. */
130 struct cache_item_metadata cache_item_metadata
;
133 /* Create a directory named 'path' if it does not already exist.
135 * Returns: 0 if path already exists as a directory or if created.
136 * -1 in all other cases.
139 mkdir_if_needed(const char *path
)
143 /* If the path exists already, then our work is done if it's a
144 * directory, but it's an error if it is not.
146 if (stat(path
, &sb
) == 0) {
147 if (S_ISDIR(sb
.st_mode
)) {
150 fprintf(stderr
, "Cannot use %s for shader cache (not a directory)"
151 "---disabling.\n", path
);
156 int ret
= mkdir(path
, 0755);
157 if (ret
== 0 || (ret
== -1 && errno
== EEXIST
))
160 fprintf(stderr
, "Failed to create %s for shader cache (%s)---disabling.\n",
161 path
, strerror(errno
));
166 /* Concatenate an existing path and a new name to form a new path. If the new
167 * path does not exist as a directory, create it then return the resulting
168 * name of the new path (ralloc'ed off of 'ctx').
170 * Returns NULL on any error, such as:
172 * <path> does not exist or is not a directory
173 * <path>/<name> exists but is not a directory
174 * <path>/<name> cannot be created as a directory
177 concatenate_and_mkdir(void *ctx
, const char *path
, const char *name
)
182 if (stat(path
, &sb
) != 0 || ! S_ISDIR(sb
.st_mode
))
185 new_path
= ralloc_asprintf(ctx
, "%s/%s", path
, name
);
187 if (mkdir_if_needed(new_path
) == 0)
193 #define DRV_KEY_CPY(_dst, _src, _src_size) \
195 memcpy(_dst, _src, _src_size); \
200 disk_cache_create(const char *gpu_name
, const char *driver_id
,
201 uint64_t driver_flags
)
204 struct disk_cache
*cache
= NULL
;
205 char *path
, *max_size_str
;
211 uint8_t cache_version
= CACHE_VERSION
;
212 size_t cv_size
= sizeof(cache_version
);
214 /* If running as a users other than the real user disable cache */
215 if (geteuid() != getuid())
218 /* A ralloc context for transient data during this invocation. */
219 local
= ralloc_context(NULL
);
223 /* At user request, disable shader cache entirely. */
224 if (env_var_as_boolean("MESA_GLSL_CACHE_DISABLE", false))
227 cache
= rzalloc(NULL
, struct disk_cache
);
231 /* Assume failure. */
232 cache
->path_init_failed
= true;
234 /* Determine path for cache based on the first defined name as follows:
236 * $MESA_GLSL_CACHE_DIR
237 * $XDG_CACHE_HOME/mesa_shader_cache
238 * <pwd.pw_dir>/.cache/mesa_shader_cache
240 path
= getenv("MESA_GLSL_CACHE_DIR");
242 if (mkdir_if_needed(path
) == -1)
245 path
= concatenate_and_mkdir(local
, path
, CACHE_DIR_NAME
);
251 char *xdg_cache_home
= getenv("XDG_CACHE_HOME");
253 if (xdg_cache_home
) {
254 if (mkdir_if_needed(xdg_cache_home
) == -1)
257 path
= concatenate_and_mkdir(local
, xdg_cache_home
, CACHE_DIR_NAME
);
266 struct passwd pwd
, *result
;
268 buf_size
= sysconf(_SC_GETPW_R_SIZE_MAX
);
272 /* Loop until buf_size is large enough to query the directory */
274 buf
= ralloc_size(local
, buf_size
);
276 getpwuid_r(getuid(), &pwd
, buf
, buf_size
, &result
);
280 if (errno
== ERANGE
) {
289 path
= concatenate_and_mkdir(local
, pwd
.pw_dir
, ".cache");
293 path
= concatenate_and_mkdir(local
, path
, CACHE_DIR_NAME
);
298 cache
->path
= ralloc_strdup(cache
, path
);
299 if (cache
->path
== NULL
)
302 path
= ralloc_asprintf(local
, "%s/index", cache
->path
);
306 fd
= open(path
, O_RDWR
| O_CREAT
| O_CLOEXEC
, 0644);
310 if (fstat(fd
, &sb
) == -1)
313 /* Force the index file to be the expected size. */
314 size
= sizeof(*cache
->size
) + CACHE_INDEX_MAX_KEYS
* CACHE_KEY_SIZE
;
315 if (sb
.st_size
!= size
) {
316 if (ftruncate(fd
, size
) == -1)
320 /* We map this shared so that other processes see updates that we
323 * Note: We do use atomic addition to ensure that multiple
324 * processes don't scramble the cache size recorded in the
325 * index. But we don't use any locking to prevent multiple
326 * processes from updating the same entry simultaneously. The idea
327 * is that if either result lands entirely in the index, then
328 * that's equivalent to a well-ordered write followed by an
329 * eviction and a write. On the other hand, if the simultaneous
330 * writes result in a corrupt entry, that's not really any
331 * different than both entries being evicted, (since within the
332 * guarantees of the cryptographic hash, a corrupt entry is
333 * unlikely to ever match a real cache key).
335 cache
->index_mmap
= mmap(NULL
, size
, PROT_READ
| PROT_WRITE
,
337 if (cache
->index_mmap
== MAP_FAILED
)
339 cache
->index_mmap_size
= size
;
341 cache
->size
= (uint64_t *) cache
->index_mmap
;
342 cache
->stored_keys
= cache
->index_mmap
+ sizeof(uint64_t);
346 max_size_str
= getenv("MESA_GLSL_CACHE_MAX_SIZE");
349 max_size
= strtoul(max_size_str
, &end
, 10);
350 if (end
== max_size_str
) {
360 max_size
*= 1024*1024;
366 max_size
*= 1024*1024*1024;
372 /* Default to 1GB for maximum cache size. */
374 max_size
= 1024*1024*1024;
377 cache
->max_size
= max_size
;
379 /* 4 threads were chosen below because just about all modern CPUs currently
380 * available that run Mesa have *at least* 4 cores. For these CPUs allowing
381 * more threads can result in the queue being processed faster, thus
382 * avoiding excessive memory use due to a backlog of cache entrys building
383 * up in the queue. Since we set the UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY
384 * flag this should have little negative impact on low core systems.
386 * The queue will resize automatically when it's full, so adding new jobs
389 util_queue_init(&cache
->cache_queue
, "disk$", 32, 4,
390 UTIL_QUEUE_INIT_RESIZE_IF_FULL
|
391 UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY
|
392 UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY
);
394 cache
->path_init_failed
= false;
401 cache
->driver_keys_blob_size
= cv_size
;
403 /* Create driver id keys */
404 size_t id_size
= strlen(driver_id
) + 1;
405 size_t gpu_name_size
= strlen(gpu_name
) + 1;
406 cache
->driver_keys_blob_size
+= id_size
;
407 cache
->driver_keys_blob_size
+= gpu_name_size
;
409 /* We sometimes store entire structs that contains a pointers in the cache,
410 * use pointer size as a key to avoid hard to debug issues.
412 uint8_t ptr_size
= sizeof(void *);
413 size_t ptr_size_size
= sizeof(ptr_size
);
414 cache
->driver_keys_blob_size
+= ptr_size_size
;
416 size_t driver_flags_size
= sizeof(driver_flags
);
417 cache
->driver_keys_blob_size
+= driver_flags_size
;
419 cache
->driver_keys_blob
=
420 ralloc_size(cache
, cache
->driver_keys_blob_size
);
421 if (!cache
->driver_keys_blob
)
424 uint8_t *drv_key_blob
= cache
->driver_keys_blob
;
425 DRV_KEY_CPY(drv_key_blob
, &cache_version
, cv_size
)
426 DRV_KEY_CPY(drv_key_blob
, driver_id
, id_size
)
427 DRV_KEY_CPY(drv_key_blob
, gpu_name
, gpu_name_size
)
428 DRV_KEY_CPY(drv_key_blob
, &ptr_size
, ptr_size_size
)
429 DRV_KEY_CPY(drv_key_blob
, &driver_flags
, driver_flags_size
)
431 /* Seed our rand function */
432 s_rand_xorshift128plus(cache
->seed_xorshift128plus
, true);
447 disk_cache_destroy(struct disk_cache
*cache
)
449 if (cache
&& !cache
->path_init_failed
) {
450 util_queue_finish(&cache
->cache_queue
);
451 util_queue_destroy(&cache
->cache_queue
);
452 munmap(cache
->index_mmap
, cache
->index_mmap_size
);
459 disk_cache_wait_for_idle(struct disk_cache
*cache
)
461 util_queue_finish(&cache
->cache_queue
);
464 /* Return a filename within the cache's directory corresponding to 'key'. The
465 * returned filename is ralloced with 'cache' as the parent context.
467 * Returns NULL if out of memory.
470 get_cache_file(struct disk_cache
*cache
, const cache_key key
)
475 if (cache
->path_init_failed
)
478 _mesa_sha1_format(buf
, key
);
479 if (asprintf(&filename
, "%s/%c%c/%s", cache
->path
, buf
[0],
480 buf
[1], buf
+ 2) == -1)
486 /* Create the directory that will be needed for the cache file for \key.
488 * Obviously, the implementation here must closely match
489 * _get_cache_file above.
492 make_cache_file_directory(struct disk_cache
*cache
, const cache_key key
)
497 _mesa_sha1_format(buf
, key
);
498 if (asprintf(&dir
, "%s/%c%c", cache
->path
, buf
[0], buf
[1]) == -1)
501 mkdir_if_needed(dir
);
505 /* Given a directory path and predicate function, find the entry with
506 * the oldest access time in that directory for which the predicate
509 * Returns: A malloc'ed string for the path to the chosen file, (or
510 * NULL on any error). The caller should free the string when
514 choose_lru_file_matching(const char *dir_path
,
515 bool (*predicate
)(const char *dir_path
,
517 const char *, const size_t))
520 struct dirent
*entry
;
522 char *lru_name
= NULL
;
523 time_t lru_atime
= 0;
525 dir
= opendir(dir_path
);
530 entry
= readdir(dir
);
535 if (fstatat(dirfd(dir
), entry
->d_name
, &sb
, 0) == 0) {
536 if (!lru_atime
|| (sb
.st_atime
< lru_atime
)) {
537 size_t len
= strlen(entry
->d_name
);
539 if (!predicate(dir_path
, &sb
, entry
->d_name
, len
))
542 char *tmp
= realloc(lru_name
, len
+ 1);
545 memcpy(lru_name
, entry
->d_name
, len
+ 1);
546 lru_atime
= sb
.st_atime
;
552 if (lru_name
== NULL
) {
557 if (asprintf(&filename
, "%s/%s", dir_path
, lru_name
) < 0)
566 /* Is entry a regular file, and not having a name with a trailing
570 is_regular_non_tmp_file(const char *path
, const struct stat
*sb
,
571 const char *d_name
, const size_t len
)
573 if (!S_ISREG(sb
->st_mode
))
576 if (len
>= 4 && strcmp(&d_name
[len
-4], ".tmp") == 0)
582 /* Returns the size of the deleted file, (or 0 on any error). */
584 unlink_lru_file_from_directory(const char *path
)
589 filename
= choose_lru_file_matching(path
, is_regular_non_tmp_file
);
590 if (filename
== NULL
)
593 if (stat(filename
, &sb
) == -1) {
601 return sb
.st_blocks
* 512;
604 /* Is entry a directory with a two-character name, (and not the
605 * special name of ".."). We also return false if the dir is empty.
608 is_two_character_sub_directory(const char *path
, const struct stat
*sb
,
609 const char *d_name
, const size_t len
)
611 if (!S_ISDIR(sb
->st_mode
))
617 if (strcmp(d_name
, "..") == 0)
621 if (asprintf(&subdir
, "%s/%s", path
, d_name
) == -1)
623 DIR *dir
= opendir(subdir
);
629 unsigned subdir_entries
= 0;
631 while ((d
= readdir(dir
)) != NULL
) {
632 if(++subdir_entries
> 2)
637 /* If dir only contains '.' and '..' it must be empty */
638 if (subdir_entries
<= 2)
645 evict_lru_item(struct disk_cache
*cache
)
649 /* With a reasonably-sized, full cache, (and with keys generated
650 * from a cryptographic hash), we can choose two random hex digits
651 * and reasonably expect the directory to exist with a file in it.
652 * Provides pseudo-LRU eviction to reduce checking all cache files.
654 uint64_t rand64
= rand_xorshift128plus(cache
->seed_xorshift128plus
);
655 if (asprintf(&dir_path
, "%s/%02" PRIx64
, cache
->path
, rand64
& 0xff) < 0)
658 size_t size
= unlink_lru_file_from_directory(dir_path
);
663 p_atomic_add(cache
->size
, - (uint64_t)size
);
667 /* In the case where the random choice of directory didn't find
668 * something, we choose the least recently accessed from the
669 * existing directories.
671 * Really, the only reason this code exists is to allow the unit
672 * tests to work, (which use an artificially-small cache to be able
673 * to force a single cached item to be evicted).
675 dir_path
= choose_lru_file_matching(cache
->path
,
676 is_two_character_sub_directory
);
677 if (dir_path
== NULL
)
680 size
= unlink_lru_file_from_directory(dir_path
);
685 p_atomic_add(cache
->size
, - (uint64_t)size
);
689 disk_cache_remove(struct disk_cache
*cache
, const cache_key key
)
693 char *filename
= get_cache_file(cache
, key
);
694 if (filename
== NULL
) {
698 if (stat(filename
, &sb
) == -1) {
707 p_atomic_add(cache
->size
, - (uint64_t)sb
.st_blocks
* 512);
711 read_all(int fd
, void *buf
, size_t count
)
717 for (done
= 0; done
< count
; done
+= read_ret
) {
718 read_ret
= read(fd
, in
+ done
, count
- done
);
719 if (read_ret
== -1 || read_ret
== 0)
726 write_all(int fd
, const void *buf
, size_t count
)
728 const char *out
= buf
;
732 for (done
= 0; done
< count
; done
+= written
) {
733 written
= write(fd
, out
+ done
, count
- done
);
740 /* From the zlib docs:
741 * "If the memory is available, buffers sizes on the order of 128K or 256K
742 * bytes should be used."
744 #define BUFSIZE 256 * 1024
747 * Compresses cache entry in memory and writes it to disk. Returns the size
748 * of the data written to disk.
751 deflate_and_write_to_disk(const void *in_data
, size_t in_data_size
, int dest
,
752 const char *filename
)
755 /* from the zstd docs (https://facebook.github.io/zstd/zstd_manual.html):
756 * compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
758 size_t out_size
= ZSTD_compressBound(in_data_size
);
759 void * out
= malloc(out_size
);
761 size_t ret
= ZSTD_compress(out
, out_size
, in_data
, in_data_size
,
762 ZSTD_COMPRESSION_LEVEL
);
763 if (ZSTD_isError(ret
)) {
767 ssize_t written
= write_all(dest
, out
, ret
);
777 /* allocate deflate state */
779 strm
.zalloc
= Z_NULL
;
781 strm
.opaque
= Z_NULL
;
782 strm
.next_in
= (uint8_t *) in_data
;
783 strm
.avail_in
= in_data_size
;
785 int ret
= deflateInit(&strm
, Z_BEST_COMPRESSION
);
789 /* compress until end of in_data */
790 size_t compressed_size
= 0;
793 out
= malloc(BUFSIZE
* sizeof(unsigned char));
798 int remaining
= in_data_size
- BUFSIZE
;
799 flush
= remaining
> 0 ? Z_NO_FLUSH
: Z_FINISH
;
800 in_data_size
-= BUFSIZE
;
802 /* Run deflate() on input until the output buffer is not full (which
803 * means there is no more data to deflate).
806 strm
.avail_out
= BUFSIZE
;
809 ret
= deflate(&strm
, flush
); /* no bad return value */
810 assert(ret
!= Z_STREAM_ERROR
); /* state not clobbered */
812 size_t have
= BUFSIZE
- strm
.avail_out
;
813 compressed_size
+= have
;
815 ssize_t written
= write_all(dest
, out
, have
);
817 (void)deflateEnd(&strm
);
821 } while (strm
.avail_out
== 0);
823 /* all input should be used */
824 assert(strm
.avail_in
== 0);
826 } while (flush
!= Z_FINISH
);
828 /* stream should be complete */
829 assert(ret
== Z_STREAM_END
);
831 /* clean up and return */
832 (void)deflateEnd(&strm
);
834 return compressed_size
;
838 static struct disk_cache_put_job
*
839 create_put_job(struct disk_cache
*cache
, const cache_key key
,
840 const void *data
, size_t size
,
841 struct cache_item_metadata
*cache_item_metadata
)
843 struct disk_cache_put_job
*dc_job
= (struct disk_cache_put_job
*)
844 malloc(sizeof(struct disk_cache_put_job
) + size
);
847 dc_job
->cache
= cache
;
848 memcpy(dc_job
->key
, key
, sizeof(cache_key
));
849 dc_job
->data
= dc_job
+ 1;
850 memcpy(dc_job
->data
, data
, size
);
853 /* Copy the cache item metadata */
854 if (cache_item_metadata
) {
855 dc_job
->cache_item_metadata
.type
= cache_item_metadata
->type
;
856 if (cache_item_metadata
->type
== CACHE_ITEM_TYPE_GLSL
) {
857 dc_job
->cache_item_metadata
.num_keys
=
858 cache_item_metadata
->num_keys
;
859 dc_job
->cache_item_metadata
.keys
= (cache_key
*)
860 malloc(cache_item_metadata
->num_keys
* sizeof(cache_key
));
862 if (!dc_job
->cache_item_metadata
.keys
)
865 memcpy(dc_job
->cache_item_metadata
.keys
,
866 cache_item_metadata
->keys
,
867 sizeof(cache_key
) * cache_item_metadata
->num_keys
);
870 dc_job
->cache_item_metadata
.type
= CACHE_ITEM_TYPE_UNKNOWN
;
871 dc_job
->cache_item_metadata
.keys
= NULL
;
884 destroy_put_job(void *job
, int thread_index
)
887 struct disk_cache_put_job
*dc_job
= (struct disk_cache_put_job
*) job
;
888 free(dc_job
->cache_item_metadata
.keys
);
894 struct cache_entry_file_data
{
896 uint32_t uncompressed_size
;
900 cache_put(void *job
, int thread_index
)
904 int fd
= -1, fd_final
= -1, err
, ret
;
906 char *filename
= NULL
, *filename_tmp
= NULL
;
907 struct disk_cache_put_job
*dc_job
= (struct disk_cache_put_job
*) job
;
909 filename
= get_cache_file(dc_job
->cache
, dc_job
->key
);
910 if (filename
== NULL
)
913 /* If the cache is too large, evict something else first. */
914 while (*dc_job
->cache
->size
+ dc_job
->size
> dc_job
->cache
->max_size
&&
916 evict_lru_item(dc_job
->cache
);
920 /* Write to a temporary file to allow for an atomic rename to the
921 * final destination filename, (to prevent any readers from seeing
922 * a partially written file).
924 if (asprintf(&filename_tmp
, "%s.tmp", filename
) == -1)
927 fd
= open(filename_tmp
, O_WRONLY
| O_CLOEXEC
| O_CREAT
, 0644);
929 /* Make the two-character subdirectory within the cache as needed. */
934 make_cache_file_directory(dc_job
->cache
, dc_job
->key
);
936 fd
= open(filename_tmp
, O_WRONLY
| O_CLOEXEC
| O_CREAT
, 0644);
941 /* With the temporary file open, we take an exclusive flock on
942 * it. If the flock fails, then another process still has the file
943 * open with the flock held. So just let that file be responsible
944 * for writing the file.
947 err
= flock(fd
, LOCK_EX
| LOCK_NB
);
949 struct flock lock
= {
951 .l_len
= 0, /* entire file */
955 err
= fcntl(fd
, F_SETLK
, &lock
);
960 /* Now that we have the lock on the open temporary file, we can
961 * check to see if the destination file already exists. If so,
962 * another process won the race between when we saw that the file
963 * didn't exist and now. In this case, we don't do anything more,
964 * (to ensure the size accounting of the cache doesn't get off).
966 fd_final
= open(filename
, O_RDONLY
| O_CLOEXEC
);
967 if (fd_final
!= -1) {
968 unlink(filename_tmp
);
972 /* OK, we're now on the hook to write out a file that we know is
973 * not in the cache, and is also not being written out to the cache
974 * by some other process.
977 /* Write the driver_keys_blob, this can be used find information about the
978 * mesa version that produced the entry or deal with hash collisions,
979 * should that ever become a real problem.
981 ret
= write_all(fd
, dc_job
->cache
->driver_keys_blob
,
982 dc_job
->cache
->driver_keys_blob_size
);
984 unlink(filename_tmp
);
988 /* Write the cache item metadata. This data can be used to deal with
989 * hash collisions, as well as providing useful information to 3rd party
990 * tools reading the cache files.
992 ret
= write_all(fd
, &dc_job
->cache_item_metadata
.type
,
995 unlink(filename_tmp
);
999 if (dc_job
->cache_item_metadata
.type
== CACHE_ITEM_TYPE_GLSL
) {
1000 ret
= write_all(fd
, &dc_job
->cache_item_metadata
.num_keys
,
1003 unlink(filename_tmp
);
1007 ret
= write_all(fd
, dc_job
->cache_item_metadata
.keys
[0],
1008 dc_job
->cache_item_metadata
.num_keys
*
1011 unlink(filename_tmp
);
1016 /* Create CRC of the data. We will read this when restoring the cache and
1017 * use it to check for corruption.
1019 struct cache_entry_file_data cf_data
;
1020 cf_data
.crc32
= util_hash_crc32(dc_job
->data
, dc_job
->size
);
1021 cf_data
.uncompressed_size
= dc_job
->size
;
1023 size_t cf_data_size
= sizeof(cf_data
);
1024 ret
= write_all(fd
, &cf_data
, cf_data_size
);
1026 unlink(filename_tmp
);
1030 /* Now, finally, write out the contents to the temporary file, then
1031 * rename them atomically to the destination filename, and also
1032 * perform an atomic increment of the total cache size.
1034 size_t file_size
= deflate_and_write_to_disk(dc_job
->data
, dc_job
->size
,
1036 if (file_size
== 0) {
1037 unlink(filename_tmp
);
1040 ret
= rename(filename_tmp
, filename
);
1042 unlink(filename_tmp
);
1047 if (stat(filename
, &sb
) == -1) {
1048 /* Something went wrong remove the file */
1053 p_atomic_add(dc_job
->cache
->size
, sb
.st_blocks
* 512);
1058 /* This close finally releases the flock, (now that the final file
1059 * has been renamed into place and the size has been added).
1068 disk_cache_put(struct disk_cache
*cache
, const cache_key key
,
1069 const void *data
, size_t size
,
1070 struct cache_item_metadata
*cache_item_metadata
)
1072 if (cache
->blob_put_cb
) {
1073 cache
->blob_put_cb(key
, CACHE_KEY_SIZE
, data
, size
);
1077 if (cache
->path_init_failed
)
1080 struct disk_cache_put_job
*dc_job
=
1081 create_put_job(cache
, key
, data
, size
, cache_item_metadata
);
1084 util_queue_fence_init(&dc_job
->fence
);
1085 util_queue_add_job(&cache
->cache_queue
, dc_job
, &dc_job
->fence
,
1086 cache_put
, destroy_put_job
, dc_job
->size
);
1091 * Decompresses cache entry, returns true if successful.
1094 inflate_cache_data(uint8_t *in_data
, size_t in_data_size
,
1095 uint8_t *out_data
, size_t out_data_size
)
1098 size_t ret
= ZSTD_decompress(out_data
, out_data_size
, in_data
, in_data_size
);
1099 return !ZSTD_isError(ret
);
1103 /* allocate inflate state */
1104 strm
.zalloc
= Z_NULL
;
1105 strm
.zfree
= Z_NULL
;
1106 strm
.opaque
= Z_NULL
;
1107 strm
.next_in
= in_data
;
1108 strm
.avail_in
= in_data_size
;
1109 strm
.next_out
= out_data
;
1110 strm
.avail_out
= out_data_size
;
1112 int ret
= inflateInit(&strm
);
1116 ret
= inflate(&strm
, Z_NO_FLUSH
);
1117 assert(ret
!= Z_STREAM_ERROR
); /* state not clobbered */
1119 /* Unless there was an error we should have decompressed everything in one
1120 * go as we know the uncompressed file size.
1122 if (ret
!= Z_STREAM_END
) {
1123 (void)inflateEnd(&strm
);
1126 assert(strm
.avail_out
== 0);
1128 /* clean up and return */
1129 (void)inflateEnd(&strm
);
1135 disk_cache_get(struct disk_cache
*cache
, const cache_key key
, size_t *size
)
1139 char *filename
= NULL
;
1140 uint8_t *data
= NULL
;
1141 uint8_t *uncompressed_data
= NULL
;
1142 uint8_t *file_header
= NULL
;
1147 if (cache
->blob_get_cb
) {
1148 /* This is what Android EGL defines as the maxValueSize in egl_cache_t
1149 * class implementation.
1151 const signed long max_blob_size
= 64 * 1024;
1152 void *blob
= malloc(max_blob_size
);
1157 cache
->blob_get_cb(key
, CACHE_KEY_SIZE
, blob
, max_blob_size
);
1169 filename
= get_cache_file(cache
, key
);
1170 if (filename
== NULL
)
1173 fd
= open(filename
, O_RDONLY
| O_CLOEXEC
);
1177 if (fstat(fd
, &sb
) == -1)
1180 data
= malloc(sb
.st_size
);
1184 size_t ck_size
= cache
->driver_keys_blob_size
;
1185 file_header
= malloc(ck_size
);
1189 if (sb
.st_size
< ck_size
)
1192 ret
= read_all(fd
, file_header
, ck_size
);
1196 /* Check for extremely unlikely hash collisions */
1197 if (memcmp(cache
->driver_keys_blob
, file_header
, ck_size
) != 0) {
1198 assert(!"Mesa cache keys mismatch!");
1202 size_t cache_item_md_size
= sizeof(uint32_t);
1204 ret
= read_all(fd
, &md_type
, cache_item_md_size
);
1208 if (md_type
== CACHE_ITEM_TYPE_GLSL
) {
1210 cache_item_md_size
+= sizeof(uint32_t);
1211 ret
= read_all(fd
, &num_keys
, sizeof(uint32_t));
1215 /* The cache item metadata is currently just used for distributing
1216 * precompiled shaders, they are not used by Mesa so just skip them for
1218 * TODO: pass the metadata back to the caller and do some basic
1221 cache_item_md_size
+= num_keys
* sizeof(cache_key
);
1222 ret
= lseek(fd
, num_keys
* sizeof(cache_key
), SEEK_CUR
);
1227 /* Load the CRC that was created when the file was written. */
1228 struct cache_entry_file_data cf_data
;
1229 size_t cf_data_size
= sizeof(cf_data
);
1230 ret
= read_all(fd
, &cf_data
, cf_data_size
);
1234 /* Load the actual cache data. */
1235 size_t cache_data_size
=
1236 sb
.st_size
- cf_data_size
- ck_size
- cache_item_md_size
;
1237 ret
= read_all(fd
, data
, cache_data_size
);
1241 /* Uncompress the cache data */
1242 uncompressed_data
= malloc(cf_data
.uncompressed_size
);
1243 if (!inflate_cache_data(data
, cache_data_size
, uncompressed_data
,
1244 cf_data
.uncompressed_size
))
1247 /* Check the data for corruption */
1248 if (cf_data
.crc32
!= util_hash_crc32(uncompressed_data
,
1249 cf_data
.uncompressed_size
))
1258 *size
= cf_data
.uncompressed_size
;
1260 return uncompressed_data
;
1265 if (uncompressed_data
)
1266 free(uncompressed_data
);
1278 disk_cache_put_key(struct disk_cache
*cache
, const cache_key key
)
1280 const uint32_t *key_chunk
= (const uint32_t *) key
;
1281 int i
= CPU_TO_LE32(*key_chunk
) & CACHE_INDEX_KEY_MASK
;
1282 unsigned char *entry
;
1284 if (cache
->blob_put_cb
) {
1285 cache
->blob_put_cb(key
, CACHE_KEY_SIZE
, key_chunk
, sizeof(uint32_t));
1289 if (cache
->path_init_failed
)
1292 entry
= &cache
->stored_keys
[i
* CACHE_KEY_SIZE
];
1294 memcpy(entry
, key
, CACHE_KEY_SIZE
);
1297 /* This function lets us test whether a given key was previously
1298 * stored in the cache with disk_cache_put_key(). The implement is
1299 * efficient by not using syscalls or hitting the disk. It's not
1300 * race-free, but the races are benign. If we race with someone else
1301 * calling disk_cache_put_key, then that's just an extra cache miss and an
1305 disk_cache_has_key(struct disk_cache
*cache
, const cache_key key
)
1307 const uint32_t *key_chunk
= (const uint32_t *) key
;
1308 int i
= CPU_TO_LE32(*key_chunk
) & CACHE_INDEX_KEY_MASK
;
1309 unsigned char *entry
;
1311 if (cache
->blob_get_cb
) {
1313 return cache
->blob_get_cb(key
, CACHE_KEY_SIZE
, &blob
, sizeof(uint32_t));
1316 if (cache
->path_init_failed
)
1319 entry
= &cache
->stored_keys
[i
* CACHE_KEY_SIZE
];
1321 return memcmp(entry
, key
, CACHE_KEY_SIZE
) == 0;
1325 disk_cache_compute_key(struct disk_cache
*cache
, const void *data
, size_t size
,
1328 struct mesa_sha1 ctx
;
1330 _mesa_sha1_init(&ctx
);
1331 _mesa_sha1_update(&ctx
, cache
->driver_keys_blob
,
1332 cache
->driver_keys_blob_size
);
1333 _mesa_sha1_update(&ctx
, data
, size
);
1334 _mesa_sha1_final(&ctx
, key
);
1338 disk_cache_set_callbacks(struct disk_cache
*cache
, disk_cache_put_cb put
,
1339 disk_cache_get_cb get
)
1341 cache
->blob_put_cb
= put
;
1342 cache
->blob_get_cb
= get
;
1345 #endif /* ENABLE_SHADER_CACHE */