2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #ifdef ENABLE_SHADER_CACHE
32 #include <sys/types.h>
42 #include "util/crc32.h"
43 #include "util/u_atomic.h"
44 #include "util/u_queue.h"
45 #include "util/mesa-sha1.h"
46 #include "util/ralloc.h"
47 #include "main/errors.h"
49 #include "disk_cache.h"
51 /* Number of bits to mask off from a cache key to get an index. */
52 #define CACHE_INDEX_KEY_BITS 16
54 /* Mask for computing an index from a key. */
55 #define CACHE_INDEX_KEY_MASK ((1 << CACHE_INDEX_KEY_BITS) - 1)
57 /* The number of keys that can be stored in the index. */
58 #define CACHE_INDEX_MAX_KEYS (1 << CACHE_INDEX_KEY_BITS)
61 /* The path to the cache directory. */
64 /* Thread queue for compressing and writing cache entries to disk */
65 struct util_queue cache_queue
;
67 /* A pointer to the mmapped index file within the cache directory. */
69 size_t index_mmap_size
;
71 /* Pointer to total size of all objects in cache (within index_mmap) */
74 /* Pointer to stored keys, (within index_mmap). */
77 /* Maximum size of all cached objects (in bytes). */
81 /* Create a directory named 'path' if it does not already exist.
83 * Returns: 0 if path already exists as a directory or if created.
84 * -1 in all other cases.
87 mkdir_if_needed(const char *path
)
91 /* If the path exists already, then our work is done if it's a
92 * directory, but it's an error if it is not.
94 if (stat(path
, &sb
) == 0) {
95 if (S_ISDIR(sb
.st_mode
)) {
98 fprintf(stderr
, "Cannot use %s for shader cache (not a directory)"
99 "---disabling.\n", path
);
104 int ret
= mkdir(path
, 0755);
105 if (ret
== 0 || (ret
== -1 && errno
== EEXIST
))
108 fprintf(stderr
, "Failed to create %s for shader cache (%s)---disabling.\n",
109 path
, strerror(errno
));
114 /* Concatenate an existing path and a new name to form a new path. If the new
115 * path does not exist as a directory, create it then return the resulting
116 * name of the new path (ralloc'ed off of 'ctx').
118 * Returns NULL on any error, such as:
120 * <path> does not exist or is not a directory
121 * <path>/<name> exists but is not a directory
122 * <path>/<name> cannot be created as a directory
125 concatenate_and_mkdir(void *ctx
, const char *path
, const char *name
)
130 if (stat(path
, &sb
) != 0 || ! S_ISDIR(sb
.st_mode
))
133 new_path
= ralloc_asprintf(ctx
, "%s/%s", path
, name
);
135 if (mkdir_if_needed(new_path
) == 0)
142 remove_dir(const char *fpath
, const struct stat
*sb
,
143 int typeflag
, struct FTW
*ftwbuf
)
145 if (S_ISREG(sb
->st_mode
))
147 else if (S_ISDIR(sb
->st_mode
))
154 remove_old_cache_directories(void *mem_ctx
, const char *path
,
155 const char *timestamp
)
157 DIR *dir
= opendir(path
);
159 struct dirent
* d_entry
;
160 while((d_entry
= readdir(dir
)) != NULL
)
163 ralloc_asprintf(mem_ctx
, "%s/%s", path
, d_entry
->d_name
);
166 if (stat(full_path
, &sb
) == 0 && S_ISDIR(sb
.st_mode
) &&
167 strcmp(d_entry
->d_name
, timestamp
) != 0 &&
168 strcmp(d_entry
->d_name
, "..") != 0 &&
169 strcmp(d_entry
->d_name
, ".") != 0) {
170 nftw(full_path
, remove_dir
, 20, FTW_DEPTH
);
178 create_mesa_cache_dir(void *mem_ctx
, const char *path
, const char *timestamp
,
179 const char *gpu_name
)
181 char *new_path
= concatenate_and_mkdir(mem_ctx
, path
, "mesa");
182 if (new_path
== NULL
)
185 /* Create a parent architecture directory so that we don't remove cache
186 * files for other architectures. In theory we could share the cache
187 * between architectures but we have no way of knowing if they were created
188 * by a compatible Mesa version.
190 new_path
= concatenate_and_mkdir(mem_ctx
, new_path
, get_arch_bitness_str());
191 if (new_path
== NULL
)
194 /* Remove cache directories for old Mesa versions */
195 remove_old_cache_directories(mem_ctx
, new_path
, timestamp
);
197 new_path
= concatenate_and_mkdir(mem_ctx
, new_path
, timestamp
);
198 if (new_path
== NULL
)
201 new_path
= concatenate_and_mkdir(mem_ctx
, new_path
, gpu_name
);
202 if (new_path
== NULL
)
209 disk_cache_create(const char *gpu_name
, const char *timestamp
)
212 struct disk_cache
*cache
= NULL
;
213 char *path
, *max_size_str
;
219 /* If running as a users other than the real user disable cache */
220 if (geteuid() != getuid())
223 /* A ralloc context for transient data during this invocation. */
224 local
= ralloc_context(NULL
);
228 /* At user request, disable shader cache entirely. */
229 if (getenv("MESA_GLSL_CACHE_DISABLE"))
232 /* Determine path for cache based on the first defined name as follows:
234 * $MESA_GLSL_CACHE_DIR
235 * $XDG_CACHE_HOME/mesa
236 * <pwd.pw_dir>/.cache/mesa
238 path
= getenv("MESA_GLSL_CACHE_DIR");
240 if (mkdir_if_needed(path
) == -1)
243 path
= create_mesa_cache_dir(local
, path
, timestamp
,
250 char *xdg_cache_home
= getenv("XDG_CACHE_HOME");
252 if (xdg_cache_home
) {
253 if (mkdir_if_needed(xdg_cache_home
) == -1)
256 path
= create_mesa_cache_dir(local
, xdg_cache_home
, timestamp
,
266 struct passwd pwd
, *result
;
268 buf_size
= sysconf(_SC_GETPW_R_SIZE_MAX
);
272 /* Loop until buf_size is large enough to query the directory */
274 buf
= ralloc_size(local
, buf_size
);
276 getpwuid_r(getuid(), &pwd
, buf
, buf_size
, &result
);
280 if (errno
== ERANGE
) {
289 path
= concatenate_and_mkdir(local
, pwd
.pw_dir
, ".cache");
293 path
= create_mesa_cache_dir(local
, path
, timestamp
, gpu_name
);
298 cache
= ralloc(NULL
, struct disk_cache
);
302 cache
->path
= ralloc_strdup(cache
, path
);
303 if (cache
->path
== NULL
)
306 path
= ralloc_asprintf(local
, "%s/index", cache
->path
);
310 fd
= open(path
, O_RDWR
| O_CREAT
| O_CLOEXEC
, 0644);
314 if (fstat(fd
, &sb
) == -1)
317 /* Force the index file to be the expected size. */
318 size
= sizeof(*cache
->size
) + CACHE_INDEX_MAX_KEYS
* CACHE_KEY_SIZE
;
319 if (sb
.st_size
!= size
) {
320 if (ftruncate(fd
, size
) == -1)
324 /* We map this shared so that other processes see updates that we
327 * Note: We do use atomic addition to ensure that multiple
328 * processes don't scramble the cache size recorded in the
329 * index. But we don't use any locking to prevent multiple
330 * processes from updating the same entry simultaneously. The idea
331 * is that if either result lands entirely in the index, then
332 * that's equivalent to a well-ordered write followed by an
333 * eviction and a write. On the other hand, if the simultaneous
334 * writes result in a corrupt entry, that's not really any
335 * different than both entries being evicted, (since within the
336 * guarantees of the cryptographic hash, a corrupt entry is
337 * unlikely to ever match a real cache key).
339 cache
->index_mmap
= mmap(NULL
, size
, PROT_READ
| PROT_WRITE
,
341 if (cache
->index_mmap
== MAP_FAILED
)
343 cache
->index_mmap_size
= size
;
347 cache
->size
= (uint64_t *) cache
->index_mmap
;
348 cache
->stored_keys
= cache
->index_mmap
+ sizeof(uint64_t);
352 max_size_str
= getenv("MESA_GLSL_CACHE_MAX_SIZE");
355 max_size
= strtoul(max_size_str
, &end
, 10);
356 if (end
== max_size_str
) {
366 max_size
*= 1024*1024;
372 max_size
*= 1024*1024*1024;
378 /* Default to 1GB for maximum cache size. */
380 max_size
= 1024*1024*1024;
382 cache
->max_size
= max_size
;
384 /* A limit of 32 jobs was choosen as observations of Deus Ex start-up times
385 * showed that we reached at most 11 jobs on an Intel i5-6400 CPU@2.70GHz
386 * (a fairly modest desktop CPU). 1 thread was chosen because we don't
387 * really care about getting things to disk quickly just that it's not
388 * blocking other tasks.
390 util_queue_init(&cache
->cache_queue
, "disk_cache", 32, 1);
407 disk_cache_destroy(struct disk_cache
*cache
)
410 util_queue_destroy(&cache
->cache_queue
);
411 munmap(cache
->index_mmap
, cache
->index_mmap_size
);
417 /* Return a filename within the cache's directory corresponding to 'key'. The
418 * returned filename is ralloced with 'cache' as the parent context.
420 * Returns NULL if out of memory.
423 get_cache_file(struct disk_cache
*cache
, const cache_key key
)
428 _mesa_sha1_format(buf
, key
);
429 if (asprintf(&filename
, "%s/%c%c/%s", cache
->path
, buf
[0],
430 buf
[1], buf
+ 2) == -1)
436 /* Create the directory that will be needed for the cache file for \key.
438 * Obviously, the implementation here must closely match
439 * _get_cache_file above.
442 make_cache_file_directory(struct disk_cache
*cache
, const cache_key key
)
447 _mesa_sha1_format(buf
, key
);
448 if (asprintf(&dir
, "%s/%c%c", cache
->path
, buf
[0], buf
[1]) == -1)
451 mkdir_if_needed(dir
);
455 /* Given a directory path and predicate function, count all entries in
456 * that directory for which the predicate returns true. Then choose a
457 * random entry from among those counted.
459 * Returns: A malloc'ed string for the path to the chosen file, (or
460 * NULL on any error). The caller should free the string when
464 choose_random_file_matching(const char *dir_path
,
465 bool (*predicate
)(const struct dirent
*,
466 const char *dir_path
))
469 struct dirent
*entry
;
470 unsigned int count
, victim
;
473 dir
= opendir(dir_path
);
480 entry
= readdir(dir
);
483 if (!predicate(entry
, dir_path
))
494 victim
= rand() % count
;
500 entry
= readdir(dir
);
503 if (!predicate(entry
, dir_path
))
516 if (asprintf(&filename
, "%s/%s", dir_path
, entry
->d_name
) < 0)
524 /* Is entry a regular file, and not having a name with a trailing
528 is_regular_non_tmp_file(const struct dirent
*entry
, const char *path
)
531 if (asprintf(&filename
, "%s/%s", path
, entry
->d_name
) == -1)
535 int res
= stat(filename
, &sb
);
538 if (res
== -1 || !S_ISREG(sb
.st_mode
))
541 size_t len
= strlen (entry
->d_name
);
542 if (len
>= 4 && strcmp(&entry
->d_name
[len
-4], ".tmp") == 0)
548 /* Returns the size of the deleted file, (or 0 on any error). */
550 unlink_random_file_from_directory(const char *path
)
555 filename
= choose_random_file_matching(path
, is_regular_non_tmp_file
);
556 if (filename
== NULL
)
559 if (stat(filename
, &sb
) == -1) {
571 /* Is entry a directory with a two-character name, (and not the
572 * special name of "..")
575 is_two_character_sub_directory(const struct dirent
*entry
, const char *path
)
578 if (asprintf(&subdir
, "%s/%s", path
, entry
->d_name
) == -1)
582 int res
= stat(subdir
, &sb
);
585 if (res
== -1 || !S_ISDIR(sb
.st_mode
))
588 if (strlen(entry
->d_name
) != 2)
591 if (strcmp(entry
->d_name
, "..") == 0)
598 evict_random_item(struct disk_cache
*cache
)
600 const char hex
[] = "0123456789abcde";
605 /* With a reasonably-sized, full cache, (and with keys generated
606 * from a cryptographic hash), we can choose two random hex digits
607 * and reasonably expect the directory to exist with a file in it.
612 if (asprintf(&dir_path
, "%s/%c%c", cache
->path
, hex
[a
], hex
[b
]) < 0)
615 size
= unlink_random_file_from_directory(dir_path
);
620 p_atomic_add(cache
->size
, - (uint64_t)size
);
624 /* In the case where the random choice of directory didn't find
625 * something, we choose randomly from the existing directories.
627 * Really, the only reason this code exists is to allow the unit
628 * tests to work, (which use an artificially-small cache to be able
629 * to force a single cached item to be evicted).
631 dir_path
= choose_random_file_matching(cache
->path
,
632 is_two_character_sub_directory
);
633 if (dir_path
== NULL
)
636 size
= unlink_random_file_from_directory(dir_path
);
641 p_atomic_add(cache
->size
, - (uint64_t)size
);
645 disk_cache_remove(struct disk_cache
*cache
, const cache_key key
)
649 char *filename
= get_cache_file(cache
, key
);
650 if (filename
== NULL
) {
654 if (stat(filename
, &sb
) == -1) {
663 p_atomic_add(cache
->size
, - (uint64_t)sb
.st_size
);
666 /* From the zlib docs:
667 * "If the memory is available, buffers sizes on the order of 128K or 256K
668 * bytes should be used."
670 #define BUFSIZE 256 * 1024
673 * Compresses cache entry in memory and writes it to disk. Returns the size
674 * of the data written to disk.
677 deflate_and_write_to_disk(const void *in_data
, size_t in_data_size
, int dest
,
678 const char *filename
)
680 unsigned char out
[BUFSIZE
];
682 /* allocate deflate state */
684 strm
.zalloc
= Z_NULL
;
686 strm
.opaque
= Z_NULL
;
687 strm
.next_in
= (uint8_t *) in_data
;
688 strm
.avail_in
= in_data_size
;
690 int ret
= deflateInit(&strm
, Z_BEST_COMPRESSION
);
694 /* compress until end of in_data */
695 size_t compressed_size
= 0;
698 int remaining
= in_data_size
- BUFSIZE
;
699 flush
= remaining
> 0 ? Z_NO_FLUSH
: Z_FINISH
;
700 in_data_size
-= BUFSIZE
;
702 /* Run deflate() on input until the output buffer is not full (which
703 * means there is no more data to deflate).
706 strm
.avail_out
= BUFSIZE
;
709 ret
= deflate(&strm
, flush
); /* no bad return value */
710 assert(ret
!= Z_STREAM_ERROR
); /* state not clobbered */
712 size_t have
= BUFSIZE
- strm
.avail_out
;
713 compressed_size
+= have
;
716 for (size_t len
= 0; len
< have
; len
+= written
) {
717 written
= write(dest
, out
+ len
, have
- len
);
719 (void)deflateEnd(&strm
);
723 } while (strm
.avail_out
== 0);
725 /* all input should be used */
726 assert(strm
.avail_in
== 0);
728 } while (flush
!= Z_FINISH
);
730 /* stream should be complete */
731 assert(ret
== Z_STREAM_END
);
733 /* clean up and return */
734 (void)deflateEnd(&strm
);
735 return compressed_size
;
738 struct cache_entry_file_data
{
740 uint32_t uncompressed_size
;
744 disk_cache_put(struct disk_cache
*cache
,
749 int fd
= -1, fd_final
= -1, err
, ret
;
751 char *filename
= NULL
, *filename_tmp
= NULL
;
753 filename
= get_cache_file(cache
, key
);
754 if (filename
== NULL
)
757 /* Write to a temporary file to allow for an atomic rename to the
758 * final destination filename, (to prevent any readers from seeing
759 * a partially written file).
761 if (asprintf(&filename_tmp
, "%s.tmp", filename
) == -1)
764 fd
= open(filename_tmp
, O_WRONLY
| O_CLOEXEC
| O_CREAT
, 0644);
766 /* Make the two-character subdirectory within the cache as needed. */
771 make_cache_file_directory(cache
, key
);
773 fd
= open(filename_tmp
, O_WRONLY
| O_CLOEXEC
| O_CREAT
, 0644);
778 /* With the temporary file open, we take an exclusive flock on
779 * it. If the flock fails, then another process still has the file
780 * open with the flock held. So just let that file be responsible
781 * for writing the file.
783 err
= flock(fd
, LOCK_EX
| LOCK_NB
);
787 /* Now that we have the lock on the open temporary file, we can
788 * check to see if the destination file already exists. If so,
789 * another process won the race between when we saw that the file
790 * didn't exist and now. In this case, we don't do anything more,
791 * (to ensure the size accounting of the cache doesn't get off).
793 fd_final
= open(filename
, O_RDONLY
| O_CLOEXEC
);
797 /* OK, we're now on the hook to write out a file that we know is
798 * not in the cache, and is also not being written out to the cache
799 * by some other process.
801 * Before we do that, if the cache is too large, evict something
804 if (*cache
->size
+ size
> cache
->max_size
)
805 evict_random_item(cache
);
807 /* Create CRC of the data and store at the start of the file. We will
808 * read this when restoring the cache and use it to check for corruption.
810 struct cache_entry_file_data cf_data
;
811 cf_data
.crc32
= util_hash_crc32(data
, size
);
812 cf_data
.uncompressed_size
= size
;
814 size_t cf_data_size
= sizeof(cf_data
);
815 for (len
= 0; len
< cf_data_size
; len
+= ret
) {
816 ret
= write(fd
, ((uint8_t *) &cf_data
) + len
, cf_data_size
- len
);
818 unlink(filename_tmp
);
823 /* Now, finally, write out the contents to the temporary file, then
824 * rename them atomically to the destination filename, and also
825 * perform an atomic increment of the total cache size.
827 size_t file_size
= deflate_and_write_to_disk(data
, size
, fd
, filename_tmp
);
828 if (file_size
== 0) {
829 unlink(filename_tmp
);
832 rename(filename_tmp
, filename
);
834 file_size
+= cf_data_size
;
835 p_atomic_add(cache
->size
, file_size
);
840 /* This close finally releases the flock, (now that the final dile
841 * has been renamed into place and the size has been added).
852 * Decompresses cache entry, returns true if successful.
855 inflate_cache_data(uint8_t *in_data
, size_t in_data_size
,
856 uint8_t *out_data
, size_t out_data_size
)
860 /* allocate inflate state */
861 strm
.zalloc
= Z_NULL
;
863 strm
.opaque
= Z_NULL
;
864 strm
.next_in
= in_data
;
865 strm
.avail_in
= in_data_size
;
866 strm
.next_out
= out_data
;
867 strm
.avail_out
= out_data_size
;
869 int ret
= inflateInit(&strm
);
873 ret
= inflate(&strm
, Z_NO_FLUSH
);
874 assert(ret
!= Z_STREAM_ERROR
); /* state not clobbered */
876 /* Unless there was an error we should have decompressed everything in one
877 * go as we know the uncompressed file size.
879 if (ret
!= Z_STREAM_END
) {
880 (void)inflateEnd(&strm
);
883 assert(strm
.avail_out
== 0);
885 /* clean up and return */
886 (void)inflateEnd(&strm
);
891 disk_cache_get(struct disk_cache
*cache
, const cache_key key
, size_t *size
)
893 int fd
= -1, ret
, len
;
895 char *filename
= NULL
;
896 uint8_t *data
= NULL
;
897 uint8_t *uncompressed_data
= NULL
;
902 filename
= get_cache_file(cache
, key
);
903 if (filename
== NULL
)
906 fd
= open(filename
, O_RDONLY
| O_CLOEXEC
);
910 if (fstat(fd
, &sb
) == -1)
913 data
= malloc(sb
.st_size
);
917 /* Load the CRC that was created when the file was written. */
918 struct cache_entry_file_data cf_data
;
919 size_t cf_data_size
= sizeof(cf_data
);
920 assert(sb
.st_size
> cf_data_size
);
921 for (len
= 0; len
< cf_data_size
; len
+= ret
) {
922 ret
= read(fd
, ((uint8_t *) &cf_data
) + len
, cf_data_size
- len
);
927 /* Load the actual cache data. */
928 size_t cache_data_size
= sb
.st_size
- cf_data_size
;
929 for (len
= 0; len
< cache_data_size
; len
+= ret
) {
930 ret
= read(fd
, data
+ len
, cache_data_size
- len
);
935 /* Uncompress the cache data */
936 uncompressed_data
= malloc(cf_data
.uncompressed_size
);
937 if (!inflate_cache_data(data
, cache_data_size
, uncompressed_data
,
938 cf_data
.uncompressed_size
))
941 /* Check the data for corruption */
942 if (cf_data
.crc32
!= util_hash_crc32(uncompressed_data
,
943 cf_data
.uncompressed_size
))
951 *size
= cf_data
.uncompressed_size
;
953 return uncompressed_data
;
958 if (uncompressed_data
)
959 free(uncompressed_data
);
969 disk_cache_put_key(struct disk_cache
*cache
, const cache_key key
)
971 const uint32_t *key_chunk
= (const uint32_t *) key
;
972 int i
= *key_chunk
& CACHE_INDEX_KEY_MASK
;
973 unsigned char *entry
;
975 entry
= &cache
->stored_keys
[i
+ CACHE_KEY_SIZE
];
977 memcpy(entry
, key
, CACHE_KEY_SIZE
);
980 /* This function lets us test whether a given key was previously
981 * stored in the cache with disk_cache_put_key(). The implement is
982 * efficient by not using syscalls or hitting the disk. It's not
983 * race-free, but the races are benign. If we race with someone else
984 * calling disk_cache_put_key, then that's just an extra cache miss and an
988 disk_cache_has_key(struct disk_cache
*cache
, const cache_key key
)
990 const uint32_t *key_chunk
= (const uint32_t *) key
;
991 int i
= *key_chunk
& CACHE_INDEX_KEY_MASK
;
992 unsigned char *entry
;
994 entry
= &cache
->stored_keys
[i
+ CACHE_KEY_SIZE
];
996 return memcmp(entry
, key
, CACHE_KEY_SIZE
) == 0;
999 #endif /* ENABLE_SHADER_CACHE */