1 /**************************************************************************
3 * Copyright 2007-2008 VMware, Inc.
4 * Copyright 2015 Advanced Micro Devices, Inc.
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 **************************************************************************/
30 #include "util/u_memory.h"
31 #include "util/os_time.h"
35 * Actually destroy the buffer.
38 destroy_buffer_locked(struct pb_cache_entry
*entry
)
40 struct pb_cache
*mgr
= entry
->mgr
;
41 struct pb_buffer
*buf
= entry
->buffer
;
43 assert(!pipe_is_referenced(&buf
->reference
));
44 if (entry
->head
.next
) {
45 LIST_DEL(&entry
->head
);
46 assert(mgr
->num_buffers
);
48 mgr
->cache_size
-= buf
->size
;
50 mgr
->destroy_buffer(buf
);
54 * Free as many cache buffers from the list head as possible.
57 release_expired_buffers_locked(struct list_head
*cache
,
60 struct list_head
*curr
, *next
;
61 struct pb_cache_entry
*entry
;
65 while (curr
!= cache
) {
66 entry
= LIST_ENTRY(struct pb_cache_entry
, curr
, head
);
68 if (!os_time_timeout(entry
->start
, entry
->end
, current_time
))
71 destroy_buffer_locked(entry
);
79 * Add a buffer to the cache. This is typically done when the buffer is
83 pb_cache_add_buffer(struct pb_cache_entry
*entry
)
85 struct pb_cache
*mgr
= entry
->mgr
;
86 struct list_head
*cache
= &mgr
->buckets
[entry
->bucket_index
];
87 struct pb_buffer
*buf
= entry
->buffer
;
90 mtx_lock(&mgr
->mutex
);
91 assert(!pipe_is_referenced(&buf
->reference
));
93 int64_t current_time
= os_time_get();
95 for (i
= 0; i
< mgr
->num_heaps
; i
++)
96 release_expired_buffers_locked(&mgr
->buckets
[i
], current_time
);
98 /* Directly release any buffer that exceeds the limit. */
99 if (mgr
->cache_size
+ buf
->size
> mgr
->max_cache_size
) {
100 mgr
->destroy_buffer(buf
);
101 mtx_unlock(&mgr
->mutex
);
105 entry
->start
= os_time_get();
106 entry
->end
= entry
->start
+ mgr
->usecs
;
107 LIST_ADDTAIL(&entry
->head
, cache
);
109 mgr
->cache_size
+= buf
->size
;
110 mtx_unlock(&mgr
->mutex
);
114 * \return 1 if compatible and can be reclaimed
116 * -1 if compatible and can't be reclaimed
119 pb_cache_is_buffer_compat(struct pb_cache_entry
*entry
,
120 pb_size size
, unsigned alignment
, unsigned usage
)
122 struct pb_cache
*mgr
= entry
->mgr
;
123 struct pb_buffer
*buf
= entry
->buffer
;
125 if (!pb_check_usage(usage
, buf
->usage
))
128 /* be lenient with size */
129 if (buf
->size
< size
||
130 buf
->size
> (unsigned) (mgr
->size_factor
* size
))
133 if (usage
& mgr
->bypass_usage
)
136 if (!pb_check_alignment(alignment
, buf
->alignment
))
139 return mgr
->can_reclaim(buf
) ? 1 : -1;
143 * Find a compatible buffer in the cache, return it, and remove it
147 pb_cache_reclaim_buffer(struct pb_cache
*mgr
, pb_size size
,
148 unsigned alignment
, unsigned usage
,
149 unsigned bucket_index
)
151 struct pb_cache_entry
*entry
;
152 struct pb_cache_entry
*cur_entry
;
153 struct list_head
*cur
, *next
;
157 assert(bucket_index
< mgr
->num_heaps
);
158 struct list_head
*cache
= &mgr
->buckets
[bucket_index
];
160 mtx_lock(&mgr
->mutex
);
166 /* search in the expired buffers, freeing them in the process */
168 while (cur
!= cache
) {
169 cur_entry
= LIST_ENTRY(struct pb_cache_entry
, cur
, head
);
171 if (!entry
&& (ret
= pb_cache_is_buffer_compat(cur_entry
, size
,
172 alignment
, usage
)) > 0)
174 else if (os_time_timeout(cur_entry
->start
, cur_entry
->end
, now
))
175 destroy_buffer_locked(cur_entry
);
177 /* This buffer (and all hereafter) are still hot in cache */
180 /* the buffer is busy (and probably all remaining ones too) */
188 /* keep searching in the hot buffers */
189 if (!entry
&& ret
!= -1) {
190 while (cur
!= cache
) {
191 cur_entry
= LIST_ENTRY(struct pb_cache_entry
, cur
, head
);
192 ret
= pb_cache_is_buffer_compat(cur_entry
, size
, alignment
, usage
);
200 /* no need to check the timeout here */
206 /* found a compatible buffer, return it */
208 struct pb_buffer
*buf
= entry
->buffer
;
210 mgr
->cache_size
-= buf
->size
;
211 LIST_DEL(&entry
->head
);
213 mtx_unlock(&mgr
->mutex
);
214 /* Increase refcount */
215 pipe_reference_init(&buf
->reference
, 1);
219 mtx_unlock(&mgr
->mutex
);
224 * Empty the cache. Useful when there is not enough memory.
227 pb_cache_release_all_buffers(struct pb_cache
*mgr
)
229 struct list_head
*curr
, *next
;
230 struct pb_cache_entry
*buf
;
233 mtx_lock(&mgr
->mutex
);
234 for (i
= 0; i
< mgr
->num_heaps
; i
++) {
235 struct list_head
*cache
= &mgr
->buckets
[i
];
239 while (curr
!= cache
) {
240 buf
= LIST_ENTRY(struct pb_cache_entry
, curr
, head
);
241 destroy_buffer_locked(buf
);
246 mtx_unlock(&mgr
->mutex
);
250 pb_cache_init_entry(struct pb_cache
*mgr
, struct pb_cache_entry
*entry
,
251 struct pb_buffer
*buf
, unsigned bucket_index
)
253 assert(bucket_index
< mgr
->num_heaps
);
255 memset(entry
, 0, sizeof(*entry
));
258 entry
->bucket_index
= bucket_index
;
262 * Initialize a caching buffer manager.
264 * @param mgr The cache buffer manager
265 * @param num_heaps Number of separate caches/buckets indexed by bucket_index
266 * for faster buffer matching (alternative to slower
267 * "usage"-based matching).
268 * @param usecs Unused buffers may be released from the cache after this
270 * @param size_factor Declare buffers that are size_factor times bigger than
271 * the requested size as cache hits.
272 * @param bypass_usage Bitmask. If (requested usage & bypass_usage) != 0,
273 * buffer allocation requests are rejected.
274 * @param maximum_cache_size Maximum size of all unused buffers the cache can
276 * @param destroy_buffer Function that destroys a buffer for good.
277 * @param can_reclaim Whether a buffer can be reclaimed (e.g. is not busy)
280 pb_cache_init(struct pb_cache
*mgr
, uint num_heaps
,
281 uint usecs
, float size_factor
,
282 unsigned bypass_usage
, uint64_t maximum_cache_size
,
283 void (*destroy_buffer
)(struct pb_buffer
*buf
),
284 bool (*can_reclaim
)(struct pb_buffer
*buf
))
288 mgr
->buckets
= CALLOC(num_heaps
, sizeof(struct list_head
));
292 for (i
= 0; i
< num_heaps
; i
++)
293 list_inithead(&mgr
->buckets
[i
]);
295 (void) mtx_init(&mgr
->mutex
, mtx_plain
);
297 mgr
->max_cache_size
= maximum_cache_size
;
298 mgr
->num_heaps
= num_heaps
;
300 mgr
->num_buffers
= 0;
301 mgr
->bypass_usage
= bypass_usage
;
302 mgr
->size_factor
= size_factor
;
303 mgr
->destroy_buffer
= destroy_buffer
;
304 mgr
->can_reclaim
= can_reclaim
;
308 * Deinitialize the manager completely.
311 pb_cache_deinit(struct pb_cache
*mgr
)
313 pb_cache_release_all_buffers(mgr
);
314 mtx_destroy(&mgr
->mutex
);