1 /**********************************************************
2 * Copyright 2008-2009 VMware, Inc. All rights reserved.
4 * Permission is hereby granted, free of charge, to any person
5 * obtaining a copy of this software and associated documentation
6 * files (the "Software"), to deal in the Software without
7 * restriction, including without limitation the rights to use, copy,
8 * modify, merge, publish, distribute, sublicense, and/or sell copies
9 * of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 **********************************************************/
26 #include "util/u_math.h"
27 #include "util/u_memory.h"
28 #include "util/crc32.h"
30 #include "svga_debug.h"
31 #include "svga_format.h"
32 #include "svga_winsys.h"
33 #include "svga_screen.h"
34 #include "svga_screen_cache.h"
35 #include "svga_context.h"
38 #define SVGA_SURFACE_CACHE_ENABLED 1
42 * Return the size of the surface described by the key (in bytes).
45 svga_surface_size(const struct svga_host_surface_cache_key
*key
)
47 unsigned bw
, bh
, bpb
, total_size
, i
;
49 assert(key
->numMipLevels
> 0);
50 assert(key
->numFaces
> 0);
51 assert(key
->arraySize
> 0);
53 if (key
->format
== SVGA3D_BUFFER
) {
54 /* Special case: we don't want to count vertex/index buffers
55 * against the cache size limit, so view them as zero-sized.
60 svga_format_size(key
->format
, &bw
, &bh
, &bpb
);
64 for (i
= 0; i
< key
->numMipLevels
; i
++) {
65 unsigned w
= u_minify(key
->size
.width
, i
);
66 unsigned h
= u_minify(key
->size
.height
, i
);
67 unsigned d
= u_minify(key
->size
.depth
, i
);
68 unsigned img_size
= ((w
+ bw
- 1) / bw
) * ((h
+ bh
- 1) / bh
) * d
* bpb
;
69 total_size
+= img_size
;
72 total_size
*= key
->numFaces
* key
->arraySize
* MAX2(1, key
->sampleCount
);
79 * Compute the bucket for this key.
81 static inline unsigned
82 svga_screen_cache_bucket(const struct svga_host_surface_cache_key
*key
)
84 return util_hash_crc32(key
, sizeof *key
) % SVGA_HOST_SURFACE_CACHE_BUCKETS
;
89 * Search the cache for a surface that matches the key. If a match is
90 * found, remove it from the cache and return the surface pointer.
91 * Return NULL otherwise.
93 static struct svga_winsys_surface
*
94 svga_screen_cache_lookup(struct svga_screen
*svgascreen
,
95 const struct svga_host_surface_cache_key
*key
)
97 struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
98 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
99 struct svga_host_surface_cache_entry
*entry
;
100 struct svga_winsys_surface
*handle
= NULL
;
101 struct list_head
*curr
, *next
;
105 assert(key
->cachable
);
107 bucket
= svga_screen_cache_bucket(key
);
109 mtx_lock(&cache
->mutex
);
111 curr
= cache
->bucket
[bucket
].next
;
113 while (curr
!= &cache
->bucket
[bucket
]) {
116 entry
= LIST_ENTRY(struct svga_host_surface_cache_entry
, curr
, bucket_head
);
118 assert(entry
->handle
);
120 /* If the key matches and the fence is signalled (the surface is no
121 * longer needed) the lookup was successful. We found a surface that
123 * We unlink the surface from the cache entry and we add the entry to
126 if (memcmp(&entry
->key
, key
, sizeof *key
) == 0 &&
127 sws
->fence_signalled(sws
, entry
->fence
, 0) == 0) {
130 assert(sws
->surface_is_flushed(sws
, entry
->handle
));
132 handle
= entry
->handle
; /* Reference is transfered here. */
133 entry
->handle
= NULL
;
135 /* Remove from hash table */
136 list_del(&entry
->bucket_head
);
138 /* remove from LRU list */
139 list_del(&entry
->head
);
141 /* Add the cache entry (but not the surface!) to the empty list */
142 list_add(&entry
->head
, &cache
->empty
);
144 /* update the cache size */
145 surf_size
= svga_surface_size(&entry
->key
);
146 assert(surf_size
<= cache
->total_size
);
147 if (surf_size
> cache
->total_size
)
148 cache
->total_size
= 0; /* should never happen, but be safe */
150 cache
->total_size
-= surf_size
;
159 mtx_unlock(&cache
->mutex
);
161 if (SVGA_DEBUG
& DEBUG_DMA
)
162 debug_printf("%s: cache %s after %u tries (bucket %d)\n", __FUNCTION__
,
163 handle
? "hit" : "miss", tries
, bucket
);
170 * Free the least recently used entries in the surface cache until the
171 * cache size is <= the target size OR there are no unused entries left
172 * to discard. We don't do any flushing to try to free up additional
176 svga_screen_cache_shrink(struct svga_screen
*svgascreen
,
177 unsigned target_size
)
179 struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
180 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
181 struct svga_host_surface_cache_entry
*entry
= NULL
, *next_entry
;
183 /* Walk over the list of unused buffers in reverse order: from oldest
186 LIST_FOR_EACH_ENTRY_SAFE_REV(entry
, next_entry
, &cache
->unused
, head
) {
187 if (entry
->key
.format
!= SVGA3D_BUFFER
) {
188 /* we don't want to discard vertex/index buffers */
190 cache
->total_size
-= svga_surface_size(&entry
->key
);
192 assert(entry
->handle
);
193 sws
->surface_reference(sws
, &entry
->handle
, NULL
);
195 list_del(&entry
->bucket_head
);
196 list_del(&entry
->head
);
197 list_add(&entry
->head
, &cache
->empty
);
199 if (cache
->total_size
<= target_size
) {
209 * Add a surface to the cache. This is done when the driver deletes
210 * the surface. Note: transfers a handle reference.
213 svga_screen_cache_add(struct svga_screen
*svgascreen
,
214 const struct svga_host_surface_cache_key
*key
,
215 struct svga_winsys_surface
**p_handle
)
217 struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
218 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
219 struct svga_host_surface_cache_entry
*entry
= NULL
;
220 struct svga_winsys_surface
*handle
= *p_handle
;
223 assert(key
->cachable
);
228 surf_size
= svga_surface_size(key
);
231 mtx_lock(&cache
->mutex
);
233 if (surf_size
>= SVGA_HOST_SURFACE_CACHE_BYTES
) {
234 /* this surface is too large to cache, just free it */
235 sws
->surface_reference(sws
, &handle
, NULL
);
236 mtx_unlock(&cache
->mutex
);
240 if (cache
->total_size
+ surf_size
> SVGA_HOST_SURFACE_CACHE_BYTES
) {
241 /* Adding this surface would exceed the cache size.
242 * Try to discard least recently used entries until we hit the
243 * new target cache size.
245 unsigned target_size
= SVGA_HOST_SURFACE_CACHE_BYTES
- surf_size
;
247 svga_screen_cache_shrink(svgascreen
, target_size
);
249 if (cache
->total_size
> target_size
) {
250 /* we weren't able to shrink the cache as much as we wanted so
251 * just discard this surface.
253 sws
->surface_reference(sws
, &handle
, NULL
);
254 mtx_unlock(&cache
->mutex
);
259 if (!list_is_empty(&cache
->empty
)) {
260 /* An empty entry has no surface associated with it.
261 * Use the first empty entry.
263 entry
= LIST_ENTRY(struct svga_host_surface_cache_entry
,
264 cache
->empty
.next
, head
);
266 /* Remove from LRU list */
267 list_del(&entry
->head
);
269 else if (!list_is_empty(&cache
->unused
)) {
270 /* free the last used buffer and reuse its entry */
271 entry
= LIST_ENTRY(struct svga_host_surface_cache_entry
,
272 cache
->unused
.prev
, head
);
273 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
274 "unref sid %p (make space)\n", entry
->handle
);
276 cache
->total_size
-= svga_surface_size(&entry
->key
);
278 sws
->surface_reference(sws
, &entry
->handle
, NULL
);
280 /* Remove from hash table */
281 list_del(&entry
->bucket_head
);
283 /* Remove from LRU list */
284 list_del(&entry
->head
);
288 assert(entry
->handle
== NULL
);
289 entry
->handle
= handle
;
290 memcpy(&entry
->key
, key
, sizeof entry
->key
);
292 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
293 "cache sid %p\n", entry
->handle
);
295 /* If we don't have gb objects, we don't need to invalidate. */
296 if (sws
->have_gb_objects
)
297 list_add(&entry
->head
, &cache
->validated
);
299 list_add(&entry
->head
, &cache
->invalidated
);
301 cache
->total_size
+= surf_size
;
304 /* Couldn't cache the buffer -- this really shouldn't happen */
305 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
306 "unref sid %p (couldn't find space)\n", handle
);
307 sws
->surface_reference(sws
, &handle
, NULL
);
310 mtx_unlock(&cache
->mutex
);
314 /* Maximum number of invalidate surface commands in a command buffer */
315 # define SVGA_MAX_SURFACE_TO_INVALIDATE 1000
318 * Called during the screen flush to move all buffers not in a validate list
319 * into the unused list.
322 svga_screen_cache_flush(struct svga_screen
*svgascreen
,
323 struct svga_context
*svga
,
324 struct pipe_fence_handle
*fence
)
326 struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
327 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
328 struct svga_host_surface_cache_entry
*entry
;
329 struct list_head
*curr
, *next
;
332 mtx_lock(&cache
->mutex
);
334 /* Loop over entries in the invalidated list */
335 curr
= cache
->invalidated
.next
;
337 while (curr
!= &cache
->invalidated
) {
338 entry
= LIST_ENTRY(struct svga_host_surface_cache_entry
, curr
, head
);
340 assert(entry
->handle
);
342 if (sws
->surface_is_flushed(sws
, entry
->handle
)) {
343 /* remove entry from the invalidated list */
344 list_del(&entry
->head
);
346 sws
->fence_reference(sws
, &entry
->fence
, fence
);
348 /* Add entry to the unused list */
349 list_add(&entry
->head
, &cache
->unused
);
351 /* Add entry to the hash table bucket */
352 bucket
= svga_screen_cache_bucket(&entry
->key
);
353 list_add(&entry
->bucket_head
, &cache
->bucket
[bucket
]);
361 curr
= cache
->validated
.next
;
363 while (curr
!= &cache
->validated
) {
364 entry
= LIST_ENTRY(struct svga_host_surface_cache_entry
, curr
, head
);
366 assert(entry
->handle
);
367 assert(svga_have_gb_objects(svga
));
369 if (sws
->surface_is_flushed(sws
, entry
->handle
)) {
370 /* remove entry from the validated list */
371 list_del(&entry
->head
);
373 /* It is now safe to invalidate the surface content.
374 * It will be done using the current context.
376 if (SVGA_TRY(SVGA3D_InvalidateGBSurface(svga
->swc
, entry
->handle
))
378 ASSERTED
enum pipe_error ret
;
380 /* Even though surface invalidation here is done after the command
381 * buffer is flushed, it is still possible that it will
382 * fail because there might be just enough of this command that is
383 * filling up the command buffer, so in this case we will call
384 * the winsys flush directly to flush the buffer.
385 * Note, we don't want to call svga_context_flush() here because
386 * this function itself is called inside svga_context_flush().
388 svga_retry_enter(svga
);
389 svga
->swc
->flush(svga
->swc
, NULL
);
391 ret
= SVGA3D_InvalidateGBSurface(svga
->swc
, entry
->handle
);
392 svga_retry_exit(svga
);
393 assert(ret
== PIPE_OK
);
396 /* add the entry to the invalidated list */
398 list_add(&entry
->head
, &cache
->invalidated
);
406 mtx_unlock(&cache
->mutex
);
409 * In some rare cases (when running ARK survival), we hit the max number
410 * of surface relocations with invalidated surfaces during context flush.
411 * So if the number of invalidated surface exceeds a certain limit (1000),
412 * we'll do another winsys flush.
414 if (nsurf
> SVGA_MAX_SURFACE_TO_INVALIDATE
) {
415 svga
->swc
->flush(svga
->swc
, NULL
);
421 * Free all the surfaces in the cache.
422 * Called when destroying the svga screen object.
425 svga_screen_cache_cleanup(struct svga_screen
*svgascreen
)
427 struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
428 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
431 for (i
= 0; i
< SVGA_HOST_SURFACE_CACHE_SIZE
; ++i
) {
432 if (cache
->entries
[i
].handle
) {
433 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
434 "unref sid %p (shutdown)\n", cache
->entries
[i
].handle
);
435 sws
->surface_reference(sws
, &cache
->entries
[i
].handle
, NULL
);
437 cache
->total_size
-= svga_surface_size(&cache
->entries
[i
].key
);
440 if (cache
->entries
[i
].fence
)
441 sws
->fence_reference(sws
, &cache
->entries
[i
].fence
, NULL
);
444 mtx_destroy(&cache
->mutex
);
449 svga_screen_cache_init(struct svga_screen
*svgascreen
)
451 struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
454 assert(cache
->total_size
== 0);
456 (void) mtx_init(&cache
->mutex
, mtx_plain
);
458 for (i
= 0; i
< SVGA_HOST_SURFACE_CACHE_BUCKETS
; ++i
)
459 list_inithead(&cache
->bucket
[i
]);
461 list_inithead(&cache
->unused
);
463 list_inithead(&cache
->validated
);
465 list_inithead(&cache
->invalidated
);
467 list_inithead(&cache
->empty
);
468 for (i
= 0; i
< SVGA_HOST_SURFACE_CACHE_SIZE
; ++i
)
469 list_addtail(&cache
->entries
[i
].head
, &cache
->empty
);
476 * Allocate a new host-side surface. If the surface is marked as cachable,
477 * first try re-using a surface in the cache of freed surfaces. Otherwise,
478 * allocate a new surface.
479 * \param bind_flags bitmask of PIPE_BIND_x flags
480 * \param usage one of PIPE_USAGE_x values
481 * \param validated return True if the surface is a reused surface
483 struct svga_winsys_surface
*
484 svga_screen_surface_create(struct svga_screen
*svgascreen
,
485 unsigned bind_flags
, enum pipe_resource_usage usage
,
487 struct svga_host_surface_cache_key
*key
)
489 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
490 struct svga_winsys_surface
*handle
= NULL
;
491 boolean cachable
= SVGA_SURFACE_CACHE_ENABLED
&& key
->cachable
;
493 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
494 "%s sz %dx%dx%d mips %d faces %d arraySize %d cachable %d\n",
505 /* Try to re-cycle a previously freed, cached surface */
506 if (key
->format
== SVGA3D_BUFFER
) {
507 SVGA3dSurfaceAllFlags hint_flag
;
509 /* For buffers, round the buffer size up to the nearest power
510 * of two to increase the probability of cache hits. Keep
511 * texture surface dimensions unchanged.
514 while (size
< key
->size
.width
)
516 key
->size
.width
= size
;
518 /* Determine whether the buffer is static or dynamic.
519 * This is a bit of a heuristic which can be tuned as needed.
521 if (usage
== PIPE_USAGE_DEFAULT
||
522 usage
== PIPE_USAGE_IMMUTABLE
) {
523 hint_flag
= SVGA3D_SURFACE_HINT_STATIC
;
525 else if (bind_flags
& PIPE_BIND_INDEX_BUFFER
) {
526 /* Index buffers don't change too often. Mark them as static.
528 hint_flag
= SVGA3D_SURFACE_HINT_STATIC
;
531 /* Since we're reusing buffers we're effectively transforming all
532 * of them into dynamic buffers.
534 * It would be nice to not cache long lived static buffers. But there
535 * is no way to detect the long lived from short lived ones yet. A
536 * good heuristic would be buffer size.
538 hint_flag
= SVGA3D_SURFACE_HINT_DYNAMIC
;
541 key
->flags
&= ~(SVGA3D_SURFACE_HINT_STATIC
|
542 SVGA3D_SURFACE_HINT_DYNAMIC
);
543 key
->flags
|= hint_flag
;
546 handle
= svga_screen_cache_lookup(svgascreen
, key
);
548 if (key
->format
== SVGA3D_BUFFER
)
549 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
550 "reuse sid %p sz %d (buffer)\n", handle
,
553 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
554 "reuse sid %p sz %dx%dx%d mips %d faces %d arraySize %d\n", handle
,
566 /* Unable to recycle surface, allocate a new one */
570 usage
|= SVGA_SURFACE_USAGE_SHARED
;
572 usage
|= SVGA_SURFACE_USAGE_SCANOUT
;
574 usage
|= SVGA_SURFACE_USAGE_COHERENT
;
576 handle
= sws
->surface_create(sws
,
581 key
->numFaces
* key
->arraySize
,
585 SVGA_DBG(DEBUG_CACHE
|DEBUG_DMA
,
586 " CREATE sid %p sz %dx%dx%d\n",
600 * Release a surface. We don't actually free the surface- we put
601 * it into the cache of freed surfaces (if it's cachable).
604 svga_screen_surface_destroy(struct svga_screen
*svgascreen
,
605 const struct svga_host_surface_cache_key
*key
,
606 struct svga_winsys_surface
**p_handle
)
608 struct svga_winsys_screen
*sws
= svgascreen
->sws
;
610 /* We only set the cachable flag for surfaces of which we are the
611 * exclusive owner. So just hold onto our existing reference in
614 if (SVGA_SURFACE_CACHE_ENABLED
&& key
->cachable
) {
615 svga_screen_cache_add(svgascreen
, key
, p_handle
);
619 "unref sid %p (uncachable)\n", *p_handle
);
620 sws
->surface_reference(sws
, p_handle
, NULL
);
626 * Print/dump the contents of the screen cache. For debugging.
629 svga_screen_cache_dump(const struct svga_screen
*svgascreen
)
631 const struct svga_host_surface_cache
*cache
= &svgascreen
->cache
;
635 debug_printf("svga3d surface cache:\n");
636 for (bucket
= 0; bucket
< SVGA_HOST_SURFACE_CACHE_BUCKETS
; bucket
++) {
637 struct list_head
*curr
;
638 curr
= cache
->bucket
[bucket
].next
;
639 while (curr
&& curr
!= &cache
->bucket
[bucket
]) {
640 struct svga_host_surface_cache_entry
*entry
=
641 LIST_ENTRY(struct svga_host_surface_cache_entry
,
643 if (entry
->key
.format
== SVGA3D_BUFFER
) {
644 debug_printf(" %p: buffer %u bytes\n",
646 entry
->key
.size
.width
);
649 debug_printf(" %p: %u x %u x %u format %u\n",
651 entry
->key
.size
.width
,
652 entry
->key
.size
.height
,
653 entry
->key
.size
.depth
,
661 debug_printf("%u surfaces, %u bytes\n", count
, cache
->total_size
);