src/gallium/drivers/svga/svga_screen_cache.c

   1 /**********************************************************
   2  * Copyright 2008-2009 VMware, Inc.  All rights reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person
   5  * obtaining a copy of this software and associated documentation
   6  * files (the "Software"), to deal in the Software without
   7  * restriction, including without limitation the rights to use, copy,
   8  * modify, merge, publish, distribute, sublicense, and/or sell copies
   9  * of the Software, and to permit persons to whom the Software is
  10  * furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be
  13  * included in all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  **********************************************************/
  25
  26 #include "util/u_math.h"
  27 #include "util/u_memory.h"
  28 #include "util/crc32.h"
  29
  30 #include "svga_debug.h"
  31 #include "svga_format.h"
  32 #include "svga_winsys.h"
  33 #include "svga_screen.h"
  34 #include "svga_screen_cache.h"
  35 #include "svga_context.h"
  36
  37
  38 #define SVGA_SURFACE_CACHE_ENABLED 1
  39
  40
  41 /**
  42  * Return the size of the surface described by the key (in bytes).
  43  */
  44 static unsigned
  45 surface_size(const struct svga_host_surface_cache_key *key)
  46 {
  47    unsigned bw, bh, bpb, total_size, i;
  48
  49    assert(key->numMipLevels > 0);
  50    assert(key->numFaces > 0);
  51    assert(key->arraySize > 0);
  52
  53    if (key->format == SVGA3D_BUFFER) {
  54       /* Special case: we don't want to count vertex/index buffers
  55        * against the cache size limit, so view them as zero-sized.
  56        */
  57       return 0;
  58    }
  59
  60    svga_format_size(key->format, &bw, &bh, &bpb);
  61
  62    total_size = 0;
  63
  64    for (i = 0; i < key->numMipLevels; i++) {
  65       unsigned w = u_minify(key->size.width, i);
  66       unsigned h = u_minify(key->size.height, i);
  67       unsigned d = u_minify(key->size.depth, i);
  68       unsigned img_size = ((w + bw - 1) / bw) * ((h + bh - 1) / bh) * d * bpb;
  69       total_size += img_size;
  70    }
  71
  72    total_size *= key->numFaces * key->arraySize * MAX2(1, key->sampleCount);
  73
  74    return total_size;
  75 }
  76
  77
  78 /**
  79  * Compute the bucket for this key.
  80  */
  81 static inline unsigned
  82 svga_screen_cache_bucket(const struct svga_host_surface_cache_key *key)
  83 {
  84    return util_hash_crc32(key, sizeof *key) % SVGA_HOST_SURFACE_CACHE_BUCKETS;
  85 }
  86
  87
  88 /**
  89  * Search the cache for a surface that matches the key.  If a match is
  90  * found, remove it from the cache and return the surface pointer.
  91  * Return NULL otherwise.
  92  */
  93 static struct svga_winsys_surface *
  94 svga_screen_cache_lookup(struct svga_screen *svgascreen,
  95                          const struct svga_host_surface_cache_key *key)
  96 {
  97    struct svga_host_surface_cache *cache = &svgascreen->cache;
  98    struct svga_winsys_screen *sws = svgascreen->sws;
  99    struct svga_host_surface_cache_entry *entry;
 100    struct svga_winsys_surface *handle = NULL;
 101    struct list_head *curr, *next;
 102    unsigned bucket;
 103    unsigned tries = 0;
 104
 105    assert(key->cachable);
 106
 107    bucket = svga_screen_cache_bucket(key);
 108
 109    mtx_lock(&cache->mutex);
 110
 111    curr = cache->bucket[bucket].next;
 112    next = curr->next;
 113    while (curr != &cache->bucket[bucket]) {
 114       ++tries;
 115
 116       entry = LIST_ENTRY(struct svga_host_surface_cache_entry, curr, bucket_head);
 117
 118       assert(entry->handle);
 119
 120       /* If the key matches and the fence is signalled (the surface is no
 121        * longer needed) the lookup was successful.  We found a surface that
 122        * can be reused.
 123        * We unlink the surface from the cache entry and we add the entry to
 124        * the 'empty' list.
 125        */
 126       if (memcmp(&entry->key, key, sizeof *key) == 0 &&
 127           sws->fence_signalled(sws, entry->fence, 0) == 0) {
 128          unsigned surf_size;
 129
 130          assert(sws->surface_is_flushed(sws, entry->handle));
 131
 132          handle = entry->handle; /* Reference is transfered here. */
 133          entry->handle = NULL;
 134
 135          /* Remove from hash table */
 136          LIST_DEL(&entry->bucket_head);
 137
 138          /* remove from LRU list */
 139          LIST_DEL(&entry->head);
 140
 141          /* Add the cache entry (but not the surface!) to the empty list */
 142          LIST_ADD(&entry->head, &cache->empty);
 143
 144          /* update the cache size */
 145          surf_size = surface_size(&entry->key);
 146          assert(surf_size <= cache->total_size);
 147          if (surf_size > cache->total_size)
 148             cache->total_size = 0; /* should never happen, but be safe */
 149          else
 150             cache->total_size -= surf_size;
 151
 152          break;
 153       }
 154
 155       curr = next;
 156       next = curr->next;
 157    }
 158
 159    mtx_unlock(&cache->mutex);
 160
 161    if (SVGA_DEBUG & DEBUG_DMA)
 162       debug_printf("%s: cache %s after %u tries (bucket %d)\n", __FUNCTION__,
 163                    handle ? "hit" : "miss", tries, bucket);
 164
 165    return handle;
 166 }
 167
 168
 169 /**
 170  * Free the least recently used entries in the surface cache until the
 171  * cache size is <= the target size OR there are no unused entries left
 172  * to discard.  We don't do any flushing to try to free up additional
 173  * surfaces.
 174  */
 175 static void
 176 svga_screen_cache_shrink(struct svga_screen *svgascreen,
 177                          unsigned target_size)
 178 {
 179    struct svga_host_surface_cache *cache = &svgascreen->cache;
 180    struct svga_winsys_screen *sws = svgascreen->sws;
 181    struct svga_host_surface_cache_entry *entry = NULL, *next_entry;
 182
 183    /* Walk over the list of unused buffers in reverse order: from oldest
 184     * to newest.
 185     */
 186    LIST_FOR_EACH_ENTRY_SAFE_REV(entry, next_entry, &cache->unused, head) {
 187       if (entry->key.format != SVGA3D_BUFFER) {
 188          /* we don't want to discard vertex/index buffers */
 189
 190          cache->total_size -= surface_size(&entry->key);
 191
 192          assert(entry->handle);
 193          sws->surface_reference(sws, &entry->handle, NULL);
 194
 195          LIST_DEL(&entry->bucket_head);
 196          LIST_DEL(&entry->head);
 197          LIST_ADD(&entry->head, &cache->empty);
 198
 199          if (cache->total_size <= target_size) {
 200             /* all done */
 201             break;
 202          }
 203       }
 204    }
 205 }
 206
 207
 208 /**
 209  * Add a surface to the cache.  This is done when the driver deletes
 210  * the surface.  Note: transfers a handle reference.
 211  */
 212 static void
 213 svga_screen_cache_add(struct svga_screen *svgascreen,
 214                       const struct svga_host_surface_cache_key *key,
 215                       struct svga_winsys_surface **p_handle)
 216 {
 217    struct svga_host_surface_cache *cache = &svgascreen->cache;
 218    struct svga_winsys_screen *sws = svgascreen->sws;
 219    struct svga_host_surface_cache_entry *entry = NULL;
 220    struct svga_winsys_surface *handle = *p_handle;
 221    unsigned surf_size;
 222
 223    assert(key->cachable);
 224
 225    if (!handle)
 226       return;
 227
 228    surf_size = surface_size(key);
 229
 230    *p_handle = NULL;
 231    mtx_lock(&cache->mutex);
 232
 233    if (surf_size >= SVGA_HOST_SURFACE_CACHE_BYTES) {
 234       /* this surface is too large to cache, just free it */
 235       sws->surface_reference(sws, &handle, NULL);
 236       mtx_unlock(&cache->mutex);
 237       return;
 238    }
 239
 240    if (cache->total_size + surf_size > SVGA_HOST_SURFACE_CACHE_BYTES) {
 241       /* Adding this surface would exceed the cache size.
 242        * Try to discard least recently used entries until we hit the
 243        * new target cache size.
 244        */
 245       unsigned target_size = SVGA_HOST_SURFACE_CACHE_BYTES - surf_size;
 246
 247       svga_screen_cache_shrink(svgascreen, target_size);
 248
 249       if (cache->total_size > target_size) {
 250          /* we weren't able to shrink the cache as much as we wanted so
 251           * just discard this surface.
 252           */
 253          sws->surface_reference(sws, &handle, NULL);
 254          mtx_unlock(&cache->mutex);
 255          return;
 256       }
 257    }
 258
 259    if (!LIST_IS_EMPTY(&cache->empty)) {
 260       /* An empty entry has no surface associated with it.
 261        * Use the first empty entry.
 262        */
 263       entry = LIST_ENTRY(struct svga_host_surface_cache_entry,
 264                          cache->empty.next, head);
 265
 266       /* Remove from LRU list */
 267       LIST_DEL(&entry->head);
 268    }
 269    else if (!LIST_IS_EMPTY(&cache->unused)) {
 270       /* free the last used buffer and reuse its entry */
 271       entry = LIST_ENTRY(struct svga_host_surface_cache_entry,
 272                          cache->unused.prev, head);
 273       SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 274                "unref sid %p (make space)\n", entry->handle);
 275
 276       cache->total_size -= surface_size(&entry->key);
 277
 278       sws->surface_reference(sws, &entry->handle, NULL);
 279
 280       /* Remove from hash table */
 281       LIST_DEL(&entry->bucket_head);
 282
 283       /* Remove from LRU list */
 284       LIST_DEL(&entry->head);
 285    }
 286
 287    if (entry) {
 288       assert(entry->handle == NULL);
 289       entry->handle = handle;
 290       memcpy(&entry->key, key, sizeof entry->key);
 291
 292       SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 293                "cache sid %p\n", entry->handle);
 294       LIST_ADD(&entry->head, &cache->validated);
 295
 296       cache->total_size += surf_size;
 297    }
 298    else {
 299       /* Couldn't cache the buffer -- this really shouldn't happen */
 300       SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 301                "unref sid %p (couldn't find space)\n", handle);
 302       sws->surface_reference(sws, &handle, NULL);
 303    }
 304
 305    mtx_unlock(&cache->mutex);
 306 }
 307
 308
 309 /**
 310  * Called during the screen flush to move all buffers not in a validate list
 311  * into the unused list.
 312  */
 313 void
 314 svga_screen_cache_flush(struct svga_screen *svgascreen,
 315                         struct svga_context *svga,
 316                         struct pipe_fence_handle *fence)
 317 {
 318    struct svga_host_surface_cache *cache = &svgascreen->cache;
 319    struct svga_winsys_screen *sws = svgascreen->sws;
 320    struct svga_host_surface_cache_entry *entry;
 321    struct list_head *curr, *next;
 322    unsigned bucket;
 323
 324    mtx_lock(&cache->mutex);
 325
 326    /* Loop over entries in the invalidated list */
 327    curr = cache->invalidated.next;
 328    next = curr->next;
 329    while (curr != &cache->invalidated) {
 330       entry = LIST_ENTRY(struct svga_host_surface_cache_entry, curr, head);
 331
 332       assert(entry->handle);
 333
 334       if (sws->surface_is_flushed(sws, entry->handle)) {
 335          /* remove entry from the invalidated list */
 336          LIST_DEL(&entry->head);
 337
 338          sws->fence_reference(sws, &entry->fence, fence);
 339
 340          /* Add entry to the unused list */
 341          LIST_ADD(&entry->head, &cache->unused);
 342
 343          /* Add entry to the hash table bucket */
 344          bucket = svga_screen_cache_bucket(&entry->key);
 345          LIST_ADD(&entry->bucket_head, &cache->bucket[bucket]);
 346       }
 347
 348       curr = next;
 349       next = curr->next;
 350    }
 351
 352    curr = cache->validated.next;
 353    next = curr->next;
 354    while (curr != &cache->validated) {
 355       entry = LIST_ENTRY(struct svga_host_surface_cache_entry, curr, head);
 356
 357       assert(entry->handle);
 358
 359       if (sws->surface_is_flushed(sws, entry->handle)) {
 360          /* remove entry from the validated list */
 361          LIST_DEL(&entry->head);
 362
 363          /* It is now safe to invalidate the surface content.
 364           * It will be done using the current context.
 365           */
 366          if (svga->swc->surface_invalidate(svga->swc, entry->handle) != PIPE_OK) {
 367             MAYBE_UNUSED enum pipe_error ret;
 368
 369             /* Even though surface invalidation here is done after the command
 370              * buffer is flushed, it is still possible that it will
 371              * fail because there might be just enough of this command that is
 372              * filling up the command buffer, so in this case we will call
 373              * the winsys flush directly to flush the buffer.
 374              * Note, we don't want to call svga_context_flush() here because
 375              * this function itself is called inside svga_context_flush().
 376              */
 377             svga->swc->flush(svga->swc, NULL);
 378             ret = svga->swc->surface_invalidate(svga->swc, entry->handle);
 379             assert(ret == PIPE_OK);
 380          }
 381
 382          /* add the entry to the invalidated list */
 383          LIST_ADD(&entry->head, &cache->invalidated);
 384       }
 385
 386       curr = next;
 387       next = curr->next;
 388    }
 389
 390    mtx_unlock(&cache->mutex);
 391 }
 392
 393
 394 /**
 395  * Free all the surfaces in the cache.
 396  * Called when destroying the svga screen object.
 397  */
 398 void
 399 svga_screen_cache_cleanup(struct svga_screen *svgascreen)
 400 {
 401    struct svga_host_surface_cache *cache = &svgascreen->cache;
 402    struct svga_winsys_screen *sws = svgascreen->sws;
 403    unsigned i;
 404
 405    for (i = 0; i < SVGA_HOST_SURFACE_CACHE_SIZE; ++i) {
 406       if (cache->entries[i].handle) {
 407          SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 408                   "unref sid %p (shutdown)\n", cache->entries[i].handle);
 409          sws->surface_reference(sws, &cache->entries[i].handle, NULL);
 410
 411          cache->total_size -= surface_size(&cache->entries[i].key);
 412       }
 413
 414       if (cache->entries[i].fence)
 415          sws->fence_reference(sws, &cache->entries[i].fence, NULL);
 416    }
 417
 418    mtx_destroy(&cache->mutex);
 419 }
 420
 421
 422 enum pipe_error
 423 svga_screen_cache_init(struct svga_screen *svgascreen)
 424 {
 425    struct svga_host_surface_cache *cache = &svgascreen->cache;
 426    unsigned i;
 427
 428    assert(cache->total_size == 0);
 429
 430    (void) mtx_init(&cache->mutex, mtx_plain);
 431
 432    for (i = 0; i < SVGA_HOST_SURFACE_CACHE_BUCKETS; ++i)
 433       LIST_INITHEAD(&cache->bucket[i]);
 434
 435    LIST_INITHEAD(&cache->unused);
 436
 437    LIST_INITHEAD(&cache->validated);
 438
 439    LIST_INITHEAD(&cache->invalidated);
 440
 441    LIST_INITHEAD(&cache->empty);
 442    for (i = 0; i < SVGA_HOST_SURFACE_CACHE_SIZE; ++i)
 443       LIST_ADDTAIL(&cache->entries[i].head, &cache->empty);
 444
 445    return PIPE_OK;
 446 }
 447
 448
 449 /**
 450  * Allocate a new host-side surface.  If the surface is marked as cachable,
 451  * first try re-using a surface in the cache of freed surfaces.  Otherwise,
 452  * allocate a new surface.
 453  * \param bind_flags  bitmask of PIPE_BIND_x flags
 454  * \param usage  one of PIPE_USAGE_x values
 455  * \param validated return True if the surface is a reused surface
 456  */
 457 struct svga_winsys_surface *
 458 svga_screen_surface_create(struct svga_screen *svgascreen,
 459                            unsigned bind_flags, enum pipe_resource_usage usage,
 460                            boolean *validated,
 461                            struct svga_host_surface_cache_key *key)
 462 {
 463    struct svga_winsys_screen *sws = svgascreen->sws;
 464    struct svga_winsys_surface *handle = NULL;
 465    boolean cachable = SVGA_SURFACE_CACHE_ENABLED && key->cachable;
 466
 467    SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 468             "%s sz %dx%dx%d mips %d faces %d arraySize %d cachable %d\n",
 469             __FUNCTION__,
 470             key->size.width,
 471             key->size.height,
 472             key->size.depth,
 473             key->numMipLevels,
 474             key->numFaces,
 475             key->arraySize,
 476             key->cachable);
 477
 478    if (cachable) {
 479       /* Try to re-cycle a previously freed, cached surface */
 480       if (key->format == SVGA3D_BUFFER) {
 481          SVGA3dSurfaceFlags hint_flag;
 482
 483          /* For buffers, round the buffer size up to the nearest power
 484           * of two to increase the probability of cache hits.  Keep
 485           * texture surface dimensions unchanged.
 486           */
 487          uint32_t size = 1;
 488          while (size < key->size.width)
 489             size <<= 1;
 490          key->size.width = size;
 491
 492          /* Determine whether the buffer is static or dynamic.
 493           * This is a bit of a heuristic which can be tuned as needed.
 494           */
 495          if (usage == PIPE_USAGE_DEFAULT ||
 496              usage == PIPE_USAGE_IMMUTABLE) {
 497             hint_flag = SVGA3D_SURFACE_HINT_STATIC;
 498          }
 499          else if (bind_flags & PIPE_BIND_INDEX_BUFFER) {
 500             /* Index buffers don't change too often.  Mark them as static.
 501              */
 502             hint_flag = SVGA3D_SURFACE_HINT_STATIC;
 503          }
 504          else {
 505             /* Since we're reusing buffers we're effectively transforming all
 506              * of them into dynamic buffers.
 507              *
 508              * It would be nice to not cache long lived static buffers. But there
 509              * is no way to detect the long lived from short lived ones yet. A
 510              * good heuristic would be buffer size.
 511              */
 512             hint_flag = SVGA3D_SURFACE_HINT_DYNAMIC;
 513          }
 514
 515          key->flags &= ~(SVGA3D_SURFACE_HINT_STATIC |
 516                          SVGA3D_SURFACE_HINT_DYNAMIC);
 517          key->flags |= hint_flag;
 518       }
 519
 520       handle = svga_screen_cache_lookup(svgascreen, key);
 521       if (handle) {
 522          if (key->format == SVGA3D_BUFFER)
 523             SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 524                      "reuse sid %p sz %d (buffer)\n", handle,
 525                      key->size.width);
 526          else
 527             SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 528                      "reuse sid %p sz %dx%dx%d mips %d faces %d arraySize %d\n", handle,
 529                      key->size.width,
 530                      key->size.height,
 531                      key->size.depth,
 532                      key->numMipLevels,
 533                      key->numFaces,
 534                      key->arraySize);
 535          *validated = TRUE;
 536       }
 537    }
 538
 539    if (!handle) {
 540       /* Unable to recycle surface, allocate a new one */
 541       unsigned usage = 0;
 542
 543       if (!key->cachable)
 544          usage |= SVGA_SURFACE_USAGE_SHARED;
 545       if (key->scanout)
 546          usage |= SVGA_SURFACE_USAGE_SCANOUT;
 547
 548       handle = sws->surface_create(sws,
 549                                    key->flags,
 550                                    key->format,
 551                                    usage,
 552                                    key->size,
 553                                    key->numFaces * key->arraySize,
 554                                    key->numMipLevels,
 555                                    key->sampleCount);
 556       if (handle)
 557          SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
 558                   "  CREATE sid %p sz %dx%dx%d\n",
 559                   handle,
 560                   key->size.width,
 561                   key->size.height,
 562                   key->size.depth);
 563
 564       *validated = FALSE;
 565    }
 566
 567    return handle;
 568 }
 569
 570
 571 /**
 572  * Release a surface.  We don't actually free the surface- we put
 573  * it into the cache of freed surfaces (if it's cachable).
 574  */
 575 void
 576 svga_screen_surface_destroy(struct svga_screen *svgascreen,
 577                             const struct svga_host_surface_cache_key *key,
 578                             struct svga_winsys_surface **p_handle)
 579 {
 580    struct svga_winsys_screen *sws = svgascreen->sws;
 581
 582    /* We only set the cachable flag for surfaces of which we are the
 583     * exclusive owner.  So just hold onto our existing reference in
 584     * that case.
 585     */
 586    if (SVGA_SURFACE_CACHE_ENABLED && key->cachable) {
 587       svga_screen_cache_add(svgascreen, key, p_handle);
 588    }
 589    else {
 590       SVGA_DBG(DEBUG_DMA,
 591                "unref sid %p (uncachable)\n", *p_handle);
 592       sws->surface_reference(sws, p_handle, NULL);
 593    }
 594 }
 595
 596
 597 /**
 598  * Print/dump the contents of the screen cache.  For debugging.
 599  */
 600 void
 601 svga_screen_cache_dump(const struct svga_screen *svgascreen)
 602 {
 603    const struct svga_host_surface_cache *cache = &svgascreen->cache;
 604    unsigned bucket;
 605    unsigned count = 0;
 606
 607    debug_printf("svga3d surface cache:\n");
 608    for (bucket = 0; bucket < SVGA_HOST_SURFACE_CACHE_BUCKETS; bucket++) {
 609       struct list_head *curr;
 610       curr = cache->bucket[bucket].next;
 611       while (curr && curr != &cache->bucket[bucket]) {
 612          struct svga_host_surface_cache_entry *entry =
 613             LIST_ENTRY(struct svga_host_surface_cache_entry,
 614                        curr, bucket_head);
 615          if (entry->key.format == SVGA3D_BUFFER) {
 616             debug_printf("  %p: buffer %u bytes\n",
 617                          entry->handle,
 618                          entry->key.size.width);
 619          }
 620          else {
 621             debug_printf("  %p: %u x %u x %u format %u\n",
 622                          entry->handle,
 623                          entry->key.size.width,
 624                          entry->key.size.height,
 625                          entry->key.size.depth,
 626                          entry->key.format);
 627          }
 628          curr = curr->next;
 629          count++;
 630       }
 631    }
 632
 633    debug_printf("%u surfaces, %u bytes\n", count, cache->total_size);
 634 }