src/gallium/auxiliary/pipebuffer/pb_bufmgr_cache.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2007-2008 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * \file
  30  * Buffer cache.
  31  *
  32  * \author Jose Fonseca <jfonseca-at-vmware-dot-com>
  33  * \author Thomas Hellström <thellstom-at-vmware-dot-com>
  34  */
  35
  36
  37 #include "pipe/p_compiler.h"
  38 #include "util/u_debug.h"
  39 #include "os/os_thread.h"
  40 #include "util/u_memory.h"
  41 #include "util/list.h"
  42 #include "util/u_time.h"
  43
  44 #include "pb_buffer.h"
  45 #include "pb_bufmgr.h"
  46
  47
  48 /**
  49  * Convenience macro (type safe).
  50  */
  51 #define SUPER(__derived) (&(__derived)->base)
  52
  53
  54 struct pb_cache_manager;
  55
  56
  57 /**
  58  * Wrapper around a pipe buffer which adds delayed destruction.
  59  */
  60 struct pb_cache_buffer
  61 {
  62    struct pb_buffer base;
  63
  64    struct pb_buffer *buffer;
  65    struct pb_cache_manager *mgr;
  66
  67    /** Caching time interval */
  68    int64_t start, end;
  69
  70    struct list_head head;
  71 };
  72
  73
  74 struct pb_cache_manager
  75 {
  76    struct pb_manager base;
  77
  78    struct pb_manager *provider;
  79    unsigned usecs;
  80
  81    pipe_mutex mutex;
  82
  83    struct list_head delayed;
  84    pb_size numDelayed;
  85    float size_factor;
  86    unsigned bypass_usage;
  87    uint64_t cache_size, max_cache_size;
  88 };
  89
  90
  91 static inline struct pb_cache_buffer *
  92 pb_cache_buffer(struct pb_buffer *buf)
  93 {
  94    assert(buf);
  95    return (struct pb_cache_buffer *)buf;
  96 }
  97
  98
  99 static inline struct pb_cache_manager *
 100 pb_cache_manager(struct pb_manager *mgr)
 101 {
 102    assert(mgr);
 103    return (struct pb_cache_manager *)mgr;
 104 }
 105
 106
 107 static void
 108 _pb_cache_manager_remove_buffer_locked(struct pb_cache_buffer *buf)
 109 {
 110    struct pb_cache_manager *mgr = buf->mgr;
 111
 112    if (buf->head.next) {
 113       LIST_DEL(&buf->head);
 114       assert(mgr->numDelayed);
 115       --mgr->numDelayed;
 116       mgr->cache_size -= buf->base.size;
 117    }
 118    buf->mgr = NULL;
 119 }
 120
 121 void
 122 pb_cache_manager_remove_buffer(struct pb_buffer *pb_buf)
 123 {
 124    struct pb_cache_buffer *buf = (struct pb_cache_buffer*)pb_buf;
 125    struct pb_cache_manager *mgr = buf->mgr;
 126
 127    if (!mgr)
 128       return;
 129
 130    pipe_mutex_lock(mgr->mutex);
 131    _pb_cache_manager_remove_buffer_locked(buf);
 132    pipe_mutex_unlock(mgr->mutex);
 133 }
 134
 135 /**
 136  * Actually destroy the buffer.
 137  */
 138 static inline void
 139 _pb_cache_buffer_destroy(struct pb_cache_buffer *buf)
 140 {
 141    if (buf->mgr)
 142       _pb_cache_manager_remove_buffer_locked(buf);
 143    assert(!pipe_is_referenced(&buf->base.reference));
 144    pb_reference(&buf->buffer, NULL);
 145    FREE(buf);
 146 }
 147
 148
 149 /**
 150  * Free as many cache buffers from the list head as possible.
 151  */
 152 static void
 153 _pb_cache_buffer_list_check_free(struct pb_cache_manager *mgr)
 154 {
 155    struct list_head *curr, *next;
 156    struct pb_cache_buffer *buf;
 157    int64_t now;
 158
 159    now = os_time_get();
 160
 161    curr = mgr->delayed.next;
 162    next = curr->next;
 163    while(curr != &mgr->delayed) {
 164       buf = LIST_ENTRY(struct pb_cache_buffer, curr, head);
 165
 166       if(!os_time_timeout(buf->start, buf->end, now))
 167          break;
 168
 169       _pb_cache_buffer_destroy(buf);
 170
 171       curr = next;
 172       next = curr->next;
 173    }
 174 }
 175
 176
 177 static void
 178 pb_cache_buffer_destroy(struct pb_buffer *_buf)
 179 {
 180    struct pb_cache_buffer *buf = pb_cache_buffer(_buf);
 181    struct pb_cache_manager *mgr = buf->mgr;
 182
 183    if (!mgr) {
 184       pb_reference(&buf->buffer, NULL);
 185       FREE(buf);
 186       return;
 187    }
 188
 189    pipe_mutex_lock(mgr->mutex);
 190    assert(!pipe_is_referenced(&buf->base.reference));
 191
 192    _pb_cache_buffer_list_check_free(mgr);
 193
 194    /* Directly release any buffer that exceeds the limit. */
 195    if (mgr->cache_size + buf->base.size > mgr->max_cache_size) {
 196       pb_reference(&buf->buffer, NULL);
 197       FREE(buf);
 198       pipe_mutex_unlock(mgr->mutex);
 199       return;
 200    }
 201
 202    buf->start = os_time_get();
 203    buf->end = buf->start + mgr->usecs;
 204    LIST_ADDTAIL(&buf->head, &mgr->delayed);
 205    ++mgr->numDelayed;
 206    mgr->cache_size += buf->base.size;
 207    pipe_mutex_unlock(mgr->mutex);
 208 }
 209
 210
 211 static void *
 212 pb_cache_buffer_map(struct pb_buffer *_buf,
 213                     unsigned flags, void *flush_ctx)
 214 {
 215    struct pb_cache_buffer *buf = pb_cache_buffer(_buf);
 216    return pb_map(buf->buffer, flags, flush_ctx);
 217 }
 218
 219
 220 static void
 221 pb_cache_buffer_unmap(struct pb_buffer *_buf)
 222 {
 223    struct pb_cache_buffer *buf = pb_cache_buffer(_buf);
 224    pb_unmap(buf->buffer);
 225 }
 226
 227
 228 static enum pipe_error
 229 pb_cache_buffer_validate(struct pb_buffer *_buf,
 230                          struct pb_validate *vl,
 231                          unsigned flags)
 232 {
 233    struct pb_cache_buffer *buf = pb_cache_buffer(_buf);
 234    return pb_validate(buf->buffer, vl, flags);
 235 }
 236
 237
 238 static void
 239 pb_cache_buffer_fence(struct pb_buffer *_buf,
 240                       struct pipe_fence_handle *fence)
 241 {
 242    struct pb_cache_buffer *buf = pb_cache_buffer(_buf);
 243    pb_fence(buf->buffer, fence);
 244 }
 245
 246
 247 static void
 248 pb_cache_buffer_get_base_buffer(struct pb_buffer *_buf,
 249                               struct pb_buffer **base_buf,
 250                               pb_size *offset)
 251 {
 252    struct pb_cache_buffer *buf = pb_cache_buffer(_buf);
 253    pb_get_base_buffer(buf->buffer, base_buf, offset);
 254 }
 255
 256
 257 const struct pb_vtbl
 258 pb_cache_buffer_vtbl = {
 259       pb_cache_buffer_destroy,
 260       pb_cache_buffer_map,
 261       pb_cache_buffer_unmap,
 262       pb_cache_buffer_validate,
 263       pb_cache_buffer_fence,
 264       pb_cache_buffer_get_base_buffer
 265 };
 266
 267
 268 static inline int
 269 pb_cache_is_buffer_compat(struct pb_cache_buffer *buf,
 270                           pb_size size,
 271                           const struct pb_desc *desc)
 272 {
 273    if (desc->usage & buf->mgr->bypass_usage)
 274       return 0;
 275
 276    if(buf->base.size < size)
 277       return 0;
 278
 279    /* be lenient with size */
 280    if(buf->base.size > (unsigned) (buf->mgr->size_factor * size))
 281       return 0;
 282
 283    if(!pb_check_alignment(desc->alignment, buf->base.alignment))
 284       return 0;
 285
 286    if(!pb_check_usage(desc->usage, buf->base.usage))
 287       return 0;
 288
 289    if (buf->mgr->provider->is_buffer_busy) {
 290       if (buf->mgr->provider->is_buffer_busy(buf->mgr->provider, buf->buffer))
 291          return -1;
 292    } else {
 293       void *ptr = pb_map(buf->buffer, PB_USAGE_DONTBLOCK, NULL);
 294
 295       if (!ptr)
 296          return -1;
 297
 298       pb_unmap(buf->buffer);
 299    }
 300
 301    return 1;
 302 }
 303
 304
 305 static struct pb_buffer *
 306 pb_cache_manager_create_buffer(struct pb_manager *_mgr,
 307                                pb_size size,
 308                                const struct pb_desc *desc)
 309 {
 310    struct pb_cache_manager *mgr = pb_cache_manager(_mgr);
 311    struct pb_cache_buffer *buf;
 312    struct pb_cache_buffer *curr_buf;
 313    struct list_head *curr, *next;
 314    int64_t now;
 315    int ret = 0;
 316
 317    pipe_mutex_lock(mgr->mutex);
 318
 319    buf = NULL;
 320    curr = mgr->delayed.next;
 321    next = curr->next;
 322
 323    /* search in the expired buffers, freeing them in the process */
 324    now = os_time_get();
 325    while(curr != &mgr->delayed) {
 326       curr_buf = LIST_ENTRY(struct pb_cache_buffer, curr, head);
 327       if(!buf && (ret = pb_cache_is_buffer_compat(curr_buf, size, desc) > 0))
 328          buf = curr_buf;
 329       else if(os_time_timeout(curr_buf->start, curr_buf->end, now))
 330          _pb_cache_buffer_destroy(curr_buf);
 331       else
 332          /* This buffer (and all hereafter) are still hot in cache */
 333          break;
 334       if (ret == -1)
 335          break;
 336       curr = next;
 337       next = curr->next;
 338    }
 339
 340    /* keep searching in the hot buffers */
 341    if(!buf && ret != -1) {
 342       while(curr != &mgr->delayed) {
 343          curr_buf = LIST_ENTRY(struct pb_cache_buffer, curr, head);
 344          ret = pb_cache_is_buffer_compat(curr_buf, size, desc);
 345          if (ret > 0) {
 346             buf = curr_buf;
 347             break;
 348          }
 349          if (ret == -1)
 350             break;
 351          /* no need to check the timeout here */
 352          curr = next;
 353          next = curr->next;
 354       }
 355    }
 356
 357    if (buf) {
 358       mgr->cache_size -= buf->base.size;
 359       LIST_DEL(&buf->head);
 360       --mgr->numDelayed;
 361       pipe_mutex_unlock(mgr->mutex);
 362       /* Increase refcount */
 363       pipe_reference_init(&buf->base.reference, 1);
 364       return &buf->base;
 365    }
 366
 367    pipe_mutex_unlock(mgr->mutex);
 368
 369    buf = CALLOC_STRUCT(pb_cache_buffer);
 370    if (!buf)
 371       return NULL;
 372
 373    buf->buffer = mgr->provider->create_buffer(mgr->provider, size, desc);
 374
 375    /* Empty the cache and try again. */
 376    if (!buf->buffer) {
 377       mgr->base.flush(&mgr->base);
 378       buf->buffer = mgr->provider->create_buffer(mgr->provider, size, desc);
 379    }
 380
 381    if(!buf->buffer) {
 382       FREE(buf);
 383       return NULL;
 384    }
 385
 386    assert(pipe_is_referenced(&buf->buffer->reference));
 387    assert(pb_check_alignment(desc->alignment, buf->buffer->alignment));
 388    assert(pb_check_usage(desc->usage & ~mgr->bypass_usage, buf->buffer->usage));
 389    assert(buf->buffer->size >= size);
 390
 391    pipe_reference_init(&buf->base.reference, 1);
 392    buf->base.alignment = buf->buffer->alignment;
 393    buf->base.usage = buf->buffer->usage;
 394    buf->base.size = buf->buffer->size;
 395
 396    buf->base.vtbl = &pb_cache_buffer_vtbl;
 397    buf->mgr = mgr;
 398
 399    return &buf->base;
 400 }
 401
 402
 403 static void
 404 pb_cache_manager_flush(struct pb_manager *_mgr)
 405 {
 406    struct pb_cache_manager *mgr = pb_cache_manager(_mgr);
 407    struct list_head *curr, *next;
 408    struct pb_cache_buffer *buf;
 409
 410    pipe_mutex_lock(mgr->mutex);
 411    curr = mgr->delayed.next;
 412    next = curr->next;
 413    while(curr != &mgr->delayed) {
 414       buf = LIST_ENTRY(struct pb_cache_buffer, curr, head);
 415       _pb_cache_buffer_destroy(buf);
 416       curr = next;
 417       next = curr->next;
 418    }
 419    pipe_mutex_unlock(mgr->mutex);
 420
 421    assert(mgr->provider->flush);
 422    if(mgr->provider->flush)
 423       mgr->provider->flush(mgr->provider);
 424 }
 425
 426
 427 static void
 428 pb_cache_manager_destroy(struct pb_manager *mgr)
 429 {
 430    pb_cache_manager_flush(mgr);
 431    FREE(mgr);
 432 }
 433
 434 /**
 435  * Create a caching buffer manager
 436  *
 437  * @param provider The buffer manager to which cache miss buffer requests
 438  * should be redirected.
 439  * @param usecs Unused buffers may be released from the cache after this
 440  * time
 441  * @param size_factor Declare buffers that are size_factor times bigger than
 442  * the requested size as cache hits.
 443  * @param bypass_usage Bitmask. If (requested usage & bypass_usage) != 0,
 444  * buffer allocation requests are redirected to the provider.
 445  * @param maximum_cache_size  Maximum size of all unused buffers the cache can
 446  * hold.
 447  */
 448 struct pb_manager *
 449 pb_cache_manager_create(struct pb_manager *provider,
 450                         unsigned usecs,
 451                         float size_factor,
 452                         unsigned bypass_usage,
 453                         uint64_t maximum_cache_size)
 454 {
 455    struct pb_cache_manager *mgr;
 456
 457    if (!provider)
 458       return NULL;
 459
 460    mgr = CALLOC_STRUCT(pb_cache_manager);
 461    if (!mgr)
 462       return NULL;
 463
 464    mgr->base.destroy = pb_cache_manager_destroy;
 465    mgr->base.create_buffer = pb_cache_manager_create_buffer;
 466    mgr->base.flush = pb_cache_manager_flush;
 467    mgr->provider = provider;
 468    mgr->usecs = usecs;
 469    mgr->size_factor = size_factor;
 470    mgr->bypass_usage = bypass_usage;
 471    LIST_INITHEAD(&mgr->delayed);
 472    mgr->numDelayed = 0;
 473    mgr->max_cache_size = maximum_cache_size;
 474    pipe_mutex_init(mgr->mutex);
 475
 476    return &mgr->base;
 477 }