src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc_buf pointers need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58
  59 #include "main/imports.h"
  60 #include "brw_state.h"
  61 #include "intel_batchbuffer.h"
  62
  63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  64  * structs:
  65  */
  66 #include "brw_wm.h"
  67 #include "brw_vs.h"
  68 #include "brw_clip.h"
  69 #include "brw_sf.h"
  70 #include "brw_gs.h"
  71
  72
  73 static GLuint
  74 hash_key(struct brw_cache_item *item)
  75 {
  76    GLuint *ikey = (GLuint *)item->key;
  77    GLuint hash = item->cache_id, i;
  78
  79    assert(item->key_size % 4 == 0);
  80
  81    /* I'm sure this can be improved on:
  82     */
  83    for (i = 0; i < item->key_size/4; i++) {
  84       hash ^= ikey[i];
  85       hash = (hash << 5) | (hash >> 27);
  86    }
  87
  88    /* Include the BO pointers as key data as well */
  89    ikey = (GLuint *)item->reloc_bufs;
  90    for (i = 0; i < item->nr_reloc_bufs * sizeof(drm_intel_bo *) / 4; i++) {
  91       hash ^= ikey[i];
  92       hash = (hash << 5) | (hash >> 27);
  93    }
  94
  95    return hash;
  96 }
  97
  98
  99 /**
 100  * Marks a new buffer as being chosen for the given cache id.
 101  */
 102 static void
 103 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 104                   dri_bo *bo)
 105 {
 106    if (bo == cache->last_bo[cache_id])
 107       return; /* no change */
 108
 109    dri_bo_unreference(cache->last_bo[cache_id]);
 110    cache->last_bo[cache_id] = bo;
 111    dri_bo_reference(cache->last_bo[cache_id]);
 112    cache->brw->state.dirty.cache |= 1 << cache_id;
 113 }
 114
 115 static int
 116 brw_cache_item_equals(const struct brw_cache_item *a,
 117                       const struct brw_cache_item *b)
 118 {
 119    return a->cache_id == b->cache_id &&
 120       a->hash == b->hash &&
 121       a->key_size == b->key_size &&
 122       (memcmp(a->key, b->key, a->key_size) == 0) &&
 123       a->nr_reloc_bufs == b->nr_reloc_bufs &&
 124       (memcmp(a->reloc_bufs, b->reloc_bufs,
 125               a->nr_reloc_bufs * sizeof(dri_bo *)) == 0);
 126 }
 127
 128 static struct brw_cache_item *
 129 search_cache(struct brw_cache *cache, GLuint hash,
 130              struct brw_cache_item *lookup)
 131 {
 132    struct brw_cache_item *c;
 133
 134 #if 0
 135    int bucketcount = 0;
 136
 137    for (c = cache->items[hash % cache->size]; c; c = c->next)
 138       bucketcount++;
 139
 140    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
 141            cache->size, bucketcount, cache->n_items);
 142 #endif
 143
 144    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 145       if (brw_cache_item_equals(lookup, c))
 146          return c;
 147    }
 148
 149    return NULL;
 150 }
 151
 152
 153 static void
 154 rehash(struct brw_cache *cache)
 155 {
 156    struct brw_cache_item **items;
 157    struct brw_cache_item *c, *next;
 158    GLuint size, i;
 159
 160    size = cache->size * 3;
 161    items = (struct brw_cache_item**) _mesa_calloc(size * sizeof(*items));
 162
 163    for (i = 0; i < cache->size; i++)
 164       for (c = cache->items[i]; c; c = next) {
 165          next = c->next;
 166          c->next = items[c->hash % size];
 167          items[c->hash % size] = c;
 168       }
 169
 170    FREE(cache->items);
 171    cache->items = items;
 172    cache->size = size;
 173 }
 174
 175
 176 /**
 177  * Returns the buffer object matching cache_id and key, or NULL.
 178  */
 179 dri_bo *
 180 brw_search_cache(struct brw_cache *cache,
 181                  enum brw_cache_id cache_id,
 182                  const void *key,
 183                  GLuint key_size,
 184                  dri_bo **reloc_bufs, GLuint nr_reloc_bufs,
 185                  void *aux_return)
 186 {
 187    struct brw_cache_item *item;
 188    struct brw_cache_item lookup;
 189    GLuint hash;
 190
 191    lookup.cache_id = cache_id;
 192    lookup.key = key;
 193    lookup.key_size = key_size;
 194    lookup.reloc_bufs = reloc_bufs;
 195    lookup.nr_reloc_bufs = nr_reloc_bufs;
 196    hash = hash_key(&lookup);
 197    lookup.hash = hash;
 198
 199    item = search_cache(cache, hash, &lookup);
 200
 201    if (item == NULL)
 202       return NULL;
 203
 204    if (aux_return)
 205       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 206
 207    update_cache_last(cache, cache_id, item->bo);
 208
 209    dri_bo_reference(item->bo);
 210    return item->bo;
 211 }
 212
 213
 214 drm_intel_bo *
 215 brw_upload_cache_with_auxdata(struct brw_cache *cache,
 216                               enum brw_cache_id cache_id,
 217                               const void *key,
 218                               GLuint key_size,
 219                               dri_bo **reloc_bufs,
 220                               GLuint nr_reloc_bufs,
 221                               const void *data,
 222                               GLuint data_size,
 223                               const void *aux,
 224                               GLuint aux_size,
 225                               void *aux_return)
 226 {
 227    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 228    GLuint hash;
 229    GLuint relocs_size = nr_reloc_bufs * sizeof(dri_bo *);
 230    void *tmp;
 231    dri_bo *bo;
 232    int i;
 233
 234    item->cache_id = cache_id;
 235    item->key = key;
 236    item->key_size = key_size;
 237    item->reloc_bufs = reloc_bufs;
 238    item->nr_reloc_bufs = nr_reloc_bufs;
 239    hash = hash_key(item);
 240    item->hash = hash;
 241
 242    /* Create the buffer object to contain the data */
 243    bo = dri_bo_alloc(cache->brw->intel.bufmgr,
 244                      cache->name[cache_id], data_size, 1 << 6);
 245
 246
 247    /* Set up the memory containing the key, aux_data, and reloc_bufs */
 248    tmp = _mesa_malloc(key_size + aux_size + relocs_size);
 249
 250    memcpy(tmp, key, key_size);
 251    memcpy(tmp + key_size, aux, aux_size);
 252    memcpy(tmp + key_size + aux_size, reloc_bufs, relocs_size);
 253    for (i = 0; i < nr_reloc_bufs; i++) {
 254       if (reloc_bufs[i] != NULL)
 255          dri_bo_reference(reloc_bufs[i]);
 256    }
 257
 258    item->key = tmp;
 259    item->reloc_bufs = tmp + key_size + aux_size;
 260
 261    item->bo = bo;
 262    dri_bo_reference(bo);
 263
 264    if (cache->n_items > cache->size * 1.5)
 265       rehash(cache);
 266
 267    hash %= cache->size;
 268    item->next = cache->items[hash];
 269    cache->items[hash] = item;
 270    cache->n_items++;
 271
 272    if (aux_return) {
 273       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 274    }
 275
 276    if (INTEL_DEBUG & DEBUG_STATE)
 277       _mesa_printf("upload %s: %d bytes to cache id %d\n",
 278                    cache->name[cache_id],
 279                    data_size, cache_id);
 280
 281    /* Copy data to the buffer */
 282    dri_bo_subdata(bo, 0, data_size, data);
 283
 284    update_cache_last(cache, cache_id, bo);
 285
 286    return bo;
 287 }
 288
 289 drm_intel_bo *
 290 brw_upload_cache(struct brw_cache *cache,
 291                  enum brw_cache_id cache_id,
 292                  const void *key,
 293                  GLuint key_size,
 294                  dri_bo **reloc_bufs,
 295                  GLuint nr_reloc_bufs,
 296                  const void *data,
 297                  GLuint data_size)
 298 {
 299    return brw_upload_cache_with_auxdata(cache, cache_id,
 300                                         key, key_size,
 301                                         reloc_bufs, nr_reloc_bufs,
 302                                         data, data_size,
 303                                         NULL, 0,
 304                                         NULL);
 305 }
 306
 307 /**
 308  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 309  *
 310  * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
 311  * better to use, as the potentially changing offsets in the data-used-as-key
 312  * will result in excessive cache misses.
 313  *
 314  * If aux data is involved, use search/upload instead.
 315
 316  */
 317 dri_bo *
 318 brw_cache_data(struct brw_cache *cache,
 319                enum brw_cache_id cache_id,
 320                const void *data,
 321                GLuint data_size,
 322                dri_bo **reloc_bufs,
 323                GLuint nr_reloc_bufs)
 324 {
 325    dri_bo *bo;
 326    struct brw_cache_item *item, lookup;
 327    GLuint hash;
 328
 329    lookup.cache_id = cache_id;
 330    lookup.key = data;
 331    lookup.key_size = data_size;
 332    lookup.reloc_bufs = reloc_bufs;
 333    lookup.nr_reloc_bufs = nr_reloc_bufs;
 334    hash = hash_key(&lookup);
 335    lookup.hash = hash;
 336
 337    item = search_cache(cache, hash, &lookup);
 338    if (item) {
 339       update_cache_last(cache, cache_id, item->bo);
 340       dri_bo_reference(item->bo);
 341       return item->bo;
 342    }
 343
 344    bo = brw_upload_cache(cache, cache_id,
 345                          data, data_size,
 346                          reloc_bufs, nr_reloc_bufs,
 347                          data, data_size);
 348
 349    return bo;
 350 }
 351
 352 enum pool_type {
 353    DW_SURFACE_STATE,
 354    DW_GENERAL_STATE
 355 };
 356
 357
 358 static void
 359 brw_init_cache_id(struct brw_cache *cache,
 360                   const char *name,
 361                   enum brw_cache_id id)
 362 {
 363    cache->name[id] = strdup(name);
 364 }
 365
 366
 367 static void
 368 brw_init_non_surface_cache(struct brw_context *brw)
 369 {
 370    struct brw_cache *cache = &brw->cache;
 371
 372    cache->brw = brw;
 373
 374    cache->size = 7;
 375    cache->n_items = 0;
 376    cache->items = (struct brw_cache_item **)
 377       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 378
 379    brw_init_cache_id(cache, "CC_VP", BRW_CC_VP);
 380    brw_init_cache_id(cache, "CC_UNIT", BRW_CC_UNIT);
 381    brw_init_cache_id(cache, "WM_PROG", BRW_WM_PROG);
 382    brw_init_cache_id(cache, "SAMPLER_DEFAULT_COLOR", BRW_SAMPLER_DEFAULT_COLOR);
 383    brw_init_cache_id(cache, "SAMPLER", BRW_SAMPLER);
 384    brw_init_cache_id(cache, "WM_UNIT", BRW_WM_UNIT);
 385    brw_init_cache_id(cache, "SF_PROG", BRW_SF_PROG);
 386    brw_init_cache_id(cache, "SF_VP", BRW_SF_VP);
 387
 388    brw_init_cache_id(cache, "SF_UNIT", BRW_SF_UNIT);
 389
 390    brw_init_cache_id(cache, "VS_UNIT", BRW_VS_UNIT);
 391
 392    brw_init_cache_id(cache, "VS_PROG", BRW_VS_PROG);
 393
 394    brw_init_cache_id(cache, "CLIP_UNIT", BRW_CLIP_UNIT);
 395
 396    brw_init_cache_id(cache, "CLIP_PROG", BRW_CLIP_PROG);
 397
 398    brw_init_cache_id(cache, "GS_UNIT", BRW_GS_UNIT);
 399
 400    brw_init_cache_id(cache, "GS_PROG", BRW_GS_PROG);
 401 }
 402
 403
 404 static void
 405 brw_init_surface_cache(struct brw_context *brw)
 406 {
 407    struct brw_cache *cache = &brw->surface_cache;
 408
 409    cache->brw = brw;
 410
 411    cache->size = 7;
 412    cache->n_items = 0;
 413    cache->items = (struct brw_cache_item **)
 414       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 415
 416    brw_init_cache_id(cache, "SS_SURFACE", BRW_SS_SURFACE);
 417    brw_init_cache_id(cache, "SS_SURF_BIND", BRW_SS_SURF_BIND);
 418 }
 419
 420
 421 void
 422 brw_init_caches(struct brw_context *brw)
 423 {
 424    brw_init_non_surface_cache(brw);
 425    brw_init_surface_cache(brw);
 426 }
 427
 428
 429 static void
 430 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 431 {
 432    struct brw_cache_item *c, *next;
 433    GLuint i;
 434
 435    if (INTEL_DEBUG & DEBUG_STATE)
 436       _mesa_printf("%s\n", __FUNCTION__);
 437
 438    for (i = 0; i < cache->size; i++) {
 439       for (c = cache->items[i]; c; c = next) {
 440          int j;
 441
 442          next = c->next;
 443          for (j = 0; j < c->nr_reloc_bufs; j++)
 444             dri_bo_unreference(c->reloc_bufs[j]);
 445          dri_bo_unreference(c->bo);
 446          free((void *)c->key);
 447          free(c);
 448       }
 449       cache->items[i] = NULL;
 450    }
 451
 452    cache->n_items = 0;
 453
 454    if (brw->curbe.last_buf) {
 455       _mesa_free(brw->curbe.last_buf);
 456       brw->curbe.last_buf = NULL;
 457    }
 458
 459    brw->state.dirty.mesa |= ~0;
 460    brw->state.dirty.brw |= ~0;
 461    brw->state.dirty.cache |= ~0;
 462 }
 463
 464 /* Clear all entries from the cache that point to the given bo.
 465  *
 466  * This lets us release memory for reuse earlier for known-dead buffers,
 467  * at the cost of walking the entire hash table.
 468  */
 469 void
 470 brw_state_cache_bo_delete(struct brw_cache *cache, dri_bo *bo)
 471 {
 472    struct brw_cache_item **prev;
 473    GLuint i;
 474
 475    if (INTEL_DEBUG & DEBUG_STATE)
 476       _mesa_printf("%s\n", __FUNCTION__);
 477
 478    for (i = 0; i < cache->size; i++) {
 479       for (prev = &cache->items[i]; *prev;) {
 480          struct brw_cache_item *c = *prev;
 481
 482          if (drm_intel_bo_references(c->bo, bo)) {
 483             int j;
 484
 485             *prev = c->next;
 486
 487             for (j = 0; j < c->nr_reloc_bufs; j++)
 488                dri_bo_unreference(c->reloc_bufs[j]);
 489             dri_bo_unreference(c->bo);
 490             free((void *)c->key);
 491             free(c);
 492             cache->n_items--;
 493          } else {
 494             prev = &c->next;
 495          }
 496       }
 497    }
 498 }
 499
 500 void
 501 brw_state_cache_check_size(struct brw_context *brw)
 502 {
 503    if (INTEL_DEBUG & DEBUG_STATE)
 504       _mesa_printf("%s (n_items=%d)\n", __FUNCTION__, brw->cache.n_items);
 505
 506    /* un-tuned guess.  We've got around 20 state objects for a total of around
 507     * 32k, so 1000 of them is around 1.5MB.
 508     */
 509    if (brw->cache.n_items > 1000)
 510       brw_clear_cache(brw, &brw->cache);
 511
 512    if (brw->surface_cache.n_items > 1000)
 513       brw_clear_cache(brw, &brw->surface_cache);
 514 }
 515
 516
 517 static void
 518 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 519 {
 520    GLuint i;
 521
 522    if (INTEL_DEBUG & DEBUG_STATE)
 523       _mesa_printf("%s\n", __FUNCTION__);
 524
 525    brw_clear_cache(brw, cache);
 526    for (i = 0; i < BRW_MAX_CACHE; i++) {
 527       dri_bo_unreference(cache->last_bo[i]);
 528       free(cache->name[i]);
 529    }
 530    free(cache->items);
 531    cache->items = NULL;
 532    cache->size = 0;
 533 }
 534
 535
 536 void
 537 brw_destroy_caches(struct brw_context *brw)
 538 {
 539    brw_destroy_cache(brw, &brw->cache);
 540    brw_destroy_cache(brw, &brw->surface_cache);
 541 }