src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc_buf pointers need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58
  59 #include "main/imports.h"
  60 #include "brw_state.h"
  61 #include "intel_batchbuffer.h"
  62
  63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  64  * structs:
  65  */
  66 #include "brw_wm.h"
  67 #include "brw_vs.h"
  68 #include "brw_clip.h"
  69 #include "brw_sf.h"
  70 #include "brw_gs.h"
  71
  72
  73 static GLuint
  74 hash_key(const void *key, GLuint key_size,
  75          dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
  76 {
  77    GLuint *ikey = (GLuint *)key;
  78    GLuint hash = 0, i;
  79
  80    assert(key_size % 4 == 0);
  81
  82    /* I'm sure this can be improved on:
  83     */
  84    for (i = 0; i < key_size/4; i++) {
  85       hash ^= ikey[i];
  86       hash = (hash << 5) | (hash >> 27);
  87    }
  88
  89    /* Include the BO pointers as key data as well */
  90    ikey = (GLuint *)reloc_bufs;
  91    key_size = nr_reloc_bufs * sizeof(dri_bo *);
  92    for (i = 0; i < key_size/4; i++) {
  93       hash ^= ikey[i];
  94       hash = (hash << 5) | (hash >> 27);
  95    }
  96
  97    return hash;
  98 }
  99
 100
 101 /**
 102  * Marks a new buffer as being chosen for the given cache id.
 103  */
 104 static void
 105 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 106                   dri_bo *bo)
 107 {
 108    if (bo == cache->last_bo[cache_id])
 109       return; /* no change */
 110
 111    dri_bo_unreference(cache->last_bo[cache_id]);
 112    cache->last_bo[cache_id] = bo;
 113    dri_bo_reference(cache->last_bo[cache_id]);
 114    cache->brw->state.dirty.cache |= 1 << cache_id;
 115 }
 116
 117
 118 static struct brw_cache_item *
 119 search_cache(struct brw_cache *cache, enum brw_cache_id cache_id,
 120              GLuint hash, const void *key, GLuint key_size,
 121              dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
 122 {
 123    struct brw_cache_item *c;
 124
 125 #if 0
 126    int bucketcount = 0;
 127
 128    for (c = cache->items[hash % cache->size]; c; c = c->next)
 129       bucketcount++;
 130
 131    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
 132            cache->size, bucketcount, cache->n_items);
 133 #endif
 134
 135    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 136       if (c->cache_id == cache_id &&
 137           c->hash == hash &&
 138           c->key_size == key_size &&
 139           memcmp(c->key, key, key_size) == 0 &&
 140           c->nr_reloc_bufs == nr_reloc_bufs &&
 141           memcmp(c->reloc_bufs, reloc_bufs,
 142                  nr_reloc_bufs * sizeof(dri_bo *)) == 0)
 143          return c;
 144    }
 145
 146    return NULL;
 147 }
 148
 149
 150 static void
 151 rehash(struct brw_cache *cache)
 152 {
 153    struct brw_cache_item **items;
 154    struct brw_cache_item *c, *next;
 155    GLuint size, i;
 156
 157    size = cache->size * 3;
 158    items = (struct brw_cache_item**) _mesa_calloc(size * sizeof(*items));
 159
 160    for (i = 0; i < cache->size; i++)
 161       for (c = cache->items[i]; c; c = next) {
 162          next = c->next;
 163          c->next = items[c->hash % size];
 164          items[c->hash % size] = c;
 165       }
 166
 167    FREE(cache->items);
 168    cache->items = items;
 169    cache->size = size;
 170 }
 171
 172
 173 /**
 174  * Returns the buffer object matching cache_id and key, or NULL.
 175  */
 176 dri_bo *
 177 brw_search_cache(struct brw_cache *cache,
 178                  enum brw_cache_id cache_id,
 179                  const void *key,
 180                  GLuint key_size,
 181                  dri_bo **reloc_bufs, GLuint nr_reloc_bufs,
 182                  void *aux_return)
 183 {
 184    struct brw_cache_item *item;
 185    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 186
 187    item = search_cache(cache, cache_id, hash, key, key_size,
 188                        reloc_bufs, nr_reloc_bufs);
 189
 190    if (item == NULL)
 191       return NULL;
 192
 193    if (aux_return)
 194       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 195
 196    update_cache_last(cache, cache_id, item->bo);
 197
 198    dri_bo_reference(item->bo);
 199    return item->bo;
 200 }
 201
 202
 203 dri_bo *
 204 brw_upload_cache( struct brw_cache *cache,
 205                   enum brw_cache_id cache_id,
 206                   const void *key,
 207                   GLuint key_size,
 208                   dri_bo **reloc_bufs,
 209                   GLuint nr_reloc_bufs,
 210                   const void *data,
 211                   GLuint data_size,
 212                   const void *aux,
 213                   void *aux_return )
 214 {
 215    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 216    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 217    GLuint relocs_size = nr_reloc_bufs * sizeof(dri_bo *);
 218    GLuint aux_size = cache->aux_size[cache_id];
 219    void *tmp;
 220    dri_bo *bo;
 221    int i;
 222
 223    /* Create the buffer object to contain the data */
 224    bo = dri_bo_alloc(cache->brw->intel.bufmgr,
 225                      cache->name[cache_id], data_size, 1 << 6);
 226
 227
 228    /* Set up the memory containing the key, aux_data, and reloc_bufs */
 229    tmp = _mesa_malloc(key_size + aux_size + relocs_size);
 230
 231    memcpy(tmp, key, key_size);
 232    memcpy(tmp + key_size, aux, cache->aux_size[cache_id]);
 233    memcpy(tmp + key_size + aux_size, reloc_bufs, relocs_size);
 234    for (i = 0; i < nr_reloc_bufs; i++) {
 235       if (reloc_bufs[i] != NULL)
 236          dri_bo_reference(reloc_bufs[i]);
 237    }
 238
 239    item->cache_id = cache_id;
 240    item->key = tmp;
 241    item->hash = hash;
 242    item->key_size = key_size;
 243    item->reloc_bufs = tmp + key_size + aux_size;
 244    item->nr_reloc_bufs = nr_reloc_bufs;
 245
 246    item->bo = bo;
 247    dri_bo_reference(bo);
 248
 249    if (cache->n_items > cache->size * 1.5)
 250       rehash(cache);
 251
 252    hash %= cache->size;
 253    item->next = cache->items[hash];
 254    cache->items[hash] = item;
 255    cache->n_items++;
 256
 257    if (aux_return) {
 258       assert(cache->aux_size[cache_id]);
 259       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 260    }
 261
 262    if (INTEL_DEBUG & DEBUG_STATE)
 263       _mesa_printf("upload %s: %d bytes to cache id %d\n",
 264                    cache->name[cache_id],
 265                    data_size, cache_id);
 266
 267    /* Copy data to the buffer */
 268    dri_bo_subdata(bo, 0, data_size, data);
 269
 270    update_cache_last(cache, cache_id, bo);
 271
 272    return bo;
 273 }
 274
 275
 276 /**
 277  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 278  *
 279  * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
 280  * better to use, as the potentially changing offsets in the data-used-as-key
 281  * will result in excessive cache misses.
 282  *
 283  * If aux data is involved, use search/upload instead.
 284
 285  */
 286 dri_bo *
 287 brw_cache_data(struct brw_cache *cache,
 288                enum brw_cache_id cache_id,
 289                const void *data,
 290                GLuint data_size,
 291                dri_bo **reloc_bufs,
 292                GLuint nr_reloc_bufs)
 293 {
 294    dri_bo *bo;
 295    struct brw_cache_item *item;
 296    GLuint hash = hash_key(data, data_size, reloc_bufs, nr_reloc_bufs);
 297
 298    item = search_cache(cache, cache_id, hash, data, data_size,
 299                        reloc_bufs, nr_reloc_bufs);
 300    if (item) {
 301       update_cache_last(cache, cache_id, item->bo);
 302       dri_bo_reference(item->bo);
 303       return item->bo;
 304    }
 305
 306    bo = brw_upload_cache(cache, cache_id,
 307                          data, data_size,
 308                          reloc_bufs, nr_reloc_bufs,
 309                          data, data_size,
 310                          NULL, NULL);
 311
 312    return bo;
 313 }
 314
 315 enum pool_type {
 316    DW_SURFACE_STATE,
 317    DW_GENERAL_STATE
 318 };
 319
 320
 321 static void
 322 brw_init_cache_id(struct brw_cache *cache,
 323                   const char *name,
 324                   enum brw_cache_id id,
 325                   GLuint aux_size)
 326 {
 327    cache->name[id] = strdup(name);
 328    cache->aux_size[id] = aux_size;
 329 }
 330
 331
 332 static void
 333 brw_init_non_surface_cache(struct brw_context *brw)
 334 {
 335    struct brw_cache *cache = &brw->cache;
 336
 337    cache->brw = brw;
 338
 339    cache->size = 7;
 340    cache->n_items = 0;
 341    cache->items = (struct brw_cache_item **)
 342       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 343
 344    brw_init_cache_id(cache,
 345                      "CC_VP",
 346                      BRW_CC_VP,
 347                      0);
 348
 349    brw_init_cache_id(cache,
 350                      "CC_UNIT",
 351                      BRW_CC_UNIT,
 352                      0);
 353
 354    brw_init_cache_id(cache,
 355                      "WM_PROG",
 356                      BRW_WM_PROG,
 357                      sizeof(struct brw_wm_prog_data));
 358
 359    brw_init_cache_id(cache,
 360                      "SAMPLER_DEFAULT_COLOR",
 361                      BRW_SAMPLER_DEFAULT_COLOR,
 362                      0);
 363
 364    brw_init_cache_id(cache,
 365                      "SAMPLER",
 366                      BRW_SAMPLER,
 367                      0);
 368
 369    brw_init_cache_id(cache,
 370                      "WM_UNIT",
 371                      BRW_WM_UNIT,
 372                      0);
 373
 374    brw_init_cache_id(cache,
 375                      "SF_PROG",
 376                      BRW_SF_PROG,
 377                      sizeof(struct brw_sf_prog_data));
 378
 379    brw_init_cache_id(cache,
 380                      "SF_VP",
 381                      BRW_SF_VP,
 382                      0);
 383
 384    brw_init_cache_id(cache,
 385                      "SF_UNIT",
 386                      BRW_SF_UNIT,
 387                      0);
 388
 389    brw_init_cache_id(cache,
 390                      "VS_UNIT",
 391                      BRW_VS_UNIT,
 392                      0);
 393
 394    brw_init_cache_id(cache,
 395                      "VS_PROG",
 396                      BRW_VS_PROG,
 397                      sizeof(struct brw_vs_prog_data));
 398
 399    brw_init_cache_id(cache,
 400                      "CLIP_UNIT",
 401                      BRW_CLIP_UNIT,
 402                      0);
 403
 404    brw_init_cache_id(cache,
 405                      "CLIP_PROG",
 406                      BRW_CLIP_PROG,
 407                      sizeof(struct brw_clip_prog_data));
 408
 409    brw_init_cache_id(cache,
 410                      "GS_UNIT",
 411                      BRW_GS_UNIT,
 412                      0);
 413
 414    brw_init_cache_id(cache,
 415                      "GS_PROG",
 416                      BRW_GS_PROG,
 417                      sizeof(struct brw_gs_prog_data));
 418 }
 419
 420
 421 static void
 422 brw_init_surface_cache(struct brw_context *brw)
 423 {
 424    struct brw_cache *cache = &brw->surface_cache;
 425
 426    cache->brw = brw;
 427
 428    cache->size = 7;
 429    cache->n_items = 0;
 430    cache->items = (struct brw_cache_item **)
 431       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 432
 433    brw_init_cache_id(cache,
 434                      "SS_SURFACE",
 435                      BRW_SS_SURFACE,
 436                      0);
 437
 438    brw_init_cache_id(cache,
 439                      "SS_SURF_BIND",
 440                      BRW_SS_SURF_BIND,
 441                      0);
 442 }
 443
 444
 445 void
 446 brw_init_caches(struct brw_context *brw)
 447 {
 448    brw_init_non_surface_cache(brw);
 449    brw_init_surface_cache(brw);
 450 }
 451
 452
 453 static void
 454 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 455 {
 456    struct brw_cache_item *c, *next;
 457    GLuint i;
 458
 459    if (INTEL_DEBUG & DEBUG_STATE)
 460       _mesa_printf("%s\n", __FUNCTION__);
 461
 462    for (i = 0; i < cache->size; i++) {
 463       for (c = cache->items[i]; c; c = next) {
 464          int j;
 465
 466          next = c->next;
 467          for (j = 0; j < c->nr_reloc_bufs; j++)
 468             dri_bo_unreference(c->reloc_bufs[j]);
 469          dri_bo_unreference(c->bo);
 470          free((void *)c->key);
 471          free(c);
 472       }
 473       cache->items[i] = NULL;
 474    }
 475
 476    cache->n_items = 0;
 477
 478    if (brw->curbe.last_buf) {
 479       _mesa_free(brw->curbe.last_buf);
 480       brw->curbe.last_buf = NULL;
 481    }
 482
 483    brw->state.dirty.mesa |= ~0;
 484    brw->state.dirty.brw |= ~0;
 485    brw->state.dirty.cache |= ~0;
 486 }
 487
 488 /* Clear all entries from the cache that point to the given bo.
 489  *
 490  * This lets us release memory for reuse earlier for known-dead buffers,
 491  * at the cost of walking the entire hash table.
 492  */
 493 void
 494 brw_state_cache_bo_delete(struct brw_cache *cache, dri_bo *bo)
 495 {
 496    struct brw_cache_item **prev;
 497    GLuint i;
 498
 499    if (INTEL_DEBUG & DEBUG_STATE)
 500       _mesa_printf("%s\n", __FUNCTION__);
 501
 502    for (i = 0; i < cache->size; i++) {
 503       for (prev = &cache->items[i]; *prev;) {
 504          struct brw_cache_item *c = *prev;
 505
 506          if (drm_intel_bo_references(c->bo, bo)) {
 507             int j;
 508
 509             *prev = c->next;
 510
 511             for (j = 0; j < c->nr_reloc_bufs; j++)
 512                dri_bo_unreference(c->reloc_bufs[j]);
 513             dri_bo_unreference(c->bo);
 514             free((void *)c->key);
 515             free(c);
 516             cache->n_items--;
 517          } else {
 518             prev = &c->next;
 519          }
 520       }
 521    }
 522 }
 523
 524 void
 525 brw_state_cache_check_size(struct brw_context *brw)
 526 {
 527    if (INTEL_DEBUG & DEBUG_STATE)
 528       _mesa_printf("%s (n_items=%d)\n", __FUNCTION__, brw->cache.n_items);
 529
 530    /* un-tuned guess.  We've got around 20 state objects for a total of around
 531     * 32k, so 1000 of them is around 1.5MB.
 532     */
 533    if (brw->cache.n_items > 1000)
 534       brw_clear_cache(brw, &brw->cache);
 535
 536    if (brw->surface_cache.n_items > 1000)
 537       brw_clear_cache(brw, &brw->surface_cache);
 538 }
 539
 540
 541 static void
 542 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 543 {
 544    GLuint i;
 545
 546    if (INTEL_DEBUG & DEBUG_STATE)
 547       _mesa_printf("%s\n", __FUNCTION__);
 548
 549    brw_clear_cache(brw, cache);
 550    for (i = 0; i < BRW_MAX_CACHE; i++) {
 551       dri_bo_unreference(cache->last_bo[i]);
 552       free(cache->name[i]);
 553    }
 554    free(cache->items);
 555    cache->items = NULL;
 556    cache->size = 0;
 557 }
 558
 559
 560 void
 561 brw_destroy_caches(struct brw_context *brw)
 562 {
 563    brw_destroy_cache(brw, &brw->cache);
 564    brw_destroy_cache(brw, &brw->surface_cache);
 565 }