src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc_buf pointers need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58
  59 #include "main/imports.h"
  60 #include "brw_state.h"
  61 #include "intel_batchbuffer.h"
  62
  63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  64  * structs:
  65  */
  66 #include "brw_wm.h"
  67 #include "brw_vs.h"
  68 #include "brw_clip.h"
  69 #include "brw_sf.h"
  70 #include "brw_gs.h"
  71
  72
  73 static GLuint
  74 hash_key(const void *key, GLuint key_size,
  75          dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
  76 {
  77    GLuint *ikey = (GLuint *)key;
  78    GLuint hash = 0, i;
  79
  80    assert(key_size % 4 == 0);
  81
  82    /* I'm sure this can be improved on:
  83     */
  84    for (i = 0; i < key_size/4; i++) {
  85       hash ^= ikey[i];
  86       hash = (hash << 5) | (hash >> 27);
  87    }
  88
  89    /* Include the BO pointers as key data as well */
  90    ikey = (GLuint *)reloc_bufs;
  91    key_size = nr_reloc_bufs * sizeof(dri_bo *);
  92    for (i = 0; i < key_size/4; i++) {
  93       hash ^= ikey[i];
  94       hash = (hash << 5) | (hash >> 27);
  95    }
  96
  97    return hash;
  98 }
  99
 100
 101 /**
 102  * Marks a new buffer as being chosen for the given cache id.
 103  */
 104 static void
 105 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 106                   dri_bo *bo)
 107 {
 108    if (bo == cache->last_bo[cache_id])
 109       return; /* no change */
 110
 111    dri_bo_unreference(cache->last_bo[cache_id]);
 112    cache->last_bo[cache_id] = bo;
 113    dri_bo_reference(cache->last_bo[cache_id]);
 114    cache->brw->state.dirty.cache |= 1 << cache_id;
 115 }
 116
 117
 118 static struct brw_cache_item *
 119 search_cache(struct brw_cache *cache, enum brw_cache_id cache_id,
 120              GLuint hash, const void *key, GLuint key_size,
 121              dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
 122 {
 123    struct brw_cache_item *c;
 124
 125 #if 0
 126    int bucketcount = 0;
 127
 128    for (c = cache->items[hash % cache->size]; c; c = c->next)
 129       bucketcount++;
 130
 131    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
 132            cache->size, bucketcount, cache->n_items);
 133 #endif
 134
 135    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 136       if (c->cache_id == cache_id &&
 137           c->hash == hash &&
 138           c->key_size == key_size &&
 139           memcmp(c->key, key, key_size) == 0 &&
 140           c->nr_reloc_bufs == nr_reloc_bufs &&
 141           memcmp(c->reloc_bufs, reloc_bufs,
 142                  nr_reloc_bufs * sizeof(dri_bo *)) == 0)
 143          return c;
 144    }
 145
 146    return NULL;
 147 }
 148
 149
 150 static void
 151 rehash(struct brw_cache *cache)
 152 {
 153    struct brw_cache_item **items;
 154    struct brw_cache_item *c, *next;
 155    GLuint size, i;
 156
 157    size = cache->size * 3;
 158    items = (struct brw_cache_item**) _mesa_calloc(size * sizeof(*items));
 159
 160    for (i = 0; i < cache->size; i++)
 161       for (c = cache->items[i]; c; c = next) {
 162          next = c->next;
 163          c->next = items[c->hash % size];
 164          items[c->hash % size] = c;
 165       }
 166
 167    FREE(cache->items);
 168    cache->items = items;
 169    cache->size = size;
 170 }
 171
 172
 173 /**
 174  * Returns the buffer object matching cache_id and key, or NULL.
 175  */
 176 dri_bo *
 177 brw_search_cache(struct brw_cache *cache,
 178                  enum brw_cache_id cache_id,
 179                  const void *key,
 180                  GLuint key_size,
 181                  dri_bo **reloc_bufs, GLuint nr_reloc_bufs,
 182                  void *aux_return)
 183 {
 184    struct brw_cache_item *item;
 185    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 186
 187    item = search_cache(cache, cache_id, hash, key, key_size,
 188                        reloc_bufs, nr_reloc_bufs);
 189
 190    if (item == NULL)
 191       return NULL;
 192
 193    if (aux_return)
 194       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 195
 196    update_cache_last(cache, cache_id, item->bo);
 197
 198    dri_bo_reference(item->bo);
 199    return item->bo;
 200 }
 201
 202
 203 dri_bo *
 204 brw_upload_cache( struct brw_cache *cache,
 205                   enum brw_cache_id cache_id,
 206                   const void *key,
 207                   GLuint key_size,
 208                   dri_bo **reloc_bufs,
 209                   GLuint nr_reloc_bufs,
 210                   const void *data,
 211                   GLuint data_size,
 212                   const void *aux,
 213                   void *aux_return )
 214 {
 215    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 216    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 217    GLuint relocs_size = nr_reloc_bufs * sizeof(dri_bo *);
 218    GLuint aux_size = cache->aux_size[cache_id];
 219    void *tmp;
 220    dri_bo *bo;
 221    int i;
 222
 223    /* Create the buffer object to contain the data */
 224    bo = dri_bo_alloc(cache->brw->intel.bufmgr,
 225                      cache->name[cache_id], data_size, 1 << 6);
 226
 227
 228    /* Set up the memory containing the key, aux_data, and reloc_bufs */
 229    tmp = _mesa_malloc(key_size + aux_size + relocs_size);
 230
 231    memcpy(tmp, key, key_size);
 232    memcpy(tmp + key_size, aux, cache->aux_size[cache_id]);
 233    memcpy(tmp + key_size + aux_size, reloc_bufs, relocs_size);
 234    for (i = 0; i < nr_reloc_bufs; i++) {
 235       if (reloc_bufs[i] != NULL)
 236          dri_bo_reference(reloc_bufs[i]);
 237    }
 238
 239    item->cache_id = cache_id;
 240    item->key = tmp;
 241    item->hash = hash;
 242    item->key_size = key_size;
 243    item->reloc_bufs = tmp + key_size + aux_size;
 244    item->nr_reloc_bufs = nr_reloc_bufs;
 245
 246    item->bo = bo;
 247    dri_bo_reference(bo);
 248    item->data_size = data_size;
 249
 250    if (cache->n_items > cache->size * 1.5)
 251       rehash(cache);
 252
 253    hash %= cache->size;
 254    item->next = cache->items[hash];
 255    cache->items[hash] = item;
 256    cache->n_items++;
 257
 258    if (aux_return) {
 259       assert(cache->aux_size[cache_id]);
 260       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 261    }
 262
 263    if (INTEL_DEBUG & DEBUG_STATE)
 264       _mesa_printf("upload %s: %d bytes to cache id %d\n",
 265                    cache->name[cache_id],
 266                    data_size, cache_id);
 267
 268    /* Copy data to the buffer */
 269    dri_bo_subdata(bo, 0, data_size, data);
 270
 271    update_cache_last(cache, cache_id, bo);
 272
 273    return bo;
 274 }
 275
 276
 277 /**
 278  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 279  *
 280  * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
 281  * better to use, as the potentially changing offsets in the data-used-as-key
 282  * will result in excessive cache misses.
 283  *
 284  * If aux data is involved, use search/upload instead.
 285
 286  */
 287 dri_bo *
 288 brw_cache_data(struct brw_cache *cache,
 289                enum brw_cache_id cache_id,
 290                const void *data,
 291                GLuint data_size,
 292                dri_bo **reloc_bufs,
 293                GLuint nr_reloc_bufs)
 294 {
 295    dri_bo *bo;
 296    struct brw_cache_item *item;
 297    GLuint hash = hash_key(data, data_size, reloc_bufs, nr_reloc_bufs);
 298
 299    item = search_cache(cache, cache_id, hash, data, data_size,
 300                        reloc_bufs, nr_reloc_bufs);
 301    if (item) {
 302       update_cache_last(cache, cache_id, item->bo);
 303       dri_bo_reference(item->bo);
 304       return item->bo;
 305    }
 306
 307    bo = brw_upload_cache(cache, cache_id,
 308                          data, data_size,
 309                          reloc_bufs, nr_reloc_bufs,
 310                          data, data_size,
 311                          NULL, NULL);
 312
 313    return bo;
 314 }
 315
 316 enum pool_type {
 317    DW_SURFACE_STATE,
 318    DW_GENERAL_STATE
 319 };
 320
 321
 322 static void
 323 brw_init_cache_id(struct brw_cache *cache,
 324                   const char *name,
 325                   enum brw_cache_id id,
 326                   GLuint aux_size)
 327 {
 328    cache->name[id] = strdup(name);
 329    cache->aux_size[id] = aux_size;
 330 }
 331
 332
 333 static void
 334 brw_init_non_surface_cache(struct brw_context *brw)
 335 {
 336    struct brw_cache *cache = &brw->cache;
 337
 338    cache->brw = brw;
 339
 340    cache->size = 7;
 341    cache->n_items = 0;
 342    cache->items = (struct brw_cache_item **)
 343       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 344
 345    brw_init_cache_id(cache,
 346                      "CC_VP",
 347                      BRW_CC_VP,
 348                      0);
 349
 350    brw_init_cache_id(cache,
 351                      "CC_UNIT",
 352                      BRW_CC_UNIT,
 353                      0);
 354
 355    brw_init_cache_id(cache,
 356                      "WM_PROG",
 357                      BRW_WM_PROG,
 358                      sizeof(struct brw_wm_prog_data));
 359
 360    brw_init_cache_id(cache,
 361                      "SAMPLER_DEFAULT_COLOR",
 362                      BRW_SAMPLER_DEFAULT_COLOR,
 363                      0);
 364
 365    brw_init_cache_id(cache,
 366                      "SAMPLER",
 367                      BRW_SAMPLER,
 368                      0);
 369
 370    brw_init_cache_id(cache,
 371                      "WM_UNIT",
 372                      BRW_WM_UNIT,
 373                      0);
 374
 375    brw_init_cache_id(cache,
 376                      "SF_PROG",
 377                      BRW_SF_PROG,
 378                      sizeof(struct brw_sf_prog_data));
 379
 380    brw_init_cache_id(cache,
 381                      "SF_VP",
 382                      BRW_SF_VP,
 383                      0);
 384
 385    brw_init_cache_id(cache,
 386                      "SF_UNIT",
 387                      BRW_SF_UNIT,
 388                      0);
 389
 390    brw_init_cache_id(cache,
 391                      "VS_UNIT",
 392                      BRW_VS_UNIT,
 393                      0);
 394
 395    brw_init_cache_id(cache,
 396                      "VS_PROG",
 397                      BRW_VS_PROG,
 398                      sizeof(struct brw_vs_prog_data));
 399
 400    brw_init_cache_id(cache,
 401                      "CLIP_UNIT",
 402                      BRW_CLIP_UNIT,
 403                      0);
 404
 405    brw_init_cache_id(cache,
 406                      "CLIP_PROG",
 407                      BRW_CLIP_PROG,
 408                      sizeof(struct brw_clip_prog_data));
 409
 410    brw_init_cache_id(cache,
 411                      "GS_UNIT",
 412                      BRW_GS_UNIT,
 413                      0);
 414
 415    brw_init_cache_id(cache,
 416                      "GS_PROG",
 417                      BRW_GS_PROG,
 418                      sizeof(struct brw_gs_prog_data));
 419 }
 420
 421
 422 static void
 423 brw_init_surface_cache(struct brw_context *brw)
 424 {
 425    struct brw_cache *cache = &brw->surface_cache;
 426
 427    cache->brw = brw;
 428
 429    cache->size = 7;
 430    cache->n_items = 0;
 431    cache->items = (struct brw_cache_item **)
 432       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 433
 434    brw_init_cache_id(cache,
 435                      "SS_SURFACE",
 436                      BRW_SS_SURFACE,
 437                      0);
 438
 439    brw_init_cache_id(cache,
 440                      "SS_SURF_BIND",
 441                      BRW_SS_SURF_BIND,
 442                      0);
 443 }
 444
 445
 446 void
 447 brw_init_caches(struct brw_context *brw)
 448 {
 449    brw_init_non_surface_cache(brw);
 450    brw_init_surface_cache(brw);
 451 }
 452
 453
 454 static void
 455 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 456 {
 457    struct brw_cache_item *c, *next;
 458    GLuint i;
 459
 460    if (INTEL_DEBUG & DEBUG_STATE)
 461       _mesa_printf("%s\n", __FUNCTION__);
 462
 463    for (i = 0; i < cache->size; i++) {
 464       for (c = cache->items[i]; c; c = next) {
 465          int j;
 466
 467          next = c->next;
 468          for (j = 0; j < c->nr_reloc_bufs; j++)
 469             dri_bo_unreference(c->reloc_bufs[j]);
 470          dri_bo_unreference(c->bo);
 471          free((void *)c->key);
 472          free(c);
 473       }
 474       cache->items[i] = NULL;
 475    }
 476
 477    cache->n_items = 0;
 478
 479    if (brw->curbe.last_buf) {
 480       _mesa_free(brw->curbe.last_buf);
 481       brw->curbe.last_buf = NULL;
 482    }
 483
 484    brw->state.dirty.mesa |= ~0;
 485    brw->state.dirty.brw |= ~0;
 486    brw->state.dirty.cache |= ~0;
 487 }
 488
 489 /* Clear all entries from the cache that point to the given bo.
 490  *
 491  * This lets us release memory for reuse earlier for known-dead buffers,
 492  * at the cost of walking the entire hash table.
 493  */
 494 void
 495 brw_state_cache_bo_delete(struct brw_cache *cache, dri_bo *bo)
 496 {
 497    struct brw_cache_item **prev;
 498    GLuint i;
 499
 500    if (INTEL_DEBUG & DEBUG_STATE)
 501       _mesa_printf("%s\n", __FUNCTION__);
 502
 503    for (i = 0; i < cache->size; i++) {
 504       for (prev = &cache->items[i]; *prev;) {
 505          struct brw_cache_item *c = *prev;
 506
 507          if (drm_intel_bo_references(c->bo, bo)) {
 508             int j;
 509
 510             *prev = c->next;
 511
 512             for (j = 0; j < c->nr_reloc_bufs; j++)
 513                dri_bo_unreference(c->reloc_bufs[j]);
 514             dri_bo_unreference(c->bo);
 515             free((void *)c->key);
 516             free(c);
 517             cache->n_items--;
 518          } else {
 519             prev = &c->next;
 520          }
 521       }
 522    }
 523 }
 524
 525 void
 526 brw_state_cache_check_size(struct brw_context *brw)
 527 {
 528    if (INTEL_DEBUG & DEBUG_STATE)
 529       _mesa_printf("%s (n_items=%d)\n", __FUNCTION__, brw->cache.n_items);
 530
 531    /* un-tuned guess.  We've got around 20 state objects for a total of around
 532     * 32k, so 1000 of them is around 1.5MB.
 533     */
 534    if (brw->cache.n_items > 1000)
 535       brw_clear_cache(brw, &brw->cache);
 536
 537    if (brw->surface_cache.n_items > 1000)
 538       brw_clear_cache(brw, &brw->surface_cache);
 539 }
 540
 541
 542 static void
 543 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 544 {
 545    GLuint i;
 546
 547    if (INTEL_DEBUG & DEBUG_STATE)
 548       _mesa_printf("%s\n", __FUNCTION__);
 549
 550    brw_clear_cache(brw, cache);
 551    for (i = 0; i < BRW_MAX_CACHE; i++) {
 552       dri_bo_unreference(cache->last_bo[i]);
 553       free(cache->name[i]);
 554    }
 555    free(cache->items);
 556    cache->items = NULL;
 557    cache->size = 0;
 558 }
 559
 560
 561 void
 562 brw_destroy_caches(struct brw_context *brw)
 563 {
 564    brw_destroy_cache(brw, &brw->cache);
 565    brw_destroy_cache(brw, &brw->surface_cache);
 566 }