src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc_buf pointers need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58
  59 #include "main/imports.h"
  60 #include "brw_state.h"
  61 #include "intel_batchbuffer.h"
  62
  63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  64  * structs:
  65  */
  66 #include "brw_wm.h"
  67 #include "brw_vs.h"
  68 #include "brw_clip.h"
  69 #include "brw_sf.h"
  70 #include "brw_gs.h"
  71
  72
  73 static GLuint
  74 hash_key(const void *key, GLuint key_size,
  75          dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
  76 {
  77    GLuint *ikey = (GLuint *)key;
  78    GLuint hash = 0, i;
  79
  80    assert(key_size % 4 == 0);
  81
  82    /* I'm sure this can be improved on:
  83     */
  84    for (i = 0; i < key_size/4; i++) {
  85       hash ^= ikey[i];
  86       hash = (hash << 5) | (hash >> 27);
  87    }
  88
  89    /* Include the BO pointers as key data as well */
  90    ikey = (GLuint *)reloc_bufs;
  91    key_size = nr_reloc_bufs * sizeof(dri_bo *);
  92    for (i = 0; i < key_size/4; i++) {
  93       hash ^= ikey[i];
  94       hash = (hash << 5) | (hash >> 27);
  95    }
  96
  97    return hash;
  98 }
  99
 100
 101 /**
 102  * Marks a new buffer as being chosen for the given cache id.
 103  */
 104 static void
 105 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 106                   dri_bo *bo)
 107 {
 108    if (bo == cache->last_bo[cache_id])
 109       return; /* no change */
 110
 111    dri_bo_unreference(cache->last_bo[cache_id]);
 112    cache->last_bo[cache_id] = bo;
 113    dri_bo_reference(cache->last_bo[cache_id]);
 114    cache->brw->state.dirty.cache |= 1 << cache_id;
 115 }
 116
 117
 118 static struct brw_cache_item *
 119 search_cache(struct brw_cache *cache, enum brw_cache_id cache_id,
 120              GLuint hash, const void *key, GLuint key_size,
 121              dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
 122 {
 123    struct brw_cache_item *c;
 124
 125 #if 0
 126    int bucketcount = 0;
 127
 128    for (c = cache->items[hash % cache->size]; c; c = c->next)
 129       bucketcount++;
 130
 131    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
 132            cache->size, bucketcount, cache->n_items);
 133 #endif
 134
 135    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 136       if (c->cache_id == cache_id &&
 137           c->hash == hash &&
 138           c->key_size == key_size &&
 139           memcmp(c->key, key, key_size) == 0 &&
 140           c->nr_reloc_bufs == nr_reloc_bufs &&
 141           memcmp(c->reloc_bufs, reloc_bufs,
 142                  nr_reloc_bufs * sizeof(dri_bo *)) == 0)
 143          return c;
 144    }
 145
 146    return NULL;
 147 }
 148
 149
 150 static void
 151 rehash(struct brw_cache *cache)
 152 {
 153    struct brw_cache_item **items;
 154    struct brw_cache_item *c, *next;
 155    GLuint size, i;
 156
 157    size = cache->size * 3;
 158    items = (struct brw_cache_item**) _mesa_calloc(size * sizeof(*items));
 159
 160    for (i = 0; i < cache->size; i++)
 161       for (c = cache->items[i]; c; c = next) {
 162          next = c->next;
 163          c->next = items[c->hash % size];
 164          items[c->hash % size] = c;
 165       }
 166
 167    FREE(cache->items);
 168    cache->items = items;
 169    cache->size = size;
 170 }
 171
 172
 173 /**
 174  * Returns the buffer object matching cache_id and key, or NULL.
 175  */
 176 dri_bo *
 177 brw_search_cache(struct brw_cache *cache,
 178                  enum brw_cache_id cache_id,
 179                  const void *key,
 180                  GLuint key_size,
 181                  dri_bo **reloc_bufs, GLuint nr_reloc_bufs,
 182                  void *aux_return)
 183 {
 184    struct brw_cache_item *item;
 185    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 186
 187    item = search_cache(cache, cache_id, hash, key, key_size,
 188                        reloc_bufs, nr_reloc_bufs);
 189
 190    if (item == NULL)
 191       return NULL;
 192
 193    if (aux_return)
 194       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 195
 196    update_cache_last(cache, cache_id, item->bo);
 197
 198    dri_bo_reference(item->bo);
 199    return item->bo;
 200 }
 201
 202
 203 dri_bo *
 204 brw_upload_cache( struct brw_cache *cache,
 205                   enum brw_cache_id cache_id,
 206                   const void *key,
 207                   GLuint key_size,
 208                   dri_bo **reloc_bufs,
 209                   GLuint nr_reloc_bufs,
 210                   const void *data,
 211                   GLuint data_size,
 212                   const void *aux,
 213                   void *aux_return )
 214 {
 215    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 216    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 217    GLuint relocs_size = nr_reloc_bufs * sizeof(dri_bo *);
 218    GLuint aux_size = cache->aux_size[cache_id];
 219    void *tmp;
 220    dri_bo *bo;
 221    int i;
 222
 223    /* Create the buffer object to contain the data */
 224    bo = dri_bo_alloc(cache->brw->intel.bufmgr,
 225                      cache->name[cache_id], data_size, 1 << 6);
 226
 227
 228    /* Set up the memory containing the key, aux_data, and reloc_bufs */
 229    tmp = _mesa_malloc(key_size + aux_size + relocs_size);
 230
 231    memcpy(tmp, key, key_size);
 232    memcpy(tmp + key_size, aux, cache->aux_size[cache_id]);
 233    memcpy(tmp + key_size + aux_size, reloc_bufs, relocs_size);
 234    for (i = 0; i < nr_reloc_bufs; i++) {
 235       if (reloc_bufs[i] != NULL)
 236          dri_bo_reference(reloc_bufs[i]);
 237    }
 238
 239    item->cache_id = cache_id;
 240    item->key = tmp;
 241    item->hash = hash;
 242    item->key_size = key_size;
 243    item->reloc_bufs = tmp + key_size + aux_size;
 244    item->nr_reloc_bufs = nr_reloc_bufs;
 245
 246    item->bo = bo;
 247    dri_bo_reference(bo);
 248    item->data_size = data_size;
 249
 250    if (cache->n_items > cache->size * 1.5)
 251       rehash(cache);
 252
 253    hash %= cache->size;
 254    item->next = cache->items[hash];
 255    cache->items[hash] = item;
 256    cache->n_items++;
 257
 258    if (aux_return) {
 259       assert(cache->aux_size[cache_id]);
 260       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 261    }
 262
 263    if (INTEL_DEBUG & DEBUG_STATE)
 264       _mesa_printf("upload %s: %d bytes to cache id %d\n",
 265                    cache->name[cache_id],
 266                    data_size, cache_id);
 267
 268    /* Copy data to the buffer */
 269    dri_bo_subdata(bo, 0, data_size, data);
 270
 271    update_cache_last(cache, cache_id, bo);
 272
 273    return bo;
 274 }
 275
 276
 277 /**
 278  * This doesn't really work with aux data.  Use search/upload instead
 279  */
 280 dri_bo *
 281 brw_cache_data_sz(struct brw_cache *cache,
 282                   enum brw_cache_id cache_id,
 283                   const void *data,
 284                   GLuint data_size,
 285                   dri_bo **reloc_bufs,
 286                   GLuint nr_reloc_bufs)
 287 {
 288    dri_bo *bo;
 289    struct brw_cache_item *item;
 290    GLuint hash = hash_key(data, data_size, reloc_bufs, nr_reloc_bufs);
 291
 292    item = search_cache(cache, cache_id, hash, data, data_size,
 293                        reloc_bufs, nr_reloc_bufs);
 294    if (item) {
 295       update_cache_last(cache, cache_id, item->bo);
 296       dri_bo_reference(item->bo);
 297       return item->bo;
 298    }
 299
 300    bo = brw_upload_cache(cache, cache_id,
 301                          data, data_size,
 302                          reloc_bufs, nr_reloc_bufs,
 303                          data, data_size,
 304                          NULL, NULL);
 305
 306    return bo;
 307 }
 308
 309
 310 /**
 311  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 312  *
 313  * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
 314  * better to use, as the potentially changing offsets in the data-used-as-key
 315  * will result in excessive cache misses.
 316  */
 317 dri_bo *
 318 brw_cache_data(struct brw_cache *cache,
 319                enum brw_cache_id cache_id,
 320                const void *data,
 321                dri_bo **reloc_bufs,
 322                GLuint nr_reloc_bufs)
 323 {
 324    return brw_cache_data_sz(cache, cache_id, data, cache->key_size[cache_id],
 325                             reloc_bufs, nr_reloc_bufs);
 326 }
 327
 328 enum pool_type {
 329    DW_SURFACE_STATE,
 330    DW_GENERAL_STATE
 331 };
 332
 333
 334 static void
 335 brw_init_cache_id(struct brw_cache *cache,
 336                   const char *name,
 337                   enum brw_cache_id id,
 338                   GLuint key_size,
 339                   GLuint aux_size)
 340 {
 341    cache->name[id] = strdup(name);
 342    cache->key_size[id] = key_size;
 343    cache->aux_size[id] = aux_size;
 344 }
 345
 346
 347 static void
 348 brw_init_non_surface_cache(struct brw_context *brw)
 349 {
 350    struct brw_cache *cache = &brw->cache;
 351
 352    cache->brw = brw;
 353
 354    cache->size = 7;
 355    cache->n_items = 0;
 356    cache->items = (struct brw_cache_item **)
 357       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 358
 359    brw_init_cache_id(cache,
 360                      "CC_VP",
 361                      BRW_CC_VP,
 362                      sizeof(struct brw_cc_viewport),
 363                      0);
 364
 365    brw_init_cache_id(cache,
 366                      "CC_UNIT",
 367                      BRW_CC_UNIT,
 368                      sizeof(struct brw_cc_unit_state),
 369                      0);
 370
 371    brw_init_cache_id(cache,
 372                      "WM_PROG",
 373                      BRW_WM_PROG,
 374                      sizeof(struct brw_wm_prog_key),
 375                      sizeof(struct brw_wm_prog_data));
 376
 377    brw_init_cache_id(cache,
 378                      "SAMPLER_DEFAULT_COLOR",
 379                      BRW_SAMPLER_DEFAULT_COLOR,
 380                      sizeof(struct brw_sampler_default_color),
 381                      0);
 382
 383    brw_init_cache_id(cache,
 384                      "SAMPLER",
 385                      BRW_SAMPLER,
 386                      0,         /* variable key/data size */
 387                      0);
 388
 389    brw_init_cache_id(cache,
 390                      "WM_UNIT",
 391                      BRW_WM_UNIT,
 392                      sizeof(struct brw_wm_unit_state),
 393                      0);
 394
 395    brw_init_cache_id(cache,
 396                      "SF_PROG",
 397                      BRW_SF_PROG,
 398                      sizeof(struct brw_sf_prog_key),
 399                      sizeof(struct brw_sf_prog_data));
 400
 401    brw_init_cache_id(cache,
 402                      "SF_VP",
 403                      BRW_SF_VP,
 404                      sizeof(struct brw_sf_viewport),
 405                      0);
 406
 407    brw_init_cache_id(cache,
 408                      "SF_UNIT",
 409                      BRW_SF_UNIT,
 410                      sizeof(struct brw_sf_unit_state),
 411                      0);
 412
 413    brw_init_cache_id(cache,
 414                      "VS_UNIT",
 415                      BRW_VS_UNIT,
 416                      sizeof(struct brw_vs_unit_state),
 417                      0);
 418
 419    brw_init_cache_id(cache,
 420                      "VS_PROG",
 421                      BRW_VS_PROG,
 422                      sizeof(struct brw_vs_prog_key),
 423                      sizeof(struct brw_vs_prog_data));
 424
 425    brw_init_cache_id(cache,
 426                      "CLIP_UNIT",
 427                      BRW_CLIP_UNIT,
 428                      sizeof(struct brw_clip_unit_state),
 429                      0);
 430
 431    brw_init_cache_id(cache,
 432                      "CLIP_PROG",
 433                      BRW_CLIP_PROG,
 434                      sizeof(struct brw_clip_prog_key),
 435                      sizeof(struct brw_clip_prog_data));
 436
 437    brw_init_cache_id(cache,
 438                      "GS_UNIT",
 439                      BRW_GS_UNIT,
 440                      sizeof(struct brw_gs_unit_state),
 441                      0);
 442
 443    brw_init_cache_id(cache,
 444                      "GS_PROG",
 445                      BRW_GS_PROG,
 446                      sizeof(struct brw_gs_prog_key),
 447                      sizeof(struct brw_gs_prog_data));
 448 }
 449
 450
 451 static void
 452 brw_init_surface_cache(struct brw_context *brw)
 453 {
 454    struct brw_cache *cache = &brw->surface_cache;
 455
 456    cache->brw = brw;
 457
 458    cache->size = 7;
 459    cache->n_items = 0;
 460    cache->items = (struct brw_cache_item **)
 461       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 462
 463    brw_init_cache_id(cache,
 464                      "SS_SURFACE",
 465                      BRW_SS_SURFACE,
 466                      sizeof(struct brw_surface_state),
 467                      0);
 468
 469    brw_init_cache_id(cache,
 470                      "SS_SURF_BIND",
 471                      BRW_SS_SURF_BIND,
 472                      0,
 473                      0);
 474 }
 475
 476
 477 void
 478 brw_init_caches(struct brw_context *brw)
 479 {
 480    brw_init_non_surface_cache(brw);
 481    brw_init_surface_cache(brw);
 482 }
 483
 484
 485 static void
 486 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 487 {
 488    struct brw_cache_item *c, *next;
 489    GLuint i;
 490
 491    if (INTEL_DEBUG & DEBUG_STATE)
 492       _mesa_printf("%s\n", __FUNCTION__);
 493
 494    for (i = 0; i < cache->size; i++) {
 495       for (c = cache->items[i]; c; c = next) {
 496          int j;
 497
 498          next = c->next;
 499          for (j = 0; j < c->nr_reloc_bufs; j++)
 500             dri_bo_unreference(c->reloc_bufs[j]);
 501          dri_bo_unreference(c->bo);
 502          free((void *)c->key);
 503          free(c);
 504       }
 505       cache->items[i] = NULL;
 506    }
 507
 508    cache->n_items = 0;
 509
 510    if (brw->curbe.last_buf) {
 511       _mesa_free(brw->curbe.last_buf);
 512       brw->curbe.last_buf = NULL;
 513    }
 514
 515    brw->state.dirty.mesa |= ~0;
 516    brw->state.dirty.brw |= ~0;
 517    brw->state.dirty.cache |= ~0;
 518 }
 519
 520 /* Clear all entries from the cache that point to the given bo.
 521  *
 522  * This lets us release memory for reuse earlier for known-dead buffers,
 523  * at the cost of walking the entire hash table.
 524  */
 525 void
 526 brw_state_cache_bo_delete(struct brw_cache *cache, dri_bo *bo)
 527 {
 528    struct brw_cache_item **prev;
 529    GLuint i;
 530
 531    if (INTEL_DEBUG & DEBUG_STATE)
 532       _mesa_printf("%s\n", __FUNCTION__);
 533
 534    for (i = 0; i < cache->size; i++) {
 535       for (prev = &cache->items[i]; *prev;) {
 536          struct brw_cache_item *c = *prev;
 537
 538          if (drm_intel_bo_references(c->bo, bo)) {
 539             int j;
 540
 541             *prev = c->next;
 542
 543             for (j = 0; j < c->nr_reloc_bufs; j++)
 544                dri_bo_unreference(c->reloc_bufs[j]);
 545             dri_bo_unreference(c->bo);
 546             free((void *)c->key);
 547             free(c);
 548             cache->n_items--;
 549          } else {
 550             prev = &c->next;
 551          }
 552       }
 553    }
 554 }
 555
 556 void
 557 brw_state_cache_check_size(struct brw_context *brw)
 558 {
 559    if (INTEL_DEBUG & DEBUG_STATE)
 560       _mesa_printf("%s (n_items=%d)\n", __FUNCTION__, brw->cache.n_items);
 561
 562    /* un-tuned guess.  We've got around 20 state objects for a total of around
 563     * 32k, so 1000 of them is around 1.5MB.
 564     */
 565    if (brw->cache.n_items > 1000)
 566       brw_clear_cache(brw, &brw->cache);
 567
 568    if (brw->surface_cache.n_items > 1000)
 569       brw_clear_cache(brw, &brw->surface_cache);
 570 }
 571
 572
 573 static void
 574 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 575 {
 576    GLuint i;
 577
 578    if (INTEL_DEBUG & DEBUG_STATE)
 579       _mesa_printf("%s\n", __FUNCTION__);
 580
 581    brw_clear_cache(brw, cache);
 582    for (i = 0; i < BRW_MAX_CACHE; i++) {
 583       dri_bo_unreference(cache->last_bo[i]);
 584       free(cache->name[i]);
 585    }
 586    free(cache->items);
 587    cache->items = NULL;
 588    cache->size = 0;
 589 }
 590
 591
 592 void
 593 brw_destroy_caches(struct brw_context *brw)
 594 {
 595    brw_destroy_cache(brw, &brw->cache);
 596    brw_destroy_cache(brw, &brw->surface_cache);
 597 }