src/gallium/drivers/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc structs need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58 #include "util/u_memory.h"
  59
  60 #include "brw_debug.h"
  61 #include "brw_state.h"
  62
  63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  64  * structs:
  65  */
  66 #include "brw_wm.h"
  67 #include "brw_vs.h"
  68 #include "brw_clip.h"
  69 #include "brw_sf.h"
  70 #include "brw_gs.h"
  71
  72
  73 static GLuint
  74 hash_key(const void *key, GLuint key_size,
  75          struct brw_winsys_reloc *relocs, GLuint nr_relocs)
  76 {
  77    GLuint *ikey = (GLuint *)key;
  78    GLuint hash = 0, i;
  79
  80    assert(key_size % 4 == 0);
  81
  82    /* I'm sure this can be improved on:
  83     */
  84    for (i = 0; i < key_size/4; i++) {
  85       hash ^= ikey[i];
  86       hash = (hash << 5) | (hash >> 27);
  87    }
  88
  89    /* Include the BO pointers as key data as well */
  90    ikey = (GLuint *)relocs;
  91    key_size = nr_relocs * sizeof(struct brw_winsys_reloc);
  92    for (i = 0; i < key_size/4; i++) {
  93       hash ^= ikey[i];
  94       hash = (hash << 5) | (hash >> 27);
  95    }
  96
  97    return hash;
  98 }
  99
 100
 101 /**
 102  * Marks a new buffer as being chosen for the given cache id.
 103  */
 104 static void
 105 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 106                   struct brw_winsys_buffer *bo)
 107 {
 108    if (bo == cache->last_bo[cache_id])
 109       return; /* no change */
 110
 111    bo_reference( &cache->last_bo[cache_id],  bo );
 112
 113    cache->brw->state.dirty.cache |= 1 << cache_id;
 114 }
 115
 116
 117 static struct brw_cache_item *
 118 search_cache(struct brw_cache *cache, enum brw_cache_id cache_id,
 119              GLuint hash, const void *key, GLuint key_size,
 120              struct brw_winsys_reloc *relocs, GLuint nr_relocs)
 121 {
 122    struct brw_cache_item *c;
 123
 124 #if 0
 125    int bucketcount = 0;
 126
 127    for (c = cache->items[hash % cache->size]; c; c = c->next)
 128       bucketcount++;
 129
 130    debug_printf("bucket %d/%d = %d/%d items\n", hash % cache->size,
 131            cache->size, bucketcount, cache->n_items);
 132 #endif
 133
 134    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 135       if (c->cache_id == cache_id &&
 136           c->hash == hash &&
 137           c->key_size == key_size &&
 138           memcmp(c->key, key, key_size) == 0 &&
 139           c->nr_relocs == nr_relocs &&
 140           memcmp(c->relocs, relocs, nr_relocs * sizeof *relocs) == 0)
 141          return c;
 142    }
 143
 144    return NULL;
 145 }
 146
 147
 148 static void
 149 rehash(struct brw_cache *cache)
 150 {
 151    struct brw_cache_item **items;
 152    struct brw_cache_item *c, *next;
 153    GLuint size, i;
 154
 155    size = cache->size * 3;
 156    items = (struct brw_cache_item**) CALLOC(size, sizeof(*items));
 157
 158    for (i = 0; i < cache->size; i++)
 159       for (c = cache->items[i]; c; c = next) {
 160          next = c->next;
 161          c->next = items[c->hash % size];
 162          items[c->hash % size] = c;
 163       }
 164
 165    FREE(cache->items);
 166    cache->items = items;
 167    cache->size = size;
 168 }
 169
 170
 171 /**
 172  * Returns the buffer object matching cache_id and key, or NULL.
 173  */
 174 boolean
 175 brw_search_cache(struct brw_cache *cache,
 176                  enum brw_cache_id cache_id,
 177                  const void *key,
 178                  GLuint key_size,
 179                  struct brw_winsys_reloc *relocs,
 180                  GLuint nr_relocs,
 181                  void *aux_return,
 182                  struct brw_winsys_buffer **bo_out)
 183 {
 184    struct brw_cache_item *item;
 185    GLuint hash = hash_key(key, key_size, relocs, nr_relocs);
 186
 187    item = search_cache(cache, cache_id, hash, key, key_size,
 188                        relocs, nr_relocs);
 189
 190    if (item) {
 191       if (aux_return)
 192          *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 193
 194       update_cache_last(cache, cache_id, item->bo);
 195       bo_reference(bo_out, item->bo);
 196       return TRUE;
 197    }
 198
 199    return FALSE;
 200 }
 201
 202
 203 enum pipe_error
 204 brw_upload_cache( struct brw_cache *cache,
 205                   enum brw_cache_id cache_id,
 206                   const void *key,
 207                   GLuint key_size,
 208                   struct brw_winsys_reloc *relocs,
 209                   GLuint nr_relocs,
 210                   const void *data,
 211                   GLuint data_size,
 212                   const void *aux,
 213                   void *aux_return,
 214                   struct brw_winsys_buffer **bo_out)
 215 {
 216    struct brw_cache_item *item;
 217    GLuint hash = hash_key(key, key_size, relocs, nr_relocs);
 218    GLuint relocs_size = nr_relocs * sizeof relocs[0];
 219    GLuint aux_size = cache->aux_size[cache_id];
 220    enum pipe_error ret;
 221    void *tmp;
 222    int i;
 223
 224    /* Create the buffer object to contain the data.  For now, use a
 225     * single buffer type to describe all cached state atoms.  Later,
 226     * may want to take advantage of hardware distinctions between
 227     * these various entities.
 228     */
 229    ret = cache->sws->bo_alloc(cache->sws,
 230                               cache->buffer_type,
 231                               data_size, 1 << 6,
 232                               bo_out);
 233    if (ret)
 234       return ret;
 235
 236    item = CALLOC_STRUCT(brw_cache_item);
 237
 238    /* Set up the memory containing the key, aux_data, and relocs */
 239    tmp = MALLOC(key_size + aux_size + relocs_size);
 240
 241    memcpy(tmp, key, key_size);
 242    memcpy((char *)tmp + key_size, aux, cache->aux_size[cache_id]);
 243    memcpy((char *)tmp + key_size + aux_size, relocs, relocs_size);
 244    for (i = 0; i < nr_relocs; i++) {
 245       p_atomic_inc(&relocs[i].bo->reference.count);
 246    }
 247
 248    item->cache_id = cache_id;
 249    item->key = tmp;
 250    item->hash = hash;
 251    item->key_size = key_size;
 252    item->relocs = (struct brw_winsys_reloc *)((char *)tmp + key_size + aux_size);
 253    item->nr_relocs = nr_relocs;
 254    bo_reference( &item->bo, *bo_out );
 255    item->data_size = data_size;
 256
 257    if (cache->n_items > cache->size * 1.5)
 258       rehash(cache);
 259
 260    hash %= cache->size;
 261    item->next = cache->items[hash];
 262    cache->items[hash] = item;
 263    cache->n_items++;
 264
 265    if (aux_return) {
 266       assert(cache->aux_size[cache_id]);
 267       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 268    }
 269
 270    if (BRW_DEBUG & DEBUG_STATE)
 271       debug_printf("upload %s: %d bytes to cache id %d\n",
 272                    cache->name[cache_id],
 273                    data_size, cache_id);
 274
 275    /* Copy data to the buffer */
 276    ret = cache->sws->bo_subdata(item->bo,
 277                                 cache_id,
 278                                 0, data_size, data,
 279                                 relocs, nr_relocs);
 280    if (ret)
 281       return ret;
 282
 283    update_cache_last(cache, cache_id, item->bo);
 284
 285    return PIPE_OK;
 286 }
 287
 288
 289 /**
 290  * This doesn't really work with aux data.  Use search/upload instead
 291  */
 292 enum pipe_error
 293 brw_cache_data_sz(struct brw_cache *cache,
 294                   enum brw_cache_id cache_id,
 295                   const void *data,
 296                   GLuint data_size,
 297                   struct brw_winsys_reloc *relocs,
 298                   GLuint nr_relocs,
 299                   struct brw_winsys_buffer **bo_out)
 300 {
 301    struct brw_cache_item *item;
 302    GLuint hash = hash_key(data, data_size, relocs, nr_relocs);
 303
 304    item = search_cache(cache, cache_id, hash, data, data_size,
 305                        relocs, nr_relocs);
 306    if (item) {
 307       update_cache_last(cache, cache_id, item->bo);
 308
 309       bo_reference(bo_out, item->bo);
 310       return PIPE_OK;
 311    }
 312
 313    return brw_upload_cache(cache, cache_id,
 314                            data, data_size,
 315                            relocs, nr_relocs,
 316                            data, data_size,
 317                            NULL, NULL,
 318                            bo_out);
 319 }
 320
 321
 322 /**
 323  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 324  *
 325  * If nr_relocs is nonzero, brw_search_cache()/brw_upload_cache() would be
 326  * better to use, as the potentially changing offsets in the data-used-as-key
 327  * will result in excessive cache misses.
 328  *
 329  * XXX: above is no longer true -- can we remove some code?
 330  */
 331 enum pipe_error
 332 brw_cache_data(struct brw_cache *cache,
 333                enum brw_cache_id cache_id,
 334                const void *data,
 335                struct brw_winsys_reloc *relocs,
 336                GLuint nr_relocs,
 337                struct brw_winsys_buffer **bo_out)
 338 {
 339    return brw_cache_data_sz(cache, cache_id, data, cache->key_size[cache_id],
 340                             relocs, nr_relocs, bo_out);
 341 }
 342
 343
 344 static void
 345 brw_init_cache_id(struct brw_cache *cache,
 346                   const char *name,
 347                   enum brw_cache_id id,
 348                   GLuint key_size,
 349                   GLuint aux_size)
 350 {
 351    cache->name[id] = strdup(name);
 352    cache->key_size[id] = key_size;
 353    cache->aux_size[id] = aux_size;
 354 }
 355
 356
 357 static void
 358 brw_init_general_state_cache(struct brw_context *brw)
 359 {
 360    struct brw_cache *cache = &brw->cache;
 361
 362    cache->brw = brw;
 363    cache->sws = brw->sws;
 364
 365    cache->buffer_type = BRW_BUFFER_TYPE_GENERAL_STATE;
 366
 367    cache->size = 7;
 368    cache->n_items = 0;
 369    cache->items = (struct brw_cache_item **)
 370       CALLOC(cache->size, sizeof(struct brw_cache_item));
 371
 372    brw_init_cache_id(cache,
 373                      "CC_VP",
 374                      BRW_CC_VP,
 375                      sizeof(struct brw_cc_viewport),
 376                      0);
 377
 378    brw_init_cache_id(cache,
 379                      "CC_UNIT",
 380                      BRW_CC_UNIT,
 381                      sizeof(struct brw_cc_unit_state),
 382                      0);
 383
 384    brw_init_cache_id(cache,
 385                      "WM_PROG",
 386                      BRW_WM_PROG,
 387                      sizeof(struct brw_wm_prog_key),
 388                      sizeof(struct brw_wm_prog_data));
 389
 390    brw_init_cache_id(cache,
 391                      "SAMPLER_DEFAULT_COLOR",
 392                      BRW_SAMPLER_DEFAULT_COLOR,
 393                      sizeof(struct brw_sampler_default_color),
 394                      0);
 395
 396    brw_init_cache_id(cache,
 397                      "SAMPLER",
 398                      BRW_SAMPLER,
 399                      0,         /* variable key/data size */
 400                      0);
 401
 402    brw_init_cache_id(cache,
 403                      "WM_UNIT",
 404                      BRW_WM_UNIT,
 405                      sizeof(struct brw_wm_unit_state),
 406                      0);
 407
 408    brw_init_cache_id(cache,
 409                      "SF_PROG",
 410                      BRW_SF_PROG,
 411                      sizeof(struct brw_sf_prog_key),
 412                      sizeof(struct brw_sf_prog_data));
 413
 414    brw_init_cache_id(cache,
 415                      "SF_VP",
 416                      BRW_SF_VP,
 417                      sizeof(struct brw_sf_viewport),
 418                      0);
 419
 420    brw_init_cache_id(cache,
 421                      "SF_UNIT",
 422                      BRW_SF_UNIT,
 423                      sizeof(struct brw_sf_unit_state),
 424                      0);
 425
 426    brw_init_cache_id(cache,
 427                      "VS_UNIT",
 428                      BRW_VS_UNIT,
 429                      sizeof(struct brw_vs_unit_state),
 430                      0);
 431
 432    brw_init_cache_id(cache,
 433                      "VS_PROG",
 434                      BRW_VS_PROG,
 435                      sizeof(struct brw_vs_prog_key),
 436                      sizeof(struct brw_vs_prog_data));
 437
 438    brw_init_cache_id(cache,
 439                      "CLIP_UNIT",
 440                      BRW_CLIP_UNIT,
 441                      sizeof(struct brw_clip_unit_state),
 442                      0);
 443
 444    brw_init_cache_id(cache,
 445                      "CLIP_PROG",
 446                      BRW_CLIP_PROG,
 447                      sizeof(struct brw_clip_prog_key),
 448                      sizeof(struct brw_clip_prog_data));
 449
 450    brw_init_cache_id(cache,
 451                      "GS_UNIT",
 452                      BRW_GS_UNIT,
 453                      sizeof(struct brw_gs_unit_state),
 454                      0);
 455
 456    brw_init_cache_id(cache,
 457                      "GS_PROG",
 458                      BRW_GS_PROG,
 459                      sizeof(struct brw_gs_prog_key),
 460                      sizeof(struct brw_gs_prog_data));
 461 }
 462
 463
 464 static void
 465 brw_init_surface_state_cache(struct brw_context *brw)
 466 {
 467    struct brw_cache *cache = &brw->surface_cache;
 468
 469    cache->brw = brw;
 470    cache->sws = brw->sws;
 471
 472    cache->buffer_type = BRW_BUFFER_TYPE_SURFACE_STATE;
 473
 474    cache->size = 7;
 475    cache->n_items = 0;
 476    cache->items = (struct brw_cache_item **)
 477       CALLOC(cache->size, sizeof(struct brw_cache_item));
 478
 479    brw_init_cache_id(cache,
 480                      "SS_SURFACE",
 481                      BRW_SS_SURFACE,
 482                      sizeof(struct brw_surface_state),
 483                      0);
 484
 485    brw_init_cache_id(cache,
 486                      "SS_SURF_BIND",
 487                      BRW_SS_SURF_BIND,
 488                      0,
 489                      0);
 490 }
 491
 492
 493 void
 494 brw_init_caches(struct brw_context *brw)
 495 {
 496    brw_init_general_state_cache(brw);
 497    brw_init_surface_state_cache(brw);
 498 }
 499
 500
 501 static void
 502 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 503 {
 504    struct brw_cache_item *c, *next;
 505    GLuint i;
 506
 507    if (BRW_DEBUG & DEBUG_STATE)
 508       debug_printf("%s\n", __FUNCTION__);
 509
 510    for (i = 0; i < cache->size; i++) {
 511       for (c = cache->items[i]; c; c = next) {
 512          int j;
 513
 514          next = c->next;
 515
 516          for (j = 0; j < c->nr_relocs; j++)
 517             bo_reference(&c->relocs[j].bo, NULL);
 518
 519          bo_reference(&c->bo, NULL);
 520          FREE((void *)c->key);
 521          FREE(c);
 522       }
 523       cache->items[i] = NULL;
 524    }
 525
 526    cache->n_items = 0;
 527
 528    if (brw->curbe.last_buf) {
 529       FREE(brw->curbe.last_buf);
 530       brw->curbe.last_buf = NULL;
 531    }
 532
 533    brw->state.dirty.mesa |= ~0;
 534    brw->state.dirty.brw |= ~0;
 535    brw->state.dirty.cache |= ~0;
 536 }
 537
 538 /* Clear all entries from the cache that point to the given bo.
 539  *
 540  * This lets us release memory for reuse earlier for known-dead buffers,
 541  * at the cost of walking the entire hash table.
 542  */
 543 void
 544 brw_state_cache_bo_delete(struct brw_cache *cache, struct brw_winsys_buffer *bo)
 545 {
 546    struct brw_cache_item **prev;
 547    GLuint i;
 548
 549    if (BRW_DEBUG & DEBUG_STATE)
 550       debug_printf("%s\n", __FUNCTION__);
 551
 552    for (i = 0; i < cache->size; i++) {
 553       for (prev = &cache->items[i]; *prev;) {
 554          struct brw_cache_item *c = *prev;
 555
 556          if (cache->sws->bo_references(c->bo, bo)) {
 557             int j;
 558
 559             *prev = c->next;
 560
 561             for (j = 0; j < c->nr_relocs; j++)
 562                bo_reference(&c->relocs[j].bo, NULL);
 563
 564             bo_reference(&c->bo, NULL);
 565
 566             FREE((void *)c->key);
 567             FREE(c);
 568             cache->n_items--;
 569          } else {
 570             prev = &c->next;
 571          }
 572       }
 573    }
 574 }
 575
 576 void
 577 brw_state_cache_check_size(struct brw_context *brw)
 578 {
 579    if (BRW_DEBUG & DEBUG_STATE)
 580       debug_printf("%s (n_items=%d)\n", __FUNCTION__, brw->cache.n_items);
 581
 582    /* un-tuned guess.  We've got around 20 state objects for a total of around
 583     * 32k, so 1000 of them is around 1.5MB.
 584     */
 585    if (brw->cache.n_items > 1000)
 586       brw_clear_cache(brw, &brw->cache);
 587
 588    if (brw->surface_cache.n_items > 1000)
 589       brw_clear_cache(brw, &brw->surface_cache);
 590 }
 591
 592
 593 static void
 594 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 595 {
 596    GLuint i;
 597
 598    if (BRW_DEBUG & DEBUG_STATE)
 599       debug_printf("%s\n", __FUNCTION__);
 600
 601    brw_clear_cache(brw, cache);
 602    for (i = 0; i < BRW_MAX_CACHE; i++) {
 603       bo_reference(&cache->last_bo[i], NULL);
 604       FREE(cache->name[i]);
 605    }
 606    FREE(cache->items);
 607    cache->items = NULL;
 608    cache->size = 0;
 609 }
 610
 611
 612 void
 613 brw_destroy_caches(struct brw_context *brw)
 614 {
 615    brw_destroy_cache(brw, &brw->cache);
 616    brw_destroy_cache(brw, &brw->surface_cache);
 617 }