src/gallium/drivers/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc_buf pointers need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58
  59 #include "brw_state.h"
  60 #include "brw_batchbuffer.h"
  61
  62 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  63  * structs:
  64  */
  65 #include "brw_wm.h"
  66 #include "brw_vs.h"
  67 #include "brw_clip.h"
  68 #include "brw_sf.h"
  69 #include "brw_gs.h"
  70
  71
  72 static GLuint
  73 hash_key(const void *key, GLuint key_size,
  74          struct brw_winsys_buffer **reloc_bufs, GLuint nr_reloc_bufs)
  75 {
  76    GLuint *ikey = (GLuint *)key;
  77    GLuint hash = 0, i;
  78
  79    assert(key_size % 4 == 0);
  80
  81    /* I'm sure this can be improved on:
  82     */
  83    for (i = 0; i < key_size/4; i++) {
  84       hash ^= ikey[i];
  85       hash = (hash << 5) | (hash >> 27);
  86    }
  87
  88    /* Include the BO pointers as key data as well */
  89    ikey = (GLuint *)reloc_bufs;
  90    key_size = nr_reloc_bufs * sizeof(struct brw_winsys_buffer *);
  91    for (i = 0; i < key_size/4; i++) {
  92       hash ^= ikey[i];
  93       hash = (hash << 5) | (hash >> 27);
  94    }
  95
  96    return hash;
  97 }
  98
  99
 100 /**
 101  * Marks a new buffer as being chosen for the given cache id.
 102  */
 103 static void
 104 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 105                   struct brw_winsys_buffer *bo)
 106 {
 107    if (bo == cache->last_bo[cache_id])
 108       return; /* no change */
 109
 110    brw->sws->bo_unreference(cache->last_bo[cache_id]);
 111    cache->last_bo[cache_id] = bo;
 112    brw->sws->bo_reference(cache->last_bo[cache_id]);
 113    cache->brw->state.dirty.cache |= 1 << cache_id;
 114 }
 115
 116
 117 static struct brw_cache_item *
 118 search_cache(struct brw_cache *cache, enum brw_cache_id cache_id,
 119              GLuint hash, const void *key, GLuint key_size,
 120              struct brw_winsys_buffer **reloc_bufs, GLuint nr_reloc_bufs)
 121 {
 122    struct brw_cache_item *c;
 123
 124 #if 0
 125    int bucketcount = 0;
 126
 127    for (c = cache->items[hash % cache->size]; c; c = c->next)
 128       bucketcount++;
 129
 130    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
 131            cache->size, bucketcount, cache->n_items);
 132 #endif
 133
 134    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 135       if (c->cache_id == cache_id &&
 136           c->hash == hash &&
 137           c->key_size == key_size &&
 138           memcmp(c->key, key, key_size) == 0 &&
 139           c->nr_reloc_bufs == nr_reloc_bufs &&
 140           memcmp(c->reloc_bufs, reloc_bufs,
 141                  nr_reloc_bufs * sizeof(struct brw_winsys_buffer *)) == 0)
 142          return c;
 143    }
 144
 145    return NULL;
 146 }
 147
 148
 149 static void
 150 rehash(struct brw_cache *cache)
 151 {
 152    struct brw_cache_item **items;
 153    struct brw_cache_item *c, *next;
 154    GLuint size, i;
 155
 156    size = cache->size * 3;
 157    items = (struct brw_cache_item**) _mesa_calloc(size * sizeof(*items));
 158
 159    for (i = 0; i < cache->size; i++)
 160       for (c = cache->items[i]; c; c = next) {
 161          next = c->next;
 162          c->next = items[c->hash % size];
 163          items[c->hash % size] = c;
 164       }
 165
 166    FREE(cache->items);
 167    cache->items = items;
 168    cache->size = size;
 169 }
 170
 171
 172 /**
 173  * Returns the buffer object matching cache_id and key, or NULL.
 174  */
 175 struct brw_winsys_buffer *
 176 brw_search_cache(struct brw_cache *cache,
 177                  enum brw_cache_id cache_id,
 178                  const void *key,
 179                  GLuint key_size,
 180                  struct brw_winsys_buffer **reloc_bufs, GLuint nr_reloc_bufs,
 181                  void *aux_return)
 182 {
 183    struct brw_cache_item *item;
 184    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 185
 186    item = search_cache(cache, cache_id, hash, key, key_size,
 187                        reloc_bufs, nr_reloc_bufs);
 188
 189    if (item == NULL)
 190       return NULL;
 191
 192    if (aux_return)
 193       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 194
 195    update_cache_last(cache, cache_id, item->bo);
 196
 197    brw->sws->bo_reference(item->bo);
 198    return item->bo;
 199 }
 200
 201
 202 struct brw_winsys_buffer *
 203 brw_upload_cache( struct brw_cache *cache,
 204                   enum brw_cache_id cache_id,
 205                   const void *key,
 206                   GLuint key_size,
 207                   struct brw_winsys_buffer **reloc_bufs,
 208                   GLuint nr_reloc_bufs,
 209                   const void *data,
 210                   GLuint data_size,
 211                   const void *aux,
 212                   void *aux_return )
 213 {
 214    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 215    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 216    GLuint relocs_size = nr_reloc_bufs * sizeof(struct brw_winsys_buffer *);
 217    GLuint aux_size = cache->aux_size[cache_id];
 218    void *tmp;
 219    struct brw_winsys_buffer *bo;
 220    int i;
 221
 222    /* Create the buffer object to contain the data */
 223    bo = dri_bo_alloc(cache->brw->intel.bufmgr,
 224                      cache->name[cache_id], data_size, 1 << 6);
 225
 226
 227    /* Set up the memory containing the key, aux_data, and reloc_bufs */
 228    tmp = _mesa_malloc(key_size + aux_size + relocs_size);
 229
 230    memcpy(tmp, key, key_size);
 231    memcpy(tmp + key_size, aux, cache->aux_size[cache_id]);
 232    memcpy(tmp + key_size + aux_size, reloc_bufs, relocs_size);
 233    for (i = 0; i < nr_reloc_bufs; i++) {
 234       if (reloc_bufs[i] != NULL)
 235          brw->sws->bo_reference(reloc_bufs[i]);
 236    }
 237
 238    item->cache_id = cache_id;
 239    item->key = tmp;
 240    item->hash = hash;
 241    item->key_size = key_size;
 242    item->reloc_bufs = tmp + key_size + aux_size;
 243    item->nr_reloc_bufs = nr_reloc_bufs;
 244
 245    item->bo = bo;
 246    brw->sws->bo_reference(bo);
 247    item->data_size = data_size;
 248
 249    if (cache->n_items > cache->size * 1.5)
 250       rehash(cache);
 251
 252    hash %= cache->size;
 253    item->next = cache->items[hash];
 254    cache->items[hash] = item;
 255    cache->n_items++;
 256
 257    if (aux_return) {
 258       assert(cache->aux_size[cache_id]);
 259       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 260    }
 261
 262    if (INTEL_DEBUG & DEBUG_STATE)
 263       _mesa_printf("upload %s: %d bytes to cache id %d\n",
 264                    cache->name[cache_id],
 265                    data_size, cache_id);
 266
 267    /* Copy data to the buffer */
 268    dri_bo_subdata(bo, 0, data_size, data);
 269
 270    update_cache_last(cache, cache_id, bo);
 271
 272    return bo;
 273 }
 274
 275
 276 /**
 277  * This doesn't really work with aux data.  Use search/upload instead
 278  */
 279 struct brw_winsys_buffer *
 280 brw_cache_data_sz(struct brw_cache *cache,
 281                   enum brw_cache_id cache_id,
 282                   const void *data,
 283                   GLuint data_size,
 284                   struct brw_winsys_buffer **reloc_bufs,
 285                   GLuint nr_reloc_bufs)
 286 {
 287    struct brw_winsys_buffer *bo;
 288    struct brw_cache_item *item;
 289    GLuint hash = hash_key(data, data_size, reloc_bufs, nr_reloc_bufs);
 290
 291    item = search_cache(cache, cache_id, hash, data, data_size,
 292                        reloc_bufs, nr_reloc_bufs);
 293    if (item) {
 294       update_cache_last(cache, cache_id, item->bo);
 295       brw->sws->bo_reference(item->bo);
 296       return item->bo;
 297    }
 298
 299    bo = brw_upload_cache(cache, cache_id,
 300                          data, data_size,
 301                          reloc_bufs, nr_reloc_bufs,
 302                          data, data_size,
 303                          NULL, NULL);
 304
 305    return bo;
 306 }
 307
 308
 309 /**
 310  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 311  *
 312  * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
 313  * better to use, as the potentially changing offsets in the data-used-as-key
 314  * will result in excessive cache misses.
 315  */
 316 struct brw_winsys_buffer *
 317 brw_cache_data(struct brw_cache *cache,
 318                enum brw_cache_id cache_id,
 319                const void *data,
 320                struct brw_winsys_buffer **reloc_bufs,
 321                GLuint nr_reloc_bufs)
 322 {
 323    return brw_cache_data_sz(cache, cache_id, data, cache->key_size[cache_id],
 324                             reloc_bufs, nr_reloc_bufs);
 325 }
 326
 327 enum pool_type {
 328    DW_SURFACE_STATE,
 329    DW_GENERAL_STATE
 330 };
 331
 332
 333 static void
 334 brw_init_cache_id(struct brw_cache *cache,
 335                   const char *name,
 336                   enum brw_cache_id id,
 337                   GLuint key_size,
 338                   GLuint aux_size)
 339 {
 340    cache->name[id] = strdup(name);
 341    cache->key_size[id] = key_size;
 342    cache->aux_size[id] = aux_size;
 343 }
 344
 345
 346 static void
 347 brw_init_non_surface_cache(struct brw_context *brw)
 348 {
 349    struct brw_cache *cache = &brw->cache;
 350
 351    cache->brw = brw;
 352
 353    cache->size = 7;
 354    cache->n_items = 0;
 355    cache->items = (struct brw_cache_item **)
 356       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 357
 358    brw_init_cache_id(cache,
 359                      "CC_VP",
 360                      BRW_CC_VP,
 361                      sizeof(struct brw_cc_viewport),
 362                      0);
 363
 364    brw_init_cache_id(cache,
 365                      "CC_UNIT",
 366                      BRW_CC_UNIT,
 367                      sizeof(struct brw_cc_unit_state),
 368                      0);
 369
 370    brw_init_cache_id(cache,
 371                      "WM_PROG",
 372                      BRW_WM_PROG,
 373                      sizeof(struct brw_wm_prog_key),
 374                      sizeof(struct brw_wm_prog_data));
 375
 376    brw_init_cache_id(cache,
 377                      "SAMPLER_DEFAULT_COLOR",
 378                      BRW_SAMPLER_DEFAULT_COLOR,
 379                      sizeof(struct brw_sampler_default_color),
 380                      0);
 381
 382    brw_init_cache_id(cache,
 383                      "SAMPLER",
 384                      BRW_SAMPLER,
 385                      0,         /* variable key/data size */
 386                      0);
 387
 388    brw_init_cache_id(cache,
 389                      "WM_UNIT",
 390                      BRW_WM_UNIT,
 391                      sizeof(struct brw_wm_unit_state),
 392                      0);
 393
 394    brw_init_cache_id(cache,
 395                      "SF_PROG",
 396                      BRW_SF_PROG,
 397                      sizeof(struct brw_sf_prog_key),
 398                      sizeof(struct brw_sf_prog_data));
 399
 400    brw_init_cache_id(cache,
 401                      "SF_VP",
 402                      BRW_SF_VP,
 403                      sizeof(struct brw_sf_viewport),
 404                      0);
 405
 406    brw_init_cache_id(cache,
 407                      "SF_UNIT",
 408                      BRW_SF_UNIT,
 409                      sizeof(struct brw_sf_unit_state),
 410                      0);
 411
 412    brw_init_cache_id(cache,
 413                      "VS_UNIT",
 414                      BRW_VS_UNIT,
 415                      sizeof(struct brw_vs_unit_state),
 416                      0);
 417
 418    brw_init_cache_id(cache,
 419                      "VS_PROG",
 420                      BRW_VS_PROG,
 421                      sizeof(struct brw_vs_prog_key),
 422                      sizeof(struct brw_vs_prog_data));
 423
 424    brw_init_cache_id(cache,
 425                      "CLIP_UNIT",
 426                      BRW_CLIP_UNIT,
 427                      sizeof(struct brw_clip_unit_state),
 428                      0);
 429
 430    brw_init_cache_id(cache,
 431                      "CLIP_PROG",
 432                      BRW_CLIP_PROG,
 433                      sizeof(struct brw_clip_prog_key),
 434                      sizeof(struct brw_clip_prog_data));
 435
 436    brw_init_cache_id(cache,
 437                      "GS_UNIT",
 438                      BRW_GS_UNIT,
 439                      sizeof(struct brw_gs_unit_state),
 440                      0);
 441
 442    brw_init_cache_id(cache,
 443                      "GS_PROG",
 444                      BRW_GS_PROG,
 445                      sizeof(struct brw_gs_prog_key),
 446                      sizeof(struct brw_gs_prog_data));
 447 }
 448
 449
 450 static void
 451 brw_init_surface_cache(struct brw_context *brw)
 452 {
 453    struct brw_cache *cache = &brw->surface_cache;
 454
 455    cache->brw = brw;
 456
 457    cache->size = 7;
 458    cache->n_items = 0;
 459    cache->items = (struct brw_cache_item **)
 460       _mesa_calloc(cache->size * sizeof(struct brw_cache_item));
 461
 462    brw_init_cache_id(cache,
 463                      "SS_SURFACE",
 464                      BRW_SS_SURFACE,
 465                      sizeof(struct brw_surface_state),
 466                      0);
 467
 468    brw_init_cache_id(cache,
 469                      "SS_SURF_BIND",
 470                      BRW_SS_SURF_BIND,
 471                      0,
 472                      0);
 473 }
 474
 475
 476 void
 477 brw_init_caches(struct brw_context *brw)
 478 {
 479    brw_init_non_surface_cache(brw);
 480    brw_init_surface_cache(brw);
 481 }
 482
 483
 484 static void
 485 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 486 {
 487    struct brw_cache_item *c, *next;
 488    GLuint i;
 489
 490    if (INTEL_DEBUG & DEBUG_STATE)
 491       _mesa_printf("%s\n", __FUNCTION__);
 492
 493    for (i = 0; i < cache->size; i++) {
 494       for (c = cache->items[i]; c; c = next) {
 495          int j;
 496
 497          next = c->next;
 498          for (j = 0; j < c->nr_reloc_bufs; j++)
 499             brw->sws->bo_unreference(c->reloc_bufs[j]);
 500          brw->sws->bo_unreference(c->bo);
 501          free((void *)c->key);
 502          free(c);
 503       }
 504       cache->items[i] = NULL;
 505    }
 506
 507    cache->n_items = 0;
 508
 509    if (brw->curbe.last_buf) {
 510       _mesa_free(brw->curbe.last_buf);
 511       brw->curbe.last_buf = NULL;
 512    }
 513
 514    brw->state.dirty.mesa |= ~0;
 515    brw->state.dirty.brw |= ~0;
 516    brw->state.dirty.cache |= ~0;
 517 }
 518
 519 /* Clear all entries from the cache that point to the given bo.
 520  *
 521  * This lets us release memory for reuse earlier for known-dead buffers,
 522  * at the cost of walking the entire hash table.
 523  */
 524 void
 525 brw_state_cache_bo_delete(struct brw_cache *cache, struct brw_winsys_buffer *bo)
 526 {
 527    struct brw_cache_item **prev;
 528    GLuint i;
 529
 530    if (INTEL_DEBUG & DEBUG_STATE)
 531       _mesa_printf("%s\n", __FUNCTION__);
 532
 533    for (i = 0; i < cache->size; i++) {
 534       for (prev = &cache->items[i]; *prev;) {
 535          struct brw_cache_item *c = *prev;
 536
 537          if (cache->sws->bo_references(c->bo, bo)) {
 538             int j;
 539
 540             *prev = c->next;
 541
 542             for (j = 0; j < c->nr_reloc_bufs; j++)
 543                brw->sws->bo_unreference(c->reloc_bufs[j]);
 544             brw->sws->bo_unreference(c->bo);
 545             free((void *)c->key);
 546             free(c);
 547             cache->n_items--;
 548          } else {
 549             prev = &c->next;
 550          }
 551       }
 552    }
 553 }
 554
 555 void
 556 brw_state_cache_check_size(struct brw_context *brw)
 557 {
 558    if (INTEL_DEBUG & DEBUG_STATE)
 559       _mesa_printf("%s (n_items=%d)\n", __FUNCTION__, brw->cache.n_items);
 560
 561    /* un-tuned guess.  We've got around 20 state objects for a total of around
 562     * 32k, so 1000 of them is around 1.5MB.
 563     */
 564    if (brw->cache.n_items > 1000)
 565       brw_clear_cache(brw, &brw->cache);
 566
 567    if (brw->surface_cache.n_items > 1000)
 568       brw_clear_cache(brw, &brw->surface_cache);
 569 }
 570
 571
 572 static void
 573 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 574 {
 575    GLuint i;
 576
 577    if (INTEL_DEBUG & DEBUG_STATE)
 578       _mesa_printf("%s\n", __FUNCTION__);
 579
 580    brw_clear_cache(brw, cache);
 581    for (i = 0; i < BRW_MAX_CACHE; i++) {
 582       brw->sws->bo_unreference(cache->last_bo[i]);
 583       free(cache->name[i]);
 584    }
 585    free(cache->items);
 586    cache->items = NULL;
 587    cache->size = 0;
 588 }
 589
 590
 591 void
 592 brw_destroy_caches(struct brw_context *brw)
 593 {
 594    brw_destroy_cache(brw, &brw->cache);
 595    brw_destroy_cache(brw, &brw->surface_cache);
 596 }