src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keith@tungstengraphics.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The consumers
  35  * can query the hash table of state using a cache_id, opaque key data,
  36  * and list of buffers that will be used in relocations, and receive the
  37  * corresponding state buffer object of state (plus associated auxiliary
  38  * data) in return.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the key data.
  41  * The cache_id and relocation target buffers associated with the state
  42  * buffer are included as auxiliary key data, but are not part of the hash
  43  * value (this should be fixed, but will likely be fixed instead by making
  44  * consumers use structured keys).
  45  *
  46  * Replacement is not implemented.  Instead, when the cache gets too big, at
  47  * a safe point (unlock) we throw out all of the cache data and let it
  48  * regenerate for the next rendering operation.
  49  *
  50  * The reloc_buf pointers need to be included as key data, otherwise the
  51  * non-unique values stuffed in the offset in key data through
  52  * brw_cache_data() may result in successful probe for state buffers
  53  * even when the buffer being referenced doesn't match.  The result would be
  54  * that the same state cache entry is used twice for different buffers,
  55  * only one of the two buffers referenced gets put into the offset, and the
  56  * incorrect program is run for the other instance.
  57  */
  58
  59 #include "brw_state.h"
  60 #include "intel_batchbuffer.h"
  61 #include "main/imports.h"
  62
  63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
  64  * structs:
  65  */
  66 #include "brw_wm.h"
  67 #include "brw_vs.h"
  68 #include "brw_clip.h"
  69 #include "brw_sf.h"
  70 #include "brw_gs.h"
  71
  72 static GLuint hash_key( const void *key, GLuint key_size,
  73                         dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
  74 {
  75    GLuint *ikey = (GLuint *)key;
  76    GLuint hash = 0, i;
  77
  78    assert(key_size % 4 == 0);
  79
  80    /* I'm sure this can be improved on:
  81     */
  82    for (i = 0; i < key_size/4; i++) {
  83       hash ^= ikey[i];
  84       hash = (hash << 5) | (hash >> 27);
  85    }
  86
  87    /* Include the BO pointers as key data as well */
  88    ikey = (GLuint *)reloc_bufs;
  89    key_size = nr_reloc_bufs * sizeof(dri_bo *);
  90    for (i = 0; i < key_size/4; i++) {
  91       hash ^= ikey[i];
  92       hash = (hash << 5) | (hash >> 27);
  93    }
  94
  95    return hash;
  96 }
  97
  98 /**
  99  * Marks a new buffer as being chosen for the given cache id.
 100  */
 101 static void
 102 update_cache_last(struct brw_cache *cache, enum brw_cache_id cache_id,
 103                   dri_bo *bo)
 104 {
 105    if (bo == cache->last_bo[cache_id])
 106       return; /* no change */
 107
 108    dri_bo_unreference(cache->last_bo[cache_id]);
 109    cache->last_bo[cache_id] = bo;
 110    dri_bo_reference(cache->last_bo[cache_id]);
 111    cache->brw->state.dirty.cache |= 1 << cache_id;
 112 }
 113
 114 static struct brw_cache_item *
 115 search_cache(struct brw_cache *cache, enum brw_cache_id cache_id,
 116              GLuint hash, const void *key, GLuint key_size,
 117              dri_bo **reloc_bufs, GLuint nr_reloc_bufs)
 118 {
 119    struct brw_cache_item *c;
 120
 121 #if 0
 122    int bucketcount = 0;
 123
 124    for (c = cache->items[hash % cache->size]; c; c = c->next)
 125       bucketcount++;
 126
 127    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
 128            cache->size, bucketcount, cache->n_items);
 129 #endif
 130
 131    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 132       if (c->cache_id == cache_id &&
 133           c->hash == hash &&
 134           c->key_size == key_size &&
 135           memcmp(c->key, key, key_size) == 0 &&
 136           c->nr_reloc_bufs == nr_reloc_bufs &&
 137           memcmp(c->reloc_bufs, reloc_bufs,
 138                  nr_reloc_bufs * sizeof(dri_bo *)) == 0)
 139          return c;
 140    }
 141
 142    return NULL;
 143 }
 144
 145
 146 static void rehash( struct brw_cache *cache )
 147 {
 148    struct brw_cache_item **items;
 149    struct brw_cache_item *c, *next;
 150    GLuint size, i;
 151
 152    size = cache->size * 3;
 153    items = (struct brw_cache_item**) _mesa_calloc(size * sizeof(*items));
 154
 155    for (i = 0; i < cache->size; i++)
 156       for (c = cache->items[i]; c; c = next) {
 157          next = c->next;
 158          c->next = items[c->hash % size];
 159          items[c->hash % size] = c;
 160       }
 161
 162    FREE(cache->items);
 163    cache->items = items;
 164    cache->size = size;
 165 }
 166
 167 /**
 168  * Returns the buffer object matching cache_id and key, or NULL.
 169  */
 170 dri_bo *brw_search_cache( struct brw_cache *cache,
 171                           enum brw_cache_id cache_id,
 172                           const void *key,
 173                           GLuint key_size,
 174                           dri_bo **reloc_bufs, GLuint nr_reloc_bufs,
 175                           void *aux_return )
 176 {
 177    struct brw_cache_item *item;
 178    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 179
 180    item = search_cache(cache, cache_id, hash, key, key_size,
 181                        reloc_bufs, nr_reloc_bufs);
 182
 183    if (item == NULL)
 184       return NULL;
 185
 186    if (aux_return)
 187       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 188
 189    update_cache_last(cache, cache_id, item->bo);
 190
 191    dri_bo_reference(item->bo);
 192    return item->bo;
 193 }
 194
 195 dri_bo *
 196 brw_upload_cache( struct brw_cache *cache,
 197                   enum brw_cache_id cache_id,
 198                   const void *key,
 199                   GLuint key_size,
 200                   dri_bo **reloc_bufs,
 201                   GLuint nr_reloc_bufs,
 202                   const void *data,
 203                   GLuint data_size,
 204                   const void *aux,
 205                   void *aux_return )
 206 {
 207    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 208    GLuint hash = hash_key(key, key_size, reloc_bufs, nr_reloc_bufs);
 209    GLuint relocs_size = nr_reloc_bufs * sizeof(dri_bo *);
 210    GLuint aux_size = cache->aux_size[cache_id];
 211    void *tmp;
 212    dri_bo *bo;
 213    int i;
 214
 215    /* Create the buffer object to contain the data */
 216    bo = dri_bo_alloc(cache->brw->intel.bufmgr,
 217                      cache->name[cache_id], data_size, 1 << 6);
 218
 219
 220    /* Set up the memory containing the key, aux_data, and reloc_bufs */
 221    tmp = _mesa_malloc(key_size + aux_size + relocs_size);
 222
 223    memcpy(tmp, key, key_size);
 224    memcpy(tmp + key_size, aux, cache->aux_size[cache_id]);
 225    memcpy(tmp + key_size + aux_size, reloc_bufs, relocs_size);
 226    for (i = 0; i < nr_reloc_bufs; i++) {
 227       if (reloc_bufs[i] != NULL)
 228          dri_bo_reference(reloc_bufs[i]);
 229    }
 230
 231    item->cache_id = cache_id;
 232    item->key = tmp;
 233    item->hash = hash;
 234    item->key_size = key_size;
 235    item->reloc_bufs = tmp + key_size + aux_size;
 236    item->nr_reloc_bufs = nr_reloc_bufs;
 237
 238    item->bo = bo;
 239    dri_bo_reference(bo);
 240    item->data_size = data_size;
 241
 242    if (cache->n_items > cache->size * 1.5)
 243       rehash(cache);
 244
 245    hash %= cache->size;
 246    item->next = cache->items[hash];
 247    cache->items[hash] = item;
 248    cache->n_items++;
 249
 250    if (aux_return) {
 251       assert(cache->aux_size[cache_id]);
 252       *(void **)aux_return = (void *)((char *)item->key + item->key_size);
 253    }
 254
 255    if (INTEL_DEBUG & DEBUG_STATE)
 256       _mesa_printf("upload %s: %d bytes to cache id %d\n",
 257                    cache->name[cache_id],
 258                    data_size, cache_id);
 259
 260    /* Copy data to the buffer */
 261    dri_bo_subdata(bo, 0, data_size, data);
 262
 263    update_cache_last(cache, cache_id, bo);
 264
 265    return bo;
 266 }
 267
 268 /* This doesn't really work with aux data.  Use search/upload instead
 269  */
 270 dri_bo *
 271 brw_cache_data_sz(struct brw_cache *cache,
 272                   enum brw_cache_id cache_id,
 273                   const void *data,
 274                   GLuint data_size,
 275                   dri_bo **reloc_bufs,
 276                   GLuint nr_reloc_bufs)
 277 {
 278    dri_bo *bo;
 279    struct brw_cache_item *item;
 280    GLuint hash = hash_key(data, data_size, reloc_bufs, nr_reloc_bufs);
 281
 282    item = search_cache(cache, cache_id, hash, data, data_size,
 283                        reloc_bufs, nr_reloc_bufs);
 284    if (item) {
 285       update_cache_last(cache, cache_id, item->bo);
 286       dri_bo_reference(item->bo);
 287       return item->bo;
 288    }
 289
 290    bo = brw_upload_cache(cache, cache_id,
 291                          data, data_size,
 292                          reloc_bufs, nr_reloc_bufs,
 293                          data, data_size,
 294                          NULL, NULL);
 295
 296    return bo;
 297 }
 298
 299 /**
 300  * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
 301  *
 302  * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
 303  * better to use, as the potentially changing offsets in the data-used-as-key
 304  * will result in excessive cache misses.
 305  */
 306 dri_bo *
 307 brw_cache_data(struct brw_cache *cache,
 308                enum brw_cache_id cache_id,
 309                const void *data,
 310                dri_bo **reloc_bufs,
 311                GLuint nr_reloc_bufs)
 312 {
 313    return brw_cache_data_sz(cache, cache_id, data, cache->key_size[cache_id],
 314                             reloc_bufs, nr_reloc_bufs);
 315 }
 316
 317 enum pool_type {
 318    DW_SURFACE_STATE,
 319    DW_GENERAL_STATE
 320 };
 321
 322 static void
 323 brw_init_cache_id( struct brw_context *brw,
 324                 const char *name,
 325                 enum brw_cache_id id,
 326                 GLuint key_size,
 327                 GLuint aux_size)
 328 {
 329    struct brw_cache *cache = &brw->cache;
 330
 331    cache->name[id] = strdup(name);
 332    cache->key_size[id] = key_size;
 333    cache->aux_size[id] = aux_size;
 334 }
 335
 336 void brw_init_cache( struct brw_context *brw )
 337 {
 338    struct brw_cache *cache = &brw->cache;
 339
 340    cache->brw = brw;
 341
 342    cache->size = 7;
 343    cache->n_items = 0;
 344    cache->items = (struct brw_cache_item **)
 345       _mesa_calloc(cache->size *
 346                    sizeof(struct brw_cache_item));
 347
 348    brw_init_cache_id(brw,
 349                      "CC_VP",
 350                      BRW_CC_VP,
 351                      sizeof(struct brw_cc_viewport),
 352                      0);
 353
 354    brw_init_cache_id(brw,
 355                      "CC_UNIT",
 356                      BRW_CC_UNIT,
 357                      sizeof(struct brw_cc_unit_state),
 358                      0);
 359
 360    brw_init_cache_id(brw,
 361                      "WM_PROG",
 362                      BRW_WM_PROG,
 363                      sizeof(struct brw_wm_prog_key),
 364                      sizeof(struct brw_wm_prog_data));
 365
 366    brw_init_cache_id(brw,
 367                      "SAMPLER_DEFAULT_COLOR",
 368                      BRW_SAMPLER_DEFAULT_COLOR,
 369                      sizeof(struct brw_sampler_default_color),
 370                      0);
 371
 372    brw_init_cache_id(brw,
 373                      "SAMPLER",
 374                      BRW_SAMPLER,
 375                      0,         /* variable key/data size */
 376                      0);
 377
 378    brw_init_cache_id(brw,
 379                      "WM_UNIT",
 380                      BRW_WM_UNIT,
 381                      sizeof(struct brw_wm_unit_state),
 382                      0);
 383
 384    brw_init_cache_id(brw,
 385                      "SF_PROG",
 386                      BRW_SF_PROG,
 387                      sizeof(struct brw_sf_prog_key),
 388                      sizeof(struct brw_sf_prog_data));
 389
 390    brw_init_cache_id(brw,
 391                      "SF_VP",
 392                      BRW_SF_VP,
 393                      sizeof(struct brw_sf_viewport),
 394                      0);
 395
 396    brw_init_cache_id(brw,
 397                      "SF_UNIT",
 398                      BRW_SF_UNIT,
 399                      sizeof(struct brw_sf_unit_state),
 400                      0);
 401
 402    brw_init_cache_id(brw,
 403                      "VS_UNIT",
 404                      BRW_VS_UNIT,
 405                      sizeof(struct brw_vs_unit_state),
 406                      0);
 407
 408    brw_init_cache_id(brw,
 409                      "VS_PROG",
 410                      BRW_VS_PROG,
 411                      sizeof(struct brw_vs_prog_key),
 412                      sizeof(struct brw_vs_prog_data));
 413
 414    brw_init_cache_id(brw,
 415                      "CLIP_UNIT",
 416                      BRW_CLIP_UNIT,
 417                      sizeof(struct brw_clip_unit_state),
 418                      0);
 419
 420    brw_init_cache_id(brw,
 421                      "CLIP_PROG",
 422                      BRW_CLIP_PROG,
 423                      sizeof(struct brw_clip_prog_key),
 424                      sizeof(struct brw_clip_prog_data));
 425
 426    brw_init_cache_id(brw,
 427                      "GS_UNIT",
 428                      BRW_GS_UNIT,
 429                      sizeof(struct brw_gs_unit_state),
 430                      0);
 431
 432    brw_init_cache_id(brw,
 433                      "GS_PROG",
 434                      BRW_GS_PROG,
 435                      sizeof(struct brw_gs_prog_key),
 436                      sizeof(struct brw_gs_prog_data));
 437
 438    brw_init_cache_id(brw,
 439                      "SS_SURFACE",
 440                      BRW_SS_SURFACE,
 441                      sizeof(struct brw_surface_state),
 442                      0);
 443
 444    brw_init_cache_id(brw,
 445                      "SS_SURF_BIND",
 446                      BRW_SS_SURF_BIND,
 447                      0,
 448                      0);
 449 }
 450
 451 static void
 452 brw_clear_cache( struct brw_context *brw )
 453 {
 454    struct brw_cache_item *c, *next;
 455    GLuint i;
 456
 457    if (INTEL_DEBUG & DEBUG_STATE)
 458       _mesa_printf("%s\n", __FUNCTION__);
 459
 460    for (i = 0; i < brw->cache.size; i++) {
 461       for (c = brw->cache.items[i]; c; c = next) {
 462          int j;
 463
 464          next = c->next;
 465          for (j = 0; j < c->nr_reloc_bufs; j++)
 466             dri_bo_unreference(c->reloc_bufs[j]);
 467          dri_bo_unreference(c->bo);
 468          free((void *)c->key);
 469          free(c);
 470       }
 471       brw->cache.items[i] = NULL;
 472    }
 473
 474    brw->cache.n_items = 0;
 475
 476    if (brw->curbe.last_buf) {
 477       _mesa_free(brw->curbe.last_buf);
 478       brw->curbe.last_buf = NULL;
 479    }
 480
 481    brw->state.dirty.mesa |= ~0;
 482    brw->state.dirty.brw |= ~0;
 483    brw->state.dirty.cache |= ~0;
 484 }
 485
 486 void brw_state_cache_check_size( struct brw_context *brw )
 487 {
 488    /* un-tuned guess.  We've got around 20 state objects for a total of around
 489     * 32k, so 1000 of them is around 1.5MB.
 490     */
 491    if (brw->cache.n_items > 1000)
 492       brw_clear_cache(brw);
 493 }
 494
 495 void brw_destroy_cache( struct brw_context *brw )
 496 {
 497    GLuint i;
 498
 499    brw_clear_cache(brw);
 500    for (i = 0; i < BRW_MAX_CACHE; i++) {
 501       dri_bo_unreference(brw->cache.last_bo[i]);
 502       free(brw->cache.name[i]);
 503    }
 504    free(brw->cache.items);
 505    brw->cache.items = NULL;
 506    brw->cache.size = 0;
 507 }