src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The
  35  * consumers can query the hash table of state using a cache_id,
  36  * opaque key data, and receive the corresponding state buffer object
  37  * of state (plus associated auxiliary data) in return.  Objects in
  38  * the cache may not have relocations (pointers to other BOs) in them.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the
  41  * key data.
  42  *
  43  * Replacement is not implemented.  Instead, when the cache gets too
  44  * big we throw out all of the cache data and let it get regenerated.
  45  */
  46
  47 #include "main/imports.h"
  48 #include "intel_batchbuffer.h"
  49 #include "brw_state.h"
  50 #include "brw_vs.h"
  51 #include "brw_wm.h"
  52 #include "brw_vs.h"
  53 #include "brw_vec4_gs.h"
  54
  55 #define FILE_DEBUG_FLAG DEBUG_STATE
  56
  57 static GLuint
  58 hash_key(struct brw_cache_item *item)
  59 {
  60    GLuint *ikey = (GLuint *)item->key;
  61    GLuint hash = item->cache_id, i;
  62
  63    assert(item->key_size % 4 == 0);
  64
  65    /* I'm sure this can be improved on:
  66     */
  67    for (i = 0; i < item->key_size/4; i++) {
  68       hash ^= ikey[i];
  69       hash = (hash << 5) | (hash >> 27);
  70    }
  71
  72    return hash;
  73 }
  74
  75 static int
  76 brw_cache_item_equals(const struct brw_cache_item *a,
  77                       const struct brw_cache_item *b)
  78 {
  79    return a->cache_id == b->cache_id &&
  80       a->hash == b->hash &&
  81       a->key_size == b->key_size &&
  82       (memcmp(a->key, b->key, a->key_size) == 0);
  83 }
  84
  85 static struct brw_cache_item *
  86 search_cache(struct brw_cache *cache, GLuint hash,
  87              struct brw_cache_item *lookup)
  88 {
  89    struct brw_cache_item *c;
  90
  91 #if 0
  92    int bucketcount = 0;
  93
  94    for (c = cache->items[hash % cache->size]; c; c = c->next)
  95       bucketcount++;
  96
  97    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
  98            cache->size, bucketcount, cache->n_items);
  99 #endif
 100
 101    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 102       if (brw_cache_item_equals(lookup, c))
 103          return c;
 104    }
 105
 106    return NULL;
 107 }
 108
 109
 110 static void
 111 rehash(struct brw_cache *cache)
 112 {
 113    struct brw_cache_item **items;
 114    struct brw_cache_item *c, *next;
 115    GLuint size, i;
 116
 117    size = cache->size * 3;
 118    items = calloc(size, sizeof(*items));
 119
 120    for (i = 0; i < cache->size; i++)
 121       for (c = cache->items[i]; c; c = next) {
 122          next = c->next;
 123          c->next = items[c->hash % size];
 124          items[c->hash % size] = c;
 125       }
 126
 127    free(cache->items);
 128    cache->items = items;
 129    cache->size = size;
 130 }
 131
 132
 133 /**
 134  * Returns the buffer object matching cache_id and key, or NULL.
 135  */
 136 bool
 137 brw_search_cache(struct brw_cache *cache,
 138                  enum brw_cache_id cache_id,
 139                  const void *key, GLuint key_size,
 140                  uint32_t *inout_offset, void *out_aux)
 141 {
 142    struct brw_context *brw = cache->brw;
 143    struct brw_cache_item *item;
 144    struct brw_cache_item lookup;
 145    GLuint hash;
 146
 147    lookup.cache_id = cache_id;
 148    lookup.key = key;
 149    lookup.key_size = key_size;
 150    hash = hash_key(&lookup);
 151    lookup.hash = hash;
 152
 153    item = search_cache(cache, hash, &lookup);
 154
 155    if (item == NULL)
 156       return false;
 157
 158    *(void **)out_aux = ((char *)item->key + item->key_size);
 159
 160    if (item->offset != *inout_offset) {
 161       brw->state.dirty.cache |= (1 << cache_id);
 162       *inout_offset = item->offset;
 163    }
 164
 165    return true;
 166 }
 167
 168 static void
 169 brw_cache_new_bo(struct brw_cache *cache, uint32_t new_size)
 170 {
 171    struct brw_context *brw = cache->brw;
 172    drm_intel_bo *new_bo;
 173
 174    new_bo = drm_intel_bo_alloc(brw->bufmgr, "program cache", new_size, 64);
 175
 176    /* Copy any existing data that needs to be saved. */
 177    if (cache->next_offset != 0) {
 178       drm_intel_bo_map(cache->bo, false);
 179       drm_intel_bo_subdata(new_bo, 0, cache->next_offset, cache->bo->virtual);
 180       drm_intel_bo_unmap(cache->bo);
 181    }
 182
 183    drm_intel_bo_unreference(cache->bo);
 184    cache->bo = new_bo;
 185    cache->bo_used_by_gpu = false;
 186
 187    /* Since we have a new BO in place, we need to signal the units
 188     * that depend on it (state base address on gen5+, or unit state before).
 189     */
 190    brw->state.dirty.brw |= BRW_NEW_PROGRAM_CACHE;
 191 }
 192
 193 /**
 194  * Attempts to find an item in the cache with identical data and aux
 195  * data to use
 196  */
 197 static bool
 198 brw_try_upload_using_copy(struct brw_cache *cache,
 199                           struct brw_cache_item *result_item,
 200                           const void *data,
 201                           const void *aux)
 202 {
 203    int i;
 204    struct brw_cache_item *item;
 205
 206    for (i = 0; i < cache->size; i++) {
 207       for (item = cache->items[i]; item; item = item->next) {
 208          const void *item_aux = item->key + item->key_size;
 209          int ret;
 210
 211          if (item->cache_id != result_item->cache_id ||
 212              item->size != result_item->size ||
 213              item->aux_size != result_item->aux_size) {
 214             continue;
 215          }
 216
 217          if (cache->aux_compare[result_item->cache_id]) {
 218             if (!cache->aux_compare[result_item->cache_id](item_aux, aux))
 219                continue;
 220          } else if (memcmp(item_aux, aux, item->aux_size) != 0) {
 221             continue;
 222          }
 223
 224          drm_intel_bo_map(cache->bo, false);
 225          ret = memcmp(cache->bo->virtual + item->offset, data, item->size);
 226          drm_intel_bo_unmap(cache->bo);
 227          if (ret)
 228             continue;
 229
 230          result_item->offset = item->offset;
 231
 232          return true;
 233       }
 234    }
 235
 236    return false;
 237 }
 238
 239 static void
 240 brw_upload_item_data(struct brw_cache *cache,
 241                      struct brw_cache_item *item,
 242                      const void *data)
 243 {
 244    struct brw_context *brw = cache->brw;
 245
 246    /* Allocate space in the cache BO for our new program. */
 247    if (cache->next_offset + item->size > cache->bo->size) {
 248       uint32_t new_size = cache->bo->size * 2;
 249
 250       while (cache->next_offset + item->size > new_size)
 251          new_size *= 2;
 252
 253       brw_cache_new_bo(cache, new_size);
 254    }
 255
 256    /* If we would block on writing to an in-use program BO, just
 257     * recreate it.
 258     */
 259    if (cache->bo_used_by_gpu) {
 260       perf_debug("Copying busy program cache buffer.\n");
 261       brw_cache_new_bo(cache, cache->bo->size);
 262    }
 263
 264    item->offset = cache->next_offset;
 265
 266    /* Programs are always 64-byte aligned, so set up the next one now */
 267    cache->next_offset = ALIGN(item->offset + item->size, 64);
 268 }
 269
 270 void
 271 brw_upload_cache(struct brw_cache *cache,
 272                  enum brw_cache_id cache_id,
 273                  const void *key,
 274                  GLuint key_size,
 275                  const void *data,
 276                  GLuint data_size,
 277                  const void *aux,
 278                  GLuint aux_size,
 279                  uint32_t *out_offset,
 280                  void *out_aux)
 281 {
 282    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 283    GLuint hash;
 284    void *tmp;
 285
 286    item->cache_id = cache_id;
 287    item->size = data_size;
 288    item->key = key;
 289    item->key_size = key_size;
 290    item->aux_size = aux_size;
 291    hash = hash_key(item);
 292    item->hash = hash;
 293
 294    /* If we can find a matching prog/prog_data combo in the cache
 295     * already, then reuse the existing stuff.  This will mean not
 296     * flagging CACHE_NEW_* when transitioning between the two
 297     * equivalent hash keys.  This is notably useful for programs
 298     * generating shaders at runtime, where multiple shaders may
 299     * compile to the thing in our backend.
 300     */
 301    if (!brw_try_upload_using_copy(cache, item, data, aux)) {
 302       brw_upload_item_data(cache, item, data);
 303    }
 304
 305    /* Set up the memory containing the key and aux_data */
 306    tmp = malloc(key_size + aux_size);
 307
 308    memcpy(tmp, key, key_size);
 309    memcpy(tmp + key_size, aux, aux_size);
 310
 311    item->key = tmp;
 312
 313    if (cache->n_items > cache->size * 1.5)
 314       rehash(cache);
 315
 316    hash %= cache->size;
 317    item->next = cache->items[hash];
 318    cache->items[hash] = item;
 319    cache->n_items++;
 320
 321    /* Copy data to the buffer */
 322    drm_intel_bo_subdata(cache->bo, item->offset, data_size, data);
 323
 324    *out_offset = item->offset;
 325    *(void **)out_aux = (void *)((char *)item->key + item->key_size);
 326    cache->brw->state.dirty.cache |= 1 << cache_id;
 327 }
 328
 329 void
 330 brw_init_caches(struct brw_context *brw)
 331 {
 332    struct brw_cache *cache = &brw->cache;
 333
 334    cache->brw = brw;
 335
 336    cache->size = 7;
 337    cache->n_items = 0;
 338    cache->items =
 339       calloc(cache->size, sizeof(struct brw_cache_item *));
 340
 341    cache->bo = drm_intel_bo_alloc(brw->bufmgr,
 342                                   "program cache",
 343                                   4096, 64);
 344
 345    cache->aux_compare[BRW_VS_PROG] = brw_vs_prog_data_compare;
 346    cache->aux_compare[BRW_GS_PROG] = brw_gs_prog_data_compare;
 347    cache->aux_compare[BRW_WM_PROG] = brw_wm_prog_data_compare;
 348    cache->aux_free[BRW_VS_PROG] = brw_stage_prog_data_free;
 349    cache->aux_free[BRW_GS_PROG] = brw_stage_prog_data_free;
 350    cache->aux_free[BRW_WM_PROG] = brw_stage_prog_data_free;
 351 }
 352
 353 static void
 354 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 355 {
 356    struct brw_cache_item *c, *next;
 357    GLuint i;
 358
 359    DBG("%s\n", __FUNCTION__);
 360
 361    for (i = 0; i < cache->size; i++) {
 362       for (c = cache->items[i]; c; c = next) {
 363          next = c->next;
 364          if (cache->aux_free[c->cache_id]) {
 365             const void *item_aux = c->key + c->key_size;
 366             cache->aux_free[c->cache_id](item_aux);
 367          }
 368          free((void *)c->key);
 369          free(c);
 370       }
 371       cache->items[i] = NULL;
 372    }
 373
 374    cache->n_items = 0;
 375
 376    /* Start putting programs into the start of the BO again, since
 377     * we'll never find the old results.
 378     */
 379    cache->next_offset = 0;
 380
 381    /* We need to make sure that the programs get regenerated, since
 382     * any offsets leftover in brw_context will no longer be valid.
 383     */
 384    brw->state.dirty.mesa |= ~0;
 385    brw->state.dirty.brw |= ~0ull;
 386    brw->state.dirty.cache |= ~0;
 387    intel_batchbuffer_flush(brw);
 388 }
 389
 390 void
 391 brw_state_cache_check_size(struct brw_context *brw)
 392 {
 393    /* un-tuned guess.  Each object is generally a page, so 2000 of them is 8 MB of
 394     * state cache.
 395     */
 396    if (brw->cache.n_items > 2000) {
 397       perf_debug("Exceeded state cache size limit.  Clearing the set "
 398                  "of compiled programs, which will trigger recompiles\n");
 399       brw_clear_cache(brw, &brw->cache);
 400    }
 401 }
 402
 403
 404 static void
 405 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 406 {
 407
 408    DBG("%s\n", __FUNCTION__);
 409
 410    drm_intel_bo_unreference(cache->bo);
 411    cache->bo = NULL;
 412    brw_clear_cache(brw, cache);
 413    free(cache->items);
 414    cache->items = NULL;
 415    cache->size = 0;
 416 }
 417
 418
 419 void
 420 brw_destroy_caches(struct brw_context *brw)
 421 {
 422    brw_destroy_cache(brw, &brw->cache);
 423 }