src/mesa/drivers/dri/i965/brw_state_cache.c

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32 /** @file brw_state_cache.c
  33  *
  34  * This file implements a simple static state cache for 965.  The
  35  * consumers can query the hash table of state using a cache_id,
  36  * opaque key data, and receive the corresponding state buffer object
  37  * of state (plus associated auxiliary data) in return.  Objects in
  38  * the cache may not have relocations (pointers to other BOs) in them.
  39  *
  40  * The inner workings are a simple hash table based on a CRC of the
  41  * key data.
  42  *
  43  * Replacement is not implemented.  Instead, when the cache gets too
  44  * big we throw out all of the cache data and let it get regenerated.
  45  */
  46
  47 #include "main/imports.h"
  48 #include "intel_batchbuffer.h"
  49 #include "brw_state.h"
  50 #include "brw_vs.h"
  51 #include "brw_wm.h"
  52 #include "brw_vs.h"
  53 #include "brw_vec4_gs.h"
  54
  55 #define FILE_DEBUG_FLAG DEBUG_STATE
  56
  57 static GLuint
  58 hash_key(struct brw_cache_item *item)
  59 {
  60    GLuint *ikey = (GLuint *)item->key;
  61    GLuint hash = item->cache_id, i;
  62
  63    assert(item->key_size % 4 == 0);
  64
  65    /* I'm sure this can be improved on:
  66     */
  67    for (i = 0; i < item->key_size/4; i++) {
  68       hash ^= ikey[i];
  69       hash = (hash << 5) | (hash >> 27);
  70    }
  71
  72    return hash;
  73 }
  74
  75 static int
  76 brw_cache_item_equals(const struct brw_cache_item *a,
  77                       const struct brw_cache_item *b)
  78 {
  79    return a->cache_id == b->cache_id &&
  80       a->hash == b->hash &&
  81       a->key_size == b->key_size &&
  82       (memcmp(a->key, b->key, a->key_size) == 0);
  83 }
  84
  85 static struct brw_cache_item *
  86 search_cache(struct brw_cache *cache, GLuint hash,
  87              struct brw_cache_item *lookup)
  88 {
  89    struct brw_cache_item *c;
  90
  91 #if 0
  92    int bucketcount = 0;
  93
  94    for (c = cache->items[hash % cache->size]; c; c = c->next)
  95       bucketcount++;
  96
  97    fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size,
  98            cache->size, bucketcount, cache->n_items);
  99 #endif
 100
 101    for (c = cache->items[hash % cache->size]; c; c = c->next) {
 102       if (brw_cache_item_equals(lookup, c))
 103          return c;
 104    }
 105
 106    return NULL;
 107 }
 108
 109
 110 static void
 111 rehash(struct brw_cache *cache)
 112 {
 113    struct brw_cache_item **items;
 114    struct brw_cache_item *c, *next;
 115    GLuint size, i;
 116
 117    size = cache->size * 3;
 118    items = calloc(size, sizeof(*items));
 119
 120    for (i = 0; i < cache->size; i++)
 121       for (c = cache->items[i]; c; c = next) {
 122          next = c->next;
 123          c->next = items[c->hash % size];
 124          items[c->hash % size] = c;
 125       }
 126
 127    free(cache->items);
 128    cache->items = items;
 129    cache->size = size;
 130 }
 131
 132
 133 /**
 134  * Returns the buffer object matching cache_id and key, or NULL.
 135  */
 136 bool
 137 brw_search_cache(struct brw_cache *cache,
 138                  enum brw_cache_id cache_id,
 139                  const void *key, GLuint key_size,
 140                  uint32_t *inout_offset, void *out_aux)
 141 {
 142    struct brw_context *brw = cache->brw;
 143    struct brw_cache_item *item;
 144    struct brw_cache_item lookup;
 145    GLuint hash;
 146
 147    lookup.cache_id = cache_id;
 148    lookup.key = key;
 149    lookup.key_size = key_size;
 150    hash = hash_key(&lookup);
 151    lookup.hash = hash;
 152
 153    item = search_cache(cache, hash, &lookup);
 154
 155    if (item == NULL)
 156       return false;
 157
 158    *(void **)out_aux = ((char *)item->key + item->key_size);
 159
 160    if (item->offset != *inout_offset) {
 161       brw->state.dirty.cache |= (1 << cache_id);
 162       *inout_offset = item->offset;
 163    }
 164
 165    return true;
 166 }
 167
 168 static void
 169 brw_cache_new_bo(struct brw_cache *cache, uint32_t new_size)
 170 {
 171    struct brw_context *brw = cache->brw;
 172    drm_intel_bo *new_bo;
 173
 174    new_bo = drm_intel_bo_alloc(brw->bufmgr, "program cache", new_size, 64);
 175
 176    /* Copy any existing data that needs to be saved. */
 177    if (cache->next_offset != 0) {
 178       drm_intel_bo_map(cache->bo, false);
 179       drm_intel_bo_subdata(new_bo, 0, cache->next_offset, cache->bo->virtual);
 180       drm_intel_bo_unmap(cache->bo);
 181    }
 182
 183    drm_intel_bo_unreference(cache->bo);
 184    cache->bo = new_bo;
 185    cache->bo_used_by_gpu = false;
 186
 187    /* Since we have a new BO in place, we need to signal the units
 188     * that depend on it (state base address on gen5+, or unit state before).
 189     */
 190    brw->state.dirty.brw |= BRW_NEW_PROGRAM_CACHE;
 191 }
 192
 193 /**
 194  * Attempts to find an item in the cache with identical data and aux
 195  * data to use
 196  */
 197 static bool
 198 brw_try_upload_using_copy(struct brw_cache *cache,
 199                           struct brw_cache_item *result_item,
 200                           const void *data,
 201                           const void *aux)
 202 {
 203    int i;
 204    struct brw_cache_item *item;
 205
 206    for (i = 0; i < cache->size; i++) {
 207       for (item = cache->items[i]; item; item = item->next) {
 208          const void *item_aux = item->key + item->key_size;
 209          int ret;
 210
 211          if (item->cache_id != result_item->cache_id ||
 212              item->size != result_item->size ||
 213              item->aux_size != result_item->aux_size) {
 214             continue;
 215          }
 216
 217          if (cache->aux_compare[result_item->cache_id]) {
 218             if (!cache->aux_compare[result_item->cache_id](item_aux, aux))
 219                continue;
 220          } else if (memcmp(item_aux, aux, item->aux_size) != 0) {
 221             continue;
 222          }
 223
 224          drm_intel_bo_map(cache->bo, false);
 225          ret = memcmp(cache->bo->virtual + item->offset, data, item->size);
 226          drm_intel_bo_unmap(cache->bo);
 227          if (ret)
 228             continue;
 229
 230          result_item->offset = item->offset;
 231
 232          return true;
 233       }
 234    }
 235
 236    return false;
 237 }
 238
 239 static void
 240 brw_upload_item_data(struct brw_cache *cache,
 241                      struct brw_cache_item *item,
 242                      const void *data)
 243 {
 244    /* Allocate space in the cache BO for our new program. */
 245    if (cache->next_offset + item->size > cache->bo->size) {
 246       uint32_t new_size = cache->bo->size * 2;
 247
 248       while (cache->next_offset + item->size > new_size)
 249          new_size *= 2;
 250
 251       brw_cache_new_bo(cache, new_size);
 252    }
 253
 254    /* If we would block on writing to an in-use program BO, just
 255     * recreate it.
 256     */
 257    if (cache->bo_used_by_gpu) {
 258       brw_cache_new_bo(cache, cache->bo->size);
 259    }
 260
 261    item->offset = cache->next_offset;
 262
 263    /* Programs are always 64-byte aligned, so set up the next one now */
 264    cache->next_offset = ALIGN(item->offset + item->size, 64);
 265 }
 266
 267 void
 268 brw_upload_cache(struct brw_cache *cache,
 269                  enum brw_cache_id cache_id,
 270                  const void *key,
 271                  GLuint key_size,
 272                  const void *data,
 273                  GLuint data_size,
 274                  const void *aux,
 275                  GLuint aux_size,
 276                  uint32_t *out_offset,
 277                  void *out_aux)
 278 {
 279    struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item);
 280    GLuint hash;
 281    void *tmp;
 282
 283    item->cache_id = cache_id;
 284    item->size = data_size;
 285    item->key = key;
 286    item->key_size = key_size;
 287    item->aux_size = aux_size;
 288    hash = hash_key(item);
 289    item->hash = hash;
 290
 291    /* If we can find a matching prog/prog_data combo in the cache
 292     * already, then reuse the existing stuff.  This will mean not
 293     * flagging CACHE_NEW_* when transitioning between the two
 294     * equivalent hash keys.  This is notably useful for programs
 295     * generating shaders at runtime, where multiple shaders may
 296     * compile to the thing in our backend.
 297     */
 298    if (!brw_try_upload_using_copy(cache, item, data, aux)) {
 299       brw_upload_item_data(cache, item, data);
 300    }
 301
 302    /* Set up the memory containing the key and aux_data */
 303    tmp = malloc(key_size + aux_size);
 304
 305    memcpy(tmp, key, key_size);
 306    memcpy(tmp + key_size, aux, aux_size);
 307
 308    item->key = tmp;
 309
 310    if (cache->n_items > cache->size * 1.5)
 311       rehash(cache);
 312
 313    hash %= cache->size;
 314    item->next = cache->items[hash];
 315    cache->items[hash] = item;
 316    cache->n_items++;
 317
 318    /* Copy data to the buffer */
 319    drm_intel_bo_subdata(cache->bo, item->offset, data_size, data);
 320
 321    *out_offset = item->offset;
 322    *(void **)out_aux = (void *)((char *)item->key + item->key_size);
 323    cache->brw->state.dirty.cache |= 1 << cache_id;
 324 }
 325
 326 void
 327 brw_init_caches(struct brw_context *brw)
 328 {
 329    struct brw_cache *cache = &brw->cache;
 330
 331    cache->brw = brw;
 332
 333    cache->size = 7;
 334    cache->n_items = 0;
 335    cache->items =
 336       calloc(cache->size, sizeof(struct brw_cache_item *));
 337
 338    cache->bo = drm_intel_bo_alloc(brw->bufmgr,
 339                                   "program cache",
 340                                   4096, 64);
 341
 342    cache->aux_compare[BRW_VS_PROG] = brw_vs_prog_data_compare;
 343    cache->aux_compare[BRW_GS_PROG] = brw_gs_prog_data_compare;
 344    cache->aux_compare[BRW_WM_PROG] = brw_wm_prog_data_compare;
 345    cache->aux_free[BRW_VS_PROG] = brw_stage_prog_data_free;
 346    cache->aux_free[BRW_GS_PROG] = brw_stage_prog_data_free;
 347    cache->aux_free[BRW_WM_PROG] = brw_stage_prog_data_free;
 348 }
 349
 350 static void
 351 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache)
 352 {
 353    struct brw_cache_item *c, *next;
 354    GLuint i;
 355
 356    DBG("%s\n", __FUNCTION__);
 357
 358    for (i = 0; i < cache->size; i++) {
 359       for (c = cache->items[i]; c; c = next) {
 360          next = c->next;
 361          if (cache->aux_free[c->cache_id]) {
 362             const void *item_aux = c->key + c->key_size;
 363             cache->aux_free[c->cache_id](item_aux);
 364          }
 365          free((void *)c->key);
 366          free(c);
 367       }
 368       cache->items[i] = NULL;
 369    }
 370
 371    cache->n_items = 0;
 372
 373    /* Start putting programs into the start of the BO again, since
 374     * we'll never find the old results.
 375     */
 376    cache->next_offset = 0;
 377
 378    /* We need to make sure that the programs get regenerated, since
 379     * any offsets leftover in brw_context will no longer be valid.
 380     */
 381    brw->state.dirty.mesa |= ~0;
 382    brw->state.dirty.brw |= ~0ull;
 383    brw->state.dirty.cache |= ~0;
 384    intel_batchbuffer_flush(brw);
 385 }
 386
 387 void
 388 brw_state_cache_check_size(struct brw_context *brw)
 389 {
 390    /* un-tuned guess.  Each object is generally a page, so 2000 of them is 8 MB of
 391     * state cache.
 392     */
 393    if (brw->cache.n_items > 2000) {
 394       perf_debug("Exceeded state cache size limit.  Clearing the set "
 395                  "of compiled programs, which will trigger recompiles\n");
 396       brw_clear_cache(brw, &brw->cache);
 397    }
 398 }
 399
 400
 401 static void
 402 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache)
 403 {
 404
 405    DBG("%s\n", __FUNCTION__);
 406
 407    drm_intel_bo_unreference(cache->bo);
 408    cache->bo = NULL;
 409    brw_clear_cache(brw, cache);
 410    free(cache->items);
 411    cache->items = NULL;
 412    cache->size = 0;
 413 }
 414
 415
 416 void
 417 brw_destroy_caches(struct brw_context *brw)
 418 {
 419    brw_destroy_cache(brw, &brw->cache);
 420 }