2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keith@tungstengraphics.com>
32 /** @file brw_state_cache.c
34 * This file implements a simple static state cache for 965. The consumers
35 * can query the hash table of state using a cache_id, opaque key data,
36 * and list of buffers that will be used in relocations, and receive the
37 * corresponding state buffer object of state (plus associated auxiliary
40 * The inner workings are a simple hash table based on a CRC of the key data.
41 * The cache_id and relocation target buffers associated with the state
42 * buffer are included as auxiliary key data, but are not part of the hash
43 * value (this should be fixed, but will likely be fixed instead by making
44 * consumers use structured keys).
46 * Replacement is not implemented. Instead, when the cache gets too big, at
47 * a safe point (unlock) we throw out all of the cache data and let it
48 * regenerate for the next rendering operation.
50 * The reloc_buf pointers need to be included as key data, otherwise the
51 * non-unique values stuffed in the offset in key data through
52 * brw_cache_data() may result in successful probe for state buffers
53 * even when the buffer being referenced doesn't match. The result would be
54 * that the same state cache entry is used twice for different buffers,
55 * only one of the two buffers referenced gets put into the offset, and the
56 * incorrect program is run for the other instance.
59 #include "main/imports.h"
60 #include "brw_state.h"
61 #include "intel_batchbuffer.h"
66 hash_key(struct brw_cache_item
*item
)
68 GLuint
*ikey
= (GLuint
*)item
->key
;
69 GLuint hash
= item
->cache_id
, i
;
71 assert(item
->key_size
% 4 == 0);
73 /* I'm sure this can be improved on:
75 for (i
= 0; i
< item
->key_size
/4; i
++) {
77 hash
= (hash
<< 5) | (hash
>> 27);
80 /* Include the BO pointers as key data as well */
81 ikey
= (GLuint
*)item
->reloc_bufs
;
82 for (i
= 0; i
< item
->nr_reloc_bufs
* sizeof(drm_intel_bo
*) / 4; i
++) {
84 hash
= (hash
<< 5) | (hash
>> 27);
92 * Marks a new buffer as being chosen for the given cache id.
95 update_cache_last(struct brw_cache
*cache
, enum brw_cache_id cache_id
,
98 if (bo
== cache
->last_bo
[cache_id
])
99 return; /* no change */
101 dri_bo_unreference(cache
->last_bo
[cache_id
]);
102 cache
->last_bo
[cache_id
] = bo
;
103 dri_bo_reference(cache
->last_bo
[cache_id
]);
104 cache
->brw
->state
.dirty
.cache
|= 1 << cache_id
;
108 brw_cache_item_equals(const struct brw_cache_item
*a
,
109 const struct brw_cache_item
*b
)
111 return a
->cache_id
== b
->cache_id
&&
112 a
->hash
== b
->hash
&&
113 a
->key_size
== b
->key_size
&&
114 (memcmp(a
->key
, b
->key
, a
->key_size
) == 0) &&
115 a
->nr_reloc_bufs
== b
->nr_reloc_bufs
&&
116 (memcmp(a
->reloc_bufs
, b
->reloc_bufs
,
117 a
->nr_reloc_bufs
* sizeof(dri_bo
*)) == 0);
120 static struct brw_cache_item
*
121 search_cache(struct brw_cache
*cache
, GLuint hash
,
122 struct brw_cache_item
*lookup
)
124 struct brw_cache_item
*c
;
129 for (c
= cache
->items
[hash
% cache
->size
]; c
; c
= c
->next
)
132 fprintf(stderr
, "bucket %d/%d = %d/%d items\n", hash
% cache
->size
,
133 cache
->size
, bucketcount
, cache
->n_items
);
136 for (c
= cache
->items
[hash
% cache
->size
]; c
; c
= c
->next
) {
137 if (brw_cache_item_equals(lookup
, c
))
146 rehash(struct brw_cache
*cache
)
148 struct brw_cache_item
**items
;
149 struct brw_cache_item
*c
, *next
;
152 size
= cache
->size
* 3;
153 items
= (struct brw_cache_item
**) _mesa_calloc(size
* sizeof(*items
));
155 for (i
= 0; i
< cache
->size
; i
++)
156 for (c
= cache
->items
[i
]; c
; c
= next
) {
158 c
->next
= items
[c
->hash
% size
];
159 items
[c
->hash
% size
] = c
;
163 cache
->items
= items
;
169 * Returns the buffer object matching cache_id and key, or NULL.
172 brw_search_cache(struct brw_cache
*cache
,
173 enum brw_cache_id cache_id
,
176 dri_bo
**reloc_bufs
, GLuint nr_reloc_bufs
,
179 struct brw_cache_item
*item
;
180 struct brw_cache_item lookup
;
183 lookup
.cache_id
= cache_id
;
185 lookup
.key_size
= key_size
;
186 lookup
.reloc_bufs
= reloc_bufs
;
187 lookup
.nr_reloc_bufs
= nr_reloc_bufs
;
188 hash
= hash_key(&lookup
);
191 item
= search_cache(cache
, hash
, &lookup
);
197 *(void **)aux_return
= (void *)((char *)item
->key
+ item
->key_size
);
199 update_cache_last(cache
, cache_id
, item
->bo
);
201 dri_bo_reference(item
->bo
);
207 brw_upload_cache_with_auxdata(struct brw_cache
*cache
,
208 enum brw_cache_id cache_id
,
212 GLuint nr_reloc_bufs
,
219 struct brw_cache_item
*item
= CALLOC_STRUCT(brw_cache_item
);
221 GLuint relocs_size
= nr_reloc_bufs
* sizeof(dri_bo
*);
226 item
->cache_id
= cache_id
;
228 item
->key_size
= key_size
;
229 item
->reloc_bufs
= reloc_bufs
;
230 item
->nr_reloc_bufs
= nr_reloc_bufs
;
231 hash
= hash_key(item
);
234 /* Create the buffer object to contain the data */
235 bo
= dri_bo_alloc(cache
->brw
->intel
.bufmgr
,
236 cache
->name
[cache_id
], data_size
, 1 << 6);
239 /* Set up the memory containing the key, aux_data, and reloc_bufs */
240 tmp
= _mesa_malloc(key_size
+ aux_size
+ relocs_size
);
242 memcpy(tmp
, key
, key_size
);
243 memcpy(tmp
+ key_size
, aux
, aux_size
);
244 memcpy(tmp
+ key_size
+ aux_size
, reloc_bufs
, relocs_size
);
245 for (i
= 0; i
< nr_reloc_bufs
; i
++) {
246 if (reloc_bufs
[i
] != NULL
)
247 dri_bo_reference(reloc_bufs
[i
]);
251 item
->reloc_bufs
= tmp
+ key_size
+ aux_size
;
254 dri_bo_reference(bo
);
256 if (cache
->n_items
> cache
->size
* 1.5)
260 item
->next
= cache
->items
[hash
];
261 cache
->items
[hash
] = item
;
265 *(void **)aux_return
= (void *)((char *)item
->key
+ item
->key_size
);
268 if (INTEL_DEBUG
& DEBUG_STATE
)
269 _mesa_printf("upload %s: %d bytes to cache id %d\n",
270 cache
->name
[cache_id
],
271 data_size
, cache_id
);
273 /* Copy data to the buffer */
274 dri_bo_subdata(bo
, 0, data_size
, data
);
276 update_cache_last(cache
, cache_id
, bo
);
282 brw_upload_cache(struct brw_cache
*cache
,
283 enum brw_cache_id cache_id
,
287 GLuint nr_reloc_bufs
,
291 return brw_upload_cache_with_auxdata(cache
, cache_id
,
293 reloc_bufs
, nr_reloc_bufs
,
300 * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
302 * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
303 * better to use, as the potentially changing offsets in the data-used-as-key
304 * will result in excessive cache misses.
306 * If aux data is involved, use search/upload instead.
310 brw_cache_data(struct brw_cache
*cache
,
311 enum brw_cache_id cache_id
,
315 GLuint nr_reloc_bufs
)
318 struct brw_cache_item
*item
, lookup
;
321 lookup
.cache_id
= cache_id
;
323 lookup
.key_size
= data_size
;
324 lookup
.reloc_bufs
= reloc_bufs
;
325 lookup
.nr_reloc_bufs
= nr_reloc_bufs
;
326 hash
= hash_key(&lookup
);
329 item
= search_cache(cache
, hash
, &lookup
);
331 update_cache_last(cache
, cache_id
, item
->bo
);
332 dri_bo_reference(item
->bo
);
336 bo
= brw_upload_cache(cache
, cache_id
,
338 reloc_bufs
, nr_reloc_bufs
,
351 brw_init_cache_id(struct brw_cache
*cache
,
353 enum brw_cache_id id
)
355 cache
->name
[id
] = strdup(name
);
360 brw_init_non_surface_cache(struct brw_context
*brw
)
362 struct brw_cache
*cache
= &brw
->cache
;
368 cache
->items
= (struct brw_cache_item
**)
369 _mesa_calloc(cache
->size
* sizeof(struct brw_cache_item
));
371 brw_init_cache_id(cache
, "CC_VP", BRW_CC_VP
);
372 brw_init_cache_id(cache
, "CC_UNIT", BRW_CC_UNIT
);
373 brw_init_cache_id(cache
, "WM_PROG", BRW_WM_PROG
);
374 brw_init_cache_id(cache
, "SAMPLER_DEFAULT_COLOR", BRW_SAMPLER_DEFAULT_COLOR
);
375 brw_init_cache_id(cache
, "SAMPLER", BRW_SAMPLER
);
376 brw_init_cache_id(cache
, "WM_UNIT", BRW_WM_UNIT
);
377 brw_init_cache_id(cache
, "SF_PROG", BRW_SF_PROG
);
378 brw_init_cache_id(cache
, "SF_VP", BRW_SF_VP
);
380 brw_init_cache_id(cache
, "SF_UNIT", BRW_SF_UNIT
);
382 brw_init_cache_id(cache
, "VS_UNIT", BRW_VS_UNIT
);
384 brw_init_cache_id(cache
, "VS_PROG", BRW_VS_PROG
);
386 brw_init_cache_id(cache
, "CLIP_UNIT", BRW_CLIP_UNIT
);
388 brw_init_cache_id(cache
, "CLIP_PROG", BRW_CLIP_PROG
);
390 brw_init_cache_id(cache
, "GS_UNIT", BRW_GS_UNIT
);
392 brw_init_cache_id(cache
, "GS_PROG", BRW_GS_PROG
);
397 brw_init_surface_cache(struct brw_context
*brw
)
399 struct brw_cache
*cache
= &brw
->surface_cache
;
405 cache
->items
= (struct brw_cache_item
**)
406 _mesa_calloc(cache
->size
* sizeof(struct brw_cache_item
));
408 brw_init_cache_id(cache
, "SS_SURFACE", BRW_SS_SURFACE
);
409 brw_init_cache_id(cache
, "SS_SURF_BIND", BRW_SS_SURF_BIND
);
414 brw_init_caches(struct brw_context
*brw
)
416 brw_init_non_surface_cache(brw
);
417 brw_init_surface_cache(brw
);
422 brw_clear_cache(struct brw_context
*brw
, struct brw_cache
*cache
)
424 struct brw_cache_item
*c
, *next
;
427 if (INTEL_DEBUG
& DEBUG_STATE
)
428 _mesa_printf("%s\n", __FUNCTION__
);
430 for (i
= 0; i
< cache
->size
; i
++) {
431 for (c
= cache
->items
[i
]; c
; c
= next
) {
435 for (j
= 0; j
< c
->nr_reloc_bufs
; j
++)
436 dri_bo_unreference(c
->reloc_bufs
[j
]);
437 dri_bo_unreference(c
->bo
);
438 free((void *)c
->key
);
441 cache
->items
[i
] = NULL
;
446 if (brw
->curbe
.last_buf
) {
447 _mesa_free(brw
->curbe
.last_buf
);
448 brw
->curbe
.last_buf
= NULL
;
451 brw
->state
.dirty
.mesa
|= ~0;
452 brw
->state
.dirty
.brw
|= ~0;
453 brw
->state
.dirty
.cache
|= ~0;
456 /* Clear all entries from the cache that point to the given bo.
458 * This lets us release memory for reuse earlier for known-dead buffers,
459 * at the cost of walking the entire hash table.
462 brw_state_cache_bo_delete(struct brw_cache
*cache
, dri_bo
*bo
)
464 struct brw_cache_item
**prev
;
467 if (INTEL_DEBUG
& DEBUG_STATE
)
468 _mesa_printf("%s\n", __FUNCTION__
);
470 for (i
= 0; i
< cache
->size
; i
++) {
471 for (prev
= &cache
->items
[i
]; *prev
;) {
472 struct brw_cache_item
*c
= *prev
;
474 if (drm_intel_bo_references(c
->bo
, bo
)) {
479 for (j
= 0; j
< c
->nr_reloc_bufs
; j
++)
480 dri_bo_unreference(c
->reloc_bufs
[j
]);
481 dri_bo_unreference(c
->bo
);
482 free((void *)c
->key
);
493 brw_state_cache_check_size(struct brw_context
*brw
)
495 if (INTEL_DEBUG
& DEBUG_STATE
)
496 _mesa_printf("%s (n_items=%d)\n", __FUNCTION__
, brw
->cache
.n_items
);
498 /* un-tuned guess. We've got around 20 state objects for a total of around
499 * 32k, so 1000 of them is around 1.5MB.
501 if (brw
->cache
.n_items
> 1000)
502 brw_clear_cache(brw
, &brw
->cache
);
504 if (brw
->surface_cache
.n_items
> 1000)
505 brw_clear_cache(brw
, &brw
->surface_cache
);
510 brw_destroy_cache(struct brw_context
*brw
, struct brw_cache
*cache
)
514 if (INTEL_DEBUG
& DEBUG_STATE
)
515 _mesa_printf("%s\n", __FUNCTION__
);
517 brw_clear_cache(brw
, cache
);
518 for (i
= 0; i
< BRW_MAX_CACHE
; i
++) {
519 dri_bo_unreference(cache
->last_bo
[i
]);
520 free(cache
->name
[i
]);
529 brw_destroy_caches(struct brw_context
*brw
)
531 brw_destroy_cache(brw
, &brw
->cache
);
532 brw_destroy_cache(brw
, &brw
->surface_cache
);