2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keith@tungstengraphics.com>
32 /** @file brw_state_cache.c
34 * This file implements a simple static state cache for 965. The consumers
35 * can query the hash table of state using a cache_id, opaque key data,
36 * and list of buffers that will be used in relocations, and receive the
37 * corresponding state buffer object of state (plus associated auxiliary
40 * The inner workings are a simple hash table based on a CRC of the key data.
41 * The cache_id and relocation target buffers associated with the state
42 * buffer are included as auxiliary key data, but are not part of the hash
43 * value (this should be fixed, but will likely be fixed instead by making
44 * consumers use structured keys).
46 * Replacement is not implemented. Instead, when the cache gets too big, at
47 * a safe point (unlock) we throw out all of the cache data and let it
48 * regenerate for the next rendering operation.
50 * The reloc_buf pointers need to be included as key data, otherwise the
51 * non-unique values stuffed in the offset in key data through
52 * brw_cache_data() may result in successful probe for state buffers
53 * even when the buffer being referenced doesn't match. The result would be
54 * that the same state cache entry is used twice for different buffers,
55 * only one of the two buffers referenced gets put into the offset, and the
56 * incorrect program is run for the other instance.
59 #include "main/imports.h"
60 #include "brw_state.h"
61 #include "intel_batchbuffer.h"
63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
74 hash_key(const void *key
, GLuint key_size
,
75 dri_bo
**reloc_bufs
, GLuint nr_reloc_bufs
)
77 GLuint
*ikey
= (GLuint
*)key
;
80 assert(key_size
% 4 == 0);
82 /* I'm sure this can be improved on:
84 for (i
= 0; i
< key_size
/4; i
++) {
86 hash
= (hash
<< 5) | (hash
>> 27);
89 /* Include the BO pointers as key data as well */
90 ikey
= (GLuint
*)reloc_bufs
;
91 key_size
= nr_reloc_bufs
* sizeof(dri_bo
*);
92 for (i
= 0; i
< key_size
/4; i
++) {
94 hash
= (hash
<< 5) | (hash
>> 27);
102 * Marks a new buffer as being chosen for the given cache id.
105 update_cache_last(struct brw_cache
*cache
, enum brw_cache_id cache_id
,
108 if (bo
== cache
->last_bo
[cache_id
])
109 return; /* no change */
111 dri_bo_unreference(cache
->last_bo
[cache_id
]);
112 cache
->last_bo
[cache_id
] = bo
;
113 dri_bo_reference(cache
->last_bo
[cache_id
]);
114 cache
->brw
->state
.dirty
.cache
|= 1 << cache_id
;
118 static struct brw_cache_item
*
119 search_cache(struct brw_cache
*cache
, enum brw_cache_id cache_id
,
120 GLuint hash
, const void *key
, GLuint key_size
,
121 dri_bo
**reloc_bufs
, GLuint nr_reloc_bufs
)
123 struct brw_cache_item
*c
;
128 for (c
= cache
->items
[hash
% cache
->size
]; c
; c
= c
->next
)
131 fprintf(stderr
, "bucket %d/%d = %d/%d items\n", hash
% cache
->size
,
132 cache
->size
, bucketcount
, cache
->n_items
);
135 for (c
= cache
->items
[hash
% cache
->size
]; c
; c
= c
->next
) {
136 if (c
->cache_id
== cache_id
&&
138 c
->key_size
== key_size
&&
139 memcmp(c
->key
, key
, key_size
) == 0 &&
140 c
->nr_reloc_bufs
== nr_reloc_bufs
&&
141 memcmp(c
->reloc_bufs
, reloc_bufs
,
142 nr_reloc_bufs
* sizeof(dri_bo
*)) == 0)
151 rehash(struct brw_cache
*cache
)
153 struct brw_cache_item
**items
;
154 struct brw_cache_item
*c
, *next
;
157 size
= cache
->size
* 3;
158 items
= (struct brw_cache_item
**) _mesa_calloc(size
* sizeof(*items
));
160 for (i
= 0; i
< cache
->size
; i
++)
161 for (c
= cache
->items
[i
]; c
; c
= next
) {
163 c
->next
= items
[c
->hash
% size
];
164 items
[c
->hash
% size
] = c
;
168 cache
->items
= items
;
174 * Returns the buffer object matching cache_id and key, or NULL.
177 brw_search_cache(struct brw_cache
*cache
,
178 enum brw_cache_id cache_id
,
181 dri_bo
**reloc_bufs
, GLuint nr_reloc_bufs
,
184 struct brw_cache_item
*item
;
185 GLuint hash
= hash_key(key
, key_size
, reloc_bufs
, nr_reloc_bufs
);
187 item
= search_cache(cache
, cache_id
, hash
, key
, key_size
,
188 reloc_bufs
, nr_reloc_bufs
);
194 *(void **)aux_return
= (void *)((char *)item
->key
+ item
->key_size
);
196 update_cache_last(cache
, cache_id
, item
->bo
);
198 dri_bo_reference(item
->bo
);
204 brw_upload_cache_with_auxdata(struct brw_cache
*cache
,
205 enum brw_cache_id cache_id
,
209 GLuint nr_reloc_bufs
,
216 struct brw_cache_item
*item
= CALLOC_STRUCT(brw_cache_item
);
217 GLuint hash
= hash_key(key
, key_size
, reloc_bufs
, nr_reloc_bufs
);
218 GLuint relocs_size
= nr_reloc_bufs
* sizeof(dri_bo
*);
223 /* Create the buffer object to contain the data */
224 bo
= dri_bo_alloc(cache
->brw
->intel
.bufmgr
,
225 cache
->name
[cache_id
], data_size
, 1 << 6);
228 /* Set up the memory containing the key, aux_data, and reloc_bufs */
229 tmp
= _mesa_malloc(key_size
+ aux_size
+ relocs_size
);
231 memcpy(tmp
, key
, key_size
);
232 memcpy(tmp
+ key_size
, aux
, aux_size
);
233 memcpy(tmp
+ key_size
+ aux_size
, reloc_bufs
, relocs_size
);
234 for (i
= 0; i
< nr_reloc_bufs
; i
++) {
235 if (reloc_bufs
[i
] != NULL
)
236 dri_bo_reference(reloc_bufs
[i
]);
239 item
->cache_id
= cache_id
;
242 item
->key_size
= key_size
;
243 item
->reloc_bufs
= tmp
+ key_size
+ aux_size
;
244 item
->nr_reloc_bufs
= nr_reloc_bufs
;
247 dri_bo_reference(bo
);
249 if (cache
->n_items
> cache
->size
* 1.5)
253 item
->next
= cache
->items
[hash
];
254 cache
->items
[hash
] = item
;
258 *(void **)aux_return
= (void *)((char *)item
->key
+ item
->key_size
);
261 if (INTEL_DEBUG
& DEBUG_STATE
)
262 _mesa_printf("upload %s: %d bytes to cache id %d\n",
263 cache
->name
[cache_id
],
264 data_size
, cache_id
);
266 /* Copy data to the buffer */
267 dri_bo_subdata(bo
, 0, data_size
, data
);
269 update_cache_last(cache
, cache_id
, bo
);
275 brw_upload_cache(struct brw_cache
*cache
,
276 enum brw_cache_id cache_id
,
280 GLuint nr_reloc_bufs
,
284 return brw_upload_cache_with_auxdata(cache
, cache_id
,
286 reloc_bufs
, nr_reloc_bufs
,
293 * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
295 * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
296 * better to use, as the potentially changing offsets in the data-used-as-key
297 * will result in excessive cache misses.
299 * If aux data is involved, use search/upload instead.
303 brw_cache_data(struct brw_cache
*cache
,
304 enum brw_cache_id cache_id
,
308 GLuint nr_reloc_bufs
)
311 struct brw_cache_item
*item
;
312 GLuint hash
= hash_key(data
, data_size
, reloc_bufs
, nr_reloc_bufs
);
314 item
= search_cache(cache
, cache_id
, hash
, data
, data_size
,
315 reloc_bufs
, nr_reloc_bufs
);
317 update_cache_last(cache
, cache_id
, item
->bo
);
318 dri_bo_reference(item
->bo
);
322 bo
= brw_upload_cache(cache
, cache_id
,
324 reloc_bufs
, nr_reloc_bufs
,
337 brw_init_cache_id(struct brw_cache
*cache
,
339 enum brw_cache_id id
)
341 cache
->name
[id
] = strdup(name
);
346 brw_init_non_surface_cache(struct brw_context
*brw
)
348 struct brw_cache
*cache
= &brw
->cache
;
354 cache
->items
= (struct brw_cache_item
**)
355 _mesa_calloc(cache
->size
* sizeof(struct brw_cache_item
));
357 brw_init_cache_id(cache
, "CC_VP", BRW_CC_VP
);
358 brw_init_cache_id(cache
, "CC_UNIT", BRW_CC_UNIT
);
359 brw_init_cache_id(cache
, "WM_PROG", BRW_WM_PROG
);
360 brw_init_cache_id(cache
, "SAMPLER_DEFAULT_COLOR", BRW_SAMPLER_DEFAULT_COLOR
);
361 brw_init_cache_id(cache
, "SAMPLER", BRW_SAMPLER
);
362 brw_init_cache_id(cache
, "WM_UNIT", BRW_WM_UNIT
);
363 brw_init_cache_id(cache
, "SF_PROG", BRW_SF_PROG
);
364 brw_init_cache_id(cache
, "SF_VP", BRW_SF_VP
);
366 brw_init_cache_id(cache
, "SF_UNIT", BRW_SF_UNIT
);
368 brw_init_cache_id(cache
, "VS_UNIT", BRW_VS_UNIT
);
370 brw_init_cache_id(cache
, "VS_PROG", BRW_VS_PROG
);
372 brw_init_cache_id(cache
, "CLIP_UNIT", BRW_CLIP_UNIT
);
374 brw_init_cache_id(cache
, "CLIP_PROG", BRW_CLIP_PROG
);
376 brw_init_cache_id(cache
, "GS_UNIT", BRW_GS_UNIT
);
378 brw_init_cache_id(cache
, "GS_PROG", BRW_GS_PROG
);
383 brw_init_surface_cache(struct brw_context
*brw
)
385 struct brw_cache
*cache
= &brw
->surface_cache
;
391 cache
->items
= (struct brw_cache_item
**)
392 _mesa_calloc(cache
->size
* sizeof(struct brw_cache_item
));
394 brw_init_cache_id(cache
, "SS_SURFACE", BRW_SS_SURFACE
);
395 brw_init_cache_id(cache
, "SS_SURF_BIND", BRW_SS_SURF_BIND
);
400 brw_init_caches(struct brw_context
*brw
)
402 brw_init_non_surface_cache(brw
);
403 brw_init_surface_cache(brw
);
408 brw_clear_cache(struct brw_context
*brw
, struct brw_cache
*cache
)
410 struct brw_cache_item
*c
, *next
;
413 if (INTEL_DEBUG
& DEBUG_STATE
)
414 _mesa_printf("%s\n", __FUNCTION__
);
416 for (i
= 0; i
< cache
->size
; i
++) {
417 for (c
= cache
->items
[i
]; c
; c
= next
) {
421 for (j
= 0; j
< c
->nr_reloc_bufs
; j
++)
422 dri_bo_unreference(c
->reloc_bufs
[j
]);
423 dri_bo_unreference(c
->bo
);
424 free((void *)c
->key
);
427 cache
->items
[i
] = NULL
;
432 if (brw
->curbe
.last_buf
) {
433 _mesa_free(brw
->curbe
.last_buf
);
434 brw
->curbe
.last_buf
= NULL
;
437 brw
->state
.dirty
.mesa
|= ~0;
438 brw
->state
.dirty
.brw
|= ~0;
439 brw
->state
.dirty
.cache
|= ~0;
442 /* Clear all entries from the cache that point to the given bo.
444 * This lets us release memory for reuse earlier for known-dead buffers,
445 * at the cost of walking the entire hash table.
448 brw_state_cache_bo_delete(struct brw_cache
*cache
, dri_bo
*bo
)
450 struct brw_cache_item
**prev
;
453 if (INTEL_DEBUG
& DEBUG_STATE
)
454 _mesa_printf("%s\n", __FUNCTION__
);
456 for (i
= 0; i
< cache
->size
; i
++) {
457 for (prev
= &cache
->items
[i
]; *prev
;) {
458 struct brw_cache_item
*c
= *prev
;
460 if (drm_intel_bo_references(c
->bo
, bo
)) {
465 for (j
= 0; j
< c
->nr_reloc_bufs
; j
++)
466 dri_bo_unreference(c
->reloc_bufs
[j
]);
467 dri_bo_unreference(c
->bo
);
468 free((void *)c
->key
);
479 brw_state_cache_check_size(struct brw_context
*brw
)
481 if (INTEL_DEBUG
& DEBUG_STATE
)
482 _mesa_printf("%s (n_items=%d)\n", __FUNCTION__
, brw
->cache
.n_items
);
484 /* un-tuned guess. We've got around 20 state objects for a total of around
485 * 32k, so 1000 of them is around 1.5MB.
487 if (brw
->cache
.n_items
> 1000)
488 brw_clear_cache(brw
, &brw
->cache
);
490 if (brw
->surface_cache
.n_items
> 1000)
491 brw_clear_cache(brw
, &brw
->surface_cache
);
496 brw_destroy_cache(struct brw_context
*brw
, struct brw_cache
*cache
)
500 if (INTEL_DEBUG
& DEBUG_STATE
)
501 _mesa_printf("%s\n", __FUNCTION__
);
503 brw_clear_cache(brw
, cache
);
504 for (i
= 0; i
< BRW_MAX_CACHE
; i
++) {
505 dri_bo_unreference(cache
->last_bo
[i
]);
506 free(cache
->name
[i
]);
515 brw_destroy_caches(struct brw_context
*brw
)
517 brw_destroy_cache(brw
, &brw
->cache
);
518 brw_destroy_cache(brw
, &brw
->surface_cache
);