2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keith@tungstengraphics.com>
32 /** @file brw_state_cache.c
34 * This file implements a simple static state cache for 965. The consumers
35 * can query the hash table of state using a cache_id, opaque key data,
36 * and list of buffers that will be used in relocations, and receive the
37 * corresponding state buffer object of state (plus associated auxiliary
40 * The inner workings are a simple hash table based on a CRC of the key data.
41 * The cache_id and relocation target buffers associated with the state
42 * buffer are included as auxiliary key data, but are not part of the hash
43 * value (this should be fixed, but will likely be fixed instead by making
44 * consumers use structured keys).
46 * Replacement is not implemented. Instead, when the cache gets too big, at
47 * a safe point (unlock) we throw out all of the cache data and let it
48 * regenerate for the next rendering operation.
50 * The reloc_buf pointers need to be included as key data, otherwise the
51 * non-unique values stuffed in the offset in key data through
52 * brw_cache_data() may result in successful probe for state buffers
53 * even when the buffer being referenced doesn't match. The result would be
54 * that the same state cache entry is used twice for different buffers,
55 * only one of the two buffers referenced gets put into the offset, and the
56 * incorrect program is run for the other instance.
59 #include "main/imports.h"
60 #include "brw_state.h"
61 #include "intel_batchbuffer.h"
63 /* XXX: Fixme - have to include these to get the sizes of the prog_key
74 hash_key(struct brw_cache_item
*item
)
76 GLuint
*ikey
= (GLuint
*)item
->key
;
77 GLuint hash
= item
->cache_id
, i
;
79 assert(item
->key_size
% 4 == 0);
81 /* I'm sure this can be improved on:
83 for (i
= 0; i
< item
->key_size
/4; i
++) {
85 hash
= (hash
<< 5) | (hash
>> 27);
88 /* Include the BO pointers as key data as well */
89 ikey
= (GLuint
*)item
->reloc_bufs
;
90 for (i
= 0; i
< item
->nr_reloc_bufs
* sizeof(drm_intel_bo
*) / 4; i
++) {
92 hash
= (hash
<< 5) | (hash
>> 27);
100 * Marks a new buffer as being chosen for the given cache id.
103 update_cache_last(struct brw_cache
*cache
, enum brw_cache_id cache_id
,
106 if (bo
== cache
->last_bo
[cache_id
])
107 return; /* no change */
109 dri_bo_unreference(cache
->last_bo
[cache_id
]);
110 cache
->last_bo
[cache_id
] = bo
;
111 dri_bo_reference(cache
->last_bo
[cache_id
]);
112 cache
->brw
->state
.dirty
.cache
|= 1 << cache_id
;
116 brw_cache_item_equals(const struct brw_cache_item
*a
,
117 const struct brw_cache_item
*b
)
119 return a
->cache_id
== b
->cache_id
&&
120 a
->hash
== b
->hash
&&
121 a
->key_size
== b
->key_size
&&
122 (memcmp(a
->key
, b
->key
, a
->key_size
) == 0) &&
123 a
->nr_reloc_bufs
== b
->nr_reloc_bufs
&&
124 (memcmp(a
->reloc_bufs
, b
->reloc_bufs
,
125 a
->nr_reloc_bufs
* sizeof(dri_bo
*)) == 0);
128 static struct brw_cache_item
*
129 search_cache(struct brw_cache
*cache
, GLuint hash
,
130 struct brw_cache_item
*lookup
)
132 struct brw_cache_item
*c
;
137 for (c
= cache
->items
[hash
% cache
->size
]; c
; c
= c
->next
)
140 fprintf(stderr
, "bucket %d/%d = %d/%d items\n", hash
% cache
->size
,
141 cache
->size
, bucketcount
, cache
->n_items
);
144 for (c
= cache
->items
[hash
% cache
->size
]; c
; c
= c
->next
) {
145 if (brw_cache_item_equals(lookup
, c
))
154 rehash(struct brw_cache
*cache
)
156 struct brw_cache_item
**items
;
157 struct brw_cache_item
*c
, *next
;
160 size
= cache
->size
* 3;
161 items
= (struct brw_cache_item
**) _mesa_calloc(size
* sizeof(*items
));
163 for (i
= 0; i
< cache
->size
; i
++)
164 for (c
= cache
->items
[i
]; c
; c
= next
) {
166 c
->next
= items
[c
->hash
% size
];
167 items
[c
->hash
% size
] = c
;
171 cache
->items
= items
;
177 * Returns the buffer object matching cache_id and key, or NULL.
180 brw_search_cache(struct brw_cache
*cache
,
181 enum brw_cache_id cache_id
,
184 dri_bo
**reloc_bufs
, GLuint nr_reloc_bufs
,
187 struct brw_cache_item
*item
;
188 struct brw_cache_item lookup
;
191 lookup
.cache_id
= cache_id
;
193 lookup
.key_size
= key_size
;
194 lookup
.reloc_bufs
= reloc_bufs
;
195 lookup
.nr_reloc_bufs
= nr_reloc_bufs
;
196 hash
= hash_key(&lookup
);
199 item
= search_cache(cache
, hash
, &lookup
);
205 *(void **)aux_return
= (void *)((char *)item
->key
+ item
->key_size
);
207 update_cache_last(cache
, cache_id
, item
->bo
);
209 dri_bo_reference(item
->bo
);
215 brw_upload_cache_with_auxdata(struct brw_cache
*cache
,
216 enum brw_cache_id cache_id
,
220 GLuint nr_reloc_bufs
,
227 struct brw_cache_item
*item
= CALLOC_STRUCT(brw_cache_item
);
229 GLuint relocs_size
= nr_reloc_bufs
* sizeof(dri_bo
*);
234 item
->cache_id
= cache_id
;
236 item
->key_size
= key_size
;
237 item
->reloc_bufs
= reloc_bufs
;
238 item
->nr_reloc_bufs
= nr_reloc_bufs
;
239 hash
= hash_key(item
);
242 /* Create the buffer object to contain the data */
243 bo
= dri_bo_alloc(cache
->brw
->intel
.bufmgr
,
244 cache
->name
[cache_id
], data_size
, 1 << 6);
247 /* Set up the memory containing the key, aux_data, and reloc_bufs */
248 tmp
= _mesa_malloc(key_size
+ aux_size
+ relocs_size
);
250 memcpy(tmp
, key
, key_size
);
251 memcpy(tmp
+ key_size
, aux
, aux_size
);
252 memcpy(tmp
+ key_size
+ aux_size
, reloc_bufs
, relocs_size
);
253 for (i
= 0; i
< nr_reloc_bufs
; i
++) {
254 if (reloc_bufs
[i
] != NULL
)
255 dri_bo_reference(reloc_bufs
[i
]);
259 item
->reloc_bufs
= tmp
+ key_size
+ aux_size
;
262 dri_bo_reference(bo
);
264 if (cache
->n_items
> cache
->size
* 1.5)
268 item
->next
= cache
->items
[hash
];
269 cache
->items
[hash
] = item
;
273 *(void **)aux_return
= (void *)((char *)item
->key
+ item
->key_size
);
276 if (INTEL_DEBUG
& DEBUG_STATE
)
277 _mesa_printf("upload %s: %d bytes to cache id %d\n",
278 cache
->name
[cache_id
],
279 data_size
, cache_id
);
281 /* Copy data to the buffer */
282 dri_bo_subdata(bo
, 0, data_size
, data
);
284 update_cache_last(cache
, cache_id
, bo
);
290 brw_upload_cache(struct brw_cache
*cache
,
291 enum brw_cache_id cache_id
,
295 GLuint nr_reloc_bufs
,
299 return brw_upload_cache_with_auxdata(cache
, cache_id
,
301 reloc_bufs
, nr_reloc_bufs
,
308 * Wrapper around brw_cache_data_sz using the cache_id's canonical key size.
310 * If nr_reloc_bufs is nonzero, brw_search_cache()/brw_upload_cache() would be
311 * better to use, as the potentially changing offsets in the data-used-as-key
312 * will result in excessive cache misses.
314 * If aux data is involved, use search/upload instead.
318 brw_cache_data(struct brw_cache
*cache
,
319 enum brw_cache_id cache_id
,
323 GLuint nr_reloc_bufs
)
326 struct brw_cache_item
*item
, lookup
;
329 lookup
.cache_id
= cache_id
;
331 lookup
.key_size
= data_size
;
332 lookup
.reloc_bufs
= reloc_bufs
;
333 lookup
.nr_reloc_bufs
= nr_reloc_bufs
;
334 hash
= hash_key(&lookup
);
337 item
= search_cache(cache
, hash
, &lookup
);
339 update_cache_last(cache
, cache_id
, item
->bo
);
340 dri_bo_reference(item
->bo
);
344 bo
= brw_upload_cache(cache
, cache_id
,
346 reloc_bufs
, nr_reloc_bufs
,
359 brw_init_cache_id(struct brw_cache
*cache
,
361 enum brw_cache_id id
)
363 cache
->name
[id
] = strdup(name
);
368 brw_init_non_surface_cache(struct brw_context
*brw
)
370 struct brw_cache
*cache
= &brw
->cache
;
376 cache
->items
= (struct brw_cache_item
**)
377 _mesa_calloc(cache
->size
* sizeof(struct brw_cache_item
));
379 brw_init_cache_id(cache
, "CC_VP", BRW_CC_VP
);
380 brw_init_cache_id(cache
, "CC_UNIT", BRW_CC_UNIT
);
381 brw_init_cache_id(cache
, "WM_PROG", BRW_WM_PROG
);
382 brw_init_cache_id(cache
, "SAMPLER_DEFAULT_COLOR", BRW_SAMPLER_DEFAULT_COLOR
);
383 brw_init_cache_id(cache
, "SAMPLER", BRW_SAMPLER
);
384 brw_init_cache_id(cache
, "WM_UNIT", BRW_WM_UNIT
);
385 brw_init_cache_id(cache
, "SF_PROG", BRW_SF_PROG
);
386 brw_init_cache_id(cache
, "SF_VP", BRW_SF_VP
);
388 brw_init_cache_id(cache
, "SF_UNIT", BRW_SF_UNIT
);
390 brw_init_cache_id(cache
, "VS_UNIT", BRW_VS_UNIT
);
392 brw_init_cache_id(cache
, "VS_PROG", BRW_VS_PROG
);
394 brw_init_cache_id(cache
, "CLIP_UNIT", BRW_CLIP_UNIT
);
396 brw_init_cache_id(cache
, "CLIP_PROG", BRW_CLIP_PROG
);
398 brw_init_cache_id(cache
, "GS_UNIT", BRW_GS_UNIT
);
400 brw_init_cache_id(cache
, "GS_PROG", BRW_GS_PROG
);
405 brw_init_surface_cache(struct brw_context
*brw
)
407 struct brw_cache
*cache
= &brw
->surface_cache
;
413 cache
->items
= (struct brw_cache_item
**)
414 _mesa_calloc(cache
->size
* sizeof(struct brw_cache_item
));
416 brw_init_cache_id(cache
, "SS_SURFACE", BRW_SS_SURFACE
);
417 brw_init_cache_id(cache
, "SS_SURF_BIND", BRW_SS_SURF_BIND
);
422 brw_init_caches(struct brw_context
*brw
)
424 brw_init_non_surface_cache(brw
);
425 brw_init_surface_cache(brw
);
430 brw_clear_cache(struct brw_context
*brw
, struct brw_cache
*cache
)
432 struct brw_cache_item
*c
, *next
;
435 if (INTEL_DEBUG
& DEBUG_STATE
)
436 _mesa_printf("%s\n", __FUNCTION__
);
438 for (i
= 0; i
< cache
->size
; i
++) {
439 for (c
= cache
->items
[i
]; c
; c
= next
) {
443 for (j
= 0; j
< c
->nr_reloc_bufs
; j
++)
444 dri_bo_unreference(c
->reloc_bufs
[j
]);
445 dri_bo_unreference(c
->bo
);
446 free((void *)c
->key
);
449 cache
->items
[i
] = NULL
;
454 if (brw
->curbe
.last_buf
) {
455 _mesa_free(brw
->curbe
.last_buf
);
456 brw
->curbe
.last_buf
= NULL
;
459 brw
->state
.dirty
.mesa
|= ~0;
460 brw
->state
.dirty
.brw
|= ~0;
461 brw
->state
.dirty
.cache
|= ~0;
464 /* Clear all entries from the cache that point to the given bo.
466 * This lets us release memory for reuse earlier for known-dead buffers,
467 * at the cost of walking the entire hash table.
470 brw_state_cache_bo_delete(struct brw_cache
*cache
, dri_bo
*bo
)
472 struct brw_cache_item
**prev
;
475 if (INTEL_DEBUG
& DEBUG_STATE
)
476 _mesa_printf("%s\n", __FUNCTION__
);
478 for (i
= 0; i
< cache
->size
; i
++) {
479 for (prev
= &cache
->items
[i
]; *prev
;) {
480 struct brw_cache_item
*c
= *prev
;
482 if (drm_intel_bo_references(c
->bo
, bo
)) {
487 for (j
= 0; j
< c
->nr_reloc_bufs
; j
++)
488 dri_bo_unreference(c
->reloc_bufs
[j
]);
489 dri_bo_unreference(c
->bo
);
490 free((void *)c
->key
);
501 brw_state_cache_check_size(struct brw_context
*brw
)
503 if (INTEL_DEBUG
& DEBUG_STATE
)
504 _mesa_printf("%s (n_items=%d)\n", __FUNCTION__
, brw
->cache
.n_items
);
506 /* un-tuned guess. We've got around 20 state objects for a total of around
507 * 32k, so 1000 of them is around 1.5MB.
509 if (brw
->cache
.n_items
> 1000)
510 brw_clear_cache(brw
, &brw
->cache
);
512 if (brw
->surface_cache
.n_items
> 1000)
513 brw_clear_cache(brw
, &brw
->surface_cache
);
518 brw_destroy_cache(struct brw_context
*brw
, struct brw_cache
*cache
)
522 if (INTEL_DEBUG
& DEBUG_STATE
)
523 _mesa_printf("%s\n", __FUNCTION__
);
525 brw_clear_cache(brw
, cache
);
526 for (i
= 0; i
< BRW_MAX_CACHE
; i
++) {
527 dri_bo_unreference(cache
->last_bo
[i
]);
528 free(cache
->name
[i
]);
537 brw_destroy_caches(struct brw_context
*brw
)
539 brw_destroy_cache(brw
, &brw
->cache
);
540 brw_destroy_cache(brw
, &brw
->surface_cache
);