From 91f9bb99c5e0db38a1ac26dd9e6a967312f5e8b0 Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Tue, 21 Jan 2020 14:28:06 -0800 Subject: [PATCH] freedreno: add gmem state cache Which also has the benefit of getting rid of fd_context::gmem. Signed-off-by: Rob Clark Part-of: --- .../drivers/freedreno/freedreno_context.h | 8 - .../drivers/freedreno/freedreno_gmem.c | 314 +++++++++++++----- .../drivers/freedreno/freedreno_gmem.h | 32 +- .../drivers/freedreno/freedreno_screen.c | 2 + .../drivers/freedreno/freedreno_screen.h | 2 + 5 files changed, 256 insertions(+), 102 deletions(-) diff --git a/src/gallium/drivers/freedreno/freedreno_context.h b/src/gallium/drivers/freedreno/freedreno_context.h index 1a60f978597..d10de1a8504 100644 --- a/src/gallium/drivers/freedreno/freedreno_context.h +++ b/src/gallium/drivers/freedreno/freedreno_context.h @@ -259,14 +259,6 @@ struct fd_context { */ struct pipe_scissor_state disabled_scissor; - /* Current gmem/tiling configuration.. gets updated on render_tiles() - * if out of date with current maximal-scissor/cpp: - * - * (NOTE: this is kind of related to the batch, but moving it there - * means we'd always have to recalc tiles ever batch) - */ - struct fd_gmem_stateobj gmem; - /* Per vsc pipe bo's (a2xx-a5xx): */ struct fd_bo *vsc_pipe_bo[32]; diff --git a/src/gallium/drivers/freedreno/freedreno_gmem.c b/src/gallium/drivers/freedreno/freedreno_gmem.c index 840ba86d6eb..3a14590dff5 100644 --- a/src/gallium/drivers/freedreno/freedreno_gmem.c +++ b/src/gallium/drivers/freedreno/freedreno_gmem.c @@ -25,6 +25,7 @@ */ #include "pipe/p_state.h" +#include "util/hash_table.h" #include "util/u_string.h" #include "util/u_memory.h" #include "util/u_inlines.h" @@ -68,6 +69,44 @@ #define BIN_DEBUG 0 +/* + * GMEM Cache: + * + * Caches GMEM state based on a given framebuffer state. The key is + * meant to be the minimal set of data that results in a unique gmem + * configuration, avoiding multiple keys arriving at the same gmem + * state. For example, the render target format is not part of the + * key, only the size per pixel. And the max_scissor bounds is not + * part of they key, only the minx/miny (after clamping to tile + * alignment) and width/height. This ensures that slightly different + * max_scissor which would result in the same gmem state, do not + * become different keys that map to the same state. + */ + +struct gmem_key { + uint16_t minx, miny; + uint16_t width, height; + uint8_t gmem_page_align; /* alignment in multiples of 0x1000 to reduce key size */ + uint8_t nr_cbufs; + uint8_t cbuf_cpp[MAX_RENDER_TARGETS]; + uint8_t zsbuf_cpp[2]; +}; + +static uint32_t +gmem_key_hash(const void *_key) +{ + const struct gmem_key *key = _key; + return _mesa_hash_data(key, sizeof(*key)); +} + +static bool +gmem_key_equals(const void *_a, const void *_b) +{ + const struct gmem_key *a = _a; + const struct gmem_key *b = _b; + return memcmp(a, b, sizeof(*a)) == 0; +} + static uint32_t bin_width(struct fd_screen *screen) { if (is_a4xx(screen) || is_a5xx(screen) || is_a6xx(screen)) @@ -78,148 +117,97 @@ static uint32_t bin_width(struct fd_screen *screen) } static uint32_t -total_size(uint8_t cbuf_cpp[], uint8_t zsbuf_cpp[2], - uint32_t bin_w, uint32_t bin_h, uint32_t gmem_align, - struct fd_gmem_stateobj *gmem) +total_size(struct gmem_key *key, uint32_t bin_w, uint32_t bin_h, + struct fd_gmem_stateobj *gmem) { + uint32_t gmem_align = key->gmem_page_align * 0x1000; uint32_t total = 0, i; for (i = 0; i < MAX_RENDER_TARGETS; i++) { - if (cbuf_cpp[i]) { + if (key->cbuf_cpp[i]) { gmem->cbuf_base[i] = align(total, gmem_align); - total = gmem->cbuf_base[i] + cbuf_cpp[i] * bin_w * bin_h; + total = gmem->cbuf_base[i] + key->cbuf_cpp[i] * bin_w * bin_h; } } - if (zsbuf_cpp[0]) { + if (key->zsbuf_cpp[0]) { gmem->zsbuf_base[0] = align(total, gmem_align); - total = gmem->zsbuf_base[0] + zsbuf_cpp[0] * bin_w * bin_h; + total = gmem->zsbuf_base[0] + key->zsbuf_cpp[0] * bin_w * bin_h; } - if (zsbuf_cpp[1]) { + if (key->zsbuf_cpp[1]) { gmem->zsbuf_base[1] = align(total, gmem_align); - total = gmem->zsbuf_base[1] + zsbuf_cpp[1] * bin_w * bin_h; + total = gmem->zsbuf_base[1] + key->zsbuf_cpp[1] * bin_w * bin_h; } return total; } -static void -calculate_tiles(struct fd_batch *batch) +static struct fd_gmem_stateobj * +gmem_stateobj_init(struct fd_screen *screen, struct gmem_key *key) { - struct fd_context *ctx = batch->ctx; - struct fd_screen *screen = ctx->screen; - struct fd_gmem_stateobj *gmem = &ctx->gmem; - struct pipe_scissor_state *scissor = &batch->max_scissor; - struct pipe_framebuffer_state *pfb = &batch->framebuffer; + struct fd_gmem_stateobj *gmem = + rzalloc(screen->gmem_cache.ht, struct fd_gmem_stateobj); + pipe_reference_init(&gmem->reference, 1); + gmem->screen = screen; + gmem->key = key; + list_inithead(&gmem->node); + const uint32_t gmem_alignw = screen->gmem_alignw; const uint32_t gmem_alignh = screen->gmem_alignh; const unsigned npipes = screen->num_vsc_pipes; const uint32_t gmem_size = screen->gmemsize_bytes; - uint32_t minx, miny, width, height; uint32_t nbins_x = 1, nbins_y = 1; uint32_t bin_w, bin_h; - uint32_t gmem_align = 0x4000; uint32_t max_width = bin_width(screen); - uint8_t cbuf_cpp[MAX_RENDER_TARGETS] = {0}, zsbuf_cpp[2] = {0}; uint32_t i, j, t, xoff, yoff; uint32_t tpp_x, tpp_y; - bool has_zs = !!(batch->gmem_reason & (FD_GMEM_DEPTH_ENABLED | - FD_GMEM_STENCIL_ENABLED | FD_GMEM_CLEARS_DEPTH_STENCIL)); int tile_n[npipes]; - if (has_zs) { - struct fd_resource *rsc = fd_resource(pfb->zsbuf->texture); - zsbuf_cpp[0] = rsc->layout.cpp; - if (rsc->stencil) - zsbuf_cpp[1] = rsc->stencil->layout.cpp; - } else { - /* we might have a zsbuf, but it isn't used */ - batch->restore &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL); - batch->resolve &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL); - } - for (i = 0; i < pfb->nr_cbufs; i++) { - if (pfb->cbufs[i]) - cbuf_cpp[i] = util_format_get_blocksize(pfb->cbufs[i]->format); - else - cbuf_cpp[i] = 4; - /* if MSAA, color buffers are super-sampled in GMEM: */ - cbuf_cpp[i] *= pfb->samples; - } - - if (!memcmp(gmem->zsbuf_cpp, zsbuf_cpp, sizeof(zsbuf_cpp)) && - !memcmp(gmem->cbuf_cpp, cbuf_cpp, sizeof(cbuf_cpp)) && - !memcmp(&gmem->scissor, scissor, sizeof(gmem->scissor))) { - /* everything is up-to-date */ - return; - } - - if (fd_mesa_debug & FD_DBG_NOSCIS) { - minx = 0; - miny = 0; - width = pfb->width; - height = pfb->height; - } else { - /* round down to multiple of alignment: */ - minx = scissor->minx & ~(gmem_alignw - 1); - miny = scissor->miny & ~(gmem_alignh - 1); - width = scissor->maxx - minx; - height = scissor->maxy - miny; - } - - bin_w = align(width, gmem_alignw); - bin_h = align(height, gmem_alignh); + bin_w = align(key->width, gmem_alignw); + bin_h = align(key->height, gmem_alignh); /* first, find a bin width that satisfies the maximum width * restrictions: */ while (bin_w > max_width) { nbins_x++; - bin_w = align(width / nbins_x, gmem_alignw); + bin_w = align(key->width / nbins_x, gmem_alignw); } if (fd_mesa_debug & FD_DBG_MSGS) { debug_printf("binning input: cbuf cpp:"); - for (i = 0; i < pfb->nr_cbufs; i++) - debug_printf(" %d", cbuf_cpp[i]); + for (i = 0; i < key->nr_cbufs; i++) + debug_printf(" %d", key->cbuf_cpp[i]); debug_printf(", zsbuf cpp: %d; %dx%d\n", - zsbuf_cpp[0], width, height); - } - - if (is_a20x(screen) && batch->cleared) { - /* under normal circumstances the requirement would be 4K - * but the fast clear path requires an alignment of 32K - */ - gmem_align = 0x8000; + key->zsbuf_cpp[0], key->width, key->height); } /* then find a bin width/height that satisfies the memory * constraints: */ - while (total_size(cbuf_cpp, zsbuf_cpp, bin_w, bin_h, gmem_align, gmem) > - gmem_size) { + while (total_size(key, bin_w, bin_h, gmem) > gmem_size) { if (bin_w > bin_h) { nbins_x++; - bin_w = align(width / nbins_x, gmem_alignw); + bin_w = align(key->width / nbins_x, gmem_alignw); } else { nbins_y++; - bin_h = align(height / nbins_y, gmem_alignh); + bin_h = align(key->height / nbins_y, gmem_alignh); } } DBG("using %d bins of size %dx%d", nbins_x*nbins_y, bin_w, bin_h); - gmem->scissor = *scissor; - memcpy(gmem->cbuf_cpp, cbuf_cpp, sizeof(cbuf_cpp)); - memcpy(gmem->zsbuf_cpp, zsbuf_cpp, sizeof(zsbuf_cpp)); + memcpy(gmem->cbuf_cpp, key->cbuf_cpp, sizeof(key->cbuf_cpp)); + memcpy(gmem->zsbuf_cpp, key->zsbuf_cpp, sizeof(key->zsbuf_cpp)); gmem->bin_h = bin_h; gmem->bin_w = bin_w; gmem->nbins_x = nbins_x; gmem->nbins_y = nbins_y; - gmem->minx = minx; - gmem->miny = miny; - gmem->width = width; - gmem->height = height; + gmem->minx = key->minx; + gmem->miny = key->miny; + gmem->width = key->width; + gmem->height = key->height; /* * Assign tiles and pipes: @@ -231,7 +219,7 @@ calculate_tiles(struct fd_batch *batch) #define div_round_up(v, a) (((v) + (a) - 1) / (a)) /* figure out number of tiles per pipe: */ - if (is_a20x(ctx->screen)) { + if (is_a20x(screen)) { /* for a20x we want to minimize the number of "pipes" * binning data has 3 bits for x/y (8x8) but the edges are used to * cull off-screen vertices with hw binning, so we have 6x6 pipes @@ -291,15 +279,15 @@ calculate_tiles(struct fd_batch *batch) /* configure tiles: */ t = 0; - yoff = miny; + yoff = key->miny; memset(tile_n, 0, sizeof(tile_n)); for (i = 0; i < nbins_y; i++) { uint32_t bw, bh; - xoff = minx; + xoff = key->minx; /* clip bin height: */ - bh = MIN2(bin_h, miny + height - yoff); + bh = MIN2(bin_h, key->miny + key->height - yoff); for (j = 0; j < nbins_x; j++) { struct fd_tile *tile = &gmem->tile[t]; @@ -312,8 +300,8 @@ calculate_tiles(struct fd_batch *batch) assert(p < gmem->num_vsc_pipes); /* clip bin width: */ - bw = MIN2(bin_w, minx + width - xoff); - tile->n = !is_a20x(ctx->screen) ? tile_n[p]++ : + bw = MIN2(bin_w, key->minx + key->width - xoff); + tile->n = !is_a20x(screen) ? tile_n[p]++ : ((i % tpp_y + 1) << 3 | (j % tpp_x + 1)); tile->p = p; tile->bin_w = bw; @@ -344,8 +332,128 @@ calculate_tiles(struct fd_batch *batch) printf("\n"); } } + + return gmem; +} + +void +__fd_gmem_destroy(struct fd_gmem_stateobj *gmem) +{ + struct fd_gmem_cache *cache = &gmem->screen->gmem_cache; + + pipe_mutex_assert_locked(gmem->screen->lock); + + _mesa_hash_table_remove_key(cache->ht, gmem->key); + list_del(&gmem->node); + + ralloc_free(gmem->key); + ralloc_free(gmem); } +static struct gmem_key * +key_init(struct fd_batch *batch) +{ + struct fd_screen *screen = batch->ctx->screen; + struct pipe_framebuffer_state *pfb = &batch->framebuffer; + bool has_zs = pfb->zsbuf && !!(batch->gmem_reason & (FD_GMEM_DEPTH_ENABLED | + FD_GMEM_STENCIL_ENABLED | FD_GMEM_CLEARS_DEPTH_STENCIL)); + struct gmem_key *key = rzalloc(screen->gmem_cache.ht, struct gmem_key); + + if (has_zs) { + struct fd_resource *rsc = fd_resource(pfb->zsbuf->texture); + key->zsbuf_cpp[0] = rsc->layout.cpp; + if (rsc->stencil) + key->zsbuf_cpp[1] = rsc->stencil->layout.cpp; + } else { + /* we might have a zsbuf, but it isn't used */ + batch->restore &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL); + batch->resolve &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL); + } + + key->nr_cbufs = pfb->nr_cbufs; + for (unsigned i = 0; i < pfb->nr_cbufs; i++) { + if (pfb->cbufs[i]) + key->cbuf_cpp[i] = util_format_get_blocksize(pfb->cbufs[i]->format); + else + key->cbuf_cpp[i] = 4; + /* if MSAA, color buffers are super-sampled in GMEM: */ + key->cbuf_cpp[i] *= pfb->samples; + } + + if (fd_mesa_debug & FD_DBG_NOSCIS) { + key->minx = 0; + key->miny = 0; + key->width = pfb->width; + key->height = pfb->height; + } else { + struct pipe_scissor_state *scissor = &batch->max_scissor; + + /* round down to multiple of alignment: */ + key->minx = scissor->minx & ~(screen->gmem_alignw - 1); + key->miny = scissor->miny & ~(screen->gmem_alignh - 1); + key->width = scissor->maxx - key->minx; + key->height = scissor->maxy - key->miny; + } + + if (is_a20x(screen) && batch->cleared) { + /* under normal circumstances the requirement would be 4K + * but the fast clear path requires an alignment of 32K + */ + key->gmem_page_align = 8; + } else { + // TODO re-check this across gens.. maybe it should only + // be a single page in some cases: + key->gmem_page_align = 4; + } + + return key; +} + +static struct fd_gmem_stateobj * +lookup_gmem_state(struct fd_batch *batch) +{ + struct fd_screen *screen = batch->ctx->screen; + struct fd_gmem_cache *cache = &screen->gmem_cache; + struct fd_gmem_stateobj *gmem = NULL; + struct gmem_key *key = key_init(batch); + uint32_t hash = gmem_key_hash(key); + + mtx_lock(&screen->lock); + + struct hash_entry *entry = + _mesa_hash_table_search_pre_hashed(cache->ht, hash, key); + if (entry) { + ralloc_free(key); + goto found; + } + + /* limit the # of cached gmem states, discarding the least + * recently used state if needed: + */ + if (cache->ht->entries >= 20) { + struct fd_gmem_stateobj *last = + list_last_entry(&cache->lru, struct fd_gmem_stateobj, node); + fd_gmem_reference(&last, NULL); + } + + entry = _mesa_hash_table_insert_pre_hashed(cache->ht, + hash, key, gmem_stateobj_init(screen, key)); + +found: + fd_gmem_reference(&gmem, entry->data); + /* Move to the head of the LRU: */ + list_delinit(&gmem->node); + list_add(&gmem->node, &cache->lru); + + mtx_unlock(&screen->lock); + + return gmem; +} + +/* + * GMEM render pass + */ + static void render_tiles(struct fd_batch *batch, struct fd_gmem_stateobj *gmem) { @@ -483,9 +591,8 @@ fd_gmem_render_tiles(struct fd_batch *batch) render_sysmem(batch); ctx->stats.batch_sysmem++; } else { - struct fd_gmem_stateobj *gmem = &ctx->gmem; + struct fd_gmem_stateobj *gmem = lookup_gmem_state(batch); batch->gmem_state = gmem; - calculate_tiles(batch); DBG("%p: rendering %dx%d tiles %ux%u (%s/%s)", batch, pfb->width, pfb->height, gmem->nbins_x, gmem->nbins_y, util_format_short_name(pipe_surface_format(pfb->cbufs[0])), @@ -493,6 +600,12 @@ fd_gmem_render_tiles(struct fd_batch *batch) if (ctx->query_prepare) ctx->query_prepare(batch, gmem->nbins_x * gmem->nbins_y); render_tiles(batch, gmem); + batch->gmem_state = NULL; + + mtx_lock(&ctx->screen->lock); + fd_gmem_reference(&gmem, NULL); + mtx_unlock(&ctx->screen->lock); + ctx->stats.batch_gmem++; } @@ -513,3 +626,20 @@ fd_gmem_needs_restore(struct fd_batch *batch, const struct fd_tile *tile, return true; } + +void +fd_gmem_screen_init(struct pipe_screen *pscreen) +{ + struct fd_gmem_cache *cache = &fd_screen(pscreen)->gmem_cache; + + cache->ht = _mesa_hash_table_create(NULL, gmem_key_hash, gmem_key_equals); + list_inithead(&cache->lru); +} + +void +fd_gmem_screen_fini(struct pipe_screen *pscreen) +{ + struct fd_gmem_cache *cache = &fd_screen(pscreen)->gmem_cache; + + _mesa_hash_table_destroy(cache->ht, NULL); +} diff --git a/src/gallium/drivers/freedreno/freedreno_gmem.h b/src/gallium/drivers/freedreno/freedreno_gmem.h index 00c446603f6..84b0172fa22 100644 --- a/src/gallium/drivers/freedreno/freedreno_gmem.h +++ b/src/gallium/drivers/freedreno/freedreno_gmem.h @@ -27,7 +27,8 @@ #ifndef FREEDRENO_GMEM_H_ #define FREEDRENO_GMEM_H_ -#include "pipe/p_context.h" +#include "pipe/p_state.h" +#include "util/list.h" #include "freedreno_util.h" @@ -45,7 +46,10 @@ struct fd_tile { }; struct fd_gmem_stateobj { - struct pipe_scissor_state scissor; + struct pipe_reference reference; + struct fd_screen *screen; + void *key; + uint32_t cbuf_base[MAX_RENDER_TARGETS]; uint32_t zsbuf_base[2]; uint8_t cbuf_cpp[MAX_RENDER_TARGETS]; @@ -59,6 +63,26 @@ struct fd_gmem_stateobj { struct fd_vsc_pipe vsc_pipe[32]; struct fd_tile tile[512]; + + struct list_head node; +}; + +void __fd_gmem_destroy(struct fd_gmem_stateobj *gmem); + +static inline void +fd_gmem_reference(struct fd_gmem_stateobj **ptr, struct fd_gmem_stateobj *gmem) +{ + struct fd_gmem_stateobj *old_gmem = *ptr; + + if (pipe_reference(&(*ptr)->reference, &gmem->reference)) + __fd_gmem_destroy(old_gmem); + + *ptr = gmem; +} + +struct fd_gmem_cache { + struct hash_table *ht; + struct list_head lru; }; struct fd_batch; @@ -68,4 +92,8 @@ void fd_gmem_render_tiles(struct fd_batch *batch); bool fd_gmem_needs_restore(struct fd_batch *batch, const struct fd_tile *tile, uint32_t buffers); +struct pipe_screen; +void fd_gmem_screen_init(struct pipe_screen *pscreen); +void fd_gmem_screen_fini(struct pipe_screen *pscreen); + #endif /* FREEDRENO_GMEM_H_ */ diff --git a/src/gallium/drivers/freedreno/freedreno_screen.c b/src/gallium/drivers/freedreno/freedreno_screen.c index 9bdc0cc260c..5c0feb9e0f6 100644 --- a/src/gallium/drivers/freedreno/freedreno_screen.c +++ b/src/gallium/drivers/freedreno/freedreno_screen.c @@ -154,6 +154,7 @@ fd_screen_destroy(struct pipe_screen *pscreen) FREE(screen->ro); fd_bc_fini(&screen->batch_cache); + fd_gmem_screen_fini(pscreen); slab_destroy_parent(&screen->transfer_pool); @@ -969,6 +970,7 @@ fd_screen_create(struct fd_device *dev, struct renderonly *ro) fd_resource_screen_init(pscreen); fd_query_screen_init(pscreen); + fd_gmem_screen_init(pscreen); pscreen->get_name = fd_screen_get_name; pscreen->get_vendor = fd_screen_get_vendor; diff --git a/src/gallium/drivers/freedreno/freedreno_screen.h b/src/gallium/drivers/freedreno/freedreno_screen.h index b60f96831a6..22513065e5d 100644 --- a/src/gallium/drivers/freedreno/freedreno_screen.h +++ b/src/gallium/drivers/freedreno/freedreno_screen.h @@ -38,6 +38,7 @@ #include "renderonly/renderonly.h" #include "freedreno_batch_cache.h" +#include "freedreno_gmem.h" #include "freedreno_util.h" struct fd_bo; @@ -113,6 +114,7 @@ struct fd_screen { int64_t cpu_gpu_time_delta; struct fd_batch_cache batch_cache; + struct fd_gmem_cache gmem_cache; bool reorder; -- 2.30.2