X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Ffreedreno%2Ffreedreno_gmem.c;h=bbffda66e470792afe28b07391e0d40c0d5579b4;hb=afd47ea83be5521c4ae50ba15e84649e1339ab9f;hp=37a2f33365daae527cb56fe5bf709f3e2a59527e;hpb=b852c3bf67cf1a047b8d17391506b19b5d1bdb70;p=mesa.git

diff --git a/src/gallium/drivers/freedreno/freedreno_gmem.c b/src/gallium/drivers/freedreno/freedreno_gmem.c
index 37a2f33365d..bbffda66e47 100644
--- a/src/gallium/drivers/freedreno/freedreno_gmem.c
+++ b/src/gallium/drivers/freedreno/freedreno_gmem.c
@@ -1,5 +1,3 @@
-/* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
-
 /*
  * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
  *
@@ -27,14 +25,17 @@
  */
 
 #include "pipe/p_state.h"
+#include "util/hash_table.h"
+#include "util/u_dump.h"
 #include "util/u_string.h"
 #include "util/u_memory.h"
 #include "util/u_inlines.h"
-#include "util/u_format.h"
+#include "util/format/u_format.h"
 
 #include "freedreno_gmem.h"
 #include "freedreno_context.h"
 #include "freedreno_fence.h"
+#include "freedreno_log.h"
 #include "freedreno_resource.h"
 #include "freedreno_query_hw.h"
 #include "freedreno_util.h"
@@ -68,140 +69,225 @@
  * resolve.
  */
 
+#define BIN_DEBUG 0
+
+/*
+ * GMEM Cache:
+ *
+ * Caches GMEM state based on a given framebuffer state.  The key is
+ * meant to be the minimal set of data that results in a unique gmem
+ * configuration, avoiding multiple keys arriving at the same gmem
+ * state.  For example, the render target format is not part of the
+ * key, only the size per pixel.  And the max_scissor bounds is not
+ * part of they key, only the minx/miny (after clamping to tile
+ * alignment) and width/height.  This ensures that slightly different
+ * max_scissor which would result in the same gmem state, do not
+ * become different keys that map to the same state.
+ */
+
+struct gmem_key {
+	uint16_t minx, miny;
+	uint16_t width, height;
+	uint8_t gmem_page_align;      /* alignment in multiples of 0x1000 to reduce key size */
+	uint8_t nr_cbufs;
+	uint8_t cbuf_cpp[MAX_RENDER_TARGETS];
+	uint8_t zsbuf_cpp[2];
+};
+
+static uint32_t
+gmem_key_hash(const void *_key)
+{
+	const struct gmem_key *key = _key;
+	return _mesa_hash_data(key, sizeof(*key));
+}
+
+static bool
+gmem_key_equals(const void *_a, const void *_b)
+{
+	const struct gmem_key *a = _a;
+	const struct gmem_key *b = _b;
+	return memcmp(a, b, sizeof(*a)) == 0;
+}
+
+static void
+dump_gmem_key(const struct gmem_key *key)
+{
+	printf("{ .minx=%u, .miny=%u, .width=%u, .height=%u",
+			key->minx, key->miny, key->width, key->height);
+	printf(", .gmem_page_align=%u, .nr_cbufs=%u",
+			key->gmem_page_align, key->nr_cbufs);
+	printf(", .cbuf_cpp = {");
+	for (unsigned i = 0; i < ARRAY_SIZE(key->cbuf_cpp); i++)
+		printf("%u,", key->cbuf_cpp[i]);
+	printf("}, .zsbuf_cpp = {");
+	for (unsigned i = 0; i < ARRAY_SIZE(key->zsbuf_cpp); i++)
+		printf("%u,", key->zsbuf_cpp[i]);
+	printf("}},\n");
+}
+
+static void
+dump_gmem_state(const struct fd_gmem_stateobj *gmem)
+{
+	unsigned total = 0;
+	printf("GMEM LAYOUT: bin=%ux%u, nbins=%ux%u\n",
+			gmem->bin_w, gmem->bin_h, gmem->nbins_x, gmem->nbins_y);
+	for (int i = 0; i < ARRAY_SIZE(gmem->cbuf_base); i++) {
+		if (!gmem->cbuf_cpp[i])
+			continue;
+
+		unsigned size = gmem->cbuf_cpp[i] * gmem->bin_w * gmem->bin_h;
+		printf("  cbuf[%d]: base=0x%06x, size=0x%x, cpp=%u\n", i,
+				gmem->cbuf_base[i], size, gmem->cbuf_cpp[i]);
+
+		total = gmem->cbuf_base[i] + size;
+	}
+
+	for (int i = 0; i < ARRAY_SIZE(gmem->zsbuf_base); i++) {
+		if (!gmem->zsbuf_cpp[i])
+			continue;
+
+		unsigned size = gmem->zsbuf_cpp[i] * gmem->bin_w * gmem->bin_h;
+		printf("  zsbuf[%d]: base=0x%06x, size=0x%x, cpp=%u\n", i,
+				gmem->zsbuf_base[i], size, gmem->zsbuf_cpp[i]);
+
+		total = gmem->zsbuf_base[i] + size;
+	}
+
+	printf("total: 0x%06x (of 0x%06x)\n", total,
+			gmem->screen->gmemsize_bytes);
+}
+
 static uint32_t bin_width(struct fd_screen *screen)
 {
-	if (is_a4xx(screen) || is_a5xx(screen))
+	if (is_a4xx(screen) || is_a5xx(screen) || is_a6xx(screen))
 		return 1024;
 	if (is_a3xx(screen))
 		return 992;
 	return 512;
 }
 
-static uint32_t
-total_size(uint8_t cbuf_cpp[], uint8_t zsbuf_cpp[2],
-		   uint32_t bin_w, uint32_t bin_h, struct fd_gmem_stateobj *gmem)
+static unsigned
+div_align(unsigned num, unsigned denom, unsigned al)
+{
+	return align(DIV_ROUND_UP(num, denom), al);
+}
+
+static bool
+layout_gmem(struct gmem_key *key, uint32_t nbins_x, uint32_t nbins_y,
+		struct fd_gmem_stateobj *gmem)
 {
+	struct fd_screen *screen = gmem->screen;
+	uint32_t gmem_align = key->gmem_page_align * 0x1000;
 	uint32_t total = 0, i;
 
+	if ((nbins_x == 0) || (nbins_y == 0))
+		return false;
+
+	uint32_t bin_w, bin_h;
+	bin_w = div_align(key->width, nbins_x, screen->gmem_alignw);
+	bin_h = div_align(key->height, nbins_y, screen->gmem_alignh);
+
+	gmem->bin_w = bin_w;
+	gmem->bin_h = bin_h;
+
+	/* due to aligning bin_w/h, we could end up with one too
+	 * many bins in either dimension, so recalculate:
+	 */
+	gmem->nbins_x = DIV_ROUND_UP(key->width, bin_w);
+	gmem->nbins_y = DIV_ROUND_UP(key->height, bin_h);
+
 	for (i = 0; i < MAX_RENDER_TARGETS; i++) {
-		if (cbuf_cpp[i]) {
-			gmem->cbuf_base[i] = align(total, 0x4000);
-			total = gmem->cbuf_base[i] + cbuf_cpp[i] * bin_w * bin_h;
+		if (key->cbuf_cpp[i]) {
+			gmem->cbuf_base[i] = align(total, gmem_align);
+			total = gmem->cbuf_base[i] + key->cbuf_cpp[i] * bin_w * bin_h;
 		}
 	}
 
-	if (zsbuf_cpp[0]) {
-		gmem->zsbuf_base[0] = align(total, 0x4000);
-		total = gmem->zsbuf_base[0] + zsbuf_cpp[0] * bin_w * bin_h;
+	if (key->zsbuf_cpp[0]) {
+		gmem->zsbuf_base[0] = align(total, gmem_align);
+		total = gmem->zsbuf_base[0] + key->zsbuf_cpp[0] * bin_w * bin_h;
 	}
 
-	if (zsbuf_cpp[1]) {
-		gmem->zsbuf_base[1] = align(total, 0x4000);
-		total = gmem->zsbuf_base[1] + zsbuf_cpp[1] * bin_w * bin_h;
+	if (key->zsbuf_cpp[1]) {
+		gmem->zsbuf_base[1] = align(total, gmem_align);
+		total = gmem->zsbuf_base[1] + key->zsbuf_cpp[1] * bin_w * bin_h;
 	}
 
-	return total;
+	return total <= screen->gmemsize_bytes;
 }
 
-static void
-calculate_tiles(struct fd_batch *batch)
+static struct fd_gmem_stateobj *
+gmem_stateobj_init(struct fd_screen *screen, struct gmem_key *key)
 {
-	struct fd_context *ctx = batch->ctx;
-	struct fd_gmem_stateobj *gmem = &ctx->gmem;
-	struct pipe_scissor_state *scissor = &batch->max_scissor;
-	struct pipe_framebuffer_state *pfb = &batch->framebuffer;
-	const uint32_t gmem_alignw = ctx->screen->gmem_alignw;
-	const uint32_t gmem_alignh = ctx->screen->gmem_alignh;
-	const unsigned npipes = ctx->screen->num_vsc_pipes;
-	const uint32_t gmem_size = ctx->screen->gmemsize_bytes;
-	uint32_t minx, miny, width, height;
+	struct fd_gmem_stateobj *gmem =
+			rzalloc(screen->gmem_cache.ht, struct fd_gmem_stateobj);
+	pipe_reference_init(&gmem->reference, 1);
+	gmem->screen = screen;
+	gmem->key = key;
+	list_inithead(&gmem->node);
+
+	const unsigned npipes = screen->num_vsc_pipes;
 	uint32_t nbins_x = 1, nbins_y = 1;
-	uint32_t bin_w, bin_h;
-	uint32_t max_width = bin_width(ctx->screen);
-	uint8_t cbuf_cpp[MAX_RENDER_TARGETS] = {0}, zsbuf_cpp[2] = {0};
+	uint32_t max_width = bin_width(screen);
 	uint32_t i, j, t, xoff, yoff;
 	uint32_t tpp_x, tpp_y;
-	bool has_zs = !!(batch->resolve & (FD_BUFFER_DEPTH | FD_BUFFER_STENCIL));
 	int tile_n[npipes];
 
-	if (has_zs) {
-		struct fd_resource *rsc = fd_resource(pfb->zsbuf->texture);
-		zsbuf_cpp[0] = rsc->cpp;
-		if (rsc->stencil)
-			zsbuf_cpp[1] = rsc->stencil->cpp;
-	}
-	for (i = 0; i < pfb->nr_cbufs; i++) {
-		if (pfb->cbufs[i])
-			cbuf_cpp[i] = util_format_get_blocksize(pfb->cbufs[i]->format);
-		else
-			cbuf_cpp[i] = 4;
-	}
-
-	if (!memcmp(gmem->zsbuf_cpp, zsbuf_cpp, sizeof(zsbuf_cpp)) &&
-		!memcmp(gmem->cbuf_cpp, cbuf_cpp, sizeof(cbuf_cpp)) &&
-		!memcmp(&gmem->scissor, scissor, sizeof(gmem->scissor))) {
-		/* everything is up-to-date */
-		return;
-	}
-
-	if (fd_mesa_debug & FD_DBG_NOSCIS) {
-		minx = 0;
-		miny = 0;
-		width = pfb->width;
-		height = pfb->height;
-	} else {
-		/* round down to multiple of alignment: */
-		minx = scissor->minx & ~(gmem_alignw - 1);
-		miny = scissor->miny & ~(gmem_alignh - 1);
-		width = scissor->maxx - minx;
-		height = scissor->maxy - miny;
+	if (fd_mesa_debug & FD_DBG_MSGS) {
+		debug_printf("binning input: cbuf cpp:");
+		for (i = 0; i < key->nr_cbufs; i++)
+			debug_printf(" %d", key->cbuf_cpp[i]);
+		debug_printf(", zsbuf cpp: %d; %dx%d\n",
+				key->zsbuf_cpp[0], key->width, key->height);
 	}
 
-	bin_w = align(width, gmem_alignw);
-	bin_h = align(height, gmem_alignh);
-
 	/* first, find a bin width that satisfies the maximum width
 	 * restrictions:
 	 */
-	while (bin_w > max_width) {
+	while (div_align(key->width, nbins_x, screen->gmem_alignw) > max_width) {
 		nbins_x++;
-		bin_w = align(width / nbins_x, gmem_alignw);
-	}
-
-	if (fd_mesa_debug & FD_DBG_MSGS) {
-		debug_printf("binning input: cbuf cpp:");
-		for (i = 0; i < pfb->nr_cbufs; i++)
-			debug_printf(" %d", cbuf_cpp[i]);
-		debug_printf(", zsbuf cpp: %d; %dx%d\n",
-				zsbuf_cpp[0], width, height);
 	}
 
 	/* then find a bin width/height that satisfies the memory
 	 * constraints:
 	 */
-	while (total_size(cbuf_cpp, zsbuf_cpp, bin_w, bin_h, gmem) > gmem_size) {
-		if (bin_w > bin_h) {
+	while (!layout_gmem(key, nbins_x, nbins_y, gmem)) {
+		if (nbins_y > nbins_x) {
 			nbins_x++;
-			bin_w = align(width / nbins_x, gmem_alignw);
 		} else {
 			nbins_y++;
-			bin_h = align(height / nbins_y, gmem_alignh);
 		}
 	}
 
-	DBG("using %d bins of size %dx%d", nbins_x*nbins_y, bin_w, bin_h);
+	/* Lets see if we can tweak the layout a bit and come up with
+	 * something better:
+	 */
+	if ((((nbins_x - 1) * (nbins_y + 1)) < (nbins_x * nbins_y)) &&
+			layout_gmem(key, nbins_x - 1, nbins_y + 1, gmem)) {
+		nbins_x--;
+		nbins_y++;
+	} else if ((((nbins_x + 1) * (nbins_y - 1)) < (nbins_x * nbins_y)) &&
+			layout_gmem(key, nbins_x + 1, nbins_y - 1, gmem)) {
+		nbins_x++;
+		nbins_y--;
+	}
 
-	gmem->scissor = *scissor;
-	memcpy(gmem->cbuf_cpp, cbuf_cpp, sizeof(cbuf_cpp));
-	memcpy(gmem->zsbuf_cpp, zsbuf_cpp, sizeof(zsbuf_cpp));
-	gmem->bin_h = bin_h;
-	gmem->bin_w = bin_w;
-	gmem->nbins_x = nbins_x;
-	gmem->nbins_y = nbins_y;
-	gmem->minx = minx;
-	gmem->miny = miny;
-	gmem->width = width;
-	gmem->height = height;
+	layout_gmem(key, nbins_x, nbins_y, gmem);
+
+	DBG("using %d bins of size %dx%d", gmem->nbins_x * gmem->nbins_y,
+			gmem->bin_w, gmem->bin_h);
+
+	memcpy(gmem->cbuf_cpp, key->cbuf_cpp, sizeof(key->cbuf_cpp));
+	memcpy(gmem->zsbuf_cpp, key->zsbuf_cpp, sizeof(key->zsbuf_cpp));
+	gmem->minx = key->minx;
+	gmem->miny = key->miny;
+	gmem->width = key->width;
+	gmem->height = key->height;
+
+	if (BIN_DEBUG) {
+		dump_gmem_state(gmem);
+		dump_gmem_key(key);
+	}
 
 	/*
 	 * Assign tiles and pipes:
@@ -213,12 +299,21 @@ calculate_tiles(struct fd_batch *batch)
 
 #define div_round_up(v, a)  (((v) + (a) - 1) / (a))
 	/* figure out number of tiles per pipe: */
-	tpp_x = tpp_y = 1;
-	while (div_round_up(nbins_y, tpp_y) > 8)
-		tpp_y += 2;
-	while ((div_round_up(nbins_y, tpp_y) *
-			div_round_up(nbins_x, tpp_x)) > 8)
-		tpp_x += 1;
+	if (is_a20x(screen)) {
+		/* for a20x we want to minimize the number of "pipes"
+		 * binning data has 3 bits for x/y (8x8) but the edges are used to
+		 * cull off-screen vertices with hw binning, so we have 6x6 pipes
+		 */
+		tpp_x = 6;
+		tpp_y = 6;
+	} else {
+		tpp_x = tpp_y = 1;
+		while (div_round_up(nbins_y, tpp_y) > npipes)
+			tpp_y += 2;
+		while ((div_round_up(nbins_y, tpp_y) *
+				div_round_up(nbins_x, tpp_x)) > npipes)
+			tpp_x += 1;
+	}
 
 	gmem->maxpw = tpp_x;
 	gmem->maxph = tpp_y;
@@ -226,7 +321,7 @@ calculate_tiles(struct fd_batch *batch)
 	/* configure pipes: */
 	xoff = yoff = 0;
 	for (i = 0; i < npipes; i++) {
-		struct fd_vsc_pipe *pipe = &ctx->vsc_pipe[i];
+		struct fd_vsc_pipe *pipe = &gmem->vsc_pipe[i];
 
 		if (xoff >= nbins_x) {
 			xoff = 0;
@@ -245,51 +340,60 @@ calculate_tiles(struct fd_batch *batch)
 		xoff += tpp_x;
 	}
 
+	/* number of pipes to use for a20x */
+	gmem->num_vsc_pipes = MAX2(1, i);
+
 	for (; i < npipes; i++) {
-		struct fd_vsc_pipe *pipe = &ctx->vsc_pipe[i];
+		struct fd_vsc_pipe *pipe = &gmem->vsc_pipe[i];
 		pipe->x = pipe->y = pipe->w = pipe->h = 0;
 	}
 
-#if 0 /* debug */
-	printf("%dx%d ... tpp=%dx%d\n", nbins_x, nbins_y, tpp_x, tpp_y);
-	for (i = 0; i < 8; i++) {
-		struct fd_vsc_pipe *pipe = &ctx->pipe[i];
-		printf("pipe[%d]: %ux%u @ %u,%u\n", i,
-				pipe->w, pipe->h, pipe->x, pipe->y);
+	if (BIN_DEBUG) {
+		printf("%dx%d ... tpp=%dx%d\n", nbins_x, nbins_y, tpp_x, tpp_y);
+		for (i = 0; i < ARRAY_SIZE(gmem->vsc_pipe); i++) {
+			struct fd_vsc_pipe *pipe = &gmem->vsc_pipe[i];
+			printf("pipe[%d]: %ux%u @ %u,%u\n", i,
+					pipe->w, pipe->h, pipe->x, pipe->y);
+		}
 	}
-#endif
 
 	/* configure tiles: */
 	t = 0;
-	yoff = miny;
+	yoff = key->miny;
 	memset(tile_n, 0, sizeof(tile_n));
 	for (i = 0; i < nbins_y; i++) {
 		uint32_t bw, bh;
 
-		xoff = minx;
+		xoff = key->minx;
 
 		/* clip bin height: */
-		bh = MIN2(bin_h, miny + height - yoff);
+		bh = MIN2(gmem->bin_h, key->miny + key->height - yoff);
 
 		for (j = 0; j < nbins_x; j++) {
-			struct fd_tile *tile = &ctx->tile[t];
+			struct fd_tile *tile = &gmem->tile[t];
 			uint32_t p;
 
-			assert(t < ARRAY_SIZE(ctx->tile));
+			assert(t < ARRAY_SIZE(gmem->tile));
 
 			/* pipe number: */
 			p = ((i / tpp_y) * div_round_up(nbins_x, tpp_x)) + (j / tpp_x);
+			assert(p < gmem->num_vsc_pipes);
 
 			/* clip bin width: */
-			bw = MIN2(bin_w, minx + width - xoff);
-
-			tile->n = tile_n[p]++;
+			bw = MIN2(gmem->bin_w, key->minx + key->width - xoff);
+			tile->n = !is_a20x(screen) ? tile_n[p]++ :
+				((i % tpp_y + 1) << 3 | (j % tpp_x + 1));
 			tile->p = p;
 			tile->bin_w = bw;
 			tile->bin_h = bh;
 			tile->xoff = xoff;
 			tile->yoff = yoff;
 
+			if (BIN_DEBUG) {
+				printf("tile[%d]: p=%u, bin=%ux%u+%u+%u\n", t,
+						p, bw, bh, xoff, yoff);
+			}
+
 			t++;
 
 			xoff += bw;
@@ -298,34 +402,160 @@ calculate_tiles(struct fd_batch *batch)
 		yoff += bh;
 	}
 
-#if 0 /* debug */
-	t = 0;
-	for (i = 0; i < nbins_y; i++) {
-		for (j = 0; j < nbins_x; j++) {
-			struct fd_tile *tile = &ctx->tile[t++];
-			printf("|p:%u n:%u|", tile->p, tile->n);
+	if (BIN_DEBUG) {
+		t = 0;
+		for (i = 0; i < nbins_y; i++) {
+			for (j = 0; j < nbins_x; j++) {
+				struct fd_tile *tile = &gmem->tile[t++];
+				printf("|p:%u n:%u|", tile->p, tile->n);
+			}
+			printf("\n");
 		}
-		printf("\n");
 	}
-#endif
+
+	return gmem;
 }
 
+void
+__fd_gmem_destroy(struct fd_gmem_stateobj *gmem)
+{
+	struct fd_gmem_cache *cache = &gmem->screen->gmem_cache;
+
+	fd_screen_assert_locked(gmem->screen);
+
+	_mesa_hash_table_remove_key(cache->ht, gmem->key);
+	list_del(&gmem->node);
+
+	ralloc_free(gmem->key);
+	ralloc_free(gmem);
+}
+
+static struct gmem_key *
+gmem_key_init(struct fd_batch *batch, bool assume_zs, bool no_scis_opt)
+{
+	struct fd_screen *screen = batch->ctx->screen;
+	struct pipe_framebuffer_state *pfb = &batch->framebuffer;
+	bool has_zs = pfb->zsbuf && !!(batch->gmem_reason & (FD_GMEM_DEPTH_ENABLED |
+		FD_GMEM_STENCIL_ENABLED | FD_GMEM_CLEARS_DEPTH_STENCIL));
+	struct gmem_key *key = rzalloc(screen->gmem_cache.ht, struct gmem_key);
+
+	if (has_zs || assume_zs) {
+		struct fd_resource *rsc = fd_resource(pfb->zsbuf->texture);
+		key->zsbuf_cpp[0] = rsc->layout.cpp;
+		if (rsc->stencil)
+			key->zsbuf_cpp[1] = rsc->stencil->layout.cpp;
+	} else {
+		/* we might have a zsbuf, but it isn't used */
+		batch->restore &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
+		batch->resolve &= ~(FD_BUFFER_DEPTH | FD_BUFFER_STENCIL);
+	}
+
+	key->nr_cbufs = pfb->nr_cbufs;
+	for (unsigned i = 0; i < pfb->nr_cbufs; i++) {
+		if (pfb->cbufs[i])
+			key->cbuf_cpp[i] = util_format_get_blocksize(pfb->cbufs[i]->format);
+		else
+			key->cbuf_cpp[i] = 4;
+		/* if MSAA, color buffers are super-sampled in GMEM: */
+		key->cbuf_cpp[i] *= pfb->samples;
+	}
+
+	/* NOTE: on a6xx, the max-scissor-rect is handled in fd6_gmem, and
+	 * we just rely on CP_COND_EXEC to skip bins with no geometry.
+	 */
+	if ((fd_mesa_debug & FD_DBG_NOSCIS) || no_scis_opt || is_a6xx(screen)) {
+		key->minx = 0;
+		key->miny = 0;
+		key->width = pfb->width;
+		key->height = pfb->height;
+	} else {
+		struct pipe_scissor_state *scissor = &batch->max_scissor;
+
+		/* round down to multiple of alignment: */
+		key->minx = scissor->minx & ~(screen->gmem_alignw - 1);
+		key->miny = scissor->miny & ~(screen->gmem_alignh - 1);
+		key->width = scissor->maxx - key->minx;
+		key->height = scissor->maxy - key->miny;
+	}
+
+	if (is_a20x(screen) && batch->cleared) {
+		/* under normal circumstances the requirement would be 4K
+		 * but the fast clear path requires an alignment of 32K
+		 */
+		key->gmem_page_align = 8;
+	} else if (is_a6xx(screen)) {
+		key->gmem_page_align = 1;
+	} else {
+		// TODO re-check this across gens.. maybe it should only
+		// be a single page in some cases:
+		key->gmem_page_align = 4;
+	}
+
+	return key;
+}
+
+static struct fd_gmem_stateobj *
+lookup_gmem_state(struct fd_batch *batch, bool assume_zs, bool no_scis_opt)
+{
+	struct fd_screen *screen = batch->ctx->screen;
+	struct fd_gmem_cache *cache = &screen->gmem_cache;
+	struct fd_gmem_stateobj *gmem = NULL;
+	struct gmem_key *key = gmem_key_init(batch, assume_zs, no_scis_opt);
+	uint32_t hash = gmem_key_hash(key);
+
+	fd_screen_lock(screen);
+
+	struct hash_entry *entry =
+		_mesa_hash_table_search_pre_hashed(cache->ht, hash, key);
+	if (entry) {
+		ralloc_free(key);
+		goto found;
+	}
+
+	/* limit the # of cached gmem states, discarding the least
+	 * recently used state if needed:
+	 */
+	if (cache->ht->entries >= 20) {
+		struct fd_gmem_stateobj *last =
+			list_last_entry(&cache->lru, struct fd_gmem_stateobj, node);
+		fd_gmem_reference(&last, NULL);
+	}
+
+	entry = _mesa_hash_table_insert_pre_hashed(cache->ht,
+			hash, key, gmem_stateobj_init(screen, key));
+
+found:
+	fd_gmem_reference(&gmem, entry->data);
+	/* Move to the head of the LRU: */
+	list_delinit(&gmem->node);
+	list_add(&gmem->node, &cache->lru);
+
+	fd_screen_unlock(screen);
+
+	return gmem;
+}
+
+/*
+ * GMEM render pass
+ */
+
 static void
-render_tiles(struct fd_batch *batch)
+render_tiles(struct fd_batch *batch, struct fd_gmem_stateobj *gmem)
 {
 	struct fd_context *ctx = batch->ctx;
-	struct fd_gmem_stateobj *gmem = &ctx->gmem;
 	int i;
 
+	mtx_lock(&ctx->gmem_lock);
+
 	ctx->emit_tile_init(batch);
 
 	if (batch->restore)
 		ctx->stats.batch_restore++;
 
 	for (i = 0; i < (gmem->nbins_x * gmem->nbins_y); i++) {
-		struct fd_tile *tile = &ctx->tile[i];
+		struct fd_tile *tile = &gmem->tile[i];
 
-		DBG("bin_h=%d, yoff=%d, bin_w=%d, xoff=%d",
+		fd_log(batch, "bin_h=%d, yoff=%d, bin_w=%d, xoff=%d",
 			tile->bin_h, tile->yoff, tile->bin_w, tile->xoff);
 
 		ctx->emit_tile_prep(batch, tile);
@@ -340,7 +570,13 @@ render_tiles(struct fd_batch *batch)
 			ctx->query_prepare_tile(batch, i, batch->gmem);
 
 		/* emit IB to drawcmds: */
-		ctx->emit_ib(batch->gmem, batch->draw);
+		fd_log(batch, "TILE[%d]: START DRAW IB", i);
+		if (ctx->emit_tile) {
+			ctx->emit_tile(batch, tile);
+		} else {
+			ctx->screen->emit_ib(batch->gmem, batch->draw);
+		}
+		fd_log(batch, "TILE[%d]: END DRAW IB", i);
 		fd_reset_wfi(batch);
 
 		/* emit gmem2mem to transfer tile back to system memory: */
@@ -349,6 +585,8 @@ render_tiles(struct fd_batch *batch)
 
 	if (ctx->emit_tile_fini)
 		ctx->emit_tile_fini(batch);
+
+	mtx_unlock(&ctx->gmem_lock);
 }
 
 static void
@@ -362,7 +600,9 @@ render_sysmem(struct fd_batch *batch)
 		ctx->query_prepare_tile(batch, 0, batch->gmem);
 
 	/* emit IB to drawcmds: */
-	ctx->emit_ib(batch->gmem, batch->draw);
+	fd_log(batch, "SYSMEM: START DRAW IB");
+	ctx->screen->emit_ib(batch->gmem, batch->draw);
+	fd_log(batch, "SYSMEM: END DRAW IB");
 	fd_reset_wfi(batch);
 
 	if (ctx->emit_sysmem_fini)
@@ -372,16 +612,18 @@ render_sysmem(struct fd_batch *batch)
 static void
 flush_ring(struct fd_batch *batch)
 {
-	/* for compute/blit batch, there is no batch->gmem, only batch->draw: */
-	struct fd_ringbuffer *ring = batch->nondraw ? batch->draw : batch->gmem;
 	uint32_t timestamp;
 	int out_fence_fd = -1;
 
-	fd_ringbuffer_flush2(ring, batch->in_fence_fd,
-			batch->needs_out_fence_fd ? &out_fence_fd : NULL);
+	if (unlikely(fd_mesa_debug & FD_DBG_NOHW))
+		return;
+
+	fd_submit_flush(batch->submit, batch->in_fence_fd,
+			batch->needs_out_fence_fd ? &out_fence_fd : NULL,
+			&timestamp);
 
-	timestamp = fd_ringbuffer_timestamp(ring);
 	fd_fence_populate(batch->fence, timestamp, out_fence_fd);
+	fd_log_flush(batch);
 }
 
 void
@@ -393,9 +635,11 @@ fd_gmem_render_tiles(struct fd_batch *batch)
 
 	if (ctx->emit_sysmem_prep && !batch->nondraw) {
 		if (batch->cleared || batch->gmem_reason ||
-				((batch->num_draws > 5) && !batch->blit)) {
-			DBG("GMEM: cleared=%x, gmem_reason=%x, num_draws=%u",
-				batch->cleared, batch->gmem_reason, batch->num_draws);
+				((batch->num_draws > 5) && !batch->blit) ||
+				(pfb->samples > 1)) {
+			fd_log(batch, "GMEM: cleared=%x, gmem_reason=%x, num_draws=%u, samples=%u",
+				batch->cleared, batch->gmem_reason, batch->num_draws,
+				pfb->samples);
 		} else if (!(fd_mesa_debug & FD_DBG_NOBYPASS)) {
 			sysmem = true;
 		}
@@ -406,65 +650,89 @@ fd_gmem_render_tiles(struct fd_batch *batch)
 		}
 	}
 
+	if (fd_mesa_debug & FD_DBG_NOGMEM)
+		sysmem = true;
+
+	/* Layered rendering always needs bypass. */
+	for (unsigned i = 0; i < pfb->nr_cbufs; i++) {
+		struct pipe_surface *psurf = pfb->cbufs[i];
+		if (!psurf)
+			continue;
+		if (psurf->u.tex.first_layer < psurf->u.tex.last_layer)
+			sysmem = true;
+	}
+
+	/* Tessellation doesn't seem to support tiled rendering so fall back to
+	 * bypass.
+	 */
+	if (batch->tessellation) {
+		debug_assert(ctx->emit_sysmem_prep);
+		sysmem = true;
+	}
+
 	fd_reset_wfi(batch);
 
 	ctx->stats.batch_total++;
 
+	if (unlikely(fd_mesa_debug & FD_DBG_LOG) && !batch->nondraw) {
+		fd_log_stream(batch, stream, util_dump_framebuffer_state(stream, pfb));
+		for (unsigned i = 0; i < pfb->nr_cbufs; i++) {
+			fd_log_stream(batch, stream, util_dump_surface(stream, pfb->cbufs[i]));
+		}
+		fd_log_stream(batch, stream, util_dump_surface(stream, pfb->zsbuf));
+	}
+
 	if (batch->nondraw) {
 		DBG("%p: rendering non-draw", batch);
 		ctx->stats.batch_nondraw++;
 	} else if (sysmem) {
-		DBG("%p: rendering sysmem %ux%u (%s/%s)",
+		fd_log(batch, "%p: rendering sysmem %ux%u (%s/%s), num_draws=%u",
 			batch, pfb->width, pfb->height,
 			util_format_short_name(pipe_surface_format(pfb->cbufs[0])),
-			util_format_short_name(pipe_surface_format(pfb->zsbuf)));
+			util_format_short_name(pipe_surface_format(pfb->zsbuf)),
+			batch->num_draws);
 		if (ctx->query_prepare)
 			ctx->query_prepare(batch, 1);
 		render_sysmem(batch);
 		ctx->stats.batch_sysmem++;
 	} else {
-		struct fd_gmem_stateobj *gmem = &ctx->gmem;
-		calculate_tiles(batch);
-		DBG("%p: rendering %dx%d tiles %ux%u (%s/%s)",
+		struct fd_gmem_stateobj *gmem = lookup_gmem_state(batch, false, false);
+		batch->gmem_state = gmem;
+		fd_log(batch, "%p: rendering %dx%d tiles %ux%u (%s/%s)",
 			batch, pfb->width, pfb->height, gmem->nbins_x, gmem->nbins_y,
 			util_format_short_name(pipe_surface_format(pfb->cbufs[0])),
 			util_format_short_name(pipe_surface_format(pfb->zsbuf)));
 		if (ctx->query_prepare)
 			ctx->query_prepare(batch, gmem->nbins_x * gmem->nbins_y);
-		render_tiles(batch);
+		render_tiles(batch, gmem);
+		batch->gmem_state = NULL;
+
+		fd_screen_lock(ctx->screen);
+		fd_gmem_reference(&gmem, NULL);
+		fd_screen_unlock(ctx->screen);
+
 		ctx->stats.batch_gmem++;
 	}
 
 	flush_ring(batch);
 }
 
-/* special case for when we need to create a fence but have no rendering
- * to flush.. just emit a no-op string-marker packet.
+/* Determine a worst-case estimate (ie. assuming we don't eliminate an
+ * unused depth/stencil) number of bins per vsc pipe.
  */
-void
-fd_gmem_render_noop(struct fd_batch *batch)
+unsigned
+fd_gmem_estimate_bins_per_pipe(struct fd_batch *batch)
 {
-	struct fd_context *ctx = batch->ctx;
-	struct pipe_context *pctx = &ctx->base;
+	struct pipe_framebuffer_state *pfb = &batch->framebuffer;
+	struct fd_screen *screen = batch->ctx->screen;
+	struct fd_gmem_stateobj *gmem = lookup_gmem_state(batch, !!pfb->zsbuf, true);
+	unsigned nbins = gmem->maxpw * gmem->maxph;
 
-	pctx->emit_string_marker(pctx, "noop", 4);
-	/* emit IB to drawcmds (which contain the string marker): */
-	ctx->emit_ib(batch->gmem, batch->draw);
-	flush_ring(batch);
-}
+	fd_screen_lock(screen);
+	fd_gmem_reference(&gmem, NULL);
+	fd_screen_unlock(screen);
 
-/* tile needs restore if it isn't completely contained within the
- * cleared scissor:
- */
-static bool
-skip_restore(struct pipe_scissor_state *scissor, struct fd_tile *tile)
-{
-	unsigned minx = tile->xoff;
-	unsigned maxx = tile->xoff + tile->bin_w;
-	unsigned miny = tile->yoff;
-	unsigned maxy = tile->yoff + tile->bin_h;
-	return (minx >= scissor->minx) && (maxx <= scissor->maxx) &&
-			(miny >= scissor->miny) && (maxy <= scissor->maxy);
+	return nbins;
 }
 
 /* When deciding whether a tile needs mem2gmem, we need to take into
@@ -473,27 +741,28 @@ skip_restore(struct pipe_scissor_state *scissor, struct fd_tile *tile)
  * case would be a single clear.
  */
 bool
-fd_gmem_needs_restore(struct fd_batch *batch, struct fd_tile *tile,
+fd_gmem_needs_restore(struct fd_batch *batch, const struct fd_tile *tile,
 		uint32_t buffers)
 {
 	if (!(batch->restore & buffers))
 		return false;
 
-	/* if buffers partially cleared, then slow-path to figure out
-	 * if this particular tile needs restoring:
-	 */
-	if ((buffers & FD_BUFFER_COLOR) &&
-			(batch->partial_cleared & FD_BUFFER_COLOR) &&
-			skip_restore(&batch->cleared_scissor.color, tile))
-		return false;
-	if ((buffers & FD_BUFFER_DEPTH) &&
-			(batch->partial_cleared & FD_BUFFER_DEPTH) &&
-			skip_restore(&batch->cleared_scissor.depth, tile))
-		return false;
-	if ((buffers & FD_BUFFER_STENCIL) &&
-			(batch->partial_cleared & FD_BUFFER_STENCIL) &&
-			skip_restore(&batch->cleared_scissor.stencil, tile))
-		return false;
-
 	return true;
 }
+
+void
+fd_gmem_screen_init(struct pipe_screen *pscreen)
+{
+	struct fd_gmem_cache *cache = &fd_screen(pscreen)->gmem_cache;
+
+	cache->ht = _mesa_hash_table_create(NULL, gmem_key_hash, gmem_key_equals);
+	list_inithead(&cache->lru);
+}
+
+void
+fd_gmem_screen_fini(struct pipe_screen *pscreen)
+{
+	struct fd_gmem_cache *cache = &fd_screen(pscreen)->gmem_cache;
+
+	_mesa_hash_table_destroy(cache->ht, NULL);
+}