2 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * Rob Clark <robclark@freedesktop.org>
27 #include "pipe/p_state.h"
28 #include "util/u_string.h"
29 #include "util/u_memory.h"
30 #include "util/u_inlines.h"
31 #include "util/format/u_format.h"
33 #include "freedreno_gmem.h"
34 #include "freedreno_context.h"
35 #include "freedreno_fence.h"
36 #include "freedreno_resource.h"
37 #include "freedreno_query_hw.h"
38 #include "freedreno_util.h"
41 * GMEM is the small (ie. 256KiB for a200, 512KiB for a220, etc) tile buffer
42 * inside the GPU. All rendering happens to GMEM. Larger render targets
43 * are split into tiles that are small enough for the color (and depth and/or
44 * stencil, if enabled) buffers to fit within GMEM. Before rendering a tile,
45 * if there was not a clear invalidating the previous tile contents, we need
46 * to restore the previous tiles contents (system mem -> GMEM), and after all
47 * the draw calls, before moving to the next tile, we need to save the tile
48 * contents (GMEM -> system mem).
50 * The code in this file handles dealing with GMEM and tiling.
52 * The structure of the ringbuffer ends up being:
54 * +--<---<-- IB ---<---+---<---+---<---<---<--+
57 * ------------------------------------------------------
58 * | clear/draw cmds | Tile0 | Tile1 | .... | TileN |
59 * ------------------------------------------------------
62 * address submitted in issueibcmds
64 * Where the per-tile section handles scissor setup, mem2gmem restore (if
65 * needed), IB to draw cmds earlier in the ringbuffer, and then gmem2mem
71 static uint32_t bin_width(struct fd_screen
*screen
)
73 if (is_a4xx(screen
) || is_a5xx(screen
) || is_a6xx(screen
))
81 total_size(uint8_t cbuf_cpp
[], uint8_t zsbuf_cpp
[2],
82 uint32_t bin_w
, uint32_t bin_h
, uint32_t gmem_align
,
83 struct fd_gmem_stateobj
*gmem
)
85 uint32_t total
= 0, i
;
87 for (i
= 0; i
< MAX_RENDER_TARGETS
; i
++) {
89 gmem
->cbuf_base
[i
] = align(total
, gmem_align
);
90 total
= gmem
->cbuf_base
[i
] + cbuf_cpp
[i
] * bin_w
* bin_h
;
95 gmem
->zsbuf_base
[0] = align(total
, gmem_align
);
96 total
= gmem
->zsbuf_base
[0] + zsbuf_cpp
[0] * bin_w
* bin_h
;
100 gmem
->zsbuf_base
[1] = align(total
, gmem_align
);
101 total
= gmem
->zsbuf_base
[1] + zsbuf_cpp
[1] * bin_w
* bin_h
;
108 calculate_tiles(struct fd_batch
*batch
)
110 struct fd_context
*ctx
= batch
->ctx
;
111 struct fd_screen
*screen
= ctx
->screen
;
112 struct fd_gmem_stateobj
*gmem
= &ctx
->gmem
;
113 struct pipe_scissor_state
*scissor
= &batch
->max_scissor
;
114 struct pipe_framebuffer_state
*pfb
= &batch
->framebuffer
;
115 const uint32_t gmem_alignw
= screen
->gmem_alignw
;
116 const uint32_t gmem_alignh
= screen
->gmem_alignh
;
117 const unsigned npipes
= screen
->num_vsc_pipes
;
118 const uint32_t gmem_size
= screen
->gmemsize_bytes
;
119 uint32_t minx
, miny
, width
, height
;
120 uint32_t nbins_x
= 1, nbins_y
= 1;
121 uint32_t bin_w
, bin_h
;
122 uint32_t gmem_align
= 0x4000;
123 uint32_t max_width
= bin_width(screen
);
124 uint8_t cbuf_cpp
[MAX_RENDER_TARGETS
] = {0}, zsbuf_cpp
[2] = {0};
125 uint32_t i
, j
, t
, xoff
, yoff
;
126 uint32_t tpp_x
, tpp_y
;
127 bool has_zs
= !!(batch
->gmem_reason
& (FD_GMEM_DEPTH_ENABLED
|
128 FD_GMEM_STENCIL_ENABLED
| FD_GMEM_CLEARS_DEPTH_STENCIL
));
132 struct fd_resource
*rsc
= fd_resource(pfb
->zsbuf
->texture
);
133 zsbuf_cpp
[0] = rsc
->layout
.cpp
;
135 zsbuf_cpp
[1] = rsc
->stencil
->layout
.cpp
;
137 /* we might have a zsbuf, but it isn't used */
138 batch
->restore
&= ~(FD_BUFFER_DEPTH
| FD_BUFFER_STENCIL
);
139 batch
->resolve
&= ~(FD_BUFFER_DEPTH
| FD_BUFFER_STENCIL
);
141 for (i
= 0; i
< pfb
->nr_cbufs
; i
++) {
143 cbuf_cpp
[i
] = util_format_get_blocksize(pfb
->cbufs
[i
]->format
);
146 /* if MSAA, color buffers are super-sampled in GMEM: */
147 cbuf_cpp
[i
] *= pfb
->samples
;
150 if (!memcmp(gmem
->zsbuf_cpp
, zsbuf_cpp
, sizeof(zsbuf_cpp
)) &&
151 !memcmp(gmem
->cbuf_cpp
, cbuf_cpp
, sizeof(cbuf_cpp
)) &&
152 !memcmp(&gmem
->scissor
, scissor
, sizeof(gmem
->scissor
))) {
153 /* everything is up-to-date */
157 if (fd_mesa_debug
& FD_DBG_NOSCIS
) {
161 height
= pfb
->height
;
163 /* round down to multiple of alignment: */
164 minx
= scissor
->minx
& ~(gmem_alignw
- 1);
165 miny
= scissor
->miny
& ~(gmem_alignh
- 1);
166 width
= scissor
->maxx
- minx
;
167 height
= scissor
->maxy
- miny
;
170 bin_w
= align(width
, gmem_alignw
);
171 bin_h
= align(height
, gmem_alignh
);
173 /* first, find a bin width that satisfies the maximum width
176 while (bin_w
> max_width
) {
178 bin_w
= align(width
/ nbins_x
, gmem_alignw
);
181 if (fd_mesa_debug
& FD_DBG_MSGS
) {
182 debug_printf("binning input: cbuf cpp:");
183 for (i
= 0; i
< pfb
->nr_cbufs
; i
++)
184 debug_printf(" %d", cbuf_cpp
[i
]);
185 debug_printf(", zsbuf cpp: %d; %dx%d\n",
186 zsbuf_cpp
[0], width
, height
);
189 if (is_a20x(screen
) && batch
->cleared
) {
190 /* under normal circumstances the requirement would be 4K
191 * but the fast clear path requires an alignment of 32K
196 /* then find a bin width/height that satisfies the memory
199 while (total_size(cbuf_cpp
, zsbuf_cpp
, bin_w
, bin_h
, gmem_align
, gmem
) >
203 bin_w
= align(width
/ nbins_x
, gmem_alignw
);
206 bin_h
= align(height
/ nbins_y
, gmem_alignh
);
210 DBG("using %d bins of size %dx%d", nbins_x
*nbins_y
, bin_w
, bin_h
);
212 gmem
->scissor
= *scissor
;
213 memcpy(gmem
->cbuf_cpp
, cbuf_cpp
, sizeof(cbuf_cpp
));
214 memcpy(gmem
->zsbuf_cpp
, zsbuf_cpp
, sizeof(zsbuf_cpp
));
217 gmem
->nbins_x
= nbins_x
;
218 gmem
->nbins_y
= nbins_y
;
222 gmem
->height
= height
;
225 * Assign tiles and pipes:
227 * At some point it might be worth playing with different
228 * strategies and seeing if that makes much impact on
232 #define div_round_up(v, a) (((v) + (a) - 1) / (a))
233 /* figure out number of tiles per pipe: */
234 if (is_a20x(ctx
->screen
)) {
235 /* for a20x we want to minimize the number of "pipes"
236 * binning data has 3 bits for x/y (8x8) but the edges are used to
237 * cull off-screen vertices with hw binning, so we have 6x6 pipes
243 while (div_round_up(nbins_y
, tpp_y
) > npipes
)
245 while ((div_round_up(nbins_y
, tpp_y
) *
246 div_round_up(nbins_x
, tpp_x
)) > npipes
)
253 /* configure pipes: */
255 for (i
= 0; i
< npipes
; i
++) {
256 struct fd_vsc_pipe
*pipe
= &gmem
->vsc_pipe
[i
];
258 if (xoff
>= nbins_x
) {
263 if (yoff
>= nbins_y
) {
269 pipe
->w
= MIN2(tpp_x
, nbins_x
- xoff
);
270 pipe
->h
= MIN2(tpp_y
, nbins_y
- yoff
);
275 /* number of pipes to use for a20x */
276 gmem
->num_vsc_pipes
= MAX2(1, i
);
278 for (; i
< npipes
; i
++) {
279 struct fd_vsc_pipe
*pipe
= &gmem
->vsc_pipe
[i
];
280 pipe
->x
= pipe
->y
= pipe
->w
= pipe
->h
= 0;
284 printf("%dx%d ... tpp=%dx%d\n", nbins_x
, nbins_y
, tpp_x
, tpp_y
);
285 for (i
= 0; i
< ARRAY_SIZE(gmem
->vsc_pipe
); i
++) {
286 struct fd_vsc_pipe
*pipe
= &gmem
->vsc_pipe
[i
];
287 printf("pipe[%d]: %ux%u @ %u,%u\n", i
,
288 pipe
->w
, pipe
->h
, pipe
->x
, pipe
->y
);
292 /* configure tiles: */
295 memset(tile_n
, 0, sizeof(tile_n
));
296 for (i
= 0; i
< nbins_y
; i
++) {
301 /* clip bin height: */
302 bh
= MIN2(bin_h
, miny
+ height
- yoff
);
304 for (j
= 0; j
< nbins_x
; j
++) {
305 struct fd_tile
*tile
= &gmem
->tile
[t
];
308 assert(t
< ARRAY_SIZE(gmem
->tile
));
311 p
= ((i
/ tpp_y
) * div_round_up(nbins_x
, tpp_x
)) + (j
/ tpp_x
);
312 assert(p
< gmem
->num_vsc_pipes
);
314 /* clip bin width: */
315 bw
= MIN2(bin_w
, minx
+ width
- xoff
);
316 tile
->n
= !is_a20x(ctx
->screen
) ? tile_n
[p
]++ :
317 ((i
% tpp_y
+ 1) << 3 | (j
% tpp_x
+ 1));
325 printf("tile[%d]: p=%u, bin=%ux%u+%u+%u\n", t
,
326 p
, bw
, bh
, xoff
, yoff
);
339 for (i
= 0; i
< nbins_y
; i
++) {
340 for (j
= 0; j
< nbins_x
; j
++) {
341 struct fd_tile
*tile
= &gmem
->tile
[t
++];
342 printf("|p:%u n:%u|", tile
->p
, tile
->n
);
350 render_tiles(struct fd_batch
*batch
)
352 struct fd_context
*ctx
= batch
->ctx
;
353 struct fd_gmem_stateobj
*gmem
= &ctx
->gmem
;
356 ctx
->emit_tile_init(batch
);
359 ctx
->stats
.batch_restore
++;
361 for (i
= 0; i
< (gmem
->nbins_x
* gmem
->nbins_y
); i
++) {
362 struct fd_tile
*tile
= &gmem
->tile
[i
];
364 DBG("bin_h=%d, yoff=%d, bin_w=%d, xoff=%d",
365 tile
->bin_h
, tile
->yoff
, tile
->bin_w
, tile
->xoff
);
367 ctx
->emit_tile_prep(batch
, tile
);
369 if (batch
->restore
) {
370 ctx
->emit_tile_mem2gmem(batch
, tile
);
373 ctx
->emit_tile_renderprep(batch
, tile
);
375 if (ctx
->query_prepare_tile
)
376 ctx
->query_prepare_tile(batch
, i
, batch
->gmem
);
378 /* emit IB to drawcmds: */
379 if (ctx
->emit_tile
) {
380 ctx
->emit_tile(batch
, tile
);
382 ctx
->screen
->emit_ib(batch
->gmem
, batch
->draw
);
386 /* emit gmem2mem to transfer tile back to system memory: */
387 ctx
->emit_tile_gmem2mem(batch
, tile
);
390 if (ctx
->emit_tile_fini
)
391 ctx
->emit_tile_fini(batch
);
395 render_sysmem(struct fd_batch
*batch
)
397 struct fd_context
*ctx
= batch
->ctx
;
399 ctx
->emit_sysmem_prep(batch
);
401 if (ctx
->query_prepare_tile
)
402 ctx
->query_prepare_tile(batch
, 0, batch
->gmem
);
404 /* emit IB to drawcmds: */
405 ctx
->screen
->emit_ib(batch
->gmem
, batch
->draw
);
408 if (ctx
->emit_sysmem_fini
)
409 ctx
->emit_sysmem_fini(batch
);
413 flush_ring(struct fd_batch
*batch
)
416 int out_fence_fd
= -1;
418 fd_submit_flush(batch
->submit
, batch
->in_fence_fd
,
419 batch
->needs_out_fence_fd
? &out_fence_fd
: NULL
,
422 fd_fence_populate(batch
->fence
, timestamp
, out_fence_fd
);
426 fd_gmem_render_tiles(struct fd_batch
*batch
)
428 struct fd_context
*ctx
= batch
->ctx
;
429 struct pipe_framebuffer_state
*pfb
= &batch
->framebuffer
;
432 if (ctx
->emit_sysmem_prep
&& !batch
->nondraw
) {
433 if (batch
->cleared
|| batch
->gmem_reason
||
434 ((batch
->num_draws
> 5) && !batch
->blit
) ||
435 (pfb
->samples
> 1)) {
436 DBG("GMEM: cleared=%x, gmem_reason=%x, num_draws=%u, samples=%u",
437 batch
->cleared
, batch
->gmem_reason
, batch
->num_draws
,
439 } else if (!(fd_mesa_debug
& FD_DBG_NOBYPASS
)) {
443 /* For ARB_framebuffer_no_attachments: */
444 if ((pfb
->nr_cbufs
== 0) && !pfb
->zsbuf
) {
449 if (fd_mesa_debug
& FD_DBG_NOGMEM
)
452 /* Layered rendering always needs bypass. */
453 for (unsigned i
= 0; i
< pfb
->nr_cbufs
; i
++) {
454 struct pipe_surface
*psurf
= pfb
->cbufs
[i
];
457 if (psurf
->u
.tex
.first_layer
< psurf
->u
.tex
.last_layer
)
461 /* Tessellation doesn't seem to support tiled rendering so fall back to
464 if (batch
->tessellation
) {
465 debug_assert(ctx
->emit_sysmem_prep
);
471 ctx
->stats
.batch_total
++;
473 if (batch
->nondraw
) {
474 DBG("%p: rendering non-draw", batch
);
475 ctx
->stats
.batch_nondraw
++;
477 DBG("%p: rendering sysmem %ux%u (%s/%s), num_draws=%u",
478 batch
, pfb
->width
, pfb
->height
,
479 util_format_short_name(pipe_surface_format(pfb
->cbufs
[0])),
480 util_format_short_name(pipe_surface_format(pfb
->zsbuf
)),
482 if (ctx
->query_prepare
)
483 ctx
->query_prepare(batch
, 1);
484 render_sysmem(batch
);
485 ctx
->stats
.batch_sysmem
++;
487 struct fd_gmem_stateobj
*gmem
= &ctx
->gmem
;
488 calculate_tiles(batch
);
489 DBG("%p: rendering %dx%d tiles %ux%u (%s/%s)",
490 batch
, pfb
->width
, pfb
->height
, gmem
->nbins_x
, gmem
->nbins_y
,
491 util_format_short_name(pipe_surface_format(pfb
->cbufs
[0])),
492 util_format_short_name(pipe_surface_format(pfb
->zsbuf
)));
493 if (ctx
->query_prepare
)
494 ctx
->query_prepare(batch
, gmem
->nbins_x
* gmem
->nbins_y
);
496 ctx
->stats
.batch_gmem
++;
502 /* When deciding whether a tile needs mem2gmem, we need to take into
503 * account the scissor rect(s) that were cleared. To simplify we only
504 * consider the last scissor rect for each buffer, since the common
505 * case would be a single clear.
508 fd_gmem_needs_restore(struct fd_batch
*batch
, struct fd_tile
*tile
,
511 if (!(batch
->restore
& buffers
))