src/gallium/drivers/v3d/v3dx_rcl.c

   1 /*
   2  * Copyright © 2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "util/u_format.h"
  25 #include "v3d_context.h"
  26 #include "v3d_tiling.h"
  27 #include "broadcom/common/v3d_macros.h"
  28 #include "broadcom/cle/v3dx_pack.h"
  29
  30 #define PIPE_CLEAR_COLOR_BUFFERS (PIPE_CLEAR_COLOR0 |                   \
  31                                   PIPE_CLEAR_COLOR1 |                   \
  32                                   PIPE_CLEAR_COLOR2 |                   \
  33                                   PIPE_CLEAR_COLOR3)                    \
  34
  35 #define PIPE_FIRST_COLOR_BUFFER_BIT (ffs(PIPE_CLEAR_COLOR0) - 1)
  36
  37 /* The HW queues up the load until the tile coordinates show up, but can only
  38  * track one at a time.  If we need to do more than one load, then we need to
  39  * flush out the previous load by emitting the tile coordinates and doing a
  40  * dummy store.
  41  */
  42 static void
  43 flush_last_load(struct v3d_cl *cl)
  44 {
  45         if (V3D_VERSION >= 40)
  46                 return;
  47
  48         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
  49         cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
  50                 store.buffer_to_store = NONE;
  51         }
  52 }
  53
  54 static void
  55 load_general(struct v3d_cl *cl, struct pipe_surface *psurf, int buffer,
  56              uint32_t pipe_bit, uint32_t *loads_pending)
  57 {
  58         struct v3d_surface *surf = v3d_surface(psurf);
  59         bool separate_stencil = surf->separate_stencil && buffer == STENCIL;
  60         if (separate_stencil) {
  61                 psurf = surf->separate_stencil;
  62                 surf = v3d_surface(psurf);
  63         }
  64
  65         struct v3d_resource *rsc = v3d_resource(psurf->texture);
  66
  67         cl_emit(cl, LOAD_TILE_BUFFER_GENERAL, load) {
  68                 load.buffer_to_load = buffer;
  69                 load.address = cl_address(rsc->bo, surf->offset);
  70
  71 #if V3D_VERSION >= 40
  72                 load.memory_format = surf->tiling;
  73                 if (separate_stencil)
  74                         load.input_image_format = V3D_OUTPUT_IMAGE_FORMAT_S8;
  75                 else
  76                         load.input_image_format = surf->format;
  77
  78                 if (surf->tiling == VC5_TILING_UIF_NO_XOR ||
  79                     surf->tiling == VC5_TILING_UIF_XOR) {
  80                         load.height_in_ub_or_stride =
  81                                 surf->padded_height_of_output_image_in_uif_blocks;
  82                 } else if (surf->tiling == VC5_TILING_RASTER) {
  83                         struct v3d_resource_slice *slice =
  84                                 &rsc->slices[psurf->u.tex.level];
  85                         load.height_in_ub_or_stride = slice->stride;
  86                 }
  87
  88                 if (psurf->texture->nr_samples > 1)
  89                         load.decimate_mode = V3D_DECIMATE_MODE_ALL_SAMPLES;
  90                 else
  91                         load.decimate_mode = V3D_DECIMATE_MODE_SAMPLE_0;
  92
  93 #else /* V3D_VERSION < 40 */
  94                 /* Can't do raw ZSTENCIL loads -- need to load/store them to
  95                  * separate buffers for Z and stencil.
  96                  */
  97                 assert(buffer != ZSTENCIL);
  98                 load.raw_mode = true;
  99                 load.padded_height_of_output_image_in_uif_blocks =
 100                         surf->padded_height_of_output_image_in_uif_blocks;
 101 #endif /* V3D_VERSION < 40 */
 102         }
 103
 104         *loads_pending &= ~pipe_bit;
 105         if (*loads_pending)
 106                 flush_last_load(cl);
 107 }
 108
 109 static void
 110 store_general(struct v3d_job *job,
 111               struct v3d_cl *cl, struct pipe_surface *psurf, int buffer,
 112               int pipe_bit, uint32_t *stores_pending, bool general_color_clear)
 113 {
 114         struct v3d_surface *surf = v3d_surface(psurf);
 115         bool separate_stencil = surf->separate_stencil && buffer == STENCIL;
 116         if (separate_stencil) {
 117                 psurf = surf->separate_stencil;
 118                 surf = v3d_surface(psurf);
 119         }
 120
 121         *stores_pending &= ~pipe_bit;
 122         bool last_store = !(*stores_pending);
 123
 124         struct v3d_resource *rsc = v3d_resource(psurf->texture);
 125
 126         rsc->writes++;
 127
 128         cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
 129                 store.buffer_to_store = buffer;
 130                 store.address = cl_address(rsc->bo, surf->offset);
 131
 132 #if V3D_VERSION >= 40
 133                 store.clear_buffer_being_stored = false;
 134
 135                 if (separate_stencil)
 136                         store.output_image_format = V3D_OUTPUT_IMAGE_FORMAT_S8;
 137                 else
 138                         store.output_image_format = surf->format;
 139
 140                 store.memory_format = surf->tiling;
 141
 142                 if (surf->tiling == VC5_TILING_UIF_NO_XOR ||
 143                     surf->tiling == VC5_TILING_UIF_XOR) {
 144                         store.height_in_ub_or_stride =
 145                                 surf->padded_height_of_output_image_in_uif_blocks;
 146                 } else if (surf->tiling == VC5_TILING_RASTER) {
 147                         struct v3d_resource_slice *slice =
 148                                 &rsc->slices[psurf->u.tex.level];
 149                         store.height_in_ub_or_stride = slice->stride;
 150                 }
 151
 152                 if (psurf->texture->nr_samples > 1)
 153                         store.decimate_mode = V3D_DECIMATE_MODE_ALL_SAMPLES;
 154                 else
 155                         store.decimate_mode = V3D_DECIMATE_MODE_SAMPLE_0;
 156
 157 #else /* V3D_VERSION < 40 */
 158                 /* Can't do raw ZSTENCIL stores -- need to load/store them to
 159                  * separate buffers for Z and stencil.
 160                  */
 161                 assert(buffer != ZSTENCIL);
 162                 store.raw_mode = true;
 163                 if (!last_store) {
 164                         store.disable_colour_buffers_clear_on_write = true;
 165                         store.disable_z_buffer_clear_on_write = true;
 166                         store.disable_stencil_buffer_clear_on_write = true;
 167                 } else {
 168                         store.disable_colour_buffers_clear_on_write =
 169                                 !(((pipe_bit & PIPE_CLEAR_COLOR_BUFFERS) &&
 170                                    general_color_clear &&
 171                                    (job->cleared & pipe_bit)));
 172                         store.disable_z_buffer_clear_on_write =
 173                                 !(job->cleared & PIPE_CLEAR_DEPTH);
 174                         store.disable_stencil_buffer_clear_on_write =
 175                                 !(job->cleared & PIPE_CLEAR_STENCIL);
 176                 }
 177                 store.padded_height_of_output_image_in_uif_blocks =
 178                         surf->padded_height_of_output_image_in_uif_blocks;
 179 #endif /* V3D_VERSION < 40 */
 180         }
 181
 182         /* There must be a TILE_COORDINATES_IMPLICIT between each store. */
 183         if (V3D_VERSION < 40 && !last_store) {
 184                 cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 185         }
 186 }
 187
 188 static int
 189 zs_buffer_from_pipe_bits(int pipe_clear_bits)
 190 {
 191         switch (pipe_clear_bits & PIPE_CLEAR_DEPTHSTENCIL) {
 192         case PIPE_CLEAR_DEPTHSTENCIL:
 193                 return ZSTENCIL;
 194         case PIPE_CLEAR_DEPTH:
 195                 return Z;
 196         case PIPE_CLEAR_STENCIL:
 197                 return STENCIL;
 198         default:
 199                 return NONE;
 200         }
 201 }
 202
 203 static void
 204 v3d_rcl_emit_loads(struct v3d_job *job, struct v3d_cl *cl)
 205 {
 206         uint32_t loads_pending = job->resolve & ~job->cleared;
 207
 208         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 209                 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
 210                 if (!(loads_pending & bit))
 211                         continue;
 212
 213                 struct pipe_surface *psurf = job->cbufs[i];
 214                 if (!psurf || (V3D_VERSION < 40 &&
 215                                psurf->texture->nr_samples <= 1)) {
 216                         continue;
 217                 }
 218
 219                 load_general(cl, psurf, RENDER_TARGET_0 + i,
 220                              bit, &loads_pending);
 221         }
 222
 223         if ((loads_pending & PIPE_CLEAR_DEPTHSTENCIL) &&
 224             (V3D_VERSION >= 40 ||
 225              (job->zsbuf && job->zsbuf->texture->nr_samples > 1))) {
 226                 struct v3d_resource *rsc = v3d_resource(job->zsbuf->texture);
 227
 228                 if (rsc->separate_stencil &&
 229                     (loads_pending & PIPE_CLEAR_STENCIL)) {
 230                         load_general(cl, job->zsbuf,
 231                                      STENCIL,
 232                                      PIPE_CLEAR_STENCIL,
 233                                      &loads_pending);
 234                 }
 235
 236                 if (loads_pending & PIPE_CLEAR_DEPTHSTENCIL) {
 237                         load_general(cl, job->zsbuf,
 238                                      zs_buffer_from_pipe_bits(loads_pending),
 239                                      loads_pending & PIPE_CLEAR_DEPTHSTENCIL,
 240                                      &loads_pending);
 241                 }
 242         }
 243
 244 #if V3D_VERSION < 40
 245         /* The initial reload will be queued until we get the
 246          * tile coordinates.
 247          */
 248         if (loads_pending) {
 249                 cl_emit(cl, RELOAD_TILE_COLOUR_BUFFER, load) {
 250                         load.disable_colour_buffer_load =
 251                                 (~loads_pending &
 252                                  PIPE_CLEAR_COLOR_BUFFERS) >>
 253                                 PIPE_FIRST_COLOR_BUFFER_BIT;
 254                         load.enable_z_load =
 255                                 loads_pending & PIPE_CLEAR_DEPTH;
 256                         load.enable_stencil_load =
 257                                 loads_pending & PIPE_CLEAR_STENCIL;
 258                 }
 259         }
 260 #else /* V3D_VERSION >= 40 */
 261         assert(!loads_pending);
 262         cl_emit(cl, END_OF_LOADS, end);
 263 #endif
 264 }
 265
 266 static void
 267 v3d_rcl_emit_stores(struct v3d_job *job, struct v3d_cl *cl)
 268 {
 269 #if V3D_VERSION < 40
 270         MAYBE_UNUSED bool needs_color_clear = job->cleared & PIPE_CLEAR_COLOR_BUFFERS;
 271         MAYBE_UNUSED bool needs_z_clear = job->cleared & PIPE_CLEAR_DEPTH;
 272         MAYBE_UNUSED bool needs_s_clear = job->cleared & PIPE_CLEAR_STENCIL;
 273
 274         /* For clearing color in a TLB general on V3D 3.3:
 275          *
 276          * - NONE buffer store clears all TLB color buffers.
 277          * - color buffer store clears just the TLB color buffer being stored.
 278          * - Z/S buffers store may not clear the TLB color buffer.
 279          *
 280          * And on V3D 4.1, we only have one flag for "clear the buffer being
 281          * stored" in the general packet, and a separate packet to clear all
 282          * color TLB buffers.
 283          *
 284          * As a result, we only bother flagging TLB color clears in a general
 285          * packet when we don't have to emit a separate packet to clear all
 286          * TLB color buffers.
 287          */
 288         bool general_color_clear = (needs_color_clear &&
 289                                     (job->cleared & PIPE_CLEAR_COLOR_BUFFERS) ==
 290                                     (job->resolve & PIPE_CLEAR_COLOR_BUFFERS));
 291 #else
 292         bool general_color_clear = false;
 293 #endif
 294
 295         uint32_t stores_pending = job->resolve;
 296
 297         /* For V3D 4.1, use general stores for all TLB stores.
 298          *
 299          * For V3D 3.3, we only use general stores to do raw stores for any
 300          * MSAA surfaces.  These output UIF tiled images where each 4x MSAA
 301          * pixel is a 2x2 quad, and the format will be that of the
 302          * internal_type/internal_bpp, rather than the format from GL's
 303          * perspective.  Non-MSAA surfaces will use
 304          * STORE_MULTI_SAMPLE_RESOLVED_TILE_COLOR_BUFFER_EXTENDED.
 305          */
 306         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 307                 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
 308                 if (!(job->resolve & bit))
 309                         continue;
 310
 311                 struct pipe_surface *psurf = job->cbufs[i];
 312                 if (!psurf ||
 313                     (V3D_VERSION < 40 && psurf->texture->nr_samples <= 1)) {
 314                         continue;
 315                 }
 316
 317                 store_general(job, cl, psurf, RENDER_TARGET_0 + i, bit,
 318                               &stores_pending, general_color_clear);
 319         }
 320
 321         if (job->resolve & PIPE_CLEAR_DEPTHSTENCIL && job->zsbuf &&
 322             !(V3D_VERSION < 40 && job->zsbuf->texture->nr_samples <= 1)) {
 323                 struct v3d_resource *rsc = v3d_resource(job->zsbuf->texture);
 324                 if (rsc->separate_stencil) {
 325                         if (job->resolve & PIPE_CLEAR_DEPTH) {
 326                                 store_general(job, cl, job->zsbuf, Z,
 327                                               PIPE_CLEAR_DEPTH,
 328                                               &stores_pending,
 329                                               general_color_clear);
 330                         }
 331
 332                         if (job->resolve & PIPE_CLEAR_STENCIL) {
 333                                 store_general(job, cl, job->zsbuf, STENCIL,
 334                                               PIPE_CLEAR_STENCIL,
 335                                               &stores_pending,
 336                                               general_color_clear);
 337                         }
 338                 } else {
 339                         store_general(job, cl, job->zsbuf,
 340                                       zs_buffer_from_pipe_bits(job->resolve),
 341                                       job->resolve & PIPE_CLEAR_DEPTHSTENCIL,
 342                                       &stores_pending, general_color_clear);
 343                 }
 344         }
 345
 346 #if V3D_VERSION < 40
 347         if (stores_pending) {
 348                 cl_emit(cl, STORE_MULTI_SAMPLE_RESOLVED_TILE_COLOR_BUFFER_EXTENDED, store) {
 349
 350                         store.disable_color_buffer_write =
 351                                 (~stores_pending >>
 352                                  PIPE_FIRST_COLOR_BUFFER_BIT) & 0xf;
 353                         store.enable_z_write = stores_pending & PIPE_CLEAR_DEPTH;
 354                         store.enable_stencil_write = stores_pending & PIPE_CLEAR_STENCIL;
 355
 356                         /* Note that when set this will clear all of the color
 357                          * buffers.
 358                          */
 359                         store.disable_colour_buffers_clear_on_write =
 360                                 !needs_color_clear;
 361                         store.disable_z_buffer_clear_on_write =
 362                                 !needs_z_clear;
 363                         store.disable_stencil_buffer_clear_on_write =
 364                                 !needs_s_clear;
 365                 };
 366         } else if (needs_color_clear && !general_color_clear) {
 367                 /* If we didn't do our color clears in the general packet,
 368                  * then emit a packet to clear all the TLB color buffers now.
 369                  */
 370                 cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
 371                         store.buffer_to_store = NONE;
 372                 }
 373         }
 374 #else /* V3D_VERSION >= 40 */
 375         assert(!stores_pending);
 376
 377         /* GFXH-1461/GFXH-1689: The per-buffer store command's clear
 378          * buffer bit is broken for depth/stencil.  In addition, the
 379          * clear packet's Z/S bit is broken, but the RTs bit ends up
 380          * clearing Z/S.
 381          */
 382         if (job->cleared) {
 383                 cl_emit(cl, CLEAR_TILE_BUFFERS, clear) {
 384                         clear.clear_z_stencil_buffer = true;
 385                         clear.clear_all_render_targets = true;
 386                 }
 387         }
 388 #endif /* V3D_VERSION >= 40 */
 389 }
 390
 391 static void
 392 v3d_rcl_emit_generic_per_tile_list(struct v3d_job *job, int last_cbuf)
 393 {
 394         /* Emit the generic list in our indirect state -- the rcl will just
 395          * have pointers into it.
 396          */
 397         struct v3d_cl *cl = &job->indirect;
 398         v3d_cl_ensure_space(cl, 200, 1);
 399         struct v3d_cl_reloc tile_list_start = cl_get_address(cl);
 400
 401         if (V3D_VERSION >= 40) {
 402                 /* V3D 4.x only requires a single tile coordinates, and
 403                  * END_OF_LOADS switches us between loading and rendering.
 404                  */
 405                 cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 406         }
 407
 408         v3d_rcl_emit_loads(job, cl);
 409
 410         if (V3D_VERSION < 40) {
 411                 /* Tile Coordinates triggers the last reload and sets where
 412                  * the stores go. There must be one per store packet.
 413                  */
 414                 cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 415         }
 416
 417         /* The binner starts out writing tiles assuming that the initial mode
 418          * is triangles, so make sure that's the case.
 419          */
 420         cl_emit(cl, PRIMITIVE_LIST_FORMAT, fmt) {
 421                 fmt.data_type = LIST_INDEXED;
 422                 fmt.primitive_type = LIST_TRIANGLES;
 423         }
 424
 425         cl_emit(cl, BRANCH_TO_IMPLICIT_TILE_LIST, branch);
 426
 427         v3d_rcl_emit_stores(job, cl);
 428
 429 #if V3D_VERSION >= 40
 430         cl_emit(cl, END_OF_TILE_MARKER, end);
 431 #endif
 432
 433         cl_emit(cl, RETURN_FROM_SUB_LIST, ret);
 434
 435         cl_emit(&job->rcl, START_ADDRESS_OF_GENERIC_TILE_LIST, branch) {
 436                 branch.start = tile_list_start;
 437                 branch.end = cl_get_address(cl);
 438         }
 439 }
 440
 441 #if V3D_VERSION >= 40
 442 static void
 443 v3d_setup_render_target(struct v3d_job *job, int cbuf,
 444                         uint32_t *rt_bpp, uint32_t *rt_type, uint32_t *rt_clamp)
 445 {
 446         if (!job->cbufs[cbuf])
 447                 return;
 448
 449         struct v3d_surface *surf = v3d_surface(job->cbufs[cbuf]);
 450         *rt_bpp = surf->internal_bpp;
 451         *rt_type = surf->internal_type;
 452         *rt_clamp = V3D_RENDER_TARGET_CLAMP_NONE;
 453 }
 454
 455 #else /* V3D_VERSION < 40 */
 456
 457 static void
 458 v3d_emit_z_stencil_config(struct v3d_job *job, struct v3d_surface *surf,
 459                           struct v3d_resource *rsc, bool is_separate_stencil)
 460 {
 461         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CONFIG, zs) {
 462                 zs.address = cl_address(rsc->bo, surf->offset);
 463
 464                 if (!is_separate_stencil) {
 465                         zs.internal_type = surf->internal_type;
 466                         zs.output_image_format = surf->format;
 467                 } else {
 468                         zs.z_stencil_id = 1; /* Separate stencil */
 469                 }
 470
 471                 zs.padded_height_of_output_image_in_uif_blocks =
 472                         surf->padded_height_of_output_image_in_uif_blocks;
 473
 474                 assert(surf->tiling != VC5_TILING_RASTER);
 475                 zs.memory_format = surf->tiling;
 476         }
 477
 478         if (job->resolve & (is_separate_stencil ?
 479                             PIPE_CLEAR_STENCIL :
 480                             PIPE_CLEAR_DEPTHSTENCIL)) {
 481                 rsc->writes++;
 482         }
 483 }
 484 #endif /* V3D_VERSION < 40 */
 485
 486 #define div_round_up(a, b) (((a) + (b) - 1) / b)
 487
 488 void
 489 v3dX(emit_rcl)(struct v3d_job *job)
 490 {
 491         /* The RCL list should be empty. */
 492         assert(!job->rcl.bo);
 493
 494         v3d_cl_ensure_space_with_branch(&job->rcl, 200 + 256 *
 495                                         cl_packet_length(SUPERTILE_COORDINATES));
 496         job->submit.rcl_start = job->rcl.bo->offset;
 497         v3d_job_add_bo(job, job->rcl.bo);
 498
 499         int nr_cbufs = 0;
 500         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 501                 if (job->cbufs[i])
 502                         nr_cbufs = i + 1;
 503         }
 504
 505         /* Comon config must be the first TILE_RENDERING_MODE_CONFIGURATION
 506          * and Z_STENCIL_CLEAR_VALUES must be last.  The ones in between are
 507          * optional updates to the previous HW state.
 508          */
 509         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_COMMON_CONFIGURATION,
 510                 config) {
 511 #if V3D_VERSION < 40
 512                 config.enable_z_store = job->resolve & PIPE_CLEAR_DEPTH;
 513                 config.enable_stencil_store = job->resolve & PIPE_CLEAR_STENCIL;
 514 #else /* V3D_VERSION >= 40 */
 515                 if (job->zsbuf) {
 516                         struct v3d_surface *surf = v3d_surface(job->zsbuf);
 517                         config.internal_depth_type = surf->internal_type;
 518                 }
 519 #endif /* V3D_VERSION >= 40 */
 520
 521                 /* XXX: Early D/S clear */
 522
 523                 switch (job->first_ez_state) {
 524                 case VC5_EZ_UNDECIDED:
 525                 case VC5_EZ_LT_LE:
 526                         config.early_z_disable = false;
 527                         config.early_z_test_and_update_direction =
 528                                 EARLY_Z_DIRECTION_LT_LE;
 529                         break;
 530                 case VC5_EZ_GT_GE:
 531                         config.early_z_disable = false;
 532                         config.early_z_test_and_update_direction =
 533                                 EARLY_Z_DIRECTION_GT_GE;
 534                         break;
 535                 case VC5_EZ_DISABLED:
 536                         config.early_z_disable = true;
 537                 }
 538
 539                 config.image_width_pixels = job->draw_width;
 540                 config.image_height_pixels = job->draw_height;
 541
 542                 config.number_of_render_targets = MAX2(nr_cbufs, 1);
 543
 544                 config.multisample_mode_4x = job->msaa;
 545
 546                 config.maximum_bpp_of_all_render_targets = job->internal_bpp;
 547         }
 548
 549         for (int i = 0; i < nr_cbufs; i++) {
 550                 struct pipe_surface *psurf = job->cbufs[i];
 551                 if (!psurf)
 552                         continue;
 553                 struct v3d_surface *surf = v3d_surface(psurf);
 554                 struct v3d_resource *rsc = v3d_resource(psurf->texture);
 555
 556                 MAYBE_UNUSED uint32_t config_pad = 0;
 557                 uint32_t clear_pad = 0;
 558
 559                 /* XXX: Set the pad for raster. */
 560                 if (surf->tiling == VC5_TILING_UIF_NO_XOR ||
 561                     surf->tiling == VC5_TILING_UIF_XOR) {
 562                         int uif_block_height = v3d_utile_height(rsc->cpp) * 2;
 563                         uint32_t implicit_padded_height = (align(job->draw_height, uif_block_height) /
 564                                                            uif_block_height);
 565                         if (surf->padded_height_of_output_image_in_uif_blocks -
 566                             implicit_padded_height < 15) {
 567                                 config_pad = (surf->padded_height_of_output_image_in_uif_blocks -
 568                                               implicit_padded_height);
 569                         } else {
 570                                 config_pad = 15;
 571                                 clear_pad = surf->padded_height_of_output_image_in_uif_blocks;
 572                         }
 573                 }
 574
 575 #if V3D_VERSION < 40
 576                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_RENDER_TARGET_CONFIG, rt) {
 577                         rt.address = cl_address(rsc->bo, surf->offset);
 578                         rt.internal_type = surf->internal_type;
 579                         rt.output_image_format = surf->format;
 580                         rt.memory_format = surf->tiling;
 581                         rt.internal_bpp = surf->internal_bpp;
 582                         rt.render_target_number = i;
 583                         rt.pad = config_pad;
 584
 585                         if (job->resolve & PIPE_CLEAR_COLOR0 << i)
 586                                 rsc->writes++;
 587                 }
 588 #endif /* V3D_VERSION < 40 */
 589
 590                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART1,
 591                         clear) {
 592                         clear.clear_color_low_32_bits = job->clear_color[i][0];
 593                         clear.clear_color_next_24_bits = job->clear_color[i][1] & 0xffffff;
 594                         clear.render_target_number = i;
 595                 };
 596
 597                 if (surf->internal_bpp >= V3D_INTERNAL_BPP_64) {
 598                         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART2,
 599                                 clear) {
 600                                 clear.clear_color_mid_low_32_bits =
 601                                         ((job->clear_color[i][1] >> 24) |
 602                                          (job->clear_color[i][2] << 8));
 603                                 clear.clear_color_mid_high_24_bits =
 604                                         ((job->clear_color[i][2] >> 24) |
 605                                          ((job->clear_color[i][3] & 0xffff) << 8));
 606                                 clear.render_target_number = i;
 607                         };
 608                 }
 609
 610                 if (surf->internal_bpp >= V3D_INTERNAL_BPP_128 || clear_pad) {
 611                         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART3,
 612                                 clear) {
 613                                 clear.uif_padded_height_in_uif_blocks = clear_pad;
 614                                 clear.clear_color_high_16_bits = job->clear_color[i][3] >> 16;
 615                                 clear.render_target_number = i;
 616                         };
 617                 }
 618         }
 619
 620 #if V3D_VERSION >= 40
 621         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_RENDER_TARGET_CONFIG, rt) {
 622                 v3d_setup_render_target(job, 0,
 623                                         &rt.render_target_0_internal_bpp,
 624                                         &rt.render_target_0_internal_type,
 625                                         &rt.render_target_0_clamp);
 626                 v3d_setup_render_target(job, 1,
 627                                         &rt.render_target_1_internal_bpp,
 628                                         &rt.render_target_1_internal_type,
 629                                         &rt.render_target_1_clamp);
 630                 v3d_setup_render_target(job, 2,
 631                                         &rt.render_target_2_internal_bpp,
 632                                         &rt.render_target_2_internal_type,
 633                                         &rt.render_target_2_clamp);
 634                 v3d_setup_render_target(job, 3,
 635                                         &rt.render_target_3_internal_bpp,
 636                                         &rt.render_target_3_internal_type,
 637                                         &rt.render_target_3_clamp);
 638         }
 639 #endif
 640
 641 #if V3D_VERSION < 40
 642         /* TODO: Don't bother emitting if we don't load/clear Z/S. */
 643         if (job->zsbuf) {
 644                 struct pipe_surface *psurf = job->zsbuf;
 645                 struct v3d_surface *surf = v3d_surface(psurf);
 646                 struct v3d_resource *rsc = v3d_resource(psurf->texture);
 647
 648                 v3d_emit_z_stencil_config(job, surf, rsc, false);
 649
 650                 /* Emit the separate stencil packet if we have a resource for
 651                  * it.  The HW will only load/store this buffer if the
 652                  * Z/Stencil config doesn't have stencil in its format.
 653                  */
 654                 if (surf->separate_stencil) {
 655                         v3d_emit_z_stencil_config(job,
 656                                                   v3d_surface(surf->separate_stencil),
 657                                                   rsc->separate_stencil, true);
 658                 }
 659         }
 660 #endif /* V3D_VERSION < 40 */
 661
 662         /* Ends rendering mode config. */
 663         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CLEAR_VALUES,
 664                 clear) {
 665                 clear.z_clear_value = job->clear_z;
 666                 clear.stencil_vg_mask_clear_value = job->clear_s;
 667         };
 668
 669         /* Always set initial block size before the first branch, which needs
 670          * to match the value from binning mode config.
 671          */
 672         cl_emit(&job->rcl, TILE_LIST_INITIAL_BLOCK_SIZE, init) {
 673                 init.use_auto_chained_tile_lists = true;
 674                 init.size_of_first_block_in_chained_tile_lists =
 675                         TILE_ALLOCATION_BLOCK_SIZE_64B;
 676         }
 677
 678         uint32_t supertile_w = 1, supertile_h = 1;
 679
 680         /* If doing multicore binning, we would need to initialize each core's
 681          * tile list here.
 682          */
 683         cl_emit(&job->rcl, MULTICORE_RENDERING_TILE_LIST_SET_BASE, list) {
 684                 list.address = cl_address(job->tile_alloc, 0);
 685         }
 686
 687         cl_emit(&job->rcl, MULTICORE_RENDERING_SUPERTILE_CONFIGURATION, config) {
 688                 uint32_t frame_w_in_supertiles, frame_h_in_supertiles;
 689                 const uint32_t max_supertiles = 256;
 690
 691                 /* Size up our supertiles until we get under the limit. */
 692                 for (;;) {
 693                         frame_w_in_supertiles = div_round_up(job->draw_tiles_x,
 694                                                              supertile_w);
 695                         frame_h_in_supertiles = div_round_up(job->draw_tiles_y,
 696                                                              supertile_h);
 697                         if (frame_w_in_supertiles * frame_h_in_supertiles <
 698                             max_supertiles) {
 699                                 break;
 700                         }
 701
 702                         if (supertile_w < supertile_h)
 703                                 supertile_w++;
 704                         else
 705                                 supertile_h++;
 706                 }
 707
 708                 config.total_frame_width_in_tiles = job->draw_tiles_x;
 709                 config.total_frame_height_in_tiles = job->draw_tiles_y;
 710
 711                 config.supertile_width_in_tiles = supertile_w;
 712                 config.supertile_height_in_tiles = supertile_h;
 713
 714                 config.total_frame_width_in_supertiles = frame_w_in_supertiles;
 715                 config.total_frame_height_in_supertiles = frame_h_in_supertiles;
 716         }
 717
 718         /* Start by clearing the tile buffer. */
 719         cl_emit(&job->rcl, TILE_COORDINATES, coords) {
 720                 coords.tile_column_number = 0;
 721                 coords.tile_row_number = 0;
 722         }
 723
 724         /* Emit an initial clear of the tile buffers.  This is necessary for
 725          * any buffers that should be cleared (since clearing normally happens
 726          * at the *end* of the generic tile list), but it's also nice to clear
 727          * everything so the first tile doesn't inherit any contents from some
 728          * previous frame.
 729          *
 730          * Also, implement the GFXH-1742 workaround.  There's a race in the HW
 731          * between the RCL updating the TLB's internal type/size and the
 732          * spawning of the QPU instances using the TLB's current internal
 733          * type/size.  To make sure the QPUs get the right state,, we need 1
 734          * dummy store in between internal type/size changes on V3D 3.x, and 2
 735          * dummy stores on 4.x.
 736          */
 737 #if V3D_VERSION < 40
 738         cl_emit(&job->rcl, STORE_TILE_BUFFER_GENERAL, store) {
 739                 store.buffer_to_store = NONE;
 740         }
 741 #else
 742         for (int i = 0; i < 2; i++) {
 743                 if (i > 0)
 744                         cl_emit(&job->rcl, TILE_COORDINATES, coords);
 745                 cl_emit(&job->rcl, END_OF_LOADS, end);
 746                 cl_emit(&job->rcl, STORE_TILE_BUFFER_GENERAL, store) {
 747                         store.buffer_to_store = NONE;
 748                 }
 749                 if (i == 0) {
 750                         cl_emit(&job->rcl, CLEAR_TILE_BUFFERS, clear) {
 751                                 clear.clear_z_stencil_buffer = true;
 752                                 clear.clear_all_render_targets = true;
 753                         }
 754                 }
 755                 cl_emit(&job->rcl, END_OF_TILE_MARKER, end);
 756         }
 757 #endif
 758
 759         cl_emit(&job->rcl, FLUSH_VCD_CACHE, flush);
 760
 761         v3d_rcl_emit_generic_per_tile_list(job, nr_cbufs - 1);
 762
 763         cl_emit(&job->rcl, WAIT_ON_SEMAPHORE, sem);
 764
 765         /* XXX: Use Morton order */
 766         uint32_t supertile_w_in_pixels = job->tile_width * supertile_w;
 767         uint32_t supertile_h_in_pixels = job->tile_height * supertile_h;
 768         uint32_t min_x_supertile = job->draw_min_x / supertile_w_in_pixels;
 769         uint32_t min_y_supertile = job->draw_min_y / supertile_h_in_pixels;
 770
 771         uint32_t max_x_supertile = 0;
 772         uint32_t max_y_supertile = 0;
 773         if (job->draw_max_x != 0 && job->draw_max_y != 0) {
 774                 max_x_supertile = (job->draw_max_x - 1) / supertile_w_in_pixels;
 775                 max_y_supertile = (job->draw_max_y - 1) / supertile_h_in_pixels;
 776         }
 777
 778         for (int y = min_y_supertile; y <= max_y_supertile; y++) {
 779                 for (int x = min_x_supertile; x <= max_x_supertile; x++) {
 780                         cl_emit(&job->rcl, SUPERTILE_COORDINATES, coords) {
 781                                 coords.column_number_in_supertiles = x;
 782                                 coords.row_number_in_supertiles = y;
 783                         }
 784                 }
 785         }
 786
 787         cl_emit(&job->rcl, END_OF_RENDERING, end);
 788 }