src/gallium/drivers/vc5/vc5_rcl.c

   1 /*
   2  * Copyright © 2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "util/u_format.h"
  25 #include "vc5_context.h"
  26 #include "vc5_tiling.h"
  27 #include "broadcom/cle/v3d_packet_v33_pack.h"
  28
  29 static void
  30 load_raw(struct vc5_cl *cl, struct pipe_surface *psurf, int buffer)
  31 {
  32         struct vc5_surface *surf = vc5_surface(psurf);
  33         struct vc5_resource *rsc = vc5_resource(psurf->texture);
  34
  35         cl_emit(cl, LOAD_TILE_BUFFER_GENERAL, load) {
  36                 load.raw_mode = true;
  37                 load.buffer_to_load = buffer;
  38                 load.address = cl_address(rsc->bo, surf->offset);
  39                 load.padded_height_of_output_image_in_uif_blocks =
  40                         surf->padded_height_of_output_image_in_uif_blocks;
  41         }
  42 }
  43
  44 static void
  45 store_raw(struct vc5_cl *cl, struct pipe_surface *psurf, int buffer,
  46           bool color_clear, bool z_clear, bool s_clear)
  47 {
  48         struct vc5_surface *surf = vc5_surface(psurf);
  49         struct vc5_resource *rsc = vc5_resource(psurf->texture);
  50
  51         cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
  52                 store.raw_mode = true;
  53                 store.buffer_to_store = buffer;
  54                 store.address = cl_address(rsc->bo, surf->offset);
  55                 store.disable_colour_buffers_clear_on_write = !color_clear;
  56                 store.disable_z_buffer_clear_on_write = !z_clear;
  57                 store.disable_stencil_buffer_clear_on_write = !s_clear;
  58                 store.padded_height_of_output_image_in_uif_blocks =
  59                         surf->padded_height_of_output_image_in_uif_blocks;
  60         }
  61 }
  62
  63 static int
  64 zs_buffer_from_pipe_bits(int pipe_clear_bits)
  65 {
  66         switch (pipe_clear_bits & PIPE_CLEAR_DEPTHSTENCIL) {
  67         case PIPE_CLEAR_DEPTHSTENCIL:
  68                 return ZSTENCIL;
  69         case PIPE_CLEAR_DEPTH:
  70                 return Z;
  71         case PIPE_CLEAR_STENCIL:
  72                 return STENCIL;
  73         default:
  74                 return NONE;
  75         }
  76 }
  77
  78 /* The HW queues up the load until the tile coordinates show up, but can only
  79  * track one at a time.  If we need to do more than one load, then we need to
  80  * flush out the previous load by emitting the tile coordinates and doing a
  81  * dummy store.
  82  */
  83 static void
  84 flush_last_load(struct vc5_cl *cl)
  85 {
  86         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
  87         cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
  88                 store.buffer_to_store = NONE;
  89         }
  90 }
  91
  92 static void
  93 vc5_rcl_emit_generic_per_tile_list(struct vc5_job *job, int last_cbuf)
  94 {
  95         /* Emit the generic list in our indirect state -- the rcl will just
  96          * have pointers into it.
  97          */
  98         struct vc5_cl *cl = &job->indirect;
  99         vc5_cl_ensure_space(cl, 200, 1);
 100         struct vc5_cl_reloc tile_list_start = cl_get_address(cl);
 101
 102         const uint32_t pipe_clear_color_buffers = (PIPE_CLEAR_COLOR0 |
 103                                                    PIPE_CLEAR_COLOR1 |
 104                                                    PIPE_CLEAR_COLOR2 |
 105                                                    PIPE_CLEAR_COLOR3);
 106         const uint32_t first_color_buffer_bit = (ffs(PIPE_CLEAR_COLOR0) - 1);
 107
 108         uint32_t read_but_not_cleared = job->resolve & ~job->cleared;
 109
 110         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 111                 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
 112                 if (!(read_but_not_cleared & bit))
 113                         continue;
 114
 115                 struct pipe_surface *psurf = job->cbufs[i];
 116                 if (!psurf || psurf->texture->nr_samples <= 1)
 117                         continue;
 118
 119                 load_raw(cl, psurf, RENDER_TARGET_0 + i);
 120                 read_but_not_cleared &= ~bit;
 121
 122                 if (read_but_not_cleared)
 123                         flush_last_load(cl);
 124         }
 125
 126         if (job->zsbuf && job->zsbuf->texture->nr_samples > 1 &&
 127             read_but_not_cleared & PIPE_CLEAR_DEPTHSTENCIL) {
 128                 load_raw(cl, job->zsbuf,
 129                          zs_buffer_from_pipe_bits(read_but_not_cleared));
 130                 read_but_not_cleared &= ~PIPE_CLEAR_DEPTHSTENCIL;
 131                 if (read_but_not_cleared)
 132                         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 133         }
 134
 135         /* The initial reload will be queued until we get the
 136          * tile coordinates.
 137          */
 138         if (read_but_not_cleared) {
 139                 cl_emit(cl, RELOAD_TILE_COLOUR_BUFFER, load) {
 140                         load.disable_colour_buffer_load =
 141                                 (~read_but_not_cleared & pipe_clear_color_buffers) >>
 142                                 first_color_buffer_bit;
 143                         load.enable_z_load =
 144                                 read_but_not_cleared & PIPE_CLEAR_DEPTH;
 145                         load.enable_stencil_load =
 146                                 read_but_not_cleared & PIPE_CLEAR_STENCIL;
 147                 }
 148         }
 149
 150         /* Tile Coordinates triggers the reload and sets where the stores
 151          * go. There must be one per store packet.
 152          */
 153         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 154
 155         /* The binner starts out writing tiles assuming that the initial mode
 156          * is triangles, so make sure that's the case.
 157          */
 158         cl_emit(cl, PRIMITIVE_LIST_FORMAT, fmt) {
 159                 fmt.data_type = LIST_INDEXED;
 160                 fmt.primitive_type = LIST_TRIANGLES;
 161         }
 162
 163         cl_emit(cl, BRANCH_TO_IMPLICIT_TILE_LIST, branch);
 164
 165         bool needs_color_clear = job->cleared & pipe_clear_color_buffers;
 166         bool needs_z_clear = job->cleared & PIPE_CLEAR_DEPTH;
 167         bool needs_s_clear = job->cleared & PIPE_CLEAR_STENCIL;
 168         /* Note that only the color RT being stored will be cleared by a
 169          * STORE_GENERAL, or all of them if the buffer is NONE.
 170          */
 171         bool msaa_color_clear = (needs_color_clear &&
 172                                  (job->cleared & pipe_clear_color_buffers) ==
 173                                  (job->resolve & pipe_clear_color_buffers));
 174
 175         uint32_t stores_pending = job->resolve;
 176
 177         /* Use raw stores for any MSAA surfaces.  These output UIF tiled
 178          * images where each 4x MSAA pixel is a 2x2 quad, and the format will
 179          * be that of the internal_type/internal_bpp, rather than the format
 180          * from GL's perspective.
 181          */
 182         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 183                 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
 184                 if (!(job->resolve & bit))
 185                         continue;
 186
 187                 struct pipe_surface *psurf = job->cbufs[i];
 188                 if (!psurf || psurf->texture->nr_samples <= 1)
 189                         continue;
 190
 191                 stores_pending &= ~bit;
 192                 store_raw(cl, psurf, RENDER_TARGET_0 + i,
 193                           !stores_pending && msaa_color_clear,
 194                           !stores_pending && needs_z_clear,
 195                           !stores_pending && needs_s_clear);
 196
 197                 if (stores_pending)
 198                         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 199         }
 200
 201         if (job->resolve & PIPE_CLEAR_DEPTHSTENCIL && job->zsbuf &&
 202             job->zsbuf->texture->nr_samples > 1) {
 203                 stores_pending &= ~PIPE_CLEAR_DEPTHSTENCIL;
 204                 store_raw(cl, job->zsbuf,
 205                           zs_buffer_from_pipe_bits(job->resolve),
 206                           false,
 207                           !stores_pending && needs_z_clear,
 208                           !stores_pending && needs_s_clear);
 209
 210                 if (stores_pending)
 211                         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 212         }
 213
 214         if (stores_pending) {
 215                 cl_emit(cl, STORE_MULTI_SAMPLE_RESOLVED_TILE_COLOR_BUFFER_EXTENDED, store) {
 216
 217                         store.disable_color_buffer_write =
 218                                 (~stores_pending >> first_color_buffer_bit) & 0xf;
 219                         store.enable_z_write = stores_pending & PIPE_CLEAR_DEPTH;
 220                         store.enable_stencil_write = stores_pending & PIPE_CLEAR_STENCIL;
 221
 222                         /* Note that when set this will clear all of the color
 223                          * buffers.
 224                          */
 225                         store.disable_colour_buffers_clear_on_write =
 226                                 !needs_color_clear;
 227                         store.disable_z_buffer_clear_on_write =
 228                                 !needs_z_clear;
 229                         store.disable_stencil_buffer_clear_on_write =
 230                                 !needs_s_clear;
 231                 };
 232         } else if (needs_color_clear && !msaa_color_clear) {
 233                 /* If we had MSAA color stores that didn't match the set of
 234                  * MSAA color clears, then we need to clear the color buffers
 235                  * now.
 236                  */
 237                 cl_emit(&job->rcl, STORE_TILE_BUFFER_GENERAL, store) {
 238                         store.buffer_to_store = NONE;
 239                 }
 240         }
 241
 242         cl_emit(cl, RETURN_FROM_SUB_LIST, ret);
 243
 244         cl_emit(&job->rcl, START_ADDRESS_OF_GENERIC_TILE_LIST, branch) {
 245                 branch.start = tile_list_start;
 246                 branch.end = cl_get_address(cl);
 247         }
 248 }
 249
 250 #define div_round_up(a, b) (((a) + (b) - 1) / b)
 251
 252 void
 253 vc5_emit_rcl(struct vc5_job *job)
 254 {
 255         /* The RCL list should be empty. */
 256         assert(!job->rcl.bo);
 257
 258         vc5_cl_ensure_space_with_branch(&job->rcl, 200 + 256 *
 259                                         cl_packet_length(SUPERTILE_COORDINATES));
 260         job->submit.rcl_start = job->rcl.bo->offset;
 261         vc5_job_add_bo(job, job->rcl.bo);
 262
 263         int nr_cbufs = 0;
 264         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 265                 if (job->cbufs[i])
 266                         nr_cbufs = i + 1;
 267         }
 268
 269         /* Comon config must be the first TILE_RENDERING_MODE_CONFIGURATION
 270          * and Z_STENCIL_CLEAR_VALUES must be last.  The ones in between are
 271          * optional updates to the previous HW state.
 272          */
 273         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_COMMON_CONFIGURATION,
 274                 config) {
 275                 config.enable_z_store = job->resolve & PIPE_CLEAR_DEPTH;
 276                 config.enable_stencil_store = job->resolve & PIPE_CLEAR_STENCIL;
 277
 278                 config.early_z_disable = !job->uses_early_z;
 279
 280                 config.image_width_pixels = job->draw_width;
 281                 config.image_height_pixels = job->draw_height;
 282
 283                 config.number_of_render_targets_minus_1 =
 284                         MAX2(nr_cbufs, 1) - 1;
 285
 286                 config.multisample_mode_4x = job->msaa;
 287
 288                 config.maximum_bpp_of_all_render_targets = job->internal_bpp;
 289         }
 290
 291         for (int i = 0; i < nr_cbufs; i++) {
 292                 struct pipe_surface *psurf = job->cbufs[i];
 293                 if (!psurf)
 294                         continue;
 295                 struct vc5_surface *surf = vc5_surface(psurf);
 296                 struct vc5_resource *rsc = vc5_resource(psurf->texture);
 297
 298                 uint32_t config_pad = 0;
 299                 uint32_t clear_pad = 0;
 300
 301                 /* XXX: Set the pad for raster. */
 302                 if (surf->tiling == VC5_TILING_UIF_NO_XOR ||
 303                     surf->tiling == VC5_TILING_UIF_XOR) {
 304                         int uif_block_height = vc5_utile_height(rsc->cpp) * 2;
 305                         uint32_t implicit_padded_height = (align(job->draw_height, uif_block_height) /
 306                                                            uif_block_height);
 307                         if (surf->padded_height_of_output_image_in_uif_blocks -
 308                             implicit_padded_height < 15) {
 309                                 config_pad = (surf->padded_height_of_output_image_in_uif_blocks -
 310                                               implicit_padded_height);
 311                         } else {
 312                                 config_pad = 15;
 313                                 clear_pad = surf->padded_height_of_output_image_in_uif_blocks;
 314                         }
 315                 }
 316
 317                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_RENDER_TARGET_CONFIG, rt) {
 318                         rt.address = cl_address(rsc->bo, surf->offset);
 319                         rt.internal_type = surf->internal_type;
 320                         rt.output_image_format = surf->format;
 321                         rt.memory_format = surf->tiling;
 322                         rt.internal_bpp = surf->internal_bpp;
 323                         rt.render_target_number = i;
 324                         rt.pad = config_pad;
 325
 326                         if (job->resolve & PIPE_CLEAR_COLOR0 << i)
 327                                 rsc->writes++;
 328                 }
 329
 330                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART1,
 331                         clear) {
 332                         clear.clear_color_low_32_bits = job->clear_color[i][0];
 333                         clear.clear_color_next_24_bits = job->clear_color[i][1] & 0xffffff;
 334                         clear.render_target_number = i;
 335                 };
 336
 337                 if (surf->internal_bpp >= INTERNAL_BPP_64) {
 338                         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART2,
 339                                 clear) {
 340                                 clear.clear_color_mid_low_32_bits =
 341                                         ((job->clear_color[i][1] >> 24) |
 342                                          (job->clear_color[i][2] << 8));
 343                                 clear.clear_color_mid_high_24_bits =
 344                                         ((job->clear_color[i][2] >> 24) |
 345                                          ((job->clear_color[i][3] & 0xffff) << 8));
 346                                 clear.render_target_number = i;
 347                         };
 348                 }
 349
 350                 if (surf->internal_bpp >= INTERNAL_BPP_128 || clear_pad) {
 351                         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART3,
 352                                 clear) {
 353                                 clear.uif_padded_height_in_uif_blocks = clear_pad;
 354                                 clear.clear_color_high_16_bits = job->clear_color[i][3] >> 16;
 355                                 clear.render_target_number = i;
 356                         };
 357                 }
 358         }
 359
 360         /* TODO: Don't bother emitting if we don't load/clear Z/S. */
 361         if (job->zsbuf) {
 362                 struct pipe_surface *psurf = job->zsbuf;
 363                 struct vc5_surface *surf = vc5_surface(psurf);
 364                 struct vc5_resource *rsc = vc5_resource(psurf->texture);
 365
 366                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CONFIG, zs) {
 367                         zs.address = cl_address(rsc->bo, surf->offset);
 368
 369                         zs.internal_type = surf->internal_type;
 370                         zs.output_image_format = surf->format;
 371                         zs.padded_height_of_output_image_in_uif_blocks =
 372                                 surf->padded_height_of_output_image_in_uif_blocks;
 373
 374                         assert(surf->tiling != VC5_TILING_RASTER);
 375                         zs.memory_format = surf->tiling;
 376                 }
 377
 378                 if (job->resolve & PIPE_CLEAR_DEPTHSTENCIL)
 379                         rsc->writes++;
 380
 381                 /* Emit the separate stencil packet if we have a resource for
 382                  * it.  The HW will only load/store this buffer if the
 383                  * Z/Stencil config doesn't have stencil in its format.
 384                  */
 385                 if (rsc->separate_stencil) {
 386                         cl_emit(&job->rcl,
 387                                 TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CONFIG,
 388                                 zs) {
 389                                 zs.address =
 390                                         cl_address(rsc->separate_stencil->bo,
 391                                                    surf->separate_stencil_offset);
 392
 393                                 zs.z_stencil_id = 1; /* Separate stencil */
 394
 395                                 zs.padded_height_of_output_image_in_uif_blocks =
 396                                         surf->separate_stencil_padded_height_of_output_image_in_uif_blocks;
 397
 398                                 assert(surf->tiling != VC5_TILING_RASTER);
 399                                 zs.memory_format = surf->separate_stencil_tiling;
 400                         }
 401                 }
 402         }
 403
 404         /* Ends rendering mode config. */
 405         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CLEAR_VALUES,
 406                 clear) {
 407                 clear.z_clear_value = job->clear_z;
 408                 clear.stencil_vg_mask_clear_value = job->clear_s;
 409         };
 410
 411         /* Always set initial block size before the first branch, which needs
 412          * to match the value from binning mode config.
 413          */
 414         cl_emit(&job->rcl, TILE_LIST_INITIAL_BLOCK_SIZE, init) {
 415                 init.use_auto_chained_tile_lists = true;
 416                 init.size_of_first_block_in_chained_tile_lists =
 417                         TILE_ALLOCATION_BLOCK_SIZE_64B;
 418         }
 419
 420         uint32_t supertile_w = 1, supertile_h = 1;
 421
 422         /* If doing multicore binning, we would need to initialize each core's
 423          * tile list here.
 424          */
 425         cl_emit(&job->rcl, MULTICORE_RENDERING_TILE_LIST_SET_BASE, list) {
 426                 list.address = cl_address(job->tile_alloc, 0);
 427         }
 428
 429         cl_emit(&job->rcl, MULTICORE_RENDERING_SUPERTILE_CONFIGURATION, config) {
 430                 uint32_t frame_w_in_supertiles, frame_h_in_supertiles;
 431                 const uint32_t max_supertiles = 256;
 432
 433                 /* Size up our supertiles until we get under the limit. */
 434                 for (;;) {
 435                         frame_w_in_supertiles = div_round_up(job->draw_tiles_x,
 436                                                              supertile_w);
 437                         frame_h_in_supertiles = div_round_up(job->draw_tiles_y,
 438                                                              supertile_h);
 439                         if (frame_w_in_supertiles * frame_h_in_supertiles <
 440                             max_supertiles) {
 441                                 break;
 442                         }
 443
 444                         if (supertile_w < supertile_h)
 445                                 supertile_w++;
 446                         else
 447                                 supertile_h++;
 448                 }
 449
 450                 config.total_frame_width_in_tiles = job->draw_tiles_x;
 451                 config.total_frame_height_in_tiles = job->draw_tiles_y;
 452
 453                 config.supertile_width_in_tiles_minus_1 = supertile_w - 1;
 454                 config.supertile_height_in_tiles_minus_1 = supertile_h - 1;
 455
 456                 config.total_frame_width_in_supertiles = frame_w_in_supertiles;
 457                 config.total_frame_height_in_supertiles = frame_h_in_supertiles;
 458         }
 459
 460         /* Start by clearing the tile buffer. */
 461         cl_emit(&job->rcl, TILE_COORDINATES, coords) {
 462                 coords.tile_column_number = 0;
 463                 coords.tile_row_number = 0;
 464         }
 465
 466         cl_emit(&job->rcl, STORE_TILE_BUFFER_GENERAL, store) {
 467                 store.buffer_to_store = NONE;
 468         }
 469
 470         cl_emit(&job->rcl, FLUSH_VCD_CACHE, flush);
 471
 472         vc5_rcl_emit_generic_per_tile_list(job, nr_cbufs - 1);
 473
 474         cl_emit(&job->rcl, WAIT_ON_SEMAPHORE, sem);
 475
 476         /* XXX: Use Morton order */
 477         uint32_t supertile_w_in_pixels = job->tile_width * supertile_w;
 478         uint32_t supertile_h_in_pixels = job->tile_height * supertile_h;
 479         uint32_t min_x_supertile = job->draw_min_x / supertile_w_in_pixels;
 480         uint32_t min_y_supertile = job->draw_min_y / supertile_h_in_pixels;
 481         uint32_t max_x_supertile = (job->draw_max_x - 1) / supertile_w_in_pixels;
 482         uint32_t max_y_supertile = (job->draw_max_y - 1) / supertile_h_in_pixels;
 483
 484         for (int y = min_y_supertile; y <= max_y_supertile; y++) {
 485                 for (int x = min_x_supertile; x <= max_x_supertile; x++) {
 486                         cl_emit(&job->rcl, SUPERTILE_COORDINATES, coords) {
 487                                 coords.column_number_in_supertiles = x;
 488                                 coords.row_number_in_supertiles = y;
 489                         }
 490                 }
 491         }
 492
 493         cl_emit(&job->rcl, END_OF_RENDERING, end);
 494 }