src/gallium/drivers/vc5/vc5_rcl.c

   1 /*
   2  * Copyright © 2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "util/u_format.h"
  25 #include "vc5_context.h"
  26 #include "vc5_tiling.h"
  27 #include "broadcom/cle/v3d_packet_v33_pack.h"
  28
  29 static void
  30 load_raw(struct vc5_cl *cl, struct pipe_surface *psurf, int buffer)
  31 {
  32         struct vc5_surface *surf = vc5_surface(psurf);
  33         struct vc5_resource *rsc = vc5_resource(psurf->texture);
  34
  35         cl_emit(cl, LOAD_TILE_BUFFER_GENERAL, load) {
  36                 load.raw_mode = true;
  37                 load.buffer_to_load = buffer;
  38                 load.address = cl_address(rsc->bo, surf->offset);
  39
  40                 struct vc5_resource_slice *slice =
  41                         &rsc->slices[psurf->u.tex.level];
  42                 load.padded_height_of_output_image_in_uif_blocks =
  43                         (slice->size / slice->stride) /
  44                         (2 * vc5_utile_height(rsc->cpp));
  45         }
  46 }
  47
  48 static void
  49 store_raw(struct vc5_cl *cl, struct pipe_surface *psurf, int buffer,
  50           bool color_clear, bool z_clear, bool s_clear)
  51 {
  52         struct vc5_surface *surf = vc5_surface(psurf);
  53         struct vc5_resource *rsc = vc5_resource(psurf->texture);
  54
  55         cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
  56                 store.raw_mode = true;
  57                 store.buffer_to_store = buffer;
  58                 store.address = cl_address(rsc->bo, surf->offset);
  59                 store.disable_colour_buffers_clear_on_write = !color_clear;
  60                 store.disable_z_buffer_clear_on_write = !z_clear;
  61                 store.disable_stencil_buffer_clear_on_write = !s_clear;
  62
  63                 struct vc5_resource_slice *slice =
  64                         &rsc->slices[psurf->u.tex.level];
  65                 store.padded_height_of_output_image_in_uif_blocks =
  66                         (slice->size / slice->stride) /
  67                         (2 * vc5_utile_height(rsc->cpp));
  68         }
  69 }
  70
  71 static int
  72 zs_buffer_from_pipe_bits(int pipe_clear_bits)
  73 {
  74         switch (pipe_clear_bits & PIPE_CLEAR_DEPTHSTENCIL) {
  75         case PIPE_CLEAR_DEPTHSTENCIL:
  76                 return ZSTENCIL;
  77         case PIPE_CLEAR_DEPTH:
  78                 return Z;
  79         case PIPE_CLEAR_STENCIL:
  80                 return STENCIL;
  81         default:
  82                 return NONE;
  83         }
  84 }
  85
  86 /* The HW queues up the load until the tile coordinates show up, but can only
  87  * track one at a time.  If we need to do more than one load, then we need to
  88  * flush out the previous load by emitting the tile coordinates and doing a
  89  * dummy store.
  90  */
  91 static void
  92 flush_last_load(struct vc5_cl *cl)
  93 {
  94         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
  95         cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) {
  96                 store.buffer_to_store = NONE;
  97         }
  98 }
  99
 100 static void
 101 vc5_rcl_emit_generic_per_tile_list(struct vc5_job *job, int last_cbuf)
 102 {
 103         /* Emit the generic list in our indirect state -- the rcl will just
 104          * have pointers into it.
 105          */
 106         struct vc5_cl *cl = &job->indirect;
 107         vc5_cl_ensure_space(cl, 200, 1);
 108         struct vc5_cl_reloc tile_list_start = cl_get_address(cl);
 109
 110         const uint32_t pipe_clear_color_buffers = (PIPE_CLEAR_COLOR0 |
 111                                                    PIPE_CLEAR_COLOR1 |
 112                                                    PIPE_CLEAR_COLOR2 |
 113                                                    PIPE_CLEAR_COLOR3);
 114         const uint32_t first_color_buffer_bit = (ffs(PIPE_CLEAR_COLOR0) - 1);
 115
 116         uint32_t read_but_not_cleared = job->resolve & ~job->cleared;
 117
 118         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 119                 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
 120                 if (!(read_but_not_cleared & bit))
 121                         continue;
 122
 123                 struct pipe_surface *psurf = job->cbufs[i];
 124                 if (!psurf || psurf->texture->nr_samples <= 1)
 125                         continue;
 126
 127                 load_raw(cl, psurf, RENDER_TARGET_0 + i);
 128                 read_but_not_cleared &= ~bit;
 129
 130                 if (read_but_not_cleared)
 131                         flush_last_load(cl);
 132         }
 133
 134         if (job->zsbuf && job->zsbuf->texture->nr_samples > 1 &&
 135             read_but_not_cleared & PIPE_CLEAR_DEPTHSTENCIL) {
 136                 load_raw(cl, job->zsbuf,
 137                          zs_buffer_from_pipe_bits(read_but_not_cleared));
 138                 read_but_not_cleared &= ~PIPE_CLEAR_DEPTHSTENCIL;
 139                 if (read_but_not_cleared)
 140                         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 141         }
 142
 143         /* The initial reload will be queued until we get the
 144          * tile coordinates.
 145          */
 146         if (read_but_not_cleared) {
 147                 cl_emit(cl, RELOAD_TILE_COLOUR_BUFFER, load) {
 148                         load.disable_colour_buffer_load =
 149                                 (~read_but_not_cleared & pipe_clear_color_buffers) >>
 150                                 first_color_buffer_bit;
 151                         load.enable_z_load =
 152                                 read_but_not_cleared & PIPE_CLEAR_DEPTH;
 153                         load.enable_stencil_load =
 154                                 read_but_not_cleared & PIPE_CLEAR_STENCIL;
 155                 }
 156         }
 157
 158         /* Tile Coordinates triggers the reload and sets where the stores
 159          * go. There must be one per store packet.
 160          */
 161         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 162
 163         cl_emit(cl, BRANCH_TO_IMPLICIT_TILE_LIST, branch);
 164
 165         bool needs_color_clear = job->cleared & pipe_clear_color_buffers;
 166         bool needs_z_clear = job->cleared & PIPE_CLEAR_DEPTH;
 167         bool needs_s_clear = job->cleared & PIPE_CLEAR_STENCIL;
 168
 169         uint32_t stores_pending = job->resolve;
 170
 171         /* Use raw stores for any MSAA surfaces.  These output UIF tiled
 172          * images where each 4x MSAA pixel is a 2x2 quad, and the format will
 173          * be that of the internal_type/internal_bpp, rather than the format
 174          * from GL's perspective.
 175          */
 176         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 177                 uint32_t bit = PIPE_CLEAR_COLOR0 << i;
 178                 if (!(job->resolve & bit))
 179                         continue;
 180
 181                 struct pipe_surface *psurf = job->cbufs[i];
 182                 if (!psurf || psurf->texture->nr_samples <= 1)
 183                         continue;
 184
 185                 stores_pending &= ~bit;
 186                 store_raw(cl, psurf, RENDER_TARGET_0 + i,
 187                           !stores_pending && needs_color_clear,
 188                           !stores_pending && needs_z_clear,
 189                           !stores_pending && needs_s_clear);
 190
 191                 if (stores_pending)
 192                         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 193         }
 194
 195         if (job->resolve & PIPE_CLEAR_DEPTHSTENCIL && job->zsbuf &&
 196             job->zsbuf->texture->nr_samples > 1) {
 197                 stores_pending &= ~PIPE_CLEAR_DEPTHSTENCIL;
 198                 store_raw(cl, job->zsbuf,
 199                           zs_buffer_from_pipe_bits(job->resolve),
 200                           !stores_pending && needs_color_clear,
 201                           !stores_pending && needs_z_clear,
 202                           !stores_pending && needs_s_clear);
 203
 204                 if (stores_pending)
 205                         cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords);
 206         }
 207
 208         if (stores_pending) {
 209                 cl_emit(cl, STORE_MULTI_SAMPLE_RESOLVED_TILE_COLOR_BUFFER_EXTENDED, store) {
 210
 211                         store.disable_color_buffer_write =
 212                                 (~stores_pending >> first_color_buffer_bit) & 0xf;
 213                         store.enable_z_write = stores_pending & PIPE_CLEAR_DEPTH;
 214                         store.enable_stencil_write = stores_pending & PIPE_CLEAR_STENCIL;
 215
 216                         store.disable_colour_buffers_clear_on_write =
 217                                 !needs_color_clear;
 218                         store.disable_z_buffer_clear_on_write =
 219                                 !needs_z_clear;
 220                         store.disable_stencil_buffer_clear_on_write =
 221                                 !needs_s_clear;
 222                 };
 223         }
 224
 225         cl_emit(cl, RETURN_FROM_SUB_LIST, ret);
 226
 227         cl_emit(&job->rcl, START_ADDRESS_OF_GENERIC_TILE_LIST, branch) {
 228                 branch.start = tile_list_start;
 229                 branch.end = cl_get_address(cl);
 230         }
 231 }
 232
 233 #define div_round_up(a, b) (((a) + (b) - 1) / b)
 234
 235 void
 236 vc5_emit_rcl(struct vc5_job *job)
 237 {
 238         /* The RCL list should be empty. */
 239         assert(!job->rcl.bo);
 240
 241         vc5_cl_ensure_space_with_branch(&job->rcl, 200 + 256 *
 242                                         cl_packet_length(SUPERTILE_COORDINATES));
 243         job->submit.rcl_start = job->rcl.bo->offset;
 244         vc5_job_add_bo(job, job->rcl.bo);
 245
 246         int nr_cbufs = 0;
 247         for (int i = 0; i < VC5_MAX_DRAW_BUFFERS; i++) {
 248                 if (job->cbufs[i])
 249                         nr_cbufs = i + 1;
 250         }
 251
 252         /* Comon config must be the first TILE_RENDERING_MODE_CONFIGURATION
 253          * and Z_STENCIL_CLEAR_VALUES must be last.  The ones in between are
 254          * optional updates to the previous HW state.
 255          */
 256         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_COMMON_CONFIGURATION,
 257                 config) {
 258                 config.enable_z_store = job->resolve & PIPE_CLEAR_DEPTH;
 259                 config.enable_stencil_store = job->resolve & PIPE_CLEAR_STENCIL;
 260
 261                 config.early_z_disable = !job->uses_early_z;
 262
 263                 config.image_width_pixels = job->draw_width;
 264                 config.image_height_pixels = job->draw_height;
 265
 266                 config.number_of_render_targets_minus_1 =
 267                         MAX2(nr_cbufs, 1) - 1;
 268
 269                 config.multisample_mode_4x = job->msaa;
 270
 271                 config.maximum_bpp_of_all_render_targets = job->internal_bpp;
 272         }
 273
 274         for (int i = 0; i < nr_cbufs; i++) {
 275                 struct pipe_surface *psurf = job->cbufs[i];
 276                 if (!psurf)
 277                         continue;
 278                 struct vc5_surface *surf = vc5_surface(psurf);
 279
 280                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_RENDER_TARGET_CONFIG, rt) {
 281                         struct vc5_resource *rsc = vc5_resource(psurf->texture);
 282                         rt.address = cl_address(rsc->bo, surf->offset);
 283                         rt.internal_type = surf->internal_type;
 284                         rt.output_image_format = surf->format;
 285                         rt.memory_format = surf->tiling;
 286                         rt.internal_bpp = surf->internal_bpp;
 287                         rt.render_target_number = i;
 288
 289                         if (job->resolve & PIPE_CLEAR_COLOR0 << i)
 290                                 rsc->writes++;
 291                 }
 292
 293                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART1,
 294                         clear) {
 295                         clear.clear_color_low_32_bits = job->clear_color[i][0];
 296                         clear.clear_color_next_24_bits = job->clear_color[i][1] & 0xffffff;
 297                         clear.render_target_number = i;
 298                 };
 299
 300                 if (surf->internal_bpp >= INTERNAL_BPP_64) {
 301                         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART2,
 302                                 clear) {
 303                                 clear.clear_color_mid_low_32_bits =
 304                                         ((job->clear_color[i][1] >> 24) |
 305                                          (job->clear_color[i][2] << 8));
 306                                 clear.clear_color_mid_high_24_bits =
 307                                         ((job->clear_color[i][2] >> 24) |
 308                                          ((job->clear_color[i][3] & 0xffff) << 8));
 309                                 clear.render_target_number = i;
 310                         };
 311                 }
 312
 313                 if (surf->internal_bpp >= INTERNAL_BPP_128) {
 314                         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_CLEAR_COLORS_PART3,
 315                                 clear) {
 316                                 clear.clear_color_high_16_bits = job->clear_color[i][3] >> 16;
 317                                 clear.render_target_number = i;
 318                         };
 319                 }
 320         }
 321
 322         /* TODO: Don't bother emitting if we don't load/clear Z/S. */
 323         if (job->zsbuf) {
 324                 struct pipe_surface *psurf = job->zsbuf;
 325                 struct vc5_surface *surf = vc5_surface(psurf);
 326                 struct vc5_resource *rsc = vc5_resource(psurf->texture);
 327
 328                 cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CONFIG, zs) {
 329                         zs.address = cl_address(rsc->bo, surf->offset);
 330
 331                         zs.internal_type = surf->internal_type;
 332                         zs.output_image_format = surf->format;
 333
 334                         struct vc5_resource_slice *slice = &rsc->slices[psurf->u.tex.level];
 335                         /* XXX */
 336                         zs.padded_height_of_output_image_in_uif_blocks =
 337                                 (slice->size / slice->stride) / (2 * vc5_utile_height(rsc->cpp));
 338
 339                         assert(surf->tiling != VC5_TILING_RASTER);
 340                         zs.memory_format = surf->tiling;
 341                 }
 342
 343                 if (job->resolve & PIPE_CLEAR_DEPTHSTENCIL)
 344                         rsc->writes++;
 345         }
 346
 347         /* Ends rendering mode config. */
 348         cl_emit(&job->rcl, TILE_RENDERING_MODE_CONFIGURATION_Z_STENCIL_CLEAR_VALUES,
 349                 clear) {
 350                 clear.z_clear_value = job->clear_z;
 351                 clear.stencil_vg_mask_clear_value = job->clear_s;
 352         };
 353
 354         /* Always set initial block size before the first branch, which needs
 355          * to match the value from binning mode config.
 356          */
 357         cl_emit(&job->rcl, TILE_LIST_INITIAL_BLOCK_SIZE, init) {
 358                 init.use_auto_chained_tile_lists = true;
 359                 init.size_of_first_block_in_chained_tile_lists =
 360                         TILE_ALLOCATION_BLOCK_SIZE_64B;
 361         }
 362
 363         uint32_t supertile_w = 1, supertile_h = 1;
 364
 365         /* If doing multicore binning, we would need to initialize each core's
 366          * tile list here.
 367          */
 368         cl_emit(&job->rcl, MULTICORE_RENDERING_TILE_LIST_SET_BASE, list) {
 369                 list.address = cl_address(job->tile_alloc, 0);
 370         }
 371
 372         cl_emit(&job->rcl, MULTICORE_RENDERING_SUPERTILE_CONFIGURATION, config) {
 373                 uint32_t frame_w_in_supertiles, frame_h_in_supertiles;
 374                 const uint32_t max_supertiles = 256;
 375
 376                 /* Size up our supertiles until we get under the limit. */
 377                 for (;;) {
 378                         frame_w_in_supertiles = div_round_up(job->draw_tiles_x,
 379                                                              supertile_w);
 380                         frame_h_in_supertiles = div_round_up(job->draw_tiles_y,
 381                                                              supertile_h);
 382                         if (frame_w_in_supertiles * frame_h_in_supertiles <
 383                             max_supertiles) {
 384                                 break;
 385                         }
 386
 387                         if (supertile_w < supertile_h)
 388                                 supertile_w++;
 389                         else
 390                                 supertile_h++;
 391                 }
 392
 393                 config.total_frame_width_in_tiles = job->draw_tiles_x;
 394                 config.total_frame_height_in_tiles = job->draw_tiles_y;
 395
 396                 config.supertile_width_in_tiles_minus_1 = supertile_w - 1;
 397                 config.supertile_height_in_tiles_minus_1 = supertile_h - 1;
 398
 399                 config.total_frame_width_in_supertiles = frame_w_in_supertiles;
 400                 config.total_frame_height_in_supertiles = frame_h_in_supertiles;
 401         }
 402
 403         /* Start by clearing the tile buffer. */
 404         cl_emit(&job->rcl, TILE_COORDINATES, coords) {
 405                 coords.tile_column_number = 0;
 406                 coords.tile_row_number = 0;
 407         }
 408
 409         cl_emit(&job->rcl, STORE_TILE_BUFFER_GENERAL, store) {
 410                 store.buffer_to_store = NONE;
 411         }
 412
 413         cl_emit(&job->rcl, FLUSH_VCD_CACHE, flush);
 414
 415         vc5_rcl_emit_generic_per_tile_list(job, nr_cbufs - 1);
 416
 417         cl_emit(&job->rcl, WAIT_ON_SEMAPHORE, sem);
 418
 419         /* XXX: Use Morton order */
 420         uint32_t supertile_w_in_pixels = job->tile_width * supertile_w;
 421         uint32_t supertile_h_in_pixels = job->tile_height * supertile_h;
 422         uint32_t min_x_supertile = job->draw_min_x / supertile_w_in_pixels;
 423         uint32_t min_y_supertile = job->draw_min_y / supertile_h_in_pixels;
 424         uint32_t max_x_supertile = (job->draw_max_x - 1) / supertile_w_in_pixels;
 425         uint32_t max_y_supertile = (job->draw_max_y - 1) / supertile_h_in_pixels;
 426
 427         for (int y = min_y_supertile; y <= max_y_supertile; y++) {
 428                 for (int x = min_x_supertile; x <= max_x_supertile; x++) {
 429                         cl_emit(&job->rcl, SUPERTILE_COORDINATES, coords) {
 430                                 coords.column_number_in_supertiles = x;
 431                                 coords.row_number_in_supertiles = y;
 432                         }
 433                 }
 434         }
 435
 436         cl_emit(&job->rcl, END_OF_RENDERING, end);
 437 }