src/gallium/drivers/vc4/kernel/vc4_render_cl.c

   1 /*
   2  * Copyright © 2014-2015 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * DOC: Render command list generation
  26  *
  27  * In the VC4 driver, render command list generation is performed by the
  28  * kernel instead of userspace.  We do this because validating a
  29  * user-submitted command list is hard to get right and has high CPU overhead,
  30  * while the number of valid configurations for render command lists is
  31  * actually fairly low.
  32  */
  33
  34 #include "vc4_drv.h"
  35 #include "vc4_packet.h"
  36
  37 struct vc4_rcl_setup {
  38         struct drm_gem_cma_object *color_read;
  39         struct drm_gem_cma_object *color_ms_write;
  40         struct drm_gem_cma_object *zs_read;
  41         struct drm_gem_cma_object *zs_write;
  42
  43         struct drm_gem_cma_object *rcl;
  44         u32 next_offset;
  45 };
  46
  47 static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
  48 {
  49         *(u8 *)(setup->rcl->vaddr + setup->next_offset) = val;
  50         setup->next_offset += 1;
  51 }
  52
  53 static inline void rcl_u16(struct vc4_rcl_setup *setup, u16 val)
  54 {
  55         *(u16 *)(setup->rcl->vaddr + setup->next_offset) = val;
  56         setup->next_offset += 2;
  57 }
  58
  59 static inline void rcl_u32(struct vc4_rcl_setup *setup, u32 val)
  60 {
  61         *(u32 *)(setup->rcl->vaddr + setup->next_offset) = val;
  62         setup->next_offset += 4;
  63 }
  64
  65
  66 /*
  67  * Emits a no-op STORE_TILE_BUFFER_GENERAL.
  68  *
  69  * If we emit a PACKET_TILE_COORDINATES, it must be followed by a store of
  70  * some sort before another load is triggered.
  71  */
  72 static void vc4_store_before_load(struct vc4_rcl_setup *setup)
  73 {
  74         rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
  75         rcl_u16(setup,
  76                 VC4_SET_FIELD(VC4_LOADSTORE_TILE_BUFFER_NONE,
  77                               VC4_LOADSTORE_TILE_BUFFER_BUFFER) |
  78                 VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR |
  79                 VC4_STORE_TILE_BUFFER_DISABLE_ZS_CLEAR |
  80                 VC4_STORE_TILE_BUFFER_DISABLE_VG_MASK_CLEAR);
  81         rcl_u32(setup, 0); /* no address, since we're in None mode */
  82 }
  83
  84 /*
  85  * Emits a PACKET_TILE_COORDINATES if one isn't already pending.
  86  *
  87  * The tile coordinates packet triggers a pending load if there is one, are
  88  * used for clipping during rendering, and determine where loads/stores happen
  89  * relative to their base address.
  90  */
  91 static void vc4_tile_coordinates(struct vc4_rcl_setup *setup,
  92                                  uint32_t x, uint32_t y)
  93 {
  94         rcl_u8(setup, VC4_PACKET_TILE_COORDINATES);
  95         rcl_u8(setup, x);
  96         rcl_u8(setup, y);
  97 }
  98
  99 static void emit_tile(struct vc4_exec_info *exec,
 100                       struct vc4_rcl_setup *setup,
 101                       uint8_t x, uint8_t y, bool first, bool last)
 102 {
 103         struct drm_vc4_submit_cl *args = exec->args;
 104         bool has_bin = args->bin_cl_size != 0;
 105
 106         /* Note that the load doesn't actually occur until the
 107          * tile coords packet is processed, and only one load
 108          * may be outstanding at a time.
 109          */
 110         if (setup->color_read) {
 111                 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
 112                 rcl_u16(setup, args->color_read.bits);
 113                 rcl_u32(setup,
 114                         setup->color_read->paddr + args->color_read.offset);
 115         }
 116
 117         if (setup->zs_read) {
 118                 if (setup->color_read) {
 119                         /* Exec previous load. */
 120                         vc4_tile_coordinates(setup, x, y);
 121                         vc4_store_before_load(setup);
 122                 }
 123
 124                 rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
 125                 rcl_u16(setup, args->zs_read.bits);
 126                 rcl_u32(setup, setup->zs_read->paddr + args->zs_read.offset);
 127         }
 128
 129         /* Clipping depends on tile coordinates having been
 130          * emitted, so we always need one here.
 131          */
 132         vc4_tile_coordinates(setup, x, y);
 133
 134         /* Wait for the binner before jumping to the first
 135          * tile's lists.
 136          */
 137         if (first && has_bin)
 138                 rcl_u8(setup, VC4_PACKET_WAIT_ON_SEMAPHORE);
 139
 140         if (has_bin) {
 141                 rcl_u8(setup, VC4_PACKET_BRANCH_TO_SUB_LIST);
 142                 rcl_u32(setup, (exec->tile_bo->paddr +
 143                                 exec->tile_alloc_offset +
 144                                 (y * exec->bin_tiles_x + x) * 32));
 145         }
 146
 147         if (setup->zs_write) {
 148                 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
 149                 rcl_u16(setup, args->zs_write.bits |
 150                         (setup->color_ms_write ?
 151                          VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR : 0));
 152                 rcl_u32(setup,
 153                         (setup->zs_write->paddr + args->zs_write.offset) |
 154                         ((last && !setup->color_ms_write) ?
 155                          VC4_LOADSTORE_TILE_BUFFER_EOF : 0));
 156         }
 157
 158         if (setup->color_ms_write) {
 159                 if (setup->zs_write) {
 160                         /* Reset after previous store */
 161                         vc4_tile_coordinates(setup, x, y);
 162                 }
 163
 164                 if (last)
 165                         rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER_AND_EOF);
 166                 else
 167                         rcl_u8(setup, VC4_PACKET_STORE_MS_TILE_BUFFER);
 168         }
 169 }
 170
 171 static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 172                              struct vc4_rcl_setup *setup)
 173 {
 174         struct drm_vc4_submit_cl *args = exec->args;
 175         bool has_bin = args->bin_cl_size != 0;
 176         uint8_t min_x_tile = args->min_x_tile;
 177         uint8_t min_y_tile = args->min_y_tile;
 178         uint8_t max_x_tile = args->max_x_tile;
 179         uint8_t max_y_tile = args->max_y_tile;
 180         uint8_t xtiles = max_x_tile - min_x_tile + 1;
 181         uint8_t ytiles = max_y_tile - min_y_tile + 1;
 182         uint8_t x, y;
 183         uint32_t size, loop_body_size;
 184
 185         size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
 186         loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
 187
 188         if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
 189                 size += VC4_PACKET_CLEAR_COLORS_SIZE +
 190                         VC4_PACKET_TILE_COORDINATES_SIZE +
 191                         VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
 192         }
 193
 194         if (setup->color_read) {
 195                 loop_body_size += (VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE);
 196         }
 197         if (setup->zs_read) {
 198                 if (setup->color_read) {
 199                         loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
 200                         loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
 201                 }
 202                 loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
 203         }
 204
 205         if (has_bin) {
 206                 size += VC4_PACKET_WAIT_ON_SEMAPHORE_SIZE;
 207                 loop_body_size += VC4_PACKET_BRANCH_TO_SUB_LIST_SIZE;
 208         }
 209
 210         if (setup->zs_write)
 211                 loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
 212         if (setup->color_ms_write) {
 213                 if (setup->zs_write)
 214                         loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
 215                 loop_body_size += VC4_PACKET_STORE_MS_TILE_BUFFER_SIZE;
 216         }
 217         size += xtiles * ytiles * loop_body_size;
 218
 219         setup->rcl = drm_gem_cma_create(dev, size);
 220         if (!setup->rcl)
 221                 return -ENOMEM;
 222         list_addtail(&to_vc4_bo(&setup->rcl->base)->unref_head,
 223                      &exec->unref_list);
 224
 225         rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
 226         rcl_u32(setup,
 227                 (setup->color_ms_write ?
 228                  (setup->color_ms_write->paddr +
 229                   args->color_ms_write.offset) :
 230                  0));
 231         rcl_u16(setup, args->width);
 232         rcl_u16(setup, args->height);
 233         rcl_u16(setup, args->color_ms_write.bits);
 234
 235         /* The tile buffer gets cleared when the previous tile is stored.  If
 236          * the clear values changed between frames, then the tile buffer has
 237          * stale clear values in it, so we have to do a store in None mode (no
 238          * writes) so that we trigger the tile buffer clear.
 239          */
 240         if (args->flags & VC4_SUBMIT_CL_USE_CLEAR_COLOR) {
 241                 rcl_u8(setup, VC4_PACKET_CLEAR_COLORS);
 242                 rcl_u32(setup, args->clear_color[0]);
 243                 rcl_u32(setup, args->clear_color[1]);
 244                 rcl_u32(setup, args->clear_z);
 245                 rcl_u8(setup, args->clear_s);
 246
 247                 vc4_tile_coordinates(setup, 0, 0);
 248
 249                 rcl_u8(setup, VC4_PACKET_STORE_TILE_BUFFER_GENERAL);
 250                 rcl_u16(setup, VC4_LOADSTORE_TILE_BUFFER_NONE);
 251                 rcl_u32(setup, 0); /* no address, since we're in None mode */
 252         }
 253
 254         for (y = min_y_tile; y <= max_y_tile; y++) {
 255                 for (x = min_x_tile; x <= max_x_tile; x++) {
 256                         bool first = (x == min_x_tile && y == min_y_tile);
 257                         bool last = (x == max_x_tile && y == max_y_tile);
 258                         emit_tile(exec, setup, x, y, first, last);
 259                 }
 260         }
 261
 262         BUG_ON(setup->next_offset != size);
 263         exec->ct1ca = setup->rcl->paddr;
 264         exec->ct1ea = setup->rcl->paddr + setup->next_offset;
 265
 266         return 0;
 267 }
 268
 269 static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 270                                  struct drm_gem_cma_object **obj,
 271                                  struct drm_vc4_submit_rcl_surface *surf)
 272 {
 273         uint8_t tiling = VC4_GET_FIELD(surf->bits,
 274                                        VC4_LOADSTORE_TILE_BUFFER_TILING);
 275         uint8_t buffer = VC4_GET_FIELD(surf->bits,
 276                                        VC4_LOADSTORE_TILE_BUFFER_BUFFER);
 277         uint8_t format = VC4_GET_FIELD(surf->bits,
 278                                        VC4_LOADSTORE_TILE_BUFFER_FORMAT);
 279         int cpp;
 280
 281         if (surf->pad != 0) {
 282                 DRM_ERROR("Padding unset\n");
 283                 return -EINVAL;
 284         }
 285
 286         if (surf->hindex == ~0)
 287                 return 0;
 288
 289         *obj = vc4_use_bo(exec, surf->hindex, VC4_MODE_RENDER);
 290         if (!*obj)
 291                 return -EINVAL;
 292
 293         if (surf->bits & ~(VC4_LOADSTORE_TILE_BUFFER_TILING_MASK |
 294                            VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK |
 295                            VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK)) {
 296                 DRM_ERROR("Unknown bits in load/store: 0x%04x\n",
 297                           surf->bits);
 298                 return -EINVAL;
 299         }
 300
 301         if (tiling > VC4_TILING_FORMAT_LT) {
 302                 DRM_ERROR("Bad tiling format\n");
 303                 return -EINVAL;
 304         }
 305
 306         if (buffer == VC4_LOADSTORE_TILE_BUFFER_ZS) {
 307                 if (format != 0) {
 308                         DRM_ERROR("No color format should be set for ZS\n");
 309                         return -EINVAL;
 310                 }
 311                 cpp = 4;
 312         } else if (buffer == VC4_LOADSTORE_TILE_BUFFER_COLOR) {
 313                 switch (format) {
 314                 case VC4_LOADSTORE_TILE_BUFFER_BGR565:
 315                 case VC4_LOADSTORE_TILE_BUFFER_BGR565_DITHER:
 316                         cpp = 2;
 317                         break;
 318                 case VC4_LOADSTORE_TILE_BUFFER_RGBA8888:
 319                         cpp = 4;
 320                         break;
 321                 default:
 322                         DRM_ERROR("Bad tile buffer format\n");
 323                         return -EINVAL;
 324                 }
 325         } else {
 326                 DRM_ERROR("Bad load/store buffer %d.\n", buffer);
 327                 return -EINVAL;
 328         }
 329
 330         if (surf->offset & 0xf) {
 331                 DRM_ERROR("load/store buffer must be 16b aligned.\n");
 332                 return -EINVAL;
 333         }
 334
 335         if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
 336                                 exec->args->width, exec->args->height, cpp)) {
 337                 return -EINVAL;
 338         }
 339
 340         return 0;
 341 }
 342
 343 static int
 344 vc4_rcl_ms_surface_setup(struct vc4_exec_info *exec,
 345                          struct drm_gem_cma_object **obj,
 346                          struct drm_vc4_submit_rcl_surface *surf)
 347 {
 348         uint8_t tiling = VC4_GET_FIELD(surf->bits,
 349                                        VC4_RENDER_CONFIG_MEMORY_FORMAT);
 350         uint8_t format = VC4_GET_FIELD(surf->bits,
 351                                        VC4_RENDER_CONFIG_FORMAT);
 352         int cpp;
 353
 354         if (surf->pad != 0) {
 355                 DRM_ERROR("Padding unset\n");
 356                 return -EINVAL;
 357         }
 358
 359         if (surf->bits & ~(VC4_RENDER_CONFIG_MEMORY_FORMAT_MASK |
 360                            VC4_RENDER_CONFIG_FORMAT_MASK)) {
 361                 DRM_ERROR("Unknown bits in render config: 0x%04x\n",
 362                           surf->bits);
 363                 return -EINVAL;
 364         }
 365
 366         if (surf->hindex == ~0)
 367                 return 0;
 368
 369         *obj = vc4_use_bo(exec, surf->hindex, VC4_MODE_RENDER);
 370         if (!*obj)
 371                 return -EINVAL;
 372
 373         if (tiling > VC4_TILING_FORMAT_LT) {
 374                 DRM_ERROR("Bad tiling format\n");
 375                 return -EINVAL;
 376         }
 377
 378         switch (format) {
 379         case VC4_RENDER_CONFIG_FORMAT_BGR565_DITHERED:
 380         case VC4_RENDER_CONFIG_FORMAT_BGR565:
 381                 cpp = 2;
 382                 break;
 383         case VC4_RENDER_CONFIG_FORMAT_RGBA8888:
 384                 cpp = 4;
 385                 break;
 386         default:
 387                 DRM_ERROR("Bad tile buffer format\n");
 388                 return -EINVAL;
 389         }
 390
 391         if (!vc4_check_tex_size(exec, *obj, surf->offset, tiling,
 392                                 exec->args->width, exec->args->height, cpp)) {
 393                 return -EINVAL;
 394         }
 395
 396         return 0;
 397 }
 398
 399 int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
 400 {
 401         struct vc4_rcl_setup setup = {0};
 402         struct drm_vc4_submit_cl *args = exec->args;
 403         bool has_bin = args->bin_cl_size != 0;
 404         int ret;
 405
 406         if (args->min_x_tile > args->max_x_tile ||
 407             args->min_y_tile > args->max_y_tile) {
 408                 DRM_ERROR("Bad render tile set (%d,%d)-(%d,%d)\n",
 409                           args->min_x_tile, args->min_y_tile,
 410                           args->max_x_tile, args->max_y_tile);
 411                 return -EINVAL;
 412         }
 413
 414         if (has_bin &&
 415             (args->max_x_tile > exec->bin_tiles_x ||
 416              args->max_y_tile > exec->bin_tiles_y)) {
 417                 DRM_ERROR("Render tiles (%d,%d) outside of bin config (%d,%d)\n",
 418                           args->max_x_tile, args->max_y_tile,
 419                           exec->bin_tiles_x, exec->bin_tiles_y);
 420                 return -EINVAL;
 421         }
 422
 423         ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read);
 424         if (ret)
 425                 return ret;
 426
 427         ret = vc4_rcl_ms_surface_setup(exec, &setup.color_ms_write,
 428                                        &args->color_ms_write);
 429         if (ret)
 430                 return ret;
 431
 432         ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read);
 433         if (ret)
 434                 return ret;
 435
 436         ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write);
 437         if (ret)
 438                 return ret;
 439
 440         /* We shouldn't even have the job submitted to us if there's no
 441          * surface to write out.
 442          */
 443         if (!setup.color_ms_write && !setup.zs_write) {
 444                 DRM_ERROR("RCL requires color or Z/S write\n");
 445                 return -EINVAL;
 446         }
 447
 448         return vc4_create_rcl_bo(dev, exec, &setup);
 449 }