src/gallium/drivers/radeonsi/cik_sdma.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  * Copyright 2015 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * on the rights to use, copy, modify, merge, publish, distribute, sub
  10  * license, and/or sell copies of the Software, and to permit persons to whom
  11  * the Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  20  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  21  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  22  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  23  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  */
  25
  26 #include "sid.h"
  27 #include "si_pipe.h"
  28
  29 static void cik_sdma_copy_buffer(struct si_context *ctx,
  30                                  struct pipe_resource *dst,
  31                                  struct pipe_resource *src,
  32                                  uint64_t dst_offset,
  33                                  uint64_t src_offset,
  34                                  uint64_t size)
  35 {
  36         struct radeon_cmdbuf *cs = ctx->dma_cs;
  37         unsigned i, ncopy, csize;
  38         struct si_resource *sdst = si_resource(dst);
  39         struct si_resource *ssrc = si_resource(src);
  40
  41         /* Mark the buffer range of destination as valid (initialized),
  42          * so that transfer_map knows it should wait for the GPU when mapping
  43          * that range. */
  44         util_range_add(&sdst->valid_buffer_range, dst_offset,
  45                        dst_offset + size);
  46
  47         dst_offset += sdst->gpu_address;
  48         src_offset += ssrc->gpu_address;
  49
  50         ncopy = DIV_ROUND_UP(size, CIK_SDMA_COPY_MAX_SIZE);
  51         si_need_dma_space(ctx, ncopy * 7, sdst, ssrc);
  52
  53         for (i = 0; i < ncopy; i++) {
  54                 csize = MIN2(size, CIK_SDMA_COPY_MAX_SIZE);
  55                 radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY,
  56                                                 CIK_SDMA_COPY_SUB_OPCODE_LINEAR,
  57                                                 0));
  58                 radeon_emit(cs, ctx->chip_class >= GFX9 ? csize - 1 : csize);
  59                 radeon_emit(cs, 0); /* src/dst endian swap */
  60                 radeon_emit(cs, src_offset);
  61                 radeon_emit(cs, src_offset >> 32);
  62                 radeon_emit(cs, dst_offset);
  63                 radeon_emit(cs, dst_offset >> 32);
  64                 dst_offset += csize;
  65                 src_offset += csize;
  66                 size -= csize;
  67         }
  68 }
  69
  70 static unsigned minify_as_blocks(unsigned width, unsigned level, unsigned blk_w)
  71 {
  72         width = u_minify(width, level);
  73         return DIV_ROUND_UP(width, blk_w);
  74 }
  75
  76 static unsigned encode_tile_info(struct si_context *sctx,
  77                                  struct si_texture *tex, unsigned level,
  78                                  bool set_bpp)
  79 {
  80         struct radeon_info *info = &sctx->screen->info;
  81         unsigned tile_index = tex->surface.u.legacy.tiling_index[level];
  82         unsigned macro_tile_index = tex->surface.u.legacy.macro_tile_index;
  83         unsigned tile_mode = info->si_tile_mode_array[tile_index];
  84         unsigned macro_tile_mode = info->cik_macrotile_mode_array[macro_tile_index];
  85
  86         return (set_bpp ? util_logbase2(tex->surface.bpe) : 0) |
  87                 (G_009910_ARRAY_MODE(tile_mode) << 3) |
  88                 (G_009910_MICRO_TILE_MODE_NEW(tile_mode) << 8) |
  89                 /* Non-depth modes don't have TILE_SPLIT set. */
  90                 ((util_logbase2(tex->surface.u.legacy.tile_split >> 6)) << 11) |
  91                 (G_009990_BANK_WIDTH(macro_tile_mode) << 15) |
  92                 (G_009990_BANK_HEIGHT(macro_tile_mode) << 18) |
  93                 (G_009990_NUM_BANKS(macro_tile_mode) << 21) |
  94                 (G_009990_MACRO_TILE_ASPECT(macro_tile_mode) << 24) |
  95                 (G_009910_PIPE_CONFIG(tile_mode) << 26);
  96 }
  97
  98 static bool cik_sdma_copy_texture(struct si_context *sctx,
  99                                   struct pipe_resource *dst,
 100                                   unsigned dst_level,
 101                                   unsigned dstx, unsigned dsty, unsigned dstz,
 102                                   struct pipe_resource *src,
 103                                   unsigned src_level,
 104                                   const struct pipe_box *src_box)
 105 {
 106         struct radeon_info *info = &sctx->screen->info;
 107         struct si_texture *ssrc = (struct si_texture*)src;
 108         struct si_texture *sdst = (struct si_texture*)dst;
 109         unsigned bpp = sdst->surface.bpe;
 110         uint64_t dst_address = sdst->buffer.gpu_address +
 111                                sdst->surface.u.legacy.level[dst_level].offset;
 112         uint64_t src_address = ssrc->buffer.gpu_address +
 113                                ssrc->surface.u.legacy.level[src_level].offset;
 114         unsigned dst_mode = sdst->surface.u.legacy.level[dst_level].mode;
 115         unsigned src_mode = ssrc->surface.u.legacy.level[src_level].mode;
 116         unsigned dst_tile_index = sdst->surface.u.legacy.tiling_index[dst_level];
 117         unsigned src_tile_index = ssrc->surface.u.legacy.tiling_index[src_level];
 118         unsigned dst_tile_mode = info->si_tile_mode_array[dst_tile_index];
 119         unsigned src_tile_mode = info->si_tile_mode_array[src_tile_index];
 120         unsigned dst_micro_mode = G_009910_MICRO_TILE_MODE_NEW(dst_tile_mode);
 121         unsigned src_micro_mode = G_009910_MICRO_TILE_MODE_NEW(src_tile_mode);
 122         unsigned dst_tile_swizzle = dst_mode == RADEON_SURF_MODE_2D ?
 123                                             sdst->surface.tile_swizzle : 0;
 124         unsigned src_tile_swizzle = src_mode == RADEON_SURF_MODE_2D ?
 125                                             ssrc->surface.tile_swizzle : 0;
 126         unsigned dst_pitch = sdst->surface.u.legacy.level[dst_level].nblk_x;
 127         unsigned src_pitch = ssrc->surface.u.legacy.level[src_level].nblk_x;
 128         uint64_t dst_slice_pitch = ((uint64_t)sdst->surface.u.legacy.level[dst_level].slice_size_dw * 4) / bpp;
 129         uint64_t src_slice_pitch = ((uint64_t)ssrc->surface.u.legacy.level[src_level].slice_size_dw * 4) / bpp;
 130         unsigned dst_width = minify_as_blocks(sdst->buffer.b.b.width0,
 131                                               dst_level, sdst->surface.blk_w);
 132         unsigned src_width = minify_as_blocks(ssrc->buffer.b.b.width0,
 133                                               src_level, ssrc->surface.blk_w);
 134         unsigned dst_height = minify_as_blocks(sdst->buffer.b.b.height0,
 135                                                dst_level, sdst->surface.blk_h);
 136         unsigned src_height = minify_as_blocks(ssrc->buffer.b.b.height0,
 137                                                src_level, ssrc->surface.blk_h);
 138         unsigned srcx = src_box->x / ssrc->surface.blk_w;
 139         unsigned srcy = src_box->y / ssrc->surface.blk_h;
 140         unsigned srcz = src_box->z;
 141         unsigned copy_width = DIV_ROUND_UP(src_box->width, ssrc->surface.blk_w);
 142         unsigned copy_height = DIV_ROUND_UP(src_box->height, ssrc->surface.blk_h);
 143         unsigned copy_depth = src_box->depth;
 144
 145         assert(src_level <= src->last_level);
 146         assert(dst_level <= dst->last_level);
 147         assert(sdst->surface.u.legacy.level[dst_level].offset +
 148                dst_slice_pitch * bpp * (dstz + src_box->depth) <=
 149                sdst->buffer.buf->size);
 150         assert(ssrc->surface.u.legacy.level[src_level].offset +
 151                src_slice_pitch * bpp * (srcz + src_box->depth) <=
 152                ssrc->buffer.buf->size);
 153
 154         if (!si_prepare_for_dma_blit(sctx, sdst, dst_level, dstx, dsty,
 155                                      dstz, ssrc, src_level, src_box))
 156                 return false;
 157
 158         dstx /= sdst->surface.blk_w;
 159         dsty /= sdst->surface.blk_h;
 160
 161         if (srcx >= (1 << 14) ||
 162             srcy >= (1 << 14) ||
 163             srcz >= (1 << 11) ||
 164             dstx >= (1 << 14) ||
 165             dsty >= (1 << 14) ||
 166             dstz >= (1 << 11))
 167                 return false;
 168
 169         dst_address |= dst_tile_swizzle << 8;
 170         src_address |= src_tile_swizzle << 8;
 171
 172         /* Linear -> linear sub-window copy. */
 173         if (dst_mode == RADEON_SURF_MODE_LINEAR_ALIGNED &&
 174             src_mode == RADEON_SURF_MODE_LINEAR_ALIGNED &&
 175             /* check if everything fits into the bitfields */
 176             src_pitch <= (1 << 14) &&
 177             dst_pitch <= (1 << 14) &&
 178             src_slice_pitch <= (1 << 28) &&
 179             dst_slice_pitch <= (1 << 28) &&
 180             copy_width <= (1 << 14) &&
 181             copy_height <= (1 << 14) &&
 182             copy_depth <= (1 << 11) &&
 183             /* HW limitation - GFX7: */
 184             (sctx->chip_class != GFX7 ||
 185              (copy_width < (1 << 14) &&
 186               copy_height < (1 << 14) &&
 187               copy_depth < (1 << 11))) &&
 188             /* HW limitation - some GFX7 parts: */
 189             ((sctx->family != CHIP_BONAIRE &&
 190               sctx->family != CHIP_KAVERI) ||
 191              (srcx + copy_width != (1 << 14) &&
 192               srcy + copy_height != (1 << 14)))) {
 193                 struct radeon_cmdbuf *cs = sctx->dma_cs;
 194
 195                 si_need_dma_space(sctx, 13, &sdst->buffer, &ssrc->buffer);
 196
 197                 radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY,
 198                                                 CIK_SDMA_COPY_SUB_OPCODE_LINEAR_SUB_WINDOW, 0) |
 199                             (util_logbase2(bpp) << 29));
 200                 radeon_emit(cs, src_address);
 201                 radeon_emit(cs, src_address >> 32);
 202                 radeon_emit(cs, srcx | (srcy << 16));
 203                 radeon_emit(cs, srcz | ((src_pitch - 1) << 16));
 204                 radeon_emit(cs, src_slice_pitch - 1);
 205                 radeon_emit(cs, dst_address);
 206                 radeon_emit(cs, dst_address >> 32);
 207                 radeon_emit(cs, dstx | (dsty << 16));
 208                 radeon_emit(cs, dstz | ((dst_pitch - 1) << 16));
 209                 radeon_emit(cs, dst_slice_pitch - 1);
 210                 if (sctx->chip_class == GFX7) {
 211                         radeon_emit(cs, copy_width | (copy_height << 16));
 212                         radeon_emit(cs, copy_depth);
 213                 } else {
 214                         radeon_emit(cs, (copy_width - 1) | ((copy_height - 1) << 16));
 215                         radeon_emit(cs, (copy_depth - 1));
 216                 }
 217                 return true;
 218         }
 219
 220         /* Tiled <-> linear sub-window copy. */
 221         if ((src_mode >= RADEON_SURF_MODE_1D) != (dst_mode >= RADEON_SURF_MODE_1D)) {
 222                 struct si_texture *tiled = src_mode >= RADEON_SURF_MODE_1D ? ssrc : sdst;
 223                 struct si_texture *linear = tiled == ssrc ? sdst : ssrc;
 224                 unsigned tiled_level =  tiled   == ssrc ? src_level : dst_level;
 225                 unsigned linear_level = linear  == ssrc ? src_level : dst_level;
 226                 unsigned tiled_x =      tiled   == ssrc ? srcx : dstx;
 227                 unsigned linear_x =     linear  == ssrc ? srcx : dstx;
 228                 unsigned tiled_y =      tiled   == ssrc ? srcy : dsty;
 229                 unsigned linear_y =     linear  == ssrc ? srcy : dsty;
 230                 unsigned tiled_z =      tiled   == ssrc ? srcz : dstz;
 231                 unsigned linear_z =     linear  == ssrc ? srcz : dstz;
 232                 unsigned tiled_width =  tiled   == ssrc ? src_width : dst_width;
 233                 unsigned linear_width = linear  == ssrc ? src_width : dst_width;
 234                 unsigned tiled_pitch =  tiled   == ssrc ? src_pitch : dst_pitch;
 235                 unsigned linear_pitch = linear  == ssrc ? src_pitch : dst_pitch;
 236                 unsigned tiled_slice_pitch  = tiled  == ssrc ? src_slice_pitch : dst_slice_pitch;
 237                 unsigned linear_slice_pitch = linear == ssrc ? src_slice_pitch : dst_slice_pitch;
 238                 uint64_t tiled_address =  tiled  == ssrc ? src_address : dst_address;
 239                 uint64_t linear_address = linear == ssrc ? src_address : dst_address;
 240                 unsigned tiled_micro_mode = tiled == ssrc ? src_micro_mode : dst_micro_mode;
 241
 242                 assert(tiled_pitch % 8 == 0);
 243                 assert(tiled_slice_pitch % 64 == 0);
 244                 unsigned pitch_tile_max = tiled_pitch / 8 - 1;
 245                 unsigned slice_tile_max = tiled_slice_pitch / 64 - 1;
 246                 unsigned xalign = MAX2(1, 4 / bpp);
 247                 unsigned copy_width_aligned = copy_width;
 248
 249                 /* If the region ends at the last pixel and is unaligned, we
 250                  * can copy the remainder of the line that is not visible to
 251                  * make it aligned.
 252                  */
 253                 if (copy_width % xalign != 0 &&
 254                     linear_x + copy_width == linear_width &&
 255                     tiled_x  + copy_width == tiled_width &&
 256                     linear_x + align(copy_width, xalign) <= linear_pitch &&
 257                     tiled_x  + align(copy_width, xalign) <= tiled_pitch)
 258                         copy_width_aligned = align(copy_width, xalign);
 259
 260                 /* HW limitations. */
 261                 if ((sctx->family == CHIP_BONAIRE ||
 262                      sctx->family == CHIP_KAVERI) &&
 263                     linear_pitch - 1 == 0x3fff &&
 264                     bpp == 16)
 265                         return false;
 266
 267                 if (sctx->chip_class == GFX7 &&
 268                     (copy_width_aligned == (1 << 14) ||
 269                      copy_height == (1 << 14) ||
 270                      copy_depth == (1 << 11)))
 271                         return false;
 272
 273                 if ((sctx->family == CHIP_BONAIRE ||
 274                      sctx->family == CHIP_KAVERI ||
 275                      sctx->family == CHIP_KABINI) &&
 276                     (tiled_x + copy_width == (1 << 14) ||
 277                      tiled_y + copy_height == (1 << 14)))
 278                         return false;
 279
 280                 /* The hw can read outside of the given linear buffer bounds,
 281                  * or access those pages but not touch the memory in case
 282                  * of writes. (it still causes a VM fault)
 283                  *
 284                  * Out-of-bounds memory access or page directory access must
 285                  * be prevented.
 286                  */
 287                 int64_t start_linear_address, end_linear_address;
 288                 unsigned granularity;
 289
 290                 /* Deduce the size of reads from the linear surface. */
 291                 switch (tiled_micro_mode) {
 292                 case V_009910_ADDR_SURF_DISPLAY_MICRO_TILING:
 293                         granularity = bpp == 1 ? 64 / (8*bpp) :
 294                                                  128 / (8*bpp);
 295                         break;
 296                 case V_009910_ADDR_SURF_THIN_MICRO_TILING:
 297                 case V_009910_ADDR_SURF_DEPTH_MICRO_TILING:
 298                         if (0 /* TODO: THICK microtiling */)
 299                                 granularity = bpp == 1 ? 32 / (8*bpp) :
 300                                               bpp == 2 ? 64 / (8*bpp) :
 301                                               bpp <= 8 ? 128 / (8*bpp) :
 302                                                          256 / (8*bpp);
 303                         else
 304                                 granularity = bpp <= 2 ? 64 / (8*bpp) :
 305                                               bpp <= 8 ? 128 / (8*bpp) :
 306                                                          256 / (8*bpp);
 307                         break;
 308                 default:
 309                         return false;
 310                 }
 311
 312                 /* The linear reads start at tiled_x & ~(granularity - 1).
 313                  * If linear_x == 0 && tiled_x % granularity != 0, the hw
 314                  * starts reading from an address preceding linear_address!!!
 315                  */
 316                 start_linear_address =
 317                         linear->surface.u.legacy.level[linear_level].offset +
 318                         bpp * (linear_z * linear_slice_pitch +
 319                                linear_y * linear_pitch +
 320                                linear_x);
 321                 start_linear_address -= (int)(bpp * (tiled_x % granularity));
 322
 323                 end_linear_address =
 324                         linear->surface.u.legacy.level[linear_level].offset +
 325                         bpp * ((linear_z + copy_depth - 1) * linear_slice_pitch +
 326                                (linear_y + copy_height - 1) * linear_pitch +
 327                                (linear_x + copy_width));
 328
 329                 if ((tiled_x + copy_width) % granularity)
 330                         end_linear_address += granularity -
 331                                               (tiled_x + copy_width) % granularity;
 332
 333                 if (start_linear_address < 0 ||
 334                     end_linear_address > linear->surface.surf_size)
 335                         return false;
 336
 337                 /* Check requirements. */
 338                 if (tiled_address % 256 == 0 &&
 339                     linear_address % 4 == 0 &&
 340                     linear_pitch % xalign == 0 &&
 341                     linear_x % xalign == 0 &&
 342                     tiled_x % xalign == 0 &&
 343                     copy_width_aligned % xalign == 0 &&
 344                     tiled_micro_mode != V_009910_ADDR_SURF_ROTATED_MICRO_TILING &&
 345                     /* check if everything fits into the bitfields */
 346                     tiled->surface.u.legacy.tile_split <= 4096 &&
 347                     pitch_tile_max < (1 << 11) &&
 348                     slice_tile_max < (1 << 22) &&
 349                     linear_pitch <= (1 << 14) &&
 350                     linear_slice_pitch <= (1 << 28) &&
 351                     copy_width_aligned <= (1 << 14) &&
 352                     copy_height <= (1 << 14) &&
 353                     copy_depth <= (1 << 11)) {
 354                         struct radeon_cmdbuf *cs = sctx->dma_cs;
 355                         uint32_t direction = linear == sdst ? 1u << 31 : 0;
 356
 357                         si_need_dma_space(sctx, 14, &sdst->buffer, &ssrc->buffer);
 358
 359                         radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY,
 360                                                         CIK_SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW, 0) |
 361                                         direction);
 362                         radeon_emit(cs, tiled_address);
 363                         radeon_emit(cs, tiled_address >> 32);
 364                         radeon_emit(cs, tiled_x | (tiled_y << 16));
 365                         radeon_emit(cs, tiled_z | (pitch_tile_max << 16));
 366                         radeon_emit(cs, slice_tile_max);
 367                         radeon_emit(cs, encode_tile_info(sctx, tiled, tiled_level, true));
 368                         radeon_emit(cs, linear_address);
 369                         radeon_emit(cs, linear_address >> 32);
 370                         radeon_emit(cs, linear_x | (linear_y << 16));
 371                         radeon_emit(cs, linear_z | ((linear_pitch - 1) << 16));
 372                         radeon_emit(cs, linear_slice_pitch - 1);
 373                         if (sctx->chip_class == GFX7) {
 374                                 radeon_emit(cs, copy_width_aligned | (copy_height << 16));
 375                                 radeon_emit(cs, copy_depth);
 376                         } else {
 377                                 radeon_emit(cs, (copy_width_aligned - 1) | ((copy_height - 1) << 16));
 378                                 radeon_emit(cs, (copy_depth - 1));
 379                         }
 380                         return true;
 381                 }
 382         }
 383
 384         /* Tiled -> Tiled sub-window copy. */
 385         if (dst_mode >= RADEON_SURF_MODE_1D &&
 386             src_mode >= RADEON_SURF_MODE_1D &&
 387             /* check if these fit into the bitfields */
 388             src_address % 256 == 0 &&
 389             dst_address % 256 == 0 &&
 390             ssrc->surface.u.legacy.tile_split <= 4096 &&
 391             sdst->surface.u.legacy.tile_split <= 4096 &&
 392             dstx % 8 == 0 &&
 393             dsty % 8 == 0 &&
 394             srcx % 8 == 0 &&
 395             srcy % 8 == 0 &&
 396             /* this can either be equal, or display->rotated (GFX8+ only) */
 397             (src_micro_mode == dst_micro_mode ||
 398              (sctx->chip_class >= GFX8 &&
 399               src_micro_mode == V_009910_ADDR_SURF_DISPLAY_MICRO_TILING &&
 400               dst_micro_mode == V_009910_ADDR_SURF_ROTATED_MICRO_TILING))) {
 401                 assert(src_pitch % 8 == 0);
 402                 assert(dst_pitch % 8 == 0);
 403                 assert(src_slice_pitch % 64 == 0);
 404                 assert(dst_slice_pitch % 64 == 0);
 405                 unsigned src_pitch_tile_max = src_pitch / 8 - 1;
 406                 unsigned dst_pitch_tile_max = dst_pitch / 8 - 1;
 407                 unsigned src_slice_tile_max = src_slice_pitch / 64 - 1;
 408                 unsigned dst_slice_tile_max = dst_slice_pitch / 64 - 1;
 409                 unsigned copy_width_aligned = copy_width;
 410                 unsigned copy_height_aligned = copy_height;
 411
 412                 /* If the region ends at the last pixel and is unaligned, we
 413                  * can copy the remainder of the tile that is not visible to
 414                  * make it aligned.
 415                  */
 416                 if (copy_width % 8 != 0 &&
 417                     srcx + copy_width == src_width &&
 418                     dstx + copy_width == dst_width)
 419                         copy_width_aligned = align(copy_width, 8);
 420
 421                 if (copy_height % 8 != 0 &&
 422                     srcy + copy_height == src_height &&
 423                     dsty + copy_height == dst_height)
 424                         copy_height_aligned = align(copy_height, 8);
 425
 426                 /* check if these fit into the bitfields */
 427                 if (src_pitch_tile_max < (1 << 11) &&
 428                     dst_pitch_tile_max < (1 << 11) &&
 429                     src_slice_tile_max < (1 << 22) &&
 430                     dst_slice_tile_max < (1 << 22) &&
 431                     copy_width_aligned <= (1 << 14) &&
 432                     copy_height_aligned <= (1 << 14) &&
 433                     copy_depth <= (1 << 11) &&
 434                     copy_width_aligned % 8 == 0 &&
 435                     copy_height_aligned % 8 == 0 &&
 436                     /* HW limitation - GFX7: */
 437                     (sctx->chip_class != GFX7 ||
 438                      (copy_width_aligned < (1 << 14) &&
 439                       copy_height_aligned < (1 << 14) &&
 440                       copy_depth < (1 << 11))) &&
 441                     /* HW limitation - some GFX7 parts: */
 442                     ((sctx->family != CHIP_BONAIRE &&
 443                       sctx->family != CHIP_KAVERI &&
 444                       sctx->family != CHIP_KABINI) ||
 445                      (srcx + copy_width_aligned != (1 << 14) &&
 446                       srcy + copy_height_aligned != (1 << 14) &&
 447                       dstx + copy_width != (1 << 14)))) {
 448                         struct radeon_cmdbuf *cs = sctx->dma_cs;
 449
 450                         si_need_dma_space(sctx, 15, &sdst->buffer, &ssrc->buffer);
 451
 452                         radeon_emit(cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY,
 453                                                         CIK_SDMA_COPY_SUB_OPCODE_T2T_SUB_WINDOW, 0));
 454                         radeon_emit(cs, src_address);
 455                         radeon_emit(cs, src_address >> 32);
 456                         radeon_emit(cs, srcx | (srcy << 16));
 457                         radeon_emit(cs, srcz | (src_pitch_tile_max << 16));
 458                         radeon_emit(cs, src_slice_tile_max);
 459                         radeon_emit(cs, encode_tile_info(sctx, ssrc, src_level, true));
 460                         radeon_emit(cs, dst_address);
 461                         radeon_emit(cs, dst_address >> 32);
 462                         radeon_emit(cs, dstx | (dsty << 16));
 463                         radeon_emit(cs, dstz | (dst_pitch_tile_max << 16));
 464                         radeon_emit(cs, dst_slice_tile_max);
 465                         radeon_emit(cs, encode_tile_info(sctx, sdst, dst_level, false));
 466                         if (sctx->chip_class == GFX7) {
 467                                 radeon_emit(cs, copy_width_aligned |
 468                                                 (copy_height_aligned << 16));
 469                                 radeon_emit(cs, copy_depth);
 470                         } else {
 471                                 radeon_emit(cs, (copy_width_aligned - 8) |
 472                                                 ((copy_height_aligned - 8) << 16));
 473                                 radeon_emit(cs, (copy_depth - 1));
 474                         }
 475                         return true;
 476                 }
 477         }
 478
 479         return false;
 480 }
 481
 482 static void cik_sdma_copy(struct pipe_context *ctx,
 483                           struct pipe_resource *dst,
 484                           unsigned dst_level,
 485                           unsigned dstx, unsigned dsty, unsigned dstz,
 486                           struct pipe_resource *src,
 487                           unsigned src_level,
 488                           const struct pipe_box *src_box)
 489 {
 490         struct si_context *sctx = (struct si_context *)ctx;
 491
 492         if (!sctx->dma_cs ||
 493             src->flags & PIPE_RESOURCE_FLAG_SPARSE ||
 494             dst->flags & PIPE_RESOURCE_FLAG_SPARSE)
 495                 goto fallback;
 496
 497         /* If src is a buffer and dst is a texture, we are uploading metadata. */
 498         if (src->target == PIPE_BUFFER) {
 499                 cik_sdma_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
 500                 return;
 501         }
 502
 503         /* SDMA causes corruption. See:
 504          *   https://bugs.freedesktop.org/show_bug.cgi?id=110575
 505          *   https://bugs.freedesktop.org/show_bug.cgi?id=110635
 506          *
 507          * Keep SDMA enabled on APUs.
 508          */
 509         if ((sctx->screen->debug_flags & DBG(FORCE_DMA) ||
 510              !sctx->screen->info.has_dedicated_vram) &&
 511             (sctx->chip_class == GFX7 || sctx->chip_class == GFX8) &&
 512             cik_sdma_copy_texture(sctx, dst, dst_level, dstx, dsty, dstz,
 513                                   src, src_level, src_box))
 514                 return;
 515
 516 fallback:
 517         si_resource_copy_region(ctx, dst, dst_level, dstx, dsty, dstz,
 518                                 src, src_level, src_box);
 519 }
 520
 521 void cik_init_sdma_functions(struct si_context *sctx)
 522 {
 523         sctx->dma_copy = cik_sdma_copy;
 524 }