X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_dma.c;h=450ed82b4d6753392d8c02b32783503125a317b3;hb=f1f3640f6fe841942a7de6a74f7a5c9996077e6b;hp=ff647228f50dd803799d6c0cac8fb4cd600da47a;hpb=d17b85524dfd74824a2135d5d4112a1fae86ed17;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/si_dma.c b/src/gallium/drivers/radeonsi/si_dma.c index ff647228f50..450ed82b4d6 100644 --- a/src/gallium/drivers/radeonsi/si_dma.c +++ b/src/gallium/drivers/radeonsi/si_dma.c @@ -1,5 +1,7 @@ /* * Copyright 2010 Jerome Glisse + * Copyright 2018 Advanced Micro Devices, Inc. + * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -19,44 +21,13 @@ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: - * Jerome Glisse */ #include "sid.h" #include "si_pipe.h" -#include "radeon/r600_cs.h" #include "util/u_format.h" -static unsigned si_array_mode(unsigned mode) -{ - switch (mode) { - case RADEON_SURF_MODE_LINEAR_ALIGNED: - return V_009910_ARRAY_LINEAR_ALIGNED; - case RADEON_SURF_MODE_1D: - return V_009910_ARRAY_1D_TILED_THIN1; - case RADEON_SURF_MODE_2D: - return V_009910_ARRAY_2D_TILED_THIN1; - default: - case RADEON_SURF_MODE_LINEAR: - return V_009910_ARRAY_LINEAR_GENERAL; - } -} - -static uint32_t si_micro_tile_mode(struct si_screen *sscreen, unsigned tile_mode) -{ - if (sscreen->b.info.si_tile_mode_array_valid) { - uint32_t gb_tile_mode = sscreen->b.info.si_tile_mode_array[tile_mode]; - - return G_009910_MICRO_TILE_MODE(gb_tile_mode); - } - - /* The kernel cannod return the tile mode array. Guess? */ - return V_009910_ADDR_SURF_THIN_MICRO_TILING; -} - static void si_dma_copy_buffer(struct si_context *ctx, struct pipe_resource *dst, struct pipe_resource *src, @@ -64,50 +35,45 @@ static void si_dma_copy_buffer(struct si_context *ctx, uint64_t src_offset, uint64_t size) { - struct radeon_winsys_cs *cs = ctx->b.rings.dma.cs; - unsigned i, ncopy, csize, max_csize, sub_cmd, shift; - struct r600_resource *rdst = (struct r600_resource*)dst; - struct r600_resource *rsrc = (struct r600_resource*)src; + struct radeon_cmdbuf *cs = ctx->dma_cs; + unsigned i, ncopy, count, max_size, sub_cmd, shift; + struct si_resource *sdst = si_resource(dst); + struct si_resource *ssrc = si_resource(src); /* Mark the buffer range of destination as valid (initialized), * so that transfer_map knows it should wait for the GPU when mapping * that range. */ - util_range_add(&rdst->valid_buffer_range, dst_offset, + util_range_add(&sdst->valid_buffer_range, dst_offset, dst_offset + size); - dst_offset += rdst->gpu_address; - src_offset += rsrc->gpu_address; + dst_offset += sdst->gpu_address; + src_offset += ssrc->gpu_address; - /* see if we use dword or byte copy */ + /* see whether we should use the dword-aligned or byte-aligned copy */ if (!(dst_offset % 4) && !(src_offset % 4) && !(size % 4)) { - size >>= 2; sub_cmd = SI_DMA_COPY_DWORD_ALIGNED; shift = 2; - max_csize = SI_DMA_COPY_MAX_SIZE_DW; + max_size = SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE; } else { sub_cmd = SI_DMA_COPY_BYTE_ALIGNED; shift = 0; - max_csize = SI_DMA_COPY_MAX_SIZE; + max_size = SI_DMA_COPY_MAX_BYTE_ALIGNED_SIZE; } - ncopy = (size / max_csize) + !!(size % max_csize); - r600_need_dma_space(&ctx->b, ncopy * 5); - - r600_context_bo_reloc(&ctx->b, &ctx->b.rings.dma, rsrc, RADEON_USAGE_READ, - RADEON_PRIO_MIN); - r600_context_bo_reloc(&ctx->b, &ctx->b.rings.dma, rdst, RADEON_USAGE_WRITE, - RADEON_PRIO_MIN); + ncopy = DIV_ROUND_UP(size, max_size); + si_need_dma_space(ctx, ncopy * 5, sdst, ssrc); for (i = 0; i < ncopy; i++) { - csize = size < max_csize ? size : max_csize; - cs->buf[cs->cdw++] = SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd, csize); - cs->buf[cs->cdw++] = dst_offset & 0xffffffff; - cs->buf[cs->cdw++] = src_offset & 0xffffffff; - cs->buf[cs->cdw++] = (dst_offset >> 32UL) & 0xff; - cs->buf[cs->cdw++] = (src_offset >> 32UL) & 0xff; - dst_offset += csize << shift; - src_offset += csize << shift; - size -= csize; + count = MIN2(size, max_size); + radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd, + count >> shift)); + radeon_emit(cs, dst_offset); + radeon_emit(cs, src_offset); + radeon_emit(cs, (dst_offset >> 32UL) & 0xff); + radeon_emit(cs, (src_offset >> 32UL) & 0xff); + dst_offset += count; + src_offset += count; + size -= count; } } @@ -126,35 +92,31 @@ static void si_dma_copy_tile(struct si_context *ctx, unsigned pitch, unsigned bpp) { - struct radeon_winsys_cs *cs = ctx->b.rings.dma.cs; - struct si_screen *sscreen = ctx->screen; - struct r600_texture *rsrc = (struct r600_texture*)src; - struct r600_texture *rdst = (struct r600_texture*)dst; - struct r600_texture *rlinear, *rtiled; - unsigned linear_lvl, tiled_lvl; + struct radeon_cmdbuf *cs = ctx->dma_cs; + struct si_texture *ssrc = (struct si_texture*)src; + struct si_texture *sdst = (struct si_texture*)dst; + unsigned dst_mode = sdst->surface.u.legacy.level[dst_level].mode; + bool detile = dst_mode == RADEON_SURF_MODE_LINEAR_ALIGNED; + struct si_texture *linear = detile ? sdst : ssrc; + struct si_texture *tiled = detile ? ssrc : sdst; + unsigned linear_lvl = detile ? dst_level : src_level; + unsigned tiled_lvl = detile ? src_level : dst_level; + struct radeon_info *info = &ctx->screen->info; + unsigned index = tiled->surface.u.legacy.tiling_index[tiled_lvl]; + unsigned tile_mode = info->si_tile_mode_array[index]; unsigned array_mode, lbpp, pitch_tile_max, slice_tile_max, size; - unsigned ncopy, height, cheight, detile, i, src_mode, dst_mode; + unsigned ncopy, height, cheight, i; unsigned linear_x, linear_y, linear_z, tiled_x, tiled_y, tiled_z; unsigned sub_cmd, bank_h, bank_w, mt_aspect, nbanks, tile_split, mt; uint64_t base, addr; - unsigned pipe_config, tile_mode_index; + unsigned pipe_config; - dst_mode = rdst->surface.level[dst_level].mode; - src_mode = rsrc->surface.level[src_level].mode; - /* downcast linear aligned to linear to simplify test */ - src_mode = src_mode == RADEON_SURF_MODE_LINEAR_ALIGNED ? RADEON_SURF_MODE_LINEAR : src_mode; - dst_mode = dst_mode == RADEON_SURF_MODE_LINEAR_ALIGNED ? RADEON_SURF_MODE_LINEAR : dst_mode; - assert(dst_mode != src_mode); + assert(dst_mode != ssrc->surface.u.legacy.level[src_level].mode); sub_cmd = SI_DMA_COPY_TILED; lbpp = util_logbase2(bpp); pitch_tile_max = ((pitch / bpp) / 8) - 1; - detile = dst_mode == RADEON_SURF_MODE_LINEAR; - rlinear = detile ? rdst : rsrc; - rtiled = detile ? rsrc : rdst; - linear_lvl = detile ? dst_level : src_level; - tiled_lvl = detile ? src_level : dst_level; linear_x = detile ? dst_x : src_x; linear_y = detile ? dst_y : src_y; linear_z = detile ? dst_z : src_z; @@ -162,86 +124,78 @@ static void si_dma_copy_tile(struct si_context *ctx, tiled_y = detile ? src_y : dst_y; tiled_z = detile ? src_z : dst_z; - assert(!util_format_is_depth_and_stencil(rtiled->resource.b.b.format)); + assert(!util_format_is_depth_and_stencil(tiled->buffer.b.b.format)); - array_mode = si_array_mode(rtiled->surface.level[tiled_lvl].mode); - slice_tile_max = (rtiled->surface.level[tiled_lvl].nblk_x * - rtiled->surface.level[tiled_lvl].nblk_y) / (8*8) - 1; + array_mode = G_009910_ARRAY_MODE(tile_mode); + slice_tile_max = (tiled->surface.u.legacy.level[tiled_lvl].nblk_x * + tiled->surface.u.legacy.level[tiled_lvl].nblk_y) / (8*8) - 1; /* linear height must be the same as the slice tile max height, it's ok even * if the linear destination/source have smaller heigh as the size of the * dma packet will be using the copy_height which is always smaller or equal * to the linear height */ - height = rtiled->surface.level[tiled_lvl].nblk_y; - base = rtiled->surface.level[tiled_lvl].offset; - addr = rlinear->surface.level[linear_lvl].offset; - addr += rlinear->surface.level[linear_lvl].slice_size * linear_z; + height = tiled->surface.u.legacy.level[tiled_lvl].nblk_y; + base = tiled->surface.u.legacy.level[tiled_lvl].offset; + addr = linear->surface.u.legacy.level[linear_lvl].offset; + addr += (uint64_t)linear->surface.u.legacy.level[linear_lvl].slice_size_dw * 4 * linear_z; addr += linear_y * pitch + linear_x * bpp; - bank_h = cik_bank_wh(rtiled->surface.bankh); - bank_w = cik_bank_wh(rtiled->surface.bankw); - mt_aspect = cik_macro_tile_aspect(rtiled->surface.mtilea); - tile_split = cik_tile_split(rtiled->surface.tile_split); - tile_mode_index = si_tile_mode_index(rtiled, tiled_lvl, false); - nbanks = si_num_banks(sscreen, rtiled); - base += rtiled->resource.gpu_address; - addr += rlinear->resource.gpu_address; - - pipe_config = cik_db_pipe_config(sscreen, tile_mode_index); - mt = si_micro_tile_mode(sscreen, tile_mode_index); - size = (copy_height * pitch) / 4; - ncopy = (size / SI_DMA_COPY_MAX_SIZE_DW) + !!(size % SI_DMA_COPY_MAX_SIZE_DW); - r600_need_dma_space(&ctx->b, ncopy * 9); - - r600_context_bo_reloc(&ctx->b, &ctx->b.rings.dma, &rsrc->resource, - RADEON_USAGE_READ, RADEON_PRIO_MIN); - r600_context_bo_reloc(&ctx->b, &ctx->b.rings.dma, &rdst->resource, - RADEON_USAGE_WRITE, RADEON_PRIO_MIN); + bank_h = G_009910_BANK_HEIGHT(tile_mode); + bank_w = G_009910_BANK_WIDTH(tile_mode); + mt_aspect = G_009910_MACRO_TILE_ASPECT(tile_mode); + /* Non-depth modes don't have TILE_SPLIT set. */ + tile_split = util_logbase2(tiled->surface.u.legacy.tile_split >> 6); + nbanks = G_009910_NUM_BANKS(tile_mode); + base += tiled->buffer.gpu_address; + addr += linear->buffer.gpu_address; + + pipe_config = G_009910_PIPE_CONFIG(tile_mode); + mt = G_009910_MICRO_TILE_MODE(tile_mode); + size = copy_height * pitch; + ncopy = DIV_ROUND_UP(size, SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE); + si_need_dma_space(ctx, ncopy * 9, &sdst->buffer, &ssrc->buffer); for (i = 0; i < ncopy; i++) { cheight = copy_height; - if (((cheight * pitch) / 4) > SI_DMA_COPY_MAX_SIZE_DW) { - cheight = (SI_DMA_COPY_MAX_SIZE_DW * 4) / pitch; + if (cheight * pitch > SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE) { + cheight = SI_DMA_COPY_MAX_DWORD_ALIGNED_SIZE / pitch; } - size = (cheight * pitch) / 4; - cs->buf[cs->cdw++] = SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd, size); - cs->buf[cs->cdw++] = base >> 8; - cs->buf[cs->cdw++] = (detile << 31) | (array_mode << 27) | - (lbpp << 24) | (bank_h << 21) | - (bank_w << 18) | (mt_aspect << 16); - cs->buf[cs->cdw++] = (pitch_tile_max << 0) | ((height - 1) << 16); - cs->buf[cs->cdw++] = (slice_tile_max << 0) | (pipe_config << 26); - cs->buf[cs->cdw++] = (tiled_x << 0) | (tiled_z << 18); - cs->buf[cs->cdw++] = (tiled_y << 0) | (tile_split << 21) | (nbanks << 25) | (mt << 27); - cs->buf[cs->cdw++] = addr & 0xfffffffc; - cs->buf[cs->cdw++] = (addr >> 32UL) & 0xff; + size = cheight * pitch; + radeon_emit(cs, SI_DMA_PACKET(SI_DMA_PACKET_COPY, sub_cmd, size / 4)); + radeon_emit(cs, base >> 8); + radeon_emit(cs, (detile << 31) | (array_mode << 27) | + (lbpp << 24) | (bank_h << 21) | + (bank_w << 18) | (mt_aspect << 16)); + radeon_emit(cs, (pitch_tile_max << 0) | ((height - 1) << 16)); + radeon_emit(cs, (slice_tile_max << 0) | (pipe_config << 26)); + radeon_emit(cs, (tiled_x << 0) | (tiled_z << 18)); + radeon_emit(cs, (tiled_y << 0) | (tile_split << 21) | (nbanks << 25) | (mt << 27)); + radeon_emit(cs, addr & 0xfffffffc); + radeon_emit(cs, (addr >> 32UL) & 0xff); copy_height -= cheight; addr += cheight * pitch; tiled_y += cheight; } } -void si_dma_copy(struct pipe_context *ctx, - struct pipe_resource *dst, - unsigned dst_level, - unsigned dstx, unsigned dsty, unsigned dstz, - struct pipe_resource *src, - unsigned src_level, - const struct pipe_box *src_box) +static void si_dma_copy(struct pipe_context *ctx, + struct pipe_resource *dst, + unsigned dst_level, + unsigned dstx, unsigned dsty, unsigned dstz, + struct pipe_resource *src, + unsigned src_level, + const struct pipe_box *src_box) { struct si_context *sctx = (struct si_context *)ctx; - struct r600_texture *rsrc = (struct r600_texture*)src; - struct r600_texture *rdst = (struct r600_texture*)dst; - unsigned dst_pitch, src_pitch, bpp, dst_mode, src_mode, copy_height; + struct si_texture *ssrc = (struct si_texture*)src; + struct si_texture *sdst = (struct si_texture*)dst; + unsigned dst_pitch, src_pitch, bpp, dst_mode, src_mode; unsigned src_w, dst_w; unsigned src_x, src_y; unsigned dst_x = dstx, dst_y = dsty, dst_z = dstz; - if (sctx->b.rings.dma.cs == NULL) { - goto fallback; - } - - /* TODO: Implement DMA copy for CIK */ - if (sctx->b.chip_class >= CIK) { + if (sctx->dma_cs == NULL || + src->flags & PIPE_RESOURCE_FLAG_SPARSE || + dst->flags & PIPE_RESOURCE_FLAG_SPARSE) { goto fallback; } @@ -250,41 +204,54 @@ void si_dma_copy(struct pipe_context *ctx, return; } - if (src->format != dst->format || src_box->depth > 1 || - rdst->dirty_level_mask != 0) { - goto fallback; - } + /* XXX: Using the asynchronous DMA engine for multi-dimensional + * operations seems to cause random GPU lockups for various people. + * While the root cause for this might need to be fixed in the kernel, + * let's disable it for now. + * + * Before re-enabling this, please make sure you can hit all newly + * enabled paths in your testing, preferably with both piglit and real + * world apps, and get in touch with people on the bug reports below + * for stability testing. + * + * https://bugs.freedesktop.org/show_bug.cgi?id=85647 + * https://bugs.freedesktop.org/show_bug.cgi?id=83500 + */ + goto fallback; - if (rsrc->dirty_level_mask) { - ctx->flush_resource(ctx, src); - } + if (src_box->depth > 1 || + !si_prepare_for_dma_blit(sctx, sdst, dst_level, dstx, dsty, + dstz, ssrc, src_level, src_box)) + goto fallback; src_x = util_format_get_nblocksx(src->format, src_box->x); dst_x = util_format_get_nblocksx(src->format, dst_x); src_y = util_format_get_nblocksy(src->format, src_box->y); dst_y = util_format_get_nblocksy(src->format, dst_y); - bpp = rdst->surface.bpe; - dst_pitch = rdst->surface.level[dst_level].pitch_bytes; - src_pitch = rsrc->surface.level[src_level].pitch_bytes; - src_w = rsrc->surface.level[src_level].npix_x; - dst_w = rdst->surface.level[dst_level].npix_x; - copy_height = src_box->height / rsrc->surface.blk_h; - - dst_mode = rdst->surface.level[dst_level].mode; - src_mode = rsrc->surface.level[src_level].mode; - /* downcast linear aligned to linear to simplify test */ - src_mode = src_mode == RADEON_SURF_MODE_LINEAR_ALIGNED ? RADEON_SURF_MODE_LINEAR : src_mode; - dst_mode = dst_mode == RADEON_SURF_MODE_LINEAR_ALIGNED ? RADEON_SURF_MODE_LINEAR : dst_mode; - - if (src_pitch != dst_pitch || src_box->x || dst_x || src_w != dst_w) { + bpp = sdst->surface.bpe; + dst_pitch = sdst->surface.u.legacy.level[dst_level].nblk_x * sdst->surface.bpe; + src_pitch = ssrc->surface.u.legacy.level[src_level].nblk_x * ssrc->surface.bpe; + src_w = u_minify(ssrc->buffer.b.b.width0, src_level); + dst_w = u_minify(sdst->buffer.b.b.width0, dst_level); + + dst_mode = sdst->surface.u.legacy.level[dst_level].mode; + src_mode = ssrc->surface.u.legacy.level[src_level].mode; + + if (src_pitch != dst_pitch || src_box->x || dst_x || src_w != dst_w || + src_box->width != src_w || + src_box->height != u_minify(ssrc->buffer.b.b.height0, src_level) || + src_box->height != u_minify(sdst->buffer.b.b.height0, dst_level) || + ssrc->surface.u.legacy.level[src_level].nblk_y != + sdst->surface.u.legacy.level[dst_level].nblk_y) { /* FIXME si can do partial blit */ goto fallback; } /* the x test here are currently useless (because we don't support partial blit) * but keep them around so we don't forget about those */ - if ((src_pitch % 8) || (src_box->x % 8) || (dst_x % 8) || (src_box->y % 8) || (dst_y % 8)) { + if ((src_pitch % 8) || (src_box->x % 8) || (dst_x % 8) || + (src_box->y % 8) || (dst_y % 8) || (src_box->height % 8)) { goto fallback; } @@ -295,18 +262,19 @@ void si_dma_copy(struct pipe_context *ctx, * dst_x/y == 0 * dst_pitch == src_pitch */ - src_offset= rsrc->surface.level[src_level].offset; - src_offset += rsrc->surface.level[src_level].slice_size * src_box->z; + src_offset= ssrc->surface.u.legacy.level[src_level].offset; + src_offset += (uint64_t)ssrc->surface.u.legacy.level[src_level].slice_size_dw * 4 * src_box->z; src_offset += src_y * src_pitch + src_x * bpp; - dst_offset = rdst->surface.level[dst_level].offset; - dst_offset += rdst->surface.level[dst_level].slice_size * dst_z; + dst_offset = sdst->surface.u.legacy.level[dst_level].offset; + dst_offset += (uint64_t)sdst->surface.u.legacy.level[dst_level].slice_size_dw * 4 * dst_z; dst_offset += dst_y * dst_pitch + dst_x * bpp; si_dma_copy_buffer(sctx, dst, src, dst_offset, src_offset, - copy_height * src_pitch); + (uint64_t)ssrc->surface.u.legacy.level[src_level].slice_size_dw * 4); } else { si_dma_copy_tile(sctx, dst, dst_level, dst_x, dst_y, dst_z, src, src_level, src_x, src_y, src_box->z, - copy_height, dst_pitch, bpp); + src_box->height / ssrc->surface.blk_h, + dst_pitch, bpp); } return; @@ -314,3 +282,8 @@ fallback: si_resource_copy_region(ctx, dst, dst_level, dstx, dsty, dstz, src, src_level, src_box); } + +void si_init_dma_functions(struct si_context *sctx) +{ + sctx->dma_copy = si_dma_copy; +}