src/gallium/drivers/r600/r600_hw_context.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Jerome Glisse
  25  */
  26 #include "r600_pipe.h"
  27 #include "r600d.h"
  28 #include "util/u_memory.h"
  29 #include <errno.h>
  30 #include <unistd.h>
  31
  32
  33 void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw,
  34                         boolean count_draw_in)
  35 {
  36         struct radeon_winsys_cs *dma = ctx->b.dma.cs;
  37
  38         /* Flush the DMA IB if it's not empty. */
  39         if (dma && dma->cdw)
  40                 ctx->b.dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
  41
  42         if (!ctx->b.ws->cs_memory_below_limit(ctx->b.gfx.cs, ctx->b.vram, ctx->b.gtt)) {
  43                 ctx->b.gtt = 0;
  44                 ctx->b.vram = 0;
  45                 ctx->b.gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
  46                 return;
  47         }
  48         /* all will be accounted once relocation are emited */
  49         ctx->b.gtt = 0;
  50         ctx->b.vram = 0;
  51
  52         /* The number of dwords we already used in the CS so far. */
  53         num_dw += ctx->b.gfx.cs->cdw;
  54
  55         if (count_draw_in) {
  56                 uint64_t mask;
  57
  58                 /* The number of dwords all the dirty states would take. */
  59                 mask = ctx->dirty_atoms;
  60                 while (mask != 0)
  61                         num_dw += ctx->atoms[u_bit_scan64(&mask)]->num_dw;
  62
  63                 /* The upper-bound of how much space a draw command would take. */
  64                 num_dw += R600_MAX_FLUSH_CS_DWORDS + R600_MAX_DRAW_CS_DWORDS;
  65         }
  66
  67         /* Count in r600_suspend_queries. */
  68         num_dw += ctx->b.num_cs_dw_queries_suspend;
  69
  70         /* Count in streamout_end at the end of CS. */
  71         if (ctx->b.streamout.begin_emitted) {
  72                 num_dw += ctx->b.streamout.num_dw_for_end;
  73         }
  74
  75         /* SX_MISC */
  76         if (ctx->b.chip_class == R600) {
  77                 num_dw += 3;
  78         }
  79
  80         /* Count in framebuffer cache flushes at the end of CS. */
  81         num_dw += R600_MAX_FLUSH_CS_DWORDS;
  82
  83         /* The fence at the end of CS. */
  84         num_dw += 10;
  85
  86         /* Flush if there's not enough space. */
  87         if (num_dw > ctx->b.gfx.cs->max_dw) {
  88                 ctx->b.gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
  89         }
  90 }
  91
  92 void r600_flush_emit(struct r600_context *rctx)
  93 {
  94         struct radeon_winsys_cs *cs = rctx->b.gfx.cs;
  95         unsigned cp_coher_cntl = 0;
  96         unsigned wait_until = 0;
  97
  98         if (!rctx->b.flags) {
  99                 return;
 100         }
 101
 102         if (rctx->b.flags & R600_CONTEXT_WAIT_3D_IDLE) {
 103                 wait_until |= S_008040_WAIT_3D_IDLE(1);
 104         }
 105         if (rctx->b.flags & R600_CONTEXT_WAIT_CP_DMA_IDLE) {
 106                 wait_until |= S_008040_WAIT_CP_DMA_IDLE(1);
 107         }
 108
 109         if (wait_until) {
 110                 /* Use of WAIT_UNTIL is deprecated on Cayman+ */
 111                 if (rctx->b.family >= CHIP_CAYMAN) {
 112                         /* emit a PS partial flush on Cayman/TN */
 113                         rctx->b.flags |= R600_CONTEXT_PS_PARTIAL_FLUSH;
 114                 }
 115         }
 116
 117         if (rctx->b.flags & R600_CONTEXT_PS_PARTIAL_FLUSH) {
 118                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 119                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
 120         }
 121
 122         if (rctx->b.chip_class >= R700 &&
 123             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_CB_META)) {
 124                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 125                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_CB_META) | EVENT_INDEX(0);
 126         }
 127
 128         if (rctx->b.chip_class >= R700 &&
 129             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_DB_META)) {
 130                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 131                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_DB_META) | EVENT_INDEX(0);
 132
 133                 /* Set FULL_CACHE_ENA for DB META flushes on r7xx and later.
 134                  *
 135                  * This hack predates use of FLUSH_AND_INV_DB_META, so it's
 136                  * unclear whether it's still needed or even whether it has
 137                  * any effect.
 138                  */
 139                 cp_coher_cntl |= S_0085F0_FULL_CACHE_ENA(1);
 140         }
 141
 142         if (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV ||
 143             (rctx->b.chip_class == R600 && rctx->b.flags & R600_CONTEXT_STREAMOUT_FLUSH)) {
 144                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 145                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0);
 146         }
 147
 148         if (rctx->b.flags & R600_CONTEXT_INV_CONST_CACHE) {
 149                 /* Direct constant addressing uses the shader cache.
 150                  * Indirect contant addressing uses the vertex cache. */
 151                 cp_coher_cntl |= S_0085F0_SH_ACTION_ENA(1) |
 152                                  (rctx->has_vertex_cache ? S_0085F0_VC_ACTION_ENA(1)
 153                                                          : S_0085F0_TC_ACTION_ENA(1));
 154         }
 155         if (rctx->b.flags & R600_CONTEXT_INV_VERTEX_CACHE) {
 156                 cp_coher_cntl |= rctx->has_vertex_cache ? S_0085F0_VC_ACTION_ENA(1)
 157                                                         : S_0085F0_TC_ACTION_ENA(1);
 158         }
 159         if (rctx->b.flags & R600_CONTEXT_INV_TEX_CACHE) {
 160                 /* Textures use the texture cache.
 161                  * Texture buffer objects use the vertex cache. */
 162                 cp_coher_cntl |= S_0085F0_TC_ACTION_ENA(1) |
 163                                  (rctx->has_vertex_cache ? S_0085F0_VC_ACTION_ENA(1) : 0);
 164         }
 165
 166         /* Don't use the DB CP COHER logic on r6xx.
 167          * There are hw bugs.
 168          */
 169         if (rctx->b.chip_class >= R700 &&
 170             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_DB)) {
 171                 cp_coher_cntl |= S_0085F0_DB_ACTION_ENA(1) |
 172                                 S_0085F0_DB_DEST_BASE_ENA(1) |
 173                                 S_0085F0_SMX_ACTION_ENA(1);
 174         }
 175
 176         /* Don't use the CB CP COHER logic on r6xx.
 177          * There are hw bugs.
 178          */
 179         if (rctx->b.chip_class >= R700 &&
 180             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_CB)) {
 181                 cp_coher_cntl |= S_0085F0_CB_ACTION_ENA(1) |
 182                                 S_0085F0_CB0_DEST_BASE_ENA(1) |
 183                                 S_0085F0_CB1_DEST_BASE_ENA(1) |
 184                                 S_0085F0_CB2_DEST_BASE_ENA(1) |
 185                                 S_0085F0_CB3_DEST_BASE_ENA(1) |
 186                                 S_0085F0_CB4_DEST_BASE_ENA(1) |
 187                                 S_0085F0_CB5_DEST_BASE_ENA(1) |
 188                                 S_0085F0_CB6_DEST_BASE_ENA(1) |
 189                                 S_0085F0_CB7_DEST_BASE_ENA(1) |
 190                                 S_0085F0_SMX_ACTION_ENA(1);
 191                 if (rctx->b.chip_class >= EVERGREEN)
 192                         cp_coher_cntl |= S_0085F0_CB8_DEST_BASE_ENA(1) |
 193                                         S_0085F0_CB9_DEST_BASE_ENA(1) |
 194                                         S_0085F0_CB10_DEST_BASE_ENA(1) |
 195                                         S_0085F0_CB11_DEST_BASE_ENA(1);
 196         }
 197
 198         if (rctx->b.chip_class >= R700 &&
 199             rctx->b.flags & R600_CONTEXT_STREAMOUT_FLUSH) {
 200                 cp_coher_cntl |= S_0085F0_SO0_DEST_BASE_ENA(1) |
 201                                 S_0085F0_SO1_DEST_BASE_ENA(1) |
 202                                 S_0085F0_SO2_DEST_BASE_ENA(1) |
 203                                 S_0085F0_SO3_DEST_BASE_ENA(1) |
 204                                 S_0085F0_SMX_ACTION_ENA(1);
 205         }
 206
 207         /* Workaround for buggy flushing on some R6xx chipsets. */
 208         if ((rctx->b.flags & (R600_CONTEXT_FLUSH_AND_INV |
 209                               R600_CONTEXT_STREAMOUT_FLUSH)) &&
 210             (rctx->b.family == CHIP_RV670 ||
 211              rctx->b.family == CHIP_RS780 ||
 212              rctx->b.family == CHIP_RS880)) {
 213                 cp_coher_cntl |=  S_0085F0_CB1_DEST_BASE_ENA(1) |
 214                                   S_0085F0_DEST_BASE_0_ENA(1);
 215         }
 216
 217         if (cp_coher_cntl) {
 218                 cs->buf[cs->cdw++] = PKT3(PKT3_SURFACE_SYNC, 3, 0);
 219                 cs->buf[cs->cdw++] = cp_coher_cntl;   /* CP_COHER_CNTL */
 220                 cs->buf[cs->cdw++] = 0xffffffff;      /* CP_COHER_SIZE */
 221                 cs->buf[cs->cdw++] = 0;               /* CP_COHER_BASE */
 222                 cs->buf[cs->cdw++] = 0x0000000A;      /* POLL_INTERVAL */
 223         }
 224
 225         if (rctx->b.flags & R600_CONTEXT_START_PIPELINE_STATS) {
 226                 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 227                 radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PIPELINESTAT_START) |
 228                                 EVENT_INDEX(0));
 229         } else if (rctx->b.flags & R600_CONTEXT_STOP_PIPELINE_STATS) {
 230                 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
 231                 radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_PIPELINESTAT_STOP) |
 232                                 EVENT_INDEX(0));
 233         }
 234
 235         if (wait_until) {
 236                 /* Use of WAIT_UNTIL is deprecated on Cayman+ */
 237                 if (rctx->b.family < CHIP_CAYMAN) {
 238                         /* wait for things to settle */
 239                         radeon_set_config_reg(cs, R_008040_WAIT_UNTIL, wait_until);
 240                 }
 241         }
 242
 243         /* everything is properly flushed */
 244         rctx->b.flags = 0;
 245 }
 246
 247 void r600_context_gfx_flush(void *context, unsigned flags,
 248                             struct pipe_fence_handle **fence)
 249 {
 250         struct r600_context *ctx = context;
 251         struct radeon_winsys_cs *cs = ctx->b.gfx.cs;
 252
 253         if (cs->cdw == ctx->b.initial_gfx_cs_size && !fence)
 254                 return;
 255
 256         r600_preflush_suspend_features(&ctx->b);
 257
 258         /* flush the framebuffer cache */
 259         ctx->b.flags |= R600_CONTEXT_FLUSH_AND_INV |
 260                       R600_CONTEXT_FLUSH_AND_INV_CB |
 261                       R600_CONTEXT_FLUSH_AND_INV_DB |
 262                       R600_CONTEXT_FLUSH_AND_INV_CB_META |
 263                       R600_CONTEXT_FLUSH_AND_INV_DB_META |
 264                       R600_CONTEXT_WAIT_3D_IDLE |
 265                       R600_CONTEXT_WAIT_CP_DMA_IDLE;
 266
 267         r600_flush_emit(ctx);
 268
 269         /* old kernels and userspace don't set SX_MISC, so we must reset it to 0 here */
 270         if (ctx->b.chip_class == R600) {
 271                 radeon_set_context_reg(cs, R_028350_SX_MISC, 0);
 272         }
 273
 274         /* force to keep tiling flags */
 275         flags |= RADEON_FLUSH_KEEP_TILING_FLAGS;
 276
 277         /* Flush the CS. */
 278         ctx->b.ws->cs_flush(cs, flags, fence);
 279
 280         r600_begin_new_cs(ctx);
 281 }
 282
 283 void r600_begin_new_cs(struct r600_context *ctx)
 284 {
 285         unsigned shader;
 286
 287         ctx->b.flags = 0;
 288         ctx->b.gtt = 0;
 289         ctx->b.vram = 0;
 290
 291         /* Begin a new CS. */
 292         r600_emit_command_buffer(ctx->b.gfx.cs, &ctx->start_cs_cmd);
 293
 294         /* Re-emit states. */
 295         r600_mark_atom_dirty(ctx, &ctx->alphatest_state.atom);
 296         r600_mark_atom_dirty(ctx, &ctx->blend_color.atom);
 297         r600_mark_atom_dirty(ctx, &ctx->cb_misc_state.atom);
 298         r600_mark_atom_dirty(ctx, &ctx->clip_misc_state.atom);
 299         r600_mark_atom_dirty(ctx, &ctx->clip_state.atom);
 300         r600_mark_atom_dirty(ctx, &ctx->db_misc_state.atom);
 301         r600_mark_atom_dirty(ctx, &ctx->db_state.atom);
 302         r600_mark_atom_dirty(ctx, &ctx->framebuffer.atom);
 303         r600_mark_atom_dirty(ctx, &ctx->hw_shader_stages[R600_HW_STAGE_PS].atom);
 304         r600_mark_atom_dirty(ctx, &ctx->poly_offset_state.atom);
 305         r600_mark_atom_dirty(ctx, &ctx->vgt_state.atom);
 306         r600_mark_atom_dirty(ctx, &ctx->sample_mask.atom);
 307         ctx->b.scissors.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1;
 308         r600_mark_atom_dirty(ctx, &ctx->b.scissors.atom);
 309         ctx->b.viewports.dirty_mask = (1 << R600_MAX_VIEWPORTS) - 1;
 310         r600_mark_atom_dirty(ctx, &ctx->b.viewports.atom);
 311         if (ctx->b.chip_class <= EVERGREEN) {
 312                 r600_mark_atom_dirty(ctx, &ctx->config_state.atom);
 313         }
 314         r600_mark_atom_dirty(ctx, &ctx->stencil_ref.atom);
 315         r600_mark_atom_dirty(ctx, &ctx->vertex_fetch_shader.atom);
 316         r600_mark_atom_dirty(ctx, &ctx->hw_shader_stages[R600_HW_STAGE_ES].atom);
 317         r600_mark_atom_dirty(ctx, &ctx->shader_stages.atom);
 318         if (ctx->gs_shader) {
 319                 r600_mark_atom_dirty(ctx, &ctx->hw_shader_stages[R600_HW_STAGE_GS].atom);
 320                 r600_mark_atom_dirty(ctx, &ctx->gs_rings.atom);
 321         }
 322         if (ctx->tes_shader) {
 323                 r600_mark_atom_dirty(ctx, &ctx->hw_shader_stages[EG_HW_STAGE_HS].atom);
 324                 r600_mark_atom_dirty(ctx, &ctx->hw_shader_stages[EG_HW_STAGE_LS].atom);
 325         }
 326         r600_mark_atom_dirty(ctx, &ctx->hw_shader_stages[R600_HW_STAGE_VS].atom);
 327         r600_mark_atom_dirty(ctx, &ctx->b.streamout.enable_atom);
 328         r600_mark_atom_dirty(ctx, &ctx->b.render_cond_atom);
 329
 330         if (ctx->blend_state.cso)
 331                 r600_mark_atom_dirty(ctx, &ctx->blend_state.atom);
 332         if (ctx->dsa_state.cso)
 333                 r600_mark_atom_dirty(ctx, &ctx->dsa_state.atom);
 334         if (ctx->rasterizer_state.cso)
 335                 r600_mark_atom_dirty(ctx, &ctx->rasterizer_state.atom);
 336
 337         if (ctx->b.chip_class <= R700) {
 338                 r600_mark_atom_dirty(ctx, &ctx->seamless_cube_map.atom);
 339         }
 340
 341         ctx->vertex_buffer_state.dirty_mask = ctx->vertex_buffer_state.enabled_mask;
 342         r600_vertex_buffers_dirty(ctx);
 343
 344         /* Re-emit shader resources. */
 345         for (shader = 0; shader < PIPE_SHADER_TYPES; shader++) {
 346                 struct r600_constbuf_state *constbuf = &ctx->constbuf_state[shader];
 347                 struct r600_textures_info *samplers = &ctx->samplers[shader];
 348
 349                 constbuf->dirty_mask = constbuf->enabled_mask;
 350                 samplers->views.dirty_mask = samplers->views.enabled_mask;
 351                 samplers->states.dirty_mask = samplers->states.enabled_mask;
 352
 353                 r600_constant_buffers_dirty(ctx, constbuf);
 354                 r600_sampler_views_dirty(ctx, &samplers->views);
 355                 r600_sampler_states_dirty(ctx, &samplers->states);
 356         }
 357
 358         r600_postflush_resume_features(&ctx->b);
 359
 360         /* Re-emit the draw state. */
 361         ctx->last_primitive_type = -1;
 362         ctx->last_start_instance = -1;
 363
 364         ctx->b.initial_gfx_cs_size = ctx->b.gfx.cs->cdw;
 365 }
 366
 367 /* The max number of bytes to copy per packet. */
 368 #define CP_DMA_MAX_BYTE_COUNT ((1 << 21) - 8)
 369
 370 void r600_cp_dma_copy_buffer(struct r600_context *rctx,
 371                              struct pipe_resource *dst, uint64_t dst_offset,
 372                              struct pipe_resource *src, uint64_t src_offset,
 373                              unsigned size)
 374 {
 375         struct radeon_winsys_cs *cs = rctx->b.gfx.cs;
 376
 377         assert(size);
 378         assert(rctx->screen->b.has_cp_dma);
 379
 380         /* Mark the buffer range of destination as valid (initialized),
 381          * so that transfer_map knows it should wait for the GPU when mapping
 382          * that range. */
 383         util_range_add(&r600_resource(dst)->valid_buffer_range, dst_offset,
 384                        dst_offset + size);
 385
 386         dst_offset += r600_resource(dst)->gpu_address;
 387         src_offset += r600_resource(src)->gpu_address;
 388
 389         /* Flush the caches where the resources are bound. */
 390         rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE |
 391                          R600_CONTEXT_INV_VERTEX_CACHE |
 392                          R600_CONTEXT_INV_TEX_CACHE |
 393                          R600_CONTEXT_FLUSH_AND_INV |
 394                          R600_CONTEXT_FLUSH_AND_INV_CB |
 395                          R600_CONTEXT_FLUSH_AND_INV_DB |
 396                          R600_CONTEXT_FLUSH_AND_INV_CB_META |
 397                          R600_CONTEXT_FLUSH_AND_INV_DB_META |
 398                          R600_CONTEXT_STREAMOUT_FLUSH |
 399                          R600_CONTEXT_WAIT_3D_IDLE;
 400
 401         /* There are differences between R700 and EG in CP DMA,
 402          * but we only use the common bits here. */
 403         while (size) {
 404                 unsigned sync = 0;
 405                 unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
 406                 unsigned src_reloc, dst_reloc;
 407
 408                 r600_need_cs_space(rctx, 10 + (rctx->b.flags ? R600_MAX_FLUSH_CS_DWORDS : 0), FALSE);
 409
 410                 /* Flush the caches for the first copy only. */
 411                 if (rctx->b.flags) {
 412                         r600_flush_emit(rctx);
 413                 }
 414
 415                 /* Do the synchronization after the last copy, so that all data is written to memory. */
 416                 if (size == byte_count) {
 417                         sync = PKT3_CP_DMA_CP_SYNC;
 418                 }
 419
 420                 /* This must be done after r600_need_cs_space. */
 421                 src_reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, (struct r600_resource*)src,
 422                                                   RADEON_USAGE_READ, RADEON_PRIO_CP_DMA);
 423                 dst_reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx, (struct r600_resource*)dst,
 424                                                   RADEON_USAGE_WRITE, RADEON_PRIO_CP_DMA);
 425
 426                 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
 427                 radeon_emit(cs, src_offset);    /* SRC_ADDR_LO [31:0] */
 428                 radeon_emit(cs, sync | ((src_offset >> 32) & 0xff));            /* CP_SYNC [31] | SRC_ADDR_HI [7:0] */
 429                 radeon_emit(cs, dst_offset);    /* DST_ADDR_LO [31:0] */
 430                 radeon_emit(cs, (dst_offset >> 32) & 0xff);             /* DST_ADDR_HI [7:0] */
 431                 radeon_emit(cs, byte_count);    /* COMMAND [29:22] | BYTE_COUNT [20:0] */
 432
 433                 radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
 434                 radeon_emit(cs, src_reloc);
 435                 radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
 436                 radeon_emit(cs, dst_reloc);
 437
 438                 size -= byte_count;
 439                 src_offset += byte_count;
 440                 dst_offset += byte_count;
 441         }
 442
 443         /* Invalidate the read caches. */
 444         rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE |
 445                          R600_CONTEXT_INV_VERTEX_CACHE |
 446                          R600_CONTEXT_INV_TEX_CACHE;
 447 }
 448
 449 void r600_dma_copy_buffer(struct r600_context *rctx,
 450                           struct pipe_resource *dst,
 451                           struct pipe_resource *src,
 452                           uint64_t dst_offset,
 453                           uint64_t src_offset,
 454                           uint64_t size)
 455 {
 456         struct radeon_winsys_cs *cs = rctx->b.dma.cs;
 457         unsigned i, ncopy, csize;
 458         struct r600_resource *rdst = (struct r600_resource*)dst;
 459         struct r600_resource *rsrc = (struct r600_resource*)src;
 460
 461         /* Mark the buffer range of destination as valid (initialized),
 462          * so that transfer_map knows it should wait for the GPU when mapping
 463          * that range. */
 464         util_range_add(&rdst->valid_buffer_range, dst_offset,
 465                        dst_offset + size);
 466
 467         size >>= 2; /* convert to dwords */
 468         ncopy = (size / R600_DMA_COPY_MAX_SIZE_DW) + !!(size % R600_DMA_COPY_MAX_SIZE_DW);
 469
 470         r600_need_dma_space(&rctx->b, ncopy * 5);
 471         for (i = 0; i < ncopy; i++) {
 472                 csize = size < R600_DMA_COPY_MAX_SIZE_DW ? size : R600_DMA_COPY_MAX_SIZE_DW;
 473                 /* emit reloc before writing cs so that cs is always in consistent state */
 474                 radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rsrc, RADEON_USAGE_READ,
 475                                       RADEON_PRIO_SDMA_BUFFER);
 476                 radeon_add_to_buffer_list(&rctx->b, &rctx->b.dma, rdst, RADEON_USAGE_WRITE,
 477                                       RADEON_PRIO_SDMA_BUFFER);
 478                 cs->buf[cs->cdw++] = DMA_PACKET(DMA_PACKET_COPY, 0, 0, csize);
 479                 cs->buf[cs->cdw++] = dst_offset & 0xfffffffc;
 480                 cs->buf[cs->cdw++] = src_offset & 0xfffffffc;
 481                 cs->buf[cs->cdw++] = (dst_offset >> 32UL) & 0xff;
 482                 cs->buf[cs->cdw++] = (src_offset >> 32UL) & 0xff;
 483                 dst_offset += csize << 2;
 484                 src_offset += csize << 2;
 485                 size -= csize;
 486         }
 487 }