src/gallium/drivers/r600/r600_hw_context.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Jerome Glisse
  25  */
  26 #include "r600_pipe.h"
  27 #include "r600d.h"
  28 #include "util/u_memory.h"
  29 #include <errno.h>
  30 #include <unistd.h>
  31
  32
  33 void r600_need_cs_space(struct r600_context *ctx, unsigned num_dw,
  34                         boolean count_draw_in)
  35 {
  36
  37         if (!ctx->b.ws->cs_memory_below_limit(ctx->b.rings.gfx.cs, ctx->b.vram, ctx->b.gtt)) {
  38                 ctx->b.gtt = 0;
  39                 ctx->b.vram = 0;
  40                 ctx->b.rings.gfx.flush(ctx, RADEON_FLUSH_ASYNC);
  41                 return;
  42         }
  43         /* all will be accounted once relocation are emited */
  44         ctx->b.gtt = 0;
  45         ctx->b.vram = 0;
  46
  47         /* The number of dwords we already used in the CS so far. */
  48         num_dw += ctx->b.rings.gfx.cs->cdw;
  49
  50         if (count_draw_in) {
  51                 unsigned i;
  52
  53                 /* The number of dwords all the dirty states would take. */
  54                 for (i = 0; i < R600_NUM_ATOMS; i++) {
  55                         if (ctx->atoms[i] && ctx->atoms[i]->dirty) {
  56                                 num_dw += ctx->atoms[i]->num_dw;
  57                                 if (ctx->screen->trace_bo) {
  58                                         num_dw += R600_TRACE_CS_DWORDS;
  59                                 }
  60                         }
  61                 }
  62
  63                 /* The upper-bound of how much space a draw command would take. */
  64                 num_dw += R600_MAX_FLUSH_CS_DWORDS + R600_MAX_DRAW_CS_DWORDS;
  65                 if (ctx->screen->trace_bo) {
  66                         num_dw += R600_TRACE_CS_DWORDS;
  67                 }
  68         }
  69
  70         /* Count in queries_suspend. */
  71         num_dw += ctx->b.num_cs_dw_nontimer_queries_suspend;
  72
  73         /* Count in streamout_end at the end of CS. */
  74         if (ctx->b.streamout.begin_emitted) {
  75                 num_dw += ctx->b.streamout.num_dw_for_end;
  76         }
  77
  78         /* Count in render_condition(NULL) at the end of CS. */
  79         if (ctx->b.predicate_drawing) {
  80                 num_dw += 3;
  81         }
  82
  83         /* SX_MISC */
  84         if (ctx->b.chip_class <= R700) {
  85                 num_dw += 3;
  86         }
  87
  88         /* Count in framebuffer cache flushes at the end of CS. */
  89         num_dw += R600_MAX_FLUSH_CS_DWORDS;
  90
  91         /* The fence at the end of CS. */
  92         num_dw += 10;
  93
  94         /* Flush if there's not enough space. */
  95         if (num_dw > RADEON_MAX_CMDBUF_DWORDS) {
  96                 ctx->b.rings.gfx.flush(ctx, RADEON_FLUSH_ASYNC);
  97         }
  98 }
  99
 100 void r600_flush_emit(struct r600_context *rctx)
 101 {
 102         struct radeon_winsys_cs *cs = rctx->b.rings.gfx.cs;
 103         unsigned cp_coher_cntl = 0;
 104         unsigned wait_until = 0;
 105
 106         if (!rctx->b.flags) {
 107                 return;
 108         }
 109
 110         if (rctx->b.flags & R600_CONTEXT_WAIT_3D_IDLE) {
 111                 wait_until |= S_008040_WAIT_3D_IDLE(1);
 112         }
 113         if (rctx->b.flags & R600_CONTEXT_WAIT_CP_DMA_IDLE) {
 114                 wait_until |= S_008040_WAIT_CP_DMA_IDLE(1);
 115         }
 116
 117         if (wait_until) {
 118                 /* Use of WAIT_UNTIL is deprecated on Cayman+ */
 119                 if (rctx->b.family >= CHIP_CAYMAN) {
 120                         /* emit a PS partial flush on Cayman/TN */
 121                         rctx->b.flags |= R600_CONTEXT_PS_PARTIAL_FLUSH;
 122                 }
 123         }
 124
 125         if (rctx->b.flags & R600_CONTEXT_PS_PARTIAL_FLUSH) {
 126                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 127                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
 128         }
 129
 130         if (rctx->b.chip_class >= R700 &&
 131             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_CB_META)) {
 132                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 133                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_CB_META) | EVENT_INDEX(0);
 134         }
 135
 136         if (rctx->b.chip_class >= R700 &&
 137             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_DB_META)) {
 138                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 139                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_FLUSH_AND_INV_DB_META) | EVENT_INDEX(0);
 140
 141                 /* Set FULL_CACHE_ENA for DB META flushes on r7xx and later.
 142                  *
 143                  * This hack predates use of FLUSH_AND_INV_DB_META, so it's
 144                  * unclear whether it's still needed or even whether it has
 145                  * any effect.
 146                  */
 147                 cp_coher_cntl |= S_0085F0_FULL_CACHE_ENA(1);
 148         }
 149
 150         if (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV) {
 151                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 152                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0);
 153         }
 154
 155         if (rctx->b.flags & R600_CONTEXT_INV_CONST_CACHE) {
 156                 /* Direct constant addressing uses the shader cache.
 157                  * Indirect contant addressing uses the vertex cache. */
 158                 cp_coher_cntl |= S_0085F0_SH_ACTION_ENA(1) |
 159                                  (rctx->has_vertex_cache ? S_0085F0_VC_ACTION_ENA(1)
 160                                                          : S_0085F0_TC_ACTION_ENA(1));
 161         }
 162         if (rctx->b.flags & R600_CONTEXT_INV_VERTEX_CACHE) {
 163                 cp_coher_cntl |= rctx->has_vertex_cache ? S_0085F0_VC_ACTION_ENA(1)
 164                                                         : S_0085F0_TC_ACTION_ENA(1);
 165         }
 166         if (rctx->b.flags & R600_CONTEXT_INV_TEX_CACHE) {
 167                 /* Textures use the texture cache.
 168                  * Texture buffer objects use the vertex cache. */
 169                 cp_coher_cntl |= S_0085F0_TC_ACTION_ENA(1) |
 170                                  (rctx->has_vertex_cache ? S_0085F0_VC_ACTION_ENA(1) : 0);
 171         }
 172
 173         /* Don't use the DB CP COHER logic on r6xx.
 174          * There are hw bugs.
 175          */
 176         if (rctx->b.chip_class >= R700 &&
 177             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_DB)) {
 178                 cp_coher_cntl |= S_0085F0_DB_ACTION_ENA(1) |
 179                                 S_0085F0_DB_DEST_BASE_ENA(1) |
 180                                 S_0085F0_SMX_ACTION_ENA(1);
 181         }
 182
 183         /* Don't use the CB CP COHER logic on r6xx.
 184          * There are hw bugs.
 185          */
 186         if (rctx->b.chip_class >= R700 &&
 187             (rctx->b.flags & R600_CONTEXT_FLUSH_AND_INV_CB)) {
 188                 cp_coher_cntl |= S_0085F0_CB_ACTION_ENA(1) |
 189                                 S_0085F0_CB0_DEST_BASE_ENA(1) |
 190                                 S_0085F0_CB1_DEST_BASE_ENA(1) |
 191                                 S_0085F0_CB2_DEST_BASE_ENA(1) |
 192                                 S_0085F0_CB3_DEST_BASE_ENA(1) |
 193                                 S_0085F0_CB4_DEST_BASE_ENA(1) |
 194                                 S_0085F0_CB5_DEST_BASE_ENA(1) |
 195                                 S_0085F0_CB6_DEST_BASE_ENA(1) |
 196                                 S_0085F0_CB7_DEST_BASE_ENA(1) |
 197                                 S_0085F0_SMX_ACTION_ENA(1);
 198                 if (rctx->b.chip_class >= EVERGREEN)
 199                         cp_coher_cntl |= S_0085F0_CB8_DEST_BASE_ENA(1) |
 200                                         S_0085F0_CB9_DEST_BASE_ENA(1) |
 201                                         S_0085F0_CB10_DEST_BASE_ENA(1) |
 202                                         S_0085F0_CB11_DEST_BASE_ENA(1);
 203         }
 204
 205         if (rctx->b.flags & R600_CONTEXT_STREAMOUT_FLUSH) {
 206                 cp_coher_cntl |= S_0085F0_SO0_DEST_BASE_ENA(1) |
 207                                 S_0085F0_SO1_DEST_BASE_ENA(1) |
 208                                 S_0085F0_SO2_DEST_BASE_ENA(1) |
 209                                 S_0085F0_SO3_DEST_BASE_ENA(1) |
 210                                 S_0085F0_SMX_ACTION_ENA(1);
 211         }
 212
 213         if (cp_coher_cntl) {
 214                 cs->buf[cs->cdw++] = PKT3(PKT3_SURFACE_SYNC, 3, 0);
 215                 cs->buf[cs->cdw++] = cp_coher_cntl;   /* CP_COHER_CNTL */
 216                 cs->buf[cs->cdw++] = 0xffffffff;      /* CP_COHER_SIZE */
 217                 cs->buf[cs->cdw++] = 0;               /* CP_COHER_BASE */
 218                 cs->buf[cs->cdw++] = 0x0000000A;      /* POLL_INTERVAL */
 219         }
 220
 221         if (wait_until) {
 222                 /* Use of WAIT_UNTIL is deprecated on Cayman+ */
 223                 if (rctx->b.family < CHIP_CAYMAN) {
 224                         /* wait for things to settle */
 225                         r600_write_config_reg(cs, R_008040_WAIT_UNTIL, wait_until);
 226                 }
 227         }
 228
 229         /* everything is properly flushed */
 230         rctx->b.flags = 0;
 231 }
 232
 233 void r600_context_flush(struct r600_context *ctx, unsigned flags)
 234 {
 235         struct radeon_winsys_cs *cs = ctx->b.rings.gfx.cs;
 236
 237         ctx->b.nontimer_queries_suspended = false;
 238         ctx->b.streamout.suspended = false;
 239
 240         /* suspend queries */
 241         if (ctx->b.num_cs_dw_nontimer_queries_suspend) {
 242                 r600_suspend_nontimer_queries(&ctx->b);
 243                 ctx->b.nontimer_queries_suspended = true;
 244         }
 245
 246         if (ctx->b.streamout.begin_emitted) {
 247                 r600_emit_streamout_end(&ctx->b);
 248                 ctx->b.streamout.suspended = true;
 249         }
 250
 251         /* flush the framebuffer cache */
 252         ctx->b.flags |= R600_CONTEXT_FLUSH_AND_INV |
 253                       R600_CONTEXT_FLUSH_AND_INV_CB |
 254                       R600_CONTEXT_FLUSH_AND_INV_DB |
 255                       R600_CONTEXT_FLUSH_AND_INV_CB_META |
 256                       R600_CONTEXT_FLUSH_AND_INV_DB_META |
 257                       R600_CONTEXT_WAIT_3D_IDLE |
 258                       R600_CONTEXT_WAIT_CP_DMA_IDLE;
 259
 260         r600_flush_emit(ctx);
 261
 262         /* old kernels and userspace don't set SX_MISC, so we must reset it to 0 here */
 263         if (ctx->b.chip_class <= R700) {
 264                 r600_write_context_reg(cs, R_028350_SX_MISC, 0);
 265         }
 266
 267         /* force to keep tiling flags */
 268         if (ctx->keep_tiling_flags) {
 269                 flags |= RADEON_FLUSH_KEEP_TILING_FLAGS;
 270         }
 271
 272         /* Flush the CS. */
 273         ctx->b.ws->cs_flush(ctx->b.rings.gfx.cs, flags, ctx->screen->cs_count++);
 274 }
 275
 276 void r600_begin_new_cs(struct r600_context *ctx)
 277 {
 278         unsigned shader;
 279
 280         ctx->b.flags = 0;
 281         ctx->b.gtt = 0;
 282         ctx->b.vram = 0;
 283
 284         /* Begin a new CS. */
 285         r600_emit_command_buffer(ctx->b.rings.gfx.cs, &ctx->start_cs_cmd);
 286
 287         /* Re-emit states. */
 288         ctx->alphatest_state.atom.dirty = true;
 289         ctx->blend_color.atom.dirty = true;
 290         ctx->cb_misc_state.atom.dirty = true;
 291         ctx->clip_misc_state.atom.dirty = true;
 292         ctx->clip_state.atom.dirty = true;
 293         ctx->db_misc_state.atom.dirty = true;
 294         ctx->db_state.atom.dirty = true;
 295         ctx->framebuffer.atom.dirty = true;
 296         ctx->pixel_shader.atom.dirty = true;
 297         ctx->poly_offset_state.atom.dirty = true;
 298         ctx->vgt_state.atom.dirty = true;
 299         ctx->sample_mask.atom.dirty = true;
 300         ctx->scissor.atom.dirty = true;
 301         ctx->config_state.atom.dirty = true;
 302         ctx->stencil_ref.atom.dirty = true;
 303         ctx->vertex_fetch_shader.atom.dirty = true;
 304         ctx->vertex_shader.atom.dirty = true;
 305         ctx->viewport.atom.dirty = true;
 306
 307         if (ctx->blend_state.cso)
 308                 ctx->blend_state.atom.dirty = true;
 309         if (ctx->dsa_state.cso)
 310                 ctx->dsa_state.atom.dirty = true;
 311         if (ctx->rasterizer_state.cso)
 312                 ctx->rasterizer_state.atom.dirty = true;
 313
 314         if (ctx->b.chip_class <= R700) {
 315                 ctx->seamless_cube_map.atom.dirty = true;
 316         }
 317
 318         ctx->vertex_buffer_state.dirty_mask = ctx->vertex_buffer_state.enabled_mask;
 319         r600_vertex_buffers_dirty(ctx);
 320
 321         /* Re-emit shader resources. */
 322         for (shader = 0; shader < PIPE_SHADER_TYPES; shader++) {
 323                 struct r600_constbuf_state *constbuf = &ctx->constbuf_state[shader];
 324                 struct r600_textures_info *samplers = &ctx->samplers[shader];
 325
 326                 constbuf->dirty_mask = constbuf->enabled_mask;
 327                 samplers->views.dirty_mask = samplers->views.enabled_mask;
 328                 samplers->states.dirty_mask = samplers->states.enabled_mask;
 329
 330                 r600_constant_buffers_dirty(ctx, constbuf);
 331                 r600_sampler_views_dirty(ctx, &samplers->views);
 332                 r600_sampler_states_dirty(ctx, &samplers->states);
 333         }
 334
 335         if (ctx->b.streamout.suspended) {
 336                 ctx->b.streamout.append_bitmask = ctx->b.streamout.enabled_mask;
 337                 r600_streamout_buffers_dirty(&ctx->b);
 338         }
 339
 340         /* resume queries */
 341         if (ctx->b.nontimer_queries_suspended) {
 342                 r600_resume_nontimer_queries(&ctx->b);
 343         }
 344
 345         /* Re-emit the draw state. */
 346         ctx->last_primitive_type = -1;
 347         ctx->last_start_instance = -1;
 348
 349         ctx->initial_gfx_cs_size = ctx->b.rings.gfx.cs->cdw;
 350 }
 351
 352 /* The max number of bytes to copy per packet. */
 353 #define CP_DMA_MAX_BYTE_COUNT ((1 << 21) - 8)
 354
 355 void r600_cp_dma_copy_buffer(struct r600_context *rctx,
 356                              struct pipe_resource *dst, uint64_t dst_offset,
 357                              struct pipe_resource *src, uint64_t src_offset,
 358                              unsigned size)
 359 {
 360         struct radeon_winsys_cs *cs = rctx->b.rings.gfx.cs;
 361
 362         assert(size);
 363         assert(rctx->screen->b.has_cp_dma);
 364
 365         /* Mark the buffer range of destination as valid (initialized),
 366          * so that transfer_map knows it should wait for the GPU when mapping
 367          * that range. */
 368         util_range_add(&r600_resource(dst)->valid_buffer_range, dst_offset,
 369                        dst_offset + size);
 370
 371         dst_offset += r600_resource_va(&rctx->screen->b.b, dst);
 372         src_offset += r600_resource_va(&rctx->screen->b.b, src);
 373
 374         /* Flush the caches where the resources are bound. */
 375         rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE |
 376                          R600_CONTEXT_INV_VERTEX_CACHE |
 377                          R600_CONTEXT_INV_TEX_CACHE |
 378                          R600_CONTEXT_FLUSH_AND_INV |
 379                          R600_CONTEXT_FLUSH_AND_INV_CB |
 380                          R600_CONTEXT_FLUSH_AND_INV_DB |
 381                          R600_CONTEXT_FLUSH_AND_INV_CB_META |
 382                          R600_CONTEXT_FLUSH_AND_INV_DB_META |
 383                          R600_CONTEXT_STREAMOUT_FLUSH |
 384                          R600_CONTEXT_WAIT_3D_IDLE;
 385
 386         /* There are differences between R700 and EG in CP DMA,
 387          * but we only use the common bits here. */
 388         while (size) {
 389                 unsigned sync = 0;
 390                 unsigned byte_count = MIN2(size, CP_DMA_MAX_BYTE_COUNT);
 391                 unsigned src_reloc, dst_reloc;
 392
 393                 r600_need_cs_space(rctx, 10 + (rctx->b.flags ? R600_MAX_FLUSH_CS_DWORDS : 0), FALSE);
 394
 395                 /* Flush the caches for the first copy only. */
 396                 if (rctx->b.flags) {
 397                         r600_flush_emit(rctx);
 398                 }
 399
 400                 /* Do the synchronization after the last copy, so that all data is written to memory. */
 401                 if (size == byte_count) {
 402                         sync = PKT3_CP_DMA_CP_SYNC;
 403                 }
 404
 405                 /* This must be done after r600_need_cs_space. */
 406                 src_reloc = r600_context_bo_reloc(&rctx->b, &rctx->b.rings.gfx, (struct r600_resource*)src, RADEON_USAGE_READ);
 407                 dst_reloc = r600_context_bo_reloc(&rctx->b, &rctx->b.rings.gfx, (struct r600_resource*)dst, RADEON_USAGE_WRITE);
 408
 409                 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, 0));
 410                 radeon_emit(cs, src_offset);    /* SRC_ADDR_LO [31:0] */
 411                 radeon_emit(cs, sync | ((src_offset >> 32) & 0xff));            /* CP_SYNC [31] | SRC_ADDR_HI [7:0] */
 412                 radeon_emit(cs, dst_offset);    /* DST_ADDR_LO [31:0] */
 413                 radeon_emit(cs, (dst_offset >> 32) & 0xff);             /* DST_ADDR_HI [7:0] */
 414                 radeon_emit(cs, byte_count);    /* COMMAND [29:22] | BYTE_COUNT [20:0] */
 415
 416                 radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
 417                 radeon_emit(cs, src_reloc);
 418                 radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
 419                 radeon_emit(cs, dst_reloc);
 420
 421                 size -= byte_count;
 422                 src_offset += byte_count;
 423                 dst_offset += byte_count;
 424         }
 425
 426         /* Invalidate the read caches. */
 427         rctx->b.flags |= R600_CONTEXT_INV_CONST_CACHE |
 428                          R600_CONTEXT_INV_VERTEX_CACHE |
 429                          R600_CONTEXT_INV_TEX_CACHE;
 430 }
 431
 432 void r600_need_dma_space(struct r600_context *ctx, unsigned num_dw)
 433 {
 434         /* The number of dwords we already used in the DMA so far. */
 435         num_dw += ctx->b.rings.dma.cs->cdw;
 436         /* Flush if there's not enough space. */
 437         if (num_dw > RADEON_MAX_CMDBUF_DWORDS) {
 438                 ctx->b.rings.dma.flush(ctx, RADEON_FLUSH_ASYNC);
 439         }
 440 }
 441
 442 void r600_dma_copy(struct r600_context *rctx,
 443                 struct pipe_resource *dst,
 444                 struct pipe_resource *src,
 445                 uint64_t dst_offset,
 446                 uint64_t src_offset,
 447                 uint64_t size)
 448 {
 449         struct radeon_winsys_cs *cs = rctx->b.rings.dma.cs;
 450         unsigned i, ncopy, csize, shift;
 451         struct r600_resource *rdst = (struct r600_resource*)dst;
 452         struct r600_resource *rsrc = (struct r600_resource*)src;
 453
 454         /* Mark the buffer range of destination as valid (initialized),
 455          * so that transfer_map knows it should wait for the GPU when mapping
 456          * that range. */
 457         util_range_add(&rdst->valid_buffer_range, dst_offset,
 458                        dst_offset + size);
 459
 460         /* make sure that the dma ring is only one active */
 461         rctx->b.rings.gfx.flush(rctx, RADEON_FLUSH_ASYNC);
 462
 463         size >>= 2;
 464         shift = 2;
 465         ncopy = (size / 0xffff) + !!(size % 0xffff);
 466
 467         r600_need_dma_space(rctx, ncopy * 5);
 468         for (i = 0; i < ncopy; i++) {
 469                 csize = size < 0xffff ? size : 0xffff;
 470                 /* emit reloc before writting cs so that cs is always in consistent state */
 471                 r600_context_bo_reloc(&rctx->b, &rctx->b.rings.dma, rsrc, RADEON_USAGE_READ);
 472                 r600_context_bo_reloc(&rctx->b, &rctx->b.rings.dma, rdst, RADEON_USAGE_WRITE);
 473                 cs->buf[cs->cdw++] = DMA_PACKET(DMA_PACKET_COPY, 0, 0, csize);
 474                 cs->buf[cs->cdw++] = dst_offset & 0xfffffffc;
 475                 cs->buf[cs->cdw++] = src_offset & 0xfffffffc;
 476                 cs->buf[cs->cdw++] = (dst_offset >> 32UL) & 0xff;
 477                 cs->buf[cs->cdw++] = (src_offset >> 32UL) & 0xff;
 478                 dst_offset += csize << shift;
 479                 src_offset += csize << shift;
 480                 size -= csize;
 481         }
 482 }