2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
24 #include "tu_private.h"
27 #include "adreno_common.xml.h"
28 #include "adreno_pm4.xml.h"
30 #include "vk_format.h"
36 * - image -> image copies
38 * - compressed image formats (need to divide offset/extent)
42 blit_control(enum a6xx_color_fmt fmt
)
44 unsigned blit_cntl
= 0xf00000;
45 blit_cntl
|= A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(fmt
);
46 blit_cntl
|= A6XX_RB_2D_BLIT_CNTL_IFMT(tu6_rb_fmt_to_ifmt(fmt
));
50 static uint32_t tu6_sp_2d_src_format(VkFormat format
)
52 const struct vk_format_description
*desc
= vk_format_description(format
);
53 uint32_t reg
= 0xf000 | A6XX_SP_2D_SRC_FORMAT_COLOR_FORMAT(tu6_get_native_format(format
)->rb
);
55 int channel
= vk_format_get_first_non_void_channel(format
);
57 /* TODO special format. */
60 if (desc
->channel
[channel
].normalized
) {
61 if (desc
->channel
[channel
].type
== VK_FORMAT_TYPE_SIGNED
)
62 reg
|= A6XX_SP_2D_SRC_FORMAT_SINT
;
63 reg
|= A6XX_SP_2D_SRC_FORMAT_NORM
;
64 } else if (desc
->channel
[channel
].pure_integer
) {
65 if (desc
->channel
[channel
].type
== VK_FORMAT_TYPE_SIGNED
)
66 reg
|= A6XX_SP_2D_SRC_FORMAT_SINT
;
68 reg
|= A6XX_SP_2D_SRC_FORMAT_UINT
;
74 tu_dma_prepare(struct tu_cmd_buffer
*cmdbuf
)
76 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 10);
78 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_EVENT_WRITE
, 1);
79 tu_cs_emit(&cmdbuf
->cs
, PC_CCU_INVALIDATE_COLOR
);
81 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_EVENT_WRITE
, 1);
82 tu_cs_emit(&cmdbuf
->cs
, LRZ_FLUSH
);
84 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_SKIP_IB2_ENABLE_GLOBAL
, 1);
85 tu_cs_emit(&cmdbuf
->cs
, 0x0);
87 tu_cs_emit_wfi(&cmdbuf
->cs
);
89 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_CCU_CNTL
, 1);
90 tu_cs_emit(&cmdbuf
->cs
, 0x10000000);
94 tu_copy_buffer(struct tu_cmd_buffer
*cmdbuf
,
101 const unsigned max_size_per_iter
= 0x4000 - 0x40;
102 const unsigned max_iterations
=
103 (size
+ max_size_per_iter
) / max_size_per_iter
;
105 tu_bo_list_add(&cmdbuf
->bo_list
, src_bo
, MSM_SUBMIT_BO_READ
);
106 tu_bo_list_add(&cmdbuf
->bo_list
, dst_bo
, MSM_SUBMIT_BO_WRITE
);
108 tu_dma_prepare(cmdbuf
);
110 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 21 + 48 * max_iterations
);
112 /* buffer copy setup */
113 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_SET_MARKER
, 1);
114 tu_cs_emit(&cmdbuf
->cs
, A2XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE
));
116 const uint32_t blit_cntl
= blit_control(RB6_R8_UNORM
) | 0x20000000;
118 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_2D_BLIT_CNTL
, 1);
119 tu_cs_emit(&cmdbuf
->cs
, blit_cntl
);
121 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_BLIT_CNTL
, 1);
122 tu_cs_emit(&cmdbuf
->cs
, blit_cntl
);
125 uint64_t src_va
= src_bo
->iova
+ src_offset
;
126 uint64_t dst_va
= dst_bo
->iova
+ dst_offset
;
128 unsigned src_shift
= src_va
& 0x3f;
129 unsigned dst_shift
= dst_va
& 0x3f;
130 unsigned max_shift
= MAX2(src_shift
, dst_shift
);
135 uint32_t size_todo
= MIN2(0x4000 - max_shift
, size
);
136 unsigned pitch
= (size_todo
+ max_shift
+ 63) & ~63;
141 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_SP_PS_2D_SRC_INFO
, 13);
142 tu_cs_emit(&cmdbuf
->cs
,
143 A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(RB6_R8_UNORM
) |
144 A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(TILE6_LINEAR
) |
145 A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(WZYX
) | 0x500000);
146 tu_cs_emit(&cmdbuf
->cs
,
147 A6XX_SP_PS_2D_SRC_SIZE_WIDTH(src_shift
+ size_todo
) |
148 A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(1)); /* SP_PS_2D_SRC_SIZE */
149 tu_cs_emit_qw(&cmdbuf
->cs
, src_va
);
150 tu_cs_emit(&cmdbuf
->cs
, A6XX_SP_PS_2D_SRC_PITCH_PITCH(pitch
));
152 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
153 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
154 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
155 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
156 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
158 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
159 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
160 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
165 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_2D_DST_INFO
, 9);
166 tu_cs_emit(&cmdbuf
->cs
, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(RB6_R8_UNORM
) |
167 A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR
) |
168 A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX
));
169 tu_cs_emit_qw(&cmdbuf
->cs
, dst_va
);
171 tu_cs_emit(&cmdbuf
->cs
, A6XX_RB_2D_DST_SIZE_PITCH(pitch
));
172 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
173 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
174 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
175 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
176 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
181 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_SRC_TL_X
, 4);
182 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_TL_X_X(src_shift
));
183 tu_cs_emit(&cmdbuf
->cs
,
184 A6XX_GRAS_2D_SRC_BR_X_X(src_shift
+ size_todo
- 1));
185 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_TL_Y_Y(0));
186 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_BR_Y_Y(0));
188 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_DST_TL
, 2);
189 tu_cs_emit(&cmdbuf
->cs
,
190 A6XX_GRAS_2D_DST_TL_X(dst_shift
) | A6XX_GRAS_2D_DST_TL_Y(0));
191 tu_cs_emit(&cmdbuf
->cs
,
192 A6XX_GRAS_2D_DST_BR_X(dst_shift
+ size_todo
- 1) |
193 A6XX_GRAS_2D_DST_BR_Y(0));
195 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_EVENT_WRITE
, 1);
196 tu_cs_emit(&cmdbuf
->cs
, 0x3f);
197 tu_cs_emit_wfi(&cmdbuf
->cs
);
199 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8C01
, 1);
200 tu_cs_emit(&cmdbuf
->cs
, 0);
202 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_SP_2D_SRC_FORMAT
, 1);
203 tu_cs_emit(&cmdbuf
->cs
, 0xf180);
205 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8E04
, 1);
206 tu_cs_emit(&cmdbuf
->cs
, 0x01000000);
208 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_BLIT
, 1);
209 tu_cs_emit(&cmdbuf
->cs
, CP_BLIT_0_OP(BLIT_OP_SCALE
));
211 tu_cs_emit_wfi(&cmdbuf
->cs
);
213 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8E04
, 1);
214 tu_cs_emit(&cmdbuf
->cs
, 0);
216 src_offset
+= size_todo
;
217 dst_offset
+= size_todo
;
221 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, 0x1d, true);
222 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, FACENESS_FLUSH
, true);
223 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, CACHE_FLUSH_TS
, true);
227 tu_copy_buffer_to_image_step(struct tu_cmd_buffer
*cmdbuf
,
228 struct tu_buffer
*src_buffer
,
229 struct tu_image
*dst_image
,
230 const VkBufferImageCopy
*copy_info
,
235 const enum a6xx_color_fmt rb_fmt
= tu6_get_native_format(format
)->rb
;
237 uint64_t dst_va
= dst_image
->bo
->iova
+ dst_image
->bo_offset
+ dst_image
->layer_size
* layer
+ dst_image
->levels
[copy_info
->imageSubresource
.mipLevel
].offset
;
238 unsigned dst_pitch
= dst_image
->levels
[copy_info
->imageSubresource
.mipLevel
].pitch
*
239 vk_format_get_blocksize(format
);
242 unsigned src_offset
= 0;
243 if (copy_info
->imageExtent
.height
== 1) {
244 /* Can't find this in the spec, but not having it is sort of insane? */
245 assert(src_va
% vk_format_get_blocksize(format
) == 0);
247 src_offset
= (src_va
& 63) / vk_format_get_blocksize(format
);
250 src_pitch
= align((src_offset
+ copy_info
->imageExtent
.width
) * vk_format_get_blocksize(format
), 64);
252 unsigned src_pixel_stride
= copy_info
->bufferRowLength
253 ? copy_info
->bufferRowLength
254 : copy_info
->imageExtent
.width
;
255 src_pitch
= src_pixel_stride
* vk_format_get_blocksize(format
);
256 assert(!(src_pitch
& 63));
257 assert(!(src_va
& 63));
260 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 48);
265 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_SP_PS_2D_SRC_INFO
, 13);
266 tu_cs_emit(&cmdbuf
->cs
, A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(rb_fmt
) |
267 A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(TILE6_LINEAR
) |
268 A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(WZYX
) |
270 tu_cs_emit(&cmdbuf
->cs
,
271 A6XX_SP_PS_2D_SRC_SIZE_WIDTH(src_offset
+ copy_info
->imageExtent
.width
) |
272 A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(
273 copy_info
->imageExtent
.height
)); /* SP_PS_2D_SRC_SIZE */
274 tu_cs_emit_qw(&cmdbuf
->cs
, src_va
);
275 tu_cs_emit(&cmdbuf
->cs
, A6XX_SP_PS_2D_SRC_PITCH_PITCH(src_pitch
));
277 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
278 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
279 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
280 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
281 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
283 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
284 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
285 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
290 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_2D_DST_INFO
, 9);
291 tu_cs_emit(&cmdbuf
->cs
,
292 A6XX_RB_2D_DST_INFO_COLOR_FORMAT(rb_fmt
) |
293 A6XX_RB_2D_DST_INFO_TILE_MODE(dst_image
->tile_mode
) |
294 A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX
));
295 tu_cs_emit_qw(&cmdbuf
->cs
, dst_va
);
296 tu_cs_emit(&cmdbuf
->cs
, A6XX_RB_2D_DST_SIZE_PITCH(dst_pitch
));
297 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
298 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
299 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
300 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
301 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
303 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_SRC_TL_X
, 4);
304 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_TL_X_X(src_offset
));
305 tu_cs_emit(&cmdbuf
->cs
,
306 A6XX_GRAS_2D_SRC_BR_X_X(src_offset
+ copy_info
->imageExtent
.width
- 1));
307 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_TL_Y_Y(0));
308 tu_cs_emit(&cmdbuf
->cs
,
309 A6XX_GRAS_2D_SRC_BR_Y_Y(copy_info
->imageExtent
.height
- 1));
311 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_DST_TL
, 2);
312 tu_cs_emit(&cmdbuf
->cs
,
313 A6XX_GRAS_2D_DST_TL_X(copy_info
->imageOffset
.x
) |
314 A6XX_GRAS_2D_DST_TL_Y(copy_info
->imageOffset
.y
));
315 tu_cs_emit(&cmdbuf
->cs
,
316 A6XX_GRAS_2D_DST_BR_X(copy_info
->imageOffset
.x
+
317 copy_info
->imageExtent
.width
- 1) |
318 A6XX_GRAS_2D_DST_BR_Y(copy_info
->imageOffset
.y
+
319 copy_info
->imageExtent
.height
- 1));
321 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_EVENT_WRITE
, 1);
322 tu_cs_emit(&cmdbuf
->cs
, 0x3f);
323 tu_cs_emit_wfi(&cmdbuf
->cs
);
325 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8C01
, 1);
326 tu_cs_emit(&cmdbuf
->cs
, 0);
328 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_SP_2D_SRC_FORMAT
, 1);
329 tu_cs_emit(&cmdbuf
->cs
, tu6_sp_2d_src_format(format
));
331 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8E04
, 1);
332 tu_cs_emit(&cmdbuf
->cs
, 0x01000000);
334 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_BLIT
, 1);
335 tu_cs_emit(&cmdbuf
->cs
, CP_BLIT_0_OP(BLIT_OP_SCALE
));
337 tu_cs_emit_wfi(&cmdbuf
->cs
);
339 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8E04
, 1);
340 tu_cs_emit(&cmdbuf
->cs
, 0);
344 tu_copy_buffer_to_image(struct tu_cmd_buffer
*cmdbuf
,
345 struct tu_buffer
*src_buffer
,
346 struct tu_image
*dst_image
,
347 const VkBufferImageCopy
*copy_info
)
349 tu_bo_list_add(&cmdbuf
->bo_list
, src_buffer
->bo
, MSM_SUBMIT_BO_READ
);
350 tu_bo_list_add(&cmdbuf
->bo_list
, dst_image
->bo
, MSM_SUBMIT_BO_WRITE
);
353 tu_dma_prepare(cmdbuf
);
355 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 6);
357 /* buffer copy setup */
358 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_SET_MARKER
, 1);
359 tu_cs_emit(&cmdbuf
->cs
, A2XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE
));
361 VkFormat format
= dst_image
->vk_format
;
362 const enum a6xx_color_fmt rb_fmt
= tu6_get_native_format(format
)->rb
;
364 const uint32_t blit_cntl
= blit_control(rb_fmt
) | 0x20000000;
366 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_2D_BLIT_CNTL
, 1);
367 tu_cs_emit(&cmdbuf
->cs
, blit_cntl
);
369 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_BLIT_CNTL
, 1);
370 tu_cs_emit(&cmdbuf
->cs
, blit_cntl
);
372 unsigned src_pixel_stride
= copy_info
->bufferRowLength
373 ? copy_info
->bufferRowLength
374 : copy_info
->imageExtent
.width
;
375 unsigned cpp
= vk_format_get_blocksize(format
);
376 unsigned src_pitch
= src_pixel_stride
* cpp
;
378 for (unsigned layer_offset
= 0; layer_offset
< copy_info
->imageSubresource
.layerCount
; ++layer_offset
) {
379 unsigned layer
= copy_info
->imageSubresource
.baseArrayLayer
+ layer_offset
;
380 uint64_t src_va
= src_buffer
->bo
->iova
+ src_buffer
->bo_offset
+ copy_info
->bufferOffset
+ layer_offset
* copy_info
->bufferImageHeight
* src_pitch
;
382 if ((src_pitch
& 63) || (src_va
& 63)) {
383 /* Do a per line copy */
384 VkBufferImageCopy line_copy_info
= *copy_info
;
385 line_copy_info
.imageExtent
.height
= 1;
386 for (unsigned r
= 0; r
< copy_info
->imageExtent
.height
; ++r
) {
388 * if src_va is not aligned the line copy will need to adjust. Give it
391 unsigned max_width
= 16384 - (src_va
& 0x3f) ? 64 : 0;
392 line_copy_info
.imageOffset
.x
= copy_info
->imageOffset
.x
;
393 line_copy_info
.imageExtent
.width
= copy_info
->imageExtent
.width
;
395 for (unsigned c
= 0; c
< copy_info
->imageExtent
.width
; c
+= max_width
) {
396 tu_copy_buffer_to_image_step(cmdbuf
, src_buffer
, dst_image
, &line_copy_info
, format
, layer
, src_va
+ c
* cpp
);
398 line_copy_info
.imageOffset
.x
+= max_width
;
399 line_copy_info
.imageExtent
.width
-= max_width
;
402 line_copy_info
.imageOffset
.y
++;
406 tu_copy_buffer_to_image_step(cmdbuf
, src_buffer
, dst_image
, copy_info
, format
, layer
, src_va
);
410 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 15);
412 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, 0x1d, true);
413 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, FACENESS_FLUSH
, true);
414 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, CACHE_FLUSH_TS
, true);
418 tu_copy_image_to_buffer_step(struct tu_cmd_buffer
*cmdbuf
,
419 struct tu_image
*src_image
,
420 struct tu_buffer
*dst_buffer
,
421 const VkBufferImageCopy
*copy_info
,
426 const enum a6xx_color_fmt rb_fmt
= tu6_get_native_format(format
)->rb
;
428 uint64_t src_va
= src_image
->bo
->iova
+ src_image
->bo_offset
+ src_image
->layer_size
* layer
+ src_image
->levels
[copy_info
->imageSubresource
.mipLevel
].offset
;
429 unsigned src_pitch
= src_image
->levels
[copy_info
->imageSubresource
.mipLevel
].pitch
*
430 vk_format_get_blocksize(format
);
433 unsigned dst_offset
= 0;
434 if (copy_info
->imageExtent
.height
== 1) {
435 /* Can't find this in the spec, but not having it is sort of insane? */
436 assert(dst_va
% vk_format_get_blocksize(format
) == 0);
438 dst_offset
= (dst_va
& 63) / vk_format_get_blocksize(format
);
441 dst_pitch
= align((dst_offset
+ copy_info
->imageExtent
.width
) * vk_format_get_blocksize(format
), 64);
443 unsigned dst_pixel_stride
= copy_info
->bufferRowLength
444 ? copy_info
->bufferRowLength
445 : copy_info
->imageExtent
.width
;
446 dst_pitch
= dst_pixel_stride
* vk_format_get_blocksize(format
);
447 assert(!(dst_pitch
& 63));
448 assert(!(dst_va
& 63));
452 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 48);
457 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_SP_PS_2D_SRC_INFO
, 13);
458 tu_cs_emit(&cmdbuf
->cs
,
459 A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(rb_fmt
) |
460 A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(src_image
->tile_mode
) |
461 A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(WZYX
) | 0x500000);
462 tu_cs_emit(&cmdbuf
->cs
,
463 A6XX_SP_PS_2D_SRC_SIZE_WIDTH(src_image
->extent
.width
) |
464 A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(
465 src_image
->extent
.height
)); /* SP_PS_2D_SRC_SIZE */
466 tu_cs_emit_qw(&cmdbuf
->cs
, src_va
);
467 tu_cs_emit(&cmdbuf
->cs
, A6XX_SP_PS_2D_SRC_PITCH_PITCH(src_pitch
));
469 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
470 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
471 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
472 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
473 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
475 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
476 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
477 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
482 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_2D_DST_INFO
, 9);
483 tu_cs_emit(&cmdbuf
->cs
, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(rb_fmt
) |
484 A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR
) |
485 A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX
));
486 tu_cs_emit_qw(&cmdbuf
->cs
, dst_va
);
487 tu_cs_emit(&cmdbuf
->cs
, A6XX_RB_2D_DST_SIZE_PITCH(dst_pitch
));
488 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
489 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
490 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
491 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
492 tu_cs_emit(&cmdbuf
->cs
, 0x00000000);
494 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_SRC_TL_X
, 4);
495 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_TL_X_X(copy_info
->imageOffset
.x
));
496 tu_cs_emit(&cmdbuf
->cs
,
497 A6XX_GRAS_2D_SRC_BR_X_X(copy_info
->imageOffset
.x
+
498 copy_info
->imageExtent
.width
- 1));
499 tu_cs_emit(&cmdbuf
->cs
, A6XX_GRAS_2D_SRC_TL_Y_Y(copy_info
->imageOffset
.y
));
500 tu_cs_emit(&cmdbuf
->cs
,
501 A6XX_GRAS_2D_SRC_BR_Y_Y(copy_info
->imageOffset
.y
+
502 copy_info
->imageExtent
.height
- 1));
504 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_DST_TL
, 2);
505 tu_cs_emit(&cmdbuf
->cs
,
506 A6XX_GRAS_2D_DST_TL_X(dst_offset
) | A6XX_GRAS_2D_DST_TL_Y(0));
507 tu_cs_emit(&cmdbuf
->cs
,
508 A6XX_GRAS_2D_DST_BR_X(dst_offset
+ copy_info
->imageExtent
.width
- 1) |
509 A6XX_GRAS_2D_DST_BR_Y(copy_info
->imageExtent
.height
- 1));
511 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_EVENT_WRITE
, 1);
512 tu_cs_emit(&cmdbuf
->cs
, 0x3f);
513 tu_cs_emit_wfi(&cmdbuf
->cs
);
515 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8C01
, 1);
516 tu_cs_emit(&cmdbuf
->cs
, 0);
518 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_SP_2D_SRC_FORMAT
, 1);
519 tu_cs_emit(&cmdbuf
->cs
, tu6_sp_2d_src_format(format
));
521 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8E04
, 1);
522 tu_cs_emit(&cmdbuf
->cs
, 0x01000000);
524 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_BLIT
, 1);
525 tu_cs_emit(&cmdbuf
->cs
, CP_BLIT_0_OP(BLIT_OP_SCALE
));
527 tu_cs_emit_wfi(&cmdbuf
->cs
);
529 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_UNKNOWN_8E04
, 1);
530 tu_cs_emit(&cmdbuf
->cs
, 0);
534 tu_copy_image_to_buffer(struct tu_cmd_buffer
*cmdbuf
,
535 struct tu_image
*src_image
,
536 struct tu_buffer
*dst_buffer
,
537 const VkBufferImageCopy
*copy_info
)
539 tu_bo_list_add(&cmdbuf
->bo_list
, src_image
->bo
, MSM_SUBMIT_BO_READ
);
540 tu_bo_list_add(&cmdbuf
->bo_list
, dst_buffer
->bo
, MSM_SUBMIT_BO_WRITE
);
543 tu_dma_prepare(cmdbuf
);
545 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 6);
547 /* buffer copy setup */
548 tu_cs_emit_pkt7(&cmdbuf
->cs
, CP_SET_MARKER
, 1);
549 tu_cs_emit(&cmdbuf
->cs
, A2XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE
));
551 VkFormat format
= src_image
->vk_format
;
552 const enum a6xx_color_fmt rb_fmt
= tu6_get_native_format(format
)->rb
;
554 unsigned dst_pixel_stride
= copy_info
->bufferRowLength
555 ? copy_info
->bufferRowLength
556 : copy_info
->imageExtent
.width
;
557 unsigned cpp
= vk_format_get_blocksize(format
);
558 unsigned dst_pitch
= dst_pixel_stride
* cpp
;
561 const uint32_t blit_cntl
= blit_control(rb_fmt
) | 0x20000000;
563 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_RB_2D_BLIT_CNTL
, 1);
564 tu_cs_emit(&cmdbuf
->cs
, blit_cntl
);
566 tu_cs_emit_pkt4(&cmdbuf
->cs
, REG_A6XX_GRAS_2D_BLIT_CNTL
, 1);
567 tu_cs_emit(&cmdbuf
->cs
, blit_cntl
);
569 for (unsigned layer_offset
= 0; layer_offset
< copy_info
->imageSubresource
.layerCount
; ++layer_offset
) {
570 unsigned layer
= copy_info
->imageSubresource
.baseArrayLayer
+ layer_offset
;
571 uint64_t dst_va
= dst_buffer
->bo
->iova
+ dst_buffer
->bo_offset
+ copy_info
->bufferOffset
+ layer_offset
* copy_info
->bufferImageHeight
* dst_pitch
;
573 if ((dst_pitch
& 63) || (dst_va
& 63)) {
574 /* Do a per line copy */
575 VkBufferImageCopy line_copy_info
= *copy_info
;
576 line_copy_info
.imageExtent
.height
= 1;
577 for (unsigned r
= 0; r
< copy_info
->imageExtent
.height
; ++r
) {
579 * if dst_va is not aligned the line copy will need to adjust. Give it
582 unsigned max_width
= 16384 - (dst_va
& 0x3f) ? 64 : 0;
583 line_copy_info
.imageOffset
.x
= copy_info
->imageOffset
.x
;
584 line_copy_info
.imageExtent
.width
= copy_info
->imageExtent
.width
;
586 for (unsigned c
= 0; c
< copy_info
->imageExtent
.width
; c
+= max_width
) {
587 tu_copy_image_to_buffer_step(cmdbuf
, src_image
, dst_buffer
, &line_copy_info
, format
, layer
, dst_va
+ c
* cpp
);
589 line_copy_info
.imageOffset
.x
+= max_width
;
590 line_copy_info
.imageExtent
.width
-= max_width
;
593 line_copy_info
.imageOffset
.y
++;
597 tu_copy_image_to_buffer_step(cmdbuf
, src_image
, dst_buffer
, copy_info
, format
, layer
, dst_va
);
601 tu_cs_reserve_space(cmdbuf
->device
, &cmdbuf
->cs
, 15);
603 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, 0x1d, true);
604 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, FACENESS_FLUSH
, true);
605 tu6_emit_event_write(cmdbuf
, &cmdbuf
->cs
, CACHE_FLUSH_TS
, true);
609 tu_CmdCopyBuffer(VkCommandBuffer commandBuffer
,
612 uint32_t regionCount
,
613 const VkBufferCopy
*pRegions
)
615 TU_FROM_HANDLE(tu_cmd_buffer
, cmdbuf
, commandBuffer
);
616 TU_FROM_HANDLE(tu_buffer
, src_buffer
, srcBuffer
);
617 TU_FROM_HANDLE(tu_buffer
, dst_buffer
, destBuffer
);
619 for (unsigned i
= 0; i
< regionCount
; ++i
) {
620 uint64_t src_offset
= src_buffer
->bo_offset
+ pRegions
[i
].srcOffset
;
621 uint64_t dst_offset
= dst_buffer
->bo_offset
+ pRegions
[i
].dstOffset
;
623 tu_copy_buffer(cmdbuf
, src_buffer
->bo
, src_offset
, dst_buffer
->bo
,
624 dst_offset
, pRegions
[i
].size
);
629 tu_CmdCopyBufferToImage(VkCommandBuffer commandBuffer
,
632 VkImageLayout destImageLayout
,
633 uint32_t regionCount
,
634 const VkBufferImageCopy
*pRegions
)
636 TU_FROM_HANDLE(tu_cmd_buffer
, cmd_buffer
, commandBuffer
);
637 TU_FROM_HANDLE(tu_image
, dest_image
, destImage
);
638 TU_FROM_HANDLE(tu_buffer
, src_buffer
, srcBuffer
);
640 for (unsigned i
= 0; i
< regionCount
; ++i
) {
641 tu_copy_buffer_to_image(cmd_buffer
, src_buffer
, dest_image
,
647 tu_CmdCopyImageToBuffer(VkCommandBuffer commandBuffer
,
649 VkImageLayout srcImageLayout
,
651 uint32_t regionCount
,
652 const VkBufferImageCopy
*pRegions
)
654 TU_FROM_HANDLE(tu_cmd_buffer
, cmd_buffer
, commandBuffer
);
655 TU_FROM_HANDLE(tu_image
, src_image
, srcImage
);
656 TU_FROM_HANDLE(tu_buffer
, dst_buffer
, destBuffer
);
658 for (unsigned i
= 0; i
< regionCount
; ++i
) {
659 tu_copy_image_to_buffer(cmd_buffer
, src_image
, dst_buffer
,
665 meta_copy_image(struct tu_cmd_buffer
*cmd_buffer
,
666 struct tu_image
*src_image
,
667 VkImageLayout src_image_layout
,
668 struct tu_image
*dest_image
,
669 VkImageLayout dest_image_layout
,
670 uint32_t regionCount
,
671 const VkImageCopy
*pRegions
)
676 tu_CmdCopyImage(VkCommandBuffer commandBuffer
,
678 VkImageLayout srcImageLayout
,
680 VkImageLayout destImageLayout
,
681 uint32_t regionCount
,
682 const VkImageCopy
*pRegions
)
684 TU_FROM_HANDLE(tu_cmd_buffer
, cmd_buffer
, commandBuffer
);
685 TU_FROM_HANDLE(tu_image
, src_image
, srcImage
);
686 TU_FROM_HANDLE(tu_image
, dest_image
, destImage
);
688 meta_copy_image(cmd_buffer
, src_image
, srcImageLayout
, dest_image
,
689 destImageLayout
, regionCount
, pRegions
);