2 * Mesa 3-D graphics library
4 * Copyright (C) 2012-2013 LunarG, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Chia-I Wu <olv@lunarg.com>
28 #include "util/u_surface.h"
29 #include "util/u_transfer.h"
30 #include "util/u_format_etc.h"
34 #include "ilo_context.h"
35 #include "ilo_resource.h"
36 #include "ilo_state.h"
37 #include "ilo_transfer.h"
40 is_bo_busy(struct ilo_context
*ilo
, struct intel_bo
*bo
, bool *need_flush
)
42 const bool referenced
= intel_bo_has_reloc(ilo
->cp
->bo
, bo
);
45 *need_flush
= referenced
;
50 return intel_bo_is_busy(bo
);
54 map_bo_for_transfer(struct ilo_context
*ilo
, struct intel_bo
*bo
,
55 const struct ilo_transfer
*xfer
)
59 switch (xfer
->method
) {
60 case ILO_TRANSFER_MAP_CPU
:
61 ptr
= intel_bo_map(bo
, (xfer
->base
.usage
& PIPE_TRANSFER_WRITE
));
63 case ILO_TRANSFER_MAP_GTT
:
64 ptr
= intel_bo_map_gtt(bo
);
66 case ILO_TRANSFER_MAP_UNSYNC
:
67 ptr
= intel_bo_map_unsynchronized(bo
);
70 assert(!"unknown mapping method");
79 * Choose the best mapping method, depending on the transfer usage and whether
83 choose_transfer_method(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
85 struct pipe_resource
*res
= xfer
->base
.resource
;
86 const unsigned usage
= xfer
->base
.usage
;
87 /* prefer map() when there is the last-level cache */
88 const bool prefer_cpu
=
89 (ilo
->dev
->has_llc
|| (usage
& PIPE_TRANSFER_READ
));
90 struct ilo_texture
*tex
;
91 struct ilo_buffer
*buf
;
93 bool tiled
, need_flush
;
95 if (res
->target
== PIPE_BUFFER
) {
98 buf
= ilo_buffer(res
);
105 tex
= ilo_texture(res
);
107 tiled
= (tex
->tiling
!= INTEL_TILING_NONE
);
110 /* choose between mapping through CPU or GTT */
111 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
112 /* we do not want fencing */
113 if (tiled
|| prefer_cpu
)
114 xfer
->method
= ILO_TRANSFER_MAP_CPU
;
116 xfer
->method
= ILO_TRANSFER_MAP_GTT
;
119 if (!tiled
&& prefer_cpu
)
120 xfer
->method
= ILO_TRANSFER_MAP_CPU
;
122 xfer
->method
= ILO_TRANSFER_MAP_GTT
;
125 /* see if we can avoid stalling */
126 if (is_bo_busy(ilo
, bo
, &need_flush
)) {
127 bool will_stall
= true;
129 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
130 /* nothing we can do */
132 else if (usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) {
133 /* unsynchronized gtt mapping does not stall */
134 xfer
->method
= ILO_TRANSFER_MAP_UNSYNC
;
137 else if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
138 /* discard old bo and allocate a new one for mapping */
139 if ((tex
&& ilo_texture_alloc_bo(tex
)) ||
140 (buf
&& ilo_buffer_alloc_bo(buf
))) {
141 ilo_mark_states_with_resource_dirty(ilo
, res
);
145 else if (usage
& PIPE_TRANSFER_FLUSH_EXPLICIT
) {
147 * We could allocate and return a system buffer here. When a region of
148 * the buffer is explicitly flushed, we pwrite() the region to a
149 * temporary bo and emit pipelined copy blit.
151 * For now, do nothing.
154 else if (usage
& PIPE_TRANSFER_DISCARD_RANGE
) {
156 * We could allocate a temporary bo for mapping, and emit pipelined copy
157 * blit upon unmapping.
159 * For now, do nothing.
164 if (usage
& PIPE_TRANSFER_DONTBLOCK
)
167 /* flush to make bo busy (so that map() stalls as it should be) */
169 ilo_cp_flush(ilo
->cp
, "syncing for transfers");
173 if (tex
&& !(usage
& PIPE_TRANSFER_MAP_DIRECTLY
)) {
174 if (tex
->separate_s8
|| tex
->bo_format
== PIPE_FORMAT_S8_UINT
)
175 xfer
->method
= ILO_TRANSFER_MAP_SW_ZS
;
176 /* need to convert on-the-fly */
177 else if (tex
->bo_format
!= tex
->base
.format
)
178 xfer
->method
= ILO_TRANSFER_MAP_SW_CONVERT
;
185 tex_get_box_origin(const struct ilo_texture
*tex
,
186 unsigned level
, unsigned slice
,
187 const struct pipe_box
*box
,
188 unsigned *mem_x
, unsigned *mem_y
)
190 const struct ilo_texture_slice
*s
=
191 ilo_texture_get_slice(tex
, level
, slice
+ box
->z
);
197 assert(x
% tex
->block_width
== 0 && y
% tex
->block_height
== 0);
199 *mem_x
= x
/ tex
->block_width
* tex
->bo_cpp
;
200 *mem_y
= y
/ tex
->block_height
;
204 tex_get_box_offset(const struct ilo_texture
*tex
, unsigned level
,
205 const struct pipe_box
*box
)
207 unsigned mem_x
, mem_y
;
209 tex_get_box_origin(tex
, level
, 0, box
, &mem_x
, &mem_y
);
211 return mem_y
* tex
->bo_stride
+ mem_x
;
215 tex_get_slice_stride(const struct ilo_texture
*tex
, unsigned level
)
217 const struct ilo_texture_slice
*s0
, *s1
;
220 /* there is no 3D array texture */
221 assert(tex
->base
.array_size
== 1 || tex
->base
.depth0
== 1);
223 if (tex
->base
.array_size
== 1) {
224 /* non-array, non-3D */
225 if (tex
->base
.depth0
== 1)
228 /* only the first level has a fixed slice stride */
230 assert(!"no slice stride for 3D texture with level > 0");
235 s0
= ilo_texture_get_slice(tex
, level
, 0);
236 s1
= ilo_texture_get_slice(tex
, level
, 1);
237 qpitch
= s1
->y
- s0
->y
;
238 assert(qpitch
% tex
->block_height
== 0);
240 return (qpitch
/ tex
->block_height
) * tex
->bo_stride
;
244 tex_tile_x_swizzle(unsigned addr
)
247 * From the Ivy Bridge PRM, volume 1 part 2, page 24:
249 * "As shown in the tiling algorithm, the new address bit[6] should be:
251 * Address bit[6] <= TiledAddr bit[6] XOR
252 * TiledAddr bit[9] XOR
255 return addr
^ (((addr
>> 3) ^ (addr
>> 4)) & 0x40);
259 tex_tile_y_swizzle(unsigned addr
)
262 * From the Ivy Bridge PRM, volume 1 part 2, page 24:
264 * "As shown in the tiling algorithm, The new address bit[6] becomes:
266 * Address bit[6] <= TiledAddr bit[6] XOR
269 return addr
^ ((addr
>> 3) & 0x40);
273 tex_tile_x_offset(unsigned mem_x
, unsigned mem_y
,
274 unsigned tiles_per_row
, bool swizzle
)
277 * From the Sandy Bridge PRM, volume 1 part 2, page 21, we know that a
278 * X-major tile has 8 rows and 32 OWord columns (512 bytes). Tiles in the
279 * tiled region are numbered in row-major order, starting from zero. The
280 * tile number can thus be calculated as follows:
282 * tile = (mem_y / 8) * tiles_per_row + (mem_x / 512)
284 * OWords in that tile are also numbered in row-major order, starting from
285 * zero. The OWord number can thus be calculated as follows:
287 * oword = (mem_y % 8) * 32 + ((mem_x % 512) / 16)
289 * and the tiled offset is
291 * offset = tile * 4096 + oword * 16 + (mem_x % 16)
292 * = tile * 4096 + (mem_y % 8) * 512 + (mem_x % 512)
294 unsigned tile
, offset
;
296 tile
= (mem_y
>> 3) * tiles_per_row
+ (mem_x
>> 9);
297 offset
= tile
<< 12 | (mem_y
& 0x7) << 9 | (mem_x
& 0x1ff);
299 return (swizzle
) ? tex_tile_x_swizzle(offset
) : offset
;
303 tex_tile_y_offset(unsigned mem_x
, unsigned mem_y
,
304 unsigned tiles_per_row
, bool swizzle
)
307 * From the Sandy Bridge PRM, volume 1 part 2, page 22, we know that a
308 * Y-major tile has 32 rows and 8 OWord columns (128 bytes). Tiles in the
309 * tiled region are numbered in row-major order, starting from zero. The
310 * tile number can thus be calculated as follows:
312 * tile = (mem_y / 32) * tiles_per_row + (mem_x / 128)
314 * OWords in that tile are numbered in column-major order, starting from
315 * zero. The OWord number can thus be calculated as follows:
317 * oword = ((mem_x % 128) / 16) * 32 + (mem_y % 32)
319 * and the tiled offset is
321 * offset = tile * 4096 + oword * 16 + (mem_x % 16)
323 unsigned tile
, oword
, offset
;
325 tile
= (mem_y
>> 5) * tiles_per_row
+ (mem_x
>> 7);
326 oword
= (mem_x
& 0x70) << 1 | (mem_y
& 0x1f);
327 offset
= tile
<< 12 | oword
<< 4 | (mem_x
& 0xf);
329 return (swizzle
) ? tex_tile_y_swizzle(offset
) : offset
;
333 tex_tile_w_offset(unsigned mem_x
, unsigned mem_y
,
334 unsigned tiles_per_row
, bool swizzle
)
337 * From the Sandy Bridge PRM, volume 1 part 2, page 23, we know that a
338 * W-major tile has 8 8x8-block rows and 8 8x8-block columns. Tiles in the
339 * tiled region are numbered in row-major order, starting from zero. The
340 * tile number can thus be calculated as follows:
342 * tile = (mem_y / 64) * tiles_per_row + (mem_x / 64)
344 * 8x8-blocks in that tile are numbered in column-major order, starting
345 * from zero. The 8x8-block number can thus be calculated as follows:
347 * blk8 = ((mem_x % 64) / 8) * 8 + ((mem_y % 64) / 8)
349 * Each 8x8-block is divided into 4 4x4-blocks, in row-major order. Each
350 * 4x4-block is further divided into 4 2x2-blocks, also in row-major order.
353 * blk4 = (((mem_y % 64) / 4) & 1) * 2 + (((mem_x % 64) / 4) & 1)
354 * blk2 = (((mem_y % 64) / 2) & 1) * 2 + (((mem_x % 64) / 2) & 1)
355 * blk1 = (((mem_y % 64) ) & 1) * 2 + (((mem_x % 64) ) & 1)
357 * and the tiled offset is
359 * offset = tile * 4096 + blk8 * 64 + blk4 * 16 + blk2 * 4 + blk1
361 unsigned tile
, blk8
, blk4
, blk2
, blk1
, offset
;
363 tile
= (mem_y
>> 6) * tiles_per_row
+ (mem_x
>> 6);
364 blk8
= ((mem_x
>> 3) & 0x7) << 3 | ((mem_y
>> 3) & 0x7);
365 blk4
= ((mem_y
>> 2) & 0x1) << 1 | ((mem_x
>> 2) & 0x1);
366 blk2
= ((mem_y
>> 1) & 0x1) << 1 | ((mem_x
>> 1) & 0x1);
367 blk1
= ((mem_y
) & 0x1) << 1 | ((mem_x
) & 0x1);
368 offset
= tile
<< 12 | blk8
<< 6 | blk4
<< 4 | blk2
<< 2 | blk1
;
370 return (swizzle
) ? tex_tile_y_swizzle(offset
) : offset
;
374 tex_tile_none_offset(unsigned mem_x
, unsigned mem_y
,
375 unsigned tiles_per_row
, bool swizzle
)
377 return mem_y
* tiles_per_row
+ mem_x
;
380 typedef unsigned (*tex_tile_offset_func
)(unsigned mem_x
, unsigned mem_y
,
381 unsigned tiles_per_row
,
384 static tex_tile_offset_func
385 tex_tile_choose_offset_func(const struct ilo_texture
*tex
,
386 unsigned *tiles_per_row
)
388 switch (tex
->tiling
) {
390 *tiles_per_row
= tex
->bo_stride
/ 512;
391 return tex_tile_x_offset
;
393 *tiles_per_row
= tex
->bo_stride
/ 128;
394 return tex_tile_y_offset
;
395 case INTEL_TILING_NONE
:
398 if (tex
->bo_format
== PIPE_FORMAT_S8_UINT
) {
399 *tiles_per_row
= tex
->bo_stride
/ 64;
400 return tex_tile_w_offset
;
403 *tiles_per_row
= tex
->bo_stride
;
404 return tex_tile_none_offset
;
410 tex_staging_sys_map_bo(const struct ilo_context
*ilo
,
411 struct ilo_texture
*tex
,
412 bool for_read_back
, bool linear_view
)
414 const bool prefer_cpu
= (ilo
->dev
->has_llc
|| for_read_back
);
417 if (prefer_cpu
&& (tex
->tiling
== INTEL_TILING_NONE
|| !linear_view
))
418 ptr
= intel_bo_map(tex
->bo
, !for_read_back
);
420 ptr
= intel_bo_map_gtt(tex
->bo
);
426 tex_staging_sys_unmap_bo(const struct ilo_context
*ilo
,
427 const struct ilo_texture
*tex
)
429 intel_bo_unmap(tex
->bo
);
433 tex_staging_sys_zs_read(struct ilo_context
*ilo
,
434 struct ilo_texture
*tex
,
435 const struct ilo_transfer
*xfer
)
437 const bool swizzle
= ilo
->dev
->has_address_swizzling
;
438 const struct pipe_box
*box
= &xfer
->base
.box
;
440 tex_tile_offset_func tile_offset
;
441 unsigned tiles_per_row
;
444 src
= tex_staging_sys_map_bo(ilo
, tex
, true, false);
448 tile_offset
= tex_tile_choose_offset_func(tex
, &tiles_per_row
);
450 assert(tex
->block_width
== 1 && tex
->block_height
== 1);
452 if (tex
->separate_s8
) {
453 struct ilo_texture
*s8_tex
= tex
->separate_s8
;
454 const uint8_t *s8_src
;
455 tex_tile_offset_func s8_tile_offset
;
456 unsigned s8_tiles_per_row
;
457 int dst_cpp
, dst_s8_pos
, src_cpp_used
;
459 s8_src
= tex_staging_sys_map_bo(ilo
, s8_tex
, true, false);
461 tex_staging_sys_unmap_bo(ilo
, tex
);
465 s8_tile_offset
= tex_tile_choose_offset_func(s8_tex
, &s8_tiles_per_row
);
467 if (tex
->base
.format
== PIPE_FORMAT_Z24_UNORM_S8_UINT
) {
468 assert(tex
->bo_format
== PIPE_FORMAT_Z24X8_UNORM
);
475 assert(tex
->base
.format
== PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
);
476 assert(tex
->bo_format
== PIPE_FORMAT_Z32_FLOAT
);
483 for (slice
= 0; slice
< box
->depth
; slice
++) {
484 unsigned mem_x
, mem_y
, s8_mem_x
, s8_mem_y
;
488 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
489 box
, &mem_x
, &mem_y
);
490 tex_get_box_origin(s8_tex
, xfer
->base
.level
, slice
,
491 box
, &s8_mem_x
, &s8_mem_y
);
493 dst
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
495 for (i
= 0; i
< box
->height
; i
++) {
496 unsigned x
= mem_x
, s8_x
= s8_mem_x
;
499 for (j
= 0; j
< box
->width
; j
++) {
500 const unsigned offset
=
501 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
502 const unsigned s8_offset
=
503 s8_tile_offset(s8_x
, s8_mem_y
, s8_tiles_per_row
, swizzle
);
505 memcpy(d
, src
+ offset
, src_cpp_used
);
506 d
[dst_s8_pos
] = s8_src
[s8_offset
];
513 dst
+= xfer
->base
.stride
;
519 tex_staging_sys_unmap_bo(ilo
, s8_tex
);
522 assert(tex
->bo_format
== PIPE_FORMAT_S8_UINT
);
524 for (slice
= 0; slice
< box
->depth
; slice
++) {
525 unsigned mem_x
, mem_y
;
529 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
530 box
, &mem_x
, &mem_y
);
532 dst
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
534 for (i
= 0; i
< box
->height
; i
++) {
538 for (j
= 0; j
< box
->width
; j
++) {
539 const unsigned offset
=
540 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
548 dst
+= xfer
->base
.stride
;
554 tex_staging_sys_unmap_bo(ilo
, tex
);
560 tex_staging_sys_zs_write(struct ilo_context
*ilo
,
561 struct ilo_texture
*tex
,
562 const struct ilo_transfer
*xfer
)
564 const bool swizzle
= ilo
->dev
->has_address_swizzling
;
565 const struct pipe_box
*box
= &xfer
->base
.box
;
567 tex_tile_offset_func tile_offset
;
568 unsigned tiles_per_row
;
571 dst
= tex_staging_sys_map_bo(ilo
, tex
, false, false);
575 tile_offset
= tex_tile_choose_offset_func(tex
, &tiles_per_row
);
577 assert(tex
->block_width
== 1 && tex
->block_height
== 1);
579 if (tex
->separate_s8
) {
580 struct ilo_texture
*s8_tex
= tex
->separate_s8
;
582 tex_tile_offset_func s8_tile_offset
;
583 unsigned s8_tiles_per_row
;
584 int src_cpp
, src_s8_pos
, dst_cpp_used
;
586 s8_dst
= tex_staging_sys_map_bo(ilo
, s8_tex
, false, false);
588 tex_staging_sys_unmap_bo(ilo
, s8_tex
);
592 s8_tile_offset
= tex_tile_choose_offset_func(s8_tex
, &s8_tiles_per_row
);
594 if (tex
->base
.format
== PIPE_FORMAT_Z24_UNORM_S8_UINT
) {
595 assert(tex
->bo_format
== PIPE_FORMAT_Z24X8_UNORM
);
602 assert(tex
->base
.format
== PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
);
603 assert(tex
->bo_format
== PIPE_FORMAT_Z32_FLOAT
);
610 for (slice
= 0; slice
< box
->depth
; slice
++) {
611 unsigned mem_x
, mem_y
, s8_mem_x
, s8_mem_y
;
615 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
616 box
, &mem_x
, &mem_y
);
617 tex_get_box_origin(s8_tex
, xfer
->base
.level
, slice
,
618 box
, &s8_mem_x
, &s8_mem_y
);
620 src
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
622 for (i
= 0; i
< box
->height
; i
++) {
623 unsigned x
= mem_x
, s8_x
= s8_mem_x
;
624 const uint8_t *s
= src
;
626 for (j
= 0; j
< box
->width
; j
++) {
627 const unsigned offset
=
628 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
629 const unsigned s8_offset
=
630 s8_tile_offset(s8_x
, s8_mem_y
, s8_tiles_per_row
, swizzle
);
632 memcpy(dst
+ offset
, s
, dst_cpp_used
);
633 s8_dst
[s8_offset
] = s
[src_s8_pos
];
640 src
+= xfer
->base
.stride
;
646 tex_staging_sys_unmap_bo(ilo
, s8_tex
);
649 assert(tex
->bo_format
== PIPE_FORMAT_S8_UINT
);
651 for (slice
= 0; slice
< box
->depth
; slice
++) {
652 unsigned mem_x
, mem_y
;
656 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
657 box
, &mem_x
, &mem_y
);
659 src
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
661 for (i
= 0; i
< box
->height
; i
++) {
663 const uint8_t *s
= src
;
665 for (j
= 0; j
< box
->width
; j
++) {
666 const unsigned offset
=
667 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
675 src
+= xfer
->base
.stride
;
681 tex_staging_sys_unmap_bo(ilo
, tex
);
687 tex_staging_sys_convert_write(struct ilo_context
*ilo
,
688 struct ilo_texture
*tex
,
689 const struct ilo_transfer
*xfer
)
691 const struct pipe_box
*box
= &xfer
->base
.box
;
692 unsigned dst_slice_stride
;
696 dst
= tex_staging_sys_map_bo(ilo
, tex
, false, true);
700 dst
+= tex_get_box_offset(tex
, xfer
->base
.level
, box
);
702 /* slice stride is not always available */
704 dst_slice_stride
= tex_get_slice_stride(tex
, xfer
->base
.level
);
706 dst_slice_stride
= 0;
708 if (unlikely(tex
->bo_format
== tex
->base
.format
)) {
709 util_copy_box(dst
, tex
->bo_format
, tex
->bo_stride
, dst_slice_stride
,
710 0, 0, 0, box
->width
, box
->height
, box
->depth
,
711 xfer
->staging_sys
, xfer
->base
.stride
, xfer
->base
.layer_stride
,
714 tex_staging_sys_unmap_bo(ilo
, tex
);
719 switch (tex
->base
.format
) {
720 case PIPE_FORMAT_ETC1_RGB8
:
721 assert(tex
->bo_format
== PIPE_FORMAT_R8G8B8X8_UNORM
);
723 for (slice
= 0; slice
< box
->depth
; slice
++) {
725 xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
727 util_format_etc1_rgb8_unpack_rgba_8unorm(dst
,
728 tex
->bo_stride
, src
, xfer
->base
.stride
,
729 box
->width
, box
->height
);
731 dst
+= dst_slice_stride
;
735 assert(!"unable to convert the staging data");
739 tex_staging_sys_unmap_bo(ilo
, tex
);
745 tex_staging_sys_unmap(struct ilo_context
*ilo
,
746 struct ilo_texture
*tex
,
747 struct ilo_transfer
*xfer
)
751 if (!(xfer
->base
.usage
& PIPE_TRANSFER_WRITE
)) {
752 FREE(xfer
->staging_sys
);
756 switch (xfer
->method
) {
757 case ILO_TRANSFER_MAP_SW_CONVERT
:
758 success
= tex_staging_sys_convert_write(ilo
, tex
, xfer
);
760 case ILO_TRANSFER_MAP_SW_ZS
:
761 success
= tex_staging_sys_zs_write(ilo
, tex
, xfer
);
764 assert(!"unknown mapping method");
770 ilo_err("failed to map resource for moving staging data\n");
772 FREE(xfer
->staging_sys
);
776 tex_staging_sys_map(struct ilo_context
*ilo
,
777 struct ilo_texture
*tex
,
778 struct ilo_transfer
*xfer
)
780 const struct pipe_box
*box
= &xfer
->base
.box
;
781 const size_t stride
= util_format_get_stride(tex
->base
.format
, box
->width
);
783 util_format_get_2d_size(tex
->base
.format
, stride
, box
->height
);
784 bool read_back
= false, success
;
786 xfer
->staging_sys
= MALLOC(size
* box
->depth
);
787 if (!xfer
->staging_sys
)
790 xfer
->base
.stride
= stride
;
791 xfer
->base
.layer_stride
= size
;
792 xfer
->ptr
= xfer
->staging_sys
;
794 /* see if we need to read the resource back */
795 if (xfer
->base
.usage
& PIPE_TRANSFER_READ
) {
798 else if (xfer
->base
.usage
& PIPE_TRANSFER_WRITE
) {
799 const unsigned discard_flags
=
800 (PIPE_TRANSFER_DISCARD_RANGE
| PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
);
802 if (!(xfer
->base
.usage
& discard_flags
))
809 switch (xfer
->method
) {
810 case ILO_TRANSFER_MAP_SW_CONVERT
:
811 assert(!"no on-the-fly format conversion for mapping");
814 case ILO_TRANSFER_MAP_SW_ZS
:
815 success
= tex_staging_sys_zs_read(ilo
, tex
, xfer
);
818 assert(!"unknown mapping method");
827 tex_direct_unmap(struct ilo_context
*ilo
,
828 struct ilo_texture
*tex
,
829 struct ilo_transfer
*xfer
)
831 intel_bo_unmap(tex
->bo
);
835 tex_direct_map(struct ilo_context
*ilo
,
836 struct ilo_texture
*tex
,
837 struct ilo_transfer
*xfer
)
839 xfer
->ptr
= map_bo_for_transfer(ilo
, tex
->bo
, xfer
);
843 xfer
->ptr
+= tex_get_box_offset(tex
, xfer
->base
.level
, &xfer
->base
.box
);
845 /* note that stride is for a block row, not a texel row */
846 xfer
->base
.stride
= tex
->bo_stride
;
848 /* slice stride is not always available */
849 if (xfer
->base
.box
.depth
> 1)
850 xfer
->base
.layer_stride
= tex_get_slice_stride(tex
, xfer
->base
.level
);
852 xfer
->base
.layer_stride
= 0;
858 tex_map(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
860 struct ilo_texture
*tex
= ilo_texture(xfer
->base
.resource
);
863 if (!choose_transfer_method(ilo
, xfer
))
866 switch (xfer
->method
) {
867 case ILO_TRANSFER_MAP_CPU
:
868 case ILO_TRANSFER_MAP_GTT
:
869 case ILO_TRANSFER_MAP_UNSYNC
:
870 success
= tex_direct_map(ilo
, tex
, xfer
);
872 case ILO_TRANSFER_MAP_SW_CONVERT
:
873 case ILO_TRANSFER_MAP_SW_ZS
:
874 success
= tex_staging_sys_map(ilo
, tex
, xfer
);
877 assert(!"unknown mapping method");
886 tex_unmap(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
888 struct ilo_texture
*tex
= ilo_texture(xfer
->base
.resource
);
890 switch (xfer
->method
) {
891 case ILO_TRANSFER_MAP_CPU
:
892 case ILO_TRANSFER_MAP_GTT
:
893 case ILO_TRANSFER_MAP_UNSYNC
:
894 tex_direct_unmap(ilo
, tex
, xfer
);
896 case ILO_TRANSFER_MAP_SW_CONVERT
:
897 case ILO_TRANSFER_MAP_SW_ZS
:
898 tex_staging_sys_unmap(ilo
, tex
, xfer
);
901 assert(!"unknown mapping method");
907 buf_map(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
909 struct ilo_buffer
*buf
= ilo_buffer(xfer
->base
.resource
);
911 if (!choose_transfer_method(ilo
, xfer
))
914 xfer
->ptr
= map_bo_for_transfer(ilo
, buf
->bo
, xfer
);
918 assert(xfer
->base
.level
== 0);
919 assert(xfer
->base
.box
.y
== 0);
920 assert(xfer
->base
.box
.z
== 0);
921 assert(xfer
->base
.box
.height
== 1);
922 assert(xfer
->base
.box
.depth
== 1);
924 xfer
->ptr
+= xfer
->base
.box
.x
;
925 xfer
->base
.stride
= 0;
926 xfer
->base
.layer_stride
= 0;
932 buf_unmap(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
934 struct ilo_buffer
*buf
= ilo_buffer(xfer
->base
.resource
);
936 intel_bo_unmap(buf
->bo
);
940 buf_pwrite(struct ilo_context
*ilo
, struct ilo_buffer
*buf
,
941 unsigned usage
, int offset
, int size
, const void *data
)
945 /* see if we can avoid stalling */
946 if (is_bo_busy(ilo
, buf
->bo
, &need_flush
)) {
947 bool will_stall
= true;
949 if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
950 /* old data not needed so discard the old bo to avoid stalling */
951 if (ilo_buffer_alloc_bo(buf
)) {
952 ilo_mark_states_with_resource_dirty(ilo
, &buf
->base
);
958 * We could allocate a temporary bo to hold the data and emit
959 * pipelined copy blit to move them to buf->bo. But for now, do
964 /* flush to make bo busy (so that pwrite() stalls as it should be) */
965 if (will_stall
&& need_flush
)
966 ilo_cp_flush(ilo
->cp
, "syncing for pwrites");
969 intel_bo_pwrite(buf
->bo
, offset
, size
, data
);
973 ilo_transfer_flush_region(struct pipe_context
*pipe
,
974 struct pipe_transfer
*transfer
,
975 const struct pipe_box
*box
)
980 ilo_transfer_unmap(struct pipe_context
*pipe
,
981 struct pipe_transfer
*transfer
)
983 struct ilo_context
*ilo
= ilo_context(pipe
);
984 struct ilo_transfer
*xfer
= ilo_transfer(transfer
);
986 if (xfer
->base
.resource
->target
== PIPE_BUFFER
)
987 buf_unmap(ilo
, xfer
);
989 tex_unmap(ilo
, xfer
);
991 pipe_resource_reference(&xfer
->base
.resource
, NULL
);
993 util_slab_free(&ilo
->transfer_mempool
, xfer
);
997 ilo_transfer_map(struct pipe_context
*pipe
,
998 struct pipe_resource
*res
,
1001 const struct pipe_box
*box
,
1002 struct pipe_transfer
**transfer
)
1004 struct ilo_context
*ilo
= ilo_context(pipe
);
1005 struct ilo_transfer
*xfer
;
1008 xfer
= util_slab_alloc(&ilo
->transfer_mempool
);
1014 xfer
->base
.resource
= NULL
;
1015 pipe_resource_reference(&xfer
->base
.resource
, res
);
1016 xfer
->base
.level
= level
;
1017 xfer
->base
.usage
= usage
;
1018 xfer
->base
.box
= *box
;
1020 ilo_blit_resolve_transfer(ilo
, &xfer
->base
);
1022 if (res
->target
== PIPE_BUFFER
)
1023 success
= buf_map(ilo
, xfer
);
1025 success
= tex_map(ilo
, xfer
);
1028 pipe_resource_reference(&xfer
->base
.resource
, NULL
);
1034 *transfer
= &xfer
->base
;
1040 ilo_transfer_inline_write(struct pipe_context
*pipe
,
1041 struct pipe_resource
*res
,
1044 const struct pipe_box
*box
,
1047 unsigned layer_stride
)
1049 if (likely(res
->target
== PIPE_BUFFER
) &&
1050 !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
1051 /* they should specify just an offset and a size */
1053 assert(box
->y
== 0);
1054 assert(box
->z
== 0);
1055 assert(box
->height
== 1);
1056 assert(box
->depth
== 1);
1058 buf_pwrite(ilo_context(pipe
), ilo_buffer(res
),
1059 usage
, box
->x
, box
->width
, data
);
1062 u_default_transfer_inline_write(pipe
, res
,
1063 level
, usage
, box
, data
, stride
, layer_stride
);
1068 * Initialize transfer-related functions.
1071 ilo_init_transfer_functions(struct ilo_context
*ilo
)
1073 ilo
->base
.transfer_map
= ilo_transfer_map
;
1074 ilo
->base
.transfer_flush_region
= ilo_transfer_flush_region
;
1075 ilo
->base
.transfer_unmap
= ilo_transfer_unmap
;
1076 ilo
->base
.transfer_inline_write
= ilo_transfer_inline_write
;