2 * Mesa 3-D graphics library
4 * Copyright (C) 2012-2013 LunarG, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Chia-I Wu <olv@lunarg.com>
28 #include "util/u_surface.h"
29 #include "util/u_transfer.h"
30 #include "util/u_format_etc.h"
33 #include "ilo_context.h"
34 #include "ilo_resource.h"
35 #include "ilo_state.h"
36 #include "ilo_transfer.h"
39 is_bo_busy(struct ilo_context
*ilo
, struct intel_bo
*bo
, bool *need_flush
)
41 const bool referenced
= intel_bo_references(ilo
->cp
->bo
, bo
);
44 *need_flush
= referenced
;
49 return intel_bo_is_busy(bo
);
53 map_bo_for_transfer(struct ilo_context
*ilo
, struct intel_bo
*bo
,
54 const struct ilo_transfer
*xfer
)
58 switch (xfer
->method
) {
59 case ILO_TRANSFER_MAP_CPU
:
60 err
= intel_bo_map(bo
, (xfer
->base
.usage
& PIPE_TRANSFER_WRITE
));
62 case ILO_TRANSFER_MAP_GTT
:
63 err
= intel_bo_map_gtt(bo
);
65 case ILO_TRANSFER_MAP_UNSYNC
:
66 err
= intel_bo_map_unsynchronized(bo
);
69 assert(!"unknown mapping method");
78 * Choose the best mapping method, depending on the transfer usage and whether
82 choose_transfer_method(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
84 struct pipe_resource
*res
= xfer
->base
.resource
;
85 const unsigned usage
= xfer
->base
.usage
;
86 /* prefer map() when there is the last-level cache */
87 const bool prefer_cpu
=
88 (ilo
->dev
->has_llc
|| (usage
& PIPE_TRANSFER_READ
));
89 struct ilo_texture
*tex
;
90 struct ilo_buffer
*buf
;
92 bool tiled
, need_flush
;
94 if (res
->target
== PIPE_BUFFER
) {
97 buf
= ilo_buffer(res
);
104 tex
= ilo_texture(res
);
106 tiled
= (tex
->tiling
!= INTEL_TILING_NONE
);
109 /* choose between mapping through CPU or GTT */
110 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
111 /* we do not want fencing */
112 if (tiled
|| prefer_cpu
)
113 xfer
->method
= ILO_TRANSFER_MAP_CPU
;
115 xfer
->method
= ILO_TRANSFER_MAP_GTT
;
118 if (!tiled
&& prefer_cpu
)
119 xfer
->method
= ILO_TRANSFER_MAP_CPU
;
121 xfer
->method
= ILO_TRANSFER_MAP_GTT
;
124 /* see if we can avoid stalling */
125 if (is_bo_busy(ilo
, bo
, &need_flush
)) {
126 bool will_stall
= true;
128 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
129 /* nothing we can do */
131 else if (usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) {
132 /* unsynchronized gtt mapping does not stall */
133 xfer
->method
= ILO_TRANSFER_MAP_UNSYNC
;
136 else if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
137 /* discard old bo and allocate a new one for mapping */
138 if ((tex
&& ilo_texture_alloc_bo(tex
)) ||
139 (buf
&& ilo_buffer_alloc_bo(buf
))) {
140 ilo_mark_states_with_resource_dirty(ilo
, res
);
144 else if (usage
& PIPE_TRANSFER_FLUSH_EXPLICIT
) {
146 * We could allocate and return a system buffer here. When a region of
147 * the buffer is explicitly flushed, we pwrite() the region to a
148 * temporary bo and emit pipelined copy blit.
150 * For now, do nothing.
153 else if (usage
& PIPE_TRANSFER_DISCARD_RANGE
) {
155 * We could allocate a temporary bo for mapping, and emit pipelined copy
156 * blit upon unmapping.
158 * For now, do nothing.
163 if (usage
& PIPE_TRANSFER_DONTBLOCK
)
166 /* flush to make bo busy (so that map() stalls as it should be) */
168 ilo_cp_flush(ilo
->cp
, "syncing for transfers");
172 if (tex
&& !(usage
& PIPE_TRANSFER_MAP_DIRECTLY
)) {
173 if (tex
->separate_s8
|| tex
->bo_format
== PIPE_FORMAT_S8_UINT
)
174 xfer
->method
= ILO_TRANSFER_MAP_SW_ZS
;
175 /* need to convert on-the-fly */
176 else if (tex
->bo_format
!= tex
->base
.format
)
177 xfer
->method
= ILO_TRANSFER_MAP_SW_CONVERT
;
184 tex_get_box_origin(const struct ilo_texture
*tex
,
185 unsigned level
, unsigned slice
,
186 const struct pipe_box
*box
,
187 unsigned *mem_x
, unsigned *mem_y
)
189 const struct ilo_texture_slice
*s
=
190 ilo_texture_get_slice(tex
, level
, slice
+ box
->z
);
196 assert(x
% tex
->block_width
== 0 && y
% tex
->block_height
== 0);
198 *mem_x
= x
/ tex
->block_width
* tex
->bo_cpp
;
199 *mem_y
= y
/ tex
->block_height
;
203 tex_get_box_offset(const struct ilo_texture
*tex
, unsigned level
,
204 const struct pipe_box
*box
)
206 unsigned mem_x
, mem_y
;
208 tex_get_box_origin(tex
, level
, 0, box
, &mem_x
, &mem_y
);
210 return mem_y
* tex
->bo_stride
+ mem_x
;
214 tex_get_slice_stride(const struct ilo_texture
*tex
, unsigned level
)
216 const struct ilo_texture_slice
*s0
, *s1
;
219 /* there is no 3D array texture */
220 assert(tex
->base
.array_size
== 1 || tex
->base
.depth0
== 1);
222 if (tex
->base
.array_size
== 1) {
223 /* non-array, non-3D */
224 if (tex
->base
.depth0
== 1)
227 /* only the first level has a fixed slice stride */
229 assert(!"no slice stride for 3D texture with level > 0");
234 s0
= ilo_texture_get_slice(tex
, level
, 0);
235 s1
= ilo_texture_get_slice(tex
, level
, 1);
236 qpitch
= s1
->y
- s0
->y
;
237 assert(qpitch
% tex
->block_height
== 0);
239 return (qpitch
/ tex
->block_height
) * tex
->bo_stride
;
243 tex_tile_x_swizzle(unsigned addr
)
246 * From the Ivy Bridge PRM, volume 1 part 2, page 24:
248 * "As shown in the tiling algorithm, the new address bit[6] should be:
250 * Address bit[6] <= TiledAddr bit[6] XOR
251 * TiledAddr bit[9] XOR
254 return addr
^ (((addr
>> 3) ^ (addr
>> 4)) & 0x40);
258 tex_tile_y_swizzle(unsigned addr
)
261 * From the Ivy Bridge PRM, volume 1 part 2, page 24:
263 * "As shown in the tiling algorithm, The new address bit[6] becomes:
265 * Address bit[6] <= TiledAddr bit[6] XOR
268 return addr
^ ((addr
>> 3) & 0x40);
272 tex_tile_x_offset(unsigned mem_x
, unsigned mem_y
,
273 unsigned tiles_per_row
, bool swizzle
)
276 * From the Sandy Bridge PRM, volume 1 part 2, page 21, we know that a
277 * X-major tile has 8 rows and 32 OWord columns (512 bytes). Tiles in the
278 * tiled region are numbered in row-major order, starting from zero. The
279 * tile number can thus be calculated as follows:
281 * tile = (mem_y / 8) * tiles_per_row + (mem_x / 512)
283 * OWords in that tile are also numbered in row-major order, starting from
284 * zero. The OWord number can thus be calculated as follows:
286 * oword = (mem_y % 8) * 32 + ((mem_x % 512) / 16)
288 * and the tiled offset is
290 * offset = tile * 4096 + oword * 16 + (mem_x % 16)
291 * = tile * 4096 + (mem_y % 8) * 512 + (mem_x % 512)
293 unsigned tile
, offset
;
295 tile
= (mem_y
>> 3) * tiles_per_row
+ (mem_x
>> 9);
296 offset
= tile
<< 12 | (mem_y
& 0x7) << 9 | (mem_x
& 0x1ff);
298 return (swizzle
) ? tex_tile_x_swizzle(offset
) : offset
;
302 tex_tile_y_offset(unsigned mem_x
, unsigned mem_y
,
303 unsigned tiles_per_row
, bool swizzle
)
306 * From the Sandy Bridge PRM, volume 1 part 2, page 22, we know that a
307 * Y-major tile has 32 rows and 8 OWord columns (128 bytes). Tiles in the
308 * tiled region are numbered in row-major order, starting from zero. The
309 * tile number can thus be calculated as follows:
311 * tile = (mem_y / 32) * tiles_per_row + (mem_x / 128)
313 * OWords in that tile are numbered in column-major order, starting from
314 * zero. The OWord number can thus be calculated as follows:
316 * oword = ((mem_x % 128) / 16) * 32 + (mem_y % 32)
318 * and the tiled offset is
320 * offset = tile * 4096 + oword * 16 + (mem_x % 16)
322 unsigned tile
, oword
, offset
;
324 tile
= (mem_y
>> 5) * tiles_per_row
+ (mem_x
>> 7);
325 oword
= (mem_x
& 0x70) << 1 | (mem_y
& 0x1f);
326 offset
= tile
<< 12 | oword
<< 4 | (mem_x
& 0xf);
328 return (swizzle
) ? tex_tile_y_swizzle(offset
) : offset
;
332 tex_tile_w_offset(unsigned mem_x
, unsigned mem_y
,
333 unsigned tiles_per_row
, bool swizzle
)
336 * From the Sandy Bridge PRM, volume 1 part 2, page 23, we know that a
337 * W-major tile has 8 8x8-block rows and 8 8x8-block columns. Tiles in the
338 * tiled region are numbered in row-major order, starting from zero. The
339 * tile number can thus be calculated as follows:
341 * tile = (mem_y / 64) * tiles_per_row + (mem_x / 64)
343 * 8x8-blocks in that tile are numbered in column-major order, starting
344 * from zero. The 8x8-block number can thus be calculated as follows:
346 * blk8 = ((mem_x % 64) / 8) * 8 + ((mem_y % 64) / 8)
348 * Each 8x8-block is divided into 4 4x4-blocks, in row-major order. Each
349 * 4x4-block is further divided into 4 2x2-blocks, also in row-major order.
352 * blk4 = (((mem_y % 64) / 4) & 1) * 2 + (((mem_x % 64) / 4) & 1)
353 * blk2 = (((mem_y % 64) / 2) & 1) * 2 + (((mem_x % 64) / 2) & 1)
354 * blk1 = (((mem_y % 64) ) & 1) * 2 + (((mem_x % 64) ) & 1)
356 * and the tiled offset is
358 * offset = tile * 4096 + blk8 * 64 + blk4 * 16 + blk2 * 4 + blk1
360 unsigned tile
, blk8
, blk4
, blk2
, blk1
, offset
;
362 tile
= (mem_y
>> 6) * tiles_per_row
+ (mem_x
>> 6);
363 blk8
= ((mem_x
>> 3) & 0x7) << 3 | ((mem_y
>> 3) & 0x7);
364 blk4
= ((mem_y
>> 2) & 0x1) << 1 | ((mem_x
>> 2) & 0x1);
365 blk2
= ((mem_y
>> 1) & 0x1) << 1 | ((mem_x
>> 1) & 0x1);
366 blk1
= ((mem_y
) & 0x1) << 1 | ((mem_x
) & 0x1);
367 offset
= tile
<< 12 | blk8
<< 6 | blk4
<< 4 | blk2
<< 2 | blk1
;
369 return (swizzle
) ? tex_tile_y_swizzle(offset
) : offset
;
373 tex_tile_none_offset(unsigned mem_x
, unsigned mem_y
,
374 unsigned tiles_per_row
, bool swizzle
)
376 return mem_y
* tiles_per_row
+ mem_x
;
379 typedef unsigned (*tex_tile_offset_func
)(unsigned mem_x
, unsigned mem_y
,
380 unsigned tiles_per_row
,
383 static tex_tile_offset_func
384 tex_tile_choose_offset_func(const struct ilo_texture
*tex
,
385 unsigned *tiles_per_row
)
387 switch (tex
->tiling
) {
389 *tiles_per_row
= tex
->bo_stride
/ 512;
390 return tex_tile_x_offset
;
392 *tiles_per_row
= tex
->bo_stride
/ 128;
393 return tex_tile_y_offset
;
394 case INTEL_TILING_NONE
:
397 if (tex
->bo_format
== PIPE_FORMAT_S8_UINT
) {
398 *tiles_per_row
= tex
->bo_stride
/ 64;
399 return tex_tile_w_offset
;
402 *tiles_per_row
= tex
->bo_stride
;
403 return tex_tile_none_offset
;
409 tex_staging_sys_zs_read(struct ilo_context
*ilo
,
410 struct ilo_texture
*tex
,
411 const struct ilo_transfer
*xfer
)
413 const bool swizzle
= ilo
->dev
->has_address_swizzling
;
414 const struct pipe_box
*box
= &xfer
->base
.box
;
415 const uint8_t *src
= intel_bo_get_virtual(tex
->bo
);
416 tex_tile_offset_func tile_offset
;
417 unsigned tiles_per_row
;
420 tile_offset
= tex_tile_choose_offset_func(tex
, &tiles_per_row
);
422 assert(tex
->block_width
== 1 && tex
->block_height
== 1);
424 if (tex
->separate_s8
) {
425 struct ilo_texture
*s8_tex
= tex
->separate_s8
;
426 const uint8_t *s8_src
= intel_bo_get_virtual(s8_tex
->bo
);
427 tex_tile_offset_func s8_tile_offset
;
428 unsigned s8_tiles_per_row
;
429 int dst_cpp
, dst_s8_pos
, src_cpp_used
;
431 s8_tile_offset
= tex_tile_choose_offset_func(s8_tex
, &s8_tiles_per_row
);
433 if (tex
->base
.format
== PIPE_FORMAT_Z24_UNORM_S8_UINT
) {
434 assert(tex
->bo_format
== PIPE_FORMAT_Z24X8_UNORM
);
441 assert(tex
->base
.format
== PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
);
442 assert(tex
->bo_format
== PIPE_FORMAT_Z32_FLOAT
);
449 for (slice
= 0; slice
< box
->depth
; slice
++) {
450 unsigned mem_x
, mem_y
, s8_mem_x
, s8_mem_y
;
454 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
455 box
, &mem_x
, &mem_y
);
456 tex_get_box_origin(s8_tex
, xfer
->base
.level
, slice
,
457 box
, &s8_mem_x
, &s8_mem_y
);
459 dst
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
461 for (i
= 0; i
< box
->height
; i
++) {
462 unsigned x
= mem_x
, s8_x
= s8_mem_x
;
465 for (j
= 0; j
< box
->width
; j
++) {
466 const unsigned offset
=
467 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
468 const unsigned s8_offset
=
469 s8_tile_offset(s8_x
, s8_mem_y
, s8_tiles_per_row
, swizzle
);
471 memcpy(d
, src
+ offset
, src_cpp_used
);
472 d
[dst_s8_pos
] = s8_src
[s8_offset
];
479 dst
+= xfer
->base
.stride
;
486 assert(tex
->bo_format
== PIPE_FORMAT_S8_UINT
);
488 for (slice
= 0; slice
< box
->depth
; slice
++) {
489 unsigned mem_x
, mem_y
;
493 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
494 box
, &mem_x
, &mem_y
);
496 dst
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
498 for (i
= 0; i
< box
->height
; i
++) {
502 for (j
= 0; j
< box
->width
; j
++) {
503 const unsigned offset
=
504 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
512 dst
+= xfer
->base
.stride
;
520 tex_staging_sys_zs_write(struct ilo_context
*ilo
,
521 struct ilo_texture
*tex
,
522 const struct ilo_transfer
*xfer
)
524 const bool swizzle
= ilo
->dev
->has_address_swizzling
;
525 const struct pipe_box
*box
= &xfer
->base
.box
;
526 uint8_t *dst
= intel_bo_get_virtual(tex
->bo
);
527 tex_tile_offset_func tile_offset
;
528 unsigned tiles_per_row
;
531 tile_offset
= tex_tile_choose_offset_func(tex
, &tiles_per_row
);
533 assert(tex
->block_width
== 1 && tex
->block_height
== 1);
535 if (tex
->separate_s8
) {
536 struct ilo_texture
*s8_tex
= tex
->separate_s8
;
537 uint8_t *s8_dst
= intel_bo_get_virtual(s8_tex
->bo
);
538 tex_tile_offset_func s8_tile_offset
;
539 unsigned s8_tiles_per_row
;
540 int src_cpp
, src_s8_pos
, dst_cpp_used
;
542 s8_tile_offset
= tex_tile_choose_offset_func(s8_tex
, &s8_tiles_per_row
);
544 if (tex
->base
.format
== PIPE_FORMAT_Z24_UNORM_S8_UINT
) {
545 assert(tex
->bo_format
== PIPE_FORMAT_Z24X8_UNORM
);
552 assert(tex
->base
.format
== PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
);
553 assert(tex
->bo_format
== PIPE_FORMAT_Z32_FLOAT
);
560 for (slice
= 0; slice
< box
->depth
; slice
++) {
561 unsigned mem_x
, mem_y
, s8_mem_x
, s8_mem_y
;
565 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
566 box
, &mem_x
, &mem_y
);
567 tex_get_box_origin(s8_tex
, xfer
->base
.level
, slice
,
568 box
, &s8_mem_x
, &s8_mem_y
);
570 src
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
572 for (i
= 0; i
< box
->height
; i
++) {
573 unsigned x
= mem_x
, s8_x
= s8_mem_x
;
574 const uint8_t *s
= src
;
576 for (j
= 0; j
< box
->width
; j
++) {
577 const unsigned offset
=
578 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
579 const unsigned s8_offset
=
580 s8_tile_offset(s8_x
, s8_mem_y
, s8_tiles_per_row
, swizzle
);
582 memcpy(dst
+ offset
, s
, dst_cpp_used
);
583 s8_dst
[s8_offset
] = s
[src_s8_pos
];
590 src
+= xfer
->base
.stride
;
597 assert(tex
->bo_format
== PIPE_FORMAT_S8_UINT
);
599 for (slice
= 0; slice
< box
->depth
; slice
++) {
600 unsigned mem_x
, mem_y
;
604 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
605 box
, &mem_x
, &mem_y
);
607 src
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
609 for (i
= 0; i
< box
->height
; i
++) {
611 const uint8_t *s
= src
;
613 for (j
= 0; j
< box
->width
; j
++) {
614 const unsigned offset
=
615 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
623 src
+= xfer
->base
.stride
;
631 tex_staging_sys_convert_write(struct ilo_context
*ilo
,
632 struct ilo_texture
*tex
,
633 const struct ilo_transfer
*xfer
)
635 const struct pipe_box
*box
= &xfer
->base
.box
;
636 unsigned dst_slice_stride
;
640 dst
= intel_bo_get_virtual(tex
->bo
);
641 dst
+= tex_get_box_offset(tex
, xfer
->base
.level
, box
);
643 /* slice stride is not always available */
645 dst_slice_stride
= tex_get_slice_stride(tex
, xfer
->base
.level
);
647 dst_slice_stride
= 0;
649 if (unlikely(tex
->bo_format
== tex
->base
.format
)) {
650 util_copy_box(dst
, tex
->bo_format
, tex
->bo_stride
, dst_slice_stride
,
651 0, 0, 0, box
->width
, box
->height
, box
->depth
,
652 xfer
->staging_sys
, xfer
->base
.stride
, xfer
->base
.layer_stride
,
657 switch (tex
->base
.format
) {
658 case PIPE_FORMAT_ETC1_RGB8
:
659 assert(tex
->bo_format
== PIPE_FORMAT_R8G8B8X8_UNORM
);
661 for (slice
= 0; slice
< box
->depth
; slice
++) {
663 xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
665 util_format_etc1_rgb8_unpack_rgba_8unorm(dst
,
666 tex
->bo_stride
, src
, xfer
->base
.stride
,
667 box
->width
, box
->height
);
669 dst
+= dst_slice_stride
;
673 assert(!"unable to convert the staging data");
679 tex_staging_sys_map_bo(const struct ilo_context
*ilo
,
680 const struct ilo_texture
*tex
,
681 bool for_read_back
, bool linear_view
)
683 const bool prefer_cpu
= (ilo
->dev
->has_llc
|| for_read_back
);
686 if (prefer_cpu
&& (tex
->tiling
== INTEL_TILING_NONE
|| !linear_view
))
687 err
= intel_bo_map(tex
->bo
, !for_read_back
);
689 err
= intel_bo_map_gtt(tex
->bo
);
691 if (!tex
->separate_s8
)
694 err
= intel_bo_map(tex
->separate_s8
->bo
, !for_read_back
);
696 intel_bo_unmap(tex
->bo
);
702 tex_staging_sys_unmap_bo(const struct ilo_context
*ilo
,
703 const struct ilo_texture
*tex
)
705 if (tex
->separate_s8
)
706 intel_bo_unmap(tex
->separate_s8
->bo
);
708 intel_bo_unmap(tex
->bo
);
712 tex_staging_sys_unmap(struct ilo_context
*ilo
,
713 struct ilo_texture
*tex
,
714 struct ilo_transfer
*xfer
)
718 if (!(xfer
->base
.usage
& PIPE_TRANSFER_WRITE
)) {
719 FREE(xfer
->staging_sys
);
723 switch (xfer
->method
) {
724 case ILO_TRANSFER_MAP_SW_CONVERT
:
725 success
= tex_staging_sys_map_bo(ilo
, tex
, false, true);
727 tex_staging_sys_convert_write(ilo
, tex
, xfer
);
728 tex_staging_sys_unmap_bo(ilo
, tex
);
731 case ILO_TRANSFER_MAP_SW_ZS
:
732 success
= tex_staging_sys_map_bo(ilo
, tex
, false, false);
734 tex_staging_sys_zs_write(ilo
, tex
, xfer
);
735 tex_staging_sys_unmap_bo(ilo
, tex
);
739 assert(!"unknown mapping method");
745 ilo_err("failed to map resource for moving staging data\n");
747 FREE(xfer
->staging_sys
);
751 tex_staging_sys_map(struct ilo_context
*ilo
,
752 struct ilo_texture
*tex
,
753 struct ilo_transfer
*xfer
)
755 const struct pipe_box
*box
= &xfer
->base
.box
;
756 const size_t stride
= util_format_get_stride(tex
->base
.format
, box
->width
);
758 util_format_get_2d_size(tex
->base
.format
, stride
, box
->height
);
759 bool read_back
= false, success
;
761 xfer
->staging_sys
= MALLOC(size
* box
->depth
);
762 if (!xfer
->staging_sys
)
765 xfer
->base
.stride
= stride
;
766 xfer
->base
.layer_stride
= size
;
767 xfer
->ptr
= xfer
->staging_sys
;
769 /* see if we need to read the resource back */
770 if (xfer
->base
.usage
& PIPE_TRANSFER_READ
) {
773 else if (xfer
->base
.usage
& PIPE_TRANSFER_WRITE
) {
774 const unsigned discard_flags
=
775 (PIPE_TRANSFER_DISCARD_RANGE
| PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
);
777 if (!(xfer
->base
.usage
& discard_flags
))
784 switch (xfer
->method
) {
785 case ILO_TRANSFER_MAP_SW_CONVERT
:
786 assert(!"no on-the-fly format conversion for mapping");
789 case ILO_TRANSFER_MAP_SW_ZS
:
790 success
= tex_staging_sys_map_bo(ilo
, tex
, true, false);
792 tex_staging_sys_zs_read(ilo
, tex
, xfer
);
793 tex_staging_sys_unmap_bo(ilo
, tex
);
797 assert(!"unknown mapping method");
806 tex_direct_unmap(struct ilo_context
*ilo
,
807 struct ilo_texture
*tex
,
808 struct ilo_transfer
*xfer
)
810 intel_bo_unmap(tex
->bo
);
814 tex_direct_map(struct ilo_context
*ilo
,
815 struct ilo_texture
*tex
,
816 struct ilo_transfer
*xfer
)
818 if (!map_bo_for_transfer(ilo
, tex
->bo
, xfer
))
821 /* note that stride is for a block row, not a texel row */
822 xfer
->base
.stride
= tex
->bo_stride
;
824 /* slice stride is not always available */
825 if (xfer
->base
.box
.depth
> 1)
826 xfer
->base
.layer_stride
= tex_get_slice_stride(tex
, xfer
->base
.level
);
828 xfer
->base
.layer_stride
= 0;
830 xfer
->ptr
= intel_bo_get_virtual(tex
->bo
);
831 xfer
->ptr
+= tex_get_box_offset(tex
, xfer
->base
.level
, &xfer
->base
.box
);
837 tex_map(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
839 struct ilo_texture
*tex
= ilo_texture(xfer
->base
.resource
);
842 if (!choose_transfer_method(ilo
, xfer
))
845 switch (xfer
->method
) {
846 case ILO_TRANSFER_MAP_CPU
:
847 case ILO_TRANSFER_MAP_GTT
:
848 case ILO_TRANSFER_MAP_UNSYNC
:
849 success
= tex_direct_map(ilo
, tex
, xfer
);
851 case ILO_TRANSFER_MAP_SW_CONVERT
:
852 case ILO_TRANSFER_MAP_SW_ZS
:
853 success
= tex_staging_sys_map(ilo
, tex
, xfer
);
856 assert(!"unknown mapping method");
865 tex_unmap(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
867 struct ilo_texture
*tex
= ilo_texture(xfer
->base
.resource
);
869 switch (xfer
->method
) {
870 case ILO_TRANSFER_MAP_CPU
:
871 case ILO_TRANSFER_MAP_GTT
:
872 case ILO_TRANSFER_MAP_UNSYNC
:
873 tex_direct_unmap(ilo
, tex
, xfer
);
875 case ILO_TRANSFER_MAP_SW_CONVERT
:
876 case ILO_TRANSFER_MAP_SW_ZS
:
877 tex_staging_sys_unmap(ilo
, tex
, xfer
);
880 assert(!"unknown mapping method");
886 buf_map(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
888 struct ilo_buffer
*buf
= ilo_buffer(xfer
->base
.resource
);
890 if (!choose_transfer_method(ilo
, xfer
))
893 if (!map_bo_for_transfer(ilo
, buf
->bo
, xfer
))
896 assert(xfer
->base
.level
== 0);
897 assert(xfer
->base
.box
.y
== 0);
898 assert(xfer
->base
.box
.z
== 0);
899 assert(xfer
->base
.box
.height
== 1);
900 assert(xfer
->base
.box
.depth
== 1);
902 xfer
->base
.stride
= 0;
903 xfer
->base
.layer_stride
= 0;
905 xfer
->ptr
= intel_bo_get_virtual(buf
->bo
);
906 xfer
->ptr
+= xfer
->base
.box
.x
;
912 buf_unmap(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
914 struct ilo_buffer
*buf
= ilo_buffer(xfer
->base
.resource
);
916 intel_bo_unmap(buf
->bo
);
920 buf_pwrite(struct ilo_context
*ilo
, struct ilo_buffer
*buf
,
921 unsigned usage
, int offset
, int size
, const void *data
)
925 /* see if we can avoid stalling */
926 if (is_bo_busy(ilo
, buf
->bo
, &need_flush
)) {
927 bool will_stall
= true;
929 if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
930 /* old data not needed so discard the old bo to avoid stalling */
931 if (ilo_buffer_alloc_bo(buf
)) {
932 ilo_mark_states_with_resource_dirty(ilo
, &buf
->base
);
938 * We could allocate a temporary bo to hold the data and emit
939 * pipelined copy blit to move them to buf->bo. But for now, do
944 /* flush to make bo busy (so that pwrite() stalls as it should be) */
945 if (will_stall
&& need_flush
)
946 ilo_cp_flush(ilo
->cp
, "syncing for pwrites");
949 intel_bo_pwrite(buf
->bo
, offset
, size
, data
);
953 ilo_transfer_flush_region(struct pipe_context
*pipe
,
954 struct pipe_transfer
*transfer
,
955 const struct pipe_box
*box
)
960 ilo_transfer_unmap(struct pipe_context
*pipe
,
961 struct pipe_transfer
*transfer
)
963 struct ilo_context
*ilo
= ilo_context(pipe
);
964 struct ilo_transfer
*xfer
= ilo_transfer(transfer
);
966 if (xfer
->base
.resource
->target
== PIPE_BUFFER
)
967 buf_unmap(ilo
, xfer
);
969 tex_unmap(ilo
, xfer
);
971 pipe_resource_reference(&xfer
->base
.resource
, NULL
);
973 util_slab_free(&ilo
->transfer_mempool
, xfer
);
977 ilo_transfer_map(struct pipe_context
*pipe
,
978 struct pipe_resource
*res
,
981 const struct pipe_box
*box
,
982 struct pipe_transfer
**transfer
)
984 struct ilo_context
*ilo
= ilo_context(pipe
);
985 struct ilo_transfer
*xfer
;
988 xfer
= util_slab_alloc(&ilo
->transfer_mempool
);
994 xfer
->base
.resource
= NULL
;
995 pipe_resource_reference(&xfer
->base
.resource
, res
);
996 xfer
->base
.level
= level
;
997 xfer
->base
.usage
= usage
;
998 xfer
->base
.box
= *box
;
1000 if (res
->target
== PIPE_BUFFER
)
1001 success
= buf_map(ilo
, xfer
);
1003 success
= tex_map(ilo
, xfer
);
1006 pipe_resource_reference(&xfer
->base
.resource
, NULL
);
1012 *transfer
= &xfer
->base
;
1018 ilo_transfer_inline_write(struct pipe_context
*pipe
,
1019 struct pipe_resource
*res
,
1022 const struct pipe_box
*box
,
1025 unsigned layer_stride
)
1027 if (likely(res
->target
== PIPE_BUFFER
) &&
1028 !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
1029 /* they should specify just an offset and a size */
1031 assert(box
->y
== 0);
1032 assert(box
->z
== 0);
1033 assert(box
->height
== 1);
1034 assert(box
->depth
== 1);
1036 buf_pwrite(ilo_context(pipe
), ilo_buffer(res
),
1037 usage
, box
->x
, box
->width
, data
);
1040 u_default_transfer_inline_write(pipe
, res
,
1041 level
, usage
, box
, data
, stride
, layer_stride
);
1046 * Initialize transfer-related functions.
1049 ilo_init_transfer_functions(struct ilo_context
*ilo
)
1051 ilo
->base
.transfer_map
= ilo_transfer_map
;
1052 ilo
->base
.transfer_flush_region
= ilo_transfer_flush_region
;
1053 ilo
->base
.transfer_unmap
= ilo_transfer_unmap
;
1054 ilo
->base
.transfer_inline_write
= ilo_transfer_inline_write
;