2 * Mesa 3-D graphics library
4 * Copyright (C) 2012-2013 LunarG, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
25 * Chia-I Wu <olv@lunarg.com>
28 #include "util/u_surface.h"
29 #include "util/u_transfer.h"
30 #include "util/u_format_etc.h"
33 #include "ilo_context.h"
34 #include "ilo_resource.h"
35 #include "ilo_state.h"
36 #include "ilo_transfer.h"
39 is_bo_busy(struct ilo_context
*ilo
, struct intel_bo
*bo
, bool *need_flush
)
41 const bool referenced
= intel_bo_references(ilo
->cp
->bo
, bo
);
44 *need_flush
= referenced
;
49 return intel_bo_is_busy(bo
);
53 map_bo_for_transfer(struct ilo_context
*ilo
, struct intel_bo
*bo
,
54 const struct ilo_transfer
*xfer
)
58 switch (xfer
->method
) {
59 case ILO_TRANSFER_MAP_CPU
:
60 err
= intel_bo_map(bo
, (xfer
->base
.usage
& PIPE_TRANSFER_WRITE
));
62 case ILO_TRANSFER_MAP_GTT
:
63 err
= intel_bo_map_gtt(bo
);
65 case ILO_TRANSFER_MAP_UNSYNC
:
66 err
= intel_bo_map_unsynchronized(bo
);
69 assert(!"unknown mapping method");
78 * Choose the best mapping method, depending on the transfer usage and whether
82 choose_transfer_method(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
84 struct pipe_resource
*res
= xfer
->base
.resource
;
85 const unsigned usage
= xfer
->base
.usage
;
86 /* prefer map() when there is the last-level cache */
87 const bool prefer_cpu
=
88 (ilo
->dev
->has_llc
|| (usage
& PIPE_TRANSFER_READ
));
89 struct ilo_texture
*tex
;
90 struct ilo_buffer
*buf
;
92 bool tiled
, need_flush
;
94 if (res
->target
== PIPE_BUFFER
) {
97 buf
= ilo_buffer(res
);
104 tex
= ilo_texture(res
);
106 tiled
= (tex
->tiling
!= INTEL_TILING_NONE
);
109 /* choose between mapping through CPU or GTT */
110 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
111 /* we do not want fencing */
112 if (tiled
|| prefer_cpu
)
113 xfer
->method
= ILO_TRANSFER_MAP_CPU
;
115 xfer
->method
= ILO_TRANSFER_MAP_GTT
;
118 if (!tiled
&& prefer_cpu
)
119 xfer
->method
= ILO_TRANSFER_MAP_CPU
;
121 xfer
->method
= ILO_TRANSFER_MAP_GTT
;
124 /* see if we can avoid stalling */
125 if (is_bo_busy(ilo
, bo
, &need_flush
)) {
126 bool will_stall
= true;
128 if (usage
& PIPE_TRANSFER_MAP_DIRECTLY
) {
129 /* nothing we can do */
131 else if (usage
& PIPE_TRANSFER_UNSYNCHRONIZED
) {
132 /* unsynchronized gtt mapping does not stall */
133 xfer
->method
= ILO_TRANSFER_MAP_UNSYNC
;
136 else if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
137 /* discard old bo and allocate a new one for mapping */
138 if ((tex
&& ilo_texture_alloc_bo(tex
)) ||
139 (buf
&& ilo_buffer_alloc_bo(buf
))) {
140 ilo_mark_states_with_resource_dirty(ilo
, res
);
144 else if (usage
& PIPE_TRANSFER_FLUSH_EXPLICIT
) {
146 * We could allocate and return a system buffer here. When a region of
147 * the buffer is explicitly flushed, we pwrite() the region to a
148 * temporary bo and emit pipelined copy blit.
150 * For now, do nothing.
153 else if (usage
& PIPE_TRANSFER_DISCARD_RANGE
) {
155 * We could allocate a temporary bo for mapping, and emit pipelined copy
156 * blit upon unmapping.
158 * For now, do nothing.
163 if (usage
& PIPE_TRANSFER_DONTBLOCK
)
166 /* flush to make bo busy (so that map() stalls as it should be) */
168 ilo_cp_flush(ilo
->cp
);
172 if (tex
&& !(usage
& PIPE_TRANSFER_MAP_DIRECTLY
)) {
173 if (tex
->separate_s8
|| tex
->bo_format
== PIPE_FORMAT_S8_UINT
)
174 xfer
->method
= ILO_TRANSFER_MAP_SW_ZS
;
175 /* need to convert on-the-fly */
176 else if (tex
->bo_format
!= tex
->base
.format
)
177 xfer
->method
= ILO_TRANSFER_MAP_SW_CONVERT
;
184 tex_get_box_origin(const struct ilo_texture
*tex
,
185 unsigned level
, unsigned slice
,
186 const struct pipe_box
*box
,
187 unsigned *mem_x
, unsigned *mem_y
)
191 x
= tex
->slice_offsets
[level
][slice
+ box
->z
].x
+ box
->x
;
192 y
= tex
->slice_offsets
[level
][slice
+ box
->z
].y
+ box
->y
;
194 assert(x
% tex
->block_width
== 0 && y
% tex
->block_height
== 0);
196 *mem_x
= x
/ tex
->block_width
* tex
->bo_cpp
;
197 *mem_y
= y
/ tex
->block_height
;
201 tex_get_box_offset(const struct ilo_texture
*tex
, unsigned level
,
202 const struct pipe_box
*box
)
204 unsigned mem_x
, mem_y
;
206 tex_get_box_origin(tex
, level
, 0, box
, &mem_x
, &mem_y
);
208 return mem_y
* tex
->bo_stride
+ mem_x
;
212 tex_get_slice_stride(const struct ilo_texture
*tex
, unsigned level
)
216 /* there is no 3D array texture */
217 assert(tex
->base
.array_size
== 1 || tex
->base
.depth0
== 1);
219 if (tex
->base
.array_size
== 1) {
220 /* non-array, non-3D */
221 if (tex
->base
.depth0
== 1)
224 /* only the first level has a fixed slice stride */
226 assert(!"no slice stride for 3D texture with level > 0");
231 qpitch
= tex
->slice_offsets
[level
][1].y
- tex
->slice_offsets
[level
][0].y
;
232 assert(qpitch
% tex
->block_height
== 0);
234 return (qpitch
/ tex
->block_height
) * tex
->bo_stride
;
238 tex_tile_x_swizzle(unsigned addr
)
241 * From the Ivy Bridge PRM, volume 1 part 2, page 24:
243 * "As shown in the tiling algorithm, the new address bit[6] should be:
245 * Address bit[6] <= TiledAddr bit[6] XOR
246 * TiledAddr bit[9] XOR
249 return addr
^ (((addr
>> 3) ^ (addr
>> 4)) & 0x40);
253 tex_tile_y_swizzle(unsigned addr
)
256 * From the Ivy Bridge PRM, volume 1 part 2, page 24:
258 * "As shown in the tiling algorithm, The new address bit[6] becomes:
260 * Address bit[6] <= TiledAddr bit[6] XOR
263 return addr
^ ((addr
>> 3) & 0x40);
267 tex_tile_x_offset(unsigned mem_x
, unsigned mem_y
,
268 unsigned tiles_per_row
, bool swizzle
)
271 * From the Sandy Bridge PRM, volume 1 part 2, page 21, we know that a
272 * X-major tile has 8 rows and 32 OWord columns (512 bytes). Tiles in the
273 * tiled region are numbered in row-major order, starting from zero. The
274 * tile number can thus be calculated as follows:
276 * tile = (mem_y / 8) * tiles_per_row + (mem_x / 512)
278 * OWords in that tile are also numbered in row-major order, starting from
279 * zero. The OWord number can thus be calculated as follows:
281 * oword = (mem_y % 8) * 32 + ((mem_x % 512) / 16)
283 * and the tiled offset is
285 * offset = tile * 4096 + oword * 16 + (mem_x % 16)
286 * = tile * 4096 + (mem_y % 8) * 512 + (mem_x % 512)
288 unsigned tile
, offset
;
290 tile
= (mem_y
>> 3) * tiles_per_row
+ (mem_x
>> 9);
291 offset
= tile
<< 12 | (mem_y
& 0x7) << 9 | (mem_x
& 0x1ff);
293 return (swizzle
) ? tex_tile_x_swizzle(offset
) : offset
;
297 tex_tile_y_offset(unsigned mem_x
, unsigned mem_y
,
298 unsigned tiles_per_row
, bool swizzle
)
301 * From the Sandy Bridge PRM, volume 1 part 2, page 22, we know that a
302 * Y-major tile has 32 rows and 8 OWord columns (128 bytes). Tiles in the
303 * tiled region are numbered in row-major order, starting from zero. The
304 * tile number can thus be calculated as follows:
306 * tile = (mem_y / 32) * tiles_per_row + (mem_x / 128)
308 * OWords in that tile are numbered in column-major order, starting from
309 * zero. The OWord number can thus be calculated as follows:
311 * oword = ((mem_x % 128) / 16) * 32 + (mem_y % 32)
313 * and the tiled offset is
315 * offset = tile * 4096 + oword * 16 + (mem_x % 16)
317 unsigned tile
, oword
, offset
;
319 tile
= (mem_y
>> 5) * tiles_per_row
+ (mem_x
>> 7);
320 oword
= (mem_x
& 0x70) << 1 | (mem_y
& 0x1f);
321 offset
= tile
<< 12 | oword
<< 4 | (mem_x
& 0xf);
323 return (swizzle
) ? tex_tile_y_swizzle(offset
) : offset
;
327 tex_tile_w_offset(unsigned mem_x
, unsigned mem_y
,
328 unsigned tiles_per_row
, bool swizzle
)
331 * From the Sandy Bridge PRM, volume 1 part 2, page 23, we know that a
332 * W-major tile has 8 8x8-block rows and 8 8x8-block columns. Tiles in the
333 * tiled region are numbered in row-major order, starting from zero. The
334 * tile number can thus be calculated as follows:
336 * tile = (mem_y / 64) * tiles_per_row + (mem_x / 64)
338 * 8x8-blocks in that tile are numbered in column-major order, starting
339 * from zero. The 8x8-block number can thus be calculated as follows:
341 * blk8 = ((mem_x % 64) / 8) * 8 + ((mem_y % 64) / 8)
343 * Each 8x8-block is divided into 4 4x4-blocks, in row-major order. Each
344 * 4x4-block is further divided into 4 2x2-blocks, also in row-major order.
347 * blk4 = (((mem_y % 64) / 4) & 1) * 2 + (((mem_x % 64) / 4) & 1)
348 * blk2 = (((mem_y % 64) / 2) & 1) * 2 + (((mem_x % 64) / 2) & 1)
349 * blk1 = (((mem_y % 64) ) & 1) * 2 + (((mem_x % 64) ) & 1)
351 * and the tiled offset is
353 * offset = tile * 4096 + blk8 * 64 + blk4 * 16 + blk2 * 4 + blk1
355 unsigned tile
, blk8
, blk4
, blk2
, blk1
, offset
;
357 tile
= (mem_y
>> 6) * tiles_per_row
+ (mem_x
>> 6);
358 blk8
= ((mem_x
>> 3) & 0x7) << 3 | ((mem_y
>> 3) & 0x7);
359 blk4
= ((mem_y
>> 2) & 0x1) << 1 | ((mem_x
>> 2) & 0x1);
360 blk2
= ((mem_y
>> 1) & 0x1) << 1 | ((mem_x
>> 1) & 0x1);
361 blk1
= ((mem_y
) & 0x1) << 1 | ((mem_x
) & 0x1);
362 offset
= tile
<< 12 | blk8
<< 6 | blk4
<< 4 | blk2
<< 2 | blk1
;
364 return (swizzle
) ? tex_tile_y_swizzle(offset
) : offset
;
368 tex_tile_none_offset(unsigned mem_x
, unsigned mem_y
,
369 unsigned tiles_per_row
, bool swizzle
)
371 return mem_y
* tiles_per_row
+ mem_x
;
374 typedef unsigned (*tex_tile_offset_func
)(unsigned mem_x
, unsigned mem_y
,
375 unsigned tiles_per_row
,
378 static tex_tile_offset_func
379 tex_tile_choose_offset_func(const struct ilo_texture
*tex
,
380 unsigned *tiles_per_row
)
382 switch (tex
->tiling
) {
384 *tiles_per_row
= tex
->bo_stride
/ 512;
385 return tex_tile_x_offset
;
387 *tiles_per_row
= tex
->bo_stride
/ 128;
388 return tex_tile_y_offset
;
389 case INTEL_TILING_NONE
:
392 if (tex
->bo_format
== PIPE_FORMAT_S8_UINT
) {
393 *tiles_per_row
= tex
->bo_stride
/ 64;
394 return tex_tile_w_offset
;
397 *tiles_per_row
= tex
->bo_stride
;
398 return tex_tile_none_offset
;
404 tex_staging_sys_zs_read(struct ilo_context
*ilo
,
405 struct ilo_texture
*tex
,
406 const struct ilo_transfer
*xfer
)
408 const bool swizzle
= ilo
->dev
->has_address_swizzling
;
409 const struct pipe_box
*box
= &xfer
->base
.box
;
410 const uint8_t *src
= intel_bo_get_virtual(tex
->bo
);
411 tex_tile_offset_func tile_offset
;
412 unsigned tiles_per_row
;
415 tile_offset
= tex_tile_choose_offset_func(tex
, &tiles_per_row
);
417 assert(tex
->block_width
== 1 && tex
->block_height
== 1);
419 if (tex
->separate_s8
) {
420 struct ilo_texture
*s8_tex
= tex
->separate_s8
;
421 const uint8_t *s8_src
= intel_bo_get_virtual(s8_tex
->bo
);
422 tex_tile_offset_func s8_tile_offset
;
423 unsigned s8_tiles_per_row
;
424 int dst_cpp
, dst_s8_pos
, src_cpp_used
;
426 s8_tile_offset
= tex_tile_choose_offset_func(s8_tex
, &s8_tiles_per_row
);
428 if (tex
->base
.format
== PIPE_FORMAT_Z24_UNORM_S8_UINT
) {
429 assert(tex
->bo_format
== PIPE_FORMAT_Z24X8_UNORM
);
436 assert(tex
->base
.format
== PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
);
437 assert(tex
->bo_format
== PIPE_FORMAT_Z32_FLOAT
);
444 for (slice
= 0; slice
< box
->depth
; slice
++) {
445 unsigned mem_x
, mem_y
, s8_mem_x
, s8_mem_y
;
449 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
450 box
, &mem_x
, &mem_y
);
451 tex_get_box_origin(s8_tex
, xfer
->base
.level
, slice
,
452 box
, &s8_mem_x
, &s8_mem_y
);
454 dst
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
456 for (i
= 0; i
< box
->height
; i
++) {
457 unsigned x
= mem_x
, s8_x
= s8_mem_x
;
460 for (j
= 0; j
< box
->width
; j
++) {
461 const unsigned offset
=
462 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
463 const unsigned s8_offset
=
464 s8_tile_offset(s8_x
, s8_mem_y
, s8_tiles_per_row
, swizzle
);
466 memcpy(d
, src
+ offset
, src_cpp_used
);
467 d
[dst_s8_pos
] = s8_src
[s8_offset
];
474 dst
+= xfer
->base
.stride
;
481 assert(tex
->bo_format
== PIPE_FORMAT_S8_UINT
);
483 for (slice
= 0; slice
< box
->depth
; slice
++) {
484 unsigned mem_x
, mem_y
;
488 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
489 box
, &mem_x
, &mem_y
);
491 dst
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
493 for (i
= 0; i
< box
->height
; i
++) {
497 for (j
= 0; j
< box
->width
; j
++) {
498 const unsigned offset
=
499 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
507 dst
+= xfer
->base
.stride
;
515 tex_staging_sys_zs_write(struct ilo_context
*ilo
,
516 struct ilo_texture
*tex
,
517 const struct ilo_transfer
*xfer
)
519 const bool swizzle
= ilo
->dev
->has_address_swizzling
;
520 const struct pipe_box
*box
= &xfer
->base
.box
;
521 uint8_t *dst
= intel_bo_get_virtual(tex
->bo
);
522 tex_tile_offset_func tile_offset
;
523 unsigned tiles_per_row
;
526 tile_offset
= tex_tile_choose_offset_func(tex
, &tiles_per_row
);
528 assert(tex
->block_width
== 1 && tex
->block_height
== 1);
530 if (tex
->separate_s8
) {
531 struct ilo_texture
*s8_tex
= tex
->separate_s8
;
532 uint8_t *s8_dst
= intel_bo_get_virtual(s8_tex
->bo
);
533 tex_tile_offset_func s8_tile_offset
;
534 unsigned s8_tiles_per_row
;
535 int src_cpp
, src_s8_pos
, dst_cpp_used
;
537 s8_tile_offset
= tex_tile_choose_offset_func(s8_tex
, &s8_tiles_per_row
);
539 if (tex
->base
.format
== PIPE_FORMAT_Z24_UNORM_S8_UINT
) {
540 assert(tex
->bo_format
== PIPE_FORMAT_Z24X8_UNORM
);
547 assert(tex
->base
.format
== PIPE_FORMAT_Z32_FLOAT_S8X24_UINT
);
548 assert(tex
->bo_format
== PIPE_FORMAT_Z32_FLOAT
);
555 for (slice
= 0; slice
< box
->depth
; slice
++) {
556 unsigned mem_x
, mem_y
, s8_mem_x
, s8_mem_y
;
560 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
561 box
, &mem_x
, &mem_y
);
562 tex_get_box_origin(s8_tex
, xfer
->base
.level
, slice
,
563 box
, &s8_mem_x
, &s8_mem_y
);
565 src
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
567 for (i
= 0; i
< box
->height
; i
++) {
568 unsigned x
= mem_x
, s8_x
= s8_mem_x
;
569 const uint8_t *s
= src
;
571 for (j
= 0; j
< box
->width
; j
++) {
572 const unsigned offset
=
573 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
574 const unsigned s8_offset
=
575 s8_tile_offset(s8_x
, s8_mem_y
, s8_tiles_per_row
, swizzle
);
577 memcpy(dst
+ offset
, s
, dst_cpp_used
);
578 s8_dst
[s8_offset
] = s
[src_s8_pos
];
585 src
+= xfer
->base
.stride
;
592 assert(tex
->bo_format
== PIPE_FORMAT_S8_UINT
);
594 for (slice
= 0; slice
< box
->depth
; slice
++) {
595 unsigned mem_x
, mem_y
;
599 tex_get_box_origin(tex
, xfer
->base
.level
, slice
,
600 box
, &mem_x
, &mem_y
);
602 src
= xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
604 for (i
= 0; i
< box
->height
; i
++) {
606 const uint8_t *s
= src
;
608 for (j
= 0; j
< box
->width
; j
++) {
609 const unsigned offset
=
610 tile_offset(x
, mem_y
, tiles_per_row
, swizzle
);
618 src
+= xfer
->base
.stride
;
626 tex_staging_sys_convert_write(struct ilo_context
*ilo
,
627 struct ilo_texture
*tex
,
628 const struct ilo_transfer
*xfer
)
630 const struct pipe_box
*box
= &xfer
->base
.box
;
631 unsigned dst_slice_stride
;
635 dst
= intel_bo_get_virtual(tex
->bo
);
636 dst
+= tex_get_box_offset(tex
, xfer
->base
.level
, box
);
638 /* slice stride is not always available */
640 dst_slice_stride
= tex_get_slice_stride(tex
, xfer
->base
.level
);
642 dst_slice_stride
= 0;
644 if (unlikely(tex
->bo_format
== tex
->base
.format
)) {
645 util_copy_box(dst
, tex
->bo_format
, tex
->bo_stride
, dst_slice_stride
,
646 0, 0, 0, box
->width
, box
->height
, box
->depth
,
647 xfer
->staging_sys
, xfer
->base
.stride
, xfer
->base
.layer_stride
,
652 switch (tex
->base
.format
) {
653 case PIPE_FORMAT_ETC1_RGB8
:
654 assert(tex
->bo_format
== PIPE_FORMAT_R8G8B8X8_UNORM
);
656 for (slice
= 0; slice
< box
->depth
; slice
++) {
658 xfer
->staging_sys
+ xfer
->base
.layer_stride
* slice
;
660 util_format_etc1_rgb8_unpack_rgba_8unorm(dst
,
661 tex
->bo_stride
, src
, xfer
->base
.stride
,
662 box
->width
, box
->height
);
664 dst
+= dst_slice_stride
;
668 assert(!"unable to convert the staging data");
674 tex_staging_sys_map_bo(const struct ilo_context
*ilo
,
675 const struct ilo_texture
*tex
,
676 bool for_read_back
, bool linear_view
)
678 const bool prefer_cpu
= (ilo
->dev
->has_llc
|| for_read_back
);
681 if (prefer_cpu
&& (tex
->tiling
== INTEL_TILING_NONE
|| !linear_view
))
682 err
= intel_bo_map(tex
->bo
, !for_read_back
);
684 err
= intel_bo_map_gtt(tex
->bo
);
686 if (!tex
->separate_s8
)
689 err
= intel_bo_map(tex
->separate_s8
->bo
, !for_read_back
);
691 intel_bo_unmap(tex
->bo
);
697 tex_staging_sys_unmap_bo(const struct ilo_context
*ilo
,
698 const struct ilo_texture
*tex
)
700 if (tex
->separate_s8
)
701 intel_bo_unmap(tex
->separate_s8
->bo
);
703 intel_bo_unmap(tex
->bo
);
707 tex_staging_sys_unmap(struct ilo_context
*ilo
,
708 struct ilo_texture
*tex
,
709 struct ilo_transfer
*xfer
)
713 if (!(xfer
->base
.usage
& PIPE_TRANSFER_WRITE
)) {
714 FREE(xfer
->staging_sys
);
718 switch (xfer
->method
) {
719 case ILO_TRANSFER_MAP_SW_CONVERT
:
720 success
= tex_staging_sys_map_bo(ilo
, tex
, false, true);
722 tex_staging_sys_convert_write(ilo
, tex
, xfer
);
723 tex_staging_sys_unmap_bo(ilo
, tex
);
726 case ILO_TRANSFER_MAP_SW_ZS
:
727 success
= tex_staging_sys_map_bo(ilo
, tex
, false, false);
729 tex_staging_sys_zs_write(ilo
, tex
, xfer
);
730 tex_staging_sys_unmap_bo(ilo
, tex
);
734 assert(!"unknown mapping method");
740 ilo_err("failed to map resource for moving staging data\n");
742 FREE(xfer
->staging_sys
);
746 tex_staging_sys_map(struct ilo_context
*ilo
,
747 struct ilo_texture
*tex
,
748 struct ilo_transfer
*xfer
)
750 const struct pipe_box
*box
= &xfer
->base
.box
;
751 const size_t stride
= util_format_get_stride(tex
->base
.format
, box
->width
);
753 util_format_get_2d_size(tex
->base
.format
, stride
, box
->height
);
754 bool read_back
= false, success
;
756 xfer
->staging_sys
= MALLOC(size
* box
->depth
);
757 if (!xfer
->staging_sys
)
760 xfer
->base
.stride
= stride
;
761 xfer
->base
.layer_stride
= size
;
762 xfer
->ptr
= xfer
->staging_sys
;
764 /* see if we need to read the resource back */
765 if (xfer
->base
.usage
& PIPE_TRANSFER_READ
) {
768 else if (xfer
->base
.usage
& PIPE_TRANSFER_WRITE
) {
769 const unsigned discard_flags
=
770 (PIPE_TRANSFER_DISCARD_RANGE
| PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
);
772 if (!(xfer
->base
.usage
& discard_flags
))
779 switch (xfer
->method
) {
780 case ILO_TRANSFER_MAP_SW_CONVERT
:
781 assert(!"no on-the-fly format conversion for mapping");
784 case ILO_TRANSFER_MAP_SW_ZS
:
785 success
= tex_staging_sys_map_bo(ilo
, tex
, true, false);
787 tex_staging_sys_zs_read(ilo
, tex
, xfer
);
788 tex_staging_sys_unmap_bo(ilo
, tex
);
792 assert(!"unknown mapping method");
801 tex_direct_unmap(struct ilo_context
*ilo
,
802 struct ilo_texture
*tex
,
803 struct ilo_transfer
*xfer
)
805 intel_bo_unmap(tex
->bo
);
809 tex_direct_map(struct ilo_context
*ilo
,
810 struct ilo_texture
*tex
,
811 struct ilo_transfer
*xfer
)
813 if (!map_bo_for_transfer(ilo
, tex
->bo
, xfer
))
816 /* note that stride is for a block row, not a texel row */
817 xfer
->base
.stride
= tex
->bo_stride
;
819 /* slice stride is not always available */
820 if (xfer
->base
.box
.depth
> 1)
821 xfer
->base
.layer_stride
= tex_get_slice_stride(tex
, xfer
->base
.level
);
823 xfer
->base
.layer_stride
= 0;
825 xfer
->ptr
= intel_bo_get_virtual(tex
->bo
);
826 xfer
->ptr
+= tex_get_box_offset(tex
, xfer
->base
.level
, &xfer
->base
.box
);
832 tex_map(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
834 struct ilo_texture
*tex
= ilo_texture(xfer
->base
.resource
);
837 if (!choose_transfer_method(ilo
, xfer
))
840 switch (xfer
->method
) {
841 case ILO_TRANSFER_MAP_CPU
:
842 case ILO_TRANSFER_MAP_GTT
:
843 case ILO_TRANSFER_MAP_UNSYNC
:
844 success
= tex_direct_map(ilo
, tex
, xfer
);
846 case ILO_TRANSFER_MAP_SW_CONVERT
:
847 case ILO_TRANSFER_MAP_SW_ZS
:
848 success
= tex_staging_sys_map(ilo
, tex
, xfer
);
851 assert(!"unknown mapping method");
860 tex_unmap(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
862 struct ilo_texture
*tex
= ilo_texture(xfer
->base
.resource
);
864 switch (xfer
->method
) {
865 case ILO_TRANSFER_MAP_CPU
:
866 case ILO_TRANSFER_MAP_GTT
:
867 case ILO_TRANSFER_MAP_UNSYNC
:
868 tex_direct_unmap(ilo
, tex
, xfer
);
870 case ILO_TRANSFER_MAP_SW_CONVERT
:
871 case ILO_TRANSFER_MAP_SW_ZS
:
872 tex_staging_sys_unmap(ilo
, tex
, xfer
);
875 assert(!"unknown mapping method");
881 buf_map(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
883 struct ilo_buffer
*buf
= ilo_buffer(xfer
->base
.resource
);
885 if (!choose_transfer_method(ilo
, xfer
))
888 if (!map_bo_for_transfer(ilo
, buf
->bo
, xfer
))
891 assert(xfer
->base
.level
== 0);
892 assert(xfer
->base
.box
.y
== 0);
893 assert(xfer
->base
.box
.z
== 0);
894 assert(xfer
->base
.box
.height
== 1);
895 assert(xfer
->base
.box
.depth
== 1);
897 xfer
->base
.stride
= 0;
898 xfer
->base
.layer_stride
= 0;
900 xfer
->ptr
= intel_bo_get_virtual(buf
->bo
);
901 xfer
->ptr
+= xfer
->base
.box
.x
;
907 buf_unmap(struct ilo_context
*ilo
, struct ilo_transfer
*xfer
)
909 struct ilo_buffer
*buf
= ilo_buffer(xfer
->base
.resource
);
911 intel_bo_unmap(buf
->bo
);
915 buf_pwrite(struct ilo_context
*ilo
, struct ilo_buffer
*buf
,
916 unsigned usage
, int offset
, int size
, const void *data
)
920 /* see if we can avoid stalling */
921 if (is_bo_busy(ilo
, buf
->bo
, &need_flush
)) {
922 bool will_stall
= true;
924 if (usage
& PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
) {
925 /* old data not needed so discard the old bo to avoid stalling */
926 if (ilo_buffer_alloc_bo(buf
)) {
927 ilo_mark_states_with_resource_dirty(ilo
, &buf
->base
);
933 * We could allocate a temporary bo to hold the data and emit
934 * pipelined copy blit to move them to buf->bo. But for now, do
939 /* flush to make bo busy (so that pwrite() stalls as it should be) */
940 if (will_stall
&& need_flush
)
941 ilo_cp_flush(ilo
->cp
);
944 intel_bo_pwrite(buf
->bo
, offset
, size
, data
);
948 ilo_transfer_flush_region(struct pipe_context
*pipe
,
949 struct pipe_transfer
*transfer
,
950 const struct pipe_box
*box
)
955 ilo_transfer_unmap(struct pipe_context
*pipe
,
956 struct pipe_transfer
*transfer
)
958 struct ilo_context
*ilo
= ilo_context(pipe
);
959 struct ilo_transfer
*xfer
= ilo_transfer(transfer
);
961 if (xfer
->base
.resource
->target
== PIPE_BUFFER
)
962 buf_unmap(ilo
, xfer
);
964 tex_unmap(ilo
, xfer
);
966 pipe_resource_reference(&xfer
->base
.resource
, NULL
);
968 util_slab_free(&ilo
->transfer_mempool
, xfer
);
972 ilo_transfer_map(struct pipe_context
*pipe
,
973 struct pipe_resource
*res
,
976 const struct pipe_box
*box
,
977 struct pipe_transfer
**transfer
)
979 struct ilo_context
*ilo
= ilo_context(pipe
);
980 struct ilo_transfer
*xfer
;
983 xfer
= util_slab_alloc(&ilo
->transfer_mempool
);
989 xfer
->base
.resource
= NULL
;
990 pipe_resource_reference(&xfer
->base
.resource
, res
);
991 xfer
->base
.level
= level
;
992 xfer
->base
.usage
= usage
;
993 xfer
->base
.box
= *box
;
995 if (res
->target
== PIPE_BUFFER
)
996 success
= buf_map(ilo
, xfer
);
998 success
= tex_map(ilo
, xfer
);
1001 pipe_resource_reference(&xfer
->base
.resource
, NULL
);
1007 *transfer
= &xfer
->base
;
1013 ilo_transfer_inline_write(struct pipe_context
*pipe
,
1014 struct pipe_resource
*res
,
1017 const struct pipe_box
*box
,
1020 unsigned layer_stride
)
1022 if (likely(res
->target
== PIPE_BUFFER
) &&
1023 !(usage
& PIPE_TRANSFER_UNSYNCHRONIZED
)) {
1024 /* they should specify just an offset and a size */
1026 assert(box
->y
== 0);
1027 assert(box
->z
== 0);
1028 assert(box
->height
== 1);
1029 assert(box
->depth
== 1);
1031 buf_pwrite(ilo_context(pipe
), ilo_buffer(res
),
1032 usage
, box
->x
, box
->width
, data
);
1035 u_default_transfer_inline_write(pipe
, res
,
1036 level
, usage
, box
, data
, stride
, layer_stride
);
1041 * Initialize transfer-related functions.
1044 ilo_init_transfer_functions(struct ilo_context
*ilo
)
1046 ilo
->base
.transfer_map
= ilo_transfer_map
;
1047 ilo
->base
.transfer_flush_region
= ilo_transfer_flush_region
;
1048 ilo
->base
.transfer_unmap
= ilo_transfer_unmap
;
1049 ilo
->base
.transfer_inline_write
= ilo_transfer_inline_write
;