2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
28 #include "tu_private.h"
30 #include "util/debug.h"
31 #include "util/u_atomic.h"
32 #include "vk_format.h"
34 #include "drm-uapi/drm_fourcc.h"
37 image_level_linear(struct tu_image
*image
, int level
, bool ubwc
)
39 unsigned w
= u_minify(image
->extent
.width
, level
);
40 /* all levels are tiled/compressed with UBWC */
41 return ubwc
? false : (w
< 16);
45 tu6_get_image_tile_mode(struct tu_image
*image
, int level
)
47 if (image_level_linear(image
, level
, !!image
->layout
.ubwc_size
))
50 return image
->layout
.tile_mode
;
54 tu_image_create(VkDevice _device
,
55 const VkImageCreateInfo
*pCreateInfo
,
56 const VkAllocationCallbacks
*alloc
,
60 TU_FROM_HANDLE(tu_device
, device
, _device
);
61 struct tu_image
*image
= NULL
;
62 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO
);
64 tu_assert(pCreateInfo
->mipLevels
> 0);
65 tu_assert(pCreateInfo
->arrayLayers
> 0);
66 tu_assert(pCreateInfo
->samples
> 0);
67 tu_assert(pCreateInfo
->extent
.width
> 0);
68 tu_assert(pCreateInfo
->extent
.height
> 0);
69 tu_assert(pCreateInfo
->extent
.depth
> 0);
71 image
= vk_zalloc2(&device
->alloc
, alloc
, sizeof(*image
), 8,
72 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
74 return vk_error(device
->instance
, VK_ERROR_OUT_OF_HOST_MEMORY
);
76 image
->type
= pCreateInfo
->imageType
;
78 image
->vk_format
= pCreateInfo
->format
;
79 image
->tiling
= pCreateInfo
->tiling
;
80 image
->usage
= pCreateInfo
->usage
;
81 image
->flags
= pCreateInfo
->flags
;
82 image
->extent
= pCreateInfo
->extent
;
83 image
->level_count
= pCreateInfo
->mipLevels
;
84 image
->layer_count
= pCreateInfo
->arrayLayers
;
85 image
->samples
= pCreateInfo
->samples
;
87 image
->exclusive
= pCreateInfo
->sharingMode
== VK_SHARING_MODE_EXCLUSIVE
;
88 if (pCreateInfo
->sharingMode
== VK_SHARING_MODE_CONCURRENT
) {
89 for (uint32_t i
= 0; i
< pCreateInfo
->queueFamilyIndexCount
; ++i
)
90 if (pCreateInfo
->pQueueFamilyIndices
[i
] ==
91 VK_QUEUE_FAMILY_EXTERNAL
)
92 image
->queue_family_mask
|= (1u << TU_MAX_QUEUE_FAMILIES
) - 1u;
94 image
->queue_family_mask
|=
95 1u << pCreateInfo
->pQueueFamilyIndices
[i
];
99 vk_find_struct_const(pCreateInfo
->pNext
,
100 EXTERNAL_MEMORY_IMAGE_CREATE_INFO
) != NULL
;
102 image
->layout
.tile_mode
= TILE6_3
;
103 bool ubwc_enabled
= true;
105 /* disable tiling when linear is requested and for compressed formats */
106 if (pCreateInfo
->tiling
== VK_IMAGE_TILING_LINEAR
||
107 modifier
== DRM_FORMAT_MOD_LINEAR
||
108 vk_format_is_compressed(image
->vk_format
)) {
109 image
->layout
.tile_mode
= TILE6_LINEAR
;
110 ubwc_enabled
= false;
113 /* using UBWC with D24S8 breaks the "stencil read" copy path (why?)
114 * (causes any deqp tests that need to check stencil to fail)
115 * disable UBWC for this format until we properly support copy aspect masks
117 if (image
->vk_format
== VK_FORMAT_D24_UNORM_S8_UINT
)
118 ubwc_enabled
= false;
120 /* UBWC can't be used with E5B9G9R9 */
121 if (image
->vk_format
== VK_FORMAT_E5B9G9R9_UFLOAT_PACK32
)
122 ubwc_enabled
= false;
124 if (image
->extent
.depth
> 1) {
125 tu_finishme("UBWC with 3D textures");
126 ubwc_enabled
= false;
129 /* Disable UBWC for storage images.
131 * The closed GL driver skips UBWC for storage images (and additionally
132 * uses linear for writeonly images). We seem to have image tiling working
133 * in freedreno in general, so turnip matches that. freedreno also enables
134 * UBWC on images, but it's not really tested due to the lack of
135 * UBWC-enabled mipmaps in freedreno currently. Just match the closed GL
136 * behavior of no UBWC.
138 if (image
->usage
& VK_IMAGE_USAGE_STORAGE_BIT
)
139 ubwc_enabled
= false;
141 uint32_t ubwc_blockwidth
, ubwc_blockheight
;
142 fdl6_get_ubwc_blockwidth(&image
->layout
,
143 &ubwc_blockwidth
, &ubwc_blockheight
);
144 if (!ubwc_blockwidth
) {
145 tu_finishme("UBWC for cpp=%d", image
->layout
.cpp
);
146 ubwc_enabled
= false;
149 /* expect UBWC enabled if we asked for it */
150 assert(modifier
!= DRM_FORMAT_MOD_QCOM_COMPRESSED
|| ubwc_enabled
);
152 fdl6_layout(&image
->layout
, vk_format_to_pipe_format(image
->vk_format
),
154 pCreateInfo
->extent
.width
,
155 pCreateInfo
->extent
.height
,
156 pCreateInfo
->extent
.depth
,
157 pCreateInfo
->mipLevels
,
158 pCreateInfo
->arrayLayers
,
159 pCreateInfo
->imageType
== VK_IMAGE_TYPE_3D
,
162 *pImage
= tu_image_to_handle(image
);
167 static enum a6xx_tex_fetchsize
168 tu6_fetchsize(VkFormat format
)
170 if (vk_format_description(format
)->layout
== UTIL_FORMAT_LAYOUT_ASTC
)
171 return TFETCH6_16_BYTE
;
173 switch (vk_format_get_blocksize(format
) / vk_format_get_blockwidth(format
)) {
174 case 1: return TFETCH6_1_BYTE
;
175 case 2: return TFETCH6_2_BYTE
;
176 case 4: return TFETCH6_4_BYTE
;
177 case 8: return TFETCH6_8_BYTE
;
178 case 16: return TFETCH6_16_BYTE
;
180 unreachable("bad block size");
185 tu6_texswiz(const VkComponentMapping
*comps
,
187 VkImageAspectFlagBits aspect_mask
)
189 unsigned char swiz
[4] = {comps
->r
, comps
->g
, comps
->b
, comps
->a
};
190 unsigned char vk_swizzle
[] = {
191 [VK_COMPONENT_SWIZZLE_ZERO
] = A6XX_TEX_ZERO
,
192 [VK_COMPONENT_SWIZZLE_ONE
] = A6XX_TEX_ONE
,
193 [VK_COMPONENT_SWIZZLE_R
] = A6XX_TEX_X
,
194 [VK_COMPONENT_SWIZZLE_G
] = A6XX_TEX_Y
,
195 [VK_COMPONENT_SWIZZLE_B
] = A6XX_TEX_Z
,
196 [VK_COMPONENT_SWIZZLE_A
] = A6XX_TEX_W
,
198 const unsigned char *fmt_swiz
= vk_format_description(format
)->swizzle
;
200 for (unsigned i
= 0; i
< 4; i
++) {
201 swiz
[i
] = (swiz
[i
] == VK_COMPONENT_SWIZZLE_IDENTITY
) ? i
: vk_swizzle
[swiz
[i
]];
202 /* if format has 0/1 in channel, use that (needed for bc1_rgb) */
204 if (aspect_mask
== VK_IMAGE_ASPECT_STENCIL_BIT
&&
205 format
== VK_FORMAT_D24_UNORM_S8_UINT
)
206 swiz
[i
] = A6XX_TEX_Y
;
207 switch (fmt_swiz
[swiz
[i
]]) {
208 case PIPE_SWIZZLE_0
: swiz
[i
] = A6XX_TEX_ZERO
; break;
209 case PIPE_SWIZZLE_1
: swiz
[i
] = A6XX_TEX_ONE
; break;
214 return A6XX_TEX_CONST_0_SWIZ_X(swiz
[0]) |
215 A6XX_TEX_CONST_0_SWIZ_Y(swiz
[1]) |
216 A6XX_TEX_CONST_0_SWIZ_Z(swiz
[2]) |
217 A6XX_TEX_CONST_0_SWIZ_W(swiz
[3]);
220 static enum a6xx_tex_type
221 tu6_tex_type(VkImageViewType type
)
225 case VK_IMAGE_VIEW_TYPE_1D
:
226 case VK_IMAGE_VIEW_TYPE_1D_ARRAY
:
228 case VK_IMAGE_VIEW_TYPE_2D
:
229 case VK_IMAGE_VIEW_TYPE_2D_ARRAY
:
231 case VK_IMAGE_VIEW_TYPE_3D
:
233 case VK_IMAGE_VIEW_TYPE_CUBE
:
234 case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY
:
235 return A6XX_TEX_CUBE
;
240 tu_image_view_init(struct tu_image_view
*iview
,
241 struct tu_device
*device
,
242 const VkImageViewCreateInfo
*pCreateInfo
)
244 TU_FROM_HANDLE(tu_image
, image
, pCreateInfo
->image
);
245 const VkImageSubresourceRange
*range
= &pCreateInfo
->subresourceRange
;
247 switch (image
->type
) {
248 case VK_IMAGE_TYPE_1D
:
249 case VK_IMAGE_TYPE_2D
:
250 assert(range
->baseArrayLayer
+ tu_get_layerCount(image
, range
) <=
253 case VK_IMAGE_TYPE_3D
:
254 assert(range
->baseArrayLayer
+ tu_get_layerCount(image
, range
) <=
255 tu_minify(image
->extent
.depth
, range
->baseMipLevel
));
258 unreachable("bad VkImageType");
261 iview
->image
= image
;
262 iview
->type
= pCreateInfo
->viewType
;
263 iview
->vk_format
= pCreateInfo
->format
;
264 iview
->aspect_mask
= pCreateInfo
->subresourceRange
.aspectMask
;
267 iview
->extent
= image
->extent
;
269 iview
->base_layer
= range
->baseArrayLayer
;
270 iview
->layer_count
= tu_get_layerCount(image
, range
);
271 iview
->base_mip
= range
->baseMipLevel
;
272 iview
->level_count
= tu_get_levelCount(image
, range
);
274 memset(iview
->descriptor
, 0, sizeof(iview
->descriptor
));
276 const struct tu_native_format
*fmt
= tu6_get_native_format(iview
->vk_format
);
277 uint64_t base_addr
= tu_image_base(image
, iview
->base_mip
, iview
->base_layer
);
278 uint64_t ubwc_addr
= tu_image_ubwc_base(image
, iview
->base_mip
, iview
->base_layer
);
280 uint32_t pitch
= tu_image_stride(image
, iview
->base_mip
) / vk_format_get_blockwidth(iview
->vk_format
);
281 enum a6xx_tile_mode tile_mode
= tu6_get_image_tile_mode(image
, iview
->base_mip
);
282 uint32_t width
= u_minify(image
->extent
.width
, iview
->base_mip
);
283 uint32_t height
= u_minify(image
->extent
.height
, iview
->base_mip
);
284 uint32_t depth
= pCreateInfo
->viewType
== VK_IMAGE_VIEW_TYPE_3D
?
285 u_minify(image
->extent
.depth
, iview
->base_mip
) : iview
->layer_count
;
287 unsigned fmt_tex
= fmt
->tex
;
288 if (iview
->aspect_mask
== VK_IMAGE_ASPECT_STENCIL_BIT
&&
289 iview
->vk_format
== VK_FORMAT_D24_UNORM_S8_UINT
)
290 fmt_tex
= TFMT6_S8Z24_UINT
;
292 iview
->descriptor
[0] =
293 A6XX_TEX_CONST_0_TILE_MODE(tile_mode
) |
294 COND(vk_format_is_srgb(iview
->vk_format
), A6XX_TEX_CONST_0_SRGB
) |
295 A6XX_TEX_CONST_0_FMT(fmt_tex
) |
296 A6XX_TEX_CONST_0_SAMPLES(tu_msaa_samples(image
->samples
)) |
297 A6XX_TEX_CONST_0_SWAP(image
->layout
.tile_mode
? WZYX
: fmt
->swap
) |
298 tu6_texswiz(&pCreateInfo
->components
, iview
->vk_format
, iview
->aspect_mask
) |
299 A6XX_TEX_CONST_0_MIPLVLS(iview
->level_count
- 1);
300 iview
->descriptor
[1] = A6XX_TEX_CONST_1_WIDTH(width
) | A6XX_TEX_CONST_1_HEIGHT(height
);
301 iview
->descriptor
[2] =
302 A6XX_TEX_CONST_2_FETCHSIZE(tu6_fetchsize(iview
->vk_format
)) |
303 A6XX_TEX_CONST_2_PITCH(pitch
) |
304 A6XX_TEX_CONST_2_TYPE(tu6_tex_type(pCreateInfo
->viewType
));
305 iview
->descriptor
[3] = A6XX_TEX_CONST_3_ARRAY_PITCH(tu_layer_size(image
, iview
->base_mip
));
306 iview
->descriptor
[4] = base_addr
;
307 iview
->descriptor
[5] = (base_addr
>> 32) | A6XX_TEX_CONST_5_DEPTH(depth
);
309 if (image
->layout
.ubwc_size
) {
310 uint32_t block_width
, block_height
;
311 fdl6_get_ubwc_blockwidth(&image
->layout
,
312 &block_width
, &block_height
);
314 iview
->descriptor
[3] |= A6XX_TEX_CONST_3_FLAG
| A6XX_TEX_CONST_3_TILE_ALL
;
315 iview
->descriptor
[7] = ubwc_addr
;
316 iview
->descriptor
[8] = ubwc_addr
>> 32;
317 iview
->descriptor
[9] |= A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH(tu_image_ubwc_size(image
, iview
->base_mip
) >> 2);
318 iview
->descriptor
[10] |=
319 A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH(tu_image_ubwc_pitch(image
, iview
->base_mip
)) |
320 A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW(util_logbase2_ceil(DIV_ROUND_UP(width
, block_width
))) |
321 A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH(util_logbase2_ceil(DIV_ROUND_UP(height
, block_height
)));
324 if (pCreateInfo
->viewType
== VK_IMAGE_VIEW_TYPE_3D
) {
325 iview
->descriptor
[3] |=
326 A6XX_TEX_CONST_3_MIN_LAYERSZ(image
->layout
.slices
[image
->level_count
- 1].size0
);
329 if (image
->usage
& VK_IMAGE_USAGE_STORAGE_BIT
) {
330 memset(iview
->storage_descriptor
, 0, sizeof(iview
->storage_descriptor
));
332 iview
->storage_descriptor
[0] =
333 A6XX_IBO_0_FMT(fmt
->tex
) |
334 A6XX_IBO_0_TILE_MODE(tile_mode
);
335 iview
->storage_descriptor
[1] =
336 A6XX_IBO_1_WIDTH(width
) |
337 A6XX_IBO_1_HEIGHT(height
);
338 iview
->storage_descriptor
[2] =
339 A6XX_IBO_2_PITCH(pitch
) |
340 A6XX_IBO_2_TYPE(tu6_tex_type(pCreateInfo
->viewType
));
341 iview
->storage_descriptor
[3] = A6XX_IBO_3_ARRAY_PITCH(tu_layer_size(image
, iview
->base_mip
));
343 iview
->storage_descriptor
[4] = base_addr
;
344 iview
->storage_descriptor
[5] = (base_addr
>> 32) | A6XX_IBO_5_DEPTH(depth
);
346 if (image
->layout
.ubwc_size
) {
347 iview
->storage_descriptor
[3] |= A6XX_IBO_3_FLAG
| A6XX_IBO_3_UNK27
;
348 iview
->storage_descriptor
[7] |= ubwc_addr
;
349 iview
->storage_descriptor
[8] |= ubwc_addr
>> 32;
350 iview
->storage_descriptor
[9] = A6XX_IBO_9_FLAG_BUFFER_ARRAY_PITCH(tu_image_ubwc_size(image
, iview
->base_mip
) >> 2);
351 iview
->storage_descriptor
[10] =
352 A6XX_IBO_10_FLAG_BUFFER_PITCH(tu_image_ubwc_pitch(image
, iview
->base_mip
));
358 tu_image_queue_family_mask(const struct tu_image
*image
,
360 uint32_t queue_family
)
362 if (!image
->exclusive
)
363 return image
->queue_family_mask
;
364 if (family
== VK_QUEUE_FAMILY_EXTERNAL
)
365 return (1u << TU_MAX_QUEUE_FAMILIES
) - 1u;
366 if (family
== VK_QUEUE_FAMILY_IGNORED
)
367 return 1u << queue_family
;
372 tu_CreateImage(VkDevice device
,
373 const VkImageCreateInfo
*pCreateInfo
,
374 const VkAllocationCallbacks
*pAllocator
,
378 const VkNativeBufferANDROID
*gralloc_info
=
379 vk_find_struct_const(pCreateInfo
->pNext
, NATIVE_BUFFER_ANDROID
);
382 return tu_image_from_gralloc(device
, pCreateInfo
, gralloc_info
,
386 uint64_t modifier
= DRM_FORMAT_MOD_INVALID
;
387 if (pCreateInfo
->tiling
== VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT
) {
388 const VkImageDrmFormatModifierListCreateInfoEXT
*mod_info
=
389 vk_find_struct_const(pCreateInfo
->pNext
,
390 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT
);
392 modifier
= DRM_FORMAT_MOD_LINEAR
;
393 for (unsigned i
= 0; i
< mod_info
->drmFormatModifierCount
; i
++) {
394 if (mod_info
->pDrmFormatModifiers
[i
] == DRM_FORMAT_MOD_QCOM_COMPRESSED
)
395 modifier
= DRM_FORMAT_MOD_QCOM_COMPRESSED
;
399 return tu_image_create(device
, pCreateInfo
, pAllocator
, pImage
, modifier
);
403 tu_DestroyImage(VkDevice _device
,
405 const VkAllocationCallbacks
*pAllocator
)
407 TU_FROM_HANDLE(tu_device
, device
, _device
);
408 TU_FROM_HANDLE(tu_image
, image
, _image
);
413 if (image
->owned_memory
!= VK_NULL_HANDLE
)
414 tu_FreeMemory(_device
, image
->owned_memory
, pAllocator
);
416 vk_free2(&device
->alloc
, pAllocator
, image
);
420 tu_GetImageSubresourceLayout(VkDevice _device
,
422 const VkImageSubresource
*pSubresource
,
423 VkSubresourceLayout
*pLayout
)
425 TU_FROM_HANDLE(tu_image
, image
, _image
);
427 const struct fdl_slice
*slice
= image
->layout
.slices
+ pSubresource
->mipLevel
;
429 pLayout
->offset
= fdl_surface_offset(&image
->layout
,
430 pSubresource
->mipLevel
,
431 pSubresource
->arrayLayer
);
432 pLayout
->size
= slice
->size0
;
434 slice
->pitch
* vk_format_get_blocksize(image
->vk_format
);
435 pLayout
->arrayPitch
= image
->layout
.layer_size
;
436 pLayout
->depthPitch
= slice
->size0
;
438 if (image
->layout
.ubwc_size
) {
439 /* UBWC starts at offset 0 */
441 /* UBWC scanout won't match what the kernel wants if we have levels/layers */
442 assert(image
->level_count
== 1 && image
->layer_count
== 1);
447 tu_CreateImageView(VkDevice _device
,
448 const VkImageViewCreateInfo
*pCreateInfo
,
449 const VkAllocationCallbacks
*pAllocator
,
452 TU_FROM_HANDLE(tu_device
, device
, _device
);
453 struct tu_image_view
*view
;
455 view
= vk_alloc2(&device
->alloc
, pAllocator
, sizeof(*view
), 8,
456 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
458 return vk_error(device
->instance
, VK_ERROR_OUT_OF_HOST_MEMORY
);
460 tu_image_view_init(view
, device
, pCreateInfo
);
462 *pView
= tu_image_view_to_handle(view
);
468 tu_DestroyImageView(VkDevice _device
,
470 const VkAllocationCallbacks
*pAllocator
)
472 TU_FROM_HANDLE(tu_device
, device
, _device
);
473 TU_FROM_HANDLE(tu_image_view
, iview
, _iview
);
477 vk_free2(&device
->alloc
, pAllocator
, iview
);
481 tu_buffer_view_init(struct tu_buffer_view
*view
,
482 struct tu_device
*device
,
483 const VkBufferViewCreateInfo
*pCreateInfo
)
485 TU_FROM_HANDLE(tu_buffer
, buffer
, pCreateInfo
->buffer
);
487 view
->range
= pCreateInfo
->range
== VK_WHOLE_SIZE
488 ? buffer
->size
- pCreateInfo
->offset
489 : pCreateInfo
->range
;
490 view
->vk_format
= pCreateInfo
->format
;
494 tu_CreateBufferView(VkDevice _device
,
495 const VkBufferViewCreateInfo
*pCreateInfo
,
496 const VkAllocationCallbacks
*pAllocator
,
499 TU_FROM_HANDLE(tu_device
, device
, _device
);
500 struct tu_buffer_view
*view
;
502 view
= vk_alloc2(&device
->alloc
, pAllocator
, sizeof(*view
), 8,
503 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
505 return vk_error(device
->instance
, VK_ERROR_OUT_OF_HOST_MEMORY
);
507 tu_buffer_view_init(view
, device
, pCreateInfo
);
509 *pView
= tu_buffer_view_to_handle(view
);
515 tu_DestroyBufferView(VkDevice _device
,
516 VkBufferView bufferView
,
517 const VkAllocationCallbacks
*pAllocator
)
519 TU_FROM_HANDLE(tu_device
, device
, _device
);
520 TU_FROM_HANDLE(tu_buffer_view
, view
, bufferView
);
525 vk_free2(&device
->alloc
, pAllocator
, view
);