turnip: Add limited support for storage images.
[mesa.git] / src / freedreno / vulkan / tu_image.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 */
27
28 #include "tu_private.h"
29
30 #include "util/debug.h"
31 #include "util/u_atomic.h"
32 #include "vk_format.h"
33 #include "vk_util.h"
34 #include "drm-uapi/drm_fourcc.h"
35
36 static inline bool
37 image_level_linear(struct tu_image *image, int level, bool ubwc)
38 {
39 unsigned w = u_minify(image->extent.width, level);
40 /* all levels are tiled/compressed with UBWC */
41 return ubwc ? false : (w < 16);
42 }
43
44 enum a6xx_tile_mode
45 tu6_get_image_tile_mode(struct tu_image *image, int level)
46 {
47 if (image_level_linear(image, level, !!image->layout.ubwc_size))
48 return TILE6_LINEAR;
49 else
50 return image->layout.tile_mode;
51 }
52
53 VkResult
54 tu_image_create(VkDevice _device,
55 const VkImageCreateInfo *pCreateInfo,
56 const VkAllocationCallbacks *alloc,
57 VkImage *pImage,
58 uint64_t modifier)
59 {
60 TU_FROM_HANDLE(tu_device, device, _device);
61 struct tu_image *image = NULL;
62 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO);
63
64 tu_assert(pCreateInfo->mipLevels > 0);
65 tu_assert(pCreateInfo->arrayLayers > 0);
66 tu_assert(pCreateInfo->samples > 0);
67 tu_assert(pCreateInfo->extent.width > 0);
68 tu_assert(pCreateInfo->extent.height > 0);
69 tu_assert(pCreateInfo->extent.depth > 0);
70
71 image = vk_zalloc2(&device->alloc, alloc, sizeof(*image), 8,
72 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
73 if (!image)
74 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
75
76 image->type = pCreateInfo->imageType;
77
78 image->vk_format = pCreateInfo->format;
79 image->tiling = pCreateInfo->tiling;
80 image->usage = pCreateInfo->usage;
81 image->flags = pCreateInfo->flags;
82 image->extent = pCreateInfo->extent;
83 image->level_count = pCreateInfo->mipLevels;
84 image->layer_count = pCreateInfo->arrayLayers;
85 image->samples = pCreateInfo->samples;
86
87 image->exclusive = pCreateInfo->sharingMode == VK_SHARING_MODE_EXCLUSIVE;
88 if (pCreateInfo->sharingMode == VK_SHARING_MODE_CONCURRENT) {
89 for (uint32_t i = 0; i < pCreateInfo->queueFamilyIndexCount; ++i)
90 if (pCreateInfo->pQueueFamilyIndices[i] ==
91 VK_QUEUE_FAMILY_EXTERNAL)
92 image->queue_family_mask |= (1u << TU_MAX_QUEUE_FAMILIES) - 1u;
93 else
94 image->queue_family_mask |=
95 1u << pCreateInfo->pQueueFamilyIndices[i];
96 }
97
98 image->shareable =
99 vk_find_struct_const(pCreateInfo->pNext,
100 EXTERNAL_MEMORY_IMAGE_CREATE_INFO) != NULL;
101
102 image->layout.tile_mode = TILE6_3;
103 bool ubwc_enabled = true;
104
105 /* disable tiling when linear is requested and for compressed formats */
106 if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR ||
107 modifier == DRM_FORMAT_MOD_LINEAR ||
108 vk_format_is_compressed(image->vk_format)) {
109 image->layout.tile_mode = TILE6_LINEAR;
110 ubwc_enabled = false;
111 }
112
113 /* using UBWC with D24S8 breaks the "stencil read" copy path (why?)
114 * (causes any deqp tests that need to check stencil to fail)
115 * disable UBWC for this format until we properly support copy aspect masks
116 */
117 if (image->vk_format == VK_FORMAT_D24_UNORM_S8_UINT)
118 ubwc_enabled = false;
119
120 /* UBWC can't be used with E5B9G9R9 */
121 if (image->vk_format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32)
122 ubwc_enabled = false;
123
124 if (image->extent.depth > 1) {
125 tu_finishme("UBWC with 3D textures");
126 ubwc_enabled = false;
127 }
128
129 uint32_t ubwc_blockwidth, ubwc_blockheight;
130 fdl6_get_ubwc_blockwidth(&image->layout,
131 &ubwc_blockwidth, &ubwc_blockheight);
132 if (!ubwc_blockwidth) {
133 tu_finishme("UBWC for cpp=%d", image->layout.cpp);
134 ubwc_enabled = false;
135 }
136
137 /* expect UBWC enabled if we asked for it */
138 assert(modifier != DRM_FORMAT_MOD_QCOM_COMPRESSED || ubwc_enabled);
139
140 fdl6_layout(&image->layout, vk_format_to_pipe_format(image->vk_format),
141 image->samples,
142 pCreateInfo->extent.width,
143 pCreateInfo->extent.height,
144 pCreateInfo->extent.depth,
145 pCreateInfo->mipLevels,
146 pCreateInfo->arrayLayers,
147 pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
148 ubwc_enabled);
149
150 *pImage = tu_image_to_handle(image);
151
152 return VK_SUCCESS;
153 }
154
155 static enum a6xx_tex_fetchsize
156 tu6_fetchsize(VkFormat format)
157 {
158 if (vk_format_description(format)->layout == UTIL_FORMAT_LAYOUT_ASTC)
159 return TFETCH6_16_BYTE;
160
161 switch (vk_format_get_blocksize(format) / vk_format_get_blockwidth(format)) {
162 case 1: return TFETCH6_1_BYTE;
163 case 2: return TFETCH6_2_BYTE;
164 case 4: return TFETCH6_4_BYTE;
165 case 8: return TFETCH6_8_BYTE;
166 case 16: return TFETCH6_16_BYTE;
167 default:
168 unreachable("bad block size");
169 }
170 }
171
172 static uint32_t
173 tu6_texswiz(const VkComponentMapping *comps,
174 VkFormat format,
175 VkImageAspectFlagBits aspect_mask)
176 {
177 unsigned char swiz[4] = {comps->r, comps->g, comps->b, comps->a};
178 unsigned char vk_swizzle[] = {
179 [VK_COMPONENT_SWIZZLE_ZERO] = A6XX_TEX_ZERO,
180 [VK_COMPONENT_SWIZZLE_ONE] = A6XX_TEX_ONE,
181 [VK_COMPONENT_SWIZZLE_R] = A6XX_TEX_X,
182 [VK_COMPONENT_SWIZZLE_G] = A6XX_TEX_Y,
183 [VK_COMPONENT_SWIZZLE_B] = A6XX_TEX_Z,
184 [VK_COMPONENT_SWIZZLE_A] = A6XX_TEX_W,
185 };
186 const unsigned char *fmt_swiz = vk_format_description(format)->swizzle;
187
188 for (unsigned i = 0; i < 4; i++) {
189 swiz[i] = (swiz[i] == VK_COMPONENT_SWIZZLE_IDENTITY) ? i : vk_swizzle[swiz[i]];
190 /* if format has 0/1 in channel, use that (needed for bc1_rgb) */
191 if (swiz[i] < 4) {
192 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT &&
193 format == VK_FORMAT_D24_UNORM_S8_UINT)
194 swiz[i] = A6XX_TEX_Y;
195 switch (fmt_swiz[swiz[i]]) {
196 case PIPE_SWIZZLE_0: swiz[i] = A6XX_TEX_ZERO; break;
197 case PIPE_SWIZZLE_1: swiz[i] = A6XX_TEX_ONE; break;
198 }
199 }
200 }
201
202 return A6XX_TEX_CONST_0_SWIZ_X(swiz[0]) |
203 A6XX_TEX_CONST_0_SWIZ_Y(swiz[1]) |
204 A6XX_TEX_CONST_0_SWIZ_Z(swiz[2]) |
205 A6XX_TEX_CONST_0_SWIZ_W(swiz[3]);
206 }
207
208 static enum a6xx_tex_type
209 tu6_tex_type(VkImageViewType type)
210 {
211 switch (type) {
212 default:
213 case VK_IMAGE_VIEW_TYPE_1D:
214 case VK_IMAGE_VIEW_TYPE_1D_ARRAY:
215 return A6XX_TEX_1D;
216 case VK_IMAGE_VIEW_TYPE_2D:
217 case VK_IMAGE_VIEW_TYPE_2D_ARRAY:
218 return A6XX_TEX_2D;
219 case VK_IMAGE_VIEW_TYPE_3D:
220 return A6XX_TEX_3D;
221 case VK_IMAGE_VIEW_TYPE_CUBE:
222 case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY:
223 return A6XX_TEX_CUBE;
224 }
225 }
226
227 void
228 tu_image_view_init(struct tu_image_view *iview,
229 struct tu_device *device,
230 const VkImageViewCreateInfo *pCreateInfo)
231 {
232 TU_FROM_HANDLE(tu_image, image, pCreateInfo->image);
233 const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
234
235 switch (image->type) {
236 case VK_IMAGE_TYPE_1D:
237 case VK_IMAGE_TYPE_2D:
238 assert(range->baseArrayLayer + tu_get_layerCount(image, range) <=
239 image->layer_count);
240 break;
241 case VK_IMAGE_TYPE_3D:
242 assert(range->baseArrayLayer + tu_get_layerCount(image, range) <=
243 tu_minify(image->extent.depth, range->baseMipLevel));
244 break;
245 default:
246 unreachable("bad VkImageType");
247 }
248
249 iview->image = image;
250 iview->type = pCreateInfo->viewType;
251 iview->vk_format = pCreateInfo->format;
252 iview->aspect_mask = pCreateInfo->subresourceRange.aspectMask;
253
254 // should we minify?
255 iview->extent = image->extent;
256
257 iview->base_layer = range->baseArrayLayer;
258 iview->layer_count = tu_get_layerCount(image, range);
259 iview->base_mip = range->baseMipLevel;
260 iview->level_count = tu_get_levelCount(image, range);
261
262 memset(iview->descriptor, 0, sizeof(iview->descriptor));
263
264 const struct tu_native_format *fmt = tu6_get_native_format(iview->vk_format);
265 uint64_t base_addr = tu_image_base(image, iview->base_mip, iview->base_layer);
266 uint64_t ubwc_addr = tu_image_ubwc_base(image, iview->base_mip, iview->base_layer);
267
268 uint32_t pitch = tu_image_stride(image, iview->base_mip) / vk_format_get_blockwidth(iview->vk_format);
269 enum a6xx_tile_mode tile_mode = tu6_get_image_tile_mode(image, iview->base_mip);
270 uint32_t width = u_minify(image->extent.width, iview->base_mip);
271 uint32_t height = u_minify(image->extent.height, iview->base_mip);
272 uint32_t depth = pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_3D ?
273 u_minify(image->extent.depth, iview->base_mip) : iview->layer_count;
274
275 unsigned fmt_tex = fmt->tex;
276 if (iview->aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT &&
277 iview->vk_format == VK_FORMAT_D24_UNORM_S8_UINT)
278 fmt_tex = TFMT6_S8Z24_UINT;
279
280 iview->descriptor[0] =
281 A6XX_TEX_CONST_0_TILE_MODE(tile_mode) |
282 COND(vk_format_is_srgb(iview->vk_format), A6XX_TEX_CONST_0_SRGB) |
283 A6XX_TEX_CONST_0_FMT(fmt_tex) |
284 A6XX_TEX_CONST_0_SAMPLES(tu_msaa_samples(image->samples)) |
285 A6XX_TEX_CONST_0_SWAP(image->layout.tile_mode ? WZYX : fmt->swap) |
286 tu6_texswiz(&pCreateInfo->components, iview->vk_format, iview->aspect_mask) |
287 A6XX_TEX_CONST_0_MIPLVLS(iview->level_count - 1);
288 iview->descriptor[1] = A6XX_TEX_CONST_1_WIDTH(width) | A6XX_TEX_CONST_1_HEIGHT(height);
289 iview->descriptor[2] =
290 A6XX_TEX_CONST_2_FETCHSIZE(tu6_fetchsize(iview->vk_format)) |
291 A6XX_TEX_CONST_2_PITCH(pitch) |
292 A6XX_TEX_CONST_2_TYPE(tu6_tex_type(pCreateInfo->viewType));
293 iview->descriptor[3] = A6XX_TEX_CONST_3_ARRAY_PITCH(tu_layer_size(image, iview->base_mip));
294 iview->descriptor[4] = base_addr;
295 iview->descriptor[5] = (base_addr >> 32) | A6XX_TEX_CONST_5_DEPTH(depth);
296
297 if (image->layout.ubwc_size) {
298 uint32_t block_width, block_height;
299 fdl6_get_ubwc_blockwidth(&image->layout,
300 &block_width, &block_height);
301
302 iview->descriptor[3] |= A6XX_TEX_CONST_3_FLAG | A6XX_TEX_CONST_3_TILE_ALL;
303 iview->descriptor[7] = ubwc_addr;
304 iview->descriptor[8] = ubwc_addr >> 32;
305 iview->descriptor[9] |= A6XX_TEX_CONST_9_FLAG_BUFFER_ARRAY_PITCH(tu_image_ubwc_size(image, iview->base_mip) >> 2);
306 iview->descriptor[10] |=
307 A6XX_TEX_CONST_10_FLAG_BUFFER_PITCH(tu_image_ubwc_pitch(image, iview->base_mip)) |
308 A6XX_TEX_CONST_10_FLAG_BUFFER_LOGW(util_logbase2_ceil(DIV_ROUND_UP(width, block_width))) |
309 A6XX_TEX_CONST_10_FLAG_BUFFER_LOGH(util_logbase2_ceil(DIV_ROUND_UP(height, block_height)));
310 }
311
312 if (pCreateInfo->viewType == VK_IMAGE_VIEW_TYPE_3D) {
313 iview->descriptor[3] |=
314 A6XX_TEX_CONST_3_MIN_LAYERSZ(image->layout.slices[image->level_count - 1].size0);
315 }
316
317 if (image->usage & VK_IMAGE_USAGE_STORAGE_BIT) {
318 memset(iview->storage_descriptor, 0, sizeof(iview->storage_descriptor));
319
320 iview->storage_descriptor[0] =
321 A6XX_IBO_0_FMT(fmt->tex) |
322 A6XX_IBO_0_TILE_MODE(tile_mode);
323 iview->storage_descriptor[1] =
324 A6XX_IBO_1_WIDTH(width) |
325 A6XX_IBO_1_HEIGHT(height);
326 iview->storage_descriptor[2] =
327 A6XX_IBO_2_PITCH(pitch) |
328 A6XX_IBO_2_TYPE(tu6_tex_type(pCreateInfo->viewType));
329 iview->storage_descriptor[3] = A6XX_IBO_3_ARRAY_PITCH(tu_layer_size(image, iview->base_mip));
330
331 iview->storage_descriptor[4] = base_addr;
332 iview->storage_descriptor[5] = (base_addr >> 32) | A6XX_IBO_5_DEPTH(depth);
333
334 if (image->layout.ubwc_size) {
335 iview->storage_descriptor[3] |= A6XX_IBO_3_FLAG | A6XX_IBO_3_UNK27;
336 iview->storage_descriptor[7] |= ubwc_addr;
337 iview->storage_descriptor[8] |= ubwc_addr >> 32;
338 iview->storage_descriptor[9] = A6XX_IBO_9_FLAG_BUFFER_ARRAY_PITCH(tu_image_ubwc_size(image, iview->base_mip) >> 2);
339 iview->storage_descriptor[10] =
340 A6XX_IBO_10_FLAG_BUFFER_PITCH(tu_image_ubwc_pitch(image, iview->base_mip));
341 }
342 }
343 }
344
345 unsigned
346 tu_image_queue_family_mask(const struct tu_image *image,
347 uint32_t family,
348 uint32_t queue_family)
349 {
350 if (!image->exclusive)
351 return image->queue_family_mask;
352 if (family == VK_QUEUE_FAMILY_EXTERNAL)
353 return (1u << TU_MAX_QUEUE_FAMILIES) - 1u;
354 if (family == VK_QUEUE_FAMILY_IGNORED)
355 return 1u << queue_family;
356 return 1u << family;
357 }
358
359 VkResult
360 tu_CreateImage(VkDevice device,
361 const VkImageCreateInfo *pCreateInfo,
362 const VkAllocationCallbacks *pAllocator,
363 VkImage *pImage)
364 {
365 #ifdef ANDROID
366 const VkNativeBufferANDROID *gralloc_info =
367 vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
368
369 if (gralloc_info)
370 return tu_image_from_gralloc(device, pCreateInfo, gralloc_info,
371 pAllocator, pImage);
372 #endif
373
374 uint64_t modifier = DRM_FORMAT_MOD_INVALID;
375 if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
376 const VkImageDrmFormatModifierListCreateInfoEXT *mod_info =
377 vk_find_struct_const(pCreateInfo->pNext,
378 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
379
380 modifier = DRM_FORMAT_MOD_LINEAR;
381 for (unsigned i = 0; i < mod_info->drmFormatModifierCount; i++) {
382 if (mod_info->pDrmFormatModifiers[i] == DRM_FORMAT_MOD_QCOM_COMPRESSED)
383 modifier = DRM_FORMAT_MOD_QCOM_COMPRESSED;
384 }
385 }
386
387 return tu_image_create(device, pCreateInfo, pAllocator, pImage, modifier);
388 }
389
390 void
391 tu_DestroyImage(VkDevice _device,
392 VkImage _image,
393 const VkAllocationCallbacks *pAllocator)
394 {
395 TU_FROM_HANDLE(tu_device, device, _device);
396 TU_FROM_HANDLE(tu_image, image, _image);
397
398 if (!image)
399 return;
400
401 if (image->owned_memory != VK_NULL_HANDLE)
402 tu_FreeMemory(_device, image->owned_memory, pAllocator);
403
404 vk_free2(&device->alloc, pAllocator, image);
405 }
406
407 void
408 tu_GetImageSubresourceLayout(VkDevice _device,
409 VkImage _image,
410 const VkImageSubresource *pSubresource,
411 VkSubresourceLayout *pLayout)
412 {
413 TU_FROM_HANDLE(tu_image, image, _image);
414
415 const struct fdl_slice *slice = image->layout.slices + pSubresource->mipLevel;
416
417 pLayout->offset = fdl_surface_offset(&image->layout,
418 pSubresource->mipLevel,
419 pSubresource->arrayLayer);
420 pLayout->size = slice->size0;
421 pLayout->rowPitch =
422 slice->pitch * vk_format_get_blocksize(image->vk_format);
423 pLayout->arrayPitch = image->layout.layer_size;
424 pLayout->depthPitch = slice->size0;
425
426 if (image->layout.ubwc_size) {
427 /* UBWC starts at offset 0 */
428 pLayout->offset = 0;
429 /* UBWC scanout won't match what the kernel wants if we have levels/layers */
430 assert(image->level_count == 1 && image->layer_count == 1);
431 }
432 }
433
434 VkResult
435 tu_CreateImageView(VkDevice _device,
436 const VkImageViewCreateInfo *pCreateInfo,
437 const VkAllocationCallbacks *pAllocator,
438 VkImageView *pView)
439 {
440 TU_FROM_HANDLE(tu_device, device, _device);
441 struct tu_image_view *view;
442
443 view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
444 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
445 if (view == NULL)
446 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
447
448 tu_image_view_init(view, device, pCreateInfo);
449
450 *pView = tu_image_view_to_handle(view);
451
452 return VK_SUCCESS;
453 }
454
455 void
456 tu_DestroyImageView(VkDevice _device,
457 VkImageView _iview,
458 const VkAllocationCallbacks *pAllocator)
459 {
460 TU_FROM_HANDLE(tu_device, device, _device);
461 TU_FROM_HANDLE(tu_image_view, iview, _iview);
462
463 if (!iview)
464 return;
465 vk_free2(&device->alloc, pAllocator, iview);
466 }
467
468 void
469 tu_buffer_view_init(struct tu_buffer_view *view,
470 struct tu_device *device,
471 const VkBufferViewCreateInfo *pCreateInfo)
472 {
473 TU_FROM_HANDLE(tu_buffer, buffer, pCreateInfo->buffer);
474
475 view->range = pCreateInfo->range == VK_WHOLE_SIZE
476 ? buffer->size - pCreateInfo->offset
477 : pCreateInfo->range;
478 view->vk_format = pCreateInfo->format;
479 }
480
481 VkResult
482 tu_CreateBufferView(VkDevice _device,
483 const VkBufferViewCreateInfo *pCreateInfo,
484 const VkAllocationCallbacks *pAllocator,
485 VkBufferView *pView)
486 {
487 TU_FROM_HANDLE(tu_device, device, _device);
488 struct tu_buffer_view *view;
489
490 view = vk_alloc2(&device->alloc, pAllocator, sizeof(*view), 8,
491 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
492 if (!view)
493 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
494
495 tu_buffer_view_init(view, device, pCreateInfo);
496
497 *pView = tu_buffer_view_to_handle(view);
498
499 return VK_SUCCESS;
500 }
501
502 void
503 tu_DestroyBufferView(VkDevice _device,
504 VkBufferView bufferView,
505 const VkAllocationCallbacks *pAllocator)
506 {
507 TU_FROM_HANDLE(tu_device, device, _device);
508 TU_FROM_HANDLE(tu_buffer_view, view, bufferView);
509
510 if (!view)
511 return;
512
513 vk_free2(&device->alloc, pAllocator, view);
514 }