projects / mesa.git / blob
? search:


10571531621e9342bd0f83fff8002ed0a89de222
[mesa.git] / src / amd / vulkan / radv_device.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * IN THE SOFTWARE.
26 */
27
28 #include "dirent.h"
29 #include <errno.h>
30 #include <fcntl.h>
31 #include <linux/audit.h>
32 #include <linux/bpf.h>
33 #include <linux/filter.h>
34 #include <linux/seccomp.h>
35 #include <linux/unistd.h>
36 #include <stdbool.h>
37 #include <stddef.h>
38 #include <stdio.h>
39 #include <string.h>
40 #include <sys/prctl.h>
41 #include <sys/wait.h>
42 #include <unistd.h>
43 #include <fcntl.h>
44
45 #include "radv_debug.h"
46 #include "radv_private.h"
47 #include "radv_shader.h"
48 #include "radv_cs.h"
49 #include "util/disk_cache.h"
50 #include "vk_util.h"
51 #include <xf86drm.h>
52 #include <amdgpu.h>
53 #include "drm-uapi/amdgpu_drm.h"
54 #include "winsys/amdgpu/radv_amdgpu_winsys_public.h"
55 #include "winsys/null/radv_null_winsys_public.h"
56 #include "ac_llvm_util.h"
57 #include "vk_format.h"
58 #include "sid.h"
59 #include "git_sha1.h"
60 #include "util/build_id.h"
61 #include "util/debug.h"
62 #include "util/mesa-sha1.h"
63 #include "util/timespec.h"
64 #include "util/u_atomic.h"
65 #include "compiler/glsl_types.h"
66 #include "util/xmlpool.h"
67
68 static struct radv_timeline_point *
69 radv_timeline_find_point_at_least_locked(struct radv_device *device,
70 struct radv_timeline *timeline,
71 uint64_t p);
72
73 static struct radv_timeline_point *
74 radv_timeline_add_point_locked(struct radv_device *device,
75 struct radv_timeline *timeline,
76 uint64_t p);
77
78 static void
79 radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline,
80 struct list_head *processing_list);
81
82 static
83 void radv_destroy_semaphore_part(struct radv_device *device,
84 struct radv_semaphore_part *part);
85
86 static int
87 radv_device_get_cache_uuid(enum radeon_family family, void *uuid)
88 {
89 struct mesa_sha1 ctx;
90 unsigned char sha1[20];
91 unsigned ptr_size = sizeof(void*);
92
93 memset(uuid, 0, VK_UUID_SIZE);
94 _mesa_sha1_init(&ctx);
95
96 if (!disk_cache_get_function_identifier(radv_device_get_cache_uuid, &ctx) ||
97 !disk_cache_get_function_identifier(LLVMInitializeAMDGPUTargetInfo, &ctx))
98 return -1;
99
100 _mesa_sha1_update(&ctx, &family, sizeof(family));
101 _mesa_sha1_update(&ctx, &ptr_size, sizeof(ptr_size));
102 _mesa_sha1_final(&ctx, sha1);
103
104 memcpy(uuid, sha1, VK_UUID_SIZE);
105 return 0;
106 }
107
108 static void
109 radv_get_driver_uuid(void *uuid)
110 {
111 ac_compute_driver_uuid(uuid, VK_UUID_SIZE);
112 }
113
114 static void
115 radv_get_device_uuid(struct radeon_info *info, void *uuid)
116 {
117 ac_compute_device_uuid(info, uuid, VK_UUID_SIZE);
118 }
119
120 static uint64_t
121 radv_get_visible_vram_size(struct radv_physical_device *device)
122 {
123 return MIN2(device->rad_info.vram_size, device->rad_info.vram_vis_size);
124 }
125
126 static uint64_t
127 radv_get_vram_size(struct radv_physical_device *device)
128 {
129 return device->rad_info.vram_size - radv_get_visible_vram_size(device);
130 }
131
132 static void
133 radv_physical_device_init_mem_types(struct radv_physical_device *device)
134 {
135 uint64_t visible_vram_size = radv_get_visible_vram_size(device);
136 uint64_t vram_size = radv_get_vram_size(device);
137 int vram_index = -1, visible_vram_index = -1, gart_index = -1;
138 device->memory_properties.memoryHeapCount = 0;
139 if (vram_size > 0) {
140 vram_index = device->memory_properties.memoryHeapCount++;
141 device->memory_properties.memoryHeaps[vram_index] = (VkMemoryHeap) {
142 .size = vram_size,
143 .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
144 };
145 }
146
147 if (device->rad_info.gart_size > 0) {
148 gart_index = device->memory_properties.memoryHeapCount++;
149 device->memory_properties.memoryHeaps[gart_index] = (VkMemoryHeap) {
150 .size = device->rad_info.gart_size,
151 .flags = 0,
152 };
153 }
154
155 if (visible_vram_size) {
156 visible_vram_index = device->memory_properties.memoryHeapCount++;
157 device->memory_properties.memoryHeaps[visible_vram_index] = (VkMemoryHeap) {
158 .size = visible_vram_size,
159 .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
160 };
161 }
162
163 unsigned type_count = 0;
164
165 if (device->rad_info.has_dedicated_vram) {
166 if (vram_index >= 0) {
167 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
168 device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS;
169 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
170 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
171 .heapIndex = vram_index,
172 };
173 }
174 } else {
175 if (visible_vram_index >= 0) {
176 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
177 device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS;
178 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
179 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
180 .heapIndex = visible_vram_index,
181 };
182 }
183 }
184
185 if (gart_index >= 0) {
186 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
187 device->memory_flags[type_count] = RADEON_FLAG_GTT_WC | RADEON_FLAG_CPU_ACCESS;
188 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
189 .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
190 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
191 .heapIndex = gart_index,
192 };
193 }
194 if (visible_vram_index >= 0) {
195 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
196 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
197 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
198 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
199 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
200 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
201 .heapIndex = visible_vram_index,
202 };
203 }
204
205 if (gart_index >= 0) {
206 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
207 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
208 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
209 .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
210 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
211 VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
212 .heapIndex = gart_index,
213 };
214 }
215 device->memory_properties.memoryTypeCount = type_count;
216
217 if (device->rad_info.has_l2_uncached) {
218 for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
219 VkMemoryType mem_type = device->memory_properties.memoryTypes[i];
220
221 if ((mem_type.propertyFlags & (VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
222 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) ||
223 mem_type.propertyFlags == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
224
225 VkMemoryPropertyFlags property_flags = mem_type.propertyFlags |
226 VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD |
227 VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD;
228
229 device->memory_domains[type_count] = device->memory_domains[i];
230 device->memory_flags[type_count] = device->memory_flags[i] | RADEON_FLAG_VA_UNCACHED;
231 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
232 .propertyFlags = property_flags,
233 .heapIndex = mem_type.heapIndex,
234 };
235 }
236 }
237 device->memory_properties.memoryTypeCount = type_count;
238 }
239 }
240
241 static VkResult
242 radv_physical_device_init(struct radv_physical_device *device,
243 struct radv_instance *instance,
244 drmDevicePtr drm_device)
245 {
246 VkResult result;
247 int fd = -1;
248 int master_fd = -1;
249
250 if (drm_device) {
251 const char *path = drm_device->nodes[DRM_NODE_RENDER];
252 drmVersionPtr version;
253
254 fd = open(path, O_RDWR | O_CLOEXEC);
255 if (fd < 0) {
256 if (instance->debug_flags & RADV_DEBUG_STARTUP)
257 radv_logi("Could not open device '%s'", path);
258
259 return vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER);
260 }
261
262 version = drmGetVersion(fd);
263 if (!version) {
264 close(fd);
265
266 if (instance->debug_flags & RADV_DEBUG_STARTUP)
267 radv_logi("Could not get the kernel driver version for device '%s'", path);
268
269 return vk_errorf(instance, VK_ERROR_INCOMPATIBLE_DRIVER,
270 "failed to get version %s: %m", path);
271 }
272
273 if (strcmp(version->name, "amdgpu")) {
274 drmFreeVersion(version);
275 close(fd);
276
277 if (instance->debug_flags & RADV_DEBUG_STARTUP)
278 radv_logi("Device '%s' is not using the amdgpu kernel driver.", path);
279
280 return VK_ERROR_INCOMPATIBLE_DRIVER;
281 }
282 drmFreeVersion(version);
283
284 if (instance->debug_flags & RADV_DEBUG_STARTUP)
285 radv_logi("Found compatible device '%s'.", path);
286 }
287
288 device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
289 device->instance = instance;
290
291 if (drm_device) {
292 device->ws = radv_amdgpu_winsys_create(fd, instance->debug_flags,
293 instance->perftest_flags);
294 } else {
295 device->ws = radv_null_winsys_create();
296 }
297
298 if (!device->ws) {
299 result = vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER);
300 goto fail;
301 }
302
303 if (drm_device && instance->enabled_extensions.KHR_display) {
304 master_fd = open(drm_device->nodes[DRM_NODE_PRIMARY], O_RDWR | O_CLOEXEC);
305 if (master_fd >= 0) {
306 uint32_t accel_working = 0;
307 struct drm_amdgpu_info request = {
308 .return_pointer = (uintptr_t)&accel_working,
309 .return_size = sizeof(accel_working),
310 .query = AMDGPU_INFO_ACCEL_WORKING
311 };
312
313 if (drmCommandWrite(master_fd, DRM_AMDGPU_INFO, &request, sizeof (struct drm_amdgpu_info)) < 0 || !accel_working) {
314 close(master_fd);
315 master_fd = -1;
316 }
317 }
318 }
319
320 device->master_fd = master_fd;
321 device->local_fd = fd;
322 device->ws->query_info(device->ws, &device->rad_info);
323
324 device->use_aco = instance->perftest_flags & RADV_PERFTEST_ACO;
325
326 snprintf(device->name, sizeof(device->name),
327 "AMD RADV%s %s (LLVM " MESA_LLVM_VERSION_STRING ")", device->use_aco ? "/ACO" : "",
328 device->rad_info.name);
329
330 if (radv_device_get_cache_uuid(device->rad_info.family, device->cache_uuid)) {
331 device->ws->destroy(device->ws);
332 result = vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED,
333 "cannot generate UUID");
334 goto fail;
335 }
336
337 /* These flags affect shader compilation. */
338 uint64_t shader_env_flags = (device->use_aco ? 0x2 : 0);
339
340 /* The gpu id is already embedded in the uuid so we just pass "radv"
341 * when creating the cache.
342 */
343 char buf[VK_UUID_SIZE * 2 + 1];
344 disk_cache_format_hex_id(buf, device->cache_uuid, VK_UUID_SIZE * 2);
345 device->disk_cache = disk_cache_create(device->name, buf, shader_env_flags);
346
347 if (device->rad_info.chip_class < GFX8)
348 fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n");
349
350 radv_get_driver_uuid(&device->driver_uuid);
351 radv_get_device_uuid(&device->rad_info, &device->device_uuid);
352
353 device->out_of_order_rast_allowed = device->rad_info.has_out_of_order_rast &&
354 !(device->instance->debug_flags & RADV_DEBUG_NO_OUT_OF_ORDER);
355
356 device->dcc_msaa_allowed =
357 (device->instance->perftest_flags & RADV_PERFTEST_DCC_MSAA);
358
359 device->use_shader_ballot = (device->use_aco && device->rad_info.chip_class >= GFX8) ||
360 (device->instance->perftest_flags & RADV_PERFTEST_SHADER_BALLOT);
361
362 device->use_ngg = device->rad_info.chip_class >= GFX10 &&
363 device->rad_info.family != CHIP_NAVI14 &&
364 !(device->instance->debug_flags & RADV_DEBUG_NO_NGG);
365
366 /* TODO: Implement NGG GS with ACO. */
367 device->use_ngg_gs = device->use_ngg && !device->use_aco;
368 device->use_ngg_streamout = false;
369
370 /* Determine the number of threads per wave for all stages. */
371 device->cs_wave_size = 64;
372 device->ps_wave_size = 64;
373 device->ge_wave_size = 64;
374
375 if (device->rad_info.chip_class >= GFX10) {
376 if (device->instance->perftest_flags & RADV_PERFTEST_CS_WAVE_32)
377 device->cs_wave_size = 32;
378
379 /* For pixel shaders, wave64 is recommanded. */
380 if (device->instance->perftest_flags & RADV_PERFTEST_PS_WAVE_32)
381 device->ps_wave_size = 32;
382
383 if (device->instance->perftest_flags & RADV_PERFTEST_GE_WAVE_32)
384 device->ge_wave_size = 32;
385 }
386
387 radv_physical_device_init_mem_types(device);
388 radv_fill_device_extension_table(device, &device->supported_extensions);
389
390 if (drm_device)
391 device->bus_info = *drm_device->businfo.pci;
392
393 if ((device->instance->debug_flags & RADV_DEBUG_INFO))
394 ac_print_gpu_info(&device->rad_info);
395
396 /* The WSI is structured as a layer on top of the driver, so this has
397 * to be the last part of initialization (at least until we get other
398 * semi-layers).
399 */
400 result = radv_init_wsi(device);
401 if (result != VK_SUCCESS) {
402 device->ws->destroy(device->ws);
403 vk_error(instance, result);
404 goto fail;
405 }
406
407 return VK_SUCCESS;
408
409 fail:
410 close(fd);
411 if (master_fd != -1)
412 close(master_fd);
413 return result;
414 }
415
416 static void
417 radv_physical_device_finish(struct radv_physical_device *device)
418 {
419 radv_finish_wsi(device);
420 device->ws->destroy(device->ws);
421 disk_cache_destroy(device->disk_cache);
422 close(device->local_fd);
423 if (device->master_fd != -1)
424 close(device->master_fd);
425 }
426
427 static void *
428 default_alloc_func(void *pUserData, size_t size, size_t align,
429 VkSystemAllocationScope allocationScope)
430 {
431 return malloc(size);
432 }
433
434 static void *
435 default_realloc_func(void *pUserData, void *pOriginal, size_t size,
436 size_t align, VkSystemAllocationScope allocationScope)
437 {
438 return realloc(pOriginal, size);
439 }
440
441 static void
442 default_free_func(void *pUserData, void *pMemory)
443 {
444 free(pMemory);
445 }
446
447 static const VkAllocationCallbacks default_alloc = {
448 .pUserData = NULL,
449 .pfnAllocation = default_alloc_func,
450 .pfnReallocation = default_realloc_func,
451 .pfnFree = default_free_func,
452 };
453
454 static const struct debug_control radv_debug_options[] = {
455 {"nofastclears", RADV_DEBUG_NO_FAST_CLEARS},
456 {"nodcc", RADV_DEBUG_NO_DCC},
457 {"shaders", RADV_DEBUG_DUMP_SHADERS},
458 {"nocache", RADV_DEBUG_NO_CACHE},
459 {"shaderstats", RADV_DEBUG_DUMP_SHADER_STATS},
460 {"nohiz", RADV_DEBUG_NO_HIZ},
461 {"nocompute", RADV_DEBUG_NO_COMPUTE_QUEUE},
462 {"allbos", RADV_DEBUG_ALL_BOS},
463 {"noibs", RADV_DEBUG_NO_IBS},
464 {"spirv", RADV_DEBUG_DUMP_SPIRV},
465 {"vmfaults", RADV_DEBUG_VM_FAULTS},
466 {"zerovram", RADV_DEBUG_ZERO_VRAM},
467 {"syncshaders", RADV_DEBUG_SYNC_SHADERS},
468 {"preoptir", RADV_DEBUG_PREOPTIR},
469 {"nodynamicbounds", RADV_DEBUG_NO_DYNAMIC_BOUNDS},
470 {"nooutoforder", RADV_DEBUG_NO_OUT_OF_ORDER},
471 {"info", RADV_DEBUG_INFO},
472 {"errors", RADV_DEBUG_ERRORS},
473 {"startup", RADV_DEBUG_STARTUP},
474 {"checkir", RADV_DEBUG_CHECKIR},
475 {"nothreadllvm", RADV_DEBUG_NOTHREADLLVM},
476 {"nobinning", RADV_DEBUG_NOBINNING},
477 {"noloadstoreopt", RADV_DEBUG_NO_LOAD_STORE_OPT},
478 {"nongg", RADV_DEBUG_NO_NGG},
479 {"noshaderballot", RADV_DEBUG_NO_SHADER_BALLOT},
480 {"allentrypoints", RADV_DEBUG_ALL_ENTRYPOINTS},
481 {"metashaders", RADV_DEBUG_DUMP_META_SHADERS},
482 {"nomemorycache", RADV_DEBUG_NO_MEMORY_CACHE},
483 {NULL, 0}
484 };
485
486 const char *
487 radv_get_debug_option_name(int id)
488 {
489 assert(id < ARRAY_SIZE(radv_debug_options) - 1);
490 return radv_debug_options[id].string;
491 }
492
493 static const struct debug_control radv_perftest_options[] = {
494 {"localbos", RADV_PERFTEST_LOCAL_BOS},
495 {"dccmsaa", RADV_PERFTEST_DCC_MSAA},
496 {"bolist", RADV_PERFTEST_BO_LIST},
497 {"shader_ballot", RADV_PERFTEST_SHADER_BALLOT},
498 {"tccompatcmask", RADV_PERFTEST_TC_COMPAT_CMASK},
499 {"cswave32", RADV_PERFTEST_CS_WAVE_32},
500 {"pswave32", RADV_PERFTEST_PS_WAVE_32},
501 {"gewave32", RADV_PERFTEST_GE_WAVE_32},
502 {"dfsm", RADV_PERFTEST_DFSM},
503 {"aco", RADV_PERFTEST_ACO},
504 {NULL, 0}
505 };
506
507 const char *
508 radv_get_perftest_option_name(int id)
509 {
510 assert(id < ARRAY_SIZE(radv_perftest_options) - 1);
511 return radv_perftest_options[id].string;
512 }
513
514 static void
515 radv_handle_per_app_options(struct radv_instance *instance,
516 const VkApplicationInfo *info)
517 {
518 const char *name = info ? info->pApplicationName : NULL;
519
520 if (!name)
521 return;
522
523 if (!strcmp(name, "DOOM_VFR")) {
524 /* Work around a Doom VFR game bug */
525 instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS;
526 } else if (!strcmp(name, "MonsterHunterWorld.exe")) {
527 /* Workaround for a WaW hazard when LLVM moves/merges
528 * load/store memory operations.
529 * See https://reviews.llvm.org/D61313
530 */
531 if (LLVM_VERSION_MAJOR < 9)
532 instance->debug_flags |= RADV_DEBUG_NO_LOAD_STORE_OPT;
533 } else if (!strcmp(name, "Wolfenstein: Youngblood")) {
534 if (!(instance->debug_flags & RADV_DEBUG_NO_SHADER_BALLOT) &&
535 !(instance->perftest_flags & RADV_PERFTEST_ACO)) {
536 /* Force enable VK_AMD_shader_ballot because it looks
537 * safe and it gives a nice boost (+20% on Vega 56 at
538 * this time). It also prevents corruption on LLVM.
539 */
540 instance->perftest_flags |= RADV_PERFTEST_SHADER_BALLOT;
541 }
542 } else if (!strcmp(name, "Fledge")) {
543 /*
544 * Zero VRAM for "The Surge 2"
545 *
546 * This avoid a hang when when rendering any level. Likely
547 * uninitialized data in an indirect draw.
548 */
549 instance->debug_flags |= RADV_DEBUG_ZERO_VRAM;
550 } else if (!strcmp(name, "No Man's Sky")) {
551 /* Work around a NMS game bug */
552 instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE;
553 }
554 }
555
556 static int radv_get_instance_extension_index(const char *name)
557 {
558 for (unsigned i = 0; i < RADV_INSTANCE_EXTENSION_COUNT; ++i) {
559 if (strcmp(name, radv_instance_extensions[i].extensionName) == 0)
560 return i;
561 }
562 return -1;
563 }
564
565 static const char radv_dri_options_xml[] =
566 DRI_CONF_BEGIN
567 DRI_CONF_SECTION_PERFORMANCE
568 DRI_CONF_ADAPTIVE_SYNC("true")
569 DRI_CONF_VK_X11_OVERRIDE_MIN_IMAGE_COUNT(0)
570 DRI_CONF_VK_X11_STRICT_IMAGE_COUNT("false")
571 DRI_CONF_SECTION_END
572
573 DRI_CONF_SECTION_DEBUG
574 DRI_CONF_VK_WSI_FORCE_BGRA8_UNORM_FIRST("false")
575 DRI_CONF_SECTION_END
576 DRI_CONF_END;
577
578 static void radv_init_dri_options(struct radv_instance *instance)
579 {
580 driParseOptionInfo(&instance->available_dri_options, radv_dri_options_xml);
581 driParseConfigFiles(&instance->dri_options,
582 &instance->available_dri_options,
583 0, "radv", NULL,
584 instance->engineName,
585 instance->engineVersion);
586 }
587
588 VkResult radv_CreateInstance(
589 const VkInstanceCreateInfo* pCreateInfo,
590 const VkAllocationCallbacks* pAllocator,
591 VkInstance* pInstance)
592 {
593 struct radv_instance *instance;
594 VkResult result;
595
596 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO);
597
598 uint32_t client_version;
599 if (pCreateInfo->pApplicationInfo &&
600 pCreateInfo->pApplicationInfo->apiVersion != 0) {
601 client_version = pCreateInfo->pApplicationInfo->apiVersion;
602 } else {
603 client_version = VK_API_VERSION_1_0;
604 }
605
606 const char *engine_name = NULL;
607 uint32_t engine_version = 0;
608 if (pCreateInfo->pApplicationInfo) {
609 engine_name = pCreateInfo->pApplicationInfo->pEngineName;
610 engine_version = pCreateInfo->pApplicationInfo->engineVersion;
611 }
612
613 instance = vk_zalloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
614 VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
615 if (!instance)
616 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
617
618 instance->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
619
620 if (pAllocator)
621 instance->alloc = *pAllocator;
622 else
623 instance->alloc = default_alloc;
624
625 instance->apiVersion = client_version;
626 instance->physicalDeviceCount = -1;
627
628 /* Get secure compile thread count. NOTE: We cap this at 32 */
629 #define MAX_SC_PROCS 32
630 char *num_sc_threads = getenv("RADV_SECURE_COMPILE_THREADS");
631 if (num_sc_threads)
632 instance->num_sc_threads = MIN2(strtoul(num_sc_threads, NULL, 10), MAX_SC_PROCS);
633
634 instance->debug_flags = parse_debug_string(getenv("RADV_DEBUG"),
635 radv_debug_options);
636
637 /* Disable memory cache when secure compile is set */
638 if (radv_device_use_secure_compile(instance))
639 instance->debug_flags |= RADV_DEBUG_NO_MEMORY_CACHE;
640
641 instance->perftest_flags = parse_debug_string(getenv("RADV_PERFTEST"),
642 radv_perftest_options);
643
644 if (instance->perftest_flags & RADV_PERFTEST_ACO)
645 fprintf(stderr, "WARNING: Experimental compiler backend enabled. Here be dragons! Incorrect rendering, GPU hangs and/or resets are likely\n");
646
647 if (instance->debug_flags & RADV_DEBUG_STARTUP)
648 radv_logi("Created an instance");
649
650 for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
651 const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i];
652 int index = radv_get_instance_extension_index(ext_name);
653
654 if (index < 0 || !radv_supported_instance_extensions.extensions[index]) {
655 vk_free2(&default_alloc, pAllocator, instance);
656 return vk_error(instance, VK_ERROR_EXTENSION_NOT_PRESENT);
657 }
658
659 instance->enabled_extensions.extensions[index] = true;
660 }
661
662 bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
663
664 for (unsigned i = 0; i < ARRAY_SIZE(instance->dispatch.entrypoints); i++) {
665 /* Vulkan requires that entrypoints for extensions which have
666 * not been enabled must not be advertised.
667 */
668 if (!unchecked &&
669 !radv_instance_entrypoint_is_enabled(i, instance->apiVersion,
670 &instance->enabled_extensions)) {
671 instance->dispatch.entrypoints[i] = NULL;
672 } else {
673 instance->dispatch.entrypoints[i] =
674 radv_instance_dispatch_table.entrypoints[i];
675 }
676 }
677
678 for (unsigned i = 0; i < ARRAY_SIZE(instance->physical_device_dispatch.entrypoints); i++) {
679 /* Vulkan requires that entrypoints for extensions which have
680 * not been enabled must not be advertised.
681 */
682 if (!unchecked &&
683 !radv_physical_device_entrypoint_is_enabled(i, instance->apiVersion,
684 &instance->enabled_extensions)) {
685 instance->physical_device_dispatch.entrypoints[i] = NULL;
686 } else {
687 instance->physical_device_dispatch.entrypoints[i] =
688 radv_physical_device_dispatch_table.entrypoints[i];
689 }
690 }
691
692 for (unsigned i = 0; i < ARRAY_SIZE(instance->device_dispatch.entrypoints); i++) {
693 /* Vulkan requires that entrypoints for extensions which have
694 * not been enabled must not be advertised.
695 */
696 if (!unchecked &&
697 !radv_device_entrypoint_is_enabled(i, instance->apiVersion,
698 &instance->enabled_extensions, NULL)) {
699 instance->device_dispatch.entrypoints[i] = NULL;
700 } else {
701 instance->device_dispatch.entrypoints[i] =
702 radv_device_dispatch_table.entrypoints[i];
703 }
704 }
705
706 result = vk_debug_report_instance_init(&instance->debug_report_callbacks);
707 if (result != VK_SUCCESS) {
708 vk_free2(&default_alloc, pAllocator, instance);
709 return vk_error(instance, result);
710 }
711
712 instance->engineName = vk_strdup(&instance->alloc, engine_name,
713 VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
714 instance->engineVersion = engine_version;
715
716 glsl_type_singleton_init_or_ref();
717
718 VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false));
719
720 radv_init_dri_options(instance);
721 radv_handle_per_app_options(instance, pCreateInfo->pApplicationInfo);
722
723 *pInstance = radv_instance_to_handle(instance);
724
725 return VK_SUCCESS;
726 }
727
728 void radv_DestroyInstance(
729 VkInstance _instance,
730 const VkAllocationCallbacks* pAllocator)
731 {
732 RADV_FROM_HANDLE(radv_instance, instance, _instance);
733
734 if (!instance)
735 return;
736
737 for (int i = 0; i < instance->physicalDeviceCount; ++i) {
738 radv_physical_device_finish(instance->physicalDevices + i);
739 }
740
741 vk_free(&instance->alloc, instance->engineName);
742
743 VG(VALGRIND_DESTROY_MEMPOOL(instance));
744
745 glsl_type_singleton_decref();
746
747 driDestroyOptionCache(&instance->dri_options);
748 driDestroyOptionInfo(&instance->available_dri_options);
749
750 vk_debug_report_instance_destroy(&instance->debug_report_callbacks);
751
752 vk_free(&instance->alloc, instance);
753 }
754
755 static VkResult
756 radv_enumerate_devices(struct radv_instance *instance)
757 {
758 /* TODO: Check for more devices ? */
759 drmDevicePtr devices[8];
760 VkResult result = VK_ERROR_INCOMPATIBLE_DRIVER;
761 int max_devices;
762
763 instance->physicalDeviceCount = 0;
764
765 if (getenv("RADV_FORCE_FAMILY")) {
766 /* When RADV_FORCE_FAMILY is set, the driver creates a nul
767 * device that allows to test the compiler without having an
768 * AMDGPU instance.
769 */
770 result = radv_physical_device_init(instance->physicalDevices +
771 instance->physicalDeviceCount,
772 instance, NULL);
773
774 ++instance->physicalDeviceCount;
775 return VK_SUCCESS;
776 }
777
778 max_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices));
779
780 if (instance->debug_flags & RADV_DEBUG_STARTUP)
781 radv_logi("Found %d drm nodes", max_devices);
782
783 if (max_devices < 1)
784 return vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER);
785
786 for (unsigned i = 0; i < (unsigned)max_devices; i++) {
787 if (devices[i]->available_nodes & 1 << DRM_NODE_RENDER &&
788 devices[i]->bustype == DRM_BUS_PCI &&
789 devices[i]->deviceinfo.pci->vendor_id == ATI_VENDOR_ID) {
790
791 result = radv_physical_device_init(instance->physicalDevices +
792 instance->physicalDeviceCount,
793 instance,
794 devices[i]);
795 if (result == VK_SUCCESS)
796 ++instance->physicalDeviceCount;
797 else if (result != VK_ERROR_INCOMPATIBLE_DRIVER)
798 break;
799 }
800 }
801 drmFreeDevices(devices, max_devices);
802
803 return result;
804 }
805
806 VkResult radv_EnumeratePhysicalDevices(
807 VkInstance _instance,
808 uint32_t* pPhysicalDeviceCount,
809 VkPhysicalDevice* pPhysicalDevices)
810 {
811 RADV_FROM_HANDLE(radv_instance, instance, _instance);
812 VkResult result;
813
814 if (instance->physicalDeviceCount < 0) {
815 result = radv_enumerate_devices(instance);
816 if (result != VK_SUCCESS &&
817 result != VK_ERROR_INCOMPATIBLE_DRIVER)
818 return result;
819 }
820
821 if (!pPhysicalDevices) {
822 *pPhysicalDeviceCount = instance->physicalDeviceCount;
823 } else {
824 *pPhysicalDeviceCount = MIN2(*pPhysicalDeviceCount, instance->physicalDeviceCount);
825 for (unsigned i = 0; i < *pPhysicalDeviceCount; ++i)
826 pPhysicalDevices[i] = radv_physical_device_to_handle(instance->physicalDevices + i);
827 }
828
829 return *pPhysicalDeviceCount < instance->physicalDeviceCount ? VK_INCOMPLETE
830 : VK_SUCCESS;
831 }
832
833 VkResult radv_EnumeratePhysicalDeviceGroups(
834 VkInstance _instance,
835 uint32_t* pPhysicalDeviceGroupCount,
836 VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties)
837 {
838 RADV_FROM_HANDLE(radv_instance, instance, _instance);
839 VkResult result;
840
841 if (instance->physicalDeviceCount < 0) {
842 result = radv_enumerate_devices(instance);
843 if (result != VK_SUCCESS &&
844 result != VK_ERROR_INCOMPATIBLE_DRIVER)
845 return result;
846 }
847
848 if (!pPhysicalDeviceGroupProperties) {
849 *pPhysicalDeviceGroupCount = instance->physicalDeviceCount;
850 } else {
851 *pPhysicalDeviceGroupCount = MIN2(*pPhysicalDeviceGroupCount, instance->physicalDeviceCount);
852 for (unsigned i = 0; i < *pPhysicalDeviceGroupCount; ++i) {
853 pPhysicalDeviceGroupProperties[i].physicalDeviceCount = 1;
854 pPhysicalDeviceGroupProperties[i].physicalDevices[0] = radv_physical_device_to_handle(instance->physicalDevices + i);
855 pPhysicalDeviceGroupProperties[i].subsetAllocation = false;
856 }
857 }
858 return *pPhysicalDeviceGroupCount < instance->physicalDeviceCount ? VK_INCOMPLETE
859 : VK_SUCCESS;
860 }
861
862 void radv_GetPhysicalDeviceFeatures(
863 VkPhysicalDevice physicalDevice,
864 VkPhysicalDeviceFeatures* pFeatures)
865 {
866 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
867 memset(pFeatures, 0, sizeof(*pFeatures));
868
869 *pFeatures = (VkPhysicalDeviceFeatures) {
870 .robustBufferAccess = true,
871 .fullDrawIndexUint32 = true,
872 .imageCubeArray = true,
873 .independentBlend = true,
874 .geometryShader = true,
875 .tessellationShader = true,
876 .sampleRateShading = true,
877 .dualSrcBlend = true,
878 .logicOp = true,
879 .multiDrawIndirect = true,
880 .drawIndirectFirstInstance = true,
881 .depthClamp = true,
882 .depthBiasClamp = true,
883 .fillModeNonSolid = true,
884 .depthBounds = true,
885 .wideLines = true,
886 .largePoints = true,
887 .alphaToOne = true,
888 .multiViewport = true,
889 .samplerAnisotropy = true,
890 .textureCompressionETC2 = radv_device_supports_etc(pdevice),
891 .textureCompressionASTC_LDR = false,
892 .textureCompressionBC = true,
893 .occlusionQueryPrecise = true,
894 .pipelineStatisticsQuery = true,
895 .vertexPipelineStoresAndAtomics = true,
896 .fragmentStoresAndAtomics = true,
897 .shaderTessellationAndGeometryPointSize = true,
898 .shaderImageGatherExtended = true,
899 .shaderStorageImageExtendedFormats = true,
900 .shaderStorageImageMultisample = true,
901 .shaderUniformBufferArrayDynamicIndexing = true,
902 .shaderSampledImageArrayDynamicIndexing = true,
903 .shaderStorageBufferArrayDynamicIndexing = true,
904 .shaderStorageImageArrayDynamicIndexing = true,
905 .shaderStorageImageReadWithoutFormat = true,
906 .shaderStorageImageWriteWithoutFormat = true,
907 .shaderClipDistance = true,
908 .shaderCullDistance = true,
909 .shaderFloat64 = true,
910 .shaderInt64 = true,
911 .shaderInt16 = pdevice->rad_info.chip_class >= GFX9,
912 .sparseBinding = true,
913 .variableMultisampleRate = true,
914 .inheritedQueries = true,
915 };
916 }
917
918 void radv_GetPhysicalDeviceFeatures2(
919 VkPhysicalDevice physicalDevice,
920 VkPhysicalDeviceFeatures2 *pFeatures)
921 {
922 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
923 vk_foreach_struct(ext, pFeatures->pNext) {
924 switch (ext->sType) {
925 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VARIABLE_POINTERS_FEATURES: {
926 VkPhysicalDeviceVariablePointersFeatures *features = (void *)ext;
927 features->variablePointersStorageBuffer = true;
928 features->variablePointers = true;
929 break;
930 }
931 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_FEATURES: {
932 VkPhysicalDeviceMultiviewFeatures *features = (VkPhysicalDeviceMultiviewFeatures*)ext;
933 features->multiview = true;
934 features->multiviewGeometryShader = true;
935 features->multiviewTessellationShader = true;
936 break;
937 }
938 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES: {
939 VkPhysicalDeviceShaderDrawParametersFeatures *features =
940 (VkPhysicalDeviceShaderDrawParametersFeatures*)ext;
941 features->shaderDrawParameters = true;
942 break;
943 }
944 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_FEATURES: {
945 VkPhysicalDeviceProtectedMemoryFeatures *features =
946 (VkPhysicalDeviceProtectedMemoryFeatures*)ext;
947 features->protectedMemory = false;
948 break;
949 }
950 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES: {
951 VkPhysicalDevice16BitStorageFeatures *features =
952 (VkPhysicalDevice16BitStorageFeatures*)ext;
953 bool enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
954 features->storageBuffer16BitAccess = enable;
955 features->uniformAndStorageBuffer16BitAccess = enable;
956 features->storagePushConstant16 = enable;
957 features->storageInputOutput16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco && LLVM_VERSION_MAJOR >= 9;
958 break;
959 }
960 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES: {
961 VkPhysicalDeviceSamplerYcbcrConversionFeatures *features =
962 (VkPhysicalDeviceSamplerYcbcrConversionFeatures*)ext;
963 features->samplerYcbcrConversion = true;
964 break;
965 }
966 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES: {
967 VkPhysicalDeviceDescriptorIndexingFeatures *features =
968 (VkPhysicalDeviceDescriptorIndexingFeatures*)ext;
969 features->shaderInputAttachmentArrayDynamicIndexing = true;
970 features->shaderUniformTexelBufferArrayDynamicIndexing = true;
971 features->shaderStorageTexelBufferArrayDynamicIndexing = true;
972 features->shaderUniformBufferArrayNonUniformIndexing = true;
973 features->shaderSampledImageArrayNonUniformIndexing = true;
974 features->shaderStorageBufferArrayNonUniformIndexing = true;
975 features->shaderStorageImageArrayNonUniformIndexing = true;
976 features->shaderInputAttachmentArrayNonUniformIndexing = true;
977 features->shaderUniformTexelBufferArrayNonUniformIndexing = true;
978 features->shaderStorageTexelBufferArrayNonUniformIndexing = true;
979 features->descriptorBindingUniformBufferUpdateAfterBind = true;
980 features->descriptorBindingSampledImageUpdateAfterBind = true;
981 features->descriptorBindingStorageImageUpdateAfterBind = true;
982 features->descriptorBindingStorageBufferUpdateAfterBind = true;
983 features->descriptorBindingUniformTexelBufferUpdateAfterBind = true;
984 features->descriptorBindingStorageTexelBufferUpdateAfterBind = true;
985 features->descriptorBindingUpdateUnusedWhilePending = true;
986 features->descriptorBindingPartiallyBound = true;
987 features->descriptorBindingVariableDescriptorCount = true;
988 features->runtimeDescriptorArray = true;
989 break;
990 }
991 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONDITIONAL_RENDERING_FEATURES_EXT: {
992 VkPhysicalDeviceConditionalRenderingFeaturesEXT *features =
993 (VkPhysicalDeviceConditionalRenderingFeaturesEXT*)ext;
994 features->conditionalRendering = true;
995 features->inheritedConditionalRendering = false;
996 break;
997 }
998 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_FEATURES_EXT: {
999 VkPhysicalDeviceVertexAttributeDivisorFeaturesEXT *features =
1000 (VkPhysicalDeviceVertexAttributeDivisorFeaturesEXT *)ext;
1001 features->vertexAttributeInstanceRateDivisor = true;
1002 features->vertexAttributeInstanceRateZeroDivisor = true;
1003 break;
1004 }
1005 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_FEATURES_EXT: {
1006 VkPhysicalDeviceTransformFeedbackFeaturesEXT *features =
1007 (VkPhysicalDeviceTransformFeedbackFeaturesEXT*)ext;
1008 features->transformFeedback = true;
1009 features->geometryStreams = !pdevice->use_ngg_streamout;
1010 break;
1011 }
1012 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SCALAR_BLOCK_LAYOUT_FEATURES: {
1013 VkPhysicalDeviceScalarBlockLayoutFeatures *features =
1014 (VkPhysicalDeviceScalarBlockLayoutFeatures *)ext;
1015 features->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7;
1016 break;
1017 }
1018 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT: {
1019 VkPhysicalDeviceMemoryPriorityFeaturesEXT *features =
1020 (VkPhysicalDeviceMemoryPriorityFeaturesEXT *)ext;
1021 features->memoryPriority = true;
1022 break;
1023 }
1024 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_EXT: {
1025 VkPhysicalDeviceBufferDeviceAddressFeaturesEXT *features =
1026 (VkPhysicalDeviceBufferDeviceAddressFeaturesEXT *)ext;
1027 features->bufferDeviceAddress = true;
1028 features->bufferDeviceAddressCaptureReplay = false;
1029 features->bufferDeviceAddressMultiDevice = false;
1030 break;
1031 }
1032 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES: {
1033 VkPhysicalDeviceBufferDeviceAddressFeatures *features =
1034 (VkPhysicalDeviceBufferDeviceAddressFeatures *)ext;
1035 features->bufferDeviceAddress = true;
1036 features->bufferDeviceAddressCaptureReplay = false;
1037 features->bufferDeviceAddressMultiDevice = false;
1038 break;
1039 }
1040 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_ENABLE_FEATURES_EXT: {
1041 VkPhysicalDeviceDepthClipEnableFeaturesEXT *features =
1042 (VkPhysicalDeviceDepthClipEnableFeaturesEXT *)ext;
1043 features->depthClipEnable = true;
1044 break;
1045 }
1046 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES: {
1047 VkPhysicalDeviceHostQueryResetFeatures *features =
1048 (VkPhysicalDeviceHostQueryResetFeatures *)ext;
1049 features->hostQueryReset = true;
1050 break;
1051 }
1052 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES: {
1053 VkPhysicalDevice8BitStorageFeatures *features =
1054 (VkPhysicalDevice8BitStorageFeatures *)ext;
1055 bool enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1056 features->storageBuffer8BitAccess = enable;
1057 features->uniformAndStorageBuffer8BitAccess = enable;
1058 features->storagePushConstant8 = enable;
1059 break;
1060 }
1061 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES: {
1062 VkPhysicalDeviceShaderFloat16Int8Features *features =
1063 (VkPhysicalDeviceShaderFloat16Int8Features*)ext;
1064 features->shaderFloat16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco;
1065 features->shaderInt8 = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1066 break;
1067 }
1068 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES: {
1069 VkPhysicalDeviceShaderAtomicInt64Features *features =
1070 (VkPhysicalDeviceShaderAtomicInt64Features *)ext;
1071 features->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1072 features->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1073 break;
1074 }
1075 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DEMOTE_TO_HELPER_INVOCATION_FEATURES_EXT: {
1076 VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *features =
1077 (VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *)ext;
1078 features->shaderDemoteToHelperInvocation = pdevice->use_aco;
1079 break;
1080 }
1081 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_FEATURES_EXT: {
1082 VkPhysicalDeviceInlineUniformBlockFeaturesEXT *features =
1083 (VkPhysicalDeviceInlineUniformBlockFeaturesEXT *)ext;
1084
1085 features->inlineUniformBlock = true;
1086 features->descriptorBindingInlineUniformBlockUpdateAfterBind = true;
1087 break;
1088 }
1089 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COMPUTE_SHADER_DERIVATIVES_FEATURES_NV: {
1090 VkPhysicalDeviceComputeShaderDerivativesFeaturesNV *features =
1091 (VkPhysicalDeviceComputeShaderDerivativesFeaturesNV *)ext;
1092 features->computeDerivativeGroupQuads = false;
1093 features->computeDerivativeGroupLinear = true;
1094 break;
1095 }
1096 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_YCBCR_IMAGE_ARRAYS_FEATURES_EXT: {
1097 VkPhysicalDeviceYcbcrImageArraysFeaturesEXT *features =
1098 (VkPhysicalDeviceYcbcrImageArraysFeaturesEXT*)ext;
1099 features->ycbcrImageArrays = true;
1100 break;
1101 }
1102 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_UNIFORM_BUFFER_STANDARD_LAYOUT_FEATURES: {
1103 VkPhysicalDeviceUniformBufferStandardLayoutFeatures *features =
1104 (VkPhysicalDeviceUniformBufferStandardLayoutFeatures *)ext;
1105 features->uniformBufferStandardLayout = true;
1106 break;
1107 }
1108 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INDEX_TYPE_UINT8_FEATURES_EXT: {
1109 VkPhysicalDeviceIndexTypeUint8FeaturesEXT *features =
1110 (VkPhysicalDeviceIndexTypeUint8FeaturesEXT *)ext;
1111 features->indexTypeUint8 = pdevice->rad_info.chip_class >= GFX8;
1112 break;
1113 }
1114 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGELESS_FRAMEBUFFER_FEATURES: {
1115 VkPhysicalDeviceImagelessFramebufferFeatures *features =
1116 (VkPhysicalDeviceImagelessFramebufferFeatures *)ext;
1117 features->imagelessFramebuffer = true;
1118 break;
1119 }
1120 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR: {
1121 VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR *features =
1122 (VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR *)ext;
1123 features->pipelineExecutableInfo = true;
1124 break;
1125 }
1126 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR: {
1127 VkPhysicalDeviceShaderClockFeaturesKHR *features =
1128 (VkPhysicalDeviceShaderClockFeaturesKHR *)ext;
1129 features->shaderSubgroupClock = true;
1130 features->shaderDeviceClock = false;
1131 break;
1132 }
1133 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_FEATURES_EXT: {
1134 VkPhysicalDeviceTexelBufferAlignmentFeaturesEXT *features =
1135 (VkPhysicalDeviceTexelBufferAlignmentFeaturesEXT *)ext;
1136 features->texelBufferAlignment = true;
1137 break;
1138 }
1139 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES: {
1140 VkPhysicalDeviceTimelineSemaphoreFeatures *features =
1141 (VkPhysicalDeviceTimelineSemaphoreFeatures *) ext;
1142 features->timelineSemaphore = true;
1143 break;
1144 }
1145 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT: {
1146 VkPhysicalDeviceSubgroupSizeControlFeaturesEXT *features =
1147 (VkPhysicalDeviceSubgroupSizeControlFeaturesEXT *)ext;
1148 features->subgroupSizeControl = true;
1149 features->computeFullSubgroups = true;
1150 break;
1151 }
1152 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COHERENT_MEMORY_FEATURES_AMD: {
1153 VkPhysicalDeviceCoherentMemoryFeaturesAMD *features =
1154 (VkPhysicalDeviceCoherentMemoryFeaturesAMD *)ext;
1155 features->deviceCoherentMemory = pdevice->rad_info.has_l2_uncached;
1156 break;
1157 }
1158 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_EXTENDED_TYPES_FEATURES: {
1159 VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *features =
1160 (VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *)ext;
1161 features->shaderSubgroupExtendedTypes = !pdevice->use_aco;
1162 break;
1163 }
1164 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SEPARATE_DEPTH_STENCIL_LAYOUTS_FEATURES_KHR: {
1165 VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *features =
1166 (VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *)ext;
1167 features->separateDepthStencilLayouts = true;
1168 break;
1169 }
1170 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES: {
1171 VkPhysicalDeviceVulkan11Features *features =
1172 (VkPhysicalDeviceVulkan11Features *)ext;
1173 bool storage16_enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1174 features->storageBuffer16BitAccess = storage16_enable;
1175 features->uniformAndStorageBuffer16BitAccess = storage16_enable;
1176 features->storagePushConstant16 = storage16_enable;
1177 features->storageInputOutput16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco && LLVM_VERSION_MAJOR >= 9;
1178 features->multiview = true;
1179 features->multiviewGeometryShader = true;
1180 features->multiviewTessellationShader = true;
1181 features->variablePointersStorageBuffer = true;
1182 features->variablePointers = true;
1183 features->protectedMemory = false;
1184 features->samplerYcbcrConversion = true;
1185 features->shaderDrawParameters = true;
1186 break;
1187 }
1188 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES: {
1189 VkPhysicalDeviceVulkan12Features *features =
1190 (VkPhysicalDeviceVulkan12Features *)ext;
1191 bool int8_enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1192 features->samplerMirrorClampToEdge = true;
1193 features->drawIndirectCount = true;
1194 features->storageBuffer8BitAccess = int8_enable;
1195 features->uniformAndStorageBuffer8BitAccess = int8_enable;
1196 features->storagePushConstant8 = int8_enable;
1197 features->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1198 features->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1199 features->shaderFloat16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco;
1200 features->shaderInt8 = int8_enable;
1201 features->descriptorIndexing = true;
1202 features->shaderInputAttachmentArrayDynamicIndexing = true;
1203 features->shaderUniformTexelBufferArrayDynamicIndexing = true;
1204 features->shaderStorageTexelBufferArrayDynamicIndexing = true;
1205 features->shaderUniformBufferArrayNonUniformIndexing = true;
1206 features->shaderSampledImageArrayNonUniformIndexing = true;
1207 features->shaderStorageBufferArrayNonUniformIndexing = true;
1208 features->shaderStorageImageArrayNonUniformIndexing = true;
1209 features->shaderInputAttachmentArrayNonUniformIndexing = true;
1210 features->shaderUniformTexelBufferArrayNonUniformIndexing = true;
1211 features->shaderStorageTexelBufferArrayNonUniformIndexing = true;
1212 features->descriptorBindingUniformBufferUpdateAfterBind = true;
1213 features->descriptorBindingSampledImageUpdateAfterBind = true;
1214 features->descriptorBindingStorageImageUpdateAfterBind = true;
1215 features->descriptorBindingStorageBufferUpdateAfterBind = true;
1216 features->descriptorBindingUniformTexelBufferUpdateAfterBind = true;
1217 features->descriptorBindingStorageTexelBufferUpdateAfterBind = true;
1218 features->descriptorBindingUpdateUnusedWhilePending = true;
1219 features->descriptorBindingPartiallyBound = true;
1220 features->descriptorBindingVariableDescriptorCount = true;
1221 features->runtimeDescriptorArray = true;
1222 features->samplerFilterMinmax = true;
1223 features->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7;
1224 features->imagelessFramebuffer = true;
1225 features->uniformBufferStandardLayout = true;
1226 features->shaderSubgroupExtendedTypes = !pdevice->use_aco;
1227 features->separateDepthStencilLayouts = true;
1228 features->hostQueryReset = true;
1229 features->timelineSemaphore = pdevice->rad_info.has_syncobj_wait_for_submit;
1230 features->bufferDeviceAddress = true;
1231 features->bufferDeviceAddressCaptureReplay = false;
1232 features->bufferDeviceAddressMultiDevice = false;
1233 features->vulkanMemoryModel = false;
1234 features->vulkanMemoryModelDeviceScope = false;
1235 features->vulkanMemoryModelAvailabilityVisibilityChains = false;
1236 features->shaderOutputViewportIndex = true;
1237 features->shaderOutputLayer = true;
1238 features->subgroupBroadcastDynamicId = true;
1239 break;
1240 }
1241 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_FEATURES_EXT: {
1242 VkPhysicalDeviceLineRasterizationFeaturesEXT *features =
1243 (VkPhysicalDeviceLineRasterizationFeaturesEXT *)ext;
1244 features->rectangularLines = false;
1245 features->bresenhamLines = true;
1246 features->smoothLines = false;
1247 features->stippledRectangularLines = false;
1248 features->stippledBresenhamLines = true;
1249 features->stippledSmoothLines = false;
1250 break;
1251 }
1252 case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: {
1253 VkDeviceMemoryOverallocationCreateInfoAMD *features =
1254 (VkDeviceMemoryOverallocationCreateInfoAMD *)ext;
1255 features->overallocationBehavior = true;
1256 break;
1257 }
1258 default:
1259 break;
1260 }
1261 }
1262 return radv_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features);
1263 }
1264
1265 static size_t
1266 radv_max_descriptor_set_size()
1267 {
1268 /* make sure that the entire descriptor set is addressable with a signed
1269 * 32-bit int. So the sum of all limits scaled by descriptor size has to
1270 * be at most 2 GiB. the combined image & samples object count as one of
1271 * both. This limit is for the pipeline layout, not for the set layout, but
1272 * there is no set limit, so we just set a pipeline limit. I don't think
1273 * any app is going to hit this soon. */
1274 return ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS
1275 - MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
1276 (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
1277 32 /* storage buffer, 32 due to potential space wasted on alignment */ +
1278 32 /* sampler, largest when combined with image */ +
1279 64 /* sampled image */ +
1280 64 /* storage image */);
1281 }
1282
1283 void radv_GetPhysicalDeviceProperties(
1284 VkPhysicalDevice physicalDevice,
1285 VkPhysicalDeviceProperties* pProperties)
1286 {
1287 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1288 VkSampleCountFlags sample_counts = 0xf;
1289
1290 size_t max_descriptor_set_size = radv_max_descriptor_set_size();
1291
1292 VkPhysicalDeviceLimits limits = {
1293 .maxImageDimension1D = (1 << 14),
1294 .maxImageDimension2D = (1 << 14),
1295 .maxImageDimension3D = (1 << 11),
1296 .maxImageDimensionCube = (1 << 14),
1297 .maxImageArrayLayers = (1 << 11),
1298 .maxTexelBufferElements = UINT32_MAX,
1299 .maxUniformBufferRange = UINT32_MAX,
1300 .maxStorageBufferRange = UINT32_MAX,
1301 .maxPushConstantsSize = MAX_PUSH_CONSTANTS_SIZE,
1302 .maxMemoryAllocationCount = UINT32_MAX,
1303 .maxSamplerAllocationCount = 64 * 1024,
1304 .bufferImageGranularity = 64, /* A cache line */
1305 .sparseAddressSpaceSize = RADV_MAX_MEMORY_ALLOCATION_SIZE, /* buffer max size */
1306 .maxBoundDescriptorSets = MAX_SETS,
1307 .maxPerStageDescriptorSamplers = max_descriptor_set_size,
1308 .maxPerStageDescriptorUniformBuffers = max_descriptor_set_size,
1309 .maxPerStageDescriptorStorageBuffers = max_descriptor_set_size,
1310 .maxPerStageDescriptorSampledImages = max_descriptor_set_size,
1311 .maxPerStageDescriptorStorageImages = max_descriptor_set_size,
1312 .maxPerStageDescriptorInputAttachments = max_descriptor_set_size,
1313 .maxPerStageResources = max_descriptor_set_size,
1314 .maxDescriptorSetSamplers = max_descriptor_set_size,
1315 .maxDescriptorSetUniformBuffers = max_descriptor_set_size,
1316 .maxDescriptorSetUniformBuffersDynamic = MAX_DYNAMIC_UNIFORM_BUFFERS,
1317 .maxDescriptorSetStorageBuffers = max_descriptor_set_size,
1318 .maxDescriptorSetStorageBuffersDynamic = MAX_DYNAMIC_STORAGE_BUFFERS,
1319 .maxDescriptorSetSampledImages = max_descriptor_set_size,
1320 .maxDescriptorSetStorageImages = max_descriptor_set_size,
1321 .maxDescriptorSetInputAttachments = max_descriptor_set_size,
1322 .maxVertexInputAttributes = MAX_VERTEX_ATTRIBS,
1323 .maxVertexInputBindings = MAX_VBS,
1324 .maxVertexInputAttributeOffset = 2047,
1325 .maxVertexInputBindingStride = 2048,
1326 .maxVertexOutputComponents = 128,
1327 .maxTessellationGenerationLevel = 64,
1328 .maxTessellationPatchSize = 32,
1329 .maxTessellationControlPerVertexInputComponents = 128,
1330 .maxTessellationControlPerVertexOutputComponents = 128,
1331 .maxTessellationControlPerPatchOutputComponents = 120,
1332 .maxTessellationControlTotalOutputComponents = 4096,
1333 .maxTessellationEvaluationInputComponents = 128,
1334 .maxTessellationEvaluationOutputComponents = 128,
1335 .maxGeometryShaderInvocations = 127,
1336 .maxGeometryInputComponents = 64,
1337 .maxGeometryOutputComponents = 128,
1338 .maxGeometryOutputVertices = 256,
1339 .maxGeometryTotalOutputComponents = 1024,
1340 .maxFragmentInputComponents = 128,
1341 .maxFragmentOutputAttachments = 8,
1342 .maxFragmentDualSrcAttachments = 1,
1343 .maxFragmentCombinedOutputResources = 8,
1344 .maxComputeSharedMemorySize = 32768,
1345 .maxComputeWorkGroupCount = { 65535, 65535, 65535 },
1346 .maxComputeWorkGroupInvocations = 1024,
1347 .maxComputeWorkGroupSize = {
1348 1024,
1349 1024,
1350 1024
1351 },
1352 .subPixelPrecisionBits = 8,
1353 .subTexelPrecisionBits = 8,
1354 .mipmapPrecisionBits = 8,
1355 .maxDrawIndexedIndexValue = UINT32_MAX,
1356 .maxDrawIndirectCount = UINT32_MAX,
1357 .maxSamplerLodBias = 16,
1358 .maxSamplerAnisotropy = 16,
1359 .maxViewports = MAX_VIEWPORTS,
1360 .maxViewportDimensions = { (1 << 14), (1 << 14) },
1361 .viewportBoundsRange = { INT16_MIN, INT16_MAX },
1362 .viewportSubPixelBits = 8,
1363 .minMemoryMapAlignment = 4096, /* A page */
1364 .minTexelBufferOffsetAlignment = 4,
1365 .minUniformBufferOffsetAlignment = 4,
1366 .minStorageBufferOffsetAlignment = 4,
1367 .minTexelOffset = -32,
1368 .maxTexelOffset = 31,
1369 .minTexelGatherOffset = -32,
1370 .maxTexelGatherOffset = 31,
1371 .minInterpolationOffset = -2,
1372 .maxInterpolationOffset = 2,
1373 .subPixelInterpolationOffsetBits = 8,
1374 .maxFramebufferWidth = (1 << 14),
1375 .maxFramebufferHeight = (1 << 14),
1376 .maxFramebufferLayers = (1 << 10),
1377 .framebufferColorSampleCounts = sample_counts,
1378 .framebufferDepthSampleCounts = sample_counts,
1379 .framebufferStencilSampleCounts = sample_counts,
1380 .framebufferNoAttachmentsSampleCounts = sample_counts,
1381 .maxColorAttachments = MAX_RTS,
1382 .sampledImageColorSampleCounts = sample_counts,
1383 .sampledImageIntegerSampleCounts = sample_counts,
1384 .sampledImageDepthSampleCounts = sample_counts,
1385 .sampledImageStencilSampleCounts = sample_counts,
1386 .storageImageSampleCounts = sample_counts,
1387 .maxSampleMaskWords = 1,
1388 .timestampComputeAndGraphics = true,
1389 .timestampPeriod = 1000000.0 / pdevice->rad_info.clock_crystal_freq,
1390 .maxClipDistances = 8,
1391 .maxCullDistances = 8,
1392 .maxCombinedClipAndCullDistances = 8,
1393 .discreteQueuePriorities = 2,
1394 .pointSizeRange = { 0.0, 8192.0 },
1395 .lineWidthRange = { 0.0, 8192.0 },
1396 .pointSizeGranularity = (1.0 / 8.0),
1397 .lineWidthGranularity = (1.0 / 8.0),
1398 .strictLines = false, /* FINISHME */
1399 .standardSampleLocations = true,
1400 .optimalBufferCopyOffsetAlignment = 128,
1401 .optimalBufferCopyRowPitchAlignment = 128,
1402 .nonCoherentAtomSize = 64,
1403 };
1404
1405 *pProperties = (VkPhysicalDeviceProperties) {
1406 .apiVersion = radv_physical_device_api_version(pdevice),
1407 .driverVersion = vk_get_driver_version(),
1408 .vendorID = ATI_VENDOR_ID,
1409 .deviceID = pdevice->rad_info.pci_id,
1410 .deviceType = pdevice->rad_info.has_dedicated_vram ? VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU : VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
1411 .limits = limits,
1412 .sparseProperties = {0},
1413 };
1414
1415 strcpy(pProperties->deviceName, pdevice->name);
1416 memcpy(pProperties->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE);
1417 }
1418
1419 static void
1420 radv_get_physical_device_properties_1_1(struct radv_physical_device *pdevice,
1421 VkPhysicalDeviceVulkan11Properties *p)
1422 {
1423 assert(p->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES);
1424
1425 memcpy(p->deviceUUID, pdevice->device_uuid, VK_UUID_SIZE);
1426 memcpy(p->driverUUID, pdevice->driver_uuid, VK_UUID_SIZE);
1427 memset(p->deviceLUID, 0, VK_LUID_SIZE);
1428 /* The LUID is for Windows. */
1429 p->deviceLUIDValid = false;
1430 p->deviceNodeMask = 0;
1431
1432 p->subgroupSize = RADV_SUBGROUP_SIZE;
1433 p->subgroupSupportedStages = VK_SHADER_STAGE_ALL_GRAPHICS |
1434 VK_SHADER_STAGE_COMPUTE_BIT;
1435 p->subgroupSupportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT |
1436 VK_SUBGROUP_FEATURE_VOTE_BIT |
1437 VK_SUBGROUP_FEATURE_ARITHMETIC_BIT |
1438 VK_SUBGROUP_FEATURE_BALLOT_BIT |
1439 VK_SUBGROUP_FEATURE_CLUSTERED_BIT |
1440 VK_SUBGROUP_FEATURE_QUAD_BIT;
1441
1442 if (((pdevice->rad_info.chip_class == GFX6 ||
1443 pdevice->rad_info.chip_class == GFX7) && !pdevice->use_aco) ||
1444 pdevice->rad_info.chip_class >= GFX8) {
1445 p->subgroupSupportedOperations |= VK_SUBGROUP_FEATURE_SHUFFLE_BIT |
1446 VK_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT;
1447 }
1448 p->subgroupQuadOperationsInAllStages = true;
1449
1450 p->pointClippingBehavior = VK_POINT_CLIPPING_BEHAVIOR_ALL_CLIP_PLANES;
1451 p->maxMultiviewViewCount = MAX_VIEWS;
1452 p->maxMultiviewInstanceIndex = INT_MAX;
1453 p->protectedNoFault = false;
1454 p->maxPerSetDescriptors = RADV_MAX_PER_SET_DESCRIPTORS;
1455 p->maxMemoryAllocationSize = RADV_MAX_MEMORY_ALLOCATION_SIZE;
1456 }
1457
1458 static void
1459 radv_get_physical_device_properties_1_2(struct radv_physical_device *pdevice,
1460 VkPhysicalDeviceVulkan12Properties *p)
1461 {
1462 assert(p->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES);
1463
1464 p->driverID = VK_DRIVER_ID_MESA_RADV;
1465 snprintf(p->driverName, VK_MAX_DRIVER_NAME_SIZE, "radv");
1466 snprintf(p->driverInfo, VK_MAX_DRIVER_INFO_SIZE,
1467 "Mesa " PACKAGE_VERSION MESA_GIT_SHA1
1468 " (LLVM " MESA_LLVM_VERSION_STRING ")");
1469 p->conformanceVersion = (VkConformanceVersion) {
1470 .major = 1,
1471 .minor = 2,
1472 .subminor = 0,
1473 .patch = 0,
1474 };
1475
1476 /* On AMD hardware, denormals and rounding modes for fp16/fp64 are
1477 * controlled by the same config register.
1478 */
1479 if (pdevice->rad_info.has_double_rate_fp16) {
1480 p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR;
1481 p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR;
1482 } else {
1483 p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL_KHR;
1484 p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL_KHR;
1485 }
1486
1487 /* Do not allow both preserving and flushing denorms because different
1488 * shaders in the same pipeline can have different settings and this
1489 * won't work for merged shaders. To make it work, this requires LLVM
1490 * support for changing the register. The same logic applies for the
1491 * rounding modes because they are configured with the same config
1492 * register. TODO: we can enable a lot of these for ACO when it
1493 * supports all stages.
1494 */
1495 p->shaderDenormFlushToZeroFloat32 = true;
1496 p->shaderDenormPreserveFloat32 = false;
1497 p->shaderRoundingModeRTEFloat32 = true;
1498 p->shaderRoundingModeRTZFloat32 = false;
1499 p->shaderSignedZeroInfNanPreserveFloat32 = true;
1500
1501 p->shaderDenormFlushToZeroFloat16 = false;
1502 p->shaderDenormPreserveFloat16 = pdevice->rad_info.has_double_rate_fp16;
1503 p->shaderRoundingModeRTEFloat16 = pdevice->rad_info.has_double_rate_fp16;
1504 p->shaderRoundingModeRTZFloat16 = false;
1505 p->shaderSignedZeroInfNanPreserveFloat16 = pdevice->rad_info.has_double_rate_fp16;
1506
1507 p->shaderDenormFlushToZeroFloat64 = false;
1508 p->shaderDenormPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8;
1509 p->shaderRoundingModeRTEFloat64 = pdevice->rad_info.chip_class >= GFX8;
1510 p->shaderRoundingModeRTZFloat64 = false;
1511 p->shaderSignedZeroInfNanPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8;
1512
1513 p->maxUpdateAfterBindDescriptorsInAllPools = UINT32_MAX / 64;
1514 p->shaderUniformBufferArrayNonUniformIndexingNative = false;
1515 p->shaderSampledImageArrayNonUniformIndexingNative = false;
1516 p->shaderStorageBufferArrayNonUniformIndexingNative = false;
1517 p->shaderStorageImageArrayNonUniformIndexingNative = false;
1518 p->shaderInputAttachmentArrayNonUniformIndexingNative = false;
1519 p->robustBufferAccessUpdateAfterBind = false;
1520 p->quadDivergentImplicitLod = false;
1521
1522 size_t max_descriptor_set_size = ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS -
1523 MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
1524 (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
1525 32 /* storage buffer, 32 due to potential space wasted on alignment */ +
1526 32 /* sampler, largest when combined with image */ +
1527 64 /* sampled image */ +
1528 64 /* storage image */);
1529 p->maxPerStageDescriptorUpdateAfterBindSamplers = max_descriptor_set_size;
1530 p->maxPerStageDescriptorUpdateAfterBindUniformBuffers = max_descriptor_set_size;
1531 p->maxPerStageDescriptorUpdateAfterBindStorageBuffers = max_descriptor_set_size;
1532 p->maxPerStageDescriptorUpdateAfterBindSampledImages = max_descriptor_set_size;
1533 p->maxPerStageDescriptorUpdateAfterBindStorageImages = max_descriptor_set_size;
1534 p->maxPerStageDescriptorUpdateAfterBindInputAttachments = max_descriptor_set_size;
1535 p->maxPerStageUpdateAfterBindResources = max_descriptor_set_size;
1536 p->maxDescriptorSetUpdateAfterBindSamplers = max_descriptor_set_size;
1537 p->maxDescriptorSetUpdateAfterBindUniformBuffers = max_descriptor_set_size;
1538 p->maxDescriptorSetUpdateAfterBindUniformBuffersDynamic = MAX_DYNAMIC_UNIFORM_BUFFERS;
1539 p->maxDescriptorSetUpdateAfterBindStorageBuffers = max_descriptor_set_size;
1540 p->maxDescriptorSetUpdateAfterBindStorageBuffersDynamic = MAX_DYNAMIC_STORAGE_BUFFERS;
1541 p->maxDescriptorSetUpdateAfterBindSampledImages = max_descriptor_set_size;
1542 p->maxDescriptorSetUpdateAfterBindStorageImages = max_descriptor_set_size;
1543 p->maxDescriptorSetUpdateAfterBindInputAttachments = max_descriptor_set_size;
1544
1545 /* We support all of the depth resolve modes */
1546 p->supportedDepthResolveModes = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR |
1547 VK_RESOLVE_MODE_AVERAGE_BIT_KHR |
1548 VK_RESOLVE_MODE_MIN_BIT_KHR |
1549 VK_RESOLVE_MODE_MAX_BIT_KHR;
1550
1551 /* Average doesn't make sense for stencil so we don't support that */
1552 p->supportedStencilResolveModes = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR |
1553 VK_RESOLVE_MODE_MIN_BIT_KHR |
1554 VK_RESOLVE_MODE_MAX_BIT_KHR;
1555
1556 p->independentResolveNone = true;
1557 p->independentResolve = true;
1558
1559 /* GFX6-8 only support single channel min/max filter. */
1560 p->filterMinmaxImageComponentMapping = pdevice->rad_info.chip_class >= GFX9;
1561 p->filterMinmaxSingleComponentFormats = true;
1562
1563 p->maxTimelineSemaphoreValueDifference = UINT64_MAX;
1564
1565 p->framebufferIntegerColorSampleCounts = VK_SAMPLE_COUNT_1_BIT;
1566 }
1567
1568 void radv_GetPhysicalDeviceProperties2(
1569 VkPhysicalDevice physicalDevice,
1570 VkPhysicalDeviceProperties2 *pProperties)
1571 {
1572 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1573 radv_GetPhysicalDeviceProperties(physicalDevice, &pProperties->properties);
1574
1575 VkPhysicalDeviceVulkan11Properties core_1_1 = {
1576 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES,
1577 };
1578 radv_get_physical_device_properties_1_1(pdevice, &core_1_1);
1579
1580 VkPhysicalDeviceVulkan12Properties core_1_2 = {
1581 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES,
1582 };
1583 radv_get_physical_device_properties_1_2(pdevice, &core_1_2);
1584
1585 #define CORE_RENAMED_PROPERTY(major, minor, ext_property, core_property) \
1586 memcpy(&properties->ext_property, &core_##major##_##minor.core_property, \
1587 sizeof(core_##major##_##minor.core_property))
1588
1589 #define CORE_PROPERTY(major, minor, property) \
1590 CORE_RENAMED_PROPERTY(major, minor, property, property)
1591
1592 vk_foreach_struct(ext, pProperties->pNext) {
1593 switch (ext->sType) {
1594 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR: {
1595 VkPhysicalDevicePushDescriptorPropertiesKHR *properties =
1596 (VkPhysicalDevicePushDescriptorPropertiesKHR *) ext;
1597 properties->maxPushDescriptors = MAX_PUSH_DESCRIPTORS;
1598 break;
1599 }
1600 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES: {
1601 VkPhysicalDeviceIDProperties *properties = (VkPhysicalDeviceIDProperties*)ext;
1602 CORE_PROPERTY(1, 1, deviceUUID);
1603 CORE_PROPERTY(1, 1, driverUUID);
1604 CORE_PROPERTY(1, 1, deviceLUID);
1605 CORE_PROPERTY(1, 1, deviceLUIDValid);
1606 break;
1607 }
1608 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_PROPERTIES: {
1609 VkPhysicalDeviceMultiviewProperties *properties = (VkPhysicalDeviceMultiviewProperties*)ext;
1610 CORE_PROPERTY(1, 1, maxMultiviewViewCount);
1611 CORE_PROPERTY(1, 1, maxMultiviewInstanceIndex);
1612 break;
1613 }
1614 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_POINT_CLIPPING_PROPERTIES: {
1615 VkPhysicalDevicePointClippingProperties *properties =
1616 (VkPhysicalDevicePointClippingProperties*)ext;
1617 CORE_PROPERTY(1, 1, pointClippingBehavior);
1618 break;
1619 }
1620 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DISCARD_RECTANGLE_PROPERTIES_EXT: {
1621 VkPhysicalDeviceDiscardRectanglePropertiesEXT *properties =
1622 (VkPhysicalDeviceDiscardRectanglePropertiesEXT*)ext;
1623 properties->maxDiscardRectangles = MAX_DISCARD_RECTANGLES;
1624 break;
1625 }
1626 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT: {
1627 VkPhysicalDeviceExternalMemoryHostPropertiesEXT *properties =
1628 (VkPhysicalDeviceExternalMemoryHostPropertiesEXT *) ext;
1629 properties->minImportedHostPointerAlignment = 4096;
1630 break;
1631 }
1632 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES: {
1633 VkPhysicalDeviceSubgroupProperties *properties =
1634 (VkPhysicalDeviceSubgroupProperties*)ext;
1635 CORE_PROPERTY(1, 1, subgroupSize);
1636 CORE_RENAMED_PROPERTY(1, 1, supportedStages,
1637 subgroupSupportedStages);
1638 CORE_RENAMED_PROPERTY(1, 1, supportedOperations,
1639 subgroupSupportedOperations);
1640 CORE_RENAMED_PROPERTY(1, 1, quadOperationsInAllStages,
1641 subgroupQuadOperationsInAllStages);
1642 break;
1643 }
1644 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES: {
1645 VkPhysicalDeviceMaintenance3Properties *properties =
1646 (VkPhysicalDeviceMaintenance3Properties*)ext;
1647 CORE_PROPERTY(1, 1, maxPerSetDescriptors);
1648 CORE_PROPERTY(1, 1, maxMemoryAllocationSize);
1649 break;
1650 }
1651 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_FILTER_MINMAX_PROPERTIES: {
1652 VkPhysicalDeviceSamplerFilterMinmaxProperties *properties =
1653 (VkPhysicalDeviceSamplerFilterMinmaxProperties *)ext;
1654 CORE_PROPERTY(1, 2, filterMinmaxImageComponentMapping);
1655 CORE_PROPERTY(1, 2, filterMinmaxSingleComponentFormats);
1656 break;
1657 }
1658 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CORE_PROPERTIES_AMD: {
1659 VkPhysicalDeviceShaderCorePropertiesAMD *properties =
1660 (VkPhysicalDeviceShaderCorePropertiesAMD *)ext;
1661
1662 /* Shader engines. */
1663 properties->shaderEngineCount =
1664 pdevice->rad_info.max_se;
1665 properties->shaderArraysPerEngineCount =
1666 pdevice->rad_info.max_sh_per_se;
1667 properties->computeUnitsPerShaderArray =
1668 pdevice->rad_info.num_good_cu_per_sh;
1669 properties->simdPerComputeUnit =
1670 pdevice->rad_info.num_simd_per_compute_unit;
1671 properties->wavefrontsPerSimd =
1672 pdevice->rad_info.max_wave64_per_simd;
1673 properties->wavefrontSize = 64;
1674
1675 /* SGPR. */
1676 properties->sgprsPerSimd =
1677 pdevice->rad_info.num_physical_sgprs_per_simd;
1678 properties->minSgprAllocation =
1679 pdevice->rad_info.min_sgpr_alloc;
1680 properties->maxSgprAllocation =
1681 pdevice->rad_info.max_sgpr_alloc;
1682 properties->sgprAllocationGranularity =
1683 pdevice->rad_info.sgpr_alloc_granularity;
1684
1685 /* VGPR. */
1686 properties->vgprsPerSimd =
1687 pdevice->rad_info.num_physical_wave64_vgprs_per_simd;
1688 properties->minVgprAllocation =
1689 pdevice->rad_info.min_wave64_vgpr_alloc;
1690 properties->maxVgprAllocation =
1691 pdevice->rad_info.max_vgpr_alloc;
1692 properties->vgprAllocationGranularity =
1693 pdevice->rad_info.wave64_vgpr_alloc_granularity;
1694 break;
1695 }
1696 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CORE_PROPERTIES_2_AMD: {
1697 VkPhysicalDeviceShaderCoreProperties2AMD *properties =
1698 (VkPhysicalDeviceShaderCoreProperties2AMD *)ext;
1699
1700 properties->shaderCoreFeatures = 0;
1701 properties->activeComputeUnitCount =
1702 pdevice->rad_info.num_good_compute_units;
1703 break;
1704 }
1705 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_PROPERTIES_EXT: {
1706 VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT *properties =
1707 (VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT *)ext;
1708 properties->maxVertexAttribDivisor = UINT32_MAX;
1709 break;
1710 }
1711 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES: {
1712 VkPhysicalDeviceDescriptorIndexingProperties *properties =
1713 (VkPhysicalDeviceDescriptorIndexingProperties*)ext;
1714 CORE_PROPERTY(1, 2, maxUpdateAfterBindDescriptorsInAllPools);
1715 CORE_PROPERTY(1, 2, shaderUniformBufferArrayNonUniformIndexingNative);
1716 CORE_PROPERTY(1, 2, shaderSampledImageArrayNonUniformIndexingNative);
1717 CORE_PROPERTY(1, 2, shaderStorageBufferArrayNonUniformIndexingNative);
1718 CORE_PROPERTY(1, 2, shaderStorageImageArrayNonUniformIndexingNative);
1719 CORE_PROPERTY(1, 2, shaderInputAttachmentArrayNonUniformIndexingNative);
1720 CORE_PROPERTY(1, 2, robustBufferAccessUpdateAfterBind);
1721 CORE_PROPERTY(1, 2, quadDivergentImplicitLod);
1722 CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindSamplers);
1723 CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindUniformBuffers);
1724 CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindStorageBuffers);
1725 CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindSampledImages);
1726 CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindStorageImages);
1727 CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindInputAttachments);
1728 CORE_PROPERTY(1, 2, maxPerStageUpdateAfterBindResources);
1729 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindSamplers);
1730 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindUniformBuffers);
1731 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindUniformBuffersDynamic);
1732 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageBuffers);
1733 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageBuffersDynamic);
1734 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindSampledImages);
1735 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageImages);
1736 CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindInputAttachments);
1737 break;
1738 }
1739 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_PROPERTIES: {
1740 VkPhysicalDeviceProtectedMemoryProperties *properties =
1741 (VkPhysicalDeviceProtectedMemoryProperties *)ext;
1742 CORE_PROPERTY(1, 1, protectedNoFault);
1743 break;
1744 }
1745 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONSERVATIVE_RASTERIZATION_PROPERTIES_EXT: {
1746 VkPhysicalDeviceConservativeRasterizationPropertiesEXT *properties =
1747 (VkPhysicalDeviceConservativeRasterizationPropertiesEXT *)ext;
1748 properties->primitiveOverestimationSize = 0;
1749 properties->maxExtraPrimitiveOverestimationSize = 0;
1750 properties->extraPrimitiveOverestimationSizeGranularity = 0;
1751 properties->primitiveUnderestimation = false;
1752 properties->conservativePointAndLineRasterization = false;
1753 properties->degenerateTrianglesRasterized = false;
1754 properties->degenerateLinesRasterized = false;
1755 properties->fullyCoveredFragmentShaderInputVariable = false;
1756 properties->conservativeRasterizationPostDepthCoverage = false;
1757 break;
1758 }
1759 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT: {
1760 VkPhysicalDevicePCIBusInfoPropertiesEXT *properties =
1761 (VkPhysicalDevicePCIBusInfoPropertiesEXT *)ext;
1762 properties->pciDomain = pdevice->bus_info.domain;
1763 properties->pciBus = pdevice->bus_info.bus;
1764 properties->pciDevice = pdevice->bus_info.dev;
1765 properties->pciFunction = pdevice->bus_info.func;
1766 break;
1767 }
1768 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES: {
1769 VkPhysicalDeviceDriverProperties *properties =
1770 (VkPhysicalDeviceDriverProperties *) ext;
1771 CORE_PROPERTY(1, 2, driverID);
1772 CORE_PROPERTY(1, 2, driverName);
1773 CORE_PROPERTY(1, 2, driverInfo);
1774 CORE_PROPERTY(1, 2, conformanceVersion);
1775 break;
1776 }
1777 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_PROPERTIES_EXT: {
1778 VkPhysicalDeviceTransformFeedbackPropertiesEXT *properties =
1779 (VkPhysicalDeviceTransformFeedbackPropertiesEXT *)ext;
1780 properties->maxTransformFeedbackStreams = MAX_SO_STREAMS;
1781 properties->maxTransformFeedbackBuffers = MAX_SO_BUFFERS;
1782 properties->maxTransformFeedbackBufferSize = UINT32_MAX;
1783 properties->maxTransformFeedbackStreamDataSize = 512;
1784 properties->maxTransformFeedbackBufferDataSize = UINT32_MAX;
1785 properties->maxTransformFeedbackBufferDataStride = 512;
1786 properties->transformFeedbackQueries = !pdevice->use_ngg_streamout;
1787 properties->transformFeedbackStreamsLinesTriangles = !pdevice->use_ngg_streamout;
1788 properties->transformFeedbackRasterizationStreamSelect = false;
1789 properties->transformFeedbackDraw = true;
1790 break;
1791 }
1792 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_PROPERTIES_EXT: {
1793 VkPhysicalDeviceInlineUniformBlockPropertiesEXT *props =
1794 (VkPhysicalDeviceInlineUniformBlockPropertiesEXT *)ext;
1795
1796 props->maxInlineUniformBlockSize = MAX_INLINE_UNIFORM_BLOCK_SIZE;
1797 props->maxPerStageDescriptorInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_SETS;
1798 props->maxPerStageDescriptorUpdateAfterBindInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_SETS;
1799 props->maxDescriptorSetInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_COUNT;
1800 props->maxDescriptorSetUpdateAfterBindInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_COUNT;
1801 break;
1802 }
1803 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLE_LOCATIONS_PROPERTIES_EXT: {
1804 VkPhysicalDeviceSampleLocationsPropertiesEXT *properties =
1805 (VkPhysicalDeviceSampleLocationsPropertiesEXT *)ext;
1806 properties->sampleLocationSampleCounts = VK_SAMPLE_COUNT_2_BIT |
1807 VK_SAMPLE_COUNT_4_BIT |
1808 VK_SAMPLE_COUNT_8_BIT;
1809 properties->maxSampleLocationGridSize = (VkExtent2D){ 2 , 2 };
1810 properties->sampleLocationCoordinateRange[0] = 0.0f;
1811 properties->sampleLocationCoordinateRange[1] = 0.9375f;
1812 properties->sampleLocationSubPixelBits = 4;
1813 properties->variableSampleLocations = false;
1814 break;
1815 }
1816 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES: {
1817 VkPhysicalDeviceDepthStencilResolveProperties *properties =
1818 (VkPhysicalDeviceDepthStencilResolveProperties *)ext;
1819 CORE_PROPERTY(1, 2, supportedDepthResolveModes);
1820 CORE_PROPERTY(1, 2, supportedStencilResolveModes);
1821 CORE_PROPERTY(1, 2, independentResolveNone);
1822 CORE_PROPERTY(1, 2, independentResolve);
1823 break;
1824 }
1825 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_PROPERTIES_EXT: {
1826 VkPhysicalDeviceTexelBufferAlignmentPropertiesEXT *properties =
1827 (VkPhysicalDeviceTexelBufferAlignmentPropertiesEXT *)ext;
1828 properties->storageTexelBufferOffsetAlignmentBytes = 4;
1829 properties->storageTexelBufferOffsetSingleTexelAlignment = true;
1830 properties->uniformTexelBufferOffsetAlignmentBytes = 4;
1831 properties->uniformTexelBufferOffsetSingleTexelAlignment = true;
1832 break;
1833 }
1834 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES : {
1835 VkPhysicalDeviceFloatControlsProperties *properties =
1836 (VkPhysicalDeviceFloatControlsProperties *)ext;
1837 CORE_PROPERTY(1, 2, denormBehaviorIndependence);
1838 CORE_PROPERTY(1, 2, roundingModeIndependence);
1839 CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat16);
1840 CORE_PROPERTY(1, 2, shaderDenormPreserveFloat16);
1841 CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat16);
1842 CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat16);
1843 CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat16);
1844 CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat32);
1845 CORE_PROPERTY(1, 2, shaderDenormPreserveFloat32);
1846 CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat32);
1847 CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat32);
1848 CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat32);
1849 CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat64);
1850 CORE_PROPERTY(1, 2, shaderDenormPreserveFloat64);
1851 CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat64);
1852 CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat64);
1853 CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat64);
1854 break;
1855 }
1856 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_PROPERTIES: {
1857 VkPhysicalDeviceTimelineSemaphoreProperties *properties =
1858 (VkPhysicalDeviceTimelineSemaphoreProperties *) ext;
1859 CORE_PROPERTY(1, 2, maxTimelineSemaphoreValueDifference);
1860 break;
1861 }
1862 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT: {
1863 VkPhysicalDeviceSubgroupSizeControlPropertiesEXT *props =
1864 (VkPhysicalDeviceSubgroupSizeControlPropertiesEXT *)ext;
1865 props->minSubgroupSize = 64;
1866 props->maxSubgroupSize = 64;
1867 props->maxComputeWorkgroupSubgroups = UINT32_MAX;
1868 props->requiredSubgroupSizeStages = 0;
1869
1870 if (pdevice->rad_info.chip_class >= GFX10) {
1871 /* Only GFX10+ supports wave32. */
1872 props->minSubgroupSize = 32;
1873 props->requiredSubgroupSizeStages = VK_SHADER_STAGE_COMPUTE_BIT;
1874 }
1875 break;
1876 }
1877 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES:
1878 radv_get_physical_device_properties_1_1(pdevice, (void *)ext);
1879 break;
1880 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES:
1881 radv_get_physical_device_properties_1_2(pdevice, (void *)ext);
1882 break;
1883 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_PROPERTIES_EXT: {
1884 VkPhysicalDeviceLineRasterizationPropertiesEXT *props =
1885 (VkPhysicalDeviceLineRasterizationPropertiesEXT *)ext;
1886 props->lineSubPixelPrecisionBits = 4;
1887 break;
1888 }
1889 default:
1890 break;
1891 }
1892 }
1893 }
1894
1895 static void radv_get_physical_device_queue_family_properties(
1896 struct radv_physical_device* pdevice,
1897 uint32_t* pCount,
1898 VkQueueFamilyProperties** pQueueFamilyProperties)
1899 {
1900 int num_queue_families = 1;
1901 int idx;
1902 if (pdevice->rad_info.num_rings[RING_COMPUTE] > 0 &&
1903 !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE))
1904 num_queue_families++;
1905
1906 if (pQueueFamilyProperties == NULL) {
1907 *pCount = num_queue_families;
1908 return;
1909 }
1910
1911 if (!*pCount)
1912 return;
1913
1914 idx = 0;
1915 if (*pCount >= 1) {
1916 *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
1917 .queueFlags = VK_QUEUE_GRAPHICS_BIT |
1918 VK_QUEUE_COMPUTE_BIT |
1919 VK_QUEUE_TRANSFER_BIT |
1920 VK_QUEUE_SPARSE_BINDING_BIT,
1921 .queueCount = 1,
1922 .timestampValidBits = 64,
1923 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
1924 };
1925 idx++;
1926 }
1927
1928 if (pdevice->rad_info.num_rings[RING_COMPUTE] > 0 &&
1929 !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE)) {
1930 if (*pCount > idx) {
1931 *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
1932 .queueFlags = VK_QUEUE_COMPUTE_BIT |
1933 VK_QUEUE_TRANSFER_BIT |
1934 VK_QUEUE_SPARSE_BINDING_BIT,
1935 .queueCount = pdevice->rad_info.num_rings[RING_COMPUTE],
1936 .timestampValidBits = 64,
1937 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
1938 };
1939 idx++;
1940 }
1941 }
1942 *pCount = idx;
1943 }
1944
1945 void radv_GetPhysicalDeviceQueueFamilyProperties(
1946 VkPhysicalDevice physicalDevice,
1947 uint32_t* pCount,
1948 VkQueueFamilyProperties* pQueueFamilyProperties)
1949 {
1950 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1951 if (!pQueueFamilyProperties) {
1952 radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
1953 return;
1954 }
1955 VkQueueFamilyProperties *properties[] = {
1956 pQueueFamilyProperties + 0,
1957 pQueueFamilyProperties + 1,
1958 pQueueFamilyProperties + 2,
1959 };
1960 radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
1961 assert(*pCount <= 3);
1962 }
1963
1964 void radv_GetPhysicalDeviceQueueFamilyProperties2(
1965 VkPhysicalDevice physicalDevice,
1966 uint32_t* pCount,
1967 VkQueueFamilyProperties2 *pQueueFamilyProperties)
1968 {
1969 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1970 if (!pQueueFamilyProperties) {
1971 radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
1972 return;
1973 }
1974 VkQueueFamilyProperties *properties[] = {
1975 &pQueueFamilyProperties[0].queueFamilyProperties,
1976 &pQueueFamilyProperties[1].queueFamilyProperties,
1977 &pQueueFamilyProperties[2].queueFamilyProperties,
1978 };
1979 radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
1980 assert(*pCount <= 3);
1981 }
1982
1983 void radv_GetPhysicalDeviceMemoryProperties(
1984 VkPhysicalDevice physicalDevice,
1985 VkPhysicalDeviceMemoryProperties *pMemoryProperties)
1986 {
1987 RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
1988
1989 *pMemoryProperties = physical_device->memory_properties;
1990 }
1991
1992 static void
1993 radv_get_memory_budget_properties(VkPhysicalDevice physicalDevice,
1994 VkPhysicalDeviceMemoryBudgetPropertiesEXT *memoryBudget)
1995 {
1996 RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
1997 VkPhysicalDeviceMemoryProperties *memory_properties = &device->memory_properties;
1998 uint64_t visible_vram_size = radv_get_visible_vram_size(device);
1999 uint64_t vram_size = radv_get_vram_size(device);
2000 uint64_t gtt_size = device->rad_info.gart_size;
2001 uint64_t heap_budget, heap_usage;
2002
2003 /* For all memory heaps, the computation of budget is as follow:
2004 * heap_budget = heap_size - global_heap_usage + app_heap_usage
2005 *
2006 * The Vulkan spec 1.1.97 says that the budget should include any
2007 * currently allocated device memory.
2008 *
2009 * Note that the application heap usages are not really accurate (eg.
2010 * in presence of shared buffers).
2011 */
2012 for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
2013 uint32_t heap_index = device->memory_properties.memoryTypes[i].heapIndex;
2014
2015 if ((device->memory_domains[i] & RADEON_DOMAIN_VRAM) && (device->memory_flags[i] & RADEON_FLAG_NO_CPU_ACCESS)) {
2016 heap_usage = device->ws->query_value(device->ws,
2017 RADEON_ALLOCATED_VRAM);
2018
2019 heap_budget = vram_size -
2020 device->ws->query_value(device->ws, RADEON_VRAM_USAGE) +
2021 heap_usage;
2022
2023 memoryBudget->heapBudget[heap_index] = heap_budget;
2024 memoryBudget->heapUsage[heap_index] = heap_usage;
2025 } else if (device->memory_domains[i] & RADEON_DOMAIN_VRAM) {
2026 heap_usage = device->ws->query_value(device->ws,
2027 RADEON_ALLOCATED_VRAM_VIS);
2028
2029 heap_budget = visible_vram_size -
2030 device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE) +
2031 heap_usage;
2032
2033 memoryBudget->heapBudget[heap_index] = heap_budget;
2034 memoryBudget->heapUsage[heap_index] = heap_usage;
2035 } else {
2036 assert(device->memory_domains[i] & RADEON_DOMAIN_GTT);
2037
2038 heap_usage = device->ws->query_value(device->ws,
2039 RADEON_ALLOCATED_GTT);
2040
2041 heap_budget = gtt_size -
2042 device->ws->query_value(device->ws, RADEON_GTT_USAGE) +
2043 heap_usage;
2044
2045 memoryBudget->heapBudget[heap_index] = heap_budget;
2046 memoryBudget->heapUsage[heap_index] = heap_usage;
2047 }
2048 }
2049
2050 /* The heapBudget and heapUsage values must be zero for array elements
2051 * greater than or equal to
2052 * VkPhysicalDeviceMemoryProperties::memoryHeapCount.
2053 */
2054 for (uint32_t i = memory_properties->memoryHeapCount; i < VK_MAX_MEMORY_HEAPS; i++) {
2055 memoryBudget->heapBudget[i] = 0;
2056 memoryBudget->heapUsage[i] = 0;
2057 }
2058 }
2059
2060 void radv_GetPhysicalDeviceMemoryProperties2(
2061 VkPhysicalDevice physicalDevice,
2062 VkPhysicalDeviceMemoryProperties2 *pMemoryProperties)
2063 {
2064 radv_GetPhysicalDeviceMemoryProperties(physicalDevice,
2065 &pMemoryProperties->memoryProperties);
2066
2067 VkPhysicalDeviceMemoryBudgetPropertiesEXT *memory_budget =
2068 vk_find_struct(pMemoryProperties->pNext,
2069 PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT);
2070 if (memory_budget)
2071 radv_get_memory_budget_properties(physicalDevice, memory_budget);
2072 }
2073
2074 VkResult radv_GetMemoryHostPointerPropertiesEXT(
2075 VkDevice _device,
2076 VkExternalMemoryHandleTypeFlagBits handleType,
2077 const void *pHostPointer,
2078 VkMemoryHostPointerPropertiesEXT *pMemoryHostPointerProperties)
2079 {
2080 RADV_FROM_HANDLE(radv_device, device, _device);
2081
2082 switch (handleType)
2083 {
2084 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT: {
2085 const struct radv_physical_device *physical_device = device->physical_device;
2086 uint32_t memoryTypeBits = 0;
2087 for (int i = 0; i < physical_device->memory_properties.memoryTypeCount; i++) {
2088 if (physical_device->memory_domains[i] == RADEON_DOMAIN_GTT &&
2089 !(physical_device->memory_flags[i] & RADEON_FLAG_GTT_WC)) {
2090 memoryTypeBits = (1 << i);
2091 break;
2092 }
2093 }
2094 pMemoryHostPointerProperties->memoryTypeBits = memoryTypeBits;
2095 return VK_SUCCESS;
2096 }
2097 default:
2098 return VK_ERROR_INVALID_EXTERNAL_HANDLE;
2099 }
2100 }
2101
2102 static enum radeon_ctx_priority
2103 radv_get_queue_global_priority(const VkDeviceQueueGlobalPriorityCreateInfoEXT *pObj)
2104 {
2105 /* Default to MEDIUM when a specific global priority isn't requested */
2106 if (!pObj)
2107 return RADEON_CTX_PRIORITY_MEDIUM;
2108
2109 switch(pObj->globalPriority) {
2110 case VK_QUEUE_GLOBAL_PRIORITY_REALTIME_EXT:
2111 return RADEON_CTX_PRIORITY_REALTIME;
2112 case VK_QUEUE_GLOBAL_PRIORITY_HIGH_EXT:
2113 return RADEON_CTX_PRIORITY_HIGH;
2114 case VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_EXT:
2115 return RADEON_CTX_PRIORITY_MEDIUM;
2116 case VK_QUEUE_GLOBAL_PRIORITY_LOW_EXT:
2117 return RADEON_CTX_PRIORITY_LOW;
2118 default:
2119 unreachable("Illegal global priority value");
2120 return RADEON_CTX_PRIORITY_INVALID;
2121 }
2122 }
2123
2124 static int
2125 radv_queue_init(struct radv_device *device, struct radv_queue *queue,
2126 uint32_t queue_family_index, int idx,
2127 VkDeviceQueueCreateFlags flags,
2128 const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority)
2129 {
2130 queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
2131 queue->device = device;
2132 queue->queue_family_index = queue_family_index;
2133 queue->queue_idx = idx;
2134 queue->priority = radv_get_queue_global_priority(global_priority);
2135 queue->flags = flags;
2136
2137 queue->hw_ctx = device->ws->ctx_create(device->ws, queue->priority);
2138 if (!queue->hw_ctx)
2139 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2140
2141 list_inithead(&queue->pending_submissions);
2142 pthread_mutex_init(&queue->pending_mutex, NULL);
2143
2144 return VK_SUCCESS;
2145 }
2146
2147 static void
2148 radv_queue_finish(struct radv_queue *queue)
2149 {
2150 pthread_mutex_destroy(&queue->pending_mutex);
2151
2152 if (queue->hw_ctx)
2153 queue->device->ws->ctx_destroy(queue->hw_ctx);
2154
2155 if (queue->initial_full_flush_preamble_cs)
2156 queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
2157 if (queue->initial_preamble_cs)
2158 queue->device->ws->cs_destroy(queue->initial_preamble_cs);
2159 if (queue->continue_preamble_cs)
2160 queue->device->ws->cs_destroy(queue->continue_preamble_cs);
2161 if (queue->descriptor_bo)
2162 queue->device->ws->buffer_destroy(queue->descriptor_bo);
2163 if (queue->scratch_bo)
2164 queue->device->ws->buffer_destroy(queue->scratch_bo);
2165 if (queue->esgs_ring_bo)
2166 queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
2167 if (queue->gsvs_ring_bo)
2168 queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
2169 if (queue->tess_rings_bo)
2170 queue->device->ws->buffer_destroy(queue->tess_rings_bo);
2171 if (queue->gds_bo)
2172 queue->device->ws->buffer_destroy(queue->gds_bo);
2173 if (queue->gds_oa_bo)
2174 queue->device->ws->buffer_destroy(queue->gds_oa_bo);
2175 if (queue->compute_scratch_bo)
2176 queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
2177 }
2178
2179 static void
2180 radv_bo_list_init(struct radv_bo_list *bo_list)
2181 {
2182 pthread_mutex_init(&bo_list->mutex, NULL);
2183 bo_list->list.count = bo_list->capacity = 0;
2184 bo_list->list.bos = NULL;
2185 }
2186
2187 static void
2188 radv_bo_list_finish(struct radv_bo_list *bo_list)
2189 {
2190 free(bo_list->list.bos);
2191 pthread_mutex_destroy(&bo_list->mutex);
2192 }
2193
2194 VkResult radv_bo_list_add(struct radv_device *device,
2195 struct radeon_winsys_bo *bo)
2196 {
2197 struct radv_bo_list *bo_list = &device->bo_list;
2198
2199 if (bo->is_local)
2200 return VK_SUCCESS;
2201
2202 if (unlikely(!device->use_global_bo_list))
2203 return VK_SUCCESS;
2204
2205 pthread_mutex_lock(&bo_list->mutex);
2206 if (bo_list->list.count == bo_list->capacity) {
2207 unsigned capacity = MAX2(4, bo_list->capacity * 2);
2208 void *data = realloc(bo_list->list.bos, capacity * sizeof(struct radeon_winsys_bo*));
2209
2210 if (!data) {
2211 pthread_mutex_unlock(&bo_list->mutex);
2212 return VK_ERROR_OUT_OF_HOST_MEMORY;
2213 }
2214
2215 bo_list->list.bos = (struct radeon_winsys_bo**)data;
2216 bo_list->capacity = capacity;
2217 }
2218
2219 bo_list->list.bos[bo_list->list.count++] = bo;
2220 pthread_mutex_unlock(&bo_list->mutex);
2221 return VK_SUCCESS;
2222 }
2223
2224 void radv_bo_list_remove(struct radv_device *device,
2225 struct radeon_winsys_bo *bo)
2226 {
2227 struct radv_bo_list *bo_list = &device->bo_list;
2228
2229 if (bo->is_local)
2230 return;
2231
2232 if (unlikely(!device->use_global_bo_list))
2233 return;
2234
2235 pthread_mutex_lock(&bo_list->mutex);
2236 /* Loop the list backwards so we find the most recently added
2237 * memory first. */
2238 for(unsigned i = bo_list->list.count; i-- > 0;) {
2239 if (bo_list->list.bos[i] == bo) {
2240 bo_list->list.bos[i] = bo_list->list.bos[bo_list->list.count - 1];
2241 --bo_list->list.count;
2242 break;
2243 }
2244 }
2245 pthread_mutex_unlock(&bo_list->mutex);
2246 }
2247
2248 static void
2249 radv_device_init_gs_info(struct radv_device *device)
2250 {
2251 device->gs_table_depth = ac_get_gs_table_depth(device->physical_device->rad_info.chip_class,
2252 device->physical_device->rad_info.family);
2253 }
2254
2255 static int radv_get_device_extension_index(const char *name)
2256 {
2257 for (unsigned i = 0; i < RADV_DEVICE_EXTENSION_COUNT; ++i) {
2258 if (strcmp(name, radv_device_extensions[i].extensionName) == 0)
2259 return i;
2260 }
2261 return -1;
2262 }
2263
2264 static int
2265 radv_get_int_debug_option(const char *name, int default_value)
2266 {
2267 const char *str;
2268 int result;
2269
2270 str = getenv(name);
2271 if (!str) {
2272 result = default_value;
2273 } else {
2274 char *endptr;
2275
2276 result = strtol(str, &endptr, 0);
2277 if (str == endptr) {
2278 /* No digits founs. */
2279 result = default_value;
2280 }
2281 }
2282
2283 return result;
2284 }
2285
2286 static int install_seccomp_filter() {
2287
2288 struct sock_filter filter[] = {
2289 /* Check arch is 64bit x86 */
2290 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, arch))),
2291 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 0, 12),
2292
2293 /* Futex is required for mutex locks */
2294 #if defined __NR__newselect
2295 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2296 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR__newselect, 11, 0),
2297 #elif defined __NR_select
2298 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2299 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_select, 11, 0),
2300 #else
2301 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2302 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_pselect6, 11, 0),
2303 #endif
2304
2305 /* Allow system exit calls for the forked process */
2306 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2307 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_exit_group, 9, 0),
2308
2309 /* Allow system read calls */
2310 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2311 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_read, 7, 0),
2312
2313 /* Allow system write calls */
2314 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2315 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_write, 5, 0),
2316
2317 /* Allow system brk calls (we need this for malloc) */
2318 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2319 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_brk, 3, 0),
2320
2321 /* Futex is required for mutex locks */
2322 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2323 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_futex, 1, 0),
2324
2325 /* Return error if we hit a system call not on the whitelist */
2326 BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA)),
2327
2328 /* Allow whitelisted system calls */
2329 BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW),
2330 };
2331
2332 struct sock_fprog prog = {
2333 .len = (unsigned short)(sizeof(filter) / sizeof(filter[0])),
2334 .filter = filter,
2335 };
2336
2337 if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0))
2338 return -1;
2339
2340 if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog))
2341 return -1;
2342
2343 return 0;
2344 }
2345
2346 /* Helper function with timeout support for reading from the pipe between
2347 * processes used for secure compile.
2348 */
2349 bool radv_sc_read(int fd, void *buf, size_t size, bool timeout)
2350 {
2351 fd_set fds;
2352 struct timeval tv;
2353
2354 FD_ZERO(&fds);
2355 FD_SET(fd, &fds);
2356
2357 while (true) {
2358 /* We can't rely on the value of tv after calling select() so
2359 * we must reset it on each iteration of the loop.
2360 */
2361 tv.tv_sec = 5;
2362 tv.tv_usec = 0;
2363
2364 int rval = select(fd + 1, &fds, NULL, NULL, timeout ? &tv : NULL);
2365
2366 if (rval == -1) {
2367 /* select error */
2368 return false;
2369 } else if (rval) {
2370 ssize_t bytes_read = read(fd, buf, size);
2371 if (bytes_read < 0)
2372 return false;
2373
2374 buf += bytes_read;
2375 size -= bytes_read;
2376 if (size == 0)
2377 return true;
2378 } else {
2379 /* select timeout */
2380 return false;
2381 }
2382 }
2383 }
2384
2385 static bool radv_close_all_fds(const int *keep_fds, int keep_fd_count)
2386 {
2387 DIR *d;
2388 struct dirent *dir;
2389 d = opendir("/proc/self/fd");
2390 if (!d)
2391 return false;
2392 int dir_fd = dirfd(d);
2393
2394 while ((dir = readdir(d)) != NULL) {
2395 if (dir->d_name[0] == '.')
2396 continue;
2397
2398 int fd = atoi(dir->d_name);
2399 if (fd == dir_fd)
2400 continue;
2401
2402 bool keep = false;
2403 for (int i = 0; !keep && i < keep_fd_count; ++i)
2404 if (keep_fds[i] == fd)
2405 keep = true;
2406
2407 if (keep)
2408 continue;
2409
2410 close(fd);
2411 }
2412 closedir(d);
2413 return true;
2414 }
2415
2416 static bool secure_compile_open_fifo_fds(struct radv_secure_compile_state *sc,
2417 int *fd_server, int *fd_client,
2418 unsigned process, bool make_fifo)
2419 {
2420 bool result = false;
2421 char *fifo_server_path = NULL;
2422 char *fifo_client_path = NULL;
2423
2424 if (asprintf(&fifo_server_path, "/tmp/radv_server_%s_%u", sc->uid, process) == -1)
2425 goto open_fifo_exit;
2426
2427 if (asprintf(&fifo_client_path, "/tmp/radv_client_%s_%u", sc->uid, process) == -1)
2428 goto open_fifo_exit;
2429
2430 if (make_fifo) {
2431 int file1 = mkfifo(fifo_server_path, 0666);
2432 if(file1 < 0)
2433 goto open_fifo_exit;
2434
2435 int file2 = mkfifo(fifo_client_path, 0666);
2436 if(file2 < 0)
2437 goto open_fifo_exit;
2438 }
2439
2440 *fd_server = open(fifo_server_path, O_RDWR);
2441 if(*fd_server < 1)
2442 goto open_fifo_exit;
2443
2444 *fd_client = open(fifo_client_path, O_RDWR);
2445 if(*fd_client < 1) {
2446 close(*fd_server);
2447 goto open_fifo_exit;
2448 }
2449
2450 result = true;
2451
2452 open_fifo_exit:
2453 free(fifo_server_path);
2454 free(fifo_client_path);
2455
2456 return result;
2457 }
2458
2459 static void run_secure_compile_device(struct radv_device *device, unsigned process,
2460 int fd_idle_device_output)
2461 {
2462 int fd_secure_input;
2463 int fd_secure_output;
2464 bool fifo_result = secure_compile_open_fifo_fds(device->sc_state,
2465 &fd_secure_input,
2466 &fd_secure_output,
2467 process, false);
2468
2469 enum radv_secure_compile_type sc_type;
2470
2471 const int needed_fds[] = {
2472 fd_secure_input,
2473 fd_secure_output,
2474 fd_idle_device_output,
2475 };
2476
2477 if (!fifo_result || !radv_close_all_fds(needed_fds, ARRAY_SIZE(needed_fds)) ||
2478 install_seccomp_filter() == -1) {
2479 sc_type = RADV_SC_TYPE_INIT_FAILURE;
2480 } else {
2481 sc_type = RADV_SC_TYPE_INIT_SUCCESS;
2482 device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input;
2483 device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output;
2484 }
2485
2486 write(fd_idle_device_output, &sc_type, sizeof(sc_type));
2487
2488 if (sc_type == RADV_SC_TYPE_INIT_FAILURE)
2489 goto secure_compile_exit;
2490
2491 while (true) {
2492 radv_sc_read(fd_secure_input, &sc_type, sizeof(sc_type), false);
2493
2494 if (sc_type == RADV_SC_TYPE_COMPILE_PIPELINE) {
2495 struct radv_pipeline *pipeline;
2496 bool sc_read = true;
2497
2498 pipeline = vk_zalloc2(&device->alloc, NULL, sizeof(*pipeline), 8,
2499 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2500
2501 pipeline->device = device;
2502
2503 /* Read pipeline layout */
2504 struct radv_pipeline_layout layout;
2505 sc_read = radv_sc_read(fd_secure_input, &layout, sizeof(struct radv_pipeline_layout), true);
2506 sc_read &= radv_sc_read(fd_secure_input, &layout.num_sets, sizeof(uint32_t), true);
2507 if (!sc_read)
2508 goto secure_compile_exit;
2509
2510 for (uint32_t set = 0; set < layout.num_sets; set++) {
2511 uint32_t layout_size;
2512 sc_read &= radv_sc_read(fd_secure_input, &layout_size, sizeof(uint32_t), true);
2513 if (!sc_read)
2514 goto secure_compile_exit;
2515
2516 layout.set[set].layout = malloc(layout_size);
2517 layout.set[set].layout->layout_size = layout_size;
2518 sc_read &= radv_sc_read(fd_secure_input, layout.set[set].layout,
2519 layout.set[set].layout->layout_size, true);
2520 }
2521
2522 pipeline->layout = &layout;
2523
2524 /* Read pipeline key */
2525 struct radv_pipeline_key key;
2526 sc_read &= radv_sc_read(fd_secure_input, &key, sizeof(struct radv_pipeline_key), true);
2527
2528 /* Read pipeline create flags */
2529 VkPipelineCreateFlags flags;
2530 sc_read &= radv_sc_read(fd_secure_input, &flags, sizeof(VkPipelineCreateFlags), true);
2531
2532 /* Read stage and shader information */
2533 uint32_t num_stages;
2534 const VkPipelineShaderStageCreateInfo *pStages[MESA_SHADER_STAGES] = { 0, };
2535 sc_read &= radv_sc_read(fd_secure_input, &num_stages, sizeof(uint32_t), true);
2536 if (!sc_read)
2537 goto secure_compile_exit;
2538
2539 for (uint32_t i = 0; i < num_stages; i++) {
2540
2541 /* Read stage */
2542 gl_shader_stage stage;
2543 sc_read &= radv_sc_read(fd_secure_input, &stage, sizeof(gl_shader_stage), true);
2544
2545 VkPipelineShaderStageCreateInfo *pStage = calloc(1, sizeof(VkPipelineShaderStageCreateInfo));
2546
2547 /* Read entry point name */
2548 size_t name_size;
2549 sc_read &= radv_sc_read(fd_secure_input, &name_size, sizeof(size_t), true);
2550 if (!sc_read)
2551 goto secure_compile_exit;
2552
2553 char *ep_name = malloc(name_size);
2554 sc_read &= radv_sc_read(fd_secure_input, ep_name, name_size, true);
2555 pStage->pName = ep_name;
2556
2557 /* Read shader module */
2558 size_t module_size;
2559 sc_read &= radv_sc_read(fd_secure_input, &module_size, sizeof(size_t), true);
2560 if (!sc_read)
2561 goto secure_compile_exit;
2562
2563 struct radv_shader_module *module = malloc(module_size);
2564 sc_read &= radv_sc_read(fd_secure_input, module, module_size, true);
2565 pStage->module = radv_shader_module_to_handle(module);
2566
2567 /* Read specialization info */
2568 bool has_spec_info;
2569 sc_read &= radv_sc_read(fd_secure_input, &has_spec_info, sizeof(bool), true);
2570 if (!sc_read)
2571 goto secure_compile_exit;
2572
2573 if (has_spec_info) {
2574 VkSpecializationInfo *specInfo = malloc(sizeof(VkSpecializationInfo));
2575 pStage->pSpecializationInfo = specInfo;
2576
2577 sc_read &= radv_sc_read(fd_secure_input, &specInfo->dataSize, sizeof(size_t), true);
2578 if (!sc_read)
2579 goto secure_compile_exit;
2580
2581 void *si_data = malloc(specInfo->dataSize);
2582 sc_read &= radv_sc_read(fd_secure_input, si_data, specInfo->dataSize, true);
2583 specInfo->pData = si_data;
2584
2585 sc_read &= radv_sc_read(fd_secure_input, &specInfo->mapEntryCount, sizeof(uint32_t), true);
2586 if (!sc_read)
2587 goto secure_compile_exit;
2588
2589 VkSpecializationMapEntry *mapEntries = malloc(sizeof(VkSpecializationMapEntry) * specInfo->mapEntryCount);
2590 for (uint32_t j = 0; j < specInfo->mapEntryCount; j++) {
2591 sc_read &= radv_sc_read(fd_secure_input, &mapEntries[j], sizeof(VkSpecializationMapEntry), true);
2592 if (!sc_read)
2593 goto secure_compile_exit;
2594 }
2595
2596 specInfo->pMapEntries = mapEntries;
2597 }
2598
2599 pStages[stage] = pStage;
2600 }
2601
2602 /* Compile the shaders */
2603 VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_SHADER_STAGES] = { 0 };
2604 radv_create_shaders(pipeline, device, NULL, &key, pStages, flags, NULL, stage_feedbacks);
2605
2606 /* free memory allocated above */
2607 for (uint32_t set = 0; set < layout.num_sets; set++)
2608 free(layout.set[set].layout);
2609
2610 for (uint32_t i = 0; i < MESA_SHADER_STAGES; i++) {
2611 if (!pStages[i])
2612 continue;
2613
2614 free((void *) pStages[i]->pName);
2615 free(radv_shader_module_from_handle(pStages[i]->module));
2616 if (pStages[i]->pSpecializationInfo) {
2617 free((void *) pStages[i]->pSpecializationInfo->pData);
2618 free((void *) pStages[i]->pSpecializationInfo->pMapEntries);
2619 free((void *) pStages[i]->pSpecializationInfo);
2620 }
2621 free((void *) pStages[i]);
2622 }
2623
2624 vk_free(&device->alloc, pipeline);
2625
2626 sc_type = RADV_SC_TYPE_COMPILE_PIPELINE_FINISHED;
2627 write(fd_secure_output, &sc_type, sizeof(sc_type));
2628
2629 } else if (sc_type == RADV_SC_TYPE_DESTROY_DEVICE) {
2630 goto secure_compile_exit;
2631 }
2632 }
2633
2634 secure_compile_exit:
2635 close(fd_secure_input);
2636 close(fd_secure_output);
2637 close(fd_idle_device_output);
2638 _exit(0);
2639 }
2640
2641 static enum radv_secure_compile_type fork_secure_compile_device(struct radv_device *device, unsigned process)
2642 {
2643 int fd_secure_input[2];
2644 int fd_secure_output[2];
2645
2646 /* create pipe descriptors (used to communicate between processes) */
2647 if (pipe(fd_secure_input) == -1 || pipe(fd_secure_output) == -1)
2648 return RADV_SC_TYPE_INIT_FAILURE;
2649
2650
2651 int sc_pid;
2652 if ((sc_pid = fork()) == 0) {
2653 device->sc_state->secure_compile_thread_counter = process;
2654 run_secure_compile_device(device, process, fd_secure_output[1]);
2655 } else {
2656 if (sc_pid == -1)
2657 return RADV_SC_TYPE_INIT_FAILURE;
2658
2659 /* Read the init result returned from the secure process */
2660 enum radv_secure_compile_type sc_type;
2661 bool sc_read = radv_sc_read(fd_secure_output[0], &sc_type, sizeof(sc_type), true);
2662
2663 if (sc_type == RADV_SC_TYPE_INIT_FAILURE || !sc_read) {
2664 close(fd_secure_input[0]);
2665 close(fd_secure_input[1]);
2666 close(fd_secure_output[1]);
2667 close(fd_secure_output[0]);
2668 int status;
2669 waitpid(sc_pid, &status, 0);
2670
2671 return RADV_SC_TYPE_INIT_FAILURE;
2672 } else {
2673 assert(sc_type == RADV_SC_TYPE_INIT_SUCCESS);
2674 write(device->sc_state->secure_compile_processes[process].fd_secure_output, &sc_type, sizeof(sc_type));
2675
2676 close(fd_secure_input[0]);
2677 close(fd_secure_input[1]);
2678 close(fd_secure_output[1]);
2679 close(fd_secure_output[0]);
2680
2681 int status;
2682 waitpid(sc_pid, &status, 0);
2683 }
2684 }
2685
2686 return RADV_SC_TYPE_INIT_SUCCESS;
2687 }
2688
2689 /* Run a bare bones fork of a device that was forked right after its creation.
2690 * This device will have low overhead when it is forked again before each
2691 * pipeline compilation. This device sits idle and its only job is to fork
2692 * itself.
2693 */
2694 static void run_secure_compile_idle_device(struct radv_device *device, unsigned process,
2695 int fd_secure_input, int fd_secure_output)
2696 {
2697 enum radv_secure_compile_type sc_type = RADV_SC_TYPE_INIT_SUCCESS;
2698 device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input;
2699 device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output;
2700
2701 write(fd_secure_output, &sc_type, sizeof(sc_type));
2702
2703 while (true) {
2704 radv_sc_read(fd_secure_input, &sc_type, sizeof(sc_type), false);
2705
2706 if (sc_type == RADV_SC_TYPE_FORK_DEVICE) {
2707 sc_type = fork_secure_compile_device(device, process);
2708
2709 if (sc_type == RADV_SC_TYPE_INIT_FAILURE)
2710 goto secure_compile_exit;
2711
2712 } else if (sc_type == RADV_SC_TYPE_DESTROY_DEVICE) {
2713 goto secure_compile_exit;
2714 }
2715 }
2716
2717 secure_compile_exit:
2718 close(fd_secure_input);
2719 close(fd_secure_output);
2720 _exit(0);
2721 }
2722
2723 static void destroy_secure_compile_device(struct radv_device *device, unsigned process)
2724 {
2725 int fd_secure_input = device->sc_state->secure_compile_processes[process].fd_secure_input;
2726
2727 enum radv_secure_compile_type sc_type = RADV_SC_TYPE_DESTROY_DEVICE;
2728 write(fd_secure_input, &sc_type, sizeof(sc_type));
2729
2730 close(device->sc_state->secure_compile_processes[process].fd_secure_input);
2731 close(device->sc_state->secure_compile_processes[process].fd_secure_output);
2732
2733 int status;
2734 waitpid(device->sc_state->secure_compile_processes[process].sc_pid, &status, 0);
2735 }
2736
2737 static VkResult fork_secure_compile_idle_device(struct radv_device *device)
2738 {
2739 device->sc_state = vk_zalloc(&device->alloc,
2740 sizeof(struct radv_secure_compile_state),
2741 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2742
2743 mtx_init(&device->sc_state->secure_compile_mutex, mtx_plain);
2744
2745 pid_t upid = getpid();
2746 time_t seconds = time(NULL);
2747
2748 char *uid;
2749 if (asprintf(&uid, "%ld_%ld", (long) upid, (long) seconds) == -1)
2750 return VK_ERROR_INITIALIZATION_FAILED;
2751
2752 device->sc_state->uid = uid;
2753
2754 uint8_t sc_threads = device->instance->num_sc_threads;
2755 int fd_secure_input[MAX_SC_PROCS][2];
2756 int fd_secure_output[MAX_SC_PROCS][2];
2757
2758 /* create pipe descriptors (used to communicate between processes) */
2759 for (unsigned i = 0; i < sc_threads; i++) {
2760 if (pipe(fd_secure_input[i]) == -1 ||
2761 pipe(fd_secure_output[i]) == -1) {
2762 return VK_ERROR_INITIALIZATION_FAILED;
2763 }
2764 }
2765
2766 device->sc_state->secure_compile_processes = vk_zalloc(&device->alloc,
2767 sizeof(struct radv_secure_compile_process) * sc_threads, 8,
2768 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2769
2770 for (unsigned process = 0; process < sc_threads; process++) {
2771 if ((device->sc_state->secure_compile_processes[process].sc_pid = fork()) == 0) {
2772 device->sc_state->secure_compile_thread_counter = process;
2773 run_secure_compile_idle_device(device, process, fd_secure_input[process][0], fd_secure_output[process][1]);
2774 } else {
2775 if (device->sc_state->secure_compile_processes[process].sc_pid == -1)
2776 return VK_ERROR_INITIALIZATION_FAILED;
2777
2778 /* Read the init result returned from the secure process */
2779 enum radv_secure_compile_type sc_type;
2780 bool sc_read = radv_sc_read(fd_secure_output[process][0], &sc_type, sizeof(sc_type), true);
2781
2782 bool fifo_result;
2783 if (sc_read && sc_type == RADV_SC_TYPE_INIT_SUCCESS) {
2784 fifo_result = secure_compile_open_fifo_fds(device->sc_state,
2785 &device->sc_state->secure_compile_processes[process].fd_server,
2786 &device->sc_state->secure_compile_processes[process].fd_client,
2787 process, true);
2788
2789 device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input[process][1];
2790 device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output[process][0];
2791 }
2792
2793 if (sc_type == RADV_SC_TYPE_INIT_FAILURE || !sc_read || !fifo_result) {
2794 close(fd_secure_input[process][0]);
2795 close(fd_secure_input[process][1]);
2796 close(fd_secure_output[process][1]);
2797 close(fd_secure_output[process][0]);
2798 int status;
2799 waitpid(device->sc_state->secure_compile_processes[process].sc_pid, &status, 0);
2800
2801 /* Destroy any forks that were created sucessfully */
2802 for (unsigned i = 0; i < process; i++) {
2803 destroy_secure_compile_device(device, i);
2804 }
2805
2806 return VK_ERROR_INITIALIZATION_FAILED;
2807 }
2808 }
2809 }
2810 return VK_SUCCESS;
2811 }
2812
2813 static void
2814 radv_device_init_dispatch(struct radv_device *device)
2815 {
2816 const struct radv_instance *instance = device->physical_device->instance;
2817 const struct radv_device_dispatch_table *dispatch_table_layer = NULL;
2818 bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
2819 int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
2820
2821 if (radv_thread_trace >= 0) {
2822 /* Use device entrypoints from the SQTT layer if enabled. */
2823 dispatch_table_layer = &sqtt_device_dispatch_table;
2824 }
2825
2826 for (unsigned i = 0; i < ARRAY_SIZE(device->dispatch.entrypoints); i++) {
2827 /* Vulkan requires that entrypoints for extensions which have not been
2828 * enabled must not be advertised.
2829 */
2830 if (!unchecked &&
2831 !radv_device_entrypoint_is_enabled(i, instance->apiVersion,
2832 &instance->enabled_extensions,
2833 &device->enabled_extensions)) {
2834 device->dispatch.entrypoints[i] = NULL;
2835 } else if (dispatch_table_layer &&
2836 dispatch_table_layer->entrypoints[i]) {
2837 device->dispatch.entrypoints[i] =
2838 dispatch_table_layer->entrypoints[i];
2839 } else {
2840 device->dispatch.entrypoints[i] =
2841 radv_device_dispatch_table.entrypoints[i];
2842 }
2843 }
2844 }
2845
2846 static VkResult
2847 radv_create_pthread_cond(pthread_cond_t *cond)
2848 {
2849 pthread_condattr_t condattr;
2850 if (pthread_condattr_init(&condattr)) {
2851 return VK_ERROR_INITIALIZATION_FAILED;
2852 }
2853
2854 if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
2855 pthread_condattr_destroy(&condattr);
2856 return VK_ERROR_INITIALIZATION_FAILED;
2857 }
2858 if (pthread_cond_init(cond, &condattr)) {
2859 pthread_condattr_destroy(&condattr);
2860 return VK_ERROR_INITIALIZATION_FAILED;
2861 }
2862 pthread_condattr_destroy(&condattr);
2863 return VK_SUCCESS;
2864 }
2865
2866 static VkResult
2867 check_physical_device_features(VkPhysicalDevice physicalDevice,
2868 const VkPhysicalDeviceFeatures *features)
2869 {
2870 RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2871 VkPhysicalDeviceFeatures supported_features;
2872 radv_GetPhysicalDeviceFeatures(physicalDevice, &supported_features);
2873 VkBool32 *supported_feature = (VkBool32 *)&supported_features;
2874 VkBool32 *enabled_feature = (VkBool32 *)features;
2875 unsigned num_features = sizeof(VkPhysicalDeviceFeatures) / sizeof(VkBool32);
2876 for (uint32_t i = 0; i < num_features; i++) {
2877 if (enabled_feature[i] && !supported_feature[i])
2878 return vk_error(physical_device->instance, VK_ERROR_FEATURE_NOT_PRESENT);
2879 }
2880
2881 return VK_SUCCESS;
2882 }
2883
2884 VkResult radv_CreateDevice(
2885 VkPhysicalDevice physicalDevice,
2886 const VkDeviceCreateInfo* pCreateInfo,
2887 const VkAllocationCallbacks* pAllocator,
2888 VkDevice* pDevice)
2889 {
2890 RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2891 VkResult result;
2892 struct radv_device *device;
2893
2894 bool keep_shader_info = false;
2895 bool robust_buffer_access = false;
2896 bool overallocation_disallowed = false;
2897
2898 /* Check enabled features */
2899 if (pCreateInfo->pEnabledFeatures) {
2900 result = check_physical_device_features(physicalDevice,
2901 pCreateInfo->pEnabledFeatures);
2902 if (result != VK_SUCCESS)
2903 return result;
2904
2905 if (pCreateInfo->pEnabledFeatures->robustBufferAccess)
2906 robust_buffer_access = true;
2907 }
2908
2909 vk_foreach_struct_const(ext, pCreateInfo->pNext) {
2910 switch (ext->sType) {
2911 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2: {
2912 const VkPhysicalDeviceFeatures2 *features = (const void *)ext;
2913 result = check_physical_device_features(physicalDevice,
2914 &features->features);
2915 if (result != VK_SUCCESS)
2916 return result;
2917
2918 if (features->features.robustBufferAccess)
2919 robust_buffer_access = true;
2920 break;
2921 }
2922 case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: {
2923 const VkDeviceMemoryOverallocationCreateInfoAMD *overallocation = (const void *)ext;
2924 if (overallocation->overallocationBehavior == VK_MEMORY_OVERALLOCATION_BEHAVIOR_DISALLOWED_AMD)
2925 overallocation_disallowed = true;
2926 break;
2927 }
2928 default:
2929 break;
2930 }
2931 }
2932
2933 device = vk_zalloc2(&physical_device->instance->alloc, pAllocator,
2934 sizeof(*device), 8,
2935 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2936 if (!device)
2937 return vk_error(physical_device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2938
2939 device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
2940 device->instance = physical_device->instance;
2941 device->physical_device = physical_device;
2942
2943 device->ws = physical_device->ws;
2944 if (pAllocator)
2945 device->alloc = *pAllocator;
2946 else
2947 device->alloc = physical_device->instance->alloc;
2948
2949 for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
2950 const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i];
2951 int index = radv_get_device_extension_index(ext_name);
2952 if (index < 0 || !physical_device->supported_extensions.extensions[index]) {
2953 vk_free(&device->alloc, device);
2954 return vk_error(physical_device->instance, VK_ERROR_EXTENSION_NOT_PRESENT);
2955 }
2956
2957 device->enabled_extensions.extensions[index] = true;
2958 }
2959
2960 radv_device_init_dispatch(device);
2961
2962 keep_shader_info = device->enabled_extensions.AMD_shader_info;
2963
2964 /* With update after bind we can't attach bo's to the command buffer
2965 * from the descriptor set anymore, so we have to use a global BO list.
2966 */
2967 device->use_global_bo_list =
2968 (device->instance->perftest_flags & RADV_PERFTEST_BO_LIST) ||
2969 device->enabled_extensions.EXT_descriptor_indexing ||
2970 device->enabled_extensions.EXT_buffer_device_address ||
2971 device->enabled_extensions.KHR_buffer_device_address;
2972
2973 device->robust_buffer_access = robust_buffer_access;
2974
2975 mtx_init(&device->shader_slab_mutex, mtx_plain);
2976 list_inithead(&device->shader_slabs);
2977
2978 device->overallocation_disallowed = overallocation_disallowed;
2979 mtx_init(&device->overallocation_mutex, mtx_plain);
2980
2981 radv_bo_list_init(&device->bo_list);
2982
2983 for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
2984 const VkDeviceQueueCreateInfo *queue_create = &pCreateInfo->pQueueCreateInfos[i];
2985 uint32_t qfi = queue_create->queueFamilyIndex;
2986 const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority =
2987 vk_find_struct_const(queue_create->pNext, DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT);
2988
2989 assert(!global_priority || device->physical_device->rad_info.has_ctx_priority);
2990
2991 device->queues[qfi] = vk_alloc(&device->alloc,
2992 queue_create->queueCount * sizeof(struct radv_queue), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2993 if (!device->queues[qfi]) {
2994 result = VK_ERROR_OUT_OF_HOST_MEMORY;
2995 goto fail;
2996 }
2997
2998 memset(device->queues[qfi], 0, queue_create->queueCount * sizeof(struct radv_queue));
2999
3000 device->queue_count[qfi] = queue_create->queueCount;
3001
3002 for (unsigned q = 0; q < queue_create->queueCount; q++) {
3003 result = radv_queue_init(device, &device->queues[qfi][q],
3004 qfi, q, queue_create->flags,
3005 global_priority);
3006 if (result != VK_SUCCESS)
3007 goto fail;
3008 }
3009 }
3010
3011 device->pbb_allowed = device->physical_device->rad_info.chip_class >= GFX9 &&
3012 !(device->instance->debug_flags & RADV_DEBUG_NOBINNING);
3013
3014 /* Disable DFSM by default. As of 2019-09-15 Talos on Low is still 3% slower on Raven. */
3015 device->dfsm_allowed = device->pbb_allowed &&
3016 (device->instance->perftest_flags & RADV_PERFTEST_DFSM);
3017
3018 device->always_use_syncobj = device->physical_device->rad_info.has_syncobj_wait_for_submit;
3019
3020 /* The maximum number of scratch waves. Scratch space isn't divided
3021 * evenly between CUs. The number is only a function of the number of CUs.
3022 * We can decrease the constant to decrease the scratch buffer size.
3023 *
3024 * sctx->scratch_waves must be >= the maximum possible size of
3025 * 1 threadgroup, so that the hw doesn't hang from being unable
3026 * to start any.
3027 *
3028 * The recommended value is 4 per CU at most. Higher numbers don't
3029 * bring much benefit, but they still occupy chip resources (think
3030 * async compute). I've seen ~2% performance difference between 4 and 32.
3031 */
3032 uint32_t max_threads_per_block = 2048;
3033 device->scratch_waves = MAX2(32 * physical_device->rad_info.num_good_compute_units,
3034 max_threads_per_block / 64);
3035
3036 device->dispatch_initiator = S_00B800_COMPUTE_SHADER_EN(1);
3037
3038 if (device->physical_device->rad_info.chip_class >= GFX7) {
3039 /* If the KMD allows it (there is a KMD hw register for it),
3040 * allow launching waves out-of-order.
3041 */
3042 device->dispatch_initiator |= S_00B800_ORDER_MODE(1);
3043 }
3044
3045 radv_device_init_gs_info(device);
3046
3047 device->tess_offchip_block_dw_size =
3048 device->physical_device->rad_info.family == CHIP_HAWAII ? 4096 : 8192;
3049
3050 if (getenv("RADV_TRACE_FILE")) {
3051 const char *filename = getenv("RADV_TRACE_FILE");
3052
3053 keep_shader_info = true;
3054
3055 if (!radv_init_trace(device))
3056 goto fail;
3057
3058 fprintf(stderr, "*****************************************************************************\n");
3059 fprintf(stderr, "* WARNING: RADV_TRACE_FILE is costly and should only be used for debugging! *\n");
3060 fprintf(stderr, "*****************************************************************************\n");
3061
3062 fprintf(stderr, "Trace file will be dumped to %s\n", filename);
3063 radv_dump_enabled_options(device, stderr);
3064 }
3065
3066 int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
3067 if (radv_thread_trace >= 0) {
3068 fprintf(stderr, "*************************************************\n");
3069 fprintf(stderr, "* WARNING: Thread trace support is experimental *\n");
3070 fprintf(stderr, "*************************************************\n");
3071
3072 if (device->physical_device->rad_info.chip_class < GFX8) {
3073 fprintf(stderr, "GPU hardware not supported: refer to "
3074 "the RGP documentation for the list of "
3075 "supported GPUs!\n");
3076 abort();
3077 }
3078
3079 /* Default buffer size set to 1MB per SE. */
3080 device->thread_trace_buffer_size =
3081 radv_get_int_debug_option("RADV_THREAD_TRACE_BUFFER_SIZE", 1024 * 1024);
3082 device->thread_trace_start_frame = radv_thread_trace;
3083
3084 if (!radv_thread_trace_init(device))
3085 goto fail;
3086 }
3087
3088 /* Temporarily disable secure compile while we create meta shaders, etc */
3089 uint8_t sc_threads = device->instance->num_sc_threads;
3090 if (sc_threads)
3091 device->instance->num_sc_threads = 0;
3092
3093 device->keep_shader_info = keep_shader_info;
3094 result = radv_device_init_meta(device);
3095 if (result != VK_SUCCESS)
3096 goto fail;
3097
3098 radv_device_init_msaa(device);
3099
3100 for (int family = 0; family < RADV_MAX_QUEUE_FAMILIES; ++family) {
3101 device->empty_cs[family] = device->ws->cs_create(device->ws, family);
3102 switch (family) {
3103 case RADV_QUEUE_GENERAL:
3104 radeon_emit(device->empty_cs[family], PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
3105 radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_LOAD_ENABLE(1));
3106 radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_SHADOW_ENABLE(1));
3107 break;
3108 case RADV_QUEUE_COMPUTE:
3109 radeon_emit(device->empty_cs[family], PKT3(PKT3_NOP, 0, 0));
3110 radeon_emit(device->empty_cs[family], 0);
3111 break;
3112 }
3113 device->ws->cs_finalize(device->empty_cs[family]);
3114 }
3115
3116 if (device->physical_device->rad_info.chip_class >= GFX7)
3117 cik_create_gfx_config(device);
3118
3119 VkPipelineCacheCreateInfo ci;
3120 ci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
3121 ci.pNext = NULL;
3122 ci.flags = 0;
3123 ci.pInitialData = NULL;
3124 ci.initialDataSize = 0;
3125 VkPipelineCache pc;
3126 result = radv_CreatePipelineCache(radv_device_to_handle(device),
3127 &ci, NULL, &pc);
3128 if (result != VK_SUCCESS)
3129 goto fail_meta;
3130
3131 device->mem_cache = radv_pipeline_cache_from_handle(pc);
3132
3133 result = radv_create_pthread_cond(&device->timeline_cond);
3134 if (result != VK_SUCCESS)
3135 goto fail_mem_cache;
3136
3137 device->force_aniso =
3138 MIN2(16, radv_get_int_debug_option("RADV_TEX_ANISO", -1));
3139 if (device->force_aniso >= 0) {
3140 fprintf(stderr, "radv: Forcing anisotropy filter to %ix\n",
3141 1 << util_logbase2(device->force_aniso));
3142 }
3143
3144 /* Fork device for secure compile as required */
3145 device->instance->num_sc_threads = sc_threads;
3146 if (radv_device_use_secure_compile(device->instance)) {
3147
3148 result = fork_secure_compile_idle_device(device);
3149 if (result != VK_SUCCESS)
3150 goto fail_meta;
3151 }
3152
3153 *pDevice = radv_device_to_handle(device);
3154 return VK_SUCCESS;
3155
3156 fail_mem_cache:
3157 radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
3158 fail_meta:
3159 radv_device_finish_meta(device);
3160 fail:
3161 radv_bo_list_finish(&device->bo_list);
3162
3163 radv_thread_trace_finish(device);
3164
3165 if (device->trace_bo)
3166 device->ws->buffer_destroy(device->trace_bo);
3167
3168 if (device->gfx_init)
3169 device->ws->buffer_destroy(device->gfx_init);
3170
3171 for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
3172 for (unsigned q = 0; q < device->queue_count[i]; q++)
3173 radv_queue_finish(&device->queues[i][q]);
3174 if (device->queue_count[i])
3175 vk_free(&device->alloc, device->queues[i]);
3176 }
3177
3178 vk_free(&device->alloc, device);
3179 return result;
3180 }
3181
3182 void radv_DestroyDevice(
3183 VkDevice _device,
3184 const VkAllocationCallbacks* pAllocator)
3185 {
3186 RADV_FROM_HANDLE(radv_device, device, _device);
3187
3188 if (!device)
3189 return;
3190
3191 if (device->trace_bo)
3192 device->ws->buffer_destroy(device->trace_bo);
3193
3194 if (device->gfx_init)
3195 device->ws->buffer_destroy(device->gfx_init);
3196
3197 for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
3198 for (unsigned q = 0; q < device->queue_count[i]; q++)
3199 radv_queue_finish(&device->queues[i][q]);
3200 if (device->queue_count[i])
3201 vk_free(&device->alloc, device->queues[i]);
3202 if (device->empty_cs[i])
3203 device->ws->cs_destroy(device->empty_cs[i]);
3204 }
3205 radv_device_finish_meta(device);
3206
3207 VkPipelineCache pc = radv_pipeline_cache_to_handle(device->mem_cache);
3208 radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
3209
3210 radv_destroy_shader_slabs(device);
3211
3212 pthread_cond_destroy(&device->timeline_cond);
3213 radv_bo_list_finish(&device->bo_list);
3214
3215 radv_thread_trace_finish(device);
3216
3217 if (radv_device_use_secure_compile(device->instance)) {
3218 for (unsigned i = 0; i < device->instance->num_sc_threads; i++ ) {
3219 destroy_secure_compile_device(device, i);
3220 }
3221 }
3222
3223 if (device->sc_state) {
3224 free(device->sc_state->uid);
3225 vk_free(&device->alloc, device->sc_state->secure_compile_processes);
3226 }
3227 vk_free(&device->alloc, device->sc_state);
3228 vk_free(&device->alloc, device);
3229 }
3230
3231 VkResult radv_EnumerateInstanceLayerProperties(
3232 uint32_t* pPropertyCount,
3233 VkLayerProperties* pProperties)
3234 {
3235 if (pProperties == NULL) {
3236 *pPropertyCount = 0;
3237 return VK_SUCCESS;
3238 }
3239
3240 /* None supported at this time */
3241 return vk_error(NULL, VK_ERROR_LAYER_NOT_PRESENT);
3242 }
3243
3244 VkResult radv_EnumerateDeviceLayerProperties(
3245 VkPhysicalDevice physicalDevice,
3246 uint32_t* pPropertyCount,
3247 VkLayerProperties* pProperties)
3248 {
3249 if (pProperties == NULL) {
3250 *pPropertyCount = 0;
3251 return VK_SUCCESS;
3252 }
3253
3254 /* None supported at this time */
3255 return vk_error(NULL, VK_ERROR_LAYER_NOT_PRESENT);
3256 }
3257
3258 void radv_GetDeviceQueue2(
3259 VkDevice _device,
3260 const VkDeviceQueueInfo2* pQueueInfo,
3261 VkQueue* pQueue)
3262 {
3263 RADV_FROM_HANDLE(radv_device, device, _device);
3264 struct radv_queue *queue;
3265
3266 queue = &device->queues[pQueueInfo->queueFamilyIndex][pQueueInfo->queueIndex];
3267 if (pQueueInfo->flags != queue->flags) {
3268 /* From the Vulkan 1.1.70 spec:
3269 *
3270 * "The queue returned by vkGetDeviceQueue2 must have the same
3271 * flags value from this structure as that used at device
3272 * creation time in a VkDeviceQueueCreateInfo instance. If no
3273 * matching flags were specified at device creation time then
3274 * pQueue will return VK_NULL_HANDLE."
3275 */
3276 *pQueue = VK_NULL_HANDLE;
3277 return;
3278 }
3279
3280 *pQueue = radv_queue_to_handle(queue);
3281 }
3282
3283 void radv_GetDeviceQueue(
3284 VkDevice _device,
3285 uint32_t queueFamilyIndex,
3286 uint32_t queueIndex,
3287 VkQueue* pQueue)
3288 {
3289 const VkDeviceQueueInfo2 info = (VkDeviceQueueInfo2) {
3290 .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2,
3291 .queueFamilyIndex = queueFamilyIndex,
3292 .queueIndex = queueIndex
3293 };
3294
3295 radv_GetDeviceQueue2(_device, &info, pQueue);
3296 }
3297
3298 static void
3299 fill_geom_tess_rings(struct radv_queue *queue,
3300 uint32_t *map,
3301 bool add_sample_positions,
3302 uint32_t esgs_ring_size,
3303 struct radeon_winsys_bo *esgs_ring_bo,
3304 uint32_t gsvs_ring_size,
3305 struct radeon_winsys_bo *gsvs_ring_bo,
3306 uint32_t tess_factor_ring_size,
3307 uint32_t tess_offchip_ring_offset,
3308 uint32_t tess_offchip_ring_size,
3309 struct radeon_winsys_bo *tess_rings_bo)
3310 {
3311 uint32_t *desc = &map[4];
3312
3313 if (esgs_ring_bo) {
3314 uint64_t esgs_va = radv_buffer_get_va(esgs_ring_bo);
3315
3316 /* stride 0, num records - size, add tid, swizzle, elsize4,
3317 index stride 64 */
3318 desc[0] = esgs_va;
3319 desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32) |
3320 S_008F04_SWIZZLE_ENABLE(true);
3321 desc[2] = esgs_ring_size;
3322 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3323 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3324 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3325 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3326 S_008F0C_INDEX_STRIDE(3) |
3327 S_008F0C_ADD_TID_ENABLE(1);
3328
3329 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3330 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3331 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3332 S_008F0C_RESOURCE_LEVEL(1);
3333 } else {
3334 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3335 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
3336 S_008F0C_ELEMENT_SIZE(1);
3337 }
3338
3339 /* GS entry for ES->GS ring */
3340 /* stride 0, num records - size, elsize0,
3341 index stride 0 */
3342 desc[4] = esgs_va;
3343 desc[5] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32);
3344 desc[6] = esgs_ring_size;
3345 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3346 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3347 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3348 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3349
3350 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3351 desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3352 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3353 S_008F0C_RESOURCE_LEVEL(1);
3354 } else {
3355 desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3356 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3357 }
3358 }
3359
3360 desc += 8;
3361
3362 if (gsvs_ring_bo) {
3363 uint64_t gsvs_va = radv_buffer_get_va(gsvs_ring_bo);
3364
3365 /* VS entry for GS->VS ring */
3366 /* stride 0, num records - size, elsize0,
3367 index stride 0 */
3368 desc[0] = gsvs_va;
3369 desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32);
3370 desc[2] = gsvs_ring_size;
3371 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3372 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3373 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3374 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3375
3376 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3377 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3378 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3379 S_008F0C_RESOURCE_LEVEL(1);
3380 } else {
3381 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3382 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3383 }
3384
3385 /* stride gsvs_itemsize, num records 64
3386 elsize 4, index stride 16 */
3387 /* shader will patch stride and desc[2] */
3388 desc[4] = gsvs_va;
3389 desc[5] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32) |
3390 S_008F04_SWIZZLE_ENABLE(1);
3391 desc[6] = 0;
3392 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3393 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3394 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3395 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3396 S_008F0C_INDEX_STRIDE(1) |
3397 S_008F0C_ADD_TID_ENABLE(true);
3398
3399 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3400 desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3401 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3402 S_008F0C_RESOURCE_LEVEL(1);
3403 } else {
3404 desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3405 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
3406 S_008F0C_ELEMENT_SIZE(1);
3407 }
3408
3409 }
3410
3411 desc += 8;
3412
3413 if (tess_rings_bo) {
3414 uint64_t tess_va = radv_buffer_get_va(tess_rings_bo);
3415 uint64_t tess_offchip_va = tess_va + tess_offchip_ring_offset;
3416
3417 desc[0] = tess_va;
3418 desc[1] = S_008F04_BASE_ADDRESS_HI(tess_va >> 32);
3419 desc[2] = tess_factor_ring_size;
3420 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3421 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3422 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3423 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3424
3425 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3426 desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3427 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3428 S_008F0C_RESOURCE_LEVEL(1);
3429 } else {
3430 desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3431 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3432 }
3433
3434 desc[4] = tess_offchip_va;
3435 desc[5] = S_008F04_BASE_ADDRESS_HI(tess_offchip_va >> 32);
3436 desc[6] = tess_offchip_ring_size;
3437 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3438 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3439 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3440 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3441
3442 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3443 desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3444 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3445 S_008F0C_RESOURCE_LEVEL(1);
3446 } else {
3447 desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3448 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3449 }
3450 }
3451
3452 desc += 8;
3453
3454 if (add_sample_positions) {
3455 /* add sample positions after all rings */
3456 memcpy(desc, queue->device->sample_locations_1x, 8);
3457 desc += 2;
3458 memcpy(desc, queue->device->sample_locations_2x, 16);
3459 desc += 4;
3460 memcpy(desc, queue->device->sample_locations_4x, 32);
3461 desc += 8;
3462 memcpy(desc, queue->device->sample_locations_8x, 64);
3463 }
3464 }
3465
3466 static unsigned
3467 radv_get_hs_offchip_param(struct radv_device *device, uint32_t *max_offchip_buffers_p)
3468 {
3469 bool double_offchip_buffers = device->physical_device->rad_info.chip_class >= GFX7 &&
3470 device->physical_device->rad_info.family != CHIP_CARRIZO &&
3471 device->physical_device->rad_info.family != CHIP_STONEY;
3472 unsigned max_offchip_buffers_per_se = double_offchip_buffers ? 128 : 64;
3473 unsigned max_offchip_buffers;
3474 unsigned offchip_granularity;
3475 unsigned hs_offchip_param;
3476
3477 /*
3478 * Per RadeonSI:
3479 * This must be one less than the maximum number due to a hw limitation.
3480 * Various hardware bugs need thGFX7
3481 *
3482 * Per AMDVLK:
3483 * Vega10 should limit max_offchip_buffers to 508 (4 * 127).
3484 * Gfx7 should limit max_offchip_buffers to 508
3485 * Gfx6 should limit max_offchip_buffers to 126 (2 * 63)
3486 *
3487 * Follow AMDVLK here.
3488 */
3489 if (device->physical_device->rad_info.chip_class >= GFX10) {
3490 max_offchip_buffers_per_se = 256;
3491 } else if (device->physical_device->rad_info.family == CHIP_VEGA10 ||
3492 device->physical_device->rad_info.chip_class == GFX7 ||
3493 device->physical_device->rad_info.chip_class == GFX6)
3494 --max_offchip_buffers_per_se;
3495
3496 max_offchip_buffers = max_offchip_buffers_per_se *
3497 device->physical_device->rad_info.max_se;
3498
3499 /* Hawaii has a bug with offchip buffers > 256 that can be worked
3500 * around by setting 4K granularity.
3501 */
3502 if (device->tess_offchip_block_dw_size == 4096) {
3503 assert(device->physical_device->rad_info.family == CHIP_HAWAII);
3504 offchip_granularity = V_03093C_X_4K_DWORDS;
3505 } else {
3506 assert(device->tess_offchip_block_dw_size == 8192);
3507 offchip_granularity = V_03093C_X_8K_DWORDS;
3508 }
3509
3510 switch (device->physical_device->rad_info.chip_class) {
3511 case GFX6:
3512 max_offchip_buffers = MIN2(max_offchip_buffers, 126);
3513 break;
3514 case GFX7:
3515 case GFX8:
3516 case GFX9:
3517 max_offchip_buffers = MIN2(max_offchip_buffers, 508);
3518 break;
3519 case GFX10:
3520 break;
3521 default:
3522 break;
3523 }
3524
3525 *max_offchip_buffers_p = max_offchip_buffers;
3526 if (device->physical_device->rad_info.chip_class >= GFX7) {
3527 if (device->physical_device->rad_info.chip_class >= GFX8)
3528 --max_offchip_buffers;
3529 hs_offchip_param =
3530 S_03093C_OFFCHIP_BUFFERING(max_offchip_buffers) |
3531 S_03093C_OFFCHIP_GRANULARITY(offchip_granularity);
3532 } else {
3533 hs_offchip_param =
3534 S_0089B0_OFFCHIP_BUFFERING(max_offchip_buffers);
3535 }
3536 return hs_offchip_param;
3537 }
3538
3539 static void
3540 radv_emit_gs_ring_sizes(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3541 struct radeon_winsys_bo *esgs_ring_bo,
3542 uint32_t esgs_ring_size,
3543 struct radeon_winsys_bo *gsvs_ring_bo,
3544 uint32_t gsvs_ring_size)
3545 {
3546 if (!esgs_ring_bo && !gsvs_ring_bo)
3547 return;
3548
3549 if (esgs_ring_bo)
3550 radv_cs_add_buffer(queue->device->ws, cs, esgs_ring_bo);
3551
3552 if (gsvs_ring_bo)
3553 radv_cs_add_buffer(queue->device->ws, cs, gsvs_ring_bo);
3554
3555 if (queue->device->physical_device->rad_info.chip_class >= GFX7) {
3556 radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2);
3557 radeon_emit(cs, esgs_ring_size >> 8);
3558 radeon_emit(cs, gsvs_ring_size >> 8);
3559 } else {
3560 radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2);
3561 radeon_emit(cs, esgs_ring_size >> 8);
3562 radeon_emit(cs, gsvs_ring_size >> 8);
3563 }
3564 }
3565
3566 static void
3567 radv_emit_tess_factor_ring(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3568 unsigned hs_offchip_param, unsigned tf_ring_size,
3569 struct radeon_winsys_bo *tess_rings_bo)
3570 {
3571 uint64_t tf_va;
3572
3573 if (!tess_rings_bo)
3574 return;
3575
3576 tf_va = radv_buffer_get_va(tess_rings_bo);
3577
3578 radv_cs_add_buffer(queue->device->ws, cs, tess_rings_bo);
3579
3580 if (queue->device->physical_device->rad_info.chip_class >= GFX7) {
3581 radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE,
3582 S_030938_SIZE(tf_ring_size / 4));
3583 radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE,
3584 tf_va >> 8);
3585
3586 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3587 radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI_UMD,
3588 S_030984_BASE_HI(tf_va >> 40));
3589 } else if (queue->device->physical_device->rad_info.chip_class == GFX9) {
3590 radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI,
3591 S_030944_BASE_HI(tf_va >> 40));
3592 }
3593 radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM,
3594 hs_offchip_param);
3595 } else {
3596 radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE,
3597 S_008988_SIZE(tf_ring_size / 4));
3598 radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE,
3599 tf_va >> 8);
3600 radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM,
3601 hs_offchip_param);
3602 }
3603 }
3604
3605 static void
3606 radv_emit_graphics_scratch(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3607 uint32_t size_per_wave, uint32_t waves,
3608 struct radeon_winsys_bo *scratch_bo)
3609 {
3610 if (queue->queue_family_index != RADV_QUEUE_GENERAL)
3611 return;
3612
3613 if (!scratch_bo)
3614 return;
3615
3616 radv_cs_add_buffer(queue->device->ws, cs, scratch_bo);
3617
3618 radeon_set_context_reg(cs, R_0286E8_SPI_TMPRING_SIZE,
3619 S_0286E8_WAVES(waves) |
3620 S_0286E8_WAVESIZE(round_up_u32(size_per_wave, 1024)));
3621 }
3622
3623 static void
3624 radv_emit_compute_scratch(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3625 uint32_t size_per_wave, uint32_t waves,
3626 struct radeon_winsys_bo *compute_scratch_bo)
3627 {
3628 uint64_t scratch_va;
3629
3630 if (!compute_scratch_bo)
3631 return;
3632
3633 scratch_va = radv_buffer_get_va(compute_scratch_bo);
3634
3635 radv_cs_add_buffer(queue->device->ws, cs, compute_scratch_bo);
3636
3637 radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2);
3638 radeon_emit(cs, scratch_va);
3639 radeon_emit(cs, S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
3640 S_008F04_SWIZZLE_ENABLE(1));
3641
3642 radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE,
3643 S_00B860_WAVES(waves) |
3644 S_00B860_WAVESIZE(round_up_u32(size_per_wave, 1024)));
3645 }
3646
3647 static void
3648 radv_emit_global_shader_pointers(struct radv_queue *queue,
3649 struct radeon_cmdbuf *cs,
3650 struct radeon_winsys_bo *descriptor_bo)
3651 {
3652 uint64_t va;
3653
3654 if (!descriptor_bo)
3655 return;
3656
3657 va = radv_buffer_get_va(descriptor_bo);
3658
3659 radv_cs_add_buffer(queue->device->ws, cs, descriptor_bo);
3660
3661 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3662 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3663 R_00B130_SPI_SHADER_USER_DATA_VS_0,
3664 R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
3665 R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
3666
3667 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3668 radv_emit_shader_pointer(queue->device, cs, regs[i],
3669 va, true);
3670 }
3671 } else if (queue->device->physical_device->rad_info.chip_class == GFX9) {
3672 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3673 R_00B130_SPI_SHADER_USER_DATA_VS_0,
3674 R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
3675 R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
3676
3677 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3678 radv_emit_shader_pointer(queue->device, cs, regs[i],
3679 va, true);
3680 }
3681 } else {
3682 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3683 R_00B130_SPI_SHADER_USER_DATA_VS_0,
3684 R_00B230_SPI_SHADER_USER_DATA_GS_0,
3685 R_00B330_SPI_SHADER_USER_DATA_ES_0,
3686 R_00B430_SPI_SHADER_USER_DATA_HS_0,
3687 R_00B530_SPI_SHADER_USER_DATA_LS_0};
3688
3689 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3690 radv_emit_shader_pointer(queue->device, cs, regs[i],
3691 va, true);
3692 }
3693 }
3694 }
3695
3696 static void
3697 radv_init_graphics_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
3698 {
3699 struct radv_device *device = queue->device;
3700
3701 if (device->gfx_init) {
3702 uint64_t va = radv_buffer_get_va(device->gfx_init);
3703
3704 radeon_emit(cs, PKT3(PKT3_INDIRECT_BUFFER_CIK, 2, 0));
3705 radeon_emit(cs, va);
3706 radeon_emit(cs, va >> 32);
3707 radeon_emit(cs, device->gfx_init_size_dw & 0xffff);
3708
3709 radv_cs_add_buffer(device->ws, cs, device->gfx_init);
3710 } else {
3711 si_emit_graphics(device, cs);
3712 }
3713 }
3714
3715 static void
3716 radv_init_compute_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
3717 {
3718 struct radv_physical_device *physical_device = queue->device->physical_device;
3719 si_emit_compute(physical_device, cs);
3720 }
3721
3722 static VkResult
3723 radv_get_preamble_cs(struct radv_queue *queue,
3724 uint32_t scratch_size_per_wave,
3725 uint32_t scratch_waves,
3726 uint32_t compute_scratch_size_per_wave,
3727 uint32_t compute_scratch_waves,
3728 uint32_t esgs_ring_size,
3729 uint32_t gsvs_ring_size,
3730 bool needs_tess_rings,
3731 bool needs_gds,
3732 bool needs_gds_oa,
3733 bool needs_sample_positions,
3734 struct radeon_cmdbuf **initial_full_flush_preamble_cs,
3735 struct radeon_cmdbuf **initial_preamble_cs,
3736 struct radeon_cmdbuf **continue_preamble_cs)
3737 {
3738 struct radeon_winsys_bo *scratch_bo = NULL;
3739 struct radeon_winsys_bo *descriptor_bo = NULL;
3740 struct radeon_winsys_bo *compute_scratch_bo = NULL;
3741 struct radeon_winsys_bo *esgs_ring_bo = NULL;
3742 struct radeon_winsys_bo *gsvs_ring_bo = NULL;
3743 struct radeon_winsys_bo *tess_rings_bo = NULL;
3744 struct radeon_winsys_bo *gds_bo = NULL;
3745 struct radeon_winsys_bo *gds_oa_bo = NULL;
3746 struct radeon_cmdbuf *dest_cs[3] = {0};
3747 bool add_tess_rings = false, add_gds = false, add_gds_oa = false, add_sample_positions = false;
3748 unsigned tess_factor_ring_size = 0, tess_offchip_ring_size = 0;
3749 unsigned max_offchip_buffers;
3750 unsigned hs_offchip_param = 0;
3751 unsigned tess_offchip_ring_offset;
3752 uint32_t ring_bo_flags = RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING;
3753 if (!queue->has_tess_rings) {
3754 if (needs_tess_rings)
3755 add_tess_rings = true;
3756 }
3757 if (!queue->has_gds) {
3758 if (needs_gds)
3759 add_gds = true;
3760 }
3761 if (!queue->has_gds_oa) {
3762 if (needs_gds_oa)
3763 add_gds_oa = true;
3764 }
3765 if (!queue->has_sample_positions) {
3766 if (needs_sample_positions)
3767 add_sample_positions = true;
3768 }
3769 tess_factor_ring_size = 32768 * queue->device->physical_device->rad_info.max_se;
3770 hs_offchip_param = radv_get_hs_offchip_param(queue->device,
3771 &max_offchip_buffers);
3772 tess_offchip_ring_offset = align(tess_factor_ring_size, 64 * 1024);
3773 tess_offchip_ring_size = max_offchip_buffers *
3774 queue->device->tess_offchip_block_dw_size * 4;
3775
3776 scratch_size_per_wave = MAX2(scratch_size_per_wave, queue->scratch_size_per_wave);
3777 if (scratch_size_per_wave)
3778 scratch_waves = MIN2(scratch_waves, UINT32_MAX / scratch_size_per_wave);
3779 else
3780 scratch_waves = 0;
3781
3782 compute_scratch_size_per_wave = MAX2(compute_scratch_size_per_wave, queue->compute_scratch_size_per_wave);
3783 if (compute_scratch_size_per_wave)
3784 compute_scratch_waves = MIN2(compute_scratch_waves, UINT32_MAX / compute_scratch_size_per_wave);
3785 else
3786 compute_scratch_waves = 0;
3787
3788 if (scratch_size_per_wave <= queue->scratch_size_per_wave &&
3789 scratch_waves <= queue->scratch_waves &&
3790 compute_scratch_size_per_wave <= queue->compute_scratch_size_per_wave &&
3791 compute_scratch_waves <= queue->compute_scratch_waves &&
3792 esgs_ring_size <= queue->esgs_ring_size &&
3793 gsvs_ring_size <= queue->gsvs_ring_size &&
3794 !add_tess_rings && !add_gds && !add_gds_oa && !add_sample_positions &&
3795 queue->initial_preamble_cs) {
3796 *initial_full_flush_preamble_cs = queue->initial_full_flush_preamble_cs;
3797 *initial_preamble_cs = queue->initial_preamble_cs;
3798 *continue_preamble_cs = queue->continue_preamble_cs;
3799 if (!scratch_size_per_wave && !compute_scratch_size_per_wave &&
3800 !esgs_ring_size && !gsvs_ring_size && !needs_tess_rings &&
3801 !needs_gds && !needs_gds_oa && !needs_sample_positions)
3802 *continue_preamble_cs = NULL;
3803 return VK_SUCCESS;
3804 }
3805
3806 uint32_t scratch_size = scratch_size_per_wave * scratch_waves;
3807 uint32_t queue_scratch_size = queue->scratch_size_per_wave * queue->scratch_waves;
3808 if (scratch_size > queue_scratch_size) {
3809 scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
3810 scratch_size,
3811 4096,
3812 RADEON_DOMAIN_VRAM,
3813 ring_bo_flags,
3814 RADV_BO_PRIORITY_SCRATCH);
3815 if (!scratch_bo)
3816 goto fail;
3817 } else
3818 scratch_bo = queue->scratch_bo;
3819
3820 uint32_t compute_scratch_size = compute_scratch_size_per_wave * compute_scratch_waves;
3821 uint32_t compute_queue_scratch_size = queue->compute_scratch_size_per_wave * queue->compute_scratch_waves;
3822 if (compute_scratch_size > compute_queue_scratch_size) {
3823 compute_scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
3824 compute_scratch_size,
3825 4096,
3826 RADEON_DOMAIN_VRAM,
3827 ring_bo_flags,
3828 RADV_BO_PRIORITY_SCRATCH);
3829 if (!compute_scratch_bo)
3830 goto fail;
3831
3832 } else
3833 compute_scratch_bo = queue->compute_scratch_bo;
3834
3835 if (esgs_ring_size > queue->esgs_ring_size) {
3836 esgs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
3837 esgs_ring_size,
3838 4096,
3839 RADEON_DOMAIN_VRAM,
3840 ring_bo_flags,
3841 RADV_BO_PRIORITY_SCRATCH);
3842 if (!esgs_ring_bo)
3843 goto fail;
3844 } else {
3845 esgs_ring_bo = queue->esgs_ring_bo;
3846 esgs_ring_size = queue->esgs_ring_size;
3847 }
3848
3849 if (gsvs_ring_size > queue->gsvs_ring_size) {
3850 gsvs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
3851 gsvs_ring_size,
3852 4096,
3853 RADEON_DOMAIN_VRAM,
3854 ring_bo_flags,
3855 RADV_BO_PRIORITY_SCRATCH);
3856 if (!gsvs_ring_bo)
3857 goto fail;
3858 } else {
3859 gsvs_ring_bo = queue->gsvs_ring_bo;
3860 gsvs_ring_size = queue->gsvs_ring_size;
3861 }
3862
3863 if (add_tess_rings) {
3864 tess_rings_bo = queue->device->ws->buffer_create(queue->device->ws,
3865 tess_offchip_ring_offset + tess_offchip_ring_size,
3866 256,
3867 RADEON_DOMAIN_VRAM,
3868 ring_bo_flags,
3869 RADV_BO_PRIORITY_SCRATCH);
3870 if (!tess_rings_bo)
3871 goto fail;
3872 } else {
3873 tess_rings_bo = queue->tess_rings_bo;
3874 }
3875
3876 if (add_gds) {
3877 assert(queue->device->physical_device->rad_info.chip_class >= GFX10);
3878
3879 /* 4 streamout GDS counters.
3880 * We need 256B (64 dw) of GDS, otherwise streamout hangs.
3881 */
3882 gds_bo = queue->device->ws->buffer_create(queue->device->ws,
3883 256, 4,
3884 RADEON_DOMAIN_GDS,
3885 ring_bo_flags,
3886 RADV_BO_PRIORITY_SCRATCH);
3887 if (!gds_bo)
3888 goto fail;
3889 } else {
3890 gds_bo = queue->gds_bo;
3891 }
3892
3893 if (add_gds_oa) {
3894 assert(queue->device->physical_device->rad_info.chip_class >= GFX10);
3895
3896 gds_oa_bo = queue->device->ws->buffer_create(queue->device->ws,
3897 4, 1,
3898 RADEON_DOMAIN_OA,
3899 ring_bo_flags,
3900 RADV_BO_PRIORITY_SCRATCH);
3901 if (!gds_oa_bo)
3902 goto fail;
3903 } else {
3904 gds_oa_bo = queue->gds_oa_bo;
3905 }
3906
3907 if (scratch_bo != queue->scratch_bo ||
3908 esgs_ring_bo != queue->esgs_ring_bo ||
3909 gsvs_ring_bo != queue->gsvs_ring_bo ||
3910 tess_rings_bo != queue->tess_rings_bo ||
3911 add_sample_positions) {
3912 uint32_t size = 0;
3913 if (gsvs_ring_bo || esgs_ring_bo ||
3914 tess_rings_bo || add_sample_positions) {
3915 size = 112; /* 2 dword + 2 padding + 4 dword * 6 */
3916 if (add_sample_positions)
3917 size += 128; /* 64+32+16+8 = 120 bytes */
3918 }
3919 else if (scratch_bo)
3920 size = 8; /* 2 dword */
3921
3922 descriptor_bo = queue->device->ws->buffer_create(queue->device->ws,
3923 size,
3924 4096,
3925 RADEON_DOMAIN_VRAM,
3926 RADEON_FLAG_CPU_ACCESS |
3927 RADEON_FLAG_NO_INTERPROCESS_SHARING |
3928 RADEON_FLAG_READ_ONLY,
3929 RADV_BO_PRIORITY_DESCRIPTOR);
3930 if (!descriptor_bo)
3931 goto fail;
3932 } else
3933 descriptor_bo = queue->descriptor_bo;
3934
3935 if (descriptor_bo != queue->descriptor_bo) {
3936 uint32_t *map = (uint32_t*)queue->device->ws->buffer_map(descriptor_bo);
3937
3938 if (scratch_bo) {
3939 uint64_t scratch_va = radv_buffer_get_va(scratch_bo);
3940 uint32_t rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
3941 S_008F04_SWIZZLE_ENABLE(1);
3942 map[0] = scratch_va;
3943 map[1] = rsrc1;
3944 }
3945
3946 if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || add_sample_positions)
3947 fill_geom_tess_rings(queue, map, add_sample_positions,
3948 esgs_ring_size, esgs_ring_bo,
3949 gsvs_ring_size, gsvs_ring_bo,
3950 tess_factor_ring_size,
3951 tess_offchip_ring_offset,
3952 tess_offchip_ring_size,
3953 tess_rings_bo);
3954
3955 queue->device->ws->buffer_unmap(descriptor_bo);
3956 }
3957
3958 for(int i = 0; i < 3; ++i) {
3959 struct radeon_cmdbuf *cs = NULL;
3960 cs = queue->device->ws->cs_create(queue->device->ws,
3961 queue->queue_family_index ? RING_COMPUTE : RING_GFX);
3962 if (!cs)
3963 goto fail;
3964
3965 dest_cs[i] = cs;
3966
3967 if (scratch_bo)
3968 radv_cs_add_buffer(queue->device->ws, cs, scratch_bo);
3969
3970 /* Emit initial configuration. */
3971 switch (queue->queue_family_index) {
3972 case RADV_QUEUE_GENERAL:
3973 radv_init_graphics_state(cs, queue);
3974 break;
3975 case RADV_QUEUE_COMPUTE:
3976 radv_init_compute_state(cs, queue);
3977 break;
3978 case RADV_QUEUE_TRANSFER:
3979 break;
3980 }
3981
3982 if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo) {
3983 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
3984 radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
3985
3986 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
3987 radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
3988 }
3989
3990 radv_emit_gs_ring_sizes(queue, cs, esgs_ring_bo, esgs_ring_size,
3991 gsvs_ring_bo, gsvs_ring_size);
3992 radv_emit_tess_factor_ring(queue, cs, hs_offchip_param,
3993 tess_factor_ring_size, tess_rings_bo);
3994 radv_emit_global_shader_pointers(queue, cs, descriptor_bo);
3995 radv_emit_compute_scratch(queue, cs, compute_scratch_size_per_wave,
3996 compute_scratch_waves, compute_scratch_bo);
3997 radv_emit_graphics_scratch(queue, cs, scratch_size_per_wave,
3998 scratch_waves, scratch_bo);
3999
4000 if (gds_bo)
4001 radv_cs_add_buffer(queue->device->ws, cs, gds_bo);
4002 if (gds_oa_bo)
4003 radv_cs_add_buffer(queue->device->ws, cs, gds_oa_bo);
4004
4005 if (queue->device->trace_bo)
4006 radv_cs_add_buffer(queue->device->ws, cs, queue->device->trace_bo);
4007
4008 if (i == 0) {
4009 si_cs_emit_cache_flush(cs,
4010 queue->device->physical_device->rad_info.chip_class,
4011 NULL, 0,
4012 queue->queue_family_index == RING_COMPUTE &&
4013 queue->device->physical_device->rad_info.chip_class >= GFX7,
4014 (queue->queue_family_index == RADV_QUEUE_COMPUTE ? RADV_CMD_FLAG_CS_PARTIAL_FLUSH : (RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH)) |
4015 RADV_CMD_FLAG_INV_ICACHE |
4016 RADV_CMD_FLAG_INV_SCACHE |
4017 RADV_CMD_FLAG_INV_VCACHE |
4018 RADV_CMD_FLAG_INV_L2 |
4019 RADV_CMD_FLAG_START_PIPELINE_STATS, 0);
4020 } else if (i == 1) {
4021 si_cs_emit_cache_flush(cs,
4022 queue->device->physical_device->rad_info.chip_class,
4023 NULL, 0,
4024 queue->queue_family_index == RING_COMPUTE &&
4025 queue->device->physical_device->rad_info.chip_class >= GFX7,
4026 RADV_CMD_FLAG_INV_ICACHE |
4027 RADV_CMD_FLAG_INV_SCACHE |
4028 RADV_CMD_FLAG_INV_VCACHE |
4029 RADV_CMD_FLAG_INV_L2 |
4030 RADV_CMD_FLAG_START_PIPELINE_STATS, 0);
4031 }
4032
4033 if (!queue->device->ws->cs_finalize(cs))
4034 goto fail;
4035 }
4036
4037 if (queue->initial_full_flush_preamble_cs)
4038 queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
4039
4040 if (queue->initial_preamble_cs)
4041 queue->device->ws->cs_destroy(queue->initial_preamble_cs);
4042
4043 if (queue->continue_preamble_cs)
4044 queue->device->ws->cs_destroy(queue->continue_preamble_cs);
4045
4046 queue->initial_full_flush_preamble_cs = dest_cs[0];
4047 queue->initial_preamble_cs = dest_cs[1];
4048 queue->continue_preamble_cs = dest_cs[2];
4049
4050 if (scratch_bo != queue->scratch_bo) {
4051 if (queue->scratch_bo)
4052 queue->device->ws->buffer_destroy(queue->scratch_bo);
4053 queue->scratch_bo = scratch_bo;
4054 }
4055 queue->scratch_size_per_wave = scratch_size_per_wave;
4056 queue->scratch_waves = scratch_waves;
4057
4058 if (compute_scratch_bo != queue->compute_scratch_bo) {
4059 if (queue->compute_scratch_bo)
4060 queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
4061 queue->compute_scratch_bo = compute_scratch_bo;
4062 }
4063 queue->compute_scratch_size_per_wave = compute_scratch_size_per_wave;
4064 queue->compute_scratch_waves = compute_scratch_waves;
4065
4066 if (esgs_ring_bo != queue->esgs_ring_bo) {
4067 if (queue->esgs_ring_bo)
4068 queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
4069 queue->esgs_ring_bo = esgs_ring_bo;
4070 queue->esgs_ring_size = esgs_ring_size;
4071 }
4072
4073 if (gsvs_ring_bo != queue->gsvs_ring_bo) {
4074 if (queue->gsvs_ring_bo)
4075 queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
4076 queue->gsvs_ring_bo = gsvs_ring_bo;
4077 queue->gsvs_ring_size = gsvs_ring_size;
4078 }
4079
4080 if (tess_rings_bo != queue->tess_rings_bo) {
4081 queue->tess_rings_bo = tess_rings_bo;
4082 queue->has_tess_rings = true;
4083 }
4084
4085 if (gds_bo != queue->gds_bo) {
4086 queue->gds_bo = gds_bo;
4087 queue->has_gds = true;
4088 }
4089
4090 if (gds_oa_bo != queue->gds_oa_bo) {
4091 queue->gds_oa_bo = gds_oa_bo;
4092 queue->has_gds_oa = true;
4093 }
4094
4095 if (descriptor_bo != queue->descriptor_bo) {
4096 if (queue->descriptor_bo)
4097 queue->device->ws->buffer_destroy(queue->descriptor_bo);
4098
4099 queue->descriptor_bo = descriptor_bo;
4100 }
4101
4102 if (add_sample_positions)
4103 queue->has_sample_positions = true;
4104
4105 *initial_full_flush_preamble_cs = queue->initial_full_flush_preamble_cs;
4106 *initial_preamble_cs = queue->initial_preamble_cs;
4107 *continue_preamble_cs = queue->continue_preamble_cs;
4108 if (!scratch_size && !compute_scratch_size && !esgs_ring_size && !gsvs_ring_size)
4109 *continue_preamble_cs = NULL;
4110 return VK_SUCCESS;
4111 fail:
4112 for (int i = 0; i < ARRAY_SIZE(dest_cs); ++i)
4113 if (dest_cs[i])
4114 queue->device->ws->cs_destroy(dest_cs[i]);
4115 if (descriptor_bo && descriptor_bo != queue->descriptor_bo)
4116 queue->device->ws->buffer_destroy(descriptor_bo);
4117 if (scratch_bo && scratch_bo != queue->scratch_bo)
4118 queue->device->ws->buffer_destroy(scratch_bo);
4119 if (compute_scratch_bo && compute_scratch_bo != queue->compute_scratch_bo)
4120 queue->device->ws->buffer_destroy(compute_scratch_bo);
4121 if (esgs_ring_bo && esgs_ring_bo != queue->esgs_ring_bo)
4122 queue->device->ws->buffer_destroy(esgs_ring_bo);
4123 if (gsvs_ring_bo && gsvs_ring_bo != queue->gsvs_ring_bo)
4124 queue->device->ws->buffer_destroy(gsvs_ring_bo);
4125 if (tess_rings_bo && tess_rings_bo != queue->tess_rings_bo)
4126 queue->device->ws->buffer_destroy(tess_rings_bo);
4127 if (gds_bo && gds_bo != queue->gds_bo)
4128 queue->device->ws->buffer_destroy(gds_bo);
4129 if (gds_oa_bo && gds_oa_bo != queue->gds_oa_bo)
4130 queue->device->ws->buffer_destroy(gds_oa_bo);
4131
4132 return vk_error(queue->device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
4133 }
4134
4135 static VkResult radv_alloc_sem_counts(struct radv_device *device,
4136 struct radv_winsys_sem_counts *counts,
4137 int num_sems,
4138 struct radv_semaphore_part **sems,
4139 const uint64_t *timeline_values,
4140 VkFence _fence,
4141 bool is_signal)
4142 {
4143 int syncobj_idx = 0, sem_idx = 0;
4144
4145 if (num_sems == 0 && _fence == VK_NULL_HANDLE)
4146 return VK_SUCCESS;
4147
4148 for (uint32_t i = 0; i < num_sems; i++) {
4149 switch(sems[i]->kind) {
4150 case RADV_SEMAPHORE_SYNCOBJ:
4151 counts->syncobj_count++;
4152 break;
4153 case RADV_SEMAPHORE_WINSYS:
4154 counts->sem_count++;
4155 break;
4156 case RADV_SEMAPHORE_NONE:
4157 break;
4158 case RADV_SEMAPHORE_TIMELINE:
4159 counts->syncobj_count++;
4160 break;
4161 }
4162 }
4163
4164 if (_fence != VK_NULL_HANDLE) {
4165 RADV_FROM_HANDLE(radv_fence, fence, _fence);
4166 if (fence->temp_syncobj || fence->syncobj)
4167 counts->syncobj_count++;
4168 }
4169
4170 if (counts->syncobj_count) {
4171 counts->syncobj = (uint32_t *)malloc(sizeof(uint32_t) * counts->syncobj_count);
4172 if (!counts->syncobj)
4173 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
4174 }
4175
4176 if (counts->sem_count) {
4177 counts->sem = (struct radeon_winsys_sem **)malloc(sizeof(struct radeon_winsys_sem *) * counts->sem_count);
4178 if (!counts->sem) {
4179 free(counts->syncobj);
4180 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
4181 }
4182 }
4183
4184 for (uint32_t i = 0; i < num_sems; i++) {
4185 switch(sems[i]->kind) {
4186 case RADV_SEMAPHORE_NONE:
4187 unreachable("Empty semaphore");
4188 break;
4189 case RADV_SEMAPHORE_SYNCOBJ:
4190 counts->syncobj[syncobj_idx++] = sems[i]->syncobj;
4191 break;
4192 case RADV_SEMAPHORE_WINSYS:
4193 counts->sem[sem_idx++] = sems[i]->ws_sem;
4194 break;
4195 case RADV_SEMAPHORE_TIMELINE: {
4196 pthread_mutex_lock(&sems[i]->timeline.mutex);
4197 struct radv_timeline_point *point = NULL;
4198 if (is_signal) {
4199 point = radv_timeline_add_point_locked(device, &sems[i]->timeline, timeline_values[i]);
4200 } else {
4201 point = radv_timeline_find_point_at_least_locked(device, &sems[i]->timeline, timeline_values[i]);
4202 }
4203
4204 pthread_mutex_unlock(&sems[i]->timeline.mutex);
4205
4206 if (point) {
4207 counts->syncobj[syncobj_idx++] = point->syncobj;
4208 } else {
4209 /* Explicitly remove the semaphore so we might not find
4210 * a point later post-submit. */
4211 sems[i] = NULL;
4212 }
4213 break;
4214 }
4215 }
4216 }
4217
4218 if (_fence != VK_NULL_HANDLE) {
4219 RADV_FROM_HANDLE(radv_fence, fence, _fence);
4220 if (fence->temp_syncobj)
4221 counts->syncobj[syncobj_idx++] = fence->temp_syncobj;
4222 else if (fence->syncobj)
4223 counts->syncobj[syncobj_idx++] = fence->syncobj;
4224 }
4225
4226 assert(syncobj_idx <= counts->syncobj_count);
4227 counts->syncobj_count = syncobj_idx;
4228
4229 return VK_SUCCESS;
4230 }
4231
4232 static void
4233 radv_free_sem_info(struct radv_winsys_sem_info *sem_info)
4234 {
4235 free(sem_info->wait.syncobj);
4236 free(sem_info->wait.sem);
4237 free(sem_info->signal.syncobj);
4238 free(sem_info->signal.sem);
4239 }
4240
4241
4242 static void radv_free_temp_syncobjs(struct radv_device *device,
4243 int num_sems,
4244 struct radv_semaphore_part *sems)
4245 {
4246 for (uint32_t i = 0; i < num_sems; i++) {
4247 radv_destroy_semaphore_part(device, sems + i);
4248 }
4249 }
4250
4251 static VkResult
4252 radv_alloc_sem_info(struct radv_device *device,
4253 struct radv_winsys_sem_info *sem_info,
4254 int num_wait_sems,
4255 struct radv_semaphore_part **wait_sems,
4256 const uint64_t *wait_values,
4257 int num_signal_sems,
4258 struct radv_semaphore_part **signal_sems,
4259 const uint64_t *signal_values,
4260 VkFence fence)
4261 {
4262 VkResult ret;
4263 memset(sem_info, 0, sizeof(*sem_info));
4264
4265 ret = radv_alloc_sem_counts(device, &sem_info->wait, num_wait_sems, wait_sems, wait_values, VK_NULL_HANDLE, false);
4266 if (ret)
4267 return ret;
4268 ret = radv_alloc_sem_counts(device, &sem_info->signal, num_signal_sems, signal_sems, signal_values, fence, true);
4269 if (ret)
4270 radv_free_sem_info(sem_info);
4271
4272 /* caller can override these */
4273 sem_info->cs_emit_wait = true;
4274 sem_info->cs_emit_signal = true;
4275 return ret;
4276 }
4277
4278 static void
4279 radv_finalize_timelines(struct radv_device *device,
4280 uint32_t num_wait_sems,
4281 struct radv_semaphore_part **wait_sems,
4282 const uint64_t *wait_values,
4283 uint32_t num_signal_sems,
4284 struct radv_semaphore_part **signal_sems,
4285 const uint64_t *signal_values,
4286 struct list_head *processing_list)
4287 {
4288 for (uint32_t i = 0; i < num_wait_sems; ++i) {
4289 if (wait_sems[i] && wait_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4290 pthread_mutex_lock(&wait_sems[i]->timeline.mutex);
4291 struct radv_timeline_point *point =
4292 radv_timeline_find_point_at_least_locked(device, &wait_sems[i]->timeline, wait_values[i]);
4293 point->wait_count -= 2;
4294 pthread_mutex_unlock(&wait_sems[i]->timeline.mutex);
4295 }
4296 }
4297 for (uint32_t i = 0; i < num_signal_sems; ++i) {
4298 if (signal_sems[i] && signal_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4299 pthread_mutex_lock(&signal_sems[i]->timeline.mutex);
4300 struct radv_timeline_point *point =
4301 radv_timeline_find_point_at_least_locked(device, &signal_sems[i]->timeline, signal_values[i]);
4302 signal_sems[i]->timeline.highest_submitted =
4303 MAX2(signal_sems[i]->timeline.highest_submitted, point->value);
4304 point->wait_count -= 2;
4305 radv_timeline_trigger_waiters_locked(&signal_sems[i]->timeline, processing_list);
4306 pthread_mutex_unlock(&signal_sems[i]->timeline.mutex);
4307 }
4308 }
4309 }
4310
4311 static void
4312 radv_sparse_buffer_bind_memory(struct radv_device *device,
4313 const VkSparseBufferMemoryBindInfo *bind)
4314 {
4315 RADV_FROM_HANDLE(radv_buffer, buffer, bind->buffer);
4316
4317 for (uint32_t i = 0; i < bind->bindCount; ++i) {
4318 struct radv_device_memory *mem = NULL;
4319
4320 if (bind->pBinds[i].memory != VK_NULL_HANDLE)
4321 mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
4322
4323 device->ws->buffer_virtual_bind(buffer->bo,
4324 bind->pBinds[i].resourceOffset,
4325 bind->pBinds[i].size,
4326 mem ? mem->bo : NULL,
4327 bind->pBinds[i].memoryOffset);
4328 }
4329 }
4330
4331 static void
4332 radv_sparse_image_opaque_bind_memory(struct radv_device *device,
4333 const VkSparseImageOpaqueMemoryBindInfo *bind)
4334 {
4335 RADV_FROM_HANDLE(radv_image, image, bind->image);
4336
4337 for (uint32_t i = 0; i < bind->bindCount; ++i) {
4338 struct radv_device_memory *mem = NULL;
4339
4340 if (bind->pBinds[i].memory != VK_NULL_HANDLE)
4341 mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
4342
4343 device->ws->buffer_virtual_bind(image->bo,
4344 bind->pBinds[i].resourceOffset,
4345 bind->pBinds[i].size,
4346 mem ? mem->bo : NULL,
4347 bind->pBinds[i].memoryOffset);
4348 }
4349 }
4350
4351 static VkResult
4352 radv_get_preambles(struct radv_queue *queue,
4353 const VkCommandBuffer *cmd_buffers,
4354 uint32_t cmd_buffer_count,
4355 struct radeon_cmdbuf **initial_full_flush_preamble_cs,
4356 struct radeon_cmdbuf **initial_preamble_cs,
4357 struct radeon_cmdbuf **continue_preamble_cs)
4358 {
4359 uint32_t scratch_size_per_wave = 0, waves_wanted = 0;
4360 uint32_t compute_scratch_size_per_wave = 0, compute_waves_wanted = 0;
4361 uint32_t esgs_ring_size = 0, gsvs_ring_size = 0;
4362 bool tess_rings_needed = false;
4363 bool gds_needed = false;
4364 bool gds_oa_needed = false;
4365 bool sample_positions_needed = false;
4366
4367 for (uint32_t j = 0; j < cmd_buffer_count; j++) {
4368 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer,
4369 cmd_buffers[j]);
4370
4371 scratch_size_per_wave = MAX2(scratch_size_per_wave, cmd_buffer->scratch_size_per_wave_needed);
4372 waves_wanted = MAX2(waves_wanted, cmd_buffer->scratch_waves_wanted);
4373 compute_scratch_size_per_wave = MAX2(compute_scratch_size_per_wave,
4374 cmd_buffer->compute_scratch_size_per_wave_needed);
4375 compute_waves_wanted = MAX2(compute_waves_wanted,
4376 cmd_buffer->compute_scratch_waves_wanted);
4377 esgs_ring_size = MAX2(esgs_ring_size, cmd_buffer->esgs_ring_size_needed);
4378 gsvs_ring_size = MAX2(gsvs_ring_size, cmd_buffer->gsvs_ring_size_needed);
4379 tess_rings_needed |= cmd_buffer->tess_rings_needed;
4380 gds_needed |= cmd_buffer->gds_needed;
4381 gds_oa_needed |= cmd_buffer->gds_oa_needed;
4382 sample_positions_needed |= cmd_buffer->sample_positions_needed;
4383 }
4384
4385 return radv_get_preamble_cs(queue, scratch_size_per_wave, waves_wanted,
4386 compute_scratch_size_per_wave, compute_waves_wanted,
4387 esgs_ring_size, gsvs_ring_size, tess_rings_needed,
4388 gds_needed, gds_oa_needed, sample_positions_needed,
4389 initial_full_flush_preamble_cs,
4390 initial_preamble_cs, continue_preamble_cs);
4391 }
4392
4393 struct radv_deferred_queue_submission {
4394 struct radv_queue *queue;
4395 VkCommandBuffer *cmd_buffers;
4396 uint32_t cmd_buffer_count;
4397
4398 /* Sparse bindings that happen on a queue. */
4399 VkSparseBufferMemoryBindInfo *buffer_binds;
4400 uint32_t buffer_bind_count;
4401 VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
4402 uint32_t image_opaque_bind_count;
4403
4404 bool flush_caches;
4405 VkShaderStageFlags wait_dst_stage_mask;
4406 struct radv_semaphore_part **wait_semaphores;
4407 uint32_t wait_semaphore_count;
4408 struct radv_semaphore_part **signal_semaphores;
4409 uint32_t signal_semaphore_count;
4410 VkFence fence;
4411
4412 uint64_t *wait_values;
4413 uint64_t *signal_values;
4414
4415 struct radv_semaphore_part *temporary_semaphore_parts;
4416 uint32_t temporary_semaphore_part_count;
4417
4418 struct list_head queue_pending_list;
4419 uint32_t submission_wait_count;
4420 struct radv_timeline_waiter *wait_nodes;
4421
4422 struct list_head processing_list;
4423 };
4424
4425 struct radv_queue_submission {
4426 const VkCommandBuffer *cmd_buffers;
4427 uint32_t cmd_buffer_count;
4428
4429 /* Sparse bindings that happen on a queue. */
4430 const VkSparseBufferMemoryBindInfo *buffer_binds;
4431 uint32_t buffer_bind_count;
4432 const VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
4433 uint32_t image_opaque_bind_count;
4434
4435 bool flush_caches;
4436 VkPipelineStageFlags wait_dst_stage_mask;
4437 const VkSemaphore *wait_semaphores;
4438 uint32_t wait_semaphore_count;
4439 const VkSemaphore *signal_semaphores;
4440 uint32_t signal_semaphore_count;
4441 VkFence fence;
4442
4443 const uint64_t *wait_values;
4444 uint32_t wait_value_count;
4445 const uint64_t *signal_values;
4446 uint32_t signal_value_count;
4447 };
4448
4449 static VkResult
4450 radv_create_deferred_submission(struct radv_queue *queue,
4451 const struct radv_queue_submission *submission,
4452 struct radv_deferred_queue_submission **out)
4453 {
4454 struct radv_deferred_queue_submission *deferred = NULL;
4455 size_t size = sizeof(struct radv_deferred_queue_submission);
4456
4457 uint32_t temporary_count = 0;
4458 for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4459 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphores[i]);
4460 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE)
4461 ++temporary_count;
4462 }
4463
4464 size += submission->cmd_buffer_count * sizeof(VkCommandBuffer);
4465 size += submission->buffer_bind_count * sizeof(VkSparseBufferMemoryBindInfo);
4466 size += submission->image_opaque_bind_count * sizeof(VkSparseImageOpaqueMemoryBindInfo);
4467 size += submission->wait_semaphore_count * sizeof(struct radv_semaphore_part *);
4468 size += temporary_count * sizeof(struct radv_semaphore_part);
4469 size += submission->signal_semaphore_count * sizeof(struct radv_semaphore_part *);
4470 size += submission->wait_value_count * sizeof(uint64_t);
4471 size += submission->signal_value_count * sizeof(uint64_t);
4472 size += submission->wait_semaphore_count * sizeof(struct radv_timeline_waiter);
4473
4474 deferred = calloc(1, size);
4475 if (!deferred)
4476 return VK_ERROR_OUT_OF_HOST_MEMORY;
4477
4478 deferred->queue = queue;
4479
4480 deferred->cmd_buffers = (void*)(deferred + 1);
4481 deferred->cmd_buffer_count = submission->cmd_buffer_count;
4482 memcpy(deferred->cmd_buffers, submission->cmd_buffers,
4483 submission->cmd_buffer_count * sizeof(*deferred->cmd_buffers));
4484
4485 deferred->buffer_binds = (void*)(deferred->cmd_buffers + submission->cmd_buffer_count);
4486 deferred->buffer_bind_count = submission->buffer_bind_count;
4487 memcpy(deferred->buffer_binds, submission->buffer_binds,
4488 submission->buffer_bind_count * sizeof(*deferred->buffer_binds));
4489
4490 deferred->image_opaque_binds = (void*)(deferred->buffer_binds + submission->buffer_bind_count);
4491 deferred->image_opaque_bind_count = submission->image_opaque_bind_count;
4492 memcpy(deferred->image_opaque_binds, submission->image_opaque_binds,
4493 submission->image_opaque_bind_count * sizeof(*deferred->image_opaque_binds));
4494
4495 deferred->flush_caches = submission->flush_caches;
4496 deferred->wait_dst_stage_mask = submission->wait_dst_stage_mask;
4497
4498 deferred->wait_semaphores = (void*)(deferred->image_opaque_binds + deferred->image_opaque_bind_count);
4499 deferred->wait_semaphore_count = submission->wait_semaphore_count;
4500
4501 deferred->signal_semaphores = (void*)(deferred->wait_semaphores + deferred->wait_semaphore_count);
4502 deferred->signal_semaphore_count = submission->signal_semaphore_count;
4503
4504 deferred->fence = submission->fence;
4505
4506 deferred->temporary_semaphore_parts = (void*)(deferred->signal_semaphores + deferred->signal_semaphore_count);
4507 deferred->temporary_semaphore_part_count = temporary_count;
4508
4509 uint32_t temporary_idx = 0;
4510 for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4511 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphores[i]);
4512 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) {
4513 deferred->wait_semaphores[i] = &deferred->temporary_semaphore_parts[temporary_idx];
4514 deferred->temporary_semaphore_parts[temporary_idx] = semaphore->temporary;
4515 semaphore->temporary.kind = RADV_SEMAPHORE_NONE;
4516 ++temporary_idx;
4517 } else
4518 deferred->wait_semaphores[i] = &semaphore->permanent;
4519 }
4520
4521 for (uint32_t i = 0; i < submission->signal_semaphore_count; ++i) {
4522 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->signal_semaphores[i]);
4523 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) {
4524 deferred->signal_semaphores[i] = &semaphore->temporary;
4525 } else {
4526 deferred->signal_semaphores[i] = &semaphore->permanent;
4527 }
4528 }
4529
4530 deferred->wait_values = (void*)(deferred->temporary_semaphore_parts + temporary_count);
4531 memcpy(deferred->wait_values, submission->wait_values, submission->wait_value_count * sizeof(uint64_t));
4532 deferred->signal_values = deferred->wait_values + submission->wait_value_count;
4533 memcpy(deferred->signal_values, submission->signal_values, submission->signal_value_count * sizeof(uint64_t));
4534
4535 deferred->wait_nodes = (void*)(deferred->signal_values + submission->signal_value_count);
4536 /* This is worst-case. radv_queue_enqueue_submission will fill in further, but this
4537 * ensure the submission is not accidentally triggered early when adding wait timelines. */
4538 deferred->submission_wait_count = 1 + submission->wait_semaphore_count;
4539
4540 *out = deferred;
4541 return VK_SUCCESS;
4542 }
4543
4544 static void
4545 radv_queue_enqueue_submission(struct radv_deferred_queue_submission *submission,
4546 struct list_head *processing_list)
4547 {
4548 uint32_t wait_cnt = 0;
4549 struct radv_timeline_waiter *waiter = submission->wait_nodes;
4550 for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4551 if (submission->wait_semaphores[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4552 pthread_mutex_lock(&submission->wait_semaphores[i]->timeline.mutex);
4553 if (submission->wait_semaphores[i]->timeline.highest_submitted < submission->wait_values[i]) {
4554 ++wait_cnt;
4555 waiter->value = submission->wait_values[i];
4556 waiter->submission = submission;
4557 list_addtail(&waiter->list, &submission->wait_semaphores[i]->timeline.waiters);
4558 ++waiter;
4559 }
4560 pthread_mutex_unlock(&submission->wait_semaphores[i]->timeline.mutex);
4561 }
4562 }
4563
4564 pthread_mutex_lock(&submission->queue->pending_mutex);
4565
4566 bool is_first = list_is_empty(&submission->queue->pending_submissions);
4567 list_addtail(&submission->queue_pending_list, &submission->queue->pending_submissions);
4568
4569 pthread_mutex_unlock(&submission->queue->pending_mutex);
4570
4571 /* If there is already a submission in the queue, that will decrement the counter by 1 when
4572 * submitted, but if the queue was empty, we decrement ourselves as there is no previous
4573 * submission. */
4574 uint32_t decrement = submission->wait_semaphore_count - wait_cnt + (is_first ? 1 : 0);
4575 if (__atomic_sub_fetch(&submission->submission_wait_count, decrement, __ATOMIC_ACQ_REL) == 0) {
4576 list_addtail(&submission->processing_list, processing_list);
4577 }
4578 }
4579
4580 static void
4581 radv_queue_submission_update_queue(struct radv_deferred_queue_submission *submission,
4582 struct list_head *processing_list)
4583 {
4584 pthread_mutex_lock(&submission->queue->pending_mutex);
4585 list_del(&submission->queue_pending_list);
4586
4587 /* trigger the next submission in the queue. */
4588 if (!list_is_empty(&submission->queue->pending_submissions)) {
4589 struct radv_deferred_queue_submission *next_submission =
4590 list_first_entry(&submission->queue->pending_submissions,
4591 struct radv_deferred_queue_submission,
4592 queue_pending_list);
4593 if (p_atomic_dec_zero(&next_submission->submission_wait_count)) {
4594 list_addtail(&next_submission->processing_list, processing_list);
4595 }
4596 }
4597 pthread_mutex_unlock(&submission->queue->pending_mutex);
4598
4599 pthread_cond_broadcast(&submission->queue->device->timeline_cond);
4600 }
4601
4602 static VkResult
4603 radv_queue_submit_deferred(struct radv_deferred_queue_submission *submission,
4604 struct list_head *processing_list)
4605 {
4606 RADV_FROM_HANDLE(radv_fence, fence, submission->fence);
4607 struct radv_queue *queue = submission->queue;
4608 struct radeon_winsys_ctx *ctx = queue->hw_ctx;
4609 uint32_t max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT;
4610 struct radeon_winsys_fence *base_fence = fence ? fence->fence : NULL;
4611 bool do_flush = submission->flush_caches || submission->wait_dst_stage_mask;
4612 bool can_patch = true;
4613 uint32_t advance;
4614 struct radv_winsys_sem_info sem_info;
4615 VkResult result;
4616 int ret;
4617 struct radeon_cmdbuf *initial_preamble_cs = NULL;
4618 struct radeon_cmdbuf *initial_flush_preamble_cs = NULL;
4619 struct radeon_cmdbuf *continue_preamble_cs = NULL;
4620
4621 result = radv_get_preambles(queue, submission->cmd_buffers,
4622 submission->cmd_buffer_count,
4623 &initial_preamble_cs,
4624 &initial_flush_preamble_cs,
4625 &continue_preamble_cs);
4626 if (result != VK_SUCCESS)
4627 goto fail;
4628
4629 result = radv_alloc_sem_info(queue->device,
4630 &sem_info,
4631 submission->wait_semaphore_count,
4632 submission->wait_semaphores,
4633 submission->wait_values,
4634 submission->signal_semaphore_count,
4635 submission->signal_semaphores,
4636 submission->signal_values,
4637 submission->fence);
4638 if (result != VK_SUCCESS)
4639 goto fail;
4640
4641 for (uint32_t i = 0; i < submission->buffer_bind_count; ++i) {
4642 radv_sparse_buffer_bind_memory(queue->device,
4643 submission->buffer_binds + i);
4644 }
4645
4646 for (uint32_t i = 0; i < submission->image_opaque_bind_count; ++i) {
4647 radv_sparse_image_opaque_bind_memory(queue->device,
4648 submission->image_opaque_binds + i);
4649 }
4650
4651 if (!submission->cmd_buffer_count) {
4652 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx,
4653 &queue->device->empty_cs[queue->queue_family_index],
4654 1, NULL, NULL,
4655 &sem_info, NULL,
4656 false, base_fence);
4657 if (ret) {
4658 radv_loge("failed to submit CS\n");
4659 abort();
4660 }
4661
4662 goto success;
4663 } else {
4664 struct radeon_cmdbuf **cs_array = malloc(sizeof(struct radeon_cmdbuf *) *
4665 (submission->cmd_buffer_count));
4666
4667 for (uint32_t j = 0; j < submission->cmd_buffer_count; j++) {
4668 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, submission->cmd_buffers[j]);
4669 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
4670
4671 cs_array[j] = cmd_buffer->cs;
4672 if ((cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT))
4673 can_patch = false;
4674
4675 cmd_buffer->status = RADV_CMD_BUFFER_STATUS_PENDING;
4676 }
4677
4678 for (uint32_t j = 0; j < submission->cmd_buffer_count; j += advance) {
4679 struct radeon_cmdbuf *initial_preamble = (do_flush && !j) ? initial_flush_preamble_cs : initial_preamble_cs;
4680 const struct radv_winsys_bo_list *bo_list = NULL;
4681
4682 advance = MIN2(max_cs_submission,
4683 submission->cmd_buffer_count - j);
4684
4685 if (queue->device->trace_bo)
4686 *queue->device->trace_id_ptr = 0;
4687
4688 sem_info.cs_emit_wait = j == 0;
4689 sem_info.cs_emit_signal = j + advance == submission->cmd_buffer_count;
4690
4691 if (unlikely(queue->device->use_global_bo_list)) {
4692 pthread_mutex_lock(&queue->device->bo_list.mutex);
4693 bo_list = &queue->device->bo_list.list;
4694 }
4695
4696 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx, cs_array + j,
4697 advance, initial_preamble, continue_preamble_cs,
4698 &sem_info, bo_list,
4699 can_patch, base_fence);
4700
4701 if (unlikely(queue->device->use_global_bo_list))
4702 pthread_mutex_unlock(&queue->device->bo_list.mutex);
4703
4704 if (ret) {
4705 radv_loge("failed to submit CS\n");
4706 abort();
4707 }
4708 if (queue->device->trace_bo) {
4709 radv_check_gpu_hangs(queue, cs_array[j]);
4710 }
4711 }
4712
4713 free(cs_array);
4714 }
4715
4716 success:
4717 radv_free_temp_syncobjs(queue->device,
4718 submission->temporary_semaphore_part_count,
4719 submission->temporary_semaphore_parts);
4720 radv_finalize_timelines(queue->device,
4721 submission->wait_semaphore_count,
4722 submission->wait_semaphores,
4723 submission->wait_values,
4724 submission->signal_semaphore_count,
4725 submission->signal_semaphores,
4726 submission->signal_values,
4727 processing_list);
4728 /* Has to happen after timeline finalization to make sure the
4729 * condition variable is only triggered when timelines and queue have
4730 * been updated. */
4731 radv_queue_submission_update_queue(submission, processing_list);
4732 radv_free_sem_info(&sem_info);
4733 free(submission);
4734 return VK_SUCCESS;
4735
4736 fail:
4737 radv_free_temp_syncobjs(queue->device,
4738 submission->temporary_semaphore_part_count,
4739 submission->temporary_semaphore_parts);
4740 free(submission);
4741 return VK_ERROR_DEVICE_LOST;
4742 }
4743
4744 static VkResult
4745 radv_process_submissions(struct list_head *processing_list)
4746 {
4747 while(!list_is_empty(processing_list)) {
4748 struct radv_deferred_queue_submission *submission =
4749 list_first_entry(processing_list, struct radv_deferred_queue_submission, processing_list);
4750 list_del(&submission->processing_list);
4751
4752 VkResult result = radv_queue_submit_deferred(submission, processing_list);
4753 if (result != VK_SUCCESS)
4754 return result;
4755 }
4756 return VK_SUCCESS;
4757 }
4758
4759 static VkResult radv_queue_submit(struct radv_queue *queue,
4760 const struct radv_queue_submission *submission)
4761 {
4762 struct radv_deferred_queue_submission *deferred = NULL;
4763
4764 VkResult result = radv_create_deferred_submission(queue, submission, &deferred);
4765 if (result != VK_SUCCESS)
4766 return result;
4767
4768 struct list_head processing_list;
4769 list_inithead(&processing_list);
4770
4771 radv_queue_enqueue_submission(deferred, &processing_list);
4772 return radv_process_submissions(&processing_list);
4773 }
4774
4775 bool
4776 radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs)
4777 {
4778 struct radeon_winsys_ctx *ctx = queue->hw_ctx;
4779 struct radv_winsys_sem_info sem_info;
4780 VkResult result;
4781 int ret;
4782
4783 result = radv_alloc_sem_info(queue->device, &sem_info, 0, NULL, 0, 0,
4784 0, NULL, VK_NULL_HANDLE);
4785 if (result != VK_SUCCESS)
4786 return false;
4787
4788 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx, &cs, 1, NULL,
4789 NULL, &sem_info, NULL, false, NULL);
4790 radv_free_sem_info(&sem_info);
4791 return !ret;
4792 }
4793
4794 /* Signals fence as soon as all the work currently put on queue is done. */
4795 static VkResult radv_signal_fence(struct radv_queue *queue,
4796 VkFence fence)
4797 {
4798 return radv_queue_submit(queue, &(struct radv_queue_submission) {
4799 .fence = fence
4800 });
4801 }
4802
4803 static bool radv_submit_has_effects(const VkSubmitInfo *info)
4804 {
4805 return info->commandBufferCount ||
4806 info->waitSemaphoreCount ||
4807 info->signalSemaphoreCount;
4808 }
4809
4810 VkResult radv_QueueSubmit(
4811 VkQueue _queue,
4812 uint32_t submitCount,
4813 const VkSubmitInfo* pSubmits,
4814 VkFence fence)
4815 {
4816 RADV_FROM_HANDLE(radv_queue, queue, _queue);
4817 VkResult result;
4818 uint32_t fence_idx = 0;
4819 bool flushed_caches = false;
4820
4821 if (fence != VK_NULL_HANDLE) {
4822 for (uint32_t i = 0; i < submitCount; ++i)
4823 if (radv_submit_has_effects(pSubmits + i))
4824 fence_idx = i;
4825 } else
4826 fence_idx = UINT32_MAX;
4827
4828 for (uint32_t i = 0; i < submitCount; i++) {
4829 if (!radv_submit_has_effects(pSubmits + i) && fence_idx != i)
4830 continue;
4831
4832 VkPipelineStageFlags wait_dst_stage_mask = 0;
4833 for (unsigned j = 0; j < pSubmits[i].waitSemaphoreCount; ++j) {
4834 wait_dst_stage_mask |= pSubmits[i].pWaitDstStageMask[j];
4835 }
4836
4837 const VkTimelineSemaphoreSubmitInfo *timeline_info =
4838 vk_find_struct_const(pSubmits[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
4839
4840 result = radv_queue_submit(queue, &(struct radv_queue_submission) {
4841 .cmd_buffers = pSubmits[i].pCommandBuffers,
4842 .cmd_buffer_count = pSubmits[i].commandBufferCount,
4843 .wait_dst_stage_mask = wait_dst_stage_mask,
4844 .flush_caches = !flushed_caches,
4845 .wait_semaphores = pSubmits[i].pWaitSemaphores,
4846 .wait_semaphore_count = pSubmits[i].waitSemaphoreCount,
4847 .signal_semaphores = pSubmits[i].pSignalSemaphores,
4848 .signal_semaphore_count = pSubmits[i].signalSemaphoreCount,
4849 .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
4850 .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
4851 .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
4852 .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
4853 .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
4854 });
4855 if (result != VK_SUCCESS)
4856 return result;
4857
4858 flushed_caches = true;
4859 }
4860
4861 if (fence != VK_NULL_HANDLE && !submitCount) {
4862 result = radv_signal_fence(queue, fence);
4863 if (result != VK_SUCCESS)
4864 return result;
4865 }
4866
4867 return VK_SUCCESS;
4868 }
4869
4870 VkResult radv_QueueWaitIdle(
4871 VkQueue _queue)
4872 {
4873 RADV_FROM_HANDLE(radv_queue, queue, _queue);
4874
4875 pthread_mutex_lock(&queue->pending_mutex);
4876 while (!list_is_empty(&queue->pending_submissions)) {
4877 pthread_cond_wait(&queue->device->timeline_cond, &queue->pending_mutex);
4878 }
4879 pthread_mutex_unlock(&queue->pending_mutex);
4880
4881 queue->device->ws->ctx_wait_idle(queue->hw_ctx,
4882 radv_queue_family_to_ring(queue->queue_family_index),
4883 queue->queue_idx);
4884 return VK_SUCCESS;
4885 }
4886
4887 VkResult radv_DeviceWaitIdle(
4888 VkDevice _device)
4889 {
4890 RADV_FROM_HANDLE(radv_device, device, _device);
4891
4892 for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
4893 for (unsigned q = 0; q < device->queue_count[i]; q++) {
4894 radv_QueueWaitIdle(radv_queue_to_handle(&device->queues[i][q]));
4895 }
4896 }
4897 return VK_SUCCESS;
4898 }
4899
4900 VkResult radv_EnumerateInstanceExtensionProperties(
4901 const char* pLayerName,
4902 uint32_t* pPropertyCount,
4903 VkExtensionProperties* pProperties)
4904 {
4905 VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
4906
4907 for (int i = 0; i < RADV_INSTANCE_EXTENSION_COUNT; i++) {
4908 if (radv_supported_instance_extensions.extensions[i]) {
4909 vk_outarray_append(&out, prop) {
4910 *prop = radv_instance_extensions[i];
4911 }
4912 }
4913 }
4914
4915 return vk_outarray_status(&out);
4916 }
4917
4918 VkResult radv_EnumerateDeviceExtensionProperties(
4919 VkPhysicalDevice physicalDevice,
4920 const char* pLayerName,
4921 uint32_t* pPropertyCount,
4922 VkExtensionProperties* pProperties)
4923 {
4924 RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
4925 VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
4926
4927 for (int i = 0; i < RADV_DEVICE_EXTENSION_COUNT; i++) {
4928 if (device->supported_extensions.extensions[i]) {
4929 vk_outarray_append(&out, prop) {
4930 *prop = radv_device_extensions[i];
4931 }
4932 }
4933 }
4934
4935 return vk_outarray_status(&out);
4936 }
4937
4938 PFN_vkVoidFunction radv_GetInstanceProcAddr(
4939 VkInstance _instance,
4940 const char* pName)
4941 {
4942 RADV_FROM_HANDLE(radv_instance, instance, _instance);
4943
4944 /* The Vulkan 1.0 spec for vkGetInstanceProcAddr has a table of exactly
4945 * when we have to return valid function pointers, NULL, or it's left
4946 * undefined. See the table for exact details.
4947 */
4948 if (pName == NULL)
4949 return NULL;
4950
4951 #define LOOKUP_RADV_ENTRYPOINT(entrypoint) \
4952 if (strcmp(pName, "vk" #entrypoint) == 0) \
4953 return (PFN_vkVoidFunction)radv_##entrypoint
4954
4955 LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceExtensionProperties);
4956 LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceLayerProperties);
4957 LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceVersion);
4958 LOOKUP_RADV_ENTRYPOINT(CreateInstance);
4959
4960 /* GetInstanceProcAddr() can also be called with a NULL instance.
4961 * See https://gitlab.khronos.org/vulkan/vulkan/issues/2057
4962 */
4963 LOOKUP_RADV_ENTRYPOINT(GetInstanceProcAddr);
4964
4965 #undef LOOKUP_RADV_ENTRYPOINT
4966
4967 if (instance == NULL)
4968 return NULL;
4969
4970 int idx = radv_get_instance_entrypoint_index(pName);
4971 if (idx >= 0)
4972 return instance->dispatch.entrypoints[idx];
4973
4974 idx = radv_get_physical_device_entrypoint_index(pName);
4975 if (idx >= 0)
4976 return instance->physical_device_dispatch.entrypoints[idx];
4977
4978 idx = radv_get_device_entrypoint_index(pName);
4979 if (idx >= 0)
4980 return instance->device_dispatch.entrypoints[idx];
4981
4982 return NULL;
4983 }
4984
4985 /* The loader wants us to expose a second GetInstanceProcAddr function
4986 * to work around certain LD_PRELOAD issues seen in apps.
4987 */
4988 PUBLIC
4989 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
4990 VkInstance instance,
4991 const char* pName);
4992
4993 PUBLIC
4994 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
4995 VkInstance instance,
4996 const char* pName)
4997 {
4998 return radv_GetInstanceProcAddr(instance, pName);
4999 }
5000
5001 PUBLIC
5002 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetPhysicalDeviceProcAddr(
5003 VkInstance _instance,
5004 const char* pName);
5005
5006 PUBLIC
5007 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetPhysicalDeviceProcAddr(
5008 VkInstance _instance,
5009 const char* pName)
5010 {
5011 RADV_FROM_HANDLE(radv_instance, instance, _instance);
5012
5013 if (!pName || !instance)
5014 return NULL;
5015
5016 int idx = radv_get_physical_device_entrypoint_index(pName);
5017 if (idx < 0)
5018 return NULL;
5019
5020 return instance->physical_device_dispatch.entrypoints[idx];
5021 }
5022
5023 PFN_vkVoidFunction radv_GetDeviceProcAddr(
5024 VkDevice _device,
5025 const char* pName)
5026 {
5027 RADV_FROM_HANDLE(radv_device, device, _device);
5028
5029 if (!device || !pName)
5030 return NULL;
5031
5032 int idx = radv_get_device_entrypoint_index(pName);
5033 if (idx < 0)
5034 return NULL;
5035
5036 return device->dispatch.entrypoints[idx];
5037 }
5038
5039 bool radv_get_memory_fd(struct radv_device *device,
5040 struct radv_device_memory *memory,
5041 int *pFD)
5042 {
5043 struct radeon_bo_metadata metadata;
5044
5045 if (memory->image) {
5046 if (memory->image->tiling != VK_IMAGE_TILING_LINEAR)
5047 radv_init_metadata(device, memory->image, &metadata);
5048 device->ws->buffer_set_metadata(memory->bo, &metadata);
5049 }
5050
5051 return device->ws->buffer_get_fd(device->ws, memory->bo,
5052 pFD);
5053 }
5054
5055
5056 static void radv_free_memory(struct radv_device *device,
5057 const VkAllocationCallbacks* pAllocator,
5058 struct radv_device_memory *mem)
5059 {
5060 if (mem == NULL)
5061 return;
5062
5063 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
5064 if (mem->android_hardware_buffer)
5065 AHardwareBuffer_release(mem->android_hardware_buffer);
5066 #endif
5067
5068 if (mem->bo) {
5069 if (device->overallocation_disallowed) {
5070 mtx_lock(&device->overallocation_mutex);
5071 device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
5072 mtx_unlock(&device->overallocation_mutex);
5073 }
5074
5075 radv_bo_list_remove(device, mem->bo);
5076 device->ws->buffer_destroy(mem->bo);
5077 mem->bo = NULL;
5078 }
5079
5080 vk_free2(&device->alloc, pAllocator, mem);
5081 }
5082
5083 static VkResult radv_alloc_memory(struct radv_device *device,
5084 const VkMemoryAllocateInfo* pAllocateInfo,
5085 const VkAllocationCallbacks* pAllocator,
5086 VkDeviceMemory* pMem)
5087 {
5088 struct radv_device_memory *mem;
5089 VkResult result;
5090 enum radeon_bo_domain domain;
5091 uint32_t flags = 0;
5092
5093 assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
5094
5095 const VkImportMemoryFdInfoKHR *import_info =
5096 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
5097 const VkMemoryDedicatedAllocateInfo *dedicate_info =
5098 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
5099 const VkExportMemoryAllocateInfo *export_info =
5100 vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
5101 const struct VkImportAndroidHardwareBufferInfoANDROID *ahb_import_info =
5102 vk_find_struct_const(pAllocateInfo->pNext,
5103 IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID);
5104 const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
5105 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
5106
5107 const struct wsi_memory_allocate_info *wsi_info =
5108 vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
5109
5110 if (pAllocateInfo->allocationSize == 0 && !ahb_import_info &&
5111 !(export_info && (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID))) {
5112 /* Apparently, this is allowed */
5113 *pMem = VK_NULL_HANDLE;
5114 return VK_SUCCESS;
5115 }
5116
5117 mem = vk_zalloc2(&device->alloc, pAllocator, sizeof(*mem), 8,
5118 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5119 if (mem == NULL)
5120 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5121
5122 if (wsi_info && wsi_info->implicit_sync)
5123 flags |= RADEON_FLAG_IMPLICIT_SYNC;
5124
5125 if (dedicate_info) {
5126 mem->image = radv_image_from_handle(dedicate_info->image);
5127 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
5128 } else {
5129 mem->image = NULL;
5130 mem->buffer = NULL;
5131 }
5132
5133 float priority_float = 0.5;
5134 const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
5135 vk_find_struct_const(pAllocateInfo->pNext,
5136 MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
5137 if (priority_ext)
5138 priority_float = priority_ext->priority;
5139
5140 unsigned priority = MIN2(RADV_BO_PRIORITY_APPLICATION_MAX - 1,
5141 (int)(priority_float * RADV_BO_PRIORITY_APPLICATION_MAX));
5142
5143 mem->user_ptr = NULL;
5144 mem->bo = NULL;
5145
5146 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
5147 mem->android_hardware_buffer = NULL;
5148 #endif
5149
5150 if (ahb_import_info) {
5151 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
5152 if (result != VK_SUCCESS)
5153 goto fail;
5154 } else if(export_info && (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID)) {
5155 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
5156 if (result != VK_SUCCESS)
5157 goto fail;
5158 } else if (import_info) {
5159 assert(import_info->handleType ==
5160 VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
5161 import_info->handleType ==
5162 VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
5163 mem->bo = device->ws->buffer_from_fd(device->ws, import_info->fd,
5164 priority, NULL);
5165 if (!mem->bo) {
5166 result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
5167 goto fail;
5168 } else {
5169 close(import_info->fd);
5170 }
5171 } else if (host_ptr_info) {
5172 assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
5173 mem->bo = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer,
5174 pAllocateInfo->allocationSize,
5175 priority);
5176 if (!mem->bo) {
5177 result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
5178 goto fail;
5179 } else {
5180 mem->user_ptr = host_ptr_info->pHostPointer;
5181 }
5182 } else {
5183 uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
5184 uint32_t heap_index;
5185
5186 heap_index = device->physical_device->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
5187 domain = device->physical_device->memory_domains[pAllocateInfo->memoryTypeIndex];
5188 flags = device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex];
5189
5190 if (!dedicate_info && !import_info && (!export_info || !export_info->handleTypes)) {
5191 flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
5192 if (device->use_global_bo_list) {
5193 flags |= RADEON_FLAG_PREFER_LOCAL_BO;
5194 }
5195 }
5196
5197 if (device->overallocation_disallowed) {
5198 uint64_t total_size =
5199 device->physical_device->memory_properties.memoryHeaps[heap_index].size;
5200
5201 mtx_lock(&device->overallocation_mutex);
5202 if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
5203 mtx_unlock(&device->overallocation_mutex);
5204 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
5205 goto fail;
5206 }
5207 device->allocated_memory_size[heap_index] += alloc_size;
5208 mtx_unlock(&device->overallocation_mutex);
5209 }
5210
5211 mem->bo = device->ws->buffer_create(device->ws, alloc_size, device->physical_device->rad_info.max_alignment,
5212 domain, flags, priority);
5213
5214 if (!mem->bo) {
5215 if (device->overallocation_disallowed) {
5216 mtx_lock(&device->overallocation_mutex);
5217 device->allocated_memory_size[heap_index] -= alloc_size;
5218 mtx_unlock(&device->overallocation_mutex);
5219 }
5220 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
5221 goto fail;
5222 }
5223
5224 mem->heap_index = heap_index;
5225 mem->alloc_size = alloc_size;
5226 }
5227
5228 if (!wsi_info) {
5229 result = radv_bo_list_add(device, mem->bo);
5230 if (result != VK_SUCCESS)
5231 goto fail;
5232 }
5233
5234 *pMem = radv_device_memory_to_handle(mem);
5235
5236 return VK_SUCCESS;
5237
5238 fail:
5239 radv_free_memory(device, pAllocator,mem);
5240
5241 return result;
5242 }
5243
5244 VkResult radv_AllocateMemory(
5245 VkDevice _device,
5246 const VkMemoryAllocateInfo* pAllocateInfo,
5247 const VkAllocationCallbacks* pAllocator,
5248 VkDeviceMemory* pMem)
5249 {
5250 RADV_FROM_HANDLE(radv_device, device, _device);
5251 return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem);
5252 }
5253
5254 void radv_FreeMemory(
5255 VkDevice _device,
5256 VkDeviceMemory _mem,
5257 const VkAllocationCallbacks* pAllocator)
5258 {
5259 RADV_FROM_HANDLE(radv_device, device, _device);
5260 RADV_FROM_HANDLE(radv_device_memory, mem, _mem);
5261
5262 radv_free_memory(device, pAllocator, mem);
5263 }
5264
5265 VkResult radv_MapMemory(
5266 VkDevice _device,
5267 VkDeviceMemory _memory,
5268 VkDeviceSize offset,
5269 VkDeviceSize size,
5270 VkMemoryMapFlags flags,
5271 void** ppData)
5272 {
5273 RADV_FROM_HANDLE(radv_device, device, _device);
5274 RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
5275
5276 if (mem == NULL) {
5277 *ppData = NULL;
5278 return VK_SUCCESS;
5279 }
5280
5281 if (mem->user_ptr)
5282 *ppData = mem->user_ptr;
5283 else
5284 *ppData = device->ws->buffer_map(mem->bo);
5285
5286 if (*ppData) {
5287 *ppData += offset;
5288 return VK_SUCCESS;
5289 }
5290
5291 return vk_error(device->instance, VK_ERROR_MEMORY_MAP_FAILED);
5292 }
5293
5294 void radv_UnmapMemory(
5295 VkDevice _device,
5296 VkDeviceMemory _memory)
5297 {
5298 RADV_FROM_HANDLE(radv_device, device, _device);
5299 RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
5300
5301 if (mem == NULL)
5302 return;
5303
5304 if (mem->user_ptr == NULL)
5305 device->ws->buffer_unmap(mem->bo);
5306 }
5307
5308 VkResult radv_FlushMappedMemoryRanges(
5309 VkDevice _device,
5310 uint32_t memoryRangeCount,
5311 const VkMappedMemoryRange* pMemoryRanges)
5312 {
5313 return VK_SUCCESS;
5314 }
5315
5316 VkResult radv_InvalidateMappedMemoryRanges(
5317 VkDevice _device,
5318 uint32_t memoryRangeCount,
5319 const VkMappedMemoryRange* pMemoryRanges)
5320 {
5321 return VK_SUCCESS;
5322 }
5323
5324 void radv_GetBufferMemoryRequirements(
5325 VkDevice _device,
5326 VkBuffer _buffer,
5327 VkMemoryRequirements* pMemoryRequirements)
5328 {
5329 RADV_FROM_HANDLE(radv_device, device, _device);
5330 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
5331
5332 pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
5333
5334 if (buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)
5335 pMemoryRequirements->alignment = 4096;
5336 else
5337 pMemoryRequirements->alignment = 16;
5338
5339 pMemoryRequirements->size = align64(buffer->size, pMemoryRequirements->alignment);
5340 }
5341
5342 void radv_GetBufferMemoryRequirements2(
5343 VkDevice device,
5344 const VkBufferMemoryRequirementsInfo2 *pInfo,
5345 VkMemoryRequirements2 *pMemoryRequirements)
5346 {
5347 radv_GetBufferMemoryRequirements(device, pInfo->buffer,
5348 &pMemoryRequirements->memoryRequirements);
5349 vk_foreach_struct(ext, pMemoryRequirements->pNext) {
5350 switch (ext->sType) {
5351 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
5352 VkMemoryDedicatedRequirements *req =
5353 (VkMemoryDedicatedRequirements *) ext;
5354 req->requiresDedicatedAllocation = false;
5355 req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
5356 break;
5357 }
5358 default:
5359 break;
5360 }
5361 }
5362 }
5363
5364 void radv_GetImageMemoryRequirements(
5365 VkDevice _device,
5366 VkImage _image,
5367 VkMemoryRequirements* pMemoryRequirements)
5368 {
5369 RADV_FROM_HANDLE(radv_device, device, _device);
5370 RADV_FROM_HANDLE(radv_image, image, _image);
5371
5372 pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
5373
5374 pMemoryRequirements->size = image->size;
5375 pMemoryRequirements->alignment = image->alignment;
5376 }
5377
5378 void radv_GetImageMemoryRequirements2(
5379 VkDevice device,
5380 const VkImageMemoryRequirementsInfo2 *pInfo,
5381 VkMemoryRequirements2 *pMemoryRequirements)
5382 {
5383 radv_GetImageMemoryRequirements(device, pInfo->image,
5384 &pMemoryRequirements->memoryRequirements);
5385
5386 RADV_FROM_HANDLE(radv_image, image, pInfo->image);
5387
5388 vk_foreach_struct(ext, pMemoryRequirements->pNext) {
5389 switch (ext->sType) {
5390 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
5391 VkMemoryDedicatedRequirements *req =
5392 (VkMemoryDedicatedRequirements *) ext;
5393 req->requiresDedicatedAllocation = image->shareable &&
5394 image->tiling != VK_IMAGE_TILING_LINEAR;
5395 req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
5396 break;
5397 }
5398 default:
5399 break;
5400 }
5401 }
5402 }
5403
5404 void radv_GetImageSparseMemoryRequirements(
5405 VkDevice device,
5406 VkImage image,
5407 uint32_t* pSparseMemoryRequirementCount,
5408 VkSparseImageMemoryRequirements* pSparseMemoryRequirements)
5409 {
5410 stub();
5411 }
5412
5413 void radv_GetImageSparseMemoryRequirements2(
5414 VkDevice device,
5415 const VkImageSparseMemoryRequirementsInfo2 *pInfo,
5416 uint32_t* pSparseMemoryRequirementCount,
5417 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
5418 {
5419 stub();
5420 }
5421
5422 void radv_GetDeviceMemoryCommitment(
5423 VkDevice device,
5424 VkDeviceMemory memory,
5425 VkDeviceSize* pCommittedMemoryInBytes)
5426 {
5427 *pCommittedMemoryInBytes = 0;
5428 }
5429
5430 VkResult radv_BindBufferMemory2(VkDevice device,
5431 uint32_t bindInfoCount,
5432 const VkBindBufferMemoryInfo *pBindInfos)
5433 {
5434 for (uint32_t i = 0; i < bindInfoCount; ++i) {
5435 RADV_FROM_HANDLE(radv_device_memory, mem, pBindInfos[i].memory);
5436 RADV_FROM_HANDLE(radv_buffer, buffer, pBindInfos[i].buffer);
5437
5438 if (mem) {
5439 buffer->bo = mem->bo;
5440 buffer->offset = pBindInfos[i].memoryOffset;
5441 } else {
5442 buffer->bo = NULL;
5443 }
5444 }
5445 return VK_SUCCESS;
5446 }
5447
5448 VkResult radv_BindBufferMemory(
5449 VkDevice device,
5450 VkBuffer buffer,
5451 VkDeviceMemory memory,
5452 VkDeviceSize memoryOffset)
5453 {
5454 const VkBindBufferMemoryInfo info = {
5455 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
5456 .buffer = buffer,
5457 .memory = memory,
5458 .memoryOffset = memoryOffset
5459 };
5460
5461 return radv_BindBufferMemory2(device, 1, &info);
5462 }
5463
5464 VkResult radv_BindImageMemory2(VkDevice device,
5465 uint32_t bindInfoCount,
5466 const VkBindImageMemoryInfo *pBindInfos)
5467 {
5468 for (uint32_t i = 0; i < bindInfoCount; ++i) {
5469 RADV_FROM_HANDLE(radv_device_memory, mem, pBindInfos[i].memory);
5470 RADV_FROM_HANDLE(radv_image, image, pBindInfos[i].image);
5471
5472 if (mem) {
5473 image->bo = mem->bo;
5474 image->offset = pBindInfos[i].memoryOffset;
5475 } else {
5476 image->bo = NULL;
5477 image->offset = 0;
5478 }
5479 }
5480 return VK_SUCCESS;
5481 }
5482
5483
5484 VkResult radv_BindImageMemory(
5485 VkDevice device,
5486 VkImage image,
5487 VkDeviceMemory memory,
5488 VkDeviceSize memoryOffset)
5489 {
5490 const VkBindImageMemoryInfo info = {
5491 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
5492 .image = image,
5493 .memory = memory,
5494 .memoryOffset = memoryOffset
5495 };
5496
5497 return radv_BindImageMemory2(device, 1, &info);
5498 }
5499
5500 static bool radv_sparse_bind_has_effects(const VkBindSparseInfo *info)
5501 {
5502 return info->bufferBindCount ||
5503 info->imageOpaqueBindCount ||
5504 info->imageBindCount ||
5505 info->waitSemaphoreCount ||
5506 info->signalSemaphoreCount;
5507 }
5508
5509 VkResult radv_QueueBindSparse(
5510 VkQueue _queue,
5511 uint32_t bindInfoCount,
5512 const VkBindSparseInfo* pBindInfo,
5513 VkFence fence)
5514 {
5515 RADV_FROM_HANDLE(radv_queue, queue, _queue);
5516 VkResult result;
5517 uint32_t fence_idx = 0;
5518
5519 if (fence != VK_NULL_HANDLE) {
5520 for (uint32_t i = 0; i < bindInfoCount; ++i)
5521 if (radv_sparse_bind_has_effects(pBindInfo + i))
5522 fence_idx = i;
5523 } else
5524 fence_idx = UINT32_MAX;
5525
5526 for (uint32_t i = 0; i < bindInfoCount; ++i) {
5527 if (i != fence_idx && !radv_sparse_bind_has_effects(pBindInfo + i))
5528 continue;
5529
5530 const VkTimelineSemaphoreSubmitInfo *timeline_info =
5531 vk_find_struct_const(pBindInfo[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
5532
5533 VkResult result = radv_queue_submit(queue, &(struct radv_queue_submission) {
5534 .buffer_binds = pBindInfo[i].pBufferBinds,
5535 .buffer_bind_count = pBindInfo[i].bufferBindCount,
5536 .image_opaque_binds = pBindInfo[i].pImageOpaqueBinds,
5537 .image_opaque_bind_count = pBindInfo[i].imageOpaqueBindCount,
5538 .wait_semaphores = pBindInfo[i].pWaitSemaphores,
5539 .wait_semaphore_count = pBindInfo[i].waitSemaphoreCount,
5540 .signal_semaphores = pBindInfo[i].pSignalSemaphores,
5541 .signal_semaphore_count = pBindInfo[i].signalSemaphoreCount,
5542 .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
5543 .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
5544 .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
5545 .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
5546 .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
5547 });
5548
5549 if (result != VK_SUCCESS)
5550 return result;
5551 }
5552
5553 if (fence != VK_NULL_HANDLE && !bindInfoCount) {
5554 result = radv_signal_fence(queue, fence);
5555 if (result != VK_SUCCESS)
5556 return result;
5557 }
5558
5559 return VK_SUCCESS;
5560 }
5561
5562 VkResult radv_CreateFence(
5563 VkDevice _device,
5564 const VkFenceCreateInfo* pCreateInfo,
5565 const VkAllocationCallbacks* pAllocator,
5566 VkFence* pFence)
5567 {
5568 RADV_FROM_HANDLE(radv_device, device, _device);
5569 const VkExportFenceCreateInfo *export =
5570 vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
5571 VkExternalFenceHandleTypeFlags handleTypes =
5572 export ? export->handleTypes : 0;
5573
5574 struct radv_fence *fence = vk_alloc2(&device->alloc, pAllocator,
5575 sizeof(*fence), 8,
5576 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5577
5578 if (!fence)
5579 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5580
5581 fence->fence_wsi = NULL;
5582 fence->temp_syncobj = 0;
5583 if (device->always_use_syncobj || handleTypes) {
5584 int ret = device->ws->create_syncobj(device->ws, &fence->syncobj);
5585 if (ret) {
5586 vk_free2(&device->alloc, pAllocator, fence);
5587 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5588 }
5589 if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) {
5590 device->ws->signal_syncobj(device->ws, fence->syncobj);
5591 }
5592 fence->fence = NULL;
5593 } else {
5594 fence->fence = device->ws->create_fence();
5595 if (!fence->fence) {
5596 vk_free2(&device->alloc, pAllocator, fence);
5597 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5598 }
5599 fence->syncobj = 0;
5600 if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT)
5601 device->ws->signal_fence(fence->fence);
5602 }
5603
5604 *pFence = radv_fence_to_handle(fence);
5605
5606 return VK_SUCCESS;
5607 }
5608
5609 void radv_DestroyFence(
5610 VkDevice _device,
5611 VkFence _fence,
5612 const VkAllocationCallbacks* pAllocator)
5613 {
5614 RADV_FROM_HANDLE(radv_device, device, _device);
5615 RADV_FROM_HANDLE(radv_fence, fence, _fence);
5616
5617 if (!fence)
5618 return;
5619
5620 if (fence->temp_syncobj)
5621 device->ws->destroy_syncobj(device->ws, fence->temp_syncobj);
5622 if (fence->syncobj)
5623 device->ws->destroy_syncobj(device->ws, fence->syncobj);
5624 if (fence->fence)
5625 device->ws->destroy_fence(fence->fence);
5626 if (fence->fence_wsi)
5627 fence->fence_wsi->destroy(fence->fence_wsi);
5628 vk_free2(&device->alloc, pAllocator, fence);
5629 }
5630
5631
5632 uint64_t radv_get_current_time(void)
5633 {
5634 struct timespec tv;
5635 clock_gettime(CLOCK_MONOTONIC, &tv);
5636 return tv.tv_nsec + tv.tv_sec*1000000000ull;
5637 }
5638
5639 static uint64_t radv_get_absolute_timeout(uint64_t timeout)
5640 {
5641 uint64_t current_time = radv_get_current_time();
5642
5643 timeout = MIN2(UINT64_MAX - current_time, timeout);
5644
5645 return current_time + timeout;
5646 }
5647
5648
5649 static bool radv_all_fences_plain_and_submitted(struct radv_device *device,
5650 uint32_t fenceCount, const VkFence *pFences)
5651 {
5652 for (uint32_t i = 0; i < fenceCount; ++i) {
5653 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5654 if (fence->fence == NULL || fence->syncobj ||
5655 fence->temp_syncobj || fence->fence_wsi ||
5656 (!device->ws->is_fence_waitable(fence->fence)))
5657 return false;
5658 }
5659 return true;
5660 }
5661
5662 static bool radv_all_fences_syncobj(uint32_t fenceCount, const VkFence *pFences)
5663 {
5664 for (uint32_t i = 0; i < fenceCount; ++i) {
5665 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5666 if (fence->syncobj == 0 && fence->temp_syncobj == 0)
5667 return false;
5668 }
5669 return true;
5670 }
5671
5672 VkResult radv_WaitForFences(
5673 VkDevice _device,
5674 uint32_t fenceCount,
5675 const VkFence* pFences,
5676 VkBool32 waitAll,
5677 uint64_t timeout)
5678 {
5679 RADV_FROM_HANDLE(radv_device, device, _device);
5680 timeout = radv_get_absolute_timeout(timeout);
5681
5682 if (device->always_use_syncobj &&
5683 radv_all_fences_syncobj(fenceCount, pFences))
5684 {
5685 uint32_t *handles = malloc(sizeof(uint32_t) * fenceCount);
5686 if (!handles)
5687 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5688
5689 for (uint32_t i = 0; i < fenceCount; ++i) {
5690 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5691 handles[i] = fence->temp_syncobj ? fence->temp_syncobj : fence->syncobj;
5692 }
5693
5694 bool success = device->ws->wait_syncobj(device->ws, handles, fenceCount, waitAll, timeout);
5695
5696 free(handles);
5697 return success ? VK_SUCCESS : VK_TIMEOUT;
5698 }
5699
5700 if (!waitAll && fenceCount > 1) {
5701 /* Not doing this by default for waitAll, due to needing to allocate twice. */
5702 if (device->physical_device->rad_info.drm_minor >= 10 && radv_all_fences_plain_and_submitted(device, fenceCount, pFences)) {
5703 uint32_t wait_count = 0;
5704 struct radeon_winsys_fence **fences = malloc(sizeof(struct radeon_winsys_fence *) * fenceCount);
5705 if (!fences)
5706 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5707
5708 for (uint32_t i = 0; i < fenceCount; ++i) {
5709 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5710
5711 if (device->ws->fence_wait(device->ws, fence->fence, false, 0)) {
5712 free(fences);
5713 return VK_SUCCESS;
5714 }
5715
5716 fences[wait_count++] = fence->fence;
5717 }
5718
5719 bool success = device->ws->fences_wait(device->ws, fences, wait_count,
5720 waitAll, timeout - radv_get_current_time());
5721
5722 free(fences);
5723 return success ? VK_SUCCESS : VK_TIMEOUT;
5724 }
5725
5726 while(radv_get_current_time() <= timeout) {
5727 for (uint32_t i = 0; i < fenceCount; ++i) {
5728 if (radv_GetFenceStatus(_device, pFences[i]) == VK_SUCCESS)
5729 return VK_SUCCESS;
5730 }
5731 }
5732 return VK_TIMEOUT;
5733 }
5734
5735 for (uint32_t i = 0; i < fenceCount; ++i) {
5736 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5737 bool expired = false;
5738
5739 if (fence->temp_syncobj) {
5740 if (!device->ws->wait_syncobj(device->ws, &fence->temp_syncobj, 1, true, timeout))
5741 return VK_TIMEOUT;
5742 continue;
5743 }
5744
5745 if (fence->syncobj) {
5746 if (!device->ws->wait_syncobj(device->ws, &fence->syncobj, 1, true, timeout))
5747 return VK_TIMEOUT;
5748 continue;
5749 }
5750
5751 if (fence->fence) {
5752 if (!device->ws->is_fence_waitable(fence->fence)) {
5753 while(!device->ws->is_fence_waitable(fence->fence) &&
5754 radv_get_current_time() <= timeout)
5755 /* Do nothing */;
5756 }
5757
5758 expired = device->ws->fence_wait(device->ws,
5759 fence->fence,
5760 true, timeout);
5761 if (!expired)
5762 return VK_TIMEOUT;
5763 }
5764
5765 if (fence->fence_wsi) {
5766 VkResult result = fence->fence_wsi->wait(fence->fence_wsi, timeout);
5767 if (result != VK_SUCCESS)
5768 return result;
5769 }
5770 }
5771
5772 return VK_SUCCESS;
5773 }
5774
5775 VkResult radv_ResetFences(VkDevice _device,
5776 uint32_t fenceCount,
5777 const VkFence *pFences)
5778 {
5779 RADV_FROM_HANDLE(radv_device, device, _device);
5780
5781 for (unsigned i = 0; i < fenceCount; ++i) {
5782 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5783 if (fence->fence)
5784 device->ws->reset_fence(fence->fence);
5785
5786 /* Per spec, we first restore the permanent payload, and then reset, so
5787 * having a temp syncobj should not skip resetting the permanent syncobj. */
5788 if (fence->temp_syncobj) {
5789 device->ws->destroy_syncobj(device->ws, fence->temp_syncobj);
5790 fence->temp_syncobj = 0;
5791 }
5792
5793 if (fence->syncobj) {
5794 device->ws->reset_syncobj(device->ws, fence->syncobj);
5795 }
5796 }
5797
5798 return VK_SUCCESS;
5799 }
5800
5801 VkResult radv_GetFenceStatus(VkDevice _device, VkFence _fence)
5802 {
5803 RADV_FROM_HANDLE(radv_device, device, _device);
5804 RADV_FROM_HANDLE(radv_fence, fence, _fence);
5805
5806 if (fence->temp_syncobj) {
5807 bool success = device->ws->wait_syncobj(device->ws, &fence->temp_syncobj, 1, true, 0);
5808 return success ? VK_SUCCESS : VK_NOT_READY;
5809 }
5810
5811 if (fence->syncobj) {
5812 bool success = device->ws->wait_syncobj(device->ws, &fence->syncobj, 1, true, 0);
5813 return success ? VK_SUCCESS : VK_NOT_READY;
5814 }
5815
5816 if (fence->fence) {
5817 if (!device->ws->fence_wait(device->ws, fence->fence, false, 0))
5818 return VK_NOT_READY;
5819 }
5820 if (fence->fence_wsi) {
5821 VkResult result = fence->fence_wsi->wait(fence->fence_wsi, 0);
5822
5823 if (result != VK_SUCCESS) {
5824 if (result == VK_TIMEOUT)
5825 return VK_NOT_READY;
5826 return result;
5827 }
5828 }
5829 return VK_SUCCESS;
5830 }
5831
5832
5833 // Queue semaphore functions
5834
5835 static void
5836 radv_create_timeline(struct radv_timeline *timeline, uint64_t value)
5837 {
5838 timeline->highest_signaled = value;
5839 timeline->highest_submitted = value;
5840 list_inithead(&timeline->points);
5841 list_inithead(&timeline->free_points);
5842 list_inithead(&timeline->waiters);
5843 pthread_mutex_init(&timeline->mutex, NULL);
5844 }
5845
5846 static void
5847 radv_destroy_timeline(struct radv_device *device,
5848 struct radv_timeline *timeline)
5849 {
5850 list_for_each_entry_safe(struct radv_timeline_point, point,
5851 &timeline->free_points, list) {
5852 list_del(&point->list);
5853 device->ws->destroy_syncobj(device->ws, point->syncobj);
5854 free(point);
5855 }
5856 list_for_each_entry_safe(struct radv_timeline_point, point,
5857 &timeline->points, list) {
5858 list_del(&point->list);
5859 device->ws->destroy_syncobj(device->ws, point->syncobj);
5860 free(point);
5861 }
5862 pthread_mutex_destroy(&timeline->mutex);
5863 }
5864
5865 static void
5866 radv_timeline_gc_locked(struct radv_device *device,
5867 struct radv_timeline *timeline)
5868 {
5869 list_for_each_entry_safe(struct radv_timeline_point, point,
5870 &timeline->points, list) {
5871 if (point->wait_count || point->value > timeline->highest_submitted)
5872 return;
5873
5874 if (device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, 0)) {
5875 timeline->highest_signaled = point->value;
5876 list_del(&point->list);
5877 list_add(&point->list, &timeline->free_points);
5878 }
5879 }
5880 }
5881
5882 static struct radv_timeline_point *
5883 radv_timeline_find_point_at_least_locked(struct radv_device *device,
5884 struct radv_timeline *timeline,
5885 uint64_t p)
5886 {
5887 radv_timeline_gc_locked(device, timeline);
5888
5889 if (p <= timeline->highest_signaled)
5890 return NULL;
5891
5892 list_for_each_entry(struct radv_timeline_point, point,
5893 &timeline->points, list) {
5894 if (point->value >= p) {
5895 ++point->wait_count;
5896 return point;
5897 }
5898 }
5899 return NULL;
5900 }
5901
5902 static struct radv_timeline_point *
5903 radv_timeline_add_point_locked(struct radv_device *device,
5904 struct radv_timeline *timeline,
5905 uint64_t p)
5906 {
5907 radv_timeline_gc_locked(device, timeline);
5908
5909 struct radv_timeline_point *ret = NULL;
5910 struct radv_timeline_point *prev = NULL;
5911
5912 if (p <= timeline->highest_signaled)
5913 return NULL;
5914
5915 list_for_each_entry(struct radv_timeline_point, point,
5916 &timeline->points, list) {
5917 if (point->value == p) {
5918 return NULL;
5919 }
5920
5921 if (point->value < p)
5922 prev = point;
5923 }
5924
5925 if (list_is_empty(&timeline->free_points)) {
5926 ret = malloc(sizeof(struct radv_timeline_point));
5927 device->ws->create_syncobj(device->ws, &ret->syncobj);
5928 } else {
5929 ret = list_first_entry(&timeline->free_points, struct radv_timeline_point, list);
5930 list_del(&ret->list);
5931
5932 device->ws->reset_syncobj(device->ws, ret->syncobj);
5933 }
5934
5935 ret->value = p;
5936 ret->wait_count = 1;
5937
5938 if (prev) {
5939 list_add(&ret->list, &prev->list);
5940 } else {
5941 list_addtail(&ret->list, &timeline->points);
5942 }
5943 return ret;
5944 }
5945
5946
5947 static VkResult
5948 radv_timeline_wait_locked(struct radv_device *device,
5949 struct radv_timeline *timeline,
5950 uint64_t value,
5951 uint64_t abs_timeout)
5952 {
5953 while(timeline->highest_submitted < value) {
5954 struct timespec abstime;
5955 timespec_from_nsec(&abstime, abs_timeout);
5956
5957 pthread_cond_timedwait(&device->timeline_cond, &timeline->mutex, &abstime);
5958
5959 if (radv_get_current_time() >= abs_timeout && timeline->highest_submitted < value)
5960 return VK_TIMEOUT;
5961 }
5962
5963 struct radv_timeline_point *point = radv_timeline_find_point_at_least_locked(device, timeline, value);
5964 if (!point)
5965 return VK_SUCCESS;
5966
5967 pthread_mutex_unlock(&timeline->mutex);
5968
5969 bool success = device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, abs_timeout);
5970
5971 pthread_mutex_lock(&timeline->mutex);
5972 point->wait_count--;
5973 return success ? VK_SUCCESS : VK_TIMEOUT;
5974 }
5975
5976 static void
5977 radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline,
5978 struct list_head *processing_list)
5979 {
5980 list_for_each_entry_safe(struct radv_timeline_waiter, waiter,
5981 &timeline->waiters, list) {
5982 if (waiter->value > timeline->highest_submitted)
5983 continue;
5984
5985 if (p_atomic_dec_zero(&waiter->submission->submission_wait_count)) {
5986 list_addtail(&waiter->submission->processing_list, processing_list);
5987 }
5988 list_del(&waiter->list);
5989 }
5990 }
5991
5992 static
5993 void radv_destroy_semaphore_part(struct radv_device *device,
5994 struct radv_semaphore_part *part)
5995 {
5996 switch(part->kind) {
5997 case RADV_SEMAPHORE_NONE:
5998 break;
5999 case RADV_SEMAPHORE_WINSYS:
6000 device->ws->destroy_sem(part->ws_sem);
6001 break;
6002 case RADV_SEMAPHORE_TIMELINE:
6003 radv_destroy_timeline(device, &part->timeline);
6004 break;
6005 case RADV_SEMAPHORE_SYNCOBJ:
6006 device->ws->destroy_syncobj(device->ws, part->syncobj);
6007 break;
6008 }
6009 part->kind = RADV_SEMAPHORE_NONE;
6010 }
6011
6012 static VkSemaphoreTypeKHR
6013 radv_get_semaphore_type(const void *pNext, uint64_t *initial_value)
6014 {
6015 const VkSemaphoreTypeCreateInfo *type_info =
6016 vk_find_struct_const(pNext, SEMAPHORE_TYPE_CREATE_INFO);
6017
6018 if (!type_info)
6019 return VK_SEMAPHORE_TYPE_BINARY;
6020
6021 if (initial_value)
6022 *initial_value = type_info->initialValue;
6023 return type_info->semaphoreType;
6024 }
6025
6026 VkResult radv_CreateSemaphore(
6027 VkDevice _device,
6028 const VkSemaphoreCreateInfo* pCreateInfo,
6029 const VkAllocationCallbacks* pAllocator,
6030 VkSemaphore* pSemaphore)
6031 {
6032 RADV_FROM_HANDLE(radv_device, device, _device);
6033 const VkExportSemaphoreCreateInfo *export =
6034 vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO);
6035 VkExternalSemaphoreHandleTypeFlags handleTypes =
6036 export ? export->handleTypes : 0;
6037 uint64_t initial_value = 0;
6038 VkSemaphoreTypeKHR type = radv_get_semaphore_type(pCreateInfo->pNext, &initial_value);
6039
6040 struct radv_semaphore *sem = vk_alloc2(&device->alloc, pAllocator,
6041 sizeof(*sem), 8,
6042 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6043 if (!sem)
6044 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6045
6046 sem->temporary.kind = RADV_SEMAPHORE_NONE;
6047 sem->permanent.kind = RADV_SEMAPHORE_NONE;
6048
6049 if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
6050 radv_create_timeline(&sem->permanent.timeline, initial_value);
6051 sem->permanent.kind = RADV_SEMAPHORE_TIMELINE;
6052 } else if (device->always_use_syncobj || handleTypes) {
6053 assert (device->physical_device->rad_info.has_syncobj);
6054 int ret = device->ws->create_syncobj(device->ws, &sem->permanent.syncobj);
6055 if (ret) {
6056 vk_free2(&device->alloc, pAllocator, sem);
6057 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6058 }
6059 sem->permanent.kind = RADV_SEMAPHORE_SYNCOBJ;
6060 } else {
6061 sem->permanent.ws_sem = device->ws->create_sem(device->ws);
6062 if (!sem->permanent.ws_sem) {
6063 vk_free2(&device->alloc, pAllocator, sem);
6064 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6065 }
6066 sem->permanent.kind = RADV_SEMAPHORE_WINSYS;
6067 }
6068
6069 *pSemaphore = radv_semaphore_to_handle(sem);
6070 return VK_SUCCESS;
6071 }
6072
6073 void radv_DestroySemaphore(
6074 VkDevice _device,
6075 VkSemaphore _semaphore,
6076 const VkAllocationCallbacks* pAllocator)
6077 {
6078 RADV_FROM_HANDLE(radv_device, device, _device);
6079 RADV_FROM_HANDLE(radv_semaphore, sem, _semaphore);
6080 if (!_semaphore)
6081 return;
6082
6083 radv_destroy_semaphore_part(device, &sem->temporary);
6084 radv_destroy_semaphore_part(device, &sem->permanent);
6085 vk_free2(&device->alloc, pAllocator, sem);
6086 }
6087
6088 VkResult
6089 radv_GetSemaphoreCounterValue(VkDevice _device,
6090 VkSemaphore _semaphore,
6091 uint64_t* pValue)
6092 {
6093 RADV_FROM_HANDLE(radv_device, device, _device);
6094 RADV_FROM_HANDLE(radv_semaphore, semaphore, _semaphore);
6095
6096 struct radv_semaphore_part *part =
6097 semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
6098
6099 switch (part->kind) {
6100 case RADV_SEMAPHORE_TIMELINE: {
6101 pthread_mutex_lock(&part->timeline.mutex);
6102 radv_timeline_gc_locked(device, &part->timeline);
6103 *pValue = part->timeline.highest_signaled;
6104 pthread_mutex_unlock(&part->timeline.mutex);
6105 return VK_SUCCESS;
6106 }
6107 case RADV_SEMAPHORE_NONE:
6108 case RADV_SEMAPHORE_SYNCOBJ:
6109 case RADV_SEMAPHORE_WINSYS:
6110 unreachable("Invalid semaphore type");
6111 }
6112 unreachable("Unhandled semaphore type");
6113 }
6114
6115
6116 static VkResult
6117 radv_wait_timelines(struct radv_device *device,
6118 const VkSemaphoreWaitInfo* pWaitInfo,
6119 uint64_t abs_timeout)
6120 {
6121 if ((pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR) && pWaitInfo->semaphoreCount > 1) {
6122 for (;;) {
6123 for(uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6124 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6125 pthread_mutex_lock(&semaphore->permanent.timeline.mutex);
6126 VkResult result = radv_timeline_wait_locked(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], 0);
6127 pthread_mutex_unlock(&semaphore->permanent.timeline.mutex);
6128
6129 if (result == VK_SUCCESS)
6130 return VK_SUCCESS;
6131 }
6132 if (radv_get_current_time() > abs_timeout)
6133 return VK_TIMEOUT;
6134 }
6135 }
6136
6137 for(uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6138 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6139 pthread_mutex_lock(&semaphore->permanent.timeline.mutex);
6140 VkResult result = radv_timeline_wait_locked(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], abs_timeout);
6141 pthread_mutex_unlock(&semaphore->permanent.timeline.mutex);
6142
6143 if (result != VK_SUCCESS)
6144 return result;
6145 }
6146 return VK_SUCCESS;
6147 }
6148 VkResult
6149 radv_WaitSemaphores(VkDevice _device,
6150 const VkSemaphoreWaitInfo* pWaitInfo,
6151 uint64_t timeout)
6152 {
6153 RADV_FROM_HANDLE(radv_device, device, _device);
6154 uint64_t abs_timeout = radv_get_absolute_timeout(timeout);
6155 return radv_wait_timelines(device, pWaitInfo, abs_timeout);
6156 }
6157
6158 VkResult
6159 radv_SignalSemaphore(VkDevice _device,
6160 const VkSemaphoreSignalInfo* pSignalInfo)
6161 {
6162 RADV_FROM_HANDLE(radv_device, device, _device);
6163 RADV_FROM_HANDLE(radv_semaphore, semaphore, pSignalInfo->semaphore);
6164
6165 struct radv_semaphore_part *part =
6166 semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
6167
6168 switch(part->kind) {
6169 case RADV_SEMAPHORE_TIMELINE: {
6170 pthread_mutex_lock(&part->timeline.mutex);
6171 radv_timeline_gc_locked(device, &part->timeline);
6172 part->timeline.highest_submitted = MAX2(part->timeline.highest_submitted, pSignalInfo->value);
6173 part->timeline.highest_signaled = MAX2(part->timeline.highest_signaled, pSignalInfo->value);
6174
6175 struct list_head processing_list;
6176 list_inithead(&processing_list);
6177 radv_timeline_trigger_waiters_locked(&part->timeline, &processing_list);
6178 pthread_mutex_unlock(&part->timeline.mutex);
6179
6180 return radv_process_submissions(&processing_list);
6181 }
6182 case RADV_SEMAPHORE_NONE:
6183 case RADV_SEMAPHORE_SYNCOBJ:
6184 case RADV_SEMAPHORE_WINSYS:
6185 unreachable("Invalid semaphore type");
6186 }
6187 return VK_SUCCESS;
6188 }
6189
6190
6191
6192 VkResult radv_CreateEvent(
6193 VkDevice _device,
6194 const VkEventCreateInfo* pCreateInfo,
6195 const VkAllocationCallbacks* pAllocator,
6196 VkEvent* pEvent)
6197 {
6198 RADV_FROM_HANDLE(radv_device, device, _device);
6199 struct radv_event *event = vk_alloc2(&device->alloc, pAllocator,
6200 sizeof(*event), 8,
6201 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6202
6203 if (!event)
6204 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6205
6206 event->bo = device->ws->buffer_create(device->ws, 8, 8,
6207 RADEON_DOMAIN_GTT,
6208 RADEON_FLAG_VA_UNCACHED | RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING,
6209 RADV_BO_PRIORITY_FENCE);
6210 if (!event->bo) {
6211 vk_free2(&device->alloc, pAllocator, event);
6212 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6213 }
6214
6215 event->map = (uint64_t*)device->ws->buffer_map(event->bo);
6216
6217 *pEvent = radv_event_to_handle(event);
6218
6219 return VK_SUCCESS;
6220 }
6221
6222 void radv_DestroyEvent(
6223 VkDevice _device,
6224 VkEvent _event,
6225 const VkAllocationCallbacks* pAllocator)
6226 {
6227 RADV_FROM_HANDLE(radv_device, device, _device);
6228 RADV_FROM_HANDLE(radv_event, event, _event);
6229
6230 if (!event)
6231 return;
6232 device->ws->buffer_destroy(event->bo);
6233 vk_free2(&device->alloc, pAllocator, event);
6234 }
6235
6236 VkResult radv_GetEventStatus(
6237 VkDevice _device,
6238 VkEvent _event)
6239 {
6240 RADV_FROM_HANDLE(radv_event, event, _event);
6241
6242 if (*event->map == 1)
6243 return VK_EVENT_SET;
6244 return VK_EVENT_RESET;
6245 }
6246
6247 VkResult radv_SetEvent(
6248 VkDevice _device,
6249 VkEvent _event)
6250 {
6251 RADV_FROM_HANDLE(radv_event, event, _event);
6252 *event->map = 1;
6253
6254 return VK_SUCCESS;
6255 }
6256
6257 VkResult radv_ResetEvent(
6258 VkDevice _device,
6259 VkEvent _event)
6260 {
6261 RADV_FROM_HANDLE(radv_event, event, _event);
6262 *event->map = 0;
6263
6264 return VK_SUCCESS;
6265 }
6266
6267 VkResult radv_CreateBuffer(
6268 VkDevice _device,
6269 const VkBufferCreateInfo* pCreateInfo,
6270 const VkAllocationCallbacks* pAllocator,
6271 VkBuffer* pBuffer)
6272 {
6273 RADV_FROM_HANDLE(radv_device, device, _device);
6274 struct radv_buffer *buffer;
6275
6276 if (pCreateInfo->size > RADV_MAX_MEMORY_ALLOCATION_SIZE)
6277 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6278
6279 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
6280
6281 buffer = vk_alloc2(&device->alloc, pAllocator, sizeof(*buffer), 8,
6282 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6283 if (buffer == NULL)
6284 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6285
6286 buffer->size = pCreateInfo->size;
6287 buffer->usage = pCreateInfo->usage;
6288 buffer->bo = NULL;
6289 buffer->offset = 0;
6290 buffer->flags = pCreateInfo->flags;
6291
6292 buffer->shareable = vk_find_struct_const(pCreateInfo->pNext,
6293 EXTERNAL_MEMORY_BUFFER_CREATE_INFO) != NULL;
6294
6295 if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
6296 buffer->bo = device->ws->buffer_create(device->ws,
6297 align64(buffer->size, 4096),
6298 4096, 0, RADEON_FLAG_VIRTUAL,
6299 RADV_BO_PRIORITY_VIRTUAL);
6300 if (!buffer->bo) {
6301 vk_free2(&device->alloc, pAllocator, buffer);
6302 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6303 }
6304 }
6305
6306 *pBuffer = radv_buffer_to_handle(buffer);
6307
6308 return VK_SUCCESS;
6309 }
6310
6311 void radv_DestroyBuffer(
6312 VkDevice _device,
6313 VkBuffer _buffer,
6314 const VkAllocationCallbacks* pAllocator)
6315 {
6316 RADV_FROM_HANDLE(radv_device, device, _device);
6317 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
6318
6319 if (!buffer)
6320 return;
6321
6322 if (buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)
6323 device->ws->buffer_destroy(buffer->bo);
6324
6325 vk_free2(&device->alloc, pAllocator, buffer);
6326 }
6327
6328 VkDeviceAddress radv_GetBufferDeviceAddress(
6329 VkDevice device,
6330 const VkBufferDeviceAddressInfo* pInfo)
6331 {
6332 RADV_FROM_HANDLE(radv_buffer, buffer, pInfo->buffer);
6333 return radv_buffer_get_va(buffer->bo) + buffer->offset;
6334 }
6335
6336
6337 uint64_t radv_GetBufferOpaqueCaptureAddress(VkDevice device,
6338 const VkBufferDeviceAddressInfo* pInfo)
6339 {
6340 return 0;
6341 }
6342
6343 uint64_t radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,
6344 const VkDeviceMemoryOpaqueCaptureAddressInfo* pInfo)
6345 {
6346 return 0;
6347 }
6348
6349 static inline unsigned
6350 si_tile_mode_index(const struct radv_image_plane *plane, unsigned level, bool stencil)
6351 {
6352 if (stencil)
6353 return plane->surface.u.legacy.stencil_tiling_index[level];
6354 else
6355 return plane->surface.u.legacy.tiling_index[level];
6356 }
6357
6358 static uint32_t radv_surface_max_layer_count(struct radv_image_view *iview)
6359 {
6360 return iview->type == VK_IMAGE_VIEW_TYPE_3D ? iview->extent.depth : (iview->base_layer + iview->layer_count);
6361 }
6362
6363 static uint32_t
6364 radv_init_dcc_control_reg(struct radv_device *device,
6365 struct radv_image_view *iview)
6366 {
6367 unsigned max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_256B;
6368 unsigned min_compressed_block_size = V_028C78_MIN_BLOCK_SIZE_32B;
6369 unsigned max_compressed_block_size;
6370 unsigned independent_128b_blocks;
6371 unsigned independent_64b_blocks;
6372
6373 if (!radv_dcc_enabled(iview->image, iview->base_mip))
6374 return 0;
6375
6376 if (!device->physical_device->rad_info.has_dedicated_vram) {
6377 /* amdvlk: [min-compressed-block-size] should be set to 32 for
6378 * dGPU and 64 for APU because all of our APUs to date use
6379 * DIMMs which have a request granularity size of 64B while all
6380 * other chips have a 32B request size.
6381 */
6382 min_compressed_block_size = V_028C78_MIN_BLOCK_SIZE_64B;
6383 }
6384
6385 if (device->physical_device->rad_info.chip_class >= GFX10) {
6386 max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B;
6387 independent_64b_blocks = 0;
6388 independent_128b_blocks = 1;
6389 } else {
6390 independent_128b_blocks = 0;
6391
6392 if (iview->image->info.samples > 1) {
6393 if (iview->image->planes[0].surface.bpe == 1)
6394 max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B;
6395 else if (iview->image->planes[0].surface.bpe == 2)
6396 max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B;
6397 }
6398
6399 if (iview->image->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
6400 VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
6401 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) {
6402 /* If this DCC image is potentially going to be used in texture
6403 * fetches, we need some special settings.
6404 */
6405 independent_64b_blocks = 1;
6406 max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B;
6407 } else {
6408 /* MAX_UNCOMPRESSED_BLOCK_SIZE must be >=
6409 * MAX_COMPRESSED_BLOCK_SIZE. Set MAX_COMPRESSED_BLOCK_SIZE as
6410 * big as possible for better compression state.
6411 */
6412 independent_64b_blocks = 0;
6413 max_compressed_block_size = max_uncompressed_block_size;
6414 }
6415 }
6416
6417 return S_028C78_MAX_UNCOMPRESSED_BLOCK_SIZE(max_uncompressed_block_size) |
6418 S_028C78_MAX_COMPRESSED_BLOCK_SIZE(max_compressed_block_size) |
6419 S_028C78_MIN_COMPRESSED_BLOCK_SIZE(min_compressed_block_size) |
6420 S_028C78_INDEPENDENT_64B_BLOCKS(independent_64b_blocks) |
6421 S_028C78_INDEPENDENT_128B_BLOCKS(independent_128b_blocks);
6422 }
6423
6424 void
6425 radv_initialise_color_surface(struct radv_device *device,
6426 struct radv_color_buffer_info *cb,
6427 struct radv_image_view *iview)
6428 {
6429 const struct vk_format_description *desc;
6430 unsigned ntype, format, swap, endian;
6431 unsigned blend_clamp = 0, blend_bypass = 0;
6432 uint64_t va;
6433 const struct radv_image_plane *plane = &iview->image->planes[iview->plane_id];
6434 const struct radeon_surf *surf = &plane->surface;
6435
6436 desc = vk_format_description(iview->vk_format);
6437
6438 memset(cb, 0, sizeof(*cb));
6439
6440 /* Intensity is implemented as Red, so treat it that way. */
6441 cb->cb_color_attrib = S_028C74_FORCE_DST_ALPHA_1(desc->swizzle[3] == VK_SWIZZLE_1);
6442
6443 va = radv_buffer_get_va(iview->bo) + iview->image->offset + plane->offset;
6444
6445 cb->cb_color_base = va >> 8;
6446
6447 if (device->physical_device->rad_info.chip_class >= GFX9) {
6448 struct gfx9_surf_meta_flags meta;
6449 if (iview->image->dcc_offset)
6450 meta = surf->u.gfx9.dcc;
6451 else
6452 meta = surf->u.gfx9.cmask;
6453
6454 if (device->physical_device->rad_info.chip_class >= GFX10) {
6455 cb->cb_color_attrib3 |= S_028EE0_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6456 S_028EE0_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) |
6457 S_028EE0_CMASK_PIPE_ALIGNED(surf->u.gfx9.cmask.pipe_aligned) |
6458 S_028EE0_DCC_PIPE_ALIGNED(surf->u.gfx9.dcc.pipe_aligned);
6459 } else {
6460 cb->cb_color_attrib |= S_028C74_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6461 S_028C74_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) |
6462 S_028C74_RB_ALIGNED(meta.rb_aligned) |
6463 S_028C74_PIPE_ALIGNED(meta.pipe_aligned);
6464 cb->cb_mrt_epitch = S_0287A0_EPITCH(surf->u.gfx9.surf.epitch);
6465 }
6466
6467 cb->cb_color_base += surf->u.gfx9.surf_offset >> 8;
6468 cb->cb_color_base |= surf->tile_swizzle;
6469 } else {
6470 const struct legacy_surf_level *level_info = &surf->u.legacy.level[iview->base_mip];
6471 unsigned pitch_tile_max, slice_tile_max, tile_mode_index;
6472
6473 cb->cb_color_base += level_info->offset >> 8;
6474 if (level_info->mode == RADEON_SURF_MODE_2D)
6475 cb->cb_color_base |= surf->tile_swizzle;
6476
6477 pitch_tile_max = level_info->nblk_x / 8 - 1;
6478 slice_tile_max = (level_info->nblk_x * level_info->nblk_y) / 64 - 1;
6479 tile_mode_index = si_tile_mode_index(plane, iview->base_mip, false);
6480
6481 cb->cb_color_pitch = S_028C64_TILE_MAX(pitch_tile_max);
6482 cb->cb_color_slice = S_028C68_TILE_MAX(slice_tile_max);
6483 cb->cb_color_cmask_slice = surf->u.legacy.cmask_slice_tile_max;
6484
6485 cb->cb_color_attrib |= S_028C74_TILE_MODE_INDEX(tile_mode_index);
6486
6487 if (radv_image_has_fmask(iview->image)) {
6488 if (device->physical_device->rad_info.chip_class >= GFX7)
6489 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(surf->u.legacy.fmask.pitch_in_pixels / 8 - 1);
6490 cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(surf->u.legacy.fmask.tiling_index);
6491 cb->cb_color_fmask_slice = S_028C88_TILE_MAX(surf->u.legacy.fmask.slice_tile_max);
6492 } else {
6493 /* This must be set for fast clear to work without FMASK. */
6494 if (device->physical_device->rad_info.chip_class >= GFX7)
6495 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(pitch_tile_max);
6496 cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(tile_mode_index);
6497 cb->cb_color_fmask_slice = S_028C88_TILE_MAX(slice_tile_max);
6498 }
6499 }
6500
6501 /* CMASK variables */
6502 va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6503 va += iview->image->cmask_offset;
6504 cb->cb_color_cmask = va >> 8;
6505
6506 va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6507 va += iview->image->dcc_offset;
6508
6509 if (radv_dcc_enabled(iview->image, iview->base_mip) &&
6510 device->physical_device->rad_info.chip_class <= GFX8)
6511 va += plane->surface.u.legacy.level[iview->base_mip].dcc_offset;
6512
6513 unsigned dcc_tile_swizzle = surf->tile_swizzle;
6514 dcc_tile_swizzle &= (surf->dcc_alignment - 1) >> 8;
6515
6516 cb->cb_dcc_base = va >> 8;
6517 cb->cb_dcc_base |= dcc_tile_swizzle;
6518
6519 /* GFX10 field has the same base shift as the GFX6 field. */
6520 uint32_t max_slice = radv_surface_max_layer_count(iview) - 1;
6521 cb->cb_color_view = S_028C6C_SLICE_START(iview->base_layer) |
6522 S_028C6C_SLICE_MAX_GFX10(max_slice);
6523
6524 if (iview->image->info.samples > 1) {
6525 unsigned log_samples = util_logbase2(iview->image->info.samples);
6526
6527 cb->cb_color_attrib |= S_028C74_NUM_SAMPLES(log_samples) |
6528 S_028C74_NUM_FRAGMENTS(log_samples);
6529 }
6530
6531 if (radv_image_has_fmask(iview->image)) {
6532 va = radv_buffer_get_va(iview->bo) + iview->image->offset + iview->image->fmask_offset;
6533 cb->cb_color_fmask = va >> 8;
6534 cb->cb_color_fmask |= surf->fmask_tile_swizzle;
6535 } else {
6536 cb->cb_color_fmask = cb->cb_color_base;
6537 }
6538
6539 ntype = radv_translate_color_numformat(iview->vk_format,
6540 desc,
6541 vk_format_get_first_non_void_channel(iview->vk_format));
6542 format = radv_translate_colorformat(iview->vk_format);
6543 if (format == V_028C70_COLOR_INVALID || ntype == ~0u)
6544 radv_finishme("Illegal color\n");
6545 swap = radv_translate_colorswap(iview->vk_format, false);
6546 endian = radv_colorformat_endian_swap(format);
6547
6548 /* blend clamp should be set for all NORM/SRGB types */
6549 if (ntype == V_028C70_NUMBER_UNORM ||
6550 ntype == V_028C70_NUMBER_SNORM ||
6551 ntype == V_028C70_NUMBER_SRGB)
6552 blend_clamp = 1;
6553
6554 /* set blend bypass according to docs if SINT/UINT or
6555 8/24 COLOR variants */
6556 if (ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT ||
6557 format == V_028C70_COLOR_8_24 || format == V_028C70_COLOR_24_8 ||
6558 format == V_028C70_COLOR_X24_8_32_FLOAT) {
6559 blend_clamp = 0;
6560 blend_bypass = 1;
6561 }
6562 #if 0
6563 if ((ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT) &&
6564 (format == V_028C70_COLOR_8 ||
6565 format == V_028C70_COLOR_8_8 ||
6566 format == V_028C70_COLOR_8_8_8_8))
6567 ->color_is_int8 = true;
6568 #endif
6569 cb->cb_color_info = S_028C70_FORMAT(format) |
6570 S_028C70_COMP_SWAP(swap) |
6571 S_028C70_BLEND_CLAMP(blend_clamp) |
6572 S_028C70_BLEND_BYPASS(blend_bypass) |
6573 S_028C70_SIMPLE_FLOAT(1) |
6574 S_028C70_ROUND_MODE(ntype != V_028C70_NUMBER_UNORM &&
6575 ntype != V_028C70_NUMBER_SNORM &&
6576 ntype != V_028C70_NUMBER_SRGB &&
6577 format != V_028C70_COLOR_8_24 &&
6578 format != V_028C70_COLOR_24_8) |
6579 S_028C70_NUMBER_TYPE(ntype) |
6580 S_028C70_ENDIAN(endian);
6581 if (radv_image_has_fmask(iview->image)) {
6582 cb->cb_color_info |= S_028C70_COMPRESSION(1);
6583 if (device->physical_device->rad_info.chip_class == GFX6) {
6584 unsigned fmask_bankh = util_logbase2(surf->u.legacy.fmask.bankh);
6585 cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(fmask_bankh);
6586 }
6587
6588 if (radv_image_is_tc_compat_cmask(iview->image)) {
6589 /* Allow the texture block to read FMASK directly
6590 * without decompressing it. This bit must be cleared
6591 * when performing FMASK_DECOMPRESS or DCC_COMPRESS,
6592 * otherwise the operation doesn't happen.
6593 */
6594 cb->cb_color_info |= S_028C70_FMASK_COMPRESS_1FRAG_ONLY(1);
6595
6596 /* Set CMASK into a tiling format that allows the
6597 * texture block to read it.
6598 */
6599 cb->cb_color_info |= S_028C70_CMASK_ADDR_TYPE(2);
6600 }
6601 }
6602
6603 if (radv_image_has_cmask(iview->image) &&
6604 !(device->instance->debug_flags & RADV_DEBUG_NO_FAST_CLEARS))
6605 cb->cb_color_info |= S_028C70_FAST_CLEAR(1);
6606
6607 if (radv_dcc_enabled(iview->image, iview->base_mip))
6608 cb->cb_color_info |= S_028C70_DCC_ENABLE(1);
6609
6610 cb->cb_dcc_control = radv_init_dcc_control_reg(device, iview);
6611
6612 /* This must be set for fast clear to work without FMASK. */
6613 if (!radv_image_has_fmask(iview->image) &&
6614 device->physical_device->rad_info.chip_class == GFX6) {
6615 unsigned bankh = util_logbase2(surf->u.legacy.bankh);
6616 cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(bankh);
6617 }
6618
6619 if (device->physical_device->rad_info.chip_class >= GFX9) {
6620 const struct vk_format_description *format_desc = vk_format_description(iview->image->vk_format);
6621
6622 unsigned mip0_depth = iview->image->type == VK_IMAGE_TYPE_3D ?
6623 (iview->extent.depth - 1) : (iview->image->info.array_size - 1);
6624 unsigned width = iview->extent.width / (iview->plane_id ? format_desc->width_divisor : 1);
6625 unsigned height = iview->extent.height / (iview->plane_id ? format_desc->height_divisor : 1);
6626
6627 if (device->physical_device->rad_info.chip_class >= GFX10) {
6628 cb->cb_color_view |= S_028C6C_MIP_LEVEL_GFX10(iview->base_mip);
6629
6630 cb->cb_color_attrib3 |= S_028EE0_MIP0_DEPTH(mip0_depth) |
6631 S_028EE0_RESOURCE_TYPE(surf->u.gfx9.resource_type) |
6632 S_028EE0_RESOURCE_LEVEL(1);
6633 } else {
6634 cb->cb_color_view |= S_028C6C_MIP_LEVEL_GFX9(iview->base_mip);
6635 cb->cb_color_attrib |= S_028C74_MIP0_DEPTH(mip0_depth) |
6636 S_028C74_RESOURCE_TYPE(surf->u.gfx9.resource_type);
6637 }
6638
6639 cb->cb_color_attrib2 = S_028C68_MIP0_WIDTH(width - 1) |
6640 S_028C68_MIP0_HEIGHT(height - 1) |
6641 S_028C68_MAX_MIP(iview->image->info.levels - 1);
6642 }
6643 }
6644
6645 static unsigned
6646 radv_calc_decompress_on_z_planes(struct radv_device *device,
6647 struct radv_image_view *iview)
6648 {
6649 unsigned max_zplanes = 0;
6650
6651 assert(radv_image_is_tc_compat_htile(iview->image));
6652
6653 if (device->physical_device->rad_info.chip_class >= GFX9) {
6654 /* Default value for 32-bit depth surfaces. */
6655 max_zplanes = 4;
6656
6657 if (iview->vk_format == VK_FORMAT_D16_UNORM &&
6658 iview->image->info.samples > 1)
6659 max_zplanes = 2;
6660
6661 max_zplanes = max_zplanes + 1;
6662 } else {
6663 if (iview->vk_format == VK_FORMAT_D16_UNORM) {
6664 /* Do not enable Z plane compression for 16-bit depth
6665 * surfaces because isn't supported on GFX8. Only
6666 * 32-bit depth surfaces are supported by the hardware.
6667 * This allows to maintain shader compatibility and to
6668 * reduce the number of depth decompressions.
6669 */
6670 max_zplanes = 1;
6671 } else {
6672 if (iview->image->info.samples <= 1)
6673 max_zplanes = 5;
6674 else if (iview->image->info.samples <= 4)
6675 max_zplanes = 3;
6676 else
6677 max_zplanes = 2;
6678 }
6679 }
6680
6681 return max_zplanes;
6682 }
6683
6684 void
6685 radv_initialise_ds_surface(struct radv_device *device,
6686 struct radv_ds_buffer_info *ds,
6687 struct radv_image_view *iview)
6688 {
6689 unsigned level = iview->base_mip;
6690 unsigned format, stencil_format;
6691 uint64_t va, s_offs, z_offs;
6692 bool stencil_only = false;
6693 const struct radv_image_plane *plane = &iview->image->planes[0];
6694 const struct radeon_surf *surf = &plane->surface;
6695
6696 assert(vk_format_get_plane_count(iview->image->vk_format) == 1);
6697
6698 memset(ds, 0, sizeof(*ds));
6699 switch (iview->image->vk_format) {
6700 case VK_FORMAT_D24_UNORM_S8_UINT:
6701 case VK_FORMAT_X8_D24_UNORM_PACK32:
6702 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-24);
6703 ds->offset_scale = 2.0f;
6704 break;
6705 case VK_FORMAT_D16_UNORM:
6706 case VK_FORMAT_D16_UNORM_S8_UINT:
6707 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-16);
6708 ds->offset_scale = 4.0f;
6709 break;
6710 case VK_FORMAT_D32_SFLOAT:
6711 case VK_FORMAT_D32_SFLOAT_S8_UINT:
6712 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-23) |
6713 S_028B78_POLY_OFFSET_DB_IS_FLOAT_FMT(1);
6714 ds->offset_scale = 1.0f;
6715 break;
6716 case VK_FORMAT_S8_UINT:
6717 stencil_only = true;
6718 break;
6719 default:
6720 break;
6721 }
6722
6723 format = radv_translate_dbformat(iview->image->vk_format);
6724 stencil_format = surf->has_stencil ?
6725 V_028044_STENCIL_8 : V_028044_STENCIL_INVALID;
6726
6727 uint32_t max_slice = radv_surface_max_layer_count(iview) - 1;
6728 ds->db_depth_view = S_028008_SLICE_START(iview->base_layer) |
6729 S_028008_SLICE_MAX(max_slice);
6730 if (device->physical_device->rad_info.chip_class >= GFX10) {
6731 ds->db_depth_view |= S_028008_SLICE_START_HI(iview->base_layer >> 11) |
6732 S_028008_SLICE_MAX_HI(max_slice >> 11);
6733 }
6734
6735 ds->db_htile_data_base = 0;
6736 ds->db_htile_surface = 0;
6737
6738 va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6739 s_offs = z_offs = va;
6740
6741 if (device->physical_device->rad_info.chip_class >= GFX9) {
6742 assert(surf->u.gfx9.surf_offset == 0);
6743 s_offs += surf->u.gfx9.stencil_offset;
6744
6745 ds->db_z_info = S_028038_FORMAT(format) |
6746 S_028038_NUM_SAMPLES(util_logbase2(iview->image->info.samples)) |
6747 S_028038_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6748 S_028038_MAXMIP(iview->image->info.levels - 1) |
6749 S_028038_ZRANGE_PRECISION(1);
6750 ds->db_stencil_info = S_02803C_FORMAT(stencil_format) |
6751 S_02803C_SW_MODE(surf->u.gfx9.stencil.swizzle_mode);
6752
6753 if (device->physical_device->rad_info.chip_class == GFX9) {
6754 ds->db_z_info2 = S_028068_EPITCH(surf->u.gfx9.surf.epitch);
6755 ds->db_stencil_info2 = S_02806C_EPITCH(surf->u.gfx9.stencil.epitch);
6756 }
6757
6758 ds->db_depth_view |= S_028008_MIPID(level);
6759 ds->db_depth_size = S_02801C_X_MAX(iview->image->info.width - 1) |
6760 S_02801C_Y_MAX(iview->image->info.height - 1);
6761
6762 if (radv_htile_enabled(iview->image, level)) {
6763 ds->db_z_info |= S_028038_TILE_SURFACE_ENABLE(1);
6764
6765 if (radv_image_is_tc_compat_htile(iview->image)) {
6766 unsigned max_zplanes =
6767 radv_calc_decompress_on_z_planes(device, iview);
6768
6769 ds->db_z_info |= S_028038_DECOMPRESS_ON_N_ZPLANES(max_zplanes);
6770
6771 if (device->physical_device->rad_info.chip_class >= GFX10) {
6772 ds->db_z_info |= S_028040_ITERATE_FLUSH(1);
6773 ds->db_stencil_info |= S_028044_ITERATE_FLUSH(1);
6774 } else {
6775 ds->db_z_info |= S_028038_ITERATE_FLUSH(1);
6776 ds->db_stencil_info |= S_02803C_ITERATE_FLUSH(1);
6777 }
6778 }
6779
6780 if (!surf->has_stencil)
6781 /* Use all of the htile_buffer for depth if there's no stencil. */
6782 ds->db_stencil_info |= S_02803C_TILE_STENCIL_DISABLE(1);
6783 va = radv_buffer_get_va(iview->bo) + iview->image->offset +
6784 iview->image->htile_offset;
6785 ds->db_htile_data_base = va >> 8;
6786 ds->db_htile_surface = S_028ABC_FULL_CACHE(1) |
6787 S_028ABC_PIPE_ALIGNED(surf->u.gfx9.htile.pipe_aligned);
6788
6789 if (device->physical_device->rad_info.chip_class == GFX9) {
6790 ds->db_htile_surface |= S_028ABC_RB_ALIGNED(surf->u.gfx9.htile.rb_aligned);
6791 }
6792 }
6793 } else {
6794 const struct legacy_surf_level *level_info = &surf->u.legacy.level[level];
6795
6796 if (stencil_only)
6797 level_info = &surf->u.legacy.stencil_level[level];
6798
6799 z_offs += surf->u.legacy.level[level].offset;
6800 s_offs += surf->u.legacy.stencil_level[level].offset;
6801
6802 ds->db_depth_info = S_02803C_ADDR5_SWIZZLE_MASK(!radv_image_is_tc_compat_htile(iview->image));
6803 ds->db_z_info = S_028040_FORMAT(format) | S_028040_ZRANGE_PRECISION(1);
6804 ds->db_stencil_info = S_028044_FORMAT(stencil_format);
6805
6806 if (iview->image->info.samples > 1)
6807 ds->db_z_info |= S_028040_NUM_SAMPLES(util_logbase2(iview->image->info.samples));
6808
6809 if (device->physical_device->rad_info.chip_class >= GFX7) {
6810 struct radeon_info *info = &device->physical_device->rad_info;
6811 unsigned tiling_index = surf->u.legacy.tiling_index[level];
6812 unsigned stencil_index = surf->u.legacy.stencil_tiling_index[level];
6813 unsigned macro_index = surf->u.legacy.macro_tile_index;
6814 unsigned tile_mode = info->si_tile_mode_array[tiling_index];
6815 unsigned stencil_tile_mode = info->si_tile_mode_array[stencil_index];
6816 unsigned macro_mode = info->cik_macrotile_mode_array[macro_index];
6817
6818 if (stencil_only)
6819 tile_mode = stencil_tile_mode;
6820
6821 ds->db_depth_info |=
6822 S_02803C_ARRAY_MODE(G_009910_ARRAY_MODE(tile_mode)) |
6823 S_02803C_PIPE_CONFIG(G_009910_PIPE_CONFIG(tile_mode)) |
6824 S_02803C_BANK_WIDTH(G_009990_BANK_WIDTH(macro_mode)) |
6825 S_02803C_BANK_HEIGHT(G_009990_BANK_HEIGHT(macro_mode)) |
6826 S_02803C_MACRO_TILE_ASPECT(G_009990_MACRO_TILE_ASPECT(macro_mode)) |
6827 S_02803C_NUM_BANKS(G_009990_NUM_BANKS(macro_mode));
6828 ds->db_z_info |= S_028040_TILE_SPLIT(G_009910_TILE_SPLIT(tile_mode));
6829 ds->db_stencil_info |= S_028044_TILE_SPLIT(G_009910_TILE_SPLIT(stencil_tile_mode));
6830 } else {
6831 unsigned tile_mode_index = si_tile_mode_index(&iview->image->planes[0], level, false);
6832 ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
6833 tile_mode_index = si_tile_mode_index(&iview->image->planes[0], level, true);
6834 ds->db_stencil_info |= S_028044_TILE_MODE_INDEX(tile_mode_index);
6835 if (stencil_only)
6836 ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
6837 }
6838
6839 ds->db_depth_size = S_028058_PITCH_TILE_MAX((level_info->nblk_x / 8) - 1) |
6840 S_028058_HEIGHT_TILE_MAX((level_info->nblk_y / 8) - 1);
6841 ds->db_depth_slice = S_02805C_SLICE_TILE_MAX((level_info->nblk_x * level_info->nblk_y) / 64 - 1);
6842
6843 if (radv_htile_enabled(iview->image, level)) {
6844 ds->db_z_info |= S_028040_TILE_SURFACE_ENABLE(1);
6845
6846 if (!surf->has_stencil &&
6847 !radv_image_is_tc_compat_htile(iview->image))
6848 /* Use all of the htile_buffer for depth if there's no stencil. */
6849 ds->db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
6850
6851 va = radv_buffer_get_va(iview->bo) + iview->image->offset +
6852 iview->image->htile_offset;
6853 ds->db_htile_data_base = va >> 8;
6854 ds->db_htile_surface = S_028ABC_FULL_CACHE(1);
6855
6856 if (radv_image_is_tc_compat_htile(iview->image)) {
6857 unsigned max_zplanes =
6858 radv_calc_decompress_on_z_planes(device, iview);
6859
6860 ds->db_htile_surface |= S_028ABC_TC_COMPATIBLE(1);
6861 ds->db_z_info |= S_028040_DECOMPRESS_ON_N_ZPLANES(max_zplanes);
6862 }
6863 }
6864 }
6865
6866 ds->db_z_read_base = ds->db_z_write_base = z_offs >> 8;
6867 ds->db_stencil_read_base = ds->db_stencil_write_base = s_offs >> 8;
6868 }
6869
6870 VkResult radv_CreateFramebuffer(
6871 VkDevice _device,
6872 const VkFramebufferCreateInfo* pCreateInfo,
6873 const VkAllocationCallbacks* pAllocator,
6874 VkFramebuffer* pFramebuffer)
6875 {
6876 RADV_FROM_HANDLE(radv_device, device, _device);
6877 struct radv_framebuffer *framebuffer;
6878 const VkFramebufferAttachmentsCreateInfo *imageless_create_info =
6879 vk_find_struct_const(pCreateInfo->pNext,
6880 FRAMEBUFFER_ATTACHMENTS_CREATE_INFO);
6881
6882 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO);
6883
6884 size_t size = sizeof(*framebuffer);
6885 if (!imageless_create_info)
6886 size += sizeof(struct radv_image_view*) * pCreateInfo->attachmentCount;
6887 framebuffer = vk_alloc2(&device->alloc, pAllocator, size, 8,
6888 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6889 if (framebuffer == NULL)
6890 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6891
6892 framebuffer->attachment_count = pCreateInfo->attachmentCount;
6893 framebuffer->width = pCreateInfo->width;
6894 framebuffer->height = pCreateInfo->height;
6895 framebuffer->layers = pCreateInfo->layers;
6896 if (imageless_create_info) {
6897 for (unsigned i = 0; i < imageless_create_info->attachmentImageInfoCount; ++i) {
6898 const VkFramebufferAttachmentImageInfo *attachment =
6899 imageless_create_info->pAttachmentImageInfos + i;
6900 framebuffer->width = MIN2(framebuffer->width, attachment->width);
6901 framebuffer->height = MIN2(framebuffer->height, attachment->height);
6902 framebuffer->layers = MIN2(framebuffer->layers, attachment->layerCount);
6903 }
6904 } else {
6905 for (uint32_t i = 0; i < pCreateInfo->attachmentCount; i++) {
6906 VkImageView _iview = pCreateInfo->pAttachments[i];
6907 struct radv_image_view *iview = radv_image_view_from_handle(_iview);
6908 framebuffer->attachments[i] = iview;
6909 framebuffer->width = MIN2(framebuffer->width, iview->extent.width);
6910 framebuffer->height = MIN2(framebuffer->height, iview->extent.height);
6911 framebuffer->layers = MIN2(framebuffer->layers, radv_surface_max_layer_count(iview));
6912 }
6913 }
6914
6915 *pFramebuffer = radv_framebuffer_to_handle(framebuffer);
6916 return VK_SUCCESS;
6917 }
6918
6919 void radv_DestroyFramebuffer(
6920 VkDevice _device,
6921 VkFramebuffer _fb,
6922 const VkAllocationCallbacks* pAllocator)
6923 {
6924 RADV_FROM_HANDLE(radv_device, device, _device);
6925 RADV_FROM_HANDLE(radv_framebuffer, fb, _fb);
6926
6927 if (!fb)
6928 return;
6929 vk_free2(&device->alloc, pAllocator, fb);
6930 }
6931
6932 static unsigned radv_tex_wrap(VkSamplerAddressMode address_mode)
6933 {
6934 switch (address_mode) {
6935 case VK_SAMPLER_ADDRESS_MODE_REPEAT:
6936 return V_008F30_SQ_TEX_WRAP;
6937 case VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT:
6938 return V_008F30_SQ_TEX_MIRROR;
6939 case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE:
6940 return V_008F30_SQ_TEX_CLAMP_LAST_TEXEL;
6941 case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER:
6942 return V_008F30_SQ_TEX_CLAMP_BORDER;
6943 case VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE:
6944 return V_008F30_SQ_TEX_MIRROR_ONCE_LAST_TEXEL;
6945 default:
6946 unreachable("illegal tex wrap mode");
6947 break;
6948 }
6949 }
6950
6951 static unsigned
6952 radv_tex_compare(VkCompareOp op)
6953 {
6954 switch (op) {
6955 case VK_COMPARE_OP_NEVER:
6956 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
6957 case VK_COMPARE_OP_LESS:
6958 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESS;
6959 case VK_COMPARE_OP_EQUAL:
6960 return V_008F30_SQ_TEX_DEPTH_COMPARE_EQUAL;
6961 case VK_COMPARE_OP_LESS_OR_EQUAL:
6962 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESSEQUAL;
6963 case VK_COMPARE_OP_GREATER:
6964 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATER;
6965 case VK_COMPARE_OP_NOT_EQUAL:
6966 return V_008F30_SQ_TEX_DEPTH_COMPARE_NOTEQUAL;
6967 case VK_COMPARE_OP_GREATER_OR_EQUAL:
6968 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATEREQUAL;
6969 case VK_COMPARE_OP_ALWAYS:
6970 return V_008F30_SQ_TEX_DEPTH_COMPARE_ALWAYS;
6971 default:
6972 unreachable("illegal compare mode");
6973 break;
6974 }
6975 }
6976
6977 static unsigned
6978 radv_tex_filter(VkFilter filter, unsigned max_ansio)
6979 {
6980 switch (filter) {
6981 case VK_FILTER_NEAREST:
6982 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_POINT :
6983 V_008F38_SQ_TEX_XY_FILTER_POINT);
6984 case VK_FILTER_LINEAR:
6985 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_BILINEAR :
6986 V_008F38_SQ_TEX_XY_FILTER_BILINEAR);
6987 case VK_FILTER_CUBIC_IMG:
6988 default:
6989 fprintf(stderr, "illegal texture filter");
6990 return 0;
6991 }
6992 }
6993
6994 static unsigned
6995 radv_tex_mipfilter(VkSamplerMipmapMode mode)
6996 {
6997 switch (mode) {
6998 case VK_SAMPLER_MIPMAP_MODE_NEAREST:
6999 return V_008F38_SQ_TEX_Z_FILTER_POINT;
7000 case VK_SAMPLER_MIPMAP_MODE_LINEAR:
7001 return V_008F38_SQ_TEX_Z_FILTER_LINEAR;
7002 default:
7003 return V_008F38_SQ_TEX_Z_FILTER_NONE;
7004 }
7005 }
7006
7007 static unsigned
7008 radv_tex_bordercolor(VkBorderColor bcolor)
7009 {
7010 switch (bcolor) {
7011 case VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK:
7012 case VK_BORDER_COLOR_INT_TRANSPARENT_BLACK:
7013 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
7014 case VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK:
7015 case VK_BORDER_COLOR_INT_OPAQUE_BLACK:
7016 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_BLACK;
7017 case VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE:
7018 case VK_BORDER_COLOR_INT_OPAQUE_WHITE:
7019 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE;
7020 default:
7021 break;
7022 }
7023 return 0;
7024 }
7025
7026 static unsigned
7027 radv_tex_aniso_filter(unsigned filter)
7028 {
7029 if (filter < 2)
7030 return 0;
7031 if (filter < 4)
7032 return 1;
7033 if (filter < 8)
7034 return 2;
7035 if (filter < 16)
7036 return 3;
7037 return 4;
7038 }
7039
7040 static unsigned
7041 radv_tex_filter_mode(VkSamplerReductionMode mode)
7042 {
7043 switch (mode) {
7044 case VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT:
7045 return V_008F30_SQ_IMG_FILTER_MODE_BLEND;
7046 case VK_SAMPLER_REDUCTION_MODE_MIN_EXT:
7047 return V_008F30_SQ_IMG_FILTER_MODE_MIN;
7048 case VK_SAMPLER_REDUCTION_MODE_MAX_EXT:
7049 return V_008F30_SQ_IMG_FILTER_MODE_MAX;
7050 default:
7051 break;
7052 }
7053 return 0;
7054 }
7055
7056 static uint32_t
7057 radv_get_max_anisotropy(struct radv_device *device,
7058 const VkSamplerCreateInfo *pCreateInfo)
7059 {
7060 if (device->force_aniso >= 0)
7061 return device->force_aniso;
7062
7063 if (pCreateInfo->anisotropyEnable &&
7064 pCreateInfo->maxAnisotropy > 1.0f)
7065 return (uint32_t)pCreateInfo->maxAnisotropy;
7066
7067 return 0;
7068 }
7069
7070 static inline int S_FIXED(float value, unsigned frac_bits)
7071 {
7072 return value * (1 << frac_bits);
7073 }
7074
7075 static void
7076 radv_init_sampler(struct radv_device *device,
7077 struct radv_sampler *sampler,
7078 const VkSamplerCreateInfo *pCreateInfo)
7079 {
7080 uint32_t max_aniso = radv_get_max_anisotropy(device, pCreateInfo);
7081 uint32_t max_aniso_ratio = radv_tex_aniso_filter(max_aniso);
7082 bool compat_mode = device->physical_device->rad_info.chip_class == GFX8 ||
7083 device->physical_device->rad_info.chip_class == GFX9;
7084 unsigned filter_mode = V_008F30_SQ_IMG_FILTER_MODE_BLEND;
7085 unsigned depth_compare_func = V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
7086 bool trunc_coord = pCreateInfo->minFilter == VK_FILTER_NEAREST && pCreateInfo->magFilter == VK_FILTER_NEAREST;
7087
7088 const struct VkSamplerReductionModeCreateInfo *sampler_reduction =
7089 vk_find_struct_const(pCreateInfo->pNext,
7090 SAMPLER_REDUCTION_MODE_CREATE_INFO);
7091 if (sampler_reduction)
7092 filter_mode = radv_tex_filter_mode(sampler_reduction->reductionMode);
7093
7094 if (pCreateInfo->compareEnable)
7095 depth_compare_func = radv_tex_compare(pCreateInfo->compareOp);
7096
7097 sampler->state[0] = (S_008F30_CLAMP_X(radv_tex_wrap(pCreateInfo->addressModeU)) |
7098 S_008F30_CLAMP_Y(radv_tex_wrap(pCreateInfo->addressModeV)) |
7099 S_008F30_CLAMP_Z(radv_tex_wrap(pCreateInfo->addressModeW)) |
7100 S_008F30_MAX_ANISO_RATIO(max_aniso_ratio) |
7101 S_008F30_DEPTH_COMPARE_FUNC(depth_compare_func) |
7102 S_008F30_FORCE_UNNORMALIZED(pCreateInfo->unnormalizedCoordinates ? 1 : 0) |
7103 S_008F30_ANISO_THRESHOLD(max_aniso_ratio >> 1) |
7104 S_008F30_ANISO_BIAS(max_aniso_ratio) |
7105 S_008F30_DISABLE_CUBE_WRAP(0) |
7106 S_008F30_COMPAT_MODE(compat_mode) |
7107 S_008F30_FILTER_MODE(filter_mode) |
7108 S_008F30_TRUNC_COORD(trunc_coord));
7109 sampler->state[1] = (S_008F34_MIN_LOD(S_FIXED(CLAMP(pCreateInfo->minLod, 0, 15), 8)) |
7110 S_008F34_MAX_LOD(S_FIXED(CLAMP(pCreateInfo->maxLod, 0, 15), 8)) |
7111 S_008F34_PERF_MIP(max_aniso_ratio ? max_aniso_ratio + 6 : 0));
7112 sampler->state[2] = (S_008F38_LOD_BIAS(S_FIXED(CLAMP(pCreateInfo->mipLodBias, -16, 16), 8)) |
7113 S_008F38_XY_MAG_FILTER(radv_tex_filter(pCreateInfo->magFilter, max_aniso)) |
7114 S_008F38_XY_MIN_FILTER(radv_tex_filter(pCreateInfo->minFilter, max_aniso)) |
7115 S_008F38_MIP_FILTER(radv_tex_mipfilter(pCreateInfo->mipmapMode)) |
7116 S_008F38_MIP_POINT_PRECLAMP(0));
7117 sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(0) |
7118 S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(pCreateInfo->borderColor)));
7119
7120 if (device->physical_device->rad_info.chip_class >= GFX10) {
7121 sampler->state[2] |= S_008F38_ANISO_OVERRIDE_GFX10(1);
7122 } else {
7123 sampler->state[2] |=
7124 S_008F38_DISABLE_LSB_CEIL(device->physical_device->rad_info.chip_class <= GFX8) |
7125 S_008F38_FILTER_PREC_FIX(1) |
7126 S_008F38_ANISO_OVERRIDE_GFX6(device->physical_device->rad_info.chip_class >= GFX8);
7127 }
7128 }
7129
7130 VkResult radv_CreateSampler(
7131 VkDevice _device,
7132 const VkSamplerCreateInfo* pCreateInfo,
7133 const VkAllocationCallbacks* pAllocator,
7134 VkSampler* pSampler)
7135 {
7136 RADV_FROM_HANDLE(radv_device, device, _device);
7137 struct radv_sampler *sampler;
7138
7139 const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion =
7140 vk_find_struct_const(pCreateInfo->pNext,
7141 SAMPLER_YCBCR_CONVERSION_INFO);
7142
7143 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO);
7144
7145 sampler = vk_alloc2(&device->alloc, pAllocator, sizeof(*sampler), 8,
7146 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
7147 if (!sampler)
7148 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
7149
7150 radv_init_sampler(device, sampler, pCreateInfo);
7151
7152 sampler->ycbcr_sampler = ycbcr_conversion ? radv_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion): NULL;
7153 *pSampler = radv_sampler_to_handle(sampler);
7154
7155 return VK_SUCCESS;
7156 }
7157
7158 void radv_DestroySampler(
7159 VkDevice _device,
7160 VkSampler _sampler,
7161 const VkAllocationCallbacks* pAllocator)
7162 {
7163 RADV_FROM_HANDLE(radv_device, device, _device);
7164 RADV_FROM_HANDLE(radv_sampler, sampler, _sampler);
7165
7166 if (!sampler)
7167 return;
7168 vk_free2(&device->alloc, pAllocator, sampler);
7169 }
7170
7171 /* vk_icd.h does not declare this function, so we declare it here to
7172 * suppress Wmissing-prototypes.
7173 */
7174 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
7175 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion);
7176
7177 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
7178 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion)
7179 {
7180 /* For the full details on loader interface versioning, see
7181 * <https://github.com/KhronosGroup/Vulkan-LoaderAndValidationLayers/blob/master/loader/LoaderAndLayerInterface.md>.
7182 * What follows is a condensed summary, to help you navigate the large and
7183 * confusing official doc.
7184 *
7185 * - Loader interface v0 is incompatible with later versions. We don't
7186 * support it.
7187 *
7188 * - In loader interface v1:
7189 * - The first ICD entrypoint called by the loader is
7190 * vk_icdGetInstanceProcAddr(). The ICD must statically expose this
7191 * entrypoint.
7192 * - The ICD must statically expose no other Vulkan symbol unless it is
7193 * linked with -Bsymbolic.
7194 * - Each dispatchable Vulkan handle created by the ICD must be
7195 * a pointer to a struct whose first member is VK_LOADER_DATA. The
7196 * ICD must initialize VK_LOADER_DATA.loadMagic to ICD_LOADER_MAGIC.
7197 * - The loader implements vkCreate{PLATFORM}SurfaceKHR() and
7198 * vkDestroySurfaceKHR(). The ICD must be capable of working with
7199 * such loader-managed surfaces.
7200 *
7201 * - Loader interface v2 differs from v1 in:
7202 * - The first ICD entrypoint called by the loader is
7203 * vk_icdNegotiateLoaderICDInterfaceVersion(). The ICD must
7204 * statically expose this entrypoint.
7205 *
7206 * - Loader interface v3 differs from v2 in:
7207 * - The ICD must implement vkCreate{PLATFORM}SurfaceKHR(),
7208 * vkDestroySurfaceKHR(), and other API which uses VKSurfaceKHR,
7209 * because the loader no longer does so.
7210 */
7211 *pSupportedVersion = MIN2(*pSupportedVersion, 4u);
7212 return VK_SUCCESS;
7213 }
7214
7215 VkResult radv_GetMemoryFdKHR(VkDevice _device,
7216 const VkMemoryGetFdInfoKHR *pGetFdInfo,
7217 int *pFD)
7218 {
7219 RADV_FROM_HANDLE(radv_device, device, _device);
7220 RADV_FROM_HANDLE(radv_device_memory, memory, pGetFdInfo->memory);
7221
7222 assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR);
7223
7224 /* At the moment, we support only the below handle types. */
7225 assert(pGetFdInfo->handleType ==
7226 VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
7227 pGetFdInfo->handleType ==
7228 VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
7229
7230 bool ret = radv_get_memory_fd(device, memory, pFD);
7231 if (ret == false)
7232 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
7233 return VK_SUCCESS;
7234 }
7235
7236 static uint32_t radv_compute_valid_memory_types_attempt(struct radv_physical_device *dev,
7237 enum radeon_bo_domain domains,
7238 enum radeon_bo_flag flags,
7239 enum radeon_bo_flag ignore_flags)
7240 {
7241 /* Don't count GTT/CPU as relevant:
7242 *
7243 * - We're not fully consistent between the two.
7244 * - Sometimes VRAM gets VRAM|GTT.
7245 */
7246 const enum radeon_bo_domain relevant_domains = RADEON_DOMAIN_VRAM |
7247 RADEON_DOMAIN_GDS |
7248 RADEON_DOMAIN_OA;
7249 uint32_t bits = 0;
7250 for (unsigned i = 0; i < dev->memory_properties.memoryTypeCount; ++i) {
7251 if ((domains & relevant_domains) != (dev->memory_domains[i] & relevant_domains))
7252 continue;
7253
7254 if ((flags & ~ignore_flags) != (dev->memory_flags[i] & ~ignore_flags))
7255 continue;
7256
7257 bits |= 1u << i;
7258 }
7259
7260 return bits;
7261 }
7262
7263 static uint32_t radv_compute_valid_memory_types(struct radv_physical_device *dev,
7264 enum radeon_bo_domain domains,
7265 enum radeon_bo_flag flags)
7266 {
7267 enum radeon_bo_flag ignore_flags = ~(RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_GTT_WC);
7268 uint32_t bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags);
7269
7270 if (!bits) {
7271 ignore_flags |= RADEON_FLAG_NO_CPU_ACCESS;
7272 bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags);
7273 }
7274
7275 return bits;
7276 }
7277 VkResult radv_GetMemoryFdPropertiesKHR(VkDevice _device,
7278 VkExternalMemoryHandleTypeFlagBits handleType,
7279 int fd,
7280 VkMemoryFdPropertiesKHR *pMemoryFdProperties)
7281 {
7282 RADV_FROM_HANDLE(radv_device, device, _device);
7283
7284 switch (handleType) {
7285 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: {
7286 enum radeon_bo_domain domains;
7287 enum radeon_bo_flag flags;
7288 if (!device->ws->buffer_get_flags_from_fd(device->ws, fd, &domains, &flags))
7289 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7290
7291 pMemoryFdProperties->memoryTypeBits = radv_compute_valid_memory_types(device->physical_device, domains, flags);
7292 return VK_SUCCESS;
7293 }
7294 default:
7295 /* The valid usage section for this function says:
7296 *
7297 * "handleType must not be one of the handle types defined as
7298 * opaque."
7299 *
7300 * So opaque handle types fall into the default "unsupported" case.
7301 */
7302 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7303 }
7304 }
7305
7306 static VkResult radv_import_opaque_fd(struct radv_device *device,
7307 int fd,
7308 uint32_t *syncobj)
7309 {
7310 uint32_t syncobj_handle = 0;
7311 int ret = device->ws->import_syncobj(device->ws, fd, &syncobj_handle);
7312 if (ret != 0)
7313 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7314
7315 if (*syncobj)
7316 device->ws->destroy_syncobj(device->ws, *syncobj);
7317
7318 *syncobj = syncobj_handle;
7319 close(fd);
7320
7321 return VK_SUCCESS;
7322 }
7323
7324 static VkResult radv_import_sync_fd(struct radv_device *device,
7325 int fd,
7326 uint32_t *syncobj)
7327 {
7328 /* If we create a syncobj we do it locally so that if we have an error, we don't
7329 * leave a syncobj in an undetermined state in the fence. */
7330 uint32_t syncobj_handle = *syncobj;
7331 if (!syncobj_handle) {
7332 int ret = device->ws->create_syncobj(device->ws, &syncobj_handle);
7333 if (ret) {
7334 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7335 }
7336 }
7337
7338 if (fd == -1) {
7339 device->ws->signal_syncobj(device->ws, syncobj_handle);
7340 } else {
7341 int ret = device->ws->import_syncobj_from_sync_file(device->ws, syncobj_handle, fd);
7342 if (ret != 0)
7343 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7344 }
7345
7346 *syncobj = syncobj_handle;
7347 if (fd != -1)
7348 close(fd);
7349
7350 return VK_SUCCESS;
7351 }
7352
7353 VkResult radv_ImportSemaphoreFdKHR(VkDevice _device,
7354 const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
7355 {
7356 RADV_FROM_HANDLE(radv_device, device, _device);
7357 RADV_FROM_HANDLE(radv_semaphore, sem, pImportSemaphoreFdInfo->semaphore);
7358 VkResult result;
7359 struct radv_semaphore_part *dst = NULL;
7360
7361 if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) {
7362 dst = &sem->temporary;
7363 } else {
7364 dst = &sem->permanent;
7365 }
7366
7367 uint32_t syncobj = dst->kind == RADV_SEMAPHORE_SYNCOBJ ? dst->syncobj : 0;
7368
7369 switch(pImportSemaphoreFdInfo->handleType) {
7370 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
7371 result = radv_import_opaque_fd(device, pImportSemaphoreFdInfo->fd, &syncobj);
7372 break;
7373 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
7374 result = radv_import_sync_fd(device, pImportSemaphoreFdInfo->fd, &syncobj);
7375 break;
7376 default:
7377 unreachable("Unhandled semaphore handle type");
7378 }
7379
7380 if (result == VK_SUCCESS) {
7381 dst->syncobj = syncobj;
7382 dst->kind = RADV_SEMAPHORE_SYNCOBJ;
7383 }
7384
7385 return result;
7386 }
7387
7388 VkResult radv_GetSemaphoreFdKHR(VkDevice _device,
7389 const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
7390 int *pFd)
7391 {
7392 RADV_FROM_HANDLE(radv_device, device, _device);
7393 RADV_FROM_HANDLE(radv_semaphore, sem, pGetFdInfo->semaphore);
7394 int ret;
7395 uint32_t syncobj_handle;
7396
7397 if (sem->temporary.kind != RADV_SEMAPHORE_NONE) {
7398 assert(sem->temporary.kind == RADV_SEMAPHORE_SYNCOBJ);
7399 syncobj_handle = sem->temporary.syncobj;
7400 } else {
7401 assert(sem->permanent.kind == RADV_SEMAPHORE_SYNCOBJ);
7402 syncobj_handle = sem->permanent.syncobj;
7403 }
7404
7405 switch(pGetFdInfo->handleType) {
7406 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
7407 ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd);
7408 break;
7409 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
7410 ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd);
7411 if (!ret) {
7412 if (sem->temporary.kind != RADV_SEMAPHORE_NONE) {
7413 radv_destroy_semaphore_part(device, &sem->temporary);
7414 } else {
7415 device->ws->reset_syncobj(device->ws, syncobj_handle);
7416 }
7417 }
7418 break;
7419 default:
7420 unreachable("Unhandled semaphore handle type");
7421 }
7422
7423 if (ret)
7424 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7425 return VK_SUCCESS;
7426 }
7427
7428 void radv_GetPhysicalDeviceExternalSemaphoreProperties(
7429 VkPhysicalDevice physicalDevice,
7430 const VkPhysicalDeviceExternalSemaphoreInfo *pExternalSemaphoreInfo,
7431 VkExternalSemaphoreProperties *pExternalSemaphoreProperties)
7432 {
7433 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
7434 VkSemaphoreTypeKHR type = radv_get_semaphore_type(pExternalSemaphoreInfo->pNext, NULL);
7435
7436 if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
7437 pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0;
7438 pExternalSemaphoreProperties->compatibleHandleTypes = 0;
7439 pExternalSemaphoreProperties->externalSemaphoreFeatures = 0;
7440
7441 /* Require has_syncobj_wait_for_submit for the syncobj signal ioctl introduced at virtually the same time */
7442 } else if (pdevice->rad_info.has_syncobj_wait_for_submit &&
7443 (pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT ||
7444 pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT)) {
7445 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
7446 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
7447 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7448 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7449 } else if (pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) {
7450 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7451 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7452 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7453 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7454 } else {
7455 pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0;
7456 pExternalSemaphoreProperties->compatibleHandleTypes = 0;
7457 pExternalSemaphoreProperties->externalSemaphoreFeatures = 0;
7458 }
7459 }
7460
7461 VkResult radv_ImportFenceFdKHR(VkDevice _device,
7462 const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
7463 {
7464 RADV_FROM_HANDLE(radv_device, device, _device);
7465 RADV_FROM_HANDLE(radv_fence, fence, pImportFenceFdInfo->fence);
7466 uint32_t *syncobj_dst = NULL;
7467
7468
7469 if (pImportFenceFdInfo->flags & VK_FENCE_IMPORT_TEMPORARY_BIT) {
7470 syncobj_dst = &fence->temp_syncobj;
7471 } else {
7472 syncobj_dst = &fence->syncobj;
7473 }
7474
7475 switch(pImportFenceFdInfo->handleType) {
7476 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT:
7477 return radv_import_opaque_fd(device, pImportFenceFdInfo->fd, syncobj_dst);
7478 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT:
7479 return radv_import_sync_fd(device, pImportFenceFdInfo->fd, syncobj_dst);
7480 default:
7481 unreachable("Unhandled fence handle type");
7482 }
7483 }
7484
7485 VkResult radv_GetFenceFdKHR(VkDevice _device,
7486 const VkFenceGetFdInfoKHR *pGetFdInfo,
7487 int *pFd)
7488 {
7489 RADV_FROM_HANDLE(radv_device, device, _device);
7490 RADV_FROM_HANDLE(radv_fence, fence, pGetFdInfo->fence);
7491 int ret;
7492 uint32_t syncobj_handle;
7493
7494 if (fence->temp_syncobj)
7495 syncobj_handle = fence->temp_syncobj;
7496 else
7497 syncobj_handle = fence->syncobj;
7498
7499 switch(pGetFdInfo->handleType) {
7500 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT:
7501 ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd);
7502 break;
7503 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT:
7504 ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd);
7505 if (!ret) {
7506 if (fence->temp_syncobj) {
7507 close (fence->temp_syncobj);
7508 fence->temp_syncobj = 0;
7509 } else {
7510 device->ws->reset_syncobj(device->ws, syncobj_handle);
7511 }
7512 }
7513 break;
7514 default:
7515 unreachable("Unhandled fence handle type");
7516 }
7517
7518 if (ret)
7519 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7520 return VK_SUCCESS;
7521 }
7522
7523 void radv_GetPhysicalDeviceExternalFenceProperties(
7524 VkPhysicalDevice physicalDevice,
7525 const VkPhysicalDeviceExternalFenceInfo *pExternalFenceInfo,
7526 VkExternalFenceProperties *pExternalFenceProperties)
7527 {
7528 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
7529
7530 if (pdevice->rad_info.has_syncobj_wait_for_submit &&
7531 (pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT ||
7532 pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT)) {
7533 pExternalFenceProperties->exportFromImportedHandleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
7534 pExternalFenceProperties->compatibleHandleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
7535 pExternalFenceProperties->externalFenceFeatures = VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT |
7536 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7537 } else {
7538 pExternalFenceProperties->exportFromImportedHandleTypes = 0;
7539 pExternalFenceProperties->compatibleHandleTypes = 0;
7540 pExternalFenceProperties->externalFenceFeatures = 0;
7541 }
7542 }
7543
7544 VkResult
7545 radv_CreateDebugReportCallbackEXT(VkInstance _instance,
7546 const VkDebugReportCallbackCreateInfoEXT* pCreateInfo,
7547 const VkAllocationCallbacks* pAllocator,
7548 VkDebugReportCallbackEXT* pCallback)
7549 {
7550 RADV_FROM_HANDLE(radv_instance, instance, _instance);
7551 return vk_create_debug_report_callback(&instance->debug_report_callbacks,
7552 pCreateInfo, pAllocator, &instance->alloc,
7553 pCallback);
7554 }
7555
7556 void
7557 radv_DestroyDebugReportCallbackEXT(VkInstance _instance,
7558 VkDebugReportCallbackEXT _callback,
7559 const VkAllocationCallbacks* pAllocator)
7560 {
7561 RADV_FROM_HANDLE(radv_instance, instance, _instance);
7562 vk_destroy_debug_report_callback(&instance->debug_report_callbacks,
7563 _callback, pAllocator, &instance->alloc);
7564 }
7565
7566 void
7567 radv_DebugReportMessageEXT(VkInstance _instance,
7568 VkDebugReportFlagsEXT flags,
7569 VkDebugReportObjectTypeEXT objectType,
7570 uint64_t object,
7571 size_t location,
7572 int32_t messageCode,
7573 const char* pLayerPrefix,
7574 const char* pMessage)
7575 {
7576 RADV_FROM_HANDLE(radv_instance, instance, _instance);
7577 vk_debug_report(&instance->debug_report_callbacks, flags, objectType,
7578 object, location, messageCode, pLayerPrefix, pMessage);
7579 }
7580
7581 void
7582 radv_GetDeviceGroupPeerMemoryFeatures(
7583 VkDevice device,
7584 uint32_t heapIndex,
7585 uint32_t localDeviceIndex,
7586 uint32_t remoteDeviceIndex,
7587 VkPeerMemoryFeatureFlags* pPeerMemoryFeatures)
7588 {
7589 assert(localDeviceIndex == remoteDeviceIndex);
7590
7591 *pPeerMemoryFeatures = VK_PEER_MEMORY_FEATURE_COPY_SRC_BIT |
7592 VK_PEER_MEMORY_FEATURE_COPY_DST_BIT |
7593 VK_PEER_MEMORY_FEATURE_GENERIC_SRC_BIT |
7594 VK_PEER_MEMORY_FEATURE_GENERIC_DST_BIT;
7595 }
7596
7597 static const VkTimeDomainEXT radv_time_domains[] = {
7598 VK_TIME_DOMAIN_DEVICE_EXT,
7599 VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT,
7600 VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT,
7601 };
7602
7603 VkResult radv_GetPhysicalDeviceCalibrateableTimeDomainsEXT(
7604 VkPhysicalDevice physicalDevice,
7605 uint32_t *pTimeDomainCount,
7606 VkTimeDomainEXT *pTimeDomains)
7607 {
7608 int d;
7609 VK_OUTARRAY_MAKE(out, pTimeDomains, pTimeDomainCount);
7610
7611 for (d = 0; d < ARRAY_SIZE(radv_time_domains); d++) {
7612 vk_outarray_append(&out, i) {
7613 *i = radv_time_domains[d];
7614 }
7615 }
7616
7617 return vk_outarray_status(&out);
7618 }
7619
7620 static uint64_t
7621 radv_clock_gettime(clockid_t clock_id)
7622 {
7623 struct timespec current;
7624 int ret;
7625
7626 ret = clock_gettime(clock_id, &current);
7627 if (ret < 0 && clock_id == CLOCK_MONOTONIC_RAW)
7628 ret = clock_gettime(CLOCK_MONOTONIC, &current);
7629 if (ret < 0)
7630 return 0;
7631
7632 return (uint64_t) current.tv_sec * 1000000000ULL + current.tv_nsec;
7633 }
7634
7635 VkResult radv_GetCalibratedTimestampsEXT(
7636 VkDevice _device,
7637 uint32_t timestampCount,
7638 const VkCalibratedTimestampInfoEXT *pTimestampInfos,
7639 uint64_t *pTimestamps,
7640 uint64_t *pMaxDeviation)
7641 {
7642 RADV_FROM_HANDLE(radv_device, device, _device);
7643 uint32_t clock_crystal_freq = device->physical_device->rad_info.clock_crystal_freq;
7644 int d;
7645 uint64_t begin, end;
7646 uint64_t max_clock_period = 0;
7647
7648 begin = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
7649
7650 for (d = 0; d < timestampCount; d++) {
7651 switch (pTimestampInfos[d].timeDomain) {
7652 case VK_TIME_DOMAIN_DEVICE_EXT:
7653 pTimestamps[d] = device->ws->query_value(device->ws,
7654 RADEON_TIMESTAMP);
7655 uint64_t device_period = DIV_ROUND_UP(1000000, clock_crystal_freq);
7656 max_clock_period = MAX2(max_clock_period, device_period);
7657 break;
7658 case VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT:
7659 pTimestamps[d] = radv_clock_gettime(CLOCK_MONOTONIC);
7660 max_clock_period = MAX2(max_clock_period, 1);
7661 break;
7662
7663 case VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT:
7664 pTimestamps[d] = begin;
7665 break;
7666 default:
7667 pTimestamps[d] = 0;
7668 break;
7669 }
7670 }
7671
7672 end = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
7673
7674 /*
7675 * The maximum deviation is the sum of the interval over which we
7676 * perform the sampling and the maximum period of any sampled
7677 * clock. That's because the maximum skew between any two sampled
7678 * clock edges is when the sampled clock with the largest period is
7679 * sampled at the end of that period but right at the beginning of the
7680 * sampling interval and some other clock is sampled right at the
7681 * begining of its sampling period and right at the end of the
7682 * sampling interval. Let's assume the GPU has the longest clock
7683 * period and that the application is sampling GPU and monotonic:
7684 *
7685 * s e
7686 * w x y z 0 1 2 3 4 5 6 7 8 9 a b c d e f
7687 * Raw -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-
7688 *
7689 * g
7690 * 0 1 2 3
7691 * GPU -----_____-----_____-----_____-----_____
7692 *
7693 * m
7694 * x y z 0 1 2 3 4 5 6 7 8 9 a b c
7695 * Monotonic -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-
7696 *
7697 * Interval <----------------->
7698 * Deviation <-------------------------->
7699 *
7700 * s = read(raw) 2
7701 * g = read(GPU) 1
7702 * m = read(monotonic) 2
7703 * e = read(raw) b
7704 *
7705 * We round the sample interval up by one tick to cover sampling error
7706 * in the interval clock
7707 */
7708
7709 uint64_t sample_interval = end - begin + 1;
7710
7711 *pMaxDeviation = sample_interval + max_clock_period;
7712
7713 return VK_SUCCESS;
7714 }
7715
7716 void radv_GetPhysicalDeviceMultisamplePropertiesEXT(
7717 VkPhysicalDevice physicalDevice,
7718 VkSampleCountFlagBits samples,
7719 VkMultisamplePropertiesEXT* pMultisampleProperties)
7720 {
7721 if (samples & (VK_SAMPLE_COUNT_2_BIT |
7722 VK_SAMPLE_COUNT_4_BIT |
7723 VK_SAMPLE_COUNT_8_BIT)) {
7724 pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 2, 2 };
7725 } else {
7726 pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 0, 0 };
7727 }
7728 }