X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Famd%2Fvulkan%2Fradv_device.c;h=5a1a67288d3d9e0474fb4016a18e7b6cc30916d1;hb=3b7cd734e8a11a9af0cbc4c2445c7103044fc4a0;hp=3eeb3b5772ff552f48592e85dba27fea91a669e1;hpb=bec92850270a046524056b8d43bbd2554ba9f2e0;p=mesa.git diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index 3eeb3b5772f..5a1a67288d3 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -63,7 +63,7 @@ #include "util/timespec.h" #include "util/u_atomic.h" #include "compiler/glsl_types.h" -#include "util/xmlpool.h" +#include "util/driconf.h" static struct radv_timeline_point * radv_timeline_find_point_at_least_locked(struct radv_device *device, @@ -162,24 +162,13 @@ radv_physical_device_init_mem_types(struct radv_physical_device *device) unsigned type_count = 0; - if (device->rad_info.has_dedicated_vram) { - if (vram_index >= 0) { - device->memory_domains[type_count] = RADEON_DOMAIN_VRAM; - device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS; - device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) { - .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, - .heapIndex = vram_index, - }; - } - } else { - if (visible_vram_index >= 0) { - device->memory_domains[type_count] = RADEON_DOMAIN_VRAM; - device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS; - device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) { - .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, - .heapIndex = visible_vram_index, - }; - } + if (vram_index >= 0 || visible_vram_index >= 0) { + device->memory_domains[type_count] = RADEON_DOMAIN_VRAM; + device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS; + device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) { + .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + .heapIndex = vram_index >= 0 ? vram_index : visible_vram_index, + }; } if (gart_index >= 0) { @@ -238,10 +227,29 @@ radv_physical_device_init_mem_types(struct radv_physical_device *device) } } +static const char * +radv_get_compiler_string(struct radv_physical_device *pdevice) +{ + if (!pdevice->use_llvm) { + /* Some games like SotTR apply shader workarounds if the LLVM + * version is too old or if the LLVM version string is + * missing. This gives 2-5% performance with SotTR and ACO. + */ + if (driQueryOptionb(&pdevice->instance->dri_options, + "radv_report_llvm9_version_string")) { + return "ACO/LLVM 9.0.1"; + } + + return "ACO"; + } + + return "LLVM " MESA_LLVM_VERSION_STRING; +} + static VkResult -radv_physical_device_init(struct radv_physical_device *device, - struct radv_instance *instance, - drmDevicePtr drm_device) +radv_physical_device_try_create(struct radv_instance *instance, + drmDevicePtr drm_device, + struct radv_physical_device **device_out) { VkResult result; int fd = -1; @@ -285,6 +293,14 @@ radv_physical_device_init(struct radv_physical_device *device, radv_logi("Found compatible device '%s'.", path); } + struct radv_physical_device *device = + vk_zalloc2(&instance->alloc, NULL, sizeof(*device), 8, + VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + if (!device) { + result = vk_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY); + goto fail_fd; + } + device->_loader_data.loaderMagic = ICD_LOADER_MAGIC; device->instance = instance; @@ -296,8 +312,9 @@ radv_physical_device_init(struct radv_physical_device *device, } if (!device->ws) { - result = vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER); - goto fail; + result = vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, + "failed to initialize winsys"); + goto fail_alloc; } if (drm_device && instance->enabled_extensions.KHR_display) { @@ -321,21 +338,20 @@ radv_physical_device_init(struct radv_physical_device *device, device->local_fd = fd; device->ws->query_info(device->ws, &device->rad_info); - device->use_aco = instance->perftest_flags & RADV_PERFTEST_ACO; + device->use_llvm = instance->debug_flags & RADV_DEBUG_LLVM; snprintf(device->name, sizeof(device->name), - "AMD RADV%s %s (LLVM " MESA_LLVM_VERSION_STRING ")", device->use_aco ? "/ACO" : "", - device->rad_info.name); + "AMD RADV %s (%s)", + device->rad_info.name, radv_get_compiler_string(device)); if (radv_device_get_cache_uuid(device->rad_info.family, device->cache_uuid)) { - device->ws->destroy(device->ws); result = vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, "cannot generate UUID"); - goto fail; + goto fail_wsi; } /* These flags affect shader compilation. */ - uint64_t shader_env_flags = (device->use_aco ? 0x2 : 0); + uint64_t shader_env_flags = (device->use_llvm ? 0 : 0x2); /* The gpu id is already embedded in the uuid so we just pass "radv" * when creating the cache. @@ -344,7 +360,7 @@ radv_physical_device_init(struct radv_physical_device *device, disk_cache_format_hex_id(buf, device->cache_uuid, VK_UUID_SIZE * 2); device->disk_cache = disk_cache_create(device->name, buf, shader_env_flags); - if (device->rad_info.chip_class < GFX8) + if (device->rad_info.chip_class < GFX8 || !device->use_llvm) fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n"); radv_get_driver_uuid(&device->driver_uuid); @@ -356,15 +372,12 @@ radv_physical_device_init(struct radv_physical_device *device, device->dcc_msaa_allowed = (device->instance->perftest_flags & RADV_PERFTEST_DCC_MSAA); - device->use_shader_ballot = (device->use_aco && device->rad_info.chip_class >= GFX8) || - (device->instance->perftest_flags & RADV_PERFTEST_SHADER_BALLOT); - device->use_ngg = device->rad_info.chip_class >= GFX10 && device->rad_info.family != CHIP_NAVI14 && !(device->instance->debug_flags & RADV_DEBUG_NO_NGG); /* TODO: Implement NGG GS with ACO. */ - device->use_ngg_gs = device->use_ngg && !device->use_aco; + device->use_ngg_gs = device->use_ngg && device->use_llvm; device->use_ngg_streamout = false; /* Determine the number of threads per wave for all stages. */ @@ -385,7 +398,9 @@ radv_physical_device_init(struct radv_physical_device *device, } radv_physical_device_init_mem_types(device); - radv_fill_device_extension_table(device, &device->supported_extensions); + + radv_physical_device_get_supported_extensions(device, + &device->supported_extensions); if (drm_device) device->bus_info = *drm_device->businfo.pci; @@ -399,22 +414,30 @@ radv_physical_device_init(struct radv_physical_device *device, */ result = radv_init_wsi(device); if (result != VK_SUCCESS) { - device->ws->destroy(device->ws); vk_error(instance, result); - goto fail; + goto fail_disk_cache; } + *device_out = device; + return VK_SUCCESS; -fail: - close(fd); +fail_disk_cache: + disk_cache_destroy(device->disk_cache); +fail_wsi: + device->ws->destroy(device->ws); +fail_alloc: + vk_free(&instance->alloc, device); +fail_fd: + if (fd != -1) + close(fd); if (master_fd != -1) close(master_fd); return result; } static void -radv_physical_device_finish(struct radv_physical_device *device) +radv_physical_device_destroy(struct radv_physical_device *device) { radv_finish_wsi(device); device->ws->destroy(device->ws); @@ -422,6 +445,7 @@ radv_physical_device_finish(struct radv_physical_device *device) close(device->local_fd); if (device->master_fd != -1) close(device->master_fd); + vk_free(&device->instance->alloc, device); } static void * @@ -474,12 +498,11 @@ static const struct debug_control radv_debug_options[] = { {"checkir", RADV_DEBUG_CHECKIR}, {"nothreadllvm", RADV_DEBUG_NOTHREADLLVM}, {"nobinning", RADV_DEBUG_NOBINNING}, - {"noloadstoreopt", RADV_DEBUG_NO_LOAD_STORE_OPT}, {"nongg", RADV_DEBUG_NO_NGG}, - {"noshaderballot", RADV_DEBUG_NO_SHADER_BALLOT}, {"allentrypoints", RADV_DEBUG_ALL_ENTRYPOINTS}, {"metashaders", RADV_DEBUG_DUMP_META_SHADERS}, {"nomemorycache", RADV_DEBUG_NO_MEMORY_CACHE}, + {"llvm", RADV_DEBUG_LLVM}, {NULL, 0} }; @@ -494,13 +517,11 @@ static const struct debug_control radv_perftest_options[] = { {"localbos", RADV_PERFTEST_LOCAL_BOS}, {"dccmsaa", RADV_PERFTEST_DCC_MSAA}, {"bolist", RADV_PERFTEST_BO_LIST}, - {"shader_ballot", RADV_PERFTEST_SHADER_BALLOT}, {"tccompatcmask", RADV_PERFTEST_TC_COMPAT_CMASK}, {"cswave32", RADV_PERFTEST_CS_WAVE_32}, {"pswave32", RADV_PERFTEST_PS_WAVE_32}, {"gewave32", RADV_PERFTEST_GE_WAVE_32}, {"dfsm", RADV_PERFTEST_DFSM}, - {"aco", RADV_PERFTEST_ACO}, {NULL, 0} }; @@ -516,50 +537,50 @@ radv_handle_per_app_options(struct radv_instance *instance, const VkApplicationInfo *info) { const char *name = info ? info->pApplicationName : NULL; - - if (!name) - return; - - if (!strcmp(name, "DOOM_VFR")) { - /* Work around a Doom VFR game bug */ - instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS; - } else if (!strcmp(name, "MonsterHunterWorld.exe")) { - /* Workaround for a WaW hazard when LLVM moves/merges - * load/store memory operations. - * See https://reviews.llvm.org/D61313 - */ - if (LLVM_VERSION_MAJOR < 9) - instance->debug_flags |= RADV_DEBUG_NO_LOAD_STORE_OPT; - } else if (!strcmp(name, "Wolfenstein: Youngblood")) { - if (!(instance->debug_flags & RADV_DEBUG_NO_SHADER_BALLOT) && - !(instance->perftest_flags & RADV_PERFTEST_ACO)) { - /* Force enable VK_AMD_shader_ballot because it looks - * safe and it gives a nice boost (+20% on Vega 56 at - * this time). It also prevents corruption on LLVM. + const char *engine_name = info ? info->pEngineName : NULL; + + if (name) { + if (!strcmp(name, "DOOM_VFR")) { + /* Work around a Doom VFR game bug */ + instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS; + } else if (!strcmp(name, "Fledge")) { + /* + * Zero VRAM for "The Surge 2" + * + * This avoid a hang when when rendering any level. Likely + * uninitialized data in an indirect draw. */ - instance->perftest_flags |= RADV_PERFTEST_SHADER_BALLOT; + instance->debug_flags |= RADV_DEBUG_ZERO_VRAM; + } else if (!strcmp(name, "No Man's Sky")) { + /* Work around a NMS game bug */ + instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE; + } else if (!strcmp(name, "DOOMEternal")) { + /* Zero VRAM for Doom Eternal to fix rendering issues. */ + instance->debug_flags |= RADV_DEBUG_ZERO_VRAM; + } else if (!strcmp(name, "Red Dead Redemption 2")) { + /* Work around a RDR2 game bug */ + instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE; + } + } + + if (engine_name) { + if (!strcmp(engine_name, "vkd3d")) { + /* Zero VRAM for all VKD3D (DX12->VK) games to fix + * rendering issues. + */ + instance->debug_flags |= RADV_DEBUG_ZERO_VRAM; + } else if (!strcmp(engine_name, "Quantic Dream Engine")) { + /* Fix various artifacts in Detroit: Become Human */ + instance->debug_flags |= RADV_DEBUG_ZERO_VRAM; } - } else if (!strcmp(name, "Fledge")) { - /* - * Zero VRAM for "The Surge 2" - * - * This avoid a hang when when rendering any level. Likely - * uninitialized data in an indirect draw. - */ - instance->debug_flags |= RADV_DEBUG_ZERO_VRAM; - } else if (!strcmp(name, "No Man's Sky")) { - /* Work around a NMS game bug */ - instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE; } -} -static int radv_get_instance_extension_index(const char *name) -{ - for (unsigned i = 0; i < RADV_INSTANCE_EXTENSION_COUNT; ++i) { - if (strcmp(name, radv_instance_extensions[i].extensionName) == 0) - return i; - } - return -1; + instance->enable_mrt_output_nan_fixup = + driQueryOptionb(&instance->dri_options, + "radv_enable_mrt_output_nan_fixup"); + + if (driQueryOptionb(&instance->dri_options, "radv_no_dynamic_bounds")) + instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS; } static const char radv_dri_options_xml[] = @@ -568,6 +589,10 @@ DRI_CONF_BEGIN DRI_CONF_ADAPTIVE_SYNC("true") DRI_CONF_VK_X11_OVERRIDE_MIN_IMAGE_COUNT(0) DRI_CONF_VK_X11_STRICT_IMAGE_COUNT("false") + DRI_CONF_VK_X11_ENSURE_MIN_IMAGE_COUNT("false") + DRI_CONF_RADV_REPORT_LLVM9_VERSION_STRING("false") + DRI_CONF_RADV_ENABLE_MRT_OUTPUT_NAN_FIXUP("false") + DRI_CONF_RADV_NO_DYNAMIC_BOUNDS("false") DRI_CONF_SECTION_END DRI_CONF_SECTION_DEBUG @@ -593,70 +618,56 @@ VkResult radv_CreateInstance( struct radv_instance *instance; VkResult result; - assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO); - - uint32_t client_version; - if (pCreateInfo->pApplicationInfo && - pCreateInfo->pApplicationInfo->apiVersion != 0) { - client_version = pCreateInfo->pApplicationInfo->apiVersion; - } else { - client_version = VK_API_VERSION_1_0; - } - - const char *engine_name = NULL; - uint32_t engine_version = 0; - if (pCreateInfo->pApplicationInfo) { - engine_name = pCreateInfo->pApplicationInfo->pEngineName; - engine_version = pCreateInfo->pApplicationInfo->engineVersion; - } - instance = vk_zalloc2(&default_alloc, pAllocator, sizeof(*instance), 8, VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); if (!instance) return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY); - instance->_loader_data.loaderMagic = ICD_LOADER_MAGIC; + vk_object_base_init(NULL, &instance->base, VK_OBJECT_TYPE_INSTANCE); if (pAllocator) instance->alloc = *pAllocator; else instance->alloc = default_alloc; - instance->apiVersion = client_version; - instance->physicalDeviceCount = -1; + if (pCreateInfo->pApplicationInfo) { + const VkApplicationInfo *app = pCreateInfo->pApplicationInfo; + + instance->engineName = + vk_strdup(&instance->alloc, app->pEngineName, + VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + instance->engineVersion = app->engineVersion; + instance->apiVersion = app->apiVersion; + } - /* Get secure compile thread count. NOTE: We cap this at 32 */ -#define MAX_SC_PROCS 32 - char *num_sc_threads = getenv("RADV_SECURE_COMPILE_THREADS"); - if (num_sc_threads) - instance->num_sc_threads = MIN2(strtoul(num_sc_threads, NULL, 10), MAX_SC_PROCS); + if (instance->apiVersion == 0) + instance->apiVersion = VK_API_VERSION_1_0; instance->debug_flags = parse_debug_string(getenv("RADV_DEBUG"), radv_debug_options); - /* Disable memory cache when secure compile is set */ - if (radv_device_use_secure_compile(instance)) - instance->debug_flags |= RADV_DEBUG_NO_MEMORY_CACHE; - instance->perftest_flags = parse_debug_string(getenv("RADV_PERFTEST"), radv_perftest_options); - if (instance->perftest_flags & RADV_PERFTEST_ACO) - fprintf(stderr, "WARNING: Experimental compiler backend enabled. Here be dragons! Incorrect rendering, GPU hangs and/or resets are likely\n"); - if (instance->debug_flags & RADV_DEBUG_STARTUP) radv_logi("Created an instance"); for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) { - const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i]; - int index = radv_get_instance_extension_index(ext_name); + int idx; + for (idx = 0; idx < RADV_INSTANCE_EXTENSION_COUNT; idx++) { + if (!strcmp(pCreateInfo->ppEnabledExtensionNames[i], + radv_instance_extensions[idx].extensionName)) + break; + } - if (index < 0 || !radv_supported_instance_extensions.extensions[index]) { + if (idx >= RADV_INSTANCE_EXTENSION_COUNT || + !radv_instance_extensions_supported.extensions[idx]) { + vk_object_base_finish(&instance->base); vk_free2(&default_alloc, pAllocator, instance); return vk_error(instance, VK_ERROR_EXTENSION_NOT_PRESENT); } - instance->enabled_extensions.extensions[index] = true; + instance->enabled_extensions.extensions[idx] = true; } bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS; @@ -703,16 +714,16 @@ VkResult radv_CreateInstance( } } + instance->physical_devices_enumerated = false; + list_inithead(&instance->physical_devices); + result = vk_debug_report_instance_init(&instance->debug_report_callbacks); if (result != VK_SUCCESS) { + vk_object_base_finish(&instance->base); vk_free2(&default_alloc, pAllocator, instance); return vk_error(instance, result); } - instance->engineName = vk_strdup(&instance->alloc, engine_name, - VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - instance->engineVersion = engine_version; - glsl_type_singleton_init_or_ref(); VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false)); @@ -734,8 +745,9 @@ void radv_DestroyInstance( if (!instance) return; - for (int i = 0; i < instance->physicalDeviceCount; ++i) { - radv_physical_device_finish(instance->physicalDevices + i); + list_for_each_entry_safe(struct radv_physical_device, pdevice, + &instance->physical_devices, link) { + radv_physical_device_destroy(pdevice); } vk_free(&instance->alloc, instance->engineName); @@ -749,29 +761,35 @@ void radv_DestroyInstance( vk_debug_report_instance_destroy(&instance->debug_report_callbacks); + vk_object_base_finish(&instance->base); vk_free(&instance->alloc, instance); } static VkResult -radv_enumerate_devices(struct radv_instance *instance) +radv_enumerate_physical_devices(struct radv_instance *instance) { + if (instance->physical_devices_enumerated) + return VK_SUCCESS; + + instance->physical_devices_enumerated = true; + /* TODO: Check for more devices ? */ drmDevicePtr devices[8]; - VkResult result = VK_ERROR_INCOMPATIBLE_DRIVER; + VkResult result = VK_SUCCESS; int max_devices; - instance->physicalDeviceCount = 0; - if (getenv("RADV_FORCE_FAMILY")) { /* When RADV_FORCE_FAMILY is set, the driver creates a nul * device that allows to test the compiler without having an * AMDGPU instance. */ - result = radv_physical_device_init(instance->physicalDevices + - instance->physicalDeviceCount, - instance, NULL); + struct radv_physical_device *pdevice; - ++instance->physicalDeviceCount; + result = radv_physical_device_try_create(instance, NULL, &pdevice); + if (result != VK_SUCCESS) + return result; + + list_addtail(&pdevice->link, &instance->physical_devices); return VK_SUCCESS; } @@ -781,25 +799,32 @@ radv_enumerate_devices(struct radv_instance *instance) radv_logi("Found %d drm nodes", max_devices); if (max_devices < 1) - return vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER); + return vk_error(instance, VK_SUCCESS); for (unsigned i = 0; i < (unsigned)max_devices; i++) { if (devices[i]->available_nodes & 1 << DRM_NODE_RENDER && devices[i]->bustype == DRM_BUS_PCI && devices[i]->deviceinfo.pci->vendor_id == ATI_VENDOR_ID) { - result = radv_physical_device_init(instance->physicalDevices + - instance->physicalDeviceCount, - instance, - devices[i]); - if (result == VK_SUCCESS) - ++instance->physicalDeviceCount; - else if (result != VK_ERROR_INCOMPATIBLE_DRIVER) + struct radv_physical_device *pdevice; + result = radv_physical_device_try_create(instance, devices[i], + &pdevice); + /* Incompatible DRM device, skip. */ + if (result == VK_ERROR_INCOMPATIBLE_DRIVER) { + result = VK_SUCCESS; + continue; + } + + /* Error creating the physical device, report the error. */ + if (result != VK_SUCCESS) break; + + list_addtail(&pdevice->link, &instance->physical_devices); } } drmFreeDevices(devices, max_devices); + /* If we successfully enumerated any devices, call it success */ return result; } @@ -809,25 +834,20 @@ VkResult radv_EnumeratePhysicalDevices( VkPhysicalDevice* pPhysicalDevices) { RADV_FROM_HANDLE(radv_instance, instance, _instance); - VkResult result; + VK_OUTARRAY_MAKE(out, pPhysicalDevices, pPhysicalDeviceCount); - if (instance->physicalDeviceCount < 0) { - result = radv_enumerate_devices(instance); - if (result != VK_SUCCESS && - result != VK_ERROR_INCOMPATIBLE_DRIVER) - return result; - } + VkResult result = radv_enumerate_physical_devices(instance); + if (result != VK_SUCCESS) + return result; - if (!pPhysicalDevices) { - *pPhysicalDeviceCount = instance->physicalDeviceCount; - } else { - *pPhysicalDeviceCount = MIN2(*pPhysicalDeviceCount, instance->physicalDeviceCount); - for (unsigned i = 0; i < *pPhysicalDeviceCount; ++i) - pPhysicalDevices[i] = radv_physical_device_to_handle(instance->physicalDevices + i); + list_for_each_entry(struct radv_physical_device, pdevice, + &instance->physical_devices, link) { + vk_outarray_append(&out, i) { + *i = radv_physical_device_to_handle(pdevice); + } } - return *pPhysicalDeviceCount < instance->physicalDeviceCount ? VK_INCOMPLETE - : VK_SUCCESS; + return vk_outarray_status(&out); } VkResult radv_EnumeratePhysicalDeviceGroups( @@ -836,27 +856,24 @@ VkResult radv_EnumeratePhysicalDeviceGroups( VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) { RADV_FROM_HANDLE(radv_instance, instance, _instance); - VkResult result; + VK_OUTARRAY_MAKE(out, pPhysicalDeviceGroupProperties, + pPhysicalDeviceGroupCount); - if (instance->physicalDeviceCount < 0) { - result = radv_enumerate_devices(instance); - if (result != VK_SUCCESS && - result != VK_ERROR_INCOMPATIBLE_DRIVER) - return result; - } + VkResult result = radv_enumerate_physical_devices(instance); + if (result != VK_SUCCESS) + return result; - if (!pPhysicalDeviceGroupProperties) { - *pPhysicalDeviceGroupCount = instance->physicalDeviceCount; - } else { - *pPhysicalDeviceGroupCount = MIN2(*pPhysicalDeviceGroupCount, instance->physicalDeviceCount); - for (unsigned i = 0; i < *pPhysicalDeviceGroupCount; ++i) { - pPhysicalDeviceGroupProperties[i].physicalDeviceCount = 1; - pPhysicalDeviceGroupProperties[i].physicalDevices[0] = radv_physical_device_to_handle(instance->physicalDevices + i); - pPhysicalDeviceGroupProperties[i].subsetAllocation = false; + list_for_each_entry(struct radv_physical_device, pdevice, + &instance->physical_devices, link) { + vk_outarray_append(&out, p) { + p->physicalDeviceCount = 1; + memset(p->physicalDevices, 0, sizeof(p->physicalDevices)); + p->physicalDevices[0] = radv_physical_device_to_handle(pdevice); + p->subsetAllocation = false; } } - return *pPhysicalDeviceGroupCount < instance->physicalDeviceCount ? VK_INCOMPLETE - : VK_SUCCESS; + + return vk_outarray_status(&out); } void radv_GetPhysicalDeviceFeatures( @@ -908,84 +925,176 @@ void radv_GetPhysicalDeviceFeatures( .shaderCullDistance = true, .shaderFloat64 = true, .shaderInt64 = true, - .shaderInt16 = pdevice->rad_info.chip_class >= GFX9, + .shaderInt16 = true, .sparseBinding = true, .variableMultisampleRate = true, + .shaderResourceMinLod = true, .inheritedQueries = true, }; } +static void +radv_get_physical_device_features_1_1(struct radv_physical_device *pdevice, + VkPhysicalDeviceVulkan11Features *f) +{ + assert(f->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES); + + f->storageBuffer16BitAccess = true; + f->uniformAndStorageBuffer16BitAccess = true; + f->storagePushConstant16 = true; + f->storageInputOutput16 = pdevice->rad_info.has_packed_math_16bit && (LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm); + f->multiview = true; + f->multiviewGeometryShader = true; + f->multiviewTessellationShader = true; + f->variablePointersStorageBuffer = true; + f->variablePointers = true; + f->protectedMemory = false; + f->samplerYcbcrConversion = true; + f->shaderDrawParameters = true; +} + +static void +radv_get_physical_device_features_1_2(struct radv_physical_device *pdevice, + VkPhysicalDeviceVulkan12Features *f) +{ + assert(f->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES); + + f->samplerMirrorClampToEdge = true; + f->drawIndirectCount = true; + f->storageBuffer8BitAccess = true; + f->uniformAndStorageBuffer8BitAccess = true; + f->storagePushConstant8 = true; + f->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm; + f->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm; + f->shaderFloat16 = pdevice->rad_info.has_packed_math_16bit; + f->shaderInt8 = true; + + f->descriptorIndexing = true; + f->shaderInputAttachmentArrayDynamicIndexing = true; + f->shaderUniformTexelBufferArrayDynamicIndexing = true; + f->shaderStorageTexelBufferArrayDynamicIndexing = true; + f->shaderUniformBufferArrayNonUniformIndexing = true; + f->shaderSampledImageArrayNonUniformIndexing = true; + f->shaderStorageBufferArrayNonUniformIndexing = true; + f->shaderStorageImageArrayNonUniformIndexing = true; + f->shaderInputAttachmentArrayNonUniformIndexing = true; + f->shaderUniformTexelBufferArrayNonUniformIndexing = true; + f->shaderStorageTexelBufferArrayNonUniformIndexing = true; + f->descriptorBindingUniformBufferUpdateAfterBind = true; + f->descriptorBindingSampledImageUpdateAfterBind = true; + f->descriptorBindingStorageImageUpdateAfterBind = true; + f->descriptorBindingStorageBufferUpdateAfterBind = true; + f->descriptorBindingUniformTexelBufferUpdateAfterBind = true; + f->descriptorBindingStorageTexelBufferUpdateAfterBind = true; + f->descriptorBindingUpdateUnusedWhilePending = true; + f->descriptorBindingPartiallyBound = true; + f->descriptorBindingVariableDescriptorCount = true; + f->runtimeDescriptorArray = true; + + f->samplerFilterMinmax = true; + f->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7; + f->imagelessFramebuffer = true; + f->uniformBufferStandardLayout = true; + f->shaderSubgroupExtendedTypes = true; + f->separateDepthStencilLayouts = true; + f->hostQueryReset = true; + f->timelineSemaphore = pdevice->rad_info.has_syncobj_wait_for_submit; + f->bufferDeviceAddress = true; + f->bufferDeviceAddressCaptureReplay = false; + f->bufferDeviceAddressMultiDevice = false; + f->vulkanMemoryModel = false; + f->vulkanMemoryModelDeviceScope = false; + f->vulkanMemoryModelAvailabilityVisibilityChains = false; + f->shaderOutputViewportIndex = true; + f->shaderOutputLayer = true; + f->subgroupBroadcastDynamicId = true; +} + void radv_GetPhysicalDeviceFeatures2( VkPhysicalDevice physicalDevice, VkPhysicalDeviceFeatures2 *pFeatures) { RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice); + radv_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features); + + VkPhysicalDeviceVulkan11Features core_1_1 = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES, + }; + radv_get_physical_device_features_1_1(pdevice, &core_1_1); + + VkPhysicalDeviceVulkan12Features core_1_2 = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, + }; + radv_get_physical_device_features_1_2(pdevice, &core_1_2); + +#define CORE_FEATURE(major, minor, feature) \ + features->feature = core_##major##_##minor.feature + vk_foreach_struct(ext, pFeatures->pNext) { switch (ext->sType) { case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VARIABLE_POINTERS_FEATURES: { VkPhysicalDeviceVariablePointersFeatures *features = (void *)ext; - features->variablePointersStorageBuffer = true; - features->variablePointers = true; + CORE_FEATURE(1, 1, variablePointersStorageBuffer); + CORE_FEATURE(1, 1, variablePointers); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_FEATURES: { VkPhysicalDeviceMultiviewFeatures *features = (VkPhysicalDeviceMultiviewFeatures*)ext; - features->multiview = true; - features->multiviewGeometryShader = true; - features->multiviewTessellationShader = true; + CORE_FEATURE(1, 1, multiview); + CORE_FEATURE(1, 1, multiviewGeometryShader); + CORE_FEATURE(1, 1, multiviewTessellationShader); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES: { VkPhysicalDeviceShaderDrawParametersFeatures *features = (VkPhysicalDeviceShaderDrawParametersFeatures*)ext; - features->shaderDrawParameters = true; + CORE_FEATURE(1, 1, shaderDrawParameters); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_FEATURES: { VkPhysicalDeviceProtectedMemoryFeatures *features = (VkPhysicalDeviceProtectedMemoryFeatures*)ext; - features->protectedMemory = false; + CORE_FEATURE(1, 1, protectedMemory); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES: { VkPhysicalDevice16BitStorageFeatures *features = (VkPhysicalDevice16BitStorageFeatures*)ext; - bool enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8; - features->storageBuffer16BitAccess = enable; - features->uniformAndStorageBuffer16BitAccess = enable; - features->storagePushConstant16 = enable; - features->storageInputOutput16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco && LLVM_VERSION_MAJOR >= 9; + CORE_FEATURE(1, 1, storageBuffer16BitAccess); + CORE_FEATURE(1, 1, uniformAndStorageBuffer16BitAccess); + CORE_FEATURE(1, 1, storagePushConstant16); + CORE_FEATURE(1, 1, storageInputOutput16); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES: { VkPhysicalDeviceSamplerYcbcrConversionFeatures *features = (VkPhysicalDeviceSamplerYcbcrConversionFeatures*)ext; - features->samplerYcbcrConversion = true; + CORE_FEATURE(1, 1, samplerYcbcrConversion); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES: { VkPhysicalDeviceDescriptorIndexingFeatures *features = (VkPhysicalDeviceDescriptorIndexingFeatures*)ext; - features->shaderInputAttachmentArrayDynamicIndexing = true; - features->shaderUniformTexelBufferArrayDynamicIndexing = true; - features->shaderStorageTexelBufferArrayDynamicIndexing = true; - features->shaderUniformBufferArrayNonUniformIndexing = true; - features->shaderSampledImageArrayNonUniformIndexing = true; - features->shaderStorageBufferArrayNonUniformIndexing = true; - features->shaderStorageImageArrayNonUniformIndexing = true; - features->shaderInputAttachmentArrayNonUniformIndexing = true; - features->shaderUniformTexelBufferArrayNonUniformIndexing = true; - features->shaderStorageTexelBufferArrayNonUniformIndexing = true; - features->descriptorBindingUniformBufferUpdateAfterBind = true; - features->descriptorBindingSampledImageUpdateAfterBind = true; - features->descriptorBindingStorageImageUpdateAfterBind = true; - features->descriptorBindingStorageBufferUpdateAfterBind = true; - features->descriptorBindingUniformTexelBufferUpdateAfterBind = true; - features->descriptorBindingStorageTexelBufferUpdateAfterBind = true; - features->descriptorBindingUpdateUnusedWhilePending = true; - features->descriptorBindingPartiallyBound = true; - features->descriptorBindingVariableDescriptorCount = true; - features->runtimeDescriptorArray = true; + CORE_FEATURE(1, 2, shaderInputAttachmentArrayDynamicIndexing); + CORE_FEATURE(1, 2, shaderUniformTexelBufferArrayDynamicIndexing); + CORE_FEATURE(1, 2, shaderStorageTexelBufferArrayDynamicIndexing); + CORE_FEATURE(1, 2, shaderUniformBufferArrayNonUniformIndexing); + CORE_FEATURE(1, 2, shaderSampledImageArrayNonUniformIndexing); + CORE_FEATURE(1, 2, shaderStorageBufferArrayNonUniformIndexing); + CORE_FEATURE(1, 2, shaderStorageImageArrayNonUniformIndexing); + CORE_FEATURE(1, 2, shaderInputAttachmentArrayNonUniformIndexing); + CORE_FEATURE(1, 2, shaderUniformTexelBufferArrayNonUniformIndexing); + CORE_FEATURE(1, 2, shaderStorageTexelBufferArrayNonUniformIndexing); + CORE_FEATURE(1, 2, descriptorBindingUniformBufferUpdateAfterBind); + CORE_FEATURE(1, 2, descriptorBindingSampledImageUpdateAfterBind); + CORE_FEATURE(1, 2, descriptorBindingStorageImageUpdateAfterBind); + CORE_FEATURE(1, 2, descriptorBindingStorageBufferUpdateAfterBind); + CORE_FEATURE(1, 2, descriptorBindingUniformTexelBufferUpdateAfterBind); + CORE_FEATURE(1, 2, descriptorBindingStorageTexelBufferUpdateAfterBind); + CORE_FEATURE(1, 2, descriptorBindingUpdateUnusedWhilePending); + CORE_FEATURE(1, 2, descriptorBindingPartiallyBound); + CORE_FEATURE(1, 2, descriptorBindingVariableDescriptorCount); + CORE_FEATURE(1, 2, runtimeDescriptorArray); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONDITIONAL_RENDERING_FEATURES_EXT: { @@ -1012,7 +1121,7 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SCALAR_BLOCK_LAYOUT_FEATURES: { VkPhysicalDeviceScalarBlockLayoutFeatures *features = (VkPhysicalDeviceScalarBlockLayoutFeatures *)ext; - features->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7; + CORE_FEATURE(1, 2, scalarBlockLayout); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT: { @@ -1032,9 +1141,9 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES: { VkPhysicalDeviceBufferDeviceAddressFeatures *features = (VkPhysicalDeviceBufferDeviceAddressFeatures *)ext; - features->bufferDeviceAddress = true; - features->bufferDeviceAddressCaptureReplay = false; - features->bufferDeviceAddressMultiDevice = false; + CORE_FEATURE(1, 2, bufferDeviceAddress); + CORE_FEATURE(1, 2, bufferDeviceAddressCaptureReplay); + CORE_FEATURE(1, 2, bufferDeviceAddressMultiDevice); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_ENABLE_FEATURES_EXT: { @@ -1046,36 +1155,35 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES: { VkPhysicalDeviceHostQueryResetFeatures *features = (VkPhysicalDeviceHostQueryResetFeatures *)ext; - features->hostQueryReset = true; + CORE_FEATURE(1, 2, hostQueryReset); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES: { VkPhysicalDevice8BitStorageFeatures *features = (VkPhysicalDevice8BitStorageFeatures *)ext; - bool enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8; - features->storageBuffer8BitAccess = enable; - features->uniformAndStorageBuffer8BitAccess = enable; - features->storagePushConstant8 = enable; + CORE_FEATURE(1, 2, storageBuffer8BitAccess); + CORE_FEATURE(1, 2, uniformAndStorageBuffer8BitAccess); + CORE_FEATURE(1, 2, storagePushConstant8); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES: { VkPhysicalDeviceShaderFloat16Int8Features *features = (VkPhysicalDeviceShaderFloat16Int8Features*)ext; - features->shaderFloat16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco; - features->shaderInt8 = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8; + CORE_FEATURE(1, 2, shaderFloat16); + CORE_FEATURE(1, 2, shaderInt8); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES: { VkPhysicalDeviceShaderAtomicInt64Features *features = (VkPhysicalDeviceShaderAtomicInt64Features *)ext; - features->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9; - features->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9; + CORE_FEATURE(1, 2, shaderBufferInt64Atomics); + CORE_FEATURE(1, 2, shaderSharedInt64Atomics); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DEMOTE_TO_HELPER_INVOCATION_FEATURES_EXT: { VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *features = (VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *)ext; - features->shaderDemoteToHelperInvocation = pdevice->use_aco; + features->shaderDemoteToHelperInvocation = LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm; break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_FEATURES_EXT: { @@ -1102,7 +1210,7 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_UNIFORM_BUFFER_STANDARD_LAYOUT_FEATURES: { VkPhysicalDeviceUniformBufferStandardLayoutFeatures *features = (VkPhysicalDeviceUniformBufferStandardLayoutFeatures *)ext; - features->uniformBufferStandardLayout = true; + CORE_FEATURE(1, 2, uniformBufferStandardLayout); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INDEX_TYPE_UINT8_FEATURES_EXT: { @@ -1114,7 +1222,7 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGELESS_FRAMEBUFFER_FEATURES: { VkPhysicalDeviceImagelessFramebufferFeatures *features = (VkPhysicalDeviceImagelessFramebufferFeatures *)ext; - features->imagelessFramebuffer = true; + CORE_FEATURE(1, 2, imagelessFramebuffer); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR: { @@ -1127,7 +1235,7 @@ void radv_GetPhysicalDeviceFeatures2( VkPhysicalDeviceShaderClockFeaturesKHR *features = (VkPhysicalDeviceShaderClockFeaturesKHR *)ext; features->shaderSubgroupClock = true; - features->shaderDeviceClock = false; + features->shaderDeviceClock = pdevice->rad_info.chip_class >= GFX8; break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_FEATURES_EXT: { @@ -1139,7 +1247,7 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES: { VkPhysicalDeviceTimelineSemaphoreFeatures *features = (VkPhysicalDeviceTimelineSemaphoreFeatures *) ext; - features->timelineSemaphore = true; + CORE_FEATURE(1, 2, timelineSemaphore); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT: { @@ -1158,84 +1266,21 @@ void radv_GetPhysicalDeviceFeatures2( case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_EXTENDED_TYPES_FEATURES: { VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *features = (VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *)ext; - features->shaderSubgroupExtendedTypes = !pdevice->use_aco; + CORE_FEATURE(1, 2, shaderSubgroupExtendedTypes); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SEPARATE_DEPTH_STENCIL_LAYOUTS_FEATURES_KHR: { VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *features = (VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *)ext; - features->separateDepthStencilLayouts = true; + CORE_FEATURE(1, 2, separateDepthStencilLayouts); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES: { - VkPhysicalDeviceVulkan11Features *features = - (VkPhysicalDeviceVulkan11Features *)ext; - bool storage16_enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8; - features->storageBuffer16BitAccess = storage16_enable; - features->uniformAndStorageBuffer16BitAccess = storage16_enable; - features->storagePushConstant16 = storage16_enable; - features->storageInputOutput16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco && LLVM_VERSION_MAJOR >= 9; - features->multiview = true; - features->multiviewGeometryShader = true; - features->multiviewTessellationShader = true; - features->variablePointersStorageBuffer = true; - features->variablePointers = true; - features->protectedMemory = false; - features->samplerYcbcrConversion = true; - features->shaderDrawParameters = true; + radv_get_physical_device_features_1_1(pdevice, (void *)ext); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES: { - VkPhysicalDeviceVulkan12Features *features = - (VkPhysicalDeviceVulkan12Features *)ext; - bool int8_enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8; - features->samplerMirrorClampToEdge = true; - features->drawIndirectCount = true; - features->storageBuffer8BitAccess = int8_enable; - features->uniformAndStorageBuffer8BitAccess = int8_enable; - features->storagePushConstant8 = int8_enable; - features->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9; - features->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9; - features->shaderFloat16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco; - features->shaderInt8 = int8_enable; - features->descriptorIndexing = true; - features->shaderInputAttachmentArrayDynamicIndexing = true; - features->shaderUniformTexelBufferArrayDynamicIndexing = true; - features->shaderStorageTexelBufferArrayDynamicIndexing = true; - features->shaderUniformBufferArrayNonUniformIndexing = true; - features->shaderSampledImageArrayNonUniformIndexing = true; - features->shaderStorageBufferArrayNonUniformIndexing = true; - features->shaderStorageImageArrayNonUniformIndexing = true; - features->shaderInputAttachmentArrayNonUniformIndexing = true; - features->shaderUniformTexelBufferArrayNonUniformIndexing = true; - features->shaderStorageTexelBufferArrayNonUniformIndexing = true; - features->descriptorBindingUniformBufferUpdateAfterBind = true; - features->descriptorBindingSampledImageUpdateAfterBind = true; - features->descriptorBindingStorageImageUpdateAfterBind = true; - features->descriptorBindingStorageBufferUpdateAfterBind = true; - features->descriptorBindingUniformTexelBufferUpdateAfterBind = true; - features->descriptorBindingStorageTexelBufferUpdateAfterBind = true; - features->descriptorBindingUpdateUnusedWhilePending = true; - features->descriptorBindingPartiallyBound = true; - features->descriptorBindingVariableDescriptorCount = true; - features->runtimeDescriptorArray = true; - features->samplerFilterMinmax = true; - features->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7; - features->imagelessFramebuffer = true; - features->uniformBufferStandardLayout = true; - features->shaderSubgroupExtendedTypes = !pdevice->use_aco; - features->separateDepthStencilLayouts = true; - features->hostQueryReset = true; - features->timelineSemaphore = pdevice->rad_info.has_syncobj_wait_for_submit; - features->bufferDeviceAddress = true; - features->bufferDeviceAddressCaptureReplay = false; - features->bufferDeviceAddressMultiDevice = false; - features->vulkanMemoryModel = false; - features->vulkanMemoryModelDeviceScope = false; - features->vulkanMemoryModelAvailabilityVisibilityChains = false; - features->shaderOutputViewportIndex = true; - features->shaderOutputLayer = true; - features->subgroupBroadcastDynamicId = true; + radv_get_physical_device_features_1_2(pdevice, (void *)ext); break; } case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_FEATURES_EXT: { @@ -1249,11 +1294,50 @@ void radv_GetPhysicalDeviceFeatures2( features->stippledSmoothLines = false; break; } + case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: { + VkDeviceMemoryOverallocationCreateInfoAMD *features = + (VkDeviceMemoryOverallocationCreateInfoAMD *)ext; + features->overallocationBehavior = true; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT: { + VkPhysicalDeviceRobustness2FeaturesEXT *features = + (VkPhysicalDeviceRobustness2FeaturesEXT *)ext; + features->robustBufferAccess2 = true; + features->robustImageAccess2 = true; + features->nullDescriptor = true; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT: { + VkPhysicalDeviceCustomBorderColorFeaturesEXT *features = + (VkPhysicalDeviceCustomBorderColorFeaturesEXT *)ext; + features->customBorderColors = true; + features->customBorderColorWithoutFormat = true; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRIVATE_DATA_FEATURES_EXT: { + VkPhysicalDevicePrivateDataFeaturesEXT *features = + (VkPhysicalDevicePrivateDataFeaturesEXT *)ext; + features->privateData = true; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES_EXT: { + VkPhysicalDevicePipelineCreationCacheControlFeaturesEXT *features = + (VkPhysicalDevicePipelineCreationCacheControlFeaturesEXT *)ext; + features-> pipelineCreationCacheControl = true; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT: { + VkPhysicalDeviceExtendedDynamicStateFeaturesEXT *features = + (VkPhysicalDeviceExtendedDynamicStateFeaturesEXT *) ext; + features->extendedDynamicState = true; + break; + } default: break; } } - return radv_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features); +#undef CORE_FEATURE } static size_t @@ -1289,7 +1373,7 @@ void radv_GetPhysicalDeviceProperties( .maxImageDimension3D = (1 << 11), .maxImageDimensionCube = (1 << 14), .maxImageArrayLayers = (1 << 11), - .maxTexelBufferElements = 128 * 1024 * 1024, + .maxTexelBufferElements = UINT32_MAX, .maxUniformBufferRange = UINT32_MAX, .maxStorageBufferRange = UINT32_MAX, .maxPushConstantsSize = MAX_PUSH_CONSTANTS_SIZE, @@ -1385,8 +1469,8 @@ void radv_GetPhysicalDeviceProperties( .maxCullDistances = 8, .maxCombinedClipAndCullDistances = 8, .discreteQueuePriorities = 2, - .pointSizeRange = { 0.0, 8192.0 }, - .lineWidthRange = { 0.0, 8192.0 }, + .pointSizeRange = { 0.0, 8191.875 }, + .lineWidthRange = { 0.0, 8191.875 }, .pointSizeGranularity = (1.0 / 8.0), .lineWidthGranularity = (1.0 / 8.0), .strictLines = false, /* FINISHME */ @@ -1431,14 +1515,9 @@ radv_get_physical_device_properties_1_1(struct radv_physical_device *pdevice, VK_SUBGROUP_FEATURE_ARITHMETIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT | VK_SUBGROUP_FEATURE_CLUSTERED_BIT | - VK_SUBGROUP_FEATURE_QUAD_BIT; - - if (((pdevice->rad_info.chip_class == GFX6 || - pdevice->rad_info.chip_class == GFX7) && !pdevice->use_aco) || - pdevice->rad_info.chip_class >= GFX8) { - p->subgroupSupportedOperations |= VK_SUBGROUP_FEATURE_SHUFFLE_BIT | - VK_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT; - } + VK_SUBGROUP_FEATURE_QUAD_BIT | + VK_SUBGROUP_FEATURE_SHUFFLE_BIT | + VK_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT; p->subgroupQuadOperationsInAllStages = true; p->pointClippingBehavior = VK_POINT_CLIPPING_BEHAVIOR_ALL_CLIP_PLANES; @@ -1458,8 +1537,8 @@ radv_get_physical_device_properties_1_2(struct radv_physical_device *pdevice, p->driverID = VK_DRIVER_ID_MESA_RADV; snprintf(p->driverName, VK_MAX_DRIVER_NAME_SIZE, "radv"); snprintf(p->driverInfo, VK_MAX_DRIVER_INFO_SIZE, - "Mesa " PACKAGE_VERSION MESA_GIT_SHA1 - " (LLVM " MESA_LLVM_VERSION_STRING ")"); + "Mesa " PACKAGE_VERSION MESA_GIT_SHA1 " (%s)", + radv_get_compiler_string(pdevice)); p->conformanceVersion = (VkConformanceVersion) { .major = 1, .minor = 2, @@ -1470,7 +1549,7 @@ radv_get_physical_device_properties_1_2(struct radv_physical_device *pdevice, /* On AMD hardware, denormals and rounding modes for fp16/fp64 are * controlled by the same config register. */ - if (pdevice->rad_info.has_double_rate_fp16) { + if (pdevice->rad_info.has_packed_math_16bit) { p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR; p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR; } else { @@ -1493,10 +1572,10 @@ radv_get_physical_device_properties_1_2(struct radv_physical_device *pdevice, p->shaderSignedZeroInfNanPreserveFloat32 = true; p->shaderDenormFlushToZeroFloat16 = false; - p->shaderDenormPreserveFloat16 = pdevice->rad_info.has_double_rate_fp16; - p->shaderRoundingModeRTEFloat16 = pdevice->rad_info.has_double_rate_fp16; + p->shaderDenormPreserveFloat16 = pdevice->rad_info.has_packed_math_16bit; + p->shaderRoundingModeRTEFloat16 = pdevice->rad_info.has_packed_math_16bit; p->shaderRoundingModeRTZFloat16 = false; - p->shaderSignedZeroInfNanPreserveFloat16 = pdevice->rad_info.has_double_rate_fp16; + p->shaderSignedZeroInfNanPreserveFloat16 = pdevice->rad_info.has_packed_math_16bit; p->shaderDenormFlushToZeroFloat64 = false; p->shaderDenormPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8; @@ -1659,7 +1738,7 @@ void radv_GetPhysicalDeviceProperties2( properties->shaderArraysPerEngineCount = pdevice->rad_info.max_sh_per_se; properties->computeUnitsPerShaderArray = - pdevice->rad_info.num_good_cu_per_sh; + pdevice->rad_info.min_good_cu_per_sa; properties->simdPerComputeUnit = pdevice->rad_info.num_simd_per_compute_unit; properties->wavefrontsPerSimd = @@ -1880,6 +1959,19 @@ void radv_GetPhysicalDeviceProperties2( props->lineSubPixelPrecisionBits = 4; break; } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_PROPERTIES_EXT: { + VkPhysicalDeviceRobustness2PropertiesEXT *properties = + (VkPhysicalDeviceRobustness2PropertiesEXT *)ext; + properties->robustStorageBufferAccessSizeAlignment = 4; + properties->robustUniformBufferAccessSizeAlignment = 4; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_PROPERTIES_EXT: { + VkPhysicalDeviceCustomBorderColorPropertiesEXT *props = + (VkPhysicalDeviceCustomBorderColorPropertiesEXT *)ext; + props->maxCustomBorderColorSamplers = RADV_BORDER_COLOR_COUNT; + break; + } default: break; } @@ -2127,10 +2219,11 @@ radv_queue_init(struct radv_device *device, struct radv_queue *queue, queue->queue_idx = idx; queue->priority = radv_get_queue_global_priority(global_priority); queue->flags = flags; + queue->hw_ctx = NULL; - queue->hw_ctx = device->ws->ctx_create(device->ws, queue->priority); - if (!queue->hw_ctx) - return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + VkResult result = device->ws->ctx_create(device->ws, queue->priority, &queue->hw_ctx); + if (result != VK_SUCCESS) + return vk_error(device->instance, result); list_inithead(&queue->pending_submissions); pthread_mutex_init(&queue->pending_mutex, NULL); @@ -2277,533 +2370,6 @@ radv_get_int_debug_option(const char *name, int default_value) return result; } -static int install_seccomp_filter() { - - struct sock_filter filter[] = { - /* Check arch is 64bit x86 */ - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, arch))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 0, 12), - - /* Futex is required for mutex locks */ - #if defined __NR__newselect - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR__newselect, 11, 0), - #elif defined __NR_select - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_select, 11, 0), - #else - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_pselect6, 11, 0), - #endif - - /* Allow system exit calls for the forked process */ - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_exit_group, 9, 0), - - /* Allow system read calls */ - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_read, 7, 0), - - /* Allow system write calls */ - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_write, 5, 0), - - /* Allow system brk calls (we need this for malloc) */ - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_brk, 3, 0), - - /* Futex is required for mutex locks */ - BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))), - BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_futex, 1, 0), - - /* Return error if we hit a system call not on the whitelist */ - BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA)), - - /* Allow whitelisted system calls */ - BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW), - }; - - struct sock_fprog prog = { - .len = (unsigned short)(sizeof(filter) / sizeof(filter[0])), - .filter = filter, - }; - - if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) - return -1; - - if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) - return -1; - - return 0; -} - -/* Helper function with timeout support for reading from the pipe between - * processes used for secure compile. - */ -bool radv_sc_read(int fd, void *buf, size_t size, bool timeout) -{ - fd_set fds; - struct timeval tv; - - FD_ZERO(&fds); - FD_SET(fd, &fds); - - while (true) { - /* We can't rely on the value of tv after calling select() so - * we must reset it on each iteration of the loop. - */ - tv.tv_sec = 5; - tv.tv_usec = 0; - - int rval = select(fd + 1, &fds, NULL, NULL, timeout ? &tv : NULL); - - if (rval == -1) { - /* select error */ - return false; - } else if (rval) { - ssize_t bytes_read = read(fd, buf, size); - if (bytes_read < 0) - return false; - - buf += bytes_read; - size -= bytes_read; - if (size == 0) - return true; - } else { - /* select timeout */ - return false; - } - } -} - -static bool radv_close_all_fds(const int *keep_fds, int keep_fd_count) -{ - DIR *d; - struct dirent *dir; - d = opendir("/proc/self/fd"); - if (!d) - return false; - int dir_fd = dirfd(d); - - while ((dir = readdir(d)) != NULL) { - if (dir->d_name[0] == '.') - continue; - - int fd = atoi(dir->d_name); - if (fd == dir_fd) - continue; - - bool keep = false; - for (int i = 0; !keep && i < keep_fd_count; ++i) - if (keep_fds[i] == fd) - keep = true; - - if (keep) - continue; - - close(fd); - } - closedir(d); - return true; -} - -static bool secure_compile_open_fifo_fds(struct radv_secure_compile_state *sc, - int *fd_server, int *fd_client, - unsigned process, bool make_fifo) -{ - bool result = false; - char *fifo_server_path = NULL; - char *fifo_client_path = NULL; - - if (asprintf(&fifo_server_path, "/tmp/radv_server_%s_%u", sc->uid, process) == -1) - goto open_fifo_exit; - - if (asprintf(&fifo_client_path, "/tmp/radv_client_%s_%u", sc->uid, process) == -1) - goto open_fifo_exit; - - if (make_fifo) { - int file1 = mkfifo(fifo_server_path, 0666); - if(file1 < 0) - goto open_fifo_exit; - - int file2 = mkfifo(fifo_client_path, 0666); - if(file2 < 0) - goto open_fifo_exit; - } - - *fd_server = open(fifo_server_path, O_RDWR); - if(*fd_server < 1) - goto open_fifo_exit; - - *fd_client = open(fifo_client_path, O_RDWR); - if(*fd_client < 1) { - close(*fd_server); - goto open_fifo_exit; - } - - result = true; - -open_fifo_exit: - free(fifo_server_path); - free(fifo_client_path); - - return result; -} - -static void run_secure_compile_device(struct radv_device *device, unsigned process, - int fd_idle_device_output) -{ - int fd_secure_input; - int fd_secure_output; - bool fifo_result = secure_compile_open_fifo_fds(device->sc_state, - &fd_secure_input, - &fd_secure_output, - process, false); - - enum radv_secure_compile_type sc_type; - - const int needed_fds[] = { - fd_secure_input, - fd_secure_output, - fd_idle_device_output, - }; - - if (!fifo_result || !radv_close_all_fds(needed_fds, ARRAY_SIZE(needed_fds)) || - install_seccomp_filter() == -1) { - sc_type = RADV_SC_TYPE_INIT_FAILURE; - } else { - sc_type = RADV_SC_TYPE_INIT_SUCCESS; - device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input; - device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output; - } - - write(fd_idle_device_output, &sc_type, sizeof(sc_type)); - - if (sc_type == RADV_SC_TYPE_INIT_FAILURE) - goto secure_compile_exit; - - while (true) { - radv_sc_read(fd_secure_input, &sc_type, sizeof(sc_type), false); - - if (sc_type == RADV_SC_TYPE_COMPILE_PIPELINE) { - struct radv_pipeline *pipeline; - bool sc_read = true; - - pipeline = vk_zalloc2(&device->alloc, NULL, sizeof(*pipeline), 8, - VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); - - pipeline->device = device; - - /* Read pipeline layout */ - struct radv_pipeline_layout layout; - sc_read = radv_sc_read(fd_secure_input, &layout, sizeof(struct radv_pipeline_layout), true); - sc_read &= radv_sc_read(fd_secure_input, &layout.num_sets, sizeof(uint32_t), true); - if (!sc_read) - goto secure_compile_exit; - - for (uint32_t set = 0; set < layout.num_sets; set++) { - uint32_t layout_size; - sc_read &= radv_sc_read(fd_secure_input, &layout_size, sizeof(uint32_t), true); - if (!sc_read) - goto secure_compile_exit; - - layout.set[set].layout = malloc(layout_size); - layout.set[set].layout->layout_size = layout_size; - sc_read &= radv_sc_read(fd_secure_input, layout.set[set].layout, - layout.set[set].layout->layout_size, true); - } - - pipeline->layout = &layout; - - /* Read pipeline key */ - struct radv_pipeline_key key; - sc_read &= radv_sc_read(fd_secure_input, &key, sizeof(struct radv_pipeline_key), true); - - /* Read pipeline create flags */ - VkPipelineCreateFlags flags; - sc_read &= radv_sc_read(fd_secure_input, &flags, sizeof(VkPipelineCreateFlags), true); - - /* Read stage and shader information */ - uint32_t num_stages; - const VkPipelineShaderStageCreateInfo *pStages[MESA_SHADER_STAGES] = { 0, }; - sc_read &= radv_sc_read(fd_secure_input, &num_stages, sizeof(uint32_t), true); - if (!sc_read) - goto secure_compile_exit; - - for (uint32_t i = 0; i < num_stages; i++) { - - /* Read stage */ - gl_shader_stage stage; - sc_read &= radv_sc_read(fd_secure_input, &stage, sizeof(gl_shader_stage), true); - - VkPipelineShaderStageCreateInfo *pStage = calloc(1, sizeof(VkPipelineShaderStageCreateInfo)); - - /* Read entry point name */ - size_t name_size; - sc_read &= radv_sc_read(fd_secure_input, &name_size, sizeof(size_t), true); - if (!sc_read) - goto secure_compile_exit; - - char *ep_name = malloc(name_size); - sc_read &= radv_sc_read(fd_secure_input, ep_name, name_size, true); - pStage->pName = ep_name; - - /* Read shader module */ - size_t module_size; - sc_read &= radv_sc_read(fd_secure_input, &module_size, sizeof(size_t), true); - if (!sc_read) - goto secure_compile_exit; - - struct radv_shader_module *module = malloc(module_size); - sc_read &= radv_sc_read(fd_secure_input, module, module_size, true); - pStage->module = radv_shader_module_to_handle(module); - - /* Read specialization info */ - bool has_spec_info; - sc_read &= radv_sc_read(fd_secure_input, &has_spec_info, sizeof(bool), true); - if (!sc_read) - goto secure_compile_exit; - - if (has_spec_info) { - VkSpecializationInfo *specInfo = malloc(sizeof(VkSpecializationInfo)); - pStage->pSpecializationInfo = specInfo; - - sc_read &= radv_sc_read(fd_secure_input, &specInfo->dataSize, sizeof(size_t), true); - if (!sc_read) - goto secure_compile_exit; - - void *si_data = malloc(specInfo->dataSize); - sc_read &= radv_sc_read(fd_secure_input, si_data, specInfo->dataSize, true); - specInfo->pData = si_data; - - sc_read &= radv_sc_read(fd_secure_input, &specInfo->mapEntryCount, sizeof(uint32_t), true); - if (!sc_read) - goto secure_compile_exit; - - VkSpecializationMapEntry *mapEntries = malloc(sizeof(VkSpecializationMapEntry) * specInfo->mapEntryCount); - for (uint32_t j = 0; j < specInfo->mapEntryCount; j++) { - sc_read &= radv_sc_read(fd_secure_input, &mapEntries[j], sizeof(VkSpecializationMapEntry), true); - if (!sc_read) - goto secure_compile_exit; - } - - specInfo->pMapEntries = mapEntries; - } - - pStages[stage] = pStage; - } - - /* Compile the shaders */ - VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_SHADER_STAGES] = { 0 }; - radv_create_shaders(pipeline, device, NULL, &key, pStages, flags, NULL, stage_feedbacks); - - /* free memory allocated above */ - for (uint32_t set = 0; set < layout.num_sets; set++) - free(layout.set[set].layout); - - for (uint32_t i = 0; i < MESA_SHADER_STAGES; i++) { - if (!pStages[i]) - continue; - - free((void *) pStages[i]->pName); - free(radv_shader_module_from_handle(pStages[i]->module)); - if (pStages[i]->pSpecializationInfo) { - free((void *) pStages[i]->pSpecializationInfo->pData); - free((void *) pStages[i]->pSpecializationInfo->pMapEntries); - free((void *) pStages[i]->pSpecializationInfo); - } - free((void *) pStages[i]); - } - - vk_free(&device->alloc, pipeline); - - sc_type = RADV_SC_TYPE_COMPILE_PIPELINE_FINISHED; - write(fd_secure_output, &sc_type, sizeof(sc_type)); - - } else if (sc_type == RADV_SC_TYPE_DESTROY_DEVICE) { - goto secure_compile_exit; - } - } - -secure_compile_exit: - close(fd_secure_input); - close(fd_secure_output); - close(fd_idle_device_output); - _exit(0); -} - -static enum radv_secure_compile_type fork_secure_compile_device(struct radv_device *device, unsigned process) -{ - int fd_secure_input[2]; - int fd_secure_output[2]; - - /* create pipe descriptors (used to communicate between processes) */ - if (pipe(fd_secure_input) == -1 || pipe(fd_secure_output) == -1) - return RADV_SC_TYPE_INIT_FAILURE; - - - int sc_pid; - if ((sc_pid = fork()) == 0) { - device->sc_state->secure_compile_thread_counter = process; - run_secure_compile_device(device, process, fd_secure_output[1]); - } else { - if (sc_pid == -1) - return RADV_SC_TYPE_INIT_FAILURE; - - /* Read the init result returned from the secure process */ - enum radv_secure_compile_type sc_type; - bool sc_read = radv_sc_read(fd_secure_output[0], &sc_type, sizeof(sc_type), true); - - if (sc_type == RADV_SC_TYPE_INIT_FAILURE || !sc_read) { - close(fd_secure_input[0]); - close(fd_secure_input[1]); - close(fd_secure_output[1]); - close(fd_secure_output[0]); - int status; - waitpid(sc_pid, &status, 0); - - return RADV_SC_TYPE_INIT_FAILURE; - } else { - assert(sc_type == RADV_SC_TYPE_INIT_SUCCESS); - write(device->sc_state->secure_compile_processes[process].fd_secure_output, &sc_type, sizeof(sc_type)); - - close(fd_secure_input[0]); - close(fd_secure_input[1]); - close(fd_secure_output[1]); - close(fd_secure_output[0]); - - int status; - waitpid(sc_pid, &status, 0); - } - } - - return RADV_SC_TYPE_INIT_SUCCESS; -} - -/* Run a bare bones fork of a device that was forked right after its creation. - * This device will have low overhead when it is forked again before each - * pipeline compilation. This device sits idle and its only job is to fork - * itself. - */ -static void run_secure_compile_idle_device(struct radv_device *device, unsigned process, - int fd_secure_input, int fd_secure_output) -{ - enum radv_secure_compile_type sc_type = RADV_SC_TYPE_INIT_SUCCESS; - device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input; - device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output; - - write(fd_secure_output, &sc_type, sizeof(sc_type)); - - while (true) { - radv_sc_read(fd_secure_input, &sc_type, sizeof(sc_type), false); - - if (sc_type == RADV_SC_TYPE_FORK_DEVICE) { - sc_type = fork_secure_compile_device(device, process); - - if (sc_type == RADV_SC_TYPE_INIT_FAILURE) - goto secure_compile_exit; - - } else if (sc_type == RADV_SC_TYPE_DESTROY_DEVICE) { - goto secure_compile_exit; - } - } - -secure_compile_exit: - close(fd_secure_input); - close(fd_secure_output); - _exit(0); -} - -static void destroy_secure_compile_device(struct radv_device *device, unsigned process) -{ - int fd_secure_input = device->sc_state->secure_compile_processes[process].fd_secure_input; - - enum radv_secure_compile_type sc_type = RADV_SC_TYPE_DESTROY_DEVICE; - write(fd_secure_input, &sc_type, sizeof(sc_type)); - - close(device->sc_state->secure_compile_processes[process].fd_secure_input); - close(device->sc_state->secure_compile_processes[process].fd_secure_output); - - int status; - waitpid(device->sc_state->secure_compile_processes[process].sc_pid, &status, 0); -} - -static VkResult fork_secure_compile_idle_device(struct radv_device *device) -{ - device->sc_state = vk_zalloc(&device->alloc, - sizeof(struct radv_secure_compile_state), - 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); - - mtx_init(&device->sc_state->secure_compile_mutex, mtx_plain); - - pid_t upid = getpid(); - time_t seconds = time(NULL); - - char *uid; - if (asprintf(&uid, "%ld_%ld", (long) upid, (long) seconds) == -1) - return VK_ERROR_INITIALIZATION_FAILED; - - device->sc_state->uid = uid; - - uint8_t sc_threads = device->instance->num_sc_threads; - int fd_secure_input[MAX_SC_PROCS][2]; - int fd_secure_output[MAX_SC_PROCS][2]; - - /* create pipe descriptors (used to communicate between processes) */ - for (unsigned i = 0; i < sc_threads; i++) { - if (pipe(fd_secure_input[i]) == -1 || - pipe(fd_secure_output[i]) == -1) { - return VK_ERROR_INITIALIZATION_FAILED; - } - } - - device->sc_state->secure_compile_processes = vk_zalloc(&device->alloc, - sizeof(struct radv_secure_compile_process) * sc_threads, 8, - VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); - - for (unsigned process = 0; process < sc_threads; process++) { - if ((device->sc_state->secure_compile_processes[process].sc_pid = fork()) == 0) { - device->sc_state->secure_compile_thread_counter = process; - run_secure_compile_idle_device(device, process, fd_secure_input[process][0], fd_secure_output[process][1]); - } else { - if (device->sc_state->secure_compile_processes[process].sc_pid == -1) - return VK_ERROR_INITIALIZATION_FAILED; - - /* Read the init result returned from the secure process */ - enum radv_secure_compile_type sc_type; - bool sc_read = radv_sc_read(fd_secure_output[process][0], &sc_type, sizeof(sc_type), true); - - bool fifo_result; - if (sc_read && sc_type == RADV_SC_TYPE_INIT_SUCCESS) { - fifo_result = secure_compile_open_fifo_fds(device->sc_state, - &device->sc_state->secure_compile_processes[process].fd_server, - &device->sc_state->secure_compile_processes[process].fd_client, - process, true); - - device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input[process][1]; - device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output[process][0]; - } - - if (sc_type == RADV_SC_TYPE_INIT_FAILURE || !sc_read || !fifo_result) { - close(fd_secure_input[process][0]); - close(fd_secure_input[process][1]); - close(fd_secure_output[process][1]); - close(fd_secure_output[process][0]); - int status; - waitpid(device->sc_state->secure_compile_processes[process].sc_pid, &status, 0); - - /* Destroy any forks that were created sucessfully */ - for (unsigned i = 0; i < process; i++) { - destroy_secure_compile_device(device, i); - } - - return VK_ERROR_INITIALIZATION_FAILED; - } - } - } - return VK_SUCCESS; -} - static void radv_device_init_dispatch(struct radv_device *device) { @@ -2875,6 +2441,39 @@ check_physical_device_features(VkPhysicalDevice physicalDevice, return VK_SUCCESS; } +static VkResult radv_device_init_border_color(struct radv_device *device) +{ + device->border_color_data.bo = + device->ws->buffer_create(device->ws, + RADV_BORDER_COLOR_BUFFER_SIZE, + 4096, + RADEON_DOMAIN_VRAM, + RADEON_FLAG_CPU_ACCESS | + RADEON_FLAG_READ_ONLY | + RADEON_FLAG_NO_INTERPROCESS_SHARING, + RADV_BO_PRIORITY_SHADER); + + if (device->border_color_data.bo == NULL) + return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); + + device->border_color_data.colors_gpu_ptr = + device->ws->buffer_map(device->border_color_data.bo); + if (!device->border_color_data.colors_gpu_ptr) + return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); + pthread_mutex_init(&device->border_color_data.mutex, NULL); + + return VK_SUCCESS; +} + +static void radv_device_finish_border_color(struct radv_device *device) +{ + if (device->border_color_data.bo) { + device->ws->buffer_destroy(device->border_color_data.bo); + + pthread_mutex_destroy(&device->border_color_data.mutex); + } +} + VkResult radv_CreateDevice( VkPhysicalDevice physicalDevice, const VkDeviceCreateInfo* pCreateInfo, @@ -2887,6 +2486,8 @@ VkResult radv_CreateDevice( bool keep_shader_info = false; bool robust_buffer_access = false; + bool overallocation_disallowed = false; + bool custom_border_colors = false; /* Check enabled features */ if (pCreateInfo->pEnabledFeatures) { @@ -2912,6 +2513,17 @@ VkResult radv_CreateDevice( robust_buffer_access = true; break; } + case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: { + const VkDeviceMemoryOverallocationCreateInfoAMD *overallocation = (const void *)ext; + if (overallocation->overallocationBehavior == VK_MEMORY_OVERALLOCATION_BEHAVIOR_DISALLOWED_AMD) + overallocation_disallowed = true; + break; + } + case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT: { + const VkPhysicalDeviceCustomBorderColorFeaturesEXT *border_color_features = (const void *)ext; + custom_border_colors = border_color_features->customBorderColors; + break; + } default: break; } @@ -2923,21 +2535,19 @@ VkResult radv_CreateDevice( if (!device) return vk_error(physical_device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); - device->_loader_data.loaderMagic = ICD_LOADER_MAGIC; + vk_device_init(&device->vk, pCreateInfo, + &physical_device->instance->alloc, pAllocator); + device->instance = physical_device->instance; device->physical_device = physical_device; device->ws = physical_device->ws; - if (pAllocator) - device->alloc = *pAllocator; - else - device->alloc = physical_device->instance->alloc; for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) { const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i]; int index = radv_get_device_extension_index(ext_name); if (index < 0 || !physical_device->supported_extensions.extensions[index]) { - vk_free(&device->alloc, device); + vk_free(&device->vk.alloc, device); return vk_error(physical_device->instance, VK_ERROR_EXTENSION_NOT_PRESENT); } @@ -2962,6 +2572,9 @@ VkResult radv_CreateDevice( mtx_init(&device->shader_slab_mutex, mtx_plain); list_inithead(&device->shader_slabs); + device->overallocation_disallowed = overallocation_disallowed; + mtx_init(&device->overallocation_mutex, mtx_plain); + radv_bo_list_init(&device->bo_list); for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) { @@ -2972,7 +2585,7 @@ VkResult radv_CreateDevice( assert(!global_priority || device->physical_device->rad_info.has_ctx_priority); - device->queues[qfi] = vk_alloc(&device->alloc, + device->queues[qfi] = vk_alloc(&device->vk.alloc, queue_create->queueCount * sizeof(struct radv_queue), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!device->queues[qfi]) { result = VK_ERROR_OUT_OF_HOST_MEMORY; @@ -3069,11 +2682,6 @@ VkResult radv_CreateDevice( goto fail; } - /* Temporarily disable secure compile while we create meta shaders, etc */ - uint8_t sc_threads = device->instance->num_sc_threads; - if (sc_threads) - device->instance->num_sc_threads = 0; - device->keep_shader_info = keep_shader_info; result = radv_device_init_meta(device); if (result != VK_SUCCESS) @@ -3081,13 +2689,20 @@ VkResult radv_CreateDevice( radv_device_init_msaa(device); + /* If the border color extension is enabled, let's create the buffer we need. */ + if (custom_border_colors) { + result = radv_device_init_border_color(device); + if (result != VK_SUCCESS) + goto fail; + } + for (int family = 0; family < RADV_MAX_QUEUE_FAMILIES; ++family) { device->empty_cs[family] = device->ws->cs_create(device->ws, family); switch (family) { case RADV_QUEUE_GENERAL: radeon_emit(device->empty_cs[family], PKT3(PKT3_CONTEXT_CONTROL, 1, 0)); - radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_LOAD_ENABLE(1)); - radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_SHADOW_ENABLE(1)); + radeon_emit(device->empty_cs[family], CC0_UPDATE_LOAD_ENABLES(1)); + radeon_emit(device->empty_cs[family], CC1_UPDATE_SHADOW_ENABLES(1)); break; case RADV_QUEUE_COMPUTE: radeon_emit(device->empty_cs[family], PKT3(PKT3_NOP, 0, 0)); @@ -3125,15 +2740,6 @@ VkResult radv_CreateDevice( 1 << util_logbase2(device->force_aniso)); } - /* Fork device for secure compile as required */ - device->instance->num_sc_threads = sc_threads; - if (radv_device_use_secure_compile(device->instance)) { - - result = fork_secure_compile_idle_device(device); - if (result != VK_SUCCESS) - goto fail_meta; - } - *pDevice = radv_device_to_handle(device); return VK_SUCCESS; @@ -3152,14 +2758,16 @@ fail: if (device->gfx_init) device->ws->buffer_destroy(device->gfx_init); + radv_device_finish_border_color(device); + for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) { for (unsigned q = 0; q < device->queue_count[i]; q++) radv_queue_finish(&device->queues[i][q]); if (device->queue_count[i]) - vk_free(&device->alloc, device->queues[i]); + vk_free(&device->vk.alloc, device->queues[i]); } - vk_free(&device->alloc, device); + vk_free(&device->vk.alloc, device); return result; } @@ -3178,11 +2786,13 @@ void radv_DestroyDevice( if (device->gfx_init) device->ws->buffer_destroy(device->gfx_init); + radv_device_finish_border_color(device); + for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) { for (unsigned q = 0; q < device->queue_count[i]; q++) radv_queue_finish(&device->queues[i][q]); if (device->queue_count[i]) - vk_free(&device->alloc, device->queues[i]); + vk_free(&device->vk.alloc, device->queues[i]); if (device->empty_cs[i]) device->ws->cs_destroy(device->empty_cs[i]); } @@ -3198,18 +2808,7 @@ void radv_DestroyDevice( radv_thread_trace_finish(device); - if (radv_device_use_secure_compile(device->instance)) { - for (unsigned i = 0; i < device->instance->num_sc_threads; i++ ) { - destroy_secure_compile_device(device, i); - } - } - - if (device->sc_state) { - free(device->sc_state->uid); - vk_free(&device->alloc, device->sc_state->secure_compile_processes); - } - vk_free(&device->alloc, device->sc_state); - vk_free(&device->alloc, device); + vk_free(&device->vk.alloc, device); } VkResult radv_EnumerateInstanceLayerProperties( @@ -3507,7 +3106,10 @@ radv_get_hs_offchip_param(struct radv_device *device, uint32_t *max_offchip_buff } *max_offchip_buffers_p = max_offchip_buffers; - if (device->physical_device->rad_info.chip_class >= GFX7) { + if (device->physical_device->rad_info.chip_class >= GFX10_3) { + hs_offchip_param = S_03093C_OFFCHIP_BUFFERING_GFX103(max_offchip_buffers - 1) | + S_03093C_OFFCHIP_GRANULARITY_GFX103(offchip_granularity); + } else if (device->physical_device->rad_info.chip_class >= GFX7) { if (device->physical_device->rad_info.chip_class >= GFX8) --max_offchip_buffers; hs_offchip_param = @@ -3918,6 +3520,8 @@ radv_get_preamble_cs(struct radv_queue *queue, if (descriptor_bo != queue->descriptor_bo) { uint32_t *map = (uint32_t*)queue->device->ws->buffer_map(descriptor_bo); + if (!map) + goto fail; if (scratch_bo) { uint64_t scratch_va = radv_buffer_get_va(scratch_bo); @@ -3989,6 +3593,10 @@ radv_get_preamble_cs(struct radv_queue *queue, if (queue->device->trace_bo) radv_cs_add_buffer(queue->device->ws, cs, queue->device->trace_bo); + if (queue->device->border_color_data.bo) + radv_cs_add_buffer(queue->device->ws, cs, + queue->device->border_color_data.bo); + if (i == 0) { si_cs_emit_cache_flush(cs, queue->device->physical_device->rad_info.chip_class, @@ -4014,7 +3622,7 @@ radv_get_preamble_cs(struct radv_queue *queue, RADV_CMD_FLAG_START_PIPELINE_STATS, 0); } - if (!queue->device->ws->cs_finalize(cs)) + if (queue->device->ws->cs_finalize(cs) != VK_SUCCESS) goto fail; } @@ -4862,9 +4470,11 @@ VkResult radv_QueueWaitIdle( } pthread_mutex_unlock(&queue->pending_mutex); - queue->device->ws->ctx_wait_idle(queue->hw_ctx, - radv_queue_family_to_ring(queue->queue_family_index), - queue->queue_idx); + if (!queue->device->ws->ctx_wait_idle(queue->hw_ctx, + radv_queue_family_to_ring(queue->queue_family_index), + queue->queue_idx)) + return VK_ERROR_DEVICE_LOST; + return VK_SUCCESS; } @@ -4875,7 +4485,11 @@ VkResult radv_DeviceWaitIdle( for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) { for (unsigned q = 0; q < device->queue_count[i]; q++) { - radv_QueueWaitIdle(radv_queue_to_handle(&device->queues[i][q])); + VkResult result = + radv_QueueWaitIdle(radv_queue_to_handle(&device->queues[i][q])); + + if (result != VK_SUCCESS) + return result; } } return VK_SUCCESS; @@ -4889,7 +4503,7 @@ VkResult radv_EnumerateInstanceExtensionProperties( VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount); for (int i = 0; i < RADV_INSTANCE_EXTENSION_COUNT; i++) { - if (radv_supported_instance_extensions.extensions[i]) { + if (radv_instance_extensions_supported.extensions[i]) { vk_outarray_append(&out, prop) { *prop = radv_instance_extensions[i]; } @@ -5037,9 +4651,10 @@ bool radv_get_memory_fd(struct radv_device *device, } -static void radv_free_memory(struct radv_device *device, - const VkAllocationCallbacks* pAllocator, - struct radv_device_memory *mem) +void +radv_free_memory(struct radv_device *device, + const VkAllocationCallbacks* pAllocator, + struct radv_device_memory *mem) { if (mem == NULL) return; @@ -5050,12 +4665,19 @@ static void radv_free_memory(struct radv_device *device, #endif if (mem->bo) { + if (device->overallocation_disallowed) { + mtx_lock(&device->overallocation_mutex); + device->allocated_memory_size[mem->heap_index] -= mem->alloc_size; + mtx_unlock(&device->overallocation_mutex); + } + radv_bo_list_remove(device, mem->bo); device->ws->buffer_destroy(mem->bo); mem->bo = NULL; } - vk_free2(&device->alloc, pAllocator, mem); + vk_object_base_finish(&mem->base); + vk_free2(&device->vk.alloc, pAllocator, mem); } static VkResult radv_alloc_memory(struct radv_device *device, @@ -5092,11 +4714,14 @@ static VkResult radv_alloc_memory(struct radv_device *device, return VK_SUCCESS; } - mem = vk_zalloc2(&device->alloc, pAllocator, sizeof(*mem), 8, + mem = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (mem == NULL) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &mem->base, + VK_OBJECT_TYPE_DEVICE_MEMORY); + if (wsi_info && wsi_info->implicit_sync) flags |= RADEON_FLAG_IMPLICIT_SYNC; @@ -5159,8 +4784,11 @@ static VkResult radv_alloc_memory(struct radv_device *device, } } else { uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096); + uint32_t heap_index; + + heap_index = device->physical_device->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex; domain = device->physical_device->memory_domains[pAllocateInfo->memoryTypeIndex]; - flags = device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex]; + flags |= device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex]; if (!dedicate_info && !import_info && (!export_info || !export_info->handleTypes)) { flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING; @@ -5169,13 +4797,35 @@ static VkResult radv_alloc_memory(struct radv_device *device, } } + if (device->overallocation_disallowed) { + uint64_t total_size = + device->physical_device->memory_properties.memoryHeaps[heap_index].size; + + mtx_lock(&device->overallocation_mutex); + if (device->allocated_memory_size[heap_index] + alloc_size > total_size) { + mtx_unlock(&device->overallocation_mutex); + result = VK_ERROR_OUT_OF_DEVICE_MEMORY; + goto fail; + } + device->allocated_memory_size[heap_index] += alloc_size; + mtx_unlock(&device->overallocation_mutex); + } + mem->bo = device->ws->buffer_create(device->ws, alloc_size, device->physical_device->rad_info.max_alignment, domain, flags, priority); if (!mem->bo) { + if (device->overallocation_disallowed) { + mtx_lock(&device->overallocation_mutex); + device->allocated_memory_size[heap_index] -= alloc_size; + mtx_unlock(&device->overallocation_mutex); + } result = VK_ERROR_OUT_OF_DEVICE_MEMORY; goto fail; } + + mem->heap_index = heap_index; + mem->alloc_size = alloc_size; } if (!wsi_info) { @@ -5512,6 +5162,24 @@ static bool radv_sparse_bind_has_effects(const VkBindSparseInfo *info) return VK_SUCCESS; } +static void +radv_destroy_fence(struct radv_device *device, + const VkAllocationCallbacks *pAllocator, + struct radv_fence *fence) +{ + if (fence->temp_syncobj) + device->ws->destroy_syncobj(device->ws, fence->temp_syncobj); + if (fence->syncobj) + device->ws->destroy_syncobj(device->ws, fence->syncobj); + if (fence->fence) + device->ws->destroy_fence(fence->fence); + if (fence->fence_wsi) + fence->fence_wsi->destroy(fence->fence_wsi); + + vk_object_base_finish(&fence->base); + vk_free2(&device->vk.alloc, pAllocator, fence); +} + VkResult radv_CreateFence( VkDevice _device, const VkFenceCreateInfo* pCreateInfo, @@ -5524,19 +5192,21 @@ VkResult radv_CreateFence( VkExternalFenceHandleTypeFlags handleTypes = export ? export->handleTypes : 0; - struct radv_fence *fence = vk_alloc2(&device->alloc, pAllocator, + struct radv_fence *fence = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*fence), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!fence) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &fence->base, VK_OBJECT_TYPE_FENCE); + fence->fence_wsi = NULL; fence->temp_syncobj = 0; if (device->always_use_syncobj || handleTypes) { int ret = device->ws->create_syncobj(device->ws, &fence->syncobj); if (ret) { - vk_free2(&device->alloc, pAllocator, fence); + radv_destroy_fence(device, pAllocator, fence); return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); } if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) { @@ -5546,7 +5216,7 @@ VkResult radv_CreateFence( } else { fence->fence = device->ws->create_fence(); if (!fence->fence) { - vk_free2(&device->alloc, pAllocator, fence); + radv_destroy_fence(device, pAllocator, fence); return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); } fence->syncobj = 0; @@ -5570,15 +5240,7 @@ void radv_DestroyFence( if (!fence) return; - if (fence->temp_syncobj) - device->ws->destroy_syncobj(device->ws, fence->temp_syncobj); - if (fence->syncobj) - device->ws->destroy_syncobj(device->ws, fence->syncobj); - if (fence->fence) - device->ws->destroy_fence(fence->fence); - if (fence->fence_wsi) - fence->fence_wsi->destroy(fence->fence_wsi); - vk_free2(&device->alloc, pAllocator, fence); + radv_destroy_fence(device, pAllocator, fence); } @@ -5976,6 +5638,17 @@ radv_get_semaphore_type(const void *pNext, uint64_t *initial_value) return type_info->semaphoreType; } +static void +radv_destroy_semaphore(struct radv_device *device, + const VkAllocationCallbacks *pAllocator, + struct radv_semaphore *sem) +{ + radv_destroy_semaphore_part(device, &sem->temporary); + radv_destroy_semaphore_part(device, &sem->permanent); + vk_object_base_finish(&sem->base); + vk_free2(&device->vk.alloc, pAllocator, sem); +} + VkResult radv_CreateSemaphore( VkDevice _device, const VkSemaphoreCreateInfo* pCreateInfo, @@ -5990,12 +5663,15 @@ VkResult radv_CreateSemaphore( uint64_t initial_value = 0; VkSemaphoreTypeKHR type = radv_get_semaphore_type(pCreateInfo->pNext, &initial_value); - struct radv_semaphore *sem = vk_alloc2(&device->alloc, pAllocator, + struct radv_semaphore *sem = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*sem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!sem) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &sem->base, + VK_OBJECT_TYPE_SEMAPHORE); + sem->temporary.kind = RADV_SEMAPHORE_NONE; sem->permanent.kind = RADV_SEMAPHORE_NONE; @@ -6006,14 +5682,14 @@ VkResult radv_CreateSemaphore( assert (device->physical_device->rad_info.has_syncobj); int ret = device->ws->create_syncobj(device->ws, &sem->permanent.syncobj); if (ret) { - vk_free2(&device->alloc, pAllocator, sem); + radv_destroy_semaphore(device, pAllocator, sem); return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); } sem->permanent.kind = RADV_SEMAPHORE_SYNCOBJ; } else { sem->permanent.ws_sem = device->ws->create_sem(device->ws); if (!sem->permanent.ws_sem) { - vk_free2(&device->alloc, pAllocator, sem); + radv_destroy_semaphore(device, pAllocator, sem); return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); } sem->permanent.kind = RADV_SEMAPHORE_WINSYS; @@ -6033,9 +5709,7 @@ void radv_DestroySemaphore( if (!_semaphore) return; - radv_destroy_semaphore_part(device, &sem->temporary); - radv_destroy_semaphore_part(device, &sem->permanent); - vk_free2(&device->alloc, pAllocator, sem); + radv_destroy_semaphore(device, pAllocator, sem); } VkResult @@ -6140,7 +5814,16 @@ radv_SignalSemaphore(VkDevice _device, return VK_SUCCESS; } +static void radv_destroy_event(struct radv_device *device, + const VkAllocationCallbacks* pAllocator, + struct radv_event *event) +{ + if (event->bo) + device->ws->buffer_destroy(event->bo); + vk_object_base_finish(&event->base); + vk_free2(&device->vk.alloc, pAllocator, event); +} VkResult radv_CreateEvent( VkDevice _device, @@ -6149,23 +5832,29 @@ VkResult radv_CreateEvent( VkEvent* pEvent) { RADV_FROM_HANDLE(radv_device, device, _device); - struct radv_event *event = vk_alloc2(&device->alloc, pAllocator, + struct radv_event *event = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*event), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!event) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &event->base, VK_OBJECT_TYPE_EVENT); + event->bo = device->ws->buffer_create(device->ws, 8, 8, RADEON_DOMAIN_GTT, RADEON_FLAG_VA_UNCACHED | RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING, RADV_BO_PRIORITY_FENCE); if (!event->bo) { - vk_free2(&device->alloc, pAllocator, event); + radv_destroy_event(device, pAllocator, event); return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); } event->map = (uint64_t*)device->ws->buffer_map(event->bo); + if (!event->map) { + radv_destroy_event(device, pAllocator, event); + return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); + } *pEvent = radv_event_to_handle(event); @@ -6182,8 +5871,8 @@ void radv_DestroyEvent( if (!event) return; - device->ws->buffer_destroy(event->bo); - vk_free2(&device->alloc, pAllocator, event); + + radv_destroy_event(device, pAllocator, event); } VkResult radv_GetEventStatus( @@ -6217,6 +5906,18 @@ VkResult radv_ResetEvent( return VK_SUCCESS; } +static void +radv_destroy_buffer(struct radv_device *device, + const VkAllocationCallbacks *pAllocator, + struct radv_buffer *buffer) +{ + if ((buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) && buffer->bo) + device->ws->buffer_destroy(buffer->bo); + + vk_object_base_finish(&buffer->base); + vk_free2(&device->vk.alloc, pAllocator, buffer); +} + VkResult radv_CreateBuffer( VkDevice _device, const VkBufferCreateInfo* pCreateInfo, @@ -6231,11 +5932,13 @@ VkResult radv_CreateBuffer( assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO); - buffer = vk_alloc2(&device->alloc, pAllocator, sizeof(*buffer), 8, + buffer = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*buffer), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (buffer == NULL) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &buffer->base, VK_OBJECT_TYPE_BUFFER); + buffer->size = pCreateInfo->size; buffer->usage = pCreateInfo->usage; buffer->bo = NULL; @@ -6251,7 +5954,7 @@ VkResult radv_CreateBuffer( 4096, 0, RADEON_FLAG_VIRTUAL, RADV_BO_PRIORITY_VIRTUAL); if (!buffer->bo) { - vk_free2(&device->alloc, pAllocator, buffer); + radv_destroy_buffer(device, pAllocator, buffer); return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY); } } @@ -6272,10 +5975,7 @@ void radv_DestroyBuffer( if (!buffer) return; - if (buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) - device->ws->buffer_destroy(buffer->bo); - - vk_free2(&device->alloc, pAllocator, buffer); + radv_destroy_buffer(device, pAllocator, buffer); } VkDeviceAddress radv_GetBufferDeviceAddress( @@ -6398,18 +6098,20 @@ radv_initialise_color_surface(struct radv_device *device, cb->cb_color_base = va >> 8; if (device->physical_device->rad_info.chip_class >= GFX9) { - struct gfx9_surf_meta_flags meta; - if (iview->image->dcc_offset) - meta = surf->u.gfx9.dcc; - else - meta = surf->u.gfx9.cmask; - if (device->physical_device->rad_info.chip_class >= GFX10) { cb->cb_color_attrib3 |= S_028EE0_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) | S_028EE0_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) | - S_028EE0_CMASK_PIPE_ALIGNED(surf->u.gfx9.cmask.pipe_aligned) | + S_028EE0_CMASK_PIPE_ALIGNED(1) | S_028EE0_DCC_PIPE_ALIGNED(surf->u.gfx9.dcc.pipe_aligned); } else { + struct gfx9_surf_meta_flags meta = { + .rb_aligned = 1, + .pipe_aligned = 1, + }; + + if (surf->dcc_offset) + meta = surf->u.gfx9.dcc; + cb->cb_color_attrib |= S_028C74_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) | S_028C74_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) | S_028C74_RB_ALIGNED(meta.rb_aligned) | @@ -6453,11 +6155,11 @@ radv_initialise_color_surface(struct radv_device *device, /* CMASK variables */ va = radv_buffer_get_va(iview->bo) + iview->image->offset; - va += iview->image->cmask_offset; + va += surf->cmask_offset; cb->cb_color_cmask = va >> 8; va = radv_buffer_get_va(iview->bo) + iview->image->offset; - va += iview->image->dcc_offset; + va += surf->dcc_offset; if (radv_dcc_enabled(iview->image, iview->base_mip) && device->physical_device->rad_info.chip_class <= GFX8) @@ -6482,7 +6184,7 @@ radv_initialise_color_surface(struct radv_device *device, } if (radv_image_has_fmask(iview->image)) { - va = radv_buffer_get_va(iview->bo) + iview->image->offset + iview->image->fmask_offset; + va = radv_buffer_get_va(iview->bo) + iview->image->offset + surf->fmask_offset; cb->cb_color_fmask = va >> 8; cb->cb_color_fmask |= surf->fmask_tile_swizzle; } else { @@ -6734,13 +6436,13 @@ radv_initialise_ds_surface(struct radv_device *device, /* Use all of the htile_buffer for depth if there's no stencil. */ ds->db_stencil_info |= S_02803C_TILE_STENCIL_DISABLE(1); va = radv_buffer_get_va(iview->bo) + iview->image->offset + - iview->image->htile_offset; + surf->htile_offset; ds->db_htile_data_base = va >> 8; ds->db_htile_surface = S_028ABC_FULL_CACHE(1) | - S_028ABC_PIPE_ALIGNED(surf->u.gfx9.htile.pipe_aligned); + S_028ABC_PIPE_ALIGNED(1); if (device->physical_device->rad_info.chip_class == GFX9) { - ds->db_htile_surface |= S_028ABC_RB_ALIGNED(surf->u.gfx9.htile.rb_aligned); + ds->db_htile_surface |= S_028ABC_RB_ALIGNED(1); } } } else { @@ -6802,7 +6504,7 @@ radv_initialise_ds_surface(struct radv_device *device, ds->db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1); va = radv_buffer_get_va(iview->bo) + iview->image->offset + - iview->image->htile_offset; + surf->htile_offset; ds->db_htile_data_base = va >> 8; ds->db_htile_surface = S_028ABC_FULL_CACHE(1); @@ -6837,11 +6539,14 @@ VkResult radv_CreateFramebuffer( size_t size = sizeof(*framebuffer); if (!imageless_create_info) size += sizeof(struct radv_image_view*) * pCreateInfo->attachmentCount; - framebuffer = vk_alloc2(&device->alloc, pAllocator, size, 8, + framebuffer = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (framebuffer == NULL) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &framebuffer->base, + VK_OBJECT_TYPE_FRAMEBUFFER); + framebuffer->attachment_count = pCreateInfo->attachmentCount; framebuffer->width = pCreateInfo->width; framebuffer->height = pCreateInfo->height; @@ -6879,7 +6584,8 @@ void radv_DestroyFramebuffer( if (!fb) return; - vk_free2(&device->alloc, pAllocator, fb); + vk_object_base_finish(&fb->base); + vk_free2(&device->vk.alloc, pAllocator, fb); } static unsigned radv_tex_wrap(VkSamplerAddressMode address_mode) @@ -6970,6 +6676,9 @@ radv_tex_bordercolor(VkBorderColor bcolor) case VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE: case VK_BORDER_COLOR_INT_OPAQUE_WHITE: return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE; + case VK_BORDER_COLOR_FLOAT_CUSTOM_EXT: + case VK_BORDER_COLOR_INT_CUSTOM_EXT: + return V_008F3C_SQ_TEX_BORDER_COLOR_REGISTER; default: break; } @@ -7025,6 +6734,40 @@ static inline int S_FIXED(float value, unsigned frac_bits) return value * (1 << frac_bits); } +static uint32_t radv_register_border_color(struct radv_device *device, + VkClearColorValue value) +{ + uint32_t slot; + + pthread_mutex_lock(&device->border_color_data.mutex); + + for (slot = 0; slot < RADV_BORDER_COLOR_COUNT; slot++) { + if (!device->border_color_data.used[slot]) { + /* Copy to the GPU wrt endian-ness. */ + util_memcpy_cpu_to_le32(&device->border_color_data.colors_gpu_ptr[slot], + &value, + sizeof(VkClearColorValue)); + + device->border_color_data.used[slot] = true; + break; + } + } + + pthread_mutex_unlock(&device->border_color_data.mutex); + + return slot; +} + +static void radv_unregister_border_color(struct radv_device *device, + uint32_t slot) +{ + pthread_mutex_lock(&device->border_color_data.mutex); + + device->border_color_data.used[slot] = false; + + pthread_mutex_unlock(&device->border_color_data.mutex); +} + static void radv_init_sampler(struct radv_device *device, struct radv_sampler *sampler, @@ -7037,6 +6780,11 @@ radv_init_sampler(struct radv_device *device, unsigned filter_mode = V_008F30_SQ_IMG_FILTER_MODE_BLEND; unsigned depth_compare_func = V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER; bool trunc_coord = pCreateInfo->minFilter == VK_FILTER_NEAREST && pCreateInfo->magFilter == VK_FILTER_NEAREST; + bool uses_border_color = pCreateInfo->addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER || + pCreateInfo->addressModeV == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER || + pCreateInfo->addressModeW == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; + VkBorderColor border_color = uses_border_color ? pCreateInfo->borderColor : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; + uint32_t border_color_ptr; const struct VkSamplerReductionModeCreateInfo *sampler_reduction = vk_find_struct_const(pCreateInfo->pNext, @@ -7047,6 +6795,30 @@ radv_init_sampler(struct radv_device *device, if (pCreateInfo->compareEnable) depth_compare_func = radv_tex_compare(pCreateInfo->compareOp); + sampler->border_color_slot = RADV_BORDER_COLOR_COUNT; + + if (border_color == VK_BORDER_COLOR_FLOAT_CUSTOM_EXT || border_color == VK_BORDER_COLOR_INT_CUSTOM_EXT) { + const VkSamplerCustomBorderColorCreateInfoEXT *custom_border_color = + vk_find_struct_const(pCreateInfo->pNext, + SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT); + + assert(custom_border_color); + + sampler->border_color_slot = + radv_register_border_color(device, custom_border_color->customBorderColor); + + /* Did we fail to find a slot? */ + if (sampler->border_color_slot == RADV_BORDER_COLOR_COUNT) { + fprintf(stderr, "WARNING: no free border color slots, defaulting to TRANS_BLACK.\n"); + border_color = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; + } + } + + /* If we don't have a custom color, set the ptr to 0 */ + border_color_ptr = sampler->border_color_slot != RADV_BORDER_COLOR_COUNT + ? sampler->border_color_slot + : 0; + sampler->state[0] = (S_008F30_CLAMP_X(radv_tex_wrap(pCreateInfo->addressModeU)) | S_008F30_CLAMP_Y(radv_tex_wrap(pCreateInfo->addressModeV)) | S_008F30_CLAMP_Z(radv_tex_wrap(pCreateInfo->addressModeW)) | @@ -7067,8 +6839,8 @@ radv_init_sampler(struct radv_device *device, S_008F38_XY_MIN_FILTER(radv_tex_filter(pCreateInfo->minFilter, max_aniso)) | S_008F38_MIP_FILTER(radv_tex_mipfilter(pCreateInfo->mipmapMode)) | S_008F38_MIP_POINT_PRECLAMP(0)); - sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(0) | - S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(pCreateInfo->borderColor))); + sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(border_color_ptr) | + S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(border_color))); if (device->physical_device->rad_info.chip_class >= GFX10) { sampler->state[2] |= S_008F38_ANISO_OVERRIDE_GFX10(1); @@ -7095,11 +6867,14 @@ VkResult radv_CreateSampler( assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO); - sampler = vk_alloc2(&device->alloc, pAllocator, sizeof(*sampler), 8, + sampler = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*sampler), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!sampler) return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + vk_object_base_init(&device->vk, &sampler->base, + VK_OBJECT_TYPE_SAMPLER); + radv_init_sampler(device, sampler, pCreateInfo); sampler->ycbcr_sampler = ycbcr_conversion ? radv_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion): NULL; @@ -7118,7 +6893,12 @@ void radv_DestroySampler( if (!sampler) return; - vk_free2(&device->alloc, pAllocator, sampler); + + if (sampler->border_color_slot != RADV_BORDER_COLOR_COUNT) + radv_unregister_border_color(device, sampler->border_color_slot); + + vk_object_base_finish(&sampler->base); + vk_free2(&device->vk.alloc, pAllocator, sampler); } /* vk_icd.h does not declare this function, so we declare it here to @@ -7186,28 +6966,74 @@ VkResult radv_GetMemoryFdKHR(VkDevice _device, return VK_SUCCESS; } +static uint32_t radv_compute_valid_memory_types_attempt(struct radv_physical_device *dev, + enum radeon_bo_domain domains, + enum radeon_bo_flag flags, + enum radeon_bo_flag ignore_flags) +{ + /* Don't count GTT/CPU as relevant: + * + * - We're not fully consistent between the two. + * - Sometimes VRAM gets VRAM|GTT. + */ + const enum radeon_bo_domain relevant_domains = RADEON_DOMAIN_VRAM | + RADEON_DOMAIN_GDS | + RADEON_DOMAIN_OA; + uint32_t bits = 0; + for (unsigned i = 0; i < dev->memory_properties.memoryTypeCount; ++i) { + if ((domains & relevant_domains) != (dev->memory_domains[i] & relevant_domains)) + continue; + + if ((flags & ~ignore_flags) != (dev->memory_flags[i] & ~ignore_flags)) + continue; + + bits |= 1u << i; + } + + return bits; +} + +static uint32_t radv_compute_valid_memory_types(struct radv_physical_device *dev, + enum radeon_bo_domain domains, + enum radeon_bo_flag flags) +{ + enum radeon_bo_flag ignore_flags = ~(RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_GTT_WC); + uint32_t bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags); + + if (!bits) { + ignore_flags |= RADEON_FLAG_NO_CPU_ACCESS; + bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags); + } + + return bits; +} VkResult radv_GetMemoryFdPropertiesKHR(VkDevice _device, VkExternalMemoryHandleTypeFlagBits handleType, int fd, VkMemoryFdPropertiesKHR *pMemoryFdProperties) { - RADV_FROM_HANDLE(radv_device, device, _device); + RADV_FROM_HANDLE(radv_device, device, _device); - switch (handleType) { - case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: - pMemoryFdProperties->memoryTypeBits = (1 << device->physical_device->memory_properties.memoryTypeCount) - 1; - return VK_SUCCESS; + switch (handleType) { + case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: { + enum radeon_bo_domain domains; + enum radeon_bo_flag flags; + if (!device->ws->buffer_get_flags_from_fd(device->ws, fd, &domains, &flags)) + return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); - default: - /* The valid usage section for this function says: - * - * "handleType must not be one of the handle types defined as - * opaque." - * - * So opaque handle types fall into the default "unsupported" case. - */ - return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); - } + pMemoryFdProperties->memoryTypeBits = radv_compute_valid_memory_types(device->physical_device, domains, flags); + return VK_SUCCESS; + } + default: + /* The valid usage section for this function says: + * + * "handleType must not be one of the handle types defined as + * opaque." + * + * So opaque handle types fall into the default "unsupported" case. + */ + return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); + } } static VkResult radv_import_opaque_fd(struct radv_device *device, @@ -7238,7 +7064,7 @@ static VkResult radv_import_sync_fd(struct radv_device *device, if (!syncobj_handle) { int ret = device->ws->create_syncobj(device->ws, &syncobj_handle); if (ret) { - return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); + return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY); } } @@ -7312,23 +7138,24 @@ VkResult radv_GetSemaphoreFdKHR(VkDevice _device, switch(pGetFdInfo->handleType) { case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd); + if (ret) + return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS); break; case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd); - if (!ret) { - if (sem->temporary.kind != RADV_SEMAPHORE_NONE) { - radv_destroy_semaphore_part(device, &sem->temporary); - } else { - device->ws->reset_syncobj(device->ws, syncobj_handle); - } + if (ret) + return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS); + + if (sem->temporary.kind != RADV_SEMAPHORE_NONE) { + radv_destroy_semaphore_part(device, &sem->temporary); + } else { + device->ws->reset_syncobj(device->ws, syncobj_handle); } break; default: unreachable("Unhandled semaphore handle type"); } - if (ret) - return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); return VK_SUCCESS; } @@ -7406,24 +7233,25 @@ VkResult radv_GetFenceFdKHR(VkDevice _device, switch(pGetFdInfo->handleType) { case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd); + if (ret) + return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS); break; case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd); - if (!ret) { - if (fence->temp_syncobj) { - close (fence->temp_syncobj); - fence->temp_syncobj = 0; - } else { - device->ws->reset_syncobj(device->ws, syncobj_handle); - } + if (ret) + return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS); + + if (fence->temp_syncobj) { + device->ws->destroy_syncobj(device->ws, fence->temp_syncobj); + fence->temp_syncobj = 0; + } else { + device->ws->reset_syncobj(device->ws, syncobj_handle); } break; default: unreachable("Unhandled fence handle type"); } - if (ret) - return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); return VK_SUCCESS; } @@ -7633,3 +7461,48 @@ void radv_GetPhysicalDeviceMultisamplePropertiesEXT( pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 0, 0 }; } } + +VkResult radv_CreatePrivateDataSlotEXT( + VkDevice _device, + const VkPrivateDataSlotCreateInfoEXT* pCreateInfo, + const VkAllocationCallbacks* pAllocator, + VkPrivateDataSlotEXT* pPrivateDataSlot) +{ + RADV_FROM_HANDLE(radv_device, device, _device); + return vk_private_data_slot_create(&device->vk, pCreateInfo, pAllocator, + pPrivateDataSlot); +} + +void radv_DestroyPrivateDataSlotEXT( + VkDevice _device, + VkPrivateDataSlotEXT privateDataSlot, + const VkAllocationCallbacks* pAllocator) +{ + RADV_FROM_HANDLE(radv_device, device, _device); + vk_private_data_slot_destroy(&device->vk, privateDataSlot, pAllocator); +} + +VkResult radv_SetPrivateDataEXT( + VkDevice _device, + VkObjectType objectType, + uint64_t objectHandle, + VkPrivateDataSlotEXT privateDataSlot, + uint64_t data) +{ + RADV_FROM_HANDLE(radv_device, device, _device); + return vk_object_base_set_private_data(&device->vk, objectType, + objectHandle, privateDataSlot, + data); +} + +void radv_GetPrivateDataEXT( + VkDevice _device, + VkObjectType objectType, + uint64_t objectHandle, + VkPrivateDataSlotEXT privateDataSlot, + uint64_t* pData) +{ + RADV_FROM_HANDLE(radv_device, device, _device); + vk_object_base_get_private_data(&device->vk, objectType, objectHandle, + privateDataSlot, pData); +}