src/amd/vulkan/radv_device.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  25  * IN THE SOFTWARE.
  26  */
  27
  28 #include "dirent.h"
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <linux/audit.h>
  32 #include <linux/bpf.h>
  33 #include <linux/filter.h>
  34 #include <linux/seccomp.h>
  35 #include <linux/unistd.h>
  36 #include <stdatomic.h>
  37 #include <stdbool.h>
  38 #include <stddef.h>
  39 #include <stdio.h>
  40 #include <string.h>
  41 #include <sys/prctl.h>
  42 #include <sys/wait.h>
  43 #include <unistd.h>
  44 #include <fcntl.h>
  45
  46 #include "radv_debug.h"
  47 #include "radv_private.h"
  48 #include "radv_shader.h"
  49 #include "radv_cs.h"
  50 #include "util/disk_cache.h"
  51 #include "vk_util.h"
  52 #include <xf86drm.h>
  53 #include <amdgpu.h>
  54 #include "drm-uapi/amdgpu_drm.h"
  55 #include "winsys/amdgpu/radv_amdgpu_winsys_public.h"
  56 #include "winsys/null/radv_null_winsys_public.h"
  57 #include "ac_llvm_util.h"
  58 #include "vk_format.h"
  59 #include "sid.h"
  60 #include "git_sha1.h"
  61 #include "util/build_id.h"
  62 #include "util/debug.h"
  63 #include "util/mesa-sha1.h"
  64 #include "util/timespec.h"
  65 #include "util/u_atomic.h"
  66 #include "compiler/glsl_types.h"
  67 #include "util/driconf.h"
  68
  69 static struct radv_timeline_point *
  70 radv_timeline_find_point_at_least_locked(struct radv_device *device,
  71                                          struct radv_timeline *timeline,
  72                                          uint64_t p);
  73
  74 static struct radv_timeline_point *
  75 radv_timeline_add_point_locked(struct radv_device *device,
  76                                struct radv_timeline *timeline,
  77                                uint64_t p);
  78
  79 static void
  80 radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline,
  81                                      struct list_head *processing_list);
  82
  83 static
  84 void radv_destroy_semaphore_part(struct radv_device *device,
  85                                  struct radv_semaphore_part *part);
  86
  87 static VkResult
  88 radv_create_pthread_cond(pthread_cond_t *cond);
  89
  90 uint64_t radv_get_current_time(void)
  91 {
  92         struct timespec tv;
  93         clock_gettime(CLOCK_MONOTONIC, &tv);
  94         return tv.tv_nsec + tv.tv_sec*1000000000ull;
  95 }
  96
  97 static uint64_t radv_get_absolute_timeout(uint64_t timeout)
  98 {
  99         uint64_t current_time = radv_get_current_time();
 100
 101         timeout = MIN2(UINT64_MAX - current_time, timeout);
 102
 103         return current_time + timeout;
 104 }
 105
 106 static int
 107 radv_device_get_cache_uuid(enum radeon_family family, void *uuid)
 108 {
 109         struct mesa_sha1 ctx;
 110         unsigned char sha1[20];
 111         unsigned ptr_size = sizeof(void*);
 112
 113         memset(uuid, 0, VK_UUID_SIZE);
 114         _mesa_sha1_init(&ctx);
 115
 116         if (!disk_cache_get_function_identifier(radv_device_get_cache_uuid, &ctx) ||
 117             !disk_cache_get_function_identifier(LLVMInitializeAMDGPUTargetInfo, &ctx))
 118                 return -1;
 119
 120         _mesa_sha1_update(&ctx, &family, sizeof(family));
 121         _mesa_sha1_update(&ctx, &ptr_size, sizeof(ptr_size));
 122         _mesa_sha1_final(&ctx, sha1);
 123
 124         memcpy(uuid, sha1, VK_UUID_SIZE);
 125         return 0;
 126 }
 127
 128 static void
 129 radv_get_driver_uuid(void *uuid)
 130 {
 131         ac_compute_driver_uuid(uuid, VK_UUID_SIZE);
 132 }
 133
 134 static void
 135 radv_get_device_uuid(struct radeon_info *info, void *uuid)
 136 {
 137         ac_compute_device_uuid(info, uuid, VK_UUID_SIZE);
 138 }
 139
 140 static uint64_t
 141 radv_get_visible_vram_size(struct radv_physical_device *device)
 142 {
 143         return MIN2(device->rad_info.vram_size, device->rad_info.vram_vis_size);
 144 }
 145
 146 static uint64_t
 147 radv_get_vram_size(struct radv_physical_device *device)
 148 {
 149         return device->rad_info.vram_size - radv_get_visible_vram_size(device);
 150 }
 151
 152 static void
 153 radv_physical_device_init_mem_types(struct radv_physical_device *device)
 154 {
 155         uint64_t visible_vram_size = radv_get_visible_vram_size(device);
 156         uint64_t vram_size = radv_get_vram_size(device);
 157         int vram_index = -1, visible_vram_index = -1, gart_index = -1;
 158         device->memory_properties.memoryHeapCount = 0;
 159         if (vram_size > 0) {
 160                 vram_index = device->memory_properties.memoryHeapCount++;
 161                 device->memory_properties.memoryHeaps[vram_index] = (VkMemoryHeap) {
 162                         .size = vram_size,
 163                         .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
 164                 };
 165         }
 166
 167         if (device->rad_info.gart_size > 0) {
 168                 gart_index = device->memory_properties.memoryHeapCount++;
 169                 device->memory_properties.memoryHeaps[gart_index] = (VkMemoryHeap) {
 170                         .size = device->rad_info.gart_size,
 171                         .flags = 0,
 172                 };
 173         }
 174
 175         if (visible_vram_size) {
 176                 visible_vram_index = device->memory_properties.memoryHeapCount++;
 177                 device->memory_properties.memoryHeaps[visible_vram_index] = (VkMemoryHeap) {
 178                         .size = visible_vram_size,
 179                         .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
 180                 };
 181         }
 182
 183         unsigned type_count = 0;
 184
 185         if (vram_index >= 0 || visible_vram_index >= 0) {
 186                 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
 187                 device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS;
 188                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 189                         .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
 190                         .heapIndex = vram_index >= 0 ? vram_index : visible_vram_index,
 191                 };
 192         }
 193
 194         if (gart_index >= 0) {
 195                 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
 196                 device->memory_flags[type_count] = RADEON_FLAG_GTT_WC | RADEON_FLAG_CPU_ACCESS;
 197                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 198                         .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 199                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 200                         .heapIndex = gart_index,
 201                 };
 202         }
 203         if (visible_vram_index >= 0) {
 204                 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
 205                 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
 206                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 207                         .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
 208                         VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 209                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 210                         .heapIndex = visible_vram_index,
 211                 };
 212         }
 213
 214         if (gart_index >= 0) {
 215                 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
 216                 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
 217                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 218                         .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 219                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
 220                         VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
 221                         .heapIndex = gart_index,
 222                 };
 223         }
 224         device->memory_properties.memoryTypeCount = type_count;
 225
 226         if (device->rad_info.has_l2_uncached) {
 227                 for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
 228                         VkMemoryType mem_type = device->memory_properties.memoryTypes[i];
 229
 230                         if ((mem_type.propertyFlags & (VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
 231                                                        VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) ||
 232                             mem_type.propertyFlags == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
 233
 234                                 VkMemoryPropertyFlags property_flags = mem_type.propertyFlags |
 235                                         VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD |
 236                                         VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD;
 237
 238                                 device->memory_domains[type_count] = device->memory_domains[i];
 239                                 device->memory_flags[type_count] = device->memory_flags[i] | RADEON_FLAG_VA_UNCACHED;
 240                                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 241                                         .propertyFlags = property_flags,
 242                                         .heapIndex = mem_type.heapIndex,
 243                                 };
 244                         }
 245                 }
 246                 device->memory_properties.memoryTypeCount = type_count;
 247         }
 248 }
 249
 250 static const char *
 251 radv_get_compiler_string(struct radv_physical_device *pdevice)
 252 {
 253         if (!pdevice->use_llvm) {
 254                 /* Some games like SotTR apply shader workarounds if the LLVM
 255                  * version is too old or if the LLVM version string is
 256                  * missing. This gives 2-5% performance with SotTR and ACO.
 257                  */
 258                 if (driQueryOptionb(&pdevice->instance->dri_options,
 259                                     "radv_report_llvm9_version_string")) {
 260                         return "ACO/LLVM 9.0.1";
 261                 }
 262
 263                 return "ACO";
 264         }
 265
 266         return "LLVM " MESA_LLVM_VERSION_STRING;
 267 }
 268
 269 static VkResult
 270 radv_physical_device_try_create(struct radv_instance *instance,
 271                                 drmDevicePtr drm_device,
 272                                 struct radv_physical_device **device_out)
 273 {
 274         VkResult result;
 275         int fd = -1;
 276         int master_fd = -1;
 277
 278         if (drm_device) {
 279                 const char *path = drm_device->nodes[DRM_NODE_RENDER];
 280                 drmVersionPtr version;
 281
 282                 fd = open(path, O_RDWR | O_CLOEXEC);
 283                 if (fd < 0) {
 284                         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 285                                 radv_logi("Could not open device '%s'", path);
 286
 287                         return vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER);
 288                 }
 289
 290                 version = drmGetVersion(fd);
 291                 if (!version) {
 292                         close(fd);
 293
 294                         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 295                                 radv_logi("Could not get the kernel driver version for device '%s'", path);
 296
 297                         return vk_errorf(instance, VK_ERROR_INCOMPATIBLE_DRIVER,
 298                                          "failed to get version %s: %m", path);
 299                 }
 300
 301                 if (strcmp(version->name, "amdgpu")) {
 302                         drmFreeVersion(version);
 303                         close(fd);
 304
 305                         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 306                                 radv_logi("Device '%s' is not using the amdgpu kernel driver.", path);
 307
 308                         return VK_ERROR_INCOMPATIBLE_DRIVER;
 309                 }
 310                 drmFreeVersion(version);
 311
 312                 if (instance->debug_flags & RADV_DEBUG_STARTUP)
 313                                 radv_logi("Found compatible device '%s'.", path);
 314         }
 315
 316         struct radv_physical_device *device =
 317                 vk_zalloc2(&instance->alloc, NULL, sizeof(*device), 8,
 318                            VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 319         if (!device) {
 320                 result = vk_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 321                 goto fail_fd;
 322         }
 323
 324         device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 325         device->instance = instance;
 326
 327         if (drm_device) {
 328                 device->ws = radv_amdgpu_winsys_create(fd, instance->debug_flags,
 329                                                        instance->perftest_flags);
 330         } else {
 331                 device->ws = radv_null_winsys_create();
 332         }
 333
 334         if (!device->ws) {
 335                 result = vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED,
 336                                    "failed to initialize winsys");
 337                 goto fail_alloc;
 338         }
 339
 340         if (drm_device && instance->enabled_extensions.KHR_display) {
 341                 master_fd = open(drm_device->nodes[DRM_NODE_PRIMARY], O_RDWR | O_CLOEXEC);
 342                 if (master_fd >= 0) {
 343                         uint32_t accel_working = 0;
 344                         struct drm_amdgpu_info request = {
 345                                 .return_pointer = (uintptr_t)&accel_working,
 346                                 .return_size = sizeof(accel_working),
 347                                 .query = AMDGPU_INFO_ACCEL_WORKING
 348                         };
 349
 350                         if (drmCommandWrite(master_fd, DRM_AMDGPU_INFO, &request, sizeof (struct drm_amdgpu_info)) < 0 || !accel_working) {
 351                                 close(master_fd);
 352                                 master_fd = -1;
 353                         }
 354                 }
 355         }
 356
 357         device->master_fd = master_fd;
 358         device->local_fd = fd;
 359         device->ws->query_info(device->ws, &device->rad_info);
 360
 361         device->use_llvm = instance->debug_flags & RADV_DEBUG_LLVM;
 362
 363         snprintf(device->name, sizeof(device->name),
 364                  "AMD RADV %s (%s)",
 365                  device->rad_info.name, radv_get_compiler_string(device));
 366
 367         if (radv_device_get_cache_uuid(device->rad_info.family, device->cache_uuid)) {
 368                 result = vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED,
 369                                    "cannot generate UUID");
 370                 goto fail_wsi;
 371         }
 372
 373         /* These flags affect shader compilation. */
 374         uint64_t shader_env_flags = (device->use_llvm ? 0 : 0x2);
 375
 376         /* The gpu id is already embedded in the uuid so we just pass "radv"
 377          * when creating the cache.
 378          */
 379         char buf[VK_UUID_SIZE * 2 + 1];
 380         disk_cache_format_hex_id(buf, device->cache_uuid, VK_UUID_SIZE * 2);
 381         device->disk_cache = disk_cache_create(device->name, buf, shader_env_flags);
 382
 383         if (device->rad_info.chip_class < GFX8)
 384                 fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n");
 385
 386         radv_get_driver_uuid(&device->driver_uuid);
 387         radv_get_device_uuid(&device->rad_info, &device->device_uuid);
 388
 389         device->out_of_order_rast_allowed = device->rad_info.has_out_of_order_rast &&
 390                                             !(device->instance->debug_flags & RADV_DEBUG_NO_OUT_OF_ORDER);
 391
 392         device->dcc_msaa_allowed =
 393                 (device->instance->perftest_flags & RADV_PERFTEST_DCC_MSAA);
 394
 395         device->use_ngg = device->rad_info.chip_class >= GFX10 &&
 396                           device->rad_info.family != CHIP_NAVI14 &&
 397                           !(device->instance->debug_flags & RADV_DEBUG_NO_NGG);
 398
 399         /* TODO: Implement NGG GS with ACO. */
 400         device->use_ngg_gs = device->use_ngg && device->use_llvm;
 401         device->use_ngg_streamout = false;
 402
 403         /* Determine the number of threads per wave for all stages. */
 404         device->cs_wave_size = 64;
 405         device->ps_wave_size = 64;
 406         device->ge_wave_size = 64;
 407
 408         if (device->rad_info.chip_class >= GFX10) {
 409                 if (device->instance->perftest_flags & RADV_PERFTEST_CS_WAVE_32)
 410                         device->cs_wave_size = 32;
 411
 412                 /* For pixel shaders, wave64 is recommanded. */
 413                 if (device->instance->perftest_flags & RADV_PERFTEST_PS_WAVE_32)
 414                         device->ps_wave_size = 32;
 415
 416                 if (device->instance->perftest_flags & RADV_PERFTEST_GE_WAVE_32)
 417                         device->ge_wave_size = 32;
 418         }
 419
 420         radv_physical_device_init_mem_types(device);
 421
 422         radv_physical_device_get_supported_extensions(device,
 423                                                       &device->supported_extensions);
 424
 425         if (drm_device)
 426                 device->bus_info = *drm_device->businfo.pci;
 427
 428         if ((device->instance->debug_flags & RADV_DEBUG_INFO))
 429                 ac_print_gpu_info(&device->rad_info);
 430
 431         /* The WSI is structured as a layer on top of the driver, so this has
 432          * to be the last part of initialization (at least until we get other
 433          * semi-layers).
 434          */
 435         result = radv_init_wsi(device);
 436         if (result != VK_SUCCESS) {
 437                 vk_error(instance, result);
 438                 goto fail_disk_cache;
 439         }
 440
 441         *device_out = device;
 442
 443         return VK_SUCCESS;
 444
 445 fail_disk_cache:
 446         disk_cache_destroy(device->disk_cache);
 447 fail_wsi:
 448         device->ws->destroy(device->ws);
 449 fail_alloc:
 450         vk_free(&instance->alloc, device);
 451 fail_fd:
 452         if (fd != -1)
 453                 close(fd);
 454         if (master_fd != -1)
 455                 close(master_fd);
 456         return result;
 457 }
 458
 459 static void
 460 radv_physical_device_destroy(struct radv_physical_device *device)
 461 {
 462         radv_finish_wsi(device);
 463         device->ws->destroy(device->ws);
 464         disk_cache_destroy(device->disk_cache);
 465         close(device->local_fd);
 466         if (device->master_fd != -1)
 467                 close(device->master_fd);
 468         vk_free(&device->instance->alloc, device);
 469 }
 470
 471 static void *
 472 default_alloc_func(void *pUserData, size_t size, size_t align,
 473                    VkSystemAllocationScope allocationScope)
 474 {
 475         return malloc(size);
 476 }
 477
 478 static void *
 479 default_realloc_func(void *pUserData, void *pOriginal, size_t size,
 480                      size_t align, VkSystemAllocationScope allocationScope)
 481 {
 482         return realloc(pOriginal, size);
 483 }
 484
 485 static void
 486 default_free_func(void *pUserData, void *pMemory)
 487 {
 488         free(pMemory);
 489 }
 490
 491 static const VkAllocationCallbacks default_alloc = {
 492         .pUserData = NULL,
 493         .pfnAllocation = default_alloc_func,
 494         .pfnReallocation = default_realloc_func,
 495         .pfnFree = default_free_func,
 496 };
 497
 498 static const struct debug_control radv_debug_options[] = {
 499         {"nofastclears", RADV_DEBUG_NO_FAST_CLEARS},
 500         {"nodcc", RADV_DEBUG_NO_DCC},
 501         {"shaders", RADV_DEBUG_DUMP_SHADERS},
 502         {"nocache", RADV_DEBUG_NO_CACHE},
 503         {"shaderstats", RADV_DEBUG_DUMP_SHADER_STATS},
 504         {"nohiz", RADV_DEBUG_NO_HIZ},
 505         {"nocompute", RADV_DEBUG_NO_COMPUTE_QUEUE},
 506         {"allbos", RADV_DEBUG_ALL_BOS},
 507         {"noibs", RADV_DEBUG_NO_IBS},
 508         {"spirv", RADV_DEBUG_DUMP_SPIRV},
 509         {"vmfaults", RADV_DEBUG_VM_FAULTS},
 510         {"zerovram", RADV_DEBUG_ZERO_VRAM},
 511         {"syncshaders", RADV_DEBUG_SYNC_SHADERS},
 512         {"preoptir", RADV_DEBUG_PREOPTIR},
 513         {"nodynamicbounds", RADV_DEBUG_NO_DYNAMIC_BOUNDS},
 514         {"nooutoforder", RADV_DEBUG_NO_OUT_OF_ORDER},
 515         {"info", RADV_DEBUG_INFO},
 516         {"errors", RADV_DEBUG_ERRORS},
 517         {"startup", RADV_DEBUG_STARTUP},
 518         {"checkir", RADV_DEBUG_CHECKIR},
 519         {"nothreadllvm", RADV_DEBUG_NOTHREADLLVM},
 520         {"nobinning", RADV_DEBUG_NOBINNING},
 521         {"nongg", RADV_DEBUG_NO_NGG},
 522         {"allentrypoints", RADV_DEBUG_ALL_ENTRYPOINTS},
 523         {"metashaders", RADV_DEBUG_DUMP_META_SHADERS},
 524         {"nomemorycache", RADV_DEBUG_NO_MEMORY_CACHE},
 525         {"llvm", RADV_DEBUG_LLVM},
 526         {"forcecompress", RADV_DEBUG_FORCE_COMPRESS},
 527         {NULL, 0}
 528 };
 529
 530 const char *
 531 radv_get_debug_option_name(int id)
 532 {
 533         assert(id < ARRAY_SIZE(radv_debug_options) - 1);
 534         return radv_debug_options[id].string;
 535 }
 536
 537 static const struct debug_control radv_perftest_options[] = {
 538         {"localbos", RADV_PERFTEST_LOCAL_BOS},
 539         {"dccmsaa", RADV_PERFTEST_DCC_MSAA},
 540         {"bolist", RADV_PERFTEST_BO_LIST},
 541         {"tccompatcmask", RADV_PERFTEST_TC_COMPAT_CMASK},
 542         {"cswave32", RADV_PERFTEST_CS_WAVE_32},
 543         {"pswave32", RADV_PERFTEST_PS_WAVE_32},
 544         {"gewave32", RADV_PERFTEST_GE_WAVE_32},
 545         {"dfsm", RADV_PERFTEST_DFSM},
 546         {NULL, 0}
 547 };
 548
 549 const char *
 550 radv_get_perftest_option_name(int id)
 551 {
 552         assert(id < ARRAY_SIZE(radv_perftest_options) - 1);
 553         return radv_perftest_options[id].string;
 554 }
 555
 556 static void
 557 radv_handle_per_app_options(struct radv_instance *instance,
 558                             const VkApplicationInfo *info)
 559 {
 560         const char *name = info ? info->pApplicationName : NULL;
 561         const char *engine_name = info ? info->pEngineName : NULL;
 562
 563         if (name) {
 564                 if (!strcmp(name, "DOOM_VFR")) {
 565                         /* Work around a Doom VFR game bug */
 566                         instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS;
 567                 } else if (!strcmp(name, "Fledge")) {
 568                         /*
 569                          * Zero VRAM for "The Surge 2"
 570                          *
 571                          * This avoid a hang when when rendering any level. Likely
 572                          * uninitialized data in an indirect draw.
 573                          */
 574                         instance->debug_flags |= RADV_DEBUG_ZERO_VRAM;
 575                 } else if (!strcmp(name, "No Man's Sky")) {
 576                         /* Work around a NMS game bug */
 577                         instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE;
 578                 } else if (!strcmp(name, "DOOMEternal")) {
 579                         /* Zero VRAM for Doom Eternal to fix rendering issues. */
 580                         instance->debug_flags |= RADV_DEBUG_ZERO_VRAM;
 581                 } else if (!strcmp(name, "Red Dead Redemption 2")) {
 582                         /* Work around a RDR2 game bug */
 583                         instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE;
 584                 }
 585         }
 586
 587         if (engine_name) {
 588                 if (!strcmp(engine_name, "vkd3d")) {
 589                         /* Zero VRAM for all VKD3D (DX12->VK) games to fix
 590                          * rendering issues.
 591                          */
 592                         instance->debug_flags |= RADV_DEBUG_ZERO_VRAM;
 593                 } else if (!strcmp(engine_name, "Quantic Dream Engine")) {
 594                         /* Fix various artifacts in Detroit: Become Human */
 595                         instance->debug_flags |= RADV_DEBUG_ZERO_VRAM |
 596                                                  RADV_DEBUG_DISCARD_TO_DEMOTE;
 597                 }
 598         }
 599
 600         instance->enable_mrt_output_nan_fixup =
 601                 driQueryOptionb(&instance->dri_options,
 602                                 "radv_enable_mrt_output_nan_fixup");
 603
 604         if (driQueryOptionb(&instance->dri_options, "radv_no_dynamic_bounds"))
 605                 instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS;
 606 }
 607
 608 static const char radv_dri_options_xml[] =
 609 DRI_CONF_BEGIN
 610         DRI_CONF_SECTION_PERFORMANCE
 611                 DRI_CONF_ADAPTIVE_SYNC("true")
 612                 DRI_CONF_VK_X11_OVERRIDE_MIN_IMAGE_COUNT(0)
 613                 DRI_CONF_VK_X11_STRICT_IMAGE_COUNT("false")
 614                 DRI_CONF_VK_X11_ENSURE_MIN_IMAGE_COUNT("false")
 615                 DRI_CONF_RADV_REPORT_LLVM9_VERSION_STRING("false")
 616                 DRI_CONF_RADV_ENABLE_MRT_OUTPUT_NAN_FIXUP("false")
 617                 DRI_CONF_RADV_NO_DYNAMIC_BOUNDS("false")
 618                 DRI_CONF_RADV_OVERRIDE_UNIFORM_OFFSET_ALIGNMENT(0)
 619         DRI_CONF_SECTION_END
 620
 621         DRI_CONF_SECTION_DEBUG
 622                 DRI_CONF_VK_WSI_FORCE_BGRA8_UNORM_FIRST("false")
 623         DRI_CONF_SECTION_END
 624 DRI_CONF_END;
 625
 626 static void  radv_init_dri_options(struct radv_instance *instance)
 627 {
 628         driParseOptionInfo(&instance->available_dri_options, radv_dri_options_xml);
 629         driParseConfigFiles(&instance->dri_options,
 630                             &instance->available_dri_options,
 631                             0, "radv", NULL,
 632                             instance->applicationName,
 633                             instance->applicationVersion,
 634                             instance->engineName,
 635                             instance->engineVersion);
 636 }
 637
 638 VkResult radv_CreateInstance(
 639         const VkInstanceCreateInfo*                 pCreateInfo,
 640         const VkAllocationCallbacks*                pAllocator,
 641         VkInstance*                                 pInstance)
 642 {
 643         struct radv_instance *instance;
 644         VkResult result;
 645
 646         instance = vk_zalloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
 647                               VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 648         if (!instance)
 649                 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
 650
 651         vk_object_base_init(NULL, &instance->base, VK_OBJECT_TYPE_INSTANCE);
 652
 653         if (pAllocator)
 654                 instance->alloc = *pAllocator;
 655         else
 656                 instance->alloc = default_alloc;
 657
 658         if (pCreateInfo->pApplicationInfo) {
 659                 const VkApplicationInfo *app = pCreateInfo->pApplicationInfo;
 660
 661                 instance->applicationName =
 662                         vk_strdup(&instance->alloc, app->pApplicationName,
 663                                   VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 664                 instance->applicationVersion = app->applicationVersion;
 665
 666                 instance->engineName =
 667                         vk_strdup(&instance->alloc, app->pEngineName,
 668                                   VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 669                 instance->engineVersion = app->engineVersion;
 670                 instance->apiVersion = app->apiVersion;
 671         }
 672
 673         if (instance->apiVersion == 0)
 674                 instance->apiVersion = VK_API_VERSION_1_0;
 675
 676         instance->debug_flags = parse_debug_string(getenv("RADV_DEBUG"),
 677                                                    radv_debug_options);
 678
 679         const char *radv_perftest_str = getenv("RADV_PERFTEST");
 680         instance->perftest_flags = parse_debug_string(radv_perftest_str,
 681                                                       radv_perftest_options);
 682
 683         if (radv_perftest_str) {
 684                 /* Output warnings for famous RADV_PERFTEST options that no
 685                  * longer exist or are deprecated.
 686                  */
 687                 if (strstr(radv_perftest_str, "aco")) {
 688                         fprintf(stderr, "*******************************************************************************\n");
 689                         fprintf(stderr, "* WARNING: Unknown option RADV_PERFTEST='aco'. ACO is enabled by default now. *\n");
 690                         fprintf(stderr, "*******************************************************************************\n");
 691                 }
 692                 if (strstr(radv_perftest_str, "llvm")) {
 693                         fprintf(stderr, "*********************************************************************************\n");
 694                         fprintf(stderr, "* WARNING: Unknown option 'RADV_PERFTEST=llvm'. Did you mean 'RADV_DEBUG=llvm'? *\n");
 695                         fprintf(stderr, "*********************************************************************************\n");
 696                         abort();
 697                 }
 698         }
 699
 700         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 701                 radv_logi("Created an instance");
 702
 703         for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
 704                 int idx;
 705                 for (idx = 0; idx < RADV_INSTANCE_EXTENSION_COUNT; idx++) {
 706                         if (!strcmp(pCreateInfo->ppEnabledExtensionNames[i],
 707                                     radv_instance_extensions[idx].extensionName))
 708                                 break;
 709                 }
 710
 711                 if (idx >= RADV_INSTANCE_EXTENSION_COUNT ||
 712                     !radv_instance_extensions_supported.extensions[idx]) {
 713                         vk_object_base_finish(&instance->base);
 714                         vk_free2(&default_alloc, pAllocator, instance);
 715                         return vk_error(instance, VK_ERROR_EXTENSION_NOT_PRESENT);
 716                 }
 717
 718                 instance->enabled_extensions.extensions[idx] = true;
 719         }
 720
 721         bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
 722
 723         for (unsigned i = 0; i < ARRAY_SIZE(instance->dispatch.entrypoints); i++) {
 724                 /* Vulkan requires that entrypoints for extensions which have
 725                  * not been enabled must not be advertised.
 726                  */
 727                 if (!unchecked &&
 728                     !radv_instance_entrypoint_is_enabled(i, instance->apiVersion,
 729                                                          &instance->enabled_extensions)) {
 730                         instance->dispatch.entrypoints[i] = NULL;
 731                 } else {
 732                         instance->dispatch.entrypoints[i] =
 733                                 radv_instance_dispatch_table.entrypoints[i];
 734                 }
 735         }
 736
 737          for (unsigned i = 0; i < ARRAY_SIZE(instance->physical_device_dispatch.entrypoints); i++) {
 738                 /* Vulkan requires that entrypoints for extensions which have
 739                  * not been enabled must not be advertised.
 740                  */
 741                 if (!unchecked &&
 742                     !radv_physical_device_entrypoint_is_enabled(i, instance->apiVersion,
 743                                                                 &instance->enabled_extensions)) {
 744                         instance->physical_device_dispatch.entrypoints[i] = NULL;
 745                 } else {
 746                         instance->physical_device_dispatch.entrypoints[i] =
 747                                 radv_physical_device_dispatch_table.entrypoints[i];
 748                 }
 749         }
 750
 751         for (unsigned i = 0; i < ARRAY_SIZE(instance->device_dispatch.entrypoints); i++) {
 752                 /* Vulkan requires that entrypoints for extensions which have
 753                  * not been enabled must not be advertised.
 754                  */
 755                 if (!unchecked &&
 756                     !radv_device_entrypoint_is_enabled(i, instance->apiVersion,
 757                                                        &instance->enabled_extensions, NULL)) {
 758                         instance->device_dispatch.entrypoints[i] = NULL;
 759                 } else {
 760                         instance->device_dispatch.entrypoints[i] =
 761                                 radv_device_dispatch_table.entrypoints[i];
 762                 }
 763         }
 764
 765         instance->physical_devices_enumerated = false;
 766         list_inithead(&instance->physical_devices);
 767
 768         result = vk_debug_report_instance_init(&instance->debug_report_callbacks);
 769         if (result != VK_SUCCESS) {
 770                 vk_object_base_finish(&instance->base);
 771                 vk_free2(&default_alloc, pAllocator, instance);
 772                 return vk_error(instance, result);
 773         }
 774
 775         glsl_type_singleton_init_or_ref();
 776
 777         VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false));
 778
 779         radv_init_dri_options(instance);
 780         radv_handle_per_app_options(instance, pCreateInfo->pApplicationInfo);
 781
 782         *pInstance = radv_instance_to_handle(instance);
 783
 784         return VK_SUCCESS;
 785 }
 786
 787 void radv_DestroyInstance(
 788         VkInstance                                  _instance,
 789         const VkAllocationCallbacks*                pAllocator)
 790 {
 791         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 792
 793         if (!instance)
 794                 return;
 795
 796         list_for_each_entry_safe(struct radv_physical_device, pdevice,
 797                                  &instance->physical_devices, link) {
 798                 radv_physical_device_destroy(pdevice);
 799         }
 800
 801         vk_free(&instance->alloc, instance->engineName);
 802         vk_free(&instance->alloc, instance->applicationName);
 803
 804         VG(VALGRIND_DESTROY_MEMPOOL(instance));
 805
 806         glsl_type_singleton_decref();
 807
 808         driDestroyOptionCache(&instance->dri_options);
 809         driDestroyOptionInfo(&instance->available_dri_options);
 810
 811         vk_debug_report_instance_destroy(&instance->debug_report_callbacks);
 812
 813         vk_object_base_finish(&instance->base);
 814         vk_free(&instance->alloc, instance);
 815 }
 816
 817 static VkResult
 818 radv_enumerate_physical_devices(struct radv_instance *instance)
 819 {
 820         if (instance->physical_devices_enumerated)
 821                 return VK_SUCCESS;
 822
 823         instance->physical_devices_enumerated = true;
 824
 825         /* TODO: Check for more devices ? */
 826         drmDevicePtr devices[8];
 827         VkResult result = VK_SUCCESS;
 828         int max_devices;
 829
 830         if (getenv("RADV_FORCE_FAMILY")) {
 831                 /* When RADV_FORCE_FAMILY is set, the driver creates a nul
 832                  * device that allows to test the compiler without having an
 833                  * AMDGPU instance.
 834                  */
 835                 struct radv_physical_device *pdevice;
 836
 837                 result = radv_physical_device_try_create(instance, NULL, &pdevice);
 838                 if (result != VK_SUCCESS)
 839                         return result;
 840
 841                 list_addtail(&pdevice->link, &instance->physical_devices);
 842                 return VK_SUCCESS;
 843         }
 844
 845         max_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices));
 846
 847         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 848                 radv_logi("Found %d drm nodes", max_devices);
 849
 850         if (max_devices < 1)
 851                 return vk_error(instance, VK_SUCCESS);
 852
 853         for (unsigned i = 0; i < (unsigned)max_devices; i++) {
 854                 if (devices[i]->available_nodes & 1 << DRM_NODE_RENDER &&
 855                     devices[i]->bustype == DRM_BUS_PCI &&
 856                     devices[i]->deviceinfo.pci->vendor_id == ATI_VENDOR_ID) {
 857
 858                         struct radv_physical_device *pdevice;
 859                         result = radv_physical_device_try_create(instance, devices[i],
 860                                                                  &pdevice);
 861                         /* Incompatible DRM device, skip. */
 862                         if (result == VK_ERROR_INCOMPATIBLE_DRIVER) {
 863                                 result = VK_SUCCESS;
 864                                 continue;
 865                         }
 866
 867                         /* Error creating the physical device, report the error. */
 868                         if (result != VK_SUCCESS)
 869                                 break;
 870
 871                         list_addtail(&pdevice->link, &instance->physical_devices);
 872                 }
 873         }
 874         drmFreeDevices(devices, max_devices);
 875
 876         /* If we successfully enumerated any devices, call it success */
 877         return result;
 878 }
 879
 880 VkResult radv_EnumeratePhysicalDevices(
 881         VkInstance                                  _instance,
 882         uint32_t*                                   pPhysicalDeviceCount,
 883         VkPhysicalDevice*                           pPhysicalDevices)
 884 {
 885         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 886         VK_OUTARRAY_MAKE(out, pPhysicalDevices, pPhysicalDeviceCount);
 887
 888         VkResult result = radv_enumerate_physical_devices(instance);
 889         if (result != VK_SUCCESS)
 890                 return result;
 891
 892         list_for_each_entry(struct radv_physical_device, pdevice,
 893                             &instance->physical_devices, link) {
 894                 vk_outarray_append(&out, i) {
 895                         *i = radv_physical_device_to_handle(pdevice);
 896                 }
 897         }
 898
 899         return vk_outarray_status(&out);
 900 }
 901
 902 VkResult radv_EnumeratePhysicalDeviceGroups(
 903     VkInstance                                  _instance,
 904     uint32_t*                                   pPhysicalDeviceGroupCount,
 905     VkPhysicalDeviceGroupProperties*            pPhysicalDeviceGroupProperties)
 906 {
 907         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 908         VK_OUTARRAY_MAKE(out, pPhysicalDeviceGroupProperties,
 909                               pPhysicalDeviceGroupCount);
 910
 911         VkResult result = radv_enumerate_physical_devices(instance);
 912         if (result != VK_SUCCESS)
 913                 return result;
 914
 915         list_for_each_entry(struct radv_physical_device, pdevice,
 916                             &instance->physical_devices, link) {
 917                 vk_outarray_append(&out, p) {
 918                         p->physicalDeviceCount = 1;
 919                         memset(p->physicalDevices, 0, sizeof(p->physicalDevices));
 920                         p->physicalDevices[0] = radv_physical_device_to_handle(pdevice);
 921                         p->subsetAllocation = false;
 922                 }
 923         }
 924
 925         return vk_outarray_status(&out);
 926 }
 927
 928 void radv_GetPhysicalDeviceFeatures(
 929         VkPhysicalDevice                            physicalDevice,
 930         VkPhysicalDeviceFeatures*                   pFeatures)
 931 {
 932         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
 933         memset(pFeatures, 0, sizeof(*pFeatures));
 934
 935         *pFeatures = (VkPhysicalDeviceFeatures) {
 936                 .robustBufferAccess                       = true,
 937                 .fullDrawIndexUint32                      = true,
 938                 .imageCubeArray                           = true,
 939                 .independentBlend                         = true,
 940                 .geometryShader                           = true,
 941                 .tessellationShader                       = true,
 942                 .sampleRateShading                        = true,
 943                 .dualSrcBlend                             = true,
 944                 .logicOp                                  = true,
 945                 .multiDrawIndirect                        = true,
 946                 .drawIndirectFirstInstance                = true,
 947                 .depthClamp                               = true,
 948                 .depthBiasClamp                           = true,
 949                 .fillModeNonSolid                         = true,
 950                 .depthBounds                              = true,
 951                 .wideLines                                = true,
 952                 .largePoints                              = true,
 953                 .alphaToOne                               = true,
 954                 .multiViewport                            = true,
 955                 .samplerAnisotropy                        = true,
 956                 .textureCompressionETC2                   = radv_device_supports_etc(pdevice),
 957                 .textureCompressionASTC_LDR               = false,
 958                 .textureCompressionBC                     = true,
 959                 .occlusionQueryPrecise                    = true,
 960                 .pipelineStatisticsQuery                  = true,
 961                 .vertexPipelineStoresAndAtomics           = true,
 962                 .fragmentStoresAndAtomics                 = true,
 963                 .shaderTessellationAndGeometryPointSize   = true,
 964                 .shaderImageGatherExtended                = true,
 965                 .shaderStorageImageExtendedFormats        = true,
 966                 .shaderStorageImageMultisample            = true,
 967                 .shaderUniformBufferArrayDynamicIndexing  = true,
 968                 .shaderSampledImageArrayDynamicIndexing   = true,
 969                 .shaderStorageBufferArrayDynamicIndexing  = true,
 970                 .shaderStorageImageArrayDynamicIndexing   = true,
 971                 .shaderStorageImageReadWithoutFormat      = true,
 972                 .shaderStorageImageWriteWithoutFormat     = true,
 973                 .shaderClipDistance                       = true,
 974                 .shaderCullDistance                       = true,
 975                 .shaderFloat64                            = true,
 976                 .shaderInt64                              = true,
 977                 .shaderInt16                              = true,
 978                 .sparseBinding                            = true,
 979                 .variableMultisampleRate                  = true,
 980                 .shaderResourceMinLod                     = true,
 981                 .inheritedQueries                         = true,
 982         };
 983 }
 984
 985 static void
 986 radv_get_physical_device_features_1_1(struct radv_physical_device *pdevice,
 987                                       VkPhysicalDeviceVulkan11Features *f)
 988 {
 989         assert(f->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
 990
 991         f->storageBuffer16BitAccess            = true;
 992         f->uniformAndStorageBuffer16BitAccess  = true;
 993         f->storagePushConstant16               = true;
 994         f->storageInputOutput16                = pdevice->rad_info.has_packed_math_16bit && (LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm);
 995         f->multiview                           = true;
 996         f->multiviewGeometryShader             = true;
 997         f->multiviewTessellationShader         = true;
 998         f->variablePointersStorageBuffer       = true;
 999         f->variablePointers                    = true;
1000         f->protectedMemory                     = false;
1001         f->samplerYcbcrConversion              = true;
1002         f->shaderDrawParameters                = true;
1003 }
1004
1005 static void
1006 radv_get_physical_device_features_1_2(struct radv_physical_device *pdevice,
1007                                       VkPhysicalDeviceVulkan12Features *f)
1008 {
1009         assert(f->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
1010
1011         f->samplerMirrorClampToEdge = true;
1012         f->drawIndirectCount = true;
1013         f->storageBuffer8BitAccess = true;
1014         f->uniformAndStorageBuffer8BitAccess = true;
1015         f->storagePushConstant8 = true;
1016         f->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm;
1017         f->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm;
1018         f->shaderFloat16 = pdevice->rad_info.has_packed_math_16bit;
1019         f->shaderInt8 = true;
1020
1021         f->descriptorIndexing = true;
1022         f->shaderInputAttachmentArrayDynamicIndexing = true;
1023         f->shaderUniformTexelBufferArrayDynamicIndexing = true;
1024         f->shaderStorageTexelBufferArrayDynamicIndexing = true;
1025         f->shaderUniformBufferArrayNonUniformIndexing = true;
1026         f->shaderSampledImageArrayNonUniformIndexing = true;
1027         f->shaderStorageBufferArrayNonUniformIndexing = true;
1028         f->shaderStorageImageArrayNonUniformIndexing = true;
1029         f->shaderInputAttachmentArrayNonUniformIndexing = true;
1030         f->shaderUniformTexelBufferArrayNonUniformIndexing = true;
1031         f->shaderStorageTexelBufferArrayNonUniformIndexing = true;
1032         f->descriptorBindingUniformBufferUpdateAfterBind = true;
1033         f->descriptorBindingSampledImageUpdateAfterBind = true;
1034         f->descriptorBindingStorageImageUpdateAfterBind = true;
1035         f->descriptorBindingStorageBufferUpdateAfterBind = true;
1036         f->descriptorBindingUniformTexelBufferUpdateAfterBind = true;
1037         f->descriptorBindingStorageTexelBufferUpdateAfterBind = true;
1038         f->descriptorBindingUpdateUnusedWhilePending = true;
1039         f->descriptorBindingPartiallyBound = true;
1040         f->descriptorBindingVariableDescriptorCount = true;
1041         f->runtimeDescriptorArray = true;
1042
1043         f->samplerFilterMinmax = true;
1044         f->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7;
1045         f->imagelessFramebuffer = true;
1046         f->uniformBufferStandardLayout = true;
1047         f->shaderSubgroupExtendedTypes = true;
1048         f->separateDepthStencilLayouts = true;
1049         f->hostQueryReset = true;
1050         f->timelineSemaphore = pdevice->rad_info.has_syncobj_wait_for_submit;
1051         f->bufferDeviceAddress = true;
1052         f->bufferDeviceAddressCaptureReplay = false;
1053         f->bufferDeviceAddressMultiDevice = false;
1054         f->vulkanMemoryModel = true;
1055         f->vulkanMemoryModelDeviceScope = true;
1056         f->vulkanMemoryModelAvailabilityVisibilityChains = false;
1057         f->shaderOutputViewportIndex = true;
1058         f->shaderOutputLayer = true;
1059         f->subgroupBroadcastDynamicId = true;
1060 }
1061
1062 void radv_GetPhysicalDeviceFeatures2(
1063         VkPhysicalDevice                            physicalDevice,
1064         VkPhysicalDeviceFeatures2                  *pFeatures)
1065 {
1066         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1067         radv_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features);
1068
1069         VkPhysicalDeviceVulkan11Features core_1_1 = {
1070                 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES,
1071         };
1072         radv_get_physical_device_features_1_1(pdevice, &core_1_1);
1073
1074         VkPhysicalDeviceVulkan12Features core_1_2 = {
1075                 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
1076         };
1077         radv_get_physical_device_features_1_2(pdevice, &core_1_2);
1078
1079 #define CORE_FEATURE(major, minor, feature) \
1080    features->feature = core_##major##_##minor.feature
1081
1082         vk_foreach_struct(ext, pFeatures->pNext) {
1083                 switch (ext->sType) {
1084                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VARIABLE_POINTERS_FEATURES: {
1085                         VkPhysicalDeviceVariablePointersFeatures *features = (void *)ext;
1086                         CORE_FEATURE(1, 1, variablePointersStorageBuffer);
1087                         CORE_FEATURE(1, 1, variablePointers);
1088                         break;
1089                 }
1090                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_FEATURES: {
1091                         VkPhysicalDeviceMultiviewFeatures *features = (VkPhysicalDeviceMultiviewFeatures*)ext;
1092                         CORE_FEATURE(1, 1, multiview);
1093                         CORE_FEATURE(1, 1, multiviewGeometryShader);
1094                         CORE_FEATURE(1, 1, multiviewTessellationShader);
1095                         break;
1096                 }
1097                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES: {
1098                         VkPhysicalDeviceShaderDrawParametersFeatures *features =
1099                             (VkPhysicalDeviceShaderDrawParametersFeatures*)ext;
1100                         CORE_FEATURE(1, 1, shaderDrawParameters);
1101                         break;
1102                 }
1103                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_FEATURES: {
1104                         VkPhysicalDeviceProtectedMemoryFeatures *features =
1105                             (VkPhysicalDeviceProtectedMemoryFeatures*)ext;
1106                         CORE_FEATURE(1, 1, protectedMemory);
1107                         break;
1108                 }
1109                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES: {
1110                         VkPhysicalDevice16BitStorageFeatures *features =
1111                             (VkPhysicalDevice16BitStorageFeatures*)ext;
1112                         CORE_FEATURE(1, 1, storageBuffer16BitAccess);
1113                         CORE_FEATURE(1, 1, uniformAndStorageBuffer16BitAccess);
1114                         CORE_FEATURE(1, 1, storagePushConstant16);
1115                         CORE_FEATURE(1, 1, storageInputOutput16);
1116                         break;
1117                 }
1118                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES: {
1119                         VkPhysicalDeviceSamplerYcbcrConversionFeatures *features =
1120                             (VkPhysicalDeviceSamplerYcbcrConversionFeatures*)ext;
1121                         CORE_FEATURE(1, 1, samplerYcbcrConversion);
1122                         break;
1123                 }
1124                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES: {
1125                         VkPhysicalDeviceDescriptorIndexingFeatures *features =
1126                                 (VkPhysicalDeviceDescriptorIndexingFeatures*)ext;
1127                         CORE_FEATURE(1, 2, shaderInputAttachmentArrayDynamicIndexing);
1128                         CORE_FEATURE(1, 2, shaderUniformTexelBufferArrayDynamicIndexing);
1129                         CORE_FEATURE(1, 2, shaderStorageTexelBufferArrayDynamicIndexing);
1130                         CORE_FEATURE(1, 2, shaderUniformBufferArrayNonUniformIndexing);
1131                         CORE_FEATURE(1, 2, shaderSampledImageArrayNonUniformIndexing);
1132                         CORE_FEATURE(1, 2, shaderStorageBufferArrayNonUniformIndexing);
1133                         CORE_FEATURE(1, 2, shaderStorageImageArrayNonUniformIndexing);
1134                         CORE_FEATURE(1, 2, shaderInputAttachmentArrayNonUniformIndexing);
1135                         CORE_FEATURE(1, 2, shaderUniformTexelBufferArrayNonUniformIndexing);
1136                         CORE_FEATURE(1, 2, shaderStorageTexelBufferArrayNonUniformIndexing);
1137                         CORE_FEATURE(1, 2, descriptorBindingUniformBufferUpdateAfterBind);
1138                         CORE_FEATURE(1, 2, descriptorBindingSampledImageUpdateAfterBind);
1139                         CORE_FEATURE(1, 2, descriptorBindingStorageImageUpdateAfterBind);
1140                         CORE_FEATURE(1, 2, descriptorBindingStorageBufferUpdateAfterBind);
1141                         CORE_FEATURE(1, 2, descriptorBindingUniformTexelBufferUpdateAfterBind);
1142                         CORE_FEATURE(1, 2, descriptorBindingStorageTexelBufferUpdateAfterBind);
1143                         CORE_FEATURE(1, 2, descriptorBindingUpdateUnusedWhilePending);
1144                         CORE_FEATURE(1, 2, descriptorBindingPartiallyBound);
1145                         CORE_FEATURE(1, 2, descriptorBindingVariableDescriptorCount);
1146                         CORE_FEATURE(1, 2, runtimeDescriptorArray);
1147                         break;
1148                 }
1149                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONDITIONAL_RENDERING_FEATURES_EXT: {
1150                         VkPhysicalDeviceConditionalRenderingFeaturesEXT *features =
1151                                 (VkPhysicalDeviceConditionalRenderingFeaturesEXT*)ext;
1152                         features->conditionalRendering = true;
1153                         features->inheritedConditionalRendering = false;
1154                         break;
1155                 }
1156                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_FEATURES_EXT: {
1157                         VkPhysicalDeviceVertexAttributeDivisorFeaturesEXT *features =
1158                                 (VkPhysicalDeviceVertexAttributeDivisorFeaturesEXT *)ext;
1159                         features->vertexAttributeInstanceRateDivisor = true;
1160                         features->vertexAttributeInstanceRateZeroDivisor = true;
1161                         break;
1162                 }
1163                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_FEATURES_EXT: {
1164                         VkPhysicalDeviceTransformFeedbackFeaturesEXT *features =
1165                                 (VkPhysicalDeviceTransformFeedbackFeaturesEXT*)ext;
1166                         features->transformFeedback = true;
1167                         features->geometryStreams = !pdevice->use_ngg_streamout;
1168                         break;
1169                 }
1170                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SCALAR_BLOCK_LAYOUT_FEATURES: {
1171                         VkPhysicalDeviceScalarBlockLayoutFeatures *features =
1172                                 (VkPhysicalDeviceScalarBlockLayoutFeatures *)ext;
1173                         CORE_FEATURE(1, 2, scalarBlockLayout);
1174                         break;
1175                 }
1176                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT: {
1177                         VkPhysicalDeviceMemoryPriorityFeaturesEXT *features =
1178                                 (VkPhysicalDeviceMemoryPriorityFeaturesEXT *)ext;
1179                         features->memoryPriority = true;
1180                         break;
1181                 }
1182                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_EXT: {
1183                         VkPhysicalDeviceBufferDeviceAddressFeaturesEXT *features =
1184                                 (VkPhysicalDeviceBufferDeviceAddressFeaturesEXT *)ext;
1185                         features->bufferDeviceAddress = true;
1186                         features->bufferDeviceAddressCaptureReplay = false;
1187                         features->bufferDeviceAddressMultiDevice = false;
1188                         break;
1189                 }
1190                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES: {
1191                         VkPhysicalDeviceBufferDeviceAddressFeatures *features =
1192                                 (VkPhysicalDeviceBufferDeviceAddressFeatures *)ext;
1193                         CORE_FEATURE(1, 2, bufferDeviceAddress);
1194                         CORE_FEATURE(1, 2, bufferDeviceAddressCaptureReplay);
1195                         CORE_FEATURE(1, 2, bufferDeviceAddressMultiDevice);
1196                         break;
1197                 }
1198                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_ENABLE_FEATURES_EXT: {
1199                         VkPhysicalDeviceDepthClipEnableFeaturesEXT *features =
1200                                 (VkPhysicalDeviceDepthClipEnableFeaturesEXT *)ext;
1201                         features->depthClipEnable = true;
1202                         break;
1203                 }
1204                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES: {
1205                         VkPhysicalDeviceHostQueryResetFeatures *features =
1206                                 (VkPhysicalDeviceHostQueryResetFeatures *)ext;
1207                         CORE_FEATURE(1, 2, hostQueryReset);
1208                         break;
1209                 }
1210                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES: {
1211                         VkPhysicalDevice8BitStorageFeatures *features =
1212                             (VkPhysicalDevice8BitStorageFeatures *)ext;
1213                         CORE_FEATURE(1, 2, storageBuffer8BitAccess);
1214                         CORE_FEATURE(1, 2, uniformAndStorageBuffer8BitAccess);
1215                         CORE_FEATURE(1, 2, storagePushConstant8);
1216                         break;
1217                 }
1218                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES: {
1219                         VkPhysicalDeviceShaderFloat16Int8Features *features =
1220                                 (VkPhysicalDeviceShaderFloat16Int8Features*)ext;
1221                         CORE_FEATURE(1, 2, shaderFloat16);
1222                         CORE_FEATURE(1, 2, shaderInt8);
1223                         break;
1224                 }
1225                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES: {
1226                         VkPhysicalDeviceShaderAtomicInt64Features *features =
1227                                 (VkPhysicalDeviceShaderAtomicInt64Features *)ext;
1228                         CORE_FEATURE(1, 2, shaderBufferInt64Atomics);
1229                         CORE_FEATURE(1, 2, shaderSharedInt64Atomics);
1230                         break;
1231                 }
1232                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DEMOTE_TO_HELPER_INVOCATION_FEATURES_EXT: {
1233                         VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *features =
1234                                 (VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *)ext;
1235                         features->shaderDemoteToHelperInvocation = LLVM_VERSION_MAJOR >= 9 || !pdevice->use_llvm;
1236                         break;
1237                 }
1238                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_FEATURES_EXT: {
1239                         VkPhysicalDeviceInlineUniformBlockFeaturesEXT *features =
1240                                 (VkPhysicalDeviceInlineUniformBlockFeaturesEXT *)ext;
1241
1242                         features->inlineUniformBlock = true;
1243                         features->descriptorBindingInlineUniformBlockUpdateAfterBind = true;
1244                         break;
1245                 }
1246                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COMPUTE_SHADER_DERIVATIVES_FEATURES_NV: {
1247                         VkPhysicalDeviceComputeShaderDerivativesFeaturesNV *features =
1248                                 (VkPhysicalDeviceComputeShaderDerivativesFeaturesNV *)ext;
1249                         features->computeDerivativeGroupQuads = false;
1250                         features->computeDerivativeGroupLinear = true;
1251                         break;
1252                 }
1253                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_YCBCR_IMAGE_ARRAYS_FEATURES_EXT: {
1254                         VkPhysicalDeviceYcbcrImageArraysFeaturesEXT *features =
1255                                 (VkPhysicalDeviceYcbcrImageArraysFeaturesEXT*)ext;
1256                         features->ycbcrImageArrays = true;
1257                         break;
1258                 }
1259                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_UNIFORM_BUFFER_STANDARD_LAYOUT_FEATURES: {
1260                         VkPhysicalDeviceUniformBufferStandardLayoutFeatures *features =
1261                                 (VkPhysicalDeviceUniformBufferStandardLayoutFeatures *)ext;
1262                         CORE_FEATURE(1, 2, uniformBufferStandardLayout);
1263                         break;
1264                 }
1265                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INDEX_TYPE_UINT8_FEATURES_EXT: {
1266                         VkPhysicalDeviceIndexTypeUint8FeaturesEXT *features =
1267                                 (VkPhysicalDeviceIndexTypeUint8FeaturesEXT *)ext;
1268                         features->indexTypeUint8 = pdevice->rad_info.chip_class >= GFX8;
1269                         break;
1270                 }
1271                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGELESS_FRAMEBUFFER_FEATURES: {
1272                         VkPhysicalDeviceImagelessFramebufferFeatures *features =
1273                                 (VkPhysicalDeviceImagelessFramebufferFeatures *)ext;
1274                         CORE_FEATURE(1, 2, imagelessFramebuffer);
1275                         break;
1276                 }
1277                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR: {
1278                         VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR *features =
1279                                 (VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR *)ext;
1280                         features->pipelineExecutableInfo = true;
1281                         break;
1282                 }
1283                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR: {
1284                         VkPhysicalDeviceShaderClockFeaturesKHR *features =
1285                                 (VkPhysicalDeviceShaderClockFeaturesKHR *)ext;
1286                         features->shaderSubgroupClock = true;
1287                         features->shaderDeviceClock = pdevice->rad_info.chip_class >= GFX8;
1288                         break;
1289                 }
1290                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_FEATURES_EXT: {
1291                         VkPhysicalDeviceTexelBufferAlignmentFeaturesEXT *features =
1292                                 (VkPhysicalDeviceTexelBufferAlignmentFeaturesEXT *)ext;
1293                         features->texelBufferAlignment = true;
1294                         break;
1295                 }
1296                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES: {
1297                         VkPhysicalDeviceTimelineSemaphoreFeatures *features =
1298                                 (VkPhysicalDeviceTimelineSemaphoreFeatures *) ext;
1299                         CORE_FEATURE(1, 2, timelineSemaphore);
1300                         break;
1301                 }
1302                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT: {
1303                         VkPhysicalDeviceSubgroupSizeControlFeaturesEXT *features =
1304                                 (VkPhysicalDeviceSubgroupSizeControlFeaturesEXT *)ext;
1305                         features->subgroupSizeControl = true;
1306                         features->computeFullSubgroups = true;
1307                         break;
1308                 }
1309                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COHERENT_MEMORY_FEATURES_AMD: {
1310                         VkPhysicalDeviceCoherentMemoryFeaturesAMD *features =
1311                                 (VkPhysicalDeviceCoherentMemoryFeaturesAMD *)ext;
1312                         features->deviceCoherentMemory = pdevice->rad_info.has_l2_uncached;
1313                         break;
1314                 }
1315                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_EXTENDED_TYPES_FEATURES: {
1316                         VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *features =
1317                                 (VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *)ext;
1318                         CORE_FEATURE(1, 2, shaderSubgroupExtendedTypes);
1319                         break;
1320                 }
1321                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SEPARATE_DEPTH_STENCIL_LAYOUTS_FEATURES_KHR: {
1322                         VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *features =
1323                                 (VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *)ext;
1324                         CORE_FEATURE(1, 2, separateDepthStencilLayouts);
1325                         break;
1326                 }
1327                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES: {
1328                         radv_get_physical_device_features_1_1(pdevice, (void *)ext);
1329                         break;
1330                 }
1331                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES: {
1332                         radv_get_physical_device_features_1_2(pdevice, (void *)ext);
1333                         break;
1334                 }
1335                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_FEATURES_EXT: {
1336                         VkPhysicalDeviceLineRasterizationFeaturesEXT *features =
1337                                 (VkPhysicalDeviceLineRasterizationFeaturesEXT *)ext;
1338                         features->rectangularLines = false;
1339                         features->bresenhamLines = true;
1340                         features->smoothLines = false;
1341                         features->stippledRectangularLines = false;
1342                         features->stippledBresenhamLines = true;
1343                         features->stippledSmoothLines = false;
1344                         break;
1345                 }
1346                 case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: {
1347                         VkDeviceMemoryOverallocationCreateInfoAMD *features =
1348                                 (VkDeviceMemoryOverallocationCreateInfoAMD *)ext;
1349                         features->overallocationBehavior = true;
1350                         break;
1351                 }
1352                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT: {
1353                         VkPhysicalDeviceRobustness2FeaturesEXT *features =
1354                                 (VkPhysicalDeviceRobustness2FeaturesEXT *)ext;
1355                         features->robustBufferAccess2 = true;
1356                         features->robustImageAccess2 = true;
1357                         features->nullDescriptor = true;
1358                         break;
1359                 }
1360                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT: {
1361                         VkPhysicalDeviceCustomBorderColorFeaturesEXT *features =
1362                                 (VkPhysicalDeviceCustomBorderColorFeaturesEXT *)ext;
1363                         features->customBorderColors = true;
1364                         features->customBorderColorWithoutFormat = true;
1365                         break;
1366                 }
1367                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRIVATE_DATA_FEATURES_EXT: {
1368                         VkPhysicalDevicePrivateDataFeaturesEXT *features =
1369                                 (VkPhysicalDevicePrivateDataFeaturesEXT *)ext;
1370                         features->privateData = true;
1371                         break;
1372                 }
1373                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES_EXT: {
1374                         VkPhysicalDevicePipelineCreationCacheControlFeaturesEXT *features =
1375                                 (VkPhysicalDevicePipelineCreationCacheControlFeaturesEXT *)ext;
1376                         features-> pipelineCreationCacheControl = true;
1377                         break;
1378                 }
1379                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_MEMORY_MODEL_FEATURES_KHR: {
1380                         VkPhysicalDeviceVulkanMemoryModelFeaturesKHR *features =
1381                                 (VkPhysicalDeviceVulkanMemoryModelFeaturesKHR *)ext;
1382                         CORE_FEATURE(1, 2, vulkanMemoryModel);
1383                         CORE_FEATURE(1, 2, vulkanMemoryModelDeviceScope);
1384                         CORE_FEATURE(1, 2, vulkanMemoryModelAvailabilityVisibilityChains);
1385                         break;
1386                 }
1387                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT: {
1388                         VkPhysicalDeviceExtendedDynamicStateFeaturesEXT *features =
1389                                 (VkPhysicalDeviceExtendedDynamicStateFeaturesEXT *) ext;
1390                         features->extendedDynamicState = true;
1391                         break;
1392                 }
1393                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_ROBUSTNESS_FEATURES_EXT: {
1394                         VkPhysicalDeviceImageRobustnessFeaturesEXT *features =
1395                                 (VkPhysicalDeviceImageRobustnessFeaturesEXT *)ext;
1396                         features->robustImageAccess = true;
1397                         break;
1398                 }
1399                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT: {
1400                         VkPhysicalDeviceShaderAtomicFloatFeaturesEXT *features =
1401                                 (VkPhysicalDeviceShaderAtomicFloatFeaturesEXT *)ext;
1402                         features->shaderBufferFloat32Atomics = true;
1403                         features->shaderBufferFloat32AtomicAdd = false;
1404                         features->shaderBufferFloat64Atomics = true;
1405                         features->shaderBufferFloat64AtomicAdd = false;
1406                         features->shaderSharedFloat32Atomics = true;
1407                         features->shaderSharedFloat32AtomicAdd = pdevice->rad_info.chip_class >= GFX8 &&
1408                                                                  (!pdevice->use_llvm || LLVM_VERSION_MAJOR >= 10);
1409                         features->shaderSharedFloat64Atomics = true;
1410                         features->shaderSharedFloat64AtomicAdd = false;
1411                         features->shaderImageFloat32Atomics = true;
1412                         features->shaderImageFloat32AtomicAdd = false;
1413                         features->sparseImageFloat32Atomics = false;
1414                         features->sparseImageFloat32AtomicAdd = false;
1415                         break;
1416                 }
1417                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_4444_FORMATS_FEATURES_EXT: {
1418                         VkPhysicalDevice4444FormatsFeaturesEXT *features =
1419                                 (VkPhysicalDevice4444FormatsFeaturesEXT *)ext;
1420                         features->formatA4R4G4B4 = true;
1421                         features->formatA4B4G4R4 = true;
1422                         break;
1423                 }
1424                 default:
1425                         break;
1426                 }
1427         }
1428 #undef CORE_FEATURE
1429 }
1430
1431 static size_t
1432 radv_max_descriptor_set_size()
1433 {
1434         /* make sure that the entire descriptor set is addressable with a signed
1435          * 32-bit int. So the sum of all limits scaled by descriptor size has to
1436          * be at most 2 GiB. the combined image & samples object count as one of
1437          * both. This limit is for the pipeline layout, not for the set layout, but
1438          * there is no set limit, so we just set a pipeline limit. I don't think
1439          * any app is going to hit this soon. */
1440         return ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS
1441                              - MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
1442                   (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
1443                    32 /* storage buffer, 32 due to potential space wasted on alignment */ +
1444                    32 /* sampler, largest when combined with image */ +
1445                    64 /* sampled image */ +
1446                    64 /* storage image */);
1447 }
1448
1449 static uint32_t
1450 radv_uniform_buffer_offset_alignment(const struct radv_physical_device *pdevice)
1451 {
1452         uint32_t uniform_offset_alignment = driQueryOptioni(&pdevice->instance->dri_options,
1453                                                            "radv_override_uniform_offset_alignment");
1454         if (!util_is_power_of_two_or_zero(uniform_offset_alignment)) {
1455                 fprintf(stderr, "ERROR: invalid radv_override_uniform_offset_alignment setting %d:"
1456                                 "not a power of two\n", uniform_offset_alignment);
1457                 uniform_offset_alignment = 0;
1458         }
1459
1460         /* Take at least the hardware limit. */
1461         return MAX2(uniform_offset_alignment, 4);
1462 }
1463
1464 void radv_GetPhysicalDeviceProperties(
1465         VkPhysicalDevice                            physicalDevice,
1466         VkPhysicalDeviceProperties*                 pProperties)
1467 {
1468         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1469         VkSampleCountFlags sample_counts = 0xf;
1470
1471         size_t max_descriptor_set_size = radv_max_descriptor_set_size();
1472
1473         VkPhysicalDeviceLimits limits = {
1474                 .maxImageDimension1D                      = (1 << 14),
1475                 .maxImageDimension2D                      = (1 << 14),
1476                 .maxImageDimension3D                      = (1 << 11),
1477                 .maxImageDimensionCube                    = (1 << 14),
1478                 .maxImageArrayLayers                      = (1 << 11),
1479                 .maxTexelBufferElements                   = UINT32_MAX,
1480                 .maxUniformBufferRange                    = UINT32_MAX,
1481                 .maxStorageBufferRange                    = UINT32_MAX,
1482                 .maxPushConstantsSize                     = MAX_PUSH_CONSTANTS_SIZE,
1483                 .maxMemoryAllocationCount                 = UINT32_MAX,
1484                 .maxSamplerAllocationCount                = 64 * 1024,
1485                 .bufferImageGranularity                   = 64, /* A cache line */
1486                 .sparseAddressSpaceSize                   = RADV_MAX_MEMORY_ALLOCATION_SIZE, /* buffer max size */
1487                 .maxBoundDescriptorSets                   = MAX_SETS,
1488                 .maxPerStageDescriptorSamplers            = max_descriptor_set_size,
1489                 .maxPerStageDescriptorUniformBuffers      = max_descriptor_set_size,
1490                 .maxPerStageDescriptorStorageBuffers      = max_descriptor_set_size,
1491                 .maxPerStageDescriptorSampledImages       = max_descriptor_set_size,
1492                 .maxPerStageDescriptorStorageImages       = max_descriptor_set_size,
1493                 .maxPerStageDescriptorInputAttachments    = max_descriptor_set_size,
1494                 .maxPerStageResources                     = max_descriptor_set_size,
1495                 .maxDescriptorSetSamplers                 = max_descriptor_set_size,
1496                 .maxDescriptorSetUniformBuffers           = max_descriptor_set_size,
1497                 .maxDescriptorSetUniformBuffersDynamic    = MAX_DYNAMIC_UNIFORM_BUFFERS,
1498                 .maxDescriptorSetStorageBuffers           = max_descriptor_set_size,
1499                 .maxDescriptorSetStorageBuffersDynamic    = MAX_DYNAMIC_STORAGE_BUFFERS,
1500                 .maxDescriptorSetSampledImages            = max_descriptor_set_size,
1501                 .maxDescriptorSetStorageImages            = max_descriptor_set_size,
1502                 .maxDescriptorSetInputAttachments         = max_descriptor_set_size,
1503                 .maxVertexInputAttributes                 = MAX_VERTEX_ATTRIBS,
1504                 .maxVertexInputBindings                   = MAX_VBS,
1505                 .maxVertexInputAttributeOffset            = 2047,
1506                 .maxVertexInputBindingStride              = 2048,
1507                 .maxVertexOutputComponents                = 128,
1508                 .maxTessellationGenerationLevel           = 64,
1509                 .maxTessellationPatchSize                 = 32,
1510                 .maxTessellationControlPerVertexInputComponents = 128,
1511                 .maxTessellationControlPerVertexOutputComponents = 128,
1512                 .maxTessellationControlPerPatchOutputComponents = 120,
1513                 .maxTessellationControlTotalOutputComponents = 4096,
1514                 .maxTessellationEvaluationInputComponents = 128,
1515                 .maxTessellationEvaluationOutputComponents = 128,
1516                 .maxGeometryShaderInvocations             = 127,
1517                 .maxGeometryInputComponents               = 64,
1518                 .maxGeometryOutputComponents              = 128,
1519                 .maxGeometryOutputVertices                = 256,
1520                 .maxGeometryTotalOutputComponents         = 1024,
1521                 .maxFragmentInputComponents               = 128,
1522                 .maxFragmentOutputAttachments             = 8,
1523                 .maxFragmentDualSrcAttachments            = 1,
1524                 .maxFragmentCombinedOutputResources       = 8,
1525                 .maxComputeSharedMemorySize               = 32768,
1526                 .maxComputeWorkGroupCount                 = { 65535, 65535, 65535 },
1527                 .maxComputeWorkGroupInvocations           = 1024,
1528                 .maxComputeWorkGroupSize = {
1529                         1024,
1530                         1024,
1531                         1024
1532                 },
1533                 .subPixelPrecisionBits                    = 8,
1534                 .subTexelPrecisionBits                    = 8,
1535                 .mipmapPrecisionBits                      = 8,
1536                 .maxDrawIndexedIndexValue                 = UINT32_MAX,
1537                 .maxDrawIndirectCount                     = UINT32_MAX,
1538                 .maxSamplerLodBias                        = 16,
1539                 .maxSamplerAnisotropy                     = 16,
1540                 .maxViewports                             = MAX_VIEWPORTS,
1541                 .maxViewportDimensions                    = { (1 << 14), (1 << 14) },
1542                 .viewportBoundsRange                      = { INT16_MIN, INT16_MAX },
1543                 .viewportSubPixelBits                     = 8,
1544                 .minMemoryMapAlignment                    = 4096, /* A page */
1545                 .minTexelBufferOffsetAlignment            = 4,
1546                 .minUniformBufferOffsetAlignment          = radv_uniform_buffer_offset_alignment(pdevice),
1547                 .minStorageBufferOffsetAlignment          = 4,
1548                 .minTexelOffset                           = -32,
1549                 .maxTexelOffset                           = 31,
1550                 .minTexelGatherOffset                     = -32,
1551                 .maxTexelGatherOffset                     = 31,
1552                 .minInterpolationOffset                   = -2,
1553                 .maxInterpolationOffset                   = 2,
1554                 .subPixelInterpolationOffsetBits          = 8,
1555                 .maxFramebufferWidth                      = (1 << 14),
1556                 .maxFramebufferHeight                     = (1 << 14),
1557                 .maxFramebufferLayers                     = (1 << 10),
1558                 .framebufferColorSampleCounts             = sample_counts,
1559                 .framebufferDepthSampleCounts             = sample_counts,
1560                 .framebufferStencilSampleCounts           = sample_counts,
1561                 .framebufferNoAttachmentsSampleCounts     = sample_counts,
1562                 .maxColorAttachments                      = MAX_RTS,
1563                 .sampledImageColorSampleCounts            = sample_counts,
1564                 .sampledImageIntegerSampleCounts          = sample_counts,
1565                 .sampledImageDepthSampleCounts            = sample_counts,
1566                 .sampledImageStencilSampleCounts          = sample_counts,
1567                 .storageImageSampleCounts                 = sample_counts,
1568                 .maxSampleMaskWords                       = 1,
1569                 .timestampComputeAndGraphics              = true,
1570                 .timestampPeriod                          = 1000000.0 / pdevice->rad_info.clock_crystal_freq,
1571                 .maxClipDistances                         = 8,
1572                 .maxCullDistances                         = 8,
1573                 .maxCombinedClipAndCullDistances          = 8,
1574                 .discreteQueuePriorities                  = 2,
1575                 .pointSizeRange                           = { 0.0, 8191.875 },
1576                 .lineWidthRange                           = { 0.0, 8191.875 },
1577                 .pointSizeGranularity                     = (1.0 / 8.0),
1578                 .lineWidthGranularity                     = (1.0 / 8.0),
1579                 .strictLines                              = false, /* FINISHME */
1580                 .standardSampleLocations                  = true,
1581                 .optimalBufferCopyOffsetAlignment         = 128,
1582                 .optimalBufferCopyRowPitchAlignment       = 128,
1583                 .nonCoherentAtomSize                      = 64,
1584         };
1585
1586         *pProperties = (VkPhysicalDeviceProperties) {
1587                 .apiVersion = radv_physical_device_api_version(pdevice),
1588                 .driverVersion = vk_get_driver_version(),
1589                 .vendorID = ATI_VENDOR_ID,
1590                 .deviceID = pdevice->rad_info.pci_id,
1591                 .deviceType = pdevice->rad_info.has_dedicated_vram ? VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU : VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
1592                 .limits = limits,
1593                 .sparseProperties = {0},
1594         };
1595
1596         strcpy(pProperties->deviceName, pdevice->name);
1597         memcpy(pProperties->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE);
1598 }
1599
1600 static void
1601 radv_get_physical_device_properties_1_1(struct radv_physical_device *pdevice,
1602                                         VkPhysicalDeviceVulkan11Properties *p)
1603 {
1604         assert(p->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES);
1605
1606         memcpy(p->deviceUUID, pdevice->device_uuid, VK_UUID_SIZE);
1607         memcpy(p->driverUUID, pdevice->driver_uuid, VK_UUID_SIZE);
1608         memset(p->deviceLUID, 0, VK_LUID_SIZE);
1609         /* The LUID is for Windows. */
1610         p->deviceLUIDValid = false;
1611         p->deviceNodeMask = 0;
1612
1613         p->subgroupSize = RADV_SUBGROUP_SIZE;
1614         p->subgroupSupportedStages = VK_SHADER_STAGE_ALL_GRAPHICS |
1615                                      VK_SHADER_STAGE_COMPUTE_BIT;
1616         p->subgroupSupportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT |
1617                                          VK_SUBGROUP_FEATURE_VOTE_BIT |
1618                                          VK_SUBGROUP_FEATURE_ARITHMETIC_BIT |
1619                                          VK_SUBGROUP_FEATURE_BALLOT_BIT |
1620                                          VK_SUBGROUP_FEATURE_CLUSTERED_BIT |
1621                                          VK_SUBGROUP_FEATURE_QUAD_BIT |
1622                                          VK_SUBGROUP_FEATURE_SHUFFLE_BIT |
1623                                          VK_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT;
1624         p->subgroupQuadOperationsInAllStages = true;
1625
1626         p->pointClippingBehavior = VK_POINT_CLIPPING_BEHAVIOR_ALL_CLIP_PLANES;
1627         p->maxMultiviewViewCount = MAX_VIEWS;
1628         p->maxMultiviewInstanceIndex = INT_MAX;
1629         p->protectedNoFault = false;
1630         p->maxPerSetDescriptors = RADV_MAX_PER_SET_DESCRIPTORS;
1631         p->maxMemoryAllocationSize = RADV_MAX_MEMORY_ALLOCATION_SIZE;
1632 }
1633
1634 static void
1635 radv_get_physical_device_properties_1_2(struct radv_physical_device *pdevice,
1636                                         VkPhysicalDeviceVulkan12Properties *p)
1637 {
1638         assert(p->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES);
1639
1640         p->driverID = VK_DRIVER_ID_MESA_RADV;
1641         snprintf(p->driverName, VK_MAX_DRIVER_NAME_SIZE, "radv");
1642         snprintf(p->driverInfo, VK_MAX_DRIVER_INFO_SIZE,
1643                  "Mesa " PACKAGE_VERSION MESA_GIT_SHA1 " (%s)",
1644                  radv_get_compiler_string(pdevice));
1645         p->conformanceVersion = (VkConformanceVersion) {
1646                 .major = 1,
1647                 .minor = 2,
1648                 .subminor = 3,
1649                 .patch = 0,
1650         };
1651
1652         /* On AMD hardware, denormals and rounding modes for fp16/fp64 are
1653          * controlled by the same config register.
1654          */
1655         if (pdevice->rad_info.has_packed_math_16bit) {
1656                 p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR;
1657                 p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR;
1658         } else {
1659                 p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL_KHR;
1660                 p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL_KHR;
1661         }
1662
1663         /* With LLVM, do not allow both preserving and flushing denorms because
1664          * different shaders in the same pipeline can have different settings and
1665          * this won't work for merged shaders. To make it work, this requires LLVM
1666          * support for changing the register. The same logic applies for the
1667          * rounding modes because they are configured with the same config
1668          * register.
1669          */
1670         p->shaderDenormFlushToZeroFloat32 = true;
1671         p->shaderDenormPreserveFloat32 = !pdevice->use_llvm;
1672         p->shaderRoundingModeRTEFloat32 = true;
1673         p->shaderRoundingModeRTZFloat32 = !pdevice->use_llvm;
1674         p->shaderSignedZeroInfNanPreserveFloat32 = true;
1675
1676         p->shaderDenormFlushToZeroFloat16 = pdevice->rad_info.has_packed_math_16bit && !pdevice->use_llvm;
1677         p->shaderDenormPreserveFloat16 = pdevice->rad_info.has_packed_math_16bit;
1678         p->shaderRoundingModeRTEFloat16 = pdevice->rad_info.has_packed_math_16bit;
1679         p->shaderRoundingModeRTZFloat16 = pdevice->rad_info.has_packed_math_16bit && !pdevice->use_llvm;
1680         p->shaderSignedZeroInfNanPreserveFloat16 = pdevice->rad_info.has_packed_math_16bit;
1681
1682         p->shaderDenormFlushToZeroFloat64 = pdevice->rad_info.chip_class >= GFX8 && !pdevice->use_llvm;
1683         p->shaderDenormPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8;
1684         p->shaderRoundingModeRTEFloat64 = pdevice->rad_info.chip_class >= GFX8;
1685         p->shaderRoundingModeRTZFloat64 = pdevice->rad_info.chip_class >= GFX8 && !pdevice->use_llvm;
1686         p->shaderSignedZeroInfNanPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8;
1687
1688         p->maxUpdateAfterBindDescriptorsInAllPools = UINT32_MAX / 64;
1689         p->shaderUniformBufferArrayNonUniformIndexingNative = false;
1690         p->shaderSampledImageArrayNonUniformIndexingNative = false;
1691         p->shaderStorageBufferArrayNonUniformIndexingNative = false;
1692         p->shaderStorageImageArrayNonUniformIndexingNative = false;
1693         p->shaderInputAttachmentArrayNonUniformIndexingNative = false;
1694         p->robustBufferAccessUpdateAfterBind = false;
1695         p->quadDivergentImplicitLod = false;
1696
1697         size_t max_descriptor_set_size = ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS -
1698                 MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
1699                         (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
1700                          32 /* storage buffer, 32 due to potential space wasted on alignment */ +
1701                          32 /* sampler, largest when combined with image */ +
1702                          64 /* sampled image */ +
1703                          64 /* storage image */);
1704         p->maxPerStageDescriptorUpdateAfterBindSamplers = max_descriptor_set_size;
1705         p->maxPerStageDescriptorUpdateAfterBindUniformBuffers = max_descriptor_set_size;
1706         p->maxPerStageDescriptorUpdateAfterBindStorageBuffers = max_descriptor_set_size;
1707         p->maxPerStageDescriptorUpdateAfterBindSampledImages = max_descriptor_set_size;
1708         p->maxPerStageDescriptorUpdateAfterBindStorageImages = max_descriptor_set_size;
1709         p->maxPerStageDescriptorUpdateAfterBindInputAttachments = max_descriptor_set_size;
1710         p->maxPerStageUpdateAfterBindResources = max_descriptor_set_size;
1711         p->maxDescriptorSetUpdateAfterBindSamplers = max_descriptor_set_size;
1712         p->maxDescriptorSetUpdateAfterBindUniformBuffers = max_descriptor_set_size;
1713         p->maxDescriptorSetUpdateAfterBindUniformBuffersDynamic = MAX_DYNAMIC_UNIFORM_BUFFERS;
1714         p->maxDescriptorSetUpdateAfterBindStorageBuffers = max_descriptor_set_size;
1715         p->maxDescriptorSetUpdateAfterBindStorageBuffersDynamic = MAX_DYNAMIC_STORAGE_BUFFERS;
1716         p->maxDescriptorSetUpdateAfterBindSampledImages = max_descriptor_set_size;
1717         p->maxDescriptorSetUpdateAfterBindStorageImages = max_descriptor_set_size;
1718         p->maxDescriptorSetUpdateAfterBindInputAttachments = max_descriptor_set_size;
1719
1720         /* We support all of the depth resolve modes */
1721         p->supportedDepthResolveModes = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR |
1722                                             VK_RESOLVE_MODE_AVERAGE_BIT_KHR |
1723                                             VK_RESOLVE_MODE_MIN_BIT_KHR |
1724                                             VK_RESOLVE_MODE_MAX_BIT_KHR;
1725
1726         /* Average doesn't make sense for stencil so we don't support that */
1727         p->supportedStencilResolveModes = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR |
1728                                               VK_RESOLVE_MODE_MIN_BIT_KHR |
1729                                               VK_RESOLVE_MODE_MAX_BIT_KHR;
1730
1731         p->independentResolveNone = true;
1732         p->independentResolve = true;
1733
1734         /* GFX6-8 only support single channel min/max filter. */
1735         p->filterMinmaxImageComponentMapping = pdevice->rad_info.chip_class >= GFX9;
1736         p->filterMinmaxSingleComponentFormats = true;
1737
1738         p->maxTimelineSemaphoreValueDifference = UINT64_MAX;
1739
1740         p->framebufferIntegerColorSampleCounts = VK_SAMPLE_COUNT_1_BIT;
1741 }
1742
1743 void radv_GetPhysicalDeviceProperties2(
1744         VkPhysicalDevice                            physicalDevice,
1745         VkPhysicalDeviceProperties2                *pProperties)
1746 {
1747         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1748         radv_GetPhysicalDeviceProperties(physicalDevice, &pProperties->properties);
1749
1750         VkPhysicalDeviceVulkan11Properties core_1_1 = {
1751                 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES,
1752         };
1753         radv_get_physical_device_properties_1_1(pdevice, &core_1_1);
1754
1755         VkPhysicalDeviceVulkan12Properties core_1_2 = {
1756                 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES,
1757         };
1758         radv_get_physical_device_properties_1_2(pdevice, &core_1_2);
1759
1760 #define CORE_RENAMED_PROPERTY(major, minor, ext_property, core_property) \
1761    memcpy(&properties->ext_property, &core_##major##_##minor.core_property, \
1762           sizeof(core_##major##_##minor.core_property))
1763
1764 #define CORE_PROPERTY(major, minor, property) \
1765    CORE_RENAMED_PROPERTY(major, minor, property, property)
1766
1767         vk_foreach_struct(ext, pProperties->pNext) {
1768                 switch (ext->sType) {
1769                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR: {
1770                         VkPhysicalDevicePushDescriptorPropertiesKHR *properties =
1771                                 (VkPhysicalDevicePushDescriptorPropertiesKHR *) ext;
1772                         properties->maxPushDescriptors = MAX_PUSH_DESCRIPTORS;
1773                         break;
1774                 }
1775                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES: {
1776                         VkPhysicalDeviceIDProperties *properties = (VkPhysicalDeviceIDProperties*)ext;
1777                         CORE_PROPERTY(1, 1, deviceUUID);
1778                         CORE_PROPERTY(1, 1, driverUUID);
1779                         CORE_PROPERTY(1, 1, deviceLUID);
1780                         CORE_PROPERTY(1, 1, deviceLUIDValid);
1781                         break;
1782                 }
1783                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_PROPERTIES: {
1784                         VkPhysicalDeviceMultiviewProperties *properties = (VkPhysicalDeviceMultiviewProperties*)ext;
1785                         CORE_PROPERTY(1, 1, maxMultiviewViewCount);
1786                         CORE_PROPERTY(1, 1, maxMultiviewInstanceIndex);
1787                         break;
1788                 }
1789                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_POINT_CLIPPING_PROPERTIES: {
1790                         VkPhysicalDevicePointClippingProperties *properties =
1791                             (VkPhysicalDevicePointClippingProperties*)ext;
1792                         CORE_PROPERTY(1, 1, pointClippingBehavior);
1793                         break;
1794                 }
1795                 case  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DISCARD_RECTANGLE_PROPERTIES_EXT: {
1796                         VkPhysicalDeviceDiscardRectanglePropertiesEXT *properties =
1797                             (VkPhysicalDeviceDiscardRectanglePropertiesEXT*)ext;
1798                         properties->maxDiscardRectangles = MAX_DISCARD_RECTANGLES;
1799                         break;
1800                 }
1801                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT: {
1802                         VkPhysicalDeviceExternalMemoryHostPropertiesEXT *properties =
1803                             (VkPhysicalDeviceExternalMemoryHostPropertiesEXT *) ext;
1804                         properties->minImportedHostPointerAlignment = 4096;
1805                         break;
1806                 }
1807                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES: {
1808                         VkPhysicalDeviceSubgroupProperties *properties =
1809                             (VkPhysicalDeviceSubgroupProperties*)ext;
1810                         CORE_PROPERTY(1, 1, subgroupSize);
1811                         CORE_RENAMED_PROPERTY(1, 1, supportedStages,
1812                                                     subgroupSupportedStages);
1813                         CORE_RENAMED_PROPERTY(1, 1, supportedOperations,
1814                                                     subgroupSupportedOperations);
1815                         CORE_RENAMED_PROPERTY(1, 1, quadOperationsInAllStages,
1816                                                     subgroupQuadOperationsInAllStages);
1817                         break;
1818                 }
1819                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES: {
1820                         VkPhysicalDeviceMaintenance3Properties *properties =
1821                             (VkPhysicalDeviceMaintenance3Properties*)ext;
1822                         CORE_PROPERTY(1, 1, maxPerSetDescriptors);
1823                         CORE_PROPERTY(1, 1, maxMemoryAllocationSize);
1824                         break;
1825                 }
1826                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_FILTER_MINMAX_PROPERTIES: {
1827                         VkPhysicalDeviceSamplerFilterMinmaxProperties *properties =
1828                                 (VkPhysicalDeviceSamplerFilterMinmaxProperties *)ext;
1829                         CORE_PROPERTY(1, 2, filterMinmaxImageComponentMapping);
1830                         CORE_PROPERTY(1, 2, filterMinmaxSingleComponentFormats);
1831                         break;
1832                 }
1833                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CORE_PROPERTIES_AMD: {
1834                         VkPhysicalDeviceShaderCorePropertiesAMD *properties =
1835                                 (VkPhysicalDeviceShaderCorePropertiesAMD *)ext;
1836
1837                         /* Shader engines. */
1838                         properties->shaderEngineCount =
1839                                 pdevice->rad_info.max_se;
1840                         properties->shaderArraysPerEngineCount =
1841                                 pdevice->rad_info.max_sh_per_se;
1842                         properties->computeUnitsPerShaderArray =
1843                                 pdevice->rad_info.min_good_cu_per_sa;
1844                         properties->simdPerComputeUnit =
1845                                 pdevice->rad_info.num_simd_per_compute_unit;
1846                         properties->wavefrontsPerSimd =
1847                                 pdevice->rad_info.max_wave64_per_simd;
1848                         properties->wavefrontSize = 64;
1849
1850                         /* SGPR. */
1851                         properties->sgprsPerSimd =
1852                                 pdevice->rad_info.num_physical_sgprs_per_simd;
1853                         properties->minSgprAllocation =
1854                                 pdevice->rad_info.min_sgpr_alloc;
1855                         properties->maxSgprAllocation =
1856                                 pdevice->rad_info.max_sgpr_alloc;
1857                         properties->sgprAllocationGranularity =
1858                                 pdevice->rad_info.sgpr_alloc_granularity;
1859
1860                         /* VGPR. */
1861                         properties->vgprsPerSimd =
1862                                 pdevice->rad_info.num_physical_wave64_vgprs_per_simd;
1863                         properties->minVgprAllocation =
1864                                 pdevice->rad_info.min_wave64_vgpr_alloc;
1865                         properties->maxVgprAllocation =
1866                                 pdevice->rad_info.max_vgpr_alloc;
1867                         properties->vgprAllocationGranularity =
1868                                 pdevice->rad_info.wave64_vgpr_alloc_granularity;
1869                         break;
1870                 }
1871                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CORE_PROPERTIES_2_AMD: {
1872                         VkPhysicalDeviceShaderCoreProperties2AMD *properties =
1873                                 (VkPhysicalDeviceShaderCoreProperties2AMD *)ext;
1874
1875                         properties->shaderCoreFeatures = 0;
1876                         properties->activeComputeUnitCount =
1877                                 pdevice->rad_info.num_good_compute_units;
1878                         break;
1879                 }
1880                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_PROPERTIES_EXT: {
1881                         VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT *properties =
1882                                 (VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT *)ext;
1883                         properties->maxVertexAttribDivisor = UINT32_MAX;
1884                         break;
1885                 }
1886                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES: {
1887                         VkPhysicalDeviceDescriptorIndexingProperties *properties =
1888                                 (VkPhysicalDeviceDescriptorIndexingProperties*)ext;
1889                         CORE_PROPERTY(1, 2, maxUpdateAfterBindDescriptorsInAllPools);
1890                         CORE_PROPERTY(1, 2, shaderUniformBufferArrayNonUniformIndexingNative);
1891                         CORE_PROPERTY(1, 2, shaderSampledImageArrayNonUniformIndexingNative);
1892                         CORE_PROPERTY(1, 2, shaderStorageBufferArrayNonUniformIndexingNative);
1893                         CORE_PROPERTY(1, 2, shaderStorageImageArrayNonUniformIndexingNative);
1894                         CORE_PROPERTY(1, 2, shaderInputAttachmentArrayNonUniformIndexingNative);
1895                         CORE_PROPERTY(1, 2, robustBufferAccessUpdateAfterBind);
1896                         CORE_PROPERTY(1, 2, quadDivergentImplicitLod);
1897                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindSamplers);
1898                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindUniformBuffers);
1899                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindStorageBuffers);
1900                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindSampledImages);
1901                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindStorageImages);
1902                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindInputAttachments);
1903                         CORE_PROPERTY(1, 2, maxPerStageUpdateAfterBindResources);
1904                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindSamplers);
1905                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindUniformBuffers);
1906                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindUniformBuffersDynamic);
1907                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageBuffers);
1908                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageBuffersDynamic);
1909                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindSampledImages);
1910                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageImages);
1911                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindInputAttachments);
1912                         break;
1913                 }
1914                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_PROPERTIES: {
1915                         VkPhysicalDeviceProtectedMemoryProperties *properties =
1916                                 (VkPhysicalDeviceProtectedMemoryProperties *)ext;
1917                         CORE_PROPERTY(1, 1, protectedNoFault);
1918                         break;
1919                 }
1920                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONSERVATIVE_RASTERIZATION_PROPERTIES_EXT: {
1921                         VkPhysicalDeviceConservativeRasterizationPropertiesEXT *properties =
1922                                 (VkPhysicalDeviceConservativeRasterizationPropertiesEXT *)ext;
1923                         properties->primitiveOverestimationSize = 0;
1924                         properties->maxExtraPrimitiveOverestimationSize = 0;
1925                         properties->extraPrimitiveOverestimationSizeGranularity = 0;
1926                         properties->primitiveUnderestimation = false;
1927                         properties->conservativePointAndLineRasterization = false;
1928                         properties->degenerateTrianglesRasterized = false;
1929                         properties->degenerateLinesRasterized = false;
1930                         properties->fullyCoveredFragmentShaderInputVariable = false;
1931                         properties->conservativeRasterizationPostDepthCoverage = false;
1932                         break;
1933                 }
1934                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT: {
1935                         VkPhysicalDevicePCIBusInfoPropertiesEXT *properties =
1936                                 (VkPhysicalDevicePCIBusInfoPropertiesEXT *)ext;
1937                         properties->pciDomain = pdevice->bus_info.domain;
1938                         properties->pciBus = pdevice->bus_info.bus;
1939                         properties->pciDevice = pdevice->bus_info.dev;
1940                         properties->pciFunction = pdevice->bus_info.func;
1941                         break;
1942                 }
1943                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES: {
1944                         VkPhysicalDeviceDriverProperties *properties =
1945                                 (VkPhysicalDeviceDriverProperties *) ext;
1946                         CORE_PROPERTY(1, 2, driverID);
1947                         CORE_PROPERTY(1, 2, driverName);
1948                         CORE_PROPERTY(1, 2, driverInfo);
1949                         CORE_PROPERTY(1, 2, conformanceVersion);
1950                         break;
1951                 }
1952                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_PROPERTIES_EXT: {
1953                         VkPhysicalDeviceTransformFeedbackPropertiesEXT *properties =
1954                                 (VkPhysicalDeviceTransformFeedbackPropertiesEXT *)ext;
1955                         properties->maxTransformFeedbackStreams = MAX_SO_STREAMS;
1956                         properties->maxTransformFeedbackBuffers = MAX_SO_BUFFERS;
1957                         properties->maxTransformFeedbackBufferSize = UINT32_MAX;
1958                         properties->maxTransformFeedbackStreamDataSize = 512;
1959                         properties->maxTransformFeedbackBufferDataSize = UINT32_MAX;
1960                         properties->maxTransformFeedbackBufferDataStride = 512;
1961                         properties->transformFeedbackQueries = !pdevice->use_ngg_streamout;
1962                         properties->transformFeedbackStreamsLinesTriangles = !pdevice->use_ngg_streamout;
1963                         properties->transformFeedbackRasterizationStreamSelect = false;
1964                         properties->transformFeedbackDraw = true;
1965                         break;
1966                 }
1967                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_PROPERTIES_EXT: {
1968                         VkPhysicalDeviceInlineUniformBlockPropertiesEXT *props =
1969                                 (VkPhysicalDeviceInlineUniformBlockPropertiesEXT *)ext;
1970
1971                         props->maxInlineUniformBlockSize = MAX_INLINE_UNIFORM_BLOCK_SIZE;
1972                         props->maxPerStageDescriptorInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_SETS;
1973                         props->maxPerStageDescriptorUpdateAfterBindInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_SETS;
1974                         props->maxDescriptorSetInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_COUNT;
1975                         props->maxDescriptorSetUpdateAfterBindInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_COUNT;
1976                         break;
1977                 }
1978                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLE_LOCATIONS_PROPERTIES_EXT: {
1979                         VkPhysicalDeviceSampleLocationsPropertiesEXT *properties =
1980                                 (VkPhysicalDeviceSampleLocationsPropertiesEXT *)ext;
1981                         properties->sampleLocationSampleCounts = VK_SAMPLE_COUNT_2_BIT |
1982                                                                  VK_SAMPLE_COUNT_4_BIT |
1983                                                                  VK_SAMPLE_COUNT_8_BIT;
1984                         properties->maxSampleLocationGridSize = (VkExtent2D){ 2 , 2 };
1985                         properties->sampleLocationCoordinateRange[0] = 0.0f;
1986                         properties->sampleLocationCoordinateRange[1] = 0.9375f;
1987                         properties->sampleLocationSubPixelBits = 4;
1988                         properties->variableSampleLocations = false;
1989                         break;
1990                 }
1991                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES: {
1992                         VkPhysicalDeviceDepthStencilResolveProperties *properties =
1993                                 (VkPhysicalDeviceDepthStencilResolveProperties *)ext;
1994                         CORE_PROPERTY(1, 2, supportedDepthResolveModes);
1995                         CORE_PROPERTY(1, 2, supportedStencilResolveModes);
1996                         CORE_PROPERTY(1, 2, independentResolveNone);
1997                         CORE_PROPERTY(1, 2, independentResolve);
1998                         break;
1999                 }
2000                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_PROPERTIES_EXT: {
2001                         VkPhysicalDeviceTexelBufferAlignmentPropertiesEXT *properties =
2002                                 (VkPhysicalDeviceTexelBufferAlignmentPropertiesEXT *)ext;
2003                         properties->storageTexelBufferOffsetAlignmentBytes = 4;
2004                         properties->storageTexelBufferOffsetSingleTexelAlignment = true;
2005                         properties->uniformTexelBufferOffsetAlignmentBytes = 4;
2006                         properties->uniformTexelBufferOffsetSingleTexelAlignment = true;
2007                         break;
2008                 }
2009                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES : {
2010                         VkPhysicalDeviceFloatControlsProperties *properties =
2011                                 (VkPhysicalDeviceFloatControlsProperties *)ext;
2012                         CORE_PROPERTY(1, 2, denormBehaviorIndependence);
2013                         CORE_PROPERTY(1, 2, roundingModeIndependence);
2014                         CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat16);
2015                         CORE_PROPERTY(1, 2, shaderDenormPreserveFloat16);
2016                         CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat16);
2017                         CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat16);
2018                         CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat16);
2019                         CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat32);
2020                         CORE_PROPERTY(1, 2, shaderDenormPreserveFloat32);
2021                         CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat32);
2022                         CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat32);
2023                         CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat32);
2024                         CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat64);
2025                         CORE_PROPERTY(1, 2, shaderDenormPreserveFloat64);
2026                         CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat64);
2027                         CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat64);
2028                         CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat64);
2029                         break;
2030                 }
2031                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_PROPERTIES: {
2032                         VkPhysicalDeviceTimelineSemaphoreProperties *properties =
2033                                 (VkPhysicalDeviceTimelineSemaphoreProperties *) ext;
2034                         CORE_PROPERTY(1, 2, maxTimelineSemaphoreValueDifference);
2035                         break;
2036                 }
2037                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT: {
2038                         VkPhysicalDeviceSubgroupSizeControlPropertiesEXT *props =
2039                                 (VkPhysicalDeviceSubgroupSizeControlPropertiesEXT *)ext;
2040                         props->minSubgroupSize = 64;
2041                         props->maxSubgroupSize = 64;
2042                         props->maxComputeWorkgroupSubgroups = UINT32_MAX;
2043                         props->requiredSubgroupSizeStages = 0;
2044
2045                         if (pdevice->rad_info.chip_class >= GFX10) {
2046                                 /* Only GFX10+ supports wave32. */
2047                                 props->minSubgroupSize = 32;
2048                                 props->requiredSubgroupSizeStages = VK_SHADER_STAGE_COMPUTE_BIT;
2049                         }
2050                         break;
2051                 }
2052                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES:
2053                         radv_get_physical_device_properties_1_1(pdevice, (void *)ext);
2054                         break;
2055                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES:
2056                         radv_get_physical_device_properties_1_2(pdevice, (void *)ext);
2057                         break;
2058                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_PROPERTIES_EXT: {
2059                         VkPhysicalDeviceLineRasterizationPropertiesEXT *props =
2060                                 (VkPhysicalDeviceLineRasterizationPropertiesEXT *)ext;
2061                         props->lineSubPixelPrecisionBits = 4;
2062                         break;
2063                 }
2064                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_PROPERTIES_EXT: {
2065                         VkPhysicalDeviceRobustness2PropertiesEXT *properties =
2066                                 (VkPhysicalDeviceRobustness2PropertiesEXT *)ext;
2067                         properties->robustStorageBufferAccessSizeAlignment = 4;
2068                         properties->robustUniformBufferAccessSizeAlignment = 4;
2069                         break;
2070                 }
2071                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_PROPERTIES_EXT: {
2072                         VkPhysicalDeviceCustomBorderColorPropertiesEXT *props =
2073                                 (VkPhysicalDeviceCustomBorderColorPropertiesEXT *)ext;
2074                         props->maxCustomBorderColorSamplers = RADV_BORDER_COLOR_COUNT;
2075                         break;
2076                 }
2077                 default:
2078                         break;
2079                 }
2080         }
2081 }
2082
2083 static void radv_get_physical_device_queue_family_properties(
2084         struct radv_physical_device*                pdevice,
2085         uint32_t*                                   pCount,
2086         VkQueueFamilyProperties**                    pQueueFamilyProperties)
2087 {
2088         int num_queue_families = 1;
2089         int idx;
2090         if (pdevice->rad_info.num_rings[RING_COMPUTE] > 0 &&
2091             !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE))
2092                 num_queue_families++;
2093
2094         if (pQueueFamilyProperties == NULL) {
2095                 *pCount = num_queue_families;
2096                 return;
2097         }
2098
2099         if (!*pCount)
2100                 return;
2101
2102         idx = 0;
2103         if (*pCount >= 1) {
2104                 *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
2105                         .queueFlags = VK_QUEUE_GRAPHICS_BIT |
2106                                       VK_QUEUE_COMPUTE_BIT |
2107                                       VK_QUEUE_TRANSFER_BIT |
2108                                       VK_QUEUE_SPARSE_BINDING_BIT,
2109                         .queueCount = 1,
2110                         .timestampValidBits = 64,
2111                         .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
2112                 };
2113                 idx++;
2114         }
2115
2116         if (pdevice->rad_info.num_rings[RING_COMPUTE] > 0 &&
2117             !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE)) {
2118                 if (*pCount > idx) {
2119                         *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
2120                                 .queueFlags = VK_QUEUE_COMPUTE_BIT |
2121                                               VK_QUEUE_TRANSFER_BIT |
2122                                               VK_QUEUE_SPARSE_BINDING_BIT,
2123                                 .queueCount = pdevice->rad_info.num_rings[RING_COMPUTE],
2124                                 .timestampValidBits = 64,
2125                                 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
2126                         };
2127                         idx++;
2128                 }
2129         }
2130         *pCount = idx;
2131 }
2132
2133 void radv_GetPhysicalDeviceQueueFamilyProperties(
2134         VkPhysicalDevice                            physicalDevice,
2135         uint32_t*                                   pCount,
2136         VkQueueFamilyProperties*                    pQueueFamilyProperties)
2137 {
2138         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
2139         if (!pQueueFamilyProperties) {
2140                 radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
2141                 return;
2142         }
2143         VkQueueFamilyProperties *properties[] = {
2144                 pQueueFamilyProperties + 0,
2145                 pQueueFamilyProperties + 1,
2146                 pQueueFamilyProperties + 2,
2147         };
2148         radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
2149         assert(*pCount <= 3);
2150 }
2151
2152 void radv_GetPhysicalDeviceQueueFamilyProperties2(
2153         VkPhysicalDevice                            physicalDevice,
2154         uint32_t*                                   pCount,
2155         VkQueueFamilyProperties2                   *pQueueFamilyProperties)
2156 {
2157         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
2158         if (!pQueueFamilyProperties) {
2159                 radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
2160                 return;
2161         }
2162         VkQueueFamilyProperties *properties[] = {
2163                 &pQueueFamilyProperties[0].queueFamilyProperties,
2164                 &pQueueFamilyProperties[1].queueFamilyProperties,
2165                 &pQueueFamilyProperties[2].queueFamilyProperties,
2166         };
2167         radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
2168         assert(*pCount <= 3);
2169 }
2170
2171 void radv_GetPhysicalDeviceMemoryProperties(
2172         VkPhysicalDevice                            physicalDevice,
2173         VkPhysicalDeviceMemoryProperties           *pMemoryProperties)
2174 {
2175         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2176
2177         *pMemoryProperties = physical_device->memory_properties;
2178 }
2179
2180 static void
2181 radv_get_memory_budget_properties(VkPhysicalDevice physicalDevice,
2182                                   VkPhysicalDeviceMemoryBudgetPropertiesEXT *memoryBudget)
2183 {
2184         RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
2185         VkPhysicalDeviceMemoryProperties *memory_properties = &device->memory_properties;
2186         uint64_t visible_vram_size = radv_get_visible_vram_size(device);
2187         uint64_t vram_size = radv_get_vram_size(device);
2188         uint64_t gtt_size = device->rad_info.gart_size;
2189         uint64_t heap_budget, heap_usage;
2190
2191         /* For all memory heaps, the computation of budget is as follow:
2192          *      heap_budget = heap_size - global_heap_usage + app_heap_usage
2193          *
2194          * The Vulkan spec 1.1.97 says that the budget should include any
2195          * currently allocated device memory.
2196          *
2197          * Note that the application heap usages are not really accurate (eg.
2198          * in presence of shared buffers).
2199          */
2200         for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
2201                 uint32_t heap_index = device->memory_properties.memoryTypes[i].heapIndex;
2202
2203                 if ((device->memory_domains[i] & RADEON_DOMAIN_VRAM) && (device->memory_flags[i] & RADEON_FLAG_NO_CPU_ACCESS)) {
2204                         heap_usage = device->ws->query_value(device->ws,
2205                                                              RADEON_ALLOCATED_VRAM);
2206
2207                         heap_budget = vram_size -
2208                                 device->ws->query_value(device->ws, RADEON_VRAM_USAGE) +
2209                                 heap_usage;
2210
2211                         memoryBudget->heapBudget[heap_index] = heap_budget;
2212                         memoryBudget->heapUsage[heap_index] = heap_usage;
2213                 } else if (device->memory_domains[i] & RADEON_DOMAIN_VRAM) {
2214                         heap_usage = device->ws->query_value(device->ws,
2215                                                              RADEON_ALLOCATED_VRAM_VIS);
2216
2217                         heap_budget = visible_vram_size -
2218                                 device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE) +
2219                                 heap_usage;
2220
2221                         memoryBudget->heapBudget[heap_index] = heap_budget;
2222                         memoryBudget->heapUsage[heap_index] = heap_usage;
2223                 } else {
2224                         assert(device->memory_domains[i] & RADEON_DOMAIN_GTT);
2225
2226                         heap_usage = device->ws->query_value(device->ws,
2227                                                              RADEON_ALLOCATED_GTT);
2228
2229                         heap_budget = gtt_size -
2230                                 device->ws->query_value(device->ws, RADEON_GTT_USAGE) +
2231                                 heap_usage;
2232
2233                         memoryBudget->heapBudget[heap_index] = heap_budget;
2234                         memoryBudget->heapUsage[heap_index] = heap_usage;
2235                 }
2236         }
2237
2238         /* The heapBudget and heapUsage values must be zero for array elements
2239          * greater than or equal to
2240          * VkPhysicalDeviceMemoryProperties::memoryHeapCount.
2241          */
2242         for (uint32_t i = memory_properties->memoryHeapCount; i < VK_MAX_MEMORY_HEAPS; i++) {
2243                 memoryBudget->heapBudget[i] = 0;
2244                 memoryBudget->heapUsage[i] = 0;
2245         }
2246 }
2247
2248 void radv_GetPhysicalDeviceMemoryProperties2(
2249         VkPhysicalDevice                            physicalDevice,
2250         VkPhysicalDeviceMemoryProperties2          *pMemoryProperties)
2251 {
2252         radv_GetPhysicalDeviceMemoryProperties(physicalDevice,
2253                                                &pMemoryProperties->memoryProperties);
2254
2255         VkPhysicalDeviceMemoryBudgetPropertiesEXT *memory_budget =
2256                 vk_find_struct(pMemoryProperties->pNext,
2257                                PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT);
2258         if (memory_budget)
2259                 radv_get_memory_budget_properties(physicalDevice, memory_budget);
2260 }
2261
2262 VkResult radv_GetMemoryHostPointerPropertiesEXT(
2263         VkDevice                                    _device,
2264         VkExternalMemoryHandleTypeFlagBits          handleType,
2265         const void                                 *pHostPointer,
2266         VkMemoryHostPointerPropertiesEXT           *pMemoryHostPointerProperties)
2267 {
2268         RADV_FROM_HANDLE(radv_device, device, _device);
2269
2270         switch (handleType)
2271         {
2272         case VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT: {
2273                 const struct radv_physical_device *physical_device = device->physical_device;
2274                 uint32_t memoryTypeBits = 0;
2275                 for (int i = 0; i < physical_device->memory_properties.memoryTypeCount; i++) {
2276                         if (physical_device->memory_domains[i] == RADEON_DOMAIN_GTT &&
2277                             !(physical_device->memory_flags[i] & RADEON_FLAG_GTT_WC)) {
2278                                 memoryTypeBits = (1 << i);
2279                                 break;
2280                         }
2281                 }
2282                 pMemoryHostPointerProperties->memoryTypeBits = memoryTypeBits;
2283                 return VK_SUCCESS;
2284         }
2285         default:
2286                 return VK_ERROR_INVALID_EXTERNAL_HANDLE;
2287         }
2288 }
2289
2290 static enum radeon_ctx_priority
2291 radv_get_queue_global_priority(const VkDeviceQueueGlobalPriorityCreateInfoEXT *pObj)
2292 {
2293         /* Default to MEDIUM when a specific global priority isn't requested */
2294         if (!pObj)
2295                 return RADEON_CTX_PRIORITY_MEDIUM;
2296
2297         switch(pObj->globalPriority) {
2298         case VK_QUEUE_GLOBAL_PRIORITY_REALTIME_EXT:
2299                 return RADEON_CTX_PRIORITY_REALTIME;
2300         case VK_QUEUE_GLOBAL_PRIORITY_HIGH_EXT:
2301                 return RADEON_CTX_PRIORITY_HIGH;
2302         case VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_EXT:
2303                 return RADEON_CTX_PRIORITY_MEDIUM;
2304         case VK_QUEUE_GLOBAL_PRIORITY_LOW_EXT:
2305                 return RADEON_CTX_PRIORITY_LOW;
2306         default:
2307                 unreachable("Illegal global priority value");
2308                 return RADEON_CTX_PRIORITY_INVALID;
2309         }
2310 }
2311
2312 static int
2313 radv_queue_init(struct radv_device *device, struct radv_queue *queue,
2314                 uint32_t queue_family_index, int idx,
2315                 VkDeviceQueueCreateFlags flags,
2316                 const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority)
2317 {
2318         queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
2319         queue->device = device;
2320         queue->queue_family_index = queue_family_index;
2321         queue->queue_idx = idx;
2322         queue->priority = radv_get_queue_global_priority(global_priority);
2323         queue->flags = flags;
2324         queue->hw_ctx = NULL;
2325
2326         VkResult result = device->ws->ctx_create(device->ws, queue->priority, &queue->hw_ctx);
2327         if (result != VK_SUCCESS)
2328                 return vk_error(device->instance, result);
2329
2330         list_inithead(&queue->pending_submissions);
2331         pthread_mutex_init(&queue->pending_mutex, NULL);
2332
2333         pthread_mutex_init(&queue->thread_mutex, NULL);
2334         queue->thread_submission = NULL;
2335         queue->thread_running = queue->thread_exit = false;
2336         result = radv_create_pthread_cond(&queue->thread_cond);
2337         if (result != VK_SUCCESS)
2338                 return vk_error(device->instance, result);
2339
2340         return VK_SUCCESS;
2341 }
2342
2343 static void
2344 radv_queue_finish(struct radv_queue *queue)
2345 {
2346         if (queue->thread_running) {
2347                 p_atomic_set(&queue->thread_exit, true);
2348                 pthread_cond_broadcast(&queue->thread_cond);
2349                 pthread_join(queue->submission_thread, NULL);
2350         }
2351         pthread_cond_destroy(&queue->thread_cond);
2352         pthread_mutex_destroy(&queue->pending_mutex);
2353         pthread_mutex_destroy(&queue->thread_mutex);
2354
2355         if (queue->hw_ctx)
2356                 queue->device->ws->ctx_destroy(queue->hw_ctx);
2357
2358         if (queue->initial_full_flush_preamble_cs)
2359                 queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
2360         if (queue->initial_preamble_cs)
2361                 queue->device->ws->cs_destroy(queue->initial_preamble_cs);
2362         if (queue->continue_preamble_cs)
2363                 queue->device->ws->cs_destroy(queue->continue_preamble_cs);
2364         if (queue->descriptor_bo)
2365                 queue->device->ws->buffer_destroy(queue->descriptor_bo);
2366         if (queue->scratch_bo)
2367                 queue->device->ws->buffer_destroy(queue->scratch_bo);
2368         if (queue->esgs_ring_bo)
2369                 queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
2370         if (queue->gsvs_ring_bo)
2371                 queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
2372         if (queue->tess_rings_bo)
2373                 queue->device->ws->buffer_destroy(queue->tess_rings_bo);
2374         if (queue->gds_bo)
2375                 queue->device->ws->buffer_destroy(queue->gds_bo);
2376         if (queue->gds_oa_bo)
2377                 queue->device->ws->buffer_destroy(queue->gds_oa_bo);
2378         if (queue->compute_scratch_bo)
2379                 queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
2380 }
2381
2382 static void
2383 radv_bo_list_init(struct radv_bo_list *bo_list)
2384 {
2385         pthread_mutex_init(&bo_list->mutex, NULL);
2386         bo_list->list.count = bo_list->capacity = 0;
2387         bo_list->list.bos = NULL;
2388 }
2389
2390 static void
2391 radv_bo_list_finish(struct radv_bo_list *bo_list)
2392 {
2393         free(bo_list->list.bos);
2394         pthread_mutex_destroy(&bo_list->mutex);
2395 }
2396
2397 VkResult radv_bo_list_add(struct radv_device *device,
2398                           struct radeon_winsys_bo *bo)
2399 {
2400         struct radv_bo_list *bo_list = &device->bo_list;
2401
2402         if (bo->is_local)
2403                 return VK_SUCCESS;
2404
2405         if (unlikely(!device->use_global_bo_list))
2406                 return VK_SUCCESS;
2407
2408         pthread_mutex_lock(&bo_list->mutex);
2409         if (bo_list->list.count == bo_list->capacity) {
2410                 unsigned capacity = MAX2(4, bo_list->capacity * 2);
2411                 void *data = realloc(bo_list->list.bos, capacity * sizeof(struct radeon_winsys_bo*));
2412
2413                 if (!data) {
2414                         pthread_mutex_unlock(&bo_list->mutex);
2415                         return VK_ERROR_OUT_OF_HOST_MEMORY;
2416                 }
2417
2418                 bo_list->list.bos = (struct radeon_winsys_bo**)data;
2419                 bo_list->capacity = capacity;
2420         }
2421
2422         bo_list->list.bos[bo_list->list.count++] = bo;
2423         pthread_mutex_unlock(&bo_list->mutex);
2424         return VK_SUCCESS;
2425 }
2426
2427 void radv_bo_list_remove(struct radv_device *device,
2428                          struct radeon_winsys_bo *bo)
2429 {
2430         struct radv_bo_list *bo_list = &device->bo_list;
2431
2432         if (bo->is_local)
2433                 return;
2434
2435         if (unlikely(!device->use_global_bo_list))
2436                 return;
2437
2438         pthread_mutex_lock(&bo_list->mutex);
2439         /* Loop the list backwards so we find the most recently added
2440          * memory first. */
2441         for(unsigned i = bo_list->list.count; i-- > 0;) {
2442                 if (bo_list->list.bos[i] == bo) {
2443                         bo_list->list.bos[i] = bo_list->list.bos[bo_list->list.count - 1];
2444                         --bo_list->list.count;
2445                         break;
2446                 }
2447         }
2448         pthread_mutex_unlock(&bo_list->mutex);
2449 }
2450
2451 static void
2452 radv_device_init_gs_info(struct radv_device *device)
2453 {
2454         device->gs_table_depth = ac_get_gs_table_depth(device->physical_device->rad_info.chip_class,
2455                                                        device->physical_device->rad_info.family);
2456 }
2457
2458 static int radv_get_device_extension_index(const char *name)
2459 {
2460         for (unsigned i = 0; i < RADV_DEVICE_EXTENSION_COUNT; ++i) {
2461                 if (strcmp(name, radv_device_extensions[i].extensionName) == 0)
2462                         return i;
2463         }
2464         return -1;
2465 }
2466
2467 static int
2468 radv_get_int_debug_option(const char *name, int default_value)
2469 {
2470         const char *str;
2471         int result;
2472
2473         str = getenv(name);
2474         if (!str) {
2475                 result = default_value;
2476         } else {
2477                 char *endptr;
2478
2479                 result = strtol(str, &endptr, 0);
2480                 if (str == endptr) {
2481                         /* No digits founs. */
2482                         result = default_value;
2483                 }
2484         }
2485
2486         return result;
2487 }
2488
2489 static bool radv_thread_trace_enabled()
2490 {
2491         return radv_get_int_debug_option("RADV_THREAD_TRACE", -1) >= 0;
2492 }
2493
2494 static void
2495 radv_device_init_dispatch(struct radv_device *device)
2496 {
2497         const struct radv_instance *instance = device->physical_device->instance;
2498         const struct radv_device_dispatch_table *dispatch_table_layer = NULL;
2499         bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
2500
2501         if (radv_thread_trace_enabled()) {
2502                 /* Use device entrypoints from the SQTT layer if enabled. */
2503                 dispatch_table_layer = &sqtt_device_dispatch_table;
2504         }
2505
2506         for (unsigned i = 0; i < ARRAY_SIZE(device->dispatch.entrypoints); i++) {
2507                 /* Vulkan requires that entrypoints for extensions which have not been
2508                  * enabled must not be advertised.
2509                  */
2510                 if (!unchecked &&
2511                     !radv_device_entrypoint_is_enabled(i, instance->apiVersion,
2512                                                        &instance->enabled_extensions,
2513                                                        &device->enabled_extensions)) {
2514                         device->dispatch.entrypoints[i] = NULL;
2515                 } else if (dispatch_table_layer &&
2516                            dispatch_table_layer->entrypoints[i]) {
2517                         device->dispatch.entrypoints[i] =
2518                                 dispatch_table_layer->entrypoints[i];
2519                 } else {
2520                         device->dispatch.entrypoints[i] =
2521                                 radv_device_dispatch_table.entrypoints[i];
2522                 }
2523         }
2524 }
2525
2526 static VkResult
2527 radv_create_pthread_cond(pthread_cond_t *cond)
2528 {
2529         pthread_condattr_t condattr;
2530         if (pthread_condattr_init(&condattr)) {
2531                 return VK_ERROR_INITIALIZATION_FAILED;
2532         }
2533
2534         if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
2535                 pthread_condattr_destroy(&condattr);
2536                 return VK_ERROR_INITIALIZATION_FAILED;
2537         }
2538         if (pthread_cond_init(cond, &condattr)) {
2539                 pthread_condattr_destroy(&condattr);
2540                 return VK_ERROR_INITIALIZATION_FAILED;
2541         }
2542         pthread_condattr_destroy(&condattr);
2543         return VK_SUCCESS;
2544 }
2545
2546 static VkResult
2547 check_physical_device_features(VkPhysicalDevice physicalDevice,
2548                                const VkPhysicalDeviceFeatures *features)
2549 {
2550         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2551         VkPhysicalDeviceFeatures supported_features;
2552         radv_GetPhysicalDeviceFeatures(physicalDevice, &supported_features);
2553         VkBool32 *supported_feature = (VkBool32 *)&supported_features;
2554         VkBool32 *enabled_feature = (VkBool32 *)features;
2555         unsigned num_features = sizeof(VkPhysicalDeviceFeatures) / sizeof(VkBool32);
2556         for (uint32_t i = 0; i < num_features; i++) {
2557                 if (enabled_feature[i] && !supported_feature[i])
2558                         return vk_error(physical_device->instance, VK_ERROR_FEATURE_NOT_PRESENT);
2559         }
2560
2561         return VK_SUCCESS;
2562 }
2563
2564 static VkResult radv_device_init_border_color(struct radv_device *device)
2565 {
2566         device->border_color_data.bo =
2567         device->ws->buffer_create(device->ws,
2568                                         RADV_BORDER_COLOR_BUFFER_SIZE,
2569                                         4096,
2570                                         RADEON_DOMAIN_VRAM,
2571                                         RADEON_FLAG_CPU_ACCESS |
2572                                         RADEON_FLAG_READ_ONLY |
2573                                         RADEON_FLAG_NO_INTERPROCESS_SHARING,
2574                                         RADV_BO_PRIORITY_SHADER);
2575
2576         if (device->border_color_data.bo == NULL)
2577                 return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
2578
2579         device->border_color_data.colors_gpu_ptr =
2580                 device->ws->buffer_map(device->border_color_data.bo);
2581         if (!device->border_color_data.colors_gpu_ptr)
2582                 return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
2583         pthread_mutex_init(&device->border_color_data.mutex, NULL);
2584
2585         return VK_SUCCESS;
2586 }
2587
2588 static void radv_device_finish_border_color(struct radv_device *device)
2589 {
2590         if (device->border_color_data.bo) {
2591                 device->ws->buffer_destroy(device->border_color_data.bo);
2592
2593                 pthread_mutex_destroy(&device->border_color_data.mutex);
2594         }
2595 }
2596
2597 VkResult
2598 _radv_device_set_lost(struct radv_device *device,
2599                       const char *file, int line,
2600                       const char *msg, ...)
2601 {
2602         VkResult err;
2603         va_list ap;
2604
2605         p_atomic_inc(&device->lost);
2606
2607         va_start(ap, msg);
2608         err = __vk_errorv(device->physical_device->instance, device,
2609                           VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT,
2610                           VK_ERROR_DEVICE_LOST, file, line, msg, ap);
2611         va_end(ap);
2612
2613         return err;
2614 }
2615
2616 VkResult radv_CreateDevice(
2617         VkPhysicalDevice                            physicalDevice,
2618         const VkDeviceCreateInfo*                   pCreateInfo,
2619         const VkAllocationCallbacks*                pAllocator,
2620         VkDevice*                                   pDevice)
2621 {
2622         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2623         VkResult result;
2624         struct radv_device *device;
2625
2626         bool keep_shader_info = false;
2627         bool robust_buffer_access = false;
2628         bool overallocation_disallowed = false;
2629         bool custom_border_colors = false;
2630
2631         /* Check enabled features */
2632         if (pCreateInfo->pEnabledFeatures) {
2633                 result = check_physical_device_features(physicalDevice,
2634                                                         pCreateInfo->pEnabledFeatures);
2635                 if (result != VK_SUCCESS)
2636                         return result;
2637
2638                 if (pCreateInfo->pEnabledFeatures->robustBufferAccess)
2639                         robust_buffer_access = true;
2640         }
2641
2642         vk_foreach_struct_const(ext, pCreateInfo->pNext) {
2643                 switch (ext->sType) {
2644                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2: {
2645                         const VkPhysicalDeviceFeatures2 *features = (const void *)ext;
2646                         result = check_physical_device_features(physicalDevice,
2647                                                                 &features->features);
2648                         if (result != VK_SUCCESS)
2649                                 return result;
2650
2651                         if (features->features.robustBufferAccess)
2652                                 robust_buffer_access = true;
2653                         break;
2654                 }
2655                 case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: {
2656                         const VkDeviceMemoryOverallocationCreateInfoAMD *overallocation = (const void *)ext;
2657                         if (overallocation->overallocationBehavior == VK_MEMORY_OVERALLOCATION_BEHAVIOR_DISALLOWED_AMD)
2658                                 overallocation_disallowed = true;
2659                         break;
2660                 }
2661                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT: {
2662                         const VkPhysicalDeviceCustomBorderColorFeaturesEXT *border_color_features = (const void *)ext;
2663                         custom_border_colors = border_color_features->customBorderColors;
2664                         break;
2665                 }
2666                 default:
2667                         break;
2668                 }
2669         }
2670
2671         device = vk_zalloc2(&physical_device->instance->alloc, pAllocator,
2672                             sizeof(*device), 8,
2673                             VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2674         if (!device)
2675                 return vk_error(physical_device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2676
2677         vk_device_init(&device->vk, pCreateInfo,
2678                        &physical_device->instance->alloc, pAllocator);
2679
2680         device->instance = physical_device->instance;
2681         device->physical_device = physical_device;
2682
2683         device->ws = physical_device->ws;
2684
2685         for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
2686                 const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i];
2687                 int index = radv_get_device_extension_index(ext_name);
2688                 if (index < 0 || !physical_device->supported_extensions.extensions[index]) {
2689                         vk_free(&device->vk.alloc, device);
2690                         return vk_error(physical_device->instance, VK_ERROR_EXTENSION_NOT_PRESENT);
2691                 }
2692
2693                 device->enabled_extensions.extensions[index] = true;
2694         }
2695
2696         radv_device_init_dispatch(device);
2697
2698         keep_shader_info = device->enabled_extensions.AMD_shader_info;
2699
2700         /* With update after bind we can't attach bo's to the command buffer
2701          * from the descriptor set anymore, so we have to use a global BO list.
2702          */
2703         device->use_global_bo_list =
2704                 (device->instance->perftest_flags & RADV_PERFTEST_BO_LIST) ||
2705                 device->enabled_extensions.EXT_descriptor_indexing ||
2706                 device->enabled_extensions.EXT_buffer_device_address ||
2707                 device->enabled_extensions.KHR_buffer_device_address;
2708
2709         device->robust_buffer_access = robust_buffer_access;
2710
2711         mtx_init(&device->shader_slab_mutex, mtx_plain);
2712         list_inithead(&device->shader_slabs);
2713
2714         device->overallocation_disallowed = overallocation_disallowed;
2715         mtx_init(&device->overallocation_mutex, mtx_plain);
2716
2717         radv_bo_list_init(&device->bo_list);
2718
2719         for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
2720                 const VkDeviceQueueCreateInfo *queue_create = &pCreateInfo->pQueueCreateInfos[i];
2721                 uint32_t qfi = queue_create->queueFamilyIndex;
2722                 const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority =
2723                         vk_find_struct_const(queue_create->pNext, DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT);
2724
2725                 assert(!global_priority || device->physical_device->rad_info.has_ctx_priority);
2726
2727                 device->queues[qfi] = vk_alloc(&device->vk.alloc,
2728                                                queue_create->queueCount * sizeof(struct radv_queue), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2729                 if (!device->queues[qfi]) {
2730                         result = VK_ERROR_OUT_OF_HOST_MEMORY;
2731                         goto fail;
2732                 }
2733
2734                 memset(device->queues[qfi], 0, queue_create->queueCount * sizeof(struct radv_queue));
2735
2736                 device->queue_count[qfi] = queue_create->queueCount;
2737
2738                 for (unsigned q = 0; q < queue_create->queueCount; q++) {
2739                         result = radv_queue_init(device, &device->queues[qfi][q],
2740                                                  qfi, q, queue_create->flags,
2741                                                  global_priority);
2742                         if (result != VK_SUCCESS)
2743                                 goto fail;
2744                 }
2745         }
2746
2747         device->pbb_allowed = device->physical_device->rad_info.chip_class >= GFX9 &&
2748                               !(device->instance->debug_flags & RADV_DEBUG_NOBINNING);
2749
2750         /* Disable DFSM by default. As of 2019-09-15 Talos on Low is still 3% slower on Raven. */
2751         device->dfsm_allowed = device->pbb_allowed &&
2752                                (device->instance->perftest_flags & RADV_PERFTEST_DFSM);
2753
2754         device->always_use_syncobj = device->physical_device->rad_info.has_syncobj_wait_for_submit;
2755
2756         /* The maximum number of scratch waves. Scratch space isn't divided
2757          * evenly between CUs. The number is only a function of the number of CUs.
2758          * We can decrease the constant to decrease the scratch buffer size.
2759          *
2760          * sctx->scratch_waves must be >= the maximum possible size of
2761          * 1 threadgroup, so that the hw doesn't hang from being unable
2762          * to start any.
2763          *
2764          * The recommended value is 4 per CU at most. Higher numbers don't
2765          * bring much benefit, but they still occupy chip resources (think
2766          * async compute). I've seen ~2% performance difference between 4 and 32.
2767          */
2768         uint32_t max_threads_per_block = 2048;
2769         device->scratch_waves = MAX2(32 * physical_device->rad_info.num_good_compute_units,
2770                                      max_threads_per_block / 64);
2771
2772         device->dispatch_initiator = S_00B800_COMPUTE_SHADER_EN(1);
2773
2774         if (device->physical_device->rad_info.chip_class >= GFX7) {
2775                 /* If the KMD allows it (there is a KMD hw register for it),
2776                  * allow launching waves out-of-order.
2777                  */
2778                 device->dispatch_initiator |= S_00B800_ORDER_MODE(1);
2779         }
2780
2781         radv_device_init_gs_info(device);
2782
2783         device->tess_offchip_block_dw_size =
2784                 device->physical_device->rad_info.family == CHIP_HAWAII ? 4096 : 8192;
2785
2786         if (getenv("RADV_TRACE_FILE")) {
2787                 const char *filename = getenv("RADV_TRACE_FILE");
2788
2789                 keep_shader_info = true;
2790
2791                 if (!radv_init_trace(device))
2792                         goto fail;
2793
2794                 fprintf(stderr, "*****************************************************************************\n");
2795                 fprintf(stderr, "* WARNING: RADV_TRACE_FILE is costly and should only be used for debugging! *\n");
2796                 fprintf(stderr, "*****************************************************************************\n");
2797
2798                 fprintf(stderr, "Trace file will be dumped to %s\n", filename);
2799
2800                 /* Wait for idle after every draw/dispatch to identify the
2801                  * first bad call.
2802                  */
2803                 device->instance->debug_flags |= RADV_DEBUG_SYNC_SHADERS;
2804
2805                 radv_dump_enabled_options(device, stderr);
2806         }
2807
2808         if (radv_thread_trace_enabled()) {
2809                 fprintf(stderr, "*************************************************\n");
2810                 fprintf(stderr, "* WARNING: Thread trace support is experimental *\n");
2811                 fprintf(stderr, "*************************************************\n");
2812
2813                 if (device->physical_device->rad_info.chip_class < GFX8) {
2814                         fprintf(stderr, "GPU hardware not supported: refer to "
2815                                         "the RGP documentation for the list of "
2816                                         "supported GPUs!\n");
2817                         abort();
2818                 }
2819
2820                 /* Default buffer size set to 1MB per SE. */
2821                 device->thread_trace_buffer_size =
2822                         radv_get_int_debug_option("RADV_THREAD_TRACE_BUFFER_SIZE", 1024 * 1024);
2823                 device->thread_trace_start_frame = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
2824
2825                 if (!radv_thread_trace_init(device))
2826                         goto fail;
2827         }
2828
2829         if (getenv("RADV_TRAP_HANDLER")) {
2830                 /* TODO: Add support for more hardware. */
2831                 assert(device->physical_device->rad_info.chip_class == GFX8);
2832
2833                 fprintf(stderr, "**********************************************************************\n");
2834                 fprintf(stderr, "* WARNING: RADV_TRAP_HANDLER is experimental and only for debugging! *\n");
2835                 fprintf(stderr, "**********************************************************************\n");
2836
2837                 /* To get the disassembly of the faulty shaders, we have to
2838                  * keep some shader info around.
2839                  */
2840                 keep_shader_info = true;
2841
2842                 if (!radv_trap_handler_init(device))
2843                         goto fail;
2844         }
2845
2846         device->keep_shader_info = keep_shader_info;
2847         result = radv_device_init_meta(device);
2848         if (result != VK_SUCCESS)
2849                 goto fail;
2850
2851         radv_device_init_msaa(device);
2852
2853         /* If the border color extension is enabled, let's create the buffer we need. */
2854         if (custom_border_colors) {
2855                 result = radv_device_init_border_color(device);
2856                 if (result != VK_SUCCESS)
2857                         goto fail;
2858         }
2859
2860         for (int family = 0; family < RADV_MAX_QUEUE_FAMILIES; ++family) {
2861                 device->empty_cs[family] = device->ws->cs_create(device->ws, family);
2862                 if (!device->empty_cs[family])
2863                         goto fail;
2864
2865                 switch (family) {
2866                 case RADV_QUEUE_GENERAL:
2867                         radeon_emit(device->empty_cs[family], PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
2868                         radeon_emit(device->empty_cs[family], CC0_UPDATE_LOAD_ENABLES(1));
2869                         radeon_emit(device->empty_cs[family], CC1_UPDATE_SHADOW_ENABLES(1));
2870                         break;
2871                 case RADV_QUEUE_COMPUTE:
2872                         radeon_emit(device->empty_cs[family], PKT3(PKT3_NOP, 0, 0));
2873                         radeon_emit(device->empty_cs[family], 0);
2874                         break;
2875                 }
2876
2877                 result = device->ws->cs_finalize(device->empty_cs[family]);
2878                 if (result != VK_SUCCESS)
2879                         goto fail;
2880         }
2881
2882         if (device->physical_device->rad_info.chip_class >= GFX7)
2883                 cik_create_gfx_config(device);
2884
2885         VkPipelineCacheCreateInfo ci;
2886         ci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
2887         ci.pNext = NULL;
2888         ci.flags = 0;
2889         ci.pInitialData = NULL;
2890         ci.initialDataSize = 0;
2891         VkPipelineCache pc;
2892         result = radv_CreatePipelineCache(radv_device_to_handle(device),
2893                                           &ci, NULL, &pc);
2894         if (result != VK_SUCCESS)
2895                 goto fail_meta;
2896
2897         device->mem_cache = radv_pipeline_cache_from_handle(pc);
2898
2899         result = radv_create_pthread_cond(&device->timeline_cond);
2900         if (result != VK_SUCCESS)
2901                 goto fail_mem_cache;
2902
2903         device->force_aniso =
2904                 MIN2(16, radv_get_int_debug_option("RADV_TEX_ANISO", -1));
2905         if (device->force_aniso >= 0) {
2906                 fprintf(stderr, "radv: Forcing anisotropy filter to %ix\n",
2907                         1 << util_logbase2(device->force_aniso));
2908         }
2909
2910         *pDevice = radv_device_to_handle(device);
2911         return VK_SUCCESS;
2912
2913 fail_mem_cache:
2914         radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
2915 fail_meta:
2916         radv_device_finish_meta(device);
2917 fail:
2918         radv_bo_list_finish(&device->bo_list);
2919
2920         radv_thread_trace_finish(device);
2921
2922         radv_trap_handler_finish(device);
2923
2924         if (device->trace_bo)
2925                 device->ws->buffer_destroy(device->trace_bo);
2926
2927         if (device->gfx_init)
2928                 device->ws->buffer_destroy(device->gfx_init);
2929
2930         radv_device_finish_border_color(device);
2931
2932         for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
2933                 for (unsigned q = 0; q < device->queue_count[i]; q++)
2934                         radv_queue_finish(&device->queues[i][q]);
2935                 if (device->queue_count[i])
2936                         vk_free(&device->vk.alloc, device->queues[i]);
2937         }
2938
2939         vk_free(&device->vk.alloc, device);
2940         return result;
2941 }
2942
2943 void radv_DestroyDevice(
2944         VkDevice                                    _device,
2945         const VkAllocationCallbacks*                pAllocator)
2946 {
2947         RADV_FROM_HANDLE(radv_device, device, _device);
2948
2949         if (!device)
2950                 return;
2951
2952         if (device->trace_bo)
2953                 device->ws->buffer_destroy(device->trace_bo);
2954
2955         if (device->gfx_init)
2956                 device->ws->buffer_destroy(device->gfx_init);
2957
2958         radv_device_finish_border_color(device);
2959
2960         for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
2961                 for (unsigned q = 0; q < device->queue_count[i]; q++)
2962                         radv_queue_finish(&device->queues[i][q]);
2963                 if (device->queue_count[i])
2964                         vk_free(&device->vk.alloc, device->queues[i]);
2965                 if (device->empty_cs[i])
2966                         device->ws->cs_destroy(device->empty_cs[i]);
2967         }
2968         radv_device_finish_meta(device);
2969
2970         VkPipelineCache pc = radv_pipeline_cache_to_handle(device->mem_cache);
2971         radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
2972
2973         radv_trap_handler_finish(device);
2974
2975         radv_destroy_shader_slabs(device);
2976
2977         pthread_cond_destroy(&device->timeline_cond);
2978         radv_bo_list_finish(&device->bo_list);
2979
2980         radv_thread_trace_finish(device);
2981
2982         vk_free(&device->vk.alloc, device);
2983 }
2984
2985 VkResult radv_EnumerateInstanceLayerProperties(
2986         uint32_t*                                   pPropertyCount,
2987         VkLayerProperties*                          pProperties)
2988 {
2989         if (pProperties == NULL) {
2990                 *pPropertyCount = 0;
2991                 return VK_SUCCESS;
2992         }
2993
2994         /* None supported at this time */
2995         return vk_error(NULL, VK_ERROR_LAYER_NOT_PRESENT);
2996 }
2997
2998 VkResult radv_EnumerateDeviceLayerProperties(
2999         VkPhysicalDevice                            physicalDevice,
3000         uint32_t*                                   pPropertyCount,
3001         VkLayerProperties*                          pProperties)
3002 {
3003         if (pProperties == NULL) {
3004                 *pPropertyCount = 0;
3005                 return VK_SUCCESS;
3006         }
3007
3008         /* None supported at this time */
3009         return vk_error(NULL, VK_ERROR_LAYER_NOT_PRESENT);
3010 }
3011
3012 void radv_GetDeviceQueue2(
3013         VkDevice                                    _device,
3014         const VkDeviceQueueInfo2*                   pQueueInfo,
3015         VkQueue*                                    pQueue)
3016 {
3017         RADV_FROM_HANDLE(radv_device, device, _device);
3018         struct radv_queue *queue;
3019
3020         queue = &device->queues[pQueueInfo->queueFamilyIndex][pQueueInfo->queueIndex];
3021         if (pQueueInfo->flags != queue->flags) {
3022                 /* From the Vulkan 1.1.70 spec:
3023                  *
3024                  * "The queue returned by vkGetDeviceQueue2 must have the same
3025                  * flags value from this structure as that used at device
3026                  * creation time in a VkDeviceQueueCreateInfo instance. If no
3027                  * matching flags were specified at device creation time then
3028                  * pQueue will return VK_NULL_HANDLE."
3029                  */
3030                 *pQueue = VK_NULL_HANDLE;
3031                 return;
3032         }
3033
3034         *pQueue = radv_queue_to_handle(queue);
3035 }
3036
3037 void radv_GetDeviceQueue(
3038         VkDevice                                    _device,
3039         uint32_t                                    queueFamilyIndex,
3040         uint32_t                                    queueIndex,
3041         VkQueue*                                    pQueue)
3042 {
3043         const VkDeviceQueueInfo2 info = (VkDeviceQueueInfo2) {
3044                 .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2,
3045                 .queueFamilyIndex = queueFamilyIndex,
3046                 .queueIndex = queueIndex
3047         };
3048
3049         radv_GetDeviceQueue2(_device, &info, pQueue);
3050 }
3051
3052 static void
3053 fill_geom_tess_rings(struct radv_queue *queue,
3054                      uint32_t *map,
3055                      bool add_sample_positions,
3056                      uint32_t esgs_ring_size,
3057                      struct radeon_winsys_bo *esgs_ring_bo,
3058                      uint32_t gsvs_ring_size,
3059                      struct radeon_winsys_bo *gsvs_ring_bo,
3060                      uint32_t tess_factor_ring_size,
3061                      uint32_t tess_offchip_ring_offset,
3062                      uint32_t tess_offchip_ring_size,
3063                      struct radeon_winsys_bo *tess_rings_bo)
3064 {
3065         uint32_t *desc = &map[4];
3066
3067         if (esgs_ring_bo) {
3068                 uint64_t esgs_va = radv_buffer_get_va(esgs_ring_bo);
3069
3070                 /* stride 0, num records - size, add tid, swizzle, elsize4,
3071                    index stride 64 */
3072                 desc[0] = esgs_va;
3073                 desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32) |
3074                           S_008F04_SWIZZLE_ENABLE(true);
3075                 desc[2] = esgs_ring_size;
3076                 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3077                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3078                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3079                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3080                           S_008F0C_INDEX_STRIDE(3) |
3081                           S_008F0C_ADD_TID_ENABLE(1);
3082
3083                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3084                         desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3085                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3086                                    S_008F0C_RESOURCE_LEVEL(1);
3087                 } else {
3088                         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3089                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
3090                                    S_008F0C_ELEMENT_SIZE(1);
3091                 }
3092
3093                 /* GS entry for ES->GS ring */
3094                 /* stride 0, num records - size, elsize0,
3095                    index stride 0 */
3096                 desc[4] = esgs_va;
3097                 desc[5] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32);
3098                 desc[6] = esgs_ring_size;
3099                 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3100                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3101                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3102                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3103
3104                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3105                         desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3106                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3107                                    S_008F0C_RESOURCE_LEVEL(1);
3108                 } else {
3109                         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3110                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3111                 }
3112         }
3113
3114         desc += 8;
3115
3116         if (gsvs_ring_bo) {
3117                 uint64_t gsvs_va = radv_buffer_get_va(gsvs_ring_bo);
3118
3119                 /* VS entry for GS->VS ring */
3120                 /* stride 0, num records - size, elsize0,
3121                    index stride 0 */
3122                 desc[0] = gsvs_va;
3123                 desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32);
3124                 desc[2] = gsvs_ring_size;
3125                 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3126                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3127                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3128                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3129
3130                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3131                         desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3132                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3133                                    S_008F0C_RESOURCE_LEVEL(1);
3134                 } else {
3135                         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3136                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3137                 }
3138
3139                 /* stride gsvs_itemsize, num records 64
3140                    elsize 4, index stride 16 */
3141                 /* shader will patch stride and desc[2] */
3142                 desc[4] = gsvs_va;
3143                 desc[5] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32) |
3144                           S_008F04_SWIZZLE_ENABLE(1);
3145                 desc[6] = 0;
3146                 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3147                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3148                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3149                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3150                           S_008F0C_INDEX_STRIDE(1) |
3151                           S_008F0C_ADD_TID_ENABLE(true);
3152
3153                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3154                         desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3155                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3156                                    S_008F0C_RESOURCE_LEVEL(1);
3157                 } else {
3158                         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3159                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
3160                                    S_008F0C_ELEMENT_SIZE(1);
3161                 }
3162
3163         }
3164
3165         desc += 8;
3166
3167         if (tess_rings_bo) {
3168                 uint64_t tess_va = radv_buffer_get_va(tess_rings_bo);
3169                 uint64_t tess_offchip_va = tess_va + tess_offchip_ring_offset;
3170
3171                 desc[0] = tess_va;
3172                 desc[1] = S_008F04_BASE_ADDRESS_HI(tess_va >> 32);
3173                 desc[2] = tess_factor_ring_size;
3174                 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3175                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3176                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3177                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3178
3179                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3180                         desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3181                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3182                                    S_008F0C_RESOURCE_LEVEL(1);
3183                 } else {
3184                         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3185                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3186                 }
3187
3188                 desc[4] = tess_offchip_va;
3189                 desc[5] = S_008F04_BASE_ADDRESS_HI(tess_offchip_va >> 32);
3190                 desc[6] = tess_offchip_ring_size;
3191                 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3192                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3193                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3194                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3195
3196                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3197                         desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3198                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3199                                    S_008F0C_RESOURCE_LEVEL(1);
3200                 } else {
3201                         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3202                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3203                 }
3204         }
3205
3206         desc += 8;
3207
3208         if (add_sample_positions) {
3209                 /* add sample positions after all rings */
3210                 memcpy(desc, queue->device->sample_locations_1x, 8);
3211                 desc += 2;
3212                 memcpy(desc, queue->device->sample_locations_2x, 16);
3213                 desc += 4;
3214                 memcpy(desc, queue->device->sample_locations_4x, 32);
3215                 desc += 8;
3216                 memcpy(desc, queue->device->sample_locations_8x, 64);
3217         }
3218 }
3219
3220 static unsigned
3221 radv_get_hs_offchip_param(struct radv_device *device, uint32_t *max_offchip_buffers_p)
3222 {
3223         bool double_offchip_buffers = device->physical_device->rad_info.chip_class >= GFX7 &&
3224                 device->physical_device->rad_info.family != CHIP_CARRIZO &&
3225                 device->physical_device->rad_info.family != CHIP_STONEY;
3226         unsigned max_offchip_buffers_per_se = double_offchip_buffers ? 128 : 64;
3227         unsigned max_offchip_buffers;
3228         unsigned offchip_granularity;
3229         unsigned hs_offchip_param;
3230
3231         /*
3232          * Per RadeonSI:
3233          * This must be one less than the maximum number due to a hw limitation.
3234          * Various hardware bugs need thGFX7
3235          *
3236          * Per AMDVLK:
3237          * Vega10 should limit max_offchip_buffers to 508 (4 * 127).
3238          * Gfx7 should limit max_offchip_buffers to 508
3239          * Gfx6 should limit max_offchip_buffers to 126 (2 * 63)
3240          *
3241          * Follow AMDVLK here.
3242          */
3243         if (device->physical_device->rad_info.chip_class >= GFX10) {
3244                 max_offchip_buffers_per_se = 256;
3245         } else if (device->physical_device->rad_info.family == CHIP_VEGA10 ||
3246                    device->physical_device->rad_info.chip_class == GFX7 ||
3247                    device->physical_device->rad_info.chip_class == GFX6)
3248                 --max_offchip_buffers_per_se;
3249
3250         max_offchip_buffers = max_offchip_buffers_per_se *
3251                 device->physical_device->rad_info.max_se;
3252
3253         /* Hawaii has a bug with offchip buffers > 256 that can be worked
3254          * around by setting 4K granularity.
3255          */
3256         if (device->tess_offchip_block_dw_size == 4096) {
3257                 assert(device->physical_device->rad_info.family == CHIP_HAWAII);
3258                 offchip_granularity = V_03093C_X_4K_DWORDS;
3259         } else {
3260                 assert(device->tess_offchip_block_dw_size == 8192);
3261                 offchip_granularity = V_03093C_X_8K_DWORDS;
3262         }
3263
3264         switch (device->physical_device->rad_info.chip_class) {
3265         case GFX6:
3266                 max_offchip_buffers = MIN2(max_offchip_buffers, 126);
3267                 break;
3268         case GFX7:
3269         case GFX8:
3270         case GFX9:
3271                 max_offchip_buffers = MIN2(max_offchip_buffers, 508);
3272                 break;
3273         case GFX10:
3274                 break;
3275         default:
3276                 break;
3277         }
3278
3279         *max_offchip_buffers_p = max_offchip_buffers;
3280         if (device->physical_device->rad_info.chip_class >= GFX10_3) {
3281                 hs_offchip_param = S_03093C_OFFCHIP_BUFFERING_GFX103(max_offchip_buffers - 1) |
3282                                    S_03093C_OFFCHIP_GRANULARITY_GFX103(offchip_granularity);
3283         } else if (device->physical_device->rad_info.chip_class >= GFX7) {
3284                 if (device->physical_device->rad_info.chip_class >= GFX8)
3285                         --max_offchip_buffers;
3286                 hs_offchip_param =
3287                         S_03093C_OFFCHIP_BUFFERING_GFX7(max_offchip_buffers) |
3288                         S_03093C_OFFCHIP_GRANULARITY_GFX7(offchip_granularity);
3289         } else {
3290                 hs_offchip_param =
3291                         S_0089B0_OFFCHIP_BUFFERING(max_offchip_buffers);
3292         }
3293         return hs_offchip_param;
3294 }
3295
3296 static void
3297 radv_emit_gs_ring_sizes(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3298                         struct radeon_winsys_bo *esgs_ring_bo,
3299                         uint32_t esgs_ring_size,
3300                         struct radeon_winsys_bo *gsvs_ring_bo,
3301                         uint32_t gsvs_ring_size)
3302 {
3303         if (!esgs_ring_bo && !gsvs_ring_bo)
3304                 return;
3305
3306         if (esgs_ring_bo)
3307                 radv_cs_add_buffer(queue->device->ws, cs, esgs_ring_bo);
3308
3309         if (gsvs_ring_bo)
3310                 radv_cs_add_buffer(queue->device->ws, cs, gsvs_ring_bo);
3311
3312         if (queue->device->physical_device->rad_info.chip_class >= GFX7) {
3313                 radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2);
3314                 radeon_emit(cs, esgs_ring_size >> 8);
3315                 radeon_emit(cs, gsvs_ring_size >> 8);
3316         } else {
3317                 radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2);
3318                 radeon_emit(cs, esgs_ring_size >> 8);
3319                 radeon_emit(cs, gsvs_ring_size >> 8);
3320         }
3321 }
3322
3323 static void
3324 radv_emit_tess_factor_ring(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3325                            unsigned hs_offchip_param, unsigned tf_ring_size,
3326                            struct radeon_winsys_bo *tess_rings_bo)
3327 {
3328         uint64_t tf_va;
3329
3330         if (!tess_rings_bo)
3331                 return;
3332
3333         tf_va = radv_buffer_get_va(tess_rings_bo);
3334
3335         radv_cs_add_buffer(queue->device->ws, cs, tess_rings_bo);
3336
3337         if (queue->device->physical_device->rad_info.chip_class >= GFX7) {
3338                 radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE,
3339                                        S_030938_SIZE(tf_ring_size / 4));
3340                 radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE,
3341                                        tf_va >> 8);
3342
3343                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3344                         radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI_UMD,
3345                                                S_030984_BASE_HI(tf_va >> 40));
3346                 } else if (queue->device->physical_device->rad_info.chip_class == GFX9) {
3347                         radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI,
3348                                                S_030944_BASE_HI(tf_va >> 40));
3349                 }
3350                 radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM,
3351                                        hs_offchip_param);
3352         } else {
3353                 radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE,
3354                                       S_008988_SIZE(tf_ring_size / 4));
3355                 radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE,
3356                                       tf_va >> 8);
3357                 radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM,
3358                                      hs_offchip_param);
3359         }
3360 }
3361
3362 static void
3363 radv_emit_graphics_scratch(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3364                            uint32_t size_per_wave, uint32_t waves,
3365                            struct radeon_winsys_bo *scratch_bo)
3366 {
3367         if (queue->queue_family_index != RADV_QUEUE_GENERAL)
3368                 return;
3369
3370         if (!scratch_bo)
3371                 return;
3372
3373         radv_cs_add_buffer(queue->device->ws, cs, scratch_bo);
3374
3375         radeon_set_context_reg(cs, R_0286E8_SPI_TMPRING_SIZE,
3376                                S_0286E8_WAVES(waves) |
3377                                S_0286E8_WAVESIZE(round_up_u32(size_per_wave, 1024)));
3378 }
3379
3380 static void
3381 radv_emit_compute_scratch(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3382                           uint32_t size_per_wave, uint32_t waves,
3383                           struct radeon_winsys_bo *compute_scratch_bo)
3384 {
3385         uint64_t scratch_va;
3386
3387         if (!compute_scratch_bo)
3388                 return;
3389
3390         scratch_va = radv_buffer_get_va(compute_scratch_bo);
3391
3392         radv_cs_add_buffer(queue->device->ws, cs, compute_scratch_bo);
3393
3394         radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2);
3395         radeon_emit(cs, scratch_va);
3396         radeon_emit(cs, S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
3397                         S_008F04_SWIZZLE_ENABLE(1));
3398
3399         radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE,
3400                          S_00B860_WAVES(waves) |
3401                          S_00B860_WAVESIZE(round_up_u32(size_per_wave, 1024)));
3402 }
3403
3404 static void
3405 radv_emit_global_shader_pointers(struct radv_queue *queue,
3406                                  struct radeon_cmdbuf *cs,
3407                                  struct radeon_winsys_bo *descriptor_bo)
3408 {
3409         uint64_t va;
3410
3411         if (!descriptor_bo)
3412                 return;
3413
3414         va = radv_buffer_get_va(descriptor_bo);
3415
3416         radv_cs_add_buffer(queue->device->ws, cs, descriptor_bo);
3417
3418         if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3419                 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3420                                    R_00B130_SPI_SHADER_USER_DATA_VS_0,
3421                                    R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
3422                                    R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
3423
3424                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3425                         radv_emit_shader_pointer(queue->device, cs, regs[i],
3426                                                  va, true);
3427                 }
3428         } else if (queue->device->physical_device->rad_info.chip_class == GFX9) {
3429                 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3430                                    R_00B130_SPI_SHADER_USER_DATA_VS_0,
3431                                    R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
3432                                    R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
3433
3434                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3435                         radv_emit_shader_pointer(queue->device, cs, regs[i],
3436                                                  va, true);
3437                 }
3438         } else {
3439                 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3440                                    R_00B130_SPI_SHADER_USER_DATA_VS_0,
3441                                    R_00B230_SPI_SHADER_USER_DATA_GS_0,
3442                                    R_00B330_SPI_SHADER_USER_DATA_ES_0,
3443                                    R_00B430_SPI_SHADER_USER_DATA_HS_0,
3444                                    R_00B530_SPI_SHADER_USER_DATA_LS_0};
3445
3446                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3447                         radv_emit_shader_pointer(queue->device, cs, regs[i],
3448                                                  va, true);
3449                 }
3450         }
3451 }
3452
3453 static void
3454 radv_emit_trap_handler(struct radv_queue *queue,
3455                        struct radeon_cmdbuf *cs,
3456                        struct radeon_winsys_bo *tma_bo)
3457 {
3458         struct radv_device *device = queue->device;
3459         struct radeon_winsys_bo *tba_bo;
3460         uint64_t tba_va, tma_va;
3461
3462         if (!device->trap_handler_shader || !tma_bo)
3463                 return;
3464
3465         tba_bo = device->trap_handler_shader->bo;
3466
3467         tba_va = radv_buffer_get_va(tba_bo) + device->trap_handler_shader->bo_offset;
3468         tma_va = radv_buffer_get_va(tma_bo);
3469
3470         radv_cs_add_buffer(queue->device->ws, cs, tba_bo);
3471         radv_cs_add_buffer(queue->device->ws, cs, tma_bo);
3472
3473         if (queue->queue_family_index == RADV_QUEUE_GENERAL) {
3474                 uint32_t regs[] = {R_00B000_SPI_SHADER_TBA_LO_PS,
3475                                    R_00B100_SPI_SHADER_TBA_LO_VS,
3476                                    R_00B200_SPI_SHADER_TBA_LO_GS,
3477                                    R_00B300_SPI_SHADER_TBA_LO_ES,
3478                                    R_00B400_SPI_SHADER_TBA_LO_HS,
3479                                    R_00B500_SPI_SHADER_TBA_LO_LS};
3480
3481                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3482                         radeon_set_sh_reg_seq(cs, regs[i], 4);
3483                         radeon_emit(cs, tba_va >> 8);
3484                         radeon_emit(cs, tba_va >> 40);
3485                         radeon_emit(cs, tma_va >> 8);
3486                         radeon_emit(cs, tma_va >> 40);
3487                 }
3488         } else {
3489                 radeon_set_sh_reg_seq(cs, R_00B838_COMPUTE_TBA_LO, 4);
3490                 radeon_emit(cs, tba_va >> 8);
3491                 radeon_emit(cs, tba_va >> 40);
3492                 radeon_emit(cs, tma_va >> 8);
3493                 radeon_emit(cs, tma_va >> 40);
3494         }
3495 }
3496
3497 static void
3498 radv_init_graphics_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
3499 {
3500         struct radv_device *device = queue->device;
3501
3502         if (device->gfx_init) {
3503                 uint64_t va = radv_buffer_get_va(device->gfx_init);
3504
3505                 radeon_emit(cs, PKT3(PKT3_INDIRECT_BUFFER_CIK, 2, 0));
3506                 radeon_emit(cs, va);
3507                 radeon_emit(cs, va >> 32);
3508                 radeon_emit(cs, device->gfx_init_size_dw & 0xffff);
3509
3510                 radv_cs_add_buffer(device->ws, cs, device->gfx_init);
3511         } else {
3512                 si_emit_graphics(device, cs);
3513         }
3514 }
3515
3516 static void
3517 radv_init_compute_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
3518 {
3519         si_emit_compute(queue->device, cs);
3520 }
3521
3522 static VkResult
3523 radv_get_preamble_cs(struct radv_queue *queue,
3524                      uint32_t scratch_size_per_wave,
3525                      uint32_t scratch_waves,
3526                      uint32_t compute_scratch_size_per_wave,
3527                      uint32_t compute_scratch_waves,
3528                      uint32_t esgs_ring_size,
3529                      uint32_t gsvs_ring_size,
3530                      bool needs_tess_rings,
3531                      bool needs_gds,
3532                      bool needs_gds_oa,
3533                      bool needs_sample_positions,
3534                      struct radeon_cmdbuf **initial_full_flush_preamble_cs,
3535                      struct radeon_cmdbuf **initial_preamble_cs,
3536                      struct radeon_cmdbuf **continue_preamble_cs)
3537 {
3538         struct radeon_winsys_bo *scratch_bo = NULL;
3539         struct radeon_winsys_bo *descriptor_bo = NULL;
3540         struct radeon_winsys_bo *compute_scratch_bo = NULL;
3541         struct radeon_winsys_bo *esgs_ring_bo = NULL;
3542         struct radeon_winsys_bo *gsvs_ring_bo = NULL;
3543         struct radeon_winsys_bo *tess_rings_bo = NULL;
3544         struct radeon_winsys_bo *gds_bo = NULL;
3545         struct radeon_winsys_bo *gds_oa_bo = NULL;
3546         struct radeon_cmdbuf *dest_cs[3] = {0};
3547         bool add_tess_rings = false, add_gds = false, add_gds_oa = false, add_sample_positions = false;
3548         unsigned tess_factor_ring_size = 0, tess_offchip_ring_size = 0;
3549         unsigned max_offchip_buffers;
3550         unsigned hs_offchip_param = 0;
3551         unsigned tess_offchip_ring_offset;
3552         uint32_t ring_bo_flags = RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING;
3553         if (!queue->has_tess_rings) {
3554                 if (needs_tess_rings)
3555                         add_tess_rings = true;
3556         }
3557         if (!queue->has_gds) {
3558                 if (needs_gds)
3559                         add_gds = true;
3560         }
3561         if (!queue->has_gds_oa) {
3562                 if (needs_gds_oa)
3563                         add_gds_oa = true;
3564         }
3565         if (!queue->has_sample_positions) {
3566                 if (needs_sample_positions)
3567                         add_sample_positions = true;
3568         }
3569         tess_factor_ring_size = 32768 * queue->device->physical_device->rad_info.max_se;
3570         hs_offchip_param = radv_get_hs_offchip_param(queue->device,
3571                                                      &max_offchip_buffers);
3572         tess_offchip_ring_offset = align(tess_factor_ring_size, 64 * 1024);
3573         tess_offchip_ring_size = max_offchip_buffers *
3574                 queue->device->tess_offchip_block_dw_size * 4;
3575
3576         scratch_size_per_wave = MAX2(scratch_size_per_wave, queue->scratch_size_per_wave);
3577         if (scratch_size_per_wave)
3578                 scratch_waves = MIN2(scratch_waves, UINT32_MAX / scratch_size_per_wave);
3579         else
3580                 scratch_waves = 0;
3581
3582         compute_scratch_size_per_wave = MAX2(compute_scratch_size_per_wave, queue->compute_scratch_size_per_wave);
3583         if (compute_scratch_size_per_wave)
3584                 compute_scratch_waves = MIN2(compute_scratch_waves, UINT32_MAX / compute_scratch_size_per_wave);
3585         else
3586                 compute_scratch_waves = 0;
3587
3588         if (scratch_size_per_wave <= queue->scratch_size_per_wave &&
3589             scratch_waves <= queue->scratch_waves &&
3590             compute_scratch_size_per_wave <= queue->compute_scratch_size_per_wave &&
3591             compute_scratch_waves <= queue->compute_scratch_waves &&
3592             esgs_ring_size <= queue->esgs_ring_size &&
3593             gsvs_ring_size <= queue->gsvs_ring_size &&
3594             !add_tess_rings && !add_gds && !add_gds_oa && !add_sample_positions &&
3595             queue->initial_preamble_cs) {
3596                 *initial_full_flush_preamble_cs = queue->initial_full_flush_preamble_cs;
3597                 *initial_preamble_cs = queue->initial_preamble_cs;
3598                 *continue_preamble_cs = queue->continue_preamble_cs;
3599                 if (!scratch_size_per_wave && !compute_scratch_size_per_wave &&
3600                     !esgs_ring_size && !gsvs_ring_size && !needs_tess_rings &&
3601                     !needs_gds && !needs_gds_oa && !needs_sample_positions)
3602                         *continue_preamble_cs = NULL;
3603                 return VK_SUCCESS;
3604         }
3605
3606         uint32_t scratch_size = scratch_size_per_wave * scratch_waves;
3607         uint32_t queue_scratch_size = queue->scratch_size_per_wave * queue->scratch_waves;
3608         if (scratch_size > queue_scratch_size) {
3609                 scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
3610                                                               scratch_size,
3611                                                               4096,
3612                                                               RADEON_DOMAIN_VRAM,
3613                                                               ring_bo_flags,
3614                                                               RADV_BO_PRIORITY_SCRATCH);
3615                 if (!scratch_bo)
3616                         goto fail;
3617         } else
3618                 scratch_bo = queue->scratch_bo;
3619
3620         uint32_t compute_scratch_size = compute_scratch_size_per_wave * compute_scratch_waves;
3621         uint32_t compute_queue_scratch_size = queue->compute_scratch_size_per_wave * queue->compute_scratch_waves;
3622         if (compute_scratch_size > compute_queue_scratch_size) {
3623                 compute_scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
3624                                                                       compute_scratch_size,
3625                                                                       4096,
3626                                                                       RADEON_DOMAIN_VRAM,
3627                                                                       ring_bo_flags,
3628                                                                       RADV_BO_PRIORITY_SCRATCH);
3629                 if (!compute_scratch_bo)
3630                         goto fail;
3631
3632         } else
3633                 compute_scratch_bo = queue->compute_scratch_bo;
3634
3635         if (esgs_ring_size > queue->esgs_ring_size) {
3636                 esgs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
3637                                                                 esgs_ring_size,
3638                                                                 4096,
3639                                                                 RADEON_DOMAIN_VRAM,
3640                                                                 ring_bo_flags,
3641                                                                 RADV_BO_PRIORITY_SCRATCH);
3642                 if (!esgs_ring_bo)
3643                         goto fail;
3644         } else {
3645                 esgs_ring_bo = queue->esgs_ring_bo;
3646                 esgs_ring_size = queue->esgs_ring_size;
3647         }
3648
3649         if (gsvs_ring_size > queue->gsvs_ring_size) {
3650                 gsvs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
3651                                                                 gsvs_ring_size,
3652                                                                 4096,
3653                                                                 RADEON_DOMAIN_VRAM,
3654                                                                 ring_bo_flags,
3655                                                                 RADV_BO_PRIORITY_SCRATCH);
3656                 if (!gsvs_ring_bo)
3657                         goto fail;
3658         } else {
3659                 gsvs_ring_bo = queue->gsvs_ring_bo;
3660                 gsvs_ring_size = queue->gsvs_ring_size;
3661         }
3662
3663         if (add_tess_rings) {
3664                 tess_rings_bo = queue->device->ws->buffer_create(queue->device->ws,
3665                                                                  tess_offchip_ring_offset + tess_offchip_ring_size,
3666                                                                  256,
3667                                                                  RADEON_DOMAIN_VRAM,
3668                                                                  ring_bo_flags,
3669                                                                  RADV_BO_PRIORITY_SCRATCH);
3670                 if (!tess_rings_bo)
3671                         goto fail;
3672         } else {
3673                 tess_rings_bo = queue->tess_rings_bo;
3674         }
3675
3676         if (add_gds) {
3677                 assert(queue->device->physical_device->rad_info.chip_class >= GFX10);
3678
3679                 /* 4 streamout GDS counters.
3680                  * We need 256B (64 dw) of GDS, otherwise streamout hangs.
3681                  */
3682                 gds_bo = queue->device->ws->buffer_create(queue->device->ws,
3683                                                           256, 4,
3684                                                           RADEON_DOMAIN_GDS,
3685                                                           ring_bo_flags,
3686                                                           RADV_BO_PRIORITY_SCRATCH);
3687                 if (!gds_bo)
3688                         goto fail;
3689         } else {
3690                 gds_bo = queue->gds_bo;
3691         }
3692
3693         if (add_gds_oa) {
3694                 assert(queue->device->physical_device->rad_info.chip_class >= GFX10);
3695
3696                 gds_oa_bo = queue->device->ws->buffer_create(queue->device->ws,
3697                                                              4, 1,
3698                                                              RADEON_DOMAIN_OA,
3699                                                              ring_bo_flags,
3700                                                              RADV_BO_PRIORITY_SCRATCH);
3701                 if (!gds_oa_bo)
3702                         goto fail;
3703         } else {
3704                 gds_oa_bo = queue->gds_oa_bo;
3705         }
3706
3707         if (scratch_bo != queue->scratch_bo ||
3708             esgs_ring_bo != queue->esgs_ring_bo ||
3709             gsvs_ring_bo != queue->gsvs_ring_bo ||
3710             tess_rings_bo != queue->tess_rings_bo ||
3711             add_sample_positions) {
3712                 uint32_t size = 0;
3713                 if (gsvs_ring_bo || esgs_ring_bo ||
3714                     tess_rings_bo || add_sample_positions) {
3715                         size = 112; /* 2 dword + 2 padding + 4 dword * 6 */
3716                         if (add_sample_positions)
3717                                 size += 128; /* 64+32+16+8 = 120 bytes */
3718                 }
3719                 else if (scratch_bo)
3720                         size = 8; /* 2 dword */
3721
3722                 descriptor_bo = queue->device->ws->buffer_create(queue->device->ws,
3723                                                                  size,
3724                                                                  4096,
3725                                                                  RADEON_DOMAIN_VRAM,
3726                                                                  RADEON_FLAG_CPU_ACCESS |
3727                                                                  RADEON_FLAG_NO_INTERPROCESS_SHARING |
3728                                                                  RADEON_FLAG_READ_ONLY,
3729                                                                  RADV_BO_PRIORITY_DESCRIPTOR);
3730                 if (!descriptor_bo)
3731                         goto fail;
3732         } else
3733                 descriptor_bo = queue->descriptor_bo;
3734
3735         if (descriptor_bo != queue->descriptor_bo) {
3736                 uint32_t *map = (uint32_t*)queue->device->ws->buffer_map(descriptor_bo);
3737                 if (!map)
3738                         goto fail;
3739
3740                 if (scratch_bo) {
3741                         uint64_t scratch_va = radv_buffer_get_va(scratch_bo);
3742                         uint32_t rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
3743                                          S_008F04_SWIZZLE_ENABLE(1);
3744                         map[0] = scratch_va;
3745                         map[1] = rsrc1;
3746                 }
3747
3748                 if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || add_sample_positions)
3749                         fill_geom_tess_rings(queue, map, add_sample_positions,
3750                                              esgs_ring_size, esgs_ring_bo,
3751                                              gsvs_ring_size, gsvs_ring_bo,
3752                                              tess_factor_ring_size,
3753                                              tess_offchip_ring_offset,
3754                                              tess_offchip_ring_size,
3755                                              tess_rings_bo);
3756
3757                 queue->device->ws->buffer_unmap(descriptor_bo);
3758         }
3759
3760         for(int i = 0; i < 3; ++i) {
3761                 struct radeon_cmdbuf *cs = NULL;
3762                 cs = queue->device->ws->cs_create(queue->device->ws,
3763                                                   queue->queue_family_index ? RING_COMPUTE : RING_GFX);
3764                 if (!cs)
3765                         goto fail;
3766
3767                 dest_cs[i] = cs;
3768
3769                 if (scratch_bo)
3770                         radv_cs_add_buffer(queue->device->ws, cs, scratch_bo);
3771
3772                 /* Emit initial configuration. */
3773                 switch (queue->queue_family_index) {
3774                 case RADV_QUEUE_GENERAL:
3775                         radv_init_graphics_state(cs, queue);
3776                         break;
3777                 case RADV_QUEUE_COMPUTE:
3778                         radv_init_compute_state(cs, queue);
3779                         break;
3780                 case RADV_QUEUE_TRANSFER:
3781                         break;
3782                 }
3783
3784                 if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo)  {
3785                         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
3786                         radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
3787
3788                         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
3789                         radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
3790                 }
3791
3792                 radv_emit_gs_ring_sizes(queue, cs, esgs_ring_bo, esgs_ring_size,
3793                                         gsvs_ring_bo, gsvs_ring_size);
3794                 radv_emit_tess_factor_ring(queue, cs, hs_offchip_param,
3795                                            tess_factor_ring_size, tess_rings_bo);
3796                 radv_emit_global_shader_pointers(queue, cs, descriptor_bo);
3797                 radv_emit_compute_scratch(queue, cs, compute_scratch_size_per_wave,
3798                                           compute_scratch_waves, compute_scratch_bo);
3799                 radv_emit_graphics_scratch(queue, cs, scratch_size_per_wave,
3800                                            scratch_waves, scratch_bo);
3801                 radv_emit_trap_handler(queue, cs, queue->device->tma_bo);
3802
3803                 if (gds_bo)
3804                         radv_cs_add_buffer(queue->device->ws, cs, gds_bo);
3805                 if (gds_oa_bo)
3806                         radv_cs_add_buffer(queue->device->ws, cs, gds_oa_bo);
3807
3808                 if (queue->device->trace_bo)
3809                         radv_cs_add_buffer(queue->device->ws, cs, queue->device->trace_bo);
3810
3811                 if (queue->device->border_color_data.bo)
3812                         radv_cs_add_buffer(queue->device->ws, cs,
3813                                            queue->device->border_color_data.bo);
3814
3815                 if (i == 0) {
3816                         si_cs_emit_cache_flush(cs,
3817                                                queue->device->physical_device->rad_info.chip_class,
3818                                                NULL, 0,
3819                                                queue->queue_family_index == RING_COMPUTE &&
3820                                                  queue->device->physical_device->rad_info.chip_class >= GFX7,
3821                                                (queue->queue_family_index == RADV_QUEUE_COMPUTE ? RADV_CMD_FLAG_CS_PARTIAL_FLUSH : (RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH)) |
3822                                                RADV_CMD_FLAG_INV_ICACHE |
3823                                                RADV_CMD_FLAG_INV_SCACHE |
3824                                                RADV_CMD_FLAG_INV_VCACHE |
3825                                                RADV_CMD_FLAG_INV_L2 |
3826                                                RADV_CMD_FLAG_START_PIPELINE_STATS, 0);
3827                 } else if (i == 1) {
3828                         si_cs_emit_cache_flush(cs,
3829                                                queue->device->physical_device->rad_info.chip_class,
3830                                                NULL, 0,
3831                                                queue->queue_family_index == RING_COMPUTE &&
3832                                                  queue->device->physical_device->rad_info.chip_class >= GFX7,
3833                                                RADV_CMD_FLAG_INV_ICACHE |
3834                                                RADV_CMD_FLAG_INV_SCACHE |
3835                                                RADV_CMD_FLAG_INV_VCACHE |
3836                                                RADV_CMD_FLAG_INV_L2 |
3837                                                RADV_CMD_FLAG_START_PIPELINE_STATS, 0);
3838                 }
3839
3840                 if (queue->device->ws->cs_finalize(cs) != VK_SUCCESS)
3841                         goto fail;
3842         }
3843
3844         if (queue->initial_full_flush_preamble_cs)
3845                         queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
3846
3847         if (queue->initial_preamble_cs)
3848                         queue->device->ws->cs_destroy(queue->initial_preamble_cs);
3849
3850         if (queue->continue_preamble_cs)
3851                         queue->device->ws->cs_destroy(queue->continue_preamble_cs);
3852
3853         queue->initial_full_flush_preamble_cs = dest_cs[0];
3854         queue->initial_preamble_cs = dest_cs[1];
3855         queue->continue_preamble_cs = dest_cs[2];
3856
3857         if (scratch_bo != queue->scratch_bo) {
3858                 if (queue->scratch_bo)
3859                         queue->device->ws->buffer_destroy(queue->scratch_bo);
3860                 queue->scratch_bo = scratch_bo;
3861         }
3862         queue->scratch_size_per_wave = scratch_size_per_wave;
3863         queue->scratch_waves = scratch_waves;
3864
3865         if (compute_scratch_bo != queue->compute_scratch_bo) {
3866                 if (queue->compute_scratch_bo)
3867                         queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
3868                 queue->compute_scratch_bo = compute_scratch_bo;
3869         }
3870         queue->compute_scratch_size_per_wave = compute_scratch_size_per_wave;
3871         queue->compute_scratch_waves = compute_scratch_waves;
3872
3873         if (esgs_ring_bo != queue->esgs_ring_bo) {
3874                 if (queue->esgs_ring_bo)
3875                         queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
3876                 queue->esgs_ring_bo = esgs_ring_bo;
3877                 queue->esgs_ring_size = esgs_ring_size;
3878         }
3879
3880         if (gsvs_ring_bo != queue->gsvs_ring_bo) {
3881                 if (queue->gsvs_ring_bo)
3882                         queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
3883                 queue->gsvs_ring_bo = gsvs_ring_bo;
3884                 queue->gsvs_ring_size = gsvs_ring_size;
3885         }
3886
3887         if (tess_rings_bo != queue->tess_rings_bo) {
3888                 queue->tess_rings_bo = tess_rings_bo;
3889                 queue->has_tess_rings = true;
3890         }
3891
3892         if (gds_bo != queue->gds_bo) {
3893                 queue->gds_bo = gds_bo;
3894                 queue->has_gds = true;
3895         }
3896
3897         if (gds_oa_bo != queue->gds_oa_bo) {
3898                 queue->gds_oa_bo = gds_oa_bo;
3899                 queue->has_gds_oa = true;
3900         }
3901
3902         if (descriptor_bo != queue->descriptor_bo) {
3903                 if (queue->descriptor_bo)
3904                         queue->device->ws->buffer_destroy(queue->descriptor_bo);
3905
3906                 queue->descriptor_bo = descriptor_bo;
3907         }
3908
3909         if (add_sample_positions)
3910                 queue->has_sample_positions = true;
3911
3912         *initial_full_flush_preamble_cs = queue->initial_full_flush_preamble_cs;
3913         *initial_preamble_cs = queue->initial_preamble_cs;
3914         *continue_preamble_cs = queue->continue_preamble_cs;
3915         if (!scratch_size && !compute_scratch_size && !esgs_ring_size && !gsvs_ring_size)
3916                         *continue_preamble_cs = NULL;
3917         return VK_SUCCESS;
3918 fail:
3919         for (int i = 0; i < ARRAY_SIZE(dest_cs); ++i)
3920                 if (dest_cs[i])
3921                         queue->device->ws->cs_destroy(dest_cs[i]);
3922         if (descriptor_bo && descriptor_bo != queue->descriptor_bo)
3923                 queue->device->ws->buffer_destroy(descriptor_bo);
3924         if (scratch_bo && scratch_bo != queue->scratch_bo)
3925                 queue->device->ws->buffer_destroy(scratch_bo);
3926         if (compute_scratch_bo && compute_scratch_bo != queue->compute_scratch_bo)
3927                 queue->device->ws->buffer_destroy(compute_scratch_bo);
3928         if (esgs_ring_bo && esgs_ring_bo != queue->esgs_ring_bo)
3929                 queue->device->ws->buffer_destroy(esgs_ring_bo);
3930         if (gsvs_ring_bo && gsvs_ring_bo != queue->gsvs_ring_bo)
3931                 queue->device->ws->buffer_destroy(gsvs_ring_bo);
3932         if (tess_rings_bo && tess_rings_bo != queue->tess_rings_bo)
3933                 queue->device->ws->buffer_destroy(tess_rings_bo);
3934         if (gds_bo && gds_bo != queue->gds_bo)
3935                 queue->device->ws->buffer_destroy(gds_bo);
3936         if (gds_oa_bo && gds_oa_bo != queue->gds_oa_bo)
3937                 queue->device->ws->buffer_destroy(gds_oa_bo);
3938
3939         return vk_error(queue->device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
3940 }
3941
3942 static VkResult radv_alloc_sem_counts(struct radv_device *device,
3943                                       struct radv_winsys_sem_counts *counts,
3944                                       int num_sems,
3945                                       struct radv_semaphore_part **sems,
3946                                       const uint64_t *timeline_values,
3947                                       VkFence _fence,
3948                                       bool is_signal)
3949 {
3950         int syncobj_idx = 0, non_reset_idx = 0, sem_idx = 0, timeline_idx = 0;
3951
3952         if (num_sems == 0 && _fence == VK_NULL_HANDLE)
3953                 return VK_SUCCESS;
3954
3955         for (uint32_t i = 0; i < num_sems; i++) {
3956                 switch(sems[i]->kind) {
3957                 case RADV_SEMAPHORE_SYNCOBJ:
3958                         counts->syncobj_count++;
3959                         counts->syncobj_reset_count++;
3960                         break;
3961                 case RADV_SEMAPHORE_WINSYS:
3962                         counts->sem_count++;
3963                         break;
3964                 case RADV_SEMAPHORE_NONE:
3965                         break;
3966                 case RADV_SEMAPHORE_TIMELINE:
3967                         counts->syncobj_count++;
3968                         break;
3969                 case RADV_SEMAPHORE_TIMELINE_SYNCOBJ:
3970                         counts->timeline_syncobj_count++;
3971                         break;
3972                 }
3973         }
3974
3975         if (_fence != VK_NULL_HANDLE) {
3976                 RADV_FROM_HANDLE(radv_fence, fence, _fence);
3977
3978                 struct radv_fence_part *part =
3979                         fence->temporary.kind != RADV_FENCE_NONE ?
3980                         &fence->temporary : &fence->permanent;
3981                 if (part->kind == RADV_FENCE_SYNCOBJ)
3982                         counts->syncobj_count++;
3983         }
3984
3985         if (counts->syncobj_count || counts->timeline_syncobj_count) {
3986                 counts->points = (uint64_t *)malloc(
3987                         sizeof(*counts->syncobj) * counts->syncobj_count +
3988                         (sizeof(*counts->syncobj) + sizeof(*counts->points)) * counts->timeline_syncobj_count);
3989                 if (!counts->points)
3990                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
3991                 counts->syncobj = (uint32_t*)(counts->points + counts->timeline_syncobj_count);
3992         }
3993
3994         if (counts->sem_count) {
3995                 counts->sem = (struct radeon_winsys_sem **)malloc(sizeof(struct radeon_winsys_sem *) * counts->sem_count);
3996                 if (!counts->sem) {
3997                         free(counts->syncobj);
3998                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
3999                 }
4000         }
4001
4002         non_reset_idx = counts->syncobj_reset_count;
4003
4004         for (uint32_t i = 0; i < num_sems; i++) {
4005                 switch(sems[i]->kind) {
4006                 case RADV_SEMAPHORE_NONE:
4007                         unreachable("Empty semaphore");
4008                         break;
4009                 case RADV_SEMAPHORE_SYNCOBJ:
4010                         counts->syncobj[syncobj_idx++] = sems[i]->syncobj;
4011                         break;
4012                 case RADV_SEMAPHORE_WINSYS:
4013                         counts->sem[sem_idx++] = sems[i]->ws_sem;
4014                         break;
4015                 case RADV_SEMAPHORE_TIMELINE: {
4016                         pthread_mutex_lock(&sems[i]->timeline.mutex);
4017                         struct radv_timeline_point *point = NULL;
4018                         if (is_signal) {
4019                                 point = radv_timeline_add_point_locked(device, &sems[i]->timeline, timeline_values[i]);
4020                         } else {
4021                                 point = radv_timeline_find_point_at_least_locked(device, &sems[i]->timeline, timeline_values[i]);
4022                         }
4023
4024                         pthread_mutex_unlock(&sems[i]->timeline.mutex);
4025
4026                         if (point) {
4027                                 counts->syncobj[non_reset_idx++] = point->syncobj;
4028                         } else {
4029                                 /* Explicitly remove the semaphore so we might not find
4030                                  * a point later post-submit. */
4031                                 sems[i] = NULL;
4032                         }
4033                         break;
4034                 }
4035                 case RADV_SEMAPHORE_TIMELINE_SYNCOBJ:
4036                         counts->syncobj[counts->syncobj_count + timeline_idx] = sems[i]->syncobj;
4037                         counts->points[timeline_idx] = timeline_values[i];
4038                         ++timeline_idx;
4039                         break;
4040                 }
4041         }
4042
4043         if (_fence != VK_NULL_HANDLE) {
4044                 RADV_FROM_HANDLE(radv_fence, fence, _fence);
4045
4046                 struct radv_fence_part *part =
4047                         fence->temporary.kind != RADV_FENCE_NONE ?
4048                         &fence->temporary : &fence->permanent;
4049                 if (part->kind == RADV_FENCE_SYNCOBJ)
4050                         counts->syncobj[non_reset_idx++] = part->syncobj;
4051         }
4052
4053         assert(MAX2(syncobj_idx, non_reset_idx) <= counts->syncobj_count);
4054         counts->syncobj_count = MAX2(syncobj_idx, non_reset_idx);
4055
4056         return VK_SUCCESS;
4057 }
4058
4059 static void
4060 radv_free_sem_info(struct radv_winsys_sem_info *sem_info)
4061 {
4062         free(sem_info->wait.points);
4063         free(sem_info->wait.sem);
4064         free(sem_info->signal.points);
4065         free(sem_info->signal.sem);
4066 }
4067
4068
4069 static void radv_free_temp_syncobjs(struct radv_device *device,
4070                                     int num_sems,
4071                                     struct radv_semaphore_part *sems)
4072 {
4073         for (uint32_t i = 0; i < num_sems; i++) {
4074                 radv_destroy_semaphore_part(device, sems + i);
4075         }
4076 }
4077
4078 static VkResult
4079 radv_alloc_sem_info(struct radv_device *device,
4080                     struct radv_winsys_sem_info *sem_info,
4081                     int num_wait_sems,
4082                     struct radv_semaphore_part **wait_sems,
4083                     const uint64_t *wait_values,
4084                     int num_signal_sems,
4085                     struct radv_semaphore_part **signal_sems,
4086                     const uint64_t *signal_values,
4087                     VkFence fence)
4088 {
4089         VkResult ret;
4090         memset(sem_info, 0, sizeof(*sem_info));
4091
4092         ret = radv_alloc_sem_counts(device, &sem_info->wait, num_wait_sems, wait_sems, wait_values, VK_NULL_HANDLE, false);
4093         if (ret)
4094                 return ret;
4095         ret = radv_alloc_sem_counts(device, &sem_info->signal, num_signal_sems, signal_sems, signal_values, fence, true);
4096         if (ret)
4097                 radv_free_sem_info(sem_info);
4098
4099         /* caller can override these */
4100         sem_info->cs_emit_wait = true;
4101         sem_info->cs_emit_signal = true;
4102         return ret;
4103 }
4104
4105 static void
4106 radv_finalize_timelines(struct radv_device *device,
4107                         uint32_t num_wait_sems,
4108                         struct radv_semaphore_part **wait_sems,
4109                         const uint64_t *wait_values,
4110                         uint32_t num_signal_sems,
4111                         struct radv_semaphore_part **signal_sems,
4112                         const uint64_t *signal_values,
4113                         struct list_head *processing_list)
4114 {
4115         for (uint32_t i = 0; i < num_wait_sems; ++i) {
4116                 if (wait_sems[i] && wait_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4117                         pthread_mutex_lock(&wait_sems[i]->timeline.mutex);
4118                         struct radv_timeline_point *point =
4119                                 radv_timeline_find_point_at_least_locked(device, &wait_sems[i]->timeline, wait_values[i]);
4120                         point->wait_count -= 2;
4121                         pthread_mutex_unlock(&wait_sems[i]->timeline.mutex);
4122                 }
4123         }
4124         for (uint32_t i = 0; i < num_signal_sems; ++i) {
4125                 if (signal_sems[i] && signal_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4126                         pthread_mutex_lock(&signal_sems[i]->timeline.mutex);
4127                         struct radv_timeline_point *point =
4128                                 radv_timeline_find_point_at_least_locked(device, &signal_sems[i]->timeline, signal_values[i]);
4129                         signal_sems[i]->timeline.highest_submitted =
4130                                 MAX2(signal_sems[i]->timeline.highest_submitted, point->value);
4131                         point->wait_count -= 2;
4132                         radv_timeline_trigger_waiters_locked(&signal_sems[i]->timeline, processing_list);
4133                         pthread_mutex_unlock(&signal_sems[i]->timeline.mutex);
4134                 } else if (signal_sems[i] && signal_sems[i]->kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ) {
4135                         signal_sems[i]->timeline_syncobj.max_point =
4136                                 MAX2(signal_sems[i]->timeline_syncobj.max_point, signal_values[i]);
4137                 }
4138         }
4139 }
4140
4141 static VkResult
4142 radv_sparse_buffer_bind_memory(struct radv_device *device,
4143                                const VkSparseBufferMemoryBindInfo *bind)
4144 {
4145         RADV_FROM_HANDLE(radv_buffer, buffer, bind->buffer);
4146         VkResult result;
4147
4148         for (uint32_t i = 0; i < bind->bindCount; ++i) {
4149                 struct radv_device_memory *mem = NULL;
4150
4151                 if (bind->pBinds[i].memory != VK_NULL_HANDLE)
4152                         mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
4153
4154                 result = device->ws->buffer_virtual_bind(buffer->bo,
4155                                                          bind->pBinds[i].resourceOffset,
4156                                                          bind->pBinds[i].size,
4157                                                          mem ? mem->bo : NULL,
4158                                                          bind->pBinds[i].memoryOffset);
4159                 if (result != VK_SUCCESS)
4160                         return result;
4161         }
4162
4163         return VK_SUCCESS;
4164 }
4165
4166 static VkResult
4167 radv_sparse_image_opaque_bind_memory(struct radv_device *device,
4168                                      const VkSparseImageOpaqueMemoryBindInfo *bind)
4169 {
4170         RADV_FROM_HANDLE(radv_image, image, bind->image);
4171         VkResult result;
4172
4173         for (uint32_t i = 0; i < bind->bindCount; ++i) {
4174                 struct radv_device_memory *mem = NULL;
4175
4176                 if (bind->pBinds[i].memory != VK_NULL_HANDLE)
4177                         mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
4178
4179                 result = device->ws->buffer_virtual_bind(image->bo,
4180                                                          bind->pBinds[i].resourceOffset,
4181                                                          bind->pBinds[i].size,
4182                                                          mem ? mem->bo : NULL,
4183                                                          bind->pBinds[i].memoryOffset);
4184                 if (result != VK_SUCCESS)
4185                         return result;
4186         }
4187
4188         return VK_SUCCESS;
4189 }
4190
4191 static VkResult
4192 radv_get_preambles(struct radv_queue *queue,
4193                    const VkCommandBuffer *cmd_buffers,
4194                    uint32_t cmd_buffer_count,
4195                    struct radeon_cmdbuf **initial_full_flush_preamble_cs,
4196                    struct radeon_cmdbuf **initial_preamble_cs,
4197                    struct radeon_cmdbuf **continue_preamble_cs)
4198 {
4199         uint32_t scratch_size_per_wave = 0, waves_wanted = 0;
4200         uint32_t compute_scratch_size_per_wave = 0, compute_waves_wanted = 0;
4201         uint32_t esgs_ring_size = 0, gsvs_ring_size = 0;
4202         bool tess_rings_needed = false;
4203         bool gds_needed = false;
4204         bool gds_oa_needed = false;
4205         bool sample_positions_needed = false;
4206
4207         for (uint32_t j = 0; j < cmd_buffer_count; j++) {
4208                 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer,
4209                                  cmd_buffers[j]);
4210
4211                 scratch_size_per_wave = MAX2(scratch_size_per_wave, cmd_buffer->scratch_size_per_wave_needed);
4212                 waves_wanted = MAX2(waves_wanted, cmd_buffer->scratch_waves_wanted);
4213                 compute_scratch_size_per_wave = MAX2(compute_scratch_size_per_wave,
4214                                                      cmd_buffer->compute_scratch_size_per_wave_needed);
4215                 compute_waves_wanted = MAX2(compute_waves_wanted,
4216                                             cmd_buffer->compute_scratch_waves_wanted);
4217                 esgs_ring_size = MAX2(esgs_ring_size, cmd_buffer->esgs_ring_size_needed);
4218                 gsvs_ring_size = MAX2(gsvs_ring_size, cmd_buffer->gsvs_ring_size_needed);
4219                 tess_rings_needed |= cmd_buffer->tess_rings_needed;
4220                 gds_needed |= cmd_buffer->gds_needed;
4221                 gds_oa_needed |= cmd_buffer->gds_oa_needed;
4222                 sample_positions_needed |= cmd_buffer->sample_positions_needed;
4223         }
4224
4225         return radv_get_preamble_cs(queue, scratch_size_per_wave, waves_wanted,
4226                                     compute_scratch_size_per_wave, compute_waves_wanted,
4227                                     esgs_ring_size, gsvs_ring_size, tess_rings_needed,
4228                                     gds_needed, gds_oa_needed, sample_positions_needed,
4229                                     initial_full_flush_preamble_cs,
4230                                     initial_preamble_cs, continue_preamble_cs);
4231 }
4232
4233 struct radv_deferred_queue_submission {
4234         struct radv_queue *queue;
4235         VkCommandBuffer *cmd_buffers;
4236         uint32_t cmd_buffer_count;
4237
4238         /* Sparse bindings that happen on a queue. */
4239         VkSparseBufferMemoryBindInfo *buffer_binds;
4240         uint32_t buffer_bind_count;
4241         VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
4242         uint32_t image_opaque_bind_count;
4243
4244         bool flush_caches;
4245         VkShaderStageFlags wait_dst_stage_mask;
4246         struct radv_semaphore_part **wait_semaphores;
4247         uint32_t wait_semaphore_count;
4248         struct radv_semaphore_part **signal_semaphores;
4249         uint32_t signal_semaphore_count;
4250         VkFence fence;
4251
4252         uint64_t *wait_values;
4253         uint64_t *signal_values;
4254
4255         struct radv_semaphore_part *temporary_semaphore_parts;
4256         uint32_t temporary_semaphore_part_count;
4257
4258         struct list_head queue_pending_list;
4259         uint32_t submission_wait_count;
4260         struct radv_timeline_waiter *wait_nodes;
4261
4262         struct list_head processing_list;
4263 };
4264
4265 struct radv_queue_submission {
4266         const VkCommandBuffer *cmd_buffers;
4267         uint32_t cmd_buffer_count;
4268
4269         /* Sparse bindings that happen on a queue. */
4270         const VkSparseBufferMemoryBindInfo *buffer_binds;
4271         uint32_t buffer_bind_count;
4272         const VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
4273         uint32_t image_opaque_bind_count;
4274
4275         bool flush_caches;
4276         VkPipelineStageFlags wait_dst_stage_mask;
4277         const VkSemaphore *wait_semaphores;
4278         uint32_t wait_semaphore_count;
4279         const VkSemaphore *signal_semaphores;
4280         uint32_t signal_semaphore_count;
4281         VkFence fence;
4282
4283         const uint64_t *wait_values;
4284         uint32_t wait_value_count;
4285         const uint64_t *signal_values;
4286         uint32_t signal_value_count;
4287 };
4288
4289 static VkResult
4290 radv_queue_trigger_submission(struct radv_deferred_queue_submission *submission,
4291                               uint32_t decrement,
4292                               struct list_head *processing_list);
4293
4294 static VkResult
4295 radv_create_deferred_submission(struct radv_queue *queue,
4296                                 const struct radv_queue_submission *submission,
4297                                 struct radv_deferred_queue_submission **out)
4298 {
4299         struct radv_deferred_queue_submission *deferred = NULL;
4300         size_t size = sizeof(struct radv_deferred_queue_submission);
4301
4302         uint32_t temporary_count = 0;
4303         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4304                 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphores[i]);
4305                 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE)
4306                         ++temporary_count;
4307         }
4308
4309         size += submission->cmd_buffer_count * sizeof(VkCommandBuffer);
4310         size += submission->buffer_bind_count * sizeof(VkSparseBufferMemoryBindInfo);
4311         size += submission->image_opaque_bind_count * sizeof(VkSparseImageOpaqueMemoryBindInfo);
4312         size += submission->wait_semaphore_count * sizeof(struct radv_semaphore_part *);
4313         size += temporary_count * sizeof(struct radv_semaphore_part);
4314         size += submission->signal_semaphore_count * sizeof(struct radv_semaphore_part *);
4315         size += submission->wait_value_count * sizeof(uint64_t);
4316         size += submission->signal_value_count * sizeof(uint64_t);
4317         size += submission->wait_semaphore_count * sizeof(struct radv_timeline_waiter);
4318
4319         deferred = calloc(1, size);
4320         if (!deferred)
4321                 return VK_ERROR_OUT_OF_HOST_MEMORY;
4322
4323         deferred->queue = queue;
4324
4325         deferred->cmd_buffers = (void*)(deferred + 1);
4326         deferred->cmd_buffer_count = submission->cmd_buffer_count;
4327         memcpy(deferred->cmd_buffers, submission->cmd_buffers,
4328                submission->cmd_buffer_count * sizeof(*deferred->cmd_buffers));
4329
4330         deferred->buffer_binds = (void*)(deferred->cmd_buffers + submission->cmd_buffer_count);
4331         deferred->buffer_bind_count = submission->buffer_bind_count;
4332         memcpy(deferred->buffer_binds, submission->buffer_binds,
4333                submission->buffer_bind_count * sizeof(*deferred->buffer_binds));
4334
4335         deferred->image_opaque_binds = (void*)(deferred->buffer_binds + submission->buffer_bind_count);
4336         deferred->image_opaque_bind_count = submission->image_opaque_bind_count;
4337         memcpy(deferred->image_opaque_binds, submission->image_opaque_binds,
4338                submission->image_opaque_bind_count * sizeof(*deferred->image_opaque_binds));
4339
4340         deferred->flush_caches = submission->flush_caches;
4341         deferred->wait_dst_stage_mask = submission->wait_dst_stage_mask;
4342
4343         deferred->wait_semaphores = (void*)(deferred->image_opaque_binds + deferred->image_opaque_bind_count);
4344         deferred->wait_semaphore_count = submission->wait_semaphore_count;
4345
4346         deferred->signal_semaphores = (void*)(deferred->wait_semaphores + deferred->wait_semaphore_count);
4347         deferred->signal_semaphore_count = submission->signal_semaphore_count;
4348
4349         deferred->fence = submission->fence;
4350
4351         deferred->temporary_semaphore_parts = (void*)(deferred->signal_semaphores + deferred->signal_semaphore_count);
4352         deferred->temporary_semaphore_part_count = temporary_count;
4353
4354         uint32_t temporary_idx = 0;
4355         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4356                 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphores[i]);
4357                 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) {
4358                         deferred->wait_semaphores[i] = &deferred->temporary_semaphore_parts[temporary_idx];
4359                         deferred->temporary_semaphore_parts[temporary_idx] = semaphore->temporary;
4360                         semaphore->temporary.kind = RADV_SEMAPHORE_NONE;
4361                         ++temporary_idx;
4362                 } else
4363                         deferred->wait_semaphores[i] = &semaphore->permanent;
4364         }
4365
4366         for (uint32_t i = 0; i < submission->signal_semaphore_count; ++i) {
4367                 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->signal_semaphores[i]);
4368                 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) {
4369                         deferred->signal_semaphores[i] = &semaphore->temporary;
4370                 } else {
4371                         deferred->signal_semaphores[i] = &semaphore->permanent;
4372                 }
4373         }
4374
4375         deferred->wait_values = (void*)(deferred->temporary_semaphore_parts + temporary_count);
4376         memcpy(deferred->wait_values, submission->wait_values, submission->wait_value_count * sizeof(uint64_t));
4377         deferred->signal_values = deferred->wait_values + submission->wait_value_count;
4378         memcpy(deferred->signal_values, submission->signal_values, submission->signal_value_count * sizeof(uint64_t));
4379
4380         deferred->wait_nodes = (void*)(deferred->signal_values + submission->signal_value_count);
4381         /* This is worst-case. radv_queue_enqueue_submission will fill in further, but this
4382          * ensure the submission is not accidentally triggered early when adding wait timelines. */
4383         deferred->submission_wait_count = 1 + submission->wait_semaphore_count;
4384
4385         *out = deferred;
4386         return VK_SUCCESS;
4387 }
4388
4389 static VkResult
4390 radv_queue_enqueue_submission(struct radv_deferred_queue_submission *submission,
4391                               struct list_head *processing_list)
4392 {
4393         uint32_t wait_cnt = 0;
4394         struct radv_timeline_waiter *waiter = submission->wait_nodes;
4395         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4396                 if (submission->wait_semaphores[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4397                         pthread_mutex_lock(&submission->wait_semaphores[i]->timeline.mutex);
4398                         if (submission->wait_semaphores[i]->timeline.highest_submitted < submission->wait_values[i]) {
4399                                 ++wait_cnt;
4400                                 waiter->value = submission->wait_values[i];
4401                                 waiter->submission = submission;
4402                                 list_addtail(&waiter->list, &submission->wait_semaphores[i]->timeline.waiters);
4403                                 ++waiter;
4404                         }
4405                         pthread_mutex_unlock(&submission->wait_semaphores[i]->timeline.mutex);
4406                 }
4407         }
4408
4409         pthread_mutex_lock(&submission->queue->pending_mutex);
4410
4411         bool is_first = list_is_empty(&submission->queue->pending_submissions);
4412         list_addtail(&submission->queue_pending_list, &submission->queue->pending_submissions);
4413
4414         pthread_mutex_unlock(&submission->queue->pending_mutex);
4415
4416         /* If there is already a submission in the queue, that will decrement the counter by 1 when
4417          * submitted, but if the queue was empty, we decrement ourselves as there is no previous
4418          * submission. */
4419         uint32_t decrement = submission->wait_semaphore_count - wait_cnt + (is_first ? 1 : 0);
4420         return radv_queue_trigger_submission(submission, decrement, processing_list);
4421 }
4422
4423 static void
4424 radv_queue_submission_update_queue(struct radv_deferred_queue_submission *submission,
4425                                    struct list_head *processing_list)
4426 {
4427         pthread_mutex_lock(&submission->queue->pending_mutex);
4428         list_del(&submission->queue_pending_list);
4429
4430         /* trigger the next submission in the queue. */
4431         if (!list_is_empty(&submission->queue->pending_submissions)) {
4432                 struct radv_deferred_queue_submission *next_submission =
4433                         list_first_entry(&submission->queue->pending_submissions,
4434                                          struct radv_deferred_queue_submission,
4435                                          queue_pending_list);
4436                 radv_queue_trigger_submission(next_submission, 1, processing_list);
4437         }
4438         pthread_mutex_unlock(&submission->queue->pending_mutex);
4439
4440         pthread_cond_broadcast(&submission->queue->device->timeline_cond);
4441 }
4442
4443 static VkResult
4444 radv_queue_submit_deferred(struct radv_deferred_queue_submission *submission,
4445                            struct list_head *processing_list)
4446 {
4447         RADV_FROM_HANDLE(radv_fence, fence, submission->fence);
4448         struct radv_queue *queue = submission->queue;
4449         struct radeon_winsys_ctx *ctx = queue->hw_ctx;
4450         uint32_t max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT;
4451         struct radeon_winsys_fence *base_fence = NULL;
4452         bool do_flush = submission->flush_caches || submission->wait_dst_stage_mask;
4453         bool can_patch = true;
4454         uint32_t advance;
4455         struct radv_winsys_sem_info sem_info;
4456         VkResult result;
4457         struct radeon_cmdbuf *initial_preamble_cs = NULL;
4458         struct radeon_cmdbuf *initial_flush_preamble_cs = NULL;
4459         struct radeon_cmdbuf *continue_preamble_cs = NULL;
4460
4461         if (fence) {
4462                 /* Under most circumstances, out fences won't be temporary.
4463                  * However, the spec does allow it for opaque_fd.
4464                  *
4465                  * From the Vulkan 1.0.53 spec:
4466                  *
4467                  *    "If the import is temporary, the implementation must
4468                  *    restore the semaphore to its prior permanent state after
4469                  *    submitting the next semaphore wait operation."
4470                  */
4471                 struct radv_fence_part *part =
4472                         fence->temporary.kind != RADV_FENCE_NONE ?
4473                         &fence->temporary : &fence->permanent;
4474                 if (part->kind == RADV_FENCE_WINSYS)
4475                         base_fence = part->fence;
4476         }
4477
4478         result = radv_get_preambles(queue, submission->cmd_buffers,
4479                                     submission->cmd_buffer_count,
4480                                     &initial_preamble_cs,
4481                                     &initial_flush_preamble_cs,
4482                                     &continue_preamble_cs);
4483         if (result != VK_SUCCESS)
4484                 goto fail;
4485
4486         result = radv_alloc_sem_info(queue->device,
4487                                      &sem_info,
4488                                      submission->wait_semaphore_count,
4489                                      submission->wait_semaphores,
4490                                      submission->wait_values,
4491                                      submission->signal_semaphore_count,
4492                                      submission->signal_semaphores,
4493                                      submission->signal_values,
4494                                      submission->fence);
4495         if (result != VK_SUCCESS)
4496                 goto fail;
4497
4498         for (uint32_t i = 0; i < submission->buffer_bind_count; ++i) {
4499                 result = radv_sparse_buffer_bind_memory(queue->device,
4500                                                         submission->buffer_binds + i);
4501                 if (result != VK_SUCCESS)
4502                         goto fail;
4503         }
4504
4505         for (uint32_t i = 0; i < submission->image_opaque_bind_count; ++i) {
4506                 result = radv_sparse_image_opaque_bind_memory(queue->device,
4507                                                               submission->image_opaque_binds + i);
4508                 if (result != VK_SUCCESS)
4509                         goto fail;
4510         }
4511
4512         if (!submission->cmd_buffer_count) {
4513                 result = queue->device->ws->cs_submit(ctx, queue->queue_idx,
4514                                                       &queue->device->empty_cs[queue->queue_family_index],
4515                                                       1, NULL, NULL,
4516                                                       &sem_info, NULL,
4517                                                       false, base_fence);
4518                 if (result != VK_SUCCESS)
4519                         goto fail;
4520         } else {
4521                 struct radeon_cmdbuf **cs_array = malloc(sizeof(struct radeon_cmdbuf *) *
4522                                                          (submission->cmd_buffer_count));
4523
4524                 for (uint32_t j = 0; j < submission->cmd_buffer_count; j++) {
4525                         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, submission->cmd_buffers[j]);
4526                         assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
4527
4528                         cs_array[j] = cmd_buffer->cs;
4529                         if ((cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT))
4530                                 can_patch = false;
4531
4532                         cmd_buffer->status = RADV_CMD_BUFFER_STATUS_PENDING;
4533                 }
4534
4535                 for (uint32_t j = 0; j < submission->cmd_buffer_count; j += advance) {
4536                         struct radeon_cmdbuf *initial_preamble = (do_flush && !j) ? initial_flush_preamble_cs : initial_preamble_cs;
4537                         const struct radv_winsys_bo_list *bo_list = NULL;
4538
4539                         advance = MIN2(max_cs_submission,
4540                                        submission->cmd_buffer_count - j);
4541
4542                         if (queue->device->trace_bo)
4543                                 *queue->device->trace_id_ptr = 0;
4544
4545                         sem_info.cs_emit_wait = j == 0;
4546                         sem_info.cs_emit_signal = j + advance == submission->cmd_buffer_count;
4547
4548                         if (unlikely(queue->device->use_global_bo_list)) {
4549                                 pthread_mutex_lock(&queue->device->bo_list.mutex);
4550                                 bo_list = &queue->device->bo_list.list;
4551                         }
4552
4553                         result = queue->device->ws->cs_submit(ctx, queue->queue_idx, cs_array + j,
4554                                                               advance, initial_preamble, continue_preamble_cs,
4555                                                               &sem_info, bo_list,
4556                                                               can_patch, base_fence);
4557
4558                         if (unlikely(queue->device->use_global_bo_list))
4559                                 pthread_mutex_unlock(&queue->device->bo_list.mutex);
4560
4561                         if (result != VK_SUCCESS)
4562                                 goto fail;
4563
4564                         if (queue->device->trace_bo) {
4565                                 radv_check_gpu_hangs(queue, cs_array[j]);
4566                         }
4567
4568                         if (queue->device->tma_bo) {
4569                                 radv_check_trap_handler(queue);
4570                         }
4571                 }
4572
4573                 free(cs_array);
4574         }
4575
4576         radv_free_temp_syncobjs(queue->device,
4577                                 submission->temporary_semaphore_part_count,
4578                                 submission->temporary_semaphore_parts);
4579         radv_finalize_timelines(queue->device,
4580                                 submission->wait_semaphore_count,
4581                                 submission->wait_semaphores,
4582                                 submission->wait_values,
4583                                 submission->signal_semaphore_count,
4584                                 submission->signal_semaphores,
4585                                 submission->signal_values,
4586                                 processing_list);
4587         /* Has to happen after timeline finalization to make sure the
4588          * condition variable is only triggered when timelines and queue have
4589          * been updated. */
4590         radv_queue_submission_update_queue(submission, processing_list);
4591         radv_free_sem_info(&sem_info);
4592         free(submission);
4593         return VK_SUCCESS;
4594
4595 fail:
4596         if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) {
4597                 /* When something bad happened during the submission, such as
4598                  * an out of memory issue, it might be hard to recover from
4599                  * this inconsistent state. To avoid this sort of problem, we
4600                  * assume that we are in a really bad situation and return
4601                  * VK_ERROR_DEVICE_LOST to ensure the clients do not attempt
4602                  * to submit the same job again to this device.
4603                  */
4604                 result = radv_device_set_lost(queue->device, "vkQueueSubmit() failed");
4605         }
4606
4607         radv_free_temp_syncobjs(queue->device,
4608                                 submission->temporary_semaphore_part_count,
4609                                 submission->temporary_semaphore_parts);
4610         free(submission);
4611         return result;
4612 }
4613
4614 static VkResult
4615 radv_process_submissions(struct list_head *processing_list)
4616 {
4617         while(!list_is_empty(processing_list)) {
4618                 struct radv_deferred_queue_submission *submission =
4619                         list_first_entry(processing_list, struct radv_deferred_queue_submission, processing_list);
4620                 list_del(&submission->processing_list);
4621
4622                 VkResult result = radv_queue_submit_deferred(submission, processing_list);
4623                 if (result != VK_SUCCESS)
4624                         return result;
4625         }
4626         return VK_SUCCESS;
4627 }
4628
4629 static VkResult
4630 wait_for_submission_timelines_available(struct radv_deferred_queue_submission *submission,
4631                                         uint64_t timeout)
4632 {
4633         struct radv_device *device = submission->queue->device;
4634         uint32_t syncobj_count = 0;
4635         uint32_t syncobj_idx = 0;
4636
4637         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4638                 if (submission->wait_semaphores[i]->kind != RADV_SEMAPHORE_TIMELINE_SYNCOBJ)
4639                         continue;
4640
4641                 if (submission->wait_semaphores[i]->timeline_syncobj.max_point >= submission->wait_values[i])
4642                         continue;
4643                 ++syncobj_count;
4644         }
4645
4646         if (!syncobj_count)
4647                 return VK_SUCCESS;
4648
4649         uint64_t *points = malloc((sizeof(uint64_t) + sizeof(uint32_t)) * syncobj_count);
4650         if (!points)
4651                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
4652
4653         uint32_t *syncobj = (uint32_t*)(points + syncobj_count);
4654
4655         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4656                 if (submission->wait_semaphores[i]->kind != RADV_SEMAPHORE_TIMELINE_SYNCOBJ)
4657                         continue;
4658
4659                 if (submission->wait_semaphores[i]->timeline_syncobj.max_point >= submission->wait_values[i])
4660                         continue;
4661
4662                 syncobj[syncobj_idx] = submission->wait_semaphores[i]->syncobj;
4663                 points[syncobj_idx] = submission->wait_values[i];
4664                 ++syncobj_idx;
4665         }
4666         bool success = device->ws->wait_timeline_syncobj(device->ws, syncobj, points, syncobj_idx, true, true, timeout);
4667
4668         free(points);
4669         return success ? VK_SUCCESS : VK_TIMEOUT;
4670 }
4671
4672 static void* radv_queue_submission_thread_run(void *q)
4673 {
4674         struct radv_queue *queue = q;
4675
4676         pthread_mutex_lock(&queue->thread_mutex);
4677         while (!p_atomic_read(&queue->thread_exit)) {
4678                 struct radv_deferred_queue_submission *submission = queue->thread_submission;
4679                 struct list_head processing_list;
4680                 VkResult result = VK_SUCCESS;
4681                 if (!submission) {
4682                         pthread_cond_wait(&queue->thread_cond, &queue->thread_mutex);
4683                         continue;
4684                 }
4685                 pthread_mutex_unlock(&queue->thread_mutex);
4686
4687                 /* Wait at most 5 seconds so we have a chance to notice shutdown when
4688                  * a semaphore never gets signaled. If it takes longer we just retry
4689                  * the wait next iteration. */
4690                 result = wait_for_submission_timelines_available(submission,
4691                                                                  radv_get_absolute_timeout(5000000000));
4692                 if (result != VK_SUCCESS) {
4693                         pthread_mutex_lock(&queue->thread_mutex);
4694                         continue;
4695                 }
4696
4697                 /* The lock isn't held but nobody will add one until we finish
4698                  * the current submission. */
4699                 p_atomic_set(&queue->thread_submission, NULL);
4700
4701                 list_inithead(&processing_list);
4702                 list_addtail(&submission->processing_list, &processing_list);
4703                 result = radv_process_submissions(&processing_list);
4704
4705                 pthread_mutex_lock(&queue->thread_mutex);
4706         }
4707         pthread_mutex_unlock(&queue->thread_mutex);
4708         return NULL;
4709 }
4710
4711 static VkResult
4712 radv_queue_trigger_submission(struct radv_deferred_queue_submission *submission,
4713                               uint32_t decrement,
4714                               struct list_head *processing_list)
4715 {
4716         struct radv_queue *queue = submission->queue;
4717         int ret;
4718         if  (p_atomic_add_return(&submission->submission_wait_count, -decrement))
4719                 return VK_SUCCESS;
4720
4721         if (wait_for_submission_timelines_available(submission, radv_get_absolute_timeout(0)) == VK_SUCCESS) {
4722                 list_addtail(&submission->processing_list, processing_list);
4723                 return VK_SUCCESS;
4724         }
4725
4726         pthread_mutex_lock(&queue->thread_mutex);
4727
4728         /* A submission can only be ready for the thread if it doesn't have
4729          * any predecessors in the same queue, so there can only be one such
4730          * submission at a time. */
4731         assert(queue->thread_submission == NULL);
4732
4733         /* Only start the thread on demand to save resources for the many games
4734          * which only use binary semaphores. */
4735         if (!queue->thread_running) {
4736                 ret  = pthread_create(&queue->submission_thread, NULL,
4737                                       radv_queue_submission_thread_run, queue);
4738                 if (ret) {
4739                         pthread_mutex_unlock(&queue->thread_mutex);
4740                         return vk_errorf(queue->device->instance,
4741                                          VK_ERROR_DEVICE_LOST,
4742                                          "Failed to start submission thread");
4743                 }
4744                 queue->thread_running = true;
4745         }
4746
4747         queue->thread_submission = submission;
4748         pthread_mutex_unlock(&queue->thread_mutex);
4749
4750         pthread_cond_signal(&queue->thread_cond);
4751         return VK_SUCCESS;
4752 }
4753
4754 static VkResult radv_queue_submit(struct radv_queue *queue,
4755                                   const struct radv_queue_submission *submission)
4756 {
4757         struct radv_deferred_queue_submission *deferred = NULL;
4758
4759         VkResult result = radv_create_deferred_submission(queue, submission, &deferred);
4760         if (result != VK_SUCCESS)
4761                 return result;
4762
4763         struct list_head processing_list;
4764         list_inithead(&processing_list);
4765
4766         result = radv_queue_enqueue_submission(deferred, &processing_list);
4767         if (result != VK_SUCCESS) {
4768                 /* If anything is in the list we leak. */
4769                 assert(list_is_empty(&processing_list));
4770                 return result;
4771         }
4772         return radv_process_submissions(&processing_list);
4773 }
4774
4775 bool
4776 radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs)
4777 {
4778         struct radeon_winsys_ctx *ctx = queue->hw_ctx;
4779         struct radv_winsys_sem_info sem_info;
4780         VkResult result;
4781
4782         result = radv_alloc_sem_info(queue->device, &sem_info, 0, NULL, 0, 0,
4783                                      0, NULL, VK_NULL_HANDLE);
4784         if (result != VK_SUCCESS)
4785                 return false;
4786
4787         result = queue->device->ws->cs_submit(ctx, queue->queue_idx, &cs, 1,
4788                                               NULL, NULL, &sem_info, NULL,
4789                                               false, NULL);
4790         radv_free_sem_info(&sem_info);
4791         if (result != VK_SUCCESS)
4792                 return false;
4793
4794         return true;
4795
4796 }
4797
4798 /* Signals fence as soon as all the work currently put on queue is done. */
4799 static VkResult radv_signal_fence(struct radv_queue *queue,
4800                               VkFence fence)
4801 {
4802         return radv_queue_submit(queue, &(struct radv_queue_submission) {
4803                         .fence = fence
4804                 });
4805 }
4806
4807 static bool radv_submit_has_effects(const VkSubmitInfo *info)
4808 {
4809         return info->commandBufferCount ||
4810                info->waitSemaphoreCount ||
4811                info->signalSemaphoreCount;
4812 }
4813
4814 VkResult radv_QueueSubmit(
4815         VkQueue                                     _queue,
4816         uint32_t                                    submitCount,
4817         const VkSubmitInfo*                         pSubmits,
4818         VkFence                                     fence)
4819 {
4820         RADV_FROM_HANDLE(radv_queue, queue, _queue);
4821         VkResult result;
4822         uint32_t fence_idx = 0;
4823         bool flushed_caches = false;
4824
4825         if (radv_device_is_lost(queue->device))
4826                 return VK_ERROR_DEVICE_LOST;
4827
4828         if (fence != VK_NULL_HANDLE) {
4829                 for (uint32_t i = 0; i < submitCount; ++i)
4830                         if (radv_submit_has_effects(pSubmits + i))
4831                                 fence_idx = i;
4832         } else
4833                 fence_idx = UINT32_MAX;
4834
4835         for (uint32_t i = 0; i < submitCount; i++) {
4836                 if (!radv_submit_has_effects(pSubmits + i) && fence_idx != i)
4837                         continue;
4838
4839                 VkPipelineStageFlags wait_dst_stage_mask = 0;
4840                 for (unsigned j = 0; j < pSubmits[i].waitSemaphoreCount; ++j) {
4841                         wait_dst_stage_mask |= pSubmits[i].pWaitDstStageMask[j];
4842                 }
4843
4844                 const VkTimelineSemaphoreSubmitInfo *timeline_info =
4845                         vk_find_struct_const(pSubmits[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
4846
4847                 result = radv_queue_submit(queue, &(struct radv_queue_submission) {
4848                                 .cmd_buffers = pSubmits[i].pCommandBuffers,
4849                                 .cmd_buffer_count = pSubmits[i].commandBufferCount,
4850                                 .wait_dst_stage_mask = wait_dst_stage_mask,
4851                                 .flush_caches = !flushed_caches,
4852                                 .wait_semaphores = pSubmits[i].pWaitSemaphores,
4853                                 .wait_semaphore_count = pSubmits[i].waitSemaphoreCount,
4854                                 .signal_semaphores = pSubmits[i].pSignalSemaphores,
4855                                 .signal_semaphore_count = pSubmits[i].signalSemaphoreCount,
4856                                 .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
4857                                 .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
4858                                 .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
4859                                 .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
4860                                 .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
4861                         });
4862                 if (result != VK_SUCCESS)
4863                         return result;
4864
4865                 flushed_caches  = true;
4866         }
4867
4868         if (fence != VK_NULL_HANDLE && !submitCount) {
4869                 result = radv_signal_fence(queue, fence);
4870                 if (result != VK_SUCCESS)
4871                         return result;
4872         }
4873
4874         return VK_SUCCESS;
4875 }
4876
4877 static const char *
4878 radv_get_queue_family_name(struct radv_queue *queue)
4879 {
4880         switch (queue->queue_family_index) {
4881         case RADV_QUEUE_GENERAL:
4882                 return "graphics";
4883         case RADV_QUEUE_COMPUTE:
4884                 return "compute";
4885         case RADV_QUEUE_TRANSFER:
4886                 return "transfer";
4887         default:
4888                 unreachable("Unknown queue family");
4889         }
4890 }
4891
4892 VkResult radv_QueueWaitIdle(
4893         VkQueue                                     _queue)
4894 {
4895         RADV_FROM_HANDLE(radv_queue, queue, _queue);
4896
4897         if (radv_device_is_lost(queue->device))
4898                 return VK_ERROR_DEVICE_LOST;
4899
4900         pthread_mutex_lock(&queue->pending_mutex);
4901         while (!list_is_empty(&queue->pending_submissions)) {
4902                 pthread_cond_wait(&queue->device->timeline_cond, &queue->pending_mutex);
4903         }
4904         pthread_mutex_unlock(&queue->pending_mutex);
4905
4906         if (!queue->device->ws->ctx_wait_idle(queue->hw_ctx,
4907                                               radv_queue_family_to_ring(queue->queue_family_index),
4908                                               queue->queue_idx)) {
4909                 return radv_device_set_lost(queue->device,
4910                                             "Failed to wait for a '%s' queue "
4911                                             "to be idle. GPU hang ?",
4912                                             radv_get_queue_family_name(queue));
4913         }
4914
4915         return VK_SUCCESS;
4916 }
4917
4918 VkResult radv_DeviceWaitIdle(
4919         VkDevice                                    _device)
4920 {
4921         RADV_FROM_HANDLE(radv_device, device, _device);
4922
4923         for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
4924                 for (unsigned q = 0; q < device->queue_count[i]; q++) {
4925                         VkResult result =
4926                                 radv_QueueWaitIdle(radv_queue_to_handle(&device->queues[i][q]));
4927
4928                         if (result != VK_SUCCESS)
4929                                 return result;
4930                 }
4931         }
4932         return VK_SUCCESS;
4933 }
4934
4935 VkResult radv_EnumerateInstanceExtensionProperties(
4936     const char*                                 pLayerName,
4937     uint32_t*                                   pPropertyCount,
4938     VkExtensionProperties*                      pProperties)
4939 {
4940         VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
4941
4942         for (int i = 0; i < RADV_INSTANCE_EXTENSION_COUNT; i++) {
4943                 if (radv_instance_extensions_supported.extensions[i]) {
4944                         vk_outarray_append(&out, prop) {
4945                                 *prop = radv_instance_extensions[i];
4946                         }
4947                 }
4948         }
4949
4950         return vk_outarray_status(&out);
4951 }
4952
4953 VkResult radv_EnumerateDeviceExtensionProperties(
4954     VkPhysicalDevice                            physicalDevice,
4955     const char*                                 pLayerName,
4956     uint32_t*                                   pPropertyCount,
4957     VkExtensionProperties*                      pProperties)
4958 {
4959         RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
4960         VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
4961
4962         for (int i = 0; i < RADV_DEVICE_EXTENSION_COUNT; i++) {
4963                 if (device->supported_extensions.extensions[i]) {
4964                         vk_outarray_append(&out, prop) {
4965                                 *prop = radv_device_extensions[i];
4966                         }
4967                 }
4968         }
4969
4970         return vk_outarray_status(&out);
4971 }
4972
4973 PFN_vkVoidFunction radv_GetInstanceProcAddr(
4974         VkInstance                                  _instance,
4975         const char*                                 pName)
4976 {
4977         RADV_FROM_HANDLE(radv_instance, instance, _instance);
4978
4979         /* The Vulkan 1.0 spec for vkGetInstanceProcAddr has a table of exactly
4980          * when we have to return valid function pointers, NULL, or it's left
4981          * undefined.  See the table for exact details.
4982          */
4983         if (pName == NULL)
4984                 return NULL;
4985
4986 #define LOOKUP_RADV_ENTRYPOINT(entrypoint) \
4987         if (strcmp(pName, "vk" #entrypoint) == 0) \
4988                 return (PFN_vkVoidFunction)radv_##entrypoint
4989
4990         LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceExtensionProperties);
4991         LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceLayerProperties);
4992         LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceVersion);
4993         LOOKUP_RADV_ENTRYPOINT(CreateInstance);
4994
4995         /* GetInstanceProcAddr() can also be called with a NULL instance.
4996          * See https://gitlab.khronos.org/vulkan/vulkan/issues/2057
4997          */
4998         LOOKUP_RADV_ENTRYPOINT(GetInstanceProcAddr);
4999
5000 #undef LOOKUP_RADV_ENTRYPOINT
5001
5002         if (instance == NULL)
5003                 return NULL;
5004
5005         int idx = radv_get_instance_entrypoint_index(pName);
5006         if (idx >= 0)
5007                 return instance->dispatch.entrypoints[idx];
5008
5009         idx = radv_get_physical_device_entrypoint_index(pName);
5010         if (idx >= 0)
5011                 return instance->physical_device_dispatch.entrypoints[idx];
5012
5013         idx = radv_get_device_entrypoint_index(pName);
5014         if (idx >= 0)
5015                 return instance->device_dispatch.entrypoints[idx];
5016
5017         return NULL;
5018 }
5019
5020 /* The loader wants us to expose a second GetInstanceProcAddr function
5021  * to work around certain LD_PRELOAD issues seen in apps.
5022  */
5023 PUBLIC
5024 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
5025         VkInstance                                  instance,
5026         const char*                                 pName);
5027
5028 PUBLIC
5029 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
5030         VkInstance                                  instance,
5031         const char*                                 pName)
5032 {
5033         return radv_GetInstanceProcAddr(instance, pName);
5034 }
5035
5036 PUBLIC
5037 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetPhysicalDeviceProcAddr(
5038         VkInstance                                  _instance,
5039         const char*                                 pName);
5040
5041 PUBLIC
5042 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetPhysicalDeviceProcAddr(
5043         VkInstance                                  _instance,
5044         const char*                                 pName)
5045 {
5046         RADV_FROM_HANDLE(radv_instance, instance, _instance);
5047
5048         if (!pName || !instance)
5049                 return NULL;
5050
5051         int idx = radv_get_physical_device_entrypoint_index(pName);
5052         if (idx < 0)
5053                 return NULL;
5054
5055         return instance->physical_device_dispatch.entrypoints[idx];
5056 }
5057
5058 PFN_vkVoidFunction radv_GetDeviceProcAddr(
5059         VkDevice                                    _device,
5060         const char*                                 pName)
5061 {
5062         RADV_FROM_HANDLE(radv_device, device, _device);
5063
5064         if (!device || !pName)
5065                 return NULL;
5066
5067         int idx = radv_get_device_entrypoint_index(pName);
5068         if (idx < 0)
5069                 return NULL;
5070
5071         return device->dispatch.entrypoints[idx];
5072 }
5073
5074 bool radv_get_memory_fd(struct radv_device *device,
5075                         struct radv_device_memory *memory,
5076                         int *pFD)
5077 {
5078         struct radeon_bo_metadata metadata;
5079
5080         if (memory->image) {
5081                 if (memory->image->tiling != VK_IMAGE_TILING_LINEAR)
5082                         radv_init_metadata(device, memory->image, &metadata);
5083                 device->ws->buffer_set_metadata(memory->bo, &metadata);
5084         }
5085
5086         return device->ws->buffer_get_fd(device->ws, memory->bo,
5087                                          pFD);
5088 }
5089
5090
5091 void
5092 radv_free_memory(struct radv_device *device,
5093                  const VkAllocationCallbacks* pAllocator,
5094                  struct radv_device_memory *mem)
5095 {
5096         if (mem == NULL)
5097                 return;
5098
5099 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
5100         if (mem->android_hardware_buffer)
5101                 AHardwareBuffer_release(mem->android_hardware_buffer);
5102 #endif
5103
5104         if (mem->bo) {
5105                 if (device->overallocation_disallowed) {
5106                         mtx_lock(&device->overallocation_mutex);
5107                         device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
5108                         mtx_unlock(&device->overallocation_mutex);
5109                 }
5110
5111                 radv_bo_list_remove(device, mem->bo);
5112                 device->ws->buffer_destroy(mem->bo);
5113                 mem->bo = NULL;
5114         }
5115
5116         vk_object_base_finish(&mem->base);
5117         vk_free2(&device->vk.alloc, pAllocator, mem);
5118 }
5119
5120 static VkResult radv_alloc_memory(struct radv_device *device,
5121                                   const VkMemoryAllocateInfo*     pAllocateInfo,
5122                                   const VkAllocationCallbacks*    pAllocator,
5123                                   VkDeviceMemory*                 pMem)
5124 {
5125         struct radv_device_memory *mem;
5126         VkResult result;
5127         enum radeon_bo_domain domain;
5128         uint32_t flags = 0;
5129
5130         assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
5131
5132         const VkImportMemoryFdInfoKHR *import_info =
5133                 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
5134         const VkMemoryDedicatedAllocateInfo *dedicate_info =
5135                 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
5136         const VkExportMemoryAllocateInfo *export_info =
5137                 vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
5138         const struct VkImportAndroidHardwareBufferInfoANDROID *ahb_import_info =
5139                 vk_find_struct_const(pAllocateInfo->pNext,
5140                                      IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID);
5141         const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
5142                 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
5143
5144         const struct wsi_memory_allocate_info *wsi_info =
5145                 vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
5146
5147         if (pAllocateInfo->allocationSize == 0 && !ahb_import_info &&
5148             !(export_info && (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID))) {
5149                 /* Apparently, this is allowed */
5150                 *pMem = VK_NULL_HANDLE;
5151                 return VK_SUCCESS;
5152         }
5153
5154         mem = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8,
5155                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5156         if (mem == NULL)
5157                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5158
5159         vk_object_base_init(&device->vk, &mem->base,
5160                             VK_OBJECT_TYPE_DEVICE_MEMORY);
5161
5162         if (wsi_info && wsi_info->implicit_sync)
5163                 flags |= RADEON_FLAG_IMPLICIT_SYNC;
5164
5165         if (dedicate_info) {
5166                 mem->image = radv_image_from_handle(dedicate_info->image);
5167                 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
5168         } else {
5169                 mem->image = NULL;
5170                 mem->buffer = NULL;
5171         }
5172
5173         float priority_float = 0.5;
5174         const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
5175                 vk_find_struct_const(pAllocateInfo->pNext,
5176                                      MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
5177         if (priority_ext)
5178                 priority_float = priority_ext->priority;
5179
5180         unsigned priority = MIN2(RADV_BO_PRIORITY_APPLICATION_MAX - 1,
5181                                  (int)(priority_float * RADV_BO_PRIORITY_APPLICATION_MAX));
5182
5183         mem->user_ptr = NULL;
5184         mem->bo = NULL;
5185
5186 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
5187         mem->android_hardware_buffer = NULL;
5188 #endif
5189
5190         if (ahb_import_info) {
5191                 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
5192                 if (result != VK_SUCCESS)
5193                         goto fail;
5194         } else if(export_info && (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID)) {
5195                 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
5196                 if (result != VK_SUCCESS)
5197                         goto fail;
5198         } else if (import_info) {
5199                 assert(import_info->handleType ==
5200                        VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
5201                        import_info->handleType ==
5202                        VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
5203                 mem->bo = device->ws->buffer_from_fd(device->ws, import_info->fd,
5204                                                      priority, NULL);
5205                 if (!mem->bo) {
5206                         result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
5207                         goto fail;
5208                 } else {
5209                         close(import_info->fd);
5210                 }
5211
5212                 if (mem->image && mem->image->plane_count == 1 &&
5213                     !vk_format_is_depth_or_stencil(mem->image->vk_format)) {
5214                         struct radeon_bo_metadata metadata;
5215                         device->ws->buffer_get_metadata(mem->bo, &metadata);
5216
5217                         struct radv_image_create_info create_info = {
5218                                 .no_metadata_planes = true,
5219                                 .bo_metadata = &metadata
5220                         };
5221
5222                         /* This gives a basic ability to import radeonsi images
5223                          * that don't have DCC. This is not guaranteed by any
5224                          * spec and can be removed after we support modifiers. */
5225                         result = radv_image_create_layout(device, create_info, mem->image);
5226                         if (result != VK_SUCCESS) {
5227                                 device->ws->buffer_destroy(mem->bo);
5228                                 goto fail;
5229                         }
5230                 }
5231         } else if (host_ptr_info) {
5232                 assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
5233                 mem->bo = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer,
5234                                                       pAllocateInfo->allocationSize,
5235                                                       priority);
5236                 if (!mem->bo) {
5237                         result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
5238                         goto fail;
5239                 } else {
5240                         mem->user_ptr = host_ptr_info->pHostPointer;
5241                 }
5242         } else {
5243                 uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
5244                 uint32_t heap_index;
5245
5246                 heap_index = device->physical_device->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
5247                 domain = device->physical_device->memory_domains[pAllocateInfo->memoryTypeIndex];
5248                 flags |= device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex];
5249
5250                 if (!dedicate_info && !import_info && (!export_info || !export_info->handleTypes)) {
5251                         flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
5252                         if (device->use_global_bo_list) {
5253                                 flags |= RADEON_FLAG_PREFER_LOCAL_BO;
5254                         }
5255                 }
5256
5257                 if (device->overallocation_disallowed) {
5258                         uint64_t total_size =
5259                                 device->physical_device->memory_properties.memoryHeaps[heap_index].size;
5260
5261                         mtx_lock(&device->overallocation_mutex);
5262                         if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
5263                                 mtx_unlock(&device->overallocation_mutex);
5264                                 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
5265                                 goto fail;
5266                         }
5267                         device->allocated_memory_size[heap_index] += alloc_size;
5268                         mtx_unlock(&device->overallocation_mutex);
5269                 }
5270
5271                 mem->bo = device->ws->buffer_create(device->ws, alloc_size, device->physical_device->rad_info.max_alignment,
5272                                                     domain, flags, priority);
5273
5274                 if (!mem->bo) {
5275                         if (device->overallocation_disallowed) {
5276                                 mtx_lock(&device->overallocation_mutex);
5277                                 device->allocated_memory_size[heap_index] -= alloc_size;
5278                                 mtx_unlock(&device->overallocation_mutex);
5279                         }
5280                         result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
5281                         goto fail;
5282                 }
5283
5284                 mem->heap_index = heap_index;
5285                 mem->alloc_size = alloc_size;
5286         }
5287
5288         if (!wsi_info) {
5289                 result = radv_bo_list_add(device, mem->bo);
5290                 if (result != VK_SUCCESS)
5291                         goto fail;
5292         }
5293
5294         *pMem = radv_device_memory_to_handle(mem);
5295
5296         return VK_SUCCESS;
5297
5298 fail:
5299         radv_free_memory(device, pAllocator,mem);
5300
5301         return result;
5302 }
5303
5304 VkResult radv_AllocateMemory(
5305         VkDevice                                    _device,
5306         const VkMemoryAllocateInfo*                 pAllocateInfo,
5307         const VkAllocationCallbacks*                pAllocator,
5308         VkDeviceMemory*                             pMem)
5309 {
5310         RADV_FROM_HANDLE(radv_device, device, _device);
5311         return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem);
5312 }
5313
5314 void radv_FreeMemory(
5315         VkDevice                                    _device,
5316         VkDeviceMemory                              _mem,
5317         const VkAllocationCallbacks*                pAllocator)
5318 {
5319         RADV_FROM_HANDLE(radv_device, device, _device);
5320         RADV_FROM_HANDLE(radv_device_memory, mem, _mem);
5321
5322         radv_free_memory(device, pAllocator, mem);
5323 }
5324
5325 VkResult radv_MapMemory(
5326         VkDevice                                    _device,
5327         VkDeviceMemory                              _memory,
5328         VkDeviceSize                                offset,
5329         VkDeviceSize                                size,
5330         VkMemoryMapFlags                            flags,
5331         void**                                      ppData)
5332 {
5333         RADV_FROM_HANDLE(radv_device, device, _device);
5334         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
5335
5336         if (mem == NULL) {
5337                 *ppData = NULL;
5338                 return VK_SUCCESS;
5339         }
5340
5341         if (mem->user_ptr)
5342                 *ppData = mem->user_ptr;
5343         else
5344                 *ppData = device->ws->buffer_map(mem->bo);
5345
5346         if (*ppData) {
5347                 *ppData += offset;
5348                 return VK_SUCCESS;
5349         }
5350
5351         return vk_error(device->instance, VK_ERROR_MEMORY_MAP_FAILED);
5352 }
5353
5354 void radv_UnmapMemory(
5355         VkDevice                                    _device,
5356         VkDeviceMemory                              _memory)
5357 {
5358         RADV_FROM_HANDLE(radv_device, device, _device);
5359         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
5360
5361         if (mem == NULL)
5362                 return;
5363
5364         if (mem->user_ptr == NULL)
5365                 device->ws->buffer_unmap(mem->bo);
5366 }
5367
5368 VkResult radv_FlushMappedMemoryRanges(
5369         VkDevice                                    _device,
5370         uint32_t                                    memoryRangeCount,
5371         const VkMappedMemoryRange*                  pMemoryRanges)
5372 {
5373         return VK_SUCCESS;
5374 }
5375
5376 VkResult radv_InvalidateMappedMemoryRanges(
5377         VkDevice                                    _device,
5378         uint32_t                                    memoryRangeCount,
5379         const VkMappedMemoryRange*                  pMemoryRanges)
5380 {
5381         return VK_SUCCESS;
5382 }
5383
5384 void radv_GetBufferMemoryRequirements(
5385         VkDevice                                    _device,
5386         VkBuffer                                    _buffer,
5387         VkMemoryRequirements*                       pMemoryRequirements)
5388 {
5389         RADV_FROM_HANDLE(radv_device, device, _device);
5390         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
5391
5392         pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
5393
5394         if (buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)
5395                 pMemoryRequirements->alignment = 4096;
5396         else
5397                 pMemoryRequirements->alignment = 16;
5398
5399         pMemoryRequirements->size = align64(buffer->size, pMemoryRequirements->alignment);
5400 }
5401
5402 void radv_GetBufferMemoryRequirements2(
5403         VkDevice                                     device,
5404         const VkBufferMemoryRequirementsInfo2       *pInfo,
5405         VkMemoryRequirements2                       *pMemoryRequirements)
5406 {
5407         radv_GetBufferMemoryRequirements(device, pInfo->buffer,
5408                                         &pMemoryRequirements->memoryRequirements);
5409         vk_foreach_struct(ext, pMemoryRequirements->pNext) {
5410                 switch (ext->sType) {
5411                 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
5412                         VkMemoryDedicatedRequirements *req =
5413                                        (VkMemoryDedicatedRequirements *) ext;
5414                         req->requiresDedicatedAllocation = false;
5415                         req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
5416                         break;
5417                 }
5418                 default:
5419                         break;
5420                 }
5421         }
5422 }
5423
5424 void radv_GetImageMemoryRequirements(
5425         VkDevice                                    _device,
5426         VkImage                                     _image,
5427         VkMemoryRequirements*                       pMemoryRequirements)
5428 {
5429         RADV_FROM_HANDLE(radv_device, device, _device);
5430         RADV_FROM_HANDLE(radv_image, image, _image);
5431
5432         pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
5433
5434         pMemoryRequirements->size = image->size;
5435         pMemoryRequirements->alignment = image->alignment;
5436 }
5437
5438 void radv_GetImageMemoryRequirements2(
5439         VkDevice                                    device,
5440         const VkImageMemoryRequirementsInfo2       *pInfo,
5441         VkMemoryRequirements2                      *pMemoryRequirements)
5442 {
5443         radv_GetImageMemoryRequirements(device, pInfo->image,
5444                                         &pMemoryRequirements->memoryRequirements);
5445
5446         RADV_FROM_HANDLE(radv_image, image, pInfo->image);
5447
5448         vk_foreach_struct(ext, pMemoryRequirements->pNext) {
5449                 switch (ext->sType) {
5450                 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
5451                         VkMemoryDedicatedRequirements *req =
5452                                        (VkMemoryDedicatedRequirements *) ext;
5453                         req->requiresDedicatedAllocation = image->shareable &&
5454                                                            image->tiling != VK_IMAGE_TILING_LINEAR;
5455                         req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
5456                         break;
5457                 }
5458                 default:
5459                         break;
5460                 }
5461         }
5462 }
5463
5464 void radv_GetImageSparseMemoryRequirements(
5465         VkDevice                                    device,
5466         VkImage                                     image,
5467         uint32_t*                                   pSparseMemoryRequirementCount,
5468         VkSparseImageMemoryRequirements*            pSparseMemoryRequirements)
5469 {
5470         stub();
5471 }
5472
5473 void radv_GetImageSparseMemoryRequirements2(
5474         VkDevice                                    device,
5475         const VkImageSparseMemoryRequirementsInfo2 *pInfo,
5476         uint32_t*                                   pSparseMemoryRequirementCount,
5477         VkSparseImageMemoryRequirements2           *pSparseMemoryRequirements)
5478 {
5479         stub();
5480 }
5481
5482 void radv_GetDeviceMemoryCommitment(
5483         VkDevice                                    device,
5484         VkDeviceMemory                              memory,
5485         VkDeviceSize*                               pCommittedMemoryInBytes)
5486 {
5487         *pCommittedMemoryInBytes = 0;
5488 }
5489
5490 VkResult radv_BindBufferMemory2(VkDevice device,
5491                                 uint32_t bindInfoCount,
5492                                 const VkBindBufferMemoryInfo *pBindInfos)
5493 {
5494         for (uint32_t i = 0; i < bindInfoCount; ++i) {
5495                 RADV_FROM_HANDLE(radv_device_memory, mem, pBindInfos[i].memory);
5496                 RADV_FROM_HANDLE(radv_buffer, buffer, pBindInfos[i].buffer);
5497
5498                 if (mem) {
5499                         buffer->bo = mem->bo;
5500                         buffer->offset = pBindInfos[i].memoryOffset;
5501                 } else {
5502                         buffer->bo = NULL;
5503                 }
5504         }
5505         return VK_SUCCESS;
5506 }
5507
5508 VkResult radv_BindBufferMemory(
5509         VkDevice                                    device,
5510         VkBuffer                                    buffer,
5511         VkDeviceMemory                              memory,
5512         VkDeviceSize                                memoryOffset)
5513 {
5514         const VkBindBufferMemoryInfo info = {
5515                 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
5516                 .buffer = buffer,
5517                 .memory = memory,
5518                 .memoryOffset = memoryOffset
5519         };
5520
5521         return radv_BindBufferMemory2(device, 1, &info);
5522 }
5523
5524 VkResult radv_BindImageMemory2(VkDevice device,
5525                                uint32_t bindInfoCount,
5526                                const VkBindImageMemoryInfo *pBindInfos)
5527 {
5528         for (uint32_t i = 0; i < bindInfoCount; ++i) {
5529                 RADV_FROM_HANDLE(radv_device_memory, mem, pBindInfos[i].memory);
5530                 RADV_FROM_HANDLE(radv_image, image, pBindInfos[i].image);
5531
5532                 if (mem) {
5533                         image->bo = mem->bo;
5534                         image->offset = pBindInfos[i].memoryOffset;
5535                 } else {
5536                         image->bo = NULL;
5537                         image->offset = 0;
5538                 }
5539         }
5540         return VK_SUCCESS;
5541 }
5542
5543
5544 VkResult radv_BindImageMemory(
5545         VkDevice                                    device,
5546         VkImage                                     image,
5547         VkDeviceMemory                              memory,
5548         VkDeviceSize                                memoryOffset)
5549 {
5550         const VkBindImageMemoryInfo info = {
5551                 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
5552                 .image = image,
5553                 .memory = memory,
5554                 .memoryOffset = memoryOffset
5555         };
5556
5557         return radv_BindImageMemory2(device, 1, &info);
5558 }
5559
5560 static bool radv_sparse_bind_has_effects(const VkBindSparseInfo *info)
5561 {
5562         return info->bufferBindCount ||
5563                info->imageOpaqueBindCount ||
5564                info->imageBindCount ||
5565                info->waitSemaphoreCount ||
5566                info->signalSemaphoreCount;
5567 }
5568
5569  VkResult radv_QueueBindSparse(
5570         VkQueue                                     _queue,
5571         uint32_t                                    bindInfoCount,
5572         const VkBindSparseInfo*                     pBindInfo,
5573         VkFence                                     fence)
5574 {
5575         RADV_FROM_HANDLE(radv_queue, queue, _queue);
5576         VkResult result;
5577         uint32_t fence_idx = 0;
5578
5579         if (radv_device_is_lost(queue->device))
5580                 return VK_ERROR_DEVICE_LOST;
5581
5582         if (fence != VK_NULL_HANDLE) {
5583                 for (uint32_t i = 0; i < bindInfoCount; ++i)
5584                         if (radv_sparse_bind_has_effects(pBindInfo + i))
5585                                 fence_idx = i;
5586         } else
5587                 fence_idx = UINT32_MAX;
5588
5589         for (uint32_t i = 0; i < bindInfoCount; ++i) {
5590                 if (i != fence_idx && !radv_sparse_bind_has_effects(pBindInfo + i))
5591                         continue;
5592
5593                 const VkTimelineSemaphoreSubmitInfo *timeline_info =
5594                         vk_find_struct_const(pBindInfo[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
5595
5596                 VkResult result = radv_queue_submit(queue, &(struct radv_queue_submission) {
5597                                 .buffer_binds = pBindInfo[i].pBufferBinds,
5598                                 .buffer_bind_count = pBindInfo[i].bufferBindCount,
5599                                 .image_opaque_binds = pBindInfo[i].pImageOpaqueBinds,
5600                                 .image_opaque_bind_count = pBindInfo[i].imageOpaqueBindCount,
5601                                 .wait_semaphores = pBindInfo[i].pWaitSemaphores,
5602                                 .wait_semaphore_count = pBindInfo[i].waitSemaphoreCount,
5603                                 .signal_semaphores = pBindInfo[i].pSignalSemaphores,
5604                                 .signal_semaphore_count = pBindInfo[i].signalSemaphoreCount,
5605                                 .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
5606                                 .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
5607                                 .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
5608                                 .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
5609                                 .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
5610                         });
5611
5612                 if (result != VK_SUCCESS)
5613                         return result;
5614         }
5615
5616         if (fence != VK_NULL_HANDLE && !bindInfoCount) {
5617                 result = radv_signal_fence(queue, fence);
5618                 if (result != VK_SUCCESS)
5619                         return result;
5620         }
5621
5622         return VK_SUCCESS;
5623 }
5624
5625 static void
5626 radv_destroy_fence_part(struct radv_device *device,
5627                         struct radv_fence_part *part)
5628 {
5629         switch (part->kind) {
5630         case RADV_FENCE_NONE:
5631                 break;
5632         case RADV_FENCE_WINSYS:
5633                 device->ws->destroy_fence(part->fence);
5634                 break;
5635         case RADV_FENCE_SYNCOBJ:
5636                 device->ws->destroy_syncobj(device->ws, part->syncobj);
5637                 break;
5638         case RADV_FENCE_WSI:
5639                 part->fence_wsi->destroy(part->fence_wsi);
5640                 break;
5641         default:
5642                 unreachable("Invalid fence type");
5643         }
5644
5645         part->kind = RADV_FENCE_NONE;
5646 }
5647
5648 static void
5649 radv_destroy_fence(struct radv_device *device,
5650                    const VkAllocationCallbacks *pAllocator,
5651                    struct radv_fence *fence)
5652 {
5653         radv_destroy_fence_part(device, &fence->temporary);
5654         radv_destroy_fence_part(device, &fence->permanent);
5655
5656         vk_object_base_finish(&fence->base);
5657         vk_free2(&device->vk.alloc, pAllocator, fence);
5658 }
5659
5660 VkResult radv_CreateFence(
5661         VkDevice                                    _device,
5662         const VkFenceCreateInfo*                    pCreateInfo,
5663         const VkAllocationCallbacks*                pAllocator,
5664         VkFence*                                    pFence)
5665 {
5666         RADV_FROM_HANDLE(radv_device, device, _device);
5667         const VkExportFenceCreateInfo *export =
5668                 vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
5669         VkExternalFenceHandleTypeFlags handleTypes =
5670                 export ? export->handleTypes : 0;
5671         struct radv_fence *fence;
5672
5673         fence = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*fence), 8,
5674                            VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5675         if (!fence)
5676                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5677
5678         vk_object_base_init(&device->vk, &fence->base, VK_OBJECT_TYPE_FENCE);
5679
5680         if (device->always_use_syncobj || handleTypes) {
5681                 fence->permanent.kind = RADV_FENCE_SYNCOBJ;
5682
5683                 bool create_signaled = false;
5684                 if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT)
5685                         create_signaled = true;
5686
5687                 int ret = device->ws->create_syncobj(device->ws, create_signaled,
5688                                                      &fence->permanent.syncobj);
5689                 if (ret) {
5690                         radv_destroy_fence(device, pAllocator, fence);
5691                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5692                 }
5693         } else {
5694                 fence->permanent.kind = RADV_FENCE_WINSYS;
5695
5696                 fence->permanent.fence = device->ws->create_fence();
5697                 if (!fence->permanent.fence) {
5698                         vk_free2(&device->vk.alloc, pAllocator, fence);
5699                         radv_destroy_fence(device, pAllocator, fence);
5700                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5701                 }
5702                 if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT)
5703                         device->ws->signal_fence(fence->permanent.fence);
5704         }
5705
5706         *pFence = radv_fence_to_handle(fence);
5707
5708         return VK_SUCCESS;
5709 }
5710
5711
5712 void radv_DestroyFence(
5713         VkDevice                                    _device,
5714         VkFence                                     _fence,
5715         const VkAllocationCallbacks*                pAllocator)
5716 {
5717         RADV_FROM_HANDLE(radv_device, device, _device);
5718         RADV_FROM_HANDLE(radv_fence, fence, _fence);
5719
5720         if (!fence)
5721                 return;
5722
5723         radv_destroy_fence(device, pAllocator, fence);
5724 }
5725
5726 static bool radv_all_fences_plain_and_submitted(struct radv_device *device,
5727                                                 uint32_t fenceCount, const VkFence *pFences)
5728 {
5729         for (uint32_t i = 0; i < fenceCount; ++i) {
5730                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5731
5732                 struct radv_fence_part *part =
5733                         fence->temporary.kind != RADV_FENCE_NONE ?
5734                         &fence->temporary : &fence->permanent;
5735                 if (part->kind != RADV_FENCE_WINSYS ||
5736                     !device->ws->is_fence_waitable(part->fence))
5737                         return false;
5738         }
5739         return true;
5740 }
5741
5742 static bool radv_all_fences_syncobj(uint32_t fenceCount, const VkFence *pFences)
5743 {
5744         for (uint32_t i = 0; i < fenceCount; ++i) {
5745                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5746
5747                 struct radv_fence_part *part =
5748                         fence->temporary.kind != RADV_FENCE_NONE ?
5749                         &fence->temporary : &fence->permanent;
5750                 if (part->kind != RADV_FENCE_SYNCOBJ)
5751                         return false;
5752         }
5753         return true;
5754 }
5755
5756 VkResult radv_WaitForFences(
5757         VkDevice                                    _device,
5758         uint32_t                                    fenceCount,
5759         const VkFence*                              pFences,
5760         VkBool32                                    waitAll,
5761         uint64_t                                    timeout)
5762 {
5763         RADV_FROM_HANDLE(radv_device, device, _device);
5764
5765         if (radv_device_is_lost(device))
5766                 return VK_ERROR_DEVICE_LOST;
5767
5768         timeout = radv_get_absolute_timeout(timeout);
5769
5770         if (device->always_use_syncobj &&
5771             radv_all_fences_syncobj(fenceCount, pFences))
5772         {
5773                 uint32_t *handles = malloc(sizeof(uint32_t) * fenceCount);
5774                 if (!handles)
5775                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5776
5777                 for (uint32_t i = 0; i < fenceCount; ++i) {
5778                         RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5779
5780                         struct radv_fence_part *part =
5781                                 fence->temporary.kind != RADV_FENCE_NONE ?
5782                                 &fence->temporary : &fence->permanent;
5783
5784                         assert(part->kind == RADV_FENCE_SYNCOBJ);
5785                         handles[i] = part->syncobj;
5786                 }
5787
5788                 bool success = device->ws->wait_syncobj(device->ws, handles, fenceCount, waitAll, timeout);
5789
5790                 free(handles);
5791                 return success ? VK_SUCCESS : VK_TIMEOUT;
5792         }
5793
5794         if (!waitAll && fenceCount > 1) {
5795                 /* Not doing this by default for waitAll, due to needing to allocate twice. */
5796                 if (device->physical_device->rad_info.drm_minor >= 10 && radv_all_fences_plain_and_submitted(device, fenceCount, pFences)) {
5797                         uint32_t wait_count = 0;
5798                         struct radeon_winsys_fence **fences = malloc(sizeof(struct radeon_winsys_fence *) * fenceCount);
5799                         if (!fences)
5800                                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5801
5802                         for (uint32_t i = 0; i < fenceCount; ++i) {
5803                                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5804
5805                                 struct radv_fence_part *part =
5806                                         fence->temporary.kind != RADV_FENCE_NONE ?
5807                                         &fence->temporary : &fence->permanent;
5808                                 assert(part->kind == RADV_FENCE_WINSYS);
5809
5810                                 if (device->ws->fence_wait(device->ws, part->fence, false, 0)) {
5811                                         free(fences);
5812                                         return VK_SUCCESS;
5813                                 }
5814
5815                                 fences[wait_count++] = part->fence;
5816                         }
5817
5818                         bool success = device->ws->fences_wait(device->ws, fences, wait_count,
5819                                                                waitAll, timeout - radv_get_current_time());
5820
5821                         free(fences);
5822                         return success ? VK_SUCCESS : VK_TIMEOUT;
5823                 }
5824
5825                 while(radv_get_current_time() <= timeout) {
5826                         for (uint32_t i = 0; i < fenceCount; ++i) {
5827                                 if (radv_GetFenceStatus(_device, pFences[i]) == VK_SUCCESS)
5828                                         return VK_SUCCESS;
5829                         }
5830                 }
5831                 return VK_TIMEOUT;
5832         }
5833
5834         for (uint32_t i = 0; i < fenceCount; ++i) {
5835                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5836                 bool expired = false;
5837
5838                 struct radv_fence_part *part =
5839                         fence->temporary.kind != RADV_FENCE_NONE ?
5840                         &fence->temporary : &fence->permanent;
5841
5842                 switch (part->kind) {
5843                 case RADV_FENCE_NONE:
5844                         break;
5845                 case RADV_FENCE_WINSYS:
5846                         if (!device->ws->is_fence_waitable(part->fence)) {
5847                                 while (!device->ws->is_fence_waitable(part->fence) &&
5848                                       radv_get_current_time() <= timeout)
5849                                         /* Do nothing */;
5850                         }
5851
5852                         expired = device->ws->fence_wait(device->ws,
5853                                                          part->fence,
5854                                                          true, timeout);
5855                         if (!expired)
5856                                 return VK_TIMEOUT;
5857                         break;
5858                 case RADV_FENCE_SYNCOBJ:
5859                         if (!device->ws->wait_syncobj(device->ws,
5860                                                       &part->syncobj, 1, true,
5861                                                       timeout))
5862                                 return VK_TIMEOUT;
5863                         break;
5864                 case RADV_FENCE_WSI: {
5865                         VkResult result = part->fence_wsi->wait(part->fence_wsi, timeout);
5866                         if (result != VK_SUCCESS)
5867                                 return result;
5868                         break;
5869                 }
5870                 default:
5871                         unreachable("Invalid fence type");
5872                 }
5873         }
5874
5875         return VK_SUCCESS;
5876 }
5877
5878 VkResult radv_ResetFences(VkDevice _device,
5879                           uint32_t fenceCount,
5880                           const VkFence *pFences)
5881 {
5882         RADV_FROM_HANDLE(radv_device, device, _device);
5883
5884         for (unsigned i = 0; i < fenceCount; ++i) {
5885                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5886
5887                 /* From the Vulkan 1.0.53 spec:
5888                  *
5889                  *    "If any member of pFences currently has its payload
5890                  *    imported with temporary permanence, that fence’s prior
5891                  *    permanent payload is irst restored. The remaining
5892                  *    operations described therefore operate on the restored
5893                  *    payload."
5894                  */
5895                 if (fence->temporary.kind != RADV_FENCE_NONE)
5896                         radv_destroy_fence_part(device, &fence->temporary);
5897
5898                 struct radv_fence_part *part = &fence->permanent;
5899
5900                 switch (part->kind) {
5901                 case RADV_FENCE_WSI:
5902                         device->ws->reset_fence(part->fence);
5903                         break;
5904                 case RADV_FENCE_SYNCOBJ:
5905                         device->ws->reset_syncobj(device->ws, part->syncobj);
5906                         break;
5907                 default:
5908                         unreachable("Invalid fence type");
5909                 }
5910         }
5911
5912         return VK_SUCCESS;
5913 }
5914
5915 VkResult radv_GetFenceStatus(VkDevice _device, VkFence _fence)
5916 {
5917         RADV_FROM_HANDLE(radv_device, device, _device);
5918         RADV_FROM_HANDLE(radv_fence, fence, _fence);
5919
5920         struct radv_fence_part *part =
5921                 fence->temporary.kind != RADV_FENCE_NONE ?
5922                 &fence->temporary : &fence->permanent;
5923
5924         if (radv_device_is_lost(device))
5925                 return VK_ERROR_DEVICE_LOST;
5926
5927         switch (part->kind) {
5928         case RADV_FENCE_NONE:
5929                 break;
5930         case RADV_FENCE_WINSYS:
5931                 if (!device->ws->fence_wait(device->ws, part->fence, false, 0))
5932                         return VK_NOT_READY;
5933                 break;
5934         case RADV_FENCE_SYNCOBJ: {
5935                 bool success = device->ws->wait_syncobj(device->ws,
5936                                                         &part->syncobj, 1, true, 0);
5937                 if (!success)
5938                         return VK_NOT_READY;
5939                 break;
5940         }
5941         case RADV_FENCE_WSI: {
5942                 VkResult result = part->fence_wsi->wait(part->fence_wsi, 0);
5943                 if (result != VK_SUCCESS) {
5944                         if (result == VK_TIMEOUT)
5945                                 return VK_NOT_READY;
5946                         return result;
5947                 }
5948                 break;
5949         }
5950         default:
5951                 unreachable("Invalid fence type");
5952         }
5953
5954         return VK_SUCCESS;
5955 }
5956
5957
5958 // Queue semaphore functions
5959
5960 static void
5961 radv_create_timeline(struct radv_timeline *timeline, uint64_t value)
5962 {
5963         timeline->highest_signaled = value;
5964         timeline->highest_submitted = value;
5965         list_inithead(&timeline->points);
5966         list_inithead(&timeline->free_points);
5967         list_inithead(&timeline->waiters);
5968         pthread_mutex_init(&timeline->mutex, NULL);
5969 }
5970
5971 static void
5972 radv_destroy_timeline(struct radv_device *device,
5973                       struct radv_timeline *timeline)
5974 {
5975         list_for_each_entry_safe(struct radv_timeline_point, point,
5976                                  &timeline->free_points, list) {
5977                 list_del(&point->list);
5978                 device->ws->destroy_syncobj(device->ws, point->syncobj);
5979                 free(point);
5980         }
5981         list_for_each_entry_safe(struct radv_timeline_point, point,
5982                                  &timeline->points, list) {
5983                 list_del(&point->list);
5984                 device->ws->destroy_syncobj(device->ws, point->syncobj);
5985                 free(point);
5986         }
5987         pthread_mutex_destroy(&timeline->mutex);
5988 }
5989
5990 static void
5991 radv_timeline_gc_locked(struct radv_device *device,
5992                         struct radv_timeline *timeline)
5993 {
5994         list_for_each_entry_safe(struct radv_timeline_point, point,
5995                                  &timeline->points, list) {
5996                 if (point->wait_count || point->value > timeline->highest_submitted)
5997                         return;
5998
5999                 if (device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, 0)) {
6000                         timeline->highest_signaled = point->value;
6001                         list_del(&point->list);
6002                         list_add(&point->list, &timeline->free_points);
6003                 }
6004         }
6005 }
6006
6007 static struct radv_timeline_point *
6008 radv_timeline_find_point_at_least_locked(struct radv_device *device,
6009                                          struct radv_timeline *timeline,
6010                                          uint64_t p)
6011 {
6012         radv_timeline_gc_locked(device, timeline);
6013
6014         if (p <= timeline->highest_signaled)
6015                 return NULL;
6016
6017         list_for_each_entry(struct radv_timeline_point, point,
6018                             &timeline->points, list) {
6019                 if (point->value >= p) {
6020                         ++point->wait_count;
6021                         return point;
6022                 }
6023         }
6024         return NULL;
6025 }
6026
6027 static struct radv_timeline_point *
6028 radv_timeline_add_point_locked(struct radv_device *device,
6029                                struct radv_timeline *timeline,
6030                                uint64_t p)
6031 {
6032         radv_timeline_gc_locked(device, timeline);
6033
6034         struct radv_timeline_point *ret = NULL;
6035         struct radv_timeline_point *prev = NULL;
6036         int r;
6037
6038         if (p <= timeline->highest_signaled)
6039                 return NULL;
6040
6041         list_for_each_entry(struct radv_timeline_point, point,
6042                             &timeline->points, list) {
6043                 if (point->value == p) {
6044                         return NULL;
6045                 }
6046
6047                 if (point->value < p)
6048                         prev = point;
6049         }
6050
6051         if (list_is_empty(&timeline->free_points)) {
6052                 ret = malloc(sizeof(struct radv_timeline_point));
6053                 r = device->ws->create_syncobj(device->ws, false, &ret->syncobj);
6054                 if (r) {
6055                         free(ret);
6056                         return NULL;
6057                 }
6058         } else {
6059                 ret = list_first_entry(&timeline->free_points, struct radv_timeline_point, list);
6060                 list_del(&ret->list);
6061
6062                 device->ws->reset_syncobj(device->ws, ret->syncobj);
6063         }
6064
6065         ret->value = p;
6066         ret->wait_count = 1;
6067
6068         if (prev) {
6069                 list_add(&ret->list, &prev->list);
6070         } else {
6071                 list_addtail(&ret->list, &timeline->points);
6072         }
6073         return ret;
6074 }
6075
6076
6077 static VkResult
6078 radv_timeline_wait(struct radv_device *device,
6079                    struct radv_timeline *timeline,
6080                    uint64_t value,
6081                    uint64_t abs_timeout)
6082 {
6083         pthread_mutex_lock(&timeline->mutex);
6084
6085         while(timeline->highest_submitted < value) {
6086                 struct timespec abstime;
6087                 timespec_from_nsec(&abstime, abs_timeout);
6088
6089                 pthread_cond_timedwait(&device->timeline_cond, &timeline->mutex, &abstime);
6090
6091                 if (radv_get_current_time() >= abs_timeout && timeline->highest_submitted < value) {
6092                         pthread_mutex_unlock(&timeline->mutex);
6093                         return VK_TIMEOUT;
6094                 }
6095         }
6096
6097         struct radv_timeline_point *point = radv_timeline_find_point_at_least_locked(device, timeline, value);
6098         pthread_mutex_unlock(&timeline->mutex);
6099         if (!point)
6100                 return VK_SUCCESS;
6101
6102         bool success = device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, abs_timeout);
6103
6104         pthread_mutex_lock(&timeline->mutex);
6105         point->wait_count--;
6106         pthread_mutex_unlock(&timeline->mutex);
6107         return success ? VK_SUCCESS : VK_TIMEOUT;
6108 }
6109
6110 static void
6111 radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline,
6112                                      struct list_head *processing_list)
6113 {
6114         list_for_each_entry_safe(struct radv_timeline_waiter, waiter,
6115                                  &timeline->waiters, list) {
6116                 if (waiter->value > timeline->highest_submitted)
6117                         continue;
6118
6119                 radv_queue_trigger_submission(waiter->submission, 1, processing_list);
6120                 list_del(&waiter->list);
6121         }
6122 }
6123
6124 static
6125 void radv_destroy_semaphore_part(struct radv_device *device,
6126                                  struct radv_semaphore_part *part)
6127 {
6128         switch(part->kind) {
6129         case RADV_SEMAPHORE_NONE:
6130                 break;
6131         case RADV_SEMAPHORE_WINSYS:
6132                 device->ws->destroy_sem(part->ws_sem);
6133                 break;
6134         case RADV_SEMAPHORE_TIMELINE:
6135                 radv_destroy_timeline(device, &part->timeline);
6136                 break;
6137         case RADV_SEMAPHORE_SYNCOBJ:
6138         case RADV_SEMAPHORE_TIMELINE_SYNCOBJ:
6139                 device->ws->destroy_syncobj(device->ws, part->syncobj);
6140                 break;
6141         }
6142         part->kind = RADV_SEMAPHORE_NONE;
6143 }
6144
6145 static VkSemaphoreTypeKHR
6146 radv_get_semaphore_type(const void *pNext, uint64_t *initial_value)
6147 {
6148         const VkSemaphoreTypeCreateInfo *type_info =
6149                 vk_find_struct_const(pNext, SEMAPHORE_TYPE_CREATE_INFO);
6150
6151         if (!type_info)
6152                 return VK_SEMAPHORE_TYPE_BINARY;
6153
6154         if (initial_value)
6155                 *initial_value = type_info->initialValue;
6156         return type_info->semaphoreType;
6157 }
6158
6159 static void
6160 radv_destroy_semaphore(struct radv_device *device,
6161                        const VkAllocationCallbacks *pAllocator,
6162                        struct radv_semaphore *sem)
6163 {
6164         radv_destroy_semaphore_part(device, &sem->temporary);
6165         radv_destroy_semaphore_part(device, &sem->permanent);
6166         vk_object_base_finish(&sem->base);
6167         vk_free2(&device->vk.alloc, pAllocator, sem);
6168 }
6169
6170 VkResult radv_CreateSemaphore(
6171         VkDevice                                    _device,
6172         const VkSemaphoreCreateInfo*                pCreateInfo,
6173         const VkAllocationCallbacks*                pAllocator,
6174         VkSemaphore*                                pSemaphore)
6175 {
6176         RADV_FROM_HANDLE(radv_device, device, _device);
6177         const VkExportSemaphoreCreateInfo *export =
6178                 vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO);
6179         VkExternalSemaphoreHandleTypeFlags handleTypes =
6180                 export ? export->handleTypes : 0;
6181         uint64_t initial_value = 0;
6182         VkSemaphoreTypeKHR type = radv_get_semaphore_type(pCreateInfo->pNext, &initial_value);
6183
6184         struct radv_semaphore *sem = vk_alloc2(&device->vk.alloc, pAllocator,
6185                                                sizeof(*sem), 8,
6186                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6187         if (!sem)
6188                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6189
6190         vk_object_base_init(&device->vk, &sem->base,
6191                             VK_OBJECT_TYPE_SEMAPHORE);
6192
6193         sem->temporary.kind = RADV_SEMAPHORE_NONE;
6194         sem->permanent.kind = RADV_SEMAPHORE_NONE;
6195
6196         if (type == VK_SEMAPHORE_TYPE_TIMELINE &&
6197             device->physical_device->rad_info.has_timeline_syncobj) {
6198                 int ret = device->ws->create_syncobj(device->ws, false, &sem->permanent.syncobj);
6199                 if (ret) {
6200                         radv_destroy_semaphore(device, pAllocator, sem);
6201                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6202                 }
6203                 device->ws->signal_syncobj(device->ws, sem->permanent.syncobj, initial_value);
6204                 sem->permanent.timeline_syncobj.max_point = initial_value;
6205                 sem->permanent.kind = RADV_SEMAPHORE_TIMELINE_SYNCOBJ;
6206         } else if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
6207                 radv_create_timeline(&sem->permanent.timeline, initial_value);
6208                 sem->permanent.kind = RADV_SEMAPHORE_TIMELINE;
6209         } else if (device->always_use_syncobj || handleTypes) {
6210                 assert (device->physical_device->rad_info.has_syncobj);
6211                 int ret = device->ws->create_syncobj(device->ws, false,
6212                                                      &sem->permanent.syncobj);
6213                 if (ret) {
6214                         radv_destroy_semaphore(device, pAllocator, sem);
6215                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6216                 }
6217                 sem->permanent.kind = RADV_SEMAPHORE_SYNCOBJ;
6218         } else {
6219                 sem->permanent.ws_sem = device->ws->create_sem(device->ws);
6220                 if (!sem->permanent.ws_sem) {
6221                         radv_destroy_semaphore(device, pAllocator, sem);
6222                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6223                 }
6224                 sem->permanent.kind = RADV_SEMAPHORE_WINSYS;
6225         }
6226
6227         *pSemaphore = radv_semaphore_to_handle(sem);
6228         return VK_SUCCESS;
6229 }
6230
6231 void radv_DestroySemaphore(
6232         VkDevice                                    _device,
6233         VkSemaphore                                 _semaphore,
6234         const VkAllocationCallbacks*                pAllocator)
6235 {
6236         RADV_FROM_HANDLE(radv_device, device, _device);
6237         RADV_FROM_HANDLE(radv_semaphore, sem, _semaphore);
6238         if (!_semaphore)
6239                 return;
6240
6241         radv_destroy_semaphore(device, pAllocator, sem);
6242 }
6243
6244 VkResult
6245 radv_GetSemaphoreCounterValue(VkDevice _device,
6246                               VkSemaphore _semaphore,
6247                               uint64_t* pValue)
6248 {
6249         RADV_FROM_HANDLE(radv_device, device, _device);
6250         RADV_FROM_HANDLE(radv_semaphore, semaphore, _semaphore);
6251
6252         if (radv_device_is_lost(device))
6253                 return VK_ERROR_DEVICE_LOST;
6254
6255         struct radv_semaphore_part *part =
6256                 semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
6257
6258         switch (part->kind) {
6259         case RADV_SEMAPHORE_TIMELINE: {
6260                 pthread_mutex_lock(&part->timeline.mutex);
6261                 radv_timeline_gc_locked(device, &part->timeline);
6262                 *pValue = part->timeline.highest_signaled;
6263                 pthread_mutex_unlock(&part->timeline.mutex);
6264                 return VK_SUCCESS;
6265         }
6266         case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: {
6267                 return device->ws->query_syncobj(device->ws, part->syncobj, pValue);
6268         }
6269         case RADV_SEMAPHORE_NONE:
6270         case RADV_SEMAPHORE_SYNCOBJ:
6271         case RADV_SEMAPHORE_WINSYS:
6272                 unreachable("Invalid semaphore type");
6273         }
6274         unreachable("Unhandled semaphore type");
6275 }
6276
6277
6278 static VkResult
6279 radv_wait_timelines(struct radv_device *device,
6280                     const VkSemaphoreWaitInfo* pWaitInfo,
6281                     uint64_t abs_timeout)
6282 {
6283         if ((pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR) && pWaitInfo->semaphoreCount > 1) {
6284                 for (;;) {
6285                         for(uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6286                                 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6287                                 VkResult result = radv_timeline_wait(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], 0);
6288
6289                                 if (result == VK_SUCCESS)
6290                                         return VK_SUCCESS;
6291                         }
6292                         if (radv_get_current_time() > abs_timeout)
6293                                 return VK_TIMEOUT;
6294                 }
6295         }
6296
6297         for(uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6298                 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6299                 VkResult result = radv_timeline_wait(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], abs_timeout);
6300
6301                 if (result != VK_SUCCESS)
6302                         return result;
6303         }
6304         return VK_SUCCESS;
6305 }
6306 VkResult
6307 radv_WaitSemaphores(VkDevice _device,
6308                     const VkSemaphoreWaitInfo* pWaitInfo,
6309                     uint64_t timeout)
6310 {
6311         RADV_FROM_HANDLE(radv_device, device, _device);
6312
6313         if (radv_device_is_lost(device))
6314                 return VK_ERROR_DEVICE_LOST;
6315
6316         uint64_t abs_timeout = radv_get_absolute_timeout(timeout);
6317
6318         if (radv_semaphore_from_handle(pWaitInfo->pSemaphores[0])->permanent.kind == RADV_SEMAPHORE_TIMELINE)
6319                 return radv_wait_timelines(device, pWaitInfo, abs_timeout);
6320
6321         if (pWaitInfo->semaphoreCount > UINT32_MAX / sizeof(uint32_t))
6322                 return vk_errorf(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY, "semaphoreCount integer overflow");
6323
6324         bool wait_all = !(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR);
6325         uint32_t *handles = malloc(sizeof(*handles) * pWaitInfo->semaphoreCount);
6326         if (!handles)
6327                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6328
6329         for (uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6330                 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6331                 handles[i] = semaphore->permanent.syncobj;
6332         }
6333
6334         bool success = device->ws->wait_timeline_syncobj(device->ws, handles, pWaitInfo->pValues,
6335                                                          pWaitInfo->semaphoreCount, wait_all, false,
6336                                                          abs_timeout);
6337         free(handles);
6338         return success ? VK_SUCCESS : VK_TIMEOUT;
6339 }
6340
6341 VkResult
6342 radv_SignalSemaphore(VkDevice _device,
6343                      const VkSemaphoreSignalInfo* pSignalInfo)
6344 {
6345         RADV_FROM_HANDLE(radv_device, device, _device);
6346         RADV_FROM_HANDLE(radv_semaphore, semaphore, pSignalInfo->semaphore);
6347
6348         struct radv_semaphore_part *part =
6349                 semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
6350
6351         switch(part->kind) {
6352         case RADV_SEMAPHORE_TIMELINE: {
6353                 pthread_mutex_lock(&part->timeline.mutex);
6354                 radv_timeline_gc_locked(device, &part->timeline);
6355                 part->timeline.highest_submitted = MAX2(part->timeline.highest_submitted, pSignalInfo->value);
6356                 part->timeline.highest_signaled = MAX2(part->timeline.highest_signaled, pSignalInfo->value);
6357
6358                 struct list_head processing_list;
6359                 list_inithead(&processing_list);
6360                 radv_timeline_trigger_waiters_locked(&part->timeline, &processing_list);
6361                 pthread_mutex_unlock(&part->timeline.mutex);
6362
6363                 VkResult result = radv_process_submissions(&processing_list);
6364
6365                 /* This needs to happen after radv_process_submissions, so
6366                  * that any submitted submissions that are now unblocked get
6367                  * processed before we wake the application. This way we
6368                  * ensure that any binary semaphores that are now unblocked
6369                  * are usable by the application. */
6370                 pthread_cond_broadcast(&device->timeline_cond);
6371
6372                 return result;
6373         }
6374         case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: {
6375                 part->timeline_syncobj.max_point = MAX2(part->timeline_syncobj.max_point, pSignalInfo->value);
6376                 device->ws->signal_syncobj(device->ws, part->syncobj, pSignalInfo->value);
6377                 break;
6378         }
6379         case RADV_SEMAPHORE_NONE:
6380         case RADV_SEMAPHORE_SYNCOBJ:
6381         case RADV_SEMAPHORE_WINSYS:
6382                 unreachable("Invalid semaphore type");
6383         }
6384         return VK_SUCCESS;
6385 }
6386
6387 static void radv_destroy_event(struct radv_device *device,
6388                                const VkAllocationCallbacks* pAllocator,
6389                                struct radv_event *event)
6390 {
6391         if (event->bo)
6392                 device->ws->buffer_destroy(event->bo);
6393
6394         vk_object_base_finish(&event->base);
6395         vk_free2(&device->vk.alloc, pAllocator, event);
6396 }
6397
6398 VkResult radv_CreateEvent(
6399         VkDevice                                    _device,
6400         const VkEventCreateInfo*                    pCreateInfo,
6401         const VkAllocationCallbacks*                pAllocator,
6402         VkEvent*                                    pEvent)
6403 {
6404         RADV_FROM_HANDLE(radv_device, device, _device);
6405         struct radv_event *event = vk_alloc2(&device->vk.alloc, pAllocator,
6406                                                sizeof(*event), 8,
6407                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6408
6409         if (!event)
6410                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6411
6412         vk_object_base_init(&device->vk, &event->base, VK_OBJECT_TYPE_EVENT);
6413
6414         event->bo = device->ws->buffer_create(device->ws, 8, 8,
6415                                               RADEON_DOMAIN_GTT,
6416                                               RADEON_FLAG_VA_UNCACHED | RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING,
6417                                               RADV_BO_PRIORITY_FENCE);
6418         if (!event->bo) {
6419                 radv_destroy_event(device, pAllocator, event);
6420                 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6421         }
6422
6423         event->map = (uint64_t*)device->ws->buffer_map(event->bo);
6424         if (!event->map) {
6425                 radv_destroy_event(device, pAllocator, event);
6426                 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6427         }
6428
6429         *pEvent = radv_event_to_handle(event);
6430
6431         return VK_SUCCESS;
6432 }
6433
6434 void radv_DestroyEvent(
6435         VkDevice                                    _device,
6436         VkEvent                                     _event,
6437         const VkAllocationCallbacks*                pAllocator)
6438 {
6439         RADV_FROM_HANDLE(radv_device, device, _device);
6440         RADV_FROM_HANDLE(radv_event, event, _event);
6441
6442         if (!event)
6443                 return;
6444
6445         radv_destroy_event(device, pAllocator, event);
6446 }
6447
6448 VkResult radv_GetEventStatus(
6449         VkDevice                                    _device,
6450         VkEvent                                     _event)
6451 {
6452         RADV_FROM_HANDLE(radv_device, device, _device);
6453         RADV_FROM_HANDLE(radv_event, event, _event);
6454
6455         if (radv_device_is_lost(device))
6456                 return VK_ERROR_DEVICE_LOST;
6457
6458         if (*event->map == 1)
6459                 return VK_EVENT_SET;
6460         return VK_EVENT_RESET;
6461 }
6462
6463 VkResult radv_SetEvent(
6464         VkDevice                                    _device,
6465         VkEvent                                     _event)
6466 {
6467         RADV_FROM_HANDLE(radv_event, event, _event);
6468         *event->map = 1;
6469
6470         return VK_SUCCESS;
6471 }
6472
6473 VkResult radv_ResetEvent(
6474     VkDevice                                    _device,
6475     VkEvent                                     _event)
6476 {
6477         RADV_FROM_HANDLE(radv_event, event, _event);
6478         *event->map = 0;
6479
6480         return VK_SUCCESS;
6481 }
6482
6483 static void
6484 radv_destroy_buffer(struct radv_device *device,
6485                     const VkAllocationCallbacks *pAllocator,
6486                     struct radv_buffer *buffer)
6487 {
6488         if ((buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) && buffer->bo)
6489                 device->ws->buffer_destroy(buffer->bo);
6490
6491         vk_object_base_finish(&buffer->base);
6492         vk_free2(&device->vk.alloc, pAllocator, buffer);
6493 }
6494
6495 VkResult radv_CreateBuffer(
6496         VkDevice                                    _device,
6497         const VkBufferCreateInfo*                   pCreateInfo,
6498         const VkAllocationCallbacks*                pAllocator,
6499         VkBuffer*                                   pBuffer)
6500 {
6501         RADV_FROM_HANDLE(radv_device, device, _device);
6502         struct radv_buffer *buffer;
6503
6504         if (pCreateInfo->size > RADV_MAX_MEMORY_ALLOCATION_SIZE)
6505                 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6506
6507         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
6508
6509         buffer = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*buffer), 8,
6510                              VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6511         if (buffer == NULL)
6512                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6513
6514         vk_object_base_init(&device->vk, &buffer->base, VK_OBJECT_TYPE_BUFFER);
6515
6516         buffer->size = pCreateInfo->size;
6517         buffer->usage = pCreateInfo->usage;
6518         buffer->bo = NULL;
6519         buffer->offset = 0;
6520         buffer->flags = pCreateInfo->flags;
6521
6522         buffer->shareable = vk_find_struct_const(pCreateInfo->pNext,
6523                                                  EXTERNAL_MEMORY_BUFFER_CREATE_INFO) != NULL;
6524
6525         if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
6526                 buffer->bo = device->ws->buffer_create(device->ws,
6527                                                        align64(buffer->size, 4096),
6528                                                        4096, 0, RADEON_FLAG_VIRTUAL,
6529                                                        RADV_BO_PRIORITY_VIRTUAL);
6530                 if (!buffer->bo) {
6531                         radv_destroy_buffer(device, pAllocator, buffer);
6532                         return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6533                 }
6534         }
6535
6536         *pBuffer = radv_buffer_to_handle(buffer);
6537
6538         return VK_SUCCESS;
6539 }
6540
6541 void radv_DestroyBuffer(
6542         VkDevice                                    _device,
6543         VkBuffer                                    _buffer,
6544         const VkAllocationCallbacks*                pAllocator)
6545 {
6546         RADV_FROM_HANDLE(radv_device, device, _device);
6547         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
6548
6549         if (!buffer)
6550                 return;
6551
6552         radv_destroy_buffer(device, pAllocator, buffer);
6553 }
6554
6555 VkDeviceAddress radv_GetBufferDeviceAddress(
6556         VkDevice                                    device,
6557         const VkBufferDeviceAddressInfo*         pInfo)
6558 {
6559         RADV_FROM_HANDLE(radv_buffer, buffer, pInfo->buffer);
6560         return radv_buffer_get_va(buffer->bo) + buffer->offset;
6561 }
6562
6563
6564 uint64_t radv_GetBufferOpaqueCaptureAddress(VkDevice device,
6565                                             const VkBufferDeviceAddressInfo* pInfo)
6566 {
6567         return 0;
6568 }
6569
6570 uint64_t radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,
6571                                                   const VkDeviceMemoryOpaqueCaptureAddressInfo* pInfo)
6572 {
6573         return 0;
6574 }
6575
6576 static inline unsigned
6577 si_tile_mode_index(const struct radv_image_plane *plane, unsigned level, bool stencil)
6578 {
6579         if (stencil)
6580                 return plane->surface.u.legacy.stencil_tiling_index[level];
6581         else
6582                 return plane->surface.u.legacy.tiling_index[level];
6583 }
6584
6585 static uint32_t radv_surface_max_layer_count(struct radv_image_view *iview)
6586 {
6587         return iview->type == VK_IMAGE_VIEW_TYPE_3D ? iview->extent.depth : (iview->base_layer + iview->layer_count);
6588 }
6589
6590 static uint32_t
6591 radv_init_dcc_control_reg(struct radv_device *device,
6592                           struct radv_image_view *iview)
6593 {
6594         unsigned max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_256B;
6595         unsigned min_compressed_block_size = V_028C78_MIN_BLOCK_SIZE_32B;
6596         unsigned max_compressed_block_size;
6597         unsigned independent_128b_blocks;
6598         unsigned independent_64b_blocks;
6599
6600         if (!radv_dcc_enabled(iview->image, iview->base_mip))
6601                 return 0;
6602
6603         if (!device->physical_device->rad_info.has_dedicated_vram) {
6604                 /* amdvlk: [min-compressed-block-size] should be set to 32 for
6605                  * dGPU and 64 for APU because all of our APUs to date use
6606                  * DIMMs which have a request granularity size of 64B while all
6607                  * other chips have a 32B request size.
6608                  */
6609                 min_compressed_block_size = V_028C78_MIN_BLOCK_SIZE_64B;
6610         }
6611
6612         if (device->physical_device->rad_info.chip_class >= GFX10) {
6613                 max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B;
6614                 independent_64b_blocks = 0;
6615                 independent_128b_blocks = 1;
6616         } else {
6617                 independent_128b_blocks = 0;
6618
6619                 if (iview->image->info.samples > 1) {
6620                         if (iview->image->planes[0].surface.bpe == 1)
6621                                 max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B;
6622                         else if (iview->image->planes[0].surface.bpe == 2)
6623                                 max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B;
6624                 }
6625
6626                 if (iview->image->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
6627                                            VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
6628                                            VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) {
6629                         /* If this DCC image is potentially going to be used in texture
6630                          * fetches, we need some special settings.
6631                          */
6632                         independent_64b_blocks = 1;
6633                         max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B;
6634                 } else {
6635                         /* MAX_UNCOMPRESSED_BLOCK_SIZE must be >=
6636                          * MAX_COMPRESSED_BLOCK_SIZE. Set MAX_COMPRESSED_BLOCK_SIZE as
6637                          * big as possible for better compression state.
6638                          */
6639                         independent_64b_blocks = 0;
6640                         max_compressed_block_size = max_uncompressed_block_size;
6641                 }
6642         }
6643
6644         return S_028C78_MAX_UNCOMPRESSED_BLOCK_SIZE(max_uncompressed_block_size) |
6645                S_028C78_MAX_COMPRESSED_BLOCK_SIZE(max_compressed_block_size) |
6646                S_028C78_MIN_COMPRESSED_BLOCK_SIZE(min_compressed_block_size) |
6647                S_028C78_INDEPENDENT_64B_BLOCKS(independent_64b_blocks) |
6648                S_028C78_INDEPENDENT_128B_BLOCKS(independent_128b_blocks);
6649 }
6650
6651 void
6652 radv_initialise_color_surface(struct radv_device *device,
6653                               struct radv_color_buffer_info *cb,
6654                               struct radv_image_view *iview)
6655 {
6656         const struct vk_format_description *desc;
6657         unsigned ntype, format, swap, endian;
6658         unsigned blend_clamp = 0, blend_bypass = 0;
6659         uint64_t va;
6660         const struct radv_image_plane *plane = &iview->image->planes[iview->plane_id];
6661         const struct radeon_surf *surf = &plane->surface;
6662
6663         desc = vk_format_description(iview->vk_format);
6664
6665         memset(cb, 0, sizeof(*cb));
6666
6667         /* Intensity is implemented as Red, so treat it that way. */
6668         cb->cb_color_attrib = S_028C74_FORCE_DST_ALPHA_1(desc->swizzle[3] == VK_SWIZZLE_1);
6669
6670         va = radv_buffer_get_va(iview->bo) + iview->image->offset + plane->offset;
6671
6672         cb->cb_color_base = va >> 8;
6673
6674         if (device->physical_device->rad_info.chip_class >= GFX9) {
6675                 if (device->physical_device->rad_info.chip_class >= GFX10) {
6676                         cb->cb_color_attrib3 |= S_028EE0_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6677                                 S_028EE0_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) |
6678                                 S_028EE0_CMASK_PIPE_ALIGNED(1) |
6679                                 S_028EE0_DCC_PIPE_ALIGNED(surf->u.gfx9.dcc.pipe_aligned);
6680                 } else {
6681                         struct gfx9_surf_meta_flags meta = {
6682                                 .rb_aligned = 1,
6683                                 .pipe_aligned = 1,
6684                         };
6685
6686                         if (surf->dcc_offset)
6687                                 meta = surf->u.gfx9.dcc;
6688
6689                         cb->cb_color_attrib |= S_028C74_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6690                                 S_028C74_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) |
6691                                 S_028C74_RB_ALIGNED(meta.rb_aligned) |
6692                                 S_028C74_PIPE_ALIGNED(meta.pipe_aligned);
6693                         cb->cb_mrt_epitch = S_0287A0_EPITCH(surf->u.gfx9.surf.epitch);
6694                 }
6695
6696                 cb->cb_color_base += surf->u.gfx9.surf_offset >> 8;
6697                 cb->cb_color_base |= surf->tile_swizzle;
6698         } else {
6699                 const struct legacy_surf_level *level_info = &surf->u.legacy.level[iview->base_mip];
6700                 unsigned pitch_tile_max, slice_tile_max, tile_mode_index;
6701
6702                 cb->cb_color_base += level_info->offset >> 8;
6703                 if (level_info->mode == RADEON_SURF_MODE_2D)
6704                         cb->cb_color_base |= surf->tile_swizzle;
6705
6706                 pitch_tile_max = level_info->nblk_x / 8 - 1;
6707                 slice_tile_max = (level_info->nblk_x * level_info->nblk_y) / 64 - 1;
6708                 tile_mode_index = si_tile_mode_index(plane, iview->base_mip, false);
6709
6710                 cb->cb_color_pitch = S_028C64_TILE_MAX(pitch_tile_max);
6711                 cb->cb_color_slice = S_028C68_TILE_MAX(slice_tile_max);
6712                 cb->cb_color_cmask_slice = surf->u.legacy.cmask_slice_tile_max;
6713
6714                 cb->cb_color_attrib |= S_028C74_TILE_MODE_INDEX(tile_mode_index);
6715
6716                 if (radv_image_has_fmask(iview->image)) {
6717                         if (device->physical_device->rad_info.chip_class >= GFX7)
6718                                 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(surf->u.legacy.fmask.pitch_in_pixels / 8 - 1);
6719                         cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(surf->u.legacy.fmask.tiling_index);
6720                         cb->cb_color_fmask_slice = S_028C88_TILE_MAX(surf->u.legacy.fmask.slice_tile_max);
6721                 } else {
6722                         /* This must be set for fast clear to work without FMASK. */
6723                         if (device->physical_device->rad_info.chip_class >= GFX7)
6724                                 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(pitch_tile_max);
6725                         cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(tile_mode_index);
6726                         cb->cb_color_fmask_slice = S_028C88_TILE_MAX(slice_tile_max);
6727                 }
6728         }
6729
6730         /* CMASK variables */
6731         va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6732         va += surf->cmask_offset;
6733         cb->cb_color_cmask = va >> 8;
6734
6735         va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6736         va += surf->dcc_offset;
6737
6738         if (radv_dcc_enabled(iview->image, iview->base_mip) &&
6739             device->physical_device->rad_info.chip_class <= GFX8)
6740                 va += plane->surface.u.legacy.level[iview->base_mip].dcc_offset;
6741
6742         unsigned dcc_tile_swizzle = surf->tile_swizzle;
6743         dcc_tile_swizzle &= (surf->dcc_alignment - 1) >> 8;
6744
6745         cb->cb_dcc_base = va >> 8;
6746         cb->cb_dcc_base |= dcc_tile_swizzle;
6747
6748         /* GFX10 field has the same base shift as the GFX6 field. */
6749         uint32_t max_slice = radv_surface_max_layer_count(iview) - 1;
6750         cb->cb_color_view = S_028C6C_SLICE_START(iview->base_layer) |
6751                 S_028C6C_SLICE_MAX_GFX10(max_slice);
6752
6753         if (iview->image->info.samples > 1) {
6754                 unsigned log_samples = util_logbase2(iview->image->info.samples);
6755
6756                 cb->cb_color_attrib |= S_028C74_NUM_SAMPLES(log_samples) |
6757                         S_028C74_NUM_FRAGMENTS(log_samples);
6758         }
6759
6760         if (radv_image_has_fmask(iview->image)) {
6761                 va = radv_buffer_get_va(iview->bo) + iview->image->offset + surf->fmask_offset;
6762                 cb->cb_color_fmask = va >> 8;
6763                 cb->cb_color_fmask |= surf->fmask_tile_swizzle;
6764         } else {
6765                 cb->cb_color_fmask = cb->cb_color_base;
6766         }
6767
6768         ntype = radv_translate_color_numformat(iview->vk_format,
6769                                                desc,
6770                                                vk_format_get_first_non_void_channel(iview->vk_format));
6771         format = radv_translate_colorformat(iview->vk_format);
6772         if (format == V_028C70_COLOR_INVALID || ntype == ~0u)
6773                 radv_finishme("Illegal color\n");
6774         swap = radv_translate_colorswap(iview->vk_format, false);
6775         endian = radv_colorformat_endian_swap(format);
6776
6777         /* blend clamp should be set for all NORM/SRGB types */
6778         if (ntype == V_028C70_NUMBER_UNORM ||
6779             ntype == V_028C70_NUMBER_SNORM ||
6780             ntype == V_028C70_NUMBER_SRGB)
6781                 blend_clamp = 1;
6782
6783         /* set blend bypass according to docs if SINT/UINT or
6784            8/24 COLOR variants */
6785         if (ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT ||
6786             format == V_028C70_COLOR_8_24 || format == V_028C70_COLOR_24_8 ||
6787             format == V_028C70_COLOR_X24_8_32_FLOAT) {
6788                 blend_clamp = 0;
6789                 blend_bypass = 1;
6790         }
6791 #if 0
6792         if ((ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT) &&
6793             (format == V_028C70_COLOR_8 ||
6794              format == V_028C70_COLOR_8_8 ||
6795              format == V_028C70_COLOR_8_8_8_8))
6796                 ->color_is_int8 = true;
6797 #endif
6798         cb->cb_color_info = S_028C70_FORMAT(format) |
6799                 S_028C70_COMP_SWAP(swap) |
6800                 S_028C70_BLEND_CLAMP(blend_clamp) |
6801                 S_028C70_BLEND_BYPASS(blend_bypass) |
6802                 S_028C70_SIMPLE_FLOAT(1) |
6803                 S_028C70_ROUND_MODE(ntype != V_028C70_NUMBER_UNORM &&
6804                                     ntype != V_028C70_NUMBER_SNORM &&
6805                                     ntype != V_028C70_NUMBER_SRGB &&
6806                                     format != V_028C70_COLOR_8_24 &&
6807                                     format != V_028C70_COLOR_24_8) |
6808                 S_028C70_NUMBER_TYPE(ntype) |
6809                 S_028C70_ENDIAN(endian);
6810         if (radv_image_has_fmask(iview->image)) {
6811                 cb->cb_color_info |= S_028C70_COMPRESSION(1);
6812                 if (device->physical_device->rad_info.chip_class == GFX6) {
6813                         unsigned fmask_bankh = util_logbase2(surf->u.legacy.fmask.bankh);
6814                         cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(fmask_bankh);
6815                 }
6816
6817                 if (radv_image_is_tc_compat_cmask(iview->image)) {
6818                         /* Allow the texture block to read FMASK directly
6819                          * without decompressing it. This bit must be cleared
6820                          * when performing FMASK_DECOMPRESS or DCC_COMPRESS,
6821                          * otherwise the operation doesn't happen.
6822                          */
6823                         cb->cb_color_info |= S_028C70_FMASK_COMPRESS_1FRAG_ONLY(1);
6824
6825                         /* Set CMASK into a tiling format that allows the
6826                          * texture block to read it.
6827                          */
6828                         cb->cb_color_info |= S_028C70_CMASK_ADDR_TYPE(2);
6829                 }
6830         }
6831
6832         if (radv_image_has_cmask(iview->image) &&
6833             !(device->instance->debug_flags & RADV_DEBUG_NO_FAST_CLEARS))
6834                 cb->cb_color_info |= S_028C70_FAST_CLEAR(1);
6835
6836         if (radv_dcc_enabled(iview->image, iview->base_mip))
6837                 cb->cb_color_info |= S_028C70_DCC_ENABLE(1);
6838
6839         cb->cb_dcc_control = radv_init_dcc_control_reg(device, iview);
6840
6841         /* This must be set for fast clear to work without FMASK. */
6842         if (!radv_image_has_fmask(iview->image) &&
6843             device->physical_device->rad_info.chip_class == GFX6) {
6844                 unsigned bankh = util_logbase2(surf->u.legacy.bankh);
6845                 cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(bankh);
6846         }
6847
6848         if (device->physical_device->rad_info.chip_class >= GFX9) {
6849                 const struct vk_format_description *format_desc = vk_format_description(iview->image->vk_format);
6850
6851                 unsigned mip0_depth = iview->image->type == VK_IMAGE_TYPE_3D ?
6852                   (iview->extent.depth - 1) : (iview->image->info.array_size - 1);
6853                 unsigned width = iview->extent.width / (iview->plane_id ? format_desc->width_divisor : 1);
6854                 unsigned height = iview->extent.height / (iview->plane_id ? format_desc->height_divisor : 1);
6855
6856                 if (device->physical_device->rad_info.chip_class >= GFX10) {
6857                         cb->cb_color_view |= S_028C6C_MIP_LEVEL_GFX10(iview->base_mip);
6858
6859                         cb->cb_color_attrib3 |= S_028EE0_MIP0_DEPTH(mip0_depth) |
6860                                                 S_028EE0_RESOURCE_TYPE(surf->u.gfx9.resource_type) |
6861                                                 S_028EE0_RESOURCE_LEVEL(1);
6862                 } else {
6863                         cb->cb_color_view |= S_028C6C_MIP_LEVEL_GFX9(iview->base_mip);
6864                         cb->cb_color_attrib |= S_028C74_MIP0_DEPTH(mip0_depth) |
6865                                                S_028C74_RESOURCE_TYPE(surf->u.gfx9.resource_type);
6866                 }
6867
6868                 cb->cb_color_attrib2 = S_028C68_MIP0_WIDTH(width - 1) |
6869                         S_028C68_MIP0_HEIGHT(height - 1) |
6870                         S_028C68_MAX_MIP(iview->image->info.levels - 1);
6871         }
6872 }
6873
6874 static unsigned
6875 radv_calc_decompress_on_z_planes(struct radv_device *device,
6876                                  struct radv_image_view *iview)
6877 {
6878         unsigned max_zplanes = 0;
6879
6880         assert(radv_image_is_tc_compat_htile(iview->image));
6881
6882         if (device->physical_device->rad_info.chip_class >= GFX9) {
6883                 /* Default value for 32-bit depth surfaces. */
6884                 max_zplanes = 4;
6885
6886                 if (iview->vk_format == VK_FORMAT_D16_UNORM &&
6887                     iview->image->info.samples > 1)
6888                         max_zplanes = 2;
6889
6890                 max_zplanes = max_zplanes + 1;
6891         } else {
6892                 if (iview->vk_format == VK_FORMAT_D16_UNORM) {
6893                         /* Do not enable Z plane compression for 16-bit depth
6894                          * surfaces because isn't supported on GFX8. Only
6895                          * 32-bit depth surfaces are supported by the hardware.
6896                          * This allows to maintain shader compatibility and to
6897                          * reduce the number of depth decompressions.
6898                          */
6899                         max_zplanes = 1;
6900                 } else {
6901                         if (iview->image->info.samples <= 1)
6902                                 max_zplanes = 5;
6903                         else if (iview->image->info.samples <= 4)
6904                                 max_zplanes = 3;
6905                         else
6906                                 max_zplanes = 2;
6907                 }
6908         }
6909
6910         return max_zplanes;
6911 }
6912
6913 void
6914 radv_initialise_ds_surface(struct radv_device *device,
6915                            struct radv_ds_buffer_info *ds,
6916                            struct radv_image_view *iview)
6917 {
6918         unsigned level = iview->base_mip;
6919         unsigned format, stencil_format;
6920         uint64_t va, s_offs, z_offs;
6921         bool stencil_only = false;
6922         const struct radv_image_plane *plane = &iview->image->planes[0];
6923         const struct radeon_surf *surf = &plane->surface;
6924
6925         assert(vk_format_get_plane_count(iview->image->vk_format) == 1);
6926
6927         memset(ds, 0, sizeof(*ds));
6928         switch (iview->image->vk_format) {
6929         case VK_FORMAT_D24_UNORM_S8_UINT:
6930         case VK_FORMAT_X8_D24_UNORM_PACK32:
6931                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-24);
6932                 ds->offset_scale = 2.0f;
6933                 break;
6934         case VK_FORMAT_D16_UNORM:
6935         case VK_FORMAT_D16_UNORM_S8_UINT:
6936                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-16);
6937                 ds->offset_scale = 4.0f;
6938                 break;
6939         case VK_FORMAT_D32_SFLOAT:
6940         case VK_FORMAT_D32_SFLOAT_S8_UINT:
6941                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-23) |
6942                         S_028B78_POLY_OFFSET_DB_IS_FLOAT_FMT(1);
6943                 ds->offset_scale = 1.0f;
6944                 break;
6945         case VK_FORMAT_S8_UINT:
6946                 stencil_only = true;
6947                 break;
6948         default:
6949                 break;
6950         }
6951
6952         format = radv_translate_dbformat(iview->image->vk_format);
6953         stencil_format = surf->has_stencil ?
6954                 V_028044_STENCIL_8 : V_028044_STENCIL_INVALID;
6955
6956         uint32_t max_slice = radv_surface_max_layer_count(iview) - 1;
6957         ds->db_depth_view = S_028008_SLICE_START(iview->base_layer) |
6958                 S_028008_SLICE_MAX(max_slice);
6959         if (device->physical_device->rad_info.chip_class >= GFX10) {
6960                 ds->db_depth_view |= S_028008_SLICE_START_HI(iview->base_layer >> 11) |
6961                                      S_028008_SLICE_MAX_HI(max_slice >> 11);
6962         }
6963
6964         ds->db_htile_data_base = 0;
6965         ds->db_htile_surface = 0;
6966
6967         va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6968         s_offs = z_offs = va;
6969
6970         if (device->physical_device->rad_info.chip_class >= GFX9) {
6971                 assert(surf->u.gfx9.surf_offset == 0);
6972                 s_offs += surf->u.gfx9.stencil_offset;
6973
6974                 ds->db_z_info = S_028038_FORMAT(format) |
6975                         S_028038_NUM_SAMPLES(util_logbase2(iview->image->info.samples)) |
6976                         S_028038_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6977                         S_028038_MAXMIP(iview->image->info.levels - 1) |
6978                         S_028038_ZRANGE_PRECISION(1);
6979                 ds->db_stencil_info = S_02803C_FORMAT(stencil_format) |
6980                         S_02803C_SW_MODE(surf->u.gfx9.stencil.swizzle_mode);
6981
6982                 if (device->physical_device->rad_info.chip_class == GFX9) {
6983                         ds->db_z_info2 = S_028068_EPITCH(surf->u.gfx9.surf.epitch);
6984                         ds->db_stencil_info2 = S_02806C_EPITCH(surf->u.gfx9.stencil.epitch);
6985                 }
6986
6987                 ds->db_depth_view |= S_028008_MIPID(level);
6988                 ds->db_depth_size = S_02801C_X_MAX(iview->image->info.width - 1) |
6989                         S_02801C_Y_MAX(iview->image->info.height - 1);
6990
6991                 if (radv_htile_enabled(iview->image, level)) {
6992                         ds->db_z_info |= S_028038_TILE_SURFACE_ENABLE(1);
6993
6994                         if (radv_image_is_tc_compat_htile(iview->image)) {
6995                                 unsigned max_zplanes =
6996                                         radv_calc_decompress_on_z_planes(device, iview);
6997
6998                                 ds->db_z_info |= S_028038_DECOMPRESS_ON_N_ZPLANES(max_zplanes);
6999
7000                                 if (device->physical_device->rad_info.chip_class >= GFX10) {
7001                                         ds->db_z_info |= S_028040_ITERATE_FLUSH(1);
7002                                         ds->db_stencil_info |= S_028044_ITERATE_FLUSH(1);
7003                                 } else {
7004                                         ds->db_z_info |= S_028038_ITERATE_FLUSH(1);
7005                                         ds->db_stencil_info |= S_02803C_ITERATE_FLUSH(1);
7006                                 }
7007                         }
7008
7009                         if (!surf->has_stencil)
7010                                 /* Use all of the htile_buffer for depth if there's no stencil. */
7011                                 ds->db_stencil_info |= S_02803C_TILE_STENCIL_DISABLE(1);
7012                         va = radv_buffer_get_va(iview->bo) + iview->image->offset +
7013                                 surf->htile_offset;
7014                         ds->db_htile_data_base = va >> 8;
7015                         ds->db_htile_surface = S_028ABC_FULL_CACHE(1) |
7016                                 S_028ABC_PIPE_ALIGNED(1);
7017
7018                         if (device->physical_device->rad_info.chip_class == GFX9) {
7019                                 ds->db_htile_surface |= S_028ABC_RB_ALIGNED(1);
7020                         }
7021                 }
7022         } else {
7023                 const struct legacy_surf_level *level_info = &surf->u.legacy.level[level];
7024
7025                 if (stencil_only)
7026                         level_info = &surf->u.legacy.stencil_level[level];
7027
7028                 z_offs += surf->u.legacy.level[level].offset;
7029                 s_offs += surf->u.legacy.stencil_level[level].offset;
7030
7031                 ds->db_depth_info = S_02803C_ADDR5_SWIZZLE_MASK(!radv_image_is_tc_compat_htile(iview->image));
7032                 ds->db_z_info = S_028040_FORMAT(format) | S_028040_ZRANGE_PRECISION(1);
7033                 ds->db_stencil_info = S_028044_FORMAT(stencil_format);
7034
7035                 if (iview->image->info.samples > 1)
7036                         ds->db_z_info |= S_028040_NUM_SAMPLES(util_logbase2(iview->image->info.samples));
7037
7038                 if (device->physical_device->rad_info.chip_class >= GFX7) {
7039                         struct radeon_info *info = &device->physical_device->rad_info;
7040                         unsigned tiling_index = surf->u.legacy.tiling_index[level];
7041                         unsigned stencil_index = surf->u.legacy.stencil_tiling_index[level];
7042                         unsigned macro_index = surf->u.legacy.macro_tile_index;
7043                         unsigned tile_mode = info->si_tile_mode_array[tiling_index];
7044                         unsigned stencil_tile_mode = info->si_tile_mode_array[stencil_index];
7045                         unsigned macro_mode = info->cik_macrotile_mode_array[macro_index];
7046
7047                         if (stencil_only)
7048                                 tile_mode = stencil_tile_mode;
7049
7050                         ds->db_depth_info |=
7051                                 S_02803C_ARRAY_MODE(G_009910_ARRAY_MODE(tile_mode)) |
7052                                 S_02803C_PIPE_CONFIG(G_009910_PIPE_CONFIG(tile_mode)) |
7053                                 S_02803C_BANK_WIDTH(G_009990_BANK_WIDTH(macro_mode)) |
7054                                 S_02803C_BANK_HEIGHT(G_009990_BANK_HEIGHT(macro_mode)) |
7055                                 S_02803C_MACRO_TILE_ASPECT(G_009990_MACRO_TILE_ASPECT(macro_mode)) |
7056                                 S_02803C_NUM_BANKS(G_009990_NUM_BANKS(macro_mode));
7057                         ds->db_z_info |= S_028040_TILE_SPLIT(G_009910_TILE_SPLIT(tile_mode));
7058                         ds->db_stencil_info |= S_028044_TILE_SPLIT(G_009910_TILE_SPLIT(stencil_tile_mode));
7059                 } else {
7060                         unsigned tile_mode_index = si_tile_mode_index(&iview->image->planes[0], level, false);
7061                         ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
7062                         tile_mode_index = si_tile_mode_index(&iview->image->planes[0], level, true);
7063                         ds->db_stencil_info |= S_028044_TILE_MODE_INDEX(tile_mode_index);
7064                         if (stencil_only)
7065                                 ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
7066                 }
7067
7068                 ds->db_depth_size = S_028058_PITCH_TILE_MAX((level_info->nblk_x / 8) - 1) |
7069                         S_028058_HEIGHT_TILE_MAX((level_info->nblk_y / 8) - 1);
7070                 ds->db_depth_slice = S_02805C_SLICE_TILE_MAX((level_info->nblk_x * level_info->nblk_y) / 64 - 1);
7071
7072                 if (radv_htile_enabled(iview->image, level)) {
7073                         ds->db_z_info |= S_028040_TILE_SURFACE_ENABLE(1);
7074
7075                         if (!surf->has_stencil &&
7076                             !radv_image_is_tc_compat_htile(iview->image))
7077                                 /* Use all of the htile_buffer for depth if there's no stencil. */
7078                                 ds->db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
7079
7080                         va = radv_buffer_get_va(iview->bo) + iview->image->offset +
7081                                 surf->htile_offset;
7082                         ds->db_htile_data_base = va >> 8;
7083                         ds->db_htile_surface = S_028ABC_FULL_CACHE(1);
7084
7085                         if (radv_image_is_tc_compat_htile(iview->image)) {
7086                                 unsigned max_zplanes =
7087                                         radv_calc_decompress_on_z_planes(device, iview);
7088
7089                                 ds->db_htile_surface |= S_028ABC_TC_COMPATIBLE(1);
7090                                 ds->db_z_info |= S_028040_DECOMPRESS_ON_N_ZPLANES(max_zplanes);
7091                         }
7092                 }
7093         }
7094
7095         ds->db_z_read_base = ds->db_z_write_base = z_offs >> 8;
7096         ds->db_stencil_read_base = ds->db_stencil_write_base = s_offs >> 8;
7097 }
7098
7099 VkResult radv_CreateFramebuffer(
7100         VkDevice                                    _device,
7101         const VkFramebufferCreateInfo*              pCreateInfo,
7102         const VkAllocationCallbacks*                pAllocator,
7103         VkFramebuffer*                              pFramebuffer)
7104 {
7105         RADV_FROM_HANDLE(radv_device, device, _device);
7106         struct radv_framebuffer *framebuffer;
7107         const VkFramebufferAttachmentsCreateInfo *imageless_create_info =
7108                 vk_find_struct_const(pCreateInfo->pNext,
7109                         FRAMEBUFFER_ATTACHMENTS_CREATE_INFO);
7110
7111         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO);
7112
7113         size_t size = sizeof(*framebuffer);
7114         if (!imageless_create_info)
7115                 size += sizeof(struct radv_image_view*) * pCreateInfo->attachmentCount;
7116         framebuffer = vk_alloc2(&device->vk.alloc, pAllocator, size, 8,
7117                                   VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
7118         if (framebuffer == NULL)
7119                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
7120
7121         vk_object_base_init(&device->vk, &framebuffer->base,
7122                             VK_OBJECT_TYPE_FRAMEBUFFER);
7123
7124         framebuffer->attachment_count = pCreateInfo->attachmentCount;
7125         framebuffer->width = pCreateInfo->width;
7126         framebuffer->height = pCreateInfo->height;
7127         framebuffer->layers = pCreateInfo->layers;
7128         if (imageless_create_info) {
7129                 for (unsigned i = 0; i < imageless_create_info->attachmentImageInfoCount; ++i) {
7130                         const VkFramebufferAttachmentImageInfo *attachment =
7131                                 imageless_create_info->pAttachmentImageInfos + i;
7132                         framebuffer->width = MIN2(framebuffer->width, attachment->width);
7133                         framebuffer->height = MIN2(framebuffer->height, attachment->height);
7134                         framebuffer->layers = MIN2(framebuffer->layers, attachment->layerCount);
7135                 }
7136         } else {
7137                 for (uint32_t i = 0; i < pCreateInfo->attachmentCount; i++) {
7138                         VkImageView _iview = pCreateInfo->pAttachments[i];
7139                         struct radv_image_view *iview = radv_image_view_from_handle(_iview);
7140                         framebuffer->attachments[i] = iview;
7141                         framebuffer->width = MIN2(framebuffer->width, iview->extent.width);
7142                         framebuffer->height = MIN2(framebuffer->height, iview->extent.height);
7143                         framebuffer->layers = MIN2(framebuffer->layers, radv_surface_max_layer_count(iview));
7144                 }
7145         }
7146
7147         *pFramebuffer = radv_framebuffer_to_handle(framebuffer);
7148         return VK_SUCCESS;
7149 }
7150
7151 void radv_DestroyFramebuffer(
7152         VkDevice                                    _device,
7153         VkFramebuffer                               _fb,
7154         const VkAllocationCallbacks*                pAllocator)
7155 {
7156         RADV_FROM_HANDLE(radv_device, device, _device);
7157         RADV_FROM_HANDLE(radv_framebuffer, fb, _fb);
7158
7159         if (!fb)
7160                 return;
7161         vk_object_base_finish(&fb->base);
7162         vk_free2(&device->vk.alloc, pAllocator, fb);
7163 }
7164
7165 static unsigned radv_tex_wrap(VkSamplerAddressMode address_mode)
7166 {
7167         switch (address_mode) {
7168         case VK_SAMPLER_ADDRESS_MODE_REPEAT:
7169                 return V_008F30_SQ_TEX_WRAP;
7170         case VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT:
7171                 return V_008F30_SQ_TEX_MIRROR;
7172         case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE:
7173                 return V_008F30_SQ_TEX_CLAMP_LAST_TEXEL;
7174         case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER:
7175                 return V_008F30_SQ_TEX_CLAMP_BORDER;
7176         case VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE:
7177                 return V_008F30_SQ_TEX_MIRROR_ONCE_LAST_TEXEL;
7178         default:
7179                 unreachable("illegal tex wrap mode");
7180                 break;
7181         }
7182 }
7183
7184 static unsigned
7185 radv_tex_compare(VkCompareOp op)
7186 {
7187         switch (op) {
7188         case VK_COMPARE_OP_NEVER:
7189                 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
7190         case VK_COMPARE_OP_LESS:
7191                 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESS;
7192         case VK_COMPARE_OP_EQUAL:
7193                 return V_008F30_SQ_TEX_DEPTH_COMPARE_EQUAL;
7194         case VK_COMPARE_OP_LESS_OR_EQUAL:
7195                 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESSEQUAL;
7196         case VK_COMPARE_OP_GREATER:
7197                 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATER;
7198         case VK_COMPARE_OP_NOT_EQUAL:
7199                 return V_008F30_SQ_TEX_DEPTH_COMPARE_NOTEQUAL;
7200         case VK_COMPARE_OP_GREATER_OR_EQUAL:
7201                 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATEREQUAL;
7202         case VK_COMPARE_OP_ALWAYS:
7203                 return V_008F30_SQ_TEX_DEPTH_COMPARE_ALWAYS;
7204         default:
7205                 unreachable("illegal compare mode");
7206                 break;
7207         }
7208 }
7209
7210 static unsigned
7211 radv_tex_filter(VkFilter filter, unsigned max_ansio)
7212 {
7213         switch (filter) {
7214         case VK_FILTER_NEAREST:
7215                 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_POINT :
7216                         V_008F38_SQ_TEX_XY_FILTER_POINT);
7217         case VK_FILTER_LINEAR:
7218                 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_BILINEAR :
7219                         V_008F38_SQ_TEX_XY_FILTER_BILINEAR);
7220         case VK_FILTER_CUBIC_IMG:
7221         default:
7222                 fprintf(stderr, "illegal texture filter");
7223                 return 0;
7224         }
7225 }
7226
7227 static unsigned
7228 radv_tex_mipfilter(VkSamplerMipmapMode mode)
7229 {
7230         switch (mode) {
7231         case VK_SAMPLER_MIPMAP_MODE_NEAREST:
7232                 return V_008F38_SQ_TEX_Z_FILTER_POINT;
7233         case VK_SAMPLER_MIPMAP_MODE_LINEAR:
7234                 return V_008F38_SQ_TEX_Z_FILTER_LINEAR;
7235         default:
7236                 return V_008F38_SQ_TEX_Z_FILTER_NONE;
7237         }
7238 }
7239
7240 static unsigned
7241 radv_tex_bordercolor(VkBorderColor bcolor)
7242 {
7243         switch (bcolor) {
7244         case VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK:
7245         case VK_BORDER_COLOR_INT_TRANSPARENT_BLACK:
7246                 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
7247         case VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK:
7248         case VK_BORDER_COLOR_INT_OPAQUE_BLACK:
7249                 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_BLACK;
7250         case VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE:
7251         case VK_BORDER_COLOR_INT_OPAQUE_WHITE:
7252                 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE;
7253         case VK_BORDER_COLOR_FLOAT_CUSTOM_EXT:
7254         case VK_BORDER_COLOR_INT_CUSTOM_EXT:
7255                 return V_008F3C_SQ_TEX_BORDER_COLOR_REGISTER;
7256         default:
7257                 break;
7258         }
7259         return 0;
7260 }
7261
7262 static unsigned
7263 radv_tex_aniso_filter(unsigned filter)
7264 {
7265         if (filter < 2)
7266                 return 0;
7267         if (filter < 4)
7268                 return 1;
7269         if (filter < 8)
7270                 return 2;
7271         if (filter < 16)
7272                 return 3;
7273         return 4;
7274 }
7275
7276 static unsigned
7277 radv_tex_filter_mode(VkSamplerReductionMode mode)
7278 {
7279         switch (mode) {
7280         case VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT:
7281                 return V_008F30_SQ_IMG_FILTER_MODE_BLEND;
7282         case VK_SAMPLER_REDUCTION_MODE_MIN_EXT:
7283                 return V_008F30_SQ_IMG_FILTER_MODE_MIN;
7284         case VK_SAMPLER_REDUCTION_MODE_MAX_EXT:
7285                 return V_008F30_SQ_IMG_FILTER_MODE_MAX;
7286         default:
7287                 break;
7288         }
7289         return 0;
7290 }
7291
7292 static uint32_t
7293 radv_get_max_anisotropy(struct radv_device *device,
7294                         const VkSamplerCreateInfo *pCreateInfo)
7295 {
7296         if (device->force_aniso >= 0)
7297                 return device->force_aniso;
7298
7299         if (pCreateInfo->anisotropyEnable &&
7300             pCreateInfo->maxAnisotropy > 1.0f)
7301                 return (uint32_t)pCreateInfo->maxAnisotropy;
7302
7303         return 0;
7304 }
7305
7306 static inline int S_FIXED(float value, unsigned frac_bits)
7307 {
7308         return value * (1 << frac_bits);
7309 }
7310
7311 static uint32_t radv_register_border_color(struct radv_device *device,
7312                                            VkClearColorValue   value)
7313 {
7314         uint32_t slot;
7315
7316         pthread_mutex_lock(&device->border_color_data.mutex);
7317
7318         for (slot = 0; slot < RADV_BORDER_COLOR_COUNT; slot++) {
7319                 if (!device->border_color_data.used[slot]) {
7320                         /* Copy to the GPU wrt endian-ness. */
7321                         util_memcpy_cpu_to_le32(&device->border_color_data.colors_gpu_ptr[slot],
7322                                                 &value,
7323                                                 sizeof(VkClearColorValue));
7324
7325                         device->border_color_data.used[slot] = true;
7326                         break;
7327                 }
7328         }
7329
7330         pthread_mutex_unlock(&device->border_color_data.mutex);
7331
7332         return slot;
7333 }
7334
7335 static void radv_unregister_border_color(struct radv_device *device,
7336                                          uint32_t            slot)
7337 {
7338         pthread_mutex_lock(&device->border_color_data.mutex);
7339
7340         device->border_color_data.used[slot] = false;
7341
7342         pthread_mutex_unlock(&device->border_color_data.mutex);
7343 }
7344
7345 static void
7346 radv_init_sampler(struct radv_device *device,
7347                   struct radv_sampler *sampler,
7348                   const VkSamplerCreateInfo *pCreateInfo)
7349 {
7350         uint32_t max_aniso = radv_get_max_anisotropy(device, pCreateInfo);
7351         uint32_t max_aniso_ratio = radv_tex_aniso_filter(max_aniso);
7352         bool compat_mode = device->physical_device->rad_info.chip_class == GFX8 ||
7353                            device->physical_device->rad_info.chip_class == GFX9;
7354         unsigned filter_mode = V_008F30_SQ_IMG_FILTER_MODE_BLEND;
7355         unsigned depth_compare_func = V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
7356         bool trunc_coord = pCreateInfo->minFilter == VK_FILTER_NEAREST && pCreateInfo->magFilter == VK_FILTER_NEAREST;
7357         bool uses_border_color = pCreateInfo->addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER ||
7358                                  pCreateInfo->addressModeV == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER ||
7359                                  pCreateInfo->addressModeW == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
7360         VkBorderColor border_color = uses_border_color ? pCreateInfo->borderColor : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
7361         uint32_t border_color_ptr;
7362
7363         const struct VkSamplerReductionModeCreateInfo *sampler_reduction =
7364                 vk_find_struct_const(pCreateInfo->pNext,
7365                                      SAMPLER_REDUCTION_MODE_CREATE_INFO);
7366         if (sampler_reduction)
7367                 filter_mode = radv_tex_filter_mode(sampler_reduction->reductionMode);
7368
7369         if (pCreateInfo->compareEnable)
7370                 depth_compare_func = radv_tex_compare(pCreateInfo->compareOp);
7371
7372         sampler->border_color_slot = RADV_BORDER_COLOR_COUNT;
7373
7374         if (border_color == VK_BORDER_COLOR_FLOAT_CUSTOM_EXT || border_color == VK_BORDER_COLOR_INT_CUSTOM_EXT) {
7375                 const VkSamplerCustomBorderColorCreateInfoEXT *custom_border_color =
7376                         vk_find_struct_const(pCreateInfo->pNext,
7377                                              SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT);
7378
7379                 assert(custom_border_color);
7380
7381                 sampler->border_color_slot =
7382                         radv_register_border_color(device, custom_border_color->customBorderColor);
7383
7384                 /* Did we fail to find a slot? */
7385                 if (sampler->border_color_slot == RADV_BORDER_COLOR_COUNT) {
7386                         fprintf(stderr, "WARNING: no free border color slots, defaulting to TRANS_BLACK.\n");
7387                         border_color = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
7388                 }
7389         }
7390
7391         /* If we don't have a custom color, set the ptr to 0 */
7392         border_color_ptr = sampler->border_color_slot != RADV_BORDER_COLOR_COUNT
7393                 ? sampler->border_color_slot
7394                 : 0;
7395
7396         sampler->state[0] = (S_008F30_CLAMP_X(radv_tex_wrap(pCreateInfo->addressModeU)) |
7397                              S_008F30_CLAMP_Y(radv_tex_wrap(pCreateInfo->addressModeV)) |
7398                              S_008F30_CLAMP_Z(radv_tex_wrap(pCreateInfo->addressModeW)) |
7399                              S_008F30_MAX_ANISO_RATIO(max_aniso_ratio) |
7400                              S_008F30_DEPTH_COMPARE_FUNC(depth_compare_func) |
7401                              S_008F30_FORCE_UNNORMALIZED(pCreateInfo->unnormalizedCoordinates ? 1 : 0) |
7402                              S_008F30_ANISO_THRESHOLD(max_aniso_ratio >> 1) |
7403                              S_008F30_ANISO_BIAS(max_aniso_ratio) |
7404                              S_008F30_DISABLE_CUBE_WRAP(0) |
7405                              S_008F30_COMPAT_MODE(compat_mode) |
7406                              S_008F30_FILTER_MODE(filter_mode) |
7407                              S_008F30_TRUNC_COORD(trunc_coord));
7408         sampler->state[1] = (S_008F34_MIN_LOD(S_FIXED(CLAMP(pCreateInfo->minLod, 0, 15), 8)) |
7409                              S_008F34_MAX_LOD(S_FIXED(CLAMP(pCreateInfo->maxLod, 0, 15), 8)) |
7410                              S_008F34_PERF_MIP(max_aniso_ratio ? max_aniso_ratio + 6 : 0));
7411         sampler->state[2] = (S_008F38_LOD_BIAS(S_FIXED(CLAMP(pCreateInfo->mipLodBias, -16, 16), 8)) |
7412                              S_008F38_XY_MAG_FILTER(radv_tex_filter(pCreateInfo->magFilter, max_aniso)) |
7413                              S_008F38_XY_MIN_FILTER(radv_tex_filter(pCreateInfo->minFilter, max_aniso)) |
7414                              S_008F38_MIP_FILTER(radv_tex_mipfilter(pCreateInfo->mipmapMode)) |
7415                              S_008F38_MIP_POINT_PRECLAMP(0));
7416         sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(border_color_ptr) |
7417                              S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(border_color)));
7418
7419         if (device->physical_device->rad_info.chip_class >= GFX10) {
7420                 sampler->state[2] |= S_008F38_ANISO_OVERRIDE_GFX10(1);
7421         } else {
7422                 sampler->state[2] |=
7423                         S_008F38_DISABLE_LSB_CEIL(device->physical_device->rad_info.chip_class <= GFX8) |
7424                         S_008F38_FILTER_PREC_FIX(1) |
7425                         S_008F38_ANISO_OVERRIDE_GFX8(device->physical_device->rad_info.chip_class >= GFX8);
7426         }
7427 }
7428
7429 VkResult radv_CreateSampler(
7430         VkDevice                                    _device,
7431         const VkSamplerCreateInfo*                  pCreateInfo,
7432         const VkAllocationCallbacks*                pAllocator,
7433         VkSampler*                                  pSampler)
7434 {
7435         RADV_FROM_HANDLE(radv_device, device, _device);
7436         struct radv_sampler *sampler;
7437
7438         const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion =
7439                 vk_find_struct_const(pCreateInfo->pNext,
7440                                      SAMPLER_YCBCR_CONVERSION_INFO);
7441
7442         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO);
7443
7444         sampler = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*sampler), 8,
7445                               VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
7446         if (!sampler)
7447                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
7448
7449         vk_object_base_init(&device->vk, &sampler->base,
7450                             VK_OBJECT_TYPE_SAMPLER);
7451
7452         radv_init_sampler(device, sampler, pCreateInfo);
7453
7454         sampler->ycbcr_sampler = ycbcr_conversion ? radv_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion): NULL;
7455         *pSampler = radv_sampler_to_handle(sampler);
7456
7457         return VK_SUCCESS;
7458 }
7459
7460 void radv_DestroySampler(
7461         VkDevice                                    _device,
7462         VkSampler                                   _sampler,
7463         const VkAllocationCallbacks*                pAllocator)
7464 {
7465         RADV_FROM_HANDLE(radv_device, device, _device);
7466         RADV_FROM_HANDLE(radv_sampler, sampler, _sampler);
7467
7468         if (!sampler)
7469                 return;
7470
7471         if (sampler->border_color_slot != RADV_BORDER_COLOR_COUNT)
7472                 radv_unregister_border_color(device, sampler->border_color_slot);
7473
7474         vk_object_base_finish(&sampler->base);
7475         vk_free2(&device->vk.alloc, pAllocator, sampler);
7476 }
7477
7478 /* vk_icd.h does not declare this function, so we declare it here to
7479  * suppress Wmissing-prototypes.
7480  */
7481 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
7482 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion);
7483
7484 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
7485 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion)
7486 {
7487         /* For the full details on loader interface versioning, see
7488         * <https://github.com/KhronosGroup/Vulkan-LoaderAndValidationLayers/blob/master/loader/LoaderAndLayerInterface.md>.
7489         * What follows is a condensed summary, to help you navigate the large and
7490         * confusing official doc.
7491         *
7492         *   - Loader interface v0 is incompatible with later versions. We don't
7493         *     support it.
7494         *
7495         *   - In loader interface v1:
7496         *       - The first ICD entrypoint called by the loader is
7497         *         vk_icdGetInstanceProcAddr(). The ICD must statically expose this
7498         *         entrypoint.
7499         *       - The ICD must statically expose no other Vulkan symbol unless it is
7500         *         linked with -Bsymbolic.
7501         *       - Each dispatchable Vulkan handle created by the ICD must be
7502         *         a pointer to a struct whose first member is VK_LOADER_DATA. The
7503         *         ICD must initialize VK_LOADER_DATA.loadMagic to ICD_LOADER_MAGIC.
7504         *       - The loader implements vkCreate{PLATFORM}SurfaceKHR() and
7505         *         vkDestroySurfaceKHR(). The ICD must be capable of working with
7506         *         such loader-managed surfaces.
7507         *
7508         *    - Loader interface v2 differs from v1 in:
7509         *       - The first ICD entrypoint called by the loader is
7510         *         vk_icdNegotiateLoaderICDInterfaceVersion(). The ICD must
7511         *         statically expose this entrypoint.
7512         *
7513         *    - Loader interface v3 differs from v2 in:
7514         *        - The ICD must implement vkCreate{PLATFORM}SurfaceKHR(),
7515         *          vkDestroySurfaceKHR(), and other API which uses VKSurfaceKHR,
7516         *          because the loader no longer does so.
7517         */
7518         *pSupportedVersion = MIN2(*pSupportedVersion, 4u);
7519         return VK_SUCCESS;
7520 }
7521
7522 VkResult radv_GetMemoryFdKHR(VkDevice _device,
7523                              const VkMemoryGetFdInfoKHR *pGetFdInfo,
7524                              int *pFD)
7525 {
7526         RADV_FROM_HANDLE(radv_device, device, _device);
7527         RADV_FROM_HANDLE(radv_device_memory, memory, pGetFdInfo->memory);
7528
7529         assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR);
7530
7531         /* At the moment, we support only the below handle types. */
7532         assert(pGetFdInfo->handleType ==
7533                VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
7534                pGetFdInfo->handleType ==
7535                VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
7536
7537         bool ret = radv_get_memory_fd(device, memory, pFD);
7538         if (ret == false)
7539                 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
7540         return VK_SUCCESS;
7541 }
7542
7543 static uint32_t radv_compute_valid_memory_types_attempt(struct radv_physical_device *dev,
7544                                                         enum radeon_bo_domain domains,
7545                                                         enum radeon_bo_flag flags,
7546                                                         enum radeon_bo_flag ignore_flags)
7547 {
7548         /* Don't count GTT/CPU as relevant:
7549          *
7550          * - We're not fully consistent between the two.
7551          * - Sometimes VRAM gets VRAM|GTT.
7552          */
7553         const enum radeon_bo_domain relevant_domains = RADEON_DOMAIN_VRAM |
7554                                                        RADEON_DOMAIN_GDS |
7555                                                        RADEON_DOMAIN_OA;
7556         uint32_t bits = 0;
7557         for (unsigned i = 0; i < dev->memory_properties.memoryTypeCount; ++i) {
7558                 if ((domains & relevant_domains) != (dev->memory_domains[i] & relevant_domains))
7559                         continue;
7560
7561                 if ((flags & ~ignore_flags) != (dev->memory_flags[i] & ~ignore_flags))
7562                         continue;
7563
7564                 bits |= 1u << i;
7565         }
7566
7567         return bits;
7568 }
7569
7570 static uint32_t radv_compute_valid_memory_types(struct radv_physical_device *dev,
7571                                                 enum radeon_bo_domain domains,
7572                                                 enum radeon_bo_flag flags)
7573 {
7574         enum radeon_bo_flag ignore_flags = ~(RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_GTT_WC);
7575         uint32_t bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags);
7576
7577         if (!bits) {
7578                 ignore_flags |= RADEON_FLAG_NO_CPU_ACCESS;
7579                 bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags);
7580         }
7581
7582         return bits;
7583 }
7584 VkResult radv_GetMemoryFdPropertiesKHR(VkDevice _device,
7585                                        VkExternalMemoryHandleTypeFlagBits handleType,
7586                                        int fd,
7587                                        VkMemoryFdPropertiesKHR *pMemoryFdProperties)
7588 {
7589         RADV_FROM_HANDLE(radv_device, device, _device);
7590
7591         switch (handleType) {
7592         case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: {
7593                 enum radeon_bo_domain domains;
7594                 enum radeon_bo_flag flags;
7595                 if (!device->ws->buffer_get_flags_from_fd(device->ws, fd, &domains, &flags))
7596                         return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7597
7598                 pMemoryFdProperties->memoryTypeBits = radv_compute_valid_memory_types(device->physical_device, domains, flags);
7599                 return VK_SUCCESS;
7600         }
7601         default:
7602                 /* The valid usage section for this function says:
7603                  *
7604                  *    "handleType must not be one of the handle types defined as
7605                  *    opaque."
7606                  *
7607                  * So opaque handle types fall into the default "unsupported" case.
7608                  */
7609                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7610         }
7611 }
7612
7613 static VkResult radv_import_opaque_fd(struct radv_device *device,
7614                                       int fd,
7615                                       uint32_t *syncobj)
7616 {
7617         uint32_t syncobj_handle = 0;
7618         int ret = device->ws->import_syncobj(device->ws, fd, &syncobj_handle);
7619         if (ret != 0)
7620                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7621
7622         if (*syncobj)
7623                 device->ws->destroy_syncobj(device->ws, *syncobj);
7624
7625         *syncobj = syncobj_handle;
7626         close(fd);
7627
7628         return VK_SUCCESS;
7629 }
7630
7631 static VkResult radv_import_sync_fd(struct radv_device *device,
7632                                     int fd,
7633                                     uint32_t *syncobj)
7634 {
7635         /* If we create a syncobj we do it locally so that if we have an error, we don't
7636          * leave a syncobj in an undetermined state in the fence. */
7637         uint32_t syncobj_handle =  *syncobj;
7638         if (!syncobj_handle) {
7639                 bool create_signaled = fd == -1 ? true : false;
7640
7641                 int ret = device->ws->create_syncobj(device->ws, create_signaled,
7642                                                      &syncobj_handle);
7643                 if (ret) {
7644                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
7645                 }
7646         } else {
7647                 if (fd == -1)
7648                         device->ws->signal_syncobj(device->ws, syncobj_handle, 0);
7649         }
7650
7651         if (fd != -1) {
7652                 int ret = device->ws->import_syncobj_from_sync_file(device->ws, syncobj_handle, fd);
7653                 if (ret)
7654                         return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7655                 close(fd);
7656         }
7657
7658         *syncobj = syncobj_handle;
7659
7660         return VK_SUCCESS;
7661 }
7662
7663 VkResult radv_ImportSemaphoreFdKHR(VkDevice _device,
7664                                    const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
7665 {
7666         RADV_FROM_HANDLE(radv_device, device, _device);
7667         RADV_FROM_HANDLE(radv_semaphore, sem, pImportSemaphoreFdInfo->semaphore);
7668         VkResult result;
7669         struct radv_semaphore_part *dst = NULL;
7670         bool timeline = sem->permanent.kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ;
7671
7672         if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) {
7673                 assert(!timeline);
7674                 dst = &sem->temporary;
7675         } else {
7676                 dst = &sem->permanent;
7677         }
7678
7679         uint32_t syncobj = (dst->kind == RADV_SEMAPHORE_SYNCOBJ ||
7680                             dst->kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ) ? dst->syncobj : 0;
7681
7682         switch(pImportSemaphoreFdInfo->handleType) {
7683                 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
7684                         result = radv_import_opaque_fd(device, pImportSemaphoreFdInfo->fd, &syncobj);
7685                         break;
7686                 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
7687                         assert(!timeline);
7688                         result = radv_import_sync_fd(device, pImportSemaphoreFdInfo->fd, &syncobj);
7689                         break;
7690                 default:
7691                         unreachable("Unhandled semaphore handle type");
7692         }
7693
7694         if (result == VK_SUCCESS) {
7695                 dst->syncobj = syncobj;
7696                 dst->kind = RADV_SEMAPHORE_SYNCOBJ;
7697                 if (timeline) {
7698                         dst->kind = RADV_SEMAPHORE_TIMELINE_SYNCOBJ;
7699                         dst->timeline_syncobj.max_point = 0;
7700                 }
7701         }
7702
7703         return result;
7704 }
7705
7706 VkResult radv_GetSemaphoreFdKHR(VkDevice _device,
7707                                 const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
7708                                 int *pFd)
7709 {
7710         RADV_FROM_HANDLE(radv_device, device, _device);
7711         RADV_FROM_HANDLE(radv_semaphore, sem, pGetFdInfo->semaphore);
7712         int ret;
7713         uint32_t syncobj_handle;
7714
7715         if (sem->temporary.kind != RADV_SEMAPHORE_NONE) {
7716                 assert(sem->temporary.kind == RADV_SEMAPHORE_SYNCOBJ ||
7717                        sem->temporary.kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ);
7718                 syncobj_handle = sem->temporary.syncobj;
7719         } else {
7720                 assert(sem->permanent.kind == RADV_SEMAPHORE_SYNCOBJ ||
7721                        sem->permanent.kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ);
7722                 syncobj_handle = sem->permanent.syncobj;
7723         }
7724
7725         switch(pGetFdInfo->handleType) {
7726         case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
7727                 ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd);
7728                 if (ret)
7729                         return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS);
7730                 break;
7731         case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
7732                 ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd);
7733                 if (ret)
7734                         return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS);
7735
7736                 if (sem->temporary.kind != RADV_SEMAPHORE_NONE) {
7737                         radv_destroy_semaphore_part(device, &sem->temporary);
7738                 } else {
7739                         device->ws->reset_syncobj(device->ws, syncobj_handle);
7740                 }
7741                 break;
7742         default:
7743                 unreachable("Unhandled semaphore handle type");
7744         }
7745
7746         return VK_SUCCESS;
7747 }
7748
7749 void radv_GetPhysicalDeviceExternalSemaphoreProperties(
7750         VkPhysicalDevice                            physicalDevice,
7751         const VkPhysicalDeviceExternalSemaphoreInfo *pExternalSemaphoreInfo,
7752         VkExternalSemaphoreProperties               *pExternalSemaphoreProperties)
7753 {
7754         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
7755         VkSemaphoreTypeKHR type = radv_get_semaphore_type(pExternalSemaphoreInfo->pNext, NULL);
7756
7757         if (type == VK_SEMAPHORE_TYPE_TIMELINE && pdevice->rad_info.has_timeline_syncobj &&
7758             pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) {
7759                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7760                 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7761                 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7762                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7763         } else if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
7764                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0;
7765                 pExternalSemaphoreProperties->compatibleHandleTypes = 0;
7766                 pExternalSemaphoreProperties->externalSemaphoreFeatures = 0;
7767
7768         /* Require has_syncobj_wait_for_submit for the syncobj signal ioctl introduced at virtually the same time */
7769         } else if (pdevice->rad_info.has_syncobj_wait_for_submit &&
7770                    (pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT ||
7771                     pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT)) {
7772                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
7773                 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
7774                 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7775                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7776         } else if (pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) {
7777                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7778                 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7779                 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7780                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7781         } else {
7782                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0;
7783                 pExternalSemaphoreProperties->compatibleHandleTypes = 0;
7784                 pExternalSemaphoreProperties->externalSemaphoreFeatures = 0;
7785         }
7786 }
7787
7788 VkResult radv_ImportFenceFdKHR(VkDevice _device,
7789                                    const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
7790 {
7791         RADV_FROM_HANDLE(radv_device, device, _device);
7792         RADV_FROM_HANDLE(radv_fence, fence, pImportFenceFdInfo->fence);
7793         struct radv_fence_part *dst = NULL;
7794         VkResult result;
7795
7796         if (pImportFenceFdInfo->flags & VK_FENCE_IMPORT_TEMPORARY_BIT) {
7797                 dst = &fence->temporary;
7798         } else {
7799                 dst = &fence->permanent;
7800         }
7801
7802         uint32_t syncobj = dst->kind == RADV_FENCE_SYNCOBJ ? dst->syncobj : 0;
7803
7804         switch(pImportFenceFdInfo->handleType) {
7805                 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT:
7806                         result = radv_import_opaque_fd(device, pImportFenceFdInfo->fd, &syncobj);
7807                         break;
7808                 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT:
7809                         result = radv_import_sync_fd(device, pImportFenceFdInfo->fd, &syncobj);
7810                         break;
7811                 default:
7812                         unreachable("Unhandled fence handle type");
7813         }
7814
7815         if (result == VK_SUCCESS) {
7816                 dst->syncobj = syncobj;
7817                 dst->kind = RADV_FENCE_SYNCOBJ;
7818         }
7819
7820         return result;
7821 }
7822
7823 VkResult radv_GetFenceFdKHR(VkDevice _device,
7824                                 const VkFenceGetFdInfoKHR *pGetFdInfo,
7825                                 int *pFd)
7826 {
7827         RADV_FROM_HANDLE(radv_device, device, _device);
7828         RADV_FROM_HANDLE(radv_fence, fence, pGetFdInfo->fence);
7829         int ret;
7830
7831         struct radv_fence_part *part =
7832                 fence->temporary.kind != RADV_FENCE_NONE ?
7833                 &fence->temporary : &fence->permanent;
7834
7835         switch(pGetFdInfo->handleType) {
7836         case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT:
7837                 ret = device->ws->export_syncobj(device->ws, part->syncobj, pFd);
7838                 if (ret)
7839                         return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS);
7840                 break;
7841         case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT:
7842                 ret = device->ws->export_syncobj_to_sync_file(device->ws,
7843                                                               part->syncobj, pFd);
7844                 if (ret)
7845                         return vk_error(device->instance, VK_ERROR_TOO_MANY_OBJECTS);
7846
7847                 if (part == &fence->temporary) {
7848                         radv_destroy_fence_part(device, part);
7849                 } else {
7850                         device->ws->reset_syncobj(device->ws, part->syncobj);
7851                 }
7852                 break;
7853         default:
7854                 unreachable("Unhandled fence handle type");
7855         }
7856
7857         return VK_SUCCESS;
7858 }
7859
7860 void radv_GetPhysicalDeviceExternalFenceProperties(
7861         VkPhysicalDevice                            physicalDevice,
7862         const VkPhysicalDeviceExternalFenceInfo *pExternalFenceInfo,
7863         VkExternalFenceProperties               *pExternalFenceProperties)
7864 {
7865         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
7866
7867         if (pdevice->rad_info.has_syncobj_wait_for_submit &&
7868             (pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT ||
7869              pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT)) {
7870                 pExternalFenceProperties->exportFromImportedHandleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
7871                 pExternalFenceProperties->compatibleHandleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
7872                 pExternalFenceProperties->externalFenceFeatures = VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT |
7873                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7874         } else {
7875                 pExternalFenceProperties->exportFromImportedHandleTypes = 0;
7876                 pExternalFenceProperties->compatibleHandleTypes = 0;
7877                 pExternalFenceProperties->externalFenceFeatures = 0;
7878         }
7879 }
7880
7881 VkResult
7882 radv_CreateDebugReportCallbackEXT(VkInstance _instance,
7883                                  const VkDebugReportCallbackCreateInfoEXT* pCreateInfo,
7884                                  const VkAllocationCallbacks* pAllocator,
7885                                  VkDebugReportCallbackEXT* pCallback)
7886 {
7887         RADV_FROM_HANDLE(radv_instance, instance, _instance);
7888         return vk_create_debug_report_callback(&instance->debug_report_callbacks,
7889                                                pCreateInfo, pAllocator, &instance->alloc,
7890                                                pCallback);
7891 }
7892
7893 void
7894 radv_DestroyDebugReportCallbackEXT(VkInstance _instance,
7895                                   VkDebugReportCallbackEXT _callback,
7896                                   const VkAllocationCallbacks* pAllocator)
7897 {
7898         RADV_FROM_HANDLE(radv_instance, instance, _instance);
7899         vk_destroy_debug_report_callback(&instance->debug_report_callbacks,
7900                                          _callback, pAllocator, &instance->alloc);
7901 }
7902
7903 void
7904 radv_DebugReportMessageEXT(VkInstance _instance,
7905                           VkDebugReportFlagsEXT flags,
7906                           VkDebugReportObjectTypeEXT objectType,
7907                           uint64_t object,
7908                           size_t location,
7909                           int32_t messageCode,
7910                           const char* pLayerPrefix,
7911                           const char* pMessage)
7912 {
7913         RADV_FROM_HANDLE(radv_instance, instance, _instance);
7914         vk_debug_report(&instance->debug_report_callbacks, flags, objectType,
7915                         object, location, messageCode, pLayerPrefix, pMessage);
7916 }
7917
7918 void
7919 radv_GetDeviceGroupPeerMemoryFeatures(
7920     VkDevice                                    device,
7921     uint32_t                                    heapIndex,
7922     uint32_t                                    localDeviceIndex,
7923     uint32_t                                    remoteDeviceIndex,
7924     VkPeerMemoryFeatureFlags*                   pPeerMemoryFeatures)
7925 {
7926         assert(localDeviceIndex == remoteDeviceIndex);
7927
7928         *pPeerMemoryFeatures = VK_PEER_MEMORY_FEATURE_COPY_SRC_BIT |
7929                                VK_PEER_MEMORY_FEATURE_COPY_DST_BIT |
7930                                VK_PEER_MEMORY_FEATURE_GENERIC_SRC_BIT |
7931                                VK_PEER_MEMORY_FEATURE_GENERIC_DST_BIT;
7932 }
7933
7934 static const VkTimeDomainEXT radv_time_domains[] = {
7935         VK_TIME_DOMAIN_DEVICE_EXT,
7936         VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT,
7937         VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT,
7938 };
7939
7940 VkResult radv_GetPhysicalDeviceCalibrateableTimeDomainsEXT(
7941         VkPhysicalDevice                             physicalDevice,
7942         uint32_t                                     *pTimeDomainCount,
7943         VkTimeDomainEXT                              *pTimeDomains)
7944 {
7945         int d;
7946         VK_OUTARRAY_MAKE(out, pTimeDomains, pTimeDomainCount);
7947
7948         for (d = 0; d < ARRAY_SIZE(radv_time_domains); d++) {
7949                 vk_outarray_append(&out, i) {
7950                         *i = radv_time_domains[d];
7951                 }
7952         }
7953
7954         return vk_outarray_status(&out);
7955 }
7956
7957 static uint64_t
7958 radv_clock_gettime(clockid_t clock_id)
7959 {
7960         struct timespec current;
7961         int ret;
7962
7963         ret = clock_gettime(clock_id, &current);
7964         if (ret < 0 && clock_id == CLOCK_MONOTONIC_RAW)
7965                 ret = clock_gettime(CLOCK_MONOTONIC, &current);
7966         if (ret < 0)
7967                 return 0;
7968
7969         return (uint64_t) current.tv_sec * 1000000000ULL + current.tv_nsec;
7970 }
7971
7972 VkResult radv_GetCalibratedTimestampsEXT(
7973         VkDevice                                     _device,
7974         uint32_t                                     timestampCount,
7975         const VkCalibratedTimestampInfoEXT           *pTimestampInfos,
7976         uint64_t                                     *pTimestamps,
7977         uint64_t                                     *pMaxDeviation)
7978 {
7979         RADV_FROM_HANDLE(radv_device, device, _device);
7980         uint32_t clock_crystal_freq = device->physical_device->rad_info.clock_crystal_freq;
7981         int d;
7982         uint64_t begin, end;
7983         uint64_t max_clock_period = 0;
7984
7985         begin = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
7986
7987         for (d = 0; d < timestampCount; d++) {
7988                 switch (pTimestampInfos[d].timeDomain) {
7989                 case VK_TIME_DOMAIN_DEVICE_EXT:
7990                         pTimestamps[d] = device->ws->query_value(device->ws,
7991                                                                  RADEON_TIMESTAMP);
7992                         uint64_t device_period = DIV_ROUND_UP(1000000, clock_crystal_freq);
7993                         max_clock_period = MAX2(max_clock_period, device_period);
7994                         break;
7995                 case VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT:
7996                         pTimestamps[d] = radv_clock_gettime(CLOCK_MONOTONIC);
7997                         max_clock_period = MAX2(max_clock_period, 1);
7998                         break;
7999
8000                 case VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT:
8001                         pTimestamps[d] = begin;
8002                         break;
8003                 default:
8004                         pTimestamps[d] = 0;
8005                         break;
8006                 }
8007         }
8008
8009         end = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
8010
8011         /*
8012          * The maximum deviation is the sum of the interval over which we
8013          * perform the sampling and the maximum period of any sampled
8014          * clock. That's because the maximum skew between any two sampled
8015          * clock edges is when the sampled clock with the largest period is
8016          * sampled at the end of that period but right at the beginning of the
8017          * sampling interval and some other clock is sampled right at the
8018          * begining of its sampling period and right at the end of the
8019          * sampling interval. Let's assume the GPU has the longest clock
8020          * period and that the application is sampling GPU and monotonic:
8021          *
8022          *                               s                 e
8023          *                       w x y z 0 1 2 3 4 5 6 7 8 9 a b c d e f
8024          *      Raw              -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-
8025          *
8026          *                               g
8027          *                0         1         2         3
8028          *      GPU       -----_____-----_____-----_____-----_____
8029          *
8030          *                                                m
8031          *                                          x y z 0 1 2 3 4 5 6 7 8 9 a b c
8032          *      Monotonic                           -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-
8033          *
8034          *      Interval                     <----------------->
8035          *      Deviation           <-------------------------->
8036          *
8037          *              s  = read(raw)       2
8038          *              g  = read(GPU)       1
8039          *              m  = read(monotonic) 2
8040          *              e  = read(raw)       b
8041          *
8042          * We round the sample interval up by one tick to cover sampling error
8043          * in the interval clock
8044          */
8045
8046         uint64_t sample_interval = end - begin + 1;
8047
8048         *pMaxDeviation = sample_interval + max_clock_period;
8049
8050         return VK_SUCCESS;
8051 }
8052
8053 void radv_GetPhysicalDeviceMultisamplePropertiesEXT(
8054     VkPhysicalDevice                            physicalDevice,
8055     VkSampleCountFlagBits                       samples,
8056     VkMultisamplePropertiesEXT*                 pMultisampleProperties)
8057 {
8058         if (samples & (VK_SAMPLE_COUNT_2_BIT |
8059                        VK_SAMPLE_COUNT_4_BIT |
8060                        VK_SAMPLE_COUNT_8_BIT)) {
8061                 pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 2, 2 };
8062         } else {
8063                 pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 0, 0 };
8064         }
8065 }
8066
8067 VkResult radv_CreatePrivateDataSlotEXT(
8068     VkDevice                                    _device,
8069     const VkPrivateDataSlotCreateInfoEXT*       pCreateInfo,
8070     const VkAllocationCallbacks*                pAllocator,
8071     VkPrivateDataSlotEXT*                       pPrivateDataSlot)
8072 {
8073         RADV_FROM_HANDLE(radv_device, device, _device);
8074         return vk_private_data_slot_create(&device->vk, pCreateInfo, pAllocator,
8075                                            pPrivateDataSlot);
8076 }
8077
8078 void radv_DestroyPrivateDataSlotEXT(
8079     VkDevice                                    _device,
8080     VkPrivateDataSlotEXT                        privateDataSlot,
8081     const VkAllocationCallbacks*                pAllocator)
8082 {
8083         RADV_FROM_HANDLE(radv_device, device, _device);
8084         vk_private_data_slot_destroy(&device->vk, privateDataSlot, pAllocator);
8085 }
8086
8087 VkResult radv_SetPrivateDataEXT(
8088     VkDevice                                    _device,
8089     VkObjectType                                objectType,
8090     uint64_t                                    objectHandle,
8091     VkPrivateDataSlotEXT                        privateDataSlot,
8092     uint64_t                                    data)
8093 {
8094         RADV_FROM_HANDLE(radv_device, device, _device);
8095         return vk_object_base_set_private_data(&device->vk, objectType,
8096                                                objectHandle, privateDataSlot,
8097                                                data);
8098 }
8099
8100 void radv_GetPrivateDataEXT(
8101     VkDevice                                    _device,
8102     VkObjectType                                objectType,
8103     uint64_t                                    objectHandle,
8104     VkPrivateDataSlotEXT                        privateDataSlot,
8105     uint64_t*                                   pData)
8106 {
8107         RADV_FROM_HANDLE(radv_device, device, _device);
8108         vk_object_base_get_private_data(&device->vk, objectType, objectHandle,
8109                                         privateDataSlot, pData);
8110 }