src/amd/vulkan/radv_device.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  25  * IN THE SOFTWARE.
  26  */
  27
  28 #include "dirent.h"
  29 #include <errno.h>
  30 #include <fcntl.h>
  31 #include <linux/audit.h>
  32 #include <linux/bpf.h>
  33 #include <linux/filter.h>
  34 #include <linux/seccomp.h>
  35 #include <linux/unistd.h>
  36 #include <stdbool.h>
  37 #include <stddef.h>
  38 #include <stdio.h>
  39 #include <string.h>
  40 #include <sys/prctl.h>
  41 #include <sys/wait.h>
  42 #include <unistd.h>
  43 #include <fcntl.h>
  44
  45 #include "radv_debug.h"
  46 #include "radv_private.h"
  47 #include "radv_shader.h"
  48 #include "radv_cs.h"
  49 #include "util/disk_cache.h"
  50 #include "vk_util.h"
  51 #include <xf86drm.h>
  52 #include <amdgpu.h>
  53 #include "drm-uapi/amdgpu_drm.h"
  54 #include "winsys/amdgpu/radv_amdgpu_winsys_public.h"
  55 #include "winsys/null/radv_null_winsys_public.h"
  56 #include "ac_llvm_util.h"
  57 #include "vk_format.h"
  58 #include "sid.h"
  59 #include "git_sha1.h"
  60 #include "util/build_id.h"
  61 #include "util/debug.h"
  62 #include "util/mesa-sha1.h"
  63 #include "util/timespec.h"
  64 #include "util/u_atomic.h"
  65 #include "compiler/glsl_types.h"
  66 #include "util/xmlpool.h"
  67
  68 static struct radv_timeline_point *
  69 radv_timeline_find_point_at_least_locked(struct radv_device *device,
  70                                          struct radv_timeline *timeline,
  71                                          uint64_t p);
  72
  73 static struct radv_timeline_point *
  74 radv_timeline_add_point_locked(struct radv_device *device,
  75                                struct radv_timeline *timeline,
  76                                uint64_t p);
  77
  78 static void
  79 radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline,
  80                                      struct list_head *processing_list);
  81
  82 static
  83 void radv_destroy_semaphore_part(struct radv_device *device,
  84                                  struct radv_semaphore_part *part);
  85
  86 static int
  87 radv_device_get_cache_uuid(enum radeon_family family, void *uuid)
  88 {
  89         struct mesa_sha1 ctx;
  90         unsigned char sha1[20];
  91         unsigned ptr_size = sizeof(void*);
  92
  93         memset(uuid, 0, VK_UUID_SIZE);
  94         _mesa_sha1_init(&ctx);
  95
  96         if (!disk_cache_get_function_identifier(radv_device_get_cache_uuid, &ctx) ||
  97             !disk_cache_get_function_identifier(LLVMInitializeAMDGPUTargetInfo, &ctx))
  98                 return -1;
  99
 100         _mesa_sha1_update(&ctx, &family, sizeof(family));
 101         _mesa_sha1_update(&ctx, &ptr_size, sizeof(ptr_size));
 102         _mesa_sha1_final(&ctx, sha1);
 103
 104         memcpy(uuid, sha1, VK_UUID_SIZE);
 105         return 0;
 106 }
 107
 108 static void
 109 radv_get_driver_uuid(void *uuid)
 110 {
 111         ac_compute_driver_uuid(uuid, VK_UUID_SIZE);
 112 }
 113
 114 static void
 115 radv_get_device_uuid(struct radeon_info *info, void *uuid)
 116 {
 117         ac_compute_device_uuid(info, uuid, VK_UUID_SIZE);
 118 }
 119
 120 static uint64_t
 121 radv_get_visible_vram_size(struct radv_physical_device *device)
 122 {
 123         return MIN2(device->rad_info.vram_size, device->rad_info.vram_vis_size);
 124 }
 125
 126 static uint64_t
 127 radv_get_vram_size(struct radv_physical_device *device)
 128 {
 129         return device->rad_info.vram_size - radv_get_visible_vram_size(device);
 130 }
 131
 132 static void
 133 radv_physical_device_init_mem_types(struct radv_physical_device *device)
 134 {
 135         uint64_t visible_vram_size = radv_get_visible_vram_size(device);
 136         uint64_t vram_size = radv_get_vram_size(device);
 137         int vram_index = -1, visible_vram_index = -1, gart_index = -1;
 138         device->memory_properties.memoryHeapCount = 0;
 139         if (vram_size > 0) {
 140                 vram_index = device->memory_properties.memoryHeapCount++;
 141                 device->memory_properties.memoryHeaps[vram_index] = (VkMemoryHeap) {
 142                         .size = vram_size,
 143                         .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
 144                 };
 145         }
 146
 147         if (device->rad_info.gart_size > 0) {
 148                 gart_index = device->memory_properties.memoryHeapCount++;
 149                 device->memory_properties.memoryHeaps[gart_index] = (VkMemoryHeap) {
 150                         .size = device->rad_info.gart_size,
 151                         .flags = 0,
 152                 };
 153         }
 154
 155         if (visible_vram_size) {
 156                 visible_vram_index = device->memory_properties.memoryHeapCount++;
 157                 device->memory_properties.memoryHeaps[visible_vram_index] = (VkMemoryHeap) {
 158                         .size = visible_vram_size,
 159                         .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
 160                 };
 161         }
 162
 163         unsigned type_count = 0;
 164
 165         if (device->rad_info.has_dedicated_vram) {
 166                 if (vram_index >= 0) {
 167                         device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
 168                         device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS;
 169                         device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 170                                 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
 171                                 .heapIndex = vram_index,
 172                         };
 173                 }
 174         } else {
 175                 if (visible_vram_index >= 0) {
 176                         device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
 177                         device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS;
 178                         device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 179                                 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
 180                                 .heapIndex = visible_vram_index,
 181                         };
 182                 }
 183         }
 184
 185         if (gart_index >= 0) {
 186                 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
 187                 device->memory_flags[type_count] = RADEON_FLAG_GTT_WC | RADEON_FLAG_CPU_ACCESS;
 188                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 189                         .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 190                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 191                         .heapIndex = gart_index,
 192                 };
 193         }
 194         if (visible_vram_index >= 0) {
 195                 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
 196                 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
 197                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 198                         .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
 199                         VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 200                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 201                         .heapIndex = visible_vram_index,
 202                 };
 203         }
 204
 205         if (gart_index >= 0) {
 206                 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
 207                 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
 208                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 209                         .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 210                         VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
 211                         VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
 212                         .heapIndex = gart_index,
 213                 };
 214         }
 215         device->memory_properties.memoryTypeCount = type_count;
 216
 217         if (device->rad_info.has_l2_uncached) {
 218                 for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
 219                         VkMemoryType mem_type = device->memory_properties.memoryTypes[i];
 220
 221                         if ((mem_type.propertyFlags & (VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
 222                                                        VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) ||
 223                             mem_type.propertyFlags == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
 224
 225                                 VkMemoryPropertyFlags property_flags = mem_type.propertyFlags |
 226                                         VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD |
 227                                         VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD;
 228
 229                                 device->memory_domains[type_count] = device->memory_domains[i];
 230                                 device->memory_flags[type_count] = device->memory_flags[i] | RADEON_FLAG_VA_UNCACHED;
 231                                 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
 232                                         .propertyFlags = property_flags,
 233                                         .heapIndex = mem_type.heapIndex,
 234                                 };
 235                         }
 236                 }
 237                 device->memory_properties.memoryTypeCount = type_count;
 238         }
 239 }
 240
 241 static VkResult
 242 radv_physical_device_init(struct radv_physical_device *device,
 243                           struct radv_instance *instance,
 244                           drmDevicePtr drm_device)
 245 {
 246         VkResult result;
 247         int fd = -1;
 248         int master_fd = -1;
 249
 250         if (drm_device) {
 251                 const char *path = drm_device->nodes[DRM_NODE_RENDER];
 252                 drmVersionPtr version;
 253
 254                 fd = open(path, O_RDWR | O_CLOEXEC);
 255                 if (fd < 0) {
 256                         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 257                                 radv_logi("Could not open device '%s'", path);
 258
 259                         return vk_error(instance, VK_ERROR_INCOMPATIBLE_DRIVER);
 260                 }
 261
 262                 version = drmGetVersion(fd);
 263                 if (!version) {
 264                         close(fd);
 265
 266                         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 267                                 radv_logi("Could not get the kernel driver version for device '%s'", path);
 268
 269                         return vk_errorf(instance, VK_ERROR_INCOMPATIBLE_DRIVER,
 270                                          "failed to get version %s: %m", path);
 271                 }
 272
 273                 if (strcmp(version->name, "amdgpu")) {
 274                         drmFreeVersion(version);
 275                         close(fd);
 276
 277                         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 278                                 radv_logi("Device '%s' is not using the amdgpu kernel driver.", path);
 279
 280                         return VK_ERROR_INCOMPATIBLE_DRIVER;
 281                 }
 282                 drmFreeVersion(version);
 283
 284                 if (instance->debug_flags & RADV_DEBUG_STARTUP)
 285                                 radv_logi("Found compatible device '%s'.", path);
 286         }
 287
 288         device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 289         device->instance = instance;
 290
 291         if (drm_device) {
 292                 device->ws = radv_amdgpu_winsys_create(fd, instance->debug_flags,
 293                                                        instance->perftest_flags);
 294         } else {
 295                 device->ws = radv_null_winsys_create();
 296         }
 297
 298         if (!device->ws) {
 299                 result = vk_error(instance, VK_ERROR_INITIALIZATION_FAILED);
 300                 goto fail_fd;
 301         }
 302
 303         if (drm_device && instance->enabled_extensions.KHR_display) {
 304                 master_fd = open(drm_device->nodes[DRM_NODE_PRIMARY], O_RDWR | O_CLOEXEC);
 305                 if (master_fd >= 0) {
 306                         uint32_t accel_working = 0;
 307                         struct drm_amdgpu_info request = {
 308                                 .return_pointer = (uintptr_t)&accel_working,
 309                                 .return_size = sizeof(accel_working),
 310                                 .query = AMDGPU_INFO_ACCEL_WORKING
 311                         };
 312
 313                         if (drmCommandWrite(master_fd, DRM_AMDGPU_INFO, &request, sizeof (struct drm_amdgpu_info)) < 0 || !accel_working) {
 314                                 close(master_fd);
 315                                 master_fd = -1;
 316                         }
 317                 }
 318         }
 319
 320         device->master_fd = master_fd;
 321         device->local_fd = fd;
 322         device->ws->query_info(device->ws, &device->rad_info);
 323
 324         device->use_aco = instance->perftest_flags & RADV_PERFTEST_ACO;
 325
 326         snprintf(device->name, sizeof(device->name),
 327                  "AMD RADV%s %s (LLVM " MESA_LLVM_VERSION_STRING ")", device->use_aco ? "/ACO" : "",
 328                  device->rad_info.name);
 329
 330         if (radv_device_get_cache_uuid(device->rad_info.family, device->cache_uuid)) {
 331                 result = vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED,
 332                                    "cannot generate UUID");
 333                 goto fail_wsi;
 334         }
 335
 336         /* These flags affect shader compilation. */
 337         uint64_t shader_env_flags = (device->use_aco ? 0x2 : 0);
 338
 339         /* The gpu id is already embedded in the uuid so we just pass "radv"
 340          * when creating the cache.
 341          */
 342         char buf[VK_UUID_SIZE * 2 + 1];
 343         disk_cache_format_hex_id(buf, device->cache_uuid, VK_UUID_SIZE * 2);
 344         device->disk_cache = disk_cache_create(device->name, buf, shader_env_flags);
 345
 346         if (device->rad_info.chip_class < GFX8)
 347                 fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n");
 348
 349         radv_get_driver_uuid(&device->driver_uuid);
 350         radv_get_device_uuid(&device->rad_info, &device->device_uuid);
 351
 352         device->out_of_order_rast_allowed = device->rad_info.has_out_of_order_rast &&
 353                                             !(device->instance->debug_flags & RADV_DEBUG_NO_OUT_OF_ORDER);
 354
 355         device->dcc_msaa_allowed =
 356                 (device->instance->perftest_flags & RADV_PERFTEST_DCC_MSAA);
 357
 358         device->use_shader_ballot = (device->use_aco && device->rad_info.chip_class >= GFX8) ||
 359                                     (device->instance->perftest_flags & RADV_PERFTEST_SHADER_BALLOT);
 360
 361         device->use_ngg = device->rad_info.chip_class >= GFX10 &&
 362                           device->rad_info.family != CHIP_NAVI14 &&
 363                           !(device->instance->debug_flags & RADV_DEBUG_NO_NGG);
 364
 365         /* TODO: Implement NGG GS with ACO. */
 366         device->use_ngg_gs = device->use_ngg && !device->use_aco;
 367         device->use_ngg_streamout = false;
 368
 369         /* Determine the number of threads per wave for all stages. */
 370         device->cs_wave_size = 64;
 371         device->ps_wave_size = 64;
 372         device->ge_wave_size = 64;
 373
 374         if (device->rad_info.chip_class >= GFX10) {
 375                 if (device->instance->perftest_flags & RADV_PERFTEST_CS_WAVE_32)
 376                         device->cs_wave_size = 32;
 377
 378                 /* For pixel shaders, wave64 is recommanded. */
 379                 if (device->instance->perftest_flags & RADV_PERFTEST_PS_WAVE_32)
 380                         device->ps_wave_size = 32;
 381
 382                 if (device->instance->perftest_flags & RADV_PERFTEST_GE_WAVE_32)
 383                         device->ge_wave_size = 32;
 384         }
 385
 386         radv_physical_device_init_mem_types(device);
 387         radv_fill_device_extension_table(device, &device->supported_extensions);
 388
 389         if (drm_device)
 390                 device->bus_info = *drm_device->businfo.pci;
 391
 392         if ((device->instance->debug_flags & RADV_DEBUG_INFO))
 393                 ac_print_gpu_info(&device->rad_info);
 394
 395         /* The WSI is structured as a layer on top of the driver, so this has
 396          * to be the last part of initialization (at least until we get other
 397          * semi-layers).
 398          */
 399         result = radv_init_wsi(device);
 400         if (result != VK_SUCCESS) {
 401                 vk_error(instance, result);
 402                 goto fail_disk_cache;
 403         }
 404
 405         return VK_SUCCESS;
 406
 407 fail_disk_cache:
 408         disk_cache_destroy(device->disk_cache);
 409 fail_wsi:
 410         device->ws->destroy(device->ws);
 411 fail_fd:
 412         close(fd);
 413         if (master_fd != -1)
 414                 close(master_fd);
 415         return result;
 416 }
 417
 418 static void
 419 radv_physical_device_finish(struct radv_physical_device *device)
 420 {
 421         radv_finish_wsi(device);
 422         device->ws->destroy(device->ws);
 423         disk_cache_destroy(device->disk_cache);
 424         close(device->local_fd);
 425         if (device->master_fd != -1)
 426                 close(device->master_fd);
 427 }
 428
 429 static void *
 430 default_alloc_func(void *pUserData, size_t size, size_t align,
 431                    VkSystemAllocationScope allocationScope)
 432 {
 433         return malloc(size);
 434 }
 435
 436 static void *
 437 default_realloc_func(void *pUserData, void *pOriginal, size_t size,
 438                      size_t align, VkSystemAllocationScope allocationScope)
 439 {
 440         return realloc(pOriginal, size);
 441 }
 442
 443 static void
 444 default_free_func(void *pUserData, void *pMemory)
 445 {
 446         free(pMemory);
 447 }
 448
 449 static const VkAllocationCallbacks default_alloc = {
 450         .pUserData = NULL,
 451         .pfnAllocation = default_alloc_func,
 452         .pfnReallocation = default_realloc_func,
 453         .pfnFree = default_free_func,
 454 };
 455
 456 static const struct debug_control radv_debug_options[] = {
 457         {"nofastclears", RADV_DEBUG_NO_FAST_CLEARS},
 458         {"nodcc", RADV_DEBUG_NO_DCC},
 459         {"shaders", RADV_DEBUG_DUMP_SHADERS},
 460         {"nocache", RADV_DEBUG_NO_CACHE},
 461         {"shaderstats", RADV_DEBUG_DUMP_SHADER_STATS},
 462         {"nohiz", RADV_DEBUG_NO_HIZ},
 463         {"nocompute", RADV_DEBUG_NO_COMPUTE_QUEUE},
 464         {"allbos", RADV_DEBUG_ALL_BOS},
 465         {"noibs", RADV_DEBUG_NO_IBS},
 466         {"spirv", RADV_DEBUG_DUMP_SPIRV},
 467         {"vmfaults", RADV_DEBUG_VM_FAULTS},
 468         {"zerovram", RADV_DEBUG_ZERO_VRAM},
 469         {"syncshaders", RADV_DEBUG_SYNC_SHADERS},
 470         {"preoptir", RADV_DEBUG_PREOPTIR},
 471         {"nodynamicbounds", RADV_DEBUG_NO_DYNAMIC_BOUNDS},
 472         {"nooutoforder", RADV_DEBUG_NO_OUT_OF_ORDER},
 473         {"info", RADV_DEBUG_INFO},
 474         {"errors", RADV_DEBUG_ERRORS},
 475         {"startup", RADV_DEBUG_STARTUP},
 476         {"checkir", RADV_DEBUG_CHECKIR},
 477         {"nothreadllvm", RADV_DEBUG_NOTHREADLLVM},
 478         {"nobinning", RADV_DEBUG_NOBINNING},
 479         {"noloadstoreopt", RADV_DEBUG_NO_LOAD_STORE_OPT},
 480         {"nongg", RADV_DEBUG_NO_NGG},
 481         {"noshaderballot", RADV_DEBUG_NO_SHADER_BALLOT},
 482         {"allentrypoints", RADV_DEBUG_ALL_ENTRYPOINTS},
 483         {"metashaders", RADV_DEBUG_DUMP_META_SHADERS},
 484         {"nomemorycache", RADV_DEBUG_NO_MEMORY_CACHE},
 485         {NULL, 0}
 486 };
 487
 488 const char *
 489 radv_get_debug_option_name(int id)
 490 {
 491         assert(id < ARRAY_SIZE(radv_debug_options) - 1);
 492         return radv_debug_options[id].string;
 493 }
 494
 495 static const struct debug_control radv_perftest_options[] = {
 496         {"localbos", RADV_PERFTEST_LOCAL_BOS},
 497         {"dccmsaa", RADV_PERFTEST_DCC_MSAA},
 498         {"bolist", RADV_PERFTEST_BO_LIST},
 499         {"shader_ballot", RADV_PERFTEST_SHADER_BALLOT},
 500         {"tccompatcmask", RADV_PERFTEST_TC_COMPAT_CMASK},
 501         {"cswave32", RADV_PERFTEST_CS_WAVE_32},
 502         {"pswave32", RADV_PERFTEST_PS_WAVE_32},
 503         {"gewave32", RADV_PERFTEST_GE_WAVE_32},
 504         {"dfsm", RADV_PERFTEST_DFSM},
 505         {"aco", RADV_PERFTEST_ACO},
 506         {NULL, 0}
 507 };
 508
 509 const char *
 510 radv_get_perftest_option_name(int id)
 511 {
 512         assert(id < ARRAY_SIZE(radv_perftest_options) - 1);
 513         return radv_perftest_options[id].string;
 514 }
 515
 516 static void
 517 radv_handle_per_app_options(struct radv_instance *instance,
 518                             const VkApplicationInfo *info)
 519 {
 520         const char *name = info ? info->pApplicationName : NULL;
 521
 522         if (!name)
 523                 return;
 524
 525         if (!strcmp(name, "DOOM_VFR")) {
 526                 /* Work around a Doom VFR game bug */
 527                 instance->debug_flags |= RADV_DEBUG_NO_DYNAMIC_BOUNDS;
 528         } else if (!strcmp(name, "MonsterHunterWorld.exe")) {
 529                 /* Workaround for a WaW hazard when LLVM moves/merges
 530                  * load/store memory operations.
 531                  * See https://reviews.llvm.org/D61313
 532                  */
 533                 if (LLVM_VERSION_MAJOR < 9)
 534                         instance->debug_flags |= RADV_DEBUG_NO_LOAD_STORE_OPT;
 535         } else if (!strcmp(name, "Wolfenstein: Youngblood")) {
 536                 if (!(instance->debug_flags & RADV_DEBUG_NO_SHADER_BALLOT) &&
 537                     !(instance->perftest_flags & RADV_PERFTEST_ACO)) {
 538                         /* Force enable VK_AMD_shader_ballot because it looks
 539                          * safe and it gives a nice boost (+20% on Vega 56 at
 540                          * this time). It also prevents corruption on LLVM.
 541                          */
 542                         instance->perftest_flags |= RADV_PERFTEST_SHADER_BALLOT;
 543                 }
 544         } else if (!strcmp(name, "Fledge")) {
 545                 /*
 546                  * Zero VRAM for "The Surge 2"
 547                  *
 548                  * This avoid a hang when when rendering any level. Likely
 549                  * uninitialized data in an indirect draw.
 550                  */
 551                 instance->debug_flags |= RADV_DEBUG_ZERO_VRAM;
 552         } else if (!strcmp(name, "No Man's Sky")) {
 553                 /* Work around a NMS game bug */
 554                 instance->debug_flags |= RADV_DEBUG_DISCARD_TO_DEMOTE;
 555         }
 556 }
 557
 558 static const char radv_dri_options_xml[] =
 559 DRI_CONF_BEGIN
 560         DRI_CONF_SECTION_PERFORMANCE
 561                 DRI_CONF_ADAPTIVE_SYNC("true")
 562                 DRI_CONF_VK_X11_OVERRIDE_MIN_IMAGE_COUNT(0)
 563                 DRI_CONF_VK_X11_STRICT_IMAGE_COUNT("false")
 564         DRI_CONF_SECTION_END
 565
 566         DRI_CONF_SECTION_DEBUG
 567                 DRI_CONF_VK_WSI_FORCE_BGRA8_UNORM_FIRST("false")
 568         DRI_CONF_SECTION_END
 569 DRI_CONF_END;
 570
 571 static void  radv_init_dri_options(struct radv_instance *instance)
 572 {
 573         driParseOptionInfo(&instance->available_dri_options, radv_dri_options_xml);
 574         driParseConfigFiles(&instance->dri_options,
 575                             &instance->available_dri_options,
 576                             0, "radv", NULL,
 577                             instance->engineName,
 578                             instance->engineVersion);
 579 }
 580
 581 VkResult radv_CreateInstance(
 582         const VkInstanceCreateInfo*                 pCreateInfo,
 583         const VkAllocationCallbacks*                pAllocator,
 584         VkInstance*                                 pInstance)
 585 {
 586         struct radv_instance *instance;
 587         VkResult result;
 588
 589         instance = vk_zalloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
 590                               VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 591         if (!instance)
 592                 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
 593
 594         instance->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 595
 596         if (pAllocator)
 597                 instance->alloc = *pAllocator;
 598         else
 599                 instance->alloc = default_alloc;
 600
 601         if (pCreateInfo->pApplicationInfo) {
 602                 const VkApplicationInfo *app = pCreateInfo->pApplicationInfo;
 603
 604                 instance->engineName =
 605                         vk_strdup(&instance->alloc, app->pEngineName,
 606                                   VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 607                 instance->engineVersion = app->engineVersion;
 608                 instance->apiVersion = app->apiVersion;
 609         }
 610
 611         if (instance->apiVersion == 0)
 612                 instance->apiVersion = VK_API_VERSION_1_0;
 613
 614         instance->physicalDeviceCount = -1;
 615
 616         /* Get secure compile thread count. NOTE: We cap this at 32 */
 617 #define MAX_SC_PROCS 32
 618         char *num_sc_threads = getenv("RADV_SECURE_COMPILE_THREADS");
 619         if (num_sc_threads)
 620                 instance->num_sc_threads = MIN2(strtoul(num_sc_threads, NULL, 10), MAX_SC_PROCS);
 621
 622         instance->debug_flags = parse_debug_string(getenv("RADV_DEBUG"),
 623                                                    radv_debug_options);
 624
 625         /* Disable memory cache when secure compile is set */
 626         if (radv_device_use_secure_compile(instance))
 627                 instance->debug_flags |= RADV_DEBUG_NO_MEMORY_CACHE;
 628
 629         instance->perftest_flags = parse_debug_string(getenv("RADV_PERFTEST"),
 630                                                    radv_perftest_options);
 631
 632         if (instance->perftest_flags & RADV_PERFTEST_ACO)
 633                 fprintf(stderr, "WARNING: Experimental compiler backend enabled. Here be dragons! Incorrect rendering, GPU hangs and/or resets are likely\n");
 634
 635         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 636                 radv_logi("Created an instance");
 637
 638         for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
 639                 int idx;
 640                 for (idx = 0; idx < RADV_INSTANCE_EXTENSION_COUNT; idx++) {
 641                         if (!strcmp(pCreateInfo->ppEnabledExtensionNames[i],
 642                                     radv_instance_extensions[idx].extensionName))
 643                                 break;
 644                 }
 645
 646                 if (idx >= RADV_INSTANCE_EXTENSION_COUNT ||
 647                     !radv_supported_instance_extensions.extensions[idx]) {
 648                         vk_free2(&default_alloc, pAllocator, instance);
 649                         return vk_error(instance, VK_ERROR_EXTENSION_NOT_PRESENT);
 650                 }
 651
 652                 instance->enabled_extensions.extensions[idx] = true;
 653         }
 654
 655         bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
 656
 657         for (unsigned i = 0; i < ARRAY_SIZE(instance->dispatch.entrypoints); i++) {
 658                 /* Vulkan requires that entrypoints for extensions which have
 659                  * not been enabled must not be advertised.
 660                  */
 661                 if (!unchecked &&
 662                     !radv_instance_entrypoint_is_enabled(i, instance->apiVersion,
 663                                                          &instance->enabled_extensions)) {
 664                         instance->dispatch.entrypoints[i] = NULL;
 665                 } else {
 666                         instance->dispatch.entrypoints[i] =
 667                                 radv_instance_dispatch_table.entrypoints[i];
 668                 }
 669         }
 670
 671          for (unsigned i = 0; i < ARRAY_SIZE(instance->physical_device_dispatch.entrypoints); i++) {
 672                 /* Vulkan requires that entrypoints for extensions which have
 673                  * not been enabled must not be advertised.
 674                  */
 675                 if (!unchecked &&
 676                     !radv_physical_device_entrypoint_is_enabled(i, instance->apiVersion,
 677                                                                 &instance->enabled_extensions)) {
 678                         instance->physical_device_dispatch.entrypoints[i] = NULL;
 679                 } else {
 680                         instance->physical_device_dispatch.entrypoints[i] =
 681                                 radv_physical_device_dispatch_table.entrypoints[i];
 682                 }
 683         }
 684
 685         for (unsigned i = 0; i < ARRAY_SIZE(instance->device_dispatch.entrypoints); i++) {
 686                 /* Vulkan requires that entrypoints for extensions which have
 687                  * not been enabled must not be advertised.
 688                  */
 689                 if (!unchecked &&
 690                     !radv_device_entrypoint_is_enabled(i, instance->apiVersion,
 691                                                        &instance->enabled_extensions, NULL)) {
 692                         instance->device_dispatch.entrypoints[i] = NULL;
 693                 } else {
 694                         instance->device_dispatch.entrypoints[i] =
 695                                 radv_device_dispatch_table.entrypoints[i];
 696                 }
 697         }
 698
 699         result = vk_debug_report_instance_init(&instance->debug_report_callbacks);
 700         if (result != VK_SUCCESS) {
 701                 vk_free2(&default_alloc, pAllocator, instance);
 702                 return vk_error(instance, result);
 703         }
 704
 705         glsl_type_singleton_init_or_ref();
 706
 707         VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false));
 708
 709         radv_init_dri_options(instance);
 710         radv_handle_per_app_options(instance, pCreateInfo->pApplicationInfo);
 711
 712         *pInstance = radv_instance_to_handle(instance);
 713
 714         return VK_SUCCESS;
 715 }
 716
 717 void radv_DestroyInstance(
 718         VkInstance                                  _instance,
 719         const VkAllocationCallbacks*                pAllocator)
 720 {
 721         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 722
 723         if (!instance)
 724                 return;
 725
 726         for (int i = 0; i < instance->physicalDeviceCount; ++i) {
 727                 radv_physical_device_finish(instance->physicalDevices + i);
 728         }
 729
 730         vk_free(&instance->alloc, instance->engineName);
 731
 732         VG(VALGRIND_DESTROY_MEMPOOL(instance));
 733
 734         glsl_type_singleton_decref();
 735
 736         driDestroyOptionCache(&instance->dri_options);
 737         driDestroyOptionInfo(&instance->available_dri_options);
 738
 739         vk_debug_report_instance_destroy(&instance->debug_report_callbacks);
 740
 741         vk_free(&instance->alloc, instance);
 742 }
 743
 744 static VkResult
 745 radv_enumerate_physical_devices(struct radv_instance *instance)
 746 {
 747         /* TODO: Check for more devices ? */
 748         drmDevicePtr devices[8];
 749         VkResult result = VK_SUCCESS;
 750         int max_devices;
 751
 752         instance->physicalDeviceCount = 0;
 753
 754         if (getenv("RADV_FORCE_FAMILY")) {
 755                 /* When RADV_FORCE_FAMILY is set, the driver creates a nul
 756                  * device that allows to test the compiler without having an
 757                  * AMDGPU instance.
 758                  */
 759                 result = radv_physical_device_init(instance->physicalDevices +
 760                                                    instance->physicalDeviceCount,
 761                                                    instance, NULL);
 762
 763                 ++instance->physicalDeviceCount;
 764                 return VK_SUCCESS;
 765         }
 766
 767         max_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices));
 768
 769         if (instance->debug_flags & RADV_DEBUG_STARTUP)
 770                 radv_logi("Found %d drm nodes", max_devices);
 771
 772         if (max_devices < 1)
 773                 return vk_error(instance, VK_SUCCESS);
 774
 775         for (unsigned i = 0; i < (unsigned)max_devices; i++) {
 776                 if (devices[i]->available_nodes & 1 << DRM_NODE_RENDER &&
 777                     devices[i]->bustype == DRM_BUS_PCI &&
 778                     devices[i]->deviceinfo.pci->vendor_id == ATI_VENDOR_ID) {
 779
 780                         result = radv_physical_device_init(instance->physicalDevices +
 781                                                            instance->physicalDeviceCount,
 782                                                            instance,
 783                                                            devices[i]);
 784                         /* Incompatible DRM device, skip. */
 785                         if (result == VK_ERROR_INCOMPATIBLE_DRIVER) {
 786                                 result = VK_SUCCESS;
 787                                 continue;
 788                         }
 789
 790                         /* Error creating the physical device, report the error. */
 791                         if (result != VK_SUCCESS)
 792                                 break;
 793
 794                         ++instance->physicalDeviceCount;
 795                 }
 796         }
 797         drmFreeDevices(devices, max_devices);
 798
 799         /* If we successfully enumerated any devices, call it success */
 800         return result;
 801 }
 802
 803 VkResult radv_EnumeratePhysicalDevices(
 804         VkInstance                                  _instance,
 805         uint32_t*                                   pPhysicalDeviceCount,
 806         VkPhysicalDevice*                           pPhysicalDevices)
 807 {
 808         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 809         VkResult result;
 810
 811         if (instance->physicalDeviceCount < 0) {
 812                 result = radv_enumerate_physical_devices(instance);
 813                 if (result != VK_SUCCESS)
 814                         return result;
 815         }
 816
 817         if (!pPhysicalDevices) {
 818                 *pPhysicalDeviceCount = instance->physicalDeviceCount;
 819         } else {
 820                 *pPhysicalDeviceCount = MIN2(*pPhysicalDeviceCount, instance->physicalDeviceCount);
 821                 for (unsigned i = 0; i < *pPhysicalDeviceCount; ++i)
 822                         pPhysicalDevices[i] = radv_physical_device_to_handle(instance->physicalDevices + i);
 823         }
 824
 825         return *pPhysicalDeviceCount < instance->physicalDeviceCount ? VK_INCOMPLETE
 826                                                                      : VK_SUCCESS;
 827 }
 828
 829 VkResult radv_EnumeratePhysicalDeviceGroups(
 830     VkInstance                                  _instance,
 831     uint32_t*                                   pPhysicalDeviceGroupCount,
 832     VkPhysicalDeviceGroupProperties*            pPhysicalDeviceGroupProperties)
 833 {
 834         RADV_FROM_HANDLE(radv_instance, instance, _instance);
 835         VkResult result;
 836
 837         if (instance->physicalDeviceCount < 0) {
 838                 result = radv_enumerate_physical_devices(instance);
 839                 if (result != VK_SUCCESS)
 840                         return result;
 841         }
 842
 843         if (!pPhysicalDeviceGroupProperties) {
 844                 *pPhysicalDeviceGroupCount = instance->physicalDeviceCount;
 845         } else {
 846                 *pPhysicalDeviceGroupCount = MIN2(*pPhysicalDeviceGroupCount, instance->physicalDeviceCount);
 847                 for (unsigned i = 0; i < *pPhysicalDeviceGroupCount; ++i) {
 848                         pPhysicalDeviceGroupProperties[i].physicalDeviceCount = 1;
 849                         pPhysicalDeviceGroupProperties[i].physicalDevices[0] = radv_physical_device_to_handle(instance->physicalDevices + i);
 850                         pPhysicalDeviceGroupProperties[i].subsetAllocation = false;
 851                 }
 852         }
 853         return *pPhysicalDeviceGroupCount < instance->physicalDeviceCount ? VK_INCOMPLETE
 854                                                                           : VK_SUCCESS;
 855 }
 856
 857 void radv_GetPhysicalDeviceFeatures(
 858         VkPhysicalDevice                            physicalDevice,
 859         VkPhysicalDeviceFeatures*                   pFeatures)
 860 {
 861         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
 862         memset(pFeatures, 0, sizeof(*pFeatures));
 863
 864         *pFeatures = (VkPhysicalDeviceFeatures) {
 865                 .robustBufferAccess                       = true,
 866                 .fullDrawIndexUint32                      = true,
 867                 .imageCubeArray                           = true,
 868                 .independentBlend                         = true,
 869                 .geometryShader                           = true,
 870                 .tessellationShader                       = true,
 871                 .sampleRateShading                        = true,
 872                 .dualSrcBlend                             = true,
 873                 .logicOp                                  = true,
 874                 .multiDrawIndirect                        = true,
 875                 .drawIndirectFirstInstance                = true,
 876                 .depthClamp                               = true,
 877                 .depthBiasClamp                           = true,
 878                 .fillModeNonSolid                         = true,
 879                 .depthBounds                              = true,
 880                 .wideLines                                = true,
 881                 .largePoints                              = true,
 882                 .alphaToOne                               = true,
 883                 .multiViewport                            = true,
 884                 .samplerAnisotropy                        = true,
 885                 .textureCompressionETC2                   = radv_device_supports_etc(pdevice),
 886                 .textureCompressionASTC_LDR               = false,
 887                 .textureCompressionBC                     = true,
 888                 .occlusionQueryPrecise                    = true,
 889                 .pipelineStatisticsQuery                  = true,
 890                 .vertexPipelineStoresAndAtomics           = true,
 891                 .fragmentStoresAndAtomics                 = true,
 892                 .shaderTessellationAndGeometryPointSize   = true,
 893                 .shaderImageGatherExtended                = true,
 894                 .shaderStorageImageExtendedFormats        = true,
 895                 .shaderStorageImageMultisample            = true,
 896                 .shaderUniformBufferArrayDynamicIndexing  = true,
 897                 .shaderSampledImageArrayDynamicIndexing   = true,
 898                 .shaderStorageBufferArrayDynamicIndexing  = true,
 899                 .shaderStorageImageArrayDynamicIndexing   = true,
 900                 .shaderStorageImageReadWithoutFormat      = true,
 901                 .shaderStorageImageWriteWithoutFormat     = true,
 902                 .shaderClipDistance                       = true,
 903                 .shaderCullDistance                       = true,
 904                 .shaderFloat64                            = true,
 905                 .shaderInt64                              = true,
 906                 .shaderInt16                              = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8,
 907                 .sparseBinding                            = true,
 908                 .variableMultisampleRate                  = true,
 909                 .inheritedQueries                         = true,
 910         };
 911 }
 912
 913 void radv_GetPhysicalDeviceFeatures2(
 914         VkPhysicalDevice                            physicalDevice,
 915         VkPhysicalDeviceFeatures2                  *pFeatures)
 916 {
 917         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
 918         vk_foreach_struct(ext, pFeatures->pNext) {
 919                 switch (ext->sType) {
 920                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VARIABLE_POINTERS_FEATURES: {
 921                         VkPhysicalDeviceVariablePointersFeatures *features = (void *)ext;
 922                         features->variablePointersStorageBuffer = true;
 923                         features->variablePointers = true;
 924                         break;
 925                 }
 926                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_FEATURES: {
 927                         VkPhysicalDeviceMultiviewFeatures *features = (VkPhysicalDeviceMultiviewFeatures*)ext;
 928                         features->multiview = true;
 929                         features->multiviewGeometryShader = true;
 930                         features->multiviewTessellationShader = true;
 931                         break;
 932                 }
 933                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DRAW_PARAMETERS_FEATURES: {
 934                         VkPhysicalDeviceShaderDrawParametersFeatures *features =
 935                             (VkPhysicalDeviceShaderDrawParametersFeatures*)ext;
 936                         features->shaderDrawParameters = true;
 937                         break;
 938                 }
 939                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_FEATURES: {
 940                         VkPhysicalDeviceProtectedMemoryFeatures *features =
 941                             (VkPhysicalDeviceProtectedMemoryFeatures*)ext;
 942                         features->protectedMemory = false;
 943                         break;
 944                 }
 945                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_16BIT_STORAGE_FEATURES: {
 946                         VkPhysicalDevice16BitStorageFeatures *features =
 947                             (VkPhysicalDevice16BitStorageFeatures*)ext;
 948                         bool enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
 949                         features->storageBuffer16BitAccess = enable;
 950                         features->uniformAndStorageBuffer16BitAccess = enable;
 951                         features->storagePushConstant16 = enable;
 952                         features->storageInputOutput16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco && LLVM_VERSION_MAJOR >= 9;
 953                         break;
 954                 }
 955                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES: {
 956                         VkPhysicalDeviceSamplerYcbcrConversionFeatures *features =
 957                             (VkPhysicalDeviceSamplerYcbcrConversionFeatures*)ext;
 958                         features->samplerYcbcrConversion = true;
 959                         break;
 960                 }
 961                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES: {
 962                         VkPhysicalDeviceDescriptorIndexingFeatures *features =
 963                                 (VkPhysicalDeviceDescriptorIndexingFeatures*)ext;
 964                         features->shaderInputAttachmentArrayDynamicIndexing = true;
 965                         features->shaderUniformTexelBufferArrayDynamicIndexing = true;
 966                         features->shaderStorageTexelBufferArrayDynamicIndexing = true;
 967                         features->shaderUniformBufferArrayNonUniformIndexing = true;
 968                         features->shaderSampledImageArrayNonUniformIndexing = true;
 969                         features->shaderStorageBufferArrayNonUniformIndexing = true;
 970                         features->shaderStorageImageArrayNonUniformIndexing = true;
 971                         features->shaderInputAttachmentArrayNonUniformIndexing = true;
 972                         features->shaderUniformTexelBufferArrayNonUniformIndexing = true;
 973                         features->shaderStorageTexelBufferArrayNonUniformIndexing = true;
 974                         features->descriptorBindingUniformBufferUpdateAfterBind = true;
 975                         features->descriptorBindingSampledImageUpdateAfterBind = true;
 976                         features->descriptorBindingStorageImageUpdateAfterBind = true;
 977                         features->descriptorBindingStorageBufferUpdateAfterBind = true;
 978                         features->descriptorBindingUniformTexelBufferUpdateAfterBind = true;
 979                         features->descriptorBindingStorageTexelBufferUpdateAfterBind = true;
 980                         features->descriptorBindingUpdateUnusedWhilePending = true;
 981                         features->descriptorBindingPartiallyBound = true;
 982                         features->descriptorBindingVariableDescriptorCount = true;
 983                         features->runtimeDescriptorArray = true;
 984                         break;
 985                 }
 986                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONDITIONAL_RENDERING_FEATURES_EXT: {
 987                         VkPhysicalDeviceConditionalRenderingFeaturesEXT *features =
 988                                 (VkPhysicalDeviceConditionalRenderingFeaturesEXT*)ext;
 989                         features->conditionalRendering = true;
 990                         features->inheritedConditionalRendering = false;
 991                         break;
 992                 }
 993                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_FEATURES_EXT: {
 994                         VkPhysicalDeviceVertexAttributeDivisorFeaturesEXT *features =
 995                                 (VkPhysicalDeviceVertexAttributeDivisorFeaturesEXT *)ext;
 996                         features->vertexAttributeInstanceRateDivisor = true;
 997                         features->vertexAttributeInstanceRateZeroDivisor = true;
 998                         break;
 999                 }
1000                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_FEATURES_EXT: {
1001                         VkPhysicalDeviceTransformFeedbackFeaturesEXT *features =
1002                                 (VkPhysicalDeviceTransformFeedbackFeaturesEXT*)ext;
1003                         features->transformFeedback = true;
1004                         features->geometryStreams = !pdevice->use_ngg_streamout;
1005                         break;
1006                 }
1007                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SCALAR_BLOCK_LAYOUT_FEATURES: {
1008                         VkPhysicalDeviceScalarBlockLayoutFeatures *features =
1009                                 (VkPhysicalDeviceScalarBlockLayoutFeatures *)ext;
1010                         features->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7;
1011                         break;
1012                 }
1013                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT: {
1014                         VkPhysicalDeviceMemoryPriorityFeaturesEXT *features =
1015                                 (VkPhysicalDeviceMemoryPriorityFeaturesEXT *)ext;
1016                         features->memoryPriority = true;
1017                         break;
1018                 }
1019                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_EXT: {
1020                         VkPhysicalDeviceBufferDeviceAddressFeaturesEXT *features =
1021                                 (VkPhysicalDeviceBufferDeviceAddressFeaturesEXT *)ext;
1022                         features->bufferDeviceAddress = true;
1023                         features->bufferDeviceAddressCaptureReplay = false;
1024                         features->bufferDeviceAddressMultiDevice = false;
1025                         break;
1026                 }
1027                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES: {
1028                         VkPhysicalDeviceBufferDeviceAddressFeatures *features =
1029                                 (VkPhysicalDeviceBufferDeviceAddressFeatures *)ext;
1030                         features->bufferDeviceAddress = true;
1031                         features->bufferDeviceAddressCaptureReplay = false;
1032                         features->bufferDeviceAddressMultiDevice = false;
1033                         break;
1034                 }
1035                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_CLIP_ENABLE_FEATURES_EXT: {
1036                         VkPhysicalDeviceDepthClipEnableFeaturesEXT *features =
1037                                 (VkPhysicalDeviceDepthClipEnableFeaturesEXT *)ext;
1038                         features->depthClipEnable = true;
1039                         break;
1040                 }
1041                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_HOST_QUERY_RESET_FEATURES: {
1042                         VkPhysicalDeviceHostQueryResetFeatures *features =
1043                                 (VkPhysicalDeviceHostQueryResetFeatures *)ext;
1044                         features->hostQueryReset = true;
1045                         break;
1046                 }
1047                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES: {
1048                         VkPhysicalDevice8BitStorageFeatures *features =
1049                             (VkPhysicalDevice8BitStorageFeatures *)ext;
1050                         bool enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1051                         features->storageBuffer8BitAccess = enable;
1052                         features->uniformAndStorageBuffer8BitAccess = enable;
1053                         features->storagePushConstant8 = enable;
1054                         break;
1055                 }
1056                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES: {
1057                         VkPhysicalDeviceShaderFloat16Int8Features *features =
1058                                 (VkPhysicalDeviceShaderFloat16Int8Features*)ext;
1059                         features->shaderFloat16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco;
1060                         features->shaderInt8 = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1061                         break;
1062                 }
1063                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES: {
1064                         VkPhysicalDeviceShaderAtomicInt64Features *features =
1065                                 (VkPhysicalDeviceShaderAtomicInt64Features *)ext;
1066                         features->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1067                         features->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1068                         break;
1069                 }
1070                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_DEMOTE_TO_HELPER_INVOCATION_FEATURES_EXT: {
1071                         VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *features =
1072                                 (VkPhysicalDeviceShaderDemoteToHelperInvocationFeaturesEXT *)ext;
1073                         features->shaderDemoteToHelperInvocation = pdevice->use_aco;
1074                         break;
1075                 }
1076                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_FEATURES_EXT: {
1077                         VkPhysicalDeviceInlineUniformBlockFeaturesEXT *features =
1078                                 (VkPhysicalDeviceInlineUniformBlockFeaturesEXT *)ext;
1079
1080                         features->inlineUniformBlock = true;
1081                         features->descriptorBindingInlineUniformBlockUpdateAfterBind = true;
1082                         break;
1083                 }
1084                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COMPUTE_SHADER_DERIVATIVES_FEATURES_NV: {
1085                         VkPhysicalDeviceComputeShaderDerivativesFeaturesNV *features =
1086                                 (VkPhysicalDeviceComputeShaderDerivativesFeaturesNV *)ext;
1087                         features->computeDerivativeGroupQuads = false;
1088                         features->computeDerivativeGroupLinear = true;
1089                         break;
1090                 }
1091                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_YCBCR_IMAGE_ARRAYS_FEATURES_EXT: {
1092                         VkPhysicalDeviceYcbcrImageArraysFeaturesEXT *features =
1093                                 (VkPhysicalDeviceYcbcrImageArraysFeaturesEXT*)ext;
1094                         features->ycbcrImageArrays = true;
1095                         break;
1096                 }
1097                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_UNIFORM_BUFFER_STANDARD_LAYOUT_FEATURES: {
1098                         VkPhysicalDeviceUniformBufferStandardLayoutFeatures *features =
1099                                 (VkPhysicalDeviceUniformBufferStandardLayoutFeatures *)ext;
1100                         features->uniformBufferStandardLayout = true;
1101                         break;
1102                 }
1103                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INDEX_TYPE_UINT8_FEATURES_EXT: {
1104                         VkPhysicalDeviceIndexTypeUint8FeaturesEXT *features =
1105                                 (VkPhysicalDeviceIndexTypeUint8FeaturesEXT *)ext;
1106                         features->indexTypeUint8 = pdevice->rad_info.chip_class >= GFX8;
1107                         break;
1108                 }
1109                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGELESS_FRAMEBUFFER_FEATURES: {
1110                         VkPhysicalDeviceImagelessFramebufferFeatures *features =
1111                                 (VkPhysicalDeviceImagelessFramebufferFeatures *)ext;
1112                         features->imagelessFramebuffer = true;
1113                         break;
1114                 }
1115                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR: {
1116                         VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR *features =
1117                                 (VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR *)ext;
1118                         features->pipelineExecutableInfo = true;
1119                         break;
1120                 }
1121                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR: {
1122                         VkPhysicalDeviceShaderClockFeaturesKHR *features =
1123                                 (VkPhysicalDeviceShaderClockFeaturesKHR *)ext;
1124                         features->shaderSubgroupClock = true;
1125                         features->shaderDeviceClock = false;
1126                         break;
1127                 }
1128                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_FEATURES_EXT: {
1129                         VkPhysicalDeviceTexelBufferAlignmentFeaturesEXT *features =
1130                                 (VkPhysicalDeviceTexelBufferAlignmentFeaturesEXT *)ext;
1131                         features->texelBufferAlignment = true;
1132                         break;
1133                 }
1134                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES: {
1135                         VkPhysicalDeviceTimelineSemaphoreFeatures *features =
1136                                 (VkPhysicalDeviceTimelineSemaphoreFeatures *) ext;
1137                         features->timelineSemaphore = true;
1138                         break;
1139                 }
1140                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT: {
1141                         VkPhysicalDeviceSubgroupSizeControlFeaturesEXT *features =
1142                                 (VkPhysicalDeviceSubgroupSizeControlFeaturesEXT *)ext;
1143                         features->subgroupSizeControl = true;
1144                         features->computeFullSubgroups = true;
1145                         break;
1146                 }
1147                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COHERENT_MEMORY_FEATURES_AMD: {
1148                         VkPhysicalDeviceCoherentMemoryFeaturesAMD *features =
1149                                 (VkPhysicalDeviceCoherentMemoryFeaturesAMD *)ext;
1150                         features->deviceCoherentMemory = pdevice->rad_info.has_l2_uncached;
1151                         break;
1152                 }
1153                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_SUBGROUP_EXTENDED_TYPES_FEATURES: {
1154                         VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *features =
1155                                 (VkPhysicalDeviceShaderSubgroupExtendedTypesFeatures *)ext;
1156                         features->shaderSubgroupExtendedTypes = !pdevice->use_aco;
1157                         break;
1158                 }
1159                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SEPARATE_DEPTH_STENCIL_LAYOUTS_FEATURES_KHR: {
1160                         VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *features =
1161                                 (VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *)ext;
1162                         features->separateDepthStencilLayouts = true;
1163                         break;
1164                 }
1165                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES: {
1166                         VkPhysicalDeviceVulkan11Features *features =
1167                                 (VkPhysicalDeviceVulkan11Features *)ext;
1168                         bool storage16_enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1169                         features->storageBuffer16BitAccess = storage16_enable;
1170                         features->uniformAndStorageBuffer16BitAccess = storage16_enable;
1171                         features->storagePushConstant16 = storage16_enable;
1172                         features->storageInputOutput16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco && LLVM_VERSION_MAJOR >= 9;
1173                         features->multiview = true;
1174                         features->multiviewGeometryShader = true;
1175                         features->multiviewTessellationShader = true;
1176                         features->variablePointersStorageBuffer = true;
1177                         features->variablePointers = true;
1178                         features->protectedMemory = false;
1179                         features->samplerYcbcrConversion = true;
1180                         features->shaderDrawParameters = true;
1181                         break;
1182                 }
1183                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES: {
1184                         VkPhysicalDeviceVulkan12Features *features =
1185                                 (VkPhysicalDeviceVulkan12Features *)ext;
1186                         bool int8_enable = !pdevice->use_aco || pdevice->rad_info.chip_class >= GFX8;
1187                         features->samplerMirrorClampToEdge = true;
1188                         features->drawIndirectCount = true;
1189                         features->storageBuffer8BitAccess = int8_enable;
1190                         features->uniformAndStorageBuffer8BitAccess = int8_enable;
1191                         features->storagePushConstant8 = int8_enable;
1192                         features->shaderBufferInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1193                         features->shaderSharedInt64Atomics = LLVM_VERSION_MAJOR >= 9;
1194                         features->shaderFloat16 = pdevice->rad_info.has_double_rate_fp16 && !pdevice->use_aco;
1195                         features->shaderInt8 = int8_enable;
1196                         features->descriptorIndexing = true;
1197                         features->shaderInputAttachmentArrayDynamicIndexing = true;
1198                         features->shaderUniformTexelBufferArrayDynamicIndexing = true;
1199                         features->shaderStorageTexelBufferArrayDynamicIndexing = true;
1200                         features->shaderUniformBufferArrayNonUniformIndexing = true;
1201                         features->shaderSampledImageArrayNonUniformIndexing = true;
1202                         features->shaderStorageBufferArrayNonUniformIndexing = true;
1203                         features->shaderStorageImageArrayNonUniformIndexing = true;
1204                         features->shaderInputAttachmentArrayNonUniformIndexing = true;
1205                         features->shaderUniformTexelBufferArrayNonUniformIndexing = true;
1206                         features->shaderStorageTexelBufferArrayNonUniformIndexing = true;
1207                         features->descriptorBindingUniformBufferUpdateAfterBind = true;
1208                         features->descriptorBindingSampledImageUpdateAfterBind = true;
1209                         features->descriptorBindingStorageImageUpdateAfterBind = true;
1210                         features->descriptorBindingStorageBufferUpdateAfterBind = true;
1211                         features->descriptorBindingUniformTexelBufferUpdateAfterBind = true;
1212                         features->descriptorBindingStorageTexelBufferUpdateAfterBind = true;
1213                         features->descriptorBindingUpdateUnusedWhilePending = true;
1214                         features->descriptorBindingPartiallyBound = true;
1215                         features->descriptorBindingVariableDescriptorCount = true;
1216                         features->runtimeDescriptorArray = true;
1217                         features->samplerFilterMinmax = true;
1218                         features->scalarBlockLayout = pdevice->rad_info.chip_class >= GFX7;
1219                         features->imagelessFramebuffer = true;
1220                         features->uniformBufferStandardLayout = true;
1221                         features->shaderSubgroupExtendedTypes = !pdevice->use_aco;
1222                         features->separateDepthStencilLayouts = true;
1223                         features->hostQueryReset = true;
1224                         features->timelineSemaphore = pdevice->rad_info.has_syncobj_wait_for_submit;
1225                         features->bufferDeviceAddress = true;
1226                         features->bufferDeviceAddressCaptureReplay = false;
1227                         features->bufferDeviceAddressMultiDevice = false;
1228                         features->vulkanMemoryModel = false;
1229                         features->vulkanMemoryModelDeviceScope = false;
1230                         features->vulkanMemoryModelAvailabilityVisibilityChains = false;
1231                         features->shaderOutputViewportIndex = true;
1232                         features->shaderOutputLayer = true;
1233                         features->subgroupBroadcastDynamicId = true;
1234                         break;
1235                 }
1236                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_FEATURES_EXT: {
1237                         VkPhysicalDeviceLineRasterizationFeaturesEXT *features =
1238                                 (VkPhysicalDeviceLineRasterizationFeaturesEXT *)ext;
1239                         features->rectangularLines = false;
1240                         features->bresenhamLines = true;
1241                         features->smoothLines = false;
1242                         features->stippledRectangularLines = false;
1243                         features->stippledBresenhamLines = true;
1244                         features->stippledSmoothLines = false;
1245                         break;
1246                 }
1247                 case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: {
1248                         VkDeviceMemoryOverallocationCreateInfoAMD *features =
1249                                 (VkDeviceMemoryOverallocationCreateInfoAMD *)ext;
1250                         features->overallocationBehavior = true;
1251                         break;
1252                 }
1253                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT: {
1254                         VkPhysicalDeviceRobustness2FeaturesEXT *features =
1255                                 (VkPhysicalDeviceRobustness2FeaturesEXT *)ext;
1256                         features->robustBufferAccess2 = true;
1257                         features->robustImageAccess2 = true;
1258                         features->nullDescriptor = true;
1259                         break;
1260                 }
1261                 default:
1262                         break;
1263                 }
1264         }
1265         return radv_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features);
1266 }
1267
1268 static size_t
1269 radv_max_descriptor_set_size()
1270 {
1271         /* make sure that the entire descriptor set is addressable with a signed
1272          * 32-bit int. So the sum of all limits scaled by descriptor size has to
1273          * be at most 2 GiB. the combined image & samples object count as one of
1274          * both. This limit is for the pipeline layout, not for the set layout, but
1275          * there is no set limit, so we just set a pipeline limit. I don't think
1276          * any app is going to hit this soon. */
1277         return ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS
1278                              - MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
1279                   (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
1280                    32 /* storage buffer, 32 due to potential space wasted on alignment */ +
1281                    32 /* sampler, largest when combined with image */ +
1282                    64 /* sampled image */ +
1283                    64 /* storage image */);
1284 }
1285
1286 void radv_GetPhysicalDeviceProperties(
1287         VkPhysicalDevice                            physicalDevice,
1288         VkPhysicalDeviceProperties*                 pProperties)
1289 {
1290         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1291         VkSampleCountFlags sample_counts = 0xf;
1292
1293         size_t max_descriptor_set_size = radv_max_descriptor_set_size();
1294
1295         VkPhysicalDeviceLimits limits = {
1296                 .maxImageDimension1D                      = (1 << 14),
1297                 .maxImageDimension2D                      = (1 << 14),
1298                 .maxImageDimension3D                      = (1 << 11),
1299                 .maxImageDimensionCube                    = (1 << 14),
1300                 .maxImageArrayLayers                      = (1 << 11),
1301                 .maxTexelBufferElements                   = UINT32_MAX,
1302                 .maxUniformBufferRange                    = UINT32_MAX,
1303                 .maxStorageBufferRange                    = UINT32_MAX,
1304                 .maxPushConstantsSize                     = MAX_PUSH_CONSTANTS_SIZE,
1305                 .maxMemoryAllocationCount                 = UINT32_MAX,
1306                 .maxSamplerAllocationCount                = 64 * 1024,
1307                 .bufferImageGranularity                   = 64, /* A cache line */
1308                 .sparseAddressSpaceSize                   = RADV_MAX_MEMORY_ALLOCATION_SIZE, /* buffer max size */
1309                 .maxBoundDescriptorSets                   = MAX_SETS,
1310                 .maxPerStageDescriptorSamplers            = max_descriptor_set_size,
1311                 .maxPerStageDescriptorUniformBuffers      = max_descriptor_set_size,
1312                 .maxPerStageDescriptorStorageBuffers      = max_descriptor_set_size,
1313                 .maxPerStageDescriptorSampledImages       = max_descriptor_set_size,
1314                 .maxPerStageDescriptorStorageImages       = max_descriptor_set_size,
1315                 .maxPerStageDescriptorInputAttachments    = max_descriptor_set_size,
1316                 .maxPerStageResources                     = max_descriptor_set_size,
1317                 .maxDescriptorSetSamplers                 = max_descriptor_set_size,
1318                 .maxDescriptorSetUniformBuffers           = max_descriptor_set_size,
1319                 .maxDescriptorSetUniformBuffersDynamic    = MAX_DYNAMIC_UNIFORM_BUFFERS,
1320                 .maxDescriptorSetStorageBuffers           = max_descriptor_set_size,
1321                 .maxDescriptorSetStorageBuffersDynamic    = MAX_DYNAMIC_STORAGE_BUFFERS,
1322                 .maxDescriptorSetSampledImages            = max_descriptor_set_size,
1323                 .maxDescriptorSetStorageImages            = max_descriptor_set_size,
1324                 .maxDescriptorSetInputAttachments         = max_descriptor_set_size,
1325                 .maxVertexInputAttributes                 = MAX_VERTEX_ATTRIBS,
1326                 .maxVertexInputBindings                   = MAX_VBS,
1327                 .maxVertexInputAttributeOffset            = 2047,
1328                 .maxVertexInputBindingStride              = 2048,
1329                 .maxVertexOutputComponents                = 128,
1330                 .maxTessellationGenerationLevel           = 64,
1331                 .maxTessellationPatchSize                 = 32,
1332                 .maxTessellationControlPerVertexInputComponents = 128,
1333                 .maxTessellationControlPerVertexOutputComponents = 128,
1334                 .maxTessellationControlPerPatchOutputComponents = 120,
1335                 .maxTessellationControlTotalOutputComponents = 4096,
1336                 .maxTessellationEvaluationInputComponents = 128,
1337                 .maxTessellationEvaluationOutputComponents = 128,
1338                 .maxGeometryShaderInvocations             = 127,
1339                 .maxGeometryInputComponents               = 64,
1340                 .maxGeometryOutputComponents              = 128,
1341                 .maxGeometryOutputVertices                = 256,
1342                 .maxGeometryTotalOutputComponents         = 1024,
1343                 .maxFragmentInputComponents               = 128,
1344                 .maxFragmentOutputAttachments             = 8,
1345                 .maxFragmentDualSrcAttachments            = 1,
1346                 .maxFragmentCombinedOutputResources       = 8,
1347                 .maxComputeSharedMemorySize               = 32768,
1348                 .maxComputeWorkGroupCount                 = { 65535, 65535, 65535 },
1349                 .maxComputeWorkGroupInvocations           = 1024,
1350                 .maxComputeWorkGroupSize = {
1351                         1024,
1352                         1024,
1353                         1024
1354                 },
1355                 .subPixelPrecisionBits                    = 8,
1356                 .subTexelPrecisionBits                    = 8,
1357                 .mipmapPrecisionBits                      = 8,
1358                 .maxDrawIndexedIndexValue                 = UINT32_MAX,
1359                 .maxDrawIndirectCount                     = UINT32_MAX,
1360                 .maxSamplerLodBias                        = 16,
1361                 .maxSamplerAnisotropy                     = 16,
1362                 .maxViewports                             = MAX_VIEWPORTS,
1363                 .maxViewportDimensions                    = { (1 << 14), (1 << 14) },
1364                 .viewportBoundsRange                      = { INT16_MIN, INT16_MAX },
1365                 .viewportSubPixelBits                     = 8,
1366                 .minMemoryMapAlignment                    = 4096, /* A page */
1367                 .minTexelBufferOffsetAlignment            = 4,
1368                 .minUniformBufferOffsetAlignment          = 4,
1369                 .minStorageBufferOffsetAlignment          = 4,
1370                 .minTexelOffset                           = -32,
1371                 .maxTexelOffset                           = 31,
1372                 .minTexelGatherOffset                     = -32,
1373                 .maxTexelGatherOffset                     = 31,
1374                 .minInterpolationOffset                   = -2,
1375                 .maxInterpolationOffset                   = 2,
1376                 .subPixelInterpolationOffsetBits          = 8,
1377                 .maxFramebufferWidth                      = (1 << 14),
1378                 .maxFramebufferHeight                     = (1 << 14),
1379                 .maxFramebufferLayers                     = (1 << 10),
1380                 .framebufferColorSampleCounts             = sample_counts,
1381                 .framebufferDepthSampleCounts             = sample_counts,
1382                 .framebufferStencilSampleCounts           = sample_counts,
1383                 .framebufferNoAttachmentsSampleCounts     = sample_counts,
1384                 .maxColorAttachments                      = MAX_RTS,
1385                 .sampledImageColorSampleCounts            = sample_counts,
1386                 .sampledImageIntegerSampleCounts          = sample_counts,
1387                 .sampledImageDepthSampleCounts            = sample_counts,
1388                 .sampledImageStencilSampleCounts          = sample_counts,
1389                 .storageImageSampleCounts                 = sample_counts,
1390                 .maxSampleMaskWords                       = 1,
1391                 .timestampComputeAndGraphics              = true,
1392                 .timestampPeriod                          = 1000000.0 / pdevice->rad_info.clock_crystal_freq,
1393                 .maxClipDistances                         = 8,
1394                 .maxCullDistances                         = 8,
1395                 .maxCombinedClipAndCullDistances          = 8,
1396                 .discreteQueuePriorities                  = 2,
1397                 .pointSizeRange                           = { 0.0, 8192.0 },
1398                 .lineWidthRange                           = { 0.0, 8192.0 },
1399                 .pointSizeGranularity                     = (1.0 / 8.0),
1400                 .lineWidthGranularity                     = (1.0 / 8.0),
1401                 .strictLines                              = false, /* FINISHME */
1402                 .standardSampleLocations                  = true,
1403                 .optimalBufferCopyOffsetAlignment         = 128,
1404                 .optimalBufferCopyRowPitchAlignment       = 128,
1405                 .nonCoherentAtomSize                      = 64,
1406         };
1407
1408         *pProperties = (VkPhysicalDeviceProperties) {
1409                 .apiVersion = radv_physical_device_api_version(pdevice),
1410                 .driverVersion = vk_get_driver_version(),
1411                 .vendorID = ATI_VENDOR_ID,
1412                 .deviceID = pdevice->rad_info.pci_id,
1413                 .deviceType = pdevice->rad_info.has_dedicated_vram ? VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU : VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
1414                 .limits = limits,
1415                 .sparseProperties = {0},
1416         };
1417
1418         strcpy(pProperties->deviceName, pdevice->name);
1419         memcpy(pProperties->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE);
1420 }
1421
1422 static void
1423 radv_get_physical_device_properties_1_1(struct radv_physical_device *pdevice,
1424                                         VkPhysicalDeviceVulkan11Properties *p)
1425 {
1426         assert(p->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES);
1427
1428         memcpy(p->deviceUUID, pdevice->device_uuid, VK_UUID_SIZE);
1429         memcpy(p->driverUUID, pdevice->driver_uuid, VK_UUID_SIZE);
1430         memset(p->deviceLUID, 0, VK_LUID_SIZE);
1431         /* The LUID is for Windows. */
1432         p->deviceLUIDValid = false;
1433         p->deviceNodeMask = 0;
1434
1435         p->subgroupSize = RADV_SUBGROUP_SIZE;
1436         p->subgroupSupportedStages = VK_SHADER_STAGE_ALL_GRAPHICS |
1437                                      VK_SHADER_STAGE_COMPUTE_BIT;
1438         p->subgroupSupportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT |
1439                                          VK_SUBGROUP_FEATURE_VOTE_BIT |
1440                                          VK_SUBGROUP_FEATURE_ARITHMETIC_BIT |
1441                                          VK_SUBGROUP_FEATURE_BALLOT_BIT |
1442                                          VK_SUBGROUP_FEATURE_CLUSTERED_BIT |
1443                                          VK_SUBGROUP_FEATURE_QUAD_BIT;
1444
1445         if (((pdevice->rad_info.chip_class == GFX6 ||
1446               pdevice->rad_info.chip_class == GFX7) && !pdevice->use_aco) ||
1447             pdevice->rad_info.chip_class >= GFX8) {
1448                 p->subgroupSupportedOperations |= VK_SUBGROUP_FEATURE_SHUFFLE_BIT |
1449                                                   VK_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT;
1450         }
1451         p->subgroupQuadOperationsInAllStages = true;
1452
1453         p->pointClippingBehavior = VK_POINT_CLIPPING_BEHAVIOR_ALL_CLIP_PLANES;
1454         p->maxMultiviewViewCount = MAX_VIEWS;
1455         p->maxMultiviewInstanceIndex = INT_MAX;
1456         p->protectedNoFault = false;
1457         p->maxPerSetDescriptors = RADV_MAX_PER_SET_DESCRIPTORS;
1458         p->maxMemoryAllocationSize = RADV_MAX_MEMORY_ALLOCATION_SIZE;
1459 }
1460
1461 static void
1462 radv_get_physical_device_properties_1_2(struct radv_physical_device *pdevice,
1463                                         VkPhysicalDeviceVulkan12Properties *p)
1464 {
1465         assert(p->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES);
1466
1467         p->driverID = VK_DRIVER_ID_MESA_RADV;
1468         snprintf(p->driverName, VK_MAX_DRIVER_NAME_SIZE, "radv");
1469         snprintf(p->driverInfo, VK_MAX_DRIVER_INFO_SIZE,
1470                  "Mesa " PACKAGE_VERSION MESA_GIT_SHA1
1471                  " (LLVM " MESA_LLVM_VERSION_STRING ")");
1472         p->conformanceVersion = (VkConformanceVersion) {
1473                 .major = 1,
1474                 .minor = 2,
1475                 .subminor = 0,
1476                 .patch = 0,
1477         };
1478
1479         /* On AMD hardware, denormals and rounding modes for fp16/fp64 are
1480          * controlled by the same config register.
1481          */
1482         if (pdevice->rad_info.has_double_rate_fp16) {
1483                 p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR;
1484                 p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_32_BIT_ONLY_KHR;
1485         } else {
1486                 p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL_KHR;
1487                 p->roundingModeIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL_KHR;
1488         }
1489
1490         /* Do not allow both preserving and flushing denorms because different
1491          * shaders in the same pipeline can have different settings and this
1492          * won't work for merged shaders. To make it work, this requires LLVM
1493          * support for changing the register. The same logic applies for the
1494          * rounding modes because they are configured with the same config
1495          * register. TODO: we can enable a lot of these for ACO when it
1496          * supports all stages.
1497          */
1498         p->shaderDenormFlushToZeroFloat32 = true;
1499         p->shaderDenormPreserveFloat32 = false;
1500         p->shaderRoundingModeRTEFloat32 = true;
1501         p->shaderRoundingModeRTZFloat32 = false;
1502         p->shaderSignedZeroInfNanPreserveFloat32 = true;
1503
1504         p->shaderDenormFlushToZeroFloat16 = false;
1505         p->shaderDenormPreserveFloat16 = pdevice->rad_info.has_double_rate_fp16;
1506         p->shaderRoundingModeRTEFloat16 = pdevice->rad_info.has_double_rate_fp16;
1507         p->shaderRoundingModeRTZFloat16 = false;
1508         p->shaderSignedZeroInfNanPreserveFloat16 = pdevice->rad_info.has_double_rate_fp16;
1509
1510         p->shaderDenormFlushToZeroFloat64 = false;
1511         p->shaderDenormPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8;
1512         p->shaderRoundingModeRTEFloat64 = pdevice->rad_info.chip_class >= GFX8;
1513         p->shaderRoundingModeRTZFloat64 = false;
1514         p->shaderSignedZeroInfNanPreserveFloat64 = pdevice->rad_info.chip_class >= GFX8;
1515
1516         p->maxUpdateAfterBindDescriptorsInAllPools = UINT32_MAX / 64;
1517         p->shaderUniformBufferArrayNonUniformIndexingNative = false;
1518         p->shaderSampledImageArrayNonUniformIndexingNative = false;
1519         p->shaderStorageBufferArrayNonUniformIndexingNative = false;
1520         p->shaderStorageImageArrayNonUniformIndexingNative = false;
1521         p->shaderInputAttachmentArrayNonUniformIndexingNative = false;
1522         p->robustBufferAccessUpdateAfterBind = false;
1523         p->quadDivergentImplicitLod = false;
1524
1525         size_t max_descriptor_set_size = ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS -
1526                 MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
1527                         (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
1528                          32 /* storage buffer, 32 due to potential space wasted on alignment */ +
1529                          32 /* sampler, largest when combined with image */ +
1530                          64 /* sampled image */ +
1531                          64 /* storage image */);
1532         p->maxPerStageDescriptorUpdateAfterBindSamplers = max_descriptor_set_size;
1533         p->maxPerStageDescriptorUpdateAfterBindUniformBuffers = max_descriptor_set_size;
1534         p->maxPerStageDescriptorUpdateAfterBindStorageBuffers = max_descriptor_set_size;
1535         p->maxPerStageDescriptorUpdateAfterBindSampledImages = max_descriptor_set_size;
1536         p->maxPerStageDescriptorUpdateAfterBindStorageImages = max_descriptor_set_size;
1537         p->maxPerStageDescriptorUpdateAfterBindInputAttachments = max_descriptor_set_size;
1538         p->maxPerStageUpdateAfterBindResources = max_descriptor_set_size;
1539         p->maxDescriptorSetUpdateAfterBindSamplers = max_descriptor_set_size;
1540         p->maxDescriptorSetUpdateAfterBindUniformBuffers = max_descriptor_set_size;
1541         p->maxDescriptorSetUpdateAfterBindUniformBuffersDynamic = MAX_DYNAMIC_UNIFORM_BUFFERS;
1542         p->maxDescriptorSetUpdateAfterBindStorageBuffers = max_descriptor_set_size;
1543         p->maxDescriptorSetUpdateAfterBindStorageBuffersDynamic = MAX_DYNAMIC_STORAGE_BUFFERS;
1544         p->maxDescriptorSetUpdateAfterBindSampledImages = max_descriptor_set_size;
1545         p->maxDescriptorSetUpdateAfterBindStorageImages = max_descriptor_set_size;
1546         p->maxDescriptorSetUpdateAfterBindInputAttachments = max_descriptor_set_size;
1547
1548         /* We support all of the depth resolve modes */
1549         p->supportedDepthResolveModes = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR |
1550                                             VK_RESOLVE_MODE_AVERAGE_BIT_KHR |
1551                                             VK_RESOLVE_MODE_MIN_BIT_KHR |
1552                                             VK_RESOLVE_MODE_MAX_BIT_KHR;
1553
1554         /* Average doesn't make sense for stencil so we don't support that */
1555         p->supportedStencilResolveModes = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR |
1556                                               VK_RESOLVE_MODE_MIN_BIT_KHR |
1557                                               VK_RESOLVE_MODE_MAX_BIT_KHR;
1558
1559         p->independentResolveNone = true;
1560         p->independentResolve = true;
1561
1562         /* GFX6-8 only support single channel min/max filter. */
1563         p->filterMinmaxImageComponentMapping = pdevice->rad_info.chip_class >= GFX9;
1564         p->filterMinmaxSingleComponentFormats = true;
1565
1566         p->maxTimelineSemaphoreValueDifference = UINT64_MAX;
1567
1568         p->framebufferIntegerColorSampleCounts = VK_SAMPLE_COUNT_1_BIT;
1569 }
1570
1571 void radv_GetPhysicalDeviceProperties2(
1572         VkPhysicalDevice                            physicalDevice,
1573         VkPhysicalDeviceProperties2                *pProperties)
1574 {
1575         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1576         radv_GetPhysicalDeviceProperties(physicalDevice, &pProperties->properties);
1577
1578         VkPhysicalDeviceVulkan11Properties core_1_1 = {
1579                 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES,
1580         };
1581         radv_get_physical_device_properties_1_1(pdevice, &core_1_1);
1582
1583         VkPhysicalDeviceVulkan12Properties core_1_2 = {
1584                 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES,
1585         };
1586         radv_get_physical_device_properties_1_2(pdevice, &core_1_2);
1587
1588 #define CORE_RENAMED_PROPERTY(major, minor, ext_property, core_property) \
1589    memcpy(&properties->ext_property, &core_##major##_##minor.core_property, \
1590           sizeof(core_##major##_##minor.core_property))
1591
1592 #define CORE_PROPERTY(major, minor, property) \
1593    CORE_RENAMED_PROPERTY(major, minor, property, property)
1594
1595         vk_foreach_struct(ext, pProperties->pNext) {
1596                 switch (ext->sType) {
1597                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR: {
1598                         VkPhysicalDevicePushDescriptorPropertiesKHR *properties =
1599                                 (VkPhysicalDevicePushDescriptorPropertiesKHR *) ext;
1600                         properties->maxPushDescriptors = MAX_PUSH_DESCRIPTORS;
1601                         break;
1602                 }
1603                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES: {
1604                         VkPhysicalDeviceIDProperties *properties = (VkPhysicalDeviceIDProperties*)ext;
1605                         CORE_PROPERTY(1, 1, deviceUUID);
1606                         CORE_PROPERTY(1, 1, driverUUID);
1607                         CORE_PROPERTY(1, 1, deviceLUID);
1608                         CORE_PROPERTY(1, 1, deviceLUIDValid);
1609                         break;
1610                 }
1611                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_PROPERTIES: {
1612                         VkPhysicalDeviceMultiviewProperties *properties = (VkPhysicalDeviceMultiviewProperties*)ext;
1613                         CORE_PROPERTY(1, 1, maxMultiviewViewCount);
1614                         CORE_PROPERTY(1, 1, maxMultiviewInstanceIndex);
1615                         break;
1616                 }
1617                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_POINT_CLIPPING_PROPERTIES: {
1618                         VkPhysicalDevicePointClippingProperties *properties =
1619                             (VkPhysicalDevicePointClippingProperties*)ext;
1620                         CORE_PROPERTY(1, 1, pointClippingBehavior);
1621                         break;
1622                 }
1623                 case  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DISCARD_RECTANGLE_PROPERTIES_EXT: {
1624                         VkPhysicalDeviceDiscardRectanglePropertiesEXT *properties =
1625                             (VkPhysicalDeviceDiscardRectanglePropertiesEXT*)ext;
1626                         properties->maxDiscardRectangles = MAX_DISCARD_RECTANGLES;
1627                         break;
1628                 }
1629                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT: {
1630                         VkPhysicalDeviceExternalMemoryHostPropertiesEXT *properties =
1631                             (VkPhysicalDeviceExternalMemoryHostPropertiesEXT *) ext;
1632                         properties->minImportedHostPointerAlignment = 4096;
1633                         break;
1634                 }
1635                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES: {
1636                         VkPhysicalDeviceSubgroupProperties *properties =
1637                             (VkPhysicalDeviceSubgroupProperties*)ext;
1638                         CORE_PROPERTY(1, 1, subgroupSize);
1639                         CORE_RENAMED_PROPERTY(1, 1, supportedStages,
1640                                                     subgroupSupportedStages);
1641                         CORE_RENAMED_PROPERTY(1, 1, supportedOperations,
1642                                                     subgroupSupportedOperations);
1643                         CORE_RENAMED_PROPERTY(1, 1, quadOperationsInAllStages,
1644                                                     subgroupQuadOperationsInAllStages);
1645                         break;
1646                 }
1647                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES: {
1648                         VkPhysicalDeviceMaintenance3Properties *properties =
1649                             (VkPhysicalDeviceMaintenance3Properties*)ext;
1650                         CORE_PROPERTY(1, 1, maxPerSetDescriptors);
1651                         CORE_PROPERTY(1, 1, maxMemoryAllocationSize);
1652                         break;
1653                 }
1654                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_FILTER_MINMAX_PROPERTIES: {
1655                         VkPhysicalDeviceSamplerFilterMinmaxProperties *properties =
1656                                 (VkPhysicalDeviceSamplerFilterMinmaxProperties *)ext;
1657                         CORE_PROPERTY(1, 2, filterMinmaxImageComponentMapping);
1658                         CORE_PROPERTY(1, 2, filterMinmaxSingleComponentFormats);
1659                         break;
1660                 }
1661                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CORE_PROPERTIES_AMD: {
1662                         VkPhysicalDeviceShaderCorePropertiesAMD *properties =
1663                                 (VkPhysicalDeviceShaderCorePropertiesAMD *)ext;
1664
1665                         /* Shader engines. */
1666                         properties->shaderEngineCount =
1667                                 pdevice->rad_info.max_se;
1668                         properties->shaderArraysPerEngineCount =
1669                                 pdevice->rad_info.max_sh_per_se;
1670                         properties->computeUnitsPerShaderArray =
1671                                 pdevice->rad_info.num_good_cu_per_sh;
1672                         properties->simdPerComputeUnit =
1673                                 pdevice->rad_info.num_simd_per_compute_unit;
1674                         properties->wavefrontsPerSimd =
1675                                 pdevice->rad_info.max_wave64_per_simd;
1676                         properties->wavefrontSize = 64;
1677
1678                         /* SGPR. */
1679                         properties->sgprsPerSimd =
1680                                 pdevice->rad_info.num_physical_sgprs_per_simd;
1681                         properties->minSgprAllocation =
1682                                 pdevice->rad_info.min_sgpr_alloc;
1683                         properties->maxSgprAllocation =
1684                                 pdevice->rad_info.max_sgpr_alloc;
1685                         properties->sgprAllocationGranularity =
1686                                 pdevice->rad_info.sgpr_alloc_granularity;
1687
1688                         /* VGPR. */
1689                         properties->vgprsPerSimd =
1690                                 pdevice->rad_info.num_physical_wave64_vgprs_per_simd;
1691                         properties->minVgprAllocation =
1692                                 pdevice->rad_info.min_wave64_vgpr_alloc;
1693                         properties->maxVgprAllocation =
1694                                 pdevice->rad_info.max_vgpr_alloc;
1695                         properties->vgprAllocationGranularity =
1696                                 pdevice->rad_info.wave64_vgpr_alloc_granularity;
1697                         break;
1698                 }
1699                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CORE_PROPERTIES_2_AMD: {
1700                         VkPhysicalDeviceShaderCoreProperties2AMD *properties =
1701                                 (VkPhysicalDeviceShaderCoreProperties2AMD *)ext;
1702
1703                         properties->shaderCoreFeatures = 0;
1704                         properties->activeComputeUnitCount =
1705                                 pdevice->rad_info.num_good_compute_units;
1706                         break;
1707                 }
1708                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_PROPERTIES_EXT: {
1709                         VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT *properties =
1710                                 (VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT *)ext;
1711                         properties->maxVertexAttribDivisor = UINT32_MAX;
1712                         break;
1713                 }
1714                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES: {
1715                         VkPhysicalDeviceDescriptorIndexingProperties *properties =
1716                                 (VkPhysicalDeviceDescriptorIndexingProperties*)ext;
1717                         CORE_PROPERTY(1, 2, maxUpdateAfterBindDescriptorsInAllPools);
1718                         CORE_PROPERTY(1, 2, shaderUniformBufferArrayNonUniformIndexingNative);
1719                         CORE_PROPERTY(1, 2, shaderSampledImageArrayNonUniformIndexingNative);
1720                         CORE_PROPERTY(1, 2, shaderStorageBufferArrayNonUniformIndexingNative);
1721                         CORE_PROPERTY(1, 2, shaderStorageImageArrayNonUniformIndexingNative);
1722                         CORE_PROPERTY(1, 2, shaderInputAttachmentArrayNonUniformIndexingNative);
1723                         CORE_PROPERTY(1, 2, robustBufferAccessUpdateAfterBind);
1724                         CORE_PROPERTY(1, 2, quadDivergentImplicitLod);
1725                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindSamplers);
1726                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindUniformBuffers);
1727                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindStorageBuffers);
1728                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindSampledImages);
1729                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindStorageImages);
1730                         CORE_PROPERTY(1, 2, maxPerStageDescriptorUpdateAfterBindInputAttachments);
1731                         CORE_PROPERTY(1, 2, maxPerStageUpdateAfterBindResources);
1732                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindSamplers);
1733                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindUniformBuffers);
1734                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindUniformBuffersDynamic);
1735                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageBuffers);
1736                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageBuffersDynamic);
1737                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindSampledImages);
1738                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindStorageImages);
1739                         CORE_PROPERTY(1, 2, maxDescriptorSetUpdateAfterBindInputAttachments);
1740                         break;
1741                 }
1742                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROTECTED_MEMORY_PROPERTIES: {
1743                         VkPhysicalDeviceProtectedMemoryProperties *properties =
1744                                 (VkPhysicalDeviceProtectedMemoryProperties *)ext;
1745                         CORE_PROPERTY(1, 1, protectedNoFault);
1746                         break;
1747                 }
1748                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONSERVATIVE_RASTERIZATION_PROPERTIES_EXT: {
1749                         VkPhysicalDeviceConservativeRasterizationPropertiesEXT *properties =
1750                                 (VkPhysicalDeviceConservativeRasterizationPropertiesEXT *)ext;
1751                         properties->primitiveOverestimationSize = 0;
1752                         properties->maxExtraPrimitiveOverestimationSize = 0;
1753                         properties->extraPrimitiveOverestimationSizeGranularity = 0;
1754                         properties->primitiveUnderestimation = false;
1755                         properties->conservativePointAndLineRasterization = false;
1756                         properties->degenerateTrianglesRasterized = false;
1757                         properties->degenerateLinesRasterized = false;
1758                         properties->fullyCoveredFragmentShaderInputVariable = false;
1759                         properties->conservativeRasterizationPostDepthCoverage = false;
1760                         break;
1761                 }
1762                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT: {
1763                         VkPhysicalDevicePCIBusInfoPropertiesEXT *properties =
1764                                 (VkPhysicalDevicePCIBusInfoPropertiesEXT *)ext;
1765                         properties->pciDomain = pdevice->bus_info.domain;
1766                         properties->pciBus = pdevice->bus_info.bus;
1767                         properties->pciDevice = pdevice->bus_info.dev;
1768                         properties->pciFunction = pdevice->bus_info.func;
1769                         break;
1770                 }
1771                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES: {
1772                         VkPhysicalDeviceDriverProperties *properties =
1773                                 (VkPhysicalDeviceDriverProperties *) ext;
1774                         CORE_PROPERTY(1, 2, driverID);
1775                         CORE_PROPERTY(1, 2, driverName);
1776                         CORE_PROPERTY(1, 2, driverInfo);
1777                         CORE_PROPERTY(1, 2, conformanceVersion);
1778                         break;
1779                 }
1780                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_PROPERTIES_EXT: {
1781                         VkPhysicalDeviceTransformFeedbackPropertiesEXT *properties =
1782                                 (VkPhysicalDeviceTransformFeedbackPropertiesEXT *)ext;
1783                         properties->maxTransformFeedbackStreams = MAX_SO_STREAMS;
1784                         properties->maxTransformFeedbackBuffers = MAX_SO_BUFFERS;
1785                         properties->maxTransformFeedbackBufferSize = UINT32_MAX;
1786                         properties->maxTransformFeedbackStreamDataSize = 512;
1787                         properties->maxTransformFeedbackBufferDataSize = UINT32_MAX;
1788                         properties->maxTransformFeedbackBufferDataStride = 512;
1789                         properties->transformFeedbackQueries = !pdevice->use_ngg_streamout;
1790                         properties->transformFeedbackStreamsLinesTriangles = !pdevice->use_ngg_streamout;
1791                         properties->transformFeedbackRasterizationStreamSelect = false;
1792                         properties->transformFeedbackDraw = true;
1793                         break;
1794                 }
1795                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_INLINE_UNIFORM_BLOCK_PROPERTIES_EXT: {
1796                         VkPhysicalDeviceInlineUniformBlockPropertiesEXT *props =
1797                                 (VkPhysicalDeviceInlineUniformBlockPropertiesEXT *)ext;
1798
1799                         props->maxInlineUniformBlockSize = MAX_INLINE_UNIFORM_BLOCK_SIZE;
1800                         props->maxPerStageDescriptorInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_SETS;
1801                         props->maxPerStageDescriptorUpdateAfterBindInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_SETS;
1802                         props->maxDescriptorSetInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_COUNT;
1803                         props->maxDescriptorSetUpdateAfterBindInlineUniformBlocks = MAX_INLINE_UNIFORM_BLOCK_COUNT;
1804                         break;
1805                 }
1806                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLE_LOCATIONS_PROPERTIES_EXT: {
1807                         VkPhysicalDeviceSampleLocationsPropertiesEXT *properties =
1808                                 (VkPhysicalDeviceSampleLocationsPropertiesEXT *)ext;
1809                         properties->sampleLocationSampleCounts = VK_SAMPLE_COUNT_2_BIT |
1810                                                                  VK_SAMPLE_COUNT_4_BIT |
1811                                                                  VK_SAMPLE_COUNT_8_BIT;
1812                         properties->maxSampleLocationGridSize = (VkExtent2D){ 2 , 2 };
1813                         properties->sampleLocationCoordinateRange[0] = 0.0f;
1814                         properties->sampleLocationCoordinateRange[1] = 0.9375f;
1815                         properties->sampleLocationSubPixelBits = 4;
1816                         properties->variableSampleLocations = false;
1817                         break;
1818                 }
1819                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES: {
1820                         VkPhysicalDeviceDepthStencilResolveProperties *properties =
1821                                 (VkPhysicalDeviceDepthStencilResolveProperties *)ext;
1822                         CORE_PROPERTY(1, 2, supportedDepthResolveModes);
1823                         CORE_PROPERTY(1, 2, supportedStencilResolveModes);
1824                         CORE_PROPERTY(1, 2, independentResolveNone);
1825                         CORE_PROPERTY(1, 2, independentResolve);
1826                         break;
1827                 }
1828                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_PROPERTIES_EXT: {
1829                         VkPhysicalDeviceTexelBufferAlignmentPropertiesEXT *properties =
1830                                 (VkPhysicalDeviceTexelBufferAlignmentPropertiesEXT *)ext;
1831                         properties->storageTexelBufferOffsetAlignmentBytes = 4;
1832                         properties->storageTexelBufferOffsetSingleTexelAlignment = true;
1833                         properties->uniformTexelBufferOffsetAlignmentBytes = 4;
1834                         properties->uniformTexelBufferOffsetSingleTexelAlignment = true;
1835                         break;
1836                 }
1837                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FLOAT_CONTROLS_PROPERTIES : {
1838                         VkPhysicalDeviceFloatControlsProperties *properties =
1839                                 (VkPhysicalDeviceFloatControlsProperties *)ext;
1840                         CORE_PROPERTY(1, 2, denormBehaviorIndependence);
1841                         CORE_PROPERTY(1, 2, roundingModeIndependence);
1842                         CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat16);
1843                         CORE_PROPERTY(1, 2, shaderDenormPreserveFloat16);
1844                         CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat16);
1845                         CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat16);
1846                         CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat16);
1847                         CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat32);
1848                         CORE_PROPERTY(1, 2, shaderDenormPreserveFloat32);
1849                         CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat32);
1850                         CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat32);
1851                         CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat32);
1852                         CORE_PROPERTY(1, 2, shaderDenormFlushToZeroFloat64);
1853                         CORE_PROPERTY(1, 2, shaderDenormPreserveFloat64);
1854                         CORE_PROPERTY(1, 2, shaderRoundingModeRTEFloat64);
1855                         CORE_PROPERTY(1, 2, shaderRoundingModeRTZFloat64);
1856                         CORE_PROPERTY(1, 2, shaderSignedZeroInfNanPreserveFloat64);
1857                         break;
1858                 }
1859                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_PROPERTIES: {
1860                         VkPhysicalDeviceTimelineSemaphoreProperties *properties =
1861                                 (VkPhysicalDeviceTimelineSemaphoreProperties *) ext;
1862                         CORE_PROPERTY(1, 2, maxTimelineSemaphoreValueDifference);
1863                         break;
1864                 }
1865                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT: {
1866                         VkPhysicalDeviceSubgroupSizeControlPropertiesEXT *props =
1867                                 (VkPhysicalDeviceSubgroupSizeControlPropertiesEXT *)ext;
1868                         props->minSubgroupSize = 64;
1869                         props->maxSubgroupSize = 64;
1870                         props->maxComputeWorkgroupSubgroups = UINT32_MAX;
1871                         props->requiredSubgroupSizeStages = 0;
1872
1873                         if (pdevice->rad_info.chip_class >= GFX10) {
1874                                 /* Only GFX10+ supports wave32. */
1875                                 props->minSubgroupSize = 32;
1876                                 props->requiredSubgroupSizeStages = VK_SHADER_STAGE_COMPUTE_BIT;
1877                         }
1878                         break;
1879                 }
1880                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES:
1881                         radv_get_physical_device_properties_1_1(pdevice, (void *)ext);
1882                         break;
1883                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES:
1884                         radv_get_physical_device_properties_1_2(pdevice, (void *)ext);
1885                         break;
1886                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_PROPERTIES_EXT: {
1887                         VkPhysicalDeviceLineRasterizationPropertiesEXT *props =
1888                                 (VkPhysicalDeviceLineRasterizationPropertiesEXT *)ext;
1889                         props->lineSubPixelPrecisionBits = 4;
1890                         break;
1891                 }
1892                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_PROPERTIES_EXT: {
1893                         VkPhysicalDeviceRobustness2PropertiesEXT *properties =
1894                                 (VkPhysicalDeviceRobustness2PropertiesEXT *)ext;
1895                         properties->robustStorageBufferAccessSizeAlignment = 4;
1896                         properties->robustUniformBufferAccessSizeAlignment = 4;
1897                         break;
1898                 }
1899                 default:
1900                         break;
1901                 }
1902         }
1903 }
1904
1905 static void radv_get_physical_device_queue_family_properties(
1906         struct radv_physical_device*                pdevice,
1907         uint32_t*                                   pCount,
1908         VkQueueFamilyProperties**                    pQueueFamilyProperties)
1909 {
1910         int num_queue_families = 1;
1911         int idx;
1912         if (pdevice->rad_info.num_rings[RING_COMPUTE] > 0 &&
1913             !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE))
1914                 num_queue_families++;
1915
1916         if (pQueueFamilyProperties == NULL) {
1917                 *pCount = num_queue_families;
1918                 return;
1919         }
1920
1921         if (!*pCount)
1922                 return;
1923
1924         idx = 0;
1925         if (*pCount >= 1) {
1926                 *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
1927                         .queueFlags = VK_QUEUE_GRAPHICS_BIT |
1928                                       VK_QUEUE_COMPUTE_BIT |
1929                                       VK_QUEUE_TRANSFER_BIT |
1930                                       VK_QUEUE_SPARSE_BINDING_BIT,
1931                         .queueCount = 1,
1932                         .timestampValidBits = 64,
1933                         .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
1934                 };
1935                 idx++;
1936         }
1937
1938         if (pdevice->rad_info.num_rings[RING_COMPUTE] > 0 &&
1939             !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE)) {
1940                 if (*pCount > idx) {
1941                         *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
1942                                 .queueFlags = VK_QUEUE_COMPUTE_BIT |
1943                                               VK_QUEUE_TRANSFER_BIT |
1944                                               VK_QUEUE_SPARSE_BINDING_BIT,
1945                                 .queueCount = pdevice->rad_info.num_rings[RING_COMPUTE],
1946                                 .timestampValidBits = 64,
1947                                 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
1948                         };
1949                         idx++;
1950                 }
1951         }
1952         *pCount = idx;
1953 }
1954
1955 void radv_GetPhysicalDeviceQueueFamilyProperties(
1956         VkPhysicalDevice                            physicalDevice,
1957         uint32_t*                                   pCount,
1958         VkQueueFamilyProperties*                    pQueueFamilyProperties)
1959 {
1960         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1961         if (!pQueueFamilyProperties) {
1962                 radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
1963                 return;
1964         }
1965         VkQueueFamilyProperties *properties[] = {
1966                 pQueueFamilyProperties + 0,
1967                 pQueueFamilyProperties + 1,
1968                 pQueueFamilyProperties + 2,
1969         };
1970         radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
1971         assert(*pCount <= 3);
1972 }
1973
1974 void radv_GetPhysicalDeviceQueueFamilyProperties2(
1975         VkPhysicalDevice                            physicalDevice,
1976         uint32_t*                                   pCount,
1977         VkQueueFamilyProperties2                   *pQueueFamilyProperties)
1978 {
1979         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1980         if (!pQueueFamilyProperties) {
1981                 radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
1982                 return;
1983         }
1984         VkQueueFamilyProperties *properties[] = {
1985                 &pQueueFamilyProperties[0].queueFamilyProperties,
1986                 &pQueueFamilyProperties[1].queueFamilyProperties,
1987                 &pQueueFamilyProperties[2].queueFamilyProperties,
1988         };
1989         radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
1990         assert(*pCount <= 3);
1991 }
1992
1993 void radv_GetPhysicalDeviceMemoryProperties(
1994         VkPhysicalDevice                            physicalDevice,
1995         VkPhysicalDeviceMemoryProperties           *pMemoryProperties)
1996 {
1997         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
1998
1999         *pMemoryProperties = physical_device->memory_properties;
2000 }
2001
2002 static void
2003 radv_get_memory_budget_properties(VkPhysicalDevice physicalDevice,
2004                                   VkPhysicalDeviceMemoryBudgetPropertiesEXT *memoryBudget)
2005 {
2006         RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
2007         VkPhysicalDeviceMemoryProperties *memory_properties = &device->memory_properties;
2008         uint64_t visible_vram_size = radv_get_visible_vram_size(device);
2009         uint64_t vram_size = radv_get_vram_size(device);
2010         uint64_t gtt_size = device->rad_info.gart_size;
2011         uint64_t heap_budget, heap_usage;
2012
2013         /* For all memory heaps, the computation of budget is as follow:
2014          *      heap_budget = heap_size - global_heap_usage + app_heap_usage
2015          *
2016          * The Vulkan spec 1.1.97 says that the budget should include any
2017          * currently allocated device memory.
2018          *
2019          * Note that the application heap usages are not really accurate (eg.
2020          * in presence of shared buffers).
2021          */
2022         for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
2023                 uint32_t heap_index = device->memory_properties.memoryTypes[i].heapIndex;
2024
2025                 if ((device->memory_domains[i] & RADEON_DOMAIN_VRAM) && (device->memory_flags[i] & RADEON_FLAG_NO_CPU_ACCESS)) {
2026                         heap_usage = device->ws->query_value(device->ws,
2027                                                              RADEON_ALLOCATED_VRAM);
2028
2029                         heap_budget = vram_size -
2030                                 device->ws->query_value(device->ws, RADEON_VRAM_USAGE) +
2031                                 heap_usage;
2032
2033                         memoryBudget->heapBudget[heap_index] = heap_budget;
2034                         memoryBudget->heapUsage[heap_index] = heap_usage;
2035                 } else if (device->memory_domains[i] & RADEON_DOMAIN_VRAM) {
2036                         heap_usage = device->ws->query_value(device->ws,
2037                                                              RADEON_ALLOCATED_VRAM_VIS);
2038
2039                         heap_budget = visible_vram_size -
2040                                 device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE) +
2041                                 heap_usage;
2042
2043                         memoryBudget->heapBudget[heap_index] = heap_budget;
2044                         memoryBudget->heapUsage[heap_index] = heap_usage;
2045                 } else {
2046                         assert(device->memory_domains[i] & RADEON_DOMAIN_GTT);
2047
2048                         heap_usage = device->ws->query_value(device->ws,
2049                                                              RADEON_ALLOCATED_GTT);
2050
2051                         heap_budget = gtt_size -
2052                                 device->ws->query_value(device->ws, RADEON_GTT_USAGE) +
2053                                 heap_usage;
2054
2055                         memoryBudget->heapBudget[heap_index] = heap_budget;
2056                         memoryBudget->heapUsage[heap_index] = heap_usage;
2057                 }
2058         }
2059
2060         /* The heapBudget and heapUsage values must be zero for array elements
2061          * greater than or equal to
2062          * VkPhysicalDeviceMemoryProperties::memoryHeapCount.
2063          */
2064         for (uint32_t i = memory_properties->memoryHeapCount; i < VK_MAX_MEMORY_HEAPS; i++) {
2065                 memoryBudget->heapBudget[i] = 0;
2066                 memoryBudget->heapUsage[i] = 0;
2067         }
2068 }
2069
2070 void radv_GetPhysicalDeviceMemoryProperties2(
2071         VkPhysicalDevice                            physicalDevice,
2072         VkPhysicalDeviceMemoryProperties2          *pMemoryProperties)
2073 {
2074         radv_GetPhysicalDeviceMemoryProperties(physicalDevice,
2075                                                &pMemoryProperties->memoryProperties);
2076
2077         VkPhysicalDeviceMemoryBudgetPropertiesEXT *memory_budget =
2078                 vk_find_struct(pMemoryProperties->pNext,
2079                                PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT);
2080         if (memory_budget)
2081                 radv_get_memory_budget_properties(physicalDevice, memory_budget);
2082 }
2083
2084 VkResult radv_GetMemoryHostPointerPropertiesEXT(
2085         VkDevice                                    _device,
2086         VkExternalMemoryHandleTypeFlagBits          handleType,
2087         const void                                 *pHostPointer,
2088         VkMemoryHostPointerPropertiesEXT           *pMemoryHostPointerProperties)
2089 {
2090         RADV_FROM_HANDLE(radv_device, device, _device);
2091
2092         switch (handleType)
2093         {
2094         case VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT: {
2095                 const struct radv_physical_device *physical_device = device->physical_device;
2096                 uint32_t memoryTypeBits = 0;
2097                 for (int i = 0; i < physical_device->memory_properties.memoryTypeCount; i++) {
2098                         if (physical_device->memory_domains[i] == RADEON_DOMAIN_GTT &&
2099                             !(physical_device->memory_flags[i] & RADEON_FLAG_GTT_WC)) {
2100                                 memoryTypeBits = (1 << i);
2101                                 break;
2102                         }
2103                 }
2104                 pMemoryHostPointerProperties->memoryTypeBits = memoryTypeBits;
2105                 return VK_SUCCESS;
2106         }
2107         default:
2108                 return VK_ERROR_INVALID_EXTERNAL_HANDLE;
2109         }
2110 }
2111
2112 static enum radeon_ctx_priority
2113 radv_get_queue_global_priority(const VkDeviceQueueGlobalPriorityCreateInfoEXT *pObj)
2114 {
2115         /* Default to MEDIUM when a specific global priority isn't requested */
2116         if (!pObj)
2117                 return RADEON_CTX_PRIORITY_MEDIUM;
2118
2119         switch(pObj->globalPriority) {
2120         case VK_QUEUE_GLOBAL_PRIORITY_REALTIME_EXT:
2121                 return RADEON_CTX_PRIORITY_REALTIME;
2122         case VK_QUEUE_GLOBAL_PRIORITY_HIGH_EXT:
2123                 return RADEON_CTX_PRIORITY_HIGH;
2124         case VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_EXT:
2125                 return RADEON_CTX_PRIORITY_MEDIUM;
2126         case VK_QUEUE_GLOBAL_PRIORITY_LOW_EXT:
2127                 return RADEON_CTX_PRIORITY_LOW;
2128         default:
2129                 unreachable("Illegal global priority value");
2130                 return RADEON_CTX_PRIORITY_INVALID;
2131         }
2132 }
2133
2134 static int
2135 radv_queue_init(struct radv_device *device, struct radv_queue *queue,
2136                 uint32_t queue_family_index, int idx,
2137                 VkDeviceQueueCreateFlags flags,
2138                 const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority)
2139 {
2140         queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
2141         queue->device = device;
2142         queue->queue_family_index = queue_family_index;
2143         queue->queue_idx = idx;
2144         queue->priority = radv_get_queue_global_priority(global_priority);
2145         queue->flags = flags;
2146
2147         queue->hw_ctx = device->ws->ctx_create(device->ws, queue->priority);
2148         if (!queue->hw_ctx)
2149                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2150
2151         list_inithead(&queue->pending_submissions);
2152         pthread_mutex_init(&queue->pending_mutex, NULL);
2153
2154         return VK_SUCCESS;
2155 }
2156
2157 static void
2158 radv_queue_finish(struct radv_queue *queue)
2159 {
2160         pthread_mutex_destroy(&queue->pending_mutex);
2161
2162         if (queue->hw_ctx)
2163                 queue->device->ws->ctx_destroy(queue->hw_ctx);
2164
2165         if (queue->initial_full_flush_preamble_cs)
2166                 queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
2167         if (queue->initial_preamble_cs)
2168                 queue->device->ws->cs_destroy(queue->initial_preamble_cs);
2169         if (queue->continue_preamble_cs)
2170                 queue->device->ws->cs_destroy(queue->continue_preamble_cs);
2171         if (queue->descriptor_bo)
2172                 queue->device->ws->buffer_destroy(queue->descriptor_bo);
2173         if (queue->scratch_bo)
2174                 queue->device->ws->buffer_destroy(queue->scratch_bo);
2175         if (queue->esgs_ring_bo)
2176                 queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
2177         if (queue->gsvs_ring_bo)
2178                 queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
2179         if (queue->tess_rings_bo)
2180                 queue->device->ws->buffer_destroy(queue->tess_rings_bo);
2181         if (queue->gds_bo)
2182                 queue->device->ws->buffer_destroy(queue->gds_bo);
2183         if (queue->gds_oa_bo)
2184                 queue->device->ws->buffer_destroy(queue->gds_oa_bo);
2185         if (queue->compute_scratch_bo)
2186                 queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
2187 }
2188
2189 static void
2190 radv_bo_list_init(struct radv_bo_list *bo_list)
2191 {
2192         pthread_mutex_init(&bo_list->mutex, NULL);
2193         bo_list->list.count = bo_list->capacity = 0;
2194         bo_list->list.bos = NULL;
2195 }
2196
2197 static void
2198 radv_bo_list_finish(struct radv_bo_list *bo_list)
2199 {
2200         free(bo_list->list.bos);
2201         pthread_mutex_destroy(&bo_list->mutex);
2202 }
2203
2204 VkResult radv_bo_list_add(struct radv_device *device,
2205                           struct radeon_winsys_bo *bo)
2206 {
2207         struct radv_bo_list *bo_list = &device->bo_list;
2208
2209         if (bo->is_local)
2210                 return VK_SUCCESS;
2211
2212         if (unlikely(!device->use_global_bo_list))
2213                 return VK_SUCCESS;
2214
2215         pthread_mutex_lock(&bo_list->mutex);
2216         if (bo_list->list.count == bo_list->capacity) {
2217                 unsigned capacity = MAX2(4, bo_list->capacity * 2);
2218                 void *data = realloc(bo_list->list.bos, capacity * sizeof(struct radeon_winsys_bo*));
2219
2220                 if (!data) {
2221                         pthread_mutex_unlock(&bo_list->mutex);
2222                         return VK_ERROR_OUT_OF_HOST_MEMORY;
2223                 }
2224
2225                 bo_list->list.bos = (struct radeon_winsys_bo**)data;
2226                 bo_list->capacity = capacity;
2227         }
2228
2229         bo_list->list.bos[bo_list->list.count++] = bo;
2230         pthread_mutex_unlock(&bo_list->mutex);
2231         return VK_SUCCESS;
2232 }
2233
2234 void radv_bo_list_remove(struct radv_device *device,
2235                          struct radeon_winsys_bo *bo)
2236 {
2237         struct radv_bo_list *bo_list = &device->bo_list;
2238
2239         if (bo->is_local)
2240                 return;
2241
2242         if (unlikely(!device->use_global_bo_list))
2243                 return;
2244
2245         pthread_mutex_lock(&bo_list->mutex);
2246         /* Loop the list backwards so we find the most recently added
2247          * memory first. */
2248         for(unsigned i = bo_list->list.count; i-- > 0;) {
2249                 if (bo_list->list.bos[i] == bo) {
2250                         bo_list->list.bos[i] = bo_list->list.bos[bo_list->list.count - 1];
2251                         --bo_list->list.count;
2252                         break;
2253                 }
2254         }
2255         pthread_mutex_unlock(&bo_list->mutex);
2256 }
2257
2258 static void
2259 radv_device_init_gs_info(struct radv_device *device)
2260 {
2261         device->gs_table_depth = ac_get_gs_table_depth(device->physical_device->rad_info.chip_class,
2262                                                        device->physical_device->rad_info.family);
2263 }
2264
2265 static int radv_get_device_extension_index(const char *name)
2266 {
2267         for (unsigned i = 0; i < RADV_DEVICE_EXTENSION_COUNT; ++i) {
2268                 if (strcmp(name, radv_device_extensions[i].extensionName) == 0)
2269                         return i;
2270         }
2271         return -1;
2272 }
2273
2274 static int
2275 radv_get_int_debug_option(const char *name, int default_value)
2276 {
2277         const char *str;
2278         int result;
2279
2280         str = getenv(name);
2281         if (!str) {
2282                 result = default_value;
2283         } else {
2284                 char *endptr;
2285
2286                 result = strtol(str, &endptr, 0);
2287                 if (str == endptr) {
2288                         /* No digits founs. */
2289                         result = default_value;
2290                 }
2291         }
2292
2293         return result;
2294 }
2295
2296 static int install_seccomp_filter() {
2297
2298         struct sock_filter filter[] = {
2299                 /* Check arch is 64bit x86 */
2300                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, arch))),
2301                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 0, 12),
2302
2303                 /* Futex is required for mutex locks */
2304                 #if defined __NR__newselect
2305                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2306                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR__newselect, 11, 0),
2307                 #elif defined __NR_select
2308                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2309                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_select, 11, 0),
2310                 #else
2311                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2312                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_pselect6, 11, 0),
2313                 #endif
2314
2315                 /* Allow system exit calls for the forked process */
2316                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2317                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_exit_group, 9, 0),
2318
2319                 /* Allow system read calls */
2320                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2321                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_read, 7, 0),
2322
2323                 /* Allow system write calls */
2324                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2325                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_write, 5, 0),
2326
2327                 /* Allow system brk calls (we need this for malloc) */
2328                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2329                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_brk, 3, 0),
2330
2331                 /* Futex is required for mutex locks */
2332                 BPF_STMT(BPF_LD + BPF_W + BPF_ABS, (offsetof(struct seccomp_data, nr))),
2333                 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, __NR_futex, 1, 0),
2334
2335                 /* Return error if we hit a system call not on the whitelist */
2336                 BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA)),
2337
2338                 /* Allow whitelisted system calls */
2339                 BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW),
2340         };
2341
2342         struct sock_fprog prog = {
2343                 .len = (unsigned short)(sizeof(filter) / sizeof(filter[0])),
2344                 .filter = filter,
2345         };
2346
2347         if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0))
2348                 return -1;
2349
2350         if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog))
2351                 return -1;
2352
2353         return 0;
2354 }
2355
2356 /* Helper function with timeout support for reading from the pipe between
2357  * processes used for secure compile.
2358  */
2359 bool radv_sc_read(int fd, void *buf, size_t size, bool timeout)
2360 {
2361         fd_set fds;
2362         struct timeval tv;
2363
2364         FD_ZERO(&fds);
2365         FD_SET(fd, &fds);
2366
2367         while (true) {
2368                 /* We can't rely on the value of tv after calling select() so
2369                  * we must reset it on each iteration of the loop.
2370                  */
2371                 tv.tv_sec = 5;
2372                 tv.tv_usec = 0;
2373
2374                 int rval = select(fd + 1, &fds, NULL, NULL, timeout ? &tv : NULL);
2375
2376                 if (rval == -1) {
2377                         /* select error */
2378                         return false;
2379                 } else if (rval) {
2380                         ssize_t bytes_read = read(fd, buf, size);
2381                         if (bytes_read < 0)
2382                                 return false;
2383
2384                         buf += bytes_read;
2385                         size -= bytes_read;
2386                         if (size == 0)
2387                                 return true;
2388                 } else {
2389                         /* select timeout */
2390                         return false;
2391                 }
2392         }
2393 }
2394
2395 static bool radv_close_all_fds(const int *keep_fds, int keep_fd_count)
2396 {
2397         DIR *d;
2398         struct dirent *dir;
2399         d = opendir("/proc/self/fd");
2400         if (!d)
2401                 return false;
2402         int dir_fd = dirfd(d);
2403
2404         while ((dir = readdir(d)) != NULL) {
2405                 if (dir->d_name[0] == '.')
2406                         continue;
2407
2408                 int fd = atoi(dir->d_name);
2409                 if (fd == dir_fd)
2410                         continue;
2411
2412                 bool keep = false;
2413                 for (int i = 0; !keep && i < keep_fd_count; ++i)
2414                         if (keep_fds[i] == fd)
2415                                 keep = true;
2416
2417                 if (keep)
2418                         continue;
2419
2420                 close(fd);
2421         }
2422         closedir(d);
2423         return true;
2424 }
2425
2426 static bool secure_compile_open_fifo_fds(struct radv_secure_compile_state *sc,
2427                                          int *fd_server, int *fd_client,
2428                                          unsigned process, bool make_fifo)
2429 {
2430         bool result = false;
2431         char *fifo_server_path = NULL;
2432         char *fifo_client_path = NULL;
2433
2434         if (asprintf(&fifo_server_path, "/tmp/radv_server_%s_%u", sc->uid, process) == -1)
2435                 goto open_fifo_exit;
2436
2437         if (asprintf(&fifo_client_path, "/tmp/radv_client_%s_%u", sc->uid, process) == -1)
2438                 goto open_fifo_exit;
2439
2440         if (make_fifo) {
2441                 int file1 = mkfifo(fifo_server_path, 0666);
2442                 if(file1 < 0)
2443                         goto open_fifo_exit;
2444
2445                 int file2 = mkfifo(fifo_client_path, 0666);
2446                 if(file2 < 0)
2447                         goto open_fifo_exit;
2448         }
2449
2450         *fd_server = open(fifo_server_path, O_RDWR);
2451         if(*fd_server < 1)
2452                 goto open_fifo_exit;
2453
2454         *fd_client = open(fifo_client_path, O_RDWR);
2455         if(*fd_client < 1) {
2456                 close(*fd_server);
2457                 goto open_fifo_exit;
2458         }
2459
2460         result = true;
2461
2462 open_fifo_exit:
2463         free(fifo_server_path);
2464         free(fifo_client_path);
2465
2466         return result;
2467 }
2468
2469 static void run_secure_compile_device(struct radv_device *device, unsigned process,
2470                                       int fd_idle_device_output)
2471 {
2472         int fd_secure_input;
2473         int fd_secure_output;
2474         bool fifo_result = secure_compile_open_fifo_fds(device->sc_state,
2475                                                         &fd_secure_input,
2476                                                         &fd_secure_output,
2477                                                         process, false);
2478
2479         enum radv_secure_compile_type sc_type;
2480
2481         const int needed_fds[] = {
2482                 fd_secure_input,
2483                 fd_secure_output,
2484                 fd_idle_device_output,
2485         };
2486
2487         if (!fifo_result || !radv_close_all_fds(needed_fds, ARRAY_SIZE(needed_fds)) ||
2488             install_seccomp_filter() == -1) {
2489                 sc_type = RADV_SC_TYPE_INIT_FAILURE;
2490         } else {
2491                 sc_type = RADV_SC_TYPE_INIT_SUCCESS;
2492                 device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input;
2493                 device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output;
2494         }
2495
2496         write(fd_idle_device_output, &sc_type, sizeof(sc_type));
2497
2498         if (sc_type == RADV_SC_TYPE_INIT_FAILURE)
2499                 goto secure_compile_exit;
2500
2501         while (true) {
2502                 radv_sc_read(fd_secure_input, &sc_type, sizeof(sc_type), false);
2503
2504                 if (sc_type == RADV_SC_TYPE_COMPILE_PIPELINE) {
2505                         struct radv_pipeline *pipeline;
2506                         bool sc_read = true;
2507
2508                         pipeline = vk_zalloc2(&device->alloc, NULL, sizeof(*pipeline), 8,
2509                                               VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2510
2511                         pipeline->device = device;
2512
2513                         /* Read pipeline layout */
2514                         struct radv_pipeline_layout layout;
2515                         sc_read = radv_sc_read(fd_secure_input, &layout, sizeof(struct radv_pipeline_layout), true);
2516                         sc_read &= radv_sc_read(fd_secure_input, &layout.num_sets, sizeof(uint32_t), true);
2517                         if (!sc_read)
2518                                 goto secure_compile_exit;
2519
2520                         for (uint32_t set = 0; set < layout.num_sets; set++) {
2521                                 uint32_t layout_size;
2522                                 sc_read &= radv_sc_read(fd_secure_input, &layout_size, sizeof(uint32_t), true);
2523                                 if (!sc_read)
2524                                         goto secure_compile_exit;
2525
2526                                 layout.set[set].layout = malloc(layout_size);
2527                                 layout.set[set].layout->layout_size = layout_size;
2528                                 sc_read &= radv_sc_read(fd_secure_input, layout.set[set].layout,
2529                                                         layout.set[set].layout->layout_size, true);
2530                         }
2531
2532                         pipeline->layout = &layout;
2533
2534                         /* Read pipeline key */
2535                         struct radv_pipeline_key key;
2536                         sc_read &= radv_sc_read(fd_secure_input, &key, sizeof(struct radv_pipeline_key), true);
2537
2538                         /* Read pipeline create flags */
2539                         VkPipelineCreateFlags flags;
2540                         sc_read &= radv_sc_read(fd_secure_input, &flags, sizeof(VkPipelineCreateFlags), true);
2541
2542                         /* Read stage and shader information */
2543                         uint32_t num_stages;
2544                         const VkPipelineShaderStageCreateInfo *pStages[MESA_SHADER_STAGES] = { 0, };
2545                         sc_read &= radv_sc_read(fd_secure_input, &num_stages, sizeof(uint32_t), true);
2546                         if (!sc_read)
2547                                 goto secure_compile_exit;
2548
2549                         for (uint32_t i = 0; i < num_stages; i++) {
2550
2551                                 /* Read stage */
2552                                 gl_shader_stage stage;
2553                                 sc_read &= radv_sc_read(fd_secure_input, &stage, sizeof(gl_shader_stage), true);
2554
2555                                 VkPipelineShaderStageCreateInfo *pStage = calloc(1, sizeof(VkPipelineShaderStageCreateInfo));
2556
2557                                 /* Read entry point name */
2558                                 size_t name_size;
2559                                 sc_read &= radv_sc_read(fd_secure_input, &name_size, sizeof(size_t), true);
2560                                 if (!sc_read)
2561                                         goto secure_compile_exit;
2562
2563                                 char *ep_name = malloc(name_size);
2564                                 sc_read &= radv_sc_read(fd_secure_input, ep_name, name_size, true);
2565                                 pStage->pName = ep_name;
2566
2567                                 /* Read shader module */
2568                                 size_t module_size;
2569                                 sc_read &= radv_sc_read(fd_secure_input, &module_size, sizeof(size_t), true);
2570                                 if (!sc_read)
2571                                         goto secure_compile_exit;
2572
2573                                 struct radv_shader_module *module = malloc(module_size);
2574                                 sc_read &= radv_sc_read(fd_secure_input, module, module_size, true);
2575                                 pStage->module = radv_shader_module_to_handle(module);
2576
2577                                 /* Read specialization info */
2578                                 bool has_spec_info;
2579                                 sc_read &= radv_sc_read(fd_secure_input, &has_spec_info, sizeof(bool), true);
2580                                 if (!sc_read)
2581                                         goto secure_compile_exit;
2582
2583                                 if (has_spec_info) {
2584                                         VkSpecializationInfo *specInfo = malloc(sizeof(VkSpecializationInfo));
2585                                         pStage->pSpecializationInfo = specInfo;
2586
2587                                         sc_read &= radv_sc_read(fd_secure_input, &specInfo->dataSize, sizeof(size_t), true);
2588                                         if (!sc_read)
2589                                                 goto secure_compile_exit;
2590
2591                                         void *si_data = malloc(specInfo->dataSize);
2592                                         sc_read &= radv_sc_read(fd_secure_input, si_data, specInfo->dataSize, true);
2593                                         specInfo->pData = si_data;
2594
2595                                         sc_read &= radv_sc_read(fd_secure_input, &specInfo->mapEntryCount, sizeof(uint32_t), true);
2596                                         if (!sc_read)
2597                                                 goto secure_compile_exit;
2598
2599                                         VkSpecializationMapEntry *mapEntries = malloc(sizeof(VkSpecializationMapEntry) * specInfo->mapEntryCount);
2600                                         for (uint32_t j = 0; j < specInfo->mapEntryCount; j++) {
2601                                                 sc_read &= radv_sc_read(fd_secure_input, &mapEntries[j], sizeof(VkSpecializationMapEntry), true);
2602                                                 if (!sc_read)
2603                                                         goto secure_compile_exit;
2604                                         }
2605
2606                                         specInfo->pMapEntries = mapEntries;
2607                                 }
2608
2609                                 pStages[stage] = pStage;
2610                         }
2611
2612                         /* Compile the shaders */
2613                         VkPipelineCreationFeedbackEXT *stage_feedbacks[MESA_SHADER_STAGES] = { 0 };
2614                         radv_create_shaders(pipeline, device, NULL, &key, pStages, flags, NULL, stage_feedbacks);
2615
2616                         /* free memory allocated above */
2617                         for (uint32_t set = 0; set < layout.num_sets; set++)
2618                                 free(layout.set[set].layout);
2619
2620                         for (uint32_t i = 0; i < MESA_SHADER_STAGES; i++) {
2621                                 if (!pStages[i])
2622                                         continue;
2623
2624                                 free((void *) pStages[i]->pName);
2625                                 free(radv_shader_module_from_handle(pStages[i]->module));
2626                                 if (pStages[i]->pSpecializationInfo) {
2627                                         free((void *) pStages[i]->pSpecializationInfo->pData);
2628                                         free((void *) pStages[i]->pSpecializationInfo->pMapEntries);
2629                                         free((void *) pStages[i]->pSpecializationInfo);
2630                                 }
2631                                 free((void *) pStages[i]);
2632                         }
2633
2634                         vk_free(&device->alloc, pipeline);
2635
2636                         sc_type = RADV_SC_TYPE_COMPILE_PIPELINE_FINISHED;
2637                         write(fd_secure_output, &sc_type, sizeof(sc_type));
2638
2639                 } else if (sc_type == RADV_SC_TYPE_DESTROY_DEVICE) {
2640                         goto secure_compile_exit;
2641                 }
2642         }
2643
2644 secure_compile_exit:
2645         close(fd_secure_input);
2646         close(fd_secure_output);
2647         close(fd_idle_device_output);
2648         _exit(0);
2649 }
2650
2651 static enum radv_secure_compile_type fork_secure_compile_device(struct radv_device *device, unsigned process)
2652 {
2653         int fd_secure_input[2];
2654         int fd_secure_output[2];
2655
2656         /* create pipe descriptors (used to communicate between processes) */
2657         if (pipe(fd_secure_input) == -1 || pipe(fd_secure_output) == -1)
2658                 return RADV_SC_TYPE_INIT_FAILURE;
2659
2660
2661         int sc_pid;
2662         if ((sc_pid = fork()) == 0) {
2663                 device->sc_state->secure_compile_thread_counter = process;
2664                 run_secure_compile_device(device, process, fd_secure_output[1]);
2665         } else {
2666                 if (sc_pid == -1)
2667                         return RADV_SC_TYPE_INIT_FAILURE;
2668
2669                 /* Read the init result returned from the secure process */
2670                 enum radv_secure_compile_type sc_type;
2671                 bool sc_read = radv_sc_read(fd_secure_output[0], &sc_type, sizeof(sc_type), true);
2672
2673                 if (sc_type == RADV_SC_TYPE_INIT_FAILURE || !sc_read) {
2674                         close(fd_secure_input[0]);
2675                         close(fd_secure_input[1]);
2676                         close(fd_secure_output[1]);
2677                         close(fd_secure_output[0]);
2678                         int status;
2679                         waitpid(sc_pid, &status, 0);
2680
2681                         return RADV_SC_TYPE_INIT_FAILURE;
2682                 } else {
2683                         assert(sc_type == RADV_SC_TYPE_INIT_SUCCESS);
2684                         write(device->sc_state->secure_compile_processes[process].fd_secure_output, &sc_type, sizeof(sc_type));
2685
2686                         close(fd_secure_input[0]);
2687                         close(fd_secure_input[1]);
2688                         close(fd_secure_output[1]);
2689                         close(fd_secure_output[0]);
2690
2691                         int status;
2692                         waitpid(sc_pid, &status, 0);
2693                 }
2694         }
2695
2696         return RADV_SC_TYPE_INIT_SUCCESS;
2697 }
2698
2699 /* Run a bare bones fork of a device that was forked right after its creation.
2700  * This device will have low overhead when it is forked again before each
2701  * pipeline compilation. This device sits idle and its only job is to fork
2702  * itself.
2703  */
2704 static void run_secure_compile_idle_device(struct radv_device *device, unsigned process,
2705                                             int fd_secure_input, int fd_secure_output)
2706 {
2707         enum radv_secure_compile_type sc_type = RADV_SC_TYPE_INIT_SUCCESS;
2708         device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input;
2709         device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output;
2710
2711         write(fd_secure_output, &sc_type, sizeof(sc_type));
2712
2713         while (true) {
2714                 radv_sc_read(fd_secure_input, &sc_type, sizeof(sc_type), false);
2715
2716                 if (sc_type == RADV_SC_TYPE_FORK_DEVICE) {
2717                         sc_type = fork_secure_compile_device(device, process);
2718
2719                         if (sc_type == RADV_SC_TYPE_INIT_FAILURE)
2720                                 goto secure_compile_exit;
2721
2722                 } else if (sc_type == RADV_SC_TYPE_DESTROY_DEVICE) {
2723                         goto secure_compile_exit;
2724                 }
2725         }
2726
2727 secure_compile_exit:
2728         close(fd_secure_input);
2729         close(fd_secure_output);
2730         _exit(0);
2731 }
2732
2733 static void destroy_secure_compile_device(struct radv_device *device, unsigned process)
2734 {
2735         int fd_secure_input = device->sc_state->secure_compile_processes[process].fd_secure_input;
2736
2737         enum radv_secure_compile_type sc_type = RADV_SC_TYPE_DESTROY_DEVICE;
2738         write(fd_secure_input, &sc_type, sizeof(sc_type));
2739
2740         close(device->sc_state->secure_compile_processes[process].fd_secure_input);
2741         close(device->sc_state->secure_compile_processes[process].fd_secure_output);
2742
2743         int status;
2744         waitpid(device->sc_state->secure_compile_processes[process].sc_pid, &status, 0);
2745 }
2746
2747 static VkResult fork_secure_compile_idle_device(struct radv_device *device)
2748 {
2749         device->sc_state = vk_zalloc(&device->alloc,
2750                                      sizeof(struct radv_secure_compile_state),
2751                                      8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2752
2753         mtx_init(&device->sc_state->secure_compile_mutex, mtx_plain);
2754
2755         pid_t upid = getpid();
2756         time_t seconds = time(NULL);
2757
2758         char *uid;
2759         if (asprintf(&uid, "%ld_%ld", (long) upid, (long) seconds) == -1)
2760                 return VK_ERROR_INITIALIZATION_FAILED;
2761
2762         device->sc_state->uid = uid;
2763
2764         uint8_t sc_threads = device->instance->num_sc_threads;
2765         int fd_secure_input[MAX_SC_PROCS][2];
2766         int fd_secure_output[MAX_SC_PROCS][2];
2767
2768         /* create pipe descriptors (used to communicate between processes) */
2769         for (unsigned i = 0; i < sc_threads; i++) {
2770                 if (pipe(fd_secure_input[i]) == -1 ||
2771                     pipe(fd_secure_output[i]) == -1) {
2772                         return VK_ERROR_INITIALIZATION_FAILED;
2773                 }
2774         }
2775
2776         device->sc_state->secure_compile_processes = vk_zalloc(&device->alloc,
2777                                                                 sizeof(struct radv_secure_compile_process) * sc_threads, 8,
2778                                                                 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2779
2780         for (unsigned process = 0; process < sc_threads; process++) {
2781                 if ((device->sc_state->secure_compile_processes[process].sc_pid = fork()) == 0) {
2782                         device->sc_state->secure_compile_thread_counter = process;
2783                         run_secure_compile_idle_device(device, process, fd_secure_input[process][0], fd_secure_output[process][1]);
2784                 } else {
2785                         if (device->sc_state->secure_compile_processes[process].sc_pid == -1)
2786                                 return VK_ERROR_INITIALIZATION_FAILED;
2787
2788                         /* Read the init result returned from the secure process */
2789                         enum radv_secure_compile_type sc_type;
2790                         bool sc_read = radv_sc_read(fd_secure_output[process][0], &sc_type, sizeof(sc_type), true);
2791
2792                         bool fifo_result;
2793                         if (sc_read && sc_type == RADV_SC_TYPE_INIT_SUCCESS) {
2794                                 fifo_result = secure_compile_open_fifo_fds(device->sc_state,
2795                                                                            &device->sc_state->secure_compile_processes[process].fd_server,
2796                                                                            &device->sc_state->secure_compile_processes[process].fd_client,
2797                                                                            process, true);
2798
2799                                 device->sc_state->secure_compile_processes[process].fd_secure_input = fd_secure_input[process][1];
2800                                 device->sc_state->secure_compile_processes[process].fd_secure_output = fd_secure_output[process][0];
2801                         }
2802
2803                         if (sc_type == RADV_SC_TYPE_INIT_FAILURE || !sc_read || !fifo_result) {
2804                                 close(fd_secure_input[process][0]);
2805                                 close(fd_secure_input[process][1]);
2806                                 close(fd_secure_output[process][1]);
2807                                 close(fd_secure_output[process][0]);
2808                                 int status;
2809                                 waitpid(device->sc_state->secure_compile_processes[process].sc_pid, &status, 0);
2810
2811                                 /* Destroy any forks that were created sucessfully */
2812                                 for (unsigned i = 0; i < process; i++) {
2813                                         destroy_secure_compile_device(device, i);
2814                                 }
2815
2816                                 return VK_ERROR_INITIALIZATION_FAILED;
2817                         }
2818                 }
2819         }
2820         return VK_SUCCESS;
2821 }
2822
2823 static void
2824 radv_device_init_dispatch(struct radv_device *device)
2825 {
2826         const struct radv_instance *instance = device->physical_device->instance;
2827         const struct radv_device_dispatch_table *dispatch_table_layer = NULL;
2828         bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
2829         int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
2830
2831         if (radv_thread_trace >= 0) {
2832                 /* Use device entrypoints from the SQTT layer if enabled. */
2833                 dispatch_table_layer = &sqtt_device_dispatch_table;
2834         }
2835
2836         for (unsigned i = 0; i < ARRAY_SIZE(device->dispatch.entrypoints); i++) {
2837                 /* Vulkan requires that entrypoints for extensions which have not been
2838                  * enabled must not be advertised.
2839                  */
2840                 if (!unchecked &&
2841                     !radv_device_entrypoint_is_enabled(i, instance->apiVersion,
2842                                                        &instance->enabled_extensions,
2843                                                        &device->enabled_extensions)) {
2844                         device->dispatch.entrypoints[i] = NULL;
2845                 } else if (dispatch_table_layer &&
2846                            dispatch_table_layer->entrypoints[i]) {
2847                         device->dispatch.entrypoints[i] =
2848                                 dispatch_table_layer->entrypoints[i];
2849                 } else {
2850                         device->dispatch.entrypoints[i] =
2851                                 radv_device_dispatch_table.entrypoints[i];
2852                 }
2853         }
2854 }
2855
2856 static VkResult
2857 radv_create_pthread_cond(pthread_cond_t *cond)
2858 {
2859         pthread_condattr_t condattr;
2860         if (pthread_condattr_init(&condattr)) {
2861                 return VK_ERROR_INITIALIZATION_FAILED;
2862         }
2863
2864         if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
2865                 pthread_condattr_destroy(&condattr);
2866                 return VK_ERROR_INITIALIZATION_FAILED;
2867         }
2868         if (pthread_cond_init(cond, &condattr)) {
2869                 pthread_condattr_destroy(&condattr);
2870                 return VK_ERROR_INITIALIZATION_FAILED;
2871         }
2872         pthread_condattr_destroy(&condattr);
2873         return VK_SUCCESS;
2874 }
2875
2876 static VkResult
2877 check_physical_device_features(VkPhysicalDevice physicalDevice,
2878                                const VkPhysicalDeviceFeatures *features)
2879 {
2880         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2881         VkPhysicalDeviceFeatures supported_features;
2882         radv_GetPhysicalDeviceFeatures(physicalDevice, &supported_features);
2883         VkBool32 *supported_feature = (VkBool32 *)&supported_features;
2884         VkBool32 *enabled_feature = (VkBool32 *)features;
2885         unsigned num_features = sizeof(VkPhysicalDeviceFeatures) / sizeof(VkBool32);
2886         for (uint32_t i = 0; i < num_features; i++) {
2887                 if (enabled_feature[i] && !supported_feature[i])
2888                         return vk_error(physical_device->instance, VK_ERROR_FEATURE_NOT_PRESENT);
2889         }
2890
2891         return VK_SUCCESS;
2892 }
2893
2894 VkResult radv_CreateDevice(
2895         VkPhysicalDevice                            physicalDevice,
2896         const VkDeviceCreateInfo*                   pCreateInfo,
2897         const VkAllocationCallbacks*                pAllocator,
2898         VkDevice*                                   pDevice)
2899 {
2900         RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
2901         VkResult result;
2902         struct radv_device *device;
2903
2904         bool keep_shader_info = false;
2905         bool robust_buffer_access = false;
2906         bool overallocation_disallowed = false;
2907
2908         /* Check enabled features */
2909         if (pCreateInfo->pEnabledFeatures) {
2910                 result = check_physical_device_features(physicalDevice,
2911                                                         pCreateInfo->pEnabledFeatures);
2912                 if (result != VK_SUCCESS)
2913                         return result;
2914
2915                 if (pCreateInfo->pEnabledFeatures->robustBufferAccess)
2916                         robust_buffer_access = true;
2917         }
2918
2919         vk_foreach_struct_const(ext, pCreateInfo->pNext) {
2920                 switch (ext->sType) {
2921                 case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2: {
2922                         const VkPhysicalDeviceFeatures2 *features = (const void *)ext;
2923                         result = check_physical_device_features(physicalDevice,
2924                                                                 &features->features);
2925                         if (result != VK_SUCCESS)
2926                                 return result;
2927
2928                         if (features->features.robustBufferAccess)
2929                                 robust_buffer_access = true;
2930                         break;
2931                 }
2932                 case VK_STRUCTURE_TYPE_DEVICE_MEMORY_OVERALLOCATION_CREATE_INFO_AMD: {
2933                         const VkDeviceMemoryOverallocationCreateInfoAMD *overallocation = (const void *)ext;
2934                         if (overallocation->overallocationBehavior == VK_MEMORY_OVERALLOCATION_BEHAVIOR_DISALLOWED_AMD)
2935                                 overallocation_disallowed = true;
2936                         break;
2937                 }
2938                 default:
2939                         break;
2940                 }
2941         }
2942
2943         device = vk_zalloc2(&physical_device->instance->alloc, pAllocator,
2944                             sizeof(*device), 8,
2945                             VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
2946         if (!device)
2947                 return vk_error(physical_device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2948
2949         device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
2950         device->instance = physical_device->instance;
2951         device->physical_device = physical_device;
2952
2953         device->ws = physical_device->ws;
2954         if (pAllocator)
2955                 device->alloc = *pAllocator;
2956         else
2957                 device->alloc = physical_device->instance->alloc;
2958
2959         for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
2960                 const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i];
2961                 int index = radv_get_device_extension_index(ext_name);
2962                 if (index < 0 || !physical_device->supported_extensions.extensions[index]) {
2963                         vk_free(&device->alloc, device);
2964                         return vk_error(physical_device->instance, VK_ERROR_EXTENSION_NOT_PRESENT);
2965                 }
2966
2967                 device->enabled_extensions.extensions[index] = true;
2968         }
2969
2970         radv_device_init_dispatch(device);
2971
2972         keep_shader_info = device->enabled_extensions.AMD_shader_info;
2973
2974         /* With update after bind we can't attach bo's to the command buffer
2975          * from the descriptor set anymore, so we have to use a global BO list.
2976          */
2977         device->use_global_bo_list =
2978                 (device->instance->perftest_flags & RADV_PERFTEST_BO_LIST) ||
2979                 device->enabled_extensions.EXT_descriptor_indexing ||
2980                 device->enabled_extensions.EXT_buffer_device_address ||
2981                 device->enabled_extensions.KHR_buffer_device_address;
2982
2983         device->robust_buffer_access = robust_buffer_access;
2984
2985         mtx_init(&device->shader_slab_mutex, mtx_plain);
2986         list_inithead(&device->shader_slabs);
2987
2988         device->overallocation_disallowed = overallocation_disallowed;
2989         mtx_init(&device->overallocation_mutex, mtx_plain);
2990
2991         radv_bo_list_init(&device->bo_list);
2992
2993         for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
2994                 const VkDeviceQueueCreateInfo *queue_create = &pCreateInfo->pQueueCreateInfos[i];
2995                 uint32_t qfi = queue_create->queueFamilyIndex;
2996                 const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority =
2997                         vk_find_struct_const(queue_create->pNext, DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT);
2998
2999                 assert(!global_priority || device->physical_device->rad_info.has_ctx_priority);
3000
3001                 device->queues[qfi] = vk_alloc(&device->alloc,
3002                                                queue_create->queueCount * sizeof(struct radv_queue), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
3003                 if (!device->queues[qfi]) {
3004                         result = VK_ERROR_OUT_OF_HOST_MEMORY;
3005                         goto fail;
3006                 }
3007
3008                 memset(device->queues[qfi], 0, queue_create->queueCount * sizeof(struct radv_queue));
3009
3010                 device->queue_count[qfi] = queue_create->queueCount;
3011
3012                 for (unsigned q = 0; q < queue_create->queueCount; q++) {
3013                         result = radv_queue_init(device, &device->queues[qfi][q],
3014                                                  qfi, q, queue_create->flags,
3015                                                  global_priority);
3016                         if (result != VK_SUCCESS)
3017                                 goto fail;
3018                 }
3019         }
3020
3021         device->pbb_allowed = device->physical_device->rad_info.chip_class >= GFX9 &&
3022                               !(device->instance->debug_flags & RADV_DEBUG_NOBINNING);
3023
3024         /* Disable DFSM by default. As of 2019-09-15 Talos on Low is still 3% slower on Raven. */
3025         device->dfsm_allowed = device->pbb_allowed &&
3026                                (device->instance->perftest_flags & RADV_PERFTEST_DFSM);
3027
3028         device->always_use_syncobj = device->physical_device->rad_info.has_syncobj_wait_for_submit;
3029
3030         /* The maximum number of scratch waves. Scratch space isn't divided
3031          * evenly between CUs. The number is only a function of the number of CUs.
3032          * We can decrease the constant to decrease the scratch buffer size.
3033          *
3034          * sctx->scratch_waves must be >= the maximum possible size of
3035          * 1 threadgroup, so that the hw doesn't hang from being unable
3036          * to start any.
3037          *
3038          * The recommended value is 4 per CU at most. Higher numbers don't
3039          * bring much benefit, but they still occupy chip resources (think
3040          * async compute). I've seen ~2% performance difference between 4 and 32.
3041          */
3042         uint32_t max_threads_per_block = 2048;
3043         device->scratch_waves = MAX2(32 * physical_device->rad_info.num_good_compute_units,
3044                                      max_threads_per_block / 64);
3045
3046         device->dispatch_initiator = S_00B800_COMPUTE_SHADER_EN(1);
3047
3048         if (device->physical_device->rad_info.chip_class >= GFX7) {
3049                 /* If the KMD allows it (there is a KMD hw register for it),
3050                  * allow launching waves out-of-order.
3051                  */
3052                 device->dispatch_initiator |= S_00B800_ORDER_MODE(1);
3053         }
3054
3055         radv_device_init_gs_info(device);
3056
3057         device->tess_offchip_block_dw_size =
3058                 device->physical_device->rad_info.family == CHIP_HAWAII ? 4096 : 8192;
3059
3060         if (getenv("RADV_TRACE_FILE")) {
3061                 const char *filename = getenv("RADV_TRACE_FILE");
3062
3063                 keep_shader_info = true;
3064
3065                 if (!radv_init_trace(device))
3066                         goto fail;
3067
3068                 fprintf(stderr, "*****************************************************************************\n");
3069                 fprintf(stderr, "* WARNING: RADV_TRACE_FILE is costly and should only be used for debugging! *\n");
3070                 fprintf(stderr, "*****************************************************************************\n");
3071
3072                 fprintf(stderr, "Trace file will be dumped to %s\n", filename);
3073                 radv_dump_enabled_options(device, stderr);
3074         }
3075
3076         int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
3077         if (radv_thread_trace >= 0) {
3078                 fprintf(stderr, "*************************************************\n");
3079                 fprintf(stderr, "* WARNING: Thread trace support is experimental *\n");
3080                 fprintf(stderr, "*************************************************\n");
3081
3082                 if (device->physical_device->rad_info.chip_class < GFX8) {
3083                         fprintf(stderr, "GPU hardware not supported: refer to "
3084                                         "the RGP documentation for the list of "
3085                                         "supported GPUs!\n");
3086                         abort();
3087                 }
3088
3089                 /* Default buffer size set to 1MB per SE. */
3090                 device->thread_trace_buffer_size =
3091                         radv_get_int_debug_option("RADV_THREAD_TRACE_BUFFER_SIZE", 1024 * 1024);
3092                 device->thread_trace_start_frame = radv_thread_trace;
3093
3094                 if (!radv_thread_trace_init(device))
3095                         goto fail;
3096         }
3097
3098         /* Temporarily disable secure compile while we create meta shaders, etc */
3099         uint8_t sc_threads = device->instance->num_sc_threads;
3100         if (sc_threads)
3101                 device->instance->num_sc_threads = 0;
3102
3103         device->keep_shader_info = keep_shader_info;
3104         result = radv_device_init_meta(device);
3105         if (result != VK_SUCCESS)
3106                 goto fail;
3107
3108         radv_device_init_msaa(device);
3109
3110         for (int family = 0; family < RADV_MAX_QUEUE_FAMILIES; ++family) {
3111                 device->empty_cs[family] = device->ws->cs_create(device->ws, family);
3112                 switch (family) {
3113                 case RADV_QUEUE_GENERAL:
3114                         radeon_emit(device->empty_cs[family], PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
3115                         radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_LOAD_ENABLE(1));
3116                         radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_SHADOW_ENABLE(1));
3117                         break;
3118                 case RADV_QUEUE_COMPUTE:
3119                         radeon_emit(device->empty_cs[family], PKT3(PKT3_NOP, 0, 0));
3120                         radeon_emit(device->empty_cs[family], 0);
3121                         break;
3122                 }
3123                 device->ws->cs_finalize(device->empty_cs[family]);
3124         }
3125
3126         if (device->physical_device->rad_info.chip_class >= GFX7)
3127                 cik_create_gfx_config(device);
3128
3129         VkPipelineCacheCreateInfo ci;
3130         ci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
3131         ci.pNext = NULL;
3132         ci.flags = 0;
3133         ci.pInitialData = NULL;
3134         ci.initialDataSize = 0;
3135         VkPipelineCache pc;
3136         result = radv_CreatePipelineCache(radv_device_to_handle(device),
3137                                           &ci, NULL, &pc);
3138         if (result != VK_SUCCESS)
3139                 goto fail_meta;
3140
3141         device->mem_cache = radv_pipeline_cache_from_handle(pc);
3142
3143         result = radv_create_pthread_cond(&device->timeline_cond);
3144         if (result != VK_SUCCESS)
3145                 goto fail_mem_cache;
3146
3147         device->force_aniso =
3148                 MIN2(16, radv_get_int_debug_option("RADV_TEX_ANISO", -1));
3149         if (device->force_aniso >= 0) {
3150                 fprintf(stderr, "radv: Forcing anisotropy filter to %ix\n",
3151                         1 << util_logbase2(device->force_aniso));
3152         }
3153
3154         /* Fork device for secure compile as required */
3155         device->instance->num_sc_threads = sc_threads;
3156         if (radv_device_use_secure_compile(device->instance)) {
3157
3158                 result = fork_secure_compile_idle_device(device);
3159                 if (result != VK_SUCCESS)
3160                         goto fail_meta;
3161         }
3162
3163         *pDevice = radv_device_to_handle(device);
3164         return VK_SUCCESS;
3165
3166 fail_mem_cache:
3167         radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
3168 fail_meta:
3169         radv_device_finish_meta(device);
3170 fail:
3171         radv_bo_list_finish(&device->bo_list);
3172
3173         radv_thread_trace_finish(device);
3174
3175         if (device->trace_bo)
3176                 device->ws->buffer_destroy(device->trace_bo);
3177
3178         if (device->gfx_init)
3179                 device->ws->buffer_destroy(device->gfx_init);
3180
3181         for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
3182                 for (unsigned q = 0; q < device->queue_count[i]; q++)
3183                         radv_queue_finish(&device->queues[i][q]);
3184                 if (device->queue_count[i])
3185                         vk_free(&device->alloc, device->queues[i]);
3186         }
3187
3188         vk_free(&device->alloc, device);
3189         return result;
3190 }
3191
3192 void radv_DestroyDevice(
3193         VkDevice                                    _device,
3194         const VkAllocationCallbacks*                pAllocator)
3195 {
3196         RADV_FROM_HANDLE(radv_device, device, _device);
3197
3198         if (!device)
3199                 return;
3200
3201         if (device->trace_bo)
3202                 device->ws->buffer_destroy(device->trace_bo);
3203
3204         if (device->gfx_init)
3205                 device->ws->buffer_destroy(device->gfx_init);
3206
3207         for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
3208                 for (unsigned q = 0; q < device->queue_count[i]; q++)
3209                         radv_queue_finish(&device->queues[i][q]);
3210                 if (device->queue_count[i])
3211                         vk_free(&device->alloc, device->queues[i]);
3212                 if (device->empty_cs[i])
3213                         device->ws->cs_destroy(device->empty_cs[i]);
3214         }
3215         radv_device_finish_meta(device);
3216
3217         VkPipelineCache pc = radv_pipeline_cache_to_handle(device->mem_cache);
3218         radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
3219
3220         radv_destroy_shader_slabs(device);
3221
3222         pthread_cond_destroy(&device->timeline_cond);
3223         radv_bo_list_finish(&device->bo_list);
3224
3225         radv_thread_trace_finish(device);
3226
3227         if (radv_device_use_secure_compile(device->instance)) {
3228                 for (unsigned i = 0; i < device->instance->num_sc_threads; i++ ) {
3229                         destroy_secure_compile_device(device, i);
3230                 }
3231         }
3232
3233         if (device->sc_state) {
3234                 free(device->sc_state->uid);
3235                 vk_free(&device->alloc, device->sc_state->secure_compile_processes);
3236         }
3237         vk_free(&device->alloc, device->sc_state);
3238         vk_free(&device->alloc, device);
3239 }
3240
3241 VkResult radv_EnumerateInstanceLayerProperties(
3242         uint32_t*                                   pPropertyCount,
3243         VkLayerProperties*                          pProperties)
3244 {
3245         if (pProperties == NULL) {
3246                 *pPropertyCount = 0;
3247                 return VK_SUCCESS;
3248         }
3249
3250         /* None supported at this time */
3251         return vk_error(NULL, VK_ERROR_LAYER_NOT_PRESENT);
3252 }
3253
3254 VkResult radv_EnumerateDeviceLayerProperties(
3255         VkPhysicalDevice                            physicalDevice,
3256         uint32_t*                                   pPropertyCount,
3257         VkLayerProperties*                          pProperties)
3258 {
3259         if (pProperties == NULL) {
3260                 *pPropertyCount = 0;
3261                 return VK_SUCCESS;
3262         }
3263
3264         /* None supported at this time */
3265         return vk_error(NULL, VK_ERROR_LAYER_NOT_PRESENT);
3266 }
3267
3268 void radv_GetDeviceQueue2(
3269         VkDevice                                    _device,
3270         const VkDeviceQueueInfo2*                   pQueueInfo,
3271         VkQueue*                                    pQueue)
3272 {
3273         RADV_FROM_HANDLE(radv_device, device, _device);
3274         struct radv_queue *queue;
3275
3276         queue = &device->queues[pQueueInfo->queueFamilyIndex][pQueueInfo->queueIndex];
3277         if (pQueueInfo->flags != queue->flags) {
3278                 /* From the Vulkan 1.1.70 spec:
3279                  *
3280                  * "The queue returned by vkGetDeviceQueue2 must have the same
3281                  * flags value from this structure as that used at device
3282                  * creation time in a VkDeviceQueueCreateInfo instance. If no
3283                  * matching flags were specified at device creation time then
3284                  * pQueue will return VK_NULL_HANDLE."
3285                  */
3286                 *pQueue = VK_NULL_HANDLE;
3287                 return;
3288         }
3289
3290         *pQueue = radv_queue_to_handle(queue);
3291 }
3292
3293 void radv_GetDeviceQueue(
3294         VkDevice                                    _device,
3295         uint32_t                                    queueFamilyIndex,
3296         uint32_t                                    queueIndex,
3297         VkQueue*                                    pQueue)
3298 {
3299         const VkDeviceQueueInfo2 info = (VkDeviceQueueInfo2) {
3300                 .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2,
3301                 .queueFamilyIndex = queueFamilyIndex,
3302                 .queueIndex = queueIndex
3303         };
3304
3305         radv_GetDeviceQueue2(_device, &info, pQueue);
3306 }
3307
3308 static void
3309 fill_geom_tess_rings(struct radv_queue *queue,
3310                      uint32_t *map,
3311                      bool add_sample_positions,
3312                      uint32_t esgs_ring_size,
3313                      struct radeon_winsys_bo *esgs_ring_bo,
3314                      uint32_t gsvs_ring_size,
3315                      struct radeon_winsys_bo *gsvs_ring_bo,
3316                      uint32_t tess_factor_ring_size,
3317                      uint32_t tess_offchip_ring_offset,
3318                      uint32_t tess_offchip_ring_size,
3319                      struct radeon_winsys_bo *tess_rings_bo)
3320 {
3321         uint32_t *desc = &map[4];
3322
3323         if (esgs_ring_bo) {
3324                 uint64_t esgs_va = radv_buffer_get_va(esgs_ring_bo);
3325
3326                 /* stride 0, num records - size, add tid, swizzle, elsize4,
3327                    index stride 64 */
3328                 desc[0] = esgs_va;
3329                 desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32) |
3330                           S_008F04_SWIZZLE_ENABLE(true);
3331                 desc[2] = esgs_ring_size;
3332                 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3333                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3334                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3335                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3336                           S_008F0C_INDEX_STRIDE(3) |
3337                           S_008F0C_ADD_TID_ENABLE(1);
3338
3339                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3340                         desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3341                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3342                                    S_008F0C_RESOURCE_LEVEL(1);
3343                 } else {
3344                         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3345                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
3346                                    S_008F0C_ELEMENT_SIZE(1);
3347                 }
3348
3349                 /* GS entry for ES->GS ring */
3350                 /* stride 0, num records - size, elsize0,
3351                    index stride 0 */
3352                 desc[4] = esgs_va;
3353                 desc[5] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32);
3354                 desc[6] = esgs_ring_size;
3355                 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3356                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3357                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3358                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3359
3360                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3361                         desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3362                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3363                                    S_008F0C_RESOURCE_LEVEL(1);
3364                 } else {
3365                         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3366                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3367                 }
3368         }
3369
3370         desc += 8;
3371
3372         if (gsvs_ring_bo) {
3373                 uint64_t gsvs_va = radv_buffer_get_va(gsvs_ring_bo);
3374
3375                 /* VS entry for GS->VS ring */
3376                 /* stride 0, num records - size, elsize0,
3377                    index stride 0 */
3378                 desc[0] = gsvs_va;
3379                 desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32);
3380                 desc[2] = gsvs_ring_size;
3381                 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3382                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3383                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3384                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3385
3386                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3387                         desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3388                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3389                                    S_008F0C_RESOURCE_LEVEL(1);
3390                 } else {
3391                         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3392                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3393                 }
3394
3395                 /* stride gsvs_itemsize, num records 64
3396                    elsize 4, index stride 16 */
3397                 /* shader will patch stride and desc[2] */
3398                 desc[4] = gsvs_va;
3399                 desc[5] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32) |
3400                           S_008F04_SWIZZLE_ENABLE(1);
3401                 desc[6] = 0;
3402                 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3403                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3404                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3405                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3406                           S_008F0C_INDEX_STRIDE(1) |
3407                           S_008F0C_ADD_TID_ENABLE(true);
3408
3409                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3410                         desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3411                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_DISABLED) |
3412                                    S_008F0C_RESOURCE_LEVEL(1);
3413                 } else {
3414                         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3415                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
3416                                    S_008F0C_ELEMENT_SIZE(1);
3417                 }
3418
3419         }
3420
3421         desc += 8;
3422
3423         if (tess_rings_bo) {
3424                 uint64_t tess_va = radv_buffer_get_va(tess_rings_bo);
3425                 uint64_t tess_offchip_va = tess_va + tess_offchip_ring_offset;
3426
3427                 desc[0] = tess_va;
3428                 desc[1] = S_008F04_BASE_ADDRESS_HI(tess_va >> 32);
3429                 desc[2] = tess_factor_ring_size;
3430                 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3431                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3432                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3433                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3434
3435                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3436                         desc[3] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3437                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3438                                    S_008F0C_RESOURCE_LEVEL(1);
3439                 } else {
3440                         desc[3] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3441                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3442                 }
3443
3444                 desc[4] = tess_offchip_va;
3445                 desc[5] = S_008F04_BASE_ADDRESS_HI(tess_offchip_va >> 32);
3446                 desc[6] = tess_offchip_ring_size;
3447                 desc[7] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3448                           S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3449                           S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3450                           S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3451
3452                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3453                         desc[7] |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3454                                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3455                                    S_008F0C_RESOURCE_LEVEL(1);
3456                 } else {
3457                         desc[7] |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3458                                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3459                 }
3460         }
3461
3462         desc += 8;
3463
3464         if (add_sample_positions) {
3465                 /* add sample positions after all rings */
3466                 memcpy(desc, queue->device->sample_locations_1x, 8);
3467                 desc += 2;
3468                 memcpy(desc, queue->device->sample_locations_2x, 16);
3469                 desc += 4;
3470                 memcpy(desc, queue->device->sample_locations_4x, 32);
3471                 desc += 8;
3472                 memcpy(desc, queue->device->sample_locations_8x, 64);
3473         }
3474 }
3475
3476 static unsigned
3477 radv_get_hs_offchip_param(struct radv_device *device, uint32_t *max_offchip_buffers_p)
3478 {
3479         bool double_offchip_buffers = device->physical_device->rad_info.chip_class >= GFX7 &&
3480                 device->physical_device->rad_info.family != CHIP_CARRIZO &&
3481                 device->physical_device->rad_info.family != CHIP_STONEY;
3482         unsigned max_offchip_buffers_per_se = double_offchip_buffers ? 128 : 64;
3483         unsigned max_offchip_buffers;
3484         unsigned offchip_granularity;
3485         unsigned hs_offchip_param;
3486
3487         /*
3488          * Per RadeonSI:
3489          * This must be one less than the maximum number due to a hw limitation.
3490          * Various hardware bugs need thGFX7
3491          *
3492          * Per AMDVLK:
3493          * Vega10 should limit max_offchip_buffers to 508 (4 * 127).
3494          * Gfx7 should limit max_offchip_buffers to 508
3495          * Gfx6 should limit max_offchip_buffers to 126 (2 * 63)
3496          *
3497          * Follow AMDVLK here.
3498          */
3499         if (device->physical_device->rad_info.chip_class >= GFX10) {
3500                 max_offchip_buffers_per_se = 256;
3501         } else if (device->physical_device->rad_info.family == CHIP_VEGA10 ||
3502                    device->physical_device->rad_info.chip_class == GFX7 ||
3503                    device->physical_device->rad_info.chip_class == GFX6)
3504                 --max_offchip_buffers_per_se;
3505
3506         max_offchip_buffers = max_offchip_buffers_per_se *
3507                 device->physical_device->rad_info.max_se;
3508
3509         /* Hawaii has a bug with offchip buffers > 256 that can be worked
3510          * around by setting 4K granularity.
3511          */
3512         if (device->tess_offchip_block_dw_size == 4096) {
3513                 assert(device->physical_device->rad_info.family == CHIP_HAWAII);
3514                 offchip_granularity = V_03093C_X_4K_DWORDS;
3515         } else {
3516                 assert(device->tess_offchip_block_dw_size == 8192);
3517                 offchip_granularity = V_03093C_X_8K_DWORDS;
3518         }
3519
3520         switch (device->physical_device->rad_info.chip_class) {
3521         case GFX6:
3522                 max_offchip_buffers = MIN2(max_offchip_buffers, 126);
3523                 break;
3524         case GFX7:
3525         case GFX8:
3526         case GFX9:
3527                 max_offchip_buffers = MIN2(max_offchip_buffers, 508);
3528                 break;
3529         case GFX10:
3530                 break;
3531         default:
3532                 break;
3533         }
3534
3535         *max_offchip_buffers_p = max_offchip_buffers;
3536         if (device->physical_device->rad_info.chip_class >= GFX7) {
3537                 if (device->physical_device->rad_info.chip_class >= GFX8)
3538                         --max_offchip_buffers;
3539                 hs_offchip_param =
3540                         S_03093C_OFFCHIP_BUFFERING(max_offchip_buffers) |
3541                         S_03093C_OFFCHIP_GRANULARITY(offchip_granularity);
3542         } else {
3543                 hs_offchip_param =
3544                         S_0089B0_OFFCHIP_BUFFERING(max_offchip_buffers);
3545         }
3546         return hs_offchip_param;
3547 }
3548
3549 static void
3550 radv_emit_gs_ring_sizes(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3551                         struct radeon_winsys_bo *esgs_ring_bo,
3552                         uint32_t esgs_ring_size,
3553                         struct radeon_winsys_bo *gsvs_ring_bo,
3554                         uint32_t gsvs_ring_size)
3555 {
3556         if (!esgs_ring_bo && !gsvs_ring_bo)
3557                 return;
3558
3559         if (esgs_ring_bo)
3560                 radv_cs_add_buffer(queue->device->ws, cs, esgs_ring_bo);
3561
3562         if (gsvs_ring_bo)
3563                 radv_cs_add_buffer(queue->device->ws, cs, gsvs_ring_bo);
3564
3565         if (queue->device->physical_device->rad_info.chip_class >= GFX7) {
3566                 radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2);
3567                 radeon_emit(cs, esgs_ring_size >> 8);
3568                 radeon_emit(cs, gsvs_ring_size >> 8);
3569         } else {
3570                 radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2);
3571                 radeon_emit(cs, esgs_ring_size >> 8);
3572                 radeon_emit(cs, gsvs_ring_size >> 8);
3573         }
3574 }
3575
3576 static void
3577 radv_emit_tess_factor_ring(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3578                            unsigned hs_offchip_param, unsigned tf_ring_size,
3579                            struct radeon_winsys_bo *tess_rings_bo)
3580 {
3581         uint64_t tf_va;
3582
3583         if (!tess_rings_bo)
3584                 return;
3585
3586         tf_va = radv_buffer_get_va(tess_rings_bo);
3587
3588         radv_cs_add_buffer(queue->device->ws, cs, tess_rings_bo);
3589
3590         if (queue->device->physical_device->rad_info.chip_class >= GFX7) {
3591                 radeon_set_uconfig_reg(cs, R_030938_VGT_TF_RING_SIZE,
3592                                        S_030938_SIZE(tf_ring_size / 4));
3593                 radeon_set_uconfig_reg(cs, R_030940_VGT_TF_MEMORY_BASE,
3594                                        tf_va >> 8);
3595
3596                 if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3597                         radeon_set_uconfig_reg(cs, R_030984_VGT_TF_MEMORY_BASE_HI_UMD,
3598                                                S_030984_BASE_HI(tf_va >> 40));
3599                 } else if (queue->device->physical_device->rad_info.chip_class == GFX9) {
3600                         radeon_set_uconfig_reg(cs, R_030944_VGT_TF_MEMORY_BASE_HI,
3601                                                S_030944_BASE_HI(tf_va >> 40));
3602                 }
3603                 radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM,
3604                                        hs_offchip_param);
3605         } else {
3606                 radeon_set_config_reg(cs, R_008988_VGT_TF_RING_SIZE,
3607                                       S_008988_SIZE(tf_ring_size / 4));
3608                 radeon_set_config_reg(cs, R_0089B8_VGT_TF_MEMORY_BASE,
3609                                       tf_va >> 8);
3610                 radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM,
3611                                      hs_offchip_param);
3612         }
3613 }
3614
3615 static void
3616 radv_emit_graphics_scratch(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3617                            uint32_t size_per_wave, uint32_t waves,
3618                            struct radeon_winsys_bo *scratch_bo)
3619 {
3620         if (queue->queue_family_index != RADV_QUEUE_GENERAL)
3621                 return;
3622
3623         if (!scratch_bo)
3624                 return;
3625
3626         radv_cs_add_buffer(queue->device->ws, cs, scratch_bo);
3627
3628         radeon_set_context_reg(cs, R_0286E8_SPI_TMPRING_SIZE,
3629                                S_0286E8_WAVES(waves) |
3630                                S_0286E8_WAVESIZE(round_up_u32(size_per_wave, 1024)));
3631 }
3632
3633 static void
3634 radv_emit_compute_scratch(struct radv_queue *queue, struct radeon_cmdbuf *cs,
3635                           uint32_t size_per_wave, uint32_t waves,
3636                           struct radeon_winsys_bo *compute_scratch_bo)
3637 {
3638         uint64_t scratch_va;
3639
3640         if (!compute_scratch_bo)
3641                 return;
3642
3643         scratch_va = radv_buffer_get_va(compute_scratch_bo);
3644
3645         radv_cs_add_buffer(queue->device->ws, cs, compute_scratch_bo);
3646
3647         radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2);
3648         radeon_emit(cs, scratch_va);
3649         radeon_emit(cs, S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
3650                         S_008F04_SWIZZLE_ENABLE(1));
3651
3652         radeon_set_sh_reg(cs, R_00B860_COMPUTE_TMPRING_SIZE,
3653                          S_00B860_WAVES(waves) |
3654                          S_00B860_WAVESIZE(round_up_u32(size_per_wave, 1024)));
3655 }
3656
3657 static void
3658 radv_emit_global_shader_pointers(struct radv_queue *queue,
3659                                  struct radeon_cmdbuf *cs,
3660                                  struct radeon_winsys_bo *descriptor_bo)
3661 {
3662         uint64_t va;
3663
3664         if (!descriptor_bo)
3665                 return;
3666
3667         va = radv_buffer_get_va(descriptor_bo);
3668
3669         radv_cs_add_buffer(queue->device->ws, cs, descriptor_bo);
3670
3671         if (queue->device->physical_device->rad_info.chip_class >= GFX10) {
3672                 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3673                                    R_00B130_SPI_SHADER_USER_DATA_VS_0,
3674                                    R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
3675                                    R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
3676
3677                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3678                         radv_emit_shader_pointer(queue->device, cs, regs[i],
3679                                                  va, true);
3680                 }
3681         } else if (queue->device->physical_device->rad_info.chip_class == GFX9) {
3682                 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3683                                    R_00B130_SPI_SHADER_USER_DATA_VS_0,
3684                                    R_00B208_SPI_SHADER_USER_DATA_ADDR_LO_GS,
3685                                    R_00B408_SPI_SHADER_USER_DATA_ADDR_LO_HS};
3686
3687                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3688                         radv_emit_shader_pointer(queue->device, cs, regs[i],
3689                                                  va, true);
3690                 }
3691         } else {
3692                 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
3693                                    R_00B130_SPI_SHADER_USER_DATA_VS_0,
3694                                    R_00B230_SPI_SHADER_USER_DATA_GS_0,
3695                                    R_00B330_SPI_SHADER_USER_DATA_ES_0,
3696                                    R_00B430_SPI_SHADER_USER_DATA_HS_0,
3697                                    R_00B530_SPI_SHADER_USER_DATA_LS_0};
3698
3699                 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
3700                         radv_emit_shader_pointer(queue->device, cs, regs[i],
3701                                                  va, true);
3702                 }
3703         }
3704 }
3705
3706 static void
3707 radv_init_graphics_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
3708 {
3709         struct radv_device *device = queue->device;
3710
3711         if (device->gfx_init) {
3712                 uint64_t va = radv_buffer_get_va(device->gfx_init);
3713
3714                 radeon_emit(cs, PKT3(PKT3_INDIRECT_BUFFER_CIK, 2, 0));
3715                 radeon_emit(cs, va);
3716                 radeon_emit(cs, va >> 32);
3717                 radeon_emit(cs, device->gfx_init_size_dw & 0xffff);
3718
3719                 radv_cs_add_buffer(device->ws, cs, device->gfx_init);
3720         } else {
3721                 si_emit_graphics(device, cs);
3722         }
3723 }
3724
3725 static void
3726 radv_init_compute_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
3727 {
3728         struct radv_physical_device *physical_device = queue->device->physical_device;
3729         si_emit_compute(physical_device, cs);
3730 }
3731
3732 static VkResult
3733 radv_get_preamble_cs(struct radv_queue *queue,
3734                      uint32_t scratch_size_per_wave,
3735                      uint32_t scratch_waves,
3736                      uint32_t compute_scratch_size_per_wave,
3737                      uint32_t compute_scratch_waves,
3738                      uint32_t esgs_ring_size,
3739                      uint32_t gsvs_ring_size,
3740                      bool needs_tess_rings,
3741                      bool needs_gds,
3742                      bool needs_gds_oa,
3743                      bool needs_sample_positions,
3744                      struct radeon_cmdbuf **initial_full_flush_preamble_cs,
3745                      struct radeon_cmdbuf **initial_preamble_cs,
3746                      struct radeon_cmdbuf **continue_preamble_cs)
3747 {
3748         struct radeon_winsys_bo *scratch_bo = NULL;
3749         struct radeon_winsys_bo *descriptor_bo = NULL;
3750         struct radeon_winsys_bo *compute_scratch_bo = NULL;
3751         struct radeon_winsys_bo *esgs_ring_bo = NULL;
3752         struct radeon_winsys_bo *gsvs_ring_bo = NULL;
3753         struct radeon_winsys_bo *tess_rings_bo = NULL;
3754         struct radeon_winsys_bo *gds_bo = NULL;
3755         struct radeon_winsys_bo *gds_oa_bo = NULL;
3756         struct radeon_cmdbuf *dest_cs[3] = {0};
3757         bool add_tess_rings = false, add_gds = false, add_gds_oa = false, add_sample_positions = false;
3758         unsigned tess_factor_ring_size = 0, tess_offchip_ring_size = 0;
3759         unsigned max_offchip_buffers;
3760         unsigned hs_offchip_param = 0;
3761         unsigned tess_offchip_ring_offset;
3762         uint32_t ring_bo_flags = RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING;
3763         if (!queue->has_tess_rings) {
3764                 if (needs_tess_rings)
3765                         add_tess_rings = true;
3766         }
3767         if (!queue->has_gds) {
3768                 if (needs_gds)
3769                         add_gds = true;
3770         }
3771         if (!queue->has_gds_oa) {
3772                 if (needs_gds_oa)
3773                         add_gds_oa = true;
3774         }
3775         if (!queue->has_sample_positions) {
3776                 if (needs_sample_positions)
3777                         add_sample_positions = true;
3778         }
3779         tess_factor_ring_size = 32768 * queue->device->physical_device->rad_info.max_se;
3780         hs_offchip_param = radv_get_hs_offchip_param(queue->device,
3781                                                      &max_offchip_buffers);
3782         tess_offchip_ring_offset = align(tess_factor_ring_size, 64 * 1024);
3783         tess_offchip_ring_size = max_offchip_buffers *
3784                 queue->device->tess_offchip_block_dw_size * 4;
3785
3786         scratch_size_per_wave = MAX2(scratch_size_per_wave, queue->scratch_size_per_wave);
3787         if (scratch_size_per_wave)
3788                 scratch_waves = MIN2(scratch_waves, UINT32_MAX / scratch_size_per_wave);
3789         else
3790                 scratch_waves = 0;
3791
3792         compute_scratch_size_per_wave = MAX2(compute_scratch_size_per_wave, queue->compute_scratch_size_per_wave);
3793         if (compute_scratch_size_per_wave)
3794                 compute_scratch_waves = MIN2(compute_scratch_waves, UINT32_MAX / compute_scratch_size_per_wave);
3795         else
3796                 compute_scratch_waves = 0;
3797
3798         if (scratch_size_per_wave <= queue->scratch_size_per_wave &&
3799             scratch_waves <= queue->scratch_waves &&
3800             compute_scratch_size_per_wave <= queue->compute_scratch_size_per_wave &&
3801             compute_scratch_waves <= queue->compute_scratch_waves &&
3802             esgs_ring_size <= queue->esgs_ring_size &&
3803             gsvs_ring_size <= queue->gsvs_ring_size &&
3804             !add_tess_rings && !add_gds && !add_gds_oa && !add_sample_positions &&
3805             queue->initial_preamble_cs) {
3806                 *initial_full_flush_preamble_cs = queue->initial_full_flush_preamble_cs;
3807                 *initial_preamble_cs = queue->initial_preamble_cs;
3808                 *continue_preamble_cs = queue->continue_preamble_cs;
3809                 if (!scratch_size_per_wave && !compute_scratch_size_per_wave &&
3810                     !esgs_ring_size && !gsvs_ring_size && !needs_tess_rings &&
3811                     !needs_gds && !needs_gds_oa && !needs_sample_positions)
3812                         *continue_preamble_cs = NULL;
3813                 return VK_SUCCESS;
3814         }
3815
3816         uint32_t scratch_size = scratch_size_per_wave * scratch_waves;
3817         uint32_t queue_scratch_size = queue->scratch_size_per_wave * queue->scratch_waves;
3818         if (scratch_size > queue_scratch_size) {
3819                 scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
3820                                                               scratch_size,
3821                                                               4096,
3822                                                               RADEON_DOMAIN_VRAM,
3823                                                               ring_bo_flags,
3824                                                               RADV_BO_PRIORITY_SCRATCH);
3825                 if (!scratch_bo)
3826                         goto fail;
3827         } else
3828                 scratch_bo = queue->scratch_bo;
3829
3830         uint32_t compute_scratch_size = compute_scratch_size_per_wave * compute_scratch_waves;
3831         uint32_t compute_queue_scratch_size = queue->compute_scratch_size_per_wave * queue->compute_scratch_waves;
3832         if (compute_scratch_size > compute_queue_scratch_size) {
3833                 compute_scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
3834                                                                       compute_scratch_size,
3835                                                                       4096,
3836                                                                       RADEON_DOMAIN_VRAM,
3837                                                                       ring_bo_flags,
3838                                                                       RADV_BO_PRIORITY_SCRATCH);
3839                 if (!compute_scratch_bo)
3840                         goto fail;
3841
3842         } else
3843                 compute_scratch_bo = queue->compute_scratch_bo;
3844
3845         if (esgs_ring_size > queue->esgs_ring_size) {
3846                 esgs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
3847                                                                 esgs_ring_size,
3848                                                                 4096,
3849                                                                 RADEON_DOMAIN_VRAM,
3850                                                                 ring_bo_flags,
3851                                                                 RADV_BO_PRIORITY_SCRATCH);
3852                 if (!esgs_ring_bo)
3853                         goto fail;
3854         } else {
3855                 esgs_ring_bo = queue->esgs_ring_bo;
3856                 esgs_ring_size = queue->esgs_ring_size;
3857         }
3858
3859         if (gsvs_ring_size > queue->gsvs_ring_size) {
3860                 gsvs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
3861                                                                 gsvs_ring_size,
3862                                                                 4096,
3863                                                                 RADEON_DOMAIN_VRAM,
3864                                                                 ring_bo_flags,
3865                                                                 RADV_BO_PRIORITY_SCRATCH);
3866                 if (!gsvs_ring_bo)
3867                         goto fail;
3868         } else {
3869                 gsvs_ring_bo = queue->gsvs_ring_bo;
3870                 gsvs_ring_size = queue->gsvs_ring_size;
3871         }
3872
3873         if (add_tess_rings) {
3874                 tess_rings_bo = queue->device->ws->buffer_create(queue->device->ws,
3875                                                                  tess_offchip_ring_offset + tess_offchip_ring_size,
3876                                                                  256,
3877                                                                  RADEON_DOMAIN_VRAM,
3878                                                                  ring_bo_flags,
3879                                                                  RADV_BO_PRIORITY_SCRATCH);
3880                 if (!tess_rings_bo)
3881                         goto fail;
3882         } else {
3883                 tess_rings_bo = queue->tess_rings_bo;
3884         }
3885
3886         if (add_gds) {
3887                 assert(queue->device->physical_device->rad_info.chip_class >= GFX10);
3888
3889                 /* 4 streamout GDS counters.
3890                  * We need 256B (64 dw) of GDS, otherwise streamout hangs.
3891                  */
3892                 gds_bo = queue->device->ws->buffer_create(queue->device->ws,
3893                                                           256, 4,
3894                                                           RADEON_DOMAIN_GDS,
3895                                                           ring_bo_flags,
3896                                                           RADV_BO_PRIORITY_SCRATCH);
3897                 if (!gds_bo)
3898                         goto fail;
3899         } else {
3900                 gds_bo = queue->gds_bo;
3901         }
3902
3903         if (add_gds_oa) {
3904                 assert(queue->device->physical_device->rad_info.chip_class >= GFX10);
3905
3906                 gds_oa_bo = queue->device->ws->buffer_create(queue->device->ws,
3907                                                              4, 1,
3908                                                              RADEON_DOMAIN_OA,
3909                                                              ring_bo_flags,
3910                                                              RADV_BO_PRIORITY_SCRATCH);
3911                 if (!gds_oa_bo)
3912                         goto fail;
3913         } else {
3914                 gds_oa_bo = queue->gds_oa_bo;
3915         }
3916
3917         if (scratch_bo != queue->scratch_bo ||
3918             esgs_ring_bo != queue->esgs_ring_bo ||
3919             gsvs_ring_bo != queue->gsvs_ring_bo ||
3920             tess_rings_bo != queue->tess_rings_bo ||
3921             add_sample_positions) {
3922                 uint32_t size = 0;
3923                 if (gsvs_ring_bo || esgs_ring_bo ||
3924                     tess_rings_bo || add_sample_positions) {
3925                         size = 112; /* 2 dword + 2 padding + 4 dword * 6 */
3926                         if (add_sample_positions)
3927                                 size += 128; /* 64+32+16+8 = 120 bytes */
3928                 }
3929                 else if (scratch_bo)
3930                         size = 8; /* 2 dword */
3931
3932                 descriptor_bo = queue->device->ws->buffer_create(queue->device->ws,
3933                                                                  size,
3934                                                                  4096,
3935                                                                  RADEON_DOMAIN_VRAM,
3936                                                                  RADEON_FLAG_CPU_ACCESS |
3937                                                                  RADEON_FLAG_NO_INTERPROCESS_SHARING |
3938                                                                  RADEON_FLAG_READ_ONLY,
3939                                                                  RADV_BO_PRIORITY_DESCRIPTOR);
3940                 if (!descriptor_bo)
3941                         goto fail;
3942         } else
3943                 descriptor_bo = queue->descriptor_bo;
3944
3945         if (descriptor_bo != queue->descriptor_bo) {
3946                 uint32_t *map = (uint32_t*)queue->device->ws->buffer_map(descriptor_bo);
3947
3948                 if (scratch_bo) {
3949                         uint64_t scratch_va = radv_buffer_get_va(scratch_bo);
3950                         uint32_t rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
3951                                          S_008F04_SWIZZLE_ENABLE(1);
3952                         map[0] = scratch_va;
3953                         map[1] = rsrc1;
3954                 }
3955
3956                 if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo || add_sample_positions)
3957                         fill_geom_tess_rings(queue, map, add_sample_positions,
3958                                              esgs_ring_size, esgs_ring_bo,
3959                                              gsvs_ring_size, gsvs_ring_bo,
3960                                              tess_factor_ring_size,
3961                                              tess_offchip_ring_offset,
3962                                              tess_offchip_ring_size,
3963                                              tess_rings_bo);
3964
3965                 queue->device->ws->buffer_unmap(descriptor_bo);
3966         }
3967
3968         for(int i = 0; i < 3; ++i) {
3969                 struct radeon_cmdbuf *cs = NULL;
3970                 cs = queue->device->ws->cs_create(queue->device->ws,
3971                                                   queue->queue_family_index ? RING_COMPUTE : RING_GFX);
3972                 if (!cs)
3973                         goto fail;
3974
3975                 dest_cs[i] = cs;
3976
3977                 if (scratch_bo)
3978                         radv_cs_add_buffer(queue->device->ws, cs, scratch_bo);
3979
3980                 /* Emit initial configuration. */
3981                 switch (queue->queue_family_index) {
3982                 case RADV_QUEUE_GENERAL:
3983                         radv_init_graphics_state(cs, queue);
3984                         break;
3985                 case RADV_QUEUE_COMPUTE:
3986                         radv_init_compute_state(cs, queue);
3987                         break;
3988                 case RADV_QUEUE_TRANSFER:
3989                         break;
3990                 }
3991
3992                 if (esgs_ring_bo || gsvs_ring_bo || tess_rings_bo)  {
3993                         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
3994                         radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
3995
3996                         radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
3997                         radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
3998                 }
3999
4000                 radv_emit_gs_ring_sizes(queue, cs, esgs_ring_bo, esgs_ring_size,
4001                                         gsvs_ring_bo, gsvs_ring_size);
4002                 radv_emit_tess_factor_ring(queue, cs, hs_offchip_param,
4003                                            tess_factor_ring_size, tess_rings_bo);
4004                 radv_emit_global_shader_pointers(queue, cs, descriptor_bo);
4005                 radv_emit_compute_scratch(queue, cs, compute_scratch_size_per_wave,
4006                                           compute_scratch_waves, compute_scratch_bo);
4007                 radv_emit_graphics_scratch(queue, cs, scratch_size_per_wave,
4008                                            scratch_waves, scratch_bo);
4009
4010                 if (gds_bo)
4011                         radv_cs_add_buffer(queue->device->ws, cs, gds_bo);
4012                 if (gds_oa_bo)
4013                         radv_cs_add_buffer(queue->device->ws, cs, gds_oa_bo);
4014
4015                 if (queue->device->trace_bo)
4016                         radv_cs_add_buffer(queue->device->ws, cs, queue->device->trace_bo);
4017
4018                 if (i == 0) {
4019                         si_cs_emit_cache_flush(cs,
4020                                                queue->device->physical_device->rad_info.chip_class,
4021                                                NULL, 0,
4022                                                queue->queue_family_index == RING_COMPUTE &&
4023                                                  queue->device->physical_device->rad_info.chip_class >= GFX7,
4024                                                (queue->queue_family_index == RADV_QUEUE_COMPUTE ? RADV_CMD_FLAG_CS_PARTIAL_FLUSH : (RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH)) |
4025                                                RADV_CMD_FLAG_INV_ICACHE |
4026                                                RADV_CMD_FLAG_INV_SCACHE |
4027                                                RADV_CMD_FLAG_INV_VCACHE |
4028                                                RADV_CMD_FLAG_INV_L2 |
4029                                                RADV_CMD_FLAG_START_PIPELINE_STATS, 0);
4030                 } else if (i == 1) {
4031                         si_cs_emit_cache_flush(cs,
4032                                                queue->device->physical_device->rad_info.chip_class,
4033                                                NULL, 0,
4034                                                queue->queue_family_index == RING_COMPUTE &&
4035                                                  queue->device->physical_device->rad_info.chip_class >= GFX7,
4036                                                RADV_CMD_FLAG_INV_ICACHE |
4037                                                RADV_CMD_FLAG_INV_SCACHE |
4038                                                RADV_CMD_FLAG_INV_VCACHE |
4039                                                RADV_CMD_FLAG_INV_L2 |
4040                                                RADV_CMD_FLAG_START_PIPELINE_STATS, 0);
4041                 }
4042
4043                 if (!queue->device->ws->cs_finalize(cs))
4044                         goto fail;
4045         }
4046
4047         if (queue->initial_full_flush_preamble_cs)
4048                         queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
4049
4050         if (queue->initial_preamble_cs)
4051                         queue->device->ws->cs_destroy(queue->initial_preamble_cs);
4052
4053         if (queue->continue_preamble_cs)
4054                         queue->device->ws->cs_destroy(queue->continue_preamble_cs);
4055
4056         queue->initial_full_flush_preamble_cs = dest_cs[0];
4057         queue->initial_preamble_cs = dest_cs[1];
4058         queue->continue_preamble_cs = dest_cs[2];
4059
4060         if (scratch_bo != queue->scratch_bo) {
4061                 if (queue->scratch_bo)
4062                         queue->device->ws->buffer_destroy(queue->scratch_bo);
4063                 queue->scratch_bo = scratch_bo;
4064         }
4065         queue->scratch_size_per_wave = scratch_size_per_wave;
4066         queue->scratch_waves = scratch_waves;
4067
4068         if (compute_scratch_bo != queue->compute_scratch_bo) {
4069                 if (queue->compute_scratch_bo)
4070                         queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
4071                 queue->compute_scratch_bo = compute_scratch_bo;
4072         }
4073         queue->compute_scratch_size_per_wave = compute_scratch_size_per_wave;
4074         queue->compute_scratch_waves = compute_scratch_waves;
4075
4076         if (esgs_ring_bo != queue->esgs_ring_bo) {
4077                 if (queue->esgs_ring_bo)
4078                         queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
4079                 queue->esgs_ring_bo = esgs_ring_bo;
4080                 queue->esgs_ring_size = esgs_ring_size;
4081         }
4082
4083         if (gsvs_ring_bo != queue->gsvs_ring_bo) {
4084                 if (queue->gsvs_ring_bo)
4085                         queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
4086                 queue->gsvs_ring_bo = gsvs_ring_bo;
4087                 queue->gsvs_ring_size = gsvs_ring_size;
4088         }
4089
4090         if (tess_rings_bo != queue->tess_rings_bo) {
4091                 queue->tess_rings_bo = tess_rings_bo;
4092                 queue->has_tess_rings = true;
4093         }
4094
4095         if (gds_bo != queue->gds_bo) {
4096                 queue->gds_bo = gds_bo;
4097                 queue->has_gds = true;
4098         }
4099
4100         if (gds_oa_bo != queue->gds_oa_bo) {
4101                 queue->gds_oa_bo = gds_oa_bo;
4102                 queue->has_gds_oa = true;
4103         }
4104
4105         if (descriptor_bo != queue->descriptor_bo) {
4106                 if (queue->descriptor_bo)
4107                         queue->device->ws->buffer_destroy(queue->descriptor_bo);
4108
4109                 queue->descriptor_bo = descriptor_bo;
4110         }
4111
4112         if (add_sample_positions)
4113                 queue->has_sample_positions = true;
4114
4115         *initial_full_flush_preamble_cs = queue->initial_full_flush_preamble_cs;
4116         *initial_preamble_cs = queue->initial_preamble_cs;
4117         *continue_preamble_cs = queue->continue_preamble_cs;
4118         if (!scratch_size && !compute_scratch_size && !esgs_ring_size && !gsvs_ring_size)
4119                         *continue_preamble_cs = NULL;
4120         return VK_SUCCESS;
4121 fail:
4122         for (int i = 0; i < ARRAY_SIZE(dest_cs); ++i)
4123                 if (dest_cs[i])
4124                         queue->device->ws->cs_destroy(dest_cs[i]);
4125         if (descriptor_bo && descriptor_bo != queue->descriptor_bo)
4126                 queue->device->ws->buffer_destroy(descriptor_bo);
4127         if (scratch_bo && scratch_bo != queue->scratch_bo)
4128                 queue->device->ws->buffer_destroy(scratch_bo);
4129         if (compute_scratch_bo && compute_scratch_bo != queue->compute_scratch_bo)
4130                 queue->device->ws->buffer_destroy(compute_scratch_bo);
4131         if (esgs_ring_bo && esgs_ring_bo != queue->esgs_ring_bo)
4132                 queue->device->ws->buffer_destroy(esgs_ring_bo);
4133         if (gsvs_ring_bo && gsvs_ring_bo != queue->gsvs_ring_bo)
4134                 queue->device->ws->buffer_destroy(gsvs_ring_bo);
4135         if (tess_rings_bo && tess_rings_bo != queue->tess_rings_bo)
4136                 queue->device->ws->buffer_destroy(tess_rings_bo);
4137         if (gds_bo && gds_bo != queue->gds_bo)
4138                 queue->device->ws->buffer_destroy(gds_bo);
4139         if (gds_oa_bo && gds_oa_bo != queue->gds_oa_bo)
4140                 queue->device->ws->buffer_destroy(gds_oa_bo);
4141
4142         return vk_error(queue->device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
4143 }
4144
4145 static VkResult radv_alloc_sem_counts(struct radv_device *device,
4146                                       struct radv_winsys_sem_counts *counts,
4147                                       int num_sems,
4148                                       struct radv_semaphore_part **sems,
4149                                       const uint64_t *timeline_values,
4150                                       VkFence _fence,
4151                                       bool is_signal)
4152 {
4153         int syncobj_idx = 0, sem_idx = 0;
4154
4155         if (num_sems == 0 && _fence == VK_NULL_HANDLE)
4156                 return VK_SUCCESS;
4157
4158         for (uint32_t i = 0; i < num_sems; i++) {
4159                 switch(sems[i]->kind) {
4160                 case RADV_SEMAPHORE_SYNCOBJ:
4161                         counts->syncobj_count++;
4162                         break;
4163                 case RADV_SEMAPHORE_WINSYS:
4164                         counts->sem_count++;
4165                         break;
4166                 case RADV_SEMAPHORE_NONE:
4167                         break;
4168                 case RADV_SEMAPHORE_TIMELINE:
4169                         counts->syncobj_count++;
4170                         break;
4171                 }
4172         }
4173
4174         if (_fence != VK_NULL_HANDLE) {
4175                 RADV_FROM_HANDLE(radv_fence, fence, _fence);
4176                 if (fence->temp_syncobj || fence->syncobj)
4177                         counts->syncobj_count++;
4178         }
4179
4180         if (counts->syncobj_count) {
4181                 counts->syncobj = (uint32_t *)malloc(sizeof(uint32_t) * counts->syncobj_count);
4182                 if (!counts->syncobj)
4183                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
4184         }
4185
4186         if (counts->sem_count) {
4187                 counts->sem = (struct radeon_winsys_sem **)malloc(sizeof(struct radeon_winsys_sem *) * counts->sem_count);
4188                 if (!counts->sem) {
4189                         free(counts->syncobj);
4190                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
4191                 }
4192         }
4193
4194         for (uint32_t i = 0; i < num_sems; i++) {
4195                 switch(sems[i]->kind) {
4196                 case RADV_SEMAPHORE_NONE:
4197                         unreachable("Empty semaphore");
4198                         break;
4199                 case RADV_SEMAPHORE_SYNCOBJ:
4200                         counts->syncobj[syncobj_idx++] = sems[i]->syncobj;
4201                         break;
4202                 case RADV_SEMAPHORE_WINSYS:
4203                         counts->sem[sem_idx++] = sems[i]->ws_sem;
4204                         break;
4205                 case RADV_SEMAPHORE_TIMELINE: {
4206                         pthread_mutex_lock(&sems[i]->timeline.mutex);
4207                         struct radv_timeline_point *point = NULL;
4208                         if (is_signal) {
4209                                 point = radv_timeline_add_point_locked(device, &sems[i]->timeline, timeline_values[i]);
4210                         } else {
4211                                 point = radv_timeline_find_point_at_least_locked(device, &sems[i]->timeline, timeline_values[i]);
4212                         }
4213
4214                         pthread_mutex_unlock(&sems[i]->timeline.mutex);
4215
4216                         if (point) {
4217                                 counts->syncobj[syncobj_idx++] = point->syncobj;
4218                         } else {
4219                                 /* Explicitly remove the semaphore so we might not find
4220                                  * a point later post-submit. */
4221                                 sems[i] = NULL;
4222                         }
4223                         break;
4224                 }
4225                 }
4226         }
4227
4228         if (_fence != VK_NULL_HANDLE) {
4229                 RADV_FROM_HANDLE(radv_fence, fence, _fence);
4230                 if (fence->temp_syncobj)
4231                         counts->syncobj[syncobj_idx++] = fence->temp_syncobj;
4232                 else if (fence->syncobj)
4233                         counts->syncobj[syncobj_idx++] = fence->syncobj;
4234         }
4235
4236         assert(syncobj_idx <= counts->syncobj_count);
4237         counts->syncobj_count = syncobj_idx;
4238
4239         return VK_SUCCESS;
4240 }
4241
4242 static void
4243 radv_free_sem_info(struct radv_winsys_sem_info *sem_info)
4244 {
4245         free(sem_info->wait.syncobj);
4246         free(sem_info->wait.sem);
4247         free(sem_info->signal.syncobj);
4248         free(sem_info->signal.sem);
4249 }
4250
4251
4252 static void radv_free_temp_syncobjs(struct radv_device *device,
4253                                     int num_sems,
4254                                     struct radv_semaphore_part *sems)
4255 {
4256         for (uint32_t i = 0; i < num_sems; i++) {
4257                 radv_destroy_semaphore_part(device, sems + i);
4258         }
4259 }
4260
4261 static VkResult
4262 radv_alloc_sem_info(struct radv_device *device,
4263                     struct radv_winsys_sem_info *sem_info,
4264                     int num_wait_sems,
4265                     struct radv_semaphore_part **wait_sems,
4266                     const uint64_t *wait_values,
4267                     int num_signal_sems,
4268                     struct radv_semaphore_part **signal_sems,
4269                     const uint64_t *signal_values,
4270                     VkFence fence)
4271 {
4272         VkResult ret;
4273         memset(sem_info, 0, sizeof(*sem_info));
4274
4275         ret = radv_alloc_sem_counts(device, &sem_info->wait, num_wait_sems, wait_sems, wait_values, VK_NULL_HANDLE, false);
4276         if (ret)
4277                 return ret;
4278         ret = radv_alloc_sem_counts(device, &sem_info->signal, num_signal_sems, signal_sems, signal_values, fence, true);
4279         if (ret)
4280                 radv_free_sem_info(sem_info);
4281
4282         /* caller can override these */
4283         sem_info->cs_emit_wait = true;
4284         sem_info->cs_emit_signal = true;
4285         return ret;
4286 }
4287
4288 static void
4289 radv_finalize_timelines(struct radv_device *device,
4290                         uint32_t num_wait_sems,
4291                         struct radv_semaphore_part **wait_sems,
4292                         const uint64_t *wait_values,
4293                         uint32_t num_signal_sems,
4294                         struct radv_semaphore_part **signal_sems,
4295                         const uint64_t *signal_values,
4296                         struct list_head *processing_list)
4297 {
4298         for (uint32_t i = 0; i < num_wait_sems; ++i) {
4299                 if (wait_sems[i] && wait_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4300                         pthread_mutex_lock(&wait_sems[i]->timeline.mutex);
4301                         struct radv_timeline_point *point =
4302                                 radv_timeline_find_point_at_least_locked(device, &wait_sems[i]->timeline, wait_values[i]);
4303                         point->wait_count -= 2;
4304                         pthread_mutex_unlock(&wait_sems[i]->timeline.mutex);
4305                 }
4306         }
4307         for (uint32_t i = 0; i < num_signal_sems; ++i) {
4308                 if (signal_sems[i] && signal_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4309                         pthread_mutex_lock(&signal_sems[i]->timeline.mutex);
4310                         struct radv_timeline_point *point =
4311                                 radv_timeline_find_point_at_least_locked(device, &signal_sems[i]->timeline, signal_values[i]);
4312                         signal_sems[i]->timeline.highest_submitted =
4313                                 MAX2(signal_sems[i]->timeline.highest_submitted, point->value);
4314                         point->wait_count -= 2;
4315                         radv_timeline_trigger_waiters_locked(&signal_sems[i]->timeline, processing_list);
4316                         pthread_mutex_unlock(&signal_sems[i]->timeline.mutex);
4317                 }
4318         }
4319 }
4320
4321 static void
4322 radv_sparse_buffer_bind_memory(struct radv_device *device,
4323                                const VkSparseBufferMemoryBindInfo *bind)
4324 {
4325         RADV_FROM_HANDLE(radv_buffer, buffer, bind->buffer);
4326
4327         for (uint32_t i = 0; i < bind->bindCount; ++i) {
4328                 struct radv_device_memory *mem = NULL;
4329
4330                 if (bind->pBinds[i].memory != VK_NULL_HANDLE)
4331                         mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
4332
4333                 device->ws->buffer_virtual_bind(buffer->bo,
4334                                                 bind->pBinds[i].resourceOffset,
4335                                                 bind->pBinds[i].size,
4336                                                 mem ? mem->bo : NULL,
4337                                                 bind->pBinds[i].memoryOffset);
4338         }
4339 }
4340
4341 static void
4342 radv_sparse_image_opaque_bind_memory(struct radv_device *device,
4343                                      const VkSparseImageOpaqueMemoryBindInfo *bind)
4344 {
4345         RADV_FROM_HANDLE(radv_image, image, bind->image);
4346
4347         for (uint32_t i = 0; i < bind->bindCount; ++i) {
4348                 struct radv_device_memory *mem = NULL;
4349
4350                 if (bind->pBinds[i].memory != VK_NULL_HANDLE)
4351                         mem = radv_device_memory_from_handle(bind->pBinds[i].memory);
4352
4353                 device->ws->buffer_virtual_bind(image->bo,
4354                                                 bind->pBinds[i].resourceOffset,
4355                                                 bind->pBinds[i].size,
4356                                                 mem ? mem->bo : NULL,
4357                                                 bind->pBinds[i].memoryOffset);
4358         }
4359 }
4360
4361 static VkResult
4362 radv_get_preambles(struct radv_queue *queue,
4363                    const VkCommandBuffer *cmd_buffers,
4364                    uint32_t cmd_buffer_count,
4365                    struct radeon_cmdbuf **initial_full_flush_preamble_cs,
4366                    struct radeon_cmdbuf **initial_preamble_cs,
4367                    struct radeon_cmdbuf **continue_preamble_cs)
4368 {
4369         uint32_t scratch_size_per_wave = 0, waves_wanted = 0;
4370         uint32_t compute_scratch_size_per_wave = 0, compute_waves_wanted = 0;
4371         uint32_t esgs_ring_size = 0, gsvs_ring_size = 0;
4372         bool tess_rings_needed = false;
4373         bool gds_needed = false;
4374         bool gds_oa_needed = false;
4375         bool sample_positions_needed = false;
4376
4377         for (uint32_t j = 0; j < cmd_buffer_count; j++) {
4378                 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer,
4379                                  cmd_buffers[j]);
4380
4381                 scratch_size_per_wave = MAX2(scratch_size_per_wave, cmd_buffer->scratch_size_per_wave_needed);
4382                 waves_wanted = MAX2(waves_wanted, cmd_buffer->scratch_waves_wanted);
4383                 compute_scratch_size_per_wave = MAX2(compute_scratch_size_per_wave,
4384                                                      cmd_buffer->compute_scratch_size_per_wave_needed);
4385                 compute_waves_wanted = MAX2(compute_waves_wanted,
4386                                             cmd_buffer->compute_scratch_waves_wanted);
4387                 esgs_ring_size = MAX2(esgs_ring_size, cmd_buffer->esgs_ring_size_needed);
4388                 gsvs_ring_size = MAX2(gsvs_ring_size, cmd_buffer->gsvs_ring_size_needed);
4389                 tess_rings_needed |= cmd_buffer->tess_rings_needed;
4390                 gds_needed |= cmd_buffer->gds_needed;
4391                 gds_oa_needed |= cmd_buffer->gds_oa_needed;
4392                 sample_positions_needed |= cmd_buffer->sample_positions_needed;
4393         }
4394
4395         return radv_get_preamble_cs(queue, scratch_size_per_wave, waves_wanted,
4396                                     compute_scratch_size_per_wave, compute_waves_wanted,
4397                                     esgs_ring_size, gsvs_ring_size, tess_rings_needed,
4398                                     gds_needed, gds_oa_needed, sample_positions_needed,
4399                                     initial_full_flush_preamble_cs,
4400                                     initial_preamble_cs, continue_preamble_cs);
4401 }
4402
4403 struct radv_deferred_queue_submission {
4404         struct radv_queue *queue;
4405         VkCommandBuffer *cmd_buffers;
4406         uint32_t cmd_buffer_count;
4407
4408         /* Sparse bindings that happen on a queue. */
4409         VkSparseBufferMemoryBindInfo *buffer_binds;
4410         uint32_t buffer_bind_count;
4411         VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
4412         uint32_t image_opaque_bind_count;
4413
4414         bool flush_caches;
4415         VkShaderStageFlags wait_dst_stage_mask;
4416         struct radv_semaphore_part **wait_semaphores;
4417         uint32_t wait_semaphore_count;
4418         struct radv_semaphore_part **signal_semaphores;
4419         uint32_t signal_semaphore_count;
4420         VkFence fence;
4421
4422         uint64_t *wait_values;
4423         uint64_t *signal_values;
4424
4425         struct radv_semaphore_part *temporary_semaphore_parts;
4426         uint32_t temporary_semaphore_part_count;
4427
4428         struct list_head queue_pending_list;
4429         uint32_t submission_wait_count;
4430         struct radv_timeline_waiter *wait_nodes;
4431
4432         struct list_head processing_list;
4433 };
4434
4435 struct radv_queue_submission {
4436         const VkCommandBuffer *cmd_buffers;
4437         uint32_t cmd_buffer_count;
4438
4439         /* Sparse bindings that happen on a queue. */
4440         const VkSparseBufferMemoryBindInfo *buffer_binds;
4441         uint32_t buffer_bind_count;
4442         const VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
4443         uint32_t image_opaque_bind_count;
4444
4445         bool flush_caches;
4446         VkPipelineStageFlags wait_dst_stage_mask;
4447         const VkSemaphore *wait_semaphores;
4448         uint32_t wait_semaphore_count;
4449         const VkSemaphore *signal_semaphores;
4450         uint32_t signal_semaphore_count;
4451         VkFence fence;
4452
4453         const uint64_t *wait_values;
4454         uint32_t wait_value_count;
4455         const uint64_t *signal_values;
4456         uint32_t signal_value_count;
4457 };
4458
4459 static VkResult
4460 radv_create_deferred_submission(struct radv_queue *queue,
4461                                 const struct radv_queue_submission *submission,
4462                                 struct radv_deferred_queue_submission **out)
4463 {
4464         struct radv_deferred_queue_submission *deferred = NULL;
4465         size_t size = sizeof(struct radv_deferred_queue_submission);
4466
4467         uint32_t temporary_count = 0;
4468         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4469                 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphores[i]);
4470                 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE)
4471                         ++temporary_count;
4472         }
4473
4474         size += submission->cmd_buffer_count * sizeof(VkCommandBuffer);
4475         size += submission->buffer_bind_count * sizeof(VkSparseBufferMemoryBindInfo);
4476         size += submission->image_opaque_bind_count * sizeof(VkSparseImageOpaqueMemoryBindInfo);
4477         size += submission->wait_semaphore_count * sizeof(struct radv_semaphore_part *);
4478         size += temporary_count * sizeof(struct radv_semaphore_part);
4479         size += submission->signal_semaphore_count * sizeof(struct radv_semaphore_part *);
4480         size += submission->wait_value_count * sizeof(uint64_t);
4481         size += submission->signal_value_count * sizeof(uint64_t);
4482         size += submission->wait_semaphore_count * sizeof(struct radv_timeline_waiter);
4483
4484         deferred = calloc(1, size);
4485         if (!deferred)
4486                 return VK_ERROR_OUT_OF_HOST_MEMORY;
4487
4488         deferred->queue = queue;
4489
4490         deferred->cmd_buffers = (void*)(deferred + 1);
4491         deferred->cmd_buffer_count = submission->cmd_buffer_count;
4492         memcpy(deferred->cmd_buffers, submission->cmd_buffers,
4493                submission->cmd_buffer_count * sizeof(*deferred->cmd_buffers));
4494
4495         deferred->buffer_binds = (void*)(deferred->cmd_buffers + submission->cmd_buffer_count);
4496         deferred->buffer_bind_count = submission->buffer_bind_count;
4497         memcpy(deferred->buffer_binds, submission->buffer_binds,
4498                submission->buffer_bind_count * sizeof(*deferred->buffer_binds));
4499
4500         deferred->image_opaque_binds = (void*)(deferred->buffer_binds + submission->buffer_bind_count);
4501         deferred->image_opaque_bind_count = submission->image_opaque_bind_count;
4502         memcpy(deferred->image_opaque_binds, submission->image_opaque_binds,
4503                submission->image_opaque_bind_count * sizeof(*deferred->image_opaque_binds));
4504
4505         deferred->flush_caches = submission->flush_caches;
4506         deferred->wait_dst_stage_mask = submission->wait_dst_stage_mask;
4507
4508         deferred->wait_semaphores = (void*)(deferred->image_opaque_binds + deferred->image_opaque_bind_count);
4509         deferred->wait_semaphore_count = submission->wait_semaphore_count;
4510
4511         deferred->signal_semaphores = (void*)(deferred->wait_semaphores + deferred->wait_semaphore_count);
4512         deferred->signal_semaphore_count = submission->signal_semaphore_count;
4513
4514         deferred->fence = submission->fence;
4515
4516         deferred->temporary_semaphore_parts = (void*)(deferred->signal_semaphores + deferred->signal_semaphore_count);
4517         deferred->temporary_semaphore_part_count = temporary_count;
4518
4519         uint32_t temporary_idx = 0;
4520         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4521                 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphores[i]);
4522                 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) {
4523                         deferred->wait_semaphores[i] = &deferred->temporary_semaphore_parts[temporary_idx];
4524                         deferred->temporary_semaphore_parts[temporary_idx] = semaphore->temporary;
4525                         semaphore->temporary.kind = RADV_SEMAPHORE_NONE;
4526                         ++temporary_idx;
4527                 } else
4528                         deferred->wait_semaphores[i] = &semaphore->permanent;
4529         }
4530
4531         for (uint32_t i = 0; i < submission->signal_semaphore_count; ++i) {
4532                 RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->signal_semaphores[i]);
4533                 if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) {
4534                         deferred->signal_semaphores[i] = &semaphore->temporary;
4535                 } else {
4536                         deferred->signal_semaphores[i] = &semaphore->permanent;
4537                 }
4538         }
4539
4540         deferred->wait_values = (void*)(deferred->temporary_semaphore_parts + temporary_count);
4541         memcpy(deferred->wait_values, submission->wait_values, submission->wait_value_count * sizeof(uint64_t));
4542         deferred->signal_values = deferred->wait_values + submission->wait_value_count;
4543         memcpy(deferred->signal_values, submission->signal_values, submission->signal_value_count * sizeof(uint64_t));
4544
4545         deferred->wait_nodes = (void*)(deferred->signal_values + submission->signal_value_count);
4546         /* This is worst-case. radv_queue_enqueue_submission will fill in further, but this
4547          * ensure the submission is not accidentally triggered early when adding wait timelines. */
4548         deferred->submission_wait_count = 1 + submission->wait_semaphore_count;
4549
4550         *out = deferred;
4551         return VK_SUCCESS;
4552 }
4553
4554 static void
4555 radv_queue_enqueue_submission(struct radv_deferred_queue_submission *submission,
4556                               struct list_head *processing_list)
4557 {
4558         uint32_t wait_cnt = 0;
4559         struct radv_timeline_waiter *waiter = submission->wait_nodes;
4560         for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) {
4561                 if (submission->wait_semaphores[i]->kind == RADV_SEMAPHORE_TIMELINE) {
4562                         pthread_mutex_lock(&submission->wait_semaphores[i]->timeline.mutex);
4563                         if (submission->wait_semaphores[i]->timeline.highest_submitted < submission->wait_values[i]) {
4564                                 ++wait_cnt;
4565                                 waiter->value = submission->wait_values[i];
4566                                 waiter->submission = submission;
4567                                 list_addtail(&waiter->list, &submission->wait_semaphores[i]->timeline.waiters);
4568                                 ++waiter;
4569                         }
4570                         pthread_mutex_unlock(&submission->wait_semaphores[i]->timeline.mutex);
4571                 }
4572         }
4573
4574         pthread_mutex_lock(&submission->queue->pending_mutex);
4575
4576         bool is_first = list_is_empty(&submission->queue->pending_submissions);
4577         list_addtail(&submission->queue_pending_list, &submission->queue->pending_submissions);
4578
4579         pthread_mutex_unlock(&submission->queue->pending_mutex);
4580
4581         /* If there is already a submission in the queue, that will decrement the counter by 1 when
4582          * submitted, but if the queue was empty, we decrement ourselves as there is no previous
4583          * submission. */
4584         uint32_t decrement = submission->wait_semaphore_count - wait_cnt + (is_first ? 1 : 0);
4585         if (__atomic_sub_fetch(&submission->submission_wait_count, decrement, __ATOMIC_ACQ_REL) == 0) {
4586                 list_addtail(&submission->processing_list, processing_list);
4587         }
4588 }
4589
4590 static void
4591 radv_queue_submission_update_queue(struct radv_deferred_queue_submission *submission,
4592                                    struct list_head *processing_list)
4593 {
4594         pthread_mutex_lock(&submission->queue->pending_mutex);
4595         list_del(&submission->queue_pending_list);
4596
4597         /* trigger the next submission in the queue. */
4598         if (!list_is_empty(&submission->queue->pending_submissions)) {
4599                 struct radv_deferred_queue_submission *next_submission =
4600                         list_first_entry(&submission->queue->pending_submissions,
4601                                          struct radv_deferred_queue_submission,
4602                                          queue_pending_list);
4603                 if (p_atomic_dec_zero(&next_submission->submission_wait_count)) {
4604                         list_addtail(&next_submission->processing_list, processing_list);
4605                 }
4606         }
4607         pthread_mutex_unlock(&submission->queue->pending_mutex);
4608
4609         pthread_cond_broadcast(&submission->queue->device->timeline_cond);
4610 }
4611
4612 static VkResult
4613 radv_queue_submit_deferred(struct radv_deferred_queue_submission *submission,
4614                            struct list_head *processing_list)
4615 {
4616         RADV_FROM_HANDLE(radv_fence, fence, submission->fence);
4617         struct radv_queue *queue = submission->queue;
4618         struct radeon_winsys_ctx *ctx = queue->hw_ctx;
4619         uint32_t max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT;
4620         struct radeon_winsys_fence *base_fence = fence ? fence->fence : NULL;
4621         bool do_flush = submission->flush_caches || submission->wait_dst_stage_mask;
4622         bool can_patch = true;
4623         uint32_t advance;
4624         struct radv_winsys_sem_info sem_info;
4625         VkResult result;
4626         int ret;
4627         struct radeon_cmdbuf *initial_preamble_cs = NULL;
4628         struct radeon_cmdbuf *initial_flush_preamble_cs = NULL;
4629         struct radeon_cmdbuf *continue_preamble_cs = NULL;
4630
4631         result = radv_get_preambles(queue, submission->cmd_buffers,
4632                                     submission->cmd_buffer_count,
4633                                     &initial_preamble_cs,
4634                                     &initial_flush_preamble_cs,
4635                                     &continue_preamble_cs);
4636         if (result != VK_SUCCESS)
4637                 goto fail;
4638
4639         result = radv_alloc_sem_info(queue->device,
4640                                      &sem_info,
4641                                      submission->wait_semaphore_count,
4642                                      submission->wait_semaphores,
4643                                      submission->wait_values,
4644                                      submission->signal_semaphore_count,
4645                                      submission->signal_semaphores,
4646                                      submission->signal_values,
4647                                      submission->fence);
4648         if (result != VK_SUCCESS)
4649                 goto fail;
4650
4651         for (uint32_t i = 0; i < submission->buffer_bind_count; ++i) {
4652                 radv_sparse_buffer_bind_memory(queue->device,
4653                                                submission->buffer_binds + i);
4654         }
4655
4656         for (uint32_t i = 0; i < submission->image_opaque_bind_count; ++i) {
4657                 radv_sparse_image_opaque_bind_memory(queue->device,
4658                                                      submission->image_opaque_binds + i);
4659         }
4660
4661         if (!submission->cmd_buffer_count) {
4662                 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx,
4663                                                    &queue->device->empty_cs[queue->queue_family_index],
4664                                                    1, NULL, NULL,
4665                                                    &sem_info, NULL,
4666                                                    false, base_fence);
4667                 if (ret) {
4668                         radv_loge("failed to submit CS\n");
4669                         abort();
4670                 }
4671
4672                 goto success;
4673         } else {
4674                 struct radeon_cmdbuf **cs_array = malloc(sizeof(struct radeon_cmdbuf *) *
4675                                                          (submission->cmd_buffer_count));
4676
4677                 for (uint32_t j = 0; j < submission->cmd_buffer_count; j++) {
4678                         RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, submission->cmd_buffers[j]);
4679                         assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
4680
4681                         cs_array[j] = cmd_buffer->cs;
4682                         if ((cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT))
4683                                 can_patch = false;
4684
4685                         cmd_buffer->status = RADV_CMD_BUFFER_STATUS_PENDING;
4686                 }
4687
4688                 for (uint32_t j = 0; j < submission->cmd_buffer_count; j += advance) {
4689                         struct radeon_cmdbuf *initial_preamble = (do_flush && !j) ? initial_flush_preamble_cs : initial_preamble_cs;
4690                         const struct radv_winsys_bo_list *bo_list = NULL;
4691
4692                         advance = MIN2(max_cs_submission,
4693                                        submission->cmd_buffer_count - j);
4694
4695                         if (queue->device->trace_bo)
4696                                 *queue->device->trace_id_ptr = 0;
4697
4698                         sem_info.cs_emit_wait = j == 0;
4699                         sem_info.cs_emit_signal = j + advance == submission->cmd_buffer_count;
4700
4701                         if (unlikely(queue->device->use_global_bo_list)) {
4702                                 pthread_mutex_lock(&queue->device->bo_list.mutex);
4703                                 bo_list = &queue->device->bo_list.list;
4704                         }
4705
4706                         ret = queue->device->ws->cs_submit(ctx, queue->queue_idx, cs_array + j,
4707                                                            advance, initial_preamble, continue_preamble_cs,
4708                                                            &sem_info, bo_list,
4709                                                            can_patch, base_fence);
4710
4711                         if (unlikely(queue->device->use_global_bo_list))
4712                                 pthread_mutex_unlock(&queue->device->bo_list.mutex);
4713
4714                         if (ret) {
4715                                 radv_loge("failed to submit CS\n");
4716                                 abort();
4717                         }
4718                         if (queue->device->trace_bo) {
4719                                 radv_check_gpu_hangs(queue, cs_array[j]);
4720                         }
4721                 }
4722
4723                 free(cs_array);
4724         }
4725
4726 success:
4727         radv_free_temp_syncobjs(queue->device,
4728                                 submission->temporary_semaphore_part_count,
4729                                 submission->temporary_semaphore_parts);
4730         radv_finalize_timelines(queue->device,
4731                                 submission->wait_semaphore_count,
4732                                 submission->wait_semaphores,
4733                                 submission->wait_values,
4734                                 submission->signal_semaphore_count,
4735                                 submission->signal_semaphores,
4736                                 submission->signal_values,
4737                                 processing_list);
4738         /* Has to happen after timeline finalization to make sure the
4739          * condition variable is only triggered when timelines and queue have
4740          * been updated. */
4741         radv_queue_submission_update_queue(submission, processing_list);
4742         radv_free_sem_info(&sem_info);
4743         free(submission);
4744         return VK_SUCCESS;
4745
4746 fail:
4747         radv_free_temp_syncobjs(queue->device,
4748                                 submission->temporary_semaphore_part_count,
4749                                 submission->temporary_semaphore_parts);
4750         free(submission);
4751         return VK_ERROR_DEVICE_LOST;
4752 }
4753
4754 static VkResult
4755 radv_process_submissions(struct list_head *processing_list)
4756 {
4757         while(!list_is_empty(processing_list)) {
4758                 struct radv_deferred_queue_submission *submission =
4759                         list_first_entry(processing_list, struct radv_deferred_queue_submission, processing_list);
4760                 list_del(&submission->processing_list);
4761
4762                 VkResult result = radv_queue_submit_deferred(submission, processing_list);
4763                 if (result != VK_SUCCESS)
4764                         return result;
4765         }
4766         return VK_SUCCESS;
4767 }
4768
4769 static VkResult radv_queue_submit(struct radv_queue *queue,
4770                                   const struct radv_queue_submission *submission)
4771 {
4772         struct radv_deferred_queue_submission *deferred = NULL;
4773
4774         VkResult result = radv_create_deferred_submission(queue, submission, &deferred);
4775         if (result != VK_SUCCESS)
4776                 return result;
4777
4778         struct list_head processing_list;
4779         list_inithead(&processing_list);
4780
4781         radv_queue_enqueue_submission(deferred, &processing_list);
4782         return radv_process_submissions(&processing_list);
4783 }
4784
4785 bool
4786 radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs)
4787 {
4788         struct radeon_winsys_ctx *ctx = queue->hw_ctx;
4789         struct radv_winsys_sem_info sem_info;
4790         VkResult result;
4791         int ret;
4792
4793         result = radv_alloc_sem_info(queue->device, &sem_info, 0, NULL, 0, 0,
4794                                      0, NULL, VK_NULL_HANDLE);
4795         if (result != VK_SUCCESS)
4796                 return false;
4797
4798         ret = queue->device->ws->cs_submit(ctx, queue->queue_idx, &cs, 1, NULL,
4799                                            NULL, &sem_info, NULL, false, NULL);
4800         radv_free_sem_info(&sem_info);
4801         return !ret;
4802 }
4803
4804 /* Signals fence as soon as all the work currently put on queue is done. */
4805 static VkResult radv_signal_fence(struct radv_queue *queue,
4806                               VkFence fence)
4807 {
4808         return radv_queue_submit(queue, &(struct radv_queue_submission) {
4809                         .fence = fence
4810                 });
4811 }
4812
4813 static bool radv_submit_has_effects(const VkSubmitInfo *info)
4814 {
4815         return info->commandBufferCount ||
4816                info->waitSemaphoreCount ||
4817                info->signalSemaphoreCount;
4818 }
4819
4820 VkResult radv_QueueSubmit(
4821         VkQueue                                     _queue,
4822         uint32_t                                    submitCount,
4823         const VkSubmitInfo*                         pSubmits,
4824         VkFence                                     fence)
4825 {
4826         RADV_FROM_HANDLE(radv_queue, queue, _queue);
4827         VkResult result;
4828         uint32_t fence_idx = 0;
4829         bool flushed_caches = false;
4830
4831         if (fence != VK_NULL_HANDLE) {
4832                 for (uint32_t i = 0; i < submitCount; ++i)
4833                         if (radv_submit_has_effects(pSubmits + i))
4834                                 fence_idx = i;
4835         } else
4836                 fence_idx = UINT32_MAX;
4837
4838         for (uint32_t i = 0; i < submitCount; i++) {
4839                 if (!radv_submit_has_effects(pSubmits + i) && fence_idx != i)
4840                         continue;
4841
4842                 VkPipelineStageFlags wait_dst_stage_mask = 0;
4843                 for (unsigned j = 0; j < pSubmits[i].waitSemaphoreCount; ++j) {
4844                         wait_dst_stage_mask |= pSubmits[i].pWaitDstStageMask[j];
4845                 }
4846
4847                 const VkTimelineSemaphoreSubmitInfo *timeline_info =
4848                         vk_find_struct_const(pSubmits[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
4849
4850                 result = radv_queue_submit(queue, &(struct radv_queue_submission) {
4851                                 .cmd_buffers = pSubmits[i].pCommandBuffers,
4852                                 .cmd_buffer_count = pSubmits[i].commandBufferCount,
4853                                 .wait_dst_stage_mask = wait_dst_stage_mask,
4854                                 .flush_caches = !flushed_caches,
4855                                 .wait_semaphores = pSubmits[i].pWaitSemaphores,
4856                                 .wait_semaphore_count = pSubmits[i].waitSemaphoreCount,
4857                                 .signal_semaphores = pSubmits[i].pSignalSemaphores,
4858                                 .signal_semaphore_count = pSubmits[i].signalSemaphoreCount,
4859                                 .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
4860                                 .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
4861                                 .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
4862                                 .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
4863                                 .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
4864                         });
4865                 if (result != VK_SUCCESS)
4866                         return result;
4867
4868                 flushed_caches  = true;
4869         }
4870
4871         if (fence != VK_NULL_HANDLE && !submitCount) {
4872                 result = radv_signal_fence(queue, fence);
4873                 if (result != VK_SUCCESS)
4874                         return result;
4875         }
4876
4877         return VK_SUCCESS;
4878 }
4879
4880 VkResult radv_QueueWaitIdle(
4881         VkQueue                                     _queue)
4882 {
4883         RADV_FROM_HANDLE(radv_queue, queue, _queue);
4884
4885         pthread_mutex_lock(&queue->pending_mutex);
4886         while (!list_is_empty(&queue->pending_submissions)) {
4887                 pthread_cond_wait(&queue->device->timeline_cond, &queue->pending_mutex);
4888         }
4889         pthread_mutex_unlock(&queue->pending_mutex);
4890
4891         queue->device->ws->ctx_wait_idle(queue->hw_ctx,
4892                                          radv_queue_family_to_ring(queue->queue_family_index),
4893                                          queue->queue_idx);
4894         return VK_SUCCESS;
4895 }
4896
4897 VkResult radv_DeviceWaitIdle(
4898         VkDevice                                    _device)
4899 {
4900         RADV_FROM_HANDLE(radv_device, device, _device);
4901
4902         for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
4903                 for (unsigned q = 0; q < device->queue_count[i]; q++) {
4904                         radv_QueueWaitIdle(radv_queue_to_handle(&device->queues[i][q]));
4905                 }
4906         }
4907         return VK_SUCCESS;
4908 }
4909
4910 VkResult radv_EnumerateInstanceExtensionProperties(
4911     const char*                                 pLayerName,
4912     uint32_t*                                   pPropertyCount,
4913     VkExtensionProperties*                      pProperties)
4914 {
4915         VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
4916
4917         for (int i = 0; i < RADV_INSTANCE_EXTENSION_COUNT; i++) {
4918                 if (radv_supported_instance_extensions.extensions[i]) {
4919                         vk_outarray_append(&out, prop) {
4920                                 *prop = radv_instance_extensions[i];
4921                         }
4922                 }
4923         }
4924
4925         return vk_outarray_status(&out);
4926 }
4927
4928 VkResult radv_EnumerateDeviceExtensionProperties(
4929     VkPhysicalDevice                            physicalDevice,
4930     const char*                                 pLayerName,
4931     uint32_t*                                   pPropertyCount,
4932     VkExtensionProperties*                      pProperties)
4933 {
4934         RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
4935         VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
4936
4937         for (int i = 0; i < RADV_DEVICE_EXTENSION_COUNT; i++) {
4938                 if (device->supported_extensions.extensions[i]) {
4939                         vk_outarray_append(&out, prop) {
4940                                 *prop = radv_device_extensions[i];
4941                         }
4942                 }
4943         }
4944
4945         return vk_outarray_status(&out);
4946 }
4947
4948 PFN_vkVoidFunction radv_GetInstanceProcAddr(
4949         VkInstance                                  _instance,
4950         const char*                                 pName)
4951 {
4952         RADV_FROM_HANDLE(radv_instance, instance, _instance);
4953
4954         /* The Vulkan 1.0 spec for vkGetInstanceProcAddr has a table of exactly
4955          * when we have to return valid function pointers, NULL, or it's left
4956          * undefined.  See the table for exact details.
4957          */
4958         if (pName == NULL)
4959                 return NULL;
4960
4961 #define LOOKUP_RADV_ENTRYPOINT(entrypoint) \
4962         if (strcmp(pName, "vk" #entrypoint) == 0) \
4963                 return (PFN_vkVoidFunction)radv_##entrypoint
4964
4965         LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceExtensionProperties);
4966         LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceLayerProperties);
4967         LOOKUP_RADV_ENTRYPOINT(EnumerateInstanceVersion);
4968         LOOKUP_RADV_ENTRYPOINT(CreateInstance);
4969
4970         /* GetInstanceProcAddr() can also be called with a NULL instance.
4971          * See https://gitlab.khronos.org/vulkan/vulkan/issues/2057
4972          */
4973         LOOKUP_RADV_ENTRYPOINT(GetInstanceProcAddr);
4974
4975 #undef LOOKUP_RADV_ENTRYPOINT
4976
4977         if (instance == NULL)
4978                 return NULL;
4979
4980         int idx = radv_get_instance_entrypoint_index(pName);
4981         if (idx >= 0)
4982                 return instance->dispatch.entrypoints[idx];
4983
4984         idx = radv_get_physical_device_entrypoint_index(pName);
4985         if (idx >= 0)
4986                 return instance->physical_device_dispatch.entrypoints[idx];
4987
4988         idx = radv_get_device_entrypoint_index(pName);
4989         if (idx >= 0)
4990                 return instance->device_dispatch.entrypoints[idx];
4991
4992         return NULL;
4993 }
4994
4995 /* The loader wants us to expose a second GetInstanceProcAddr function
4996  * to work around certain LD_PRELOAD issues seen in apps.
4997  */
4998 PUBLIC
4999 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
5000         VkInstance                                  instance,
5001         const char*                                 pName);
5002
5003 PUBLIC
5004 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
5005         VkInstance                                  instance,
5006         const char*                                 pName)
5007 {
5008         return radv_GetInstanceProcAddr(instance, pName);
5009 }
5010
5011 PUBLIC
5012 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetPhysicalDeviceProcAddr(
5013         VkInstance                                  _instance,
5014         const char*                                 pName);
5015
5016 PUBLIC
5017 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetPhysicalDeviceProcAddr(
5018         VkInstance                                  _instance,
5019         const char*                                 pName)
5020 {
5021         RADV_FROM_HANDLE(radv_instance, instance, _instance);
5022
5023         if (!pName || !instance)
5024                 return NULL;
5025
5026         int idx = radv_get_physical_device_entrypoint_index(pName);
5027         if (idx < 0)
5028                 return NULL;
5029
5030         return instance->physical_device_dispatch.entrypoints[idx];
5031 }
5032
5033 PFN_vkVoidFunction radv_GetDeviceProcAddr(
5034         VkDevice                                    _device,
5035         const char*                                 pName)
5036 {
5037         RADV_FROM_HANDLE(radv_device, device, _device);
5038
5039         if (!device || !pName)
5040                 return NULL;
5041
5042         int idx = radv_get_device_entrypoint_index(pName);
5043         if (idx < 0)
5044                 return NULL;
5045
5046         return device->dispatch.entrypoints[idx];
5047 }
5048
5049 bool radv_get_memory_fd(struct radv_device *device,
5050                         struct radv_device_memory *memory,
5051                         int *pFD)
5052 {
5053         struct radeon_bo_metadata metadata;
5054
5055         if (memory->image) {
5056                 if (memory->image->tiling != VK_IMAGE_TILING_LINEAR)
5057                         radv_init_metadata(device, memory->image, &metadata);
5058                 device->ws->buffer_set_metadata(memory->bo, &metadata);
5059         }
5060
5061         return device->ws->buffer_get_fd(device->ws, memory->bo,
5062                                          pFD);
5063 }
5064
5065
5066 static void radv_free_memory(struct radv_device *device,
5067                              const VkAllocationCallbacks* pAllocator,
5068                              struct radv_device_memory *mem)
5069 {
5070         if (mem == NULL)
5071                 return;
5072
5073 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
5074         if (mem->android_hardware_buffer)
5075                 AHardwareBuffer_release(mem->android_hardware_buffer);
5076 #endif
5077
5078         if (mem->bo) {
5079                 if (device->overallocation_disallowed) {
5080                         mtx_lock(&device->overallocation_mutex);
5081                         device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
5082                         mtx_unlock(&device->overallocation_mutex);
5083                 }
5084
5085                 radv_bo_list_remove(device, mem->bo);
5086                 device->ws->buffer_destroy(mem->bo);
5087                 mem->bo = NULL;
5088         }
5089
5090         vk_free2(&device->alloc, pAllocator, mem);
5091 }
5092
5093 static VkResult radv_alloc_memory(struct radv_device *device,
5094                                   const VkMemoryAllocateInfo*     pAllocateInfo,
5095                                   const VkAllocationCallbacks*    pAllocator,
5096                                   VkDeviceMemory*                 pMem)
5097 {
5098         struct radv_device_memory *mem;
5099         VkResult result;
5100         enum radeon_bo_domain domain;
5101         uint32_t flags = 0;
5102
5103         assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
5104
5105         const VkImportMemoryFdInfoKHR *import_info =
5106                 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
5107         const VkMemoryDedicatedAllocateInfo *dedicate_info =
5108                 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
5109         const VkExportMemoryAllocateInfo *export_info =
5110                 vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
5111         const struct VkImportAndroidHardwareBufferInfoANDROID *ahb_import_info =
5112                 vk_find_struct_const(pAllocateInfo->pNext,
5113                                      IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID);
5114         const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
5115                 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
5116
5117         const struct wsi_memory_allocate_info *wsi_info =
5118                 vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
5119
5120         if (pAllocateInfo->allocationSize == 0 && !ahb_import_info &&
5121             !(export_info && (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID))) {
5122                 /* Apparently, this is allowed */
5123                 *pMem = VK_NULL_HANDLE;
5124                 return VK_SUCCESS;
5125         }
5126
5127         mem = vk_zalloc2(&device->alloc, pAllocator, sizeof(*mem), 8,
5128                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5129         if (mem == NULL)
5130                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5131
5132         if (wsi_info && wsi_info->implicit_sync)
5133                 flags |= RADEON_FLAG_IMPLICIT_SYNC;
5134
5135         if (dedicate_info) {
5136                 mem->image = radv_image_from_handle(dedicate_info->image);
5137                 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
5138         } else {
5139                 mem->image = NULL;
5140                 mem->buffer = NULL;
5141         }
5142
5143         float priority_float = 0.5;
5144         const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
5145                 vk_find_struct_const(pAllocateInfo->pNext,
5146                                      MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
5147         if (priority_ext)
5148                 priority_float = priority_ext->priority;
5149
5150         unsigned priority = MIN2(RADV_BO_PRIORITY_APPLICATION_MAX - 1,
5151                                  (int)(priority_float * RADV_BO_PRIORITY_APPLICATION_MAX));
5152
5153         mem->user_ptr = NULL;
5154         mem->bo = NULL;
5155
5156 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
5157         mem->android_hardware_buffer = NULL;
5158 #endif
5159
5160         if (ahb_import_info) {
5161                 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
5162                 if (result != VK_SUCCESS)
5163                         goto fail;
5164         } else if(export_info && (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID)) {
5165                 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
5166                 if (result != VK_SUCCESS)
5167                         goto fail;
5168         } else if (import_info) {
5169                 assert(import_info->handleType ==
5170                        VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
5171                        import_info->handleType ==
5172                        VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
5173                 mem->bo = device->ws->buffer_from_fd(device->ws, import_info->fd,
5174                                                      priority, NULL);
5175                 if (!mem->bo) {
5176                         result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
5177                         goto fail;
5178                 } else {
5179                         close(import_info->fd);
5180                 }
5181         } else if (host_ptr_info) {
5182                 assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
5183                 mem->bo = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer,
5184                                                       pAllocateInfo->allocationSize,
5185                                                       priority);
5186                 if (!mem->bo) {
5187                         result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
5188                         goto fail;
5189                 } else {
5190                         mem->user_ptr = host_ptr_info->pHostPointer;
5191                 }
5192         } else {
5193                 uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
5194                 uint32_t heap_index;
5195
5196                 heap_index = device->physical_device->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
5197                 domain = device->physical_device->memory_domains[pAllocateInfo->memoryTypeIndex];
5198                 flags |= device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex];
5199
5200                 if (!dedicate_info && !import_info && (!export_info || !export_info->handleTypes)) {
5201                         flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
5202                         if (device->use_global_bo_list) {
5203                                 flags |= RADEON_FLAG_PREFER_LOCAL_BO;
5204                         }
5205                 }
5206
5207                 if (device->overallocation_disallowed) {
5208                         uint64_t total_size =
5209                                 device->physical_device->memory_properties.memoryHeaps[heap_index].size;
5210
5211                         mtx_lock(&device->overallocation_mutex);
5212                         if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
5213                                 mtx_unlock(&device->overallocation_mutex);
5214                                 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
5215                                 goto fail;
5216                         }
5217                         device->allocated_memory_size[heap_index] += alloc_size;
5218                         mtx_unlock(&device->overallocation_mutex);
5219                 }
5220
5221                 mem->bo = device->ws->buffer_create(device->ws, alloc_size, device->physical_device->rad_info.max_alignment,
5222                                                     domain, flags, priority);
5223
5224                 if (!mem->bo) {
5225                         if (device->overallocation_disallowed) {
5226                                 mtx_lock(&device->overallocation_mutex);
5227                                 device->allocated_memory_size[heap_index] -= alloc_size;
5228                                 mtx_unlock(&device->overallocation_mutex);
5229                         }
5230                         result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
5231                         goto fail;
5232                 }
5233
5234                 mem->heap_index = heap_index;
5235                 mem->alloc_size = alloc_size;
5236         }
5237
5238         if (!wsi_info) {
5239                 result = radv_bo_list_add(device, mem->bo);
5240                 if (result != VK_SUCCESS)
5241                         goto fail;
5242         }
5243
5244         *pMem = radv_device_memory_to_handle(mem);
5245
5246         return VK_SUCCESS;
5247
5248 fail:
5249         radv_free_memory(device, pAllocator,mem);
5250
5251         return result;
5252 }
5253
5254 VkResult radv_AllocateMemory(
5255         VkDevice                                    _device,
5256         const VkMemoryAllocateInfo*                 pAllocateInfo,
5257         const VkAllocationCallbacks*                pAllocator,
5258         VkDeviceMemory*                             pMem)
5259 {
5260         RADV_FROM_HANDLE(radv_device, device, _device);
5261         return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem);
5262 }
5263
5264 void radv_FreeMemory(
5265         VkDevice                                    _device,
5266         VkDeviceMemory                              _mem,
5267         const VkAllocationCallbacks*                pAllocator)
5268 {
5269         RADV_FROM_HANDLE(radv_device, device, _device);
5270         RADV_FROM_HANDLE(radv_device_memory, mem, _mem);
5271
5272         radv_free_memory(device, pAllocator, mem);
5273 }
5274
5275 VkResult radv_MapMemory(
5276         VkDevice                                    _device,
5277         VkDeviceMemory                              _memory,
5278         VkDeviceSize                                offset,
5279         VkDeviceSize                                size,
5280         VkMemoryMapFlags                            flags,
5281         void**                                      ppData)
5282 {
5283         RADV_FROM_HANDLE(radv_device, device, _device);
5284         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
5285
5286         if (mem == NULL) {
5287                 *ppData = NULL;
5288                 return VK_SUCCESS;
5289         }
5290
5291         if (mem->user_ptr)
5292                 *ppData = mem->user_ptr;
5293         else
5294                 *ppData = device->ws->buffer_map(mem->bo);
5295
5296         if (*ppData) {
5297                 *ppData += offset;
5298                 return VK_SUCCESS;
5299         }
5300
5301         return vk_error(device->instance, VK_ERROR_MEMORY_MAP_FAILED);
5302 }
5303
5304 void radv_UnmapMemory(
5305         VkDevice                                    _device,
5306         VkDeviceMemory                              _memory)
5307 {
5308         RADV_FROM_HANDLE(radv_device, device, _device);
5309         RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
5310
5311         if (mem == NULL)
5312                 return;
5313
5314         if (mem->user_ptr == NULL)
5315                 device->ws->buffer_unmap(mem->bo);
5316 }
5317
5318 VkResult radv_FlushMappedMemoryRanges(
5319         VkDevice                                    _device,
5320         uint32_t                                    memoryRangeCount,
5321         const VkMappedMemoryRange*                  pMemoryRanges)
5322 {
5323         return VK_SUCCESS;
5324 }
5325
5326 VkResult radv_InvalidateMappedMemoryRanges(
5327         VkDevice                                    _device,
5328         uint32_t                                    memoryRangeCount,
5329         const VkMappedMemoryRange*                  pMemoryRanges)
5330 {
5331         return VK_SUCCESS;
5332 }
5333
5334 void radv_GetBufferMemoryRequirements(
5335         VkDevice                                    _device,
5336         VkBuffer                                    _buffer,
5337         VkMemoryRequirements*                       pMemoryRequirements)
5338 {
5339         RADV_FROM_HANDLE(radv_device, device, _device);
5340         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
5341
5342         pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
5343
5344         if (buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)
5345                 pMemoryRequirements->alignment = 4096;
5346         else
5347                 pMemoryRequirements->alignment = 16;
5348
5349         pMemoryRequirements->size = align64(buffer->size, pMemoryRequirements->alignment);
5350 }
5351
5352 void radv_GetBufferMemoryRequirements2(
5353         VkDevice                                     device,
5354         const VkBufferMemoryRequirementsInfo2       *pInfo,
5355         VkMemoryRequirements2                       *pMemoryRequirements)
5356 {
5357         radv_GetBufferMemoryRequirements(device, pInfo->buffer,
5358                                         &pMemoryRequirements->memoryRequirements);
5359         vk_foreach_struct(ext, pMemoryRequirements->pNext) {
5360                 switch (ext->sType) {
5361                 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
5362                         VkMemoryDedicatedRequirements *req =
5363                                        (VkMemoryDedicatedRequirements *) ext;
5364                         req->requiresDedicatedAllocation = false;
5365                         req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
5366                         break;
5367                 }
5368                 default:
5369                         break;
5370                 }
5371         }
5372 }
5373
5374 void radv_GetImageMemoryRequirements(
5375         VkDevice                                    _device,
5376         VkImage                                     _image,
5377         VkMemoryRequirements*                       pMemoryRequirements)
5378 {
5379         RADV_FROM_HANDLE(radv_device, device, _device);
5380         RADV_FROM_HANDLE(radv_image, image, _image);
5381
5382         pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
5383
5384         pMemoryRequirements->size = image->size;
5385         pMemoryRequirements->alignment = image->alignment;
5386 }
5387
5388 void radv_GetImageMemoryRequirements2(
5389         VkDevice                                    device,
5390         const VkImageMemoryRequirementsInfo2       *pInfo,
5391         VkMemoryRequirements2                      *pMemoryRequirements)
5392 {
5393         radv_GetImageMemoryRequirements(device, pInfo->image,
5394                                         &pMemoryRequirements->memoryRequirements);
5395
5396         RADV_FROM_HANDLE(radv_image, image, pInfo->image);
5397
5398         vk_foreach_struct(ext, pMemoryRequirements->pNext) {
5399                 switch (ext->sType) {
5400                 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
5401                         VkMemoryDedicatedRequirements *req =
5402                                        (VkMemoryDedicatedRequirements *) ext;
5403                         req->requiresDedicatedAllocation = image->shareable &&
5404                                                            image->tiling != VK_IMAGE_TILING_LINEAR;
5405                         req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
5406                         break;
5407                 }
5408                 default:
5409                         break;
5410                 }
5411         }
5412 }
5413
5414 void radv_GetImageSparseMemoryRequirements(
5415         VkDevice                                    device,
5416         VkImage                                     image,
5417         uint32_t*                                   pSparseMemoryRequirementCount,
5418         VkSparseImageMemoryRequirements*            pSparseMemoryRequirements)
5419 {
5420         stub();
5421 }
5422
5423 void radv_GetImageSparseMemoryRequirements2(
5424         VkDevice                                    device,
5425         const VkImageSparseMemoryRequirementsInfo2 *pInfo,
5426         uint32_t*                                   pSparseMemoryRequirementCount,
5427         VkSparseImageMemoryRequirements2           *pSparseMemoryRequirements)
5428 {
5429         stub();
5430 }
5431
5432 void radv_GetDeviceMemoryCommitment(
5433         VkDevice                                    device,
5434         VkDeviceMemory                              memory,
5435         VkDeviceSize*                               pCommittedMemoryInBytes)
5436 {
5437         *pCommittedMemoryInBytes = 0;
5438 }
5439
5440 VkResult radv_BindBufferMemory2(VkDevice device,
5441                                 uint32_t bindInfoCount,
5442                                 const VkBindBufferMemoryInfo *pBindInfos)
5443 {
5444         for (uint32_t i = 0; i < bindInfoCount; ++i) {
5445                 RADV_FROM_HANDLE(radv_device_memory, mem, pBindInfos[i].memory);
5446                 RADV_FROM_HANDLE(radv_buffer, buffer, pBindInfos[i].buffer);
5447
5448                 if (mem) {
5449                         buffer->bo = mem->bo;
5450                         buffer->offset = pBindInfos[i].memoryOffset;
5451                 } else {
5452                         buffer->bo = NULL;
5453                 }
5454         }
5455         return VK_SUCCESS;
5456 }
5457
5458 VkResult radv_BindBufferMemory(
5459         VkDevice                                    device,
5460         VkBuffer                                    buffer,
5461         VkDeviceMemory                              memory,
5462         VkDeviceSize                                memoryOffset)
5463 {
5464         const VkBindBufferMemoryInfo info = {
5465                 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
5466                 .buffer = buffer,
5467                 .memory = memory,
5468                 .memoryOffset = memoryOffset
5469         };
5470
5471         return radv_BindBufferMemory2(device, 1, &info);
5472 }
5473
5474 VkResult radv_BindImageMemory2(VkDevice device,
5475                                uint32_t bindInfoCount,
5476                                const VkBindImageMemoryInfo *pBindInfos)
5477 {
5478         for (uint32_t i = 0; i < bindInfoCount; ++i) {
5479                 RADV_FROM_HANDLE(radv_device_memory, mem, pBindInfos[i].memory);
5480                 RADV_FROM_HANDLE(radv_image, image, pBindInfos[i].image);
5481
5482                 if (mem) {
5483                         image->bo = mem->bo;
5484                         image->offset = pBindInfos[i].memoryOffset;
5485                 } else {
5486                         image->bo = NULL;
5487                         image->offset = 0;
5488                 }
5489         }
5490         return VK_SUCCESS;
5491 }
5492
5493
5494 VkResult radv_BindImageMemory(
5495         VkDevice                                    device,
5496         VkImage                                     image,
5497         VkDeviceMemory                              memory,
5498         VkDeviceSize                                memoryOffset)
5499 {
5500         const VkBindImageMemoryInfo info = {
5501                 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
5502                 .image = image,
5503                 .memory = memory,
5504                 .memoryOffset = memoryOffset
5505         };
5506
5507         return radv_BindImageMemory2(device, 1, &info);
5508 }
5509
5510 static bool radv_sparse_bind_has_effects(const VkBindSparseInfo *info)
5511 {
5512         return info->bufferBindCount ||
5513                info->imageOpaqueBindCount ||
5514                info->imageBindCount ||
5515                info->waitSemaphoreCount ||
5516                info->signalSemaphoreCount;
5517 }
5518
5519  VkResult radv_QueueBindSparse(
5520         VkQueue                                     _queue,
5521         uint32_t                                    bindInfoCount,
5522         const VkBindSparseInfo*                     pBindInfo,
5523         VkFence                                     fence)
5524 {
5525         RADV_FROM_HANDLE(radv_queue, queue, _queue);
5526         VkResult result;
5527         uint32_t fence_idx = 0;
5528
5529         if (fence != VK_NULL_HANDLE) {
5530                 for (uint32_t i = 0; i < bindInfoCount; ++i)
5531                         if (radv_sparse_bind_has_effects(pBindInfo + i))
5532                                 fence_idx = i;
5533         } else
5534                 fence_idx = UINT32_MAX;
5535
5536         for (uint32_t i = 0; i < bindInfoCount; ++i) {
5537                 if (i != fence_idx && !radv_sparse_bind_has_effects(pBindInfo + i))
5538                         continue;
5539
5540                 const VkTimelineSemaphoreSubmitInfo *timeline_info =
5541                         vk_find_struct_const(pBindInfo[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
5542
5543                 VkResult result = radv_queue_submit(queue, &(struct radv_queue_submission) {
5544                                 .buffer_binds = pBindInfo[i].pBufferBinds,
5545                                 .buffer_bind_count = pBindInfo[i].bufferBindCount,
5546                                 .image_opaque_binds = pBindInfo[i].pImageOpaqueBinds,
5547                                 .image_opaque_bind_count = pBindInfo[i].imageOpaqueBindCount,
5548                                 .wait_semaphores = pBindInfo[i].pWaitSemaphores,
5549                                 .wait_semaphore_count = pBindInfo[i].waitSemaphoreCount,
5550                                 .signal_semaphores = pBindInfo[i].pSignalSemaphores,
5551                                 .signal_semaphore_count = pBindInfo[i].signalSemaphoreCount,
5552                                 .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
5553                                 .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
5554                                 .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
5555                                 .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
5556                                 .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
5557                         });
5558
5559                 if (result != VK_SUCCESS)
5560                         return result;
5561         }
5562
5563         if (fence != VK_NULL_HANDLE && !bindInfoCount) {
5564                 result = radv_signal_fence(queue, fence);
5565                 if (result != VK_SUCCESS)
5566                         return result;
5567         }
5568
5569         return VK_SUCCESS;
5570 }
5571
5572 VkResult radv_CreateFence(
5573         VkDevice                                    _device,
5574         const VkFenceCreateInfo*                    pCreateInfo,
5575         const VkAllocationCallbacks*                pAllocator,
5576         VkFence*                                    pFence)
5577 {
5578         RADV_FROM_HANDLE(radv_device, device, _device);
5579         const VkExportFenceCreateInfo *export =
5580                 vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
5581         VkExternalFenceHandleTypeFlags handleTypes =
5582                 export ? export->handleTypes : 0;
5583
5584         struct radv_fence *fence = vk_alloc2(&device->alloc, pAllocator,
5585                                                sizeof(*fence), 8,
5586                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5587
5588         if (!fence)
5589                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5590
5591         fence->fence_wsi = NULL;
5592         fence->temp_syncobj = 0;
5593         if (device->always_use_syncobj || handleTypes) {
5594                 int ret = device->ws->create_syncobj(device->ws, &fence->syncobj);
5595                 if (ret) {
5596                         vk_free2(&device->alloc, pAllocator, fence);
5597                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5598                 }
5599                 if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) {
5600                         device->ws->signal_syncobj(device->ws, fence->syncobj);
5601                 }
5602                 fence->fence = NULL;
5603         } else {
5604                 fence->fence = device->ws->create_fence();
5605                 if (!fence->fence) {
5606                         vk_free2(&device->alloc, pAllocator, fence);
5607                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5608                 }
5609                 fence->syncobj = 0;
5610                 if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT)
5611                         device->ws->signal_fence(fence->fence);
5612         }
5613
5614         *pFence = radv_fence_to_handle(fence);
5615
5616         return VK_SUCCESS;
5617 }
5618
5619 void radv_DestroyFence(
5620         VkDevice                                    _device,
5621         VkFence                                     _fence,
5622         const VkAllocationCallbacks*                pAllocator)
5623 {
5624         RADV_FROM_HANDLE(radv_device, device, _device);
5625         RADV_FROM_HANDLE(radv_fence, fence, _fence);
5626
5627         if (!fence)
5628                 return;
5629
5630         if (fence->temp_syncobj)
5631                 device->ws->destroy_syncobj(device->ws, fence->temp_syncobj);
5632         if (fence->syncobj)
5633                 device->ws->destroy_syncobj(device->ws, fence->syncobj);
5634         if (fence->fence)
5635                 device->ws->destroy_fence(fence->fence);
5636         if (fence->fence_wsi)
5637                 fence->fence_wsi->destroy(fence->fence_wsi);
5638         vk_free2(&device->alloc, pAllocator, fence);
5639 }
5640
5641
5642 uint64_t radv_get_current_time(void)
5643 {
5644         struct timespec tv;
5645         clock_gettime(CLOCK_MONOTONIC, &tv);
5646         return tv.tv_nsec + tv.tv_sec*1000000000ull;
5647 }
5648
5649 static uint64_t radv_get_absolute_timeout(uint64_t timeout)
5650 {
5651         uint64_t current_time = radv_get_current_time();
5652
5653         timeout = MIN2(UINT64_MAX - current_time, timeout);
5654
5655         return current_time + timeout;
5656 }
5657
5658
5659 static bool radv_all_fences_plain_and_submitted(struct radv_device *device,
5660                                                 uint32_t fenceCount, const VkFence *pFences)
5661 {
5662         for (uint32_t i = 0; i < fenceCount; ++i) {
5663                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5664                 if (fence->fence == NULL || fence->syncobj ||
5665                     fence->temp_syncobj || fence->fence_wsi ||
5666                     (!device->ws->is_fence_waitable(fence->fence)))
5667                         return false;
5668         }
5669         return true;
5670 }
5671
5672 static bool radv_all_fences_syncobj(uint32_t fenceCount, const VkFence *pFences)
5673 {
5674         for (uint32_t i = 0; i < fenceCount; ++i) {
5675                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5676                 if (fence->syncobj == 0 && fence->temp_syncobj == 0)
5677                         return false;
5678         }
5679         return true;
5680 }
5681
5682 VkResult radv_WaitForFences(
5683         VkDevice                                    _device,
5684         uint32_t                                    fenceCount,
5685         const VkFence*                              pFences,
5686         VkBool32                                    waitAll,
5687         uint64_t                                    timeout)
5688 {
5689         RADV_FROM_HANDLE(radv_device, device, _device);
5690         timeout = radv_get_absolute_timeout(timeout);
5691
5692         if (device->always_use_syncobj &&
5693             radv_all_fences_syncobj(fenceCount, pFences))
5694         {
5695                 uint32_t *handles = malloc(sizeof(uint32_t) * fenceCount);
5696                 if (!handles)
5697                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5698
5699                 for (uint32_t i = 0; i < fenceCount; ++i) {
5700                         RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5701                         handles[i] = fence->temp_syncobj ? fence->temp_syncobj : fence->syncobj;
5702                 }
5703
5704                 bool success = device->ws->wait_syncobj(device->ws, handles, fenceCount, waitAll, timeout);
5705
5706                 free(handles);
5707                 return success ? VK_SUCCESS : VK_TIMEOUT;
5708         }
5709
5710         if (!waitAll && fenceCount > 1) {
5711                 /* Not doing this by default for waitAll, due to needing to allocate twice. */
5712                 if (device->physical_device->rad_info.drm_minor >= 10 && radv_all_fences_plain_and_submitted(device, fenceCount, pFences)) {
5713                         uint32_t wait_count = 0;
5714                         struct radeon_winsys_fence **fences = malloc(sizeof(struct radeon_winsys_fence *) * fenceCount);
5715                         if (!fences)
5716                                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
5717
5718                         for (uint32_t i = 0; i < fenceCount; ++i) {
5719                                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5720
5721                                 if (device->ws->fence_wait(device->ws, fence->fence, false, 0)) {
5722                                         free(fences);
5723                                         return VK_SUCCESS;
5724                                 }
5725
5726                                 fences[wait_count++] = fence->fence;
5727                         }
5728
5729                         bool success = device->ws->fences_wait(device->ws, fences, wait_count,
5730                                                                waitAll, timeout - radv_get_current_time());
5731
5732                         free(fences);
5733                         return success ? VK_SUCCESS : VK_TIMEOUT;
5734                 }
5735
5736                 while(radv_get_current_time() <= timeout) {
5737                         for (uint32_t i = 0; i < fenceCount; ++i) {
5738                                 if (radv_GetFenceStatus(_device, pFences[i]) == VK_SUCCESS)
5739                                         return VK_SUCCESS;
5740                         }
5741                 }
5742                 return VK_TIMEOUT;
5743         }
5744
5745         for (uint32_t i = 0; i < fenceCount; ++i) {
5746                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5747                 bool expired = false;
5748
5749                 if (fence->temp_syncobj) {
5750                         if (!device->ws->wait_syncobj(device->ws, &fence->temp_syncobj, 1, true, timeout))
5751                                 return VK_TIMEOUT;
5752                         continue;
5753                 }
5754
5755                 if (fence->syncobj) {
5756                         if (!device->ws->wait_syncobj(device->ws, &fence->syncobj, 1, true, timeout))
5757                                 return VK_TIMEOUT;
5758                         continue;
5759                 }
5760
5761                 if (fence->fence) {
5762                         if (!device->ws->is_fence_waitable(fence->fence)) {
5763                                 while(!device->ws->is_fence_waitable(fence->fence) &&
5764                                       radv_get_current_time() <= timeout)
5765                                         /* Do nothing */;
5766                         }
5767
5768                         expired = device->ws->fence_wait(device->ws,
5769                                                          fence->fence,
5770                                                          true, timeout);
5771                         if (!expired)
5772                                 return VK_TIMEOUT;
5773                 }
5774
5775                 if (fence->fence_wsi) {
5776                         VkResult result = fence->fence_wsi->wait(fence->fence_wsi, timeout);
5777                         if (result != VK_SUCCESS)
5778                                 return result;
5779                 }
5780         }
5781
5782         return VK_SUCCESS;
5783 }
5784
5785 VkResult radv_ResetFences(VkDevice _device,
5786                           uint32_t fenceCount,
5787                           const VkFence *pFences)
5788 {
5789         RADV_FROM_HANDLE(radv_device, device, _device);
5790
5791         for (unsigned i = 0; i < fenceCount; ++i) {
5792                 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
5793                 if (fence->fence)
5794                         device->ws->reset_fence(fence->fence);
5795
5796                 /* Per spec, we first restore the permanent payload, and then reset, so
5797                  * having a temp syncobj should not skip resetting the permanent syncobj. */
5798                 if (fence->temp_syncobj) {
5799                         device->ws->destroy_syncobj(device->ws, fence->temp_syncobj);
5800                         fence->temp_syncobj = 0;
5801                 }
5802
5803                 if (fence->syncobj) {
5804                         device->ws->reset_syncobj(device->ws, fence->syncobj);
5805                 }
5806         }
5807
5808         return VK_SUCCESS;
5809 }
5810
5811 VkResult radv_GetFenceStatus(VkDevice _device, VkFence _fence)
5812 {
5813         RADV_FROM_HANDLE(radv_device, device, _device);
5814         RADV_FROM_HANDLE(radv_fence, fence, _fence);
5815
5816         if (fence->temp_syncobj) {
5817                         bool success = device->ws->wait_syncobj(device->ws, &fence->temp_syncobj, 1, true, 0);
5818                         return success ? VK_SUCCESS : VK_NOT_READY;
5819         }
5820
5821         if (fence->syncobj) {
5822                         bool success = device->ws->wait_syncobj(device->ws, &fence->syncobj, 1, true, 0);
5823                         return success ? VK_SUCCESS : VK_NOT_READY;
5824         }
5825
5826         if (fence->fence) {
5827                 if (!device->ws->fence_wait(device->ws, fence->fence, false, 0))
5828                         return VK_NOT_READY;
5829         }
5830         if (fence->fence_wsi) {
5831                 VkResult result = fence->fence_wsi->wait(fence->fence_wsi, 0);
5832
5833                 if (result != VK_SUCCESS) {
5834                         if (result == VK_TIMEOUT)
5835                                 return VK_NOT_READY;
5836                         return result;
5837                 }
5838         }
5839         return VK_SUCCESS;
5840 }
5841
5842
5843 // Queue semaphore functions
5844
5845 static void
5846 radv_create_timeline(struct radv_timeline *timeline, uint64_t value)
5847 {
5848         timeline->highest_signaled = value;
5849         timeline->highest_submitted = value;
5850         list_inithead(&timeline->points);
5851         list_inithead(&timeline->free_points);
5852         list_inithead(&timeline->waiters);
5853         pthread_mutex_init(&timeline->mutex, NULL);
5854 }
5855
5856 static void
5857 radv_destroy_timeline(struct radv_device *device,
5858                       struct radv_timeline *timeline)
5859 {
5860         list_for_each_entry_safe(struct radv_timeline_point, point,
5861                                  &timeline->free_points, list) {
5862                 list_del(&point->list);
5863                 device->ws->destroy_syncobj(device->ws, point->syncobj);
5864                 free(point);
5865         }
5866         list_for_each_entry_safe(struct radv_timeline_point, point,
5867                                  &timeline->points, list) {
5868                 list_del(&point->list);
5869                 device->ws->destroy_syncobj(device->ws, point->syncobj);
5870                 free(point);
5871         }
5872         pthread_mutex_destroy(&timeline->mutex);
5873 }
5874
5875 static void
5876 radv_timeline_gc_locked(struct radv_device *device,
5877                         struct radv_timeline *timeline)
5878 {
5879         list_for_each_entry_safe(struct radv_timeline_point, point,
5880                                  &timeline->points, list) {
5881                 if (point->wait_count || point->value > timeline->highest_submitted)
5882                         return;
5883
5884                 if (device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, 0)) {
5885                         timeline->highest_signaled = point->value;
5886                         list_del(&point->list);
5887                         list_add(&point->list, &timeline->free_points);
5888                 }
5889         }
5890 }
5891
5892 static struct radv_timeline_point *
5893 radv_timeline_find_point_at_least_locked(struct radv_device *device,
5894                                          struct radv_timeline *timeline,
5895                                          uint64_t p)
5896 {
5897         radv_timeline_gc_locked(device, timeline);
5898
5899         if (p <= timeline->highest_signaled)
5900                 return NULL;
5901
5902         list_for_each_entry(struct radv_timeline_point, point,
5903                             &timeline->points, list) {
5904                 if (point->value >= p) {
5905                         ++point->wait_count;
5906                         return point;
5907                 }
5908         }
5909         return NULL;
5910 }
5911
5912 static struct radv_timeline_point *
5913 radv_timeline_add_point_locked(struct radv_device *device,
5914                                struct radv_timeline *timeline,
5915                                uint64_t p)
5916 {
5917         radv_timeline_gc_locked(device, timeline);
5918
5919         struct radv_timeline_point *ret = NULL;
5920         struct radv_timeline_point *prev = NULL;
5921
5922         if (p <= timeline->highest_signaled)
5923                 return NULL;
5924
5925         list_for_each_entry(struct radv_timeline_point, point,
5926                             &timeline->points, list) {
5927                 if (point->value == p) {
5928                         return NULL;
5929                 }
5930
5931                 if (point->value < p)
5932                         prev = point;
5933         }
5934
5935         if (list_is_empty(&timeline->free_points)) {
5936                 ret = malloc(sizeof(struct radv_timeline_point));
5937                 device->ws->create_syncobj(device->ws, &ret->syncobj);
5938         } else {
5939                 ret = list_first_entry(&timeline->free_points, struct radv_timeline_point, list);
5940                 list_del(&ret->list);
5941
5942                 device->ws->reset_syncobj(device->ws, ret->syncobj);
5943         }
5944
5945         ret->value = p;
5946         ret->wait_count = 1;
5947
5948         if (prev) {
5949                 list_add(&ret->list, &prev->list);
5950         } else {
5951                 list_addtail(&ret->list, &timeline->points);
5952         }
5953         return ret;
5954 }
5955
5956
5957 static VkResult
5958 radv_timeline_wait_locked(struct radv_device *device,
5959                           struct radv_timeline *timeline,
5960                           uint64_t value,
5961                           uint64_t abs_timeout)
5962 {
5963         while(timeline->highest_submitted < value) {
5964                 struct timespec abstime;
5965                 timespec_from_nsec(&abstime, abs_timeout);
5966
5967                 pthread_cond_timedwait(&device->timeline_cond, &timeline->mutex, &abstime);
5968
5969                 if (radv_get_current_time() >= abs_timeout && timeline->highest_submitted < value)
5970                         return VK_TIMEOUT;
5971         }
5972
5973         struct radv_timeline_point *point = radv_timeline_find_point_at_least_locked(device, timeline, value);
5974         if (!point)
5975                 return VK_SUCCESS;
5976
5977         pthread_mutex_unlock(&timeline->mutex);
5978
5979         bool success = device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, abs_timeout);
5980
5981         pthread_mutex_lock(&timeline->mutex);
5982         point->wait_count--;
5983         return success ? VK_SUCCESS : VK_TIMEOUT;
5984 }
5985
5986 static void
5987 radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline,
5988                                      struct list_head *processing_list)
5989 {
5990         list_for_each_entry_safe(struct radv_timeline_waiter, waiter,
5991                                  &timeline->waiters, list) {
5992                 if (waiter->value > timeline->highest_submitted)
5993                         continue;
5994
5995                 if (p_atomic_dec_zero(&waiter->submission->submission_wait_count)) {
5996                         list_addtail(&waiter->submission->processing_list, processing_list);
5997                 }
5998                 list_del(&waiter->list);
5999         }
6000 }
6001
6002 static
6003 void radv_destroy_semaphore_part(struct radv_device *device,
6004                                  struct radv_semaphore_part *part)
6005 {
6006         switch(part->kind) {
6007         case RADV_SEMAPHORE_NONE:
6008                 break;
6009         case RADV_SEMAPHORE_WINSYS:
6010                 device->ws->destroy_sem(part->ws_sem);
6011                 break;
6012         case RADV_SEMAPHORE_TIMELINE:
6013                 radv_destroy_timeline(device, &part->timeline);
6014                 break;
6015         case RADV_SEMAPHORE_SYNCOBJ:
6016                 device->ws->destroy_syncobj(device->ws, part->syncobj);
6017                 break;
6018         }
6019         part->kind = RADV_SEMAPHORE_NONE;
6020 }
6021
6022 static VkSemaphoreTypeKHR
6023 radv_get_semaphore_type(const void *pNext, uint64_t *initial_value)
6024 {
6025         const VkSemaphoreTypeCreateInfo *type_info =
6026                 vk_find_struct_const(pNext, SEMAPHORE_TYPE_CREATE_INFO);
6027
6028         if (!type_info)
6029                 return VK_SEMAPHORE_TYPE_BINARY;
6030
6031         if (initial_value)
6032                 *initial_value = type_info->initialValue;
6033         return type_info->semaphoreType;
6034 }
6035
6036 VkResult radv_CreateSemaphore(
6037         VkDevice                                    _device,
6038         const VkSemaphoreCreateInfo*                pCreateInfo,
6039         const VkAllocationCallbacks*                pAllocator,
6040         VkSemaphore*                                pSemaphore)
6041 {
6042         RADV_FROM_HANDLE(radv_device, device, _device);
6043         const VkExportSemaphoreCreateInfo *export =
6044                 vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO);
6045         VkExternalSemaphoreHandleTypeFlags handleTypes =
6046                 export ? export->handleTypes : 0;
6047         uint64_t initial_value = 0;
6048         VkSemaphoreTypeKHR type = radv_get_semaphore_type(pCreateInfo->pNext, &initial_value);
6049
6050         struct radv_semaphore *sem = vk_alloc2(&device->alloc, pAllocator,
6051                                                sizeof(*sem), 8,
6052                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6053         if (!sem)
6054                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6055
6056         sem->temporary.kind = RADV_SEMAPHORE_NONE;
6057         sem->permanent.kind = RADV_SEMAPHORE_NONE;
6058
6059         if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
6060                 radv_create_timeline(&sem->permanent.timeline, initial_value);
6061                 sem->permanent.kind = RADV_SEMAPHORE_TIMELINE;
6062         } else if (device->always_use_syncobj || handleTypes) {
6063                 assert (device->physical_device->rad_info.has_syncobj);
6064                 int ret = device->ws->create_syncobj(device->ws, &sem->permanent.syncobj);
6065                 if (ret) {
6066                         vk_free2(&device->alloc, pAllocator, sem);
6067                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6068                 }
6069                 sem->permanent.kind = RADV_SEMAPHORE_SYNCOBJ;
6070         } else {
6071                 sem->permanent.ws_sem = device->ws->create_sem(device->ws);
6072                 if (!sem->permanent.ws_sem) {
6073                         vk_free2(&device->alloc, pAllocator, sem);
6074                         return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6075                 }
6076                 sem->permanent.kind = RADV_SEMAPHORE_WINSYS;
6077         }
6078
6079         *pSemaphore = radv_semaphore_to_handle(sem);
6080         return VK_SUCCESS;
6081 }
6082
6083 void radv_DestroySemaphore(
6084         VkDevice                                    _device,
6085         VkSemaphore                                 _semaphore,
6086         const VkAllocationCallbacks*                pAllocator)
6087 {
6088         RADV_FROM_HANDLE(radv_device, device, _device);
6089         RADV_FROM_HANDLE(radv_semaphore, sem, _semaphore);
6090         if (!_semaphore)
6091                 return;
6092
6093         radv_destroy_semaphore_part(device, &sem->temporary);
6094         radv_destroy_semaphore_part(device, &sem->permanent);
6095         vk_free2(&device->alloc, pAllocator, sem);
6096 }
6097
6098 VkResult
6099 radv_GetSemaphoreCounterValue(VkDevice _device,
6100                               VkSemaphore _semaphore,
6101                               uint64_t* pValue)
6102 {
6103         RADV_FROM_HANDLE(radv_device, device, _device);
6104         RADV_FROM_HANDLE(radv_semaphore, semaphore, _semaphore);
6105
6106         struct radv_semaphore_part *part =
6107                 semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
6108
6109         switch (part->kind) {
6110         case RADV_SEMAPHORE_TIMELINE: {
6111                 pthread_mutex_lock(&part->timeline.mutex);
6112                 radv_timeline_gc_locked(device, &part->timeline);
6113                 *pValue = part->timeline.highest_signaled;
6114                 pthread_mutex_unlock(&part->timeline.mutex);
6115                 return VK_SUCCESS;
6116         }
6117         case RADV_SEMAPHORE_NONE:
6118         case RADV_SEMAPHORE_SYNCOBJ:
6119         case RADV_SEMAPHORE_WINSYS:
6120                 unreachable("Invalid semaphore type");
6121         }
6122         unreachable("Unhandled semaphore type");
6123 }
6124
6125
6126 static VkResult
6127 radv_wait_timelines(struct radv_device *device,
6128                     const VkSemaphoreWaitInfo* pWaitInfo,
6129                     uint64_t abs_timeout)
6130 {
6131         if ((pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR) && pWaitInfo->semaphoreCount > 1) {
6132                 for (;;) {
6133                         for(uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6134                                 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6135                                 pthread_mutex_lock(&semaphore->permanent.timeline.mutex);
6136                                 VkResult result = radv_timeline_wait_locked(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], 0);
6137                                 pthread_mutex_unlock(&semaphore->permanent.timeline.mutex);
6138
6139                                 if (result == VK_SUCCESS)
6140                                         return VK_SUCCESS;
6141                         }
6142                         if (radv_get_current_time() > abs_timeout)
6143                                 return VK_TIMEOUT;
6144                 }
6145         }
6146
6147         for(uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) {
6148                 RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]);
6149                 pthread_mutex_lock(&semaphore->permanent.timeline.mutex);
6150                 VkResult result = radv_timeline_wait_locked(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], abs_timeout);
6151                 pthread_mutex_unlock(&semaphore->permanent.timeline.mutex);
6152
6153                 if (result != VK_SUCCESS)
6154                         return result;
6155         }
6156         return VK_SUCCESS;
6157 }
6158 VkResult
6159 radv_WaitSemaphores(VkDevice _device,
6160                     const VkSemaphoreWaitInfo* pWaitInfo,
6161                     uint64_t timeout)
6162 {
6163         RADV_FROM_HANDLE(radv_device, device, _device);
6164         uint64_t abs_timeout = radv_get_absolute_timeout(timeout);
6165         return radv_wait_timelines(device, pWaitInfo, abs_timeout);
6166 }
6167
6168 VkResult
6169 radv_SignalSemaphore(VkDevice _device,
6170                      const VkSemaphoreSignalInfo* pSignalInfo)
6171 {
6172         RADV_FROM_HANDLE(radv_device, device, _device);
6173         RADV_FROM_HANDLE(radv_semaphore, semaphore, pSignalInfo->semaphore);
6174
6175         struct radv_semaphore_part *part =
6176                 semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
6177
6178         switch(part->kind) {
6179         case RADV_SEMAPHORE_TIMELINE: {
6180                 pthread_mutex_lock(&part->timeline.mutex);
6181                 radv_timeline_gc_locked(device, &part->timeline);
6182                 part->timeline.highest_submitted = MAX2(part->timeline.highest_submitted, pSignalInfo->value);
6183                 part->timeline.highest_signaled = MAX2(part->timeline.highest_signaled, pSignalInfo->value);
6184
6185                 struct list_head processing_list;
6186                 list_inithead(&processing_list);
6187                 radv_timeline_trigger_waiters_locked(&part->timeline, &processing_list);
6188                 pthread_mutex_unlock(&part->timeline.mutex);
6189
6190                 return radv_process_submissions(&processing_list);
6191         }
6192         case RADV_SEMAPHORE_NONE:
6193         case RADV_SEMAPHORE_SYNCOBJ:
6194         case RADV_SEMAPHORE_WINSYS:
6195                 unreachable("Invalid semaphore type");
6196         }
6197         return VK_SUCCESS;
6198 }
6199
6200
6201
6202 VkResult radv_CreateEvent(
6203         VkDevice                                    _device,
6204         const VkEventCreateInfo*                    pCreateInfo,
6205         const VkAllocationCallbacks*                pAllocator,
6206         VkEvent*                                    pEvent)
6207 {
6208         RADV_FROM_HANDLE(radv_device, device, _device);
6209         struct radv_event *event = vk_alloc2(&device->alloc, pAllocator,
6210                                                sizeof(*event), 8,
6211                                                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6212
6213         if (!event)
6214                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6215
6216         event->bo = device->ws->buffer_create(device->ws, 8, 8,
6217                                               RADEON_DOMAIN_GTT,
6218                                               RADEON_FLAG_VA_UNCACHED | RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING,
6219                                               RADV_BO_PRIORITY_FENCE);
6220         if (!event->bo) {
6221                 vk_free2(&device->alloc, pAllocator, event);
6222                 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6223         }
6224
6225         event->map = (uint64_t*)device->ws->buffer_map(event->bo);
6226
6227         *pEvent = radv_event_to_handle(event);
6228
6229         return VK_SUCCESS;
6230 }
6231
6232 void radv_DestroyEvent(
6233         VkDevice                                    _device,
6234         VkEvent                                     _event,
6235         const VkAllocationCallbacks*                pAllocator)
6236 {
6237         RADV_FROM_HANDLE(radv_device, device, _device);
6238         RADV_FROM_HANDLE(radv_event, event, _event);
6239
6240         if (!event)
6241                 return;
6242         device->ws->buffer_destroy(event->bo);
6243         vk_free2(&device->alloc, pAllocator, event);
6244 }
6245
6246 VkResult radv_GetEventStatus(
6247         VkDevice                                    _device,
6248         VkEvent                                     _event)
6249 {
6250         RADV_FROM_HANDLE(radv_event, event, _event);
6251
6252         if (*event->map == 1)
6253                 return VK_EVENT_SET;
6254         return VK_EVENT_RESET;
6255 }
6256
6257 VkResult radv_SetEvent(
6258         VkDevice                                    _device,
6259         VkEvent                                     _event)
6260 {
6261         RADV_FROM_HANDLE(radv_event, event, _event);
6262         *event->map = 1;
6263
6264         return VK_SUCCESS;
6265 }
6266
6267 VkResult radv_ResetEvent(
6268     VkDevice                                    _device,
6269     VkEvent                                     _event)
6270 {
6271         RADV_FROM_HANDLE(radv_event, event, _event);
6272         *event->map = 0;
6273
6274         return VK_SUCCESS;
6275 }
6276
6277 VkResult radv_CreateBuffer(
6278         VkDevice                                    _device,
6279         const VkBufferCreateInfo*                   pCreateInfo,
6280         const VkAllocationCallbacks*                pAllocator,
6281         VkBuffer*                                   pBuffer)
6282 {
6283         RADV_FROM_HANDLE(radv_device, device, _device);
6284         struct radv_buffer *buffer;
6285
6286         if (pCreateInfo->size > RADV_MAX_MEMORY_ALLOCATION_SIZE)
6287                 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6288
6289         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
6290
6291         buffer = vk_alloc2(&device->alloc, pAllocator, sizeof(*buffer), 8,
6292                              VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6293         if (buffer == NULL)
6294                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6295
6296         buffer->size = pCreateInfo->size;
6297         buffer->usage = pCreateInfo->usage;
6298         buffer->bo = NULL;
6299         buffer->offset = 0;
6300         buffer->flags = pCreateInfo->flags;
6301
6302         buffer->shareable = vk_find_struct_const(pCreateInfo->pNext,
6303                                                  EXTERNAL_MEMORY_BUFFER_CREATE_INFO) != NULL;
6304
6305         if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
6306                 buffer->bo = device->ws->buffer_create(device->ws,
6307                                                        align64(buffer->size, 4096),
6308                                                        4096, 0, RADEON_FLAG_VIRTUAL,
6309                                                        RADV_BO_PRIORITY_VIRTUAL);
6310                 if (!buffer->bo) {
6311                         vk_free2(&device->alloc, pAllocator, buffer);
6312                         return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
6313                 }
6314         }
6315
6316         *pBuffer = radv_buffer_to_handle(buffer);
6317
6318         return VK_SUCCESS;
6319 }
6320
6321 void radv_DestroyBuffer(
6322         VkDevice                                    _device,
6323         VkBuffer                                    _buffer,
6324         const VkAllocationCallbacks*                pAllocator)
6325 {
6326         RADV_FROM_HANDLE(radv_device, device, _device);
6327         RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
6328
6329         if (!buffer)
6330                 return;
6331
6332         if (buffer->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)
6333                 device->ws->buffer_destroy(buffer->bo);
6334
6335         vk_free2(&device->alloc, pAllocator, buffer);
6336 }
6337
6338 VkDeviceAddress radv_GetBufferDeviceAddress(
6339         VkDevice                                    device,
6340         const VkBufferDeviceAddressInfo*         pInfo)
6341 {
6342         RADV_FROM_HANDLE(radv_buffer, buffer, pInfo->buffer);
6343         return radv_buffer_get_va(buffer->bo) + buffer->offset;
6344 }
6345
6346
6347 uint64_t radv_GetBufferOpaqueCaptureAddress(VkDevice device,
6348                                             const VkBufferDeviceAddressInfo* pInfo)
6349 {
6350         return 0;
6351 }
6352
6353 uint64_t radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,
6354                                                   const VkDeviceMemoryOpaqueCaptureAddressInfo* pInfo)
6355 {
6356         return 0;
6357 }
6358
6359 static inline unsigned
6360 si_tile_mode_index(const struct radv_image_plane *plane, unsigned level, bool stencil)
6361 {
6362         if (stencil)
6363                 return plane->surface.u.legacy.stencil_tiling_index[level];
6364         else
6365                 return plane->surface.u.legacy.tiling_index[level];
6366 }
6367
6368 static uint32_t radv_surface_max_layer_count(struct radv_image_view *iview)
6369 {
6370         return iview->type == VK_IMAGE_VIEW_TYPE_3D ? iview->extent.depth : (iview->base_layer + iview->layer_count);
6371 }
6372
6373 static uint32_t
6374 radv_init_dcc_control_reg(struct radv_device *device,
6375                           struct radv_image_view *iview)
6376 {
6377         unsigned max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_256B;
6378         unsigned min_compressed_block_size = V_028C78_MIN_BLOCK_SIZE_32B;
6379         unsigned max_compressed_block_size;
6380         unsigned independent_128b_blocks;
6381         unsigned independent_64b_blocks;
6382
6383         if (!radv_dcc_enabled(iview->image, iview->base_mip))
6384                 return 0;
6385
6386         if (!device->physical_device->rad_info.has_dedicated_vram) {
6387                 /* amdvlk: [min-compressed-block-size] should be set to 32 for
6388                  * dGPU and 64 for APU because all of our APUs to date use
6389                  * DIMMs which have a request granularity size of 64B while all
6390                  * other chips have a 32B request size.
6391                  */
6392                 min_compressed_block_size = V_028C78_MIN_BLOCK_SIZE_64B;
6393         }
6394
6395         if (device->physical_device->rad_info.chip_class >= GFX10) {
6396                 max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B;
6397                 independent_64b_blocks = 0;
6398                 independent_128b_blocks = 1;
6399         } else {
6400                 independent_128b_blocks = 0;
6401
6402                 if (iview->image->info.samples > 1) {
6403                         if (iview->image->planes[0].surface.bpe == 1)
6404                                 max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B;
6405                         else if (iview->image->planes[0].surface.bpe == 2)
6406                                 max_uncompressed_block_size = V_028C78_MAX_BLOCK_SIZE_128B;
6407                 }
6408
6409                 if (iview->image->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
6410                                            VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
6411                                            VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) {
6412                         /* If this DCC image is potentially going to be used in texture
6413                          * fetches, we need some special settings.
6414                          */
6415                         independent_64b_blocks = 1;
6416                         max_compressed_block_size = V_028C78_MAX_BLOCK_SIZE_64B;
6417                 } else {
6418                         /* MAX_UNCOMPRESSED_BLOCK_SIZE must be >=
6419                          * MAX_COMPRESSED_BLOCK_SIZE. Set MAX_COMPRESSED_BLOCK_SIZE as
6420                          * big as possible for better compression state.
6421                          */
6422                         independent_64b_blocks = 0;
6423                         max_compressed_block_size = max_uncompressed_block_size;
6424                 }
6425         }
6426
6427         return S_028C78_MAX_UNCOMPRESSED_BLOCK_SIZE(max_uncompressed_block_size) |
6428                S_028C78_MAX_COMPRESSED_BLOCK_SIZE(max_compressed_block_size) |
6429                S_028C78_MIN_COMPRESSED_BLOCK_SIZE(min_compressed_block_size) |
6430                S_028C78_INDEPENDENT_64B_BLOCKS(independent_64b_blocks) |
6431                S_028C78_INDEPENDENT_128B_BLOCKS(independent_128b_blocks);
6432 }
6433
6434 void
6435 radv_initialise_color_surface(struct radv_device *device,
6436                               struct radv_color_buffer_info *cb,
6437                               struct radv_image_view *iview)
6438 {
6439         const struct vk_format_description *desc;
6440         unsigned ntype, format, swap, endian;
6441         unsigned blend_clamp = 0, blend_bypass = 0;
6442         uint64_t va;
6443         const struct radv_image_plane *plane = &iview->image->planes[iview->plane_id];
6444         const struct radeon_surf *surf = &plane->surface;
6445
6446         desc = vk_format_description(iview->vk_format);
6447
6448         memset(cb, 0, sizeof(*cb));
6449
6450         /* Intensity is implemented as Red, so treat it that way. */
6451         cb->cb_color_attrib = S_028C74_FORCE_DST_ALPHA_1(desc->swizzle[3] == VK_SWIZZLE_1);
6452
6453         va = radv_buffer_get_va(iview->bo) + iview->image->offset + plane->offset;
6454
6455         cb->cb_color_base = va >> 8;
6456
6457         if (device->physical_device->rad_info.chip_class >= GFX9) {
6458                 struct gfx9_surf_meta_flags meta;
6459                 if (iview->image->dcc_offset)
6460                         meta = surf->u.gfx9.dcc;
6461                 else
6462                         meta = surf->u.gfx9.cmask;
6463
6464                 if (device->physical_device->rad_info.chip_class >= GFX10) {
6465                         cb->cb_color_attrib3 |= S_028EE0_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6466                                 S_028EE0_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) |
6467                                 S_028EE0_CMASK_PIPE_ALIGNED(surf->u.gfx9.cmask.pipe_aligned) |
6468                                 S_028EE0_DCC_PIPE_ALIGNED(surf->u.gfx9.dcc.pipe_aligned);
6469                 } else {
6470                         cb->cb_color_attrib |= S_028C74_COLOR_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6471                                 S_028C74_FMASK_SW_MODE(surf->u.gfx9.fmask.swizzle_mode) |
6472                                 S_028C74_RB_ALIGNED(meta.rb_aligned) |
6473                                 S_028C74_PIPE_ALIGNED(meta.pipe_aligned);
6474                         cb->cb_mrt_epitch = S_0287A0_EPITCH(surf->u.gfx9.surf.epitch);
6475                 }
6476
6477                 cb->cb_color_base += surf->u.gfx9.surf_offset >> 8;
6478                 cb->cb_color_base |= surf->tile_swizzle;
6479         } else {
6480                 const struct legacy_surf_level *level_info = &surf->u.legacy.level[iview->base_mip];
6481                 unsigned pitch_tile_max, slice_tile_max, tile_mode_index;
6482
6483                 cb->cb_color_base += level_info->offset >> 8;
6484                 if (level_info->mode == RADEON_SURF_MODE_2D)
6485                         cb->cb_color_base |= surf->tile_swizzle;
6486
6487                 pitch_tile_max = level_info->nblk_x / 8 - 1;
6488                 slice_tile_max = (level_info->nblk_x * level_info->nblk_y) / 64 - 1;
6489                 tile_mode_index = si_tile_mode_index(plane, iview->base_mip, false);
6490
6491                 cb->cb_color_pitch = S_028C64_TILE_MAX(pitch_tile_max);
6492                 cb->cb_color_slice = S_028C68_TILE_MAX(slice_tile_max);
6493                 cb->cb_color_cmask_slice = surf->u.legacy.cmask_slice_tile_max;
6494
6495                 cb->cb_color_attrib |= S_028C74_TILE_MODE_INDEX(tile_mode_index);
6496
6497                 if (radv_image_has_fmask(iview->image)) {
6498                         if (device->physical_device->rad_info.chip_class >= GFX7)
6499                                 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(surf->u.legacy.fmask.pitch_in_pixels / 8 - 1);
6500                         cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(surf->u.legacy.fmask.tiling_index);
6501                         cb->cb_color_fmask_slice = S_028C88_TILE_MAX(surf->u.legacy.fmask.slice_tile_max);
6502                 } else {
6503                         /* This must be set for fast clear to work without FMASK. */
6504                         if (device->physical_device->rad_info.chip_class >= GFX7)
6505                                 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(pitch_tile_max);
6506                         cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(tile_mode_index);
6507                         cb->cb_color_fmask_slice = S_028C88_TILE_MAX(slice_tile_max);
6508                 }
6509         }
6510
6511         /* CMASK variables */
6512         va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6513         va += iview->image->cmask_offset;
6514         cb->cb_color_cmask = va >> 8;
6515
6516         va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6517         va += iview->image->dcc_offset;
6518
6519         if (radv_dcc_enabled(iview->image, iview->base_mip) &&
6520             device->physical_device->rad_info.chip_class <= GFX8)
6521                 va += plane->surface.u.legacy.level[iview->base_mip].dcc_offset;
6522
6523         unsigned dcc_tile_swizzle = surf->tile_swizzle;
6524         dcc_tile_swizzle &= (surf->dcc_alignment - 1) >> 8;
6525
6526         cb->cb_dcc_base = va >> 8;
6527         cb->cb_dcc_base |= dcc_tile_swizzle;
6528
6529         /* GFX10 field has the same base shift as the GFX6 field. */
6530         uint32_t max_slice = radv_surface_max_layer_count(iview) - 1;
6531         cb->cb_color_view = S_028C6C_SLICE_START(iview->base_layer) |
6532                 S_028C6C_SLICE_MAX_GFX10(max_slice);
6533
6534         if (iview->image->info.samples > 1) {
6535                 unsigned log_samples = util_logbase2(iview->image->info.samples);
6536
6537                 cb->cb_color_attrib |= S_028C74_NUM_SAMPLES(log_samples) |
6538                         S_028C74_NUM_FRAGMENTS(log_samples);
6539         }
6540
6541         if (radv_image_has_fmask(iview->image)) {
6542                 va = radv_buffer_get_va(iview->bo) + iview->image->offset + iview->image->fmask_offset;
6543                 cb->cb_color_fmask = va >> 8;
6544                 cb->cb_color_fmask |= surf->fmask_tile_swizzle;
6545         } else {
6546                 cb->cb_color_fmask = cb->cb_color_base;
6547         }
6548
6549         ntype = radv_translate_color_numformat(iview->vk_format,
6550                                                desc,
6551                                                vk_format_get_first_non_void_channel(iview->vk_format));
6552         format = radv_translate_colorformat(iview->vk_format);
6553         if (format == V_028C70_COLOR_INVALID || ntype == ~0u)
6554                 radv_finishme("Illegal color\n");
6555         swap = radv_translate_colorswap(iview->vk_format, false);
6556         endian = radv_colorformat_endian_swap(format);
6557
6558         /* blend clamp should be set for all NORM/SRGB types */
6559         if (ntype == V_028C70_NUMBER_UNORM ||
6560             ntype == V_028C70_NUMBER_SNORM ||
6561             ntype == V_028C70_NUMBER_SRGB)
6562                 blend_clamp = 1;
6563
6564         /* set blend bypass according to docs if SINT/UINT or
6565            8/24 COLOR variants */
6566         if (ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT ||
6567             format == V_028C70_COLOR_8_24 || format == V_028C70_COLOR_24_8 ||
6568             format == V_028C70_COLOR_X24_8_32_FLOAT) {
6569                 blend_clamp = 0;
6570                 blend_bypass = 1;
6571         }
6572 #if 0
6573         if ((ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT) &&
6574             (format == V_028C70_COLOR_8 ||
6575              format == V_028C70_COLOR_8_8 ||
6576              format == V_028C70_COLOR_8_8_8_8))
6577                 ->color_is_int8 = true;
6578 #endif
6579         cb->cb_color_info = S_028C70_FORMAT(format) |
6580                 S_028C70_COMP_SWAP(swap) |
6581                 S_028C70_BLEND_CLAMP(blend_clamp) |
6582                 S_028C70_BLEND_BYPASS(blend_bypass) |
6583                 S_028C70_SIMPLE_FLOAT(1) |
6584                 S_028C70_ROUND_MODE(ntype != V_028C70_NUMBER_UNORM &&
6585                                     ntype != V_028C70_NUMBER_SNORM &&
6586                                     ntype != V_028C70_NUMBER_SRGB &&
6587                                     format != V_028C70_COLOR_8_24 &&
6588                                     format != V_028C70_COLOR_24_8) |
6589                 S_028C70_NUMBER_TYPE(ntype) |
6590                 S_028C70_ENDIAN(endian);
6591         if (radv_image_has_fmask(iview->image)) {
6592                 cb->cb_color_info |= S_028C70_COMPRESSION(1);
6593                 if (device->physical_device->rad_info.chip_class == GFX6) {
6594                         unsigned fmask_bankh = util_logbase2(surf->u.legacy.fmask.bankh);
6595                         cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(fmask_bankh);
6596                 }
6597
6598                 if (radv_image_is_tc_compat_cmask(iview->image)) {
6599                         /* Allow the texture block to read FMASK directly
6600                          * without decompressing it. This bit must be cleared
6601                          * when performing FMASK_DECOMPRESS or DCC_COMPRESS,
6602                          * otherwise the operation doesn't happen.
6603                          */
6604                         cb->cb_color_info |= S_028C70_FMASK_COMPRESS_1FRAG_ONLY(1);
6605
6606                         /* Set CMASK into a tiling format that allows the
6607                          * texture block to read it.
6608                          */
6609                         cb->cb_color_info |= S_028C70_CMASK_ADDR_TYPE(2);
6610                 }
6611         }
6612
6613         if (radv_image_has_cmask(iview->image) &&
6614             !(device->instance->debug_flags & RADV_DEBUG_NO_FAST_CLEARS))
6615                 cb->cb_color_info |= S_028C70_FAST_CLEAR(1);
6616
6617         if (radv_dcc_enabled(iview->image, iview->base_mip))
6618                 cb->cb_color_info |= S_028C70_DCC_ENABLE(1);
6619
6620         cb->cb_dcc_control = radv_init_dcc_control_reg(device, iview);
6621
6622         /* This must be set for fast clear to work without FMASK. */
6623         if (!radv_image_has_fmask(iview->image) &&
6624             device->physical_device->rad_info.chip_class == GFX6) {
6625                 unsigned bankh = util_logbase2(surf->u.legacy.bankh);
6626                 cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(bankh);
6627         }
6628
6629         if (device->physical_device->rad_info.chip_class >= GFX9) {
6630                 const struct vk_format_description *format_desc = vk_format_description(iview->image->vk_format);
6631
6632                 unsigned mip0_depth = iview->image->type == VK_IMAGE_TYPE_3D ?
6633                   (iview->extent.depth - 1) : (iview->image->info.array_size - 1);
6634                 unsigned width = iview->extent.width / (iview->plane_id ? format_desc->width_divisor : 1);
6635                 unsigned height = iview->extent.height / (iview->plane_id ? format_desc->height_divisor : 1);
6636
6637                 if (device->physical_device->rad_info.chip_class >= GFX10) {
6638                         cb->cb_color_view |= S_028C6C_MIP_LEVEL_GFX10(iview->base_mip);
6639
6640                         cb->cb_color_attrib3 |= S_028EE0_MIP0_DEPTH(mip0_depth) |
6641                                                 S_028EE0_RESOURCE_TYPE(surf->u.gfx9.resource_type) |
6642                                                 S_028EE0_RESOURCE_LEVEL(1);
6643                 } else {
6644                         cb->cb_color_view |= S_028C6C_MIP_LEVEL_GFX9(iview->base_mip);
6645                         cb->cb_color_attrib |= S_028C74_MIP0_DEPTH(mip0_depth) |
6646                                                S_028C74_RESOURCE_TYPE(surf->u.gfx9.resource_type);
6647                 }
6648
6649                 cb->cb_color_attrib2 = S_028C68_MIP0_WIDTH(width - 1) |
6650                         S_028C68_MIP0_HEIGHT(height - 1) |
6651                         S_028C68_MAX_MIP(iview->image->info.levels - 1);
6652         }
6653 }
6654
6655 static unsigned
6656 radv_calc_decompress_on_z_planes(struct radv_device *device,
6657                                  struct radv_image_view *iview)
6658 {
6659         unsigned max_zplanes = 0;
6660
6661         assert(radv_image_is_tc_compat_htile(iview->image));
6662
6663         if (device->physical_device->rad_info.chip_class >= GFX9) {
6664                 /* Default value for 32-bit depth surfaces. */
6665                 max_zplanes = 4;
6666
6667                 if (iview->vk_format == VK_FORMAT_D16_UNORM &&
6668                     iview->image->info.samples > 1)
6669                         max_zplanes = 2;
6670
6671                 max_zplanes = max_zplanes + 1;
6672         } else {
6673                 if (iview->vk_format == VK_FORMAT_D16_UNORM) {
6674                         /* Do not enable Z plane compression for 16-bit depth
6675                          * surfaces because isn't supported on GFX8. Only
6676                          * 32-bit depth surfaces are supported by the hardware.
6677                          * This allows to maintain shader compatibility and to
6678                          * reduce the number of depth decompressions.
6679                          */
6680                         max_zplanes = 1;
6681                 } else {
6682                         if (iview->image->info.samples <= 1)
6683                                 max_zplanes = 5;
6684                         else if (iview->image->info.samples <= 4)
6685                                 max_zplanes = 3;
6686                         else
6687                                 max_zplanes = 2;
6688                 }
6689         }
6690
6691         return max_zplanes;
6692 }
6693
6694 void
6695 radv_initialise_ds_surface(struct radv_device *device,
6696                            struct radv_ds_buffer_info *ds,
6697                            struct radv_image_view *iview)
6698 {
6699         unsigned level = iview->base_mip;
6700         unsigned format, stencil_format;
6701         uint64_t va, s_offs, z_offs;
6702         bool stencil_only = false;
6703         const struct radv_image_plane *plane = &iview->image->planes[0];
6704         const struct radeon_surf *surf = &plane->surface;
6705
6706         assert(vk_format_get_plane_count(iview->image->vk_format) == 1);
6707
6708         memset(ds, 0, sizeof(*ds));
6709         switch (iview->image->vk_format) {
6710         case VK_FORMAT_D24_UNORM_S8_UINT:
6711         case VK_FORMAT_X8_D24_UNORM_PACK32:
6712                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-24);
6713                 ds->offset_scale = 2.0f;
6714                 break;
6715         case VK_FORMAT_D16_UNORM:
6716         case VK_FORMAT_D16_UNORM_S8_UINT:
6717                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-16);
6718                 ds->offset_scale = 4.0f;
6719                 break;
6720         case VK_FORMAT_D32_SFLOAT:
6721         case VK_FORMAT_D32_SFLOAT_S8_UINT:
6722                 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-23) |
6723                         S_028B78_POLY_OFFSET_DB_IS_FLOAT_FMT(1);
6724                 ds->offset_scale = 1.0f;
6725                 break;
6726         case VK_FORMAT_S8_UINT:
6727                 stencil_only = true;
6728                 break;
6729         default:
6730                 break;
6731         }
6732
6733         format = radv_translate_dbformat(iview->image->vk_format);
6734         stencil_format = surf->has_stencil ?
6735                 V_028044_STENCIL_8 : V_028044_STENCIL_INVALID;
6736
6737         uint32_t max_slice = radv_surface_max_layer_count(iview) - 1;
6738         ds->db_depth_view = S_028008_SLICE_START(iview->base_layer) |
6739                 S_028008_SLICE_MAX(max_slice);
6740         if (device->physical_device->rad_info.chip_class >= GFX10) {
6741                 ds->db_depth_view |= S_028008_SLICE_START_HI(iview->base_layer >> 11) |
6742                                      S_028008_SLICE_MAX_HI(max_slice >> 11);
6743         }
6744
6745         ds->db_htile_data_base = 0;
6746         ds->db_htile_surface = 0;
6747
6748         va = radv_buffer_get_va(iview->bo) + iview->image->offset;
6749         s_offs = z_offs = va;
6750
6751         if (device->physical_device->rad_info.chip_class >= GFX9) {
6752                 assert(surf->u.gfx9.surf_offset == 0);
6753                 s_offs += surf->u.gfx9.stencil_offset;
6754
6755                 ds->db_z_info = S_028038_FORMAT(format) |
6756                         S_028038_NUM_SAMPLES(util_logbase2(iview->image->info.samples)) |
6757                         S_028038_SW_MODE(surf->u.gfx9.surf.swizzle_mode) |
6758                         S_028038_MAXMIP(iview->image->info.levels - 1) |
6759                         S_028038_ZRANGE_PRECISION(1);
6760                 ds->db_stencil_info = S_02803C_FORMAT(stencil_format) |
6761                         S_02803C_SW_MODE(surf->u.gfx9.stencil.swizzle_mode);
6762
6763                 if (device->physical_device->rad_info.chip_class == GFX9) {
6764                         ds->db_z_info2 = S_028068_EPITCH(surf->u.gfx9.surf.epitch);
6765                         ds->db_stencil_info2 = S_02806C_EPITCH(surf->u.gfx9.stencil.epitch);
6766                 }
6767
6768                 ds->db_depth_view |= S_028008_MIPID(level);
6769                 ds->db_depth_size = S_02801C_X_MAX(iview->image->info.width - 1) |
6770                         S_02801C_Y_MAX(iview->image->info.height - 1);
6771
6772                 if (radv_htile_enabled(iview->image, level)) {
6773                         ds->db_z_info |= S_028038_TILE_SURFACE_ENABLE(1);
6774
6775                         if (radv_image_is_tc_compat_htile(iview->image)) {
6776                                 unsigned max_zplanes =
6777                                         radv_calc_decompress_on_z_planes(device, iview);
6778
6779                                 ds->db_z_info |= S_028038_DECOMPRESS_ON_N_ZPLANES(max_zplanes);
6780
6781                                 if (device->physical_device->rad_info.chip_class >= GFX10) {
6782                                         ds->db_z_info |= S_028040_ITERATE_FLUSH(1);
6783                                         ds->db_stencil_info |= S_028044_ITERATE_FLUSH(1);
6784                                 } else {
6785                                         ds->db_z_info |= S_028038_ITERATE_FLUSH(1);
6786                                         ds->db_stencil_info |= S_02803C_ITERATE_FLUSH(1);
6787                                 }
6788                         }
6789
6790                         if (!surf->has_stencil)
6791                                 /* Use all of the htile_buffer for depth if there's no stencil. */
6792                                 ds->db_stencil_info |= S_02803C_TILE_STENCIL_DISABLE(1);
6793                         va = radv_buffer_get_va(iview->bo) + iview->image->offset +
6794                                 iview->image->htile_offset;
6795                         ds->db_htile_data_base = va >> 8;
6796                         ds->db_htile_surface = S_028ABC_FULL_CACHE(1) |
6797                                 S_028ABC_PIPE_ALIGNED(surf->u.gfx9.htile.pipe_aligned);
6798
6799                         if (device->physical_device->rad_info.chip_class == GFX9) {
6800                                 ds->db_htile_surface |= S_028ABC_RB_ALIGNED(surf->u.gfx9.htile.rb_aligned);
6801                         }
6802                 }
6803         } else {
6804                 const struct legacy_surf_level *level_info = &surf->u.legacy.level[level];
6805
6806                 if (stencil_only)
6807                         level_info = &surf->u.legacy.stencil_level[level];
6808
6809                 z_offs += surf->u.legacy.level[level].offset;
6810                 s_offs += surf->u.legacy.stencil_level[level].offset;
6811
6812                 ds->db_depth_info = S_02803C_ADDR5_SWIZZLE_MASK(!radv_image_is_tc_compat_htile(iview->image));
6813                 ds->db_z_info = S_028040_FORMAT(format) | S_028040_ZRANGE_PRECISION(1);
6814                 ds->db_stencil_info = S_028044_FORMAT(stencil_format);
6815
6816                 if (iview->image->info.samples > 1)
6817                         ds->db_z_info |= S_028040_NUM_SAMPLES(util_logbase2(iview->image->info.samples));
6818
6819                 if (device->physical_device->rad_info.chip_class >= GFX7) {
6820                         struct radeon_info *info = &device->physical_device->rad_info;
6821                         unsigned tiling_index = surf->u.legacy.tiling_index[level];
6822                         unsigned stencil_index = surf->u.legacy.stencil_tiling_index[level];
6823                         unsigned macro_index = surf->u.legacy.macro_tile_index;
6824                         unsigned tile_mode = info->si_tile_mode_array[tiling_index];
6825                         unsigned stencil_tile_mode = info->si_tile_mode_array[stencil_index];
6826                         unsigned macro_mode = info->cik_macrotile_mode_array[macro_index];
6827
6828                         if (stencil_only)
6829                                 tile_mode = stencil_tile_mode;
6830
6831                         ds->db_depth_info |=
6832                                 S_02803C_ARRAY_MODE(G_009910_ARRAY_MODE(tile_mode)) |
6833                                 S_02803C_PIPE_CONFIG(G_009910_PIPE_CONFIG(tile_mode)) |
6834                                 S_02803C_BANK_WIDTH(G_009990_BANK_WIDTH(macro_mode)) |
6835                                 S_02803C_BANK_HEIGHT(G_009990_BANK_HEIGHT(macro_mode)) |
6836                                 S_02803C_MACRO_TILE_ASPECT(G_009990_MACRO_TILE_ASPECT(macro_mode)) |
6837                                 S_02803C_NUM_BANKS(G_009990_NUM_BANKS(macro_mode));
6838                         ds->db_z_info |= S_028040_TILE_SPLIT(G_009910_TILE_SPLIT(tile_mode));
6839                         ds->db_stencil_info |= S_028044_TILE_SPLIT(G_009910_TILE_SPLIT(stencil_tile_mode));
6840                 } else {
6841                         unsigned tile_mode_index = si_tile_mode_index(&iview->image->planes[0], level, false);
6842                         ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
6843                         tile_mode_index = si_tile_mode_index(&iview->image->planes[0], level, true);
6844                         ds->db_stencil_info |= S_028044_TILE_MODE_INDEX(tile_mode_index);
6845                         if (stencil_only)
6846                                 ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
6847                 }
6848
6849                 ds->db_depth_size = S_028058_PITCH_TILE_MAX((level_info->nblk_x / 8) - 1) |
6850                         S_028058_HEIGHT_TILE_MAX((level_info->nblk_y / 8) - 1);
6851                 ds->db_depth_slice = S_02805C_SLICE_TILE_MAX((level_info->nblk_x * level_info->nblk_y) / 64 - 1);
6852
6853                 if (radv_htile_enabled(iview->image, level)) {
6854                         ds->db_z_info |= S_028040_TILE_SURFACE_ENABLE(1);
6855
6856                         if (!surf->has_stencil &&
6857                             !radv_image_is_tc_compat_htile(iview->image))
6858                                 /* Use all of the htile_buffer for depth if there's no stencil. */
6859                                 ds->db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
6860
6861                         va = radv_buffer_get_va(iview->bo) + iview->image->offset +
6862                                 iview->image->htile_offset;
6863                         ds->db_htile_data_base = va >> 8;
6864                         ds->db_htile_surface = S_028ABC_FULL_CACHE(1);
6865
6866                         if (radv_image_is_tc_compat_htile(iview->image)) {
6867                                 unsigned max_zplanes =
6868                                         radv_calc_decompress_on_z_planes(device, iview);
6869
6870                                 ds->db_htile_surface |= S_028ABC_TC_COMPATIBLE(1);
6871                                 ds->db_z_info |= S_028040_DECOMPRESS_ON_N_ZPLANES(max_zplanes);
6872                         }
6873                 }
6874         }
6875
6876         ds->db_z_read_base = ds->db_z_write_base = z_offs >> 8;
6877         ds->db_stencil_read_base = ds->db_stencil_write_base = s_offs >> 8;
6878 }
6879
6880 VkResult radv_CreateFramebuffer(
6881         VkDevice                                    _device,
6882         const VkFramebufferCreateInfo*              pCreateInfo,
6883         const VkAllocationCallbacks*                pAllocator,
6884         VkFramebuffer*                              pFramebuffer)
6885 {
6886         RADV_FROM_HANDLE(radv_device, device, _device);
6887         struct radv_framebuffer *framebuffer;
6888         const VkFramebufferAttachmentsCreateInfo *imageless_create_info =
6889                 vk_find_struct_const(pCreateInfo->pNext,
6890                         FRAMEBUFFER_ATTACHMENTS_CREATE_INFO);
6891
6892         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO);
6893
6894         size_t size = sizeof(*framebuffer);
6895         if (!imageless_create_info)
6896                 size += sizeof(struct radv_image_view*) * pCreateInfo->attachmentCount;
6897         framebuffer = vk_alloc2(&device->alloc, pAllocator, size, 8,
6898                                   VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
6899         if (framebuffer == NULL)
6900                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
6901
6902         framebuffer->attachment_count = pCreateInfo->attachmentCount;
6903         framebuffer->width = pCreateInfo->width;
6904         framebuffer->height = pCreateInfo->height;
6905         framebuffer->layers = pCreateInfo->layers;
6906         if (imageless_create_info) {
6907                 for (unsigned i = 0; i < imageless_create_info->attachmentImageInfoCount; ++i) {
6908                         const VkFramebufferAttachmentImageInfo *attachment =
6909                                 imageless_create_info->pAttachmentImageInfos + i;
6910                         framebuffer->width = MIN2(framebuffer->width, attachment->width);
6911                         framebuffer->height = MIN2(framebuffer->height, attachment->height);
6912                         framebuffer->layers = MIN2(framebuffer->layers, attachment->layerCount);
6913                 }
6914         } else {
6915                 for (uint32_t i = 0; i < pCreateInfo->attachmentCount; i++) {
6916                         VkImageView _iview = pCreateInfo->pAttachments[i];
6917                         struct radv_image_view *iview = radv_image_view_from_handle(_iview);
6918                         framebuffer->attachments[i] = iview;
6919                         framebuffer->width = MIN2(framebuffer->width, iview->extent.width);
6920                         framebuffer->height = MIN2(framebuffer->height, iview->extent.height);
6921                         framebuffer->layers = MIN2(framebuffer->layers, radv_surface_max_layer_count(iview));
6922                 }
6923         }
6924
6925         *pFramebuffer = radv_framebuffer_to_handle(framebuffer);
6926         return VK_SUCCESS;
6927 }
6928
6929 void radv_DestroyFramebuffer(
6930         VkDevice                                    _device,
6931         VkFramebuffer                               _fb,
6932         const VkAllocationCallbacks*                pAllocator)
6933 {
6934         RADV_FROM_HANDLE(radv_device, device, _device);
6935         RADV_FROM_HANDLE(radv_framebuffer, fb, _fb);
6936
6937         if (!fb)
6938                 return;
6939         vk_free2(&device->alloc, pAllocator, fb);
6940 }
6941
6942 static unsigned radv_tex_wrap(VkSamplerAddressMode address_mode)
6943 {
6944         switch (address_mode) {
6945         case VK_SAMPLER_ADDRESS_MODE_REPEAT:
6946                 return V_008F30_SQ_TEX_WRAP;
6947         case VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT:
6948                 return V_008F30_SQ_TEX_MIRROR;
6949         case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE:
6950                 return V_008F30_SQ_TEX_CLAMP_LAST_TEXEL;
6951         case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER:
6952                 return V_008F30_SQ_TEX_CLAMP_BORDER;
6953         case VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE:
6954                 return V_008F30_SQ_TEX_MIRROR_ONCE_LAST_TEXEL;
6955         default:
6956                 unreachable("illegal tex wrap mode");
6957                 break;
6958         }
6959 }
6960
6961 static unsigned
6962 radv_tex_compare(VkCompareOp op)
6963 {
6964         switch (op) {
6965         case VK_COMPARE_OP_NEVER:
6966                 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
6967         case VK_COMPARE_OP_LESS:
6968                 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESS;
6969         case VK_COMPARE_OP_EQUAL:
6970                 return V_008F30_SQ_TEX_DEPTH_COMPARE_EQUAL;
6971         case VK_COMPARE_OP_LESS_OR_EQUAL:
6972                 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESSEQUAL;
6973         case VK_COMPARE_OP_GREATER:
6974                 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATER;
6975         case VK_COMPARE_OP_NOT_EQUAL:
6976                 return V_008F30_SQ_TEX_DEPTH_COMPARE_NOTEQUAL;
6977         case VK_COMPARE_OP_GREATER_OR_EQUAL:
6978                 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATEREQUAL;
6979         case VK_COMPARE_OP_ALWAYS:
6980                 return V_008F30_SQ_TEX_DEPTH_COMPARE_ALWAYS;
6981         default:
6982                 unreachable("illegal compare mode");
6983                 break;
6984         }
6985 }
6986
6987 static unsigned
6988 radv_tex_filter(VkFilter filter, unsigned max_ansio)
6989 {
6990         switch (filter) {
6991         case VK_FILTER_NEAREST:
6992                 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_POINT :
6993                         V_008F38_SQ_TEX_XY_FILTER_POINT);
6994         case VK_FILTER_LINEAR:
6995                 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_BILINEAR :
6996                         V_008F38_SQ_TEX_XY_FILTER_BILINEAR);
6997         case VK_FILTER_CUBIC_IMG:
6998         default:
6999                 fprintf(stderr, "illegal texture filter");
7000                 return 0;
7001         }
7002 }
7003
7004 static unsigned
7005 radv_tex_mipfilter(VkSamplerMipmapMode mode)
7006 {
7007         switch (mode) {
7008         case VK_SAMPLER_MIPMAP_MODE_NEAREST:
7009                 return V_008F38_SQ_TEX_Z_FILTER_POINT;
7010         case VK_SAMPLER_MIPMAP_MODE_LINEAR:
7011                 return V_008F38_SQ_TEX_Z_FILTER_LINEAR;
7012         default:
7013                 return V_008F38_SQ_TEX_Z_FILTER_NONE;
7014         }
7015 }
7016
7017 static unsigned
7018 radv_tex_bordercolor(VkBorderColor bcolor)
7019 {
7020         switch (bcolor) {
7021         case VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK:
7022         case VK_BORDER_COLOR_INT_TRANSPARENT_BLACK:
7023                 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
7024         case VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK:
7025         case VK_BORDER_COLOR_INT_OPAQUE_BLACK:
7026                 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_BLACK;
7027         case VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE:
7028         case VK_BORDER_COLOR_INT_OPAQUE_WHITE:
7029                 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE;
7030         default:
7031                 break;
7032         }
7033         return 0;
7034 }
7035
7036 static unsigned
7037 radv_tex_aniso_filter(unsigned filter)
7038 {
7039         if (filter < 2)
7040                 return 0;
7041         if (filter < 4)
7042                 return 1;
7043         if (filter < 8)
7044                 return 2;
7045         if (filter < 16)
7046                 return 3;
7047         return 4;
7048 }
7049
7050 static unsigned
7051 radv_tex_filter_mode(VkSamplerReductionMode mode)
7052 {
7053         switch (mode) {
7054         case VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT:
7055                 return V_008F30_SQ_IMG_FILTER_MODE_BLEND;
7056         case VK_SAMPLER_REDUCTION_MODE_MIN_EXT:
7057                 return V_008F30_SQ_IMG_FILTER_MODE_MIN;
7058         case VK_SAMPLER_REDUCTION_MODE_MAX_EXT:
7059                 return V_008F30_SQ_IMG_FILTER_MODE_MAX;
7060         default:
7061                 break;
7062         }
7063         return 0;
7064 }
7065
7066 static uint32_t
7067 radv_get_max_anisotropy(struct radv_device *device,
7068                         const VkSamplerCreateInfo *pCreateInfo)
7069 {
7070         if (device->force_aniso >= 0)
7071                 return device->force_aniso;
7072
7073         if (pCreateInfo->anisotropyEnable &&
7074             pCreateInfo->maxAnisotropy > 1.0f)
7075                 return (uint32_t)pCreateInfo->maxAnisotropy;
7076
7077         return 0;
7078 }
7079
7080 static inline int S_FIXED(float value, unsigned frac_bits)
7081 {
7082         return value * (1 << frac_bits);
7083 }
7084
7085 static void
7086 radv_init_sampler(struct radv_device *device,
7087                   struct radv_sampler *sampler,
7088                   const VkSamplerCreateInfo *pCreateInfo)
7089 {
7090         uint32_t max_aniso = radv_get_max_anisotropy(device, pCreateInfo);
7091         uint32_t max_aniso_ratio = radv_tex_aniso_filter(max_aniso);
7092         bool compat_mode = device->physical_device->rad_info.chip_class == GFX8 ||
7093                            device->physical_device->rad_info.chip_class == GFX9;
7094         unsigned filter_mode = V_008F30_SQ_IMG_FILTER_MODE_BLEND;
7095         unsigned depth_compare_func = V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
7096         bool trunc_coord = pCreateInfo->minFilter == VK_FILTER_NEAREST && pCreateInfo->magFilter == VK_FILTER_NEAREST;
7097
7098         const struct VkSamplerReductionModeCreateInfo *sampler_reduction =
7099                 vk_find_struct_const(pCreateInfo->pNext,
7100                                      SAMPLER_REDUCTION_MODE_CREATE_INFO);
7101         if (sampler_reduction)
7102                 filter_mode = radv_tex_filter_mode(sampler_reduction->reductionMode);
7103
7104         if (pCreateInfo->compareEnable)
7105                 depth_compare_func = radv_tex_compare(pCreateInfo->compareOp);
7106
7107         sampler->state[0] = (S_008F30_CLAMP_X(radv_tex_wrap(pCreateInfo->addressModeU)) |
7108                              S_008F30_CLAMP_Y(radv_tex_wrap(pCreateInfo->addressModeV)) |
7109                              S_008F30_CLAMP_Z(radv_tex_wrap(pCreateInfo->addressModeW)) |
7110                              S_008F30_MAX_ANISO_RATIO(max_aniso_ratio) |
7111                              S_008F30_DEPTH_COMPARE_FUNC(depth_compare_func) |
7112                              S_008F30_FORCE_UNNORMALIZED(pCreateInfo->unnormalizedCoordinates ? 1 : 0) |
7113                              S_008F30_ANISO_THRESHOLD(max_aniso_ratio >> 1) |
7114                              S_008F30_ANISO_BIAS(max_aniso_ratio) |
7115                              S_008F30_DISABLE_CUBE_WRAP(0) |
7116                              S_008F30_COMPAT_MODE(compat_mode) |
7117                              S_008F30_FILTER_MODE(filter_mode) |
7118                              S_008F30_TRUNC_COORD(trunc_coord));
7119         sampler->state[1] = (S_008F34_MIN_LOD(S_FIXED(CLAMP(pCreateInfo->minLod, 0, 15), 8)) |
7120                              S_008F34_MAX_LOD(S_FIXED(CLAMP(pCreateInfo->maxLod, 0, 15), 8)) |
7121                              S_008F34_PERF_MIP(max_aniso_ratio ? max_aniso_ratio + 6 : 0));
7122         sampler->state[2] = (S_008F38_LOD_BIAS(S_FIXED(CLAMP(pCreateInfo->mipLodBias, -16, 16), 8)) |
7123                              S_008F38_XY_MAG_FILTER(radv_tex_filter(pCreateInfo->magFilter, max_aniso)) |
7124                              S_008F38_XY_MIN_FILTER(radv_tex_filter(pCreateInfo->minFilter, max_aniso)) |
7125                              S_008F38_MIP_FILTER(radv_tex_mipfilter(pCreateInfo->mipmapMode)) |
7126                              S_008F38_MIP_POINT_PRECLAMP(0));
7127         sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(0) |
7128                              S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(pCreateInfo->borderColor)));
7129
7130         if (device->physical_device->rad_info.chip_class >= GFX10) {
7131                 sampler->state[2] |= S_008F38_ANISO_OVERRIDE_GFX10(1);
7132         } else {
7133                 sampler->state[2] |=
7134                         S_008F38_DISABLE_LSB_CEIL(device->physical_device->rad_info.chip_class <= GFX8) |
7135                         S_008F38_FILTER_PREC_FIX(1) |
7136                         S_008F38_ANISO_OVERRIDE_GFX6(device->physical_device->rad_info.chip_class >= GFX8);
7137         }
7138 }
7139
7140 VkResult radv_CreateSampler(
7141         VkDevice                                    _device,
7142         const VkSamplerCreateInfo*                  pCreateInfo,
7143         const VkAllocationCallbacks*                pAllocator,
7144         VkSampler*                                  pSampler)
7145 {
7146         RADV_FROM_HANDLE(radv_device, device, _device);
7147         struct radv_sampler *sampler;
7148
7149         const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion =
7150                 vk_find_struct_const(pCreateInfo->pNext,
7151                                      SAMPLER_YCBCR_CONVERSION_INFO);
7152
7153         assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO);
7154
7155         sampler = vk_alloc2(&device->alloc, pAllocator, sizeof(*sampler), 8,
7156                               VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
7157         if (!sampler)
7158                 return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
7159
7160         radv_init_sampler(device, sampler, pCreateInfo);
7161
7162         sampler->ycbcr_sampler = ycbcr_conversion ? radv_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion): NULL;
7163         *pSampler = radv_sampler_to_handle(sampler);
7164
7165         return VK_SUCCESS;
7166 }
7167
7168 void radv_DestroySampler(
7169         VkDevice                                    _device,
7170         VkSampler                                   _sampler,
7171         const VkAllocationCallbacks*                pAllocator)
7172 {
7173         RADV_FROM_HANDLE(radv_device, device, _device);
7174         RADV_FROM_HANDLE(radv_sampler, sampler, _sampler);
7175
7176         if (!sampler)
7177                 return;
7178         vk_free2(&device->alloc, pAllocator, sampler);
7179 }
7180
7181 /* vk_icd.h does not declare this function, so we declare it here to
7182  * suppress Wmissing-prototypes.
7183  */
7184 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
7185 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion);
7186
7187 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
7188 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion)
7189 {
7190         /* For the full details on loader interface versioning, see
7191         * <https://github.com/KhronosGroup/Vulkan-LoaderAndValidationLayers/blob/master/loader/LoaderAndLayerInterface.md>.
7192         * What follows is a condensed summary, to help you navigate the large and
7193         * confusing official doc.
7194         *
7195         *   - Loader interface v0 is incompatible with later versions. We don't
7196         *     support it.
7197         *
7198         *   - In loader interface v1:
7199         *       - The first ICD entrypoint called by the loader is
7200         *         vk_icdGetInstanceProcAddr(). The ICD must statically expose this
7201         *         entrypoint.
7202         *       - The ICD must statically expose no other Vulkan symbol unless it is
7203         *         linked with -Bsymbolic.
7204         *       - Each dispatchable Vulkan handle created by the ICD must be
7205         *         a pointer to a struct whose first member is VK_LOADER_DATA. The
7206         *         ICD must initialize VK_LOADER_DATA.loadMagic to ICD_LOADER_MAGIC.
7207         *       - The loader implements vkCreate{PLATFORM}SurfaceKHR() and
7208         *         vkDestroySurfaceKHR(). The ICD must be capable of working with
7209         *         such loader-managed surfaces.
7210         *
7211         *    - Loader interface v2 differs from v1 in:
7212         *       - The first ICD entrypoint called by the loader is
7213         *         vk_icdNegotiateLoaderICDInterfaceVersion(). The ICD must
7214         *         statically expose this entrypoint.
7215         *
7216         *    - Loader interface v3 differs from v2 in:
7217         *        - The ICD must implement vkCreate{PLATFORM}SurfaceKHR(),
7218         *          vkDestroySurfaceKHR(), and other API which uses VKSurfaceKHR,
7219         *          because the loader no longer does so.
7220         */
7221         *pSupportedVersion = MIN2(*pSupportedVersion, 4u);
7222         return VK_SUCCESS;
7223 }
7224
7225 VkResult radv_GetMemoryFdKHR(VkDevice _device,
7226                              const VkMemoryGetFdInfoKHR *pGetFdInfo,
7227                              int *pFD)
7228 {
7229         RADV_FROM_HANDLE(radv_device, device, _device);
7230         RADV_FROM_HANDLE(radv_device_memory, memory, pGetFdInfo->memory);
7231
7232         assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR);
7233
7234         /* At the moment, we support only the below handle types. */
7235         assert(pGetFdInfo->handleType ==
7236                VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
7237                pGetFdInfo->handleType ==
7238                VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
7239
7240         bool ret = radv_get_memory_fd(device, memory, pFD);
7241         if (ret == false)
7242                 return vk_error(device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
7243         return VK_SUCCESS;
7244 }
7245
7246 static uint32_t radv_compute_valid_memory_types_attempt(struct radv_physical_device *dev,
7247                                                         enum radeon_bo_domain domains,
7248                                                         enum radeon_bo_flag flags,
7249                                                         enum radeon_bo_flag ignore_flags)
7250 {
7251         /* Don't count GTT/CPU as relevant:
7252          *
7253          * - We're not fully consistent between the two.
7254          * - Sometimes VRAM gets VRAM|GTT.
7255          */
7256         const enum radeon_bo_domain relevant_domains = RADEON_DOMAIN_VRAM |
7257                                                        RADEON_DOMAIN_GDS |
7258                                                        RADEON_DOMAIN_OA;
7259         uint32_t bits = 0;
7260         for (unsigned i = 0; i < dev->memory_properties.memoryTypeCount; ++i) {
7261                 if ((domains & relevant_domains) != (dev->memory_domains[i] & relevant_domains))
7262                         continue;
7263
7264                 if ((flags & ~ignore_flags) != (dev->memory_flags[i] & ~ignore_flags))
7265                         continue;
7266
7267                 bits |= 1u << i;
7268         }
7269
7270         return bits;
7271 }
7272
7273 static uint32_t radv_compute_valid_memory_types(struct radv_physical_device *dev,
7274                                                 enum radeon_bo_domain domains,
7275                                                 enum radeon_bo_flag flags)
7276 {
7277         enum radeon_bo_flag ignore_flags = ~(RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_GTT_WC);
7278         uint32_t bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags);
7279
7280         if (!bits) {
7281                 ignore_flags |= RADEON_FLAG_NO_CPU_ACCESS;
7282                 bits = radv_compute_valid_memory_types_attempt(dev, domains, flags, ignore_flags);
7283         }
7284
7285         return bits;
7286 }
7287 VkResult radv_GetMemoryFdPropertiesKHR(VkDevice _device,
7288                                        VkExternalMemoryHandleTypeFlagBits handleType,
7289                                        int fd,
7290                                        VkMemoryFdPropertiesKHR *pMemoryFdProperties)
7291 {
7292         RADV_FROM_HANDLE(radv_device, device, _device);
7293
7294         switch (handleType) {
7295         case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: {
7296                 enum radeon_bo_domain domains;
7297                 enum radeon_bo_flag flags;
7298                 if (!device->ws->buffer_get_flags_from_fd(device->ws, fd, &domains, &flags))
7299                         return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7300
7301                 pMemoryFdProperties->memoryTypeBits = radv_compute_valid_memory_types(device->physical_device, domains, flags);
7302                 return VK_SUCCESS;
7303         }
7304         default:
7305                 /* The valid usage section for this function says:
7306                  *
7307                  *    "handleType must not be one of the handle types defined as
7308                  *    opaque."
7309                  *
7310                  * So opaque handle types fall into the default "unsupported" case.
7311                  */
7312                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7313         }
7314 }
7315
7316 static VkResult radv_import_opaque_fd(struct radv_device *device,
7317                                       int fd,
7318                                       uint32_t *syncobj)
7319 {
7320         uint32_t syncobj_handle = 0;
7321         int ret = device->ws->import_syncobj(device->ws, fd, &syncobj_handle);
7322         if (ret != 0)
7323                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7324
7325         if (*syncobj)
7326                 device->ws->destroy_syncobj(device->ws, *syncobj);
7327
7328         *syncobj = syncobj_handle;
7329         close(fd);
7330
7331         return VK_SUCCESS;
7332 }
7333
7334 static VkResult radv_import_sync_fd(struct radv_device *device,
7335                                     int fd,
7336                                     uint32_t *syncobj)
7337 {
7338         /* If we create a syncobj we do it locally so that if we have an error, we don't
7339          * leave a syncobj in an undetermined state in the fence. */
7340         uint32_t syncobj_handle =  *syncobj;
7341         if (!syncobj_handle) {
7342                 int ret = device->ws->create_syncobj(device->ws, &syncobj_handle);
7343                 if (ret) {
7344                         return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7345                 }
7346         }
7347
7348         if (fd == -1) {
7349                 device->ws->signal_syncobj(device->ws, syncobj_handle);
7350         } else {
7351                 int ret = device->ws->import_syncobj_from_sync_file(device->ws, syncobj_handle, fd);
7352         if (ret != 0)
7353                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7354         }
7355
7356         *syncobj = syncobj_handle;
7357         if (fd != -1)
7358                 close(fd);
7359
7360         return VK_SUCCESS;
7361 }
7362
7363 VkResult radv_ImportSemaphoreFdKHR(VkDevice _device,
7364                                    const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
7365 {
7366         RADV_FROM_HANDLE(radv_device, device, _device);
7367         RADV_FROM_HANDLE(radv_semaphore, sem, pImportSemaphoreFdInfo->semaphore);
7368         VkResult result;
7369         struct radv_semaphore_part *dst = NULL;
7370
7371         if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) {
7372                 dst = &sem->temporary;
7373         } else {
7374                 dst = &sem->permanent;
7375         }
7376
7377         uint32_t syncobj = dst->kind == RADV_SEMAPHORE_SYNCOBJ ? dst->syncobj : 0;
7378
7379         switch(pImportSemaphoreFdInfo->handleType) {
7380                 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
7381                         result = radv_import_opaque_fd(device, pImportSemaphoreFdInfo->fd, &syncobj);
7382                         break;
7383                 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
7384                         result = radv_import_sync_fd(device, pImportSemaphoreFdInfo->fd, &syncobj);
7385                         break;
7386                 default:
7387                         unreachable("Unhandled semaphore handle type");
7388         }
7389
7390         if (result == VK_SUCCESS) {
7391                 dst->syncobj = syncobj;
7392                 dst->kind = RADV_SEMAPHORE_SYNCOBJ;
7393         }
7394
7395         return result;
7396 }
7397
7398 VkResult radv_GetSemaphoreFdKHR(VkDevice _device,
7399                                 const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
7400                                 int *pFd)
7401 {
7402         RADV_FROM_HANDLE(radv_device, device, _device);
7403         RADV_FROM_HANDLE(radv_semaphore, sem, pGetFdInfo->semaphore);
7404         int ret;
7405         uint32_t syncobj_handle;
7406
7407         if (sem->temporary.kind != RADV_SEMAPHORE_NONE) {
7408                 assert(sem->temporary.kind == RADV_SEMAPHORE_SYNCOBJ);
7409                 syncobj_handle = sem->temporary.syncobj;
7410         } else {
7411                 assert(sem->permanent.kind == RADV_SEMAPHORE_SYNCOBJ);
7412                 syncobj_handle = sem->permanent.syncobj;
7413         }
7414
7415         switch(pGetFdInfo->handleType) {
7416         case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT:
7417                 ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd);
7418                 break;
7419         case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT:
7420                 ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd);
7421                 if (!ret) {
7422                         if (sem->temporary.kind != RADV_SEMAPHORE_NONE) {
7423                                 radv_destroy_semaphore_part(device, &sem->temporary);
7424                         } else {
7425                                 device->ws->reset_syncobj(device->ws, syncobj_handle);
7426                         }
7427                 }
7428                 break;
7429         default:
7430                 unreachable("Unhandled semaphore handle type");
7431         }
7432
7433         if (ret)
7434                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7435         return VK_SUCCESS;
7436 }
7437
7438 void radv_GetPhysicalDeviceExternalSemaphoreProperties(
7439         VkPhysicalDevice                            physicalDevice,
7440         const VkPhysicalDeviceExternalSemaphoreInfo *pExternalSemaphoreInfo,
7441         VkExternalSemaphoreProperties               *pExternalSemaphoreProperties)
7442 {
7443         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
7444         VkSemaphoreTypeKHR type = radv_get_semaphore_type(pExternalSemaphoreInfo->pNext, NULL);
7445
7446         if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
7447                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0;
7448                 pExternalSemaphoreProperties->compatibleHandleTypes = 0;
7449                 pExternalSemaphoreProperties->externalSemaphoreFeatures = 0;
7450
7451         /* Require has_syncobj_wait_for_submit for the syncobj signal ioctl introduced at virtually the same time */
7452         } else if (pdevice->rad_info.has_syncobj_wait_for_submit &&
7453                    (pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT ||
7454                     pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT)) {
7455                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
7456                 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
7457                 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7458                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7459         } else if (pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) {
7460                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7461                 pExternalSemaphoreProperties->compatibleHandleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
7462                 pExternalSemaphoreProperties->externalSemaphoreFeatures = VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT |
7463                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7464         } else {
7465                 pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0;
7466                 pExternalSemaphoreProperties->compatibleHandleTypes = 0;
7467                 pExternalSemaphoreProperties->externalSemaphoreFeatures = 0;
7468         }
7469 }
7470
7471 VkResult radv_ImportFenceFdKHR(VkDevice _device,
7472                                    const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
7473 {
7474         RADV_FROM_HANDLE(radv_device, device, _device);
7475         RADV_FROM_HANDLE(radv_fence, fence, pImportFenceFdInfo->fence);
7476         uint32_t *syncobj_dst = NULL;
7477
7478
7479         if (pImportFenceFdInfo->flags & VK_FENCE_IMPORT_TEMPORARY_BIT) {
7480                 syncobj_dst = &fence->temp_syncobj;
7481         } else {
7482                 syncobj_dst = &fence->syncobj;
7483         }
7484
7485         switch(pImportFenceFdInfo->handleType) {
7486                 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT:
7487                         return radv_import_opaque_fd(device, pImportFenceFdInfo->fd, syncobj_dst);
7488                 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT:
7489                         return radv_import_sync_fd(device, pImportFenceFdInfo->fd, syncobj_dst);
7490                 default:
7491                         unreachable("Unhandled fence handle type");
7492         }
7493 }
7494
7495 VkResult radv_GetFenceFdKHR(VkDevice _device,
7496                                 const VkFenceGetFdInfoKHR *pGetFdInfo,
7497                                 int *pFd)
7498 {
7499         RADV_FROM_HANDLE(radv_device, device, _device);
7500         RADV_FROM_HANDLE(radv_fence, fence, pGetFdInfo->fence);
7501         int ret;
7502         uint32_t syncobj_handle;
7503
7504         if (fence->temp_syncobj)
7505                 syncobj_handle = fence->temp_syncobj;
7506         else
7507                 syncobj_handle = fence->syncobj;
7508
7509         switch(pGetFdInfo->handleType) {
7510         case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT:
7511                 ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd);
7512                 break;
7513         case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT:
7514                 ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd);
7515                 if (!ret) {
7516                         if (fence->temp_syncobj) {
7517                                 close (fence->temp_syncobj);
7518                                 fence->temp_syncobj = 0;
7519                         } else {
7520                                 device->ws->reset_syncobj(device->ws, syncobj_handle);
7521                         }
7522                 }
7523                 break;
7524         default:
7525                 unreachable("Unhandled fence handle type");
7526         }
7527
7528         if (ret)
7529                 return vk_error(device->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7530         return VK_SUCCESS;
7531 }
7532
7533 void radv_GetPhysicalDeviceExternalFenceProperties(
7534         VkPhysicalDevice                            physicalDevice,
7535         const VkPhysicalDeviceExternalFenceInfo *pExternalFenceInfo,
7536         VkExternalFenceProperties               *pExternalFenceProperties)
7537 {
7538         RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
7539
7540         if (pdevice->rad_info.has_syncobj_wait_for_submit &&
7541             (pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT ||
7542              pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT)) {
7543                 pExternalFenceProperties->exportFromImportedHandleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
7544                 pExternalFenceProperties->compatibleHandleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
7545                 pExternalFenceProperties->externalFenceFeatures = VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT |
7546                         VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT;
7547         } else {
7548                 pExternalFenceProperties->exportFromImportedHandleTypes = 0;
7549                 pExternalFenceProperties->compatibleHandleTypes = 0;
7550                 pExternalFenceProperties->externalFenceFeatures = 0;
7551         }
7552 }
7553
7554 VkResult
7555 radv_CreateDebugReportCallbackEXT(VkInstance _instance,
7556                                  const VkDebugReportCallbackCreateInfoEXT* pCreateInfo,
7557                                  const VkAllocationCallbacks* pAllocator,
7558                                  VkDebugReportCallbackEXT* pCallback)
7559 {
7560         RADV_FROM_HANDLE(radv_instance, instance, _instance);
7561         return vk_create_debug_report_callback(&instance->debug_report_callbacks,
7562                                                pCreateInfo, pAllocator, &instance->alloc,
7563                                                pCallback);
7564 }
7565
7566 void
7567 radv_DestroyDebugReportCallbackEXT(VkInstance _instance,
7568                                   VkDebugReportCallbackEXT _callback,
7569                                   const VkAllocationCallbacks* pAllocator)
7570 {
7571         RADV_FROM_HANDLE(radv_instance, instance, _instance);
7572         vk_destroy_debug_report_callback(&instance->debug_report_callbacks,
7573                                          _callback, pAllocator, &instance->alloc);
7574 }
7575
7576 void
7577 radv_DebugReportMessageEXT(VkInstance _instance,
7578                           VkDebugReportFlagsEXT flags,
7579                           VkDebugReportObjectTypeEXT objectType,
7580                           uint64_t object,
7581                           size_t location,
7582                           int32_t messageCode,
7583                           const char* pLayerPrefix,
7584                           const char* pMessage)
7585 {
7586         RADV_FROM_HANDLE(radv_instance, instance, _instance);
7587         vk_debug_report(&instance->debug_report_callbacks, flags, objectType,
7588                         object, location, messageCode, pLayerPrefix, pMessage);
7589 }
7590
7591 void
7592 radv_GetDeviceGroupPeerMemoryFeatures(
7593     VkDevice                                    device,
7594     uint32_t                                    heapIndex,
7595     uint32_t                                    localDeviceIndex,
7596     uint32_t                                    remoteDeviceIndex,
7597     VkPeerMemoryFeatureFlags*                   pPeerMemoryFeatures)
7598 {
7599         assert(localDeviceIndex == remoteDeviceIndex);
7600
7601         *pPeerMemoryFeatures = VK_PEER_MEMORY_FEATURE_COPY_SRC_BIT |
7602                                VK_PEER_MEMORY_FEATURE_COPY_DST_BIT |
7603                                VK_PEER_MEMORY_FEATURE_GENERIC_SRC_BIT |
7604                                VK_PEER_MEMORY_FEATURE_GENERIC_DST_BIT;
7605 }
7606
7607 static const VkTimeDomainEXT radv_time_domains[] = {
7608         VK_TIME_DOMAIN_DEVICE_EXT,
7609         VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT,
7610         VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT,
7611 };
7612
7613 VkResult radv_GetPhysicalDeviceCalibrateableTimeDomainsEXT(
7614         VkPhysicalDevice                             physicalDevice,
7615         uint32_t                                     *pTimeDomainCount,
7616         VkTimeDomainEXT                              *pTimeDomains)
7617 {
7618         int d;
7619         VK_OUTARRAY_MAKE(out, pTimeDomains, pTimeDomainCount);
7620
7621         for (d = 0; d < ARRAY_SIZE(radv_time_domains); d++) {
7622                 vk_outarray_append(&out, i) {
7623                         *i = radv_time_domains[d];
7624                 }
7625         }
7626
7627         return vk_outarray_status(&out);
7628 }
7629
7630 static uint64_t
7631 radv_clock_gettime(clockid_t clock_id)
7632 {
7633         struct timespec current;
7634         int ret;
7635
7636         ret = clock_gettime(clock_id, &current);
7637         if (ret < 0 && clock_id == CLOCK_MONOTONIC_RAW)
7638                 ret = clock_gettime(CLOCK_MONOTONIC, &current);
7639         if (ret < 0)
7640                 return 0;
7641
7642         return (uint64_t) current.tv_sec * 1000000000ULL + current.tv_nsec;
7643 }
7644
7645 VkResult radv_GetCalibratedTimestampsEXT(
7646         VkDevice                                     _device,
7647         uint32_t                                     timestampCount,
7648         const VkCalibratedTimestampInfoEXT           *pTimestampInfos,
7649         uint64_t                                     *pTimestamps,
7650         uint64_t                                     *pMaxDeviation)
7651 {
7652         RADV_FROM_HANDLE(radv_device, device, _device);
7653         uint32_t clock_crystal_freq = device->physical_device->rad_info.clock_crystal_freq;
7654         int d;
7655         uint64_t begin, end;
7656         uint64_t max_clock_period = 0;
7657
7658         begin = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
7659
7660         for (d = 0; d < timestampCount; d++) {
7661                 switch (pTimestampInfos[d].timeDomain) {
7662                 case VK_TIME_DOMAIN_DEVICE_EXT:
7663                         pTimestamps[d] = device->ws->query_value(device->ws,
7664                                                                  RADEON_TIMESTAMP);
7665                         uint64_t device_period = DIV_ROUND_UP(1000000, clock_crystal_freq);
7666                         max_clock_period = MAX2(max_clock_period, device_period);
7667                         break;
7668                 case VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT:
7669                         pTimestamps[d] = radv_clock_gettime(CLOCK_MONOTONIC);
7670                         max_clock_period = MAX2(max_clock_period, 1);
7671                         break;
7672
7673                 case VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT:
7674                         pTimestamps[d] = begin;
7675                         break;
7676                 default:
7677                         pTimestamps[d] = 0;
7678                         break;
7679                 }
7680         }
7681
7682         end = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
7683
7684         /*
7685          * The maximum deviation is the sum of the interval over which we
7686          * perform the sampling and the maximum period of any sampled
7687          * clock. That's because the maximum skew between any two sampled
7688          * clock edges is when the sampled clock with the largest period is
7689          * sampled at the end of that period but right at the beginning of the
7690          * sampling interval and some other clock is sampled right at the
7691          * begining of its sampling period and right at the end of the
7692          * sampling interval. Let's assume the GPU has the longest clock
7693          * period and that the application is sampling GPU and monotonic:
7694          *
7695          *                               s                 e
7696          *                       w x y z 0 1 2 3 4 5 6 7 8 9 a b c d e f
7697          *      Raw              -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-
7698          *
7699          *                               g
7700          *                0         1         2         3
7701          *      GPU       -----_____-----_____-----_____-----_____
7702          *
7703          *                                                m
7704          *                                          x y z 0 1 2 3 4 5 6 7 8 9 a b c
7705          *      Monotonic                           -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-
7706          *
7707          *      Interval                     <----------------->
7708          *      Deviation           <-------------------------->
7709          *
7710          *              s  = read(raw)       2
7711          *              g  = read(GPU)       1
7712          *              m  = read(monotonic) 2
7713          *              e  = read(raw)       b
7714          *
7715          * We round the sample interval up by one tick to cover sampling error
7716          * in the interval clock
7717          */
7718
7719         uint64_t sample_interval = end - begin + 1;
7720
7721         *pMaxDeviation = sample_interval + max_clock_period;
7722
7723         return VK_SUCCESS;
7724 }
7725
7726 void radv_GetPhysicalDeviceMultisamplePropertiesEXT(
7727     VkPhysicalDevice                            physicalDevice,
7728     VkSampleCountFlagBits                       samples,
7729     VkMultisamplePropertiesEXT*                 pMultisampleProperties)
7730 {
7731         if (samples & (VK_SAMPLE_COUNT_2_BIT |
7732                        VK_SAMPLE_COUNT_4_BIT |
7733                        VK_SAMPLE_COUNT_8_BIT)) {
7734                 pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 2, 2 };
7735         } else {
7736                 pMultisampleProperties->maxSampleLocationGridSize = (VkExtent2D){ 0, 0 };
7737         }
7738 }