projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radv: Increase api version to 1.0.42.
[mesa.git] / src / amd / vulkan / radv_device.c
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * IN THE SOFTWARE.
26 */
27
28 #include <stdbool.h>
29 #include <string.h>
30 #include <unistd.h>
31 #include <fcntl.h>
32 #include "radv_private.h"
33 #include "radv_cs.h"
34 #include "util/disk_cache.h"
35 #include "util/strtod.h"
36 #include "util/vk_util.h"
37 #include <xf86drm.h>
38 #include <amdgpu.h>
39 #include <amdgpu_drm.h>
40 #include "amdgpu_id.h"
41 #include "winsys/amdgpu/radv_amdgpu_winsys_public.h"
42 #include "ac_llvm_util.h"
43 #include "vk_format.h"
44 #include "sid.h"
45 #include "util/debug.h"
46
47 static int
48 radv_device_get_cache_uuid(enum radeon_family family, void *uuid)
49 {
50 uint32_t mesa_timestamp, llvm_timestamp;
51 uint16_t f = family;
52 memset(uuid, 0, VK_UUID_SIZE);
53 if (!disk_cache_get_function_timestamp(radv_device_get_cache_uuid, &mesa_timestamp) ||
54 !disk_cache_get_function_timestamp(LLVMInitializeAMDGPUTargetInfo, &llvm_timestamp))
55 return -1;
56
57 memcpy(uuid, &mesa_timestamp, 4);
58 memcpy((char*)uuid + 4, &llvm_timestamp, 4);
59 memcpy((char*)uuid + 8, &f, 2);
60 snprintf((char*)uuid + 10, VK_UUID_SIZE - 10, "radv");
61 return 0;
62 }
63
64 static const VkExtensionProperties instance_extensions[] = {
65 {
66 .extensionName = VK_KHR_SURFACE_EXTENSION_NAME,
67 .specVersion = 25,
68 },
69 #ifdef VK_USE_PLATFORM_XCB_KHR
70 {
71 .extensionName = VK_KHR_XCB_SURFACE_EXTENSION_NAME,
72 .specVersion = 6,
73 },
74 #endif
75 #ifdef VK_USE_PLATFORM_XLIB_KHR
76 {
77 .extensionName = VK_KHR_XLIB_SURFACE_EXTENSION_NAME,
78 .specVersion = 6,
79 },
80 #endif
81 #ifdef VK_USE_PLATFORM_WAYLAND_KHR
82 {
83 .extensionName = VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME,
84 .specVersion = 5,
85 },
86 #endif
87 };
88
89 static const VkExtensionProperties common_device_extensions[] = {
90 {
91 .extensionName = VK_KHR_MAINTENANCE1_EXTENSION_NAME,
92 .specVersion = 1,
93 },
94 {
95 .extensionName = VK_KHR_SAMPLER_MIRROR_CLAMP_TO_EDGE_EXTENSION_NAME,
96 .specVersion = 1,
97 },
98 {
99 .extensionName = VK_KHR_SWAPCHAIN_EXTENSION_NAME,
100 .specVersion = 68,
101 },
102 {
103 .extensionName = VK_AMD_DRAW_INDIRECT_COUNT_EXTENSION_NAME,
104 .specVersion = 1,
105 },
106 {
107 .extensionName = VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME,
108 .specVersion = 1,
109 },
110 {
111 .extensionName = VK_KHR_SHADER_DRAW_PARAMETERS_EXTENSION_NAME,
112 .specVersion = 1,
113 },
114 {
115 .extensionName = VK_NV_DEDICATED_ALLOCATION_EXTENSION_NAME,
116 .specVersion = 1,
117 },
118 };
119
120 static VkResult
121 radv_extensions_register(struct radv_instance *instance,
122 struct radv_extensions *extensions,
123 const VkExtensionProperties *new_ext,
124 uint32_t num_ext)
125 {
126 size_t new_size;
127 VkExtensionProperties *new_ptr;
128
129 assert(new_ext && num_ext > 0);
130
131 if (!new_ext)
132 return VK_ERROR_INITIALIZATION_FAILED;
133
134 new_size = (extensions->num_ext + num_ext) * sizeof(VkExtensionProperties);
135 new_ptr = vk_realloc(&instance->alloc, extensions->ext_array,
136 new_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
137
138 /* Old array continues to be valid, update nothing */
139 if (!new_ptr)
140 return VK_ERROR_OUT_OF_HOST_MEMORY;
141
142 memcpy(&new_ptr[extensions->num_ext], new_ext,
143 num_ext * sizeof(VkExtensionProperties));
144 extensions->ext_array = new_ptr;
145 extensions->num_ext += num_ext;
146
147 return VK_SUCCESS;
148 }
149
150 static void
151 radv_extensions_finish(struct radv_instance *instance,
152 struct radv_extensions *extensions)
153 {
154 assert(extensions);
155
156 if (!extensions)
157 radv_loge("Attemted to free invalid extension struct\n");
158
159 if (extensions->ext_array)
160 vk_free(&instance->alloc, extensions->ext_array);
161 }
162
163 static bool
164 is_extension_enabled(const VkExtensionProperties *extensions,
165 size_t num_ext,
166 const char *name)
167 {
168 assert(extensions && name);
169
170 for (uint32_t i = 0; i < num_ext; i++) {
171 if (strcmp(name, extensions[i].extensionName) == 0)
172 return true;
173 }
174
175 return false;
176 }
177
178 static VkResult
179 radv_physical_device_init(struct radv_physical_device *device,
180 struct radv_instance *instance,
181 const char *path)
182 {
183 VkResult result;
184 drmVersionPtr version;
185 int fd;
186
187 fd = open(path, O_RDWR | O_CLOEXEC);
188 if (fd < 0)
189 return VK_ERROR_INCOMPATIBLE_DRIVER;
190
191 version = drmGetVersion(fd);
192 if (!version) {
193 close(fd);
194 return vk_errorf(VK_ERROR_INCOMPATIBLE_DRIVER,
195 "failed to get version %s: %m", path);
196 }
197
198 if (strcmp(version->name, "amdgpu")) {
199 drmFreeVersion(version);
200 close(fd);
201 return VK_ERROR_INCOMPATIBLE_DRIVER;
202 }
203 drmFreeVersion(version);
204
205 device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
206 device->instance = instance;
207 assert(strlen(path) < ARRAY_SIZE(device->path));
208 strncpy(device->path, path, ARRAY_SIZE(device->path));
209
210 device->ws = radv_amdgpu_winsys_create(fd, instance->debug_flags);
211 if (!device->ws) {
212 result = VK_ERROR_INCOMPATIBLE_DRIVER;
213 goto fail;
214 }
215
216 device->local_fd = fd;
217 device->ws->query_info(device->ws, &device->rad_info);
218 result = radv_init_wsi(device);
219 if (result != VK_SUCCESS) {
220 device->ws->destroy(device->ws);
221 goto fail;
222 }
223
224 if (radv_device_get_cache_uuid(device->rad_info.family, device->uuid)) {
225 radv_finish_wsi(device);
226 device->ws->destroy(device->ws);
227 result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
228 "cannot generate UUID");
229 goto fail;
230 }
231
232 result = radv_extensions_register(instance,
233 &device->extensions,
234 common_device_extensions,
235 ARRAY_SIZE(common_device_extensions));
236 if (result != VK_SUCCESS)
237 goto fail;
238
239 fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n");
240 device->name = device->rad_info.name;
241
242 return VK_SUCCESS;
243
244 fail:
245 close(fd);
246 return result;
247 }
248
249 static void
250 radv_physical_device_finish(struct radv_physical_device *device)
251 {
252 radv_extensions_finish(device->instance, &device->extensions);
253 radv_finish_wsi(device);
254 device->ws->destroy(device->ws);
255 close(device->local_fd);
256 }
257
258
259 static void *
260 default_alloc_func(void *pUserData, size_t size, size_t align,
261 VkSystemAllocationScope allocationScope)
262 {
263 return malloc(size);
264 }
265
266 static void *
267 default_realloc_func(void *pUserData, void *pOriginal, size_t size,
268 size_t align, VkSystemAllocationScope allocationScope)
269 {
270 return realloc(pOriginal, size);
271 }
272
273 static void
274 default_free_func(void *pUserData, void *pMemory)
275 {
276 free(pMemory);
277 }
278
279 static const VkAllocationCallbacks default_alloc = {
280 .pUserData = NULL,
281 .pfnAllocation = default_alloc_func,
282 .pfnReallocation = default_realloc_func,
283 .pfnFree = default_free_func,
284 };
285
286 static const struct debug_control radv_debug_options[] = {
287 {"nofastclears", RADV_DEBUG_NO_FAST_CLEARS},
288 {"nodcc", RADV_DEBUG_NO_DCC},
289 {"shaders", RADV_DEBUG_DUMP_SHADERS},
290 {"nocache", RADV_DEBUG_NO_CACHE},
291 {"shaderstats", RADV_DEBUG_DUMP_SHADER_STATS},
292 {"nohiz", RADV_DEBUG_NO_HIZ},
293 {"nocompute", RADV_DEBUG_NO_COMPUTE_QUEUE},
294 {"unsafemath", RADV_DEBUG_UNSAFE_MATH},
295 {"allbos", RADV_DEBUG_ALL_BOS},
296 {"noibs", RADV_DEBUG_NO_IBS},
297 {NULL, 0}
298 };
299
300 VkResult radv_CreateInstance(
301 const VkInstanceCreateInfo* pCreateInfo,
302 const VkAllocationCallbacks* pAllocator,
303 VkInstance* pInstance)
304 {
305 struct radv_instance *instance;
306
307 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO);
308
309 uint32_t client_version;
310 if (pCreateInfo->pApplicationInfo &&
311 pCreateInfo->pApplicationInfo->apiVersion != 0) {
312 client_version = pCreateInfo->pApplicationInfo->apiVersion;
313 } else {
314 client_version = VK_MAKE_VERSION(1, 0, 0);
315 }
316
317 if (VK_MAKE_VERSION(1, 0, 0) > client_version ||
318 client_version > VK_MAKE_VERSION(1, 0, 0xfff)) {
319 return vk_errorf(VK_ERROR_INCOMPATIBLE_DRIVER,
320 "Client requested version %d.%d.%d",
321 VK_VERSION_MAJOR(client_version),
322 VK_VERSION_MINOR(client_version),
323 VK_VERSION_PATCH(client_version));
324 }
325
326 for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
327 if (!is_extension_enabled(instance_extensions,
328 ARRAY_SIZE(instance_extensions),
329 pCreateInfo->ppEnabledExtensionNames[i]))
330 return vk_error(VK_ERROR_EXTENSION_NOT_PRESENT);
331 }
332
333 instance = vk_alloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
334 VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
335 if (!instance)
336 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
337
338 memset(instance, 0, sizeof(*instance));
339
340 instance->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
341
342 if (pAllocator)
343 instance->alloc = *pAllocator;
344 else
345 instance->alloc = default_alloc;
346
347 instance->apiVersion = client_version;
348 instance->physicalDeviceCount = -1;
349
350 _mesa_locale_init();
351
352 VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false));
353
354 instance->debug_flags = parse_debug_string(getenv("RADV_DEBUG"),
355 radv_debug_options);
356
357 *pInstance = radv_instance_to_handle(instance);
358
359 return VK_SUCCESS;
360 }
361
362 void radv_DestroyInstance(
363 VkInstance _instance,
364 const VkAllocationCallbacks* pAllocator)
365 {
366 RADV_FROM_HANDLE(radv_instance, instance, _instance);
367
368 if (!instance)
369 return;
370
371 for (int i = 0; i < instance->physicalDeviceCount; ++i) {
372 radv_physical_device_finish(instance->physicalDevices + i);
373 }
374
375 VG(VALGRIND_DESTROY_MEMPOOL(instance));
376
377 _mesa_locale_fini();
378
379 vk_free(&instance->alloc, instance);
380 }
381
382 VkResult radv_EnumeratePhysicalDevices(
383 VkInstance _instance,
384 uint32_t* pPhysicalDeviceCount,
385 VkPhysicalDevice* pPhysicalDevices)
386 {
387 RADV_FROM_HANDLE(radv_instance, instance, _instance);
388 VkResult result;
389
390 if (instance->physicalDeviceCount < 0) {
391 char path[20];
392 instance->physicalDeviceCount = 0;
393 for (unsigned i = 0; i < RADV_MAX_DRM_DEVICES; i++) {
394 snprintf(path, sizeof(path), "/dev/dri/renderD%d", 128 + i);
395 result = radv_physical_device_init(instance->physicalDevices +
396 instance->physicalDeviceCount,
397 instance, path);
398 if (result == VK_SUCCESS)
399 ++instance->physicalDeviceCount;
400 else if (result != VK_ERROR_INCOMPATIBLE_DRIVER)
401 return result;
402 }
403 }
404
405 if (!pPhysicalDevices) {
406 *pPhysicalDeviceCount = instance->physicalDeviceCount;
407 } else {
408 *pPhysicalDeviceCount = MIN2(*pPhysicalDeviceCount, instance->physicalDeviceCount);
409 for (unsigned i = 0; i < *pPhysicalDeviceCount; ++i)
410 pPhysicalDevices[i] = radv_physical_device_to_handle(instance->physicalDevices + i);
411 }
412
413 return *pPhysicalDeviceCount < instance->physicalDeviceCount ? VK_INCOMPLETE
414 : VK_SUCCESS;
415 }
416
417 void radv_GetPhysicalDeviceFeatures(
418 VkPhysicalDevice physicalDevice,
419 VkPhysicalDeviceFeatures* pFeatures)
420 {
421 // RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
422
423 memset(pFeatures, 0, sizeof(*pFeatures));
424
425 *pFeatures = (VkPhysicalDeviceFeatures) {
426 .robustBufferAccess = true,
427 .fullDrawIndexUint32 = true,
428 .imageCubeArray = true,
429 .independentBlend = true,
430 .geometryShader = true,
431 .tessellationShader = false,
432 .sampleRateShading = false,
433 .dualSrcBlend = true,
434 .logicOp = true,
435 .multiDrawIndirect = true,
436 .drawIndirectFirstInstance = true,
437 .depthClamp = true,
438 .depthBiasClamp = true,
439 .fillModeNonSolid = true,
440 .depthBounds = true,
441 .wideLines = true,
442 .largePoints = true,
443 .alphaToOne = true,
444 .multiViewport = true,
445 .samplerAnisotropy = true,
446 .textureCompressionETC2 = false,
447 .textureCompressionASTC_LDR = false,
448 .textureCompressionBC = true,
449 .occlusionQueryPrecise = true,
450 .pipelineStatisticsQuery = false,
451 .vertexPipelineStoresAndAtomics = true,
452 .fragmentStoresAndAtomics = true,
453 .shaderTessellationAndGeometryPointSize = true,
454 .shaderImageGatherExtended = true,
455 .shaderStorageImageExtendedFormats = true,
456 .shaderStorageImageMultisample = false,
457 .shaderUniformBufferArrayDynamicIndexing = true,
458 .shaderSampledImageArrayDynamicIndexing = true,
459 .shaderStorageBufferArrayDynamicIndexing = true,
460 .shaderStorageImageArrayDynamicIndexing = true,
461 .shaderStorageImageReadWithoutFormat = true,
462 .shaderStorageImageWriteWithoutFormat = true,
463 .shaderClipDistance = true,
464 .shaderCullDistance = true,
465 .shaderFloat64 = true,
466 .shaderInt64 = false,
467 .shaderInt16 = false,
468 .variableMultisampleRate = false,
469 .inheritedQueries = false,
470 };
471 }
472
473 void radv_GetPhysicalDeviceFeatures2KHR(
474 VkPhysicalDevice physicalDevice,
475 VkPhysicalDeviceFeatures2KHR *pFeatures)
476 {
477 return radv_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features);
478 }
479
480 void radv_GetPhysicalDeviceProperties(
481 VkPhysicalDevice physicalDevice,
482 VkPhysicalDeviceProperties* pProperties)
483 {
484 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
485 VkSampleCountFlags sample_counts = 0xf;
486 VkPhysicalDeviceLimits limits = {
487 .maxImageDimension1D = (1 << 14),
488 .maxImageDimension2D = (1 << 14),
489 .maxImageDimension3D = (1 << 11),
490 .maxImageDimensionCube = (1 << 14),
491 .maxImageArrayLayers = (1 << 11),
492 .maxTexelBufferElements = 128 * 1024 * 1024,
493 .maxUniformBufferRange = UINT32_MAX,
494 .maxStorageBufferRange = UINT32_MAX,
495 .maxPushConstantsSize = MAX_PUSH_CONSTANTS_SIZE,
496 .maxMemoryAllocationCount = UINT32_MAX,
497 .maxSamplerAllocationCount = 64 * 1024,
498 .bufferImageGranularity = 64, /* A cache line */
499 .sparseAddressSpaceSize = 0,
500 .maxBoundDescriptorSets = MAX_SETS,
501 .maxPerStageDescriptorSamplers = 64,
502 .maxPerStageDescriptorUniformBuffers = 64,
503 .maxPerStageDescriptorStorageBuffers = 64,
504 .maxPerStageDescriptorSampledImages = 64,
505 .maxPerStageDescriptorStorageImages = 64,
506 .maxPerStageDescriptorInputAttachments = 64,
507 .maxPerStageResources = 128,
508 .maxDescriptorSetSamplers = 256,
509 .maxDescriptorSetUniformBuffers = 256,
510 .maxDescriptorSetUniformBuffersDynamic = 256,
511 .maxDescriptorSetStorageBuffers = 256,
512 .maxDescriptorSetStorageBuffersDynamic = 256,
513 .maxDescriptorSetSampledImages = 256,
514 .maxDescriptorSetStorageImages = 256,
515 .maxDescriptorSetInputAttachments = 256,
516 .maxVertexInputAttributes = 32,
517 .maxVertexInputBindings = 32,
518 .maxVertexInputAttributeOffset = 2047,
519 .maxVertexInputBindingStride = 2048,
520 .maxVertexOutputComponents = 128,
521 .maxTessellationGenerationLevel = 0,
522 .maxTessellationPatchSize = 0,
523 .maxTessellationControlPerVertexInputComponents = 0,
524 .maxTessellationControlPerVertexOutputComponents = 0,
525 .maxTessellationControlPerPatchOutputComponents = 0,
526 .maxTessellationControlTotalOutputComponents = 0,
527 .maxTessellationEvaluationInputComponents = 0,
528 .maxTessellationEvaluationOutputComponents = 0,
529 .maxGeometryShaderInvocations = 32,
530 .maxGeometryInputComponents = 64,
531 .maxGeometryOutputComponents = 128,
532 .maxGeometryOutputVertices = 256,
533 .maxGeometryTotalOutputComponents = 1024,
534 .maxFragmentInputComponents = 128,
535 .maxFragmentOutputAttachments = 8,
536 .maxFragmentDualSrcAttachments = 1,
537 .maxFragmentCombinedOutputResources = 8,
538 .maxComputeSharedMemorySize = 32768,
539 .maxComputeWorkGroupCount = { 65535, 65535, 65535 },
540 .maxComputeWorkGroupInvocations = 2048,
541 .maxComputeWorkGroupSize = {
542 2048,
543 2048,
544 2048
545 },
546 .subPixelPrecisionBits = 4 /* FIXME */,
547 .subTexelPrecisionBits = 4 /* FIXME */,
548 .mipmapPrecisionBits = 4 /* FIXME */,
549 .maxDrawIndexedIndexValue = UINT32_MAX,
550 .maxDrawIndirectCount = UINT32_MAX,
551 .maxSamplerLodBias = 16,
552 .maxSamplerAnisotropy = 16,
553 .maxViewports = MAX_VIEWPORTS,
554 .maxViewportDimensions = { (1 << 14), (1 << 14) },
555 .viewportBoundsRange = { INT16_MIN, INT16_MAX },
556 .viewportSubPixelBits = 13, /* We take a float? */
557 .minMemoryMapAlignment = 4096, /* A page */
558 .minTexelBufferOffsetAlignment = 1,
559 .minUniformBufferOffsetAlignment = 4,
560 .minStorageBufferOffsetAlignment = 4,
561 .minTexelOffset = -32,
562 .maxTexelOffset = 31,
563 .minTexelGatherOffset = -32,
564 .maxTexelGatherOffset = 31,
565 .minInterpolationOffset = -2,
566 .maxInterpolationOffset = 2,
567 .subPixelInterpolationOffsetBits = 8,
568 .maxFramebufferWidth = (1 << 14),
569 .maxFramebufferHeight = (1 << 14),
570 .maxFramebufferLayers = (1 << 10),
571 .framebufferColorSampleCounts = sample_counts,
572 .framebufferDepthSampleCounts = sample_counts,
573 .framebufferStencilSampleCounts = sample_counts,
574 .framebufferNoAttachmentsSampleCounts = sample_counts,
575 .maxColorAttachments = MAX_RTS,
576 .sampledImageColorSampleCounts = sample_counts,
577 .sampledImageIntegerSampleCounts = VK_SAMPLE_COUNT_1_BIT,
578 .sampledImageDepthSampleCounts = sample_counts,
579 .sampledImageStencilSampleCounts = sample_counts,
580 .storageImageSampleCounts = VK_SAMPLE_COUNT_1_BIT,
581 .maxSampleMaskWords = 1,
582 .timestampComputeAndGraphics = false,
583 .timestampPeriod = 100000.0 / pdevice->rad_info.clock_crystal_freq,
584 .maxClipDistances = 8,
585 .maxCullDistances = 8,
586 .maxCombinedClipAndCullDistances = 8,
587 .discreteQueuePriorities = 1,
588 .pointSizeRange = { 0.125, 255.875 },
589 .lineWidthRange = { 0.0, 7.9921875 },
590 .pointSizeGranularity = (1.0 / 8.0),
591 .lineWidthGranularity = (1.0 / 128.0),
592 .strictLines = false, /* FINISHME */
593 .standardSampleLocations = true,
594 .optimalBufferCopyOffsetAlignment = 128,
595 .optimalBufferCopyRowPitchAlignment = 128,
596 .nonCoherentAtomSize = 64,
597 };
598
599 *pProperties = (VkPhysicalDeviceProperties) {
600 .apiVersion = VK_MAKE_VERSION(1, 0, 42),
601 .driverVersion = 1,
602 .vendorID = 0x1002,
603 .deviceID = pdevice->rad_info.pci_id,
604 .deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU,
605 .limits = limits,
606 .sparseProperties = {0}, /* Broadwell doesn't do sparse. */
607 };
608
609 strcpy(pProperties->deviceName, pdevice->name);
610 memcpy(pProperties->pipelineCacheUUID, pdevice->uuid, VK_UUID_SIZE);
611 }
612
613 void radv_GetPhysicalDeviceProperties2KHR(
614 VkPhysicalDevice physicalDevice,
615 VkPhysicalDeviceProperties2KHR *pProperties)
616 {
617 return radv_GetPhysicalDeviceProperties(physicalDevice, &pProperties->properties);
618 }
619
620 static void radv_get_physical_device_queue_family_properties(
621 struct radv_physical_device* pdevice,
622 uint32_t* pCount,
623 VkQueueFamilyProperties** pQueueFamilyProperties)
624 {
625 int num_queue_families = 1;
626 int idx;
627 if (pdevice->rad_info.compute_rings > 0 &&
628 pdevice->rad_info.chip_class >= CIK &&
629 !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE))
630 num_queue_families++;
631
632 if (pQueueFamilyProperties == NULL) {
633 *pCount = num_queue_families;
634 return;
635 }
636
637 if (!*pCount)
638 return;
639
640 idx = 0;
641 if (*pCount >= 1) {
642 *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
643 .queueFlags = VK_QUEUE_GRAPHICS_BIT |
644 VK_QUEUE_COMPUTE_BIT |
645 VK_QUEUE_TRANSFER_BIT,
646 .queueCount = 1,
647 .timestampValidBits = 64,
648 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
649 };
650 idx++;
651 }
652
653 if (pdevice->rad_info.compute_rings > 0 &&
654 pdevice->rad_info.chip_class >= CIK &&
655 !(pdevice->instance->debug_flags & RADV_DEBUG_NO_COMPUTE_QUEUE)) {
656 if (*pCount > idx) {
657 *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
658 .queueFlags = VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT,
659 .queueCount = pdevice->rad_info.compute_rings,
660 .timestampValidBits = 64,
661 .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
662 };
663 idx++;
664 }
665 }
666 *pCount = idx;
667 }
668
669 void radv_GetPhysicalDeviceQueueFamilyProperties(
670 VkPhysicalDevice physicalDevice,
671 uint32_t* pCount,
672 VkQueueFamilyProperties* pQueueFamilyProperties)
673 {
674 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
675 if (!pQueueFamilyProperties) {
676 return radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
677 return;
678 }
679 VkQueueFamilyProperties *properties[] = {
680 pQueueFamilyProperties + 0,
681 pQueueFamilyProperties + 1,
682 pQueueFamilyProperties + 2,
683 };
684 radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
685 assert(*pCount <= 3);
686 }
687
688 void radv_GetPhysicalDeviceQueueFamilyProperties2KHR(
689 VkPhysicalDevice physicalDevice,
690 uint32_t* pCount,
691 VkQueueFamilyProperties2KHR *pQueueFamilyProperties)
692 {
693 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
694 if (!pQueueFamilyProperties) {
695 return radv_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
696 return;
697 }
698 VkQueueFamilyProperties *properties[] = {
699 &pQueueFamilyProperties[0].queueFamilyProperties,
700 &pQueueFamilyProperties[1].queueFamilyProperties,
701 &pQueueFamilyProperties[2].queueFamilyProperties,
702 };
703 radv_get_physical_device_queue_family_properties(pdevice, pCount, properties);
704 assert(*pCount <= 3);
705 }
706
707 void radv_GetPhysicalDeviceMemoryProperties(
708 VkPhysicalDevice physicalDevice,
709 VkPhysicalDeviceMemoryProperties *pMemoryProperties)
710 {
711 RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
712
713 STATIC_ASSERT(RADV_MEM_TYPE_COUNT <= VK_MAX_MEMORY_TYPES);
714
715 pMemoryProperties->memoryTypeCount = RADV_MEM_TYPE_COUNT;
716 pMemoryProperties->memoryTypes[RADV_MEM_TYPE_VRAM] = (VkMemoryType) {
717 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
718 .heapIndex = RADV_MEM_HEAP_VRAM,
719 };
720 pMemoryProperties->memoryTypes[RADV_MEM_TYPE_GTT_WRITE_COMBINE] = (VkMemoryType) {
721 .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
722 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
723 .heapIndex = RADV_MEM_HEAP_GTT,
724 };
725 pMemoryProperties->memoryTypes[RADV_MEM_TYPE_VRAM_CPU_ACCESS] = (VkMemoryType) {
726 .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
727 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
728 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
729 .heapIndex = RADV_MEM_HEAP_VRAM_CPU_ACCESS,
730 };
731 pMemoryProperties->memoryTypes[RADV_MEM_TYPE_GTT_CACHED] = (VkMemoryType) {
732 .propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
733 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
734 VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
735 .heapIndex = RADV_MEM_HEAP_GTT,
736 };
737
738 STATIC_ASSERT(RADV_MEM_HEAP_COUNT <= VK_MAX_MEMORY_HEAPS);
739
740 pMemoryProperties->memoryHeapCount = RADV_MEM_HEAP_COUNT;
741 pMemoryProperties->memoryHeaps[RADV_MEM_HEAP_VRAM] = (VkMemoryHeap) {
742 .size = physical_device->rad_info.vram_size -
743 physical_device->rad_info.visible_vram_size,
744 .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
745 };
746 pMemoryProperties->memoryHeaps[RADV_MEM_HEAP_VRAM_CPU_ACCESS] = (VkMemoryHeap) {
747 .size = physical_device->rad_info.visible_vram_size,
748 .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
749 };
750 pMemoryProperties->memoryHeaps[RADV_MEM_HEAP_GTT] = (VkMemoryHeap) {
751 .size = physical_device->rad_info.gart_size,
752 .flags = 0,
753 };
754 }
755
756 void radv_GetPhysicalDeviceMemoryProperties2KHR(
757 VkPhysicalDevice physicalDevice,
758 VkPhysicalDeviceMemoryProperties2KHR *pMemoryProperties)
759 {
760 return radv_GetPhysicalDeviceMemoryProperties(physicalDevice,
761 &pMemoryProperties->memoryProperties);
762 }
763
764 static int
765 radv_queue_init(struct radv_device *device, struct radv_queue *queue,
766 int queue_family_index, int idx)
767 {
768 queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
769 queue->device = device;
770 queue->queue_family_index = queue_family_index;
771 queue->queue_idx = idx;
772
773 queue->hw_ctx = device->ws->ctx_create(device->ws);
774 if (!queue->hw_ctx)
775 return VK_ERROR_OUT_OF_HOST_MEMORY;
776
777 return VK_SUCCESS;
778 }
779
780 static void
781 radv_queue_finish(struct radv_queue *queue)
782 {
783 if (queue->hw_ctx)
784 queue->device->ws->ctx_destroy(queue->hw_ctx);
785
786 if (queue->initial_preamble_cs)
787 queue->device->ws->cs_destroy(queue->initial_preamble_cs);
788 if (queue->continue_preamble_cs)
789 queue->device->ws->cs_destroy(queue->continue_preamble_cs);
790 if (queue->descriptor_bo)
791 queue->device->ws->buffer_destroy(queue->descriptor_bo);
792 if (queue->scratch_bo)
793 queue->device->ws->buffer_destroy(queue->scratch_bo);
794 if (queue->esgs_ring_bo)
795 queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
796 if (queue->gsvs_ring_bo)
797 queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
798 if (queue->compute_scratch_bo)
799 queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
800 }
801
802 static void
803 radv_device_init_gs_info(struct radv_device *device)
804 {
805 switch (device->physical_device->rad_info.family) {
806 case CHIP_OLAND:
807 case CHIP_HAINAN:
808 case CHIP_KAVERI:
809 case CHIP_KABINI:
810 case CHIP_MULLINS:
811 case CHIP_ICELAND:
812 case CHIP_CARRIZO:
813 case CHIP_STONEY:
814 device->gs_table_depth = 16;
815 return;
816 case CHIP_TAHITI:
817 case CHIP_PITCAIRN:
818 case CHIP_VERDE:
819 case CHIP_BONAIRE:
820 case CHIP_HAWAII:
821 case CHIP_TONGA:
822 case CHIP_FIJI:
823 case CHIP_POLARIS10:
824 case CHIP_POLARIS11:
825 device->gs_table_depth = 32;
826 return;
827 default:
828 unreachable("unknown GPU");
829 }
830 }
831
832 VkResult radv_CreateDevice(
833 VkPhysicalDevice physicalDevice,
834 const VkDeviceCreateInfo* pCreateInfo,
835 const VkAllocationCallbacks* pAllocator,
836 VkDevice* pDevice)
837 {
838 RADV_FROM_HANDLE(radv_physical_device, physical_device, physicalDevice);
839 VkResult result;
840 struct radv_device *device;
841
842 for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
843 if (!is_extension_enabled(physical_device->extensions.ext_array,
844 physical_device->extensions.num_ext,
845 pCreateInfo->ppEnabledExtensionNames[i]))
846 return vk_error(VK_ERROR_EXTENSION_NOT_PRESENT);
847 }
848
849 device = vk_alloc2(&physical_device->instance->alloc, pAllocator,
850 sizeof(*device), 8,
851 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
852 if (!device)
853 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
854
855 memset(device, 0, sizeof(*device));
856
857 device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
858 device->instance = physical_device->instance;
859 device->physical_device = physical_device;
860
861 device->debug_flags = device->instance->debug_flags;
862
863 device->ws = physical_device->ws;
864 if (pAllocator)
865 device->alloc = *pAllocator;
866 else
867 device->alloc = physical_device->instance->alloc;
868
869 for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
870 const VkDeviceQueueCreateInfo *queue_create = &pCreateInfo->pQueueCreateInfos[i];
871 uint32_t qfi = queue_create->queueFamilyIndex;
872
873 device->queues[qfi] = vk_alloc(&device->alloc,
874 queue_create->queueCount * sizeof(struct radv_queue), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
875 if (!device->queues[qfi]) {
876 result = VK_ERROR_OUT_OF_HOST_MEMORY;
877 goto fail;
878 }
879
880 memset(device->queues[qfi], 0, queue_create->queueCount * sizeof(struct radv_queue));
881
882 device->queue_count[qfi] = queue_create->queueCount;
883
884 for (unsigned q = 0; q < queue_create->queueCount; q++) {
885 result = radv_queue_init(device, &device->queues[qfi][q], qfi, q);
886 if (result != VK_SUCCESS)
887 goto fail;
888 }
889 }
890
891 #if HAVE_LLVM < 0x0400
892 device->llvm_supports_spill = false;
893 #else
894 device->llvm_supports_spill = true;
895 #endif
896
897 /* The maximum number of scratch waves. Scratch space isn't divided
898 * evenly between CUs. The number is only a function of the number of CUs.
899 * We can decrease the constant to decrease the scratch buffer size.
900 *
901 * sctx->scratch_waves must be >= the maximum posible size of
902 * 1 threadgroup, so that the hw doesn't hang from being unable
903 * to start any.
904 *
905 * The recommended value is 4 per CU at most. Higher numbers don't
906 * bring much benefit, but they still occupy chip resources (think
907 * async compute). I've seen ~2% performance difference between 4 and 32.
908 */
909 uint32_t max_threads_per_block = 2048;
910 device->scratch_waves = MAX2(32 * physical_device->rad_info.num_good_compute_units,
911 max_threads_per_block / 64);
912
913 radv_device_init_gs_info(device);
914
915 result = radv_device_init_meta(device);
916 if (result != VK_SUCCESS)
917 goto fail;
918
919 radv_device_init_msaa(device);
920
921 for (int family = 0; family < RADV_MAX_QUEUE_FAMILIES; ++family) {
922 device->empty_cs[family] = device->ws->cs_create(device->ws, family);
923 switch (family) {
924 case RADV_QUEUE_GENERAL:
925 radeon_emit(device->empty_cs[family], PKT3(PKT3_CONTEXT_CONTROL, 1, 0));
926 radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_LOAD_ENABLE(1));
927 radeon_emit(device->empty_cs[family], CONTEXT_CONTROL_SHADOW_ENABLE(1));
928 break;
929 case RADV_QUEUE_COMPUTE:
930 radeon_emit(device->empty_cs[family], PKT3(PKT3_NOP, 0, 0));
931 radeon_emit(device->empty_cs[family], 0);
932 break;
933 }
934 device->ws->cs_finalize(device->empty_cs[family]);
935
936 device->flush_cs[family] = device->ws->cs_create(device->ws, family);
937 switch (family) {
938 case RADV_QUEUE_GENERAL:
939 case RADV_QUEUE_COMPUTE:
940 si_cs_emit_cache_flush(device->flush_cs[family],
941 device->physical_device->rad_info.chip_class,
942 family == RADV_QUEUE_COMPUTE && device->physical_device->rad_info.chip_class >= CIK,
943 RADV_CMD_FLAG_INV_ICACHE |
944 RADV_CMD_FLAG_INV_SMEM_L1 |
945 RADV_CMD_FLAG_INV_VMEM_L1 |
946 RADV_CMD_FLAG_INV_GLOBAL_L2);
947 break;
948 }
949 device->ws->cs_finalize(device->flush_cs[family]);
950 }
951
952 if (getenv("RADV_TRACE_FILE")) {
953 device->trace_bo = device->ws->buffer_create(device->ws, 4096, 8,
954 RADEON_DOMAIN_VRAM, RADEON_FLAG_CPU_ACCESS);
955 if (!device->trace_bo)
956 goto fail;
957
958 device->trace_id_ptr = device->ws->buffer_map(device->trace_bo);
959 if (!device->trace_id_ptr)
960 goto fail;
961 }
962
963 if (device->physical_device->rad_info.chip_class >= CIK)
964 cik_create_gfx_config(device);
965
966 *pDevice = radv_device_to_handle(device);
967 return VK_SUCCESS;
968
969 fail:
970 if (device->trace_bo)
971 device->ws->buffer_destroy(device->trace_bo);
972
973 if (device->gfx_init)
974 device->ws->buffer_destroy(device->gfx_init);
975
976 for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
977 for (unsigned q = 0; q < device->queue_count[i]; q++)
978 radv_queue_finish(&device->queues[i][q]);
979 if (device->queue_count[i])
980 vk_free(&device->alloc, device->queues[i]);
981 }
982
983 vk_free(&device->alloc, device);
984 return result;
985 }
986
987 void radv_DestroyDevice(
988 VkDevice _device,
989 const VkAllocationCallbacks* pAllocator)
990 {
991 RADV_FROM_HANDLE(radv_device, device, _device);
992
993 if (!device)
994 return;
995
996 if (device->trace_bo)
997 device->ws->buffer_destroy(device->trace_bo);
998
999 if (device->gfx_init)
1000 device->ws->buffer_destroy(device->gfx_init);
1001
1002 for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
1003 for (unsigned q = 0; q < device->queue_count[i]; q++)
1004 radv_queue_finish(&device->queues[i][q]);
1005 if (device->queue_count[i])
1006 vk_free(&device->alloc, device->queues[i]);
1007 if (device->empty_cs[i])
1008 device->ws->cs_destroy(device->empty_cs[i]);
1009 if (device->flush_cs[i])
1010 device->ws->cs_destroy(device->flush_cs[i]);
1011 }
1012 radv_device_finish_meta(device);
1013
1014 vk_free(&device->alloc, device);
1015 }
1016
1017 VkResult radv_EnumerateInstanceExtensionProperties(
1018 const char* pLayerName,
1019 uint32_t* pPropertyCount,
1020 VkExtensionProperties* pProperties)
1021 {
1022 if (pProperties == NULL) {
1023 *pPropertyCount = ARRAY_SIZE(instance_extensions);
1024 return VK_SUCCESS;
1025 }
1026
1027 *pPropertyCount = MIN2(*pPropertyCount, ARRAY_SIZE(instance_extensions));
1028 typed_memcpy(pProperties, instance_extensions, *pPropertyCount);
1029
1030 if (*pPropertyCount < ARRAY_SIZE(instance_extensions))
1031 return VK_INCOMPLETE;
1032
1033 return VK_SUCCESS;
1034 }
1035
1036 VkResult radv_EnumerateDeviceExtensionProperties(
1037 VkPhysicalDevice physicalDevice,
1038 const char* pLayerName,
1039 uint32_t* pPropertyCount,
1040 VkExtensionProperties* pProperties)
1041 {
1042 RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice);
1043
1044 if (pProperties == NULL) {
1045 *pPropertyCount = pdevice->extensions.num_ext;
1046 return VK_SUCCESS;
1047 }
1048
1049 *pPropertyCount = MIN2(*pPropertyCount, pdevice->extensions.num_ext);
1050 typed_memcpy(pProperties, pdevice->extensions.ext_array, *pPropertyCount);
1051
1052 if (*pPropertyCount < pdevice->extensions.num_ext)
1053 return VK_INCOMPLETE;
1054
1055 return VK_SUCCESS;
1056 }
1057
1058 VkResult radv_EnumerateInstanceLayerProperties(
1059 uint32_t* pPropertyCount,
1060 VkLayerProperties* pProperties)
1061 {
1062 if (pProperties == NULL) {
1063 *pPropertyCount = 0;
1064 return VK_SUCCESS;
1065 }
1066
1067 /* None supported at this time */
1068 return vk_error(VK_ERROR_LAYER_NOT_PRESENT);
1069 }
1070
1071 VkResult radv_EnumerateDeviceLayerProperties(
1072 VkPhysicalDevice physicalDevice,
1073 uint32_t* pPropertyCount,
1074 VkLayerProperties* pProperties)
1075 {
1076 if (pProperties == NULL) {
1077 *pPropertyCount = 0;
1078 return VK_SUCCESS;
1079 }
1080
1081 /* None supported at this time */
1082 return vk_error(VK_ERROR_LAYER_NOT_PRESENT);
1083 }
1084
1085 void radv_GetDeviceQueue(
1086 VkDevice _device,
1087 uint32_t queueFamilyIndex,
1088 uint32_t queueIndex,
1089 VkQueue* pQueue)
1090 {
1091 RADV_FROM_HANDLE(radv_device, device, _device);
1092
1093 *pQueue = radv_queue_to_handle(&device->queues[queueFamilyIndex][queueIndex]);
1094 }
1095
1096 static void radv_dump_trace(struct radv_device *device,
1097 struct radeon_winsys_cs *cs)
1098 {
1099 const char *filename = getenv("RADV_TRACE_FILE");
1100 FILE *f = fopen(filename, "w");
1101 if (!f) {
1102 fprintf(stderr, "Failed to write trace dump to %s\n", filename);
1103 return;
1104 }
1105
1106 fprintf(f, "Trace ID: %x\n", *device->trace_id_ptr);
1107 device->ws->cs_dump(cs, f, *device->trace_id_ptr);
1108 fclose(f);
1109 }
1110
1111 static void
1112 fill_geom_rings(struct radv_queue *queue,
1113 uint32_t *map,
1114 uint32_t esgs_ring_size,
1115 struct radeon_winsys_bo *esgs_ring_bo,
1116 uint32_t gsvs_ring_size,
1117 struct radeon_winsys_bo *gsvs_ring_bo)
1118 {
1119 uint64_t esgs_va = 0, gsvs_va = 0;
1120 uint32_t *desc = &map[4];
1121
1122 if (esgs_ring_bo)
1123 esgs_va = queue->device->ws->buffer_get_va(esgs_ring_bo);
1124 if (gsvs_ring_bo)
1125 gsvs_va = queue->device->ws->buffer_get_va(gsvs_ring_bo);
1126
1127 /* stride 0, num records - size, add tid, swizzle, elsize4,
1128 index stride 64 */
1129 desc[0] = esgs_va;
1130 desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32) |
1131 S_008F04_STRIDE(0) |
1132 S_008F04_SWIZZLE_ENABLE(true);
1133 desc[2] = esgs_ring_size;
1134 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1135 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1136 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1137 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1138 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1139 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1140 S_008F0C_ELEMENT_SIZE(1) |
1141 S_008F0C_INDEX_STRIDE(3) |
1142 S_008F0C_ADD_TID_ENABLE(true);
1143
1144 desc += 4;
1145 /* GS entry for ES->GS ring */
1146 /* stride 0, num records - size, elsize0,
1147 index stride 0 */
1148 desc[0] = esgs_va;
1149 desc[1] = S_008F04_BASE_ADDRESS_HI(esgs_va >> 32)|
1150 S_008F04_STRIDE(0) |
1151 S_008F04_SWIZZLE_ENABLE(false);
1152 desc[2] = esgs_ring_size;
1153 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1154 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1155 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1156 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1157 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1158 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1159 S_008F0C_ELEMENT_SIZE(0) |
1160 S_008F0C_INDEX_STRIDE(0) |
1161 S_008F0C_ADD_TID_ENABLE(false);
1162
1163 desc += 4;
1164 /* VS entry for GS->VS ring */
1165 /* stride 0, num records - size, elsize0,
1166 index stride 0 */
1167 desc[0] = gsvs_va;
1168 desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32)|
1169 S_008F04_STRIDE(0) |
1170 S_008F04_SWIZZLE_ENABLE(false);
1171 desc[2] = gsvs_ring_size;
1172 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1173 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1174 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1175 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1176 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1177 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1178 S_008F0C_ELEMENT_SIZE(0) |
1179 S_008F0C_INDEX_STRIDE(0) |
1180 S_008F0C_ADD_TID_ENABLE(false);
1181 desc += 4;
1182
1183 /* stride gsvs_itemsize, num records 64
1184 elsize 4, index stride 16 */
1185 /* shader will patch stride and desc[2] */
1186 desc[0] = gsvs_va;
1187 desc[1] = S_008F04_BASE_ADDRESS_HI(gsvs_va >> 32)|
1188 S_008F04_STRIDE(0) |
1189 S_008F04_SWIZZLE_ENABLE(true);
1190 desc[2] = 0;
1191 desc[3] = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
1192 S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
1193 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
1194 S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
1195 S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
1196 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32) |
1197 S_008F0C_ELEMENT_SIZE(1) |
1198 S_008F0C_INDEX_STRIDE(1) |
1199 S_008F0C_ADD_TID_ENABLE(true);
1200 }
1201
1202 static VkResult
1203 radv_get_preamble_cs(struct radv_queue *queue,
1204 uint32_t scratch_size,
1205 uint32_t compute_scratch_size,
1206 uint32_t esgs_ring_size,
1207 uint32_t gsvs_ring_size,
1208 struct radeon_winsys_cs **initial_preamble_cs,
1209 struct radeon_winsys_cs **continue_preamble_cs)
1210 {
1211 struct radeon_winsys_bo *scratch_bo = NULL;
1212 struct radeon_winsys_bo *descriptor_bo = NULL;
1213 struct radeon_winsys_bo *compute_scratch_bo = NULL;
1214 struct radeon_winsys_bo *esgs_ring_bo = NULL;
1215 struct radeon_winsys_bo *gsvs_ring_bo = NULL;
1216 struct radeon_winsys_cs *dest_cs[2] = {0};
1217
1218 if (scratch_size <= queue->scratch_size &&
1219 compute_scratch_size <= queue->compute_scratch_size &&
1220 esgs_ring_size <= queue->esgs_ring_size &&
1221 gsvs_ring_size <= queue->gsvs_ring_size &&
1222 queue->initial_preamble_cs) {
1223 *initial_preamble_cs = queue->initial_preamble_cs;
1224 *continue_preamble_cs = queue->continue_preamble_cs;
1225 if (!scratch_size && !compute_scratch_size && !esgs_ring_size && !gsvs_ring_size)
1226 *continue_preamble_cs = NULL;
1227 return VK_SUCCESS;
1228 }
1229
1230 if (scratch_size > queue->scratch_size) {
1231 scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
1232 scratch_size,
1233 4096,
1234 RADEON_DOMAIN_VRAM,
1235 RADEON_FLAG_NO_CPU_ACCESS);
1236 if (!scratch_bo)
1237 goto fail;
1238 } else
1239 scratch_bo = queue->scratch_bo;
1240
1241 if (compute_scratch_size > queue->compute_scratch_size) {
1242 compute_scratch_bo = queue->device->ws->buffer_create(queue->device->ws,
1243 compute_scratch_size,
1244 4096,
1245 RADEON_DOMAIN_VRAM,
1246 RADEON_FLAG_NO_CPU_ACCESS);
1247 if (!compute_scratch_bo)
1248 goto fail;
1249
1250 } else
1251 compute_scratch_bo = queue->compute_scratch_bo;
1252
1253 if (esgs_ring_size > queue->esgs_ring_size) {
1254 esgs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
1255 esgs_ring_size,
1256 4096,
1257 RADEON_DOMAIN_VRAM,
1258 RADEON_FLAG_NO_CPU_ACCESS);
1259 if (!esgs_ring_bo)
1260 goto fail;
1261 } else {
1262 esgs_ring_bo = queue->esgs_ring_bo;
1263 esgs_ring_size = queue->esgs_ring_size;
1264 }
1265
1266 if (gsvs_ring_size > queue->gsvs_ring_size) {
1267 gsvs_ring_bo = queue->device->ws->buffer_create(queue->device->ws,
1268 gsvs_ring_size,
1269 4096,
1270 RADEON_DOMAIN_VRAM,
1271 RADEON_FLAG_NO_CPU_ACCESS);
1272 if (!gsvs_ring_bo)
1273 goto fail;
1274 } else {
1275 gsvs_ring_bo = queue->gsvs_ring_bo;
1276 gsvs_ring_size = queue->gsvs_ring_size;
1277 }
1278
1279 if (scratch_bo != queue->scratch_bo ||
1280 esgs_ring_bo != queue->esgs_ring_bo ||
1281 gsvs_ring_bo != queue->gsvs_ring_bo) {
1282 uint32_t size = 0;
1283 if (gsvs_ring_bo || esgs_ring_bo)
1284 size = 80; /* 2 dword + 2 padding + 4 dword * 4 */
1285 else if (scratch_bo)
1286 size = 8; /* 2 dword */
1287
1288 descriptor_bo = queue->device->ws->buffer_create(queue->device->ws,
1289 size,
1290 4096,
1291 RADEON_DOMAIN_VRAM,
1292 RADEON_FLAG_CPU_ACCESS);
1293 if (!descriptor_bo)
1294 goto fail;
1295 } else
1296 descriptor_bo = queue->descriptor_bo;
1297
1298 for(int i = 0; i < 2; ++i) {
1299 struct radeon_winsys_cs *cs = NULL;
1300 cs = queue->device->ws->cs_create(queue->device->ws,
1301 queue->queue_family_index ? RING_COMPUTE : RING_GFX);
1302 if (!cs)
1303 goto fail;
1304
1305 dest_cs[i] = cs;
1306
1307 if (scratch_bo)
1308 queue->device->ws->cs_add_buffer(cs, scratch_bo, 8);
1309
1310 if (esgs_ring_bo)
1311 queue->device->ws->cs_add_buffer(cs, esgs_ring_bo, 8);
1312
1313 if (gsvs_ring_bo)
1314 queue->device->ws->cs_add_buffer(cs, gsvs_ring_bo, 8);
1315
1316 if (descriptor_bo)
1317 queue->device->ws->cs_add_buffer(cs, descriptor_bo, 8);
1318
1319 if (descriptor_bo != queue->descriptor_bo) {
1320 uint32_t *map = (uint32_t*)queue->device->ws->buffer_map(descriptor_bo);
1321
1322 if (scratch_bo) {
1323 uint64_t scratch_va = queue->device->ws->buffer_get_va(scratch_bo);
1324 uint32_t rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
1325 S_008F04_SWIZZLE_ENABLE(1);
1326 map[0] = scratch_va;
1327 map[1] = rsrc1;
1328 }
1329
1330 if (esgs_ring_bo || gsvs_ring_bo)
1331 fill_geom_rings(queue, map, esgs_ring_size, esgs_ring_bo, gsvs_ring_size, gsvs_ring_bo);
1332
1333 queue->device->ws->buffer_unmap(descriptor_bo);
1334 }
1335
1336 if (esgs_ring_bo || gsvs_ring_bo) {
1337 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1338 radeon_emit(cs, EVENT_TYPE(V_028A90_VS_PARTIAL_FLUSH) | EVENT_INDEX(4));
1339 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
1340 radeon_emit(cs, EVENT_TYPE(V_028A90_VGT_FLUSH) | EVENT_INDEX(0));
1341
1342 if (queue->device->physical_device->rad_info.chip_class >= CIK) {
1343 radeon_set_uconfig_reg_seq(cs, R_030900_VGT_ESGS_RING_SIZE, 2);
1344 radeon_emit(cs, esgs_ring_size >> 8);
1345 radeon_emit(cs, gsvs_ring_size >> 8);
1346 } else {
1347 radeon_set_config_reg_seq(cs, R_0088C8_VGT_ESGS_RING_SIZE, 2);
1348 radeon_emit(cs, esgs_ring_size >> 8);
1349 radeon_emit(cs, gsvs_ring_size >> 8);
1350 }
1351 }
1352
1353 if (descriptor_bo) {
1354 uint32_t regs[] = {R_00B030_SPI_SHADER_USER_DATA_PS_0,
1355 R_00B130_SPI_SHADER_USER_DATA_VS_0,
1356 R_00B230_SPI_SHADER_USER_DATA_GS_0,
1357 R_00B330_SPI_SHADER_USER_DATA_ES_0,
1358 R_00B430_SPI_SHADER_USER_DATA_HS_0,
1359 R_00B530_SPI_SHADER_USER_DATA_LS_0};
1360
1361 uint64_t va = queue->device->ws->buffer_get_va(descriptor_bo);
1362
1363 for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
1364 radeon_set_sh_reg_seq(cs, regs[i], 2);
1365 radeon_emit(cs, va);
1366 radeon_emit(cs, va >> 32);
1367 }
1368 }
1369
1370 if (compute_scratch_bo) {
1371 uint64_t scratch_va = queue->device->ws->buffer_get_va(compute_scratch_bo);
1372 uint32_t rsrc1 = S_008F04_BASE_ADDRESS_HI(scratch_va >> 32) |
1373 S_008F04_SWIZZLE_ENABLE(1);
1374
1375 queue->device->ws->cs_add_buffer(cs, compute_scratch_bo, 8);
1376
1377 radeon_set_sh_reg_seq(cs, R_00B900_COMPUTE_USER_DATA_0, 2);
1378 radeon_emit(cs, scratch_va);
1379 radeon_emit(cs, rsrc1);
1380 }
1381
1382 if (!i) {
1383 si_cs_emit_cache_flush(cs,
1384 queue->device->physical_device->rad_info.chip_class,
1385 queue->queue_family_index == RING_COMPUTE &&
1386 queue->device->physical_device->rad_info.chip_class >= CIK,
1387 RADV_CMD_FLAG_INV_ICACHE |
1388 RADV_CMD_FLAG_INV_SMEM_L1 |
1389 RADV_CMD_FLAG_INV_VMEM_L1 |
1390 RADV_CMD_FLAG_INV_GLOBAL_L2);
1391 }
1392
1393 if (!queue->device->ws->cs_finalize(cs))
1394 goto fail;
1395 }
1396
1397 if (queue->initial_preamble_cs)
1398 queue->device->ws->cs_destroy(queue->initial_preamble_cs);
1399
1400 if (queue->continue_preamble_cs)
1401 queue->device->ws->cs_destroy(queue->continue_preamble_cs);
1402
1403 queue->initial_preamble_cs = dest_cs[0];
1404 queue->continue_preamble_cs = dest_cs[1];
1405
1406 if (scratch_bo != queue->scratch_bo) {
1407 if (queue->scratch_bo)
1408 queue->device->ws->buffer_destroy(queue->scratch_bo);
1409 queue->scratch_bo = scratch_bo;
1410 queue->scratch_size = scratch_size;
1411 }
1412
1413 if (compute_scratch_bo != queue->compute_scratch_bo) {
1414 if (queue->compute_scratch_bo)
1415 queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
1416 queue->compute_scratch_bo = compute_scratch_bo;
1417 queue->compute_scratch_size = compute_scratch_size;
1418 }
1419
1420 if (esgs_ring_bo != queue->esgs_ring_bo) {
1421 if (queue->esgs_ring_bo)
1422 queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
1423 queue->esgs_ring_bo = esgs_ring_bo;
1424 queue->esgs_ring_size = esgs_ring_size;
1425 }
1426
1427 if (gsvs_ring_bo != queue->gsvs_ring_bo) {
1428 if (queue->gsvs_ring_bo)
1429 queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
1430 queue->gsvs_ring_bo = gsvs_ring_bo;
1431 queue->gsvs_ring_size = gsvs_ring_size;
1432 }
1433
1434 if (descriptor_bo != queue->descriptor_bo) {
1435 if (queue->descriptor_bo)
1436 queue->device->ws->buffer_destroy(queue->descriptor_bo);
1437
1438 queue->descriptor_bo = descriptor_bo;
1439 }
1440
1441 *initial_preamble_cs = queue->initial_preamble_cs;
1442 *continue_preamble_cs = queue->continue_preamble_cs;
1443 if (!scratch_size && !compute_scratch_size && !esgs_ring_size && !gsvs_ring_size)
1444 *continue_preamble_cs = NULL;
1445 return VK_SUCCESS;
1446 fail:
1447 for (int i = 0; i < ARRAY_SIZE(dest_cs); ++i)
1448 if (dest_cs[i])
1449 queue->device->ws->cs_destroy(dest_cs[i]);
1450 if (descriptor_bo && descriptor_bo != queue->descriptor_bo)
1451 queue->device->ws->buffer_destroy(descriptor_bo);
1452 if (scratch_bo && scratch_bo != queue->scratch_bo)
1453 queue->device->ws->buffer_destroy(scratch_bo);
1454 if (compute_scratch_bo && compute_scratch_bo != queue->compute_scratch_bo)
1455 queue->device->ws->buffer_destroy(compute_scratch_bo);
1456 if (esgs_ring_bo && esgs_ring_bo != queue->esgs_ring_bo)
1457 queue->device->ws->buffer_destroy(esgs_ring_bo);
1458 if (gsvs_ring_bo && gsvs_ring_bo != queue->gsvs_ring_bo)
1459 queue->device->ws->buffer_destroy(gsvs_ring_bo);
1460 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1461 }
1462
1463 VkResult radv_QueueSubmit(
1464 VkQueue _queue,
1465 uint32_t submitCount,
1466 const VkSubmitInfo* pSubmits,
1467 VkFence _fence)
1468 {
1469 RADV_FROM_HANDLE(radv_queue, queue, _queue);
1470 RADV_FROM_HANDLE(radv_fence, fence, _fence);
1471 struct radeon_winsys_fence *base_fence = fence ? fence->fence : NULL;
1472 struct radeon_winsys_ctx *ctx = queue->hw_ctx;
1473 int ret;
1474 uint32_t max_cs_submission = queue->device->trace_bo ? 1 : UINT32_MAX;
1475 uint32_t scratch_size = 0;
1476 uint32_t compute_scratch_size = 0;
1477 uint32_t esgs_ring_size = 0, gsvs_ring_size = 0;
1478 struct radeon_winsys_cs *initial_preamble_cs = NULL, *continue_preamble_cs = NULL;
1479 VkResult result;
1480 bool fence_emitted = false;
1481
1482 /* Do this first so failing to allocate scratch buffers can't result in
1483 * partially executed submissions. */
1484 for (uint32_t i = 0; i < submitCount; i++) {
1485 for (uint32_t j = 0; j < pSubmits[i].commandBufferCount; j++) {
1486 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer,
1487 pSubmits[i].pCommandBuffers[j]);
1488
1489 scratch_size = MAX2(scratch_size, cmd_buffer->scratch_size_needed);
1490 compute_scratch_size = MAX2(compute_scratch_size,
1491 cmd_buffer->compute_scratch_size_needed);
1492 esgs_ring_size = MAX2(esgs_ring_size, cmd_buffer->esgs_ring_size_needed);
1493 gsvs_ring_size = MAX2(gsvs_ring_size, cmd_buffer->gsvs_ring_size_needed);
1494 }
1495 }
1496
1497 result = radv_get_preamble_cs(queue, scratch_size, compute_scratch_size,
1498 esgs_ring_size, gsvs_ring_size,
1499 &initial_preamble_cs, &continue_preamble_cs);
1500 if (result != VK_SUCCESS)
1501 return result;
1502
1503 for (uint32_t i = 0; i < submitCount; i++) {
1504 struct radeon_winsys_cs **cs_array;
1505 bool has_flush = !submitCount;
1506 bool can_patch = !has_flush;
1507 uint32_t advance;
1508
1509 if (!pSubmits[i].commandBufferCount) {
1510 if (pSubmits[i].waitSemaphoreCount || pSubmits[i].signalSemaphoreCount) {
1511 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx,
1512 &queue->device->empty_cs[queue->queue_family_index],
1513 1, NULL, NULL,
1514 (struct radeon_winsys_sem **)pSubmits[i].pWaitSemaphores,
1515 pSubmits[i].waitSemaphoreCount,
1516 (struct radeon_winsys_sem **)pSubmits[i].pSignalSemaphores,
1517 pSubmits[i].signalSemaphoreCount,
1518 false, base_fence);
1519 if (ret) {
1520 radv_loge("failed to submit CS %d\n", i);
1521 abort();
1522 }
1523 fence_emitted = true;
1524 }
1525 continue;
1526 }
1527
1528 cs_array = malloc(sizeof(struct radeon_winsys_cs *) *
1529 (pSubmits[i].commandBufferCount + has_flush));
1530
1531 if(has_flush)
1532 cs_array[0] = queue->device->flush_cs[queue->queue_family_index];
1533
1534 for (uint32_t j = 0; j < pSubmits[i].commandBufferCount; j++) {
1535 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer,
1536 pSubmits[i].pCommandBuffers[j]);
1537 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
1538
1539 cs_array[j + has_flush] = cmd_buffer->cs;
1540 if ((cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT))
1541 can_patch = false;
1542 }
1543
1544 for (uint32_t j = 0; j < pSubmits[i].commandBufferCount + has_flush; j += advance) {
1545 advance = MIN2(max_cs_submission,
1546 pSubmits[i].commandBufferCount + has_flush - j);
1547 bool b = j == 0;
1548 bool e = j + advance == pSubmits[i].commandBufferCount + has_flush;
1549
1550 if (queue->device->trace_bo)
1551 *queue->device->trace_id_ptr = 0;
1552
1553 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx, cs_array + j,
1554 advance, initial_preamble_cs, continue_preamble_cs,
1555 (struct radeon_winsys_sem **)pSubmits[i].pWaitSemaphores,
1556 b ? pSubmits[i].waitSemaphoreCount : 0,
1557 (struct radeon_winsys_sem **)pSubmits[i].pSignalSemaphores,
1558 e ? pSubmits[i].signalSemaphoreCount : 0,
1559 can_patch, base_fence);
1560
1561 if (ret) {
1562 radv_loge("failed to submit CS %d\n", i);
1563 abort();
1564 }
1565 fence_emitted = true;
1566 if (queue->device->trace_bo) {
1567 bool success = queue->device->ws->ctx_wait_idle(
1568 queue->hw_ctx,
1569 radv_queue_family_to_ring(
1570 queue->queue_family_index),
1571 queue->queue_idx);
1572
1573 if (!success) { /* Hang */
1574 radv_dump_trace(queue->device, cs_array[j]);
1575 abort();
1576 }
1577 }
1578 }
1579 free(cs_array);
1580 }
1581
1582 if (fence) {
1583 if (!fence_emitted)
1584 ret = queue->device->ws->cs_submit(ctx, queue->queue_idx,
1585 &queue->device->empty_cs[queue->queue_family_index],
1586 1, NULL, NULL, NULL, 0, NULL, 0,
1587 false, base_fence);
1588
1589 fence->submitted = true;
1590 }
1591
1592 return VK_SUCCESS;
1593 }
1594
1595 VkResult radv_QueueWaitIdle(
1596 VkQueue _queue)
1597 {
1598 RADV_FROM_HANDLE(radv_queue, queue, _queue);
1599
1600 queue->device->ws->ctx_wait_idle(queue->hw_ctx,
1601 radv_queue_family_to_ring(queue->queue_family_index),
1602 queue->queue_idx);
1603 return VK_SUCCESS;
1604 }
1605
1606 VkResult radv_DeviceWaitIdle(
1607 VkDevice _device)
1608 {
1609 RADV_FROM_HANDLE(radv_device, device, _device);
1610
1611 for (unsigned i = 0; i < RADV_MAX_QUEUE_FAMILIES; i++) {
1612 for (unsigned q = 0; q < device->queue_count[i]; q++) {
1613 radv_QueueWaitIdle(radv_queue_to_handle(&device->queues[i][q]));
1614 }
1615 }
1616 return VK_SUCCESS;
1617 }
1618
1619 PFN_vkVoidFunction radv_GetInstanceProcAddr(
1620 VkInstance instance,
1621 const char* pName)
1622 {
1623 return radv_lookup_entrypoint(pName);
1624 }
1625
1626 /* The loader wants us to expose a second GetInstanceProcAddr function
1627 * to work around certain LD_PRELOAD issues seen in apps.
1628 */
1629 PUBLIC
1630 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
1631 VkInstance instance,
1632 const char* pName);
1633
1634 PUBLIC
1635 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vk_icdGetInstanceProcAddr(
1636 VkInstance instance,
1637 const char* pName)
1638 {
1639 return radv_GetInstanceProcAddr(instance, pName);
1640 }
1641
1642 PFN_vkVoidFunction radv_GetDeviceProcAddr(
1643 VkDevice device,
1644 const char* pName)
1645 {
1646 return radv_lookup_entrypoint(pName);
1647 }
1648
1649 bool radv_get_memory_fd(struct radv_device *device,
1650 struct radv_device_memory *memory,
1651 int *pFD)
1652 {
1653 struct radeon_bo_metadata metadata;
1654
1655 if (memory->image) {
1656 radv_init_metadata(device, memory->image, &metadata);
1657 device->ws->buffer_set_metadata(memory->bo, &metadata);
1658 }
1659
1660 return device->ws->buffer_get_fd(device->ws, memory->bo,
1661 pFD);
1662 }
1663
1664 VkResult radv_AllocateMemory(
1665 VkDevice _device,
1666 const VkMemoryAllocateInfo* pAllocateInfo,
1667 const VkAllocationCallbacks* pAllocator,
1668 VkDeviceMemory* pMem)
1669 {
1670 RADV_FROM_HANDLE(radv_device, device, _device);
1671 struct radv_device_memory *mem;
1672 VkResult result;
1673 enum radeon_bo_domain domain;
1674 uint32_t flags = 0;
1675 const VkDedicatedAllocationMemoryAllocateInfoNV *dedicate_info = NULL;
1676 assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
1677
1678 if (pAllocateInfo->allocationSize == 0) {
1679 /* Apparently, this is allowed */
1680 *pMem = VK_NULL_HANDLE;
1681 return VK_SUCCESS;
1682 }
1683
1684 vk_foreach_struct(ext, pAllocateInfo->pNext) {
1685 switch (ext->sType) {
1686 case VK_STRUCTURE_TYPE_DEDICATED_ALLOCATION_MEMORY_ALLOCATE_INFO_NV:
1687 dedicate_info = (const VkDedicatedAllocationMemoryAllocateInfoNV *)ext;
1688 break;
1689 default:
1690 break;
1691 }
1692 }
1693
1694 mem = vk_alloc2(&device->alloc, pAllocator, sizeof(*mem), 8,
1695 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1696 if (mem == NULL)
1697 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1698
1699 if (dedicate_info) {
1700 mem->image = radv_image_from_handle(dedicate_info->image);
1701 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
1702 } else {
1703 mem->image = NULL;
1704 mem->buffer = NULL;
1705 }
1706
1707 uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
1708 if (pAllocateInfo->memoryTypeIndex == RADV_MEM_TYPE_GTT_WRITE_COMBINE ||
1709 pAllocateInfo->memoryTypeIndex == RADV_MEM_TYPE_GTT_CACHED)
1710 domain = RADEON_DOMAIN_GTT;
1711 else
1712 domain = RADEON_DOMAIN_VRAM;
1713
1714 if (pAllocateInfo->memoryTypeIndex == RADV_MEM_TYPE_VRAM)
1715 flags |= RADEON_FLAG_NO_CPU_ACCESS;
1716 else
1717 flags |= RADEON_FLAG_CPU_ACCESS;
1718
1719 if (pAllocateInfo->memoryTypeIndex == RADV_MEM_TYPE_GTT_WRITE_COMBINE)
1720 flags |= RADEON_FLAG_GTT_WC;
1721
1722 mem->bo = device->ws->buffer_create(device->ws, alloc_size, 65536,
1723 domain, flags);
1724
1725 if (!mem->bo) {
1726 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
1727 goto fail;
1728 }
1729 mem->type_index = pAllocateInfo->memoryTypeIndex;
1730
1731 *pMem = radv_device_memory_to_handle(mem);
1732
1733 return VK_SUCCESS;
1734
1735 fail:
1736 vk_free2(&device->alloc, pAllocator, mem);
1737
1738 return result;
1739 }
1740
1741 void radv_FreeMemory(
1742 VkDevice _device,
1743 VkDeviceMemory _mem,
1744 const VkAllocationCallbacks* pAllocator)
1745 {
1746 RADV_FROM_HANDLE(radv_device, device, _device);
1747 RADV_FROM_HANDLE(radv_device_memory, mem, _mem);
1748
1749 if (mem == NULL)
1750 return;
1751
1752 device->ws->buffer_destroy(mem->bo);
1753 mem->bo = NULL;
1754
1755 vk_free2(&device->alloc, pAllocator, mem);
1756 }
1757
1758 VkResult radv_MapMemory(
1759 VkDevice _device,
1760 VkDeviceMemory _memory,
1761 VkDeviceSize offset,
1762 VkDeviceSize size,
1763 VkMemoryMapFlags flags,
1764 void** ppData)
1765 {
1766 RADV_FROM_HANDLE(radv_device, device, _device);
1767 RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
1768
1769 if (mem == NULL) {
1770 *ppData = NULL;
1771 return VK_SUCCESS;
1772 }
1773
1774 *ppData = device->ws->buffer_map(mem->bo);
1775 if (*ppData) {
1776 *ppData += offset;
1777 return VK_SUCCESS;
1778 }
1779
1780 return VK_ERROR_MEMORY_MAP_FAILED;
1781 }
1782
1783 void radv_UnmapMemory(
1784 VkDevice _device,
1785 VkDeviceMemory _memory)
1786 {
1787 RADV_FROM_HANDLE(radv_device, device, _device);
1788 RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
1789
1790 if (mem == NULL)
1791 return;
1792
1793 device->ws->buffer_unmap(mem->bo);
1794 }
1795
1796 VkResult radv_FlushMappedMemoryRanges(
1797 VkDevice _device,
1798 uint32_t memoryRangeCount,
1799 const VkMappedMemoryRange* pMemoryRanges)
1800 {
1801 return VK_SUCCESS;
1802 }
1803
1804 VkResult radv_InvalidateMappedMemoryRanges(
1805 VkDevice _device,
1806 uint32_t memoryRangeCount,
1807 const VkMappedMemoryRange* pMemoryRanges)
1808 {
1809 return VK_SUCCESS;
1810 }
1811
1812 void radv_GetBufferMemoryRequirements(
1813 VkDevice device,
1814 VkBuffer _buffer,
1815 VkMemoryRequirements* pMemoryRequirements)
1816 {
1817 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
1818
1819 pMemoryRequirements->memoryTypeBits = (1u << RADV_MEM_TYPE_COUNT) - 1;
1820
1821 pMemoryRequirements->size = buffer->size;
1822 pMemoryRequirements->alignment = 16;
1823 }
1824
1825 void radv_GetImageMemoryRequirements(
1826 VkDevice device,
1827 VkImage _image,
1828 VkMemoryRequirements* pMemoryRequirements)
1829 {
1830 RADV_FROM_HANDLE(radv_image, image, _image);
1831
1832 pMemoryRequirements->memoryTypeBits = (1u << RADV_MEM_TYPE_COUNT) - 1;
1833
1834 pMemoryRequirements->size = image->size;
1835 pMemoryRequirements->alignment = image->alignment;
1836 }
1837
1838 void radv_GetImageSparseMemoryRequirements(
1839 VkDevice device,
1840 VkImage image,
1841 uint32_t* pSparseMemoryRequirementCount,
1842 VkSparseImageMemoryRequirements* pSparseMemoryRequirements)
1843 {
1844 stub();
1845 }
1846
1847 void radv_GetDeviceMemoryCommitment(
1848 VkDevice device,
1849 VkDeviceMemory memory,
1850 VkDeviceSize* pCommittedMemoryInBytes)
1851 {
1852 *pCommittedMemoryInBytes = 0;
1853 }
1854
1855 VkResult radv_BindBufferMemory(
1856 VkDevice device,
1857 VkBuffer _buffer,
1858 VkDeviceMemory _memory,
1859 VkDeviceSize memoryOffset)
1860 {
1861 RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
1862 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
1863
1864 if (mem) {
1865 buffer->bo = mem->bo;
1866 buffer->offset = memoryOffset;
1867 } else {
1868 buffer->bo = NULL;
1869 buffer->offset = 0;
1870 }
1871
1872 return VK_SUCCESS;
1873 }
1874
1875 VkResult radv_BindImageMemory(
1876 VkDevice device,
1877 VkImage _image,
1878 VkDeviceMemory _memory,
1879 VkDeviceSize memoryOffset)
1880 {
1881 RADV_FROM_HANDLE(radv_device_memory, mem, _memory);
1882 RADV_FROM_HANDLE(radv_image, image, _image);
1883
1884 if (mem) {
1885 image->bo = mem->bo;
1886 image->offset = memoryOffset;
1887 } else {
1888 image->bo = NULL;
1889 image->offset = 0;
1890 }
1891
1892 return VK_SUCCESS;
1893 }
1894
1895 VkResult radv_QueueBindSparse(
1896 VkQueue queue,
1897 uint32_t bindInfoCount,
1898 const VkBindSparseInfo* pBindInfo,
1899 VkFence fence)
1900 {
1901 stub_return(VK_ERROR_INCOMPATIBLE_DRIVER);
1902 }
1903
1904 VkResult radv_CreateFence(
1905 VkDevice _device,
1906 const VkFenceCreateInfo* pCreateInfo,
1907 const VkAllocationCallbacks* pAllocator,
1908 VkFence* pFence)
1909 {
1910 RADV_FROM_HANDLE(radv_device, device, _device);
1911 struct radv_fence *fence = vk_alloc2(&device->alloc, pAllocator,
1912 sizeof(*fence), 8,
1913 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1914
1915 if (!fence)
1916 return VK_ERROR_OUT_OF_HOST_MEMORY;
1917
1918 memset(fence, 0, sizeof(*fence));
1919 fence->submitted = false;
1920 fence->signalled = !!(pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT);
1921 fence->fence = device->ws->create_fence();
1922 if (!fence->fence) {
1923 vk_free2(&device->alloc, pAllocator, fence);
1924 return VK_ERROR_OUT_OF_HOST_MEMORY;
1925 }
1926
1927 *pFence = radv_fence_to_handle(fence);
1928
1929 return VK_SUCCESS;
1930 }
1931
1932 void radv_DestroyFence(
1933 VkDevice _device,
1934 VkFence _fence,
1935 const VkAllocationCallbacks* pAllocator)
1936 {
1937 RADV_FROM_HANDLE(radv_device, device, _device);
1938 RADV_FROM_HANDLE(radv_fence, fence, _fence);
1939
1940 if (!fence)
1941 return;
1942 device->ws->destroy_fence(fence->fence);
1943 vk_free2(&device->alloc, pAllocator, fence);
1944 }
1945
1946 static uint64_t radv_get_absolute_timeout(uint64_t timeout)
1947 {
1948 uint64_t current_time;
1949 struct timespec tv;
1950
1951 clock_gettime(CLOCK_MONOTONIC, &tv);
1952 current_time = tv.tv_nsec + tv.tv_sec*1000000000ull;
1953
1954 timeout = MIN2(UINT64_MAX - current_time, timeout);
1955
1956 return current_time + timeout;
1957 }
1958
1959 VkResult radv_WaitForFences(
1960 VkDevice _device,
1961 uint32_t fenceCount,
1962 const VkFence* pFences,
1963 VkBool32 waitAll,
1964 uint64_t timeout)
1965 {
1966 RADV_FROM_HANDLE(radv_device, device, _device);
1967 timeout = radv_get_absolute_timeout(timeout);
1968
1969 if (!waitAll && fenceCount > 1) {
1970 fprintf(stderr, "radv: WaitForFences without waitAll not implemented yet\n");
1971 }
1972
1973 for (uint32_t i = 0; i < fenceCount; ++i) {
1974 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
1975 bool expired = false;
1976
1977 if (fence->signalled)
1978 continue;
1979
1980 if (!fence->submitted)
1981 return VK_TIMEOUT;
1982
1983 expired = device->ws->fence_wait(device->ws, fence->fence, true, timeout);
1984 if (!expired)
1985 return VK_TIMEOUT;
1986
1987 fence->signalled = true;
1988 }
1989
1990 return VK_SUCCESS;
1991 }
1992
1993 VkResult radv_ResetFences(VkDevice device,
1994 uint32_t fenceCount,
1995 const VkFence *pFences)
1996 {
1997 for (unsigned i = 0; i < fenceCount; ++i) {
1998 RADV_FROM_HANDLE(radv_fence, fence, pFences[i]);
1999 fence->submitted = fence->signalled = false;
2000 }
2001
2002 return VK_SUCCESS;
2003 }
2004
2005 VkResult radv_GetFenceStatus(VkDevice _device, VkFence _fence)
2006 {
2007 RADV_FROM_HANDLE(radv_device, device, _device);
2008 RADV_FROM_HANDLE(radv_fence, fence, _fence);
2009
2010 if (fence->signalled)
2011 return VK_SUCCESS;
2012 if (!fence->submitted)
2013 return VK_NOT_READY;
2014
2015 if (!device->ws->fence_wait(device->ws, fence->fence, false, 0))
2016 return VK_NOT_READY;
2017
2018 return VK_SUCCESS;
2019 }
2020
2021
2022 // Queue semaphore functions
2023
2024 VkResult radv_CreateSemaphore(
2025 VkDevice _device,
2026 const VkSemaphoreCreateInfo* pCreateInfo,
2027 const VkAllocationCallbacks* pAllocator,
2028 VkSemaphore* pSemaphore)
2029 {
2030 RADV_FROM_HANDLE(radv_device, device, _device);
2031 struct radeon_winsys_sem *sem;
2032
2033 sem = device->ws->create_sem(device->ws);
2034 if (!sem)
2035 return VK_ERROR_OUT_OF_HOST_MEMORY;
2036
2037 *pSemaphore = radeon_winsys_sem_to_handle(sem);
2038 return VK_SUCCESS;
2039 }
2040
2041 void radv_DestroySemaphore(
2042 VkDevice _device,
2043 VkSemaphore _semaphore,
2044 const VkAllocationCallbacks* pAllocator)
2045 {
2046 RADV_FROM_HANDLE(radv_device, device, _device);
2047 RADV_FROM_HANDLE(radeon_winsys_sem, sem, _semaphore);
2048 if (!_semaphore)
2049 return;
2050
2051 device->ws->destroy_sem(sem);
2052 }
2053
2054 VkResult radv_CreateEvent(
2055 VkDevice _device,
2056 const VkEventCreateInfo* pCreateInfo,
2057 const VkAllocationCallbacks* pAllocator,
2058 VkEvent* pEvent)
2059 {
2060 RADV_FROM_HANDLE(radv_device, device, _device);
2061 struct radv_event *event = vk_alloc2(&device->alloc, pAllocator,
2062 sizeof(*event), 8,
2063 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2064
2065 if (!event)
2066 return VK_ERROR_OUT_OF_HOST_MEMORY;
2067
2068 event->bo = device->ws->buffer_create(device->ws, 8, 8,
2069 RADEON_DOMAIN_GTT,
2070 RADEON_FLAG_CPU_ACCESS);
2071 if (!event->bo) {
2072 vk_free2(&device->alloc, pAllocator, event);
2073 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
2074 }
2075
2076 event->map = (uint64_t*)device->ws->buffer_map(event->bo);
2077
2078 *pEvent = radv_event_to_handle(event);
2079
2080 return VK_SUCCESS;
2081 }
2082
2083 void radv_DestroyEvent(
2084 VkDevice _device,
2085 VkEvent _event,
2086 const VkAllocationCallbacks* pAllocator)
2087 {
2088 RADV_FROM_HANDLE(radv_device, device, _device);
2089 RADV_FROM_HANDLE(radv_event, event, _event);
2090
2091 if (!event)
2092 return;
2093 device->ws->buffer_destroy(event->bo);
2094 vk_free2(&device->alloc, pAllocator, event);
2095 }
2096
2097 VkResult radv_GetEventStatus(
2098 VkDevice _device,
2099 VkEvent _event)
2100 {
2101 RADV_FROM_HANDLE(radv_event, event, _event);
2102
2103 if (*event->map == 1)
2104 return VK_EVENT_SET;
2105 return VK_EVENT_RESET;
2106 }
2107
2108 VkResult radv_SetEvent(
2109 VkDevice _device,
2110 VkEvent _event)
2111 {
2112 RADV_FROM_HANDLE(radv_event, event, _event);
2113 *event->map = 1;
2114
2115 return VK_SUCCESS;
2116 }
2117
2118 VkResult radv_ResetEvent(
2119 VkDevice _device,
2120 VkEvent _event)
2121 {
2122 RADV_FROM_HANDLE(radv_event, event, _event);
2123 *event->map = 0;
2124
2125 return VK_SUCCESS;
2126 }
2127
2128 VkResult radv_CreateBuffer(
2129 VkDevice _device,
2130 const VkBufferCreateInfo* pCreateInfo,
2131 const VkAllocationCallbacks* pAllocator,
2132 VkBuffer* pBuffer)
2133 {
2134 RADV_FROM_HANDLE(radv_device, device, _device);
2135 struct radv_buffer *buffer;
2136
2137 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
2138
2139 buffer = vk_alloc2(&device->alloc, pAllocator, sizeof(*buffer), 8,
2140 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2141 if (buffer == NULL)
2142 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
2143
2144 buffer->size = pCreateInfo->size;
2145 buffer->usage = pCreateInfo->usage;
2146 buffer->bo = NULL;
2147 buffer->offset = 0;
2148
2149 *pBuffer = radv_buffer_to_handle(buffer);
2150
2151 return VK_SUCCESS;
2152 }
2153
2154 void radv_DestroyBuffer(
2155 VkDevice _device,
2156 VkBuffer _buffer,
2157 const VkAllocationCallbacks* pAllocator)
2158 {
2159 RADV_FROM_HANDLE(radv_device, device, _device);
2160 RADV_FROM_HANDLE(radv_buffer, buffer, _buffer);
2161
2162 if (!buffer)
2163 return;
2164
2165 vk_free2(&device->alloc, pAllocator, buffer);
2166 }
2167
2168 static inline unsigned
2169 si_tile_mode_index(const struct radv_image *image, unsigned level, bool stencil)
2170 {
2171 if (stencil)
2172 return image->surface.stencil_tiling_index[level];
2173 else
2174 return image->surface.tiling_index[level];
2175 }
2176
2177 static uint32_t radv_surface_layer_count(struct radv_image_view *iview)
2178 {
2179 return iview->type == VK_IMAGE_VIEW_TYPE_3D ? iview->extent.depth : iview->layer_count;
2180 }
2181
2182 static void
2183 radv_initialise_color_surface(struct radv_device *device,
2184 struct radv_color_buffer_info *cb,
2185 struct radv_image_view *iview)
2186 {
2187 const struct vk_format_description *desc;
2188 unsigned ntype, format, swap, endian;
2189 unsigned blend_clamp = 0, blend_bypass = 0;
2190 unsigned pitch_tile_max, slice_tile_max, tile_mode_index;
2191 uint64_t va;
2192 const struct radeon_surf *surf = &iview->image->surface;
2193 const struct radeon_surf_level *level_info = &surf->level[iview->base_mip];
2194
2195 desc = vk_format_description(iview->vk_format);
2196
2197 memset(cb, 0, sizeof(*cb));
2198
2199 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
2200 va += level_info->offset;
2201 cb->cb_color_base = va >> 8;
2202
2203 /* CMASK variables */
2204 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
2205 va += iview->image->cmask.offset;
2206 cb->cb_color_cmask = va >> 8;
2207 cb->cb_color_cmask_slice = iview->image->cmask.slice_tile_max;
2208
2209 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
2210 va += iview->image->dcc_offset;
2211 cb->cb_dcc_base = va >> 8;
2212
2213 uint32_t max_slice = radv_surface_layer_count(iview);
2214 cb->cb_color_view = S_028C6C_SLICE_START(iview->base_layer) |
2215 S_028C6C_SLICE_MAX(iview->base_layer + max_slice - 1);
2216
2217 cb->micro_tile_mode = iview->image->surface.micro_tile_mode;
2218 pitch_tile_max = level_info->nblk_x / 8 - 1;
2219 slice_tile_max = (level_info->nblk_x * level_info->nblk_y) / 64 - 1;
2220 tile_mode_index = si_tile_mode_index(iview->image, iview->base_mip, false);
2221
2222 cb->cb_color_pitch = S_028C64_TILE_MAX(pitch_tile_max);
2223 cb->cb_color_slice = S_028C68_TILE_MAX(slice_tile_max);
2224
2225 /* Intensity is implemented as Red, so treat it that way. */
2226 cb->cb_color_attrib = S_028C74_FORCE_DST_ALPHA_1(desc->swizzle[3] == VK_SWIZZLE_1) |
2227 S_028C74_TILE_MODE_INDEX(tile_mode_index);
2228
2229 if (iview->image->samples > 1) {
2230 unsigned log_samples = util_logbase2(iview->image->samples);
2231
2232 cb->cb_color_attrib |= S_028C74_NUM_SAMPLES(log_samples) |
2233 S_028C74_NUM_FRAGMENTS(log_samples);
2234 }
2235
2236 if (iview->image->fmask.size) {
2237 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset + iview->image->fmask.offset;
2238 if (device->physical_device->rad_info.chip_class >= CIK)
2239 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(iview->image->fmask.pitch_in_pixels / 8 - 1);
2240 cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(iview->image->fmask.tile_mode_index);
2241 cb->cb_color_fmask = va >> 8;
2242 cb->cb_color_fmask_slice = S_028C88_TILE_MAX(iview->image->fmask.slice_tile_max);
2243 } else {
2244 /* This must be set for fast clear to work without FMASK. */
2245 if (device->physical_device->rad_info.chip_class >= CIK)
2246 cb->cb_color_pitch |= S_028C64_FMASK_TILE_MAX(pitch_tile_max);
2247 cb->cb_color_attrib |= S_028C74_FMASK_TILE_MODE_INDEX(tile_mode_index);
2248 cb->cb_color_fmask = cb->cb_color_base;
2249 cb->cb_color_fmask_slice = S_028C88_TILE_MAX(slice_tile_max);
2250 }
2251
2252 ntype = radv_translate_color_numformat(iview->vk_format,
2253 desc,
2254 vk_format_get_first_non_void_channel(iview->vk_format));
2255 format = radv_translate_colorformat(iview->vk_format);
2256 if (format == V_028C70_COLOR_INVALID || ntype == ~0u)
2257 radv_finishme("Illegal color\n");
2258 swap = radv_translate_colorswap(iview->vk_format, FALSE);
2259 endian = radv_colorformat_endian_swap(format);
2260
2261 /* blend clamp should be set for all NORM/SRGB types */
2262 if (ntype == V_028C70_NUMBER_UNORM ||
2263 ntype == V_028C70_NUMBER_SNORM ||
2264 ntype == V_028C70_NUMBER_SRGB)
2265 blend_clamp = 1;
2266
2267 /* set blend bypass according to docs if SINT/UINT or
2268 8/24 COLOR variants */
2269 if (ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT ||
2270 format == V_028C70_COLOR_8_24 || format == V_028C70_COLOR_24_8 ||
2271 format == V_028C70_COLOR_X24_8_32_FLOAT) {
2272 blend_clamp = 0;
2273 blend_bypass = 1;
2274 }
2275 #if 0
2276 if ((ntype == V_028C70_NUMBER_UINT || ntype == V_028C70_NUMBER_SINT) &&
2277 (format == V_028C70_COLOR_8 ||
2278 format == V_028C70_COLOR_8_8 ||
2279 format == V_028C70_COLOR_8_8_8_8))
2280 ->color_is_int8 = true;
2281 #endif
2282 cb->cb_color_info = S_028C70_FORMAT(format) |
2283 S_028C70_COMP_SWAP(swap) |
2284 S_028C70_BLEND_CLAMP(blend_clamp) |
2285 S_028C70_BLEND_BYPASS(blend_bypass) |
2286 S_028C70_SIMPLE_FLOAT(1) |
2287 S_028C70_ROUND_MODE(ntype != V_028C70_NUMBER_UNORM &&
2288 ntype != V_028C70_NUMBER_SNORM &&
2289 ntype != V_028C70_NUMBER_SRGB &&
2290 format != V_028C70_COLOR_8_24 &&
2291 format != V_028C70_COLOR_24_8) |
2292 S_028C70_NUMBER_TYPE(ntype) |
2293 S_028C70_ENDIAN(endian);
2294 if (iview->image->samples > 1)
2295 if (iview->image->fmask.size)
2296 cb->cb_color_info |= S_028C70_COMPRESSION(1);
2297
2298 if (iview->image->cmask.size &&
2299 !(device->debug_flags & RADV_DEBUG_NO_FAST_CLEARS))
2300 cb->cb_color_info |= S_028C70_FAST_CLEAR(1);
2301
2302 if (iview->image->surface.dcc_size && level_info->dcc_enabled)
2303 cb->cb_color_info |= S_028C70_DCC_ENABLE(1);
2304
2305 if (device->physical_device->rad_info.chip_class >= VI) {
2306 unsigned max_uncompressed_block_size = 2;
2307 if (iview->image->samples > 1) {
2308 if (iview->image->surface.bpe == 1)
2309 max_uncompressed_block_size = 0;
2310 else if (iview->image->surface.bpe == 2)
2311 max_uncompressed_block_size = 1;
2312 }
2313
2314 cb->cb_dcc_control = S_028C78_MAX_UNCOMPRESSED_BLOCK_SIZE(max_uncompressed_block_size) |
2315 S_028C78_INDEPENDENT_64B_BLOCKS(1);
2316 }
2317
2318 /* This must be set for fast clear to work without FMASK. */
2319 if (!iview->image->fmask.size &&
2320 device->physical_device->rad_info.chip_class == SI) {
2321 unsigned bankh = util_logbase2(iview->image->surface.bankh);
2322 cb->cb_color_attrib |= S_028C74_FMASK_BANK_HEIGHT(bankh);
2323 }
2324 }
2325
2326 static void
2327 radv_initialise_ds_surface(struct radv_device *device,
2328 struct radv_ds_buffer_info *ds,
2329 struct radv_image_view *iview)
2330 {
2331 unsigned level = iview->base_mip;
2332 unsigned format;
2333 uint64_t va, s_offs, z_offs;
2334 const struct radeon_surf_level *level_info = &iview->image->surface.level[level];
2335 memset(ds, 0, sizeof(*ds));
2336 switch (iview->vk_format) {
2337 case VK_FORMAT_D24_UNORM_S8_UINT:
2338 case VK_FORMAT_X8_D24_UNORM_PACK32:
2339 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-24);
2340 ds->offset_scale = 2.0f;
2341 break;
2342 case VK_FORMAT_D16_UNORM:
2343 case VK_FORMAT_D16_UNORM_S8_UINT:
2344 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-16);
2345 ds->offset_scale = 4.0f;
2346 break;
2347 case VK_FORMAT_D32_SFLOAT:
2348 case VK_FORMAT_D32_SFLOAT_S8_UINT:
2349 ds->pa_su_poly_offset_db_fmt_cntl = S_028B78_POLY_OFFSET_NEG_NUM_DB_BITS(-23) |
2350 S_028B78_POLY_OFFSET_DB_IS_FLOAT_FMT(1);
2351 ds->offset_scale = 1.0f;
2352 break;
2353 default:
2354 break;
2355 }
2356
2357 format = radv_translate_dbformat(iview->vk_format);
2358 if (format == V_028040_Z_INVALID) {
2359 fprintf(stderr, "Invalid DB format: %d, disabling DB.\n", iview->vk_format);
2360 }
2361
2362 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset;
2363 s_offs = z_offs = va;
2364 z_offs += iview->image->surface.level[level].offset;
2365 s_offs += iview->image->surface.stencil_level[level].offset;
2366
2367 uint32_t max_slice = radv_surface_layer_count(iview);
2368 ds->db_depth_view = S_028008_SLICE_START(iview->base_layer) |
2369 S_028008_SLICE_MAX(iview->base_layer + max_slice - 1);
2370 ds->db_depth_info = S_02803C_ADDR5_SWIZZLE_MASK(1);
2371 ds->db_z_info = S_028040_FORMAT(format) | S_028040_ZRANGE_PRECISION(1);
2372
2373 if (iview->image->samples > 1)
2374 ds->db_z_info |= S_028040_NUM_SAMPLES(util_logbase2(iview->image->samples));
2375
2376 if (iview->image->surface.flags & RADEON_SURF_SBUFFER)
2377 ds->db_stencil_info = S_028044_FORMAT(V_028044_STENCIL_8);
2378 else
2379 ds->db_stencil_info = S_028044_FORMAT(V_028044_STENCIL_INVALID);
2380
2381 if (device->physical_device->rad_info.chip_class >= CIK) {
2382 struct radeon_info *info = &device->physical_device->rad_info;
2383 unsigned tiling_index = iview->image->surface.tiling_index[level];
2384 unsigned stencil_index = iview->image->surface.stencil_tiling_index[level];
2385 unsigned macro_index = iview->image->surface.macro_tile_index;
2386 unsigned tile_mode = info->si_tile_mode_array[tiling_index];
2387 unsigned stencil_tile_mode = info->si_tile_mode_array[stencil_index];
2388 unsigned macro_mode = info->cik_macrotile_mode_array[macro_index];
2389
2390 ds->db_depth_info |=
2391 S_02803C_ARRAY_MODE(G_009910_ARRAY_MODE(tile_mode)) |
2392 S_02803C_PIPE_CONFIG(G_009910_PIPE_CONFIG(tile_mode)) |
2393 S_02803C_BANK_WIDTH(G_009990_BANK_WIDTH(macro_mode)) |
2394 S_02803C_BANK_HEIGHT(G_009990_BANK_HEIGHT(macro_mode)) |
2395 S_02803C_MACRO_TILE_ASPECT(G_009990_MACRO_TILE_ASPECT(macro_mode)) |
2396 S_02803C_NUM_BANKS(G_009990_NUM_BANKS(macro_mode));
2397 ds->db_z_info |= S_028040_TILE_SPLIT(G_009910_TILE_SPLIT(tile_mode));
2398 ds->db_stencil_info |= S_028044_TILE_SPLIT(G_009910_TILE_SPLIT(stencil_tile_mode));
2399 } else {
2400 unsigned tile_mode_index = si_tile_mode_index(iview->image, level, false);
2401 ds->db_z_info |= S_028040_TILE_MODE_INDEX(tile_mode_index);
2402 tile_mode_index = si_tile_mode_index(iview->image, level, true);
2403 ds->db_stencil_info |= S_028044_TILE_MODE_INDEX(tile_mode_index);
2404 }
2405
2406 if (iview->image->surface.htile_size && !level) {
2407 ds->db_z_info |= S_028040_TILE_SURFACE_ENABLE(1) |
2408 S_028040_ALLOW_EXPCLEAR(1);
2409
2410 if (iview->image->surface.flags & RADEON_SURF_SBUFFER) {
2411 /* Workaround: For a not yet understood reason, the
2412 * combination of MSAA, fast stencil clear and stencil
2413 * decompress messes with subsequent stencil buffer
2414 * uses. Problem was reproduced on Verde, Bonaire,
2415 * Tonga, and Carrizo.
2416 *
2417 * Disabling EXPCLEAR works around the problem.
2418 *
2419 * Check piglit's arb_texture_multisample-stencil-clear
2420 * test if you want to try changing this.
2421 */
2422 if (iview->image->samples <= 1)
2423 ds->db_stencil_info |= S_028044_ALLOW_EXPCLEAR(1);
2424 } else
2425 /* Use all of the htile_buffer for depth if there's no stencil. */
2426 ds->db_stencil_info |= S_028044_TILE_STENCIL_DISABLE(1);
2427
2428 va = device->ws->buffer_get_va(iview->bo) + iview->image->offset +
2429 iview->image->htile_offset;
2430 ds->db_htile_data_base = va >> 8;
2431 ds->db_htile_surface = S_028ABC_FULL_CACHE(1);
2432 } else {
2433 ds->db_htile_data_base = 0;
2434 ds->db_htile_surface = 0;
2435 }
2436
2437 ds->db_z_read_base = ds->db_z_write_base = z_offs >> 8;
2438 ds->db_stencil_read_base = ds->db_stencil_write_base = s_offs >> 8;
2439
2440 ds->db_depth_size = S_028058_PITCH_TILE_MAX((level_info->nblk_x / 8) - 1) |
2441 S_028058_HEIGHT_TILE_MAX((level_info->nblk_y / 8) - 1);
2442 ds->db_depth_slice = S_02805C_SLICE_TILE_MAX((level_info->nblk_x * level_info->nblk_y) / 64 - 1);
2443 }
2444
2445 VkResult radv_CreateFramebuffer(
2446 VkDevice _device,
2447 const VkFramebufferCreateInfo* pCreateInfo,
2448 const VkAllocationCallbacks* pAllocator,
2449 VkFramebuffer* pFramebuffer)
2450 {
2451 RADV_FROM_HANDLE(radv_device, device, _device);
2452 struct radv_framebuffer *framebuffer;
2453
2454 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO);
2455
2456 size_t size = sizeof(*framebuffer) +
2457 sizeof(struct radv_attachment_info) * pCreateInfo->attachmentCount;
2458 framebuffer = vk_alloc2(&device->alloc, pAllocator, size, 8,
2459 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2460 if (framebuffer == NULL)
2461 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
2462
2463 framebuffer->attachment_count = pCreateInfo->attachmentCount;
2464 framebuffer->width = pCreateInfo->width;
2465 framebuffer->height = pCreateInfo->height;
2466 framebuffer->layers = pCreateInfo->layers;
2467 for (uint32_t i = 0; i < pCreateInfo->attachmentCount; i++) {
2468 VkImageView _iview = pCreateInfo->pAttachments[i];
2469 struct radv_image_view *iview = radv_image_view_from_handle(_iview);
2470 framebuffer->attachments[i].attachment = iview;
2471 if (iview->aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) {
2472 radv_initialise_color_surface(device, &framebuffer->attachments[i].cb, iview);
2473 } else if (iview->aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
2474 radv_initialise_ds_surface(device, &framebuffer->attachments[i].ds, iview);
2475 }
2476 framebuffer->width = MIN2(framebuffer->width, iview->extent.width);
2477 framebuffer->height = MIN2(framebuffer->height, iview->extent.height);
2478 framebuffer->layers = MIN2(framebuffer->layers, radv_surface_layer_count(iview));
2479 }
2480
2481 *pFramebuffer = radv_framebuffer_to_handle(framebuffer);
2482 return VK_SUCCESS;
2483 }
2484
2485 void radv_DestroyFramebuffer(
2486 VkDevice _device,
2487 VkFramebuffer _fb,
2488 const VkAllocationCallbacks* pAllocator)
2489 {
2490 RADV_FROM_HANDLE(radv_device, device, _device);
2491 RADV_FROM_HANDLE(radv_framebuffer, fb, _fb);
2492
2493 if (!fb)
2494 return;
2495 vk_free2(&device->alloc, pAllocator, fb);
2496 }
2497
2498 static unsigned radv_tex_wrap(VkSamplerAddressMode address_mode)
2499 {
2500 switch (address_mode) {
2501 case VK_SAMPLER_ADDRESS_MODE_REPEAT:
2502 return V_008F30_SQ_TEX_WRAP;
2503 case VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT:
2504 return V_008F30_SQ_TEX_MIRROR;
2505 case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE:
2506 return V_008F30_SQ_TEX_CLAMP_LAST_TEXEL;
2507 case VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER:
2508 return V_008F30_SQ_TEX_CLAMP_BORDER;
2509 case VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE:
2510 return V_008F30_SQ_TEX_MIRROR_ONCE_LAST_TEXEL;
2511 default:
2512 unreachable("illegal tex wrap mode");
2513 break;
2514 }
2515 }
2516
2517 static unsigned
2518 radv_tex_compare(VkCompareOp op)
2519 {
2520 switch (op) {
2521 case VK_COMPARE_OP_NEVER:
2522 return V_008F30_SQ_TEX_DEPTH_COMPARE_NEVER;
2523 case VK_COMPARE_OP_LESS:
2524 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESS;
2525 case VK_COMPARE_OP_EQUAL:
2526 return V_008F30_SQ_TEX_DEPTH_COMPARE_EQUAL;
2527 case VK_COMPARE_OP_LESS_OR_EQUAL:
2528 return V_008F30_SQ_TEX_DEPTH_COMPARE_LESSEQUAL;
2529 case VK_COMPARE_OP_GREATER:
2530 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATER;
2531 case VK_COMPARE_OP_NOT_EQUAL:
2532 return V_008F30_SQ_TEX_DEPTH_COMPARE_NOTEQUAL;
2533 case VK_COMPARE_OP_GREATER_OR_EQUAL:
2534 return V_008F30_SQ_TEX_DEPTH_COMPARE_GREATEREQUAL;
2535 case VK_COMPARE_OP_ALWAYS:
2536 return V_008F30_SQ_TEX_DEPTH_COMPARE_ALWAYS;
2537 default:
2538 unreachable("illegal compare mode");
2539 break;
2540 }
2541 }
2542
2543 static unsigned
2544 radv_tex_filter(VkFilter filter, unsigned max_ansio)
2545 {
2546 switch (filter) {
2547 case VK_FILTER_NEAREST:
2548 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_POINT :
2549 V_008F38_SQ_TEX_XY_FILTER_POINT);
2550 case VK_FILTER_LINEAR:
2551 return (max_ansio > 1 ? V_008F38_SQ_TEX_XY_FILTER_ANISO_BILINEAR :
2552 V_008F38_SQ_TEX_XY_FILTER_BILINEAR);
2553 case VK_FILTER_CUBIC_IMG:
2554 default:
2555 fprintf(stderr, "illegal texture filter");
2556 return 0;
2557 }
2558 }
2559
2560 static unsigned
2561 radv_tex_mipfilter(VkSamplerMipmapMode mode)
2562 {
2563 switch (mode) {
2564 case VK_SAMPLER_MIPMAP_MODE_NEAREST:
2565 return V_008F38_SQ_TEX_Z_FILTER_POINT;
2566 case VK_SAMPLER_MIPMAP_MODE_LINEAR:
2567 return V_008F38_SQ_TEX_Z_FILTER_LINEAR;
2568 default:
2569 return V_008F38_SQ_TEX_Z_FILTER_NONE;
2570 }
2571 }
2572
2573 static unsigned
2574 radv_tex_bordercolor(VkBorderColor bcolor)
2575 {
2576 switch (bcolor) {
2577 case VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK:
2578 case VK_BORDER_COLOR_INT_TRANSPARENT_BLACK:
2579 return V_008F3C_SQ_TEX_BORDER_COLOR_TRANS_BLACK;
2580 case VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK:
2581 case VK_BORDER_COLOR_INT_OPAQUE_BLACK:
2582 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_BLACK;
2583 case VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE:
2584 case VK_BORDER_COLOR_INT_OPAQUE_WHITE:
2585 return V_008F3C_SQ_TEX_BORDER_COLOR_OPAQUE_WHITE;
2586 default:
2587 break;
2588 }
2589 return 0;
2590 }
2591
2592 static unsigned
2593 radv_tex_aniso_filter(unsigned filter)
2594 {
2595 if (filter < 2)
2596 return 0;
2597 if (filter < 4)
2598 return 1;
2599 if (filter < 8)
2600 return 2;
2601 if (filter < 16)
2602 return 3;
2603 return 4;
2604 }
2605
2606 static void
2607 radv_init_sampler(struct radv_device *device,
2608 struct radv_sampler *sampler,
2609 const VkSamplerCreateInfo *pCreateInfo)
2610 {
2611 uint32_t max_aniso = pCreateInfo->anisotropyEnable && pCreateInfo->maxAnisotropy > 1.0 ?
2612 (uint32_t) pCreateInfo->maxAnisotropy : 0;
2613 uint32_t max_aniso_ratio = radv_tex_aniso_filter(max_aniso);
2614 bool is_vi = (device->physical_device->rad_info.chip_class >= VI);
2615
2616 sampler->state[0] = (S_008F30_CLAMP_X(radv_tex_wrap(pCreateInfo->addressModeU)) |
2617 S_008F30_CLAMP_Y(radv_tex_wrap(pCreateInfo->addressModeV)) |
2618 S_008F30_CLAMP_Z(radv_tex_wrap(pCreateInfo->addressModeW)) |
2619 S_008F30_MAX_ANISO_RATIO(max_aniso_ratio) |
2620 S_008F30_DEPTH_COMPARE_FUNC(radv_tex_compare(pCreateInfo->compareOp)) |
2621 S_008F30_FORCE_UNNORMALIZED(pCreateInfo->unnormalizedCoordinates ? 1 : 0) |
2622 S_008F30_ANISO_THRESHOLD(max_aniso_ratio >> 1) |
2623 S_008F30_ANISO_BIAS(max_aniso_ratio) |
2624 S_008F30_DISABLE_CUBE_WRAP(0) |
2625 S_008F30_COMPAT_MODE(is_vi));
2626 sampler->state[1] = (S_008F34_MIN_LOD(S_FIXED(CLAMP(pCreateInfo->minLod, 0, 15), 8)) |
2627 S_008F34_MAX_LOD(S_FIXED(CLAMP(pCreateInfo->maxLod, 0, 15), 8)) |
2628 S_008F34_PERF_MIP(max_aniso_ratio ? max_aniso_ratio + 6 : 0));
2629 sampler->state[2] = (S_008F38_LOD_BIAS(S_FIXED(CLAMP(pCreateInfo->mipLodBias, -16, 16), 8)) |
2630 S_008F38_XY_MAG_FILTER(radv_tex_filter(pCreateInfo->magFilter, max_aniso)) |
2631 S_008F38_XY_MIN_FILTER(radv_tex_filter(pCreateInfo->minFilter, max_aniso)) |
2632 S_008F38_MIP_FILTER(radv_tex_mipfilter(pCreateInfo->mipmapMode)) |
2633 S_008F38_MIP_POINT_PRECLAMP(0) |
2634 S_008F38_DISABLE_LSB_CEIL(1) |
2635 S_008F38_FILTER_PREC_FIX(1) |
2636 S_008F38_ANISO_OVERRIDE(is_vi));
2637 sampler->state[3] = (S_008F3C_BORDER_COLOR_PTR(0) |
2638 S_008F3C_BORDER_COLOR_TYPE(radv_tex_bordercolor(pCreateInfo->borderColor)));
2639 }
2640
2641 VkResult radv_CreateSampler(
2642 VkDevice _device,
2643 const VkSamplerCreateInfo* pCreateInfo,
2644 const VkAllocationCallbacks* pAllocator,
2645 VkSampler* pSampler)
2646 {
2647 RADV_FROM_HANDLE(radv_device, device, _device);
2648 struct radv_sampler *sampler;
2649
2650 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO);
2651
2652 sampler = vk_alloc2(&device->alloc, pAllocator, sizeof(*sampler), 8,
2653 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2654 if (!sampler)
2655 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
2656
2657 radv_init_sampler(device, sampler, pCreateInfo);
2658 *pSampler = radv_sampler_to_handle(sampler);
2659
2660 return VK_SUCCESS;
2661 }
2662
2663 void radv_DestroySampler(
2664 VkDevice _device,
2665 VkSampler _sampler,
2666 const VkAllocationCallbacks* pAllocator)
2667 {
2668 RADV_FROM_HANDLE(radv_device, device, _device);
2669 RADV_FROM_HANDLE(radv_sampler, sampler, _sampler);
2670
2671 if (!sampler)
2672 return;
2673 vk_free2(&device->alloc, pAllocator, sampler);
2674 }
2675
2676
2677 /* vk_icd.h does not declare this function, so we declare it here to
2678 * suppress Wmissing-prototypes.
2679 */
2680 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
2681 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion);
2682
2683 PUBLIC VKAPI_ATTR VkResult VKAPI_CALL
2684 vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion)
2685 {
2686 /* For the full details on loader interface versioning, see
2687 * <https://github.com/KhronosGroup/Vulkan-LoaderAndValidationLayers/blob/master/loader/LoaderAndLayerInterface.md>.
2688 * What follows is a condensed summary, to help you navigate the large and
2689 * confusing official doc.
2690 *
2691 * - Loader interface v0 is incompatible with later versions. We don't
2692 * support it.
2693 *
2694 * - In loader interface v1:
2695 * - The first ICD entrypoint called by the loader is
2696 * vk_icdGetInstanceProcAddr(). The ICD must statically expose this
2697 * entrypoint.
2698 * - The ICD must statically expose no other Vulkan symbol unless it is
2699 * linked with -Bsymbolic.
2700 * - Each dispatchable Vulkan handle created by the ICD must be
2701 * a pointer to a struct whose first member is VK_LOADER_DATA. The
2702 * ICD must initialize VK_LOADER_DATA.loadMagic to ICD_LOADER_MAGIC.
2703 * - The loader implements vkCreate{PLATFORM}SurfaceKHR() and
2704 * vkDestroySurfaceKHR(). The ICD must be capable of working with
2705 * such loader-managed surfaces.
2706 *
2707 * - Loader interface v2 differs from v1 in:
2708 * - The first ICD entrypoint called by the loader is
2709 * vk_icdNegotiateLoaderICDInterfaceVersion(). The ICD must
2710 * statically expose this entrypoint.
2711 *
2712 * - Loader interface v3 differs from v2 in:
2713 * - The ICD must implement vkCreate{PLATFORM}SurfaceKHR(),
2714 * vkDestroySurfaceKHR(), and other API which uses VKSurfaceKHR,
2715 * because the loader no longer does so.
2716 */
2717 *pSupportedVersion = MIN2(*pSupportedVersion, 3u);
2718 return VK_SUCCESS;
2719 }