*/
#include "dirent.h"
-#include <errno.h>
-#include <fcntl.h>
-#include <linux/audit.h>
-#include <linux/bpf.h>
-#include <linux/filter.h>
-#include <linux/seccomp.h>
-#include <linux/unistd.h>
+
#include <stdatomic.h>
#include <stdbool.h>
-#include <stddef.h>
-#include <stdio.h>
#include <string.h>
-#include <sys/prctl.h>
-#include <sys/wait.h>
#include <unistd.h>
#include <fcntl.h>
disk_cache_format_hex_id(buf, device->cache_uuid, VK_UUID_SIZE * 2);
device->disk_cache = disk_cache_create(device->name, buf, shader_env_flags);
- if (device->rad_info.chip_class < GFX8 || !device->use_llvm)
+ if (device->rad_info.chip_class < GFX8)
fprintf(stderr, "WARNING: radv is not a conformant vulkan implementation, testing use only.\n");
radv_get_driver_uuid(&device->driver_uuid);
DRI_CONF_RADV_REPORT_LLVM9_VERSION_STRING("false")
DRI_CONF_RADV_ENABLE_MRT_OUTPUT_NAN_FIXUP("false")
DRI_CONF_RADV_NO_DYNAMIC_BOUNDS("false")
+ DRI_CONF_RADV_OVERRIDE_UNIFORM_OFFSET_ALIGNMENT(0)
DRI_CONF_SECTION_END
DRI_CONF_SECTION_DEBUG
driParseConfigFiles(&instance->dri_options,
&instance->available_dri_options,
0, "radv", NULL,
+ instance->applicationName,
+ instance->applicationVersion,
instance->engineName,
instance->engineVersion);
}
if (pCreateInfo->pApplicationInfo) {
const VkApplicationInfo *app = pCreateInfo->pApplicationInfo;
+ instance->applicationName =
+ vk_strdup(&instance->alloc, app->pApplicationName,
+ VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+ instance->applicationVersion = app->applicationVersion;
+
instance->engineName =
vk_strdup(&instance->alloc, app->pEngineName,
VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
}
vk_free(&instance->alloc, instance->engineName);
+ vk_free(&instance->alloc, instance->applicationName);
VG(VALGRIND_DESTROY_MEMPOOL(instance));
64 /* storage image */);
}
+static uint32_t
+radv_uniform_buffer_offset_alignment(const struct radv_physical_device *pdevice)
+{
+ uint32_t uniform_offset_alignment = driQueryOptioni(&pdevice->instance->dri_options,
+ "radv_override_uniform_offset_alignment");
+ if (!util_is_power_of_two_or_zero(uniform_offset_alignment)) {
+ fprintf(stderr, "ERROR: invalid radv_override_uniform_offset_alignment setting %d:"
+ "not a power of two\n", uniform_offset_alignment);
+ uniform_offset_alignment = 0;
+ }
+
+ /* Take at least the hardware limit. */
+ return MAX2(uniform_offset_alignment, 4);
+}
+
void radv_GetPhysicalDeviceProperties(
VkPhysicalDevice physicalDevice,
VkPhysicalDeviceProperties* pProperties)
.viewportSubPixelBits = 8,
.minMemoryMapAlignment = 4096, /* A page */
.minTexelBufferOffsetAlignment = 4,
- .minUniformBufferOffsetAlignment = 4,
+ .minUniformBufferOffsetAlignment = radv_uniform_buffer_offset_alignment(pdevice),
.minStorageBufferOffsetAlignment = 4,
.minTexelOffset = -32,
.maxTexelOffset = 31,
p->conformanceVersion = (VkConformanceVersion) {
.major = 1,
.minor = 2,
- .subminor = 0,
+ .subminor = 3,
.patch = 0,
};
static void
radv_bo_list_init(struct radv_bo_list *bo_list)
{
- pthread_mutex_init(&bo_list->mutex, NULL);
+ pthread_rwlock_init(&bo_list->rwlock, NULL);
bo_list->list.count = bo_list->capacity = 0;
bo_list->list.bos = NULL;
}
radv_bo_list_finish(struct radv_bo_list *bo_list)
{
free(bo_list->list.bos);
- pthread_mutex_destroy(&bo_list->mutex);
+ pthread_rwlock_destroy(&bo_list->rwlock);
}
VkResult radv_bo_list_add(struct radv_device *device,
if (unlikely(!device->use_global_bo_list))
return VK_SUCCESS;
- pthread_mutex_lock(&bo_list->mutex);
+ pthread_rwlock_wrlock(&bo_list->rwlock);
if (bo_list->list.count == bo_list->capacity) {
unsigned capacity = MAX2(4, bo_list->capacity * 2);
void *data = realloc(bo_list->list.bos, capacity * sizeof(struct radeon_winsys_bo*));
if (!data) {
- pthread_mutex_unlock(&bo_list->mutex);
+ pthread_rwlock_unlock(&bo_list->rwlock);
return VK_ERROR_OUT_OF_HOST_MEMORY;
}
}
bo_list->list.bos[bo_list->list.count++] = bo;
- pthread_mutex_unlock(&bo_list->mutex);
+ pthread_rwlock_unlock(&bo_list->rwlock);
return VK_SUCCESS;
}
if (unlikely(!device->use_global_bo_list))
return;
- pthread_mutex_lock(&bo_list->mutex);
+ pthread_rwlock_wrlock(&bo_list->rwlock);
/* Loop the list backwards so we find the most recently added
* memory first. */
for(unsigned i = bo_list->list.count; i-- > 0;) {
break;
}
}
- pthread_mutex_unlock(&bo_list->mutex);
+ pthread_rwlock_unlock(&bo_list->rwlock);
}
static void
return result;
}
+static bool radv_thread_trace_enabled()
+{
+ return radv_get_int_debug_option("RADV_THREAD_TRACE", -1) >= 0 ||
+ getenv("RADV_THREAD_TRACE_TRIGGER");
+}
+
static void
radv_device_init_dispatch(struct radv_device *device)
{
const struct radv_instance *instance = device->physical_device->instance;
const struct radv_device_dispatch_table *dispatch_table_layer = NULL;
bool unchecked = instance->debug_flags & RADV_DEBUG_ALL_ENTRYPOINTS;
- int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
- if (radv_thread_trace >= 0) {
+ if (radv_thread_trace_enabled()) {
/* Use device entrypoints from the SQTT layer if enabled. */
dispatch_table_layer = &sqtt_device_dispatch_table;
}
}
}
+VkResult
+_radv_device_set_lost(struct radv_device *device,
+ const char *file, int line,
+ const char *msg, ...)
+{
+ VkResult err;
+ va_list ap;
+
+ p_atomic_inc(&device->lost);
+
+ va_start(ap, msg);
+ err = __vk_errorv(device->physical_device->instance, device,
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT,
+ VK_ERROR_DEVICE_LOST, file, line, msg, ap);
+ va_end(ap);
+
+ return err;
+}
+
VkResult radv_CreateDevice(
VkPhysicalDevice physicalDevice,
const VkDeviceCreateInfo* pCreateInfo,
fprintf(stderr, "*****************************************************************************\n");
fprintf(stderr, "Trace file will be dumped to %s\n", filename);
+
+ /* Wait for idle after every draw/dispatch to identify the
+ * first bad call.
+ */
+ device->instance->debug_flags |= RADV_DEBUG_SYNC_SHADERS;
+
radv_dump_enabled_options(device, stderr);
}
- int radv_thread_trace = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
- if (radv_thread_trace >= 0) {
+ if (radv_thread_trace_enabled()) {
fprintf(stderr, "*************************************************\n");
fprintf(stderr, "* WARNING: Thread trace support is experimental *\n");
fprintf(stderr, "*************************************************\n");
/* Default buffer size set to 1MB per SE. */
device->thread_trace_buffer_size =
radv_get_int_debug_option("RADV_THREAD_TRACE_BUFFER_SIZE", 1024 * 1024);
- device->thread_trace_start_frame = radv_thread_trace;
+ device->thread_trace_start_frame = radv_get_int_debug_option("RADV_THREAD_TRACE", -1);
+
+ const char *trigger_file = getenv("RADV_THREAD_TRACE_TRIGGER");
+ if (trigger_file)
+ device->thread_trace_trigger_file = strdup(trigger_file);
if (!radv_thread_trace_init(device))
goto fail;
}
+ if (getenv("RADV_TRAP_HANDLER")) {
+ /* TODO: Add support for more hardware. */
+ assert(device->physical_device->rad_info.chip_class == GFX8);
+
+ fprintf(stderr, "**********************************************************************\n");
+ fprintf(stderr, "* WARNING: RADV_TRAP_HANDLER is experimental and only for debugging! *\n");
+ fprintf(stderr, "**********************************************************************\n");
+
+ /* To get the disassembly of the faulty shaders, we have to
+ * keep some shader info around.
+ */
+ keep_shader_info = true;
+
+ if (!radv_trap_handler_init(device))
+ goto fail;
+ }
+
device->keep_shader_info = keep_shader_info;
result = radv_device_init_meta(device);
if (result != VK_SUCCESS)
radv_bo_list_finish(&device->bo_list);
radv_thread_trace_finish(device);
+ free(device->thread_trace_trigger_file);
+
+ radv_trap_handler_finish(device);
if (device->trace_bo)
device->ws->buffer_destroy(device->trace_bo);
VkPipelineCache pc = radv_pipeline_cache_to_handle(device->mem_cache);
radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
+ radv_trap_handler_finish(device);
+
radv_destroy_shader_slabs(device);
pthread_cond_destroy(&device->timeline_cond);
radv_bo_list_finish(&device->bo_list);
+ free(device->thread_trace_trigger_file);
radv_thread_trace_finish(device);
vk_free(&device->vk.alloc, device);
if (device->physical_device->rad_info.chip_class >= GFX8)
--max_offchip_buffers;
hs_offchip_param =
- S_03093C_OFFCHIP_BUFFERING(max_offchip_buffers) |
- S_03093C_OFFCHIP_GRANULARITY(offchip_granularity);
+ S_03093C_OFFCHIP_BUFFERING_GFX7(max_offchip_buffers) |
+ S_03093C_OFFCHIP_GRANULARITY_GFX7(offchip_granularity);
} else {
hs_offchip_param =
S_0089B0_OFFCHIP_BUFFERING(max_offchip_buffers);
}
}
+static void
+radv_emit_trap_handler(struct radv_queue *queue,
+ struct radeon_cmdbuf *cs,
+ struct radeon_winsys_bo *tma_bo)
+{
+ struct radv_device *device = queue->device;
+ struct radeon_winsys_bo *tba_bo;
+ uint64_t tba_va, tma_va;
+
+ if (!device->trap_handler_shader || !tma_bo)
+ return;
+
+ tba_bo = device->trap_handler_shader->bo;
+
+ tba_va = radv_buffer_get_va(tba_bo) + device->trap_handler_shader->bo_offset;
+ tma_va = radv_buffer_get_va(tma_bo);
+
+ radv_cs_add_buffer(queue->device->ws, cs, tba_bo);
+ radv_cs_add_buffer(queue->device->ws, cs, tma_bo);
+
+ if (queue->queue_family_index == RADV_QUEUE_GENERAL) {
+ uint32_t regs[] = {R_00B000_SPI_SHADER_TBA_LO_PS,
+ R_00B100_SPI_SHADER_TBA_LO_VS,
+ R_00B200_SPI_SHADER_TBA_LO_GS,
+ R_00B300_SPI_SHADER_TBA_LO_ES,
+ R_00B400_SPI_SHADER_TBA_LO_HS,
+ R_00B500_SPI_SHADER_TBA_LO_LS};
+
+ for (int i = 0; i < ARRAY_SIZE(regs); ++i) {
+ radeon_set_sh_reg_seq(cs, regs[i], 4);
+ radeon_emit(cs, tba_va >> 8);
+ radeon_emit(cs, tba_va >> 40);
+ radeon_emit(cs, tma_va >> 8);
+ radeon_emit(cs, tma_va >> 40);
+ }
+ } else {
+ radeon_set_sh_reg_seq(cs, R_00B838_COMPUTE_TBA_LO, 4);
+ radeon_emit(cs, tba_va >> 8);
+ radeon_emit(cs, tba_va >> 40);
+ radeon_emit(cs, tma_va >> 8);
+ radeon_emit(cs, tma_va >> 40);
+ }
+}
+
static void
radv_init_graphics_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
{
static void
radv_init_compute_state(struct radeon_cmdbuf *cs, struct radv_queue *queue)
{
- struct radv_physical_device *physical_device = queue->device->physical_device;
- si_emit_compute(physical_device, cs);
+ si_emit_compute(queue->device, cs);
}
static VkResult
compute_scratch_waves, compute_scratch_bo);
radv_emit_graphics_scratch(queue, cs, scratch_size_per_wave,
scratch_waves, scratch_bo);
+ radv_emit_trap_handler(queue, cs, queue->device->tma_bo);
if (gds_bo)
radv_cs_add_buffer(queue->device->ws, cs, gds_bo);
* submitted, but if the queue was empty, we decrement ourselves as there is no previous
* submission. */
uint32_t decrement = submission->wait_semaphore_count - wait_cnt + (is_first ? 1 : 0);
+
+ /* if decrement is zero, then we don't have a refcounted reference to the
+ * submission anymore, so it is not safe to access the submission. */
+ if (!decrement)
+ return VK_SUCCESS;
+
return radv_queue_trigger_submission(submission, decrement, processing_list);
}
sem_info.cs_emit_signal = j + advance == submission->cmd_buffer_count;
if (unlikely(queue->device->use_global_bo_list)) {
- pthread_mutex_lock(&queue->device->bo_list.mutex);
+ pthread_rwlock_rdlock(&queue->device->bo_list.rwlock);
bo_list = &queue->device->bo_list.list;
}
can_patch, base_fence);
if (unlikely(queue->device->use_global_bo_list))
- pthread_mutex_unlock(&queue->device->bo_list.mutex);
+ pthread_rwlock_unlock(&queue->device->bo_list.rwlock);
if (result != VK_SUCCESS)
goto fail;
if (queue->device->trace_bo) {
radv_check_gpu_hangs(queue, cs_array[j]);
}
+
+ if (queue->device->tma_bo) {
+ radv_check_trap_handler(queue);
+ }
}
free(cs_array);
* VK_ERROR_DEVICE_LOST to ensure the clients do not attempt
* to submit the same job again to this device.
*/
- result = VK_ERROR_DEVICE_LOST;
+ result = radv_device_set_lost(queue->device, "vkQueueSubmit() failed");
}
radv_free_temp_syncobjs(queue->device,
uint32_t fence_idx = 0;
bool flushed_caches = false;
+ if (radv_device_is_lost(queue->device))
+ return VK_ERROR_DEVICE_LOST;
+
if (fence != VK_NULL_HANDLE) {
for (uint32_t i = 0; i < submitCount; ++i)
if (radv_submit_has_effects(pSubmits + i))
{
RADV_FROM_HANDLE(radv_queue, queue, _queue);
+ if (radv_device_is_lost(queue->device))
+ return VK_ERROR_DEVICE_LOST;
+
pthread_mutex_lock(&queue->pending_mutex);
while (!list_is_empty(&queue->pending_submissions)) {
pthread_cond_wait(&queue->device->timeline_cond, &queue->pending_mutex);
if (!queue->device->ws->ctx_wait_idle(queue->hw_ctx,
radv_queue_family_to_ring(queue->queue_family_index),
queue->queue_idx)) {
- return vk_errorf(queue->device->instance, VK_ERROR_DEVICE_LOST,
- "Failed to wait for a '%s' queue to be idle. "
- "GPU hang ?", radv_get_queue_family_name(queue));
+ return radv_device_set_lost(queue->device,
+ "Failed to wait for a '%s' queue "
+ "to be idle. GPU hang ?",
+ radv_get_queue_family_name(queue));
}
return VK_SUCCESS;
} else {
close(import_info->fd);
}
+
+ if (mem->image && mem->image->plane_count == 1 &&
+ !vk_format_is_depth_or_stencil(mem->image->vk_format)) {
+ struct radeon_bo_metadata metadata;
+ device->ws->buffer_get_metadata(mem->bo, &metadata);
+
+ struct radv_image_create_info create_info = {
+ .no_metadata_planes = true,
+ .bo_metadata = &metadata
+ };
+
+ /* This gives a basic ability to import radeonsi images
+ * that don't have DCC. This is not guaranteed by any
+ * spec and can be removed after we support modifiers. */
+ result = radv_image_create_layout(device, create_info, mem->image);
+ if (result != VK_SUCCESS) {
+ device->ws->buffer_destroy(mem->bo);
+ goto fail;
+ }
+ }
} else if (host_ptr_info) {
assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
mem->bo = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer,
VkResult result;
uint32_t fence_idx = 0;
+ if (radv_device_is_lost(queue->device))
+ return VK_ERROR_DEVICE_LOST;
+
if (fence != VK_NULL_HANDLE) {
for (uint32_t i = 0; i < bindInfoCount; ++i)
if (radv_sparse_bind_has_effects(pBindInfo + i))
uint64_t timeout)
{
RADV_FROM_HANDLE(radv_device, device, _device);
+
+ if (radv_device_is_lost(device))
+ return VK_ERROR_DEVICE_LOST;
+
timeout = radv_get_absolute_timeout(timeout);
if (device->always_use_syncobj &&
fence->temporary.kind != RADV_FENCE_NONE ?
&fence->temporary : &fence->permanent;
+ if (radv_device_is_lost(device))
+ return VK_ERROR_DEVICE_LOST;
+
switch (part->kind) {
case RADV_FENCE_NONE:
break;
RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_semaphore, semaphore, _semaphore);
+ if (radv_device_is_lost(device))
+ return VK_ERROR_DEVICE_LOST;
+
struct radv_semaphore_part *part =
semaphore->temporary.kind != RADV_SEMAPHORE_NONE ? &semaphore->temporary : &semaphore->permanent;
uint64_t timeout)
{
RADV_FROM_HANDLE(radv_device, device, _device);
+
+ if (radv_device_is_lost(device))
+ return VK_ERROR_DEVICE_LOST;
+
uint64_t abs_timeout = radv_get_absolute_timeout(timeout);
if (radv_semaphore_from_handle(pWaitInfo->pSemaphores[0])->permanent.kind == RADV_SEMAPHORE_TIMELINE)
VkDevice _device,
VkEvent _event)
{
+ RADV_FROM_HANDLE(radv_device, device, _device);
RADV_FROM_HANDLE(radv_event, event, _event);
+ if (radv_device_is_lost(device))
+ return VK_ERROR_DEVICE_LOST;
+
if (*event->map == 1)
return VK_EVENT_SET;
return VK_EVENT_RESET;
sampler->state[2] |=
S_008F38_DISABLE_LSB_CEIL(device->physical_device->rad_info.chip_class <= GFX8) |
S_008F38_FILTER_PREC_FIX(1) |
- S_008F38_ANISO_OVERRIDE_GFX6(device->physical_device->rad_info.chip_class >= GFX8);
+ S_008F38_ANISO_OVERRIDE_GFX8(device->physical_device->rad_info.chip_class >= GFX8);
}
}
static const VkTimeDomainEXT radv_time_domains[] = {
VK_TIME_DOMAIN_DEVICE_EXT,
VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT,
+#ifdef CLOCK_MONOTONIC_RAW
VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT,
+#endif
};
VkResult radv_GetPhysicalDeviceCalibrateableTimeDomainsEXT(
int ret;
ret = clock_gettime(clock_id, ¤t);
+#ifdef CLOCK_MONOTONIC_RAW
if (ret < 0 && clock_id == CLOCK_MONOTONIC_RAW)
ret = clock_gettime(CLOCK_MONOTONIC, ¤t);
+#endif
if (ret < 0)
return 0;
uint64_t begin, end;
uint64_t max_clock_period = 0;
+#ifdef CLOCK_MONOTONIC_RAW
begin = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
+#else
+ begin = radv_clock_gettime(CLOCK_MONOTONIC);
+#endif
for (d = 0; d < timestampCount; d++) {
switch (pTimestampInfos[d].timeDomain) {
max_clock_period = MAX2(max_clock_period, 1);
break;
+#ifdef CLOCK_MONOTONIC_RAW
case VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_EXT:
pTimestamps[d] = begin;
break;
+#endif
default:
pTimestamps[d] = 0;
break;
}
}
+#ifdef CLOCK_MONOTONIC_RAW
end = radv_clock_gettime(CLOCK_MONOTONIC_RAW);
+#else
+ end = radv_clock_gettime(CLOCK_MONOTONIC);
+#endif
/*
* The maximum deviation is the sum of the interval over which we
projects / mesa.git / blobdiff