From 9bd7928a35c27d3d0898db83bc8db823a6dbee5e Mon Sep 17 00:00:00 2001 From: =?utf8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Sat, 26 Sep 2015 03:15:40 +0200 Subject: [PATCH] radeonsi: add an option for debugging VM faults MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Reviewed-by: Michel Dänzer --- src/gallium/drivers/radeon/r600_pipe_common.c | 1 + src/gallium/drivers/radeon/r600_pipe_common.h | 1 + src/gallium/drivers/radeonsi/si_debug.c | 113 ++++++++++++++++++ src/gallium/drivers/radeonsi/si_hw_context.c | 4 + src/gallium/drivers/radeonsi/si_pipe.c | 3 + src/gallium/drivers/radeonsi/si_pipe.h | 2 + 6 files changed, 124 insertions(+) diff --git a/src/gallium/drivers/radeon/r600_pipe_common.c b/src/gallium/drivers/radeon/r600_pipe_common.c index 08839343b74..7ac94caad9f 100644 --- a/src/gallium/drivers/radeon/r600_pipe_common.c +++ b/src/gallium/drivers/radeon/r600_pipe_common.c @@ -359,6 +359,7 @@ static const struct debug_named_value common_debug_options[] = { { "forcedma", DBG_FORCE_DMA, "Use asynchronous DMA for all operations when possible." }, { "precompile", DBG_PRECOMPILE, "Compile one shader variant at shader creation." }, { "nowc", DBG_NO_WC, "Disable GTT write combining" }, + { "check_vm", DBG_CHECK_VM, "Check VM faults and dump debug info." }, DEBUG_NAMED_VALUE_END /* must be last */ }; diff --git a/src/gallium/drivers/radeon/r600_pipe_common.h b/src/gallium/drivers/radeon/r600_pipe_common.h index 534b987a2cc..2df93e54559 100644 --- a/src/gallium/drivers/radeon/r600_pipe_common.h +++ b/src/gallium/drivers/radeon/r600_pipe_common.h @@ -98,6 +98,7 @@ #define DBG_PRECOMPILE (1llu << 39) #define DBG_INFO (1llu << 40) #define DBG_NO_WC (1llu << 41) +#define DBG_CHECK_VM (1llu << 42) #define R600_MAP_BUFFER_ALIGNMENT 64 diff --git a/src/gallium/drivers/radeonsi/si_debug.c b/src/gallium/drivers/radeonsi/si_debug.c index ccdc6921896..3d127236831 100644 --- a/src/gallium/drivers/radeonsi/si_debug.c +++ b/src/gallium/drivers/radeonsi/si_debug.c @@ -28,6 +28,7 @@ #include "si_shader.h" #include "sid.h" #include "sid_tables.h" +#include "ddebug/dd_util.h" static void si_dump_shader(struct si_shader_selector *sel, const char *name, @@ -438,7 +439,119 @@ static void si_dump_debug_state(struct pipe_context *ctx, FILE *f, fprintf(f, "Done.\n"); } +static bool si_vm_fault_occured(struct si_context *sctx, uint32_t *out_addr) +{ + char line[2000]; + unsigned sec, usec; + int progress = 0; + uint64_t timestamp = 0; + bool fault = false; + + FILE *p = popen("dmesg", "r"); + if (!p) + return false; + + while (fgets(line, sizeof(line), p)) { + char *msg, len; + + /* Get the timestamp. */ + if (sscanf(line, "[%u.%u]", &sec, &usec) != 2) { + assert(0); + continue; + } + timestamp = sec * 1000000llu + usec; + + /* If just updating the timestamp. */ + if (!out_addr) + continue; + + /* Process messages only if the timestamp is newer. */ + if (timestamp <= sctx->dmesg_timestamp) + continue; + + /* Only process the first VM fault. */ + if (fault) + continue; + + /* Remove trailing \n */ + len = strlen(line); + if (len && line[len-1] == '\n') + line[len-1] = 0; + + /* Get the message part. */ + msg = strchr(line, ']'); + if (!msg) { + assert(0); + continue; + } + msg++; + + switch (progress) { + case 0: + if (strstr(msg, "GPU fault detected:")) + progress = 1; + break; + case 1: + msg = strstr(msg, "VM_CONTEXT1_PROTECTION_FAULT_ADDR"); + if (msg) { + msg = strstr(msg, "0x"); + if (msg) { + msg += 2; + if (sscanf(msg, "%X", out_addr) == 1) + fault = true; + } + } + progress = 0; + break; + default: + progress = 0; + } + } + pclose(p); + + if (timestamp > sctx->dmesg_timestamp) + sctx->dmesg_timestamp = timestamp; + return fault; +} + +void si_check_vm_faults(struct si_context *sctx) +{ + struct pipe_screen *screen = sctx->b.b.screen; + FILE *f; + uint32_t addr; + + /* Use conservative timeout 800ms, after which we won't wait any + * longer and assume the GPU is hung. + */ + screen->fence_finish(screen, sctx->last_gfx_fence, 800*1000*1000); + + if (!si_vm_fault_occured(sctx, &addr)) + return; + + f = dd_get_debug_file(); + if (!f) + return; + + fprintf(f, "VM fault report.\n\n"); + fprintf(f, "Driver vendor: %s\n", screen->get_vendor(screen)); + fprintf(f, "Device vendor: %s\n", screen->get_device_vendor(screen)); + fprintf(f, "Device name: %s\n\n", screen->get_name(screen)); + fprintf(f, "Failing VM page: 0x%08x\n\n", addr); + + si_dump_last_ib(sctx, f); + fclose(f); + + fprintf(stderr, "Detected a VM fault, exiting...\n"); + exit(0); +} + void si_init_debug_functions(struct si_context *sctx) { sctx->b.b.dump_debug_state = si_dump_debug_state; + + /* Set the initial dmesg timestamp for this context, so that + * only new messages will be checked for VM faults. + */ + if (sctx->screen->b.debug_flags & DBG_CHECK_VM) + si_vm_fault_occured(sctx, NULL); } diff --git a/src/gallium/drivers/radeonsi/si_hw_context.c b/src/gallium/drivers/radeonsi/si_hw_context.c index 1d5d42657e4..c789292e742 100644 --- a/src/gallium/drivers/radeonsi/si_hw_context.c +++ b/src/gallium/drivers/radeonsi/si_hw_context.c @@ -103,6 +103,10 @@ void si_context_gfx_flush(void *context, unsigned flags, if (fence) ws->fence_reference(fence, ctx->last_gfx_fence); + /* Check VM faults if needed. */ + if (ctx->screen->b.debug_flags & DBG_CHECK_VM) + si_check_vm_faults(ctx); + si_begin_new_cs(ctx); } diff --git a/src/gallium/drivers/radeonsi/si_pipe.c b/src/gallium/drivers/radeonsi/si_pipe.c index 9edee50ac8a..5a2b60620e3 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.c +++ b/src/gallium/drivers/radeonsi/si_pipe.c @@ -107,6 +107,9 @@ static struct pipe_context *si_create_context(struct pipe_screen *screen, if (sctx == NULL) return NULL; + if (sscreen->b.debug_flags & DBG_CHECK_VM) + flags |= PIPE_CONTEXT_DEBUG; + sctx->b.b.screen = screen; /* this must be set first */ sctx->b.b.priv = priv; sctx->b.b.destroy = si_destroy_context; diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h index 847853e59e9..1c26022bb1b 100644 --- a/src/gallium/drivers/radeonsi/si_pipe.h +++ b/src/gallium/drivers/radeonsi/si_pipe.h @@ -276,6 +276,7 @@ struct si_context { struct r600_resource *last_trace_buf; struct r600_resource *trace_buf; unsigned trace_id; + uint64_t dmesg_timestamp; }; /* cik_sdma.c */ @@ -310,6 +311,7 @@ void si_init_cp_dma_functions(struct si_context *sctx); /* si_debug.c */ void si_init_debug_functions(struct si_context *sctx); +void si_check_vm_faults(struct si_context *sctx); /* si_dma.c */ void si_dma_copy(struct pipe_context *ctx, -- 2.30.2