#include <stdint.h>
#include <xf86drm.h>
-/*
- * this are copy from radeon_drm, once an updated libdrm is released
- * we should bump configure.ac requirement for it and remove the following
- * field
- */
-#ifndef RADEON_CHUNK_ID_FLAGS
-#define RADEON_CHUNK_ID_FLAGS 0x03
-
-/* The first dword of RADEON_CHUNK_ID_FLAGS is a uint32 of these flags: */
-#define RADEON_CS_KEEP_TILING_FLAGS 0x01
-#endif
-
-#ifndef RADEON_CS_USE_VM
-#define RADEON_CS_USE_VM 0x02
-/* The second dword of RADEON_CHUNK_ID_FLAGS is a uint32 that sets the ring type */
-#define RADEON_CS_RING_GFX 0
-#define RADEON_CS_RING_COMPUTE 1
-#endif
-
-#ifndef RADEON_CS_RING_DMA
-#define RADEON_CS_RING_DMA 2
-#endif
-
-#ifndef RADEON_CS_RING_UVD
-#define RADEON_CS_RING_UVD 3
-#endif
-
-#ifndef RADEON_CS_RING_VCE
-#define RADEON_CS_RING_VCE 4
-#endif
-
-#ifndef RADEON_CS_END_OF_FRAME
-#define RADEON_CS_END_OF_FRAME 0x04
-#endif
-
#define RELOC_DWORDS (sizeof(struct drm_radeon_cs_reloc) / sizeof(uint32_t))
+static struct pipe_fence_handle *
+radeon_cs_create_fence(struct radeon_winsys_cs *rcs);
+static void radeon_fence_reference(struct pipe_fence_handle **dst,
+ struct pipe_fence_handle *src);
+
static boolean radeon_init_cs_context(struct radeon_cs_context *csc,
struct radeon_drm_winsys *ws)
{
+ int i;
+
+ csc->buf = MALLOC(ws->ib_max_size);
+ if (!csc->buf)
+ return FALSE;
csc->fd = ws->fd;
csc->nrelocs = 512;
csc->relocs_bo = (struct radeon_bo**)
CALLOC(1, csc->nrelocs * sizeof(struct radeon_bo*));
if (!csc->relocs_bo) {
+ FREE(csc->buf);
return FALSE;
}
csc->relocs = (struct drm_radeon_cs_reloc*)
CALLOC(1, csc->nrelocs * sizeof(struct drm_radeon_cs_reloc));
if (!csc->relocs) {
+ FREE(csc->buf);
FREE(csc->relocs_bo);
return FALSE;
}
csc->chunk_array[2] = (uint64_t)(uintptr_t)&csc->chunks[2];
csc->cs.chunks = (uint64_t)(uintptr_t)csc->chunk_array;
+
+ for (i = 0; i < Elements(csc->reloc_indices_hashlist); i++) {
+ csc->reloc_indices_hashlist[i] = -1;
+ }
return TRUE;
}
csc->chunks[1].length_dw = 0;
csc->used_gart = 0;
csc->used_vram = 0;
- memset(csc->is_handle_added, 0, sizeof(csc->is_handle_added));
+
+ for (i = 0; i < Elements(csc->reloc_indices_hashlist); i++) {
+ csc->reloc_indices_hashlist[i] = -1;
+ }
}
static void radeon_destroy_cs_context(struct radeon_cs_context *csc)
radeon_cs_context_cleanup(csc);
FREE(csc->relocs_bo);
FREE(csc->relocs);
+ FREE(csc->buf);
}
-static struct radeon_winsys_cs *radeon_drm_cs_create(struct radeon_winsys *rws,
- enum ring_type ring_type,
- struct radeon_winsys_cs_handle *trace_buf)
+static struct radeon_winsys_cs *
+radeon_drm_cs_create(struct radeon_winsys *rws,
+ enum ring_type ring_type,
+ void (*flush)(void *ctx, unsigned flags,
+ struct pipe_fence_handle **fence),
+ void *flush_ctx,
+ struct radeon_winsys_cs_handle *trace_buf)
{
struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
struct radeon_drm_cs *cs;
pipe_semaphore_init(&cs->flush_completed, 1);
cs->ws = ws;
+ cs->flush_cs = flush;
+ cs->flush_data = flush_ctx;
cs->trace_buf = (struct radeon_bo*)trace_buf;
if (!radeon_init_cs_context(&cs->csc1, cs->ws)) {
cs->cst = &cs->csc2;
cs->base.buf = cs->csc->buf;
cs->base.ring_type = ring_type;
+ cs->base.max_dw = ws->ib_max_size / 4;
p_atomic_inc(&ws->num_cs);
return &cs->base;
#define OUT_CS(cs, value) (cs)->buf[(cs)->cdw++] = (value)
-static INLINE void update_reloc(struct drm_radeon_cs_reloc *reloc,
- enum radeon_bo_domain rd,
- enum radeon_bo_domain wd,
- unsigned priority,
- enum radeon_bo_domain *added_domains)
+static inline void update_reloc(struct drm_radeon_cs_reloc *reloc,
+ enum radeon_bo_domain rd,
+ enum radeon_bo_domain wd,
+ unsigned priority,
+ enum radeon_bo_domain *added_domains)
{
*added_domains = (rd | wd) & ~(reloc->read_domains | reloc->write_domain);
int radeon_get_reloc(struct radeon_cs_context *csc, struct radeon_bo *bo)
{
- struct drm_radeon_cs_reloc *reloc;
- unsigned i;
- unsigned hash = bo->handle & (sizeof(csc->is_handle_added)-1);
-
- if (csc->is_handle_added[hash]) {
- i = csc->reloc_indices_hashlist[hash];
- reloc = &csc->relocs[i];
- if (reloc->handle == bo->handle) {
+ unsigned hash = bo->handle & (Elements(csc->reloc_indices_hashlist)-1);
+ int i = csc->reloc_indices_hashlist[hash];
+
+ /* not found or found */
+ if (i == -1 || csc->relocs_bo[i] == bo)
+ return i;
+
+ /* Hash collision, look for the BO in the list of relocs linearly. */
+ for (i = csc->crelocs - 1; i >= 0; i--) {
+ if (csc->relocs_bo[i] == bo) {
+ /* Put this reloc in the hash list.
+ * This will prevent additional hash collisions if there are
+ * several consecutive get_reloc calls for the same buffer.
+ *
+ * Example: Assuming buffers A,B,C collide in the hash list,
+ * the following sequence of relocs:
+ * AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
+ * will collide here: ^ and here: ^,
+ * meaning that we should get very few collisions in the end. */
+ csc->reloc_indices_hashlist[hash] = i;
return i;
}
-
- /* Hash collision, look for the BO in the list of relocs linearly. */
- for (i = csc->crelocs; i != 0;) {
- --i;
- reloc = &csc->relocs[i];
- if (reloc->handle == bo->handle) {
- /* Put this reloc in the hash list.
- * This will prevent additional hash collisions if there are
- * several consecutive get_reloc calls for the same buffer.
- *
- * Example: Assuming buffers A,B,C collide in the hash list,
- * the following sequence of relocs:
- * AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
- * will collide here: ^ and here: ^,
- * meaning that we should get very few collisions in the end. */
- csc->reloc_indices_hashlist[hash] = i;
- /*printf("write_reloc collision, hash: %i, handle: %i\n", hash, bo->handle);*/
- return i;
- }
- }
}
-
return -1;
}
{
struct radeon_cs_context *csc = cs->csc;
struct drm_radeon_cs_reloc *reloc;
- unsigned hash = bo->handle & (sizeof(csc->is_handle_added)-1);
+ unsigned hash = bo->handle & (Elements(csc->reloc_indices_hashlist)-1);
enum radeon_bo_domain rd = usage & RADEON_USAGE_READ ? domains : 0;
enum radeon_bo_domain wd = usage & RADEON_USAGE_WRITE ? domains : 0;
- bool update_hash = TRUE;
- int i;
+ int i = -1;
priority = MIN2(priority, 15);
*added_domains = 0;
- if (csc->is_handle_added[hash]) {
- i = csc->reloc_indices_hashlist[hash];
- reloc = &csc->relocs[i];
- if (reloc->handle != bo->handle) {
- /* Hash collision, look for the BO in the list of relocs linearly. */
- for (i = csc->crelocs - 1; i >= 0; i--) {
- reloc = &csc->relocs[i];
- if (reloc->handle == bo->handle) {
- /*printf("write_reloc collision, hash: %i, handle: %i\n", hash, bo->handle);*/
- break;
- }
- }
- }
+ i = radeon_get_reloc(csc, bo);
- if (i >= 0) {
- /* On DMA ring we need to emit as many relocation as there is use of the bo
- * thus each time this function is call we should grow add again the bo to
- * the relocation buffer
- *
- * Do not update the hash table if it's dma ring, so that first hash always point
- * to first bo relocation which will the one used by the kernel. Following relocation
- * will be ignore by the kernel memory placement (but still use by the kernel to
- * update the cmd stream with proper buffer offset).
- */
- update_hash = FALSE;
- update_reloc(reloc, rd, wd, priority, added_domains);
- if (cs->base.ring_type != RING_DMA) {
- csc->reloc_indices_hashlist[hash] = i;
- return i;
- }
+ if (i >= 0) {
+ reloc = &csc->relocs[i];
+ update_reloc(reloc, rd, wd, priority, added_domains);
+
+ /* For async DMA, every add_reloc call must add a buffer to the list
+ * no matter how many duplicates there are. This is due to the fact
+ * the DMA CS checker doesn't use NOP packets for offset patching,
+ * but always uses the i-th buffer from the list to patch the i-th
+ * offset. If there are N offsets in a DMA CS, there must also be N
+ * buffers in the relocation list.
+ *
+ * This doesn't have to be done if virtual memory is enabled,
+ * because there is no offset patching with virtual memory.
+ */
+ if (cs->base.ring_type != RING_DMA || cs->ws->info.r600_virtual_address) {
+ return i;
}
}
reloc->write_domain = wd;
reloc->flags = priority;
- csc->is_handle_added[hash] = TRUE;
- if (update_hash) {
- csc->reloc_indices_hashlist[hash] = csc->crelocs;
- }
+ csc->reloc_indices_hashlist[hash] = csc->crelocs;
csc->chunks[1].length_dw += RELOC_DWORDS;
return index;
}
+static int radeon_drm_cs_get_reloc(struct radeon_winsys_cs *rcs,
+ struct radeon_winsys_cs_handle *buf)
+{
+ struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
+
+ return radeon_get_reloc(cs->csc, (struct radeon_bo*)buf);
+}
+
static boolean radeon_drm_cs_validate(struct radeon_winsys_cs *rcs)
{
struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
/* Flush if there are any relocs. Clean up otherwise. */
if (cs->csc->crelocs) {
- cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC);
+ cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
} else {
radeon_cs_context_cleanup(cs->csc);
return status;
}
-static void radeon_drm_cs_write_reloc(struct radeon_winsys_cs *rcs,
- struct radeon_winsys_cs_handle *buf)
-{
- struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
- struct radeon_bo *bo = (struct radeon_bo*)buf;
- unsigned index = radeon_get_reloc(cs->csc, bo);
-
- if (index == -1) {
- fprintf(stderr, "radeon: Cannot get a relocation in %s.\n", __func__);
- return;
- }
-
- OUT_CS(&cs->base, 0xc0001000);
- OUT_CS(&cs->base, index * RELOC_DWORDS);
-}
-
void radeon_drm_cs_emit_ioctl_oneshot(struct radeon_drm_cs *cs, struct radeon_cs_context *csc)
{
unsigned i;
DEBUG_GET_ONCE_BOOL_OPTION(noop, "RADEON_NOOP", FALSE)
-static void radeon_drm_cs_flush(struct radeon_winsys_cs *rcs, unsigned flags, uint32_t cs_trace_id)
+static void radeon_drm_cs_flush(struct radeon_winsys_cs *rcs,
+ unsigned flags,
+ struct pipe_fence_handle **fence,
+ uint32_t cs_trace_id)
{
struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
struct radeon_cs_context *tmp;
switch (cs->base.ring_type) {
case RING_DMA:
- /* pad DMA ring to 8 DWs */
- if (cs->ws->info.chip_class <= SI) {
- while (rcs->cdw & 7)
- OUT_CS(&cs->base, 0xf0000000); /* NOP packet */
- } else {
- while (rcs->cdw & 7)
- OUT_CS(&cs->base, 0x00000000); /* NOP packet */
- }
- break;
+ /* pad DMA ring to 8 DWs */
+ if (cs->ws->info.chip_class <= SI) {
+ while (rcs->cdw & 7)
+ OUT_CS(&cs->base, 0xf0000000); /* NOP packet */
+ } else {
+ while (rcs->cdw & 7)
+ OUT_CS(&cs->base, 0x00000000); /* NOP packet */
+ }
+ break;
case RING_GFX:
- /* pad DMA ring to 8 DWs to meet CP fetch alignment requirements
- * r6xx, requires at least 4 dw alignment to avoid a hw bug.
- */
- if (cs->ws->info.chip_class <= SI) {
- while (rcs->cdw & 7)
- OUT_CS(&cs->base, 0x80000000); /* type2 nop packet */
- } else {
- while (rcs->cdw & 7)
- OUT_CS(&cs->base, 0xffff1000); /* type3 nop packet */
- }
- break;
+ /* pad DMA ring to 8 DWs to meet CP fetch alignment requirements
+ * r6xx, requires at least 4 dw alignment to avoid a hw bug.
+ * hawaii with old firmware needs type2 nop packet.
+ * accel_working2 with value 3 indicates the new firmware.
+ */
+ if (cs->ws->info.chip_class <= SI ||
+ (cs->ws->info.family == CHIP_HAWAII &&
+ cs->ws->accel_working2 < 3)) {
+ while (rcs->cdw & 7)
+ OUT_CS(&cs->base, 0x80000000); /* type2 nop packet */
+ } else {
+ while (rcs->cdw & 7)
+ OUT_CS(&cs->base, 0xffff1000); /* type3 nop packet */
+ }
+ break;
case RING_UVD:
- while (rcs->cdw & 15)
- OUT_CS(&cs->base, 0x80000000); /* type2 nop packet */
- break;
+ while (rcs->cdw & 15)
+ OUT_CS(&cs->base, 0x80000000); /* type2 nop packet */
+ break;
default:
- break;
+ break;
}
- if (rcs->cdw > RADEON_MAX_CMDBUF_DWORDS) {
+ if (rcs->cdw > rcs->max_dw) {
fprintf(stderr, "radeon: command stream overflowed\n");
}
+ if (fence) {
+ radeon_fence_reference(fence, NULL);
+ *fence = radeon_cs_create_fence(rcs);
+ }
+
radeon_drm_cs_sync_flush(rcs);
- /* Flip command streams. */
+ /* Swap command streams. */
tmp = cs->csc;
cs->csc = cs->cst;
cs->cst = tmp;
cs->cst->cs_trace_id = cs_trace_id;
/* If the CS is not empty or overflowed, emit it in a separate thread. */
- if (cs->base.cdw && cs->base.cdw <= RADEON_MAX_CMDBUF_DWORDS && !debug_get_option_noop()) {
- unsigned i, crelocs = cs->cst->crelocs;
+ if (cs->base.cdw && cs->base.cdw <= cs->base.max_dw && !debug_get_option_noop()) {
+ unsigned i, crelocs;
+
+ crelocs = cs->cst->crelocs;
cs->cst->chunks[0].length_dw = cs->base.cdw;
/* Prepare a new CS. */
cs->base.buf = cs->csc->buf;
cs->base.cdw = 0;
+
+ cs->ws->num_cs_flushes++;
}
static void radeon_drm_cs_destroy(struct radeon_winsys_cs *rcs)
FREE(cs);
}
-static void radeon_drm_cs_set_flush(struct radeon_winsys_cs *rcs,
- void (*flush)(void *ctx, unsigned flags),
- void *user)
-{
- struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
-
- cs->flush_cs = flush;
- cs->flush_data = user;
-}
-
static boolean radeon_bo_is_referenced(struct radeon_winsys_cs *rcs,
struct radeon_winsys_cs_handle *_buf,
enum radeon_bo_usage usage)
/* Create a fence, which is a dummy BO. */
fence = cs->ws->base.buffer_create(&cs->ws->base, 1, 1, TRUE,
- RADEON_DOMAIN_GTT);
+ RADEON_DOMAIN_GTT, 0);
/* Add the fence as a dummy relocation. */
cs->ws->base.cs_add_reloc(rcs, cs->ws->base.buffer_get_cs_handle(fence),
RADEON_USAGE_READWRITE, RADEON_DOMAIN_GTT,
ws->base.cs_create = radeon_drm_cs_create;
ws->base.cs_destroy = radeon_drm_cs_destroy;
ws->base.cs_add_reloc = radeon_drm_cs_add_reloc;
+ ws->base.cs_get_reloc = radeon_drm_cs_get_reloc;
ws->base.cs_validate = radeon_drm_cs_validate;
ws->base.cs_memory_below_limit = radeon_drm_cs_memory_below_limit;
- ws->base.cs_write_reloc = radeon_drm_cs_write_reloc;
ws->base.cs_flush = radeon_drm_cs_flush;
- ws->base.cs_set_flush_callback = radeon_drm_cs_set_flush;
ws->base.cs_is_buffer_referenced = radeon_bo_is_referenced;
ws->base.cs_sync_flush = radeon_drm_cs_sync_flush;
- ws->base.cs_create_fence = radeon_cs_create_fence;
ws->base.fence_wait = radeon_fence_wait;
ws->base.fence_reference = radeon_fence_reference;
}
projects / mesa.git / blobdiff