From d732728590e93de54dd4f4576b394ca2442c3db8 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 24 Dec 2007 19:21:17 +1100 Subject: [PATCH] nouveau: pushbuf code, now with 50% less suck! Far more efficient, if not a bit more complicated. Hopefully not too buggy still. This commit will potentially expose some unrelated bugs, fixes for them will follow "real soon now". --- .../drivers/dri/nouveau_winsys/nouveau_dma.c | 77 ++++++----- .../drivers/dri/nouveau_winsys/nouveau_dma.h | 41 +++--- .../dri/nouveau_winsys/nouveau_drmif.h | 32 +++-- .../dri/nouveau_winsys/nouveau_pushbuf.c | 121 ++++++++++-------- .../dri/nouveau_winsys/nouveau_winsys.c | 7 +- 5 files changed, 155 insertions(+), 123 deletions(-) diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c index 3bb7c49762b..f8a8ba04f6d 100644 --- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c +++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.c @@ -28,23 +28,23 @@ #include "nouveau_dma.h" #include "nouveau_local.h" -static __inline__ uint32_t +static inline uint32_t READ_GET(struct nouveau_channel_priv *nvchan) { - return ((*nvchan->get - nvchan->dma.base) >> 2); + return *nvchan->get; } -static __inline__ void +static inline void WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val) { - uint32_t put = ((val << 2) + nvchan->dma.base); + uint32_t put = ((val << 2) + nvchan->dma->base); volatile int dum; NOUVEAU_DMA_BARRIER; dum = READ_GET(nvchan); *nvchan->put = put; - nvchan->dma.put = val; + nvchan->dma->put = val; #ifdef NOUVEAU_DMA_TRACE NOUVEAU_MSG("WRITE_PUT %d/0x%08x\n", nvchan->drm.channel, put); #endif @@ -52,16 +52,30 @@ WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val) NOUVEAU_DMA_BARRIER; } +static inline int +LOCAL_GET(struct nouveau_dma_priv *dma, uint32_t *val) +{ + uint32_t get = *val; + + if (get >= dma->base && get <= (dma->base + (dma->max << 2))) { + *val = (get - dma->base) >> 2; + return 1; + } + + return 0; +} + void nouveau_dma_channel_init(struct nouveau_channel *chan) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); int i; - nvchan->dma.base = nvchan->drm.put_base; - nvchan->dma.cur = nvchan->dma.put = 0; - nvchan->dma.max = (nvchan->drm.cmdbuf_size >> 2) - 2; - nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur; + nvchan->dma = &nvchan->dma_master; + nvchan->dma->base = nvchan->drm.put_base; + nvchan->dma->cur = nvchan->dma->put = 0; + nvchan->dma->max = (nvchan->drm.cmdbuf_size >> 2) - 2; + nvchan->dma->free = nvchan->dma->max - nvchan->dma->cur; RING_SPACE_CH(chan, RING_SKIPS); for (i = 0; i < RING_SKIPS; i++) @@ -73,54 +87,51 @@ nouveau_dma_channel_init(struct nouveau_channel *chan) return - EBUSY; \ } while(0) -#define IN_MASTER_RING(chan, ptr) ((ptr) <= (chan)->dma.max) - int nouveau_dma_wait(struct nouveau_channel *chan, int size) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); + struct nouveau_dma_priv *dma = nvchan->dma; uint32_t get, t_start; FIRE_RING_CH(chan); t_start = NOUVEAU_TIME_MSEC(); - while (nvchan->dma.free < size) { + while (dma->free < size) { CHECK_TIMEOUT(); get = READ_GET(nvchan); - if (!IN_MASTER_RING(nvchan, get)) + if (!LOCAL_GET(dma, &get)) continue; - if (nvchan->dma.put >= get) { - nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur; + if (dma->put >= get) { + dma->free = dma->max - dma->cur; - if (nvchan->dma.free < size) { + if (dma->free < size) { #ifdef NOUVEAU_DMA_DEBUG - nvchan->dma.push_free = 1; + dma->push_free = 1; #endif - OUT_RING_CH(chan, - 0x20000000 | nvchan->dma.base); + OUT_RING_CH(chan, 0x20000000 | dma->base); if (get <= RING_SKIPS) { /*corner case - will be idle*/ - if (nvchan->dma.put <= RING_SKIPS) + if (dma->put <= RING_SKIPS) WRITE_PUT(nvchan, RING_SKIPS + 1); do { CHECK_TIMEOUT(); get = READ_GET(nvchan); - if (!IN_MASTER_RING(nvchan, - get)) - continue; + if (!LOCAL_GET(dma, &get)) + get = 0; } while (get <= RING_SKIPS); } WRITE_PUT(nvchan, RING_SKIPS); - nvchan->dma.cur = nvchan->dma.put = RING_SKIPS; - nvchan->dma.free = get - (RING_SKIPS + 1); + dma->cur = dma->put = RING_SKIPS; + dma->free = get - (RING_SKIPS + 1); } } else { - nvchan->dma.free = get - nvchan->dma.cur - 1; + dma->free = get - dma->cur - 1; } } @@ -135,7 +146,7 @@ nouveau_dma_parse_pushbuf(struct nouveau_channel *chan, int get, int put) unsigned mthd_count = 0; while (get != put) { - uint32_t gpuget = (get << 2) + nvchan->dma.base; + uint32_t gpuget = (get << 2) + nvchan->drm.put_base; uint32_t data; if (get < 0 || get >= nvchan->drm.cmdbuf_size) { @@ -188,21 +199,21 @@ void nouveau_dma_kickoff(struct nouveau_channel *chan) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); + struct nouveau_dma_priv *dma = nvchan->dma; - if (nvchan->dma.cur == nvchan->dma.put) + if (dma->cur == dma->put) return; #ifdef NOUVEAU_DMA_DEBUG - if (nvchan->dma.push_free) { - NOUVEAU_ERR("Packet incomplete: %d left\n", - nvchan->dma.push_free); + if (dma->push_free) { + NOUVEAU_ERR("Packet incomplete: %d left\n", dma->push_free); return; } #endif #ifdef NOUVEAU_DMA_DUMP_POSTRELOC_PUSHBUF - nouveau_dma_parse_pushbuf(chan, nvchan->dma.put, nvchan->dma.cur); + nouveau_dma_parse_pushbuf(chan, dma->put, dma->cur); #endif - WRITE_PUT(nvchan, nvchan->dma.cur); + WRITE_PUT(nvchan, dma->cur); } diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h index 940a196bfe2..cfa6d26e828 100644 --- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h +++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_dma.h @@ -42,34 +42,37 @@ static inline void nouveau_dma_out(struct nouveau_channel *chan, uint32_t data) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); + struct nouveau_dma_priv *dma = nvchan->dma; #ifdef NOUVEAU_DMA_DEBUG - if (nvchan->dma.push_free == 0) { - NOUVEAU_ERR("No space left in packet. Error at %s\n",faulty); + if (dma->push_free == 0) { + NOUVEAU_ERR("No space left in packet at %s\n", faulty); return; } - nvchan->dma.push_free--; + dma->push_free--; #endif #ifdef NOUVEAU_DMA_TRACE { - uint32_t offset = (nvchan->dma.cur << 2) + nvchan->dma.base; + uint32_t offset = (dma->cur << 2) + dma->base; NOUVEAU_MSG("\tOUT_RING %d/0x%08x -> 0x%08x\n", nvchan->drm.channel, offset, data); } #endif - nvchan->pushbuf[nvchan->dma.cur++] = data; + nvchan->pushbuf[dma->cur + (dma->base - nvchan->drm.put_base)/4] = data; + dma->cur++; } static inline void nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); - (void)chan; + struct nouveau_dma_priv *dma = nvchan->dma; + (void)dma; #ifdef NOUVEAU_DMA_DEBUG - if (nvchan->dma.push_free < size) { + if (dma->push_free < size) { NOUVEAU_ERR("Packet too small. Free=%d, Need=%d\n", - nvchan->dma.push_free, size); + dma->push_free, size); return; } #endif @@ -79,11 +82,11 @@ nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size) ptr++; } #else - memcpy(&nvchan->pushbuf[nvchan->dma.cur], ptr, size << 2); + memcpy(&nvchan->pushbuf[dma->cur], ptr, size << 2); #ifdef NOUVEAU_DMA_DEBUG - nvchan->dma.push_free -= size; + dma->push_free -= size; #endif - nvchan->dma.cur += size; + dma->cur += size; #endif } @@ -91,14 +94,15 @@ static inline void nouveau_dma_space(struct nouveau_channel *chan, int size) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); + struct nouveau_dma_priv *dma = nvchan->dma; - if (nvchan->dma.free < size) { + if (dma->free < size) { if (nouveau_dma_wait(chan, size) && chan->hang_notify) chan->hang_notify(chan); } - nvchan->dma.free -= size; + dma->free -= size; #ifdef NOUVEAU_DMA_DEBUG - nvchan->dma.push_free = size; + dma->push_free = size; #endif } @@ -107,7 +111,8 @@ nouveau_dma_begin(struct nouveau_channel *chan, struct nouveau_grobj *grobj, int method, int size, const char* file, int line) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); - (void)nvchan; + struct nouveau_dma_priv *dma = nvchan->dma; + (void)dma; #ifdef NOUVEAU_DMA_TRACE NOUVEAU_MSG("BEGIN_RING %d/%08x/%d/0x%04x/%d\n", nvchan->drm.channel, @@ -115,9 +120,9 @@ nouveau_dma_begin(struct nouveau_channel *chan, struct nouveau_grobj *grobj, #endif #ifdef NOUVEAU_DMA_DEBUG - if (nvchan->dma.push_free) { - NOUVEAU_ERR("Previous packet incomplete: %d left. Error at %s\n", - nvchan->dma.push_free,faulty); + if (dma->push_free) { + NOUVEAU_ERR("Previous packet incomplete: %d left at %s\n", + dma->push_free, faulty); return; } sprintf(faulty,"%s:%d",file,line); diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h b/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h index 11a15d632ce..724677961e6 100644 --- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h +++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_drmif.h @@ -128,10 +128,10 @@ struct nouveau_pushbuf_bo { struct nouveau_pushbuf_priv { struct nouveau_pushbuf base; - struct nouveau_pushbuf *next; - struct nouveau_resource *res; - struct nouveau_fence *fence; + unsigned nop_jump; + unsigned start; + unsigned size; uint64_t buffers; int nr_buffers; @@ -149,13 +149,23 @@ extern int nouveau_pushbuf_init(struct nouveau_channel *); extern int -nouveau_pushbuf_flush(struct nouveau_channel *); +nouveau_pushbuf_flush(struct nouveau_channel *, unsigned min); extern int nouveau_pushbuf_emit_reloc(struct nouveau_channel *, void *ptr, struct nouveau_bo *, uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor); +struct nouveau_dma_priv { + uint32_t base; + uint32_t max; + uint32_t cur; + uint32_t put; + uint32_t free; + + int push_free; +} dma; + struct nouveau_channel_priv { struct nouveau_channel base; @@ -169,21 +179,15 @@ struct nouveau_channel_priv { volatile uint32_t *get; volatile uint32_t *ref_cnt; - struct { - uint32_t base, max; - uint32_t cur, put; - uint32_t free; - - int push_free; - } dma; + struct nouveau_dma_priv dma_master; + struct nouveau_dma_priv dma_bufmgr; + struct nouveau_dma_priv *dma; struct nouveau_fence *fence_head; struct nouveau_fence *fence_tail; uint32_t fence_sequence; - struct nouveau_resource *pb_heap; - struct nouveau_pushbuf *pb_head; - struct nouveau_pushbuf *pb_tail; + struct nouveau_pushbuf_priv pb; unsigned user_charge; }; diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c b/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c index 11754417893..148406b85a7 100644 --- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c +++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_pushbuf.c @@ -27,28 +27,34 @@ #include "nouveau_drmif.h" #include "nouveau_dma.h" -#define PB_RSVD_DWORDS 2 +#define PB_BUFMGR_DWORDS (4096 / 2) +#define PB_MIN_USER_DWORDS 2048 static int -nouveau_pushbuf_space(struct nouveau_channel *chan) +nouveau_pushbuf_space(struct nouveau_channel *chan, unsigned min) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); - struct nouveau_pushbuf_priv *nvpb; + struct nouveau_pushbuf_priv *nvpb = &nvchan->pb; - nvpb = calloc(1, sizeof(struct nouveau_pushbuf_priv)); - if (!nvpb) - return -ENOMEM; + assert((min + 1) <= nvchan->dma->max); - while (nouveau_resource_alloc(nvchan->pb_heap, 0x2100, NULL, - &nvpb->res)) { - nouveau_fence_flush(chan); - } + /* Wait for enough space in push buffer */ + min = min < PB_MIN_USER_DWORDS ? PB_MIN_USER_DWORDS : min; + min += 1; /* a bit extra for the NOP */ + if (nvchan->dma->free < min) + WAIT_RING_CH(chan, min); + + /* Insert NOP, may turn into a jump later */ + RING_SPACE_CH(chan, 1); + nvpb->nop_jump = nvchan->dma->cur; + OUT_RING_CH(chan, 0); + /* Any remaining space is available to the user */ + nvpb->start = nvchan->dma->cur; + nvpb->size = nvchan->dma->free; nvpb->base.channel = chan; - nvpb->base.remaining = (nvpb->res->size / 4) - PB_RSVD_DWORDS; - nvpb->base.cur = &nvchan->pushbuf[nvpb->res->start/4]; - nvchan->pb_tail = &nvpb->base; - nvchan->base.pushbuf = nvchan->pb_tail; + nvpb->base.remaining = nvpb->size; + nvpb->base.cur = &nvchan->pushbuf[nvpb->start]; return 0; } @@ -57,53 +63,65 @@ int nouveau_pushbuf_init(struct nouveau_channel *chan) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); + struct nouveau_dma_priv *m = &nvchan->dma_master; + struct nouveau_dma_priv *b = &nvchan->dma_bufmgr; + int i; if (!nvchan) return -EINVAL; - /* Everything except first 4KiB of the push buffer is managed by us */ - if (nouveau_resource_init(&nvchan->pb_heap, 4096, - nvchan->drm.cmdbuf_size - 4096)) - return -EINVAL; - - /* Shrink master ring to 4KiB */ - assert(nvchan->dma.cur <= (4096/4)); - nvchan->dma.max = (4096 / 4) - 2; - nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur; - - assert(!nouveau_pushbuf_space(chan)); + /* Reassign last bit of push buffer for a "separate" bufmgr + * ring buffer + */ + m->max -= PB_BUFMGR_DWORDS; + m->free -= PB_BUFMGR_DWORDS; + + b->base = m->base + ((m->max + 2) << 2); + b->max = PB_BUFMGR_DWORDS - 2; + b->cur = b->put = 0; + b->free = b->max - b->cur; + + /* Some NOPs just to be safe + *XXX: RING_SKIPS + */ + nvchan->dma = b; + RING_SPACE_CH(chan, 8); + for (i = 0; i < 8; i++) + OUT_RING_CH(chan, 0); + nvchan->dma = m; + + nouveau_pushbuf_space(chan, 0); + chan->pushbuf = &nvchan->pb.base; return 0; } -static void -nouveau_pushbuf_fence_signalled(void *priv) -{ - struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(priv); - - nouveau_fence_del(&nvpb->fence); - nouveau_resource_free(&nvpb->res); - free(nvpb); -} - /* This would be our TTM "superioctl" */ int -nouveau_pushbuf_flush(struct nouveau_channel *chan) +nouveau_pushbuf_flush(struct nouveau_channel *chan, unsigned min) { struct nouveau_channel_priv *nvchan = nouveau_channel(chan); - struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf); + struct nouveau_pushbuf_priv *nvpb = &nvchan->pb; struct nouveau_pushbuf_bo *pbbo; struct nouveau_fence *fence = NULL; int ret; - if (nvpb->base.remaining == (nvpb->res->size / 4) - PB_RSVD_DWORDS) + if (nvpb->base.remaining == nvpb->size) return 0; - nvchan->pb_tail = NULL; + + nvpb->size -= nvpb->base.remaining; + nvchan->dma->cur += nvpb->size; + nvchan->dma->free -= nvpb->size; + assert(nvchan->dma->cur <= nvchan->dma->max); ret = nouveau_fence_new(chan, &fence); if (ret) return ret; + nvchan->dma = &nvchan->dma_bufmgr; + nvchan->pushbuf[nvpb->nop_jump] = 0x20000000 | + (nvchan->dma->base + (nvchan->dma->cur << 2)); + /* Validate buffers + apply relocations */ nvchan->user_charge = 0; while ((pbbo = ptr_to_pbbo(nvpb->buffers))) { @@ -142,25 +160,19 @@ nouveau_pushbuf_flush(struct nouveau_channel *chan) } nvpb->nr_buffers = 0; - /* Emit JMP to indirect pushbuf */ + /* Switch back to user's ring */ RING_SPACE_CH(chan, 1); - OUT_RING_CH(chan, 0x20000000 | nvpb->res->start); + OUT_RING_CH(chan, 0x20000000 | ((nvpb->start << 2) + + nvchan->dma_master.base)); + nvchan->dma = &nvchan->dma_master; - /* Add JMP back to master pushbuf from indirect pushbuf */ - (*nvpb->base.cur++) = - 0x20000000 | ((nvchan->dma.cur << 2) + nvchan->dma.base); - - /* Fence */ - nvpb->fence = fence; - nouveau_fence_signal_cb(nvpb->fence, nouveau_pushbuf_fence_signalled, - nvpb); - nouveau_fence_emit(nvpb->fence); - - /* Kickoff */ + /* Fence + kickoff */ + nouveau_fence_emit(fence); FIRE_RING_CH(chan); + nouveau_fence_del(&fence); /* Allocate space for next push buffer */ - assert(!nouveau_pushbuf_space(chan)); + assert(!nouveau_pushbuf_space(chan, min)); return 0; } @@ -168,8 +180,7 @@ nouveau_pushbuf_flush(struct nouveau_channel *chan) static struct nouveau_pushbuf_bo * nouveau_pushbuf_emit_buffer(struct nouveau_channel *chan, struct nouveau_bo *bo) { - struct nouveau_channel_priv *nvchan = nouveau_channel(chan); - struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(nvchan->pb_tail); + struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf); struct nouveau_pushbuf_bo *pbbo = ptr_to_pbbo(nvpb->buffers); while (pbbo) { diff --git a/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c b/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c index b31e0dcd1dd..403647ab17e 100644 --- a/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c +++ b/src/mesa/drivers/dri/nouveau_winsys/nouveau_winsys.c @@ -40,8 +40,9 @@ nouveau_pipe_dma_beginp(struct nouveau_grobj *grobj, int mthd, int size) struct nouveau_channel *chan = grobj->channel; uint32_t *pushbuf; - if (chan->pushbuf->remaining < (size + 1)) - nouveau_pushbuf_flush(chan); + if (chan->pushbuf->remaining < (size + 1)) { + nouveau_pushbuf_flush(chan, size + 1); + } pushbuf = chan->pushbuf->cur; chan->pushbuf->cur += (size + 1); @@ -54,7 +55,7 @@ nouveau_pipe_dma_beginp(struct nouveau_grobj *grobj, int mthd, int size) void nouveau_pipe_dma_kickoff(struct nouveau_channel *chan) { - nouveau_pushbuf_flush(chan); + nouveau_pushbuf_flush(chan, 0); } static int -- 2.30.2