From 0d76bb5d4c5c867155f7fb381c46018e1560b790 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Fri, 17 Sep 2010 14:01:50 +1000 Subject: [PATCH] r600g: add upload manager support. this add support for the upload manager for uploading user vbo/index buffers. this provides a considerable speedup in q3 type games. --- src/gallium/drivers/r600/r600_buffer.c | 183 ++++++++++++++++++----- src/gallium/drivers/r600/r600_context.c | 32 +++- src/gallium/drivers/r600/r600_context.h | 12 ++ src/gallium/drivers/r600/r600_draw.c | 8 +- src/gallium/drivers/r600/r600_resource.h | 31 ++++ src/gallium/drivers/r600/r600_state.c | 4 + 6 files changed, 223 insertions(+), 47 deletions(-) diff --git a/src/gallium/drivers/r600/r600_buffer.c b/src/gallium/drivers/r600/r600_buffer.c index a38c013e167..dc3fc812e1a 100644 --- a/src/gallium/drivers/r600/r600_buffer.c +++ b/src/gallium/drivers/r600/r600_buffer.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "state_tracker/drm_driver.h" #include "r600_screen.h" #include "r600_context.h" @@ -67,66 +68,69 @@ struct pipe_resource *r600_buffer_create(struct pipe_screen *screen, const struct pipe_resource *templ) { struct r600_screen *rscreen = r600_screen(screen); - struct r600_resource *rbuffer; + struct r600_resource_buffer *rbuffer; struct radeon_ws_bo *bo; /* XXX We probably want a different alignment for buffers and textures. */ unsigned alignment = 4096; - rbuffer = CALLOC_STRUCT(r600_resource); + rbuffer = CALLOC_STRUCT(r600_resource_buffer); if (rbuffer == NULL) return NULL; - rbuffer->base.b = *templ; - pipe_reference_init(&rbuffer->base.b.reference, 1); - rbuffer->base.b.screen = screen; - rbuffer->base.vtbl = &r600_buffer_vtbl; - rbuffer->size = rbuffer->base.b.width0; - rbuffer->domain = r600_domain_from_usage(rbuffer->base.b.bind); - bo = radeon_ws_bo(rscreen->rw, rbuffer->base.b.width0, alignment, rbuffer->base.b.bind); + rbuffer->magic = R600_BUFFER_MAGIC; + rbuffer->user_buffer = NULL; + rbuffer->num_ranges = 0; + rbuffer->r.base.b = *templ; + pipe_reference_init(&rbuffer->r.base.b.reference, 1); + rbuffer->r.base.b.screen = screen; + rbuffer->r.base.vtbl = &r600_buffer_vtbl; + rbuffer->r.size = rbuffer->r.base.b.width0; + rbuffer->r.domain = r600_domain_from_usage(rbuffer->r.base.b.bind); + bo = radeon_ws_bo(rscreen->rw, rbuffer->r.base.b.width0, alignment, rbuffer->r.base.b.bind); if (bo == NULL) { FREE(rbuffer); return NULL; } - rbuffer->bo = bo; - return &rbuffer->base.b; + rbuffer->r.bo = bo; + return &rbuffer->r.base.b; } struct pipe_resource *r600_user_buffer_create(struct pipe_screen *screen, void *ptr, unsigned bytes, unsigned bind) { - struct r600_resource *rbuffer; - struct r600_screen *rscreen = r600_screen(screen); - struct pipe_resource templ; - void *data; - - memset(&templ, 0, sizeof(struct pipe_resource)); - templ.target = PIPE_BUFFER; - templ.format = PIPE_FORMAT_R8_UNORM; - templ.usage = PIPE_USAGE_IMMUTABLE; - templ.bind = bind; - templ.width0 = bytes; - templ.height0 = 1; - templ.depth0 = 1; - - rbuffer = (struct r600_resource*)r600_buffer_create(screen, &templ); - if (rbuffer == NULL) { + struct r600_resource_buffer *rbuffer; + + rbuffer = CALLOC_STRUCT(r600_resource_buffer); + if (rbuffer == NULL) return NULL; - } - data = radeon_ws_bo_map(rscreen->rw, rbuffer->bo, 0, NULL); - memcpy(data, ptr, bytes); - radeon_ws_bo_unmap(rscreen->rw, rbuffer->bo); - return &rbuffer->base.b; + + rbuffer->magic = R600_BUFFER_MAGIC; + pipe_reference_init(&rbuffer->r.base.b.reference, 1); + rbuffer->r.base.vtbl = &r600_buffer_vtbl; + rbuffer->r.base.b.screen = screen; + rbuffer->r.base.b.target = PIPE_BUFFER; + rbuffer->r.base.b.format = PIPE_FORMAT_R8_UNORM; + rbuffer->r.base.b.usage = PIPE_USAGE_IMMUTABLE; + rbuffer->r.base.b.bind = bind; + rbuffer->r.base.b.width0 = bytes; + rbuffer->r.base.b.height0 = 1; + rbuffer->r.base.b.depth0 = 1; + rbuffer->r.base.b.flags = 0; + rbuffer->num_ranges = 0; + rbuffer->r.bo = NULL; + rbuffer->user_buffer = ptr; + return &rbuffer->r.base.b; } static void r600_buffer_destroy(struct pipe_screen *screen, struct pipe_resource *buf) { - struct r600_resource *rbuffer = (struct r600_resource*)buf; + struct r600_resource_buffer *rbuffer = r600_buffer(buf); struct r600_screen *rscreen = r600_screen(screen); - if (rbuffer->bo) { - radeon_ws_bo_reference(rscreen->rw, &rbuffer->bo, NULL); + if (rbuffer->r.bo) { + radeon_ws_bo_reference(rscreen->rw, &rbuffer->r.bo, NULL); } FREE(rbuffer); } @@ -134,18 +138,40 @@ static void r600_buffer_destroy(struct pipe_screen *screen, static void *r600_buffer_transfer_map(struct pipe_context *pipe, struct pipe_transfer *transfer) { - struct r600_resource *rbuffer = (struct r600_resource*)transfer->resource; + struct r600_context *rctx = r600_context(pipe); + struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource); struct r600_screen *rscreen = r600_screen(pipe->screen); int write = 0; uint8_t *data; + int i; + boolean flush = FALSE; + + if (rbuffer->user_buffer) + return (uint8_t*)rbuffer->user_buffer + transfer->box.x; + if (transfer->usage & PIPE_TRANSFER_DISCARD) { + for (i = 0; i < rbuffer->num_ranges; i++) { + if ((transfer->box.x >= rbuffer->ranges[i].start) && + (transfer->box.x < rbuffer->ranges[i].end)) + flush = TRUE; + + if (flush) { + radeon_ws_bo_reference(rscreen->rw, &rbuffer->r.bo, NULL); + rbuffer->num_ranges = 0; + rbuffer->r.bo = radeon_ws_bo(rscreen->rw, + rbuffer->r.base.b.width0, 0, + rbuffer->r.base.b.bind); + break; + } + } + } if (transfer->usage & PIPE_TRANSFER_DONTBLOCK) { /* FIXME */ } if (transfer->usage & PIPE_TRANSFER_WRITE) { write = 1; } - data = radeon_ws_bo_map(rscreen->rw, rbuffer->bo, transfer->usage, r600_context(pipe)); + data = radeon_ws_bo_map(rscreen->rw, rbuffer->r.bo, transfer->usage, rctx); if (!data) return NULL; @@ -155,16 +181,39 @@ static void *r600_buffer_transfer_map(struct pipe_context *pipe, static void r600_buffer_transfer_unmap(struct pipe_context *pipe, struct pipe_transfer *transfer) { - struct r600_resource *rbuffer = (struct r600_resource*)transfer->resource; + struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource); struct r600_screen *rscreen = r600_screen(pipe->screen); - radeon_ws_bo_unmap(rscreen->rw, rbuffer->bo); + if (rbuffer->r.bo) + radeon_ws_bo_unmap(rscreen->rw, rbuffer->r.bo); } static void r600_buffer_transfer_flush_region(struct pipe_context *pipe, struct pipe_transfer *transfer, const struct pipe_box *box) { + struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource); + unsigned i; + unsigned offset = transfer->box.x + box->x; + unsigned length = box->width; + + assert(box->x + box->width <= transfer->box.width); + + if (rbuffer->user_buffer) + return; + + /* mark the range as used */ + for(i = 0; i < rbuffer->num_ranges; ++i) { + if(offset <= rbuffer->ranges[i].end && rbuffer->ranges[i].start <= (offset+box->width)) { + rbuffer->ranges[i].start = MIN2(rbuffer->ranges[i].start, offset); + rbuffer->ranges[i].end = MAX2(rbuffer->ranges[i].end, (offset+length)); + return; + } + } + + rbuffer->ranges[rbuffer->num_ranges].start = offset; + rbuffer->ranges[rbuffer->num_ranges].end = offset+length; + rbuffer->num_ranges++; } unsigned r600_buffer_is_referenced_by_cs(struct pipe_context *context, @@ -213,3 +262,59 @@ struct u_resource_vtbl r600_buffer_vtbl = r600_buffer_transfer_unmap, /* transfer_unmap */ u_default_transfer_inline_write /* transfer_inline_write */ }; + +int r600_upload_index_buffer(struct r600_context *rctx, + struct r600_draw *draw) +{ + struct pipe_resource *upload_buffer = NULL; + unsigned index_offset = draw->index_buffer_offset; + int ret = 0; + + if (r600_buffer_is_user_buffer(draw->index_buffer)) { + ret = u_upload_buffer(rctx->upload_ib, + index_offset, + draw->count * draw->index_size, + draw->index_buffer, + &index_offset, + &upload_buffer); + if (ret) { + goto done; + } + draw->index_buffer_offset = index_offset; + draw->index_buffer = upload_buffer; + } + +done: + return ret; +} + +int r600_upload_user_buffers(struct r600_context *rctx) +{ + enum pipe_error ret = PIPE_OK; + int i, nr; + + nr = rctx->vertex_elements->count; + + for (i = 0; i < nr; i++) { + struct pipe_vertex_buffer *vb = + &rctx->vertex_buffer[rctx->vertex_elements->elements[i].vertex_buffer_index]; + + if (r600_buffer_is_user_buffer(vb->buffer)) { + struct pipe_resource *upload_buffer = NULL; + unsigned offset = 0; /*vb->buffer_offset * 4;*/ + unsigned size = vb->buffer->width0; + unsigned upload_offset; + ret = u_upload_buffer(rctx->upload_vb, + offset, size, + vb->buffer, + &upload_offset, &upload_buffer); + if (ret) + return ret; + + pipe_resource_reference(&vb->buffer, NULL); + vb->buffer = upload_buffer; + vb->buffer_offset = upload_offset; + } + } + return ret; +} diff --git a/src/gallium/drivers/r600/r600_context.c b/src/gallium/drivers/r600/r600_context.c index cca1e356734..776dc24569b 100644 --- a/src/gallium/drivers/r600/r600_context.c +++ b/src/gallium/drivers/r600/r600_context.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "r600_screen.h" #include "r600_context.h" @@ -56,6 +57,9 @@ static void r600_destroy_context(struct pipe_context *context) free(rctx->vs_constant); free(rctx->vs_resource); + u_upload_destroy(rctx->upload_vb); + u_upload_destroy(rctx->upload_ib); + radeon_ctx_fini(rctx->ctx); FREE(rctx); } @@ -66,6 +70,10 @@ void r600_flush(struct pipe_context *ctx, unsigned flags, struct r600_context *rctx = r600_context(ctx); struct r600_query *rquery = NULL; + /* flush upload buffers */ + u_upload_flush(rctx->upload_vb); + u_upload_flush(rctx->upload_ib); + /* suspend queries */ r600_queries_suspend(ctx); @@ -123,25 +131,37 @@ struct pipe_context *r600_create_context(struct pipe_screen *screen, void *priv) rctx->vtbl->init_config(rctx); + rctx->upload_ib = u_upload_create(&rctx->context, 32 * 1024, 16, + PIPE_BIND_INDEX_BUFFER); + if (rctx->upload_ib == NULL) { + goto out_free; + } + + rctx->upload_vb = u_upload_create(&rctx->context, 128 * 1024, 16, + PIPE_BIND_VERTEX_BUFFER); + if (rctx->upload_vb == NULL) { + goto out_free; + } + rctx->vs_constant = (struct radeon_state *)calloc(R600_MAX_CONSTANT, sizeof(struct radeon_state)); if (!rctx->vs_constant) { - FREE(rctx); - return NULL; + goto out_free; } rctx->ps_constant = (struct radeon_state *)calloc(R600_MAX_CONSTANT, sizeof(struct radeon_state)); if (!rctx->ps_constant) { - FREE(rctx); - return NULL; + goto out_free; } rctx->vs_resource = (struct radeon_state *)calloc(R600_MAX_RESOURCE, sizeof(struct radeon_state)); if (!rctx->vs_resource) { - FREE(rctx); - return NULL; + goto out_free; } rctx->ctx = radeon_ctx_init(rscreen->rw); radeon_draw_init(&rctx->draw, rscreen->rw); return &rctx->context; + out_free: + FREE(rctx); + return NULL; } diff --git a/src/gallium/drivers/r600/r600_context.h b/src/gallium/drivers/r600/r600_context.h index f82c8f82fe0..3107f189c78 100644 --- a/src/gallium/drivers/r600/r600_context.h +++ b/src/gallium/drivers/r600/r600_context.h @@ -34,6 +34,8 @@ #include "radeon.h" #include "r600_shader.h" +struct u_upload_mgr; + #define R600_QUERY_STATE_STARTED (1 << 0) #define R600_QUERY_STATE_ENDED (1 << 1) #define R600_QUERY_STATE_SUSPENDED (1 << 2) @@ -249,6 +251,12 @@ struct r600_context { struct pipe_index_buffer index_buffer; struct pipe_blend_color blend_color; struct list_head query_list; + + /* upload managers */ + struct u_upload_mgr *upload_vb; + struct u_upload_mgr *upload_ib; + bool any_user_vbs; + }; /* Convenience cast wrapper. */ @@ -306,4 +314,8 @@ void eg_set_constant_buffer(struct pipe_context *ctx, uint shader, uint index, struct pipe_resource *buffer); +int r600_upload_index_buffer(struct r600_context *rctx, + struct r600_draw *draw); +int r600_upload_user_buffers(struct r600_context *rctx); + #endif diff --git a/src/gallium/drivers/r600/r600_draw.c b/src/gallium/drivers/r600/r600_draw.c index 669c9b4cdb6..5480ca002d2 100644 --- a/src/gallium/drivers/r600/r600_draw.c +++ b/src/gallium/drivers/r600/r600_draw.c @@ -126,6 +126,11 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info) memset(&draw, 0, sizeof(draw)); + if (rctx->any_user_vbs) { + r600_upload_user_buffers(rctx); + rctx->any_user_vbs = false; + } + draw.ctx = ctx; draw.mode = info->mode; draw.start = info->start; @@ -139,8 +144,7 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info) assert(rctx->index_buffer.offset % rctx->index_buffer.index_size == 0); - draw.start += rctx->index_buffer.offset / - rctx->index_buffer.index_size; + r600_upload_index_buffer(rctx, &draw); } else { draw.index_size = 0; diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h index ff05afbc302..9608a5a6234 100644 --- a/src/gallium/drivers/r600/r600_resource.h +++ b/src/gallium/drivers/r600/r600_resource.h @@ -75,4 +75,35 @@ struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen, const struct pipe_resource *base, struct winsys_handle *whandle); +#define R600_BUFFER_MAGIC 0xabcd1600 +#define R600_BUFFER_MAX_RANGES 32 + +struct r600_buffer_range { + uint32_t start; + uint32_t end; +}; + +struct r600_resource_buffer { + struct r600_resource r; + uint32_t magic; + void *user_buffer; + struct r600_buffer_range ranges[R600_BUFFER_MAX_RANGES]; + unsigned num_ranges; +}; + +/* r600_buffer */ +static INLINE struct r600_resource_buffer *r600_buffer(struct pipe_resource *buffer) +{ + if (buffer) { + assert(((struct r600_resource_buffer *)buffer)->magic == R600_BUFFER_MAGIC); + return (struct r600_resource_buffer *)buffer; + } + return NULL; +} + +static INLINE boolean r600_buffer_is_user_buffer(struct pipe_resource *buffer) +{ + return r600_buffer(buffer)->user_buffer ? true : false; +} + #endif diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c index 5d6236206f7..4dcdc492fc1 100644 --- a/src/gallium/drivers/r600/r600_state.c +++ b/src/gallium/drivers/r600/r600_state.c @@ -437,6 +437,7 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx, { struct r600_context *rctx = r600_context(ctx); unsigned i; + boolean any_user_buffers = FALSE; for (i = 0; i < rctx->nvertex_buffer; i++) { pipe_resource_reference(&rctx->vertex_buffer[i].buffer, NULL); @@ -444,8 +445,11 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx, memcpy(rctx->vertex_buffer, buffers, sizeof(struct pipe_vertex_buffer) * count); for (i = 0; i < count; i++) { rctx->vertex_buffer[i].buffer = NULL; + if (r600_buffer_is_user_buffer(buffers[i].buffer)) + any_user_buffers = TRUE; pipe_resource_reference(&rctx->vertex_buffer[i].buffer, buffers[i].buffer); } + rctx->any_user_vbs = any_user_buffers; rctx->nvertex_buffer = count; } -- 2.30.2