r600g: add upload manager support.

this add support for the upload manager for uploading user vbo/index buffers.

this provides a considerable speedup in q3 type games.

diff --git a/src/gallium/drivers/r600/r600_buffer.c b/src/gallium/drivers/r600/r600_buffer.c
index a38c013e1674b8d5caa5b2f042f587405ec3fcd8..dc3fc812e1ae34890671d25ac206c76f22bb6dc7 100644 (file)
--- a/src/gallium/drivers/r600/r600_buffer.c
+++ b/src/gallium/drivers/r600/r600_buffer.c
@@ -29,6 +29,7 @@
 #include <util/u_math.h>
 #include <util/u_inlines.h>
 #include <util/u_memory.h>
+#include <util/u_upload_mgr.h>
 #include "state_tracker/drm_driver.h"
 #include "r600_screen.h"
 #include "r600_context.h"
@@ -67,66 +68,69 @@ struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
                                         const struct pipe_resource *templ)
 {
        struct r600_screen *rscreen = r600_screen(screen);
-       struct r600_resource *rbuffer;
+       struct r600_resource_buffer *rbuffer;
        struct radeon_ws_bo *bo;
        /* XXX We probably want a different alignment for buffers and textures. */
        unsigned alignment = 4096;
 
-       rbuffer = CALLOC_STRUCT(r600_resource);
+       rbuffer = CALLOC_STRUCT(r600_resource_buffer);
        if (rbuffer == NULL)
                return NULL;
 
-       rbuffer->base.b = *templ;
-       pipe_reference_init(&rbuffer->base.b.reference, 1);
-       rbuffer->base.b.screen = screen;
-       rbuffer->base.vtbl = &r600_buffer_vtbl;
-       rbuffer->size = rbuffer->base.b.width0;
-       rbuffer->domain = r600_domain_from_usage(rbuffer->base.b.bind);
-       bo = radeon_ws_bo(rscreen->rw, rbuffer->base.b.width0, alignment, rbuffer->base.b.bind);
+       rbuffer->magic = R600_BUFFER_MAGIC;
+       rbuffer->user_buffer = NULL;
+       rbuffer->num_ranges = 0;
+       rbuffer->r.base.b = *templ;
+       pipe_reference_init(&rbuffer->r.base.b.reference, 1);
+       rbuffer->r.base.b.screen = screen;
+       rbuffer->r.base.vtbl = &r600_buffer_vtbl;
+       rbuffer->r.size = rbuffer->r.base.b.width0;
+       rbuffer->r.domain = r600_domain_from_usage(rbuffer->r.base.b.bind);
+       bo = radeon_ws_bo(rscreen->rw, rbuffer->r.base.b.width0, alignment, rbuffer->r.base.b.bind);
        if (bo == NULL) {
                FREE(rbuffer);
                return NULL;
        }
-       rbuffer->bo = bo;
-       return &rbuffer->base.b;
+       rbuffer->r.bo = bo;
+       return &rbuffer->r.base.b;
 }
 
 struct pipe_resource *r600_user_buffer_create(struct pipe_screen *screen,
                                              void *ptr, unsigned bytes,
                                              unsigned bind)
 {
-       struct r600_resource *rbuffer;
-       struct r600_screen *rscreen = r600_screen(screen);
-       struct pipe_resource templ;
-       void *data;
-
-       memset(&templ, 0, sizeof(struct pipe_resource));
-       templ.target = PIPE_BUFFER;
-       templ.format = PIPE_FORMAT_R8_UNORM;
-       templ.usage = PIPE_USAGE_IMMUTABLE;
-       templ.bind = bind;
-       templ.width0 = bytes;
-       templ.height0 = 1;
-       templ.depth0 = 1;
-
-       rbuffer = (struct r600_resource*)r600_buffer_create(screen, &templ);
-       if (rbuffer == NULL) {
+       struct r600_resource_buffer *rbuffer;
+
+       rbuffer = CALLOC_STRUCT(r600_resource_buffer);
+       if (rbuffer == NULL)
                return NULL;
-       }
-       data = radeon_ws_bo_map(rscreen->rw, rbuffer->bo, 0, NULL);
-       memcpy(data, ptr, bytes);
-       radeon_ws_bo_unmap(rscreen->rw, rbuffer->bo);
-       return &rbuffer->base.b;
+
+       rbuffer->magic = R600_BUFFER_MAGIC;
+       pipe_reference_init(&rbuffer->r.base.b.reference, 1);
+       rbuffer->r.base.vtbl = &r600_buffer_vtbl;
+       rbuffer->r.base.b.screen = screen;
+       rbuffer->r.base.b.target = PIPE_BUFFER;
+       rbuffer->r.base.b.format = PIPE_FORMAT_R8_UNORM;
+       rbuffer->r.base.b.usage = PIPE_USAGE_IMMUTABLE;
+       rbuffer->r.base.b.bind = bind;
+       rbuffer->r.base.b.width0 = bytes;
+       rbuffer->r.base.b.height0 = 1;
+       rbuffer->r.base.b.depth0 = 1;
+       rbuffer->r.base.b.flags = 0;
+       rbuffer->num_ranges = 0;
+       rbuffer->r.bo = NULL;
+       rbuffer->user_buffer = ptr;
+       return &rbuffer->r.base.b;
 }
 
 static void r600_buffer_destroy(struct pipe_screen *screen,
                                struct pipe_resource *buf)
 {
-       struct r600_resource *rbuffer = (struct r600_resource*)buf;
+       struct r600_resource_buffer *rbuffer = r600_buffer(buf);
        struct r600_screen *rscreen = r600_screen(screen);
 
-       if (rbuffer->bo) {
-               radeon_ws_bo_reference(rscreen->rw, &rbuffer->bo, NULL);
+       if (rbuffer->r.bo) {
+               radeon_ws_bo_reference(rscreen->rw, &rbuffer->r.bo, NULL);
        }
        FREE(rbuffer);
 }
@@ -134,18 +138,40 @@ static void r600_buffer_destroy(struct pipe_screen *screen,
 static void *r600_buffer_transfer_map(struct pipe_context *pipe,
                                      struct pipe_transfer *transfer)
 {
-       struct r600_resource *rbuffer = (struct r600_resource*)transfer->resource;
+       struct r600_context *rctx = r600_context(pipe);
+       struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
        struct r600_screen *rscreen = r600_screen(pipe->screen);
        int write = 0;
        uint8_t *data;
+       int i;
+       boolean flush = FALSE;
+
+       if (rbuffer->user_buffer)
+               return (uint8_t*)rbuffer->user_buffer + transfer->box.x;
 
+       if (transfer->usage & PIPE_TRANSFER_DISCARD) {
+               for (i = 0; i < rbuffer->num_ranges; i++) {
+                       if ((transfer->box.x >= rbuffer->ranges[i].start) &&
+                           (transfer->box.x < rbuffer->ranges[i].end))
+                               flush = TRUE;
+                       
+                       if (flush) {
+                               radeon_ws_bo_reference(rscreen->rw, &rbuffer->r.bo, NULL);
+                               rbuffer->num_ranges = 0;
+                               rbuffer->r.bo = radeon_ws_bo(rscreen->rw, 
+                                                            rbuffer->r.base.b.width0, 0,
+                                                            rbuffer->r.base.b.bind);
+                               break;
+                       }
+               }
+       }
        if (transfer->usage & PIPE_TRANSFER_DONTBLOCK) {
                /* FIXME */
        }
        if (transfer->usage & PIPE_TRANSFER_WRITE) {
                write = 1;
        }
-       data = radeon_ws_bo_map(rscreen->rw, rbuffer->bo, transfer->usage, r600_context(pipe));
+       data = radeon_ws_bo_map(rscreen->rw, rbuffer->r.bo, transfer->usage, rctx);
        if (!data)
                return NULL;
 
@@ -155,16 +181,39 @@ static void *r600_buffer_transfer_map(struct pipe_context *pipe,
 static void r600_buffer_transfer_unmap(struct pipe_context *pipe,
                                        struct pipe_transfer *transfer)
 {
-       struct r600_resource *rbuffer = (struct r600_resource*)transfer->resource;
+       struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
        struct r600_screen *rscreen = r600_screen(pipe->screen);
 
-       radeon_ws_bo_unmap(rscreen->rw, rbuffer->bo);
+       if (rbuffer->r.bo)
+               radeon_ws_bo_unmap(rscreen->rw, rbuffer->r.bo);
 }
 
 static void r600_buffer_transfer_flush_region(struct pipe_context *pipe,
                                              struct pipe_transfer *transfer,
                                              const struct pipe_box *box)
 {
+       struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
+       unsigned i;
+       unsigned offset = transfer->box.x + box->x;
+       unsigned length = box->width;
+
+       assert(box->x + box->width <= transfer->box.width);
+
+       if (rbuffer->user_buffer)
+               return;
+
+       /* mark the range as used */
+       for(i = 0; i < rbuffer->num_ranges; ++i) {
+               if(offset <= rbuffer->ranges[i].end && rbuffer->ranges[i].start <= (offset+box->width)) {
+                       rbuffer->ranges[i].start = MIN2(rbuffer->ranges[i].start, offset);
+                       rbuffer->ranges[i].end   = MAX2(rbuffer->ranges[i].end, (offset+length));
+                       return;
+               }
+       }
+       
+       rbuffer->ranges[rbuffer->num_ranges].start = offset;
+       rbuffer->ranges[rbuffer->num_ranges].end = offset+length;
+       rbuffer->num_ranges++;
 }
 
 unsigned r600_buffer_is_referenced_by_cs(struct pipe_context *context,
@@ -213,3 +262,59 @@ struct u_resource_vtbl r600_buffer_vtbl =
        r600_buffer_transfer_unmap,             /* transfer_unmap */
        u_default_transfer_inline_write         /* transfer_inline_write */
 };
+
+int r600_upload_index_buffer(struct r600_context *rctx,
+                            struct r600_draw *draw)
+{
+       struct pipe_resource *upload_buffer = NULL;
+       unsigned index_offset = draw->index_buffer_offset;
+       int ret = 0;
+
+       if (r600_buffer_is_user_buffer(draw->index_buffer)) {
+               ret = u_upload_buffer(rctx->upload_ib,
+                                     index_offset,
+                                     draw->count * draw->index_size,
+                                     draw->index_buffer,
+                                     &index_offset,
+                                     &upload_buffer);
+               if (ret) {
+                       goto done;
+               }
+               draw->index_buffer_offset = index_offset;
+               draw->index_buffer = upload_buffer;
+       }
+
+done:
+       return ret;
+}
+
+int r600_upload_user_buffers(struct r600_context *rctx)
+{
+       enum pipe_error ret = PIPE_OK;
+       int i, nr;
+
+       nr = rctx->vertex_elements->count;
+
+       for (i = 0; i < nr; i++) {
+               struct pipe_vertex_buffer *vb =
+                       &rctx->vertex_buffer[rctx->vertex_elements->elements[i].vertex_buffer_index];
+
+               if (r600_buffer_is_user_buffer(vb->buffer)) {
+                       struct pipe_resource *upload_buffer = NULL;
+                       unsigned offset = 0; /*vb->buffer_offset * 4;*/
+                       unsigned size = vb->buffer->width0;
+                       unsigned upload_offset;
+                       ret = u_upload_buffer(rctx->upload_vb,
+                                             offset, size,
+                                             vb->buffer,
+                                             &upload_offset, &upload_buffer);
+                       if (ret)
+                               return ret;
+
+                       pipe_resource_reference(&vb->buffer, NULL);
+                       vb->buffer = upload_buffer;
+                       vb->buffer_offset = upload_offset;
+               }
+       }
+       return ret;
+}

diff --git a/src/gallium/drivers/r600/r600_context.c b/src/gallium/drivers/r600/r600_context.c
index cca1e3567341826df29c190bb2aa15be6f0b469b..776dc24569b692b826382d8c100ed0bf19eb2fdd 100644 (file)
--- a/src/gallium/drivers/r600/r600_context.c
+++ b/src/gallium/drivers/r600/r600_context.c
@@ -28,6 +28,7 @@
 #include <util/u_inlines.h>
 #include <util/u_format.h>
 #include <util/u_memory.h>
+#include <util/u_upload_mgr.h>
 #include <util/u_blitter.h>
 #include "r600_screen.h"
 #include "r600_context.h"
@@ -56,6 +57,9 @@ static void r600_destroy_context(struct pipe_context *context)
        free(rctx->vs_constant);
        free(rctx->vs_resource);
 
+       u_upload_destroy(rctx->upload_vb);
+       u_upload_destroy(rctx->upload_ib);
+
        radeon_ctx_fini(rctx->ctx);
        FREE(rctx);
 }
@@ -66,6 +70,10 @@ void r600_flush(struct pipe_context *ctx, unsigned flags,
        struct r600_context *rctx = r600_context(ctx);
        struct r600_query *rquery = NULL;
 
+       /* flush upload buffers */
+       u_upload_flush(rctx->upload_vb);
+       u_upload_flush(rctx->upload_ib);
+
        /* suspend queries */
        r600_queries_suspend(ctx);
 
@@ -123,25 +131,37 @@ struct pipe_context *r600_create_context(struct pipe_screen *screen, void *priv)
 
        rctx->vtbl->init_config(rctx);
 
+       rctx->upload_ib = u_upload_create(&rctx->context, 32 * 1024, 16,
+                                         PIPE_BIND_INDEX_BUFFER);
+       if (rctx->upload_ib == NULL) {
+               goto out_free;
+       }
+
+       rctx->upload_vb = u_upload_create(&rctx->context, 128 * 1024, 16,
+                                         PIPE_BIND_VERTEX_BUFFER);
+       if (rctx->upload_vb == NULL) {
+               goto out_free;
+       }
+
        rctx->vs_constant = (struct radeon_state *)calloc(R600_MAX_CONSTANT, sizeof(struct radeon_state));
        if (!rctx->vs_constant) {
-               FREE(rctx);
-               return NULL;
+               goto out_free;
        }
 
        rctx->ps_constant = (struct radeon_state *)calloc(R600_MAX_CONSTANT, sizeof(struct radeon_state));
        if (!rctx->ps_constant) {
-               FREE(rctx);
-               return NULL;
+               goto out_free;
        }
 
        rctx->vs_resource = (struct radeon_state *)calloc(R600_MAX_RESOURCE, sizeof(struct radeon_state));
        if (!rctx->vs_resource) {
-               FREE(rctx);
-               return NULL;
+               goto out_free;
        }                                                  
 
        rctx->ctx = radeon_ctx_init(rscreen->rw);
        radeon_draw_init(&rctx->draw, rscreen->rw);
        return &rctx->context;
+ out_free:
+       FREE(rctx);
+       return NULL;
 }

diff --git a/src/gallium/drivers/r600/r600_context.h b/src/gallium/drivers/r600/r600_context.h
index f82c8f82fe045238be52c2970d76fbf1fec509b4..3107f189c784ba2bf88325025f2c36dc528a0886 100644 (file)
--- a/src/gallium/drivers/r600/r600_context.h
+++ b/src/gallium/drivers/r600/r600_context.h
@@ -34,6 +34,8 @@
 #include "radeon.h"
 #include "r600_shader.h"
 
+struct u_upload_mgr;
+
 #define R600_QUERY_STATE_STARTED       (1 << 0)
 #define R600_QUERY_STATE_ENDED         (1 << 1)
 #define R600_QUERY_STATE_SUSPENDED     (1 << 2)
@@ -249,6 +251,12 @@ struct r600_context {
        struct pipe_index_buffer        index_buffer;
        struct pipe_blend_color         blend_color;
        struct list_head                query_list;
+
+       /* upload managers */
+       struct u_upload_mgr *upload_vb;
+       struct u_upload_mgr *upload_ib;
+       bool any_user_vbs;
+
 };
 
 /* Convenience cast wrapper. */
@@ -306,4 +314,8 @@ void eg_set_constant_buffer(struct pipe_context *ctx,
                            uint shader, uint index,
                            struct pipe_resource *buffer);
 
+int r600_upload_index_buffer(struct r600_context *rctx,
+                             struct r600_draw *draw);
+int r600_upload_user_buffers(struct r600_context *rctx);
+
 #endif

diff --git a/src/gallium/drivers/r600/r600_draw.c b/src/gallium/drivers/r600/r600_draw.c
index 669c9b4cdb62d3c37e3f0eb1e824bf46fef05881..5480ca002d2360d65b92e62ebfe68a5abb92ccb7 100644 (file)
--- a/src/gallium/drivers/r600/r600_draw.c
+++ b/src/gallium/drivers/r600/r600_draw.c
@@ -126,6 +126,11 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 
        memset(&draw, 0, sizeof(draw));
 
+       if (rctx->any_user_vbs) {
+               r600_upload_user_buffers(rctx);
+               rctx->any_user_vbs = false;
+       }
+
        draw.ctx = ctx;
        draw.mode = info->mode;
        draw.start = info->start;
@@ -139,8 +144,7 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 
                assert(rctx->index_buffer.offset %
                                rctx->index_buffer.index_size == 0);
-               draw.start += rctx->index_buffer.offset /
-                       rctx->index_buffer.index_size;
+               r600_upload_index_buffer(rctx, &draw);
        }
        else {
                draw.index_size = 0;

diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h
index ff05afbc30217b081569c2511305920dc585a724..9608a5a62349d865181162943e764309b27b463f 100644 (file)
--- a/src/gallium/drivers/r600/r600_resource.h
+++ b/src/gallium/drivers/r600/r600_resource.h
@@ -75,4 +75,35 @@ struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
                                                const struct pipe_resource *base,
                                                struct winsys_handle *whandle);
 
+#define R600_BUFFER_MAGIC 0xabcd1600
+#define R600_BUFFER_MAX_RANGES 32
+
+struct r600_buffer_range {
+       uint32_t start;
+       uint32_t end;
+};
+
+struct r600_resource_buffer {
+       struct r600_resource r;
+       uint32_t magic;
+       void *user_buffer;
+       struct r600_buffer_range ranges[R600_BUFFER_MAX_RANGES];
+       unsigned num_ranges;
+};
+
+/* r600_buffer */
+static INLINE struct r600_resource_buffer *r600_buffer(struct pipe_resource *buffer)
+{
+       if (buffer) {
+               assert(((struct r600_resource_buffer *)buffer)->magic == R600_BUFFER_MAGIC);
+               return (struct r600_resource_buffer *)buffer;
+    }
+    return NULL;
+}
+
+static INLINE boolean r600_buffer_is_user_buffer(struct pipe_resource *buffer)
+{
+    return r600_buffer(buffer)->user_buffer ? true : false;
+}
+
 #endif

diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c
index 5d6236206f78b9d927b13ea404999defcaf993b9..4dcdc492fc1cec375293ffb5d9d157b8f169a568 100644 (file)
--- a/src/gallium/drivers/r600/r600_state.c
+++ b/src/gallium/drivers/r600/r600_state.c
@@ -437,6 +437,7 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx,
 {
        struct r600_context *rctx = r600_context(ctx);
        unsigned i;
+       boolean any_user_buffers = FALSE;
 
        for (i = 0; i < rctx->nvertex_buffer; i++) {
                pipe_resource_reference(&rctx->vertex_buffer[i].buffer, NULL);
@@ -444,8 +445,11 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx,
        memcpy(rctx->vertex_buffer, buffers, sizeof(struct pipe_vertex_buffer) * count);
        for (i = 0; i < count; i++) {
                rctx->vertex_buffer[i].buffer = NULL;
+               if (r600_buffer_is_user_buffer(buffers[i].buffer))
+                       any_user_buffers = TRUE;
                pipe_resource_reference(&rctx->vertex_buffer[i].buffer, buffers[i].buffer);
        }
+       rctx->any_user_vbs = any_user_buffers;
        rctx->nvertex_buffer = count;
 }