gallium/radeon: Initialize pipe_resource::next to NULL
[mesa.git] / src / gallium / drivers / radeon / r600_buffer_common.c
index d85b64b457fd471147a62dadc6c3123e684b7601..cbbcc293e124efc686ab110f73a6763ece56c836 100644 (file)
 #include <inttypes.h>
 #include <stdio.h>
 
-boolean r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
-                                       struct radeon_winsys_cs_handle *buf,
-                                       enum radeon_bo_usage usage)
+bool r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
+                                    struct pb_buffer *buf,
+                                    enum radeon_bo_usage usage)
 {
-       if (ctx->ws->cs_is_buffer_referenced(ctx->rings.gfx.cs, buf, usage)) {
-               return TRUE;
+       if (ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, buf, usage)) {
+               return true;
        }
-       if (ctx->rings.dma.cs && ctx->rings.dma.cs->cdw &&
-           ctx->ws->cs_is_buffer_referenced(ctx->rings.dma.cs, buf, usage)) {
-               return TRUE;
+       if (radeon_emitted(ctx->dma.cs, 0) &&
+           ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, buf, usage)) {
+               return true;
        }
-       return FALSE;
+       return false;
 }
 
 void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
@@ -52,7 +52,7 @@ void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
        bool busy = false;
 
        if (usage & PIPE_TRANSFER_UNSYNCHRONIZED) {
-               return ctx->ws->buffer_map(resource->cs_buf, NULL, usage);
+               return ctx->ws->buffer_map(resource->buf, NULL, usage);
        }
 
        if (!(usage & PIPE_TRANSFER_WRITE)) {
@@ -60,108 +60,143 @@ void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
                rusage = RADEON_USAGE_WRITE;
        }
 
-       if (ctx->rings.gfx.cs->cdw != ctx->initial_gfx_cs_size &&
-           ctx->ws->cs_is_buffer_referenced(ctx->rings.gfx.cs,
-                                            resource->cs_buf, rusage)) {
+       if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
+           ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs,
+                                            resource->buf, rusage)) {
                if (usage & PIPE_TRANSFER_DONTBLOCK) {
-                       ctx->rings.gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+                       ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
                        return NULL;
                } else {
-                       ctx->rings.gfx.flush(ctx, 0, NULL);
+                       ctx->gfx.flush(ctx, 0, NULL);
                        busy = true;
                }
        }
-       if (ctx->rings.dma.cs &&
-           ctx->rings.dma.cs->cdw &&
-           ctx->ws->cs_is_buffer_referenced(ctx->rings.dma.cs,
-                                            resource->cs_buf, rusage)) {
+       if (radeon_emitted(ctx->dma.cs, 0) &&
+           ctx->ws->cs_is_buffer_referenced(ctx->dma.cs,
+                                            resource->buf, rusage)) {
                if (usage & PIPE_TRANSFER_DONTBLOCK) {
-                       ctx->rings.dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
+                       ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
                        return NULL;
                } else {
-                       ctx->rings.dma.flush(ctx, 0, NULL);
+                       ctx->dma.flush(ctx, 0, NULL);
                        busy = true;
                }
        }
 
-       if (busy || ctx->ws->buffer_is_busy(resource->buf, rusage)) {
+       if (busy || !ctx->ws->buffer_wait(resource->buf, 0, rusage)) {
                if (usage & PIPE_TRANSFER_DONTBLOCK) {
                        return NULL;
                } else {
                        /* We will be wait for the GPU. Wait for any offloaded
                         * CS flush to complete to avoid busy-waiting in the winsys. */
-                       ctx->ws->cs_sync_flush(ctx->rings.gfx.cs);
-                       if (ctx->rings.dma.cs)
-                               ctx->ws->cs_sync_flush(ctx->rings.dma.cs);
+                       ctx->ws->cs_sync_flush(ctx->gfx.cs);
+                       if (ctx->dma.cs)
+                               ctx->ws->cs_sync_flush(ctx->dma.cs);
                }
        }
 
        /* Setting the CS to NULL will prevent doing checks we have done already. */
-       return ctx->ws->buffer_map(resource->cs_buf, NULL, usage);
+       return ctx->ws->buffer_map(resource->buf, NULL, usage);
 }
 
-bool r600_init_resource(struct r600_common_screen *rscreen,
-                       struct r600_resource *res,
-                       unsigned size, unsigned alignment,
-                       bool use_reusable_pool)
+void r600_init_resource_fields(struct r600_common_screen *rscreen,
+                              struct r600_resource *res,
+                              uint64_t size, unsigned alignment)
 {
        struct r600_texture *rtex = (struct r600_texture*)res;
-       struct pb_buffer *old_buf, *new_buf;
-       enum radeon_bo_flag flags = 0;
+
+       res->bo_size = size;
+       res->bo_alignment = alignment;
+       res->flags = 0;
 
        switch (res->b.b.usage) {
+       case PIPE_USAGE_STREAM:
+               res->flags = RADEON_FLAG_GTT_WC;
+               /* fall through */
        case PIPE_USAGE_STAGING:
-               /* Transfers are likely to occur more often with these resources. */
+               /* Transfers are likely to occur more often with these
+                * resources. */
                res->domains = RADEON_DOMAIN_GTT;
                break;
-       case PIPE_USAGE_STREAM:
        case PIPE_USAGE_DYNAMIC:
                /* Older kernels didn't always flush the HDP cache before
                 * CS execution
                 */
-               if (rscreen->info.drm_minor < 40) {
+               if (rscreen->info.drm_major == 2 &&
+                   rscreen->info.drm_minor < 40) {
                        res->domains = RADEON_DOMAIN_GTT;
-                       flags |= RADEON_FLAG_GTT_WC;
+                       res->flags |= RADEON_FLAG_GTT_WC;
                        break;
                }
-               flags |= RADEON_FLAG_CPU_ACCESS;
+               res->flags |= RADEON_FLAG_CPU_ACCESS;
                /* fall through */
        case PIPE_USAGE_DEFAULT:
        case PIPE_USAGE_IMMUTABLE:
        default:
-               /* Not listing GTT here improves performance in some apps. */
+               /* Not listing GTT here improves performance in some
+                * apps. */
                res->domains = RADEON_DOMAIN_VRAM;
-               flags |= RADEON_FLAG_GTT_WC;
+               res->flags |= RADEON_FLAG_GTT_WC;
                break;
        }
 
        if (res->b.b.target == PIPE_BUFFER &&
            res->b.b.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
                              PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
-               /* Use GTT for all persistent mappings with older kernels,
-                * because they didn't always flush the HDP cache before CS
-                * execution.
+               /* Use GTT for all persistent mappings with older
+                * kernels, because they didn't always flush the HDP
+                * cache before CS execution.
                 *
-                * Write-combined CPU mappings are fine, the kernel ensures all CPU
-                * writes finish before the GPU executes a command stream.
+                * Write-combined CPU mappings are fine, the kernel
+                * ensures all CPU writes finish before the GPU
+                * executes a command stream.
                 */
-               if (rscreen->info.drm_minor < 40)
+               if (rscreen->info.drm_major == 2 &&
+                   rscreen->info.drm_minor < 40)
                        res->domains = RADEON_DOMAIN_GTT;
                else if (res->domains & RADEON_DOMAIN_VRAM)
-                       flags |= RADEON_FLAG_CPU_ACCESS;
+                       res->flags |= RADEON_FLAG_CPU_ACCESS;
        }
 
        /* Tiled textures are unmappable. Always put them in VRAM. */
        if (res->b.b.target != PIPE_BUFFER &&
            rtex->surface.level[0].mode >= RADEON_SURF_MODE_1D) {
                res->domains = RADEON_DOMAIN_VRAM;
-               flags &= ~RADEON_FLAG_CPU_ACCESS;
+               res->flags &= ~RADEON_FLAG_CPU_ACCESS;
+               res->flags |= RADEON_FLAG_NO_CPU_ACCESS |
+                        RADEON_FLAG_GTT_WC;
        }
 
+       /* If VRAM is just stolen system memory, allow both VRAM and
+        * GTT, whichever has free space. If a buffer is evicted from
+        * VRAM to GTT, it will stay there.
+        */
+       if (!rscreen->info.has_dedicated_vram &&
+           res->domains == RADEON_DOMAIN_VRAM)
+               res->domains = RADEON_DOMAIN_VRAM_GTT;
+
+       if (rscreen->debug_flags & DBG_NO_WC)
+               res->flags &= ~RADEON_FLAG_GTT_WC;
+
+       /* Set expected VRAM and GART usage for the buffer. */
+       res->vram_usage = 0;
+       res->gart_usage = 0;
+
+       if (res->domains & RADEON_DOMAIN_VRAM)
+               res->vram_usage = size;
+       else if (res->domains & RADEON_DOMAIN_GTT)
+               res->gart_usage = size;
+}
+
+bool r600_alloc_resource(struct r600_common_screen *rscreen,
+                        struct r600_resource *res)
+{
+       struct pb_buffer *old_buf, *new_buf;
+
        /* Allocate a new resource. */
-       new_buf = rscreen->ws->buffer_create(rscreen->ws, size, alignment,
-                                            use_reusable_pool,
-                                            res->domains, flags);
+       new_buf = rscreen->ws->buffer_create(rscreen->ws, res->bo_size,
+                                            res->bo_alignment,
+                                            res->domains, res->flags);
        if (!new_buf) {
                return false;
        }
@@ -171,20 +206,21 @@ bool r600_init_resource(struct r600_common_screen *rscreen,
         * the same buffer where one of the contexts invalidates it while
         * the others are using it. */
        old_buf = res->buf;
-       res->cs_buf = rscreen->ws->buffer_get_cs_handle(new_buf); /* should be atomic */
        res->buf = new_buf; /* should be atomic */
 
-       if (rscreen->info.r600_virtual_address)
-               res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->cs_buf);
+       if (rscreen->info.has_virtual_memory)
+               res->gpu_address = rscreen->ws->buffer_get_virtual_address(res->buf);
        else
                res->gpu_address = 0;
 
        pb_reference(&old_buf, NULL);
 
        util_range_set_empty(&res->valid_buffer_range);
+       res->TC_L2_dirty = false;
 
+       /* Print debug information. */
        if (rscreen->debug_flags & DBG_VM && res->b.b.target == PIPE_BUFFER) {
-               fprintf(stderr, "VM start=0x%"PRIX64"  end=0x%"PRIX64" | Buffer %u bytes\n",
+               fprintf(stderr, "VM start=0x%"PRIX64"  end=0x%"PRIX64" | Buffer %"PRIu64" bytes\n",
                        res->gpu_address, res->gpu_address + res->buf->size,
                        res->buf->size);
        }
@@ -201,6 +237,42 @@ static void r600_buffer_destroy(struct pipe_screen *screen,
        FREE(rbuffer);
 }
 
+static bool
+r600_invalidate_buffer(struct r600_common_context *rctx,
+                      struct r600_resource *rbuffer)
+{
+       /* Shared buffers can't be reallocated. */
+       if (rbuffer->is_shared)
+               return false;
+
+       /* In AMD_pinned_memory, the user pointer association only gets
+        * broken when the buffer is explicitly re-allocated.
+        */
+       if (rctx->ws->buffer_is_user_ptr(rbuffer->buf))
+               return false;
+
+       /* Check if mapping this buffer would cause waiting for the GPU. */
+       if (r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
+           !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
+               rctx->invalidate_buffer(&rctx->b, &rbuffer->b.b);
+       } else {
+               util_range_set_empty(&rbuffer->valid_buffer_range);
+       }
+
+       return true;
+}
+
+void r600_invalidate_resource(struct pipe_context *ctx,
+                             struct pipe_resource *resource)
+{
+       struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+       struct r600_resource *rbuffer = r600_resource(resource);
+
+       /* We currently only do anyting here for buffers */
+       if (resource->target == PIPE_BUFFER)
+               (void)r600_invalidate_buffer(rctx, rbuffer);
+}
+
 static void *r600_buffer_get_transfer(struct pipe_context *ctx,
                                      struct pipe_resource *resource,
                                       unsigned level,
@@ -211,7 +283,7 @@ static void *r600_buffer_get_transfer(struct pipe_context *ctx,
                                      unsigned offset)
 {
        struct r600_common_context *rctx = (struct r600_common_context*)ctx;
-       struct r600_transfer *transfer = util_slab_alloc(&rctx->pool_transfers);
+       struct r600_transfer *transfer = slab_alloc_st(&rctx->pool_transfers);
 
        transfer->transfer.resource = resource;
        transfer->transfer.level = level;
@@ -231,7 +303,7 @@ static bool r600_can_dma_copy_buffer(struct r600_common_context *rctx,
        bool dword_aligned = !(dstx % 4) && !(srcx % 4) && !(size % 4);
 
        return rctx->screen->has_cp_dma ||
-              (dword_aligned && (rctx->rings.dma.cs ||
+              (dword_aligned && (rctx->dma.cs ||
                                  rctx->screen->has_streamout));
 
 }
@@ -254,6 +326,7 @@ static void *r600_buffer_transfer_map(struct pipe_context *ctx,
         * in which case it can be mapped unsynchronized. */
        if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED) &&
            usage & PIPE_TRANSFER_WRITE &&
+           !rbuffer->is_shared &&
            !util_ranges_intersect(&rbuffer->valid_buffer_range, box->x, box->x + box->width)) {
                usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
        }
@@ -268,58 +341,64 @@ static void *r600_buffer_transfer_map(struct pipe_context *ctx,
            !(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
                assert(usage & PIPE_TRANSFER_WRITE);
 
-               /* Check if mapping this buffer would cause waiting for the GPU. */
-               if (r600_rings_is_buffer_referenced(rctx, rbuffer->cs_buf, RADEON_USAGE_READWRITE) ||
-                   rctx->ws->buffer_is_busy(rbuffer->buf, RADEON_USAGE_READWRITE)) {
-                       rctx->invalidate_buffer(&rctx->b, &rbuffer->b.b);
+               if (r600_invalidate_buffer(rctx, rbuffer)) {
+                       /* At this point, the buffer is always idle. */
+                       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
+               } else {
+                       /* Fall back to a temporary buffer. */
+                       usage |= PIPE_TRANSFER_DISCARD_RANGE;
                }
-               /* At this point, the buffer is always idle. */
-               usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
        }
-       else if ((usage & PIPE_TRANSFER_DISCARD_RANGE) &&
-                !(usage & PIPE_TRANSFER_UNSYNCHRONIZED) &&
-                !(rscreen->debug_flags & DBG_NO_DISCARD_RANGE) &&
-                r600_can_dma_copy_buffer(rctx, box->x, 0, box->width)) {
+
+       if ((usage & PIPE_TRANSFER_DISCARD_RANGE) &&
+           !(usage & (PIPE_TRANSFER_UNSYNCHRONIZED |
+                      PIPE_TRANSFER_PERSISTENT)) &&
+           !(rscreen->debug_flags & DBG_NO_DISCARD_RANGE) &&
+           r600_can_dma_copy_buffer(rctx, box->x, 0, box->width)) {
                assert(usage & PIPE_TRANSFER_WRITE);
 
                /* Check if mapping this buffer would cause waiting for the GPU. */
-               if (r600_rings_is_buffer_referenced(rctx, rbuffer->cs_buf, RADEON_USAGE_READWRITE) ||
-                   rctx->ws->buffer_is_busy(rbuffer->buf, RADEON_USAGE_READWRITE)) {
+               if (r600_rings_is_buffer_referenced(rctx, rbuffer->buf, RADEON_USAGE_READWRITE) ||
+                   !rctx->ws->buffer_wait(rbuffer->buf, 0, RADEON_USAGE_READWRITE)) {
                        /* Do a wait-free write-only transfer using a temporary buffer. */
                        unsigned offset;
                        struct r600_resource *staging = NULL;
 
                        u_upload_alloc(rctx->uploader, 0, box->width + (box->x % R600_MAP_BUFFER_ALIGNMENT),
-                                      &offset, (struct pipe_resource**)&staging, (void**)&data);
+                                      256, &offset, (struct pipe_resource**)&staging, (void**)&data);
 
                        if (staging) {
                                data += box->x % R600_MAP_BUFFER_ALIGNMENT;
                                return r600_buffer_get_transfer(ctx, resource, level, usage, box,
                                                                ptransfer, data, staging, offset);
-                       } else {
-                               return NULL; /* error, shouldn't occur though */
                        }
+               } else {
+                       /* At this point, the buffer is always idle (we checked it above). */
+                       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
                }
-               /* At this point, the buffer is always idle (we checked it above). */
-               usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
        }
        /* Using a staging buffer in GTT for larger reads is much faster. */
        else if ((usage & PIPE_TRANSFER_READ) &&
-                !(usage & PIPE_TRANSFER_WRITE) &&
-                rbuffer->domains == RADEON_DOMAIN_VRAM &&
+                !(usage & (PIPE_TRANSFER_WRITE |
+                           PIPE_TRANSFER_PERSISTENT)) &&
+                rbuffer->domains & RADEON_DOMAIN_VRAM &&
                 r600_can_dma_copy_buffer(rctx, 0, box->x, box->width)) {
                struct r600_resource *staging;
 
                staging = (struct r600_resource*) pipe_buffer_create(
-                               ctx->screen, PIPE_BIND_TRANSFER_READ, PIPE_USAGE_STAGING,
+                               ctx->screen, 0, PIPE_USAGE_STAGING,
                                box->width + (box->x % R600_MAP_BUFFER_ALIGNMENT));
                if (staging) {
                        /* Copy the VRAM buffer to the staging buffer. */
-                       rctx->dma_copy(ctx, &staging->b.b, 0,
-                                      box->x % R600_MAP_BUFFER_ALIGNMENT,
-                                      0, 0, resource, level, box);
+                       ctx->resource_copy_region(ctx, &staging->b.b, 0,
+                                                 box->x % R600_MAP_BUFFER_ALIGNMENT,
+                                                 0, 0, resource, level, box);
 
                        data = r600_buffer_map_sync_with_rings(rctx, staging, PIPE_TRANSFER_READ);
+                       if (!data) {
+                               r600_resource_reference(&staging, NULL);
+                               return NULL;
+                       }
                        data += box->x % R600_MAP_BUFFER_ALIGNMENT;
 
                        return r600_buffer_get_transfer(ctx, resource, level, usage, box,
@@ -337,38 +416,81 @@ static void *r600_buffer_transfer_map(struct pipe_context *ctx,
                                        ptransfer, data, NULL, 0);
 }
 
-static void r600_buffer_transfer_unmap(struct pipe_context *ctx,
-                                      struct pipe_transfer *transfer)
+static void r600_buffer_do_flush_region(struct pipe_context *ctx,
+                                       struct pipe_transfer *transfer,
+                                       const struct pipe_box *box)
 {
-       struct r600_common_context *rctx = (struct r600_common_context*)ctx;
        struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
        struct r600_resource *rbuffer = r600_resource(transfer->resource);
 
        if (rtransfer->staging) {
-               if (rtransfer->transfer.usage & PIPE_TRANSFER_WRITE) {
-                       struct pipe_resource *dst, *src;
-                       unsigned soffset, doffset, size;
-                       struct pipe_box box;
+               struct pipe_resource *dst, *src;
+               unsigned soffset;
+               struct pipe_box dma_box;
 
-                       dst = transfer->resource;
-                       src = &rtransfer->staging->b.b;
-                       size = transfer->box.width;
-                       doffset = transfer->box.x;
-                       soffset = rtransfer->offset + transfer->box.x % R600_MAP_BUFFER_ALIGNMENT;
+               dst = transfer->resource;
+               src = &rtransfer->staging->b.b;
+               soffset = rtransfer->offset + box->x % R600_MAP_BUFFER_ALIGNMENT;
 
-                       u_box_1d(soffset, size, &box);
+               u_box_1d(soffset, box->width, &dma_box);
 
-                       /* Copy the staging buffer into the original one. */
-                       rctx->dma_copy(ctx, dst, 0, doffset, 0, 0, src, 0, &box);
-               }
-               pipe_resource_reference((struct pipe_resource**)&rtransfer->staging, NULL);
+               /* Copy the staging buffer into the original one. */
+               ctx->resource_copy_region(ctx, dst, 0, box->x, 0, 0, src, 0, &dma_box);
        }
 
-       if (transfer->usage & PIPE_TRANSFER_WRITE) {
-               util_range_add(&rbuffer->valid_buffer_range, transfer->box.x,
-                              transfer->box.x + transfer->box.width);
+       util_range_add(&rbuffer->valid_buffer_range, box->x,
+                      box->x + box->width);
+}
+
+static void r600_buffer_flush_region(struct pipe_context *ctx,
+                                    struct pipe_transfer *transfer,
+                                    const struct pipe_box *rel_box)
+{
+       if (transfer->usage & (PIPE_TRANSFER_WRITE |
+                              PIPE_TRANSFER_FLUSH_EXPLICIT)) {
+               struct pipe_box box;
+
+               u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
+               r600_buffer_do_flush_region(ctx, transfer, &box);
        }
-       util_slab_free(&rctx->pool_transfers, transfer);
+}
+
+static void r600_buffer_transfer_unmap(struct pipe_context *ctx,
+                                      struct pipe_transfer *transfer)
+{
+       struct r600_common_context *rctx = (struct r600_common_context*)ctx;
+       struct r600_transfer *rtransfer = (struct r600_transfer*)transfer;
+
+       if (transfer->usage & PIPE_TRANSFER_WRITE &&
+           !(transfer->usage & PIPE_TRANSFER_FLUSH_EXPLICIT))
+               r600_buffer_do_flush_region(ctx, transfer, &transfer->box);
+
+       if (rtransfer->staging)
+               r600_resource_reference(&rtransfer->staging, NULL);
+
+       slab_free_st(&rctx->pool_transfers, transfer);
+}
+
+void r600_buffer_subdata(struct pipe_context *ctx,
+                        struct pipe_resource *buffer,
+                        unsigned usage, unsigned offset,
+                        unsigned size, const void *data)
+{
+       struct pipe_transfer *transfer = NULL;
+       struct pipe_box box;
+       uint8_t *map = NULL;
+
+       u_box_1d(offset, size, &box);
+       map = r600_buffer_transfer_map(ctx, buffer, 0,
+                                      PIPE_TRANSFER_WRITE |
+                                      PIPE_TRANSFER_DISCARD_RANGE |
+                                      usage,
+                                      &box, &transfer);
+       if (!map)
+               return;
+
+       memcpy(map, data, size);
+       r600_buffer_transfer_unmap(ctx, transfer);
 }
 
 static const struct u_resource_vtbl r600_buffer_vtbl =
@@ -376,30 +498,91 @@ static const struct u_resource_vtbl r600_buffer_vtbl =
        NULL,                           /* get_handle */
        r600_buffer_destroy,            /* resource_destroy */
        r600_buffer_transfer_map,       /* transfer_map */
-       NULL,                           /* transfer_flush_region */
+       r600_buffer_flush_region,       /* transfer_flush_region */
        r600_buffer_transfer_unmap,     /* transfer_unmap */
-       NULL                            /* transfer_inline_write */
 };
 
-struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
-                                        const struct pipe_resource *templ,
-                                        unsigned alignment)
+static struct r600_resource *
+r600_alloc_buffer_struct(struct pipe_screen *screen,
+                        const struct pipe_resource *templ)
 {
-       struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
        struct r600_resource *rbuffer;
 
        rbuffer = MALLOC_STRUCT(r600_resource);
 
        rbuffer->b.b = *templ;
+       rbuffer->b.b.next = NULL;
        pipe_reference_init(&rbuffer->b.b.reference, 1);
        rbuffer->b.b.screen = screen;
        rbuffer->b.vtbl = &r600_buffer_vtbl;
        rbuffer->buf = NULL;
+       rbuffer->TC_L2_dirty = false;
+       rbuffer->is_shared = false;
        util_range_init(&rbuffer->valid_buffer_range);
+       return rbuffer;
+}
+
+struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
+                                        const struct pipe_resource *templ,
+                                        unsigned alignment)
+{
+       struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+       struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
+
+       r600_init_resource_fields(rscreen, rbuffer, templ->width0, alignment);
+
+       if (!r600_alloc_resource(rscreen, rbuffer)) {
+               FREE(rbuffer);
+               return NULL;
+       }
+       return &rbuffer->b.b;
+}
 
-       if (!r600_init_resource(rscreen, rbuffer, templ->width0, alignment, TRUE)) {
+struct pipe_resource *r600_aligned_buffer_create(struct pipe_screen *screen,
+                                                unsigned bind,
+                                                unsigned usage,
+                                                unsigned size,
+                                                unsigned alignment)
+{
+       struct pipe_resource buffer;
+
+       memset(&buffer, 0, sizeof buffer);
+       buffer.target = PIPE_BUFFER;
+       buffer.format = PIPE_FORMAT_R8_UNORM;
+       buffer.bind = bind;
+       buffer.usage = usage;
+       buffer.flags = 0;
+       buffer.width0 = size;
+       buffer.height0 = 1;
+       buffer.depth0 = 1;
+       buffer.array_size = 1;
+       return r600_buffer_create(screen, &buffer, alignment);
+}
+
+struct pipe_resource *
+r600_buffer_from_user_memory(struct pipe_screen *screen,
+                            const struct pipe_resource *templ,
+                            void *user_memory)
+{
+       struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
+       struct radeon_winsys *ws = rscreen->ws;
+       struct r600_resource *rbuffer = r600_alloc_buffer_struct(screen, templ);
+
+       rbuffer->domains = RADEON_DOMAIN_GTT;
+       util_range_add(&rbuffer->valid_buffer_range, 0, templ->width0);
+
+       /* Convert a user pointer to a buffer. */
+       rbuffer->buf = ws->buffer_from_ptr(ws, user_memory, templ->width0);
+       if (!rbuffer->buf) {
                FREE(rbuffer);
                return NULL;
        }
+
+       if (rscreen->info.has_virtual_memory)
+               rbuffer->gpu_address =
+                       ws->buffer_get_virtual_address(rbuffer->buf);
+       else
+               rbuffer->gpu_address = 0;
+
        return &rbuffer->b.b;
 }