* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*/
-/*
- * Authors:
- * Marek Olšák <maraeo@gmail.com>
- */
#include "amdgpu_cs.h"
-#include "os/os_time.h"
+#include "util/os_time.h"
+#include "util/u_hash_table.h"
#include "state_tracker/drm_driver.h"
#include <amdgpu_drm.h>
#include <xf86drm.h>
#include <stdio.h>
+#include <inttypes.h>
-static const struct pb_vtbl amdgpu_winsys_bo_vtbl;
+#ifndef AMDGPU_GEM_CREATE_VM_ALWAYS_VALID
+#define AMDGPU_GEM_CREATE_VM_ALWAYS_VALID (1 << 6)
+#endif
-static inline struct amdgpu_winsys_bo *amdgpu_winsys_bo(struct pb_buffer *bo)
-{
- assert(bo->vtbl == &amdgpu_winsys_bo_vtbl);
- return (struct amdgpu_winsys_bo *)bo;
-}
+#ifndef AMDGPU_VA_RANGE_HIGH
+#define AMDGPU_VA_RANGE_HIGH 0x2
+#endif
+
+/* Set to 1 for verbose output showing committed sparse buffer ranges. */
+#define DEBUG_SPARSE_COMMITS 0
-struct amdgpu_bomgr {
- struct pb_manager base;
- struct amdgpu_winsys *rws;
+struct amdgpu_sparse_backing_chunk {
+ uint32_t begin, end;
};
-static struct amdgpu_winsys *get_winsys(struct pb_manager *mgr)
-{
- return ((struct amdgpu_bomgr*)mgr)->rws;
-}
+static struct pb_buffer *
+amdgpu_bo_create(struct radeon_winsys *rws,
+ uint64_t size,
+ unsigned alignment,
+ enum radeon_bo_domain domain,
+ enum radeon_bo_flag flags);
-static struct amdgpu_winsys_bo *get_amdgpu_winsys_bo(struct pb_buffer *_buf)
+static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
+ enum radeon_bo_usage usage)
{
- struct amdgpu_winsys_bo *bo = NULL;
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
+ struct amdgpu_winsys *ws = bo->ws;
+ int64_t abs_timeout;
+
+ if (timeout == 0) {
+ if (p_atomic_read(&bo->num_active_ioctls))
+ return false;
- if (_buf->vtbl == &amdgpu_winsys_bo_vtbl) {
- bo = amdgpu_winsys_bo(_buf);
} else {
- struct pb_buffer *base_buf;
- pb_size offset;
- pb_get_base_buffer(_buf, &base_buf, &offset);
+ abs_timeout = os_time_get_absolute_timeout(timeout);
- if (base_buf->vtbl == &amdgpu_winsys_bo_vtbl)
- bo = amdgpu_winsys_bo(base_buf);
+ /* Wait if any ioctl is being submitted with this buffer. */
+ if (!os_wait_until_zero_abs_timeout(&bo->num_active_ioctls, abs_timeout))
+ return false;
}
- return bo;
-}
-
-static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
- enum radeon_bo_usage usage)
-{
- struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(_buf);
- struct amdgpu_winsys *ws = bo->rws;
- int i;
-
if (bo->is_shared) {
/* We can't use user fences for shared buffers, because user fences
* are local to this process only. If we want to wait for all buffer
}
if (timeout == 0) {
- /* Timeout == 0 is quite simple. */
- pipe_mutex_lock(ws->bo_fence_lock);
- for (i = 0; i < RING_LAST; i++)
- if (bo->fence[i]) {
- if (amdgpu_fence_wait(bo->fence[i], 0, false)) {
- /* Release the idle fence to avoid checking it again later. */
- amdgpu_fence_reference(&bo->fence[i], NULL);
- } else {
- pipe_mutex_unlock(ws->bo_fence_lock);
- return false;
- }
- }
- pipe_mutex_unlock(ws->bo_fence_lock);
- return true;
+ unsigned idle_fences;
+ bool buffer_idle;
+
+ simple_mtx_lock(&ws->bo_fence_lock);
+ for (idle_fences = 0; idle_fences < bo->num_fences; ++idle_fences) {
+ if (!amdgpu_fence_wait(bo->fences[idle_fences], 0, false))
+ break;
+ }
+
+ /* Release the idle fences to avoid checking them again later. */
+ for (unsigned i = 0; i < idle_fences; ++i)
+ amdgpu_fence_reference(&bo->fences[i], NULL);
+
+ memmove(&bo->fences[0], &bo->fences[idle_fences],
+ (bo->num_fences - idle_fences) * sizeof(*bo->fences));
+ bo->num_fences -= idle_fences;
+
+ buffer_idle = !bo->num_fences;
+ simple_mtx_unlock(&ws->bo_fence_lock);
+
+ return buffer_idle;
} else {
- struct pipe_fence_handle *fence[RING_LAST] = {};
- bool fence_idle[RING_LAST] = {};
bool buffer_idle = true;
- int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
-
- /* Take references to all fences, so that we can wait for them
- * without the lock. */
- pipe_mutex_lock(ws->bo_fence_lock);
- for (i = 0; i < RING_LAST; i++)
- amdgpu_fence_reference(&fence[i], bo->fence[i]);
- pipe_mutex_unlock(ws->bo_fence_lock);
-
- /* Now wait for the fences. */
- for (i = 0; i < RING_LAST; i++) {
- if (fence[i]) {
- if (amdgpu_fence_wait(fence[i], abs_timeout, true))
- fence_idle[i] = true;
- else
- buffer_idle = false;
- }
- }
- /* Release idle fences to avoid checking them again later. */
- pipe_mutex_lock(ws->bo_fence_lock);
- for (i = 0; i < RING_LAST; i++) {
- if (fence[i] == bo->fence[i] && fence_idle[i])
- amdgpu_fence_reference(&bo->fence[i], NULL);
+ simple_mtx_lock(&ws->bo_fence_lock);
+ while (bo->num_fences && buffer_idle) {
+ struct pipe_fence_handle *fence = NULL;
+ bool fence_idle = false;
+
+ amdgpu_fence_reference(&fence, bo->fences[0]);
+
+ /* Wait for the fence. */
+ simple_mtx_unlock(&ws->bo_fence_lock);
+ if (amdgpu_fence_wait(fence, abs_timeout, true))
+ fence_idle = true;
+ else
+ buffer_idle = false;
+ simple_mtx_lock(&ws->bo_fence_lock);
+
+ /* Release an idle fence to avoid checking it again later, keeping in
+ * mind that the fence array may have been modified by other threads.
+ */
+ if (fence_idle && bo->num_fences && bo->fences[0] == fence) {
+ amdgpu_fence_reference(&bo->fences[0], NULL);
+ memmove(&bo->fences[0], &bo->fences[1],
+ (bo->num_fences - 1) * sizeof(*bo->fences));
+ bo->num_fences--;
+ }
- amdgpu_fence_reference(&fence[i], NULL);
+ amdgpu_fence_reference(&fence, NULL);
}
- pipe_mutex_unlock(ws->bo_fence_lock);
+ simple_mtx_unlock(&ws->bo_fence_lock);
return buffer_idle;
}
}
static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
- struct radeon_winsys_cs_handle *buf)
+ struct pb_buffer *buf)
{
return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
}
-static void amdgpu_bo_destroy(struct pb_buffer *_buf)
+static void amdgpu_bo_remove_fences(struct amdgpu_winsys_bo *bo)
+{
+ for (unsigned i = 0; i < bo->num_fences; ++i)
+ amdgpu_fence_reference(&bo->fences[i], NULL);
+
+ FREE(bo->fences);
+ bo->num_fences = 0;
+ bo->max_fences = 0;
+}
+
+void amdgpu_bo_destroy(struct pb_buffer *_buf)
{
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
- int i;
+ struct amdgpu_winsys *ws = bo->ws;
+
+ assert(bo->bo && "must not be called for slab entries");
+
+ if (ws->debug_all_bos) {
+ simple_mtx_lock(&ws->global_bo_list_lock);
+ LIST_DEL(&bo->u.real.global_list_item);
+ ws->num_buffers--;
+ simple_mtx_unlock(&ws->global_bo_list_lock);
+ }
+
+ simple_mtx_lock(&ws->bo_export_table_lock);
+ util_hash_table_remove(ws->bo_export_table, bo->bo);
+ simple_mtx_unlock(&ws->bo_export_table_lock);
amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
- amdgpu_va_range_free(bo->va_handle);
+ amdgpu_va_range_free(bo->u.real.va_handle);
amdgpu_bo_free(bo->bo);
- for (i = 0; i < RING_LAST; i++)
- amdgpu_fence_reference(&bo->fence[i], NULL);
+ amdgpu_bo_remove_fences(bo);
if (bo->initial_domain & RADEON_DOMAIN_VRAM)
- bo->rws->allocated_vram -= align(bo->base.size, bo->rws->gart_page_size);
+ ws->allocated_vram -= align64(bo->base.size, ws->info.gart_page_size);
else if (bo->initial_domain & RADEON_DOMAIN_GTT)
- bo->rws->allocated_gtt -= align(bo->base.size, bo->rws->gart_page_size);
+ ws->allocated_gtt -= align64(bo->base.size, ws->info.gart_page_size);
+
+ if (bo->u.real.map_count >= 1) {
+ if (bo->initial_domain & RADEON_DOMAIN_VRAM)
+ ws->mapped_vram -= bo->base.size;
+ else if (bo->initial_domain & RADEON_DOMAIN_GTT)
+ ws->mapped_gtt -= bo->base.size;
+ ws->num_mapped_buffers--;
+ }
+
FREE(bo);
}
-static void *amdgpu_bo_map(struct radeon_winsys_cs_handle *buf,
- struct radeon_winsys_cs *rcs,
+static void amdgpu_bo_destroy_or_cache(struct pb_buffer *_buf)
+{
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
+
+ assert(bo->bo); /* slab buffers have a separate vtbl */
+
+ if (bo->u.real.use_reusable_pool)
+ pb_cache_add_buffer(&bo->u.real.cache_entry);
+ else
+ amdgpu_bo_destroy(_buf);
+}
+
+static void *amdgpu_bo_map(struct pb_buffer *buf,
+ struct radeon_cmdbuf *rcs,
enum pipe_transfer_usage usage)
{
struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
+ struct amdgpu_winsys_bo *real;
struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
int r;
void *cpu = NULL;
+ uint64_t offset = 0;
+
+ assert(!bo->sparse);
/* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
* Only check whether the buffer is being used for write. */
if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
RADEON_USAGE_WRITE)) {
- cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
+ cs->flush_cs(cs->flush_data,
+ RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
return NULL;
}
}
} else {
if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
- cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
+ cs->flush_cs(cs->flush_data,
+ RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
return NULL;
}
* (neither one is changing it).
*
* Only check whether the buffer is being used for write. */
- if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
- RADEON_USAGE_WRITE)) {
- cs->flush_cs(cs->flush_data, 0, NULL);
+ if (cs) {
+ if (amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
+ RADEON_USAGE_WRITE)) {
+ cs->flush_cs(cs->flush_data,
+ RADEON_FLUSH_START_NEXT_GFX_IB_NOW, NULL);
+ } else {
+ /* Try to avoid busy-waiting in amdgpu_bo_wait. */
+ if (p_atomic_read(&bo->num_active_ioctls))
+ amdgpu_cs_sync_flush(rcs);
+ }
}
+
amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
RADEON_USAGE_WRITE);
} else {
/* Mapping for write. */
- if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo))
- cs->flush_cs(cs->flush_data, 0, NULL);
+ if (cs) {
+ if (amdgpu_bo_is_referenced_by_cs(cs, bo)) {
+ cs->flush_cs(cs->flush_data,
+ RADEON_FLUSH_START_NEXT_GFX_IB_NOW, NULL);
+ } else {
+ /* Try to avoid busy-waiting in amdgpu_bo_wait. */
+ if (p_atomic_read(&bo->num_active_ioctls))
+ amdgpu_cs_sync_flush(rcs);
+ }
+ }
amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
RADEON_USAGE_READWRITE);
}
- bo->rws->buffer_wait_time += os_time_get_nano() - time;
+ bo->ws->buffer_wait_time += os_time_get_nano() - time;
}
}
/* If the buffer is created from user memory, return the user pointer. */
if (bo->user_ptr)
- return bo->user_ptr;
+ return bo->user_ptr;
+
+ if (bo->bo) {
+ real = bo;
+ } else {
+ real = bo->u.slab.real;
+ offset = bo->va - real->va;
+ }
+
+ r = amdgpu_bo_cpu_map(real->bo, &cpu);
+ if (r) {
+ /* Clear the cache and try again. */
+ pb_cache_release_all_buffers(&real->ws->bo_cache);
+ r = amdgpu_bo_cpu_map(real->bo, &cpu);
+ if (r)
+ return NULL;
+ }
- r = amdgpu_bo_cpu_map(bo->bo, &cpu);
- return r ? NULL : cpu;
+ if (p_atomic_inc_return(&real->u.real.map_count) == 1) {
+ if (real->initial_domain & RADEON_DOMAIN_VRAM)
+ real->ws->mapped_vram += real->base.size;
+ else if (real->initial_domain & RADEON_DOMAIN_GTT)
+ real->ws->mapped_gtt += real->base.size;
+ real->ws->num_mapped_buffers++;
+ }
+ return (uint8_t*)cpu + offset;
}
-static void amdgpu_bo_unmap(struct radeon_winsys_cs_handle *buf)
+static void amdgpu_bo_unmap(struct pb_buffer *buf)
{
struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
+ struct amdgpu_winsys_bo *real;
- amdgpu_bo_cpu_unmap(bo->bo);
-}
+ assert(!bo->sparse);
-static void amdgpu_bo_get_base_buffer(struct pb_buffer *buf,
- struct pb_buffer **base_buf,
- unsigned *offset)
-{
- *base_buf = buf;
- *offset = 0;
-}
+ if (bo->user_ptr)
+ return;
-static enum pipe_error amdgpu_bo_validate(struct pb_buffer *_buf,
- struct pb_validate *vl,
- unsigned flags)
-{
- /* Always pinned */
- return PIPE_OK;
-}
+ real = bo->bo ? bo : bo->u.slab.real;
-static void amdgpu_bo_fence(struct pb_buffer *buf,
- struct pipe_fence_handle *fence)
-{
+ if (p_atomic_dec_zero(&real->u.real.map_count)) {
+ if (real->initial_domain & RADEON_DOMAIN_VRAM)
+ real->ws->mapped_vram -= real->base.size;
+ else if (real->initial_domain & RADEON_DOMAIN_GTT)
+ real->ws->mapped_gtt -= real->base.size;
+ real->ws->num_mapped_buffers--;
+ }
+
+ amdgpu_bo_cpu_unmap(real->bo);
}
static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
- amdgpu_bo_destroy,
- NULL, /* never called */
- NULL, /* never called */
- amdgpu_bo_validate,
- amdgpu_bo_fence,
- amdgpu_bo_get_base_buffer,
+ amdgpu_bo_destroy_or_cache
+ /* other functions are never called */
};
-static struct pb_buffer *amdgpu_bomgr_create_bo(struct pb_manager *_mgr,
- pb_size size,
- const struct pb_desc *desc)
+static void amdgpu_add_buffer_to_global_list(struct amdgpu_winsys_bo *bo)
+{
+ struct amdgpu_winsys *ws = bo->ws;
+
+ assert(bo->bo);
+
+ if (ws->debug_all_bos) {
+ simple_mtx_lock(&ws->global_bo_list_lock);
+ LIST_ADDTAIL(&bo->u.real.global_list_item, &ws->global_bo_list);
+ ws->num_buffers++;
+ simple_mtx_unlock(&ws->global_bo_list_lock);
+ }
+}
+
+static struct amdgpu_winsys_bo *amdgpu_create_bo(struct amdgpu_winsys *ws,
+ uint64_t size,
+ unsigned alignment,
+ enum radeon_bo_domain initial_domain,
+ unsigned flags,
+ int heap)
{
- struct amdgpu_winsys *rws = get_winsys(_mgr);
- struct amdgpu_bo_desc *rdesc = (struct amdgpu_bo_desc*)desc;
struct amdgpu_bo_alloc_request request = {0};
amdgpu_bo_handle buf_handle;
uint64_t va = 0;
struct amdgpu_winsys_bo *bo;
amdgpu_va_handle va_handle;
+ unsigned va_gap_size;
int r;
- assert(rdesc->initial_domain & RADEON_DOMAIN_VRAM_GTT);
+ /* VRAM or GTT must be specified, but not both at the same time. */
+ assert(util_bitcount(initial_domain & RADEON_DOMAIN_VRAM_GTT) == 1);
+
bo = CALLOC_STRUCT(amdgpu_winsys_bo);
if (!bo) {
return NULL;
}
+ if (heap >= 0) {
+ pb_cache_init_entry(&ws->bo_cache, &bo->u.real.cache_entry, &bo->base,
+ heap);
+ }
request.alloc_size = size;
- request.phys_alignment = desc->alignment;
+ request.phys_alignment = alignment;
- if (rdesc->initial_domain & RADEON_DOMAIN_VRAM) {
+ if (initial_domain & RADEON_DOMAIN_VRAM)
request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
- if (rdesc->flags & RADEON_FLAG_CPU_ACCESS)
- request.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
- }
- if (rdesc->initial_domain & RADEON_DOMAIN_GTT) {
+ if (initial_domain & RADEON_DOMAIN_GTT)
request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
- if (rdesc->flags & RADEON_FLAG_GTT_WC)
- request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
- }
- r = amdgpu_bo_alloc(rws->dev, &request, &buf_handle);
+ /* Since VRAM and GTT have almost the same performance on APUs, we could
+ * just set GTT. However, in order to decrease GTT(RAM) usage, which is
+ * shared with the OS, allow VRAM placements too. The idea is not to use
+ * VRAM usefully, but to use it so that it's not unused and wasted.
+ */
+ if (!ws->info.has_dedicated_vram)
+ request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
+
+ if (flags & RADEON_FLAG_NO_CPU_ACCESS)
+ request.flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+ if (flags & RADEON_FLAG_GTT_WC)
+ request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+ if (flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
+ ws->info.has_local_buffers)
+ request.flags |= AMDGPU_GEM_CREATE_VM_ALWAYS_VALID;
+ if (ws->zero_all_vram_allocs &&
+ (request.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM))
+ request.flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
+
+ r = amdgpu_bo_alloc(ws->dev, &request, &buf_handle);
if (r) {
fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
- fprintf(stderr, "amdgpu: size : %d bytes\n", size);
- fprintf(stderr, "amdgpu: alignment : %d bytes\n", desc->alignment);
- fprintf(stderr, "amdgpu: domains : %d\n", rdesc->initial_domain);
+ fprintf(stderr, "amdgpu: size : %"PRIu64" bytes\n", size);
+ fprintf(stderr, "amdgpu: alignment : %u bytes\n", alignment);
+ fprintf(stderr, "amdgpu: domains : %u\n", initial_domain);
goto error_bo_alloc;
}
- r = amdgpu_va_range_alloc(rws->dev, amdgpu_gpu_va_range_general,
- size, desc->alignment, 0, &va, &va_handle, 0);
+ va_gap_size = ws->check_vm ? MAX2(4 * alignment, 64 * 1024) : 0;
+ if (size > ws->info.pte_fragment_size)
+ alignment = MAX2(alignment, ws->info.pte_fragment_size);
+ r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
+ size + va_gap_size, alignment, 0, &va, &va_handle,
+ (flags & RADEON_FLAG_32BIT ? AMDGPU_VA_RANGE_32_BIT : 0) |
+ AMDGPU_VA_RANGE_HIGH);
if (r)
goto error_va_alloc;
- r = amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP);
+ unsigned vm_flags = AMDGPU_VM_PAGE_READABLE |
+ AMDGPU_VM_PAGE_EXECUTABLE;
+
+ if (!(flags & RADEON_FLAG_READ_ONLY))
+ vm_flags |= AMDGPU_VM_PAGE_WRITEABLE;
+
+ r = amdgpu_bo_va_op_raw(ws->dev, buf_handle, 0, size, va, vm_flags,
+ AMDGPU_VA_OP_MAP);
if (r)
goto error_va_map;
pipe_reference_init(&bo->base.reference, 1);
- bo->base.alignment = desc->alignment;
- bo->base.usage = desc->usage;
+ bo->base.alignment = alignment;
+ bo->base.usage = 0;
bo->base.size = size;
bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
- bo->rws = rws;
+ bo->ws = ws;
bo->bo = buf_handle;
bo->va = va;
- bo->va_handle = va_handle;
- bo->initial_domain = rdesc->initial_domain;
- bo->unique_id = __sync_fetch_and_add(&rws->next_bo_unique_id, 1);
+ bo->u.real.va_handle = va_handle;
+ bo->initial_domain = initial_domain;
+ bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
+ bo->is_local = !!(request.flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID);
- if (rdesc->initial_domain & RADEON_DOMAIN_VRAM)
- rws->allocated_vram += align(size, rws->gart_page_size);
- else if (rdesc->initial_domain & RADEON_DOMAIN_GTT)
- rws->allocated_gtt += align(size, rws->gart_page_size);
+ if (initial_domain & RADEON_DOMAIN_VRAM)
+ ws->allocated_vram += align64(size, ws->info.gart_page_size);
+ else if (initial_domain & RADEON_DOMAIN_GTT)
+ ws->allocated_gtt += align64(size, ws->info.gart_page_size);
- return &bo->base;
+ amdgpu_bo_export(bo->bo, amdgpu_bo_handle_type_kms, &bo->u.real.kms_handle);
+
+ amdgpu_add_buffer_to_global_list(bo);
+
+ return bo;
error_va_map:
amdgpu_va_range_free(va_handle);
return NULL;
}
-static void amdgpu_bomgr_flush(struct pb_manager *mgr)
+bool amdgpu_bo_can_reclaim(struct pb_buffer *_buf)
{
- /* NOP */
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
+
+ if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
+ return false;
+ }
+
+ return amdgpu_bo_wait(_buf, 0, RADEON_USAGE_READWRITE);
}
-/* This is for the cache bufmgr. */
-static boolean amdgpu_bomgr_is_buffer_busy(struct pb_manager *_mgr,
- struct pb_buffer *_buf)
+bool amdgpu_bo_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
+{
+ struct amdgpu_winsys_bo *bo = NULL; /* fix container_of */
+ bo = container_of(entry, bo, u.slab.entry);
+
+ return amdgpu_bo_can_reclaim(&bo->base);
+}
+
+static void amdgpu_bo_slab_destroy(struct pb_buffer *_buf)
{
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
- if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
- return TRUE;
+ assert(!bo->bo);
+
+ pb_slab_free(&bo->ws->bo_slabs, &bo->u.slab.entry);
+}
+
+static const struct pb_vtbl amdgpu_winsys_bo_slab_vtbl = {
+ amdgpu_bo_slab_destroy
+ /* other functions are never called */
+};
+
+struct pb_slab *amdgpu_bo_slab_alloc(void *priv, unsigned heap,
+ unsigned entry_size,
+ unsigned group_index)
+{
+ struct amdgpu_winsys *ws = priv;
+ struct amdgpu_slab *slab = CALLOC_STRUCT(amdgpu_slab);
+ enum radeon_bo_domain domains = radeon_domain_from_heap(heap);
+ enum radeon_bo_flag flags = radeon_flags_from_heap(heap);
+ uint32_t base_id;
+
+ if (!slab)
+ return NULL;
+
+ unsigned slab_size = 1 << AMDGPU_SLAB_BO_SIZE_LOG2;
+ slab->buffer = amdgpu_winsys_bo(amdgpu_bo_create(&ws->base,
+ slab_size, slab_size,
+ domains, flags));
+ if (!slab->buffer)
+ goto fail;
+
+ assert(slab->buffer->bo);
+
+ slab->base.num_entries = slab->buffer->base.size / entry_size;
+ slab->base.num_free = slab->base.num_entries;
+ slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
+ if (!slab->entries)
+ goto fail_buffer;
+
+ LIST_INITHEAD(&slab->base.free);
+
+ base_id = __sync_fetch_and_add(&ws->next_bo_unique_id, slab->base.num_entries);
+
+ for (unsigned i = 0; i < slab->base.num_entries; ++i) {
+ struct amdgpu_winsys_bo *bo = &slab->entries[i];
+
+ bo->base.alignment = entry_size;
+ bo->base.usage = slab->buffer->base.usage;
+ bo->base.size = entry_size;
+ bo->base.vtbl = &amdgpu_winsys_bo_slab_vtbl;
+ bo->ws = ws;
+ bo->va = slab->buffer->va + i * entry_size;
+ bo->initial_domain = domains;
+ bo->unique_id = base_id + i;
+ bo->u.slab.entry.slab = &slab->base;
+ bo->u.slab.entry.group_index = group_index;
+ bo->u.slab.real = slab->buffer;
+
+ LIST_ADDTAIL(&bo->u.slab.entry.head, &slab->base.free);
}
- if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0, RADEON_USAGE_READWRITE)) {
- return TRUE;
+ return &slab->base;
+
+fail_buffer:
+ amdgpu_winsys_bo_reference(&slab->buffer, NULL);
+fail:
+ FREE(slab);
+ return NULL;
+}
+
+void amdgpu_bo_slab_free(void *priv, struct pb_slab *pslab)
+{
+ struct amdgpu_slab *slab = amdgpu_slab(pslab);
+
+ for (unsigned i = 0; i < slab->base.num_entries; ++i)
+ amdgpu_bo_remove_fences(&slab->entries[i]);
+
+ FREE(slab->entries);
+ amdgpu_winsys_bo_reference(&slab->buffer, NULL);
+ FREE(slab);
+}
+
+#if DEBUG_SPARSE_COMMITS
+static void
+sparse_dump(struct amdgpu_winsys_bo *bo, const char *func)
+{
+ fprintf(stderr, "%s: %p (size=%"PRIu64", num_va_pages=%u) @ %s\n"
+ "Commitments:\n",
+ __func__, bo, bo->base.size, bo->u.sparse.num_va_pages, func);
+
+ struct amdgpu_sparse_backing *span_backing = NULL;
+ uint32_t span_first_backing_page = 0;
+ uint32_t span_first_va_page = 0;
+ uint32_t va_page = 0;
+
+ for (;;) {
+ struct amdgpu_sparse_backing *backing = 0;
+ uint32_t backing_page = 0;
+
+ if (va_page < bo->u.sparse.num_va_pages) {
+ backing = bo->u.sparse.commitments[va_page].backing;
+ backing_page = bo->u.sparse.commitments[va_page].page;
+ }
+
+ if (span_backing &&
+ (backing != span_backing ||
+ backing_page != span_first_backing_page + (va_page - span_first_va_page))) {
+ fprintf(stderr, " %u..%u: backing=%p:%u..%u\n",
+ span_first_va_page, va_page - 1, span_backing,
+ span_first_backing_page,
+ span_first_backing_page + (va_page - span_first_va_page) - 1);
+
+ span_backing = NULL;
+ }
+
+ if (va_page >= bo->u.sparse.num_va_pages)
+ break;
+
+ if (backing && !span_backing) {
+ span_backing = backing;
+ span_first_backing_page = backing_page;
+ span_first_va_page = va_page;
+ }
+
+ va_page++;
+ }
+
+ fprintf(stderr, "Backing:\n");
+
+ list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
+ fprintf(stderr, " %p (size=%"PRIu64")\n", backing, backing->bo->base.size);
+ for (unsigned i = 0; i < backing->num_chunks; ++i)
+ fprintf(stderr, " %u..%u\n", backing->chunks[i].begin, backing->chunks[i].end);
+ }
+}
+#endif
+
+/*
+ * Attempt to allocate the given number of backing pages. Fewer pages may be
+ * allocated (depending on the fragmentation of existing backing buffers),
+ * which will be reflected by a change to *pnum_pages.
+ */
+static struct amdgpu_sparse_backing *
+sparse_backing_alloc(struct amdgpu_winsys_bo *bo, uint32_t *pstart_page, uint32_t *pnum_pages)
+{
+ struct amdgpu_sparse_backing *best_backing;
+ unsigned best_idx;
+ uint32_t best_num_pages;
+
+ best_backing = NULL;
+ best_idx = 0;
+ best_num_pages = 0;
+
+ /* This is a very simple and inefficient best-fit algorithm. */
+ list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
+ for (unsigned idx = 0; idx < backing->num_chunks; ++idx) {
+ uint32_t cur_num_pages = backing->chunks[idx].end - backing->chunks[idx].begin;
+ if ((best_num_pages < *pnum_pages && cur_num_pages > best_num_pages) ||
+ (best_num_pages > *pnum_pages && cur_num_pages < best_num_pages)) {
+ best_backing = backing;
+ best_idx = idx;
+ best_num_pages = cur_num_pages;
+ }
+ }
+ }
+
+ /* Allocate a new backing buffer if necessary. */
+ if (!best_backing) {
+ struct pb_buffer *buf;
+ uint64_t size;
+ uint32_t pages;
+
+ best_backing = CALLOC_STRUCT(amdgpu_sparse_backing);
+ if (!best_backing)
+ return NULL;
+
+ best_backing->max_chunks = 4;
+ best_backing->chunks = CALLOC(best_backing->max_chunks,
+ sizeof(*best_backing->chunks));
+ if (!best_backing->chunks) {
+ FREE(best_backing);
+ return NULL;
+ }
+
+ assert(bo->u.sparse.num_backing_pages < DIV_ROUND_UP(bo->base.size, RADEON_SPARSE_PAGE_SIZE));
+
+ size = MIN3(bo->base.size / 16,
+ 8 * 1024 * 1024,
+ bo->base.size - (uint64_t)bo->u.sparse.num_backing_pages * RADEON_SPARSE_PAGE_SIZE);
+ size = MAX2(size, RADEON_SPARSE_PAGE_SIZE);
+
+ buf = amdgpu_bo_create(&bo->ws->base, size, RADEON_SPARSE_PAGE_SIZE,
+ bo->initial_domain,
+ bo->u.sparse.flags | RADEON_FLAG_NO_SUBALLOC);
+ if (!buf) {
+ FREE(best_backing->chunks);
+ FREE(best_backing);
+ return NULL;
+ }
+
+ /* We might have gotten a bigger buffer than requested via caching. */
+ pages = buf->size / RADEON_SPARSE_PAGE_SIZE;
+
+ best_backing->bo = amdgpu_winsys_bo(buf);
+ best_backing->num_chunks = 1;
+ best_backing->chunks[0].begin = 0;
+ best_backing->chunks[0].end = pages;
+
+ list_add(&best_backing->list, &bo->u.sparse.backing);
+ bo->u.sparse.num_backing_pages += pages;
+
+ best_idx = 0;
+ best_num_pages = pages;
}
- return FALSE;
+ *pnum_pages = MIN2(*pnum_pages, best_num_pages);
+ *pstart_page = best_backing->chunks[best_idx].begin;
+ best_backing->chunks[best_idx].begin += *pnum_pages;
+
+ if (best_backing->chunks[best_idx].begin >= best_backing->chunks[best_idx].end) {
+ memmove(&best_backing->chunks[best_idx], &best_backing->chunks[best_idx + 1],
+ sizeof(*best_backing->chunks) * (best_backing->num_chunks - best_idx - 1));
+ best_backing->num_chunks--;
+ }
+
+ return best_backing;
}
-static void amdgpu_bomgr_destroy(struct pb_manager *mgr)
+static void
+sparse_free_backing_buffer(struct amdgpu_winsys_bo *bo,
+ struct amdgpu_sparse_backing *backing)
{
- FREE(mgr);
+ struct amdgpu_winsys *ws = backing->bo->ws;
+
+ bo->u.sparse.num_backing_pages -= backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE;
+
+ simple_mtx_lock(&ws->bo_fence_lock);
+ amdgpu_add_fences(backing->bo, bo->num_fences, bo->fences);
+ simple_mtx_unlock(&ws->bo_fence_lock);
+
+ list_del(&backing->list);
+ amdgpu_winsys_bo_reference(&backing->bo, NULL);
+ FREE(backing->chunks);
+ FREE(backing);
}
-struct pb_manager *amdgpu_bomgr_create(struct amdgpu_winsys *rws)
+/*
+ * Return a range of pages from the given backing buffer back into the
+ * free structure.
+ */
+static bool
+sparse_backing_free(struct amdgpu_winsys_bo *bo,
+ struct amdgpu_sparse_backing *backing,
+ uint32_t start_page, uint32_t num_pages)
{
- struct amdgpu_bomgr *mgr;
+ uint32_t end_page = start_page + num_pages;
+ unsigned low = 0;
+ unsigned high = backing->num_chunks;
+
+ /* Find the first chunk with begin >= start_page. */
+ while (low < high) {
+ unsigned mid = low + (high - low) / 2;
+
+ if (backing->chunks[mid].begin >= start_page)
+ high = mid;
+ else
+ low = mid + 1;
+ }
+
+ assert(low >= backing->num_chunks || end_page <= backing->chunks[low].begin);
+ assert(low == 0 || backing->chunks[low - 1].end <= start_page);
+
+ if (low > 0 && backing->chunks[low - 1].end == start_page) {
+ backing->chunks[low - 1].end = end_page;
+
+ if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
+ backing->chunks[low - 1].end = backing->chunks[low].end;
+ memmove(&backing->chunks[low], &backing->chunks[low + 1],
+ sizeof(*backing->chunks) * (backing->num_chunks - low - 1));
+ backing->num_chunks--;
+ }
+ } else if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
+ backing->chunks[low].begin = start_page;
+ } else {
+ if (backing->num_chunks >= backing->max_chunks) {
+ unsigned new_max_chunks = 2 * backing->max_chunks;
+ struct amdgpu_sparse_backing_chunk *new_chunks =
+ REALLOC(backing->chunks,
+ sizeof(*backing->chunks) * backing->max_chunks,
+ sizeof(*backing->chunks) * new_max_chunks);
+ if (!new_chunks)
+ return false;
+
+ backing->max_chunks = new_max_chunks;
+ backing->chunks = new_chunks;
+ }
- mgr = CALLOC_STRUCT(amdgpu_bomgr);
- if (!mgr)
+ memmove(&backing->chunks[low + 1], &backing->chunks[low],
+ sizeof(*backing->chunks) * (backing->num_chunks - low));
+ backing->chunks[low].begin = start_page;
+ backing->chunks[low].end = end_page;
+ backing->num_chunks++;
+ }
+
+ if (backing->num_chunks == 1 && backing->chunks[0].begin == 0 &&
+ backing->chunks[0].end == backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE)
+ sparse_free_backing_buffer(bo, backing);
+
+ return true;
+}
+
+static void amdgpu_bo_sparse_destroy(struct pb_buffer *_buf)
+{
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
+ int r;
+
+ assert(!bo->bo && bo->sparse);
+
+ r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
+ (uint64_t)bo->u.sparse.num_va_pages * RADEON_SPARSE_PAGE_SIZE,
+ bo->va, 0, AMDGPU_VA_OP_CLEAR);
+ if (r) {
+ fprintf(stderr, "amdgpu: clearing PRT VA region on destroy failed (%d)\n", r);
+ }
+
+ while (!list_empty(&bo->u.sparse.backing)) {
+ struct amdgpu_sparse_backing *dummy = NULL;
+ sparse_free_backing_buffer(bo,
+ container_of(bo->u.sparse.backing.next,
+ dummy, list));
+ }
+
+ amdgpu_va_range_free(bo->u.sparse.va_handle);
+ simple_mtx_destroy(&bo->u.sparse.commit_lock);
+ FREE(bo->u.sparse.commitments);
+ FREE(bo);
+}
+
+static const struct pb_vtbl amdgpu_winsys_bo_sparse_vtbl = {
+ amdgpu_bo_sparse_destroy
+ /* other functions are never called */
+};
+
+static struct pb_buffer *
+amdgpu_bo_sparse_create(struct amdgpu_winsys *ws, uint64_t size,
+ enum radeon_bo_domain domain,
+ enum radeon_bo_flag flags)
+{
+ struct amdgpu_winsys_bo *bo;
+ uint64_t map_size;
+ uint64_t va_gap_size;
+ int r;
+
+ /* We use 32-bit page numbers; refuse to attempt allocating sparse buffers
+ * that exceed this limit. This is not really a restriction: we don't have
+ * that much virtual address space anyway.
+ */
+ if (size > (uint64_t)INT32_MAX * RADEON_SPARSE_PAGE_SIZE)
return NULL;
- mgr->base.destroy = amdgpu_bomgr_destroy;
- mgr->base.create_buffer = amdgpu_bomgr_create_bo;
- mgr->base.flush = amdgpu_bomgr_flush;
- mgr->base.is_buffer_busy = amdgpu_bomgr_is_buffer_busy;
+ bo = CALLOC_STRUCT(amdgpu_winsys_bo);
+ if (!bo)
+ return NULL;
- mgr->rws = rws;
- return &mgr->base;
+ pipe_reference_init(&bo->base.reference, 1);
+ bo->base.alignment = RADEON_SPARSE_PAGE_SIZE;
+ bo->base.size = size;
+ bo->base.vtbl = &amdgpu_winsys_bo_sparse_vtbl;
+ bo->ws = ws;
+ bo->initial_domain = domain;
+ bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
+ bo->sparse = true;
+ bo->u.sparse.flags = flags & ~RADEON_FLAG_SPARSE;
+
+ bo->u.sparse.num_va_pages = DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
+ bo->u.sparse.commitments = CALLOC(bo->u.sparse.num_va_pages,
+ sizeof(*bo->u.sparse.commitments));
+ if (!bo->u.sparse.commitments)
+ goto error_alloc_commitments;
+
+ simple_mtx_init(&bo->u.sparse.commit_lock, mtx_plain);
+ LIST_INITHEAD(&bo->u.sparse.backing);
+
+ /* For simplicity, we always map a multiple of the page size. */
+ map_size = align64(size, RADEON_SPARSE_PAGE_SIZE);
+ va_gap_size = ws->check_vm ? 4 * RADEON_SPARSE_PAGE_SIZE : 0;
+ r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
+ map_size + va_gap_size, RADEON_SPARSE_PAGE_SIZE,
+ 0, &bo->va, &bo->u.sparse.va_handle,
+ AMDGPU_VA_RANGE_HIGH);
+ if (r)
+ goto error_va_alloc;
+
+ r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0, size, bo->va,
+ AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_MAP);
+ if (r)
+ goto error_va_map;
+
+ return &bo->base;
+
+error_va_map:
+ amdgpu_va_range_free(bo->u.sparse.va_handle);
+error_va_alloc:
+ simple_mtx_destroy(&bo->u.sparse.commit_lock);
+ FREE(bo->u.sparse.commitments);
+error_alloc_commitments:
+ FREE(bo);
+ return NULL;
+}
+
+static bool
+amdgpu_bo_sparse_commit(struct pb_buffer *buf, uint64_t offset, uint64_t size,
+ bool commit)
+{
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buf);
+ struct amdgpu_sparse_commitment *comm;
+ uint32_t va_page, end_va_page;
+ bool ok = true;
+ int r;
+
+ assert(bo->sparse);
+ assert(offset % RADEON_SPARSE_PAGE_SIZE == 0);
+ assert(offset <= bo->base.size);
+ assert(size <= bo->base.size - offset);
+ assert(size % RADEON_SPARSE_PAGE_SIZE == 0 || offset + size == bo->base.size);
+
+ comm = bo->u.sparse.commitments;
+ va_page = offset / RADEON_SPARSE_PAGE_SIZE;
+ end_va_page = va_page + DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
+
+ simple_mtx_lock(&bo->u.sparse.commit_lock);
+
+#if DEBUG_SPARSE_COMMITS
+ sparse_dump(bo, __func__);
+#endif
+
+ if (commit) {
+ while (va_page < end_va_page) {
+ uint32_t span_va_page;
+
+ /* Skip pages that are already committed. */
+ if (comm[va_page].backing) {
+ va_page++;
+ continue;
+ }
+
+ /* Determine length of uncommitted span. */
+ span_va_page = va_page;
+ while (va_page < end_va_page && !comm[va_page].backing)
+ va_page++;
+
+ /* Fill the uncommitted span with chunks of backing memory. */
+ while (span_va_page < va_page) {
+ struct amdgpu_sparse_backing *backing;
+ uint32_t backing_start, backing_size;
+
+ backing_size = va_page - span_va_page;
+ backing = sparse_backing_alloc(bo, &backing_start, &backing_size);
+ if (!backing) {
+ ok = false;
+ goto out;
+ }
+
+ r = amdgpu_bo_va_op_raw(bo->ws->dev, backing->bo->bo,
+ (uint64_t)backing_start * RADEON_SPARSE_PAGE_SIZE,
+ (uint64_t)backing_size * RADEON_SPARSE_PAGE_SIZE,
+ bo->va + (uint64_t)span_va_page * RADEON_SPARSE_PAGE_SIZE,
+ AMDGPU_VM_PAGE_READABLE |
+ AMDGPU_VM_PAGE_WRITEABLE |
+ AMDGPU_VM_PAGE_EXECUTABLE,
+ AMDGPU_VA_OP_REPLACE);
+ if (r) {
+ ok = sparse_backing_free(bo, backing, backing_start, backing_size);
+ assert(ok && "sufficient memory should already be allocated");
+
+ ok = false;
+ goto out;
+ }
+
+ while (backing_size) {
+ comm[span_va_page].backing = backing;
+ comm[span_va_page].page = backing_start;
+ span_va_page++;
+ backing_start++;
+ backing_size--;
+ }
+ }
+ }
+ } else {
+ r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
+ (uint64_t)(end_va_page - va_page) * RADEON_SPARSE_PAGE_SIZE,
+ bo->va + (uint64_t)va_page * RADEON_SPARSE_PAGE_SIZE,
+ AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_REPLACE);
+ if (r) {
+ ok = false;
+ goto out;
+ }
+
+ while (va_page < end_va_page) {
+ struct amdgpu_sparse_backing *backing;
+ uint32_t backing_start;
+ uint32_t span_pages;
+
+ /* Skip pages that are already uncommitted. */
+ if (!comm[va_page].backing) {
+ va_page++;
+ continue;
+ }
+
+ /* Group contiguous spans of pages. */
+ backing = comm[va_page].backing;
+ backing_start = comm[va_page].page;
+ comm[va_page].backing = NULL;
+
+ span_pages = 1;
+ va_page++;
+
+ while (va_page < end_va_page &&
+ comm[va_page].backing == backing &&
+ comm[va_page].page == backing_start + span_pages) {
+ comm[va_page].backing = NULL;
+ va_page++;
+ span_pages++;
+ }
+
+ if (!sparse_backing_free(bo, backing, backing_start, span_pages)) {
+ /* Couldn't allocate tracking data structures, so we have to leak */
+ fprintf(stderr, "amdgpu: leaking PRT backing memory\n");
+ ok = false;
+ }
+ }
+ }
+out:
+
+ simple_mtx_unlock(&bo->u.sparse.commit_lock);
+
+ return ok;
}
static unsigned eg_tile_split(unsigned tile_split)
}
}
-static void amdgpu_bo_get_tiling(struct pb_buffer *_buf,
- enum radeon_bo_layout *microtiled,
- enum radeon_bo_layout *macrotiled,
- unsigned *bankw, unsigned *bankh,
- unsigned *tile_split,
- unsigned *stencil_tile_split,
- unsigned *mtilea,
- bool *scanout)
+static void amdgpu_buffer_get_metadata(struct pb_buffer *_buf,
+ struct radeon_bo_metadata *md)
{
- struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(_buf);
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
struct amdgpu_bo_info info = {0};
- uint32_t tiling_flags;
+ uint64_t tiling_flags;
int r;
+ assert(bo->bo && "must not be called for slab entries");
+
r = amdgpu_bo_query_info(bo->bo, &info);
if (r)
return;
tiling_flags = info.metadata.tiling_info;
- *microtiled = RADEON_LAYOUT_LINEAR;
- *macrotiled = RADEON_LAYOUT_LINEAR;
-
- if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4) /* 2D_TILED_THIN1 */
- *macrotiled = RADEON_LAYOUT_TILED;
- else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
- *microtiled = RADEON_LAYOUT_TILED;
-
- if (bankw && tile_split && mtilea && tile_split) {
- *bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
- *bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
- *tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
- *mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
- }
- if (scanout)
- *scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
-}
-
-static void amdgpu_bo_set_tiling(struct pb_buffer *_buf,
- struct radeon_winsys_cs *rcs,
- enum radeon_bo_layout microtiled,
- enum radeon_bo_layout macrotiled,
- unsigned pipe_config,
- unsigned bankw, unsigned bankh,
- unsigned tile_split,
- unsigned stencil_tile_split,
- unsigned mtilea, unsigned num_banks,
- uint32_t pitch,
- bool scanout)
-{
- struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(_buf);
+ if (bo->ws->info.chip_class >= GFX9) {
+ md->u.gfx9.swizzle_mode = AMDGPU_TILING_GET(tiling_flags, SWIZZLE_MODE);
+ } else {
+ md->u.legacy.microtile = RADEON_LAYOUT_LINEAR;
+ md->u.legacy.macrotile = RADEON_LAYOUT_LINEAR;
+
+ if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4) /* 2D_TILED_THIN1 */
+ md->u.legacy.macrotile = RADEON_LAYOUT_TILED;
+ else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
+ md->u.legacy.microtile = RADEON_LAYOUT_TILED;
+
+ md->u.legacy.pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
+ md->u.legacy.bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
+ md->u.legacy.bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
+ md->u.legacy.tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
+ md->u.legacy.mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
+ md->u.legacy.num_banks = 2 << AMDGPU_TILING_GET(tiling_flags, NUM_BANKS);
+ md->u.legacy.scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
+ }
+
+ md->size_metadata = info.metadata.size_metadata;
+ memcpy(md->metadata, info.metadata.umd_metadata, sizeof(md->metadata));
+}
+
+static void amdgpu_buffer_set_metadata(struct pb_buffer *_buf,
+ struct radeon_bo_metadata *md)
+{
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
struct amdgpu_bo_metadata metadata = {0};
- uint32_t tiling_flags = 0;
+ uint64_t tiling_flags = 0;
- if (macrotiled == RADEON_LAYOUT_TILED)
- tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
- else if (microtiled == RADEON_LAYOUT_TILED)
- tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
- else
- tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
-
- tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, pipe_config);
- tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(bankw));
- tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(bankh));
- if (tile_split)
- tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(tile_split));
- tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(mtilea));
- tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(num_banks)-1);
-
- if (scanout)
- tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
- else
- tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
+ assert(bo->bo && "must not be called for slab entries");
+
+ if (bo->ws->info.chip_class >= GFX9) {
+ tiling_flags |= AMDGPU_TILING_SET(SWIZZLE_MODE, md->u.gfx9.swizzle_mode);
+ } else {
+ if (md->u.legacy.macrotile == RADEON_LAYOUT_TILED)
+ tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
+ else if (md->u.legacy.microtile == RADEON_LAYOUT_TILED)
+ tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
+ else
+ tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
+
+ tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, md->u.legacy.pipe_config);
+ tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(md->u.legacy.bankw));
+ tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(md->u.legacy.bankh));
+ if (md->u.legacy.tile_split)
+ tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(md->u.legacy.tile_split));
+ tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(md->u.legacy.mtilea));
+ tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(md->u.legacy.num_banks)-1);
+
+ if (md->u.legacy.scanout)
+ tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
+ else
+ tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
+ }
metadata.tiling_info = tiling_flags;
+ metadata.size_metadata = md->size_metadata;
+ memcpy(metadata.umd_metadata, md->metadata, sizeof(md->metadata));
amdgpu_bo_set_metadata(bo->bo, &metadata);
}
-static struct radeon_winsys_cs_handle *amdgpu_get_cs_handle(struct pb_buffer *_buf)
-{
- /* return a direct pointer to amdgpu_winsys_bo. */
- return (struct radeon_winsys_cs_handle*)get_amdgpu_winsys_bo(_buf);
-}
-
static struct pb_buffer *
amdgpu_bo_create(struct radeon_winsys *rws,
- unsigned size,
+ uint64_t size,
unsigned alignment,
- boolean use_reusable_pool,
enum radeon_bo_domain domain,
enum radeon_bo_flag flags)
{
struct amdgpu_winsys *ws = amdgpu_winsys(rws);
- struct amdgpu_bo_desc desc;
- struct pb_manager *provider;
- struct pb_buffer *buffer;
-
- /* Don't use VRAM if the GPU doesn't have much. This is only the initial
- * domain. The kernel is free to move the buffer if it wants to.
- *
- * 64MB means no VRAM by todays standards.
- */
- if (domain & RADEON_DOMAIN_VRAM && ws->info.vram_size <= 64*1024*1024) {
- domain = RADEON_DOMAIN_GTT;
- flags = RADEON_FLAG_GTT_WC;
+ struct amdgpu_winsys_bo *bo;
+ int heap = -1;
+
+ /* VRAM implies WC. This is not optional. */
+ assert(!(domain & RADEON_DOMAIN_VRAM) || flags & RADEON_FLAG_GTT_WC);
+
+ /* NO_CPU_ACCESS is valid with VRAM only. */
+ assert(domain == RADEON_DOMAIN_VRAM || !(flags & RADEON_FLAG_NO_CPU_ACCESS));
+
+ /* Sparse buffers must have NO_CPU_ACCESS set. */
+ assert(!(flags & RADEON_FLAG_SPARSE) || flags & RADEON_FLAG_NO_CPU_ACCESS);
+
+ /* Sub-allocate small buffers from slabs. */
+ if (!(flags & (RADEON_FLAG_NO_SUBALLOC | RADEON_FLAG_SPARSE)) &&
+ size <= (1 << AMDGPU_SLAB_MAX_SIZE_LOG2) &&
+ alignment <= MAX2(1 << AMDGPU_SLAB_MIN_SIZE_LOG2, util_next_power_of_two(size))) {
+ struct pb_slab_entry *entry;
+ int heap = radeon_get_heap_index(domain, flags);
+
+ if (heap < 0 || heap >= RADEON_MAX_SLAB_HEAPS)
+ goto no_slab;
+
+ entry = pb_slab_alloc(&ws->bo_slabs, size, heap);
+ if (!entry) {
+ /* Clear the cache and try again. */
+ pb_cache_release_all_buffers(&ws->bo_cache);
+
+ entry = pb_slab_alloc(&ws->bo_slabs, size, heap);
+ }
+ if (!entry)
+ return NULL;
+
+ bo = NULL;
+ bo = container_of(entry, bo, u.slab.entry);
+
+ pipe_reference_init(&bo->base.reference, 1);
+
+ return &bo->base;
}
+no_slab:
- memset(&desc, 0, sizeof(desc));
- desc.base.alignment = alignment;
+ if (flags & RADEON_FLAG_SPARSE) {
+ assert(RADEON_SPARSE_PAGE_SIZE % alignment == 0);
+
+ return amdgpu_bo_sparse_create(ws, size, domain, flags);
+ }
+
+ /* This flag is irrelevant for the cache. */
+ flags &= ~RADEON_FLAG_NO_SUBALLOC;
/* Align size to page size. This is the minimum alignment for normal
* BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
* like constant/uniform buffers, can benefit from better and more reuse.
*/
- size = align(size, ws->gart_page_size);
+ size = align64(size, ws->info.gart_page_size);
+ alignment = align(alignment, ws->info.gart_page_size);
- /* Only set one usage bit each for domains and flags, or the cache manager
- * might consider different sets of domains / flags compatible
- */
- if (domain == RADEON_DOMAIN_VRAM_GTT)
- desc.base.usage = 1 << 2;
- else
- desc.base.usage = domain >> 1;
- assert(flags < sizeof(desc.base.usage) * 8 - 3);
- desc.base.usage |= 1 << (flags + 3);
+ bool use_reusable_pool = flags & RADEON_FLAG_NO_INTERPROCESS_SHARING;
- desc.initial_domain = domain;
- desc.flags = flags;
+ if (use_reusable_pool) {
+ heap = radeon_get_heap_index(domain, flags);
+ assert(heap >= 0 && heap < RADEON_MAX_CACHED_HEAPS);
- /* Assign a buffer manager. */
- if (use_reusable_pool)
- provider = ws->cman;
- else
- provider = ws->kman;
+ /* Get a buffer from the cache. */
+ bo = (struct amdgpu_winsys_bo*)
+ pb_cache_reclaim_buffer(&ws->bo_cache, size, alignment, 0, heap);
+ if (bo)
+ return &bo->base;
+ }
- buffer = provider->create_buffer(provider, size, &desc.base);
- if (!buffer)
- return NULL;
+ /* Create a new one. */
+ bo = amdgpu_create_bo(ws, size, alignment, domain, flags, heap);
+ if (!bo) {
+ /* Clear the cache and try again. */
+ pb_slabs_reclaim(&ws->bo_slabs);
+ pb_cache_release_all_buffers(&ws->bo_cache);
+ bo = amdgpu_create_bo(ws, size, alignment, domain, flags, heap);
+ if (!bo)
+ return NULL;
+ }
- return (struct pb_buffer*)buffer;
+ bo->u.real.use_reusable_pool = use_reusable_pool;
+ return &bo->base;
}
static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
struct winsys_handle *whandle,
- unsigned *stride)
+ unsigned *stride,
+ unsigned *offset)
{
struct amdgpu_winsys *ws = amdgpu_winsys(rws);
- struct amdgpu_winsys_bo *bo;
+ struct amdgpu_winsys_bo *bo = NULL;
enum amdgpu_bo_handle_type type;
struct amdgpu_bo_import_result result = {0};
uint64_t va;
- amdgpu_va_handle va_handle;
+ amdgpu_va_handle va_handle = NULL;
struct amdgpu_bo_info info = {0};
enum radeon_bo_domain initial = 0;
int r;
- /* Initialize the structure. */
- bo = CALLOC_STRUCT(amdgpu_winsys_bo);
- if (!bo) {
- return NULL;
- }
-
switch (whandle->type) {
- case DRM_API_HANDLE_TYPE_SHARED:
+ case WINSYS_HANDLE_TYPE_SHARED:
type = amdgpu_bo_handle_type_gem_flink_name;
break;
- case DRM_API_HANDLE_TYPE_FD:
+ case WINSYS_HANDLE_TYPE_FD:
type = amdgpu_bo_handle_type_dma_buf_fd;
break;
default:
return NULL;
}
+ if (stride)
+ *stride = whandle->stride;
+ if (offset)
+ *offset = whandle->offset;
+
r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
if (r)
- goto error;
+ return NULL;
+
+ simple_mtx_lock(&ws->bo_export_table_lock);
+ bo = util_hash_table_get(ws->bo_export_table, result.buf_handle);
+
+ /* If the amdgpu_winsys_bo instance already exists, bump the reference
+ * counter and return it.
+ */
+ if (bo) {
+ p_atomic_inc(&bo->base.reference.count);
+ simple_mtx_unlock(&ws->bo_export_table_lock);
+
+ /* Release the buffer handle, because we don't need it anymore.
+ * This function is returning an existing buffer, which has its own
+ * handle.
+ */
+ amdgpu_bo_free(result.buf_handle);
+ return &bo->base;
+ }
/* Get initial domains. */
r = amdgpu_bo_query_info(result.buf_handle, &info);
if (r)
- goto error_query;
+ goto error;
r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
- result.alloc_size, 1 << 20, 0, &va, &va_handle, 0);
+ result.alloc_size, 1 << 20, 0, &va, &va_handle,
+ AMDGPU_VA_RANGE_HIGH);
if (r)
- goto error_query;
+ goto error;
+
+ bo = CALLOC_STRUCT(amdgpu_winsys_bo);
+ if (!bo)
+ goto error;
r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
if (r)
- goto error_va_map;
+ goto error;
if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
initial |= RADEON_DOMAIN_VRAM;
if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
initial |= RADEON_DOMAIN_GTT;
-
+ /* Initialize the structure. */
pipe_reference_init(&bo->base.reference, 1);
bo->base.alignment = info.phys_alignment;
- bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
bo->bo = result.buf_handle;
bo->base.size = result.alloc_size;
bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
- bo->rws = ws;
+ bo->ws = ws;
bo->va = va;
- bo->va_handle = va_handle;
+ bo->u.real.va_handle = va_handle;
bo->initial_domain = initial;
bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
bo->is_shared = true;
- if (stride)
- *stride = whandle->stride;
-
if (bo->initial_domain & RADEON_DOMAIN_VRAM)
- ws->allocated_vram += align(bo->base.size, ws->gart_page_size);
+ ws->allocated_vram += align64(bo->base.size, ws->info.gart_page_size);
else if (bo->initial_domain & RADEON_DOMAIN_GTT)
- ws->allocated_gtt += align(bo->base.size, ws->gart_page_size);
+ ws->allocated_gtt += align64(bo->base.size, ws->info.gart_page_size);
- return &bo->base;
+ amdgpu_bo_export(bo->bo, amdgpu_bo_handle_type_kms, &bo->u.real.kms_handle);
-error_va_map:
- amdgpu_va_range_free(va_handle);
+ amdgpu_add_buffer_to_global_list(bo);
-error_query:
- amdgpu_bo_free(result.buf_handle);
+ util_hash_table_set(ws->bo_export_table, bo->bo, bo);
+ simple_mtx_unlock(&ws->bo_export_table_lock);
+
+ return &bo->base;
error:
- FREE(bo);
+ simple_mtx_unlock(&ws->bo_export_table_lock);
+ if (bo)
+ FREE(bo);
+ if (va_handle)
+ amdgpu_va_range_free(va_handle);
+ amdgpu_bo_free(result.buf_handle);
return NULL;
}
-static boolean amdgpu_bo_get_handle(struct pb_buffer *buffer,
- unsigned stride,
- struct winsys_handle *whandle)
+static bool amdgpu_bo_get_handle(struct pb_buffer *buffer,
+ unsigned stride, unsigned offset,
+ unsigned slice_size,
+ struct winsys_handle *whandle)
{
- struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(buffer);
+ struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buffer);
+ struct amdgpu_winsys *ws = bo->ws;
enum amdgpu_bo_handle_type type;
int r;
- if ((void*)bo != (void*)buffer)
- pb_cache_manager_remove_buffer(buffer);
+ /* Don't allow exports of slab entries and sparse buffers. */
+ if (!bo->bo)
+ return false;
+
+ bo->u.real.use_reusable_pool = false;
switch (whandle->type) {
- case DRM_API_HANDLE_TYPE_SHARED:
+ case WINSYS_HANDLE_TYPE_SHARED:
type = amdgpu_bo_handle_type_gem_flink_name;
break;
- case DRM_API_HANDLE_TYPE_FD:
+ case WINSYS_HANDLE_TYPE_FD:
type = amdgpu_bo_handle_type_dma_buf_fd;
break;
- case DRM_API_HANDLE_TYPE_KMS:
+ case WINSYS_HANDLE_TYPE_KMS:
type = amdgpu_bo_handle_type_kms;
break;
default:
- return FALSE;
+ return false;
}
r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
if (r)
- return FALSE;
+ return false;
+
+ simple_mtx_lock(&ws->bo_export_table_lock);
+ util_hash_table_set(ws->bo_export_table, bo->bo, bo);
+ simple_mtx_unlock(&ws->bo_export_table_lock);
whandle->stride = stride;
+ whandle->offset = offset;
+ whandle->offset += slice_size * whandle->layer;
bo->is_shared = true;
- return TRUE;
+ return true;
}
static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
- void *pointer, unsigned size)
+ void *pointer, uint64_t size)
{
struct amdgpu_winsys *ws = amdgpu_winsys(rws);
amdgpu_bo_handle buf_handle;
struct amdgpu_winsys_bo *bo;
uint64_t va;
amdgpu_va_handle va_handle;
+ /* Avoid failure when the size is not page aligned */
+ uint64_t aligned_size = align64(size, ws->info.gart_page_size);
bo = CALLOC_STRUCT(amdgpu_winsys_bo);
if (!bo)
return NULL;
- if (amdgpu_create_bo_from_user_mem(ws->dev, pointer, size, &buf_handle))
+ if (amdgpu_create_bo_from_user_mem(ws->dev, pointer,
+ aligned_size, &buf_handle))
goto error;
if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
- size, 1 << 12, 0, &va, &va_handle, 0))
+ aligned_size, 1 << 12, 0, &va, &va_handle,
+ AMDGPU_VA_RANGE_HIGH))
goto error_va_alloc;
- if (amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP))
+ if (amdgpu_bo_va_op(buf_handle, 0, aligned_size, va, 0, AMDGPU_VA_OP_MAP))
goto error_va_map;
/* Initialize it. */
pipe_reference_init(&bo->base.reference, 1);
bo->bo = buf_handle;
bo->base.alignment = 0;
- bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
bo->base.size = size;
bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
- bo->rws = ws;
+ bo->ws = ws;
bo->user_ptr = pointer;
bo->va = va;
- bo->va_handle = va_handle;
+ bo->u.real.va_handle = va_handle;
bo->initial_domain = RADEON_DOMAIN_GTT;
bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
- ws->allocated_gtt += align(bo->base.size, ws->gart_page_size);
+ ws->allocated_gtt += aligned_size;
+
+ amdgpu_add_buffer_to_global_list(bo);
+
+ amdgpu_bo_export(bo->bo, amdgpu_bo_handle_type_kms, &bo->u.real.kms_handle);
return (struct pb_buffer*)bo;
return NULL;
}
-static uint64_t amdgpu_bo_get_va(struct radeon_winsys_cs_handle *buf)
+static bool amdgpu_bo_is_user_ptr(struct pb_buffer *buf)
+{
+ return ((struct amdgpu_winsys_bo*)buf)->user_ptr != NULL;
+}
+
+static bool amdgpu_bo_is_suballocated(struct pb_buffer *buf)
+{
+ struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
+
+ return !bo->bo && !bo->sparse;
+}
+
+static uint64_t amdgpu_bo_get_va(struct pb_buffer *buf)
{
return ((struct amdgpu_winsys_bo*)buf)->va;
}
-void amdgpu_bomgr_init_functions(struct amdgpu_winsys *ws)
+void amdgpu_bo_init_functions(struct amdgpu_winsys *ws)
{
- ws->base.buffer_get_cs_handle = amdgpu_get_cs_handle;
- ws->base.buffer_set_tiling = amdgpu_bo_set_tiling;
- ws->base.buffer_get_tiling = amdgpu_bo_get_tiling;
+ ws->base.buffer_set_metadata = amdgpu_buffer_set_metadata;
+ ws->base.buffer_get_metadata = amdgpu_buffer_get_metadata;
ws->base.buffer_map = amdgpu_bo_map;
ws->base.buffer_unmap = amdgpu_bo_unmap;
ws->base.buffer_wait = amdgpu_bo_wait;
ws->base.buffer_create = amdgpu_bo_create;
ws->base.buffer_from_handle = amdgpu_bo_from_handle;
ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
+ ws->base.buffer_is_user_ptr = amdgpu_bo_is_user_ptr;
+ ws->base.buffer_is_suballocated = amdgpu_bo_is_suballocated;
ws->base.buffer_get_handle = amdgpu_bo_get_handle;
+ ws->base.buffer_commit = amdgpu_bo_sparse_commit;
ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
}