src/gallium/winsys/amdgpu/drm/amdgpu_bo.c

   1 /*
   2  * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
   3  * Copyright © 2015 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  16  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  17  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  18  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * The above copyright notice and this permission notice (including the
  24  * next paragraph) shall be included in all copies or substantial portions
  25  * of the Software.
  26  */
  27 /*
  28  * Authors:
  29  *      Marek Olšák <maraeo@gmail.com>
  30  */
  31
  32 #include "amdgpu_cs.h"
  33
  34 #include "os/os_time.h"
  35 #include "state_tracker/drm_driver.h"
  36 #include <amdgpu_drm.h>
  37 #include <xf86drm.h>
  38 #include <stdio.h>
  39 #include <inttypes.h>
  40
  41 static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
  42                            enum radeon_bo_usage usage)
  43 {
  44    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
  45    struct amdgpu_winsys *ws = bo->ws;
  46    int i;
  47
  48    if (bo->is_shared) {
  49       /* We can't use user fences for shared buffers, because user fences
  50        * are local to this process only. If we want to wait for all buffer
  51        * uses in all processes, we have to use amdgpu_bo_wait_for_idle.
  52        */
  53       bool buffer_busy = true;
  54       int r;
  55
  56       r = amdgpu_bo_wait_for_idle(bo->bo, timeout, &buffer_busy);
  57       if (r)
  58          fprintf(stderr, "%s: amdgpu_bo_wait_for_idle failed %i\n", __func__,
  59                  r);
  60       return !buffer_busy;
  61    }
  62
  63    if (timeout == 0) {
  64       /* Timeout == 0 is quite simple. */
  65       pipe_mutex_lock(ws->bo_fence_lock);
  66       for (i = 0; i < RING_LAST; i++)
  67          if (bo->fence[i]) {
  68             if (amdgpu_fence_wait(bo->fence[i], 0, false)) {
  69                /* Release the idle fence to avoid checking it again later. */
  70                amdgpu_fence_reference(&bo->fence[i], NULL);
  71             } else {
  72                pipe_mutex_unlock(ws->bo_fence_lock);
  73                return false;
  74             }
  75          }
  76       pipe_mutex_unlock(ws->bo_fence_lock);
  77       return true;
  78
  79    } else {
  80       struct pipe_fence_handle *fence[RING_LAST] = {};
  81       bool fence_idle[RING_LAST] = {};
  82       bool buffer_idle = true;
  83       int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
  84
  85       /* Take references to all fences, so that we can wait for them
  86        * without the lock. */
  87       pipe_mutex_lock(ws->bo_fence_lock);
  88       for (i = 0; i < RING_LAST; i++)
  89          amdgpu_fence_reference(&fence[i], bo->fence[i]);
  90       pipe_mutex_unlock(ws->bo_fence_lock);
  91
  92       /* Now wait for the fences. */
  93       for (i = 0; i < RING_LAST; i++) {
  94          if (fence[i]) {
  95             if (amdgpu_fence_wait(fence[i], abs_timeout, true))
  96                fence_idle[i] = true;
  97             else
  98                buffer_idle = false;
  99          }
 100       }
 101
 102       /* Release idle fences to avoid checking them again later. */
 103       pipe_mutex_lock(ws->bo_fence_lock);
 104       for (i = 0; i < RING_LAST; i++) {
 105          if (fence[i] == bo->fence[i] && fence_idle[i])
 106             amdgpu_fence_reference(&bo->fence[i], NULL);
 107
 108          amdgpu_fence_reference(&fence[i], NULL);
 109       }
 110       pipe_mutex_unlock(ws->bo_fence_lock);
 111
 112       return buffer_idle;
 113    }
 114 }
 115
 116 static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
 117       struct pb_buffer *buf)
 118 {
 119    return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
 120 }
 121
 122 void amdgpu_bo_destroy(struct pb_buffer *_buf)
 123 {
 124    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 125    int i;
 126
 127    pipe_mutex_lock(bo->ws->global_bo_list_lock);
 128    LIST_DEL(&bo->global_list_item);
 129    bo->ws->num_buffers--;
 130    pipe_mutex_unlock(bo->ws->global_bo_list_lock);
 131
 132    amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
 133    amdgpu_va_range_free(bo->va_handle);
 134    amdgpu_bo_free(bo->bo);
 135
 136    for (i = 0; i < RING_LAST; i++)
 137       amdgpu_fence_reference(&bo->fence[i], NULL);
 138
 139    if (bo->initial_domain & RADEON_DOMAIN_VRAM)
 140       bo->ws->allocated_vram -= align64(bo->base.size, bo->ws->gart_page_size);
 141    else if (bo->initial_domain & RADEON_DOMAIN_GTT)
 142       bo->ws->allocated_gtt -= align64(bo->base.size, bo->ws->gart_page_size);
 143    FREE(bo);
 144 }
 145
 146 static void amdgpu_bo_destroy_or_cache(struct pb_buffer *_buf)
 147 {
 148    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 149
 150    if (bo->use_reusable_pool)
 151       pb_cache_add_buffer(&bo->cache_entry);
 152    else
 153       amdgpu_bo_destroy(_buf);
 154 }
 155
 156 static void *amdgpu_bo_map(struct pb_buffer *buf,
 157                            struct radeon_winsys_cs *rcs,
 158                            enum pipe_transfer_usage usage)
 159 {
 160    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 161    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 162    int r;
 163    void *cpu = NULL;
 164
 165    /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
 166    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 167       /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
 168       if (usage & PIPE_TRANSFER_DONTBLOCK) {
 169          if (!(usage & PIPE_TRANSFER_WRITE)) {
 170             /* Mapping for read.
 171              *
 172              * Since we are mapping for read, we don't need to wait
 173              * if the GPU is using the buffer for read too
 174              * (neither one is changing it).
 175              *
 176              * Only check whether the buffer is being used for write. */
 177             if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
 178                                                                RADEON_USAGE_WRITE)) {
 179                cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
 180                return NULL;
 181             }
 182
 183             if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
 184                                 RADEON_USAGE_WRITE)) {
 185                return NULL;
 186             }
 187          } else {
 188             if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
 189                cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
 190                return NULL;
 191             }
 192
 193             if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
 194                                 RADEON_USAGE_READWRITE)) {
 195                return NULL;
 196             }
 197          }
 198       } else {
 199          uint64_t time = os_time_get_nano();
 200
 201          if (!(usage & PIPE_TRANSFER_WRITE)) {
 202             /* Mapping for read.
 203              *
 204              * Since we are mapping for read, we don't need to wait
 205              * if the GPU is using the buffer for read too
 206              * (neither one is changing it).
 207              *
 208              * Only check whether the buffer is being used for write. */
 209             if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
 210                                                                RADEON_USAGE_WRITE)) {
 211                cs->flush_cs(cs->flush_data, 0, NULL);
 212             }
 213             amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
 214                            RADEON_USAGE_WRITE);
 215          } else {
 216             /* Mapping for write. */
 217             if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo))
 218                cs->flush_cs(cs->flush_data, 0, NULL);
 219
 220             amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
 221                            RADEON_USAGE_READWRITE);
 222          }
 223
 224          bo->ws->buffer_wait_time += os_time_get_nano() - time;
 225       }
 226    }
 227
 228    /* If the buffer is created from user memory, return the user pointer. */
 229    if (bo->user_ptr)
 230        return bo->user_ptr;
 231
 232    r = amdgpu_bo_cpu_map(bo->bo, &cpu);
 233    if (r) {
 234       /* Clear the cache and try again. */
 235       pb_cache_release_all_buffers(&bo->ws->bo_cache);
 236       r = amdgpu_bo_cpu_map(bo->bo, &cpu);
 237    }
 238    return r ? NULL : cpu;
 239 }
 240
 241 static void amdgpu_bo_unmap(struct pb_buffer *buf)
 242 {
 243    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 244
 245    amdgpu_bo_cpu_unmap(bo->bo);
 246 }
 247
 248 static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
 249    amdgpu_bo_destroy_or_cache
 250    /* other functions are never called */
 251 };
 252
 253 static void amdgpu_add_buffer_to_global_list(struct amdgpu_winsys_bo *bo)
 254 {
 255    struct amdgpu_winsys *ws = bo->ws;
 256
 257    pipe_mutex_lock(ws->global_bo_list_lock);
 258    LIST_ADDTAIL(&bo->global_list_item, &ws->global_bo_list);
 259    ws->num_buffers++;
 260    pipe_mutex_unlock(ws->global_bo_list_lock);
 261 }
 262
 263 static struct amdgpu_winsys_bo *amdgpu_create_bo(struct amdgpu_winsys *ws,
 264                                                  uint64_t size,
 265                                                  unsigned alignment,
 266                                                  unsigned usage,
 267                                                  enum radeon_bo_domain initial_domain,
 268                                                  unsigned flags)
 269 {
 270    struct amdgpu_bo_alloc_request request = {0};
 271    amdgpu_bo_handle buf_handle;
 272    uint64_t va = 0;
 273    struct amdgpu_winsys_bo *bo;
 274    amdgpu_va_handle va_handle;
 275    int r;
 276
 277    assert(initial_domain & RADEON_DOMAIN_VRAM_GTT);
 278    bo = CALLOC_STRUCT(amdgpu_winsys_bo);
 279    if (!bo) {
 280       return NULL;
 281    }
 282
 283    pb_cache_init_entry(&ws->bo_cache, &bo->cache_entry, &bo->base);
 284    request.alloc_size = size;
 285    request.phys_alignment = alignment;
 286
 287    if (initial_domain & RADEON_DOMAIN_VRAM)
 288       request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
 289    if (initial_domain & RADEON_DOMAIN_GTT)
 290       request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
 291
 292    if (flags & RADEON_FLAG_CPU_ACCESS)
 293       request.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 294    if (flags & RADEON_FLAG_NO_CPU_ACCESS)
 295       request.flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
 296    if (flags & RADEON_FLAG_GTT_WC)
 297       request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 298
 299    r = amdgpu_bo_alloc(ws->dev, &request, &buf_handle);
 300    if (r) {
 301       fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
 302       fprintf(stderr, "amdgpu:    size      : %"PRIu64" bytes\n", size);
 303       fprintf(stderr, "amdgpu:    alignment : %u bytes\n", alignment);
 304       fprintf(stderr, "amdgpu:    domains   : %u\n", initial_domain);
 305       goto error_bo_alloc;
 306    }
 307
 308    r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
 309                              size, alignment, 0, &va, &va_handle, 0);
 310    if (r)
 311       goto error_va_alloc;
 312
 313    r = amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP);
 314    if (r)
 315       goto error_va_map;
 316
 317    pipe_reference_init(&bo->base.reference, 1);
 318    bo->base.alignment = alignment;
 319    bo->base.usage = usage;
 320    bo->base.size = size;
 321    bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
 322    bo->ws = ws;
 323    bo->bo = buf_handle;
 324    bo->va = va;
 325    bo->va_handle = va_handle;
 326    bo->initial_domain = initial_domain;
 327    bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
 328
 329    if (initial_domain & RADEON_DOMAIN_VRAM)
 330       ws->allocated_vram += align64(size, ws->gart_page_size);
 331    else if (initial_domain & RADEON_DOMAIN_GTT)
 332       ws->allocated_gtt += align64(size, ws->gart_page_size);
 333
 334    amdgpu_add_buffer_to_global_list(bo);
 335
 336    return bo;
 337
 338 error_va_map:
 339    amdgpu_va_range_free(va_handle);
 340
 341 error_va_alloc:
 342    amdgpu_bo_free(buf_handle);
 343
 344 error_bo_alloc:
 345    FREE(bo);
 346    return NULL;
 347 }
 348
 349 bool amdgpu_bo_can_reclaim(struct pb_buffer *_buf)
 350 {
 351    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 352
 353    if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
 354       return false;
 355    }
 356
 357    return amdgpu_bo_wait(_buf, 0, RADEON_USAGE_READWRITE);
 358 }
 359
 360 static unsigned eg_tile_split(unsigned tile_split)
 361 {
 362    switch (tile_split) {
 363    case 0:     tile_split = 64;    break;
 364    case 1:     tile_split = 128;   break;
 365    case 2:     tile_split = 256;   break;
 366    case 3:     tile_split = 512;   break;
 367    default:
 368    case 4:     tile_split = 1024;  break;
 369    case 5:     tile_split = 2048;  break;
 370    case 6:     tile_split = 4096;  break;
 371    }
 372    return tile_split;
 373 }
 374
 375 static unsigned eg_tile_split_rev(unsigned eg_tile_split)
 376 {
 377    switch (eg_tile_split) {
 378    case 64:    return 0;
 379    case 128:   return 1;
 380    case 256:   return 2;
 381    case 512:   return 3;
 382    default:
 383    case 1024:  return 4;
 384    case 2048:  return 5;
 385    case 4096:  return 6;
 386    }
 387 }
 388
 389 static void amdgpu_buffer_get_metadata(struct pb_buffer *_buf,
 390                                        struct radeon_bo_metadata *md)
 391 {
 392    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 393    struct amdgpu_bo_info info = {0};
 394    uint32_t tiling_flags;
 395    int r;
 396
 397    r = amdgpu_bo_query_info(bo->bo, &info);
 398    if (r)
 399       return;
 400
 401    tiling_flags = info.metadata.tiling_info;
 402
 403    md->microtile = RADEON_LAYOUT_LINEAR;
 404    md->macrotile = RADEON_LAYOUT_LINEAR;
 405
 406    if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4)  /* 2D_TILED_THIN1 */
 407       md->macrotile = RADEON_LAYOUT_TILED;
 408    else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
 409       md->microtile = RADEON_LAYOUT_TILED;
 410
 411    md->bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
 412    md->bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
 413    md->tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
 414    md->mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
 415    md->scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
 416
 417    md->size_metadata = info.metadata.size_metadata;
 418    memcpy(md->metadata, info.metadata.umd_metadata, sizeof(md->metadata));
 419 }
 420
 421 static void amdgpu_buffer_set_metadata(struct pb_buffer *_buf,
 422                                        struct radeon_bo_metadata *md)
 423 {
 424    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 425    struct amdgpu_bo_metadata metadata = {0};
 426    uint32_t tiling_flags = 0;
 427
 428    if (md->macrotile == RADEON_LAYOUT_TILED)
 429       tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
 430    else if (md->microtile == RADEON_LAYOUT_TILED)
 431       tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
 432    else
 433       tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
 434
 435    tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, md->pipe_config);
 436    tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(md->bankw));
 437    tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(md->bankh));
 438    if (md->tile_split)
 439       tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(md->tile_split));
 440    tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(md->mtilea));
 441    tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(md->num_banks)-1);
 442
 443    if (md->scanout)
 444       tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
 445    else
 446       tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
 447
 448    metadata.tiling_info = tiling_flags;
 449    metadata.size_metadata = md->size_metadata;
 450    memcpy(metadata.umd_metadata, md->metadata, sizeof(md->metadata));
 451
 452    amdgpu_bo_set_metadata(bo->bo, &metadata);
 453 }
 454
 455 static struct pb_buffer *
 456 amdgpu_bo_create(struct radeon_winsys *rws,
 457                  uint64_t size,
 458                  unsigned alignment,
 459                  enum radeon_bo_domain domain,
 460                  enum radeon_bo_flag flags)
 461 {
 462    struct amdgpu_winsys *ws = amdgpu_winsys(rws);
 463    struct amdgpu_winsys_bo *bo;
 464    unsigned usage = 0;
 465
 466    /* Align size to page size. This is the minimum alignment for normal
 467     * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
 468     * like constant/uniform buffers, can benefit from better and more reuse.
 469     */
 470    size = align64(size, ws->gart_page_size);
 471
 472    /* Only set one usage bit each for domains and flags, or the cache manager
 473     * might consider different sets of domains / flags compatible
 474     */
 475    if (domain == RADEON_DOMAIN_VRAM_GTT)
 476       usage = 1 << 2;
 477    else
 478       usage = domain >> 1;
 479    assert(flags < sizeof(usage) * 8 - 3);
 480    usage |= 1 << (flags + 3);
 481
 482    /* Get a buffer from the cache. */
 483    bo = (struct amdgpu_winsys_bo*)
 484         pb_cache_reclaim_buffer(&ws->bo_cache, size, alignment, usage);
 485    if (bo)
 486       return &bo->base;
 487
 488    /* Create a new one. */
 489    bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags);
 490    if (!bo) {
 491       /* Clear the cache and try again. */
 492       pb_cache_release_all_buffers(&ws->bo_cache);
 493       bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags);
 494       if (!bo)
 495          return NULL;
 496    }
 497
 498    bo->use_reusable_pool = true;
 499    return &bo->base;
 500 }
 501
 502 static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
 503                                                struct winsys_handle *whandle,
 504                                                unsigned *stride,
 505                                                unsigned *offset)
 506 {
 507    struct amdgpu_winsys *ws = amdgpu_winsys(rws);
 508    struct amdgpu_winsys_bo *bo;
 509    enum amdgpu_bo_handle_type type;
 510    struct amdgpu_bo_import_result result = {0};
 511    uint64_t va;
 512    amdgpu_va_handle va_handle;
 513    struct amdgpu_bo_info info = {0};
 514    enum radeon_bo_domain initial = 0;
 515    int r;
 516
 517    /* Initialize the structure. */
 518    bo = CALLOC_STRUCT(amdgpu_winsys_bo);
 519    if (!bo) {
 520       return NULL;
 521    }
 522
 523    switch (whandle->type) {
 524    case DRM_API_HANDLE_TYPE_SHARED:
 525       type = amdgpu_bo_handle_type_gem_flink_name;
 526       break;
 527    case DRM_API_HANDLE_TYPE_FD:
 528       type = amdgpu_bo_handle_type_dma_buf_fd;
 529       break;
 530    default:
 531       return NULL;
 532    }
 533
 534    r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
 535    if (r)
 536       goto error;
 537
 538    /* Get initial domains. */
 539    r = amdgpu_bo_query_info(result.buf_handle, &info);
 540    if (r)
 541       goto error_query;
 542
 543    r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
 544                              result.alloc_size, 1 << 20, 0, &va, &va_handle, 0);
 545    if (r)
 546       goto error_query;
 547
 548    r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
 549    if (r)
 550       goto error_va_map;
 551
 552    if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
 553       initial |= RADEON_DOMAIN_VRAM;
 554    if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
 555       initial |= RADEON_DOMAIN_GTT;
 556
 557
 558    pipe_reference_init(&bo->base.reference, 1);
 559    bo->base.alignment = info.phys_alignment;
 560    bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
 561    bo->bo = result.buf_handle;
 562    bo->base.size = result.alloc_size;
 563    bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
 564    bo->ws = ws;
 565    bo->va = va;
 566    bo->va_handle = va_handle;
 567    bo->initial_domain = initial;
 568    bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
 569    bo->is_shared = true;
 570
 571    if (stride)
 572       *stride = whandle->stride;
 573    if (offset)
 574       *offset = whandle->offset;
 575
 576    if (bo->initial_domain & RADEON_DOMAIN_VRAM)
 577       ws->allocated_vram += align64(bo->base.size, ws->gart_page_size);
 578    else if (bo->initial_domain & RADEON_DOMAIN_GTT)
 579       ws->allocated_gtt += align64(bo->base.size, ws->gart_page_size);
 580
 581    amdgpu_add_buffer_to_global_list(bo);
 582
 583    return &bo->base;
 584
 585 error_va_map:
 586    amdgpu_va_range_free(va_handle);
 587
 588 error_query:
 589    amdgpu_bo_free(result.buf_handle);
 590
 591 error:
 592    FREE(bo);
 593    return NULL;
 594 }
 595
 596 static boolean amdgpu_bo_get_handle(struct pb_buffer *buffer,
 597                                     unsigned stride, unsigned offset,
 598                                     unsigned slice_size,
 599                                     struct winsys_handle *whandle)
 600 {
 601    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buffer);
 602    enum amdgpu_bo_handle_type type;
 603    int r;
 604
 605    bo->use_reusable_pool = false;
 606
 607    switch (whandle->type) {
 608    case DRM_API_HANDLE_TYPE_SHARED:
 609       type = amdgpu_bo_handle_type_gem_flink_name;
 610       break;
 611    case DRM_API_HANDLE_TYPE_FD:
 612       type = amdgpu_bo_handle_type_dma_buf_fd;
 613       break;
 614    case DRM_API_HANDLE_TYPE_KMS:
 615       type = amdgpu_bo_handle_type_kms;
 616       break;
 617    default:
 618       return FALSE;
 619    }
 620
 621    r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
 622    if (r)
 623       return FALSE;
 624
 625    whandle->stride = stride;
 626    whandle->offset = offset;
 627    whandle->offset += slice_size * whandle->layer;
 628    bo->is_shared = true;
 629    return TRUE;
 630 }
 631
 632 static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
 633                                             void *pointer, uint64_t size)
 634 {
 635     struct amdgpu_winsys *ws = amdgpu_winsys(rws);
 636     amdgpu_bo_handle buf_handle;
 637     struct amdgpu_winsys_bo *bo;
 638     uint64_t va;
 639     amdgpu_va_handle va_handle;
 640
 641     bo = CALLOC_STRUCT(amdgpu_winsys_bo);
 642     if (!bo)
 643         return NULL;
 644
 645     if (amdgpu_create_bo_from_user_mem(ws->dev, pointer, size, &buf_handle))
 646         goto error;
 647
 648     if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
 649                               size, 1 << 12, 0, &va, &va_handle, 0))
 650         goto error_va_alloc;
 651
 652     if (amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP))
 653         goto error_va_map;
 654
 655     /* Initialize it. */
 656     pipe_reference_init(&bo->base.reference, 1);
 657     bo->bo = buf_handle;
 658     bo->base.alignment = 0;
 659     bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
 660     bo->base.size = size;
 661     bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
 662     bo->ws = ws;
 663     bo->user_ptr = pointer;
 664     bo->va = va;
 665     bo->va_handle = va_handle;
 666     bo->initial_domain = RADEON_DOMAIN_GTT;
 667     bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
 668
 669     ws->allocated_gtt += align64(bo->base.size, ws->gart_page_size);
 670
 671     amdgpu_add_buffer_to_global_list(bo);
 672
 673     return (struct pb_buffer*)bo;
 674
 675 error_va_map:
 676     amdgpu_va_range_free(va_handle);
 677
 678 error_va_alloc:
 679     amdgpu_bo_free(buf_handle);
 680
 681 error:
 682     FREE(bo);
 683     return NULL;
 684 }
 685
 686 static bool amdgpu_bo_is_user_ptr(struct pb_buffer *buf)
 687 {
 688    return ((struct amdgpu_winsys_bo*)buf)->user_ptr != NULL;
 689 }
 690
 691 static uint64_t amdgpu_bo_get_va(struct pb_buffer *buf)
 692 {
 693    return ((struct amdgpu_winsys_bo*)buf)->va;
 694 }
 695
 696 void amdgpu_bo_init_functions(struct amdgpu_winsys *ws)
 697 {
 698    ws->base.buffer_set_metadata = amdgpu_buffer_set_metadata;
 699    ws->base.buffer_get_metadata = amdgpu_buffer_get_metadata;
 700    ws->base.buffer_map = amdgpu_bo_map;
 701    ws->base.buffer_unmap = amdgpu_bo_unmap;
 702    ws->base.buffer_wait = amdgpu_bo_wait;
 703    ws->base.buffer_create = amdgpu_bo_create;
 704    ws->base.buffer_from_handle = amdgpu_bo_from_handle;
 705    ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
 706    ws->base.buffer_is_user_ptr = amdgpu_bo_is_user_ptr;
 707    ws->base.buffer_get_handle = amdgpu_bo_get_handle;
 708    ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
 709    ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
 710 }