src/gallium/winsys/amdgpu/drm/amdgpu_bo.c

   1 /*
   2  * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
   3  * Copyright © 2015 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  16  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  17  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  18  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * The above copyright notice and this permission notice (including the
  24  * next paragraph) shall be included in all copies or substantial portions
  25  * of the Software.
  26  */
  27 /*
  28  * Authors:
  29  *      Marek Olšák <maraeo@gmail.com>
  30  */
  31
  32 #include "amdgpu_cs.h"
  33
  34 #include "os/os_time.h"
  35 #include "state_tracker/drm_driver.h"
  36 #include <amdgpu_drm.h>
  37 #include <xf86drm.h>
  38 #include <stdio.h>
  39
  40 static const struct pb_vtbl amdgpu_winsys_bo_vtbl;
  41
  42 static inline struct amdgpu_winsys_bo *amdgpu_winsys_bo(struct pb_buffer *bo)
  43 {
  44    assert(bo->vtbl == &amdgpu_winsys_bo_vtbl);
  45    return (struct amdgpu_winsys_bo *)bo;
  46 }
  47
  48 struct amdgpu_bomgr {
  49    struct pb_manager base;
  50    struct amdgpu_winsys *rws;
  51 };
  52
  53 static struct amdgpu_winsys *get_winsys(struct pb_manager *mgr)
  54 {
  55    return ((struct amdgpu_bomgr*)mgr)->rws;
  56 }
  57
  58 static struct amdgpu_winsys_bo *get_amdgpu_winsys_bo(struct pb_buffer *_buf)
  59 {
  60    struct amdgpu_winsys_bo *bo = NULL;
  61
  62    if (_buf->vtbl == &amdgpu_winsys_bo_vtbl) {
  63       bo = amdgpu_winsys_bo(_buf);
  64    } else {
  65       struct pb_buffer *base_buf;
  66       pb_size offset;
  67       pb_get_base_buffer(_buf, &base_buf, &offset);
  68
  69       if (base_buf->vtbl == &amdgpu_winsys_bo_vtbl)
  70          bo = amdgpu_winsys_bo(base_buf);
  71    }
  72
  73    return bo;
  74 }
  75
  76 static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
  77                            enum radeon_bo_usage usage)
  78 {
  79    struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(_buf);
  80    struct amdgpu_winsys *ws = bo->rws;
  81    int i;
  82
  83    if (bo->is_shared) {
  84       /* We can't use user fences for shared buffers, because user fences
  85        * are local to this process only. If we want to wait for all buffer
  86        * uses in all processes, we have to use amdgpu_bo_wait_for_idle.
  87        */
  88       bool buffer_busy = true;
  89       int r;
  90
  91       r = amdgpu_bo_wait_for_idle(bo->bo, timeout, &buffer_busy);
  92       if (r)
  93          fprintf(stderr, "%s: amdgpu_bo_wait_for_idle failed %i\n", __func__,
  94                  r);
  95       return !buffer_busy;
  96    }
  97
  98    if (timeout == 0) {
  99       /* Timeout == 0 is quite simple. */
 100       pipe_mutex_lock(ws->bo_fence_lock);
 101       for (i = 0; i < RING_LAST; i++)
 102          if (bo->fence[i]) {
 103             if (amdgpu_fence_wait(bo->fence[i], 0, false)) {
 104                /* Release the idle fence to avoid checking it again later. */
 105                amdgpu_fence_reference(&bo->fence[i], NULL);
 106             } else {
 107                pipe_mutex_unlock(ws->bo_fence_lock);
 108                return false;
 109             }
 110          }
 111       pipe_mutex_unlock(ws->bo_fence_lock);
 112       return true;
 113
 114    } else {
 115       struct pipe_fence_handle *fence[RING_LAST] = {};
 116       bool fence_idle[RING_LAST] = {};
 117       bool buffer_idle = true;
 118       int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
 119
 120       /* Take references to all fences, so that we can wait for them
 121        * without the lock. */
 122       pipe_mutex_lock(ws->bo_fence_lock);
 123       for (i = 0; i < RING_LAST; i++)
 124          amdgpu_fence_reference(&fence[i], bo->fence[i]);
 125       pipe_mutex_unlock(ws->bo_fence_lock);
 126
 127       /* Now wait for the fences. */
 128       for (i = 0; i < RING_LAST; i++) {
 129          if (fence[i]) {
 130             if (amdgpu_fence_wait(fence[i], abs_timeout, true))
 131                fence_idle[i] = true;
 132             else
 133                buffer_idle = false;
 134          }
 135       }
 136
 137       /* Release idle fences to avoid checking them again later. */
 138       pipe_mutex_lock(ws->bo_fence_lock);
 139       for (i = 0; i < RING_LAST; i++) {
 140          if (fence[i] == bo->fence[i] && fence_idle[i])
 141             amdgpu_fence_reference(&bo->fence[i], NULL);
 142
 143          amdgpu_fence_reference(&fence[i], NULL);
 144       }
 145       pipe_mutex_unlock(ws->bo_fence_lock);
 146
 147       return buffer_idle;
 148    }
 149 }
 150
 151 static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
 152       struct radeon_winsys_cs_handle *buf)
 153 {
 154    return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
 155 }
 156
 157 static void amdgpu_bo_destroy(struct pb_buffer *_buf)
 158 {
 159    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 160    int i;
 161
 162    amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
 163    amdgpu_va_range_free(bo->va_handle);
 164    amdgpu_bo_free(bo->bo);
 165
 166    for (i = 0; i < RING_LAST; i++)
 167       amdgpu_fence_reference(&bo->fence[i], NULL);
 168
 169    if (bo->initial_domain & RADEON_DOMAIN_VRAM)
 170       bo->rws->allocated_vram -= align(bo->base.size, bo->rws->gart_page_size);
 171    else if (bo->initial_domain & RADEON_DOMAIN_GTT)
 172       bo->rws->allocated_gtt -= align(bo->base.size, bo->rws->gart_page_size);
 173    FREE(bo);
 174 }
 175
 176 static void *amdgpu_bo_map(struct radeon_winsys_cs_handle *buf,
 177                            struct radeon_winsys_cs *rcs,
 178                            enum pipe_transfer_usage usage)
 179 {
 180    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 181    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 182    int r;
 183    void *cpu = NULL;
 184
 185    /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
 186    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 187       /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
 188       if (usage & PIPE_TRANSFER_DONTBLOCK) {
 189          if (!(usage & PIPE_TRANSFER_WRITE)) {
 190             /* Mapping for read.
 191              *
 192              * Since we are mapping for read, we don't need to wait
 193              * if the GPU is using the buffer for read too
 194              * (neither one is changing it).
 195              *
 196              * Only check whether the buffer is being used for write. */
 197             if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
 198                                                                RADEON_USAGE_WRITE)) {
 199                cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
 200                return NULL;
 201             }
 202
 203             if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
 204                                 RADEON_USAGE_WRITE)) {
 205                return NULL;
 206             }
 207          } else {
 208             if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
 209                cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
 210                return NULL;
 211             }
 212
 213             if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
 214                                 RADEON_USAGE_READWRITE)) {
 215                return NULL;
 216             }
 217          }
 218       } else {
 219          uint64_t time = os_time_get_nano();
 220
 221          if (!(usage & PIPE_TRANSFER_WRITE)) {
 222             /* Mapping for read.
 223              *
 224              * Since we are mapping for read, we don't need to wait
 225              * if the GPU is using the buffer for read too
 226              * (neither one is changing it).
 227              *
 228              * Only check whether the buffer is being used for write. */
 229             if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
 230                                                                RADEON_USAGE_WRITE)) {
 231                cs->flush_cs(cs->flush_data, 0, NULL);
 232             }
 233             amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
 234                            RADEON_USAGE_WRITE);
 235          } else {
 236             /* Mapping for write. */
 237             if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo))
 238                cs->flush_cs(cs->flush_data, 0, NULL);
 239
 240             amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
 241                            RADEON_USAGE_READWRITE);
 242          }
 243
 244          bo->rws->buffer_wait_time += os_time_get_nano() - time;
 245       }
 246    }
 247
 248    /* If the buffer is created from user memory, return the user pointer. */
 249    if (bo->user_ptr)
 250        return bo->user_ptr;
 251
 252    r = amdgpu_bo_cpu_map(bo->bo, &cpu);
 253    return r ? NULL : cpu;
 254 }
 255
 256 static void amdgpu_bo_unmap(struct radeon_winsys_cs_handle *buf)
 257 {
 258    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 259
 260    amdgpu_bo_cpu_unmap(bo->bo);
 261 }
 262
 263 static void amdgpu_bo_get_base_buffer(struct pb_buffer *buf,
 264                                       struct pb_buffer **base_buf,
 265                                       unsigned *offset)
 266 {
 267    *base_buf = buf;
 268    *offset = 0;
 269 }
 270
 271 static enum pipe_error amdgpu_bo_validate(struct pb_buffer *_buf,
 272                                           struct pb_validate *vl,
 273                                           unsigned flags)
 274 {
 275    /* Always pinned */
 276    return PIPE_OK;
 277 }
 278
 279 static void amdgpu_bo_fence(struct pb_buffer *buf,
 280                             struct pipe_fence_handle *fence)
 281 {
 282 }
 283
 284 static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
 285    amdgpu_bo_destroy,
 286    NULL, /* never called */
 287    NULL, /* never called */
 288    amdgpu_bo_validate,
 289    amdgpu_bo_fence,
 290    amdgpu_bo_get_base_buffer,
 291 };
 292
 293 static struct pb_buffer *amdgpu_bomgr_create_bo(struct pb_manager *_mgr,
 294                                                 pb_size size,
 295                                                 const struct pb_desc *desc)
 296 {
 297    struct amdgpu_winsys *rws = get_winsys(_mgr);
 298    struct amdgpu_bo_desc *rdesc = (struct amdgpu_bo_desc*)desc;
 299    struct amdgpu_bo_alloc_request request = {0};
 300    amdgpu_bo_handle buf_handle;
 301    uint64_t va = 0;
 302    struct amdgpu_winsys_bo *bo;
 303    amdgpu_va_handle va_handle;
 304    int r;
 305
 306    assert(rdesc->initial_domain & RADEON_DOMAIN_VRAM_GTT);
 307    bo = CALLOC_STRUCT(amdgpu_winsys_bo);
 308    if (!bo) {
 309       return NULL;
 310    }
 311
 312    request.alloc_size = size;
 313    request.phys_alignment = desc->alignment;
 314
 315    if (rdesc->initial_domain & RADEON_DOMAIN_VRAM) {
 316       request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
 317       if (rdesc->flags & RADEON_FLAG_CPU_ACCESS)
 318          request.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 319    }
 320    if (rdesc->initial_domain & RADEON_DOMAIN_GTT) {
 321       request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
 322       if (rdesc->flags & RADEON_FLAG_GTT_WC)
 323          request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 324    }
 325
 326    r = amdgpu_bo_alloc(rws->dev, &request, &buf_handle);
 327    if (r) {
 328       fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
 329       fprintf(stderr, "amdgpu:    size      : %d bytes\n", size);
 330       fprintf(stderr, "amdgpu:    alignment : %d bytes\n", desc->alignment);
 331       fprintf(stderr, "amdgpu:    domains   : %d\n", rdesc->initial_domain);
 332       goto error_bo_alloc;
 333    }
 334
 335    r = amdgpu_va_range_alloc(rws->dev, amdgpu_gpu_va_range_general,
 336                              size, desc->alignment, 0, &va, &va_handle, 0);
 337    if (r)
 338       goto error_va_alloc;
 339
 340    r = amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP);
 341    if (r)
 342       goto error_va_map;
 343
 344    pipe_reference_init(&bo->base.reference, 1);
 345    bo->base.alignment = desc->alignment;
 346    bo->base.usage = desc->usage;
 347    bo->base.size = size;
 348    bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
 349    bo->rws = rws;
 350    bo->bo = buf_handle;
 351    bo->va = va;
 352    bo->va_handle = va_handle;
 353    bo->initial_domain = rdesc->initial_domain;
 354    bo->unique_id = __sync_fetch_and_add(&rws->next_bo_unique_id, 1);
 355
 356    if (rdesc->initial_domain & RADEON_DOMAIN_VRAM)
 357       rws->allocated_vram += align(size, rws->gart_page_size);
 358    else if (rdesc->initial_domain & RADEON_DOMAIN_GTT)
 359       rws->allocated_gtt += align(size, rws->gart_page_size);
 360
 361    return &bo->base;
 362
 363 error_va_map:
 364    amdgpu_va_range_free(va_handle);
 365
 366 error_va_alloc:
 367    amdgpu_bo_free(buf_handle);
 368
 369 error_bo_alloc:
 370    FREE(bo);
 371    return NULL;
 372 }
 373
 374 static void amdgpu_bomgr_flush(struct pb_manager *mgr)
 375 {
 376    /* NOP */
 377 }
 378
 379 /* This is for the cache bufmgr. */
 380 static boolean amdgpu_bomgr_is_buffer_busy(struct pb_manager *_mgr,
 381                                            struct pb_buffer *_buf)
 382 {
 383    struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
 384
 385    if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
 386       return TRUE;
 387    }
 388
 389    if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0, RADEON_USAGE_READWRITE)) {
 390       return TRUE;
 391    }
 392
 393    return FALSE;
 394 }
 395
 396 static void amdgpu_bomgr_destroy(struct pb_manager *mgr)
 397 {
 398    FREE(mgr);
 399 }
 400
 401 struct pb_manager *amdgpu_bomgr_create(struct amdgpu_winsys *rws)
 402 {
 403    struct amdgpu_bomgr *mgr;
 404
 405    mgr = CALLOC_STRUCT(amdgpu_bomgr);
 406    if (!mgr)
 407       return NULL;
 408
 409    mgr->base.destroy = amdgpu_bomgr_destroy;
 410    mgr->base.create_buffer = amdgpu_bomgr_create_bo;
 411    mgr->base.flush = amdgpu_bomgr_flush;
 412    mgr->base.is_buffer_busy = amdgpu_bomgr_is_buffer_busy;
 413
 414    mgr->rws = rws;
 415    return &mgr->base;
 416 }
 417
 418 static unsigned eg_tile_split(unsigned tile_split)
 419 {
 420    switch (tile_split) {
 421    case 0:     tile_split = 64;    break;
 422    case 1:     tile_split = 128;   break;
 423    case 2:     tile_split = 256;   break;
 424    case 3:     tile_split = 512;   break;
 425    default:
 426    case 4:     tile_split = 1024;  break;
 427    case 5:     tile_split = 2048;  break;
 428    case 6:     tile_split = 4096;  break;
 429    }
 430    return tile_split;
 431 }
 432
 433 static unsigned eg_tile_split_rev(unsigned eg_tile_split)
 434 {
 435    switch (eg_tile_split) {
 436    case 64:    return 0;
 437    case 128:   return 1;
 438    case 256:   return 2;
 439    case 512:   return 3;
 440    default:
 441    case 1024:  return 4;
 442    case 2048:  return 5;
 443    case 4096:  return 6;
 444    }
 445 }
 446
 447 static void amdgpu_bo_get_tiling(struct pb_buffer *_buf,
 448                                  enum radeon_bo_layout *microtiled,
 449                                  enum radeon_bo_layout *macrotiled,
 450                                  unsigned *bankw, unsigned *bankh,
 451                                  unsigned *tile_split,
 452                                  unsigned *stencil_tile_split,
 453                                  unsigned *mtilea,
 454                                  bool *scanout)
 455 {
 456    struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(_buf);
 457    struct amdgpu_bo_info info = {0};
 458    uint32_t tiling_flags;
 459    int r;
 460
 461    r = amdgpu_bo_query_info(bo->bo, &info);
 462    if (r)
 463       return;
 464
 465    tiling_flags = info.metadata.tiling_info;
 466
 467    *microtiled = RADEON_LAYOUT_LINEAR;
 468    *macrotiled = RADEON_LAYOUT_LINEAR;
 469
 470    if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4)  /* 2D_TILED_THIN1 */
 471       *macrotiled = RADEON_LAYOUT_TILED;
 472    else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
 473       *microtiled = RADEON_LAYOUT_TILED;
 474
 475    if (bankw && tile_split && mtilea && tile_split) {
 476       *bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
 477       *bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
 478       *tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
 479       *mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
 480    }
 481    if (scanout)
 482       *scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
 483 }
 484
 485 static void amdgpu_bo_set_tiling(struct pb_buffer *_buf,
 486                                  struct radeon_winsys_cs *rcs,
 487                                  enum radeon_bo_layout microtiled,
 488                                  enum radeon_bo_layout macrotiled,
 489                                  unsigned pipe_config,
 490                                  unsigned bankw, unsigned bankh,
 491                                  unsigned tile_split,
 492                                  unsigned stencil_tile_split,
 493                                  unsigned mtilea, unsigned num_banks,
 494                                  uint32_t pitch,
 495                                  bool scanout)
 496 {
 497    struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(_buf);
 498    struct amdgpu_bo_metadata metadata = {0};
 499    uint32_t tiling_flags = 0;
 500
 501    if (macrotiled == RADEON_LAYOUT_TILED)
 502       tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
 503    else if (microtiled == RADEON_LAYOUT_TILED)
 504       tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
 505    else
 506       tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
 507
 508    tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, pipe_config);
 509    tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(bankw));
 510    tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(bankh));
 511    if (tile_split)
 512       tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(tile_split));
 513    tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(mtilea));
 514    tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(num_banks)-1);
 515
 516    if (scanout)
 517       tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
 518    else
 519       tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
 520
 521    metadata.tiling_info = tiling_flags;
 522
 523    amdgpu_bo_set_metadata(bo->bo, &metadata);
 524 }
 525
 526 static struct radeon_winsys_cs_handle *amdgpu_get_cs_handle(struct pb_buffer *_buf)
 527 {
 528    /* return a direct pointer to amdgpu_winsys_bo. */
 529    return (struct radeon_winsys_cs_handle*)get_amdgpu_winsys_bo(_buf);
 530 }
 531
 532 static struct pb_buffer *
 533 amdgpu_bo_create(struct radeon_winsys *rws,
 534                  unsigned size,
 535                  unsigned alignment,
 536                  boolean use_reusable_pool,
 537                  enum radeon_bo_domain domain,
 538                  enum radeon_bo_flag flags)
 539 {
 540    struct amdgpu_winsys *ws = amdgpu_winsys(rws);
 541    struct amdgpu_bo_desc desc;
 542    struct pb_manager *provider;
 543    struct pb_buffer *buffer;
 544
 545    /* Don't use VRAM if the GPU doesn't have much. This is only the initial
 546     * domain. The kernel is free to move the buffer if it wants to.
 547     *
 548     * 64MB means no VRAM by todays standards.
 549     */
 550    if (domain & RADEON_DOMAIN_VRAM && ws->info.vram_size <= 64*1024*1024) {
 551       domain = RADEON_DOMAIN_GTT;
 552       flags = RADEON_FLAG_GTT_WC;
 553    }
 554
 555    memset(&desc, 0, sizeof(desc));
 556    desc.base.alignment = alignment;
 557
 558    /* Align size to page size. This is the minimum alignment for normal
 559     * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
 560     * like constant/uniform buffers, can benefit from better and more reuse.
 561     */
 562    size = align(size, ws->gart_page_size);
 563
 564    /* Only set one usage bit each for domains and flags, or the cache manager
 565     * might consider different sets of domains / flags compatible
 566     */
 567    if (domain == RADEON_DOMAIN_VRAM_GTT)
 568       desc.base.usage = 1 << 2;
 569    else
 570       desc.base.usage = domain >> 1;
 571    assert(flags < sizeof(desc.base.usage) * 8 - 3);
 572    desc.base.usage |= 1 << (flags + 3);
 573
 574    desc.initial_domain = domain;
 575    desc.flags = flags;
 576
 577    /* Assign a buffer manager. */
 578    if (use_reusable_pool)
 579       provider = ws->cman;
 580    else
 581       provider = ws->kman;
 582
 583    buffer = provider->create_buffer(provider, size, &desc.base);
 584    if (!buffer)
 585       return NULL;
 586
 587    return (struct pb_buffer*)buffer;
 588 }
 589
 590 static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
 591                                                struct winsys_handle *whandle,
 592                                                unsigned *stride)
 593 {
 594    struct amdgpu_winsys *ws = amdgpu_winsys(rws);
 595    struct amdgpu_winsys_bo *bo;
 596    enum amdgpu_bo_handle_type type;
 597    struct amdgpu_bo_import_result result = {0};
 598    uint64_t va;
 599    amdgpu_va_handle va_handle;
 600    struct amdgpu_bo_info info = {0};
 601    enum radeon_bo_domain initial = 0;
 602    int r;
 603
 604    /* Initialize the structure. */
 605    bo = CALLOC_STRUCT(amdgpu_winsys_bo);
 606    if (!bo) {
 607       return NULL;
 608    }
 609
 610    switch (whandle->type) {
 611    case DRM_API_HANDLE_TYPE_SHARED:
 612       type = amdgpu_bo_handle_type_gem_flink_name;
 613       break;
 614    case DRM_API_HANDLE_TYPE_FD:
 615       type = amdgpu_bo_handle_type_dma_buf_fd;
 616       break;
 617    default:
 618       return NULL;
 619    }
 620
 621    r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
 622    if (r)
 623       goto error;
 624
 625    /* Get initial domains. */
 626    r = amdgpu_bo_query_info(result.buf_handle, &info);
 627    if (r)
 628       goto error_query;
 629
 630    r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
 631                              result.alloc_size, 1 << 20, 0, &va, &va_handle, 0);
 632    if (r)
 633       goto error_query;
 634
 635    r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
 636    if (r)
 637       goto error_va_map;
 638
 639    if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
 640       initial |= RADEON_DOMAIN_VRAM;
 641    if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
 642       initial |= RADEON_DOMAIN_GTT;
 643
 644
 645    pipe_reference_init(&bo->base.reference, 1);
 646    bo->base.alignment = info.phys_alignment;
 647    bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
 648    bo->bo = result.buf_handle;
 649    bo->base.size = result.alloc_size;
 650    bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
 651    bo->rws = ws;
 652    bo->va = va;
 653    bo->va_handle = va_handle;
 654    bo->initial_domain = initial;
 655    bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
 656    bo->is_shared = true;
 657
 658    if (stride)
 659       *stride = whandle->stride;
 660
 661    if (bo->initial_domain & RADEON_DOMAIN_VRAM)
 662       ws->allocated_vram += align(bo->base.size, ws->gart_page_size);
 663    else if (bo->initial_domain & RADEON_DOMAIN_GTT)
 664       ws->allocated_gtt += align(bo->base.size, ws->gart_page_size);
 665
 666    return &bo->base;
 667
 668 error_va_map:
 669    amdgpu_va_range_free(va_handle);
 670
 671 error_query:
 672    amdgpu_bo_free(result.buf_handle);
 673
 674 error:
 675    FREE(bo);
 676    return NULL;
 677 }
 678
 679 static boolean amdgpu_bo_get_handle(struct pb_buffer *buffer,
 680                                     unsigned stride,
 681                                     struct winsys_handle *whandle)
 682 {
 683    struct amdgpu_winsys_bo *bo = get_amdgpu_winsys_bo(buffer);
 684    enum amdgpu_bo_handle_type type;
 685    int r;
 686
 687    if ((void*)bo != (void*)buffer)
 688       pb_cache_manager_remove_buffer(buffer);
 689
 690    switch (whandle->type) {
 691    case DRM_API_HANDLE_TYPE_SHARED:
 692       type = amdgpu_bo_handle_type_gem_flink_name;
 693       break;
 694    case DRM_API_HANDLE_TYPE_FD:
 695       type = amdgpu_bo_handle_type_dma_buf_fd;
 696       break;
 697    case DRM_API_HANDLE_TYPE_KMS:
 698       type = amdgpu_bo_handle_type_kms;
 699       break;
 700    default:
 701       return FALSE;
 702    }
 703
 704    r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
 705    if (r)
 706       return FALSE;
 707
 708    whandle->stride = stride;
 709    bo->is_shared = true;
 710    return TRUE;
 711 }
 712
 713 static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
 714                                             void *pointer, unsigned size)
 715 {
 716     struct amdgpu_winsys *ws = amdgpu_winsys(rws);
 717     amdgpu_bo_handle buf_handle;
 718     struct amdgpu_winsys_bo *bo;
 719     uint64_t va;
 720     amdgpu_va_handle va_handle;
 721
 722     bo = CALLOC_STRUCT(amdgpu_winsys_bo);
 723     if (!bo)
 724         return NULL;
 725
 726     if (amdgpu_create_bo_from_user_mem(ws->dev, pointer, size, &buf_handle))
 727         goto error;
 728
 729     if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
 730                               size, 1 << 12, 0, &va, &va_handle, 0))
 731         goto error_va_alloc;
 732
 733     if (amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP))
 734         goto error_va_map;
 735
 736     /* Initialize it. */
 737     pipe_reference_init(&bo->base.reference, 1);
 738     bo->bo = buf_handle;
 739     bo->base.alignment = 0;
 740     bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
 741     bo->base.size = size;
 742     bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
 743     bo->rws = ws;
 744     bo->user_ptr = pointer;
 745     bo->va = va;
 746     bo->va_handle = va_handle;
 747     bo->initial_domain = RADEON_DOMAIN_GTT;
 748     bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
 749
 750     ws->allocated_gtt += align(bo->base.size, ws->gart_page_size);
 751
 752     return (struct pb_buffer*)bo;
 753
 754 error_va_map:
 755     amdgpu_va_range_free(va_handle);
 756
 757 error_va_alloc:
 758     amdgpu_bo_free(buf_handle);
 759
 760 error:
 761     FREE(bo);
 762     return NULL;
 763 }
 764
 765 static uint64_t amdgpu_bo_get_va(struct radeon_winsys_cs_handle *buf)
 766 {
 767    return ((struct amdgpu_winsys_bo*)buf)->va;
 768 }
 769
 770 void amdgpu_bomgr_init_functions(struct amdgpu_winsys *ws)
 771 {
 772    ws->base.buffer_get_cs_handle = amdgpu_get_cs_handle;
 773    ws->base.buffer_set_tiling = amdgpu_bo_set_tiling;
 774    ws->base.buffer_get_tiling = amdgpu_bo_get_tiling;
 775    ws->base.buffer_map = amdgpu_bo_map;
 776    ws->base.buffer_unmap = amdgpu_bo_unmap;
 777    ws->base.buffer_wait = amdgpu_bo_wait;
 778    ws->base.buffer_create = amdgpu_bo_create;
 779    ws->base.buffer_from_handle = amdgpu_bo_from_handle;
 780    ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
 781    ws->base.buffer_get_handle = amdgpu_bo_get_handle;
 782    ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
 783    ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
 784 }