src/gallium/winsys/amdgpu/drm/amdgpu_cs.c

   1 /*
   2  * Copyright © 2008 Jérôme Glisse
   3  * Copyright © 2010 Marek Olšák <maraeo@gmail.com>
   4  * Copyright © 2015 Advanced Micro Devices, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining
   8  * a copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  17  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  19  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * The above copyright notice and this permission notice (including the
  25  * next paragraph) shall be included in all copies or substantial portions
  26  * of the Software.
  27  */
  28 /*
  29  * Authors:
  30  *      Marek Olšák <maraeo@gmail.com>
  31  */
  32
  33 #include "amdgpu_cs.h"
  34 #include "os/os_time.h"
  35 #include <stdio.h>
  36 #include <amdgpu_drm.h>
  37
  38 #include "../../../drivers/radeonsi/sid.h"
  39
  40 /* FENCES */
  41
  42 static struct pipe_fence_handle *
  43 amdgpu_fence_create(struct amdgpu_ctx *ctx, unsigned ip_type,
  44                     unsigned ip_instance, unsigned ring)
  45 {
  46    struct amdgpu_fence *fence = CALLOC_STRUCT(amdgpu_fence);
  47
  48    fence->reference.count = 1;
  49    fence->ctx = ctx;
  50    fence->fence.context = ctx->ctx;
  51    fence->fence.ip_type = ip_type;
  52    fence->fence.ip_instance = ip_instance;
  53    fence->fence.ring = ring;
  54    fence->submission_in_progress = true;
  55    p_atomic_inc(&ctx->refcount);
  56    return (struct pipe_fence_handle *)fence;
  57 }
  58
  59 static void amdgpu_fence_submitted(struct pipe_fence_handle *fence,
  60                                 struct amdgpu_cs_request* request,
  61                                 uint64_t *user_fence_cpu_address)
  62 {
  63    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  64
  65    rfence->fence.fence = request->seq_no;
  66    rfence->user_fence_cpu_address = user_fence_cpu_address;
  67    rfence->submission_in_progress = false;
  68 }
  69
  70 static void amdgpu_fence_signalled(struct pipe_fence_handle *fence)
  71 {
  72    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  73
  74    rfence->signalled = true;
  75    rfence->submission_in_progress = false;
  76 }
  77
  78 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
  79                        bool absolute)
  80 {
  81    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  82    uint32_t expired;
  83    int64_t abs_timeout;
  84    uint64_t *user_fence_cpu;
  85    int r;
  86
  87    if (rfence->signalled)
  88       return true;
  89
  90    if (absolute)
  91       abs_timeout = timeout;
  92    else
  93       abs_timeout = os_time_get_absolute_timeout(timeout);
  94
  95    /* The fence might not have a number assigned if its IB is being
  96     * submitted in the other thread right now. Wait until the submission
  97     * is done. */
  98    if (!os_wait_until_zero_abs_timeout(&rfence->submission_in_progress,
  99                                        abs_timeout))
 100       return false;
 101
 102    user_fence_cpu = rfence->user_fence_cpu_address;
 103    if (user_fence_cpu) {
 104       if (*user_fence_cpu >= rfence->fence.fence) {
 105          rfence->signalled = true;
 106          return true;
 107       }
 108
 109       /* No timeout, just query: no need for the ioctl. */
 110       if (!absolute && !timeout)
 111          return false;
 112    }
 113
 114    /* Now use the libdrm query. */
 115    r = amdgpu_cs_query_fence_status(&rfence->fence,
 116                                     abs_timeout,
 117                                     AMDGPU_QUERY_FENCE_TIMEOUT_IS_ABSOLUTE,
 118                                     &expired);
 119    if (r) {
 120       fprintf(stderr, "amdgpu: amdgpu_cs_query_fence_status failed.\n");
 121       return FALSE;
 122    }
 123
 124    if (expired) {
 125       /* This variable can only transition from false to true, so it doesn't
 126        * matter if threads race for it. */
 127       rfence->signalled = true;
 128       return true;
 129    }
 130    return false;
 131 }
 132
 133 static bool amdgpu_fence_wait_rel_timeout(struct radeon_winsys *rws,
 134                                           struct pipe_fence_handle *fence,
 135                                           uint64_t timeout)
 136 {
 137    return amdgpu_fence_wait(fence, timeout, false);
 138 }
 139
 140 /* CONTEXTS */
 141
 142 static struct radeon_winsys_ctx *amdgpu_ctx_create(struct radeon_winsys *ws)
 143 {
 144    struct amdgpu_ctx *ctx = CALLOC_STRUCT(amdgpu_ctx);
 145    int r;
 146    struct amdgpu_bo_alloc_request alloc_buffer = {};
 147    amdgpu_bo_handle buf_handle;
 148
 149    if (!ctx)
 150       return NULL;
 151
 152    ctx->ws = amdgpu_winsys(ws);
 153    ctx->refcount = 1;
 154
 155    r = amdgpu_cs_ctx_create(ctx->ws->dev, &ctx->ctx);
 156    if (r) {
 157       fprintf(stderr, "amdgpu: amdgpu_cs_ctx_create failed. (%i)\n", r);
 158       goto error_create;
 159    }
 160
 161    alloc_buffer.alloc_size = ctx->ws->info.gart_page_size;
 162    alloc_buffer.phys_alignment = ctx->ws->info.gart_page_size;
 163    alloc_buffer.preferred_heap = AMDGPU_GEM_DOMAIN_GTT;
 164
 165    r = amdgpu_bo_alloc(ctx->ws->dev, &alloc_buffer, &buf_handle);
 166    if (r) {
 167       fprintf(stderr, "amdgpu: amdgpu_bo_alloc failed. (%i)\n", r);
 168       goto error_user_fence_alloc;
 169    }
 170
 171    r = amdgpu_bo_cpu_map(buf_handle, (void**)&ctx->user_fence_cpu_address_base);
 172    if (r) {
 173       fprintf(stderr, "amdgpu: amdgpu_bo_cpu_map failed. (%i)\n", r);
 174       goto error_user_fence_map;
 175    }
 176
 177    memset(ctx->user_fence_cpu_address_base, 0, alloc_buffer.alloc_size);
 178    ctx->user_fence_bo = buf_handle;
 179
 180    return (struct radeon_winsys_ctx*)ctx;
 181
 182 error_user_fence_map:
 183    amdgpu_bo_free(buf_handle);
 184 error_user_fence_alloc:
 185    amdgpu_cs_ctx_free(ctx->ctx);
 186 error_create:
 187    FREE(ctx);
 188    return NULL;
 189 }
 190
 191 static void amdgpu_ctx_destroy(struct radeon_winsys_ctx *rwctx)
 192 {
 193    amdgpu_ctx_unref((struct amdgpu_ctx*)rwctx);
 194 }
 195
 196 static enum pipe_reset_status
 197 amdgpu_ctx_query_reset_status(struct radeon_winsys_ctx *rwctx)
 198 {
 199    struct amdgpu_ctx *ctx = (struct amdgpu_ctx*)rwctx;
 200    uint32_t result, hangs;
 201    int r;
 202
 203    r = amdgpu_cs_query_reset_state(ctx->ctx, &result, &hangs);
 204    if (r) {
 205       fprintf(stderr, "amdgpu: amdgpu_cs_query_reset_state failed. (%i)\n", r);
 206       return PIPE_NO_RESET;
 207    }
 208
 209    switch (result) {
 210    case AMDGPU_CTX_GUILTY_RESET:
 211       return PIPE_GUILTY_CONTEXT_RESET;
 212    case AMDGPU_CTX_INNOCENT_RESET:
 213       return PIPE_INNOCENT_CONTEXT_RESET;
 214    case AMDGPU_CTX_UNKNOWN_RESET:
 215       return PIPE_UNKNOWN_CONTEXT_RESET;
 216    case AMDGPU_CTX_NO_RESET:
 217    default:
 218       return PIPE_NO_RESET;
 219    }
 220 }
 221
 222 /* COMMAND SUBMISSION */
 223
 224 static bool amdgpu_cs_has_user_fence(struct amdgpu_cs_context *cs)
 225 {
 226    return cs->request.ip_type != AMDGPU_HW_IP_UVD &&
 227           cs->request.ip_type != AMDGPU_HW_IP_VCE;
 228 }
 229
 230 static bool amdgpu_cs_has_chaining(enum ring_type ring_type)
 231 {
 232    return ring_type == RING_GFX;
 233 }
 234
 235 static unsigned amdgpu_cs_epilog_dws(enum ring_type ring_type)
 236 {
 237    if (ring_type == RING_GFX)
 238       return 4; /* for chaining */
 239
 240    return 0;
 241 }
 242
 243 int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo)
 244 {
 245    unsigned hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 246    int i = cs->buffer_indices_hashlist[hash];
 247
 248    /* not found or found */
 249    if (i == -1 || cs->buffers[i].bo == bo)
 250       return i;
 251
 252    /* Hash collision, look for the BO in the list of buffers linearly. */
 253    for (i = cs->num_buffers - 1; i >= 0; i--) {
 254       if (cs->buffers[i].bo == bo) {
 255          /* Put this buffer in the hash list.
 256           * This will prevent additional hash collisions if there are
 257           * several consecutive lookup_buffer calls for the same buffer.
 258           *
 259           * Example: Assuming buffers A,B,C collide in the hash list,
 260           * the following sequence of buffers:
 261           *         AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
 262           * will collide here: ^ and here:   ^,
 263           * meaning that we should get very few collisions in the end. */
 264          cs->buffer_indices_hashlist[hash] = i;
 265          return i;
 266       }
 267    }
 268    return -1;
 269 }
 270
 271 static unsigned amdgpu_add_buffer(struct amdgpu_cs *acs,
 272                                  struct amdgpu_winsys_bo *bo,
 273                                  enum radeon_bo_usage usage,
 274                                  enum radeon_bo_domain domains,
 275                                  unsigned priority,
 276                                  enum radeon_bo_domain *added_domains)
 277 {
 278    struct amdgpu_cs_context *cs = acs->csc;
 279    struct amdgpu_cs_buffer *buffer;
 280    unsigned hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 281    int i = -1;
 282
 283    assert(priority < 64);
 284    *added_domains = 0;
 285
 286    i = amdgpu_lookup_buffer(cs, bo);
 287
 288    if (i >= 0) {
 289       buffer = &cs->buffers[i];
 290       buffer->priority_usage |= 1llu << priority;
 291       buffer->usage |= usage;
 292       *added_domains = domains & ~buffer->domains;
 293       buffer->domains |= domains;
 294       cs->flags[i] = MAX2(cs->flags[i], priority / 4);
 295       return i;
 296    }
 297
 298    /* New buffer, check if the backing array is large enough. */
 299    if (cs->num_buffers >= cs->max_num_buffers) {
 300       uint32_t size;
 301       cs->max_num_buffers += 10;
 302
 303       size = cs->max_num_buffers * sizeof(struct amdgpu_cs_buffer);
 304       cs->buffers = realloc(cs->buffers, size);
 305
 306       size = cs->max_num_buffers * sizeof(amdgpu_bo_handle);
 307       cs->handles = realloc(cs->handles, size);
 308
 309       cs->flags = realloc(cs->flags, cs->max_num_buffers);
 310    }
 311
 312    /* Initialize the new buffer. */
 313    cs->buffers[cs->num_buffers].bo = NULL;
 314    amdgpu_winsys_bo_reference(&cs->buffers[cs->num_buffers].bo, bo);
 315    cs->handles[cs->num_buffers] = bo->bo;
 316    cs->flags[cs->num_buffers] = priority / 4;
 317    p_atomic_inc(&bo->num_cs_references);
 318    buffer = &cs->buffers[cs->num_buffers];
 319    buffer->bo = bo;
 320    buffer->priority_usage = 1llu << priority;
 321    buffer->usage = usage;
 322    buffer->domains = domains;
 323
 324    cs->buffer_indices_hashlist[hash] = cs->num_buffers;
 325
 326    *added_domains = domains;
 327    return cs->num_buffers++;
 328 }
 329
 330 static unsigned amdgpu_cs_add_buffer(struct radeon_winsys_cs *rcs,
 331                                     struct pb_buffer *buf,
 332                                     enum radeon_bo_usage usage,
 333                                     enum radeon_bo_domain domains,
 334                                     enum radeon_bo_priority priority)
 335 {
 336    /* Don't use the "domains" parameter. Amdgpu doesn't support changing
 337     * the buffer placement during command submission.
 338     */
 339    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 340    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 341    enum radeon_bo_domain added_domains;
 342    unsigned index = amdgpu_add_buffer(cs, bo, usage, bo->initial_domain,
 343                                      priority, &added_domains);
 344
 345    if (added_domains & RADEON_DOMAIN_VRAM)
 346       cs->csc->used_vram += bo->base.size;
 347    else if (added_domains & RADEON_DOMAIN_GTT)
 348       cs->csc->used_gart += bo->base.size;
 349
 350    return index;
 351 }
 352
 353 static bool amdgpu_ib_new_buffer(struct amdgpu_winsys *ws, struct amdgpu_ib *ib)
 354 {
 355    struct pb_buffer *pb;
 356    uint8_t *mapped;
 357    unsigned buffer_size;
 358
 359    /* Always create a buffer that is at least as large as the maximum seen IB
 360     * size, aligned to a power of two (and multiplied by 4 to reduce internal
 361     * fragmentation if chaining is not available). Limit to 512k dwords, which
 362     * is the largest power of two that fits into the size field of the
 363     * INDIRECT_BUFFER packet.
 364     */
 365    if (amdgpu_cs_has_chaining(amdgpu_cs_from_ib(ib)->ring_type))
 366       buffer_size = 4 *util_next_power_of_two(ib->max_ib_size);
 367    else
 368       buffer_size = 4 *util_next_power_of_two(4 * ib->max_ib_size);
 369
 370    buffer_size = MIN2(buffer_size, 4 * 512 * 1024);
 371
 372    switch (ib->ib_type) {
 373    case IB_CONST_PREAMBLE:
 374       buffer_size = MAX2(buffer_size, 4 * 1024);
 375       break;
 376    case IB_CONST:
 377       buffer_size = MAX2(buffer_size, 16 * 1024 * 4);
 378       break;
 379    case IB_MAIN:
 380       buffer_size = MAX2(buffer_size, 8 * 1024 * 4);
 381       break;
 382    default:
 383       unreachable("unhandled IB type");
 384    }
 385
 386    pb = ws->base.buffer_create(&ws->base, buffer_size,
 387                                ws->info.gart_page_size,
 388                                RADEON_DOMAIN_GTT,
 389                                RADEON_FLAG_CPU_ACCESS);
 390    if (!pb)
 391       return false;
 392
 393    mapped = ws->base.buffer_map(pb, NULL, PIPE_TRANSFER_WRITE);
 394    if (!mapped) {
 395       pb_reference(&pb, NULL);
 396       return false;
 397    }
 398
 399    pb_reference(&ib->big_ib_buffer, pb);
 400    pb_reference(&pb, NULL);
 401
 402    ib->ib_mapped = mapped;
 403    ib->used_ib_space = 0;
 404
 405    return true;
 406 }
 407
 408 static unsigned amdgpu_ib_max_submit_dwords(enum ib_type ib_type)
 409 {
 410    switch (ib_type) {
 411    case IB_MAIN:
 412       /* Smaller submits means the GPU gets busy sooner and there is less
 413        * waiting for buffers and fences. Proof:
 414        *   http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
 415        */
 416       return 20 * 1024;
 417    case IB_CONST_PREAMBLE:
 418    case IB_CONST:
 419       /* There isn't really any reason to limit CE IB size beyond the natural
 420        * limit implied by the main IB, except perhaps GTT size. Just return
 421        * an extremely large value that we never get anywhere close to.
 422        */
 423       return 16 * 1024 * 1024;
 424    default:
 425       unreachable("bad ib_type");
 426    }
 427 }
 428
 429 static bool amdgpu_get_new_ib(struct radeon_winsys *ws, struct amdgpu_cs *cs,
 430                               enum ib_type ib_type)
 431 {
 432    struct amdgpu_winsys *aws = (struct amdgpu_winsys*)ws;
 433    /* Small IBs are better than big IBs, because the GPU goes idle quicker
 434     * and there is less waiting for buffers and fences. Proof:
 435     *   http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
 436     */
 437    struct amdgpu_ib *ib = NULL;
 438    struct amdgpu_cs_ib_info *info = &cs->csc->ib[ib_type];
 439    unsigned ib_size = 0;
 440
 441    switch (ib_type) {
 442    case IB_CONST_PREAMBLE:
 443       ib = &cs->const_preamble_ib;
 444       ib_size = 256 * 4;
 445       break;
 446    case IB_CONST:
 447       ib = &cs->const_ib;
 448       ib_size = 8 * 1024 * 4;
 449       break;
 450    case IB_MAIN:
 451       ib = &cs->main;
 452       ib_size = 4 * 1024 * 4;
 453       break;
 454    default:
 455       unreachable("unhandled IB type");
 456    }
 457
 458    if (!amdgpu_cs_has_chaining(cs->ring_type)) {
 459       ib_size = MAX2(ib_size,
 460                      4 * MIN2(util_next_power_of_two(ib->max_ib_size),
 461                               amdgpu_ib_max_submit_dwords(ib_type)));
 462    }
 463
 464    ib->max_ib_size = ib->max_ib_size - ib->max_ib_size / 32;
 465
 466    ib->base.prev_dw = 0;
 467    ib->base.num_prev = 0;
 468    ib->base.current.cdw = 0;
 469    ib->base.current.buf = NULL;
 470
 471    /* Allocate a new buffer for IBs if the current buffer is all used. */
 472    if (!ib->big_ib_buffer ||
 473        ib->used_ib_space + ib_size > ib->big_ib_buffer->size) {
 474       if (!amdgpu_ib_new_buffer(aws, ib))
 475          return false;
 476    }
 477
 478    info->ib_mc_address = amdgpu_winsys_bo(ib->big_ib_buffer)->va +
 479                          ib->used_ib_space;
 480    info->size = 0;
 481    ib->ptr_ib_size = &info->size;
 482
 483    amdgpu_cs_add_buffer(&cs->main.base, ib->big_ib_buffer,
 484                         RADEON_USAGE_READ, 0, RADEON_PRIO_IB1);
 485
 486    ib->base.current.buf = (uint32_t*)(ib->ib_mapped + ib->used_ib_space);
 487
 488    ib_size = ib->big_ib_buffer->size - ib->used_ib_space;
 489    ib->base.current.max_dw = ib_size / 4 - amdgpu_cs_epilog_dws(cs->ring_type);
 490    return true;
 491 }
 492
 493 static void amdgpu_ib_finalize(struct amdgpu_ib *ib)
 494 {
 495    *ib->ptr_ib_size |= ib->base.current.cdw;
 496    ib->used_ib_space += ib->base.current.cdw * 4;
 497    ib->max_ib_size = MAX2(ib->max_ib_size, ib->base.prev_dw + ib->base.current.cdw);
 498 }
 499
 500 static boolean amdgpu_init_cs_context(struct amdgpu_cs_context *cs,
 501                                       enum ring_type ring_type)
 502 {
 503    int i;
 504
 505    switch (ring_type) {
 506    case RING_DMA:
 507       cs->request.ip_type = AMDGPU_HW_IP_DMA;
 508       break;
 509
 510    case RING_UVD:
 511       cs->request.ip_type = AMDGPU_HW_IP_UVD;
 512       break;
 513
 514    case RING_VCE:
 515       cs->request.ip_type = AMDGPU_HW_IP_VCE;
 516       break;
 517
 518    case RING_COMPUTE:
 519       cs->request.ip_type = AMDGPU_HW_IP_COMPUTE;
 520       break;
 521
 522    default:
 523    case RING_GFX:
 524       cs->request.ip_type = AMDGPU_HW_IP_GFX;
 525       break;
 526    }
 527
 528    cs->max_num_buffers = 512;
 529    cs->buffers = (struct amdgpu_cs_buffer*)
 530                   CALLOC(1, cs->max_num_buffers * sizeof(struct amdgpu_cs_buffer));
 531    if (!cs->buffers) {
 532       return FALSE;
 533    }
 534
 535    cs->handles = CALLOC(1, cs->max_num_buffers * sizeof(amdgpu_bo_handle));
 536    if (!cs->handles) {
 537       FREE(cs->buffers);
 538       return FALSE;
 539    }
 540
 541    cs->flags = CALLOC(1, cs->max_num_buffers);
 542    if (!cs->flags) {
 543       FREE(cs->handles);
 544       FREE(cs->buffers);
 545       return FALSE;
 546    }
 547
 548    for (i = 0; i < ARRAY_SIZE(cs->buffer_indices_hashlist); i++) {
 549       cs->buffer_indices_hashlist[i] = -1;
 550    }
 551
 552    cs->request.number_of_ibs = 1;
 553    cs->request.ibs = &cs->ib[IB_MAIN];
 554
 555    cs->ib[IB_CONST].flags = AMDGPU_IB_FLAG_CE;
 556    cs->ib[IB_CONST_PREAMBLE].flags = AMDGPU_IB_FLAG_CE |
 557                                      AMDGPU_IB_FLAG_PREAMBLE;
 558
 559    return TRUE;
 560 }
 561
 562 static void amdgpu_cs_context_cleanup(struct amdgpu_cs_context *cs)
 563 {
 564    unsigned i;
 565
 566    for (i = 0; i < cs->num_buffers; i++) {
 567       p_atomic_dec(&cs->buffers[i].bo->num_cs_references);
 568       amdgpu_winsys_bo_reference(&cs->buffers[i].bo, NULL);
 569       cs->handles[i] = NULL;
 570       cs->flags[i] = 0;
 571    }
 572
 573    cs->num_buffers = 0;
 574    cs->used_gart = 0;
 575    cs->used_vram = 0;
 576    amdgpu_fence_reference(&cs->fence, NULL);
 577
 578    for (i = 0; i < ARRAY_SIZE(cs->buffer_indices_hashlist); i++) {
 579       cs->buffer_indices_hashlist[i] = -1;
 580    }
 581 }
 582
 583 static void amdgpu_destroy_cs_context(struct amdgpu_cs_context *cs)
 584 {
 585    amdgpu_cs_context_cleanup(cs);
 586    FREE(cs->flags);
 587    FREE(cs->buffers);
 588    FREE(cs->handles);
 589    FREE(cs->request.dependencies);
 590 }
 591
 592
 593 static struct radeon_winsys_cs *
 594 amdgpu_cs_create(struct radeon_winsys_ctx *rwctx,
 595                  enum ring_type ring_type,
 596                  void (*flush)(void *ctx, unsigned flags,
 597                                struct pipe_fence_handle **fence),
 598                  void *flush_ctx)
 599 {
 600    struct amdgpu_ctx *ctx = (struct amdgpu_ctx*)rwctx;
 601    struct amdgpu_cs *cs;
 602
 603    cs = CALLOC_STRUCT(amdgpu_cs);
 604    if (!cs) {
 605       return NULL;
 606    }
 607
 608    pipe_semaphore_init(&cs->flush_completed, 1);
 609
 610    cs->ctx = ctx;
 611    cs->flush_cs = flush;
 612    cs->flush_data = flush_ctx;
 613    cs->ring_type = ring_type;
 614
 615    cs->main.ib_type = IB_MAIN;
 616    cs->const_ib.ib_type = IB_CONST;
 617    cs->const_preamble_ib.ib_type = IB_CONST_PREAMBLE;
 618
 619    if (!amdgpu_init_cs_context(&cs->csc1, ring_type)) {
 620       FREE(cs);
 621       return NULL;
 622    }
 623
 624    if (!amdgpu_init_cs_context(&cs->csc2, ring_type)) {
 625       amdgpu_destroy_cs_context(&cs->csc1);
 626       FREE(cs);
 627       return NULL;
 628    }
 629
 630    /* Set the first submission context as current. */
 631    cs->csc = &cs->csc1;
 632    cs->cst = &cs->csc2;
 633
 634    if (!amdgpu_get_new_ib(&ctx->ws->base, cs, IB_MAIN)) {
 635       amdgpu_destroy_cs_context(&cs->csc2);
 636       amdgpu_destroy_cs_context(&cs->csc1);
 637       FREE(cs);
 638       return NULL;
 639    }
 640
 641    p_atomic_inc(&ctx->ws->num_cs);
 642    return &cs->main.base;
 643 }
 644
 645 static struct radeon_winsys_cs *
 646 amdgpu_cs_add_const_ib(struct radeon_winsys_cs *rcs)
 647 {
 648    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 649    struct amdgpu_winsys *ws = cs->ctx->ws;
 650
 651    /* only one const IB can be added */
 652    if (cs->ring_type != RING_GFX || cs->const_ib.ib_mapped)
 653       return NULL;
 654
 655    if (!amdgpu_get_new_ib(&ws->base, cs, IB_CONST))
 656       return NULL;
 657
 658    cs->csc->request.number_of_ibs = 2;
 659    cs->csc->request.ibs = &cs->csc->ib[IB_CONST];
 660
 661    cs->cst->request.number_of_ibs = 2;
 662    cs->cst->request.ibs = &cs->cst->ib[IB_CONST];
 663
 664    return &cs->const_ib.base;
 665 }
 666
 667 static struct radeon_winsys_cs *
 668 amdgpu_cs_add_const_preamble_ib(struct radeon_winsys_cs *rcs)
 669 {
 670    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 671    struct amdgpu_winsys *ws = cs->ctx->ws;
 672
 673    /* only one const preamble IB can be added and only when the const IB has
 674     * also been mapped */
 675    if (cs->ring_type != RING_GFX || !cs->const_ib.ib_mapped ||
 676        cs->const_preamble_ib.ib_mapped)
 677       return NULL;
 678
 679    if (!amdgpu_get_new_ib(&ws->base, cs, IB_CONST_PREAMBLE))
 680       return NULL;
 681
 682    cs->csc->request.number_of_ibs = 3;
 683    cs->csc->request.ibs = &cs->csc->ib[IB_CONST_PREAMBLE];
 684
 685    cs->cst->request.number_of_ibs = 3;
 686    cs->cst->request.ibs = &cs->cst->ib[IB_CONST_PREAMBLE];
 687
 688    return &cs->const_preamble_ib.base;
 689 }
 690
 691 #define OUT_CS(cs, value) (cs)->current.buf[(cs)->current.cdw++] = (value)
 692
 693 static int amdgpu_cs_lookup_buffer(struct radeon_winsys_cs *rcs,
 694                                struct pb_buffer *buf)
 695 {
 696    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 697
 698    return amdgpu_lookup_buffer(cs->csc, (struct amdgpu_winsys_bo*)buf);
 699 }
 700
 701 static boolean amdgpu_cs_validate(struct radeon_winsys_cs *rcs)
 702 {
 703    return TRUE;
 704 }
 705
 706 static bool amdgpu_cs_check_space(struct radeon_winsys_cs *rcs, unsigned dw)
 707 {
 708    struct amdgpu_ib *ib = amdgpu_ib(rcs);
 709    struct amdgpu_cs *cs = amdgpu_cs_from_ib(ib);
 710    unsigned requested_size = rcs->prev_dw + rcs->current.cdw + dw;
 711    uint64_t va;
 712    uint32_t *new_ptr_ib_size;
 713
 714    assert(rcs->current.cdw <= rcs->current.max_dw);
 715
 716    if (requested_size > amdgpu_ib_max_submit_dwords(ib->ib_type))
 717       return false;
 718
 719    ib->max_ib_size = MAX2(ib->max_ib_size, requested_size);
 720
 721    if (rcs->current.max_dw - rcs->current.cdw >= dw)
 722       return true;
 723
 724    if (!amdgpu_cs_has_chaining(cs->ring_type))
 725       return false;
 726
 727    /* Allocate a new chunk */
 728    if (rcs->num_prev >= rcs->max_prev) {
 729       unsigned new_max_prev = MAX2(1, 2 * rcs->max_prev);
 730       struct radeon_winsys_cs_chunk *new_prev;
 731
 732       new_prev = REALLOC(rcs->prev,
 733                          sizeof(*new_prev) * rcs->max_prev,
 734                          sizeof(*new_prev) * new_max_prev);
 735       if (!new_prev)
 736          return false;
 737
 738       rcs->prev = new_prev;
 739       rcs->max_prev = new_max_prev;
 740    }
 741
 742    if (!amdgpu_ib_new_buffer(cs->ctx->ws, ib))
 743       return false;
 744
 745    assert(ib->used_ib_space == 0);
 746    va = amdgpu_winsys_bo(ib->big_ib_buffer)->va;
 747
 748    /* This space was originally reserved. */
 749    rcs->current.max_dw += 4;
 750    assert(ib->used_ib_space + 4 * rcs->current.max_dw <= ib->big_ib_buffer->size);
 751
 752    /* Pad with NOPs and add INDIRECT_BUFFER packet */
 753    while ((rcs->current.cdw & 7) != 4)
 754       OUT_CS(rcs, 0xffff1000); /* type3 nop packet */
 755
 756    OUT_CS(rcs, PKT3(ib->ib_type == IB_MAIN ? PKT3_INDIRECT_BUFFER_CIK
 757                                            : PKT3_INDIRECT_BUFFER_CONST, 2, 0));
 758    OUT_CS(rcs, va);
 759    OUT_CS(rcs, va >> 32);
 760    new_ptr_ib_size = &rcs->current.buf[rcs->current.cdw];
 761    OUT_CS(rcs, S_3F2_CHAIN(1) | S_3F2_VALID(1));
 762
 763    assert((rcs->current.cdw & 7) == 0);
 764    assert(rcs->current.cdw <= rcs->current.max_dw);
 765
 766    *ib->ptr_ib_size |= rcs->current.cdw;
 767    ib->ptr_ib_size = new_ptr_ib_size;
 768
 769    /* Hook up the new chunk */
 770    rcs->prev[rcs->num_prev].buf = rcs->current.buf;
 771    rcs->prev[rcs->num_prev].cdw = rcs->current.cdw;
 772    rcs->prev[rcs->num_prev].max_dw = rcs->current.cdw; /* no modifications */
 773    rcs->num_prev++;
 774
 775    ib->base.prev_dw += ib->base.current.cdw;
 776    ib->base.current.cdw = 0;
 777
 778    ib->base.current.buf = (uint32_t*)(ib->ib_mapped + ib->used_ib_space);
 779    ib->base.current.max_dw = ib->big_ib_buffer->size / 4 - amdgpu_cs_epilog_dws(cs->ring_type);
 780
 781    amdgpu_cs_add_buffer(&cs->main.base, ib->big_ib_buffer,
 782                         RADEON_USAGE_READ, 0, RADEON_PRIO_IB1);
 783
 784    return true;
 785 }
 786
 787 static boolean amdgpu_cs_memory_below_limit(struct radeon_winsys_cs *rcs, uint64_t vram, uint64_t gtt)
 788 {
 789    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 790    struct amdgpu_winsys *ws = cs->ctx->ws;
 791
 792    vram += cs->csc->used_vram;
 793    gtt += cs->csc->used_gart;
 794
 795    /* Anything that goes above the VRAM size should go to GTT. */
 796    if (vram > ws->info.vram_size)
 797        gtt += vram - ws->info.vram_size;
 798
 799    /* Now we just need to check if we have enough GTT. */
 800    return gtt < ws->info.gart_size * 0.7;
 801 }
 802
 803 static uint64_t amdgpu_cs_query_memory_usage(struct radeon_winsys_cs *rcs)
 804 {
 805    struct amdgpu_cs_context *cs = amdgpu_cs(rcs)->csc;
 806
 807    return cs->used_vram + cs->used_gart;
 808 }
 809
 810 static unsigned amdgpu_cs_get_buffer_list(struct radeon_winsys_cs *rcs,
 811                                           struct radeon_bo_list_item *list)
 812 {
 813     struct amdgpu_cs_context *cs = amdgpu_cs(rcs)->csc;
 814     int i;
 815
 816     if (list) {
 817         for (i = 0; i < cs->num_buffers; i++) {
 818             pb_reference(&list[i].buf, &cs->buffers[i].bo->base);
 819             list[i].vm_address = cs->buffers[i].bo->va;
 820             list[i].priority_usage = cs->buffers[i].priority_usage;
 821         }
 822     }
 823     return cs->num_buffers;
 824 }
 825
 826 DEBUG_GET_ONCE_BOOL_OPTION(all_bos, "RADEON_ALL_BOS", FALSE)
 827
 828 /* Since the kernel driver doesn't synchronize execution between different
 829  * rings automatically, we have to add fence dependencies manually.
 830  */
 831 static void amdgpu_add_fence_dependencies(struct amdgpu_cs *acs)
 832 {
 833    struct amdgpu_cs_context *cs = acs->csc;
 834    int i, j;
 835
 836    cs->request.number_of_dependencies = 0;
 837
 838    for (i = 0; i < cs->num_buffers; i++) {
 839       for (j = 0; j < RING_LAST; j++) {
 840          struct amdgpu_cs_fence *dep;
 841          unsigned idx;
 842
 843          struct amdgpu_fence *bo_fence = (void *)cs->buffers[i].bo->fence[j];
 844          if (!bo_fence)
 845             continue;
 846
 847          if (bo_fence->ctx == acs->ctx &&
 848              bo_fence->fence.ip_type == cs->request.ip_type &&
 849              bo_fence->fence.ip_instance == cs->request.ip_instance &&
 850              bo_fence->fence.ring == cs->request.ring)
 851             continue;
 852
 853          if (amdgpu_fence_wait((void *)bo_fence, 0, false))
 854             continue;
 855
 856          if (bo_fence->submission_in_progress)
 857             os_wait_until_zero(&bo_fence->submission_in_progress,
 858                                PIPE_TIMEOUT_INFINITE);
 859
 860          idx = cs->request.number_of_dependencies++;
 861          if (idx >= cs->max_dependencies) {
 862             unsigned size;
 863
 864             cs->max_dependencies = idx + 8;
 865             size = cs->max_dependencies * sizeof(struct amdgpu_cs_fence);
 866             cs->request.dependencies = realloc(cs->request.dependencies, size);
 867          }
 868
 869          dep = &cs->request.dependencies[idx];
 870          memcpy(dep, &bo_fence->fence, sizeof(*dep));
 871       }
 872    }
 873 }
 874
 875 void amdgpu_cs_submit_ib(struct amdgpu_cs *acs)
 876 {
 877    struct amdgpu_winsys *ws = acs->ctx->ws;
 878    struct amdgpu_cs_context *cs = acs->cst;
 879    int i, r;
 880
 881    cs->request.fence_info.handle = NULL;
 882    if (amdgpu_cs_has_user_fence(cs)) {
 883         cs->request.fence_info.handle = acs->ctx->user_fence_bo;
 884         cs->request.fence_info.offset = acs->ring_type;
 885    }
 886
 887    /* Create the buffer list.
 888     * Use a buffer list containing all allocated buffers if requested.
 889     */
 890    if (debug_get_option_all_bos()) {
 891       struct amdgpu_winsys_bo *bo;
 892       amdgpu_bo_handle *handles;
 893       unsigned num = 0;
 894
 895       pipe_mutex_lock(ws->global_bo_list_lock);
 896
 897       handles = malloc(sizeof(handles[0]) * ws->num_buffers);
 898       if (!handles) {
 899          pipe_mutex_unlock(ws->global_bo_list_lock);
 900          amdgpu_cs_context_cleanup(cs);
 901          return;
 902       }
 903
 904       LIST_FOR_EACH_ENTRY(bo, &ws->global_bo_list, global_list_item) {
 905          assert(num < ws->num_buffers);
 906          handles[num++] = bo->bo;
 907       }
 908
 909       r = amdgpu_bo_list_create(ws->dev, ws->num_buffers,
 910                                 handles, NULL,
 911                                 &cs->request.resources);
 912       free(handles);
 913       pipe_mutex_unlock(ws->global_bo_list_lock);
 914    } else {
 915       r = amdgpu_bo_list_create(ws->dev, cs->num_buffers,
 916                                 cs->handles, cs->flags,
 917                                 &cs->request.resources);
 918    }
 919
 920    if (r) {
 921       fprintf(stderr, "amdgpu: buffer list creation failed (%d)\n", r);
 922       cs->request.resources = NULL;
 923       amdgpu_fence_signalled(cs->fence);
 924       goto cleanup;
 925    }
 926
 927    r = amdgpu_cs_submit(acs->ctx->ctx, 0, &cs->request, 1);
 928    if (r) {
 929       if (r == -ENOMEM)
 930          fprintf(stderr, "amdgpu: Not enough memory for command submission.\n");
 931       else
 932          fprintf(stderr, "amdgpu: The CS has been rejected, "
 933                  "see dmesg for more information.\n");
 934
 935       amdgpu_fence_signalled(cs->fence);
 936    } else {
 937       /* Success. */
 938       uint64_t *user_fence = NULL;
 939       if (amdgpu_cs_has_user_fence(cs))
 940          user_fence = acs->ctx->user_fence_cpu_address_base +
 941                       cs->request.fence_info.offset;
 942       amdgpu_fence_submitted(cs->fence, &cs->request, user_fence);
 943    }
 944
 945    /* Cleanup. */
 946    if (cs->request.resources)
 947       amdgpu_bo_list_destroy(cs->request.resources);
 948
 949 cleanup:
 950    for (i = 0; i < cs->num_buffers; i++)
 951       p_atomic_dec(&cs->buffers[i].bo->num_active_ioctls);
 952
 953    amdgpu_cs_context_cleanup(cs);
 954 }
 955
 956 /* Make sure the previous submission is completed. */
 957 void amdgpu_cs_sync_flush(struct radeon_winsys_cs *rcs)
 958 {
 959    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 960
 961    /* Wait for any pending ioctl of this CS to complete. */
 962    if (cs->ctx->ws->thread) {
 963       /* wait and set the semaphore to "busy" */
 964       pipe_semaphore_wait(&cs->flush_completed);
 965       /* set the semaphore to "idle" */
 966       pipe_semaphore_signal(&cs->flush_completed);
 967    }
 968 }
 969
 970 DEBUG_GET_ONCE_BOOL_OPTION(noop, "RADEON_NOOP", FALSE)
 971
 972 static void amdgpu_cs_flush(struct radeon_winsys_cs *rcs,
 973                             unsigned flags,
 974                             struct pipe_fence_handle **fence)
 975 {
 976    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 977    struct amdgpu_winsys *ws = cs->ctx->ws;
 978
 979    rcs->current.max_dw += amdgpu_cs_epilog_dws(cs->ring_type);
 980
 981    switch (cs->ring_type) {
 982    case RING_DMA:
 983       /* pad DMA ring to 8 DWs */
 984       while (rcs->current.cdw & 7)
 985          OUT_CS(rcs, 0x00000000); /* NOP packet */
 986       break;
 987    case RING_GFX:
 988       /* pad GFX ring to 8 DWs to meet CP fetch alignment requirements */
 989       while (rcs->current.cdw & 7)
 990          OUT_CS(rcs, 0xffff1000); /* type3 nop packet */
 991
 992       /* Also pad the const IB. */
 993       if (cs->const_ib.ib_mapped)
 994          while (!cs->const_ib.base.current.cdw || (cs->const_ib.base.current.cdw & 7))
 995             OUT_CS(&cs->const_ib.base, 0xffff1000); /* type3 nop packet */
 996
 997       if (cs->const_preamble_ib.ib_mapped)
 998          while (!cs->const_preamble_ib.base.current.cdw || (cs->const_preamble_ib.base.current.cdw & 7))
 999             OUT_CS(&cs->const_preamble_ib.base, 0xffff1000);
1000       break;
1001    case RING_UVD:
1002       while (rcs->current.cdw & 15)
1003          OUT_CS(rcs, 0x80000000); /* type2 nop packet */
1004       break;
1005    default:
1006       break;
1007    }
1008
1009    if (rcs->current.cdw > rcs->current.max_dw) {
1010       fprintf(stderr, "amdgpu: command stream overflowed\n");
1011    }
1012
1013    /* If the CS is not empty or overflowed.... */
1014    if (radeon_emitted(&cs->main.base, 0) &&
1015        cs->main.base.current.cdw <= cs->main.base.current.max_dw &&
1016        !debug_get_option_noop()) {
1017       struct amdgpu_cs_context *cur = cs->csc;
1018       unsigned i, num_buffers = cur->num_buffers;
1019
1020       /* Set IB sizes. */
1021       amdgpu_ib_finalize(&cs->main);
1022
1023       if (cs->const_ib.ib_mapped)
1024          amdgpu_ib_finalize(&cs->const_ib);
1025
1026       if (cs->const_preamble_ib.ib_mapped)
1027          amdgpu_ib_finalize(&cs->const_preamble_ib);
1028
1029       /* Create a fence. */
1030       amdgpu_fence_reference(&cur->fence, NULL);
1031       cur->fence = amdgpu_fence_create(cs->ctx,
1032                                            cur->request.ip_type,
1033                                            cur->request.ip_instance,
1034                                            cur->request.ring);
1035       if (fence)
1036          amdgpu_fence_reference(fence, cur->fence);
1037
1038       /* Prepare buffers. */
1039       pipe_mutex_lock(ws->bo_fence_lock);
1040       amdgpu_add_fence_dependencies(cs);
1041       for (i = 0; i < num_buffers; i++) {
1042          p_atomic_inc(&cur->buffers[i].bo->num_active_ioctls);
1043          amdgpu_fence_reference(&cur->buffers[i].bo->fence[cs->ring_type],
1044                                 cur->fence);
1045       }
1046       pipe_mutex_unlock(ws->bo_fence_lock);
1047
1048       amdgpu_cs_sync_flush(rcs);
1049
1050       /* Swap command streams. "cst" is going to be submitted. */
1051       cs->csc = cs->cst;
1052       cs->cst = cur;
1053
1054       /* Submit. */
1055       if (ws->thread && (flags & RADEON_FLUSH_ASYNC)) {
1056          /* Set the semaphore to "busy". */
1057          pipe_semaphore_wait(&cs->flush_completed);
1058          amdgpu_ws_queue_cs(ws, cs);
1059       } else {
1060          amdgpu_cs_submit_ib(cs);
1061       }
1062    } else {
1063       amdgpu_cs_context_cleanup(cs->csc);
1064    }
1065
1066    amdgpu_get_new_ib(&ws->base, cs, IB_MAIN);
1067    if (cs->const_ib.ib_mapped)
1068       amdgpu_get_new_ib(&ws->base, cs, IB_CONST);
1069    if (cs->const_preamble_ib.ib_mapped)
1070       amdgpu_get_new_ib(&ws->base, cs, IB_CONST_PREAMBLE);
1071
1072    ws->num_cs_flushes++;
1073 }
1074
1075 static void amdgpu_cs_destroy(struct radeon_winsys_cs *rcs)
1076 {
1077    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1078
1079    amdgpu_cs_sync_flush(rcs);
1080    pipe_semaphore_destroy(&cs->flush_completed);
1081    p_atomic_dec(&cs->ctx->ws->num_cs);
1082    pb_reference(&cs->main.big_ib_buffer, NULL);
1083    FREE(cs->main.base.prev);
1084    pb_reference(&cs->const_ib.big_ib_buffer, NULL);
1085    FREE(cs->const_ib.base.prev);
1086    pb_reference(&cs->const_preamble_ib.big_ib_buffer, NULL);
1087    FREE(cs->const_preamble_ib.base.prev);
1088    amdgpu_destroy_cs_context(&cs->csc1);
1089    amdgpu_destroy_cs_context(&cs->csc2);
1090    FREE(cs);
1091 }
1092
1093 static boolean amdgpu_bo_is_referenced(struct radeon_winsys_cs *rcs,
1094                                        struct pb_buffer *_buf,
1095                                        enum radeon_bo_usage usage)
1096 {
1097    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1098    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)_buf;
1099
1100    return amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo, usage);
1101 }
1102
1103 void amdgpu_cs_init_functions(struct amdgpu_winsys *ws)
1104 {
1105    ws->base.ctx_create = amdgpu_ctx_create;
1106    ws->base.ctx_destroy = amdgpu_ctx_destroy;
1107    ws->base.ctx_query_reset_status = amdgpu_ctx_query_reset_status;
1108    ws->base.cs_create = amdgpu_cs_create;
1109    ws->base.cs_add_const_ib = amdgpu_cs_add_const_ib;
1110    ws->base.cs_add_const_preamble_ib = amdgpu_cs_add_const_preamble_ib;
1111    ws->base.cs_destroy = amdgpu_cs_destroy;
1112    ws->base.cs_add_buffer = amdgpu_cs_add_buffer;
1113    ws->base.cs_lookup_buffer = amdgpu_cs_lookup_buffer;
1114    ws->base.cs_validate = amdgpu_cs_validate;
1115    ws->base.cs_check_space = amdgpu_cs_check_space;
1116    ws->base.cs_memory_below_limit = amdgpu_cs_memory_below_limit;
1117    ws->base.cs_query_memory_usage = amdgpu_cs_query_memory_usage;
1118    ws->base.cs_get_buffer_list = amdgpu_cs_get_buffer_list;
1119    ws->base.cs_flush = amdgpu_cs_flush;
1120    ws->base.cs_is_buffer_referenced = amdgpu_bo_is_referenced;
1121    ws->base.cs_sync_flush = amdgpu_cs_sync_flush;
1122    ws->base.fence_wait = amdgpu_fence_wait_rel_timeout;
1123    ws->base.fence_reference = amdgpu_fence_reference;
1124 }