src/gallium/winsys/amdgpu/drm/amdgpu_cs.c

   1 /*
   2  * Copyright © 2008 Jérôme Glisse
   3  * Copyright © 2010 Marek Olšák <maraeo@gmail.com>
   4  * Copyright © 2015 Advanced Micro Devices, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining
   8  * a copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  17  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  19  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * The above copyright notice and this permission notice (including the
  25  * next paragraph) shall be included in all copies or substantial portions
  26  * of the Software.
  27  */
  28 /*
  29  * Authors:
  30  *      Marek Olšák <maraeo@gmail.com>
  31  */
  32
  33 #include "amdgpu_cs.h"
  34 #include "os/os_time.h"
  35 #include <stdio.h>
  36 #include <amdgpu_drm.h>
  37
  38 #include "amd/common/sid.h"
  39
  40 /* FENCES */
  41
  42 static struct pipe_fence_handle *
  43 amdgpu_fence_create(struct amdgpu_ctx *ctx, unsigned ip_type,
  44                     unsigned ip_instance, unsigned ring)
  45 {
  46    struct amdgpu_fence *fence = CALLOC_STRUCT(amdgpu_fence);
  47
  48    fence->reference.count = 1;
  49    fence->ctx = ctx;
  50    fence->fence.context = ctx->ctx;
  51    fence->fence.ip_type = ip_type;
  52    fence->fence.ip_instance = ip_instance;
  53    fence->fence.ring = ring;
  54    fence->submission_in_progress = true;
  55    p_atomic_inc(&ctx->refcount);
  56    return (struct pipe_fence_handle *)fence;
  57 }
  58
  59 static void amdgpu_fence_submitted(struct pipe_fence_handle *fence,
  60                                 struct amdgpu_cs_request* request,
  61                                 uint64_t *user_fence_cpu_address)
  62 {
  63    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  64
  65    rfence->fence.fence = request->seq_no;
  66    rfence->user_fence_cpu_address = user_fence_cpu_address;
  67    rfence->submission_in_progress = false;
  68 }
  69
  70 static void amdgpu_fence_signalled(struct pipe_fence_handle *fence)
  71 {
  72    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  73
  74    rfence->signalled = true;
  75    rfence->submission_in_progress = false;
  76 }
  77
  78 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
  79                        bool absolute)
  80 {
  81    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  82    uint32_t expired;
  83    int64_t abs_timeout;
  84    uint64_t *user_fence_cpu;
  85    int r;
  86
  87    if (rfence->signalled)
  88       return true;
  89
  90    if (absolute)
  91       abs_timeout = timeout;
  92    else
  93       abs_timeout = os_time_get_absolute_timeout(timeout);
  94
  95    /* The fence might not have a number assigned if its IB is being
  96     * submitted in the other thread right now. Wait until the submission
  97     * is done. */
  98    if (!os_wait_until_zero_abs_timeout(&rfence->submission_in_progress,
  99                                        abs_timeout))
 100       return false;
 101
 102    user_fence_cpu = rfence->user_fence_cpu_address;
 103    if (user_fence_cpu) {
 104       if (*user_fence_cpu >= rfence->fence.fence) {
 105          rfence->signalled = true;
 106          return true;
 107       }
 108
 109       /* No timeout, just query: no need for the ioctl. */
 110       if (!absolute && !timeout)
 111          return false;
 112    }
 113
 114    /* Now use the libdrm query. */
 115    r = amdgpu_cs_query_fence_status(&rfence->fence,
 116                                     abs_timeout,
 117                                     AMDGPU_QUERY_FENCE_TIMEOUT_IS_ABSOLUTE,
 118                                     &expired);
 119    if (r) {
 120       fprintf(stderr, "amdgpu: amdgpu_cs_query_fence_status failed.\n");
 121       return false;
 122    }
 123
 124    if (expired) {
 125       /* This variable can only transition from false to true, so it doesn't
 126        * matter if threads race for it. */
 127       rfence->signalled = true;
 128       return true;
 129    }
 130    return false;
 131 }
 132
 133 static bool amdgpu_fence_wait_rel_timeout(struct radeon_winsys *rws,
 134                                           struct pipe_fence_handle *fence,
 135                                           uint64_t timeout)
 136 {
 137    return amdgpu_fence_wait(fence, timeout, false);
 138 }
 139
 140 static struct pipe_fence_handle *
 141 amdgpu_cs_get_next_fence(struct radeon_winsys_cs *rcs)
 142 {
 143    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 144    struct pipe_fence_handle *fence = NULL;
 145
 146    if (cs->next_fence) {
 147       amdgpu_fence_reference(&fence, cs->next_fence);
 148       return fence;
 149    }
 150
 151    fence = amdgpu_fence_create(cs->ctx,
 152                                cs->csc->request.ip_type,
 153                                cs->csc->request.ip_instance,
 154                                cs->csc->request.ring);
 155    if (!fence)
 156       return NULL;
 157
 158    amdgpu_fence_reference(&cs->next_fence, fence);
 159    return fence;
 160 }
 161
 162 /* CONTEXTS */
 163
 164 static struct radeon_winsys_ctx *amdgpu_ctx_create(struct radeon_winsys *ws)
 165 {
 166    struct amdgpu_ctx *ctx = CALLOC_STRUCT(amdgpu_ctx);
 167    int r;
 168    struct amdgpu_bo_alloc_request alloc_buffer = {};
 169    amdgpu_bo_handle buf_handle;
 170
 171    if (!ctx)
 172       return NULL;
 173
 174    ctx->ws = amdgpu_winsys(ws);
 175    ctx->refcount = 1;
 176
 177    r = amdgpu_cs_ctx_create(ctx->ws->dev, &ctx->ctx);
 178    if (r) {
 179       fprintf(stderr, "amdgpu: amdgpu_cs_ctx_create failed. (%i)\n", r);
 180       goto error_create;
 181    }
 182
 183    alloc_buffer.alloc_size = ctx->ws->info.gart_page_size;
 184    alloc_buffer.phys_alignment = ctx->ws->info.gart_page_size;
 185    alloc_buffer.preferred_heap = AMDGPU_GEM_DOMAIN_GTT;
 186
 187    r = amdgpu_bo_alloc(ctx->ws->dev, &alloc_buffer, &buf_handle);
 188    if (r) {
 189       fprintf(stderr, "amdgpu: amdgpu_bo_alloc failed. (%i)\n", r);
 190       goto error_user_fence_alloc;
 191    }
 192
 193    r = amdgpu_bo_cpu_map(buf_handle, (void**)&ctx->user_fence_cpu_address_base);
 194    if (r) {
 195       fprintf(stderr, "amdgpu: amdgpu_bo_cpu_map failed. (%i)\n", r);
 196       goto error_user_fence_map;
 197    }
 198
 199    memset(ctx->user_fence_cpu_address_base, 0, alloc_buffer.alloc_size);
 200    ctx->user_fence_bo = buf_handle;
 201
 202    return (struct radeon_winsys_ctx*)ctx;
 203
 204 error_user_fence_map:
 205    amdgpu_bo_free(buf_handle);
 206 error_user_fence_alloc:
 207    amdgpu_cs_ctx_free(ctx->ctx);
 208 error_create:
 209    FREE(ctx);
 210    return NULL;
 211 }
 212
 213 static void amdgpu_ctx_destroy(struct radeon_winsys_ctx *rwctx)
 214 {
 215    amdgpu_ctx_unref((struct amdgpu_ctx*)rwctx);
 216 }
 217
 218 static enum pipe_reset_status
 219 amdgpu_ctx_query_reset_status(struct radeon_winsys_ctx *rwctx)
 220 {
 221    struct amdgpu_ctx *ctx = (struct amdgpu_ctx*)rwctx;
 222    uint32_t result, hangs;
 223    int r;
 224
 225    r = amdgpu_cs_query_reset_state(ctx->ctx, &result, &hangs);
 226    if (r) {
 227       fprintf(stderr, "amdgpu: amdgpu_cs_query_reset_state failed. (%i)\n", r);
 228       return PIPE_NO_RESET;
 229    }
 230
 231    switch (result) {
 232    case AMDGPU_CTX_GUILTY_RESET:
 233       return PIPE_GUILTY_CONTEXT_RESET;
 234    case AMDGPU_CTX_INNOCENT_RESET:
 235       return PIPE_INNOCENT_CONTEXT_RESET;
 236    case AMDGPU_CTX_UNKNOWN_RESET:
 237       return PIPE_UNKNOWN_CONTEXT_RESET;
 238    case AMDGPU_CTX_NO_RESET:
 239    default:
 240       return PIPE_NO_RESET;
 241    }
 242 }
 243
 244 /* COMMAND SUBMISSION */
 245
 246 static bool amdgpu_cs_has_user_fence(struct amdgpu_cs_context *cs)
 247 {
 248    return cs->request.ip_type != AMDGPU_HW_IP_UVD &&
 249           cs->request.ip_type != AMDGPU_HW_IP_VCE;
 250 }
 251
 252 static bool amdgpu_cs_has_chaining(struct amdgpu_cs *cs)
 253 {
 254    return cs->ctx->ws->info.chip_class >= CIK &&
 255           cs->ring_type == RING_GFX;
 256 }
 257
 258 static unsigned amdgpu_cs_epilog_dws(enum ring_type ring_type)
 259 {
 260    if (ring_type == RING_GFX)
 261       return 4; /* for chaining */
 262
 263    return 0;
 264 }
 265
 266 int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo)
 267 {
 268    unsigned hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 269    int i = cs->buffer_indices_hashlist[hash];
 270
 271    /* not found or found */
 272    if (i == -1 || cs->buffers[i].bo == bo)
 273       return i;
 274
 275    /* Hash collision, look for the BO in the list of buffers linearly. */
 276    for (i = cs->num_buffers - 1; i >= 0; i--) {
 277       if (cs->buffers[i].bo == bo) {
 278          /* Put this buffer in the hash list.
 279           * This will prevent additional hash collisions if there are
 280           * several consecutive lookup_buffer calls for the same buffer.
 281           *
 282           * Example: Assuming buffers A,B,C collide in the hash list,
 283           * the following sequence of buffers:
 284           *         AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
 285           * will collide here: ^ and here:   ^,
 286           * meaning that we should get very few collisions in the end. */
 287          cs->buffer_indices_hashlist[hash] = i;
 288          return i;
 289       }
 290    }
 291    return -1;
 292 }
 293
 294 static unsigned amdgpu_add_buffer(struct amdgpu_cs *acs,
 295                                  struct amdgpu_winsys_bo *bo,
 296                                  enum radeon_bo_usage usage,
 297                                  enum radeon_bo_domain domains,
 298                                  unsigned priority,
 299                                  enum radeon_bo_domain *added_domains)
 300 {
 301    struct amdgpu_cs_context *cs = acs->csc;
 302    struct amdgpu_cs_buffer *buffer;
 303    unsigned hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 304    int i = -1;
 305
 306    assert(priority < 64);
 307    *added_domains = 0;
 308
 309    i = amdgpu_lookup_buffer(cs, bo);
 310
 311    if (i >= 0) {
 312       buffer = &cs->buffers[i];
 313       buffer->priority_usage |= 1llu << priority;
 314       buffer->usage |= usage;
 315       *added_domains = domains & ~buffer->domains;
 316       buffer->domains |= domains;
 317       cs->flags[i] = MAX2(cs->flags[i], priority / 4);
 318       return i;
 319    }
 320
 321    /* New buffer, check if the backing array is large enough. */
 322    if (cs->num_buffers >= cs->max_num_buffers) {
 323       uint32_t size;
 324       cs->max_num_buffers += 10;
 325
 326       size = cs->max_num_buffers * sizeof(struct amdgpu_cs_buffer);
 327       cs->buffers = realloc(cs->buffers, size);
 328
 329       size = cs->max_num_buffers * sizeof(amdgpu_bo_handle);
 330       cs->handles = realloc(cs->handles, size);
 331
 332       cs->flags = realloc(cs->flags, cs->max_num_buffers);
 333    }
 334
 335    /* Initialize the new buffer. */
 336    cs->buffers[cs->num_buffers].bo = NULL;
 337    amdgpu_winsys_bo_reference(&cs->buffers[cs->num_buffers].bo, bo);
 338    cs->handles[cs->num_buffers] = bo->bo;
 339    cs->flags[cs->num_buffers] = priority / 4;
 340    p_atomic_inc(&bo->num_cs_references);
 341    buffer = &cs->buffers[cs->num_buffers];
 342    buffer->bo = bo;
 343    buffer->priority_usage = 1llu << priority;
 344    buffer->usage = usage;
 345    buffer->domains = domains;
 346
 347    cs->buffer_indices_hashlist[hash] = cs->num_buffers;
 348
 349    *added_domains = domains;
 350    return cs->num_buffers++;
 351 }
 352
 353 static unsigned amdgpu_cs_add_buffer(struct radeon_winsys_cs *rcs,
 354                                     struct pb_buffer *buf,
 355                                     enum radeon_bo_usage usage,
 356                                     enum radeon_bo_domain domains,
 357                                     enum radeon_bo_priority priority)
 358 {
 359    /* Don't use the "domains" parameter. Amdgpu doesn't support changing
 360     * the buffer placement during command submission.
 361     */
 362    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 363    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 364    enum radeon_bo_domain added_domains;
 365    unsigned index = amdgpu_add_buffer(cs, bo, usage, bo->initial_domain,
 366                                      priority, &added_domains);
 367
 368    if (added_domains & RADEON_DOMAIN_VRAM)
 369       cs->main.base.used_vram += bo->base.size;
 370    else if (added_domains & RADEON_DOMAIN_GTT)
 371       cs->main.base.used_gart += bo->base.size;
 372
 373    return index;
 374 }
 375
 376 static bool amdgpu_ib_new_buffer(struct amdgpu_winsys *ws, struct amdgpu_ib *ib)
 377 {
 378    struct pb_buffer *pb;
 379    uint8_t *mapped;
 380    unsigned buffer_size;
 381
 382    /* Always create a buffer that is at least as large as the maximum seen IB
 383     * size, aligned to a power of two (and multiplied by 4 to reduce internal
 384     * fragmentation if chaining is not available). Limit to 512k dwords, which
 385     * is the largest power of two that fits into the size field of the
 386     * INDIRECT_BUFFER packet.
 387     */
 388    if (amdgpu_cs_has_chaining(amdgpu_cs_from_ib(ib)))
 389       buffer_size = 4 *util_next_power_of_two(ib->max_ib_size);
 390    else
 391       buffer_size = 4 *util_next_power_of_two(4 * ib->max_ib_size);
 392
 393    buffer_size = MIN2(buffer_size, 4 * 512 * 1024);
 394
 395    switch (ib->ib_type) {
 396    case IB_CONST_PREAMBLE:
 397       buffer_size = MAX2(buffer_size, 4 * 1024);
 398       break;
 399    case IB_CONST:
 400       buffer_size = MAX2(buffer_size, 16 * 1024 * 4);
 401       break;
 402    case IB_MAIN:
 403       buffer_size = MAX2(buffer_size, 8 * 1024 * 4);
 404       break;
 405    default:
 406       unreachable("unhandled IB type");
 407    }
 408
 409    pb = ws->base.buffer_create(&ws->base, buffer_size,
 410                                ws->info.gart_page_size,
 411                                RADEON_DOMAIN_GTT,
 412                                RADEON_FLAG_CPU_ACCESS);
 413    if (!pb)
 414       return false;
 415
 416    mapped = ws->base.buffer_map(pb, NULL, PIPE_TRANSFER_WRITE);
 417    if (!mapped) {
 418       pb_reference(&pb, NULL);
 419       return false;
 420    }
 421
 422    pb_reference(&ib->big_ib_buffer, pb);
 423    pb_reference(&pb, NULL);
 424
 425    ib->ib_mapped = mapped;
 426    ib->used_ib_space = 0;
 427
 428    return true;
 429 }
 430
 431 static unsigned amdgpu_ib_max_submit_dwords(enum ib_type ib_type)
 432 {
 433    switch (ib_type) {
 434    case IB_MAIN:
 435       /* Smaller submits means the GPU gets busy sooner and there is less
 436        * waiting for buffers and fences. Proof:
 437        *   http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
 438        */
 439       return 20 * 1024;
 440    case IB_CONST_PREAMBLE:
 441    case IB_CONST:
 442       /* There isn't really any reason to limit CE IB size beyond the natural
 443        * limit implied by the main IB, except perhaps GTT size. Just return
 444        * an extremely large value that we never get anywhere close to.
 445        */
 446       return 16 * 1024 * 1024;
 447    default:
 448       unreachable("bad ib_type");
 449    }
 450 }
 451
 452 static bool amdgpu_get_new_ib(struct radeon_winsys *ws, struct amdgpu_cs *cs,
 453                               enum ib_type ib_type)
 454 {
 455    struct amdgpu_winsys *aws = (struct amdgpu_winsys*)ws;
 456    /* Small IBs are better than big IBs, because the GPU goes idle quicker
 457     * and there is less waiting for buffers and fences. Proof:
 458     *   http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
 459     */
 460    struct amdgpu_ib *ib = NULL;
 461    struct amdgpu_cs_ib_info *info = &cs->csc->ib[ib_type];
 462    unsigned ib_size = 0;
 463
 464    switch (ib_type) {
 465    case IB_CONST_PREAMBLE:
 466       ib = &cs->const_preamble_ib;
 467       ib_size = 256 * 4;
 468       break;
 469    case IB_CONST:
 470       ib = &cs->const_ib;
 471       ib_size = 8 * 1024 * 4;
 472       break;
 473    case IB_MAIN:
 474       ib = &cs->main;
 475       ib_size = 4 * 1024 * 4;
 476       break;
 477    default:
 478       unreachable("unhandled IB type");
 479    }
 480
 481    if (!amdgpu_cs_has_chaining(cs)) {
 482       ib_size = MAX2(ib_size,
 483                      4 * MIN2(util_next_power_of_two(ib->max_ib_size),
 484                               amdgpu_ib_max_submit_dwords(ib_type)));
 485    }
 486
 487    ib->max_ib_size = ib->max_ib_size - ib->max_ib_size / 32;
 488
 489    ib->base.prev_dw = 0;
 490    ib->base.num_prev = 0;
 491    ib->base.current.cdw = 0;
 492    ib->base.current.buf = NULL;
 493
 494    /* Allocate a new buffer for IBs if the current buffer is all used. */
 495    if (!ib->big_ib_buffer ||
 496        ib->used_ib_space + ib_size > ib->big_ib_buffer->size) {
 497       if (!amdgpu_ib_new_buffer(aws, ib))
 498          return false;
 499    }
 500
 501    info->ib_mc_address = amdgpu_winsys_bo(ib->big_ib_buffer)->va +
 502                          ib->used_ib_space;
 503    info->size = 0;
 504    ib->ptr_ib_size = &info->size;
 505
 506    amdgpu_cs_add_buffer(&cs->main.base, ib->big_ib_buffer,
 507                         RADEON_USAGE_READ, 0, RADEON_PRIO_IB1);
 508
 509    ib->base.current.buf = (uint32_t*)(ib->ib_mapped + ib->used_ib_space);
 510
 511    ib_size = ib->big_ib_buffer->size - ib->used_ib_space;
 512    ib->base.current.max_dw = ib_size / 4 - amdgpu_cs_epilog_dws(cs->ring_type);
 513    return true;
 514 }
 515
 516 static void amdgpu_ib_finalize(struct amdgpu_ib *ib)
 517 {
 518    *ib->ptr_ib_size |= ib->base.current.cdw;
 519    ib->used_ib_space += ib->base.current.cdw * 4;
 520    ib->max_ib_size = MAX2(ib->max_ib_size, ib->base.prev_dw + ib->base.current.cdw);
 521 }
 522
 523 static bool amdgpu_init_cs_context(struct amdgpu_cs_context *cs,
 524                                    enum ring_type ring_type)
 525 {
 526    int i;
 527
 528    switch (ring_type) {
 529    case RING_DMA:
 530       cs->request.ip_type = AMDGPU_HW_IP_DMA;
 531       break;
 532
 533    case RING_UVD:
 534       cs->request.ip_type = AMDGPU_HW_IP_UVD;
 535       break;
 536
 537    case RING_VCE:
 538       cs->request.ip_type = AMDGPU_HW_IP_VCE;
 539       break;
 540
 541    case RING_COMPUTE:
 542       cs->request.ip_type = AMDGPU_HW_IP_COMPUTE;
 543       break;
 544
 545    default:
 546    case RING_GFX:
 547       cs->request.ip_type = AMDGPU_HW_IP_GFX;
 548       break;
 549    }
 550
 551    cs->max_num_buffers = 512;
 552    cs->buffers = (struct amdgpu_cs_buffer*)
 553                   CALLOC(1, cs->max_num_buffers * sizeof(struct amdgpu_cs_buffer));
 554    if (!cs->buffers) {
 555       return false;
 556    }
 557
 558    cs->handles = CALLOC(1, cs->max_num_buffers * sizeof(amdgpu_bo_handle));
 559    if (!cs->handles) {
 560       FREE(cs->buffers);
 561       return false;
 562    }
 563
 564    cs->flags = CALLOC(1, cs->max_num_buffers);
 565    if (!cs->flags) {
 566       FREE(cs->handles);
 567       FREE(cs->buffers);
 568       return false;
 569    }
 570
 571    for (i = 0; i < ARRAY_SIZE(cs->buffer_indices_hashlist); i++) {
 572       cs->buffer_indices_hashlist[i] = -1;
 573    }
 574
 575    cs->request.number_of_ibs = 1;
 576    cs->request.ibs = &cs->ib[IB_MAIN];
 577
 578    cs->ib[IB_CONST].flags = AMDGPU_IB_FLAG_CE;
 579    cs->ib[IB_CONST_PREAMBLE].flags = AMDGPU_IB_FLAG_CE |
 580                                      AMDGPU_IB_FLAG_PREAMBLE;
 581
 582    return true;
 583 }
 584
 585 static void amdgpu_cs_context_cleanup(struct amdgpu_cs_context *cs)
 586 {
 587    unsigned i;
 588
 589    for (i = 0; i < cs->num_buffers; i++) {
 590       p_atomic_dec(&cs->buffers[i].bo->num_cs_references);
 591       amdgpu_winsys_bo_reference(&cs->buffers[i].bo, NULL);
 592       cs->handles[i] = NULL;
 593       cs->flags[i] = 0;
 594    }
 595
 596    cs->num_buffers = 0;
 597    amdgpu_fence_reference(&cs->fence, NULL);
 598
 599    for (i = 0; i < ARRAY_SIZE(cs->buffer_indices_hashlist); i++) {
 600       cs->buffer_indices_hashlist[i] = -1;
 601    }
 602 }
 603
 604 static void amdgpu_destroy_cs_context(struct amdgpu_cs_context *cs)
 605 {
 606    amdgpu_cs_context_cleanup(cs);
 607    FREE(cs->flags);
 608    FREE(cs->buffers);
 609    FREE(cs->handles);
 610    FREE(cs->request.dependencies);
 611 }
 612
 613
 614 static struct radeon_winsys_cs *
 615 amdgpu_cs_create(struct radeon_winsys_ctx *rwctx,
 616                  enum ring_type ring_type,
 617                  void (*flush)(void *ctx, unsigned flags,
 618                                struct pipe_fence_handle **fence),
 619                  void *flush_ctx)
 620 {
 621    struct amdgpu_ctx *ctx = (struct amdgpu_ctx*)rwctx;
 622    struct amdgpu_cs *cs;
 623
 624    cs = CALLOC_STRUCT(amdgpu_cs);
 625    if (!cs) {
 626       return NULL;
 627    }
 628
 629    util_queue_fence_init(&cs->flush_completed);
 630
 631    cs->ctx = ctx;
 632    cs->flush_cs = flush;
 633    cs->flush_data = flush_ctx;
 634    cs->ring_type = ring_type;
 635
 636    cs->main.ib_type = IB_MAIN;
 637    cs->const_ib.ib_type = IB_CONST;
 638    cs->const_preamble_ib.ib_type = IB_CONST_PREAMBLE;
 639
 640    if (!amdgpu_init_cs_context(&cs->csc1, ring_type)) {
 641       FREE(cs);
 642       return NULL;
 643    }
 644
 645    if (!amdgpu_init_cs_context(&cs->csc2, ring_type)) {
 646       amdgpu_destroy_cs_context(&cs->csc1);
 647       FREE(cs);
 648       return NULL;
 649    }
 650
 651    /* Set the first submission context as current. */
 652    cs->csc = &cs->csc1;
 653    cs->cst = &cs->csc2;
 654
 655    if (!amdgpu_get_new_ib(&ctx->ws->base, cs, IB_MAIN)) {
 656       amdgpu_destroy_cs_context(&cs->csc2);
 657       amdgpu_destroy_cs_context(&cs->csc1);
 658       FREE(cs);
 659       return NULL;
 660    }
 661
 662    p_atomic_inc(&ctx->ws->num_cs);
 663    return &cs->main.base;
 664 }
 665
 666 static struct radeon_winsys_cs *
 667 amdgpu_cs_add_const_ib(struct radeon_winsys_cs *rcs)
 668 {
 669    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 670    struct amdgpu_winsys *ws = cs->ctx->ws;
 671
 672    /* only one const IB can be added */
 673    if (cs->ring_type != RING_GFX || cs->const_ib.ib_mapped)
 674       return NULL;
 675
 676    if (!amdgpu_get_new_ib(&ws->base, cs, IB_CONST))
 677       return NULL;
 678
 679    cs->csc->request.number_of_ibs = 2;
 680    cs->csc->request.ibs = &cs->csc->ib[IB_CONST];
 681
 682    cs->cst->request.number_of_ibs = 2;
 683    cs->cst->request.ibs = &cs->cst->ib[IB_CONST];
 684
 685    return &cs->const_ib.base;
 686 }
 687
 688 static struct radeon_winsys_cs *
 689 amdgpu_cs_add_const_preamble_ib(struct radeon_winsys_cs *rcs)
 690 {
 691    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 692    struct amdgpu_winsys *ws = cs->ctx->ws;
 693
 694    /* only one const preamble IB can be added and only when the const IB has
 695     * also been mapped */
 696    if (cs->ring_type != RING_GFX || !cs->const_ib.ib_mapped ||
 697        cs->const_preamble_ib.ib_mapped)
 698       return NULL;
 699
 700    if (!amdgpu_get_new_ib(&ws->base, cs, IB_CONST_PREAMBLE))
 701       return NULL;
 702
 703    cs->csc->request.number_of_ibs = 3;
 704    cs->csc->request.ibs = &cs->csc->ib[IB_CONST_PREAMBLE];
 705
 706    cs->cst->request.number_of_ibs = 3;
 707    cs->cst->request.ibs = &cs->cst->ib[IB_CONST_PREAMBLE];
 708
 709    return &cs->const_preamble_ib.base;
 710 }
 711
 712 #define OUT_CS(cs, value) (cs)->current.buf[(cs)->current.cdw++] = (value)
 713
 714 static int amdgpu_cs_lookup_buffer(struct radeon_winsys_cs *rcs,
 715                                struct pb_buffer *buf)
 716 {
 717    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 718
 719    return amdgpu_lookup_buffer(cs->csc, (struct amdgpu_winsys_bo*)buf);
 720 }
 721
 722 static bool amdgpu_cs_validate(struct radeon_winsys_cs *rcs)
 723 {
 724    return true;
 725 }
 726
 727 static bool amdgpu_cs_check_space(struct radeon_winsys_cs *rcs, unsigned dw)
 728 {
 729    struct amdgpu_ib *ib = amdgpu_ib(rcs);
 730    struct amdgpu_cs *cs = amdgpu_cs_from_ib(ib);
 731    unsigned requested_size = rcs->prev_dw + rcs->current.cdw + dw;
 732    uint64_t va;
 733    uint32_t *new_ptr_ib_size;
 734
 735    assert(rcs->current.cdw <= rcs->current.max_dw);
 736
 737    if (requested_size > amdgpu_ib_max_submit_dwords(ib->ib_type))
 738       return false;
 739
 740    ib->max_ib_size = MAX2(ib->max_ib_size, requested_size);
 741
 742    if (rcs->current.max_dw - rcs->current.cdw >= dw)
 743       return true;
 744
 745    if (!amdgpu_cs_has_chaining(cs))
 746       return false;
 747
 748    /* Allocate a new chunk */
 749    if (rcs->num_prev >= rcs->max_prev) {
 750       unsigned new_max_prev = MAX2(1, 2 * rcs->max_prev);
 751       struct radeon_winsys_cs_chunk *new_prev;
 752
 753       new_prev = REALLOC(rcs->prev,
 754                          sizeof(*new_prev) * rcs->max_prev,
 755                          sizeof(*new_prev) * new_max_prev);
 756       if (!new_prev)
 757          return false;
 758
 759       rcs->prev = new_prev;
 760       rcs->max_prev = new_max_prev;
 761    }
 762
 763    if (!amdgpu_ib_new_buffer(cs->ctx->ws, ib))
 764       return false;
 765
 766    assert(ib->used_ib_space == 0);
 767    va = amdgpu_winsys_bo(ib->big_ib_buffer)->va;
 768
 769    /* This space was originally reserved. */
 770    rcs->current.max_dw += 4;
 771    assert(ib->used_ib_space + 4 * rcs->current.max_dw <= ib->big_ib_buffer->size);
 772
 773    /* Pad with NOPs and add INDIRECT_BUFFER packet */
 774    while ((rcs->current.cdw & 7) != 4)
 775       OUT_CS(rcs, 0xffff1000); /* type3 nop packet */
 776
 777    OUT_CS(rcs, PKT3(ib->ib_type == IB_MAIN ? PKT3_INDIRECT_BUFFER_CIK
 778                                            : PKT3_INDIRECT_BUFFER_CONST, 2, 0));
 779    OUT_CS(rcs, va);
 780    OUT_CS(rcs, va >> 32);
 781    new_ptr_ib_size = &rcs->current.buf[rcs->current.cdw];
 782    OUT_CS(rcs, S_3F2_CHAIN(1) | S_3F2_VALID(1));
 783
 784    assert((rcs->current.cdw & 7) == 0);
 785    assert(rcs->current.cdw <= rcs->current.max_dw);
 786
 787    *ib->ptr_ib_size |= rcs->current.cdw;
 788    ib->ptr_ib_size = new_ptr_ib_size;
 789
 790    /* Hook up the new chunk */
 791    rcs->prev[rcs->num_prev].buf = rcs->current.buf;
 792    rcs->prev[rcs->num_prev].cdw = rcs->current.cdw;
 793    rcs->prev[rcs->num_prev].max_dw = rcs->current.cdw; /* no modifications */
 794    rcs->num_prev++;
 795
 796    ib->base.prev_dw += ib->base.current.cdw;
 797    ib->base.current.cdw = 0;
 798
 799    ib->base.current.buf = (uint32_t*)(ib->ib_mapped + ib->used_ib_space);
 800    ib->base.current.max_dw = ib->big_ib_buffer->size / 4 - amdgpu_cs_epilog_dws(cs->ring_type);
 801
 802    amdgpu_cs_add_buffer(&cs->main.base, ib->big_ib_buffer,
 803                         RADEON_USAGE_READ, 0, RADEON_PRIO_IB1);
 804
 805    return true;
 806 }
 807
 808 static unsigned amdgpu_cs_get_buffer_list(struct radeon_winsys_cs *rcs,
 809                                           struct radeon_bo_list_item *list)
 810 {
 811     struct amdgpu_cs_context *cs = amdgpu_cs(rcs)->csc;
 812     int i;
 813
 814     if (list) {
 815         for (i = 0; i < cs->num_buffers; i++) {
 816             list[i].bo_size = cs->buffers[i].bo->base.size;
 817             list[i].vm_address = cs->buffers[i].bo->va;
 818             list[i].priority_usage = cs->buffers[i].priority_usage;
 819         }
 820     }
 821     return cs->num_buffers;
 822 }
 823
 824 DEBUG_GET_ONCE_BOOL_OPTION(all_bos, "RADEON_ALL_BOS", false)
 825
 826 /* Since the kernel driver doesn't synchronize execution between different
 827  * rings automatically, we have to add fence dependencies manually.
 828  */
 829 static void amdgpu_add_fence_dependencies(struct amdgpu_cs *acs)
 830 {
 831    struct amdgpu_cs_context *cs = acs->csc;
 832    int i, j;
 833
 834    cs->request.number_of_dependencies = 0;
 835
 836    for (i = 0; i < cs->num_buffers; i++) {
 837       for (j = 0; j < RING_LAST; j++) {
 838          struct amdgpu_cs_fence *dep;
 839          unsigned idx;
 840
 841          struct amdgpu_fence *bo_fence = (void *)cs->buffers[i].bo->fence[j];
 842          if (!bo_fence)
 843             continue;
 844
 845          if (bo_fence->ctx == acs->ctx &&
 846              bo_fence->fence.ip_type == cs->request.ip_type &&
 847              bo_fence->fence.ip_instance == cs->request.ip_instance &&
 848              bo_fence->fence.ring == cs->request.ring)
 849             continue;
 850
 851          if (amdgpu_fence_wait((void *)bo_fence, 0, false))
 852             continue;
 853
 854          if (bo_fence->submission_in_progress)
 855             os_wait_until_zero(&bo_fence->submission_in_progress,
 856                                PIPE_TIMEOUT_INFINITE);
 857
 858          idx = cs->request.number_of_dependencies++;
 859          if (idx >= cs->max_dependencies) {
 860             unsigned size;
 861
 862             cs->max_dependencies = idx + 8;
 863             size = cs->max_dependencies * sizeof(struct amdgpu_cs_fence);
 864             cs->request.dependencies = realloc(cs->request.dependencies, size);
 865          }
 866
 867          dep = &cs->request.dependencies[idx];
 868          memcpy(dep, &bo_fence->fence, sizeof(*dep));
 869       }
 870    }
 871 }
 872
 873 void amdgpu_cs_submit_ib(void *job, int thread_index)
 874 {
 875    struct amdgpu_cs *acs = (struct amdgpu_cs*)job;
 876    struct amdgpu_winsys *ws = acs->ctx->ws;
 877    struct amdgpu_cs_context *cs = acs->cst;
 878    int i, r;
 879
 880    cs->request.fence_info.handle = NULL;
 881    if (amdgpu_cs_has_user_fence(cs)) {
 882         cs->request.fence_info.handle = acs->ctx->user_fence_bo;
 883         cs->request.fence_info.offset = acs->ring_type;
 884    }
 885
 886    /* Create the buffer list.
 887     * Use a buffer list containing all allocated buffers if requested.
 888     */
 889    if (debug_get_option_all_bos()) {
 890       struct amdgpu_winsys_bo *bo;
 891       amdgpu_bo_handle *handles;
 892       unsigned num = 0;
 893
 894       pipe_mutex_lock(ws->global_bo_list_lock);
 895
 896       handles = malloc(sizeof(handles[0]) * ws->num_buffers);
 897       if (!handles) {
 898          pipe_mutex_unlock(ws->global_bo_list_lock);
 899          amdgpu_cs_context_cleanup(cs);
 900          cs->error_code = -ENOMEM;
 901          return;
 902       }
 903
 904       LIST_FOR_EACH_ENTRY(bo, &ws->global_bo_list, global_list_item) {
 905          assert(num < ws->num_buffers);
 906          handles[num++] = bo->bo;
 907       }
 908
 909       r = amdgpu_bo_list_create(ws->dev, ws->num_buffers,
 910                                 handles, NULL,
 911                                 &cs->request.resources);
 912       free(handles);
 913       pipe_mutex_unlock(ws->global_bo_list_lock);
 914    } else {
 915       r = amdgpu_bo_list_create(ws->dev, cs->num_buffers,
 916                                 cs->handles, cs->flags,
 917                                 &cs->request.resources);
 918    }
 919
 920    if (r) {
 921       fprintf(stderr, "amdgpu: buffer list creation failed (%d)\n", r);
 922       cs->request.resources = NULL;
 923       amdgpu_fence_signalled(cs->fence);
 924       cs->error_code = r;
 925       goto cleanup;
 926    }
 927
 928    r = amdgpu_cs_submit(acs->ctx->ctx, 0, &cs->request, 1);
 929    cs->error_code = r;
 930    if (r) {
 931       if (r == -ENOMEM)
 932          fprintf(stderr, "amdgpu: Not enough memory for command submission.\n");
 933       else
 934          fprintf(stderr, "amdgpu: The CS has been rejected, "
 935                  "see dmesg for more information (%i).\n", r);
 936
 937       amdgpu_fence_signalled(cs->fence);
 938    } else {
 939       /* Success. */
 940       uint64_t *user_fence = NULL;
 941       if (amdgpu_cs_has_user_fence(cs))
 942          user_fence = acs->ctx->user_fence_cpu_address_base +
 943                       cs->request.fence_info.offset;
 944       amdgpu_fence_submitted(cs->fence, &cs->request, user_fence);
 945    }
 946
 947    /* Cleanup. */
 948    if (cs->request.resources)
 949       amdgpu_bo_list_destroy(cs->request.resources);
 950
 951 cleanup:
 952    for (i = 0; i < cs->num_buffers; i++)
 953       p_atomic_dec(&cs->buffers[i].bo->num_active_ioctls);
 954
 955    amdgpu_cs_context_cleanup(cs);
 956 }
 957
 958 /* Make sure the previous submission is completed. */
 959 void amdgpu_cs_sync_flush(struct radeon_winsys_cs *rcs)
 960 {
 961    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 962    struct amdgpu_winsys *ws = cs->ctx->ws;
 963
 964    /* Wait for any pending ioctl of this CS to complete. */
 965    if (util_queue_is_initialized(&ws->cs_queue))
 966       util_queue_job_wait(&cs->flush_completed);
 967 }
 968
 969 DEBUG_GET_ONCE_BOOL_OPTION(noop, "RADEON_NOOP", false)
 970
 971 static int amdgpu_cs_flush(struct radeon_winsys_cs *rcs,
 972                            unsigned flags,
 973                            struct pipe_fence_handle **fence)
 974 {
 975    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 976    struct amdgpu_winsys *ws = cs->ctx->ws;
 977    int error_code = 0;
 978
 979    rcs->current.max_dw += amdgpu_cs_epilog_dws(cs->ring_type);
 980
 981    switch (cs->ring_type) {
 982    case RING_DMA:
 983       /* pad DMA ring to 8 DWs */
 984       if (ws->info.chip_class <= SI) {
 985          while (rcs->current.cdw & 7)
 986             OUT_CS(rcs, 0xf0000000); /* NOP packet */
 987       } else {
 988          while (rcs->current.cdw & 7)
 989             OUT_CS(rcs, 0x00000000); /* NOP packet */
 990       }
 991       break;
 992    case RING_GFX:
 993       /* pad GFX ring to 8 DWs to meet CP fetch alignment requirements */
 994       if (ws->info.gfx_ib_pad_with_type2) {
 995          while (rcs->current.cdw & 7)
 996             OUT_CS(rcs, 0x80000000); /* type2 nop packet */
 997       } else {
 998          while (rcs->current.cdw & 7)
 999             OUT_CS(rcs, 0xffff1000); /* type3 nop packet */
1000       }
1001
1002       /* Also pad the const IB. */
1003       if (cs->const_ib.ib_mapped)
1004          while (!cs->const_ib.base.current.cdw || (cs->const_ib.base.current.cdw & 7))
1005             OUT_CS(&cs->const_ib.base, 0xffff1000); /* type3 nop packet */
1006
1007       if (cs->const_preamble_ib.ib_mapped)
1008          while (!cs->const_preamble_ib.base.current.cdw || (cs->const_preamble_ib.base.current.cdw & 7))
1009             OUT_CS(&cs->const_preamble_ib.base, 0xffff1000);
1010       break;
1011    case RING_UVD:
1012       while (rcs->current.cdw & 15)
1013          OUT_CS(rcs, 0x80000000); /* type2 nop packet */
1014       break;
1015    default:
1016       break;
1017    }
1018
1019    if (rcs->current.cdw > rcs->current.max_dw) {
1020       fprintf(stderr, "amdgpu: command stream overflowed\n");
1021    }
1022
1023    /* If the CS is not empty or overflowed.... */
1024    if (radeon_emitted(&cs->main.base, 0) &&
1025        cs->main.base.current.cdw <= cs->main.base.current.max_dw &&
1026        !debug_get_option_noop()) {
1027       struct amdgpu_cs_context *cur = cs->csc;
1028       unsigned i, num_buffers = cur->num_buffers;
1029
1030       /* Set IB sizes. */
1031       amdgpu_ib_finalize(&cs->main);
1032
1033       if (cs->const_ib.ib_mapped)
1034          amdgpu_ib_finalize(&cs->const_ib);
1035
1036       if (cs->const_preamble_ib.ib_mapped)
1037          amdgpu_ib_finalize(&cs->const_preamble_ib);
1038
1039       /* Create a fence. */
1040       amdgpu_fence_reference(&cur->fence, NULL);
1041       if (cs->next_fence) {
1042          /* just move the reference */
1043          cur->fence = cs->next_fence;
1044          cs->next_fence = NULL;
1045       } else {
1046          cur->fence = amdgpu_fence_create(cs->ctx,
1047                                           cur->request.ip_type,
1048                                           cur->request.ip_instance,
1049                                           cur->request.ring);
1050       }
1051       if (fence)
1052          amdgpu_fence_reference(fence, cur->fence);
1053
1054       /* Prepare buffers. */
1055       pipe_mutex_lock(ws->bo_fence_lock);
1056       amdgpu_add_fence_dependencies(cs);
1057       for (i = 0; i < num_buffers; i++) {
1058          p_atomic_inc(&cur->buffers[i].bo->num_active_ioctls);
1059          amdgpu_fence_reference(&cur->buffers[i].bo->fence[cs->ring_type],
1060                                 cur->fence);
1061       }
1062       pipe_mutex_unlock(ws->bo_fence_lock);
1063
1064       amdgpu_cs_sync_flush(rcs);
1065
1066       /* Swap command streams. "cst" is going to be submitted. */
1067       cs->csc = cs->cst;
1068       cs->cst = cur;
1069
1070       /* Submit. */
1071       if ((flags & RADEON_FLUSH_ASYNC) &&
1072           util_queue_is_initialized(&ws->cs_queue)) {
1073          util_queue_add_job(&ws->cs_queue, cs, &cs->flush_completed,
1074                             amdgpu_cs_submit_ib, NULL);
1075       } else {
1076          amdgpu_cs_submit_ib(cs, 0);
1077          error_code = cs->cst->error_code;
1078       }
1079    } else {
1080       amdgpu_cs_context_cleanup(cs->csc);
1081    }
1082
1083    amdgpu_get_new_ib(&ws->base, cs, IB_MAIN);
1084    if (cs->const_ib.ib_mapped)
1085       amdgpu_get_new_ib(&ws->base, cs, IB_CONST);
1086    if (cs->const_preamble_ib.ib_mapped)
1087       amdgpu_get_new_ib(&ws->base, cs, IB_CONST_PREAMBLE);
1088
1089    cs->main.base.used_gart = 0;
1090    cs->main.base.used_vram = 0;
1091
1092    ws->num_cs_flushes++;
1093    return error_code;
1094 }
1095
1096 static void amdgpu_cs_destroy(struct radeon_winsys_cs *rcs)
1097 {
1098    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1099
1100    amdgpu_cs_sync_flush(rcs);
1101    util_queue_fence_destroy(&cs->flush_completed);
1102    p_atomic_dec(&cs->ctx->ws->num_cs);
1103    pb_reference(&cs->main.big_ib_buffer, NULL);
1104    FREE(cs->main.base.prev);
1105    pb_reference(&cs->const_ib.big_ib_buffer, NULL);
1106    FREE(cs->const_ib.base.prev);
1107    pb_reference(&cs->const_preamble_ib.big_ib_buffer, NULL);
1108    FREE(cs->const_preamble_ib.base.prev);
1109    amdgpu_destroy_cs_context(&cs->csc1);
1110    amdgpu_destroy_cs_context(&cs->csc2);
1111    amdgpu_fence_reference(&cs->next_fence, NULL);
1112    FREE(cs);
1113 }
1114
1115 static bool amdgpu_bo_is_referenced(struct radeon_winsys_cs *rcs,
1116                                     struct pb_buffer *_buf,
1117                                     enum radeon_bo_usage usage)
1118 {
1119    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1120    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)_buf;
1121
1122    return amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo, usage);
1123 }
1124
1125 void amdgpu_cs_init_functions(struct amdgpu_winsys *ws)
1126 {
1127    ws->base.ctx_create = amdgpu_ctx_create;
1128    ws->base.ctx_destroy = amdgpu_ctx_destroy;
1129    ws->base.ctx_query_reset_status = amdgpu_ctx_query_reset_status;
1130    ws->base.cs_create = amdgpu_cs_create;
1131    ws->base.cs_add_const_ib = amdgpu_cs_add_const_ib;
1132    ws->base.cs_add_const_preamble_ib = amdgpu_cs_add_const_preamble_ib;
1133    ws->base.cs_destroy = amdgpu_cs_destroy;
1134    ws->base.cs_add_buffer = amdgpu_cs_add_buffer;
1135    ws->base.cs_lookup_buffer = amdgpu_cs_lookup_buffer;
1136    ws->base.cs_validate = amdgpu_cs_validate;
1137    ws->base.cs_check_space = amdgpu_cs_check_space;
1138    ws->base.cs_get_buffer_list = amdgpu_cs_get_buffer_list;
1139    ws->base.cs_flush = amdgpu_cs_flush;
1140    ws->base.cs_get_next_fence = amdgpu_cs_get_next_fence;
1141    ws->base.cs_is_buffer_referenced = amdgpu_bo_is_referenced;
1142    ws->base.cs_sync_flush = amdgpu_cs_sync_flush;
1143    ws->base.fence_wait = amdgpu_fence_wait_rel_timeout;
1144    ws->base.fence_reference = amdgpu_fence_reference;
1145 }