src/gallium/winsys/amdgpu/drm/amdgpu_cs.c

   1 /*
   2  * Copyright © 2008 Jérôme Glisse
   3  * Copyright © 2010 Marek Olšák <maraeo@gmail.com>
   4  * Copyright © 2015 Advanced Micro Devices, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining
   8  * a copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  17  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  19  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * The above copyright notice and this permission notice (including the
  25  * next paragraph) shall be included in all copies or substantial portions
  26  * of the Software.
  27  */
  28 /*
  29  * Authors:
  30  *      Marek Olšák <maraeo@gmail.com>
  31  */
  32
  33 #include "amdgpu_cs.h"
  34 #include "os/os_time.h"
  35 #include <stdio.h>
  36 #include <amdgpu_drm.h>
  37
  38 #include "amd/common/sid.h"
  39
  40 DEBUG_GET_ONCE_BOOL_OPTION(noop, "RADEON_NOOP", false)
  41
  42 /* FENCES */
  43
  44 static struct pipe_fence_handle *
  45 amdgpu_fence_create(struct amdgpu_ctx *ctx, unsigned ip_type,
  46                     unsigned ip_instance, unsigned ring)
  47 {
  48    struct amdgpu_fence *fence = CALLOC_STRUCT(amdgpu_fence);
  49
  50    fence->reference.count = 1;
  51    fence->ctx = ctx;
  52    fence->fence.context = ctx->ctx;
  53    fence->fence.ip_type = ip_type;
  54    fence->fence.ip_instance = ip_instance;
  55    fence->fence.ring = ring;
  56    fence->submission_in_progress = true;
  57    p_atomic_inc(&ctx->refcount);
  58    return (struct pipe_fence_handle *)fence;
  59 }
  60
  61 static void amdgpu_fence_submitted(struct pipe_fence_handle *fence,
  62                                 struct amdgpu_cs_request* request,
  63                                 uint64_t *user_fence_cpu_address)
  64 {
  65    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  66
  67    rfence->fence.fence = request->seq_no;
  68    rfence->user_fence_cpu_address = user_fence_cpu_address;
  69    rfence->submission_in_progress = false;
  70 }
  71
  72 static void amdgpu_fence_signalled(struct pipe_fence_handle *fence)
  73 {
  74    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  75
  76    rfence->signalled = true;
  77    rfence->submission_in_progress = false;
  78 }
  79
  80 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
  81                        bool absolute)
  82 {
  83    struct amdgpu_fence *rfence = (struct amdgpu_fence*)fence;
  84    uint32_t expired;
  85    int64_t abs_timeout;
  86    uint64_t *user_fence_cpu;
  87    int r;
  88
  89    if (rfence->signalled)
  90       return true;
  91
  92    if (absolute)
  93       abs_timeout = timeout;
  94    else
  95       abs_timeout = os_time_get_absolute_timeout(timeout);
  96
  97    /* The fence might not have a number assigned if its IB is being
  98     * submitted in the other thread right now. Wait until the submission
  99     * is done. */
 100    if (!os_wait_until_zero_abs_timeout(&rfence->submission_in_progress,
 101                                        abs_timeout))
 102       return false;
 103
 104    user_fence_cpu = rfence->user_fence_cpu_address;
 105    if (user_fence_cpu) {
 106       if (*user_fence_cpu >= rfence->fence.fence) {
 107          rfence->signalled = true;
 108          return true;
 109       }
 110
 111       /* No timeout, just query: no need for the ioctl. */
 112       if (!absolute && !timeout)
 113          return false;
 114    }
 115
 116    /* Now use the libdrm query. */
 117    r = amdgpu_cs_query_fence_status(&rfence->fence,
 118                                     abs_timeout,
 119                                     AMDGPU_QUERY_FENCE_TIMEOUT_IS_ABSOLUTE,
 120                                     &expired);
 121    if (r) {
 122       fprintf(stderr, "amdgpu: amdgpu_cs_query_fence_status failed.\n");
 123       return false;
 124    }
 125
 126    if (expired) {
 127       /* This variable can only transition from false to true, so it doesn't
 128        * matter if threads race for it. */
 129       rfence->signalled = true;
 130       return true;
 131    }
 132    return false;
 133 }
 134
 135 static bool amdgpu_fence_wait_rel_timeout(struct radeon_winsys *rws,
 136                                           struct pipe_fence_handle *fence,
 137                                           uint64_t timeout)
 138 {
 139    return amdgpu_fence_wait(fence, timeout, false);
 140 }
 141
 142 static struct pipe_fence_handle *
 143 amdgpu_cs_get_next_fence(struct radeon_winsys_cs *rcs)
 144 {
 145    struct amdgpu_cs *cs = amdgpu_cs(rcs);
 146    struct pipe_fence_handle *fence = NULL;
 147
 148    if (debug_get_option_noop())
 149       return NULL;
 150
 151    if (cs->next_fence) {
 152       amdgpu_fence_reference(&fence, cs->next_fence);
 153       return fence;
 154    }
 155
 156    fence = amdgpu_fence_create(cs->ctx,
 157                                cs->csc->request.ip_type,
 158                                cs->csc->request.ip_instance,
 159                                cs->csc->request.ring);
 160    if (!fence)
 161       return NULL;
 162
 163    amdgpu_fence_reference(&cs->next_fence, fence);
 164    return fence;
 165 }
 166
 167 /* CONTEXTS */
 168
 169 static struct radeon_winsys_ctx *amdgpu_ctx_create(struct radeon_winsys *ws)
 170 {
 171    struct amdgpu_ctx *ctx = CALLOC_STRUCT(amdgpu_ctx);
 172    int r;
 173    struct amdgpu_bo_alloc_request alloc_buffer = {};
 174    amdgpu_bo_handle buf_handle;
 175
 176    if (!ctx)
 177       return NULL;
 178
 179    ctx->ws = amdgpu_winsys(ws);
 180    ctx->refcount = 1;
 181    ctx->initial_num_total_rejected_cs = ctx->ws->num_total_rejected_cs;
 182
 183    r = amdgpu_cs_ctx_create(ctx->ws->dev, &ctx->ctx);
 184    if (r) {
 185       fprintf(stderr, "amdgpu: amdgpu_cs_ctx_create failed. (%i)\n", r);
 186       goto error_create;
 187    }
 188
 189    alloc_buffer.alloc_size = ctx->ws->info.gart_page_size;
 190    alloc_buffer.phys_alignment = ctx->ws->info.gart_page_size;
 191    alloc_buffer.preferred_heap = AMDGPU_GEM_DOMAIN_GTT;
 192
 193    r = amdgpu_bo_alloc(ctx->ws->dev, &alloc_buffer, &buf_handle);
 194    if (r) {
 195       fprintf(stderr, "amdgpu: amdgpu_bo_alloc failed. (%i)\n", r);
 196       goto error_user_fence_alloc;
 197    }
 198
 199    r = amdgpu_bo_cpu_map(buf_handle, (void**)&ctx->user_fence_cpu_address_base);
 200    if (r) {
 201       fprintf(stderr, "amdgpu: amdgpu_bo_cpu_map failed. (%i)\n", r);
 202       goto error_user_fence_map;
 203    }
 204
 205    memset(ctx->user_fence_cpu_address_base, 0, alloc_buffer.alloc_size);
 206    ctx->user_fence_bo = buf_handle;
 207
 208    return (struct radeon_winsys_ctx*)ctx;
 209
 210 error_user_fence_map:
 211    amdgpu_bo_free(buf_handle);
 212 error_user_fence_alloc:
 213    amdgpu_cs_ctx_free(ctx->ctx);
 214 error_create:
 215    FREE(ctx);
 216    return NULL;
 217 }
 218
 219 static void amdgpu_ctx_destroy(struct radeon_winsys_ctx *rwctx)
 220 {
 221    amdgpu_ctx_unref((struct amdgpu_ctx*)rwctx);
 222 }
 223
 224 static enum pipe_reset_status
 225 amdgpu_ctx_query_reset_status(struct radeon_winsys_ctx *rwctx)
 226 {
 227    struct amdgpu_ctx *ctx = (struct amdgpu_ctx*)rwctx;
 228    uint32_t result, hangs;
 229    int r;
 230
 231    /* Return a failure due to a rejected command submission. */
 232    if (ctx->ws->num_total_rejected_cs > ctx->initial_num_total_rejected_cs) {
 233       return ctx->num_rejected_cs ? PIPE_GUILTY_CONTEXT_RESET :
 234                                     PIPE_INNOCENT_CONTEXT_RESET;
 235    }
 236
 237    /* Return a failure due to a GPU hang. */
 238    r = amdgpu_cs_query_reset_state(ctx->ctx, &result, &hangs);
 239    if (r) {
 240       fprintf(stderr, "amdgpu: amdgpu_cs_query_reset_state failed. (%i)\n", r);
 241       return PIPE_NO_RESET;
 242    }
 243
 244    switch (result) {
 245    case AMDGPU_CTX_GUILTY_RESET:
 246       return PIPE_GUILTY_CONTEXT_RESET;
 247    case AMDGPU_CTX_INNOCENT_RESET:
 248       return PIPE_INNOCENT_CONTEXT_RESET;
 249    case AMDGPU_CTX_UNKNOWN_RESET:
 250       return PIPE_UNKNOWN_CONTEXT_RESET;
 251    case AMDGPU_CTX_NO_RESET:
 252    default:
 253       return PIPE_NO_RESET;
 254    }
 255 }
 256
 257 /* COMMAND SUBMISSION */
 258
 259 static bool amdgpu_cs_has_user_fence(struct amdgpu_cs_context *cs)
 260 {
 261    return cs->request.ip_type != AMDGPU_HW_IP_UVD &&
 262           cs->request.ip_type != AMDGPU_HW_IP_VCE;
 263 }
 264
 265 static bool amdgpu_cs_has_chaining(struct amdgpu_cs *cs)
 266 {
 267    return cs->ctx->ws->info.chip_class >= CIK &&
 268           cs->ring_type == RING_GFX;
 269 }
 270
 271 static unsigned amdgpu_cs_epilog_dws(enum ring_type ring_type)
 272 {
 273    if (ring_type == RING_GFX)
 274       return 4; /* for chaining */
 275
 276    return 0;
 277 }
 278
 279 int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo)
 280 {
 281    unsigned hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 282    int i = cs->buffer_indices_hashlist[hash];
 283    struct amdgpu_cs_buffer *buffers;
 284    int num_buffers;
 285
 286    if (bo->bo) {
 287       buffers = cs->real_buffers;
 288       num_buffers = cs->num_real_buffers;
 289    } else {
 290       buffers = cs->slab_buffers;
 291       num_buffers = cs->num_slab_buffers;
 292    }
 293
 294    /* not found or found */
 295    if (i < 0 || (i < num_buffers && buffers[i].bo == bo))
 296       return i;
 297
 298    /* Hash collision, look for the BO in the list of buffers linearly. */
 299    for (i = num_buffers - 1; i >= 0; i--) {
 300       if (buffers[i].bo == bo) {
 301          /* Put this buffer in the hash list.
 302           * This will prevent additional hash collisions if there are
 303           * several consecutive lookup_buffer calls for the same buffer.
 304           *
 305           * Example: Assuming buffers A,B,C collide in the hash list,
 306           * the following sequence of buffers:
 307           *         AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
 308           * will collide here: ^ and here:   ^,
 309           * meaning that we should get very few collisions in the end. */
 310          cs->buffer_indices_hashlist[hash] = i;
 311          return i;
 312       }
 313    }
 314    return -1;
 315 }
 316
 317 static int
 318 amdgpu_lookup_or_add_real_buffer(struct amdgpu_cs *acs, struct amdgpu_winsys_bo *bo)
 319 {
 320    struct amdgpu_cs_context *cs = acs->csc;
 321    struct amdgpu_cs_buffer *buffer;
 322    unsigned hash;
 323    int idx = amdgpu_lookup_buffer(cs, bo);
 324
 325    if (idx >= 0)
 326       return idx;
 327
 328    /* New buffer, check if the backing array is large enough. */
 329    if (cs->num_real_buffers >= cs->max_real_buffers) {
 330       unsigned new_max =
 331          MAX2(cs->max_real_buffers + 16, (unsigned)(cs->max_real_buffers * 1.3));
 332       struct amdgpu_cs_buffer *new_buffers;
 333       amdgpu_bo_handle *new_handles;
 334       uint8_t *new_flags;
 335
 336       new_buffers = MALLOC(new_max * sizeof(*new_buffers));
 337       new_handles = MALLOC(new_max * sizeof(*new_handles));
 338       new_flags = MALLOC(new_max * sizeof(*new_flags));
 339
 340       if (!new_buffers || !new_handles || !new_flags) {
 341          fprintf(stderr, "amdgpu_lookup_or_add_buffer: allocation failed\n");
 342          FREE(new_buffers);
 343          FREE(new_handles);
 344          FREE(new_flags);
 345          return -1;
 346       }
 347
 348       memcpy(new_buffers, cs->real_buffers, cs->num_real_buffers * sizeof(*new_buffers));
 349       memcpy(new_handles, cs->handles, cs->num_real_buffers * sizeof(*new_handles));
 350       memcpy(new_flags, cs->flags, cs->num_real_buffers * sizeof(*new_flags));
 351
 352       FREE(cs->real_buffers);
 353       FREE(cs->handles);
 354       FREE(cs->flags);
 355
 356       cs->max_real_buffers = new_max;
 357       cs->real_buffers = new_buffers;
 358       cs->handles = new_handles;
 359       cs->flags = new_flags;
 360    }
 361
 362    idx = cs->num_real_buffers;
 363    buffer = &cs->real_buffers[idx];
 364
 365    memset(buffer, 0, sizeof(*buffer));
 366    amdgpu_winsys_bo_reference(&buffer->bo, bo);
 367    cs->handles[idx] = bo->bo;
 368    cs->flags[idx] = 0;
 369    p_atomic_inc(&bo->num_cs_references);
 370    cs->num_real_buffers++;
 371
 372    hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 373    cs->buffer_indices_hashlist[hash] = idx;
 374
 375    if (bo->initial_domain & RADEON_DOMAIN_VRAM)
 376       acs->main.base.used_vram += bo->base.size;
 377    else if (bo->initial_domain & RADEON_DOMAIN_GTT)
 378       acs->main.base.used_gart += bo->base.size;
 379
 380    return idx;
 381 }
 382
 383 static int amdgpu_lookup_or_add_slab_buffer(struct amdgpu_cs *acs,
 384                                             struct amdgpu_winsys_bo *bo)
 385 {
 386    struct amdgpu_cs_context *cs = acs->csc;
 387    struct amdgpu_cs_buffer *buffer;
 388    unsigned hash;
 389    int idx = amdgpu_lookup_buffer(cs, bo);
 390    int real_idx;
 391
 392    if (idx >= 0)
 393       return idx;
 394
 395    real_idx = amdgpu_lookup_or_add_real_buffer(acs, bo->u.slab.real);
 396    if (real_idx < 0)
 397       return -1;
 398
 399    /* New buffer, check if the backing array is large enough. */
 400    if (cs->num_slab_buffers >= cs->max_slab_buffers) {
 401       unsigned new_max =
 402          MAX2(cs->max_slab_buffers + 16, (unsigned)(cs->max_slab_buffers * 1.3));
 403       struct amdgpu_cs_buffer *new_buffers;
 404
 405       new_buffers = REALLOC(cs->slab_buffers,
 406                             cs->max_slab_buffers * sizeof(*new_buffers),
 407                             new_max * sizeof(*new_buffers));
 408       if (!new_buffers) {
 409          fprintf(stderr, "amdgpu_lookup_or_add_slab_buffer: allocation failed\n");
 410          return -1;
 411       }
 412
 413       cs->max_slab_buffers = new_max;
 414       cs->slab_buffers = new_buffers;
 415    }
 416
 417    idx = cs->num_slab_buffers;
 418    buffer = &cs->slab_buffers[idx];
 419
 420    memset(buffer, 0, sizeof(*buffer));
 421    amdgpu_winsys_bo_reference(&buffer->bo, bo);
 422    buffer->u.slab.real_idx = real_idx;
 423    p_atomic_inc(&bo->num_cs_references);
 424    cs->num_slab_buffers++;
 425
 426    hash = bo->unique_id & (ARRAY_SIZE(cs->buffer_indices_hashlist)-1);
 427    cs->buffer_indices_hashlist[hash] = idx;
 428
 429    return idx;
 430 }
 431
 432 static unsigned amdgpu_cs_add_buffer(struct radeon_winsys_cs *rcs,
 433                                     struct pb_buffer *buf,
 434                                     enum radeon_bo_usage usage,
 435                                     enum radeon_bo_domain domains,
 436                                     enum radeon_bo_priority priority)
 437 {
 438    /* Don't use the "domains" parameter. Amdgpu doesn't support changing
 439     * the buffer placement during command submission.
 440     */
 441    struct amdgpu_cs *acs = amdgpu_cs(rcs);
 442    struct amdgpu_cs_context *cs = acs->csc;
 443    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
 444    struct amdgpu_cs_buffer *buffer;
 445    int index;
 446
 447    if (!bo->bo) {
 448       index = amdgpu_lookup_or_add_slab_buffer(acs, bo);
 449       if (index < 0)
 450          return 0;
 451
 452       buffer = &cs->slab_buffers[index];
 453       buffer->usage |= usage;
 454
 455       usage &= ~RADEON_USAGE_SYNCHRONIZED;
 456       index = buffer->u.slab.real_idx;
 457    } else {
 458       index = amdgpu_lookup_or_add_real_buffer(acs, bo);
 459       if (index < 0)
 460          return 0;
 461    }
 462
 463    buffer = &cs->real_buffers[index];
 464    buffer->u.real.priority_usage |= 1llu << priority;
 465    buffer->usage |= usage;
 466    cs->flags[index] = MAX2(cs->flags[index], priority / 4);
 467    return index;
 468 }
 469
 470 static bool amdgpu_ib_new_buffer(struct amdgpu_winsys *ws, struct amdgpu_ib *ib)
 471 {
 472    struct pb_buffer *pb;
 473    uint8_t *mapped;
 474    unsigned buffer_size;
 475
 476    /* Always create a buffer that is at least as large as the maximum seen IB
 477     * size, aligned to a power of two (and multiplied by 4 to reduce internal
 478     * fragmentation if chaining is not available). Limit to 512k dwords, which
 479     * is the largest power of two that fits into the size field of the
 480     * INDIRECT_BUFFER packet.
 481     */
 482    if (amdgpu_cs_has_chaining(amdgpu_cs_from_ib(ib)))
 483       buffer_size = 4 *util_next_power_of_two(ib->max_ib_size);
 484    else
 485       buffer_size = 4 *util_next_power_of_two(4 * ib->max_ib_size);
 486
 487    buffer_size = MIN2(buffer_size, 4 * 512 * 1024);
 488
 489    switch (ib->ib_type) {
 490    case IB_CONST_PREAMBLE:
 491       buffer_size = MAX2(buffer_size, 4 * 1024);
 492       break;
 493    case IB_CONST:
 494       buffer_size = MAX2(buffer_size, 16 * 1024 * 4);
 495       break;
 496    case IB_MAIN:
 497       buffer_size = MAX2(buffer_size, 8 * 1024 * 4);
 498       break;
 499    default:
 500       unreachable("unhandled IB type");
 501    }
 502
 503    pb = ws->base.buffer_create(&ws->base, buffer_size,
 504                                ws->info.gart_page_size,
 505                                RADEON_DOMAIN_GTT,
 506                                RADEON_FLAG_CPU_ACCESS);
 507    if (!pb)
 508       return false;
 509
 510    mapped = ws->base.buffer_map(pb, NULL, PIPE_TRANSFER_WRITE);
 511    if (!mapped) {
 512       pb_reference(&pb, NULL);
 513       return false;
 514    }
 515
 516    pb_reference(&ib->big_ib_buffer, pb);
 517    pb_reference(&pb, NULL);
 518
 519    ib->ib_mapped = mapped;
 520    ib->used_ib_space = 0;
 521
 522    return true;
 523 }
 524
 525 static unsigned amdgpu_ib_max_submit_dwords(enum ib_type ib_type)
 526 {
 527    switch (ib_type) {
 528    case IB_MAIN:
 529       /* Smaller submits means the GPU gets busy sooner and there is less
 530        * waiting for buffers and fences. Proof:
 531        *   http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
 532        */
 533       return 20 * 1024;
 534    case IB_CONST_PREAMBLE:
 535    case IB_CONST:
 536       /* There isn't really any reason to limit CE IB size beyond the natural
 537        * limit implied by the main IB, except perhaps GTT size. Just return
 538        * an extremely large value that we never get anywhere close to.
 539        */
 540       return 16 * 1024 * 1024;
 541    default:
 542       unreachable("bad ib_type");
 543    }
 544 }
 545
 546 static bool amdgpu_get_new_ib(struct radeon_winsys *ws, struct amdgpu_cs *cs,
 547                               enum ib_type ib_type)
 548 {
 549    struct amdgpu_winsys *aws = (struct amdgpu_winsys*)ws;
 550    /* Small IBs are better than big IBs, because the GPU goes idle quicker
 551     * and there is less waiting for buffers and fences. Proof:
 552     *   http://www.phoronix.com/scan.php?page=article&item=mesa-111-si&num=1
 553     */
 554    struct amdgpu_ib *ib = NULL;
 555    struct amdgpu_cs_ib_info *info = &cs->csc->ib[ib_type];
 556    unsigned ib_size = 0;
 557
 558    switch (ib_type) {
 559    case IB_CONST_PREAMBLE:
 560       ib = &cs->const_preamble_ib;
 561       ib_size = 256 * 4;
 562       break;
 563    case IB_CONST:
 564       ib = &cs->const_ib;
 565       ib_size = 8 * 1024 * 4;
 566       break;
 567    case IB_MAIN:
 568       ib = &cs->main;
 569       ib_size = 4 * 1024 * 4;
 570       break;
 571    default:
 572       unreachable("unhandled IB type");
 573    }
 574
 575    if (!amdgpu_cs_has_chaining(cs)) {
 576       ib_size = MAX2(ib_size,
 577                      4 * MIN2(util_next_power_of_two(ib->max_ib_size),
 578                               amdgpu_ib_max_submit_dwords(ib_type)));
 579    }
 580
 581    ib->max_ib_size = ib->max_ib_size - ib->max_ib_size / 32;
 582
 583    ib->base.prev_dw = 0;
 584    ib->base.num_prev = 0;
 585    ib->base.current.cdw = 0;
 586    ib->base.current.buf = NULL;
 587
 588    /* Allocate a new buffer for IBs if the current buffer is all used. */
 589    if (!ib->big_ib_buffer ||
 590        ib->used_ib_space + ib_size > ib->big_ib_buffer->size) {
 591       if (!amdgpu_ib_new_buffer(aws, ib))
 592          return false;
 593    }
 594
 595    info->ib_mc_address = amdgpu_winsys_bo(ib->big_ib_buffer)->va +
 596                          ib->used_ib_space;
 597    info->size = 0;
 598    ib->ptr_ib_size = &info->size;
 599
 600    amdgpu_cs_add_buffer(&cs->main.base, ib->big_ib_buffer,
 601                         RADEON_USAGE_READ, 0, RADEON_PRIO_IB1);
 602
 603    ib->base.current.buf = (uint32_t*)(ib->ib_mapped + ib->used_ib_space);
 604
 605    ib_size = ib->big_ib_buffer->size - ib->used_ib_space;
 606    ib->base.current.max_dw = ib_size / 4 - amdgpu_cs_epilog_dws(cs->ring_type);
 607    return true;
 608 }
 609
 610 static void amdgpu_ib_finalize(struct amdgpu_ib *ib)
 611 {
 612    *ib->ptr_ib_size |= ib->base.current.cdw;
 613    ib->used_ib_space += ib->base.current.cdw * 4;
 614    ib->max_ib_size = MAX2(ib->max_ib_size, ib->base.prev_dw + ib->base.current.cdw);
 615 }
 616
 617 static bool amdgpu_init_cs_context(struct amdgpu_cs_context *cs,
 618                                    enum ring_type ring_type)
 619 {
 620    int i;
 621
 622    switch (ring_type) {
 623    case RING_DMA:
 624       cs->request.ip_type = AMDGPU_HW_IP_DMA;
 625       break;
 626
 627    case RING_UVD:
 628       cs->request.ip_type = AMDGPU_HW_IP_UVD;
 629       break;
 630
 631    case RING_VCE:
 632       cs->request.ip_type = AMDGPU_HW_IP_VCE;
 633       break;
 634
 635    case RING_COMPUTE:
 636       cs->request.ip_type = AMDGPU_HW_IP_COMPUTE;
 637       break;
 638
 639    default:
 640    case RING_GFX:
 641       cs->request.ip_type = AMDGPU_HW_IP_GFX;
 642       break;
 643    }
 644
 645    for (i = 0; i < ARRAY_SIZE(cs->buffer_indices_hashlist); i++) {
 646       cs->buffer_indices_hashlist[i] = -1;
 647    }
 648
 649    cs->request.number_of_ibs = 1;
 650    cs->request.ibs = &cs->ib[IB_MAIN];
 651
 652    cs->ib[IB_CONST].flags = AMDGPU_IB_FLAG_CE;
 653    cs->ib[IB_CONST_PREAMBLE].flags = AMDGPU_IB_FLAG_CE |
 654                                      AMDGPU_IB_FLAG_PREAMBLE;
 655
 656    return true;
 657 }
 658
 659 static void amdgpu_cs_context_cleanup(struct amdgpu_cs_context *cs)
 660 {
 661    unsigned i;
 662
 663    for (i = 0; i < cs->num_real_buffers; i++) {
 664       p_atomic_dec(&cs->real_buffers[i].bo->num_cs_references);
 665       amdgpu_winsys_bo_reference(&cs->real_buffers[i].bo, NULL);
 666    }
 667    for (i = 0; i < cs->num_slab_buffers; i++) {
 668       p_atomic_dec(&cs->slab_buffers[i].bo->num_cs_references);
 669       amdgpu_winsys_bo_reference(&cs->slab_buffers[i].bo, NULL);
 670    }
 671
 672    cs->num_real_buffers = 0;
 673    cs->num_slab_buffers = 0;
 674    amdgpu_fence_reference(&cs->fence, NULL);
 675
 676    for (i = 0; i < ARRAY_SIZE(cs->buffer_indices_hashlist); i++) {
 677       cs->buffer_indices_hashlist[i] = -1;
 678    }
 679 }
 680
 681 static void amdgpu_destroy_cs_context(struct amdgpu_cs_context *cs)
 682 {
 683    amdgpu_cs_context_cleanup(cs);
 684    FREE(cs->flags);
 685    FREE(cs->real_buffers);
 686    FREE(cs->handles);
 687    FREE(cs->slab_buffers);
 688    FREE(cs->request.dependencies);
 689 }
 690
 691
 692 static struct radeon_winsys_cs *
 693 amdgpu_cs_create(struct radeon_winsys_ctx *rwctx,
 694                  enum ring_type ring_type,
 695                  void (*flush)(void *ctx, unsigned flags,
 696                                struct pipe_fence_handle **fence),
 697                  void *flush_ctx)
 698 {
 699    struct amdgpu_ctx *ctx = (struct amdgpu_ctx*)rwctx;
 700    struct amdgpu_cs *cs;
 701
 702    cs = CALLOC_STRUCT(amdgpu_cs);
 703    if (!cs) {
 704       return NULL;
 705    }
 706
 707    util_queue_fence_init(&cs->flush_completed);
 708
 709    cs->ctx = ctx;
 710    cs->flush_cs = flush;
 711    cs->flush_data = flush_ctx;
 712    cs->ring_type = ring_type;
 713
 714    cs->main.ib_type = IB_MAIN;
 715    cs->const_ib.ib_type = IB_CONST;
 716    cs->const_preamble_ib.ib_type = IB_CONST_PREAMBLE;
 717
 718    if (!amdgpu_init_cs_context(&cs->csc1, ring_type)) {
 719       FREE(cs);
 720       return NULL;
 721    }
 722
 723    if (!amdgpu_init_cs_context(&cs->csc2, ring_type)) {
 724       amdgpu_destroy_cs_context(&cs->csc1);
 725       FREE(cs);
 726       return NULL;
 727    }
 728
 729    /* Set the first submission context as current. */
 730    cs->csc = &cs->csc1;
 731    cs->cst = &cs->csc2;
 732
 733    if (!amdgpu_get_new_ib(&ctx->ws->base, cs, IB_MAIN)) {
 734       amdgpu_destroy_cs_context(&cs->csc2);
 735       amdgpu_destroy_cs_context(&cs->csc1);
 736       FREE(cs);
 737       return NULL;
 738    }
 739
 740    p_atomic_inc(&ctx->ws->num_cs);
 741    return &cs->main.base;
 742 }
 743
 744 static struct radeon_winsys_cs *
 745 amdgpu_cs_add_const_ib(struct radeon_winsys_cs *rcs)
 746 {
 747    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 748    struct amdgpu_winsys *ws = cs->ctx->ws;
 749
 750    /* only one const IB can be added */
 751    if (cs->ring_type != RING_GFX || cs->const_ib.ib_mapped)
 752       return NULL;
 753
 754    if (!amdgpu_get_new_ib(&ws->base, cs, IB_CONST))
 755       return NULL;
 756
 757    cs->csc->request.number_of_ibs = 2;
 758    cs->csc->request.ibs = &cs->csc->ib[IB_CONST];
 759
 760    cs->cst->request.number_of_ibs = 2;
 761    cs->cst->request.ibs = &cs->cst->ib[IB_CONST];
 762
 763    return &cs->const_ib.base;
 764 }
 765
 766 static struct radeon_winsys_cs *
 767 amdgpu_cs_add_const_preamble_ib(struct radeon_winsys_cs *rcs)
 768 {
 769    struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
 770    struct amdgpu_winsys *ws = cs->ctx->ws;
 771
 772    /* only one const preamble IB can be added and only when the const IB has
 773     * also been mapped */
 774    if (cs->ring_type != RING_GFX || !cs->const_ib.ib_mapped ||
 775        cs->const_preamble_ib.ib_mapped)
 776       return NULL;
 777
 778    if (!amdgpu_get_new_ib(&ws->base, cs, IB_CONST_PREAMBLE))
 779       return NULL;
 780
 781    cs->csc->request.number_of_ibs = 3;
 782    cs->csc->request.ibs = &cs->csc->ib[IB_CONST_PREAMBLE];
 783
 784    cs->cst->request.number_of_ibs = 3;
 785    cs->cst->request.ibs = &cs->cst->ib[IB_CONST_PREAMBLE];
 786
 787    return &cs->const_preamble_ib.base;
 788 }
 789
 790 static bool amdgpu_cs_validate(struct radeon_winsys_cs *rcs)
 791 {
 792    return true;
 793 }
 794
 795 static bool amdgpu_cs_check_space(struct radeon_winsys_cs *rcs, unsigned dw)
 796 {
 797    struct amdgpu_ib *ib = amdgpu_ib(rcs);
 798    struct amdgpu_cs *cs = amdgpu_cs_from_ib(ib);
 799    unsigned requested_size = rcs->prev_dw + rcs->current.cdw + dw;
 800    uint64_t va;
 801    uint32_t *new_ptr_ib_size;
 802
 803    assert(rcs->current.cdw <= rcs->current.max_dw);
 804
 805    if (requested_size > amdgpu_ib_max_submit_dwords(ib->ib_type))
 806       return false;
 807
 808    ib->max_ib_size = MAX2(ib->max_ib_size, requested_size);
 809
 810    if (rcs->current.max_dw - rcs->current.cdw >= dw)
 811       return true;
 812
 813    if (!amdgpu_cs_has_chaining(cs))
 814       return false;
 815
 816    /* Allocate a new chunk */
 817    if (rcs->num_prev >= rcs->max_prev) {
 818       unsigned new_max_prev = MAX2(1, 2 * rcs->max_prev);
 819       struct radeon_winsys_cs_chunk *new_prev;
 820
 821       new_prev = REALLOC(rcs->prev,
 822                          sizeof(*new_prev) * rcs->max_prev,
 823                          sizeof(*new_prev) * new_max_prev);
 824       if (!new_prev)
 825          return false;
 826
 827       rcs->prev = new_prev;
 828       rcs->max_prev = new_max_prev;
 829    }
 830
 831    if (!amdgpu_ib_new_buffer(cs->ctx->ws, ib))
 832       return false;
 833
 834    assert(ib->used_ib_space == 0);
 835    va = amdgpu_winsys_bo(ib->big_ib_buffer)->va;
 836
 837    /* This space was originally reserved. */
 838    rcs->current.max_dw += 4;
 839    assert(ib->used_ib_space + 4 * rcs->current.max_dw <= ib->big_ib_buffer->size);
 840
 841    /* Pad with NOPs and add INDIRECT_BUFFER packet */
 842    while ((rcs->current.cdw & 7) != 4)
 843       radeon_emit(rcs, 0xffff1000); /* type3 nop packet */
 844
 845    radeon_emit(rcs, PKT3(ib->ib_type == IB_MAIN ? PKT3_INDIRECT_BUFFER_CIK
 846                                            : PKT3_INDIRECT_BUFFER_CONST, 2, 0));
 847    radeon_emit(rcs, va);
 848    radeon_emit(rcs, va >> 32);
 849    new_ptr_ib_size = &rcs->current.buf[rcs->current.cdw];
 850    radeon_emit(rcs, S_3F2_CHAIN(1) | S_3F2_VALID(1));
 851
 852    assert((rcs->current.cdw & 7) == 0);
 853    assert(rcs->current.cdw <= rcs->current.max_dw);
 854
 855    *ib->ptr_ib_size |= rcs->current.cdw;
 856    ib->ptr_ib_size = new_ptr_ib_size;
 857
 858    /* Hook up the new chunk */
 859    rcs->prev[rcs->num_prev].buf = rcs->current.buf;
 860    rcs->prev[rcs->num_prev].cdw = rcs->current.cdw;
 861    rcs->prev[rcs->num_prev].max_dw = rcs->current.cdw; /* no modifications */
 862    rcs->num_prev++;
 863
 864    ib->base.prev_dw += ib->base.current.cdw;
 865    ib->base.current.cdw = 0;
 866
 867    ib->base.current.buf = (uint32_t*)(ib->ib_mapped + ib->used_ib_space);
 868    ib->base.current.max_dw = ib->big_ib_buffer->size / 4 - amdgpu_cs_epilog_dws(cs->ring_type);
 869
 870    amdgpu_cs_add_buffer(&cs->main.base, ib->big_ib_buffer,
 871                         RADEON_USAGE_READ, 0, RADEON_PRIO_IB1);
 872
 873    return true;
 874 }
 875
 876 static unsigned amdgpu_cs_get_buffer_list(struct radeon_winsys_cs *rcs,
 877                                           struct radeon_bo_list_item *list)
 878 {
 879     struct amdgpu_cs_context *cs = amdgpu_cs(rcs)->csc;
 880     int i;
 881
 882     if (list) {
 883         for (i = 0; i < cs->num_real_buffers; i++) {
 884             list[i].bo_size = cs->real_buffers[i].bo->base.size;
 885             list[i].vm_address = cs->real_buffers[i].bo->va;
 886             list[i].priority_usage = cs->real_buffers[i].u.real.priority_usage;
 887         }
 888     }
 889     return cs->num_real_buffers;
 890 }
 891
 892 DEBUG_GET_ONCE_BOOL_OPTION(all_bos, "RADEON_ALL_BOS", false)
 893
 894 static void amdgpu_add_fence_dependency(struct amdgpu_cs *acs,
 895                                         struct amdgpu_cs_buffer *buffer)
 896 {
 897    struct amdgpu_cs_context *cs = acs->csc;
 898    struct amdgpu_winsys_bo *bo = buffer->bo;
 899    struct amdgpu_cs_fence *dep;
 900    unsigned new_num_fences = 0;
 901
 902    for (unsigned j = 0; j < bo->num_fences; ++j) {
 903       struct amdgpu_fence *bo_fence = (void *)bo->fences[j];
 904       unsigned idx;
 905
 906       if (bo_fence->ctx == acs->ctx &&
 907          bo_fence->fence.ip_type == cs->request.ip_type &&
 908          bo_fence->fence.ip_instance == cs->request.ip_instance &&
 909          bo_fence->fence.ring == cs->request.ring)
 910          continue;
 911
 912       if (amdgpu_fence_wait((void *)bo_fence, 0, false))
 913          continue;
 914
 915       amdgpu_fence_reference(&bo->fences[new_num_fences], bo->fences[j]);
 916       new_num_fences++;
 917
 918       if (!(buffer->usage & RADEON_USAGE_SYNCHRONIZED))
 919          continue;
 920
 921       if (bo_fence->submission_in_progress)
 922          os_wait_until_zero(&bo_fence->submission_in_progress,
 923                             PIPE_TIMEOUT_INFINITE);
 924
 925       idx = cs->request.number_of_dependencies++;
 926       if (idx >= cs->max_dependencies) {
 927          unsigned size;
 928
 929          cs->max_dependencies = idx + 8;
 930          size = cs->max_dependencies * sizeof(struct amdgpu_cs_fence);
 931          cs->request.dependencies = realloc(cs->request.dependencies, size);
 932       }
 933
 934       dep = &cs->request.dependencies[idx];
 935       memcpy(dep, &bo_fence->fence, sizeof(*dep));
 936    }
 937
 938    for (unsigned j = new_num_fences; j < bo->num_fences; ++j)
 939       amdgpu_fence_reference(&bo->fences[j], NULL);
 940
 941    bo->num_fences = new_num_fences;
 942 }
 943
 944 /* Since the kernel driver doesn't synchronize execution between different
 945  * rings automatically, we have to add fence dependencies manually.
 946  */
 947 static void amdgpu_add_fence_dependencies(struct amdgpu_cs *acs)
 948 {
 949    struct amdgpu_cs_context *cs = acs->csc;
 950    int i;
 951
 952    cs->request.number_of_dependencies = 0;
 953
 954    for (i = 0; i < cs->num_real_buffers; i++)
 955       amdgpu_add_fence_dependency(acs, &cs->real_buffers[i]);
 956    for (i = 0; i < cs->num_slab_buffers; i++)
 957       amdgpu_add_fence_dependency(acs, &cs->slab_buffers[i]);
 958 }
 959
 960 static void amdgpu_add_fence(struct amdgpu_winsys_bo *bo,
 961                              struct pipe_fence_handle *fence)
 962 {
 963    if (bo->num_fences >= bo->max_fences) {
 964       unsigned new_max_fences = MAX2(1, bo->max_fences * 2);
 965       struct pipe_fence_handle **new_fences =
 966          REALLOC(bo->fences,
 967                  bo->num_fences * sizeof(*new_fences),
 968                  new_max_fences * sizeof(*new_fences));
 969       if (new_fences) {
 970          bo->fences = new_fences;
 971          bo->max_fences = new_max_fences;
 972       } else {
 973          fprintf(stderr, "amdgpu_add_fence: allocation failure, dropping fence\n");
 974          if (!bo->num_fences)
 975             return;
 976
 977          bo->num_fences--; /* prefer to keep a more recent fence if possible */
 978          amdgpu_fence_reference(&bo->fences[bo->num_fences], NULL);
 979       }
 980    }
 981
 982    bo->fences[bo->num_fences] = NULL;
 983    amdgpu_fence_reference(&bo->fences[bo->num_fences], fence);
 984    bo->num_fences++;
 985 }
 986
 987 void amdgpu_cs_submit_ib(void *job, int thread_index)
 988 {
 989    struct amdgpu_cs *acs = (struct amdgpu_cs*)job;
 990    struct amdgpu_winsys *ws = acs->ctx->ws;
 991    struct amdgpu_cs_context *cs = acs->cst;
 992    int i, r;
 993
 994    cs->request.fence_info.handle = NULL;
 995    if (amdgpu_cs_has_user_fence(cs)) {
 996         cs->request.fence_info.handle = acs->ctx->user_fence_bo;
 997         cs->request.fence_info.offset = acs->ring_type;
 998    }
 999
1000    /* Create the buffer list.
1001     * Use a buffer list containing all allocated buffers if requested.
1002     */
1003    if (debug_get_option_all_bos()) {
1004       struct amdgpu_winsys_bo *bo;
1005       amdgpu_bo_handle *handles;
1006       unsigned num = 0;
1007
1008       pipe_mutex_lock(ws->global_bo_list_lock);
1009
1010       handles = malloc(sizeof(handles[0]) * ws->num_buffers);
1011       if (!handles) {
1012          pipe_mutex_unlock(ws->global_bo_list_lock);
1013          amdgpu_cs_context_cleanup(cs);
1014          cs->error_code = -ENOMEM;
1015          return;
1016       }
1017
1018       LIST_FOR_EACH_ENTRY(bo, &ws->global_bo_list, u.real.global_list_item) {
1019          assert(num < ws->num_buffers);
1020          handles[num++] = bo->bo;
1021       }
1022
1023       r = amdgpu_bo_list_create(ws->dev, ws->num_buffers,
1024                                 handles, NULL,
1025                                 &cs->request.resources);
1026       free(handles);
1027       pipe_mutex_unlock(ws->global_bo_list_lock);
1028    } else {
1029       r = amdgpu_bo_list_create(ws->dev, cs->num_real_buffers,
1030                                 cs->handles, cs->flags,
1031                                 &cs->request.resources);
1032    }
1033
1034    if (r) {
1035       fprintf(stderr, "amdgpu: buffer list creation failed (%d)\n", r);
1036       cs->request.resources = NULL;
1037       amdgpu_fence_signalled(cs->fence);
1038       cs->error_code = r;
1039       goto cleanup;
1040    }
1041
1042    if (acs->ctx->num_rejected_cs)
1043       r = -ECANCELED;
1044    else
1045       r = amdgpu_cs_submit(acs->ctx->ctx, 0, &cs->request, 1);
1046
1047    cs->error_code = r;
1048    if (r) {
1049       if (r == -ENOMEM)
1050          fprintf(stderr, "amdgpu: Not enough memory for command submission.\n");
1051       else if (r == -ECANCELED)
1052          fprintf(stderr, "amdgpu: The CS has been cancelled because the context is lost.\n");
1053       else
1054          fprintf(stderr, "amdgpu: The CS has been rejected, "
1055                  "see dmesg for more information (%i).\n", r);
1056
1057       amdgpu_fence_signalled(cs->fence);
1058
1059       acs->ctx->num_rejected_cs++;
1060       ws->num_total_rejected_cs++;
1061    } else {
1062       /* Success. */
1063       uint64_t *user_fence = NULL;
1064       if (amdgpu_cs_has_user_fence(cs))
1065          user_fence = acs->ctx->user_fence_cpu_address_base +
1066                       cs->request.fence_info.offset;
1067       amdgpu_fence_submitted(cs->fence, &cs->request, user_fence);
1068    }
1069
1070    /* Cleanup. */
1071    if (cs->request.resources)
1072       amdgpu_bo_list_destroy(cs->request.resources);
1073
1074 cleanup:
1075    for (i = 0; i < cs->num_real_buffers; i++)
1076       p_atomic_dec(&cs->real_buffers[i].bo->num_active_ioctls);
1077    for (i = 0; i < cs->num_slab_buffers; i++)
1078       p_atomic_dec(&cs->slab_buffers[i].bo->num_active_ioctls);
1079
1080    amdgpu_cs_context_cleanup(cs);
1081 }
1082
1083 /* Make sure the previous submission is completed. */
1084 void amdgpu_cs_sync_flush(struct radeon_winsys_cs *rcs)
1085 {
1086    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1087
1088    /* Wait for any pending ioctl of this CS to complete. */
1089    util_queue_job_wait(&cs->flush_completed);
1090 }
1091
1092 static int amdgpu_cs_flush(struct radeon_winsys_cs *rcs,
1093                            unsigned flags,
1094                            struct pipe_fence_handle **fence)
1095 {
1096    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1097    struct amdgpu_winsys *ws = cs->ctx->ws;
1098    int error_code = 0;
1099
1100    rcs->current.max_dw += amdgpu_cs_epilog_dws(cs->ring_type);
1101
1102    switch (cs->ring_type) {
1103    case RING_DMA:
1104       /* pad DMA ring to 8 DWs */
1105       if (ws->info.chip_class <= SI) {
1106          while (rcs->current.cdw & 7)
1107             radeon_emit(rcs, 0xf0000000); /* NOP packet */
1108       } else {
1109          while (rcs->current.cdw & 7)
1110             radeon_emit(rcs, 0x00000000); /* NOP packet */
1111       }
1112       break;
1113    case RING_GFX:
1114       /* pad GFX ring to 8 DWs to meet CP fetch alignment requirements */
1115       if (ws->info.gfx_ib_pad_with_type2) {
1116          while (rcs->current.cdw & 7)
1117             radeon_emit(rcs, 0x80000000); /* type2 nop packet */
1118       } else {
1119          while (rcs->current.cdw & 7)
1120             radeon_emit(rcs, 0xffff1000); /* type3 nop packet */
1121       }
1122
1123       /* Also pad the const IB. */
1124       if (cs->const_ib.ib_mapped)
1125          while (!cs->const_ib.base.current.cdw || (cs->const_ib.base.current.cdw & 7))
1126             radeon_emit(&cs->const_ib.base, 0xffff1000); /* type3 nop packet */
1127
1128       if (cs->const_preamble_ib.ib_mapped)
1129          while (!cs->const_preamble_ib.base.current.cdw || (cs->const_preamble_ib.base.current.cdw & 7))
1130             radeon_emit(&cs->const_preamble_ib.base, 0xffff1000);
1131       break;
1132    case RING_UVD:
1133       while (rcs->current.cdw & 15)
1134          radeon_emit(rcs, 0x80000000); /* type2 nop packet */
1135       break;
1136    default:
1137       break;
1138    }
1139
1140    if (rcs->current.cdw > rcs->current.max_dw) {
1141       fprintf(stderr, "amdgpu: command stream overflowed\n");
1142    }
1143
1144    /* If the CS is not empty or overflowed.... */
1145    if (radeon_emitted(&cs->main.base, 0) &&
1146        cs->main.base.current.cdw <= cs->main.base.current.max_dw &&
1147        !debug_get_option_noop()) {
1148       struct amdgpu_cs_context *cur = cs->csc;
1149       unsigned i, num_buffers;
1150
1151       /* Set IB sizes. */
1152       amdgpu_ib_finalize(&cs->main);
1153
1154       if (cs->const_ib.ib_mapped)
1155          amdgpu_ib_finalize(&cs->const_ib);
1156
1157       if (cs->const_preamble_ib.ib_mapped)
1158          amdgpu_ib_finalize(&cs->const_preamble_ib);
1159
1160       /* Create a fence. */
1161       amdgpu_fence_reference(&cur->fence, NULL);
1162       if (cs->next_fence) {
1163          /* just move the reference */
1164          cur->fence = cs->next_fence;
1165          cs->next_fence = NULL;
1166       } else {
1167          cur->fence = amdgpu_fence_create(cs->ctx,
1168                                           cur->request.ip_type,
1169                                           cur->request.ip_instance,
1170                                           cur->request.ring);
1171       }
1172       if (fence)
1173          amdgpu_fence_reference(fence, cur->fence);
1174
1175       amdgpu_cs_sync_flush(rcs);
1176
1177       /* Prepare buffers.
1178        *
1179        * This fence must be held until the submission is queued to ensure
1180        * that the order of fence dependency updates matches the order of
1181        * submissions.
1182        */
1183       pipe_mutex_lock(ws->bo_fence_lock);
1184       amdgpu_add_fence_dependencies(cs);
1185
1186       num_buffers = cur->num_real_buffers;
1187       for (i = 0; i < num_buffers; i++) {
1188          struct amdgpu_winsys_bo *bo = cur->real_buffers[i].bo;
1189          p_atomic_inc(&bo->num_active_ioctls);
1190          amdgpu_add_fence(bo, cur->fence);
1191       }
1192
1193       num_buffers = cur->num_slab_buffers;
1194       for (i = 0; i < num_buffers; i++) {
1195          struct amdgpu_winsys_bo *bo = cur->slab_buffers[i].bo;
1196          p_atomic_inc(&bo->num_active_ioctls);
1197          amdgpu_add_fence(bo, cur->fence);
1198       }
1199
1200       /* Swap command streams. "cst" is going to be submitted. */
1201       cs->csc = cs->cst;
1202       cs->cst = cur;
1203
1204       /* Submit. */
1205       util_queue_add_job(&ws->cs_queue, cs, &cs->flush_completed,
1206                          amdgpu_cs_submit_ib, NULL);
1207       /* The submission has been queued, unlock the fence now. */
1208       pipe_mutex_unlock(ws->bo_fence_lock);
1209
1210       if (!(flags & RADEON_FLUSH_ASYNC)) {
1211          amdgpu_cs_sync_flush(rcs);
1212          error_code = cur->error_code;
1213       }
1214    } else {
1215       amdgpu_cs_context_cleanup(cs->csc);
1216    }
1217
1218    amdgpu_get_new_ib(&ws->base, cs, IB_MAIN);
1219    if (cs->const_ib.ib_mapped)
1220       amdgpu_get_new_ib(&ws->base, cs, IB_CONST);
1221    if (cs->const_preamble_ib.ib_mapped)
1222       amdgpu_get_new_ib(&ws->base, cs, IB_CONST_PREAMBLE);
1223
1224    cs->main.base.used_gart = 0;
1225    cs->main.base.used_vram = 0;
1226
1227    if (cs->ring_type == RING_GFX)
1228       ws->num_gfx_IBs++;
1229    else if (cs->ring_type == RING_DMA)
1230       ws->num_sdma_IBs++;
1231
1232    return error_code;
1233 }
1234
1235 static void amdgpu_cs_destroy(struct radeon_winsys_cs *rcs)
1236 {
1237    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1238
1239    amdgpu_cs_sync_flush(rcs);
1240    util_queue_fence_destroy(&cs->flush_completed);
1241    p_atomic_dec(&cs->ctx->ws->num_cs);
1242    pb_reference(&cs->main.big_ib_buffer, NULL);
1243    FREE(cs->main.base.prev);
1244    pb_reference(&cs->const_ib.big_ib_buffer, NULL);
1245    FREE(cs->const_ib.base.prev);
1246    pb_reference(&cs->const_preamble_ib.big_ib_buffer, NULL);
1247    FREE(cs->const_preamble_ib.base.prev);
1248    amdgpu_destroy_cs_context(&cs->csc1);
1249    amdgpu_destroy_cs_context(&cs->csc2);
1250    amdgpu_fence_reference(&cs->next_fence, NULL);
1251    FREE(cs);
1252 }
1253
1254 static bool amdgpu_bo_is_referenced(struct radeon_winsys_cs *rcs,
1255                                     struct pb_buffer *_buf,
1256                                     enum radeon_bo_usage usage)
1257 {
1258    struct amdgpu_cs *cs = amdgpu_cs(rcs);
1259    struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)_buf;
1260
1261    return amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo, usage);
1262 }
1263
1264 void amdgpu_cs_init_functions(struct amdgpu_winsys *ws)
1265 {
1266    ws->base.ctx_create = amdgpu_ctx_create;
1267    ws->base.ctx_destroy = amdgpu_ctx_destroy;
1268    ws->base.ctx_query_reset_status = amdgpu_ctx_query_reset_status;
1269    ws->base.cs_create = amdgpu_cs_create;
1270    ws->base.cs_add_const_ib = amdgpu_cs_add_const_ib;
1271    ws->base.cs_add_const_preamble_ib = amdgpu_cs_add_const_preamble_ib;
1272    ws->base.cs_destroy = amdgpu_cs_destroy;
1273    ws->base.cs_add_buffer = amdgpu_cs_add_buffer;
1274    ws->base.cs_validate = amdgpu_cs_validate;
1275    ws->base.cs_check_space = amdgpu_cs_check_space;
1276    ws->base.cs_get_buffer_list = amdgpu_cs_get_buffer_list;
1277    ws->base.cs_flush = amdgpu_cs_flush;
1278    ws->base.cs_get_next_fence = amdgpu_cs_get_next_fence;
1279    ws->base.cs_is_buffer_referenced = amdgpu_bo_is_referenced;
1280    ws->base.cs_sync_flush = amdgpu_cs_sync_flush;
1281    ws->base.fence_wait = amdgpu_fence_wait_rel_timeout;
1282    ws->base.fence_reference = amdgpu_fence_reference;
1283 }