src/gallium/drivers/radeonsi/si_buffer.c

   1 /*
   2  * Copyright 2013 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "radeonsi/si_pipe.h"
  26 #include "util/u_memory.h"
  27 #include "util/u_transfer.h"
  28 #include "util/u_upload_mgr.h"
  29
  30 #include <inttypes.h>
  31 #include <stdio.h>
  32
  33 bool si_rings_is_buffer_referenced(struct si_context *sctx, struct pb_buffer *buf,
  34                                    enum radeon_bo_usage usage)
  35 {
  36    if (sctx->ws->cs_is_buffer_referenced(sctx->gfx_cs, buf, usage)) {
  37       return true;
  38    }
  39    if (radeon_emitted(sctx->sdma_cs, 0) &&
  40        sctx->ws->cs_is_buffer_referenced(sctx->sdma_cs, buf, usage)) {
  41       return true;
  42    }
  43    return false;
  44 }
  45
  46 void *si_buffer_map_sync_with_rings(struct si_context *sctx, struct si_resource *resource,
  47                                     unsigned usage)
  48 {
  49    enum radeon_bo_usage rusage = RADEON_USAGE_READWRITE;
  50    bool busy = false;
  51
  52    assert(!(resource->flags & RADEON_FLAG_SPARSE));
  53
  54    if (usage & PIPE_TRANSFER_UNSYNCHRONIZED) {
  55       return sctx->ws->buffer_map(resource->buf, NULL, usage);
  56    }
  57
  58    if (!(usage & PIPE_TRANSFER_WRITE)) {
  59       /* have to wait for the last write */
  60       rusage = RADEON_USAGE_WRITE;
  61    }
  62
  63    if (radeon_emitted(sctx->gfx_cs, sctx->initial_gfx_cs_size) &&
  64        sctx->ws->cs_is_buffer_referenced(sctx->gfx_cs, resource->buf, rusage)) {
  65       if (usage & PIPE_TRANSFER_DONTBLOCK) {
  66          si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  67          return NULL;
  68       } else {
  69          si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  70          busy = true;
  71       }
  72    }
  73    if (radeon_emitted(sctx->sdma_cs, 0) &&
  74        sctx->ws->cs_is_buffer_referenced(sctx->sdma_cs, resource->buf, rusage)) {
  75       if (usage & PIPE_TRANSFER_DONTBLOCK) {
  76          si_flush_dma_cs(sctx, PIPE_FLUSH_ASYNC, NULL);
  77          return NULL;
  78       } else {
  79          si_flush_dma_cs(sctx, 0, NULL);
  80          busy = true;
  81       }
  82    }
  83
  84    if (busy || !sctx->ws->buffer_wait(resource->buf, 0, rusage)) {
  85       if (usage & PIPE_TRANSFER_DONTBLOCK) {
  86          return NULL;
  87       } else {
  88          /* We will be wait for the GPU. Wait for any offloaded
  89           * CS flush to complete to avoid busy-waiting in the winsys. */
  90          sctx->ws->cs_sync_flush(sctx->gfx_cs);
  91          if (sctx->sdma_cs)
  92             sctx->ws->cs_sync_flush(sctx->sdma_cs);
  93       }
  94    }
  95
  96    /* Setting the CS to NULL will prevent doing checks we have done already. */
  97    return sctx->ws->buffer_map(resource->buf, NULL, usage);
  98 }
  99
 100 void si_init_resource_fields(struct si_screen *sscreen, struct si_resource *res, uint64_t size,
 101                              unsigned alignment)
 102 {
 103    struct si_texture *tex = (struct si_texture *)res;
 104
 105    res->bo_size = size;
 106    res->bo_alignment = alignment;
 107    res->flags = 0;
 108    res->texture_handle_allocated = false;
 109    res->image_handle_allocated = false;
 110
 111    switch (res->b.b.usage) {
 112    case PIPE_USAGE_STREAM:
 113       res->flags = RADEON_FLAG_GTT_WC;
 114       /* fall through */
 115    case PIPE_USAGE_STAGING:
 116       /* Transfers are likely to occur more often with these
 117        * resources. */
 118       res->domains = RADEON_DOMAIN_GTT;
 119       break;
 120    case PIPE_USAGE_DYNAMIC:
 121       /* Older kernels didn't always flush the HDP cache before
 122        * CS execution
 123        */
 124       if (!sscreen->info.kernel_flushes_hdp_before_ib) {
 125          res->domains = RADEON_DOMAIN_GTT;
 126          res->flags |= RADEON_FLAG_GTT_WC;
 127          break;
 128       }
 129       /* fall through */
 130    case PIPE_USAGE_DEFAULT:
 131    case PIPE_USAGE_IMMUTABLE:
 132    default:
 133       /* Not listing GTT here improves performance in some
 134        * apps. */
 135       res->domains = RADEON_DOMAIN_VRAM;
 136       res->flags |= RADEON_FLAG_GTT_WC;
 137       break;
 138    }
 139
 140    if (res->b.b.target == PIPE_BUFFER && res->b.b.flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) {
 141       /* Use GTT for all persistent mappings with older
 142        * kernels, because they didn't always flush the HDP
 143        * cache before CS execution.
 144        *
 145        * Write-combined CPU mappings are fine, the kernel
 146        * ensures all CPU writes finish before the GPU
 147        * executes a command stream.
 148        *
 149        * radeon doesn't have good BO move throttling, so put all
 150        * persistent buffers into GTT to prevent VRAM CPU page faults.
 151        */
 152       if (!sscreen->info.kernel_flushes_hdp_before_ib || !sscreen->info.is_amdgpu)
 153          res->domains = RADEON_DOMAIN_GTT;
 154    }
 155
 156    /* Tiled textures are unmappable. Always put them in VRAM. */
 157    if ((res->b.b.target != PIPE_BUFFER && !tex->surface.is_linear) ||
 158        res->b.b.flags & SI_RESOURCE_FLAG_UNMAPPABLE) {
 159       res->domains = RADEON_DOMAIN_VRAM;
 160       res->flags |= RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_GTT_WC;
 161    }
 162
 163    /* Displayable and shareable surfaces are not suballocated. */
 164    if (res->b.b.bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
 165       res->flags |= RADEON_FLAG_NO_SUBALLOC; /* shareable */
 166    else
 167       res->flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
 168
 169    if (sscreen->debug_flags & DBG(NO_WC))
 170       res->flags &= ~RADEON_FLAG_GTT_WC;
 171
 172    if (res->b.b.flags & SI_RESOURCE_FLAG_READ_ONLY)
 173       res->flags |= RADEON_FLAG_READ_ONLY;
 174
 175    if (res->b.b.flags & SI_RESOURCE_FLAG_32BIT)
 176       res->flags |= RADEON_FLAG_32BIT;
 177
 178    /* Set expected VRAM and GART usage for the buffer. */
 179    res->vram_usage = 0;
 180    res->gart_usage = 0;
 181    res->max_forced_staging_uploads = 0;
 182    res->b.max_forced_staging_uploads = 0;
 183
 184    if (res->domains & RADEON_DOMAIN_VRAM) {
 185       res->vram_usage = size;
 186
 187       res->max_forced_staging_uploads = res->b.max_forced_staging_uploads =
 188          sscreen->info.has_dedicated_vram && size >= sscreen->info.vram_vis_size / 4 ? 1 : 0;
 189    } else if (res->domains & RADEON_DOMAIN_GTT) {
 190       res->gart_usage = size;
 191    }
 192 }
 193
 194 bool si_alloc_resource(struct si_screen *sscreen, struct si_resource *res)
 195 {
 196    struct pb_buffer *old_buf, *new_buf;
 197
 198    /* Allocate a new resource. */
 199    new_buf = sscreen->ws->buffer_create(sscreen->ws, res->bo_size, res->bo_alignment, res->domains,
 200                                         res->flags);
 201    if (!new_buf) {
 202       return false;
 203    }
 204
 205    /* Replace the pointer such that if res->buf wasn't NULL, it won't be
 206     * NULL. This should prevent crashes with multiple contexts using
 207     * the same buffer where one of the contexts invalidates it while
 208     * the others are using it. */
 209    old_buf = res->buf;
 210    res->buf = new_buf; /* should be atomic */
 211    res->gpu_address = sscreen->ws->buffer_get_virtual_address(res->buf);
 212
 213    if (res->flags & RADEON_FLAG_32BIT) {
 214       uint64_t start = res->gpu_address;
 215       uint64_t last = start + res->bo_size - 1;
 216       (void)start;
 217       (void)last;
 218
 219       assert((start >> 32) == sscreen->info.address32_hi);
 220       assert((last >> 32) == sscreen->info.address32_hi);
 221    }
 222
 223    pb_reference(&old_buf, NULL);
 224
 225    util_range_set_empty(&res->valid_buffer_range);
 226    res->TC_L2_dirty = false;
 227
 228    /* Print debug information. */
 229    if (sscreen->debug_flags & DBG(VM) && res->b.b.target == PIPE_BUFFER) {
 230       fprintf(stderr, "VM start=0x%" PRIX64 "  end=0x%" PRIX64 " | Buffer %" PRIu64 " bytes\n",
 231               res->gpu_address, res->gpu_address + res->buf->size, res->buf->size);
 232    }
 233
 234    if (res->b.b.flags & SI_RESOURCE_FLAG_CLEAR)
 235       si_screen_clear_buffer(sscreen, &res->b.b, 0, res->bo_size, 0);
 236
 237    return true;
 238 }
 239
 240 static void si_buffer_destroy(struct pipe_screen *screen, struct pipe_resource *buf)
 241 {
 242    struct si_resource *buffer = si_resource(buf);
 243
 244    threaded_resource_deinit(buf);
 245    util_range_destroy(&buffer->valid_buffer_range);
 246    pb_reference(&buffer->buf, NULL);
 247    FREE(buffer);
 248 }
 249
 250 /* Reallocate the buffer a update all resource bindings where the buffer is
 251  * bound.
 252  *
 253  * This is used to avoid CPU-GPU synchronizations, because it makes the buffer
 254  * idle by discarding its contents.
 255  */
 256 static bool si_invalidate_buffer(struct si_context *sctx, struct si_resource *buf)
 257 {
 258    /* Shared buffers can't be reallocated. */
 259    if (buf->b.is_shared)
 260       return false;
 261
 262    /* Sparse buffers can't be reallocated. */
 263    if (buf->flags & RADEON_FLAG_SPARSE)
 264       return false;
 265
 266    /* In AMD_pinned_memory, the user pointer association only gets
 267     * broken when the buffer is explicitly re-allocated.
 268     */
 269    if (buf->b.is_user_ptr)
 270       return false;
 271
 272    /* Check if mapping this buffer would cause waiting for the GPU. */
 273    if (si_rings_is_buffer_referenced(sctx, buf->buf, RADEON_USAGE_READWRITE) ||
 274        !sctx->ws->buffer_wait(buf->buf, 0, RADEON_USAGE_READWRITE)) {
 275       /* Reallocate the buffer in the same pipe_resource. */
 276       si_alloc_resource(sctx->screen, buf);
 277       si_rebind_buffer(sctx, &buf->b.b);
 278    } else {
 279       util_range_set_empty(&buf->valid_buffer_range);
 280    }
 281
 282    return true;
 283 }
 284
 285 /* Replace the storage of dst with src. */
 286 void si_replace_buffer_storage(struct pipe_context *ctx, struct pipe_resource *dst,
 287                                struct pipe_resource *src)
 288 {
 289    struct si_context *sctx = (struct si_context *)ctx;
 290    struct si_resource *sdst = si_resource(dst);
 291    struct si_resource *ssrc = si_resource(src);
 292
 293    pb_reference(&sdst->buf, ssrc->buf);
 294    sdst->gpu_address = ssrc->gpu_address;
 295    sdst->b.b.bind = ssrc->b.b.bind;
 296    sdst->b.max_forced_staging_uploads = ssrc->b.max_forced_staging_uploads;
 297    sdst->max_forced_staging_uploads = ssrc->max_forced_staging_uploads;
 298    sdst->flags = ssrc->flags;
 299
 300    assert(sdst->vram_usage == ssrc->vram_usage);
 301    assert(sdst->gart_usage == ssrc->gart_usage);
 302    assert(sdst->bo_size == ssrc->bo_size);
 303    assert(sdst->bo_alignment == ssrc->bo_alignment);
 304    assert(sdst->domains == ssrc->domains);
 305
 306    si_rebind_buffer(sctx, dst);
 307 }
 308
 309 static void si_invalidate_resource(struct pipe_context *ctx, struct pipe_resource *resource)
 310 {
 311    struct si_context *sctx = (struct si_context *)ctx;
 312    struct si_resource *buf = si_resource(resource);
 313
 314    /* We currently only do anyting here for buffers */
 315    if (resource->target == PIPE_BUFFER)
 316       (void)si_invalidate_buffer(sctx, buf);
 317 }
 318
 319 static void *si_buffer_get_transfer(struct pipe_context *ctx, struct pipe_resource *resource,
 320                                     unsigned usage, const struct pipe_box *box,
 321                                     struct pipe_transfer **ptransfer, void *data,
 322                                     struct si_resource *staging, unsigned offset)
 323 {
 324    struct si_context *sctx = (struct si_context *)ctx;
 325    struct si_transfer *transfer;
 326
 327    if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
 328       transfer = slab_alloc(&sctx->pool_transfers_unsync);
 329    else
 330       transfer = slab_alloc(&sctx->pool_transfers);
 331
 332    transfer->b.b.resource = NULL;
 333    pipe_resource_reference(&transfer->b.b.resource, resource);
 334    transfer->b.b.level = 0;
 335    transfer->b.b.usage = usage;
 336    transfer->b.b.box = *box;
 337    transfer->b.b.stride = 0;
 338    transfer->b.b.layer_stride = 0;
 339    transfer->b.staging = NULL;
 340    transfer->offset = offset;
 341    transfer->staging = staging;
 342    *ptransfer = &transfer->b.b;
 343    return data;
 344 }
 345
 346 static void *si_buffer_transfer_map(struct pipe_context *ctx, struct pipe_resource *resource,
 347                                     unsigned level, unsigned usage, const struct pipe_box *box,
 348                                     struct pipe_transfer **ptransfer)
 349 {
 350    struct si_context *sctx = (struct si_context *)ctx;
 351    struct si_resource *buf = si_resource(resource);
 352    uint8_t *data;
 353
 354    assert(box->x + box->width <= resource->width0);
 355
 356    /* From GL_AMD_pinned_memory issues:
 357     *
 358     *     4) Is glMapBuffer on a shared buffer guaranteed to return the
 359     *        same system address which was specified at creation time?
 360     *
 361     *        RESOLVED: NO. The GL implementation might return a different
 362     *        virtual mapping of that memory, although the same physical
 363     *        page will be used.
 364     *
 365     * So don't ever use staging buffers.
 366     */
 367    if (buf->b.is_user_ptr)
 368       usage |= PIPE_TRANSFER_PERSISTENT;
 369
 370    /* See if the buffer range being mapped has never been initialized,
 371     * in which case it can be mapped unsynchronized. */
 372    if (!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED)) &&
 373        usage & PIPE_TRANSFER_WRITE && !buf->b.is_shared &&
 374        !util_ranges_intersect(&buf->valid_buffer_range, box->x, box->x + box->width)) {
 375       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 376    }
 377
 378    /* If discarding the entire range, discard the whole resource instead. */
 379    if (usage & PIPE_TRANSFER_DISCARD_RANGE && box->x == 0 && box->width == resource->width0) {
 380       usage |= PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE;
 381    }
 382
 383    /* If a buffer in VRAM is too large and the range is discarded, don't
 384     * map it directly. This makes sure that the buffer stays in VRAM.
 385     */
 386    bool force_discard_range = false;
 387    if (usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | PIPE_TRANSFER_DISCARD_RANGE) &&
 388        !(usage & PIPE_TRANSFER_PERSISTENT) &&
 389        /* Try not to decrement the counter if it's not positive. Still racy,
 390         * but it makes it harder to wrap the counter from INT_MIN to INT_MAX. */
 391        buf->max_forced_staging_uploads > 0 &&
 392        p_atomic_dec_return(&buf->max_forced_staging_uploads) >= 0) {
 393       usage &= ~(PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | PIPE_TRANSFER_UNSYNCHRONIZED);
 394       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 395       force_discard_range = true;
 396    }
 397
 398    if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE &&
 399        !(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | TC_TRANSFER_MAP_NO_INVALIDATE))) {
 400       assert(usage & PIPE_TRANSFER_WRITE);
 401
 402       if (si_invalidate_buffer(sctx, buf)) {
 403          /* At this point, the buffer is always idle. */
 404          usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 405       } else {
 406          /* Fall back to a temporary buffer. */
 407          usage |= PIPE_TRANSFER_DISCARD_RANGE;
 408       }
 409    }
 410
 411    if (usage & PIPE_TRANSFER_FLUSH_EXPLICIT &&
 412        buf->b.b.flags & SI_RESOURCE_FLAG_UPLOAD_FLUSH_EXPLICIT_VIA_SDMA) {
 413       usage &= ~(PIPE_TRANSFER_UNSYNCHRONIZED | PIPE_TRANSFER_PERSISTENT);
 414       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 415       force_discard_range = true;
 416    }
 417
 418    if (usage & PIPE_TRANSFER_DISCARD_RANGE &&
 419        ((!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | PIPE_TRANSFER_PERSISTENT))) ||
 420         (buf->flags & RADEON_FLAG_SPARSE))) {
 421       assert(usage & PIPE_TRANSFER_WRITE);
 422
 423       /* Check if mapping this buffer would cause waiting for the GPU.
 424        */
 425       if (buf->flags & RADEON_FLAG_SPARSE || force_discard_range ||
 426           si_rings_is_buffer_referenced(sctx, buf->buf, RADEON_USAGE_READWRITE) ||
 427           !sctx->ws->buffer_wait(buf->buf, 0, RADEON_USAGE_READWRITE)) {
 428          /* Do a wait-free write-only transfer using a temporary buffer. */
 429          struct u_upload_mgr *uploader;
 430          struct si_resource *staging = NULL;
 431          unsigned offset;
 432
 433          /* If we are not called from the driver thread, we have
 434           * to use the uploader from u_threaded_context, which is
 435           * local to the calling thread.
 436           */
 437          if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
 438             uploader = sctx->tc->base.stream_uploader;
 439          else
 440             uploader = sctx->b.stream_uploader;
 441
 442          u_upload_alloc(uploader, 0, box->width + (box->x % SI_MAP_BUFFER_ALIGNMENT),
 443                         sctx->screen->info.tcc_cache_line_size, &offset,
 444                         (struct pipe_resource **)&staging, (void **)&data);
 445
 446          if (staging) {
 447             data += box->x % SI_MAP_BUFFER_ALIGNMENT;
 448             return si_buffer_get_transfer(ctx, resource, usage, box, ptransfer, data, staging,
 449                                           offset);
 450          } else if (buf->flags & RADEON_FLAG_SPARSE) {
 451             return NULL;
 452          }
 453       } else {
 454          /* At this point, the buffer is always idle (we checked it above). */
 455          usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 456       }
 457    }
 458    /* Use a staging buffer in cached GTT for reads. */
 459    else if (((usage & PIPE_TRANSFER_READ) && !(usage & PIPE_TRANSFER_PERSISTENT) &&
 460              (buf->domains & RADEON_DOMAIN_VRAM || buf->flags & RADEON_FLAG_GTT_WC)) ||
 461             (buf->flags & RADEON_FLAG_SPARSE)) {
 462       struct si_resource *staging;
 463
 464       assert(!(usage & TC_TRANSFER_MAP_THREADED_UNSYNC));
 465       staging = si_resource(pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_STAGING,
 466                                                box->width + (box->x % SI_MAP_BUFFER_ALIGNMENT)));
 467       if (staging) {
 468          /* Copy the VRAM buffer to the staging buffer. */
 469          si_sdma_copy_buffer(sctx, &staging->b.b, resource, box->x % SI_MAP_BUFFER_ALIGNMENT,
 470                              box->x, box->width);
 471
 472          data = si_buffer_map_sync_with_rings(sctx, staging, usage & ~PIPE_TRANSFER_UNSYNCHRONIZED);
 473          if (!data) {
 474             si_resource_reference(&staging, NULL);
 475             return NULL;
 476          }
 477          data += box->x % SI_MAP_BUFFER_ALIGNMENT;
 478
 479          return si_buffer_get_transfer(ctx, resource, usage, box, ptransfer, data, staging, 0);
 480       } else if (buf->flags & RADEON_FLAG_SPARSE) {
 481          return NULL;
 482       }
 483    }
 484
 485    data = si_buffer_map_sync_with_rings(sctx, buf, usage);
 486    if (!data) {
 487       return NULL;
 488    }
 489    data += box->x;
 490
 491    return si_buffer_get_transfer(ctx, resource, usage, box, ptransfer, data, NULL, 0);
 492 }
 493
 494 static void si_buffer_do_flush_region(struct pipe_context *ctx, struct pipe_transfer *transfer,
 495                                       const struct pipe_box *box)
 496 {
 497    struct si_context *sctx = (struct si_context *)ctx;
 498    struct si_transfer *stransfer = (struct si_transfer *)transfer;
 499    struct si_resource *buf = si_resource(transfer->resource);
 500
 501    if (stransfer->staging) {
 502       unsigned src_offset =
 503          stransfer->offset + transfer->box.x % SI_MAP_BUFFER_ALIGNMENT + (box->x - transfer->box.x);
 504
 505       if (buf->b.b.flags & SI_RESOURCE_FLAG_UPLOAD_FLUSH_EXPLICIT_VIA_SDMA) {
 506          /* This should be true for all uploaders. */
 507          assert(transfer->box.x == 0);
 508
 509          /* Find a previous upload and extend its range. The last
 510           * upload is likely to be at the end of the list.
 511           */
 512          for (int i = sctx->num_sdma_uploads - 1; i >= 0; i--) {
 513             struct si_sdma_upload *up = &sctx->sdma_uploads[i];
 514
 515             if (up->dst != buf)
 516                continue;
 517
 518             assert(up->src == stransfer->staging);
 519             assert(box->x > up->dst_offset);
 520             up->size = box->x + box->width - up->dst_offset;
 521             return;
 522          }
 523
 524          /* Enlarge the array if it's full. */
 525          if (sctx->num_sdma_uploads == sctx->max_sdma_uploads) {
 526             unsigned size;
 527
 528             sctx->max_sdma_uploads += 4;
 529             size = sctx->max_sdma_uploads * sizeof(sctx->sdma_uploads[0]);
 530             sctx->sdma_uploads = realloc(sctx->sdma_uploads, size);
 531          }
 532
 533          /* Add a new upload. */
 534          struct si_sdma_upload *up = &sctx->sdma_uploads[sctx->num_sdma_uploads++];
 535          up->dst = up->src = NULL;
 536          si_resource_reference(&up->dst, buf);
 537          si_resource_reference(&up->src, stransfer->staging);
 538          up->dst_offset = box->x;
 539          up->src_offset = src_offset;
 540          up->size = box->width;
 541          return;
 542       }
 543
 544       /* Copy the staging buffer into the original one. */
 545       si_copy_buffer(sctx, transfer->resource, &stransfer->staging->b.b, box->x, src_offset,
 546                      box->width);
 547    }
 548
 549    util_range_add(&buf->b.b, &buf->valid_buffer_range, box->x, box->x + box->width);
 550 }
 551
 552 static void si_buffer_flush_region(struct pipe_context *ctx, struct pipe_transfer *transfer,
 553                                    const struct pipe_box *rel_box)
 554 {
 555    unsigned required_usage = PIPE_TRANSFER_WRITE | PIPE_TRANSFER_FLUSH_EXPLICIT;
 556
 557    if ((transfer->usage & required_usage) == required_usage) {
 558       struct pipe_box box;
 559
 560       u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
 561       si_buffer_do_flush_region(ctx, transfer, &box);
 562    }
 563 }
 564
 565 static void si_buffer_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *transfer)
 566 {
 567    struct si_context *sctx = (struct si_context *)ctx;
 568    struct si_transfer *stransfer = (struct si_transfer *)transfer;
 569
 570    if (transfer->usage & PIPE_TRANSFER_WRITE && !(transfer->usage & PIPE_TRANSFER_FLUSH_EXPLICIT))
 571       si_buffer_do_flush_region(ctx, transfer, &transfer->box);
 572
 573    si_resource_reference(&stransfer->staging, NULL);
 574    assert(stransfer->b.staging == NULL); /* for threaded context only */
 575    pipe_resource_reference(&transfer->resource, NULL);
 576
 577    /* Don't use pool_transfers_unsync. We are always in the driver
 578     * thread. */
 579    slab_free(&sctx->pool_transfers, transfer);
 580 }
 581
 582 static void si_buffer_subdata(struct pipe_context *ctx, struct pipe_resource *buffer,
 583                               unsigned usage, unsigned offset, unsigned size, const void *data)
 584 {
 585    struct pipe_transfer *transfer = NULL;
 586    struct pipe_box box;
 587    uint8_t *map = NULL;
 588
 589    usage |= PIPE_TRANSFER_WRITE;
 590
 591    if (!(usage & PIPE_TRANSFER_MAP_DIRECTLY))
 592       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 593
 594    u_box_1d(offset, size, &box);
 595    map = si_buffer_transfer_map(ctx, buffer, 0, usage, &box, &transfer);
 596    if (!map)
 597       return;
 598
 599    memcpy(map, data, size);
 600    si_buffer_transfer_unmap(ctx, transfer);
 601 }
 602
 603 static const struct u_resource_vtbl si_buffer_vtbl = {
 604    NULL,                     /* get_handle */
 605    si_buffer_destroy,        /* resource_destroy */
 606    si_buffer_transfer_map,   /* transfer_map */
 607    si_buffer_flush_region,   /* transfer_flush_region */
 608    si_buffer_transfer_unmap, /* transfer_unmap */
 609 };
 610
 611 static struct si_resource *si_alloc_buffer_struct(struct pipe_screen *screen,
 612                                                   const struct pipe_resource *templ)
 613 {
 614    struct si_resource *buf;
 615
 616    buf = MALLOC_STRUCT(si_resource);
 617
 618    buf->b.b = *templ;
 619    buf->b.b.next = NULL;
 620    pipe_reference_init(&buf->b.b.reference, 1);
 621    buf->b.b.screen = screen;
 622
 623    buf->b.vtbl = &si_buffer_vtbl;
 624    threaded_resource_init(&buf->b.b);
 625
 626    buf->buf = NULL;
 627    buf->bind_history = 0;
 628    buf->TC_L2_dirty = false;
 629    util_range_init(&buf->valid_buffer_range);
 630    return buf;
 631 }
 632
 633 static struct pipe_resource *si_buffer_create(struct pipe_screen *screen,
 634                                               const struct pipe_resource *templ, unsigned alignment)
 635 {
 636    struct si_screen *sscreen = (struct si_screen *)screen;
 637    struct si_resource *buf = si_alloc_buffer_struct(screen, templ);
 638
 639    if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 640       buf->b.b.flags |= SI_RESOURCE_FLAG_UNMAPPABLE;
 641
 642    si_init_resource_fields(sscreen, buf, templ->width0, alignment);
 643
 644    if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 645       buf->flags |= RADEON_FLAG_SPARSE;
 646
 647    if (!si_alloc_resource(sscreen, buf)) {
 648       FREE(buf);
 649       return NULL;
 650    }
 651    return &buf->b.b;
 652 }
 653
 654 struct pipe_resource *pipe_aligned_buffer_create(struct pipe_screen *screen, unsigned flags,
 655                                                  unsigned usage, unsigned size, unsigned alignment)
 656 {
 657    struct pipe_resource buffer;
 658
 659    memset(&buffer, 0, sizeof buffer);
 660    buffer.target = PIPE_BUFFER;
 661    buffer.format = PIPE_FORMAT_R8_UNORM;
 662    buffer.bind = 0;
 663    buffer.usage = usage;
 664    buffer.flags = flags;
 665    buffer.width0 = size;
 666    buffer.height0 = 1;
 667    buffer.depth0 = 1;
 668    buffer.array_size = 1;
 669    return si_buffer_create(screen, &buffer, alignment);
 670 }
 671
 672 struct si_resource *si_aligned_buffer_create(struct pipe_screen *screen, unsigned flags,
 673                                              unsigned usage, unsigned size, unsigned alignment)
 674 {
 675    return si_resource(pipe_aligned_buffer_create(screen, flags, usage, size, alignment));
 676 }
 677
 678 static struct pipe_resource *si_buffer_from_user_memory(struct pipe_screen *screen,
 679                                                         const struct pipe_resource *templ,
 680                                                         void *user_memory)
 681 {
 682    struct si_screen *sscreen = (struct si_screen *)screen;
 683    struct radeon_winsys *ws = sscreen->ws;
 684    struct si_resource *buf = si_alloc_buffer_struct(screen, templ);
 685
 686    buf->domains = RADEON_DOMAIN_GTT;
 687    buf->flags = 0;
 688    buf->b.is_user_ptr = true;
 689    util_range_add(&buf->b.b, &buf->valid_buffer_range, 0, templ->width0);
 690    util_range_add(&buf->b.b, &buf->b.valid_buffer_range, 0, templ->width0);
 691
 692    /* Convert a user pointer to a buffer. */
 693    buf->buf = ws->buffer_from_ptr(ws, user_memory, templ->width0);
 694    if (!buf->buf) {
 695       FREE(buf);
 696       return NULL;
 697    }
 698
 699    buf->gpu_address = ws->buffer_get_virtual_address(buf->buf);
 700    buf->vram_usage = 0;
 701    buf->gart_usage = templ->width0;
 702
 703    return &buf->b.b;
 704 }
 705
 706 static struct pipe_resource *si_resource_create(struct pipe_screen *screen,
 707                                                 const struct pipe_resource *templ)
 708 {
 709    if (templ->target == PIPE_BUFFER) {
 710       return si_buffer_create(screen, templ, 256);
 711    } else {
 712       return si_texture_create(screen, templ);
 713    }
 714 }
 715
 716 static bool si_resource_commit(struct pipe_context *pctx, struct pipe_resource *resource,
 717                                unsigned level, struct pipe_box *box, bool commit)
 718 {
 719    struct si_context *ctx = (struct si_context *)pctx;
 720    struct si_resource *res = si_resource(resource);
 721
 722    /*
 723     * Since buffer commitment changes cannot be pipelined, we need to
 724     * (a) flush any pending commands that refer to the buffer we're about
 725     *     to change, and
 726     * (b) wait for threaded submit to finish, including those that were
 727     *     triggered by some other, earlier operation.
 728     */
 729    if (radeon_emitted(ctx->gfx_cs, ctx->initial_gfx_cs_size) &&
 730        ctx->ws->cs_is_buffer_referenced(ctx->gfx_cs, res->buf, RADEON_USAGE_READWRITE)) {
 731       si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
 732    }
 733    if (radeon_emitted(ctx->sdma_cs, 0) &&
 734        ctx->ws->cs_is_buffer_referenced(ctx->sdma_cs, res->buf, RADEON_USAGE_READWRITE)) {
 735       si_flush_dma_cs(ctx, PIPE_FLUSH_ASYNC, NULL);
 736    }
 737
 738    if (ctx->sdma_cs)
 739       ctx->ws->cs_sync_flush(ctx->sdma_cs);
 740    ctx->ws->cs_sync_flush(ctx->gfx_cs);
 741
 742    assert(resource->target == PIPE_BUFFER);
 743
 744    return ctx->ws->buffer_commit(res->buf, box->x, box->width, commit);
 745 }
 746
 747 void si_init_screen_buffer_functions(struct si_screen *sscreen)
 748 {
 749    sscreen->b.resource_create = si_resource_create;
 750    sscreen->b.resource_destroy = u_resource_destroy_vtbl;
 751    sscreen->b.resource_from_user_memory = si_buffer_from_user_memory;
 752 }
 753
 754 void si_init_buffer_functions(struct si_context *sctx)
 755 {
 756    sctx->b.invalidate_resource = si_invalidate_resource;
 757    sctx->b.transfer_map = u_transfer_map_vtbl;
 758    sctx->b.transfer_flush_region = u_transfer_flush_region_vtbl;
 759    sctx->b.transfer_unmap = u_transfer_unmap_vtbl;
 760    sctx->b.texture_subdata = u_default_texture_subdata;
 761    sctx->b.buffer_subdata = si_buffer_subdata;
 762    sctx->b.resource_commit = si_resource_commit;
 763 }