src/gallium/drivers/radeonsi/si_buffer.c

   1 /*
   2  * Copyright 2013 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "radeonsi/si_pipe.h"
  26 #include "util/u_memory.h"
  27 #include "util/u_transfer.h"
  28 #include "util/u_upload_mgr.h"
  29
  30 #include <inttypes.h>
  31 #include <stdio.h>
  32
  33 bool si_rings_is_buffer_referenced(struct si_context *sctx, struct pb_buffer *buf,
  34                                    enum radeon_bo_usage usage)
  35 {
  36    if (sctx->ws->cs_is_buffer_referenced(sctx->gfx_cs, buf, usage)) {
  37       return true;
  38    }
  39    if (radeon_emitted(sctx->sdma_cs, 0) &&
  40        sctx->ws->cs_is_buffer_referenced(sctx->sdma_cs, buf, usage)) {
  41       return true;
  42    }
  43    return false;
  44 }
  45
  46 void *si_buffer_map_sync_with_rings(struct si_context *sctx, struct si_resource *resource,
  47                                     unsigned usage)
  48 {
  49    enum radeon_bo_usage rusage = RADEON_USAGE_READWRITE;
  50    bool busy = false;
  51
  52    assert(!(resource->flags & RADEON_FLAG_SPARSE));
  53
  54    if (usage & PIPE_TRANSFER_UNSYNCHRONIZED) {
  55       return sctx->ws->buffer_map(resource->buf, NULL, usage);
  56    }
  57
  58    if (!(usage & PIPE_TRANSFER_WRITE)) {
  59       /* have to wait for the last write */
  60       rusage = RADEON_USAGE_WRITE;
  61    }
  62
  63    if (radeon_emitted(sctx->gfx_cs, sctx->initial_gfx_cs_size) &&
  64        sctx->ws->cs_is_buffer_referenced(sctx->gfx_cs, resource->buf, rusage)) {
  65       if (usage & PIPE_TRANSFER_DONTBLOCK) {
  66          si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  67          return NULL;
  68       } else {
  69          si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
  70          busy = true;
  71       }
  72    }
  73    if (radeon_emitted(sctx->sdma_cs, 0) &&
  74        sctx->ws->cs_is_buffer_referenced(sctx->sdma_cs, resource->buf, rusage)) {
  75       if (usage & PIPE_TRANSFER_DONTBLOCK) {
  76          si_flush_dma_cs(sctx, PIPE_FLUSH_ASYNC, NULL);
  77          return NULL;
  78       } else {
  79          si_flush_dma_cs(sctx, 0, NULL);
  80          busy = true;
  81       }
  82    }
  83
  84    if (busy || !sctx->ws->buffer_wait(resource->buf, 0, rusage)) {
  85       if (usage & PIPE_TRANSFER_DONTBLOCK) {
  86          return NULL;
  87       } else {
  88          /* We will be wait for the GPU. Wait for any offloaded
  89           * CS flush to complete to avoid busy-waiting in the winsys. */
  90          sctx->ws->cs_sync_flush(sctx->gfx_cs);
  91          if (sctx->sdma_cs)
  92             sctx->ws->cs_sync_flush(sctx->sdma_cs);
  93       }
  94    }
  95
  96    /* Setting the CS to NULL will prevent doing checks we have done already. */
  97    return sctx->ws->buffer_map(resource->buf, NULL, usage);
  98 }
  99
 100 void si_init_resource_fields(struct si_screen *sscreen, struct si_resource *res, uint64_t size,
 101                              unsigned alignment)
 102 {
 103    struct si_texture *tex = (struct si_texture *)res;
 104
 105    res->bo_size = size;
 106    res->bo_alignment = alignment;
 107    res->flags = 0;
 108    res->texture_handle_allocated = false;
 109    res->image_handle_allocated = false;
 110
 111    switch (res->b.b.usage) {
 112    case PIPE_USAGE_STREAM:
 113       res->flags = RADEON_FLAG_GTT_WC;
 114       /* fall through */
 115    case PIPE_USAGE_STAGING:
 116       /* Transfers are likely to occur more often with these
 117        * resources. */
 118       res->domains = RADEON_DOMAIN_GTT;
 119       break;
 120    case PIPE_USAGE_DYNAMIC:
 121       /* Older kernels didn't always flush the HDP cache before
 122        * CS execution
 123        */
 124       if (!sscreen->info.kernel_flushes_hdp_before_ib) {
 125          res->domains = RADEON_DOMAIN_GTT;
 126          res->flags |= RADEON_FLAG_GTT_WC;
 127          break;
 128       }
 129       /* fall through */
 130    case PIPE_USAGE_DEFAULT:
 131    case PIPE_USAGE_IMMUTABLE:
 132    default:
 133       /* Not listing GTT here improves performance in some
 134        * apps. */
 135       res->domains = RADEON_DOMAIN_VRAM;
 136       res->flags |= RADEON_FLAG_GTT_WC;
 137       break;
 138    }
 139
 140    if (res->b.b.target == PIPE_BUFFER && res->b.b.flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) {
 141       /* Use GTT for all persistent mappings with older
 142        * kernels, because they didn't always flush the HDP
 143        * cache before CS execution.
 144        *
 145        * Write-combined CPU mappings are fine, the kernel
 146        * ensures all CPU writes finish before the GPU
 147        * executes a command stream.
 148        *
 149        * radeon doesn't have good BO move throttling, so put all
 150        * persistent buffers into GTT to prevent VRAM CPU page faults.
 151        */
 152       if (!sscreen->info.kernel_flushes_hdp_before_ib || !sscreen->info.is_amdgpu)
 153          res->domains = RADEON_DOMAIN_GTT;
 154    }
 155
 156    /* Tiled textures are unmappable. Always put them in VRAM. */
 157    if ((res->b.b.target != PIPE_BUFFER && !tex->surface.is_linear) ||
 158        res->b.b.flags & SI_RESOURCE_FLAG_UNMAPPABLE) {
 159       res->domains = RADEON_DOMAIN_VRAM;
 160       res->flags |= RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_GTT_WC;
 161    }
 162
 163    /* Displayable and shareable surfaces are not suballocated. */
 164    if (res->b.b.bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
 165       res->flags |= RADEON_FLAG_NO_SUBALLOC; /* shareable */
 166    else
 167       res->flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
 168
 169    if (sscreen->ws->ws_is_secure(sscreen->ws)) {
 170       if (res->b.b.bind & PIPE_BIND_SCANOUT)
 171          res->flags |= RADEON_FLAG_ENCRYPTED;
 172       if (res->b.b.flags & PIPE_RESOURCE_FLAG_ENCRYPTED)
 173          res->flags |= RADEON_FLAG_ENCRYPTED;
 174    }
 175
 176    if (sscreen->debug_flags & DBG(NO_WC))
 177       res->flags &= ~RADEON_FLAG_GTT_WC;
 178
 179    if (res->b.b.flags & SI_RESOURCE_FLAG_READ_ONLY)
 180       res->flags |= RADEON_FLAG_READ_ONLY;
 181
 182    if (res->b.b.flags & SI_RESOURCE_FLAG_32BIT)
 183       res->flags |= RADEON_FLAG_32BIT;
 184
 185    /* Set expected VRAM and GART usage for the buffer. */
 186    res->vram_usage = 0;
 187    res->gart_usage = 0;
 188    res->max_forced_staging_uploads = 0;
 189    res->b.max_forced_staging_uploads = 0;
 190
 191    if (res->domains & RADEON_DOMAIN_VRAM) {
 192       res->vram_usage = size;
 193
 194       res->max_forced_staging_uploads = res->b.max_forced_staging_uploads =
 195          sscreen->info.has_dedicated_vram && size >= sscreen->info.vram_vis_size / 4 ? 1 : 0;
 196    } else if (res->domains & RADEON_DOMAIN_GTT) {
 197       res->gart_usage = size;
 198    }
 199 }
 200
 201 bool si_alloc_resource(struct si_screen *sscreen, struct si_resource *res)
 202 {
 203    struct pb_buffer *old_buf, *new_buf;
 204
 205    /* Allocate a new resource. */
 206    new_buf = sscreen->ws->buffer_create(sscreen->ws, res->bo_size, res->bo_alignment, res->domains,
 207                                         res->flags);
 208    if (!new_buf) {
 209       return false;
 210    }
 211
 212    /* Replace the pointer such that if res->buf wasn't NULL, it won't be
 213     * NULL. This should prevent crashes with multiple contexts using
 214     * the same buffer where one of the contexts invalidates it while
 215     * the others are using it. */
 216    old_buf = res->buf;
 217    res->buf = new_buf; /* should be atomic */
 218    res->gpu_address = sscreen->ws->buffer_get_virtual_address(res->buf);
 219
 220    if (res->flags & RADEON_FLAG_32BIT) {
 221       uint64_t start = res->gpu_address;
 222       uint64_t last = start + res->bo_size - 1;
 223       (void)start;
 224       (void)last;
 225
 226       assert((start >> 32) == sscreen->info.address32_hi);
 227       assert((last >> 32) == sscreen->info.address32_hi);
 228    }
 229
 230    pb_reference(&old_buf, NULL);
 231
 232    util_range_set_empty(&res->valid_buffer_range);
 233    res->TC_L2_dirty = false;
 234
 235    /* Print debug information. */
 236    if (sscreen->debug_flags & DBG(VM) && res->b.b.target == PIPE_BUFFER) {
 237       fprintf(stderr, "VM start=0x%" PRIX64 "  end=0x%" PRIX64 " | Buffer %" PRIu64 " bytes\n",
 238               res->gpu_address, res->gpu_address + res->buf->size, res->buf->size);
 239    }
 240
 241    if (res->b.b.flags & SI_RESOURCE_FLAG_CLEAR)
 242       si_screen_clear_buffer(sscreen, &res->b.b, 0, res->bo_size, 0);
 243
 244    return true;
 245 }
 246
 247 static void si_buffer_destroy(struct pipe_screen *screen, struct pipe_resource *buf)
 248 {
 249    struct si_resource *buffer = si_resource(buf);
 250
 251    threaded_resource_deinit(buf);
 252    util_range_destroy(&buffer->valid_buffer_range);
 253    pb_reference(&buffer->buf, NULL);
 254    FREE(buffer);
 255 }
 256
 257 /* Reallocate the buffer a update all resource bindings where the buffer is
 258  * bound.
 259  *
 260  * This is used to avoid CPU-GPU synchronizations, because it makes the buffer
 261  * idle by discarding its contents.
 262  */
 263 static bool si_invalidate_buffer(struct si_context *sctx, struct si_resource *buf)
 264 {
 265    /* Shared buffers can't be reallocated. */
 266    if (buf->b.is_shared)
 267       return false;
 268
 269    /* Sparse buffers can't be reallocated. */
 270    if (buf->flags & RADEON_FLAG_SPARSE)
 271       return false;
 272
 273    /* In AMD_pinned_memory, the user pointer association only gets
 274     * broken when the buffer is explicitly re-allocated.
 275     */
 276    if (buf->b.is_user_ptr)
 277       return false;
 278
 279    /* Check if mapping this buffer would cause waiting for the GPU. */
 280    if (si_rings_is_buffer_referenced(sctx, buf->buf, RADEON_USAGE_READWRITE) ||
 281        !sctx->ws->buffer_wait(buf->buf, 0, RADEON_USAGE_READWRITE)) {
 282       /* Reallocate the buffer in the same pipe_resource. */
 283       si_alloc_resource(sctx->screen, buf);
 284       si_rebind_buffer(sctx, &buf->b.b);
 285    } else {
 286       util_range_set_empty(&buf->valid_buffer_range);
 287    }
 288
 289    return true;
 290 }
 291
 292 /* Replace the storage of dst with src. */
 293 void si_replace_buffer_storage(struct pipe_context *ctx, struct pipe_resource *dst,
 294                                struct pipe_resource *src)
 295 {
 296    struct si_context *sctx = (struct si_context *)ctx;
 297    struct si_resource *sdst = si_resource(dst);
 298    struct si_resource *ssrc = si_resource(src);
 299
 300    pb_reference(&sdst->buf, ssrc->buf);
 301    sdst->gpu_address = ssrc->gpu_address;
 302    sdst->b.b.bind = ssrc->b.b.bind;
 303    sdst->b.max_forced_staging_uploads = ssrc->b.max_forced_staging_uploads;
 304    sdst->max_forced_staging_uploads = ssrc->max_forced_staging_uploads;
 305    sdst->flags = ssrc->flags;
 306
 307    assert(sdst->vram_usage == ssrc->vram_usage);
 308    assert(sdst->gart_usage == ssrc->gart_usage);
 309    assert(sdst->bo_size == ssrc->bo_size);
 310    assert(sdst->bo_alignment == ssrc->bo_alignment);
 311    assert(sdst->domains == ssrc->domains);
 312
 313    si_rebind_buffer(sctx, dst);
 314 }
 315
 316 static void si_invalidate_resource(struct pipe_context *ctx, struct pipe_resource *resource)
 317 {
 318    struct si_context *sctx = (struct si_context *)ctx;
 319    struct si_resource *buf = si_resource(resource);
 320
 321    /* We currently only do anyting here for buffers */
 322    if (resource->target == PIPE_BUFFER)
 323       (void)si_invalidate_buffer(sctx, buf);
 324 }
 325
 326 static void *si_buffer_get_transfer(struct pipe_context *ctx, struct pipe_resource *resource,
 327                                     unsigned usage, const struct pipe_box *box,
 328                                     struct pipe_transfer **ptransfer, void *data,
 329                                     struct si_resource *staging, unsigned offset)
 330 {
 331    struct si_context *sctx = (struct si_context *)ctx;
 332    struct si_transfer *transfer;
 333
 334    if (usage & PIPE_TRANSFER_THREAD_SAFE)
 335       transfer = malloc(sizeof(*transfer));
 336    else if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
 337       transfer = slab_alloc(&sctx->pool_transfers_unsync);
 338    else
 339       transfer = slab_alloc(&sctx->pool_transfers);
 340
 341    transfer->b.b.resource = NULL;
 342    pipe_resource_reference(&transfer->b.b.resource, resource);
 343    transfer->b.b.level = 0;
 344    transfer->b.b.usage = usage;
 345    transfer->b.b.box = *box;
 346    transfer->b.b.stride = 0;
 347    transfer->b.b.layer_stride = 0;
 348    transfer->b.staging = NULL;
 349    transfer->offset = offset;
 350    transfer->staging = staging;
 351    *ptransfer = &transfer->b.b;
 352    return data;
 353 }
 354
 355 static void *si_buffer_transfer_map(struct pipe_context *ctx, struct pipe_resource *resource,
 356                                     unsigned level, unsigned usage, const struct pipe_box *box,
 357                                     struct pipe_transfer **ptransfer)
 358 {
 359    struct si_context *sctx = (struct si_context *)ctx;
 360    struct si_resource *buf = si_resource(resource);
 361    uint8_t *data;
 362
 363    assert(box->x + box->width <= resource->width0);
 364
 365    /* From GL_AMD_pinned_memory issues:
 366     *
 367     *     4) Is glMapBuffer on a shared buffer guaranteed to return the
 368     *        same system address which was specified at creation time?
 369     *
 370     *        RESOLVED: NO. The GL implementation might return a different
 371     *        virtual mapping of that memory, although the same physical
 372     *        page will be used.
 373     *
 374     * So don't ever use staging buffers.
 375     */
 376    if (buf->b.is_user_ptr)
 377       usage |= PIPE_TRANSFER_PERSISTENT;
 378
 379    /* See if the buffer range being mapped has never been initialized,
 380     * in which case it can be mapped unsynchronized. */
 381    if (!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED)) &&
 382        usage & PIPE_TRANSFER_WRITE && !buf->b.is_shared &&
 383        !util_ranges_intersect(&buf->valid_buffer_range, box->x, box->x + box->width)) {
 384       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 385    }
 386
 387    /* If discarding the entire range, discard the whole resource instead. */
 388    if (usage & PIPE_TRANSFER_DISCARD_RANGE && box->x == 0 && box->width == resource->width0) {
 389       usage |= PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE;
 390    }
 391
 392    /* If a buffer in VRAM is too large and the range is discarded, don't
 393     * map it directly. This makes sure that the buffer stays in VRAM.
 394     */
 395    bool force_discard_range = false;
 396    if (usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | PIPE_TRANSFER_DISCARD_RANGE) &&
 397        !(usage & PIPE_TRANSFER_PERSISTENT) &&
 398        /* Try not to decrement the counter if it's not positive. Still racy,
 399         * but it makes it harder to wrap the counter from INT_MIN to INT_MAX. */
 400        buf->max_forced_staging_uploads > 0 &&
 401        p_atomic_dec_return(&buf->max_forced_staging_uploads) >= 0) {
 402       usage &= ~(PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE | PIPE_TRANSFER_UNSYNCHRONIZED);
 403       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 404       force_discard_range = true;
 405    }
 406
 407    if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE &&
 408        !(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | TC_TRANSFER_MAP_NO_INVALIDATE))) {
 409       assert(usage & PIPE_TRANSFER_WRITE);
 410
 411       if (si_invalidate_buffer(sctx, buf)) {
 412          /* At this point, the buffer is always idle. */
 413          usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 414       } else {
 415          /* Fall back to a temporary buffer. */
 416          usage |= PIPE_TRANSFER_DISCARD_RANGE;
 417       }
 418    }
 419
 420    if (usage & PIPE_TRANSFER_FLUSH_EXPLICIT &&
 421        buf->b.b.flags & SI_RESOURCE_FLAG_UPLOAD_FLUSH_EXPLICIT_VIA_SDMA) {
 422       usage &= ~(PIPE_TRANSFER_UNSYNCHRONIZED | PIPE_TRANSFER_PERSISTENT);
 423       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 424       force_discard_range = true;
 425    }
 426
 427    if (usage & PIPE_TRANSFER_DISCARD_RANGE &&
 428        ((!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | PIPE_TRANSFER_PERSISTENT))) ||
 429         (buf->flags & RADEON_FLAG_SPARSE))) {
 430       assert(usage & PIPE_TRANSFER_WRITE);
 431
 432       /* Check if mapping this buffer would cause waiting for the GPU.
 433        */
 434       if (buf->flags & RADEON_FLAG_SPARSE || force_discard_range ||
 435           si_rings_is_buffer_referenced(sctx, buf->buf, RADEON_USAGE_READWRITE) ||
 436           !sctx->ws->buffer_wait(buf->buf, 0, RADEON_USAGE_READWRITE)) {
 437          /* Do a wait-free write-only transfer using a temporary buffer. */
 438          struct u_upload_mgr *uploader;
 439          struct si_resource *staging = NULL;
 440          unsigned offset;
 441
 442          /* If we are not called from the driver thread, we have
 443           * to use the uploader from u_threaded_context, which is
 444           * local to the calling thread.
 445           */
 446          if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC)
 447             uploader = sctx->tc->base.stream_uploader;
 448          else
 449             uploader = sctx->b.stream_uploader;
 450
 451          u_upload_alloc(uploader, 0, box->width + (box->x % SI_MAP_BUFFER_ALIGNMENT),
 452                         sctx->screen->info.tcc_cache_line_size, &offset,
 453                         (struct pipe_resource **)&staging, (void **)&data);
 454
 455          if (staging) {
 456             data += box->x % SI_MAP_BUFFER_ALIGNMENT;
 457             return si_buffer_get_transfer(ctx, resource, usage, box, ptransfer, data, staging,
 458                                           offset);
 459          } else if (buf->flags & RADEON_FLAG_SPARSE) {
 460             return NULL;
 461          }
 462       } else {
 463          /* At this point, the buffer is always idle (we checked it above). */
 464          usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 465       }
 466    }
 467    /* Use a staging buffer in cached GTT for reads. */
 468    else if (((usage & PIPE_TRANSFER_READ) && !(usage & PIPE_TRANSFER_PERSISTENT) &&
 469              (buf->domains & RADEON_DOMAIN_VRAM || buf->flags & RADEON_FLAG_GTT_WC)) ||
 470             (buf->flags & RADEON_FLAG_SPARSE)) {
 471       struct si_resource *staging;
 472
 473       assert(!(usage & (TC_TRANSFER_MAP_THREADED_UNSYNC | PIPE_TRANSFER_THREAD_SAFE)));
 474       staging = si_resource(pipe_buffer_create(ctx->screen, 0, PIPE_USAGE_STAGING,
 475                                                box->width + (box->x % SI_MAP_BUFFER_ALIGNMENT)));
 476       if (staging) {
 477          /* Copy the VRAM buffer to the staging buffer. */
 478          si_sdma_copy_buffer(sctx, &staging->b.b, resource, box->x % SI_MAP_BUFFER_ALIGNMENT,
 479                              box->x, box->width);
 480
 481          data = si_buffer_map_sync_with_rings(sctx, staging, usage & ~PIPE_TRANSFER_UNSYNCHRONIZED);
 482          if (!data) {
 483             si_resource_reference(&staging, NULL);
 484             return NULL;
 485          }
 486          data += box->x % SI_MAP_BUFFER_ALIGNMENT;
 487
 488          return si_buffer_get_transfer(ctx, resource, usage, box, ptransfer, data, staging, 0);
 489       } else if (buf->flags & RADEON_FLAG_SPARSE) {
 490          return NULL;
 491       }
 492    }
 493
 494    data = si_buffer_map_sync_with_rings(sctx, buf, usage);
 495    if (!data) {
 496       return NULL;
 497    }
 498    data += box->x;
 499
 500    return si_buffer_get_transfer(ctx, resource, usage, box, ptransfer, data, NULL, 0);
 501 }
 502
 503 static void si_buffer_do_flush_region(struct pipe_context *ctx, struct pipe_transfer *transfer,
 504                                       const struct pipe_box *box)
 505 {
 506    struct si_context *sctx = (struct si_context *)ctx;
 507    struct si_transfer *stransfer = (struct si_transfer *)transfer;
 508    struct si_resource *buf = si_resource(transfer->resource);
 509
 510    if (stransfer->staging) {
 511       unsigned src_offset =
 512          stransfer->offset + transfer->box.x % SI_MAP_BUFFER_ALIGNMENT + (box->x - transfer->box.x);
 513
 514       if (buf->b.b.flags & SI_RESOURCE_FLAG_UPLOAD_FLUSH_EXPLICIT_VIA_SDMA) {
 515          /* This should be true for all uploaders. */
 516          assert(transfer->box.x == 0);
 517
 518          /* Find a previous upload and extend its range. The last
 519           * upload is likely to be at the end of the list.
 520           */
 521          for (int i = sctx->num_sdma_uploads - 1; i >= 0; i--) {
 522             struct si_sdma_upload *up = &sctx->sdma_uploads[i];
 523
 524             if (up->dst != buf)
 525                continue;
 526
 527             assert(up->src == stransfer->staging);
 528             assert(box->x > up->dst_offset);
 529             up->size = box->x + box->width - up->dst_offset;
 530             return;
 531          }
 532
 533          /* Enlarge the array if it's full. */
 534          if (sctx->num_sdma_uploads == sctx->max_sdma_uploads) {
 535             unsigned size;
 536
 537             sctx->max_sdma_uploads += 4;
 538             size = sctx->max_sdma_uploads * sizeof(sctx->sdma_uploads[0]);
 539             sctx->sdma_uploads = realloc(sctx->sdma_uploads, size);
 540          }
 541
 542          /* Add a new upload. */
 543          struct si_sdma_upload *up = &sctx->sdma_uploads[sctx->num_sdma_uploads++];
 544          up->dst = up->src = NULL;
 545          si_resource_reference(&up->dst, buf);
 546          si_resource_reference(&up->src, stransfer->staging);
 547          up->dst_offset = box->x;
 548          up->src_offset = src_offset;
 549          up->size = box->width;
 550          return;
 551       }
 552
 553       /* Copy the staging buffer into the original one. */
 554       si_copy_buffer(sctx, transfer->resource, &stransfer->staging->b.b, box->x, src_offset,
 555                      box->width);
 556    }
 557
 558    util_range_add(&buf->b.b, &buf->valid_buffer_range, box->x, box->x + box->width);
 559 }
 560
 561 static void si_buffer_flush_region(struct pipe_context *ctx, struct pipe_transfer *transfer,
 562                                    const struct pipe_box *rel_box)
 563 {
 564    unsigned required_usage = PIPE_TRANSFER_WRITE | PIPE_TRANSFER_FLUSH_EXPLICIT;
 565
 566    if ((transfer->usage & required_usage) == required_usage) {
 567       struct pipe_box box;
 568
 569       u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
 570       si_buffer_do_flush_region(ctx, transfer, &box);
 571    }
 572 }
 573
 574 static void si_buffer_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *transfer)
 575 {
 576    struct si_context *sctx = (struct si_context *)ctx;
 577    struct si_transfer *stransfer = (struct si_transfer *)transfer;
 578
 579    if (transfer->usage & PIPE_TRANSFER_WRITE && !(transfer->usage & PIPE_TRANSFER_FLUSH_EXPLICIT))
 580       si_buffer_do_flush_region(ctx, transfer, &transfer->box);
 581
 582    si_resource_reference(&stransfer->staging, NULL);
 583    assert(stransfer->b.staging == NULL); /* for threaded context only */
 584    pipe_resource_reference(&transfer->resource, NULL);
 585
 586    if (transfer->usage & PIPE_TRANSFER_THREAD_SAFE) {
 587       free(transfer);
 588    } else {
 589       /* Don't use pool_transfers_unsync. We are always in the driver
 590        * thread. Freeing an object into a different pool is allowed.
 591        */
 592       slab_free(&sctx->pool_transfers, transfer);
 593    }
 594 }
 595
 596 static void si_buffer_subdata(struct pipe_context *ctx, struct pipe_resource *buffer,
 597                               unsigned usage, unsigned offset, unsigned size, const void *data)
 598 {
 599    struct pipe_transfer *transfer = NULL;
 600    struct pipe_box box;
 601    uint8_t *map = NULL;
 602
 603    usage |= PIPE_TRANSFER_WRITE;
 604
 605    if (!(usage & PIPE_TRANSFER_MAP_DIRECTLY))
 606       usage |= PIPE_TRANSFER_DISCARD_RANGE;
 607
 608    u_box_1d(offset, size, &box);
 609    map = si_buffer_transfer_map(ctx, buffer, 0, usage, &box, &transfer);
 610    if (!map)
 611       return;
 612
 613    memcpy(map, data, size);
 614    si_buffer_transfer_unmap(ctx, transfer);
 615 }
 616
 617 static const struct u_resource_vtbl si_buffer_vtbl = {
 618    NULL,                     /* get_handle */
 619    si_buffer_destroy,        /* resource_destroy */
 620    si_buffer_transfer_map,   /* transfer_map */
 621    si_buffer_flush_region,   /* transfer_flush_region */
 622    si_buffer_transfer_unmap, /* transfer_unmap */
 623 };
 624
 625 static struct si_resource *si_alloc_buffer_struct(struct pipe_screen *screen,
 626                                                   const struct pipe_resource *templ)
 627 {
 628    struct si_resource *buf;
 629
 630    buf = MALLOC_STRUCT(si_resource);
 631
 632    buf->b.b = *templ;
 633    buf->b.b.next = NULL;
 634    pipe_reference_init(&buf->b.b.reference, 1);
 635    buf->b.b.screen = screen;
 636
 637    buf->b.vtbl = &si_buffer_vtbl;
 638    threaded_resource_init(&buf->b.b);
 639
 640    buf->buf = NULL;
 641    buf->bind_history = 0;
 642    buf->TC_L2_dirty = false;
 643    util_range_init(&buf->valid_buffer_range);
 644    return buf;
 645 }
 646
 647 static struct pipe_resource *si_buffer_create(struct pipe_screen *screen,
 648                                               const struct pipe_resource *templ, unsigned alignment)
 649 {
 650    struct si_screen *sscreen = (struct si_screen *)screen;
 651    struct si_resource *buf = si_alloc_buffer_struct(screen, templ);
 652
 653    if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 654       buf->b.b.flags |= SI_RESOURCE_FLAG_UNMAPPABLE;
 655
 656    si_init_resource_fields(sscreen, buf, templ->width0, alignment);
 657
 658    if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE)
 659       buf->flags |= RADEON_FLAG_SPARSE;
 660
 661    if (!si_alloc_resource(sscreen, buf)) {
 662       FREE(buf);
 663       return NULL;
 664    }
 665    return &buf->b.b;
 666 }
 667
 668 struct pipe_resource *pipe_aligned_buffer_create(struct pipe_screen *screen, unsigned flags,
 669                                                  unsigned usage, unsigned size, unsigned alignment)
 670 {
 671    struct pipe_resource buffer;
 672
 673    memset(&buffer, 0, sizeof buffer);
 674    buffer.target = PIPE_BUFFER;
 675    buffer.format = PIPE_FORMAT_R8_UNORM;
 676    buffer.bind = 0;
 677    buffer.usage = usage;
 678    buffer.flags = flags;
 679    buffer.width0 = size;
 680    buffer.height0 = 1;
 681    buffer.depth0 = 1;
 682    buffer.array_size = 1;
 683    return si_buffer_create(screen, &buffer, alignment);
 684 }
 685
 686 struct si_resource *si_aligned_buffer_create(struct pipe_screen *screen, unsigned flags,
 687                                              unsigned usage, unsigned size, unsigned alignment)
 688 {
 689    return si_resource(pipe_aligned_buffer_create(screen, flags, usage, size, alignment));
 690 }
 691
 692 static struct pipe_resource *si_buffer_from_user_memory(struct pipe_screen *screen,
 693                                                         const struct pipe_resource *templ,
 694                                                         void *user_memory)
 695 {
 696    struct si_screen *sscreen = (struct si_screen *)screen;
 697    struct radeon_winsys *ws = sscreen->ws;
 698    struct si_resource *buf = si_alloc_buffer_struct(screen, templ);
 699
 700    buf->domains = RADEON_DOMAIN_GTT;
 701    buf->flags = 0;
 702    buf->b.is_user_ptr = true;
 703    util_range_add(&buf->b.b, &buf->valid_buffer_range, 0, templ->width0);
 704    util_range_add(&buf->b.b, &buf->b.valid_buffer_range, 0, templ->width0);
 705
 706    /* Convert a user pointer to a buffer. */
 707    buf->buf = ws->buffer_from_ptr(ws, user_memory, templ->width0);
 708    if (!buf->buf) {
 709       FREE(buf);
 710       return NULL;
 711    }
 712
 713    buf->gpu_address = ws->buffer_get_virtual_address(buf->buf);
 714    buf->vram_usage = 0;
 715    buf->gart_usage = templ->width0;
 716
 717    return &buf->b.b;
 718 }
 719
 720 static struct pipe_resource *si_resource_create(struct pipe_screen *screen,
 721                                                 const struct pipe_resource *templ)
 722 {
 723    if (templ->target == PIPE_BUFFER) {
 724       return si_buffer_create(screen, templ, 256);
 725    } else {
 726       return si_texture_create(screen, templ);
 727    }
 728 }
 729
 730 static bool si_resource_commit(struct pipe_context *pctx, struct pipe_resource *resource,
 731                                unsigned level, struct pipe_box *box, bool commit)
 732 {
 733    struct si_context *ctx = (struct si_context *)pctx;
 734    struct si_resource *res = si_resource(resource);
 735
 736    /*
 737     * Since buffer commitment changes cannot be pipelined, we need to
 738     * (a) flush any pending commands that refer to the buffer we're about
 739     *     to change, and
 740     * (b) wait for threaded submit to finish, including those that were
 741     *     triggered by some other, earlier operation.
 742     */
 743    if (radeon_emitted(ctx->gfx_cs, ctx->initial_gfx_cs_size) &&
 744        ctx->ws->cs_is_buffer_referenced(ctx->gfx_cs, res->buf, RADEON_USAGE_READWRITE)) {
 745       si_flush_gfx_cs(ctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
 746    }
 747    if (radeon_emitted(ctx->sdma_cs, 0) &&
 748        ctx->ws->cs_is_buffer_referenced(ctx->sdma_cs, res->buf, RADEON_USAGE_READWRITE)) {
 749       si_flush_dma_cs(ctx, PIPE_FLUSH_ASYNC, NULL);
 750    }
 751
 752    if (ctx->sdma_cs)
 753       ctx->ws->cs_sync_flush(ctx->sdma_cs);
 754    ctx->ws->cs_sync_flush(ctx->gfx_cs);
 755
 756    assert(resource->target == PIPE_BUFFER);
 757
 758    return ctx->ws->buffer_commit(res->buf, box->x, box->width, commit);
 759 }
 760
 761 void si_init_screen_buffer_functions(struct si_screen *sscreen)
 762 {
 763    sscreen->b.resource_create = si_resource_create;
 764    sscreen->b.resource_destroy = u_resource_destroy_vtbl;
 765    sscreen->b.resource_from_user_memory = si_buffer_from_user_memory;
 766 }
 767
 768 void si_init_buffer_functions(struct si_context *sctx)
 769 {
 770    sctx->b.invalidate_resource = si_invalidate_resource;
 771    sctx->b.transfer_map = u_transfer_map_vtbl;
 772    sctx->b.transfer_flush_region = u_transfer_flush_region_vtbl;
 773    sctx->b.transfer_unmap = u_transfer_unmap_vtbl;
 774    sctx->b.texture_subdata = u_default_texture_subdata;
 775    sctx->b.buffer_subdata = si_buffer_subdata;
 776    sctx->b.resource_commit = si_resource_commit;
 777 }