src/gallium/drivers/radeonsi/gfx10_query.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "si_pipe.h"
  26 #include "si_query.h"
  27 #include "sid.h"
  28 #include "util/u_memory.h"
  29 #include "util/u_suballoc.h"
  30
  31 #include <stddef.h>
  32
  33 /**
  34  * The query buffer is written to by ESGS NGG shaders with statistics about
  35  * generated and (streamout-)emitted primitives.
  36  *
  37  * The context maintains a ring of these query buffers, and queries simply
  38  * point into the ring, allowing an arbitrary number of queries to be active
  39  * without additional GPU cost.
  40  */
  41 struct gfx10_sh_query_buffer {
  42    struct list_head list;
  43    struct si_resource *buf;
  44    unsigned refcount;
  45
  46    /* Offset into the buffer in bytes; points at the first un-emitted entry. */
  47    unsigned head;
  48 };
  49
  50 /* Memory layout of the query buffer. Must be kept in sync with shaders
  51  * (including QBO shaders) and should be aligned to cachelines.
  52  *
  53  * The somewhat awkward memory layout is for compatibility with the
  54  * SET_PREDICATION packet, which also means that we're setting the high bit
  55  * of all those values unconditionally.
  56  */
  57 struct gfx10_sh_query_buffer_mem {
  58    struct {
  59       uint64_t generated_primitives_start_dummy;
  60       uint64_t emitted_primitives_start_dummy;
  61       uint64_t generated_primitives;
  62       uint64_t emitted_primitives;
  63    } stream[4];
  64    uint32_t fence; /* bottom-of-pipe fence: set to ~0 when draws have finished */
  65    uint32_t pad[31];
  66 };
  67
  68 /* Shader-based queries. */
  69 struct gfx10_sh_query {
  70    struct si_query b;
  71
  72    struct gfx10_sh_query_buffer *first;
  73    struct gfx10_sh_query_buffer *last;
  74    unsigned first_begin;
  75    unsigned last_end;
  76
  77    unsigned stream;
  78 };
  79
  80 static void emit_shader_query(struct si_context *sctx)
  81 {
  82    assert(!list_is_empty(&sctx->shader_query_buffers));
  83
  84    struct gfx10_sh_query_buffer *qbuf =
  85       list_last_entry(&sctx->shader_query_buffers, struct gfx10_sh_query_buffer, list);
  86    qbuf->head += sizeof(struct gfx10_sh_query_buffer_mem);
  87 }
  88
  89 static void gfx10_release_query_buffers(struct si_context *sctx,
  90                                         struct gfx10_sh_query_buffer *first,
  91                                         struct gfx10_sh_query_buffer *last)
  92 {
  93    while (first) {
  94       struct gfx10_sh_query_buffer *qbuf = first;
  95       if (first != last)
  96          first = LIST_ENTRY(struct gfx10_sh_query_buffer, qbuf->list.next, list);
  97       else
  98          first = NULL;
  99
 100       qbuf->refcount--;
 101       if (qbuf->refcount)
 102          continue;
 103
 104       if (qbuf->list.next == &sctx->shader_query_buffers)
 105          continue; /* keep the most recent buffer; it may not be full yet */
 106       if (qbuf->list.prev == &sctx->shader_query_buffers)
 107          continue; /* keep the oldest buffer for recycling */
 108
 109       list_del(&qbuf->list);
 110       si_resource_reference(&qbuf->buf, NULL);
 111       FREE(qbuf);
 112    }
 113 }
 114
 115 static bool gfx10_alloc_query_buffer(struct si_context *sctx)
 116 {
 117    if (si_is_atom_dirty(sctx, &sctx->atoms.s.shader_query))
 118       return true;
 119
 120    struct gfx10_sh_query_buffer *qbuf = NULL;
 121
 122    if (!list_is_empty(&sctx->shader_query_buffers)) {
 123       qbuf = list_last_entry(&sctx->shader_query_buffers, struct gfx10_sh_query_buffer, list);
 124       if (qbuf->head + sizeof(struct gfx10_sh_query_buffer_mem) <= qbuf->buf->b.b.width0)
 125          goto success;
 126
 127       qbuf = list_first_entry(&sctx->shader_query_buffers, struct gfx10_sh_query_buffer, list);
 128       if (!qbuf->refcount &&
 129           !si_rings_is_buffer_referenced(sctx, qbuf->buf->buf, RADEON_USAGE_READWRITE) &&
 130           sctx->ws->buffer_wait(qbuf->buf->buf, 0, RADEON_USAGE_READWRITE)) {
 131          /* Can immediately re-use the oldest buffer */
 132          list_del(&qbuf->list);
 133       } else {
 134          qbuf = NULL;
 135       }
 136    }
 137
 138    if (!qbuf) {
 139       qbuf = CALLOC_STRUCT(gfx10_sh_query_buffer);
 140       if (unlikely(!qbuf))
 141          return false;
 142
 143       struct si_screen *screen = sctx->screen;
 144       unsigned buf_size =
 145          MAX2(sizeof(struct gfx10_sh_query_buffer_mem), screen->info.min_alloc_size);
 146       qbuf->buf = si_resource(pipe_buffer_create(&screen->b, 0, PIPE_USAGE_STAGING, buf_size));
 147       if (unlikely(!qbuf->buf)) {
 148          FREE(qbuf);
 149          return false;
 150       }
 151    }
 152
 153    /* The buffer is currently unused by the GPU. Initialize it.
 154     *
 155     * We need to set the high bit of all the primitive counters for
 156     * compatibility with the SET_PREDICATION packet.
 157     */
 158    uint64_t *results = sctx->ws->buffer_map(qbuf->buf->buf, NULL,
 159                                             PIPE_TRANSFER_WRITE | PIPE_TRANSFER_UNSYNCHRONIZED);
 160    assert(results);
 161
 162    for (unsigned i = 0, e = qbuf->buf->b.b.width0 / sizeof(struct gfx10_sh_query_buffer_mem); i < e;
 163         ++i) {
 164       for (unsigned j = 0; j < 16; ++j)
 165          results[32 * i + j] = (uint64_t)1 << 63;
 166       results[32 * i + 16] = 0;
 167    }
 168
 169    list_addtail(&qbuf->list, &sctx->shader_query_buffers);
 170    qbuf->head = 0;
 171    qbuf->refcount = sctx->num_active_shader_queries;
 172
 173 success:;
 174    struct pipe_shader_buffer sbuf;
 175    sbuf.buffer = &qbuf->buf->b.b;
 176    sbuf.buffer_offset = qbuf->head;
 177    sbuf.buffer_size = sizeof(struct gfx10_sh_query_buffer_mem);
 178    si_set_rw_shader_buffer(sctx, GFX10_GS_QUERY_BUF, &sbuf);
 179    sctx->current_vs_state |= S_VS_STATE_STREAMOUT_QUERY_ENABLED(1);
 180
 181    si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_query);
 182    return true;
 183 }
 184
 185 static void gfx10_sh_query_destroy(struct si_context *sctx, struct si_query *rquery)
 186 {
 187    struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
 188    gfx10_release_query_buffers(sctx, query->first, query->last);
 189    FREE(query);
 190 }
 191
 192 static bool gfx10_sh_query_begin(struct si_context *sctx, struct si_query *rquery)
 193 {
 194    struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
 195
 196    gfx10_release_query_buffers(sctx, query->first, query->last);
 197    query->first = query->last = NULL;
 198
 199    if (unlikely(!gfx10_alloc_query_buffer(sctx)))
 200       return false;
 201
 202    query->first = list_last_entry(&sctx->shader_query_buffers, struct gfx10_sh_query_buffer, list);
 203    query->first_begin = query->first->head;
 204
 205    sctx->num_active_shader_queries++;
 206    query->first->refcount++;
 207
 208    return true;
 209 }
 210
 211 static bool gfx10_sh_query_end(struct si_context *sctx, struct si_query *rquery)
 212 {
 213    struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
 214
 215    if (unlikely(!query->first))
 216       return false; /* earlier out of memory error */
 217
 218    query->last = list_last_entry(&sctx->shader_query_buffers, struct gfx10_sh_query_buffer, list);
 219    query->last_end = query->last->head;
 220
 221    /* Signal the fence of the previous chunk */
 222    if (query->last_end != 0) {
 223       uint64_t fence_va = query->last->buf->gpu_address;
 224       fence_va += query->last_end - sizeof(struct gfx10_sh_query_buffer_mem);
 225       fence_va += offsetof(struct gfx10_sh_query_buffer_mem, fence);
 226       si_cp_release_mem(sctx, sctx->gfx_cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM,
 227                         EOP_INT_SEL_NONE, EOP_DATA_SEL_VALUE_32BIT, query->last->buf, fence_va,
 228                         0xffffffff, PIPE_QUERY_GPU_FINISHED);
 229    }
 230
 231    sctx->num_active_shader_queries--;
 232
 233    if (sctx->num_active_shader_queries > 0) {
 234       gfx10_alloc_query_buffer(sctx);
 235    } else {
 236       si_set_rw_shader_buffer(sctx, GFX10_GS_QUERY_BUF, NULL);
 237       sctx->current_vs_state &= C_VS_STATE_STREAMOUT_QUERY_ENABLED;
 238
 239       /* If a query_begin is followed by a query_end without a draw
 240        * in-between, we need to clear the atom to ensure that the
 241        * next query_begin will re-initialize the shader buffer. */
 242       si_set_atom_dirty(sctx, &sctx->atoms.s.shader_query, false);
 243    }
 244
 245    return true;
 246 }
 247
 248 static void gfx10_sh_query_add_result(struct gfx10_sh_query *query,
 249                                       struct gfx10_sh_query_buffer_mem *qmem,
 250                                       union pipe_query_result *result)
 251 {
 252    static const uint64_t mask = ((uint64_t)1 << 63) - 1;
 253
 254    switch (query->b.type) {
 255    case PIPE_QUERY_PRIMITIVES_EMITTED:
 256       result->u64 += qmem->stream[query->stream].emitted_primitives & mask;
 257       break;
 258    case PIPE_QUERY_PRIMITIVES_GENERATED:
 259       result->u64 += qmem->stream[query->stream].generated_primitives & mask;
 260       break;
 261    case PIPE_QUERY_SO_STATISTICS:
 262       result->so_statistics.num_primitives_written +=
 263          qmem->stream[query->stream].emitted_primitives & mask;
 264       result->so_statistics.primitives_storage_needed +=
 265          qmem->stream[query->stream].generated_primitives & mask;
 266       break;
 267    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 268       result->b |= qmem->stream[query->stream].emitted_primitives !=
 269                    qmem->stream[query->stream].generated_primitives;
 270       break;
 271    case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
 272       for (unsigned stream = 0; stream < SI_MAX_STREAMS; ++stream) {
 273          result->b |= qmem->stream[query->stream].emitted_primitives !=
 274                       qmem->stream[query->stream].generated_primitives;
 275       }
 276       break;
 277    default:
 278       assert(0);
 279    }
 280 }
 281
 282 static bool gfx10_sh_query_get_result(struct si_context *sctx, struct si_query *rquery, bool wait,
 283                                       union pipe_query_result *result)
 284 {
 285    struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
 286
 287    util_query_clear_result(result, query->b.type);
 288
 289    if (unlikely(!query->first))
 290       return false; /* earlier out of memory error */
 291    assert(query->last);
 292
 293    for (struct gfx10_sh_query_buffer *qbuf = query->last;;
 294         qbuf = LIST_ENTRY(struct gfx10_sh_query_buffer, qbuf->list.prev, list)) {
 295       unsigned usage = PIPE_TRANSFER_READ | (wait ? 0 : PIPE_TRANSFER_DONTBLOCK);
 296       void *map;
 297
 298       if (rquery->b.flushed)
 299          map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);
 300       else
 301          map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage);
 302
 303       if (!map)
 304          return false;
 305
 306       unsigned results_begin = 0;
 307       unsigned results_end = qbuf->head;
 308       if (qbuf == query->first)
 309          results_begin = query->first_begin;
 310       if (qbuf == query->last)
 311          results_end = query->last_end;
 312
 313       while (results_begin != results_end) {
 314          struct gfx10_sh_query_buffer_mem *qmem = map + results_begin;
 315          results_begin += sizeof(*qmem);
 316
 317          gfx10_sh_query_add_result(query, qmem, result);
 318       }
 319
 320       if (qbuf == query->first)
 321          break;
 322    }
 323
 324    return true;
 325 }
 326
 327 static void gfx10_sh_query_get_result_resource(struct si_context *sctx, struct si_query *rquery,
 328                                                bool wait, enum pipe_query_value_type result_type,
 329                                                int index, struct pipe_resource *resource,
 330                                                unsigned offset)
 331 {
 332    struct gfx10_sh_query *query = (struct gfx10_sh_query *)rquery;
 333    struct si_qbo_state saved_state = {};
 334    struct pipe_resource *tmp_buffer = NULL;
 335    unsigned tmp_buffer_offset = 0;
 336
 337    if (!sctx->sh_query_result_shader) {
 338       sctx->sh_query_result_shader = gfx10_create_sh_query_result_cs(sctx);
 339       if (!sctx->sh_query_result_shader)
 340          return;
 341    }
 342
 343    if (query->first != query->last) {
 344       u_suballocator_alloc(sctx->allocator_zeroed_memory, 16, 16, &tmp_buffer_offset, &tmp_buffer);
 345       if (!tmp_buffer)
 346          return;
 347    }
 348
 349    si_save_qbo_state(sctx, &saved_state);
 350
 351    /* Pre-fill the constants configuring the shader behavior. */
 352    struct {
 353       uint32_t config;
 354       uint32_t offset;
 355       uint32_t chain;
 356       uint32_t result_count;
 357    } consts;
 358    struct pipe_constant_buffer constant_buffer = {};
 359
 360    if (index >= 0) {
 361       switch (query->b.type) {
 362       case PIPE_QUERY_PRIMITIVES_GENERATED:
 363          consts.offset = sizeof(uint32_t) * query->stream;
 364          consts.config = 0;
 365          break;
 366       case PIPE_QUERY_PRIMITIVES_EMITTED:
 367          consts.offset = sizeof(uint32_t) * (4 + query->stream);
 368          consts.config = 0;
 369          break;
 370       case PIPE_QUERY_SO_STATISTICS:
 371          consts.offset = sizeof(uint32_t) * (4 * index + query->stream);
 372          consts.config = 0;
 373          break;
 374       case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 375          consts.offset = sizeof(uint32_t) * query->stream;
 376          consts.config = 2;
 377          break;
 378       case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
 379          consts.offset = 0;
 380          consts.config = 3;
 381          break;
 382       default:
 383          unreachable("bad query type");
 384       }
 385    } else {
 386       /* Check result availability. */
 387       consts.offset = 0;
 388       consts.config = 1;
 389    }
 390
 391    if (result_type == PIPE_QUERY_TYPE_I64 || result_type == PIPE_QUERY_TYPE_U64)
 392       consts.config |= 8;
 393
 394    constant_buffer.buffer_size = sizeof(consts);
 395    constant_buffer.user_buffer = &consts;
 396
 397    /* Pre-fill the SSBOs and grid. */
 398    struct pipe_shader_buffer ssbo[3];
 399    struct pipe_grid_info grid = {};
 400
 401    ssbo[1].buffer = tmp_buffer;
 402    ssbo[1].buffer_offset = tmp_buffer_offset;
 403    ssbo[1].buffer_size = 16;
 404
 405    ssbo[2] = ssbo[1];
 406
 407    sctx->b.bind_compute_state(&sctx->b, sctx->sh_query_result_shader);
 408
 409    grid.block[0] = 1;
 410    grid.block[1] = 1;
 411    grid.block[2] = 1;
 412    grid.grid[0] = 1;
 413    grid.grid[1] = 1;
 414    grid.grid[2] = 1;
 415
 416    struct gfx10_sh_query_buffer *qbuf = query->first;
 417    for (;;) {
 418       unsigned begin = qbuf == query->first ? query->first_begin : 0;
 419       unsigned end = qbuf == query->last ? query->last_end : qbuf->buf->b.b.width0;
 420       if (!end)
 421          continue;
 422
 423       ssbo[0].buffer = &qbuf->buf->b.b;
 424       ssbo[0].buffer_offset = begin;
 425       ssbo[0].buffer_size = end - begin;
 426
 427       consts.result_count = (end - begin) / sizeof(struct gfx10_sh_query_buffer_mem);
 428       consts.chain = 0;
 429       if (qbuf != query->first)
 430          consts.chain |= 1;
 431       if (qbuf != query->last)
 432          consts.chain |= 2;
 433
 434       if (qbuf == query->last) {
 435          ssbo[2].buffer = resource;
 436          ssbo[2].buffer_offset = offset;
 437          ssbo[2].buffer_size = 8;
 438       }
 439
 440       sctx->b.set_constant_buffer(&sctx->b, PIPE_SHADER_COMPUTE, 0, &constant_buffer);
 441       sctx->b.set_shader_buffers(&sctx->b, PIPE_SHADER_COMPUTE, 0, 3, ssbo, 0x6);
 442
 443       if (wait) {
 444          uint64_t va;
 445
 446          /* Wait for result availability. Wait only for readiness
 447           * of the last entry, since the fence writes should be
 448           * serialized in the CP.
 449           */
 450          va = qbuf->buf->gpu_address;
 451          va += end - sizeof(struct gfx10_sh_query_buffer_mem);
 452          va += offsetof(struct gfx10_sh_query_buffer_mem, fence);
 453
 454          si_cp_wait_mem(sctx, sctx->gfx_cs, va, 0x00000001, 0x00000001, 0);
 455       }
 456
 457       sctx->b.launch_grid(&sctx->b, &grid);
 458       sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;
 459
 460       if (qbuf == query->last)
 461          break;
 462       qbuf = LIST_ENTRY(struct gfx10_sh_query_buffer, qbuf->list.next, list);
 463    }
 464
 465    si_restore_qbo_state(sctx, &saved_state);
 466    pipe_resource_reference(&tmp_buffer, NULL);
 467 }
 468
 469 static const struct si_query_ops gfx10_sh_query_ops = {
 470    .destroy = gfx10_sh_query_destroy,
 471    .begin = gfx10_sh_query_begin,
 472    .end = gfx10_sh_query_end,
 473    .get_result = gfx10_sh_query_get_result,
 474    .get_result_resource = gfx10_sh_query_get_result_resource,
 475 };
 476
 477 struct pipe_query *gfx10_sh_query_create(struct si_screen *screen, enum pipe_query_type query_type,
 478                                          unsigned index)
 479 {
 480    struct gfx10_sh_query *query = CALLOC_STRUCT(gfx10_sh_query);
 481    if (unlikely(!query))
 482       return NULL;
 483
 484    query->b.ops = &gfx10_sh_query_ops;
 485    query->b.type = query_type;
 486    query->stream = index;
 487
 488    return (struct pipe_query *)query;
 489 }
 490
 491 void gfx10_init_query(struct si_context *sctx)
 492 {
 493    list_inithead(&sctx->shader_query_buffers);
 494    sctx->atoms.s.shader_query.emit = emit_shader_query;
 495 }
 496
 497 void gfx10_destroy_query(struct si_context *sctx)
 498 {
 499    while (!list_is_empty(&sctx->shader_query_buffers)) {
 500       struct gfx10_sh_query_buffer *qbuf =
 501          list_first_entry(&sctx->shader_query_buffers, struct gfx10_sh_query_buffer, list);
 502       list_del(&qbuf->list);
 503
 504       assert(!qbuf->refcount);
 505       si_resource_reference(&qbuf->buf, NULL);
 506       FREE(qbuf);
 507    }
 508 }