src/freedreno/vulkan/tu_query.c

   1 /*
   2  * Copyrigh 2016 Red Hat Inc.
   3  * Based on anv:
   4  * Copyright © 2015 Intel Corporation
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23  * DEALINGS IN THE SOFTWARE.
  24  */
  25
  26 #include "tu_private.h"
  27
  28 #include <assert.h>
  29 #include <fcntl.h>
  30 #include <stdbool.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33
  34 #include "registers/adreno_pm4.xml.h"
  35 #include "registers/adreno_common.xml.h"
  36 #include "registers/a6xx.xml.h"
  37
  38 #include "nir/nir_builder.h"
  39 #include "util/os_time.h"
  40
  41 #include "tu_cs.h"
  42
  43 #define NSEC_PER_SEC 1000000000ull
  44 #define WAIT_TIMEOUT 5
  45
  46 /* It seems like sample counts need to be copied over to 16-byte aligned
  47  * memory. */
  48 struct PACKED slot_value {
  49    uint64_t value;
  50    uint64_t __padding;
  51 };
  52
  53 struct PACKED occlusion_query_slot {
  54    struct slot_value available; /* 0 when unavailable, 1 when available */
  55    struct slot_value begin;
  56    struct slot_value end;
  57    struct slot_value result;
  58 };
  59
  60 /* Returns the IOVA of a given uint64_t field in a given slot of a query
  61  * pool. */
  62 #define query_iova(type, pool, query, field)                         \
  63    pool->bo.iova + pool->stride * query + offsetof(type, field) +    \
  64          offsetof(struct slot_value, value)
  65
  66 #define occlusion_query_iova(pool, query, field)                     \
  67    query_iova(struct occlusion_query_slot, pool, query, field)
  68
  69 #define query_is_available(type, slot)                               \
  70    ((type*)slot)->available.value
  71
  72 #define occlusion_query_is_available(slot)                           \
  73    query_is_available(struct occlusion_query_slot, slot)
  74
  75 /*
  76  * Returns a pointer to a given slot in a query pool.
  77  */
  78 static void* slot_address(struct tu_query_pool *pool, uint32_t query)
  79 {
  80    return (char*)pool->bo.map + query * pool->stride;
  81 }
  82
  83 VkResult
  84 tu_CreateQueryPool(VkDevice _device,
  85                    const VkQueryPoolCreateInfo *pCreateInfo,
  86                    const VkAllocationCallbacks *pAllocator,
  87                    VkQueryPool *pQueryPool)
  88 {
  89    TU_FROM_HANDLE(tu_device, device, _device);
  90    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO);
  91    assert(pCreateInfo->queryCount > 0);
  92
  93    uint32_t slot_size;
  94    switch (pCreateInfo->queryType) {
  95    case VK_QUERY_TYPE_OCCLUSION:
  96       slot_size = sizeof(struct occlusion_query_slot);
  97       break;
  98    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
  99    case VK_QUERY_TYPE_TIMESTAMP:
 100       unreachable("Unimplemented query type");
 101    default:
 102       assert(!"Invalid query type");
 103    }
 104
 105    struct tu_query_pool *pool =
 106       vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
 107                 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 108
 109    if (!pool)
 110       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 111
 112    VkResult result = tu_bo_init_new(device, &pool->bo,
 113          pCreateInfo->queryCount * slot_size);
 114    if (result != VK_SUCCESS) {
 115       vk_free2(&device->alloc, pAllocator, pool);
 116       return result;
 117    }
 118
 119    result = tu_bo_map(device, &pool->bo);
 120    if (result != VK_SUCCESS) {
 121       tu_bo_finish(device, &pool->bo);
 122       vk_free2(&device->alloc, pAllocator, pool);
 123       return result;
 124    }
 125
 126    /* Initialize all query statuses to unavailable */
 127    memset(pool->bo.map, 0, pool->bo.size);
 128
 129    pool->type = pCreateInfo->queryType;
 130    pool->stride = slot_size;
 131    pool->size = pCreateInfo->queryCount;
 132    pool->pipeline_statistics = pCreateInfo->pipelineStatistics;
 133    *pQueryPool = tu_query_pool_to_handle(pool);
 134
 135    return VK_SUCCESS;
 136 }
 137
 138 void
 139 tu_DestroyQueryPool(VkDevice _device,
 140                     VkQueryPool _pool,
 141                     const VkAllocationCallbacks *pAllocator)
 142 {
 143    TU_FROM_HANDLE(tu_device, device, _device);
 144    TU_FROM_HANDLE(tu_query_pool, pool, _pool);
 145
 146    if (!pool)
 147       return;
 148
 149    tu_bo_finish(device, &pool->bo);
 150    vk_free2(&device->alloc, pAllocator, pool);
 151 }
 152
 153 /* Wait on the the availability status of a query up until a timeout. */
 154 static VkResult
 155 wait_for_available(struct tu_device *device, struct tu_query_pool *pool,
 156                    uint32_t query)
 157 {
 158    /* TODO: Use the MSM_IOVA_WAIT ioctl to wait on the available bit in a
 159     * scheduler friendly way instead of busy polling once the patch has landed
 160     * upstream. */
 161    struct occlusion_query_slot *slot = slot_address(pool, query);
 162    uint64_t abs_timeout = os_time_get_absolute_timeout(
 163          WAIT_TIMEOUT * NSEC_PER_SEC);
 164    while(os_time_get_nano() < abs_timeout) {
 165       if (occlusion_query_is_available(slot))
 166          return VK_SUCCESS;
 167    }
 168    return vk_error(device->instance, VK_TIMEOUT);
 169 }
 170
 171 /* Writes a query value to a buffer from the CPU. */
 172 static void
 173 write_query_value_cpu(char* base,
 174                       uint32_t offset,
 175                       uint64_t value,
 176                       VkQueryResultFlags flags)
 177 {
 178    if (flags & VK_QUERY_RESULT_64_BIT) {
 179       *(uint64_t*)(base + (offset * sizeof(uint64_t))) = value;
 180    } else {
 181       *(uint32_t*)(base + (offset * sizeof(uint32_t))) = value;
 182    }
 183 }
 184
 185 static VkResult
 186 get_occlusion_query_pool_results(struct tu_device *device,
 187                                  struct tu_query_pool *pool,
 188                                  uint32_t firstQuery,
 189                                  uint32_t queryCount,
 190                                  size_t dataSize,
 191                                  void *pData,
 192                                  VkDeviceSize stride,
 193                                  VkQueryResultFlags flags)
 194 {
 195    assert(dataSize >= stride * queryCount);
 196
 197    char *result_base = pData;
 198    VkResult result = VK_SUCCESS;
 199    for (uint32_t i = 0; i < queryCount; i++) {
 200       uint32_t query = firstQuery + i;
 201       struct occlusion_query_slot *slot = slot_address(pool, query);
 202       bool available = occlusion_query_is_available(slot);
 203       if ((flags & VK_QUERY_RESULT_WAIT_BIT) && !available) {
 204          VkResult wait_result = wait_for_available(device, pool, query);
 205          if (wait_result != VK_SUCCESS)
 206             return wait_result;
 207          available = true;
 208       } else if (!(flags & VK_QUERY_RESULT_PARTIAL_BIT) && !available) {
 209          /* From the Vulkan 1.1.130 spec:
 210           *
 211           *    If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
 212           *    both not set then no result values are written to pData for
 213           *    queries that are in the unavailable state at the time of the
 214           *    call, and vkGetQueryPoolResults returns VK_NOT_READY. However,
 215           *    availability state is still written to pData for those queries
 216           *    if VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set.
 217           */
 218          result = VK_NOT_READY;
 219          if (!(flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)) {
 220             result_base += stride;
 221             continue;
 222          }
 223       }
 224
 225       if (available)
 226          write_query_value_cpu(result_base, 0, slot->result.value, flags);
 227       else if (flags & VK_QUERY_RESULT_PARTIAL_BIT)
 228           /* From the Vulkan 1.1.130 spec:
 229            *
 230            *   If VK_QUERY_RESULT_PARTIAL_BIT is set, VK_QUERY_RESULT_WAIT_BIT
 231            *   is not set, and the query’s status is unavailable, an
 232            *   intermediate result value between zero and the final result
 233            *   value is written to pData for that query.
 234            *
 235            * Just return 0 here for simplicity since it's a valid result.
 236            */
 237          write_query_value_cpu(result_base, 0, 0, flags);
 238
 239       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
 240          /* From the Vulkan 1.1.130 spec:
 241           *
 242           *    If VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set, the final
 243           *    integer value written for each query is non-zero if the query’s
 244           *    status was available or zero if the status was unavailable.
 245           */
 246          write_query_value_cpu(result_base, 1, available, flags);
 247
 248       result_base += stride;
 249    }
 250    return result;
 251 }
 252
 253 VkResult
 254 tu_GetQueryPoolResults(VkDevice _device,
 255                        VkQueryPool queryPool,
 256                        uint32_t firstQuery,
 257                        uint32_t queryCount,
 258                        size_t dataSize,
 259                        void *pData,
 260                        VkDeviceSize stride,
 261                        VkQueryResultFlags flags)
 262 {
 263    TU_FROM_HANDLE(tu_device, device, _device);
 264    TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
 265    assert(firstQuery + queryCount <= pool->size);
 266
 267    switch (pool->type) {
 268    case VK_QUERY_TYPE_OCCLUSION: {
 269       return get_occlusion_query_pool_results(device, pool, firstQuery,
 270             queryCount, dataSize, pData, stride, flags);
 271    }
 272    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
 273    case VK_QUERY_TYPE_TIMESTAMP:
 274       unreachable("Unimplemented query type");
 275    default:
 276       assert(!"Invalid query type");
 277    }
 278    return VK_SUCCESS;
 279 }
 280
 281 /* Copies a query value from one buffer to another from the GPU. */
 282 static void
 283 copy_query_value_gpu(struct tu_cmd_buffer *cmdbuf,
 284                      struct tu_cs *cs,
 285                      uint64_t src_iova,
 286                      uint64_t base_write_iova,
 287                      uint32_t offset,
 288                      VkQueryResultFlags flags) {
 289    uint32_t element_size = flags & VK_QUERY_RESULT_64_BIT ?
 290          sizeof(uint64_t) : sizeof(uint32_t);
 291    uint64_t write_iova = base_write_iova + (offset * element_size);
 292
 293    tu_cs_emit_pkt7(cs, CP_MEM_TO_MEM, 5);
 294    uint32_t mem_to_mem_flags = flags & VK_QUERY_RESULT_64_BIT ?
 295          CP_MEM_TO_MEM_0_DOUBLE : 0;
 296    tu_cs_emit(cs, mem_to_mem_flags);
 297    tu_cs_emit_qw(cs, write_iova);
 298    tu_cs_emit_qw(cs, src_iova);
 299 }
 300
 301 static void
 302 emit_copy_occlusion_query_pool_results(struct tu_cmd_buffer *cmdbuf,
 303                                        struct tu_cs *cs,
 304                                        struct tu_query_pool *pool,
 305                                        uint32_t firstQuery,
 306                                        uint32_t queryCount,
 307                                        struct tu_buffer *buffer,
 308                                        VkDeviceSize dstOffset,
 309                                        VkDeviceSize stride,
 310                                        VkQueryResultFlags flags)
 311 {
 312    /* From the Vulkan 1.1.130 spec:
 313     *
 314     *    vkCmdCopyQueryPoolResults is guaranteed to see the effect of previous
 315     *    uses of vkCmdResetQueryPool in the same queue, without any additional
 316     *    synchronization.
 317     *
 318     * To ensure that previous writes to the available bit are coherent, first
 319     * wait for all writes to complete.
 320     */
 321    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
 322
 323    for (uint32_t i = 0; i < queryCount; i++) {
 324       uint32_t query = firstQuery + i;
 325       uint64_t available_iova = occlusion_query_iova(pool, query, available);
 326       uint64_t result_iova = occlusion_query_iova(pool, query, result);
 327       uint64_t buffer_iova = tu_buffer_iova(buffer) + dstOffset + i * stride;
 328       /* Wait for the available bit to be set if executed with the
 329        * VK_QUERY_RESULT_WAIT_BIT flag. */
 330       if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 331          tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
 332          tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
 333                         CP_WAIT_REG_MEM_0_POLL_MEMORY);
 334          tu_cs_emit_qw(cs, available_iova);
 335          tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(0x1));
 336          tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
 337          tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
 338       }
 339
 340       if (flags & VK_QUERY_RESULT_PARTIAL_BIT) {
 341          /* Unconditionally copying the bo->result into the buffer here is
 342           * valid because we only set bo->result on vkCmdEndQuery. Thus, even
 343           * if the query is unavailable, this will copy the correct partial
 344           * value of 0.
 345           */
 346          copy_query_value_gpu(cmdbuf, cs, result_iova, buffer_iova,
 347                               0 /* offset */, flags);
 348       } else {
 349          /* Conditionally copy bo->result into the buffer based on whether the
 350           * query is available.
 351           *
 352           * NOTE: For the conditional packets to be executed, CP_COND_EXEC
 353           * tests that ADDR0 != 0 and ADDR1 < REF. The packet here simply tests
 354           * that 0 < available < 2, aka available == 1.
 355           */
 356          tu_cs_reserve(cs, 7 + 6);
 357          tu_cs_emit_pkt7(cs, CP_COND_EXEC, 6);
 358          tu_cs_emit_qw(cs, available_iova);
 359          tu_cs_emit_qw(cs, available_iova);
 360          tu_cs_emit(cs, CP_COND_EXEC_4_REF(0x2));
 361          tu_cs_emit(cs, 6); /* Cond execute the next 6 DWORDS */
 362
 363          /* Start of conditional execution */
 364          copy_query_value_gpu(cmdbuf, cs, result_iova, buffer_iova,
 365                               0 /* offset */, flags);
 366          /* End of conditional execution */
 367       }
 368
 369       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 370          copy_query_value_gpu(cmdbuf, cs, available_iova, buffer_iova,
 371                               1 /* offset */, flags);
 372       }
 373    }
 374
 375    tu_bo_list_add(&cmdbuf->bo_list, buffer->bo, MSM_SUBMIT_BO_WRITE);
 376 }
 377
 378 void
 379 tu_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer,
 380                            VkQueryPool queryPool,
 381                            uint32_t firstQuery,
 382                            uint32_t queryCount,
 383                            VkBuffer dstBuffer,
 384                            VkDeviceSize dstOffset,
 385                            VkDeviceSize stride,
 386                            VkQueryResultFlags flags)
 387 {
 388    TU_FROM_HANDLE(tu_cmd_buffer, cmdbuf, commandBuffer);
 389    TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
 390    TU_FROM_HANDLE(tu_buffer, buffer, dstBuffer);
 391    struct tu_cs *cs = &cmdbuf->cs;
 392    assert(firstQuery + queryCount <= pool->size);
 393
 394    switch (pool->type) {
 395    case VK_QUERY_TYPE_OCCLUSION: {
 396       return emit_copy_occlusion_query_pool_results(cmdbuf, cs, pool,
 397             firstQuery, queryCount, buffer, dstOffset, stride, flags);
 398    }
 399    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
 400    case VK_QUERY_TYPE_TIMESTAMP:
 401       unreachable("Unimplemented query type");
 402    default:
 403       assert(!"Invalid query type");
 404    }
 405 }
 406
 407 static void
 408 emit_reset_occlusion_query_pool(struct tu_cmd_buffer *cmdbuf,
 409                                 struct tu_query_pool *pool,
 410                                 uint32_t firstQuery,
 411                                 uint32_t queryCount)
 412 {
 413    struct tu_cs *cs = &cmdbuf->cs;
 414
 415    for (uint32_t i = 0; i < queryCount; i++) {
 416       uint32_t query = firstQuery + i;
 417       uint64_t available_iova = occlusion_query_iova(pool, query, available);
 418       uint64_t result_iova = occlusion_query_iova(pool, query, result);
 419       tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 4);
 420       tu_cs_emit_qw(cs, available_iova);
 421       tu_cs_emit_qw(cs, 0x0);
 422
 423       tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 4);
 424       tu_cs_emit_qw(cs, result_iova);
 425       tu_cs_emit_qw(cs, 0x0);
 426    }
 427 }
 428
 429 void
 430 tu_CmdResetQueryPool(VkCommandBuffer commandBuffer,
 431                      VkQueryPool queryPool,
 432                      uint32_t firstQuery,
 433                      uint32_t queryCount)
 434 {
 435    TU_FROM_HANDLE(tu_cmd_buffer, cmdbuf, commandBuffer);
 436    TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
 437
 438    switch (pool->type) {
 439    case VK_QUERY_TYPE_OCCLUSION:
 440       emit_reset_occlusion_query_pool(cmdbuf, pool, firstQuery, queryCount);
 441       break;
 442    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
 443    case VK_QUERY_TYPE_TIMESTAMP:
 444       unreachable("Unimplemented query type");
 445    default:
 446       assert(!"Invalid query type");
 447    }
 448
 449    tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
 450 }
 451
 452 static void
 453 emit_begin_occlusion_query(struct tu_cmd_buffer *cmdbuf,
 454                            struct tu_query_pool *pool,
 455                            uint32_t query)
 456 {
 457    /* From the Vulkan 1.1.130 spec:
 458     *
 459     *    A query must begin and end inside the same subpass of a render pass
 460     *    instance, or must both begin and end outside of a render pass
 461     *    instance.
 462     *
 463     * Unlike on an immediate-mode renderer, Turnip renders all tiles on
 464     * vkCmdEndRenderPass, not individually on each vkCmdDraw*. As such, if a
 465     * query begins/ends inside the same subpass of a render pass, we need to
 466     * record the packets on the secondary draw command stream. cmdbuf->draw_cs
 467     * is then run on every tile during render, so we just need to accumulate
 468     * sample counts in slot->result to compute the query result.
 469     */
 470    struct tu_cs *cs = cmdbuf->state.pass ? &cmdbuf->draw_cs : &cmdbuf->cs;
 471
 472    uint64_t begin_iova = occlusion_query_iova(pool, query, begin);
 473
 474    tu_cs_emit_regs(cs,
 475                    A6XX_RB_SAMPLE_COUNT_CONTROL(.copy = true));
 476
 477    tu_cs_emit_regs(cs,
 478                    A6XX_RB_SAMPLE_COUNT_ADDR_LO(begin_iova));
 479
 480    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
 481    tu_cs_emit(cs, ZPASS_DONE);
 482 }
 483
 484 void
 485 tu_CmdBeginQuery(VkCommandBuffer commandBuffer,
 486                  VkQueryPool queryPool,
 487                  uint32_t query,
 488                  VkQueryControlFlags flags)
 489 {
 490    TU_FROM_HANDLE(tu_cmd_buffer, cmdbuf, commandBuffer);
 491    TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
 492    assert(query < pool->size);
 493
 494    switch (pool->type) {
 495    case VK_QUERY_TYPE_OCCLUSION:
 496       /* In freedreno, there is no implementation difference between
 497        * GL_SAMPLES_PASSED and GL_ANY_SAMPLES_PASSED, so we can similarly
 498        * ignore the VK_QUERY_CONTROL_PRECISE_BIT flag here.
 499        */
 500       emit_begin_occlusion_query(cmdbuf, pool, query);
 501       break;
 502    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
 503    case VK_QUERY_TYPE_TIMESTAMP:
 504       unreachable("Unimplemented query type");
 505    default:
 506       assert(!"Invalid query type");
 507    }
 508
 509    tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
 510 }
 511
 512 static void
 513 emit_end_occlusion_query(struct tu_cmd_buffer *cmdbuf,
 514                          struct tu_query_pool *pool,
 515                          uint32_t query)
 516 {
 517    /* Ending an occlusion query happens in a few steps:
 518     *    1) Set the slot->end to UINT64_MAX.
 519     *    2) Set up the SAMPLE_COUNT registers and trigger a CP_EVENT_WRITE to
 520     *       write the current sample count value into slot->end.
 521     *    3) Since (2) is asynchronous, wait until slot->end is not equal to
 522     *       UINT64_MAX before continuing via CP_WAIT_REG_MEM.
 523     *    4) Accumulate the results of the query (slot->end - slot->begin) into
 524     *       slot->result.
 525     *    5) If vkCmdEndQuery is *not* called from within the scope of a render
 526     *       pass, set the slot's available bit since the query is now done.
 527     *    6) If vkCmdEndQuery *is* called from within the scope of a render
 528     *       pass, we cannot mark as available yet since the commands in
 529     *       draw_cs are not run until vkCmdEndRenderPass.
 530     */
 531    const struct tu_render_pass *pass = cmdbuf->state.pass;
 532    struct tu_cs *cs = pass ? &cmdbuf->draw_cs : &cmdbuf->cs;
 533
 534    uint64_t available_iova = occlusion_query_iova(pool, query, available);
 535    uint64_t begin_iova = occlusion_query_iova(pool, query, begin);
 536    uint64_t end_iova = occlusion_query_iova(pool, query, end);
 537    uint64_t result_iova = occlusion_query_iova(pool, query, result);
 538    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 4);
 539    tu_cs_emit_qw(cs, end_iova);
 540    tu_cs_emit_qw(cs, 0xffffffffffffffffull);
 541
 542    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
 543
 544    tu_cs_emit_regs(cs,
 545                    A6XX_RB_SAMPLE_COUNT_CONTROL(.copy = true));
 546
 547    tu_cs_emit_regs(cs,
 548                    A6XX_RB_SAMPLE_COUNT_ADDR_LO(end_iova));
 549
 550    tu_cs_emit_pkt7(cs, CP_EVENT_WRITE, 1);
 551    tu_cs_emit(cs, ZPASS_DONE);
 552
 553    tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
 554    tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_NE) |
 555                   CP_WAIT_REG_MEM_0_POLL_MEMORY);
 556    tu_cs_emit_qw(cs, end_iova);
 557    tu_cs_emit(cs, CP_WAIT_REG_MEM_3_REF(0xffffffff));
 558    tu_cs_emit(cs, CP_WAIT_REG_MEM_4_MASK(~0));
 559    tu_cs_emit(cs, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(16));
 560
 561    /* result (dst) = result (srcA) + end (srcB) - begin (srcC) */
 562    tu_cs_emit_pkt7(cs, CP_MEM_TO_MEM, 9);
 563    tu_cs_emit(cs, CP_MEM_TO_MEM_0_DOUBLE | CP_MEM_TO_MEM_0_NEG_C);
 564    tu_cs_emit_qw(cs, result_iova);
 565    tu_cs_emit_qw(cs, result_iova);
 566    tu_cs_emit_qw(cs, end_iova);
 567    tu_cs_emit_qw(cs, begin_iova);
 568
 569    tu_cs_emit_pkt7(cs, CP_WAIT_MEM_WRITES, 0);
 570
 571    if (pass)
 572       /* Technically, queries should be tracked per-subpass, but here we track
 573        * at the render pass level to simply the code a bit. This is safe
 574        * because the only commands that use the available bit are
 575        * vkCmdCopyQueryPoolResults and vkCmdResetQueryPool, both of which
 576        * cannot be invoked from inside a render pass scope.
 577        */
 578       cs = &cmdbuf->draw_epilogue_cs;
 579
 580    tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 4);
 581    tu_cs_emit_qw(cs, available_iova);
 582    tu_cs_emit_qw(cs, 0x1);
 583 }
 584
 585 void
 586 tu_CmdEndQuery(VkCommandBuffer commandBuffer,
 587                VkQueryPool queryPool,
 588                uint32_t query)
 589 {
 590    TU_FROM_HANDLE(tu_cmd_buffer, cmdbuf, commandBuffer);
 591    TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
 592    assert(query < pool->size);
 593
 594    switch (pool->type) {
 595    case VK_QUERY_TYPE_OCCLUSION:
 596       emit_end_occlusion_query(cmdbuf, pool, query);
 597       break;
 598    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
 599    case VK_QUERY_TYPE_TIMESTAMP:
 600       unreachable("Unimplemented query type");
 601    default:
 602       assert(!"Invalid query type");
 603    }
 604
 605    tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
 606 }
 607
 608 void
 609 tu_CmdWriteTimestamp(VkCommandBuffer commandBuffer,
 610                      VkPipelineStageFlagBits pipelineStage,
 611                      VkQueryPool queryPool,
 612                      uint32_t query)
 613 {
 614 }