src/intel/vulkan/genX_query.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 #include "genxml/gen_macros.h"
  33 #include "genxml/genX_pack.h"
  34
  35 VkResult genX(CreateQueryPool)(
  36     VkDevice                                    _device,
  37     const VkQueryPoolCreateInfo*                pCreateInfo,
  38     const VkAllocationCallbacks*                pAllocator,
  39     VkQueryPool*                                pQueryPool)
  40 {
  41    ANV_FROM_HANDLE(anv_device, device, _device);
  42    struct anv_query_pool *pool;
  43    VkResult result;
  44
  45    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO);
  46
  47    /* Query pool slots are made up of some number of 64-bit values packed
  48     * tightly together.  The first 64-bit value is always the "available" bit
  49     * which is 0 when the query is unavailable and 1 when it is available.
  50     * The 64-bit values that follow are determined by the type of query.
  51     */
  52    uint32_t uint64s_per_slot = 1;
  53
  54    VkQueryPipelineStatisticFlags pipeline_statistics = 0;
  55    switch (pCreateInfo->queryType) {
  56    case VK_QUERY_TYPE_OCCLUSION:
  57       /* Occlusion queries have two values: begin and end. */
  58       uint64s_per_slot += 2;
  59       break;
  60    case VK_QUERY_TYPE_TIMESTAMP:
  61       /* Timestamps just have the one timestamp value */
  62       uint64s_per_slot += 1;
  63       break;
  64    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
  65       pipeline_statistics = pCreateInfo->pipelineStatistics;
  66       /* We're going to trust this field implicitly so we need to ensure that
  67        * no unhandled extension bits leak in.
  68        */
  69       pipeline_statistics &= ANV_PIPELINE_STATISTICS_MASK;
  70
  71       /* Statistics queries have a min and max for every statistic */
  72       uint64s_per_slot += 2 * _mesa_bitcount(pipeline_statistics);
  73       break;
  74    default:
  75       assert(!"Invalid query type");
  76    }
  77
  78    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
  79                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  80    if (pool == NULL)
  81       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
  82
  83    pool->type = pCreateInfo->queryType;
  84    pool->pipeline_statistics = pipeline_statistics;
  85    pool->stride = uint64s_per_slot * sizeof(uint64_t);
  86    pool->slots = pCreateInfo->queryCount;
  87
  88    uint64_t size = pool->slots * pool->stride;
  89    result = anv_bo_init_new(&pool->bo, device, size);
  90    if (result != VK_SUCCESS)
  91       goto fail;
  92
  93    pool->bo.map = anv_gem_mmap(device, pool->bo.gem_handle, 0, size, 0);
  94
  95    *pQueryPool = anv_query_pool_to_handle(pool);
  96
  97    return VK_SUCCESS;
  98
  99  fail:
 100    vk_free2(&device->alloc, pAllocator, pool);
 101
 102    return result;
 103 }
 104
 105 void genX(DestroyQueryPool)(
 106     VkDevice                                    _device,
 107     VkQueryPool                                 _pool,
 108     const VkAllocationCallbacks*                pAllocator)
 109 {
 110    ANV_FROM_HANDLE(anv_device, device, _device);
 111    ANV_FROM_HANDLE(anv_query_pool, pool, _pool);
 112
 113    if (!pool)
 114       return;
 115
 116    anv_gem_munmap(pool->bo.map, pool->bo.size);
 117    anv_gem_close(device, pool->bo.gem_handle);
 118    vk_free2(&device->alloc, pAllocator, pool);
 119 }
 120
 121 static void
 122 cpu_write_query_result(void *dst_slot, VkQueryResultFlags flags,
 123                        uint32_t value_index, uint64_t result)
 124 {
 125    if (flags & VK_QUERY_RESULT_64_BIT) {
 126       uint64_t *dst64 = dst_slot;
 127       dst64[value_index] = result;
 128    } else {
 129       uint32_t *dst32 = dst_slot;
 130       dst32[value_index] = result;
 131    }
 132 }
 133
 134 VkResult genX(GetQueryPoolResults)(
 135     VkDevice                                    _device,
 136     VkQueryPool                                 queryPool,
 137     uint32_t                                    firstQuery,
 138     uint32_t                                    queryCount,
 139     size_t                                      dataSize,
 140     void*                                       pData,
 141     VkDeviceSize                                stride,
 142     VkQueryResultFlags                          flags)
 143 {
 144    ANV_FROM_HANDLE(anv_device, device, _device);
 145    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 146    int64_t timeout = INT64_MAX;
 147    int ret;
 148
 149    assert(pool->type == VK_QUERY_TYPE_OCCLUSION ||
 150           pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS ||
 151           pool->type == VK_QUERY_TYPE_TIMESTAMP);
 152
 153    if (pData == NULL)
 154       return VK_SUCCESS;
 155
 156    if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 157       ret = anv_gem_wait(device, pool->bo.gem_handle, &timeout);
 158       if (ret == -1) {
 159          /* We don't know the real error. */
 160          return vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY,
 161                           "gem_wait failed %m");
 162       }
 163    }
 164
 165    void *data_end = pData + dataSize;
 166
 167    if (!device->info.has_llc) {
 168       uint64_t offset = firstQuery * pool->stride;
 169       uint64_t size = queryCount * pool->stride;
 170       anv_invalidate_range(pool->bo.map + offset,
 171                            MIN2(size, pool->bo.size - offset));
 172    }
 173
 174    VkResult status = VK_SUCCESS;
 175    for (uint32_t i = 0; i < queryCount; i++) {
 176       uint64_t *slot = pool->bo.map + (firstQuery + i) * pool->stride;
 177
 178       /* Availability is always at the start of the slot */
 179       bool available = slot[0];
 180
 181       /* From the Vulkan 1.0.42 spec:
 182        *
 183        *    "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
 184        *    both not set then no result values are written to pData for
 185        *    queries that are in the unavailable state at the time of the call,
 186        *    and vkGetQueryPoolResults returns VK_NOT_READY. However,
 187        *    availability state is still written to pData for those queries if
 188        *    VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set."
 189        */
 190       bool write_results = available || (flags & VK_QUERY_RESULT_PARTIAL_BIT);
 191
 192       if (write_results) {
 193          switch (pool->type) {
 194          case VK_QUERY_TYPE_OCCLUSION: {
 195             cpu_write_query_result(pData, flags, 0, slot[2] - slot[1]);
 196             break;
 197          }
 198
 199          case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 200             uint32_t statistics = pool->pipeline_statistics;
 201             uint32_t idx = 0;
 202             while (statistics) {
 203                uint32_t stat = u_bit_scan(&statistics);
 204                uint64_t result = slot[idx * 2 + 2] - slot[idx * 2 + 1];
 205
 206                /* WaDividePSInvocationCountBy4:HSW,BDW */
 207                if ((device->info.gen == 8 || device->info.is_haswell) &&
 208                    (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT)
 209                   result >>= 2;
 210
 211                cpu_write_query_result(pData, flags, idx, result);
 212
 213                idx++;
 214             }
 215             assert(idx == _mesa_bitcount(pool->pipeline_statistics));
 216             break;
 217          }
 218
 219          case VK_QUERY_TYPE_TIMESTAMP: {
 220             cpu_write_query_result(pData, flags, 0, slot[1]);
 221             break;
 222          }
 223          default:
 224             unreachable("invalid pool type");
 225          }
 226       } else {
 227          status = VK_NOT_READY;
 228       }
 229
 230       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 231          uint32_t idx = (pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS) ?
 232                         _mesa_bitcount(pool->pipeline_statistics) : 1;
 233          cpu_write_query_result(pData, flags, idx, available);
 234       }
 235
 236       pData += stride;
 237       if (pData >= data_end)
 238          break;
 239    }
 240
 241    return status;
 242 }
 243
 244 static void
 245 emit_ps_depth_count(struct anv_cmd_buffer *cmd_buffer,
 246                     struct anv_bo *bo, uint32_t offset)
 247 {
 248    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 249       pc.DestinationAddressType  = DAT_PPGTT;
 250       pc.PostSyncOperation       = WritePSDepthCount;
 251       pc.DepthStallEnable        = true;
 252       pc.Address                 = (struct anv_address) { bo, offset };
 253
 254       if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
 255          pc.CommandStreamerStallEnable = true;
 256    }
 257 }
 258
 259 static void
 260 emit_query_availability(struct anv_cmd_buffer *cmd_buffer,
 261                         struct anv_bo *bo, uint32_t offset)
 262 {
 263    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 264       pc.DestinationAddressType  = DAT_PPGTT;
 265       pc.PostSyncOperation       = WriteImmediateData;
 266       pc.Address                 = (struct anv_address) { bo, offset };
 267       pc.ImmediateData           = 1;
 268    }
 269 }
 270
 271 void genX(CmdResetQueryPool)(
 272     VkCommandBuffer                             commandBuffer,
 273     VkQueryPool                                 queryPool,
 274     uint32_t                                    firstQuery,
 275     uint32_t                                    queryCount)
 276 {
 277    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 278    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 279
 280    for (uint32_t i = 0; i < queryCount; i++) {
 281       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdm) {
 282          sdm.Address = (struct anv_address) {
 283             .bo = &pool->bo,
 284             .offset = (firstQuery + i) * pool->stride,
 285          };
 286          sdm.ImmediateData = 0;
 287       }
 288    }
 289 }
 290
 291 static const uint32_t vk_pipeline_stat_to_reg[] = {
 292    GENX(IA_VERTICES_COUNT_num),
 293    GENX(IA_PRIMITIVES_COUNT_num),
 294    GENX(VS_INVOCATION_COUNT_num),
 295    GENX(GS_INVOCATION_COUNT_num),
 296    GENX(GS_PRIMITIVES_COUNT_num),
 297    GENX(CL_INVOCATION_COUNT_num),
 298    GENX(CL_PRIMITIVES_COUNT_num),
 299    GENX(PS_INVOCATION_COUNT_num),
 300    GENX(HS_INVOCATION_COUNT_num),
 301    GENX(DS_INVOCATION_COUNT_num),
 302    GENX(CS_INVOCATION_COUNT_num),
 303 };
 304
 305 static void
 306 emit_pipeline_stat(struct anv_cmd_buffer *cmd_buffer, uint32_t stat,
 307                    struct anv_bo *bo, uint32_t offset)
 308 {
 309    STATIC_ASSERT(ANV_PIPELINE_STATISTICS_MASK ==
 310                  (1 << ARRAY_SIZE(vk_pipeline_stat_to_reg)) - 1);
 311
 312    assert(stat < ARRAY_SIZE(vk_pipeline_stat_to_reg));
 313    uint32_t reg = vk_pipeline_stat_to_reg[stat];
 314
 315    anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
 316       lrm.RegisterAddress  = reg,
 317       lrm.MemoryAddress    = (struct anv_address) { bo, offset };
 318    }
 319    anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
 320       lrm.RegisterAddress  = reg + 4,
 321       lrm.MemoryAddress    = (struct anv_address) { bo, offset + 4 };
 322    }
 323 }
 324
 325 void genX(CmdBeginQuery)(
 326     VkCommandBuffer                             commandBuffer,
 327     VkQueryPool                                 queryPool,
 328     uint32_t                                    query,
 329     VkQueryControlFlags                         flags)
 330 {
 331    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 332    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 333
 334    /* Workaround: When meta uses the pipeline with the VS disabled, it seems
 335     * that the pipelining of the depth write breaks. What we see is that
 336     * samples from the render pass clear leaks into the first query
 337     * immediately after the clear. Doing a pipecontrol with a post-sync
 338     * operation and DepthStallEnable seems to work around the issue.
 339     */
 340    if (cmd_buffer->state.need_query_wa) {
 341       cmd_buffer->state.need_query_wa = false;
 342       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 343          pc.DepthCacheFlushEnable   = true;
 344          pc.DepthStallEnable        = true;
 345       }
 346    }
 347
 348    switch (pool->type) {
 349    case VK_QUERY_TYPE_OCCLUSION:
 350       emit_ps_depth_count(cmd_buffer, &pool->bo, query * pool->stride + 8);
 351       break;
 352
 353    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 354       /* TODO: This might only be necessary for certain stats */
 355       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 356          pc.CommandStreamerStallEnable = true;
 357          pc.StallAtPixelScoreboard = true;
 358       }
 359
 360       uint32_t statistics = pool->pipeline_statistics;
 361       uint32_t offset = query * pool->stride + 8;
 362       while (statistics) {
 363          uint32_t stat = u_bit_scan(&statistics);
 364          emit_pipeline_stat(cmd_buffer, stat, &pool->bo, offset);
 365          offset += 16;
 366       }
 367       break;
 368    }
 369
 370    default:
 371       unreachable("");
 372    }
 373 }
 374
 375 void genX(CmdEndQuery)(
 376     VkCommandBuffer                             commandBuffer,
 377     VkQueryPool                                 queryPool,
 378     uint32_t                                    query)
 379 {
 380    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 381    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 382
 383    switch (pool->type) {
 384    case VK_QUERY_TYPE_OCCLUSION:
 385       emit_ps_depth_count(cmd_buffer, &pool->bo, query * pool->stride + 16);
 386       emit_query_availability(cmd_buffer, &pool->bo, query * pool->stride);
 387       break;
 388
 389    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 390       /* TODO: This might only be necessary for certain stats */
 391       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 392          pc.CommandStreamerStallEnable = true;
 393          pc.StallAtPixelScoreboard = true;
 394       }
 395
 396       uint32_t statistics = pool->pipeline_statistics;
 397       uint32_t offset = query * pool->stride + 16;
 398       while (statistics) {
 399          uint32_t stat = u_bit_scan(&statistics);
 400          emit_pipeline_stat(cmd_buffer, stat, &pool->bo, offset);
 401          offset += 16;
 402       }
 403
 404       emit_query_availability(cmd_buffer, &pool->bo, query * pool->stride);
 405       break;
 406    }
 407
 408    default:
 409       unreachable("");
 410    }
 411 }
 412
 413 #define TIMESTAMP 0x2358
 414
 415 void genX(CmdWriteTimestamp)(
 416     VkCommandBuffer                             commandBuffer,
 417     VkPipelineStageFlagBits                     pipelineStage,
 418     VkQueryPool                                 queryPool,
 419     uint32_t                                    query)
 420 {
 421    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 422    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 423    uint32_t offset = query * pool->stride;
 424
 425    assert(pool->type == VK_QUERY_TYPE_TIMESTAMP);
 426
 427    switch (pipelineStage) {
 428    case VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT:
 429       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 430          srm.RegisterAddress  = TIMESTAMP;
 431          srm.MemoryAddress    = (struct anv_address) { &pool->bo, offset + 8 };
 432       }
 433       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 434          srm.RegisterAddress  = TIMESTAMP + 4;
 435          srm.MemoryAddress    = (struct anv_address) { &pool->bo, offset + 12 };
 436       }
 437       break;
 438
 439    default:
 440       /* Everything else is bottom-of-pipe */
 441       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 442          pc.DestinationAddressType  = DAT_PPGTT;
 443          pc.PostSyncOperation       = WriteTimestamp;
 444          pc.Address = (struct anv_address) { &pool->bo, offset + 8 };
 445
 446          if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
 447             pc.CommandStreamerStallEnable = true;
 448       }
 449       break;
 450    }
 451
 452    emit_query_availability(cmd_buffer, &pool->bo, offset);
 453 }
 454
 455 #if GEN_GEN > 7 || GEN_IS_HASWELL
 456
 457 #define alu_opcode(v)   __gen_uint((v),  20, 31)
 458 #define alu_operand1(v) __gen_uint((v),  10, 19)
 459 #define alu_operand2(v) __gen_uint((v),   0,  9)
 460 #define alu(opcode, operand1, operand2) \
 461    alu_opcode(opcode) | alu_operand1(operand1) | alu_operand2(operand2)
 462
 463 #define OPCODE_NOOP      0x000
 464 #define OPCODE_LOAD      0x080
 465 #define OPCODE_LOADINV   0x480
 466 #define OPCODE_LOAD0     0x081
 467 #define OPCODE_LOAD1     0x481
 468 #define OPCODE_ADD       0x100
 469 #define OPCODE_SUB       0x101
 470 #define OPCODE_AND       0x102
 471 #define OPCODE_OR        0x103
 472 #define OPCODE_XOR       0x104
 473 #define OPCODE_STORE     0x180
 474 #define OPCODE_STOREINV  0x580
 475
 476 #define OPERAND_R0   0x00
 477 #define OPERAND_R1   0x01
 478 #define OPERAND_R2   0x02
 479 #define OPERAND_R3   0x03
 480 #define OPERAND_R4   0x04
 481 #define OPERAND_SRCA 0x20
 482 #define OPERAND_SRCB 0x21
 483 #define OPERAND_ACCU 0x31
 484 #define OPERAND_ZF   0x32
 485 #define OPERAND_CF   0x33
 486
 487 #define CS_GPR(n) (0x2600 + (n) * 8)
 488
 489 static void
 490 emit_load_alu_reg_u64(struct anv_batch *batch, uint32_t reg,
 491                       struct anv_bo *bo, uint32_t offset)
 492 {
 493    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
 494       lrm.RegisterAddress  = reg,
 495       lrm.MemoryAddress    = (struct anv_address) { bo, offset };
 496    }
 497    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
 498       lrm.RegisterAddress  = reg + 4;
 499       lrm.MemoryAddress    = (struct anv_address) { bo, offset + 4 };
 500    }
 501 }
 502
 503 static void
 504 emit_load_alu_reg_imm32(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 505 {
 506    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
 507       lri.RegisterOffset   = reg;
 508       lri.DataDWord        = imm;
 509    }
 510 }
 511
 512 static void
 513 emit_load_alu_reg_imm64(struct anv_batch *batch, uint32_t reg, uint64_t imm)
 514 {
 515    emit_load_alu_reg_imm32(batch, reg, (uint32_t)imm);
 516    emit_load_alu_reg_imm32(batch, reg + 4, (uint32_t)(imm >> 32));
 517 }
 518
 519 static void
 520 emit_load_alu_reg_reg32(struct anv_batch *batch, uint32_t src, uint32_t dst)
 521 {
 522    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_REG), lrr) {
 523       lrr.SourceRegisterAddress      = src;
 524       lrr.DestinationRegisterAddress = dst;
 525    }
 526 }
 527
 528 /*
 529  * GPR0 = GPR0 & ((1ull << n) - 1);
 530  */
 531 static void
 532 keep_gpr0_lower_n_bits(struct anv_batch *batch, uint32_t n)
 533 {
 534    assert(n < 64);
 535    emit_load_alu_reg_imm64(batch, CS_GPR(1), (1ull << n) - 1);
 536
 537    uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
 538    dw[1] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R0);
 539    dw[2] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R1);
 540    dw[3] = alu(OPCODE_AND, 0, 0);
 541    dw[4] = alu(OPCODE_STORE, OPERAND_R0, OPERAND_ACCU);
 542 }
 543
 544 /*
 545  * GPR0 = GPR0 << 30;
 546  */
 547 static void
 548 shl_gpr0_by_30_bits(struct anv_batch *batch)
 549 {
 550    /* First we mask 34 bits of GPR0 to prevent overflow */
 551    keep_gpr0_lower_n_bits(batch, 34);
 552
 553    const uint32_t outer_count = 5;
 554    const uint32_t inner_count = 6;
 555    STATIC_ASSERT(outer_count * inner_count == 30);
 556    const uint32_t cmd_len = 1 + inner_count * 4;
 557
 558    /* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
 559     * 30 left shifts.
 560     */
 561    for (int o = 0; o < outer_count; o++) {
 562       /* Submit one MI_MATH to shift left by 6 bits */
 563       uint32_t *dw = anv_batch_emitn(batch, cmd_len, GENX(MI_MATH));
 564       dw++;
 565       for (int i = 0; i < inner_count; i++, dw += 4) {
 566          dw[0] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R0);
 567          dw[1] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R0);
 568          dw[2] = alu(OPCODE_ADD, 0, 0);
 569          dw[3] = alu(OPCODE_STORE, OPERAND_R0, OPERAND_ACCU);
 570       }
 571    }
 572 }
 573
 574 /*
 575  * GPR0 = GPR0 >> 2;
 576  *
 577  * Note that the upper 30 bits of GPR are lost!
 578  */
 579 static void
 580 shr_gpr0_by_2_bits(struct anv_batch *batch)
 581 {
 582    shl_gpr0_by_30_bits(batch);
 583    emit_load_alu_reg_reg32(batch, CS_GPR(0) + 4, CS_GPR(0));
 584    emit_load_alu_reg_imm32(batch, CS_GPR(0) + 4, 0);
 585 }
 586
 587 static void
 588 gpu_write_query_result(struct anv_batch *batch,
 589                        struct anv_buffer *dst_buffer, uint32_t dst_offset,
 590                        VkQueryResultFlags flags,
 591                        uint32_t value_index, uint32_t reg)
 592 {
 593    if (flags & VK_QUERY_RESULT_64_BIT)
 594       dst_offset += value_index * 8;
 595    else
 596       dst_offset += value_index * 4;
 597
 598    anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 599       srm.RegisterAddress  = reg;
 600       srm.MemoryAddress    = (struct anv_address) {
 601          .bo = dst_buffer->bo,
 602          .offset = dst_buffer->offset + dst_offset,
 603       };
 604    }
 605
 606    if (flags & VK_QUERY_RESULT_64_BIT) {
 607       anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 608          srm.RegisterAddress  = reg + 4;
 609          srm.MemoryAddress    = (struct anv_address) {
 610             .bo = dst_buffer->bo,
 611             .offset = dst_buffer->offset + dst_offset + 4,
 612          };
 613       }
 614    }
 615 }
 616
 617 static void
 618 compute_query_result(struct anv_batch *batch, uint32_t dst_reg,
 619                      struct anv_bo *bo, uint32_t offset)
 620 {
 621    emit_load_alu_reg_u64(batch, CS_GPR(0), bo, offset);
 622    emit_load_alu_reg_u64(batch, CS_GPR(1), bo, offset + 8);
 623
 624    /* FIXME: We need to clamp the result for 32 bit. */
 625
 626    uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
 627    dw[1] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R1);
 628    dw[2] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R0);
 629    dw[3] = alu(OPCODE_SUB, 0, 0);
 630    dw[4] = alu(OPCODE_STORE, dst_reg, OPERAND_ACCU);
 631 }
 632
 633 void genX(CmdCopyQueryPoolResults)(
 634     VkCommandBuffer                             commandBuffer,
 635     VkQueryPool                                 queryPool,
 636     uint32_t                                    firstQuery,
 637     uint32_t                                    queryCount,
 638     VkBuffer                                    destBuffer,
 639     VkDeviceSize                                destOffset,
 640     VkDeviceSize                                destStride,
 641     VkQueryResultFlags                          flags)
 642 {
 643    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 644    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 645    ANV_FROM_HANDLE(anv_buffer, buffer, destBuffer);
 646    uint32_t slot_offset;
 647
 648    if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 649       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 650          pc.CommandStreamerStallEnable = true;
 651          pc.StallAtPixelScoreboard     = true;
 652       }
 653    }
 654
 655    for (uint32_t i = 0; i < queryCount; i++) {
 656       slot_offset = (firstQuery + i) * pool->stride;
 657       switch (pool->type) {
 658       case VK_QUERY_TYPE_OCCLUSION:
 659          compute_query_result(&cmd_buffer->batch, OPERAND_R2,
 660                               &pool->bo, slot_offset + 8);
 661          gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 662                                 flags, 0, CS_GPR(2));
 663          break;
 664
 665       case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 666          uint32_t statistics = pool->pipeline_statistics;
 667          uint32_t idx = 0;
 668          while (statistics) {
 669             uint32_t stat = u_bit_scan(&statistics);
 670
 671             compute_query_result(&cmd_buffer->batch, OPERAND_R0,
 672                                  &pool->bo, slot_offset + idx * 16 + 8);
 673
 674             /* WaDividePSInvocationCountBy4:HSW,BDW */
 675             if ((cmd_buffer->device->info.gen == 8 ||
 676                  cmd_buffer->device->info.is_haswell) &&
 677                 (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT) {
 678                shr_gpr0_by_2_bits(&cmd_buffer->batch);
 679             }
 680
 681             gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 682                                    flags, idx, CS_GPR(0));
 683
 684             idx++;
 685          }
 686          assert(idx == _mesa_bitcount(pool->pipeline_statistics));
 687          break;
 688       }
 689
 690       case VK_QUERY_TYPE_TIMESTAMP:
 691          emit_load_alu_reg_u64(&cmd_buffer->batch,
 692                                CS_GPR(2), &pool->bo, slot_offset + 8);
 693          gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 694                                 flags, 0, CS_GPR(2));
 695          break;
 696
 697       default:
 698          unreachable("unhandled query type");
 699       }
 700
 701       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 702          uint32_t idx = (pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS) ?
 703                         _mesa_bitcount(pool->pipeline_statistics) : 1;
 704
 705          emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(0),
 706                                &pool->bo, slot_offset);
 707          gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 708                                 flags, idx, CS_GPR(0));
 709       }
 710
 711       destOffset += destStride;
 712    }
 713 }
 714
 715 #else
 716 void genX(CmdCopyQueryPoolResults)(
 717     VkCommandBuffer                             commandBuffer,
 718     VkQueryPool                                 queryPool,
 719     uint32_t                                    firstQuery,
 720     uint32_t                                    queryCount,
 721     VkBuffer                                    destBuffer,
 722     VkDeviceSize                                destOffset,
 723     VkDeviceSize                                destStride,
 724     VkQueryResultFlags                          flags)
 725 {
 726    anv_finishme("Queries not yet supported on Ivy Bridge");
 727 }
 728 #endif