src/intel/vulkan/genX_query.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 #include "genxml/gen_macros.h"
  33 #include "genxml/genX_pack.h"
  34
  35 VkResult genX(CreateQueryPool)(
  36     VkDevice                                    _device,
  37     const VkQueryPoolCreateInfo*                pCreateInfo,
  38     const VkAllocationCallbacks*                pAllocator,
  39     VkQueryPool*                                pQueryPool)
  40 {
  41    ANV_FROM_HANDLE(anv_device, device, _device);
  42    const struct anv_physical_device *pdevice = &device->instance->physicalDevice;
  43    struct anv_query_pool *pool;
  44    VkResult result;
  45
  46    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO);
  47
  48    /* Query pool slots are made up of some number of 64-bit values packed
  49     * tightly together.  The first 64-bit value is always the "available" bit
  50     * which is 0 when the query is unavailable and 1 when it is available.
  51     * The 64-bit values that follow are determined by the type of query.
  52     */
  53    uint32_t uint64s_per_slot = 1;
  54
  55    VkQueryPipelineStatisticFlags pipeline_statistics = 0;
  56    switch (pCreateInfo->queryType) {
  57    case VK_QUERY_TYPE_OCCLUSION:
  58       /* Occlusion queries have two values: begin and end. */
  59       uint64s_per_slot += 2;
  60       break;
  61    case VK_QUERY_TYPE_TIMESTAMP:
  62       /* Timestamps just have the one timestamp value */
  63       uint64s_per_slot += 1;
  64       break;
  65    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
  66       pipeline_statistics = pCreateInfo->pipelineStatistics;
  67       /* We're going to trust this field implicitly so we need to ensure that
  68        * no unhandled extension bits leak in.
  69        */
  70       pipeline_statistics &= ANV_PIPELINE_STATISTICS_MASK;
  71
  72       /* Statistics queries have a min and max for every statistic */
  73       uint64s_per_slot += 2 * util_bitcount(pipeline_statistics);
  74       break;
  75    default:
  76       assert(!"Invalid query type");
  77    }
  78
  79    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
  80                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  81    if (pool == NULL)
  82       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
  83
  84    pool->type = pCreateInfo->queryType;
  85    pool->pipeline_statistics = pipeline_statistics;
  86    pool->stride = uint64s_per_slot * sizeof(uint64_t);
  87    pool->slots = pCreateInfo->queryCount;
  88
  89    uint64_t size = pool->slots * pool->stride;
  90    result = anv_bo_init_new(&pool->bo, device, size);
  91    if (result != VK_SUCCESS)
  92       goto fail;
  93
  94    if (pdevice->supports_48bit_addresses)
  95       pool->bo.flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
  96
  97    if (pdevice->use_softpin)
  98       pool->bo.flags |= EXEC_OBJECT_PINNED;
  99
 100    if (pdevice->has_exec_async)
 101       pool->bo.flags |= EXEC_OBJECT_ASYNC;
 102
 103    anv_vma_alloc(device, &pool->bo);
 104
 105    /* For query pools, we set the caching mode to I915_CACHING_CACHED.  On LLC
 106     * platforms, this does nothing.  On non-LLC platforms, this means snooping
 107     * which comes at a slight cost.  However, the buffers aren't big, won't be
 108     * written frequently, and trying to handle the flushing manually without
 109     * doing too much flushing is extremely painful.
 110     */
 111    anv_gem_set_caching(device, pool->bo.gem_handle, I915_CACHING_CACHED);
 112
 113    pool->bo.map = anv_gem_mmap(device, pool->bo.gem_handle, 0, size, 0);
 114
 115    *pQueryPool = anv_query_pool_to_handle(pool);
 116
 117    return VK_SUCCESS;
 118
 119  fail:
 120    vk_free2(&device->alloc, pAllocator, pool);
 121
 122    return result;
 123 }
 124
 125 void genX(DestroyQueryPool)(
 126     VkDevice                                    _device,
 127     VkQueryPool                                 _pool,
 128     const VkAllocationCallbacks*                pAllocator)
 129 {
 130    ANV_FROM_HANDLE(anv_device, device, _device);
 131    ANV_FROM_HANDLE(anv_query_pool, pool, _pool);
 132
 133    if (!pool)
 134       return;
 135
 136    anv_gem_munmap(pool->bo.map, pool->bo.size);
 137    anv_vma_free(device, &pool->bo);
 138    anv_gem_close(device, pool->bo.gem_handle);
 139    vk_free2(&device->alloc, pAllocator, pool);
 140 }
 141
 142 static struct anv_address
 143 anv_query_address(struct anv_query_pool *pool, uint32_t query)
 144 {
 145    return (struct anv_address) {
 146       .bo = &pool->bo,
 147       .offset = query * pool->stride,
 148    };
 149 }
 150
 151 static void
 152 cpu_write_query_result(void *dst_slot, VkQueryResultFlags flags,
 153                        uint32_t value_index, uint64_t result)
 154 {
 155    if (flags & VK_QUERY_RESULT_64_BIT) {
 156       uint64_t *dst64 = dst_slot;
 157       dst64[value_index] = result;
 158    } else {
 159       uint32_t *dst32 = dst_slot;
 160       dst32[value_index] = result;
 161    }
 162 }
 163
 164 static bool
 165 query_is_available(uint64_t *slot)
 166 {
 167    return *(volatile uint64_t *)slot;
 168 }
 169
 170 static VkResult
 171 wait_for_available(struct anv_device *device,
 172                    struct anv_query_pool *pool, uint64_t *slot)
 173 {
 174    while (true) {
 175       if (query_is_available(slot))
 176          return VK_SUCCESS;
 177
 178       int ret = anv_gem_busy(device, pool->bo.gem_handle);
 179       if (ret == 1) {
 180          /* The BO is still busy, keep waiting. */
 181          continue;
 182       } else if (ret == -1) {
 183          /* We don't know the real error. */
 184          device->lost = true;
 185          return vk_errorf(device->instance, device, VK_ERROR_DEVICE_LOST,
 186                           "gem wait failed: %m");
 187       } else {
 188          assert(ret == 0);
 189          /* The BO is no longer busy. */
 190          if (query_is_available(slot)) {
 191             return VK_SUCCESS;
 192          } else {
 193             VkResult status = anv_device_query_status(device);
 194             if (status != VK_SUCCESS)
 195                return status;
 196
 197             /* If we haven't seen availability yet, then we never will.  This
 198              * can only happen if we have a client error where they call
 199              * GetQueryPoolResults on a query that they haven't submitted to
 200              * the GPU yet.  The spec allows us to do anything in this case,
 201              * but returning VK_SUCCESS doesn't seem right and we shouldn't
 202              * just keep spinning.
 203              */
 204             return VK_NOT_READY;
 205          }
 206       }
 207    }
 208 }
 209
 210 VkResult genX(GetQueryPoolResults)(
 211     VkDevice                                    _device,
 212     VkQueryPool                                 queryPool,
 213     uint32_t                                    firstQuery,
 214     uint32_t                                    queryCount,
 215     size_t                                      dataSize,
 216     void*                                       pData,
 217     VkDeviceSize                                stride,
 218     VkQueryResultFlags                          flags)
 219 {
 220    ANV_FROM_HANDLE(anv_device, device, _device);
 221    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 222
 223    assert(pool->type == VK_QUERY_TYPE_OCCLUSION ||
 224           pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS ||
 225           pool->type == VK_QUERY_TYPE_TIMESTAMP);
 226
 227    if (unlikely(device->lost))
 228       return VK_ERROR_DEVICE_LOST;
 229
 230    if (pData == NULL)
 231       return VK_SUCCESS;
 232
 233    void *data_end = pData + dataSize;
 234
 235    VkResult status = VK_SUCCESS;
 236    for (uint32_t i = 0; i < queryCount; i++) {
 237       uint64_t *slot = pool->bo.map + (firstQuery + i) * pool->stride;
 238
 239       /* Availability is always at the start of the slot */
 240       bool available = slot[0];
 241
 242       if (!available && (flags & VK_QUERY_RESULT_WAIT_BIT)) {
 243          status = wait_for_available(device, pool, slot);
 244          if (status != VK_SUCCESS)
 245             return status;
 246
 247          available = true;
 248       }
 249
 250       /* From the Vulkan 1.0.42 spec:
 251        *
 252        *    "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
 253        *    both not set then no result values are written to pData for
 254        *    queries that are in the unavailable state at the time of the call,
 255        *    and vkGetQueryPoolResults returns VK_NOT_READY. However,
 256        *    availability state is still written to pData for those queries if
 257        *    VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set."
 258        */
 259       bool write_results = available || (flags & VK_QUERY_RESULT_PARTIAL_BIT);
 260
 261       uint32_t idx = 0;
 262       switch (pool->type) {
 263       case VK_QUERY_TYPE_OCCLUSION:
 264          if (write_results)
 265             cpu_write_query_result(pData, flags, idx, slot[2] - slot[1]);
 266          idx++;
 267          break;
 268
 269       case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 270          uint32_t statistics = pool->pipeline_statistics;
 271          while (statistics) {
 272             uint32_t stat = u_bit_scan(&statistics);
 273             if (write_results) {
 274                uint64_t result = slot[idx * 2 + 2] - slot[idx * 2 + 1];
 275
 276                /* WaDividePSInvocationCountBy4:HSW,BDW */
 277                if ((device->info.gen == 8 || device->info.is_haswell) &&
 278                    (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT)
 279                   result >>= 2;
 280
 281                cpu_write_query_result(pData, flags, idx, result);
 282             }
 283             idx++;
 284          }
 285          assert(idx == util_bitcount(pool->pipeline_statistics));
 286          break;
 287       }
 288
 289       case VK_QUERY_TYPE_TIMESTAMP:
 290          if (write_results)
 291             cpu_write_query_result(pData, flags, idx, slot[1]);
 292          idx++;
 293          break;
 294
 295       default:
 296          unreachable("invalid pool type");
 297       }
 298
 299       if (!write_results)
 300          status = VK_NOT_READY;
 301
 302       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
 303          cpu_write_query_result(pData, flags, idx, available);
 304
 305       pData += stride;
 306       if (pData >= data_end)
 307          break;
 308    }
 309
 310    return status;
 311 }
 312
 313 static void
 314 emit_ps_depth_count(struct anv_cmd_buffer *cmd_buffer,
 315                     struct anv_address addr)
 316 {
 317    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 318       pc.DestinationAddressType  = DAT_PPGTT;
 319       pc.PostSyncOperation       = WritePSDepthCount;
 320       pc.DepthStallEnable        = true;
 321       pc.Address                 = addr;
 322
 323       if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
 324          pc.CommandStreamerStallEnable = true;
 325    }
 326 }
 327
 328 static void
 329 emit_query_availability(struct anv_cmd_buffer *cmd_buffer,
 330                         struct anv_address addr)
 331 {
 332    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 333       pc.DestinationAddressType  = DAT_PPGTT;
 334       pc.PostSyncOperation       = WriteImmediateData;
 335       pc.Address                 = addr;
 336       pc.ImmediateData           = 1;
 337    }
 338 }
 339
 340 /**
 341  * Goes through a series of consecutive query indices in the given pool
 342  * setting all element values to 0 and emitting them as available.
 343  */
 344 static void
 345 emit_zero_queries(struct anv_cmd_buffer *cmd_buffer,
 346                   struct anv_query_pool *pool,
 347                   uint32_t first_index, uint32_t num_queries)
 348 {
 349    for (uint32_t i = 0; i < num_queries; i++) {
 350       struct anv_address slot_addr =
 351          anv_query_address(pool, first_index + i);
 352       genX(cmd_buffer_mi_memset)(cmd_buffer, anv_address_add(slot_addr, 8),
 353                                  0, pool->stride - 8);
 354       emit_query_availability(cmd_buffer, slot_addr);
 355    }
 356 }
 357
 358 void genX(CmdResetQueryPool)(
 359     VkCommandBuffer                             commandBuffer,
 360     VkQueryPool                                 queryPool,
 361     uint32_t                                    firstQuery,
 362     uint32_t                                    queryCount)
 363 {
 364    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 365    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 366
 367    for (uint32_t i = 0; i < queryCount; i++) {
 368       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdm) {
 369          sdm.Address = anv_query_address(pool, firstQuery + i);
 370          sdm.ImmediateData = 0;
 371       }
 372    }
 373 }
 374
 375 static const uint32_t vk_pipeline_stat_to_reg[] = {
 376    GENX(IA_VERTICES_COUNT_num),
 377    GENX(IA_PRIMITIVES_COUNT_num),
 378    GENX(VS_INVOCATION_COUNT_num),
 379    GENX(GS_INVOCATION_COUNT_num),
 380    GENX(GS_PRIMITIVES_COUNT_num),
 381    GENX(CL_INVOCATION_COUNT_num),
 382    GENX(CL_PRIMITIVES_COUNT_num),
 383    GENX(PS_INVOCATION_COUNT_num),
 384    GENX(HS_INVOCATION_COUNT_num),
 385    GENX(DS_INVOCATION_COUNT_num),
 386    GENX(CS_INVOCATION_COUNT_num),
 387 };
 388
 389 static void
 390 emit_pipeline_stat(struct anv_cmd_buffer *cmd_buffer, uint32_t stat,
 391                    struct anv_address addr)
 392 {
 393    STATIC_ASSERT(ANV_PIPELINE_STATISTICS_MASK ==
 394                  (1 << ARRAY_SIZE(vk_pipeline_stat_to_reg)) - 1);
 395
 396    assert(stat < ARRAY_SIZE(vk_pipeline_stat_to_reg));
 397    uint32_t reg = vk_pipeline_stat_to_reg[stat];
 398
 399    anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
 400       lrm.RegisterAddress  = reg;
 401       lrm.MemoryAddress    = anv_address_add(addr, 0);
 402    }
 403    anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
 404       lrm.RegisterAddress  = reg + 4;
 405       lrm.MemoryAddress    = anv_address_add(addr, 4);
 406    }
 407 }
 408
 409 void genX(CmdBeginQuery)(
 410     VkCommandBuffer                             commandBuffer,
 411     VkQueryPool                                 queryPool,
 412     uint32_t                                    query,
 413     VkQueryControlFlags                         flags)
 414 {
 415    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 416    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 417    struct anv_address query_addr = anv_query_address(pool, query);
 418
 419    switch (pool->type) {
 420    case VK_QUERY_TYPE_OCCLUSION:
 421       emit_ps_depth_count(cmd_buffer, anv_address_add(query_addr, 8));
 422       break;
 423
 424    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 425       /* TODO: This might only be necessary for certain stats */
 426       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 427          pc.CommandStreamerStallEnable = true;
 428          pc.StallAtPixelScoreboard = true;
 429       }
 430
 431       uint32_t statistics = pool->pipeline_statistics;
 432       uint32_t offset = 8;
 433       while (statistics) {
 434          uint32_t stat = u_bit_scan(&statistics);
 435          emit_pipeline_stat(cmd_buffer, stat,
 436                             anv_address_add(query_addr, offset));
 437          offset += 16;
 438       }
 439       break;
 440    }
 441
 442    default:
 443       unreachable("");
 444    }
 445 }
 446
 447 void genX(CmdEndQuery)(
 448     VkCommandBuffer                             commandBuffer,
 449     VkQueryPool                                 queryPool,
 450     uint32_t                                    query)
 451 {
 452    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 453    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 454    struct anv_address query_addr = anv_query_address(pool, query);
 455
 456    switch (pool->type) {
 457    case VK_QUERY_TYPE_OCCLUSION:
 458       emit_ps_depth_count(cmd_buffer, anv_address_add(query_addr, 16));
 459       emit_query_availability(cmd_buffer, query_addr);
 460       break;
 461
 462    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 463       /* TODO: This might only be necessary for certain stats */
 464       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 465          pc.CommandStreamerStallEnable = true;
 466          pc.StallAtPixelScoreboard = true;
 467       }
 468
 469       uint32_t statistics = pool->pipeline_statistics;
 470       uint32_t offset = 16;
 471       while (statistics) {
 472          uint32_t stat = u_bit_scan(&statistics);
 473          emit_pipeline_stat(cmd_buffer, stat,
 474                             anv_address_add(query_addr, offset));
 475          offset += 16;
 476       }
 477
 478       emit_query_availability(cmd_buffer, query_addr);
 479       break;
 480    }
 481
 482    default:
 483       unreachable("");
 484    }
 485
 486    /* When multiview is active the spec requires that N consecutive query
 487     * indices are used, where N is the number of active views in the subpass.
 488     * The spec allows that we only write the results to one of the queries
 489     * but we still need to manage result availability for all the query indices.
 490     * Since we only emit a single query for all active views in the
 491     * first index, mark the other query indices as being already available
 492     * with result 0.
 493     */
 494    if (cmd_buffer->state.subpass && cmd_buffer->state.subpass->view_mask) {
 495       const uint32_t num_queries =
 496          util_bitcount(cmd_buffer->state.subpass->view_mask);
 497       if (num_queries > 1)
 498          emit_zero_queries(cmd_buffer, pool, query + 1, num_queries - 1);
 499    }
 500 }
 501
 502 #define TIMESTAMP 0x2358
 503
 504 void genX(CmdWriteTimestamp)(
 505     VkCommandBuffer                             commandBuffer,
 506     VkPipelineStageFlagBits                     pipelineStage,
 507     VkQueryPool                                 queryPool,
 508     uint32_t                                    query)
 509 {
 510    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 511    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 512    struct anv_address query_addr = anv_query_address(pool, query);
 513
 514    assert(pool->type == VK_QUERY_TYPE_TIMESTAMP);
 515
 516    switch (pipelineStage) {
 517    case VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT:
 518       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 519          srm.RegisterAddress  = TIMESTAMP;
 520          srm.MemoryAddress    = anv_address_add(query_addr, 8);
 521       }
 522       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 523          srm.RegisterAddress  = TIMESTAMP + 4;
 524          srm.MemoryAddress    = anv_address_add(query_addr, 12);
 525       }
 526       break;
 527
 528    default:
 529       /* Everything else is bottom-of-pipe */
 530       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 531          pc.DestinationAddressType  = DAT_PPGTT;
 532          pc.PostSyncOperation       = WriteTimestamp;
 533          pc.Address                 = anv_address_add(query_addr, 8);
 534
 535          if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
 536             pc.CommandStreamerStallEnable = true;
 537       }
 538       break;
 539    }
 540
 541    emit_query_availability(cmd_buffer, query_addr);
 542
 543    /* When multiview is active the spec requires that N consecutive query
 544     * indices are used, where N is the number of active views in the subpass.
 545     * The spec allows that we only write the results to one of the queries
 546     * but we still need to manage result availability for all the query indices.
 547     * Since we only emit a single query for all active views in the
 548     * first index, mark the other query indices as being already available
 549     * with result 0.
 550     */
 551    if (cmd_buffer->state.subpass && cmd_buffer->state.subpass->view_mask) {
 552       const uint32_t num_queries =
 553          util_bitcount(cmd_buffer->state.subpass->view_mask);
 554       if (num_queries > 1)
 555          emit_zero_queries(cmd_buffer, pool, query + 1, num_queries - 1);
 556    }
 557 }
 558
 559 #if GEN_GEN > 7 || GEN_IS_HASWELL
 560
 561 static uint32_t
 562 mi_alu(uint32_t opcode, uint32_t operand1, uint32_t operand2)
 563 {
 564    struct GENX(MI_MATH_ALU_INSTRUCTION) instr = {
 565       .ALUOpcode = opcode,
 566       .Operand1 = operand1,
 567       .Operand2 = operand2,
 568    };
 569
 570    uint32_t dw;
 571    GENX(MI_MATH_ALU_INSTRUCTION_pack)(NULL, &dw, &instr);
 572
 573    return dw;
 574 }
 575
 576 #define CS_GPR(n) (0x2600 + (n) * 8)
 577
 578 static void
 579 emit_load_alu_reg_u64(struct anv_batch *batch, uint32_t reg,
 580                       struct anv_address addr)
 581 {
 582    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
 583       lrm.RegisterAddress  = reg;
 584       lrm.MemoryAddress    = anv_address_add(addr, 0);
 585    }
 586    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
 587       lrm.RegisterAddress  = reg + 4;
 588       lrm.MemoryAddress    = anv_address_add(addr, 4);
 589    }
 590 }
 591
 592 static void
 593 emit_load_alu_reg_imm32(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 594 {
 595    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
 596       lri.RegisterOffset   = reg;
 597       lri.DataDWord        = imm;
 598    }
 599 }
 600
 601 static void
 602 emit_load_alu_reg_imm64(struct anv_batch *batch, uint32_t reg, uint64_t imm)
 603 {
 604    emit_load_alu_reg_imm32(batch, reg, (uint32_t)imm);
 605    emit_load_alu_reg_imm32(batch, reg + 4, (uint32_t)(imm >> 32));
 606 }
 607
 608 static void
 609 emit_load_alu_reg_reg32(struct anv_batch *batch, uint32_t src, uint32_t dst)
 610 {
 611    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_REG), lrr) {
 612       lrr.SourceRegisterAddress      = src;
 613       lrr.DestinationRegisterAddress = dst;
 614    }
 615 }
 616
 617 /*
 618  * GPR0 = GPR0 & ((1ull << n) - 1);
 619  */
 620 static void
 621 keep_gpr0_lower_n_bits(struct anv_batch *batch, uint32_t n)
 622 {
 623    assert(n < 64);
 624    emit_load_alu_reg_imm64(batch, CS_GPR(1), (1ull << n) - 1);
 625
 626    uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
 627    if (!dw) {
 628       anv_batch_set_error(batch, VK_ERROR_OUT_OF_HOST_MEMORY);
 629       return;
 630    }
 631
 632    dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, MI_ALU_REG0);
 633    dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG1);
 634    dw[3] = mi_alu(MI_ALU_AND, 0, 0);
 635    dw[4] = mi_alu(MI_ALU_STORE, MI_ALU_REG0, MI_ALU_ACCU);
 636 }
 637
 638 /*
 639  * GPR0 = GPR0 << 30;
 640  */
 641 static void
 642 shl_gpr0_by_30_bits(struct anv_batch *batch)
 643 {
 644    /* First we mask 34 bits of GPR0 to prevent overflow */
 645    keep_gpr0_lower_n_bits(batch, 34);
 646
 647    const uint32_t outer_count = 5;
 648    const uint32_t inner_count = 6;
 649    STATIC_ASSERT(outer_count * inner_count == 30);
 650    const uint32_t cmd_len = 1 + inner_count * 4;
 651
 652    /* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
 653     * 30 left shifts.
 654     */
 655    for (int o = 0; o < outer_count; o++) {
 656       /* Submit one MI_MATH to shift left by 6 bits */
 657       uint32_t *dw = anv_batch_emitn(batch, cmd_len, GENX(MI_MATH));
 658       if (!dw) {
 659          anv_batch_set_error(batch, VK_ERROR_OUT_OF_HOST_MEMORY);
 660          return;
 661       }
 662
 663       dw++;
 664       for (int i = 0; i < inner_count; i++, dw += 4) {
 665          dw[0] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, MI_ALU_REG0);
 666          dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG0);
 667          dw[2] = mi_alu(MI_ALU_ADD, 0, 0);
 668          dw[3] = mi_alu(MI_ALU_STORE, MI_ALU_REG0, MI_ALU_ACCU);
 669       }
 670    }
 671 }
 672
 673 /*
 674  * GPR0 = GPR0 >> 2;
 675  *
 676  * Note that the upper 30 bits of GPR are lost!
 677  */
 678 static void
 679 shr_gpr0_by_2_bits(struct anv_batch *batch)
 680 {
 681    shl_gpr0_by_30_bits(batch);
 682    emit_load_alu_reg_reg32(batch, CS_GPR(0) + 4, CS_GPR(0));
 683    emit_load_alu_reg_imm32(batch, CS_GPR(0) + 4, 0);
 684 }
 685
 686 static void
 687 gpu_write_query_result(struct anv_batch *batch,
 688                        struct anv_address dst_addr,
 689                        VkQueryResultFlags flags,
 690                        uint32_t value_index, uint32_t reg)
 691 {
 692    if (flags & VK_QUERY_RESULT_64_BIT)
 693       dst_addr = anv_address_add(dst_addr, value_index * 8);
 694    else
 695       dst_addr = anv_address_add(dst_addr, value_index * 4);
 696
 697    anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 698       srm.RegisterAddress  = reg;
 699       srm.MemoryAddress    = anv_address_add(dst_addr, 0);
 700    }
 701
 702    if (flags & VK_QUERY_RESULT_64_BIT) {
 703       anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 704          srm.RegisterAddress  = reg + 4;
 705          srm.MemoryAddress    = anv_address_add(dst_addr, 4);
 706       }
 707    }
 708 }
 709
 710 static void
 711 compute_query_result(struct anv_batch *batch, uint32_t dst_reg,
 712                      struct anv_address addr)
 713 {
 714    emit_load_alu_reg_u64(batch, CS_GPR(0), anv_address_add(addr, 0));
 715    emit_load_alu_reg_u64(batch, CS_GPR(1), anv_address_add(addr, 8));
 716
 717    /* FIXME: We need to clamp the result for 32 bit. */
 718
 719    uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
 720    if (!dw) {
 721       anv_batch_set_error(batch, VK_ERROR_OUT_OF_HOST_MEMORY);
 722       return;
 723    }
 724
 725    dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, MI_ALU_REG1);
 726    dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG0);
 727    dw[3] = mi_alu(MI_ALU_SUB, 0, 0);
 728    dw[4] = mi_alu(MI_ALU_STORE, dst_reg, MI_ALU_ACCU);
 729 }
 730
 731 void genX(CmdCopyQueryPoolResults)(
 732     VkCommandBuffer                             commandBuffer,
 733     VkQueryPool                                 queryPool,
 734     uint32_t                                    firstQuery,
 735     uint32_t                                    queryCount,
 736     VkBuffer                                    destBuffer,
 737     VkDeviceSize                                destOffset,
 738     VkDeviceSize                                destStride,
 739     VkQueryResultFlags                          flags)
 740 {
 741    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 742    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 743    ANV_FROM_HANDLE(anv_buffer, buffer, destBuffer);
 744
 745    if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 746       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 747          pc.CommandStreamerStallEnable = true;
 748          pc.StallAtPixelScoreboard     = true;
 749       }
 750    }
 751
 752    struct anv_address dest_addr = anv_address_add(buffer->address, destOffset);
 753    for (uint32_t i = 0; i < queryCount; i++) {
 754       struct anv_address query_addr = anv_query_address(pool, firstQuery + i);
 755       uint32_t idx = 0;
 756       switch (pool->type) {
 757       case VK_QUERY_TYPE_OCCLUSION:
 758          compute_query_result(&cmd_buffer->batch, MI_ALU_REG2,
 759                               anv_address_add(query_addr, 8));
 760          gpu_write_query_result(&cmd_buffer->batch, dest_addr,
 761                                 flags, idx++, CS_GPR(2));
 762          break;
 763
 764       case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 765          uint32_t statistics = pool->pipeline_statistics;
 766          while (statistics) {
 767             uint32_t stat = u_bit_scan(&statistics);
 768
 769             compute_query_result(&cmd_buffer->batch, MI_ALU_REG0,
 770                                  anv_address_add(query_addr, idx * 16 + 8));
 771
 772             /* WaDividePSInvocationCountBy4:HSW,BDW */
 773             if ((cmd_buffer->device->info.gen == 8 ||
 774                  cmd_buffer->device->info.is_haswell) &&
 775                 (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT) {
 776                shr_gpr0_by_2_bits(&cmd_buffer->batch);
 777             }
 778
 779             gpu_write_query_result(&cmd_buffer->batch, dest_addr,
 780                                    flags, idx++, CS_GPR(0));
 781          }
 782          assert(idx == util_bitcount(pool->pipeline_statistics));
 783          break;
 784       }
 785
 786       case VK_QUERY_TYPE_TIMESTAMP:
 787          emit_load_alu_reg_u64(&cmd_buffer->batch,
 788                                CS_GPR(2), anv_address_add(query_addr, 8));
 789          gpu_write_query_result(&cmd_buffer->batch, dest_addr,
 790                                 flags, 0, CS_GPR(2));
 791          break;
 792
 793       default:
 794          unreachable("unhandled query type");
 795       }
 796
 797       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 798          emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(0), query_addr);
 799          gpu_write_query_result(&cmd_buffer->batch, dest_addr,
 800                                 flags, idx, CS_GPR(0));
 801       }
 802
 803       dest_addr = anv_address_add(dest_addr, destStride);
 804    }
 805 }
 806
 807 #else
 808 void genX(CmdCopyQueryPoolResults)(
 809     VkCommandBuffer                             commandBuffer,
 810     VkQueryPool                                 queryPool,
 811     uint32_t                                    firstQuery,
 812     uint32_t                                    queryCount,
 813     VkBuffer                                    destBuffer,
 814     VkDeviceSize                                destOffset,
 815     VkDeviceSize                                destStride,
 816     VkQueryResultFlags                          flags)
 817 {
 818    anv_finishme("Queries not yet supported on Ivy Bridge");
 819 }
 820 #endif