src/intel/vulkan/genX_query.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 #include "genxml/gen_macros.h"
  33 #include "genxml/genX_pack.h"
  34
  35 VkResult genX(CreateQueryPool)(
  36     VkDevice                                    _device,
  37     const VkQueryPoolCreateInfo*                pCreateInfo,
  38     const VkAllocationCallbacks*                pAllocator,
  39     VkQueryPool*                                pQueryPool)
  40 {
  41    ANV_FROM_HANDLE(anv_device, device, _device);
  42    const struct anv_physical_device *pdevice = &device->instance->physicalDevice;
  43    struct anv_query_pool *pool;
  44    VkResult result;
  45
  46    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO);
  47
  48    /* Query pool slots are made up of some number of 64-bit values packed
  49     * tightly together.  The first 64-bit value is always the "available" bit
  50     * which is 0 when the query is unavailable and 1 when it is available.
  51     * The 64-bit values that follow are determined by the type of query.
  52     */
  53    uint32_t uint64s_per_slot = 1;
  54
  55    VkQueryPipelineStatisticFlags pipeline_statistics = 0;
  56    switch (pCreateInfo->queryType) {
  57    case VK_QUERY_TYPE_OCCLUSION:
  58       /* Occlusion queries have two values: begin and end. */
  59       uint64s_per_slot += 2;
  60       break;
  61    case VK_QUERY_TYPE_TIMESTAMP:
  62       /* Timestamps just have the one timestamp value */
  63       uint64s_per_slot += 1;
  64       break;
  65    case VK_QUERY_TYPE_PIPELINE_STATISTICS:
  66       pipeline_statistics = pCreateInfo->pipelineStatistics;
  67       /* We're going to trust this field implicitly so we need to ensure that
  68        * no unhandled extension bits leak in.
  69        */
  70       pipeline_statistics &= ANV_PIPELINE_STATISTICS_MASK;
  71
  72       /* Statistics queries have a min and max for every statistic */
  73       uint64s_per_slot += 2 * _mesa_bitcount(pipeline_statistics);
  74       break;
  75    default:
  76       assert(!"Invalid query type");
  77    }
  78
  79    pool = vk_alloc2(&device->alloc, pAllocator, sizeof(*pool), 8,
  80                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  81    if (pool == NULL)
  82       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
  83
  84    pool->type = pCreateInfo->queryType;
  85    pool->pipeline_statistics = pipeline_statistics;
  86    pool->stride = uint64s_per_slot * sizeof(uint64_t);
  87    pool->slots = pCreateInfo->queryCount;
  88
  89    uint64_t size = pool->slots * pool->stride;
  90    result = anv_bo_init_new(&pool->bo, device, size);
  91    if (result != VK_SUCCESS)
  92       goto fail;
  93
  94    if (pdevice->supports_48bit_addresses)
  95       pool->bo.flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
  96
  97    if (pdevice->has_exec_async)
  98       pool->bo.flags |= EXEC_OBJECT_ASYNC;
  99
 100    /* For query pools, we set the caching mode to I915_CACHING_CACHED.  On LLC
 101     * platforms, this does nothing.  On non-LLC platforms, this means snooping
 102     * which comes at a slight cost.  However, the buffers aren't big, won't be
 103     * written frequently, and trying to handle the flushing manually without
 104     * doing too much flushing is extremely painful.
 105     */
 106    anv_gem_set_caching(device, pool->bo.gem_handle, I915_CACHING_CACHED);
 107
 108    pool->bo.map = anv_gem_mmap(device, pool->bo.gem_handle, 0, size, 0);
 109
 110    *pQueryPool = anv_query_pool_to_handle(pool);
 111
 112    return VK_SUCCESS;
 113
 114  fail:
 115    vk_free2(&device->alloc, pAllocator, pool);
 116
 117    return result;
 118 }
 119
 120 void genX(DestroyQueryPool)(
 121     VkDevice                                    _device,
 122     VkQueryPool                                 _pool,
 123     const VkAllocationCallbacks*                pAllocator)
 124 {
 125    ANV_FROM_HANDLE(anv_device, device, _device);
 126    ANV_FROM_HANDLE(anv_query_pool, pool, _pool);
 127
 128    if (!pool)
 129       return;
 130
 131    anv_gem_munmap(pool->bo.map, pool->bo.size);
 132    anv_gem_close(device, pool->bo.gem_handle);
 133    vk_free2(&device->alloc, pAllocator, pool);
 134 }
 135
 136 static void
 137 cpu_write_query_result(void *dst_slot, VkQueryResultFlags flags,
 138                        uint32_t value_index, uint64_t result)
 139 {
 140    if (flags & VK_QUERY_RESULT_64_BIT) {
 141       uint64_t *dst64 = dst_slot;
 142       dst64[value_index] = result;
 143    } else {
 144       uint32_t *dst32 = dst_slot;
 145       dst32[value_index] = result;
 146    }
 147 }
 148
 149 static bool
 150 query_is_available(uint64_t *slot)
 151 {
 152    return *(volatile uint64_t *)slot;
 153 }
 154
 155 static VkResult
 156 wait_for_available(struct anv_device *device,
 157                    struct anv_query_pool *pool, uint64_t *slot)
 158 {
 159    while (true) {
 160       if (query_is_available(slot))
 161          return VK_SUCCESS;
 162
 163       int ret = anv_gem_busy(device, pool->bo.gem_handle);
 164       if (ret == 1) {
 165          /* The BO is still busy, keep waiting. */
 166          continue;
 167       } else if (ret == -1) {
 168          /* We don't know the real error. */
 169          device->lost = true;
 170          return vk_errorf(VK_ERROR_DEVICE_LOST, "gem wait failed: %m");
 171       } else {
 172          assert(ret == 0);
 173          /* The BO is no longer busy. */
 174          if (query_is_available(slot)) {
 175             return VK_SUCCESS;
 176          } else {
 177             VkResult status = anv_device_query_status(device);
 178             if (status != VK_SUCCESS)
 179                return status;
 180
 181             /* If we haven't seen availability yet, then we never will.  This
 182              * can only happen if we have a client error where they call
 183              * GetQueryPoolResults on a query that they haven't submitted to
 184              * the GPU yet.  The spec allows us to do anything in this case,
 185              * but returning VK_SUCCESS doesn't seem right and we shouldn't
 186              * just keep spinning.
 187              */
 188             return VK_NOT_READY;
 189          }
 190       }
 191    }
 192 }
 193
 194 VkResult genX(GetQueryPoolResults)(
 195     VkDevice                                    _device,
 196     VkQueryPool                                 queryPool,
 197     uint32_t                                    firstQuery,
 198     uint32_t                                    queryCount,
 199     size_t                                      dataSize,
 200     void*                                       pData,
 201     VkDeviceSize                                stride,
 202     VkQueryResultFlags                          flags)
 203 {
 204    ANV_FROM_HANDLE(anv_device, device, _device);
 205    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 206
 207    assert(pool->type == VK_QUERY_TYPE_OCCLUSION ||
 208           pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS ||
 209           pool->type == VK_QUERY_TYPE_TIMESTAMP);
 210
 211    if (unlikely(device->lost))
 212       return VK_ERROR_DEVICE_LOST;
 213
 214    if (pData == NULL)
 215       return VK_SUCCESS;
 216
 217    void *data_end = pData + dataSize;
 218
 219    VkResult status = VK_SUCCESS;
 220    for (uint32_t i = 0; i < queryCount; i++) {
 221       uint64_t *slot = pool->bo.map + (firstQuery + i) * pool->stride;
 222
 223       /* Availability is always at the start of the slot */
 224       bool available = slot[0];
 225
 226       if (!available && (flags & VK_QUERY_RESULT_WAIT_BIT)) {
 227          status = wait_for_available(device, pool, slot);
 228          if (status != VK_SUCCESS)
 229             return status;
 230
 231          available = true;
 232       }
 233
 234       /* From the Vulkan 1.0.42 spec:
 235        *
 236        *    "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
 237        *    both not set then no result values are written to pData for
 238        *    queries that are in the unavailable state at the time of the call,
 239        *    and vkGetQueryPoolResults returns VK_NOT_READY. However,
 240        *    availability state is still written to pData for those queries if
 241        *    VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set."
 242        */
 243       bool write_results = available || (flags & VK_QUERY_RESULT_PARTIAL_BIT);
 244
 245       if (write_results) {
 246          switch (pool->type) {
 247          case VK_QUERY_TYPE_OCCLUSION: {
 248             cpu_write_query_result(pData, flags, 0, slot[2] - slot[1]);
 249             break;
 250          }
 251
 252          case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 253             uint32_t statistics = pool->pipeline_statistics;
 254             uint32_t idx = 0;
 255             while (statistics) {
 256                uint32_t stat = u_bit_scan(&statistics);
 257                uint64_t result = slot[idx * 2 + 2] - slot[idx * 2 + 1];
 258
 259                /* WaDividePSInvocationCountBy4:HSW,BDW */
 260                if ((device->info.gen == 8 || device->info.is_haswell) &&
 261                    (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT)
 262                   result >>= 2;
 263
 264                cpu_write_query_result(pData, flags, idx, result);
 265
 266                idx++;
 267             }
 268             assert(idx == _mesa_bitcount(pool->pipeline_statistics));
 269             break;
 270          }
 271
 272          case VK_QUERY_TYPE_TIMESTAMP: {
 273             cpu_write_query_result(pData, flags, 0, slot[1]);
 274             break;
 275          }
 276          default:
 277             unreachable("invalid pool type");
 278          }
 279       } else {
 280          status = VK_NOT_READY;
 281       }
 282
 283       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 284          uint32_t idx = (pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS) ?
 285                         _mesa_bitcount(pool->pipeline_statistics) : 1;
 286          cpu_write_query_result(pData, flags, idx, available);
 287       }
 288
 289       pData += stride;
 290       if (pData >= data_end)
 291          break;
 292    }
 293
 294    return status;
 295 }
 296
 297 static void
 298 emit_ps_depth_count(struct anv_cmd_buffer *cmd_buffer,
 299                     struct anv_bo *bo, uint32_t offset)
 300 {
 301    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 302       pc.DestinationAddressType  = DAT_PPGTT;
 303       pc.PostSyncOperation       = WritePSDepthCount;
 304       pc.DepthStallEnable        = true;
 305       pc.Address                 = (struct anv_address) { bo, offset };
 306
 307       if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
 308          pc.CommandStreamerStallEnable = true;
 309    }
 310 }
 311
 312 static void
 313 emit_query_availability(struct anv_cmd_buffer *cmd_buffer,
 314                         struct anv_bo *bo, uint32_t offset)
 315 {
 316    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 317       pc.DestinationAddressType  = DAT_PPGTT;
 318       pc.PostSyncOperation       = WriteImmediateData;
 319       pc.Address                 = (struct anv_address) { bo, offset };
 320       pc.ImmediateData           = 1;
 321    }
 322 }
 323
 324 void genX(CmdResetQueryPool)(
 325     VkCommandBuffer                             commandBuffer,
 326     VkQueryPool                                 queryPool,
 327     uint32_t                                    firstQuery,
 328     uint32_t                                    queryCount)
 329 {
 330    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 331    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 332
 333    for (uint32_t i = 0; i < queryCount; i++) {
 334       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdm) {
 335          sdm.Address = (struct anv_address) {
 336             .bo = &pool->bo,
 337             .offset = (firstQuery + i) * pool->stride,
 338          };
 339          sdm.ImmediateData = 0;
 340       }
 341    }
 342 }
 343
 344 static const uint32_t vk_pipeline_stat_to_reg[] = {
 345    GENX(IA_VERTICES_COUNT_num),
 346    GENX(IA_PRIMITIVES_COUNT_num),
 347    GENX(VS_INVOCATION_COUNT_num),
 348    GENX(GS_INVOCATION_COUNT_num),
 349    GENX(GS_PRIMITIVES_COUNT_num),
 350    GENX(CL_INVOCATION_COUNT_num),
 351    GENX(CL_PRIMITIVES_COUNT_num),
 352    GENX(PS_INVOCATION_COUNT_num),
 353    GENX(HS_INVOCATION_COUNT_num),
 354    GENX(DS_INVOCATION_COUNT_num),
 355    GENX(CS_INVOCATION_COUNT_num),
 356 };
 357
 358 static void
 359 emit_pipeline_stat(struct anv_cmd_buffer *cmd_buffer, uint32_t stat,
 360                    struct anv_bo *bo, uint32_t offset)
 361 {
 362    STATIC_ASSERT(ANV_PIPELINE_STATISTICS_MASK ==
 363                  (1 << ARRAY_SIZE(vk_pipeline_stat_to_reg)) - 1);
 364
 365    assert(stat < ARRAY_SIZE(vk_pipeline_stat_to_reg));
 366    uint32_t reg = vk_pipeline_stat_to_reg[stat];
 367
 368    anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
 369       lrm.RegisterAddress  = reg,
 370       lrm.MemoryAddress    = (struct anv_address) { bo, offset };
 371    }
 372    anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
 373       lrm.RegisterAddress  = reg + 4,
 374       lrm.MemoryAddress    = (struct anv_address) { bo, offset + 4 };
 375    }
 376 }
 377
 378 void genX(CmdBeginQuery)(
 379     VkCommandBuffer                             commandBuffer,
 380     VkQueryPool                                 queryPool,
 381     uint32_t                                    query,
 382     VkQueryControlFlags                         flags)
 383 {
 384    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 385    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 386
 387    /* Workaround: When meta uses the pipeline with the VS disabled, it seems
 388     * that the pipelining of the depth write breaks. What we see is that
 389     * samples from the render pass clear leaks into the first query
 390     * immediately after the clear. Doing a pipecontrol with a post-sync
 391     * operation and DepthStallEnable seems to work around the issue.
 392     */
 393    if (cmd_buffer->state.need_query_wa) {
 394       cmd_buffer->state.need_query_wa = false;
 395       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 396          pc.DepthCacheFlushEnable   = true;
 397          pc.DepthStallEnable        = true;
 398       }
 399    }
 400
 401    switch (pool->type) {
 402    case VK_QUERY_TYPE_OCCLUSION:
 403       emit_ps_depth_count(cmd_buffer, &pool->bo, query * pool->stride + 8);
 404       break;
 405
 406    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 407       /* TODO: This might only be necessary for certain stats */
 408       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 409          pc.CommandStreamerStallEnable = true;
 410          pc.StallAtPixelScoreboard = true;
 411       }
 412
 413       uint32_t statistics = pool->pipeline_statistics;
 414       uint32_t offset = query * pool->stride + 8;
 415       while (statistics) {
 416          uint32_t stat = u_bit_scan(&statistics);
 417          emit_pipeline_stat(cmd_buffer, stat, &pool->bo, offset);
 418          offset += 16;
 419       }
 420       break;
 421    }
 422
 423    default:
 424       unreachable("");
 425    }
 426 }
 427
 428 void genX(CmdEndQuery)(
 429     VkCommandBuffer                             commandBuffer,
 430     VkQueryPool                                 queryPool,
 431     uint32_t                                    query)
 432 {
 433    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 434    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 435
 436    switch (pool->type) {
 437    case VK_QUERY_TYPE_OCCLUSION:
 438       emit_ps_depth_count(cmd_buffer, &pool->bo, query * pool->stride + 16);
 439       emit_query_availability(cmd_buffer, &pool->bo, query * pool->stride);
 440       break;
 441
 442    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 443       /* TODO: This might only be necessary for certain stats */
 444       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 445          pc.CommandStreamerStallEnable = true;
 446          pc.StallAtPixelScoreboard = true;
 447       }
 448
 449       uint32_t statistics = pool->pipeline_statistics;
 450       uint32_t offset = query * pool->stride + 16;
 451       while (statistics) {
 452          uint32_t stat = u_bit_scan(&statistics);
 453          emit_pipeline_stat(cmd_buffer, stat, &pool->bo, offset);
 454          offset += 16;
 455       }
 456
 457       emit_query_availability(cmd_buffer, &pool->bo, query * pool->stride);
 458       break;
 459    }
 460
 461    default:
 462       unreachable("");
 463    }
 464 }
 465
 466 #define TIMESTAMP 0x2358
 467
 468 void genX(CmdWriteTimestamp)(
 469     VkCommandBuffer                             commandBuffer,
 470     VkPipelineStageFlagBits                     pipelineStage,
 471     VkQueryPool                                 queryPool,
 472     uint32_t                                    query)
 473 {
 474    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 475    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 476    uint32_t offset = query * pool->stride;
 477
 478    assert(pool->type == VK_QUERY_TYPE_TIMESTAMP);
 479
 480    switch (pipelineStage) {
 481    case VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT:
 482       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 483          srm.RegisterAddress  = TIMESTAMP;
 484          srm.MemoryAddress    = (struct anv_address) { &pool->bo, offset + 8 };
 485       }
 486       anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 487          srm.RegisterAddress  = TIMESTAMP + 4;
 488          srm.MemoryAddress    = (struct anv_address) { &pool->bo, offset + 12 };
 489       }
 490       break;
 491
 492    default:
 493       /* Everything else is bottom-of-pipe */
 494       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 495          pc.DestinationAddressType  = DAT_PPGTT;
 496          pc.PostSyncOperation       = WriteTimestamp;
 497          pc.Address = (struct anv_address) { &pool->bo, offset + 8 };
 498
 499          if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
 500             pc.CommandStreamerStallEnable = true;
 501       }
 502       break;
 503    }
 504
 505    emit_query_availability(cmd_buffer, &pool->bo, offset);
 506 }
 507
 508 #if GEN_GEN > 7 || GEN_IS_HASWELL
 509
 510 static inline uint32_t
 511 mi_alu(uint32_t opcode, uint32_t operand1, uint32_t operand2)
 512 {
 513    struct GENX(MI_MATH_ALU_INSTRUCTION) instr = {
 514       .ALUOpcode = opcode,
 515       .Operand1 = operand1,
 516       .Operand2 = operand2,
 517    };
 518
 519    uint32_t dw;
 520    GENX(MI_MATH_ALU_INSTRUCTION_pack)(NULL, &dw, &instr);
 521
 522    return dw;
 523 }
 524
 525 #define CS_GPR(n) (0x2600 + (n) * 8)
 526
 527 static void
 528 emit_load_alu_reg_u64(struct anv_batch *batch, uint32_t reg,
 529                       struct anv_bo *bo, uint32_t offset)
 530 {
 531    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
 532       lrm.RegisterAddress  = reg,
 533       lrm.MemoryAddress    = (struct anv_address) { bo, offset };
 534    }
 535    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
 536       lrm.RegisterAddress  = reg + 4;
 537       lrm.MemoryAddress    = (struct anv_address) { bo, offset + 4 };
 538    }
 539 }
 540
 541 static void
 542 emit_load_alu_reg_imm32(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 543 {
 544    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
 545       lri.RegisterOffset   = reg;
 546       lri.DataDWord        = imm;
 547    }
 548 }
 549
 550 static void
 551 emit_load_alu_reg_imm64(struct anv_batch *batch, uint32_t reg, uint64_t imm)
 552 {
 553    emit_load_alu_reg_imm32(batch, reg, (uint32_t)imm);
 554    emit_load_alu_reg_imm32(batch, reg + 4, (uint32_t)(imm >> 32));
 555 }
 556
 557 static void
 558 emit_load_alu_reg_reg32(struct anv_batch *batch, uint32_t src, uint32_t dst)
 559 {
 560    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_REG), lrr) {
 561       lrr.SourceRegisterAddress      = src;
 562       lrr.DestinationRegisterAddress = dst;
 563    }
 564 }
 565
 566 /*
 567  * GPR0 = GPR0 & ((1ull << n) - 1);
 568  */
 569 static void
 570 keep_gpr0_lower_n_bits(struct anv_batch *batch, uint32_t n)
 571 {
 572    assert(n < 64);
 573    emit_load_alu_reg_imm64(batch, CS_GPR(1), (1ull << n) - 1);
 574
 575    uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
 576    if (!dw) {
 577       anv_batch_set_error(batch, VK_ERROR_OUT_OF_HOST_MEMORY);
 578       return;
 579    }
 580
 581    dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, MI_ALU_REG0);
 582    dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG1);
 583    dw[3] = mi_alu(MI_ALU_AND, 0, 0);
 584    dw[4] = mi_alu(MI_ALU_STORE, MI_ALU_REG0, MI_ALU_ACCU);
 585 }
 586
 587 /*
 588  * GPR0 = GPR0 << 30;
 589  */
 590 static void
 591 shl_gpr0_by_30_bits(struct anv_batch *batch)
 592 {
 593    /* First we mask 34 bits of GPR0 to prevent overflow */
 594    keep_gpr0_lower_n_bits(batch, 34);
 595
 596    const uint32_t outer_count = 5;
 597    const uint32_t inner_count = 6;
 598    STATIC_ASSERT(outer_count * inner_count == 30);
 599    const uint32_t cmd_len = 1 + inner_count * 4;
 600
 601    /* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
 602     * 30 left shifts.
 603     */
 604    for (int o = 0; o < outer_count; o++) {
 605       /* Submit one MI_MATH to shift left by 6 bits */
 606       uint32_t *dw = anv_batch_emitn(batch, cmd_len, GENX(MI_MATH));
 607       if (!dw) {
 608          anv_batch_set_error(batch, VK_ERROR_OUT_OF_HOST_MEMORY);
 609          return;
 610       }
 611
 612       dw++;
 613       for (int i = 0; i < inner_count; i++, dw += 4) {
 614          dw[0] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, MI_ALU_REG0);
 615          dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG0);
 616          dw[2] = mi_alu(MI_ALU_ADD, 0, 0);
 617          dw[3] = mi_alu(MI_ALU_STORE, MI_ALU_REG0, MI_ALU_ACCU);
 618       }
 619    }
 620 }
 621
 622 /*
 623  * GPR0 = GPR0 >> 2;
 624  *
 625  * Note that the upper 30 bits of GPR are lost!
 626  */
 627 static void
 628 shr_gpr0_by_2_bits(struct anv_batch *batch)
 629 {
 630    shl_gpr0_by_30_bits(batch);
 631    emit_load_alu_reg_reg32(batch, CS_GPR(0) + 4, CS_GPR(0));
 632    emit_load_alu_reg_imm32(batch, CS_GPR(0) + 4, 0);
 633 }
 634
 635 static void
 636 gpu_write_query_result(struct anv_batch *batch,
 637                        struct anv_buffer *dst_buffer, uint32_t dst_offset,
 638                        VkQueryResultFlags flags,
 639                        uint32_t value_index, uint32_t reg)
 640 {
 641    if (flags & VK_QUERY_RESULT_64_BIT)
 642       dst_offset += value_index * 8;
 643    else
 644       dst_offset += value_index * 4;
 645
 646    anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 647       srm.RegisterAddress  = reg;
 648       srm.MemoryAddress    = (struct anv_address) {
 649          .bo = dst_buffer->bo,
 650          .offset = dst_buffer->offset + dst_offset,
 651       };
 652    }
 653
 654    if (flags & VK_QUERY_RESULT_64_BIT) {
 655       anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
 656          srm.RegisterAddress  = reg + 4;
 657          srm.MemoryAddress    = (struct anv_address) {
 658             .bo = dst_buffer->bo,
 659             .offset = dst_buffer->offset + dst_offset + 4,
 660          };
 661       }
 662    }
 663 }
 664
 665 static void
 666 compute_query_result(struct anv_batch *batch, uint32_t dst_reg,
 667                      struct anv_bo *bo, uint32_t offset)
 668 {
 669    emit_load_alu_reg_u64(batch, CS_GPR(0), bo, offset);
 670    emit_load_alu_reg_u64(batch, CS_GPR(1), bo, offset + 8);
 671
 672    /* FIXME: We need to clamp the result for 32 bit. */
 673
 674    uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
 675    if (!dw) {
 676       anv_batch_set_error(batch, VK_ERROR_OUT_OF_HOST_MEMORY);
 677       return;
 678    }
 679
 680    dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, MI_ALU_REG1);
 681    dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG0);
 682    dw[3] = mi_alu(MI_ALU_SUB, 0, 0);
 683    dw[4] = mi_alu(MI_ALU_STORE, dst_reg, MI_ALU_ACCU);
 684 }
 685
 686 void genX(CmdCopyQueryPoolResults)(
 687     VkCommandBuffer                             commandBuffer,
 688     VkQueryPool                                 queryPool,
 689     uint32_t                                    firstQuery,
 690     uint32_t                                    queryCount,
 691     VkBuffer                                    destBuffer,
 692     VkDeviceSize                                destOffset,
 693     VkDeviceSize                                destStride,
 694     VkQueryResultFlags                          flags)
 695 {
 696    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 697    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 698    ANV_FROM_HANDLE(anv_buffer, buffer, destBuffer);
 699    uint32_t slot_offset;
 700
 701    if (flags & VK_QUERY_RESULT_WAIT_BIT) {
 702       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
 703          pc.CommandStreamerStallEnable = true;
 704          pc.StallAtPixelScoreboard     = true;
 705       }
 706    }
 707
 708    for (uint32_t i = 0; i < queryCount; i++) {
 709       slot_offset = (firstQuery + i) * pool->stride;
 710       switch (pool->type) {
 711       case VK_QUERY_TYPE_OCCLUSION:
 712          compute_query_result(&cmd_buffer->batch, MI_ALU_REG2,
 713                               &pool->bo, slot_offset + 8);
 714          gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 715                                 flags, 0, CS_GPR(2));
 716          break;
 717
 718       case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
 719          uint32_t statistics = pool->pipeline_statistics;
 720          uint32_t idx = 0;
 721          while (statistics) {
 722             uint32_t stat = u_bit_scan(&statistics);
 723
 724             compute_query_result(&cmd_buffer->batch, MI_ALU_REG0,
 725                                  &pool->bo, slot_offset + idx * 16 + 8);
 726
 727             /* WaDividePSInvocationCountBy4:HSW,BDW */
 728             if ((cmd_buffer->device->info.gen == 8 ||
 729                  cmd_buffer->device->info.is_haswell) &&
 730                 (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT) {
 731                shr_gpr0_by_2_bits(&cmd_buffer->batch);
 732             }
 733
 734             gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 735                                    flags, idx, CS_GPR(0));
 736
 737             idx++;
 738          }
 739          assert(idx == _mesa_bitcount(pool->pipeline_statistics));
 740          break;
 741       }
 742
 743       case VK_QUERY_TYPE_TIMESTAMP:
 744          emit_load_alu_reg_u64(&cmd_buffer->batch,
 745                                CS_GPR(2), &pool->bo, slot_offset + 8);
 746          gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 747                                 flags, 0, CS_GPR(2));
 748          break;
 749
 750       default:
 751          unreachable("unhandled query type");
 752       }
 753
 754       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
 755          uint32_t idx = (pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS) ?
 756                         _mesa_bitcount(pool->pipeline_statistics) : 1;
 757
 758          emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(0),
 759                                &pool->bo, slot_offset);
 760          gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
 761                                 flags, idx, CS_GPR(0));
 762       }
 763
 764       destOffset += destStride;
 765    }
 766 }
 767
 768 #else
 769 void genX(CmdCopyQueryPoolResults)(
 770     VkCommandBuffer                             commandBuffer,
 771     VkQueryPool                                 queryPool,
 772     uint32_t                                    firstQuery,
 773     uint32_t                                    queryCount,
 774     VkBuffer                                    destBuffer,
 775     VkDeviceSize                                destOffset,
 776     VkDeviceSize                                destStride,
 777     VkQueryResultFlags                          flags)
 778 {
 779    anv_finishme("Queries not yet supported on Ivy Bridge");
 780 }
 781 #endif